Add files via upload

Networking.py

@@ -0,0 +1,52 @@
+import socket
+import nmap
+
+def inMyNetwork(hosts='192.168.1.0/24', arguments='-n -sP -PE -PA21,23,80,3389'):
+    """
+    generate list of devices and there IP addresses who connected with you in the same Network
+    :param str() hosts: string for hosts as nmap use it 'scanme.nmap.org' or '198.116.0-255.1-127' or '216.163.128.20/20'
+    :param arguments: string of arguments for nmap '-sU -sX -sC'
+    """
+    nm = nmap.PortScanner()
+    nm.scan(hosts=hosts,arguments=arguments)
+    hosts_list = [(x, socket.gethostbyaddr(x)[0]) for x in nm.all_hosts() if socket.gethostbyaddr(x)]
+    for host, name in hosts_list:
+        print('{0}:{1} '.format(name, host))
+
+def ClusterGen(workers,pss,port=2222):
+    """
+    function to generate list of IP addresses for workers and parameters servers from Devices name
+    :param list[str] workers: list of workers Devices name
+    :param list[str] pss: list of parameter servers Devices name
+    :param int() port: port number
+    """
+    l = []
+    d = []
+    for worker in workers:
+        try:
+            l.append(socket.gethostbyname(worker)+":"+str(port))
+        except:
+            print("worker {} Hors ligne ".format(worker))
+    for ps in pss:
+        try:
+            d.append(socket.gethostbyname(ps)+":"+str(port))
+        except:
+            print("ps {} Hors ligne".format(ps))
+    return {'worker':l,'ps':d}
+
+
+
+
+if __name__ == '__main__':
+
+    workers = ['DESKTOP-07HFBQN','fouzi-pc']
+    pss = ['DELL-MINI']
+
+    print(ClusterGen(workers,pss))
+
+    inMyNetwork()
+
+
+
+
+

dataSetGenerator.py

@@ -0,0 +1 @@
+import numpy as npfrom os import listdirfrom glob import globfrom time import timeimport cv2import matplotlib.pyplot as pltfrom os.path import isfilefrom os import getlogindef dataSetGenerator(path,resize=True,resize_to=224,percentage=100):    """    Generate a image dataSet from a picture dataSets    the picture dataSets must be in the same structure to generate also labels    example of pictureFolder: http://weegee.vision.ucmerced.edu/datasets/landuse.html    picture dataSets      |      |----------class-1      |        .   |-------image-1      |        .   |         .      |        .   |         .      |        .   |         .      |        .   |-------image-n      |        .      |-------class-n    :param str path: the path for picture dataSets folder    :param bool resize: choose resize the pictures or not    :param int resize_to: the new size of pictures    :param int or float percentage: how many pictures you want to get from this pictureFolder    :return:give as tuple of images, labels , classes    :rtype: tuple[object[numpy.ndarray],object[numpy.ndarray],object[numpy.ndarray]]        """    try:        start_time = time()        classes = listdir(path)        image_list = []        labels = []        for classe in classes:            for filename in glob(path+'/'+classe+'/*'):                if resize: image_list.append(cv2.resize(cv2.imread(filename, cv2.COLOR_BGR2RGB),(resize_to, resize_to)))                else:image_list.append(cv2.imread(filename, cv2.COLOR_BGR2RGB))                #if resize: image_list.append(cv2.cvtColor(cv2.resize(cv2.imread(filename),(resize_to, resize_to)), cv2.COLOR_BGR2RGB))                #else:image_list.append((cv2.cvtColor(cv2.imread(filename), cv2.COLOR_BGR2RGB)))                label = np.zeros(len(classes))                label[classes.index(classe)] = 1                labels.append(label)        indice = np.random.permutation(len(image_list))[:int(len(image_list)*percentage/100)]        print("\n --- dataSet generated in  %s seconds --- \n" % (np.round(time()-start_time)))        return np.array([image_list[x] for x in indice]),np.array([labels[x] for x in indice]),np.array(classes)    except IOError as e:            print("I/O error({}): {} \nlike : {}".format(e.errno, e.strerror,path))def dataSetToNPY(path,dataSet_name,resize=True,resize_to=224,percentage=80):    """    Generate a image dataSet from a picture dataSets and save it in pny files for fast reading in teast and train    the picture dataSets must be in the same structure to generate also labels    example of pictureFolder: http://weegee.vision.ucmerced.edu/datasets/landuse.html    picture dataSets      |      |----------class-1      |        .   |-------image-1      |        .   |         .      |        .   |         .      |        .   |         .      |        .   |-------image-n      |        .      |-------class-n    :param str path: the path for picture dataSets folder    :param str dataSet_name: dataSets name for output file.npy    :param bool resize: choose resize the pictures or not    :param int resize_to: the new size of pictures    :param int or float percentage: how many pictures you want to get from this pictureFolder for training    :return: return dataset in npy files for fast Test and Train    """    try:        data,labels,classes = dataSetGenerator(path,resize,resize_to,100)        indice = np.random.permutation(len(data))        indice80 = indice[:int(len(data)*percentage/100)]        indice20 = indice[int(len(data)*percentage/100):]        np.save(dataSet_name+'_dataTrain.npy',[data[x] for x in indice80])        np.save(dataSet_name+'_labelsTrain.npy',[labels[x] for x in indice80])        np.save(dataSet_name+'_dataTest.npy',[data[x] for x in indice20])        np.save(dataSet_name+'_labelsTest.npy',[labels[x] for x in indice20])        np.save(dataSet_name+'_classes.npy',classes)    except IOError as e:            print("I/O error({}): {} \nlike : {}".format(e.errno, e.strerror,path))def picShow(data,labels,classes,just=None,predict=None,autoClose = False):    """    show a pictures and which class belong in one figure    :param list[int or float] data: list of picture that you need to show    :param list[int or float] labels: list of labels for this picture    :param list[str] classes: list of classes for this picture    :param int or None just: how much picture you want to show    :param list[str] or None predict: the list of probability of picture for each class    :param bool autoClose: auto close of plot window after seconds    :return: a figure contain this picture    """    fig =plt.figure()    if just is None: just = len(data)    for i in range(1, just+1):        true_out = classes[labels[i-1].argmax()] # the true class of picture i        sub = fig.add_subplot(np.ceil(np.sqrt(just)), np.rint(np.sqrt(just)), i)        title = "true: "+true_out        color = 'black'        if predict is not None:            classIndex = predict[i-1].argmax()            predict_out = classes[classIndex] #the predicted class of picture i            title += " predicted: "+ str(round(predict[i-1][classIndex]*100,2)) + " " + predict_out            color = 'green' if predict_out == true_out else 'red'        #sub.set_title(title,color=color, fontsize=7, fontweight='bold', y=-0.2-sub.get_ylim()[0])        sub.set_title(title,color=color, fontsize=7, fontweight='bold')        sub.axis('off')        sub.imshow(data[i-1], interpolation='nearest', aspect="auto")    if autoClose:        plt.show(0)        plt.pause(10)        plt.close()    else:        plt.show()def plotFiles(*path, xlabel='# epochs', ylabel='Error and Accu',autoClose = False):    """    Read data files and show it all in one graph figure    :rtype: object    :param list[str] or str path: the path for all the file who contain data to plot    :param str xlabel: x axes label name    :param str ylabel: y axes label name    :param bool autoClose: auto close of plot window after seconds    :return: Show data graph of files    """    for _ in path:        if isfile(_):            with open(_) as f:                plt.plot([float(i.strip('\x00')) for i in f.read().split('\n')[:-1] if float(i.strip('\x00'))], label=f.name.split("/")[-1])        else:            print("I/O error({}): {} \nlike : {}".format(IOError.errno, IOError.strerror,_))    plt.ylabel(ylabel)    plt.xlabel(xlabel)    plt.legend(loc='center left')    if autoClose:        plt.show(0)        plt.pause(10)        plt.close()    else:        plt.show()def saveArray(data,npy_path):    """    save array to file.npy    :param list[int] data: data array to save    :param str npy_path: the path of npy file for saving    """    try:        y = np.load(npy_path) if isfile(npy_path) else []        np.save(npy_path,np.append(y,data))        print("file saved", npy_path)        return npy_path    except IOError as e:        print("I/O error({}): {} \nlike : {}".format(e.errno, e.strerror,npy_path))def txtToNpy(txt_path,npy_path):    """    convert file.txt to file.npy    :param str txt_path: the path of text file to convert into npy    :param str npy_path: the path of npy file for saving    """    try:        with open(txt_path) as f:            saveArray([float(i) for i in f.read().split('\n')[:-1]],npy_path)    except IOError as e:        print("I/O error({}): {} \nlike : {}".format(e.errno, e.strerror,txt_path))def saveClasses(path,save_to,mode = 'w'):    """    read picture dataSets folder and save classes name in texty file    picture dataSets      |      |----------class-1      |        .   |-------image-1      |        .   |         .      |        .   |         .      |        .   |         .      |        .   |-------image-n      |        .      |-------class-n    :param str path: the path for picture dataSets folder    :param str save_to: path for the text file where you want save classes name    :param str mode: mode writing in text file so => a = append and w = write    :return None :    """    if mode != 'a': open(save_to, 'w')    for classe in listdir(path):        with open(save_to, 'a') as f:                    f.write(classe+'\n')def loadClasses(path):    """    load classes name from text file    :param str path:the path for text file contain the classes name    :return list[str]: retrun list of classes name    """    with open(path) as f:        return f.read().split('\n')[:-1]def getLabel(classe,classes):    """    Generate a label From lot of resources     - path for text file contain the classes name     - path for picture dataSets folder     - list contain classes name     - numpy array contain classes name    :param str classe :the name of classe    :param list[int] or list[str] or str or object[numpy.ndarray] classes: the path of text file or dataSet folder or list of integer or String    :return list[int]: label in probability form [0,0,1,0] of shape (1,n)    """    if type(classes) is str:        classes = loadClasses(classes) if isfile(classes) else listdir(classes)    elif isinstance(classes,np.ndarray):        classes =classes.tolist()    label = np.zeros(len(classes))    label[classes.index(classe)] = 1    return [label]def imread(path,resize=224):    """    read a picture    :param str path :path for pictures or folder to read    :param int resize: the new size of picture    :return list[float]: list of picture of shape (n,height,width,channel)    """    if isfile(path):        return [cv2.resize(cv2.imread(_, cv2.COLOR_BGR2RGB),(resize, resize)) for _ in [path]]    else:        return [cv2.resize(cv2.imread(path+"/"+_, cv2.COLOR_BGR2RGB),(resize, resize)) for _ in listdir(path)]def append(data,to):    """    save list() to text File    :param data: list() of Data to save    :param to: text file Were you wnat to save Data    """    for _ in data:        with open(to, 'a') as f:            f.write(_)

test_vgg16.py

@@ -0,0 +1,30 @@
+"""
+Simple tester for the vgg16
+"""
+import tensorflow as tf
+import vgg16.vgg16 as vgg16
+from dataSetGenerator import picShow
+from dataSetGenerator import imread
+from dataSetGenerator import getLabel
+from dataSetGenerator import loadClasses
+from os import getlogin
+from os import environ
+
+path = "C:/Users/{}/Desktop/UCMerced_LandUse/Images/golfcourse/golfcourse84.tif".format(getlogin())# picture path
+classe = path.split("/")[-2] # get classe name from path
+
+classes = loadClasses("Weights/UCMerced_LandUse.txt") # get classes name
+batch = imread(path) # read one picture
+label = getLabel(classe,classes)  # get the right label for the picture
+
+# with tf.Session(config=tf.ConfigProto(gpu_options=(tf.GPUOptions(per_process_gpu_memory_fraction=0.7)))) as sess:
+with tf.device('/cpu:0'):
+    with tf.Session(config=tf.ConfigProto(intra_op_parallelism_threads=int(environ['NUMBER_OF_PROCESSORS']))) as sess:
+        images = tf.placeholder(tf.float32, [None, 224, 224, 3])
+
+        vgg = vgg16.Vgg16("Weights/VGG16_21C.npy") #set the path
+        with tf.name_scope("content_vgg"):
+            vgg.build(images)
+
+        prob = sess.run(vgg.prob, {images: batch})
+        picShow(batch, label, classes, None, prob)

test_vgg19.py

@@ -0,0 +1,27 @@
+"""
+Simple tester for the vgg19
+"""
+import tensorflow as tf
+import vgg19.vgg19 as vgg19
+from dataSetGenerator import picShow
+from dataSetGenerator import imread
+from dataSetGenerator import getLabel
+from dataSetGenerator import loadClasses
+from os import getlogin
+
+path = "C:/Users/{}/Desktop/datasets/UCMerced_LandUse/Images/mediumresidential/mediumresidential84.tif".format(getlogin())# picture path
+classe = path.split("/")[-2] # get classe name from path
+classes = loadClasses("Datasets/UCMerced_LandUse.txt") # get classes name
+batch = imread(path) # read one picture
+label = getLabel(classe,classes)  # get the right label for the picture
+
+with tf.device('/device:cpu:0'):
+# with tf.device('/device:GPU:0'):
+    # with tf.Session(config=tf.ConfigProto(intra_op_parallelism_threads=int(environ['NUMBER_OF_PROCESSORS']))) as sess:
+        sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True))
+        images = tf.placeholder(tf.float32, [None, 224, 224, 3])
+        vgg = vgg19.Vgg19("Weights/VGG19_21C.npy") #set the path
+        with tf.name_scope("content_vgg"):
+                vgg.build(images)
+        prob = sess.run(vgg.prob, {images: batch})
+        picShow(batch, label, classes, None, prob)

train_vgg16.py

@@ -0,0 +1,69 @@
+"""
+Simple tester for the vgg16_trainable
+"""
+from os import environ
+
+import numpy as np
+import tensorflow as tf
+
+from dataSetGenerator import loadClasses
+from dataSetGenerator import picShow
+from vgg16 import vgg16_trainable as vgg16
+
+#path= "C:/Users/{}/Desktop/UCMerced_LandUse/Images/".format(getlogin())
+#batch, labels, classes = dataSetGenerator(path,True,224,80)
+batch = np.load("DataSets/UCMerced_LandUse_dataTrain.npy")
+labels = np.load("DataSets/UCMerced_LandUse_labelsTrain.npy")
+classes = loadClasses("DataSets/UCMerced_LandUse.txt")
+# with tf.device('/device:GPU:0'):
+#     with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True)) as sess:
+with tf.device('/cpu:0'):
+    with tf.Session(config=tf.ConfigProto(intra_op_parallelism_threads=int(environ['NUMBER_OF_PROCESSORS']))) as sess:
+
+
+        images = tf.placeholder(tf.float32, [None, 224, 224, 3])
+        true_out = tf.placeholder(tf.float32, [None, len(classes)])
+        train_mode = tf.placeholder(tf.bool)
+
+        vgg = vgg16.Vgg16('Weights/VGG16_21C.npy')
+        vgg.build(images,train_mode)
+
+        # print number of variables used: 143667240 variables, i.e. ideal size = 548MB
+        print('number of variables used:',vgg.get_var_count())
+
+        sess.run(tf.global_variables_initializer())
+
+        # test classification
+        prob = sess.run(vgg.prob, feed_dict={images: batch[:10], train_mode: False})
+        picShow(batch[:10],labels[:10], classes, None, prob,True)
+        # simple 1-step training
+        cost = tf.reduce_sum((vgg.prob - true_out) ** 2)
+        train = tf.train.GradientDescentOptimizer(0.0001).minimize(cost)
+
+        correct_prediction = tf.equal(tf.argmax(prob), tf.argmax(true_out))
+        acc = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
+
+        batch_size = 10
+        epochs = 30
+        batche_num = batch.shape[0]
+        for _ in range(epochs):
+                print("*******************  ", _, "  *******************")
+                indice = np.random.permutation(batche_num)
+                for i in range(int(batche_num/batch_size)):
+                    min_batch = indice[i*batch_size:(i+1)*batch_size]
+                    cur_cost, cur_train,cur_acc= sess.run([cost, train,acc], feed_dict={images: batch[min_batch], true_out: labels[min_batch], train_mode: True})
+                    print("Iteration %d loss:\n%s" % (i, cur_cost))
+                    with open('Data/cost16_21C.txt', 'a') as f:
+                        f.write(str(cur_cost)+'\n')
+                    with open('Data/acc16_21C.txt', 'a') as f:
+                        f.write(str(cur_acc)+'\n')
+
+                # test save
+                vgg.save_npy(sess, 'Weights/VGG16_21C.npy')
+
+        # test classification again, should have a higher probability about tiger
+        prob = sess.run(vgg.prob, feed_dict={images: batch[:10], train_mode: False})
+        picShow(batch[:10],labels[:10], classes,None,prob)
+
+        # import subprocess
+        # subprocess.call(["shutdown", "/s"])