diff --git a/Ch11/Android/MLP.py b/Ch11/Android/MLP.py new file mode 100644 index 0000000..e501bb9 --- /dev/null +++ b/Ch11/Android/MLP.py @@ -0,0 +1,538 @@ +import numpy +import itertools +import sys +import time + +class MLP: + trained_ann = {} + + def train(x, y, net_arch, max_iter=5000, tolerance=0.0000001, learning_rate=0.001, activation="sigmoid", GD_type="stochastic", + debug=False): + """ + train(x, y, w, max_iter, tolerance, learning_rate, activation="sigmoid", debug=False) + Training artificial neural network (ANN) using gradient descent (GD). + + Inputs: + x: Training data inputs. + y: Training data outputs. + net_arch: Network architecture defining number of inputs/outputs and number of neurons per each hidden layer. + max_iter: Maximum number of iterations for training the network. + tolerance: The desired minimum difference between predicted and target values. The network tries to reach this value if possible. + learning_rate: Learning rate to adapt the network weights while learning. + activation: Activation function to be used for building the hidden layers. Only one activation function can be used for all hidden layers. + GD_type: Can be either stochastic or batch. + debug: If True, informational messages are printed while training. Note that this increases the training time. + + Outputs: + trained_ann: A dictionary representing the trained neural network. It holds all information necessary for restoring the trained network. + """ + + # Number of inputs, number of neurons per each hidden layer, number of output neurons + # The network architecture defined in the net_arch list does not include the number of inputs nor the number of outputs. The lines below updates the net_arch list to include these 2 numbers. + in_size = x.shape[1] + if y.ndim == 1: + out_size = 1 + else: + out_size = y.shape[1] + + net_arch = [in_size] + net_arch # Number of inputs is added at the beginning of the list. + net_arch = net_arch + [out_size] # Number of outputs is added at the end of the list. + net_arch = numpy.array(net_arch) # Converting net_arch from a list to a NumPy array. + + net_arch, w_initial, b_initial = MLP.initialize_weights(net_arch) + w = w_initial.copy() + b = b_initial.copy() + + start_time = time.time() + + network_error = sys.maxsize + if (debug == True): + print("Training Started") + + if activation == "sigmoid": + activation_func = MLP.sigmoid + elif activation == "relu": + activation_func = MLP.relu + else: + activation_func = MLP.sigmoid + print("Sorry. Only Sigmoid and ReLU are supported at the current time. Sigmoid will still be used.") + + # pred_target_diff holds the difference between the predicted output and the target output to be compared by the tolerance. + pred_target_diff = sys.maxsize + + w_temp = [] + b_temp = [] + for k1 in range(len(w)): + w_temp.append(numpy.zeros(w[k1].shape)) + b_temp.append(numpy.zeros(b[k1].shape)) + + iter_num = 0 + while (iter_num < max_iter and pred_target_diff > tolerance): + network_error = 0 # Declare a variable to hold the network error across all samples. + pred_target_diff = 0 # Declare a variable to hold the difference between the targets and predictions across all samples. + all_samples_predictions = [] + for sample_idx in range(x.shape[0]): + curr_sample_x = x[sample_idx] # inputs of current sample. + curr_sample_y = y[sample_idx] # Target of current sample. + + sop_activ_mul = MLP.forward_path(curr_sample_x, w, b, activation_func) + + prediction = sop_activ_mul[-1][-1] # Predictions for current sample. + all_samples_predictions.append(prediction) + + sample_pred_target_diff = numpy.sum(numpy.abs(curr_sample_y - prediction)) + pred_target_diff = pred_target_diff + sample_pred_target_diff + + sample_error = MLP.error(prediction, curr_sample_y) + network_error = network_error + sample_error + + if GD_type == "stochastic": # For stochastic gradient descent. + w, b = MLP.backward_pass(curr_sample_x, curr_sample_y, w, b, net_arch, sop_activ_mul, prediction, out_size, activation, learning_rate) + elif GD_type=="batch": # For batch gradient descent. + w_sample, b_sample = MLP.backward_pass(curr_sample_x, curr_sample_y, w, b, net_arch, sop_activ_mul, prediction, out_size, activation, learning_rate) + + for k1 in range(len(w)): + w_temp[k1] = w_temp[k1] + w_sample[k1] + b_temp[k1] = b_temp[k1] + b_sample[k1] + else: + print("Sorry. Only stochastic and batch gradient descent are supported. Stochastic gradient descent will be used.") + w, b = MLP.backward_pass(curr_sample_x, curr_sample_y, w, b, net_arch, sop_activ_mul, prediction, out_size, activation, learning_rate) + + if GD_type == "batch": + for k1 in range(len(w)): + w_temp[k1] = w_temp[k1]/x.shape[0] + b_temp[k1] = b_temp[k1]/x.shape[0] + + w = w_temp.copy() + b = b_temp.copy() + + w_temp = [] + b_temp = [] + for k1 in range(len(w)): + w_temp.append(numpy.zeros(w[k1].shape)) + b_temp.append(numpy.zeros(b[k1].shape)) + + if (debug == True): + print("\nIteration : ", iter_num, "\nError : ", network_error) + iter_num = iter_num + 1 + + if (debug == True): + print("Training Finished") + end_time = time.time() + + training_time = end_time - start_time + + MLP.trained_ann["w"] = w + MLP.trained_ann["b"] = b + MLP.trained_ann["max_iter"] = max_iter + MLP.trained_ann[ + "elapsed_iter"] = iter_num + MLP.trained_ann["tolerance"] = tolerance + MLP.trained_ann["activation"] = activation + MLP.trained_ann["learning_rate"] = learning_rate + MLP.trained_ann["initial_w"] = w_initial + MLP.trained_ann["net_arch"] = net_arch + MLP.trained_ann["num_hidden_layers"] = net_arch.shape[0] - 2 + MLP.trained_ann["training_time_sec"] = training_time + MLP.trained_ann["network_error"] = network_error + MLP.trained_ann["in_size"] = in_size + MLP.trained_ann["out_size"] = out_size + + return MLP.trained_ann + + def initialize_weights(net_arch): + """ + initialize_weights(net_arch) + Initializing the neural network weights. + + Inputs: + net_arch: Network architecture defining number of inputs/outputs and number of neurons per each hidden layer. The user input might need some refine. + + Outputs: + net_arch: The final refined network architecture [if neccessary]. + w: The initial weights. + b: Initial bias values. + """ + + w = [] + w_temp = [] + b = [] + for layer_counter in numpy.arange(net_arch.shape[0] - 1): + rand_array = numpy.random.uniform(low=0.0, high=0.1, size=net_arch[layer_counter + 1]) + b.append(rand_array) + for neuron_counter in numpy.arange(net_arch[layer_counter + 1]): + rand_array = numpy.random.uniform(low=0.0, high=0.1, size=net_arch[layer_counter]) + w_temp.append(rand_array) + w_temp = numpy.array(w_temp) + w.append(w_temp) + w_temp = [] + + return net_arch, w, b + + def forward_path(x, w, b, activation_func): + """ + forward_path(x, w, activation_func) + Implementation of the forward pass. + + Inputs: + x: training inputs + w: current set of weights. + b: current bias values. + activation_func: A string representing the activation function used. + Outputs: + sop_activ_mul: An array representing the outputs of each layer + The sop_activ_mul array has number of rows equal to number of layers. For each layer, 2 things are calculated: + 1) SOP between inputs and weights. + 2) Activation output for SOP. + For example, If there are 4 layers, then the shape of the sop_activ_mul array is (4, 2). + sop_activ_mul[x][0]: SOP in layer x. + sop_activ_mul[x][1]: Activation in layer x. + """ + sop_activ_mul = [] + + sop_activ_mul_temp = [] + curr_multiplicand = x + for layer_num in range(len(w)): + sop_temp = numpy.matmul(curr_multiplicand, w[layer_num].T) + b[layer_num] + activ_temp = activation_func(sop_temp) + + curr_multiplicand = activ_temp + + sop_activ_mul_temp.append([sop_temp, activ_temp]) + sop_activ_mul.extend(sop_activ_mul_temp) + sop_activ_mul_temp = [] + return sop_activ_mul + + def sigmoid(sop): + """ + sigmoid(sop) + Implementation of the sigmoid activation function. + + Inputs: + sop: A single value representing the sum of products between the layer inputs and its weights. + + Outputs: + A single value representing the output of the sigmoid. + """ + return 1.0 / (1 + numpy.exp(-1 * sop)) + + def relu(x): + """ + relu(sop) + Implementation of the rectified linear unit (ReLU) activation function. + + Inputs: + sop: A single value representing the sum of products between the layer inputs and its weights. + + Outputs: + A single value representing the output of the relu. + """ + return numpy.maximum(0, x) + + def error(predicted, target): + """ + error(predicted, target) + Preduction error in the current iteration. + + Inputs: + predicted: The predictions of the network using its current parameters. + target: The correct outputs that the network should predict. + Outputs: + Error. + """ + return numpy.power(predicted - target, 2) + + def backward_pass(x, y, w, b, net_arch, sop_activ_mul, prediction, num_outputs, activation, learning_rate): + """ + backward_pass(x, y, w, net_arch, sop_activ_mul, prediction, num_outputs, activation, learning_rate) + Implementation of the backward pass for training the neural network using gradient descent. + + Inputs: + x: Training inputs. Used for calcualting the derivative for the weights between the input layer and the hidden layer. + y: Training data outputs. + w: Current weights which are to be updated during the backward pass using the gradient descent. + b: Bias values to be updated during the backward pass using the gradient descent. + net_arch: A NumPy array defining the network archietcture defining the number of neurons in all layers. It is used here to know the number of neurons per each layer. + sop_activ_mul: An array holding all calculations during the forward pass. This includes the sum of products between the inputs and weights of each layer and the results of the activation functions. + prediction: The predicted outputs of the network using the current weights. + num_outputs: Number of outputs per sample. + learning_rate: Learning rate to adapt the network weights while learning. + + Outputs: + w: The updated weights using gradient descent. + """ + if activation == "sigmoid": + activation_sop_deriv = MLP.sigmoid_sop_deriv + elif activation == "relu": + activation_sop_deriv = MLP.relu_sop_deriv + else: + activation_sop_deriv = MLP.sigmoid_sop_deriv + print("Sorry. Only Sigmoid and ReLU are supported at the current time. Sigmoid will still be used.") + + g1 = MLP.error_predicted_deriv(prediction, y) + + g2 = activation_sop_deriv(sop_activ_mul[-1][0]) + + output_layer_derivs = g2 * g1 + + layer_weights_derivs = [] + layer_weights_grads = [] + + if net_arch.shape[0] == 2: + layer_weights_derivs = [MLP.sop_w_deriv(sop_activ_mul[-1][1])] + layer_weights_derivs + layer_weights_grads = [layer_weights_derivs[0] * output_layer_derivs] + layer_weights_grads + + for out_neuron_idx in range(num_outputs): + w[-1][out_neuron_idx] = w[-1][out_neuron_idx] - layer_weights_grads[0][out_neuron_idx] * learning_rate + b[-1][out_neuron_idx] = b[-1][out_neuron_idx] - layer_weights_derivs[0] * output_layer_derivs[out_neuron_idx] * learning_rate + + MLP.trained_ann["derivative_chain"] = output_layer_derivs + MLP.trained_ann["weights_derivatives"] = layer_weights_derivs + MLP.trained_ann["weights_gradients"] = layer_weights_grads + + return w, b + + w_old = w.copy() + + layer_weights_derivs = [MLP.sop_w_deriv(sop_activ_mul[-1][1])] + layer_weights_derivs + + layer_weights_grads = [layer_weights_derivs[0] * output_layer_derivs] + layer_weights_grads + + # Updating the weights between the last hidden layer and the output neurons. + for out_neuron_idx in range(num_outputs): + w[-1][out_neuron_idx] = w[-1][out_neuron_idx] - layer_weights_grads[0][out_neuron_idx] * learning_rate + b[-1][out_neuron_idx] = b[-1][out_neuron_idx] - layer_weights_derivs[0] * output_layer_derivs[out_neuron_idx] * learning_rate + + SOPs_ACTIVs_deriv_individual = [] + deriv_chain_final = [] + + # Derivatives of the neurons between the last hidden layer and the output neurons. + SOPs_ACTIVs_deriv_individual.append(output_layer_derivs) + deriv_chain_final.append(output_layer_derivs) + + ############# GENERIC GRADIENT DESCENT ############# + # The idea is to prepare the individual derivatives of all neurons in all layers. These derivatives include: + # -) Derivative of activation (output) to SOP (input) + # -) Derivative of SOP (input) to activation (output) + # These derivative not include the following: + # -) Derivative of SOP (output) to the weight. It will be calculated later. + # Using the chain rule, combinations are created from these individual derivatives. + # X) The total derivative at a given neuron is the mean of products of all these combinations. + # Y) For every neuron at a given layer, the derivative of the SOP (output) to the weight is calculated. + # The gradient to update a given weight is the product between the mean (step X) and the weight derivative (step Y). + + # Derivatives of all other layers + # If the network has no or just 1 hidden layer, this loop will not be executed. # It works when there are more than 1 hidden layer. + curr_idx = -1 + for curr_lay_idx in range(len(w)-2, -1, -1): + # ACTIVs_deriv are identical for all neurons in a given layer. If a layer has 5 neurons, then the length of ACTIVs_deriv is 5. + # But SOPs_deriv returns 5 values for every neuron. Thus, there will be an array of 5 rows, one row for one neuron. + SOPs_deriv = MLP.sop_w_deriv(w_old[curr_lay_idx + 1]) + ACTIVs_deriv = activation_sop_deriv(sop_activ_mul[curr_lay_idx][0]) + + temp = [] + for curr_neuron_idx in range(net_arch[curr_idx]): + temp.append(ACTIVs_deriv * SOPs_deriv[curr_neuron_idx]) + curr_idx = curr_idx - 1 + temp = numpy.array(temp) + # Individual Derivatives of the Network Except the Weights Derivatives + SOPs_ACTIVs_deriv_individual = [temp] + SOPs_ACTIVs_deriv_individual + + temp2 = MLP.deriv_chain_prod(temp[0], deriv_chain_final[-1]) + for neuron_calc_derivs_idx in range(1, temp.shape[0]): + # Only last element in the deriv_chain_final array is used for calculating the chain. + # This reuses the previous calculations because some chains were calculated previously. + temp2 = temp2 + MLP.deriv_chain_prod(temp[neuron_calc_derivs_idx], deriv_chain_final[-1]) + deriv_chain_final.append(temp2) + + #### Calculate WEIGHTS Derivatives and Gradients + # Index 1 means output of activation function for all neurons per layer. + # At layer with index i, the derivs (layer_weights_derivs) and grads (layer_weights_grads) of its weights are at index i. + # For example, layer indexed i=0 has its derivs (layer_weights_derivs) and grads (layer_weights_grads) in index 0. + if curr_lay_idx == 0: + layer_weights_derivs = [MLP.sop_w_deriv(x)] + layer_weights_derivs + else: + layer_weights_derivs = [MLP.sop_w_deriv(sop_activ_mul[curr_lay_idx - 1][1])] + layer_weights_derivs + layer_weights_grads = [layer_weights_derivs[0]] + layer_weights_grads + + # Derivatives of the Entire Network (Except Weights Derivatives) Chains Following the Chain Rule. + deriv_chain_final = numpy.array(deriv_chain_final) + # Derivatives of the Entire Nework including the Weights Derivatives + layer_weights_derivs = numpy.array(layer_weights_derivs) + # Gradients of the Entire Network Weights + layer_weights_grads = numpy.array(layer_weights_grads) + + deriv_chain_final = numpy.flip(deriv_chain_final) + + MLP.trained_ann["derivative_chain"] = deriv_chain_final + MLP.trained_ann["weights_derivatives"] = layer_weights_derivs + MLP.trained_ann["weights_gradients"] = layer_weights_grads + + #### Updating Weights of All Layers Except Last Layer Because it was Updated Previously + for layer_idx in range(len(w) - 1): + w[layer_idx] = MLP.update_weights(w[layer_idx], layer_weights_grads[layer_idx], + deriv_chain_final[layer_idx], learning_rate) + b[layer_idx] = MLP.update_bias(b[layer_idx], deriv_chain_final[layer_idx], learning_rate) + return w, b + + def error_predicted_deriv(predicted, target): + """ + error_predicted_deriv(predicted, target) + Derivative of the error to the predicted value. + + Inputs: + predicted: The predictions of the network using its current parameters. + target: The correct outputs that the network should predict. + + Outputs: + Derivative of the error to the predicted value. + """ + return 2 * (predicted - target) + + def sigmoid_sop_deriv(sop): + """ + sigmoid_sop_deriv(sop) + Calculating the derivative of the sigmoid to the sum of products. + + Inputs: + sop: Sum of products. + + Outputs: + Derivative of the sum of products to sigmoid. + """ + return MLP.sigmoid(sop) * (1.0 - MLP.sigmoid(sop)) + + def relu_sop_deriv(sop): + """ + relu_sop_deriv(sop) + Calculating the derivative of the relu to the sum of products. + + Inputs: + sop: Sum of products. + + Outputs: + Derivative of the sum of products to relu. + """ + sop[sop <= 0] = 0.0 + sop[sop > 0] = 1.0 + return sop + + def sop_w_deriv(x): + """ + sop_w_deriv(x) + Derivative of the sum of products to the weights. + + Inputs: + x: inputs to the current layer. + + Outputs: + Derivative of the sum of products to the weights. + """ + return x + + def deriv_chain_prod(layer_derivs, previous_lay_derivs_chains): + """ + deriv_chain_prod(derivs_arrs) + Derivative chains of a given layer. + + Inputs: + layer_derivs: Derivatives of the current layer. + previous_lay_derivs_chains: Derivatives chains of the previous layer. + Outputs: + deriv_chain_prod_sum: A NumPy array representing the sum of the derivatives products in all chains. + """ + derivs_arrs = [layer_derivs] + [[previous_lay_derivs_chains]] + deriv_combinations = numpy.array(list(itertools.product(*derivs_arrs))) + num_products_in_layer = deriv_combinations.shape[0] + num_neurons_in_layer = len(derivs_arrs[0]) + num_chains_for_neuron = numpy.uint32(num_products_in_layer / num_neurons_in_layer) + + # print("\n# of Products in Layer : ", num_products_in_layer, + # "\n# of Neurons in Layer : ", num_neurons_in_layer, + # "\n# of Products per Neuron : ", num_chains_for_neuron) + + deriv_chain_prod_sum = [] + for neuron_idx in range(num_neurons_in_layer): + start_idx = neuron_idx * num_chains_for_neuron + end_idx = (neuron_idx + 1) * num_chains_for_neuron + deriv_chain = deriv_combinations[start_idx:end_idx] + deriv_chain_prod = numpy.prod(deriv_chain, axis=1) + # When there are more than 1 chain to reach a neuron, the sum of all chains is calculated and returned as a single value. + deriv_chain_prod_sum.append(numpy.concatenate(deriv_chain_prod).sum()) + # Update weight according to chains product sum (deriv_chain_prod_sum) + deriv_chain_prod_sum = numpy.array(deriv_chain_prod_sum) + return deriv_chain_prod_sum + + def update_weights(weights, layer_weights_grads, deriv_chain_final, learning_rate): + """ + update_weights(weights, gradients, learning_rate) + Updating the weights based on the calcualted gradients. + + Inputs: + weights: Weights of a layer tbe updated. + layer_weights_grads: Gradient of the current layer for updating its weights. + deriv_chain_final: Chains of derivatives from the layers higher than the current layer. + learning_rate: Learnign rate. + + Outputs: + Updated weights. + """ + for neuron_idx in range(weights.shape[0]): + weights[neuron_idx] = weights[neuron_idx] - layer_weights_grads * deriv_chain_final[ + neuron_idx] * learning_rate + return weights + + def update_bias(b, deriv_chain_final, learning_rate): + """ + update_weights(weights, gradients, learning_rate) + Updating the weights based on the calcualted gradients. + + Inputs: + b: Bias. + deriv_chain_final: Chains of derivatives from the layers higher than the current layer. + learning_rate: Learnign rate. + + Outputs: + Updated b. + """ + for neuron_idx in range(b.shape[0]): + b[neuron_idx] = b[neuron_idx] - deriv_chain_final[neuron_idx] * learning_rate + return b + + def predict(trained_ann, x): + """ + predict(trained_ann, x) + Making prediction for a new sample. + + Inputs: + trained_ann: A dictionary representing trained MLP. + x: The new sample. + Outputs: + prediction: The predicted output for the current sample. + """ + in_size = trained_ann["in_size"] + if x.ndim == 1: + if x.shape[0] != in_size: + print("Input shape ", x.shape[0], " does not match the expected network input shape ", in_size) + return + else: + if x.shape[1] != in_size: + print("Input shape ", x.shape[1], " does not match the expected network input shape ", in_size) + return + + w = trained_ann["w"] + b = trained_ann["b"] + + activation = trained_ann["activation"] + if activation == "sigmoid": + activation_func = MLP.sigmoid + elif activation == "relu": + activation_func = MLP.relu + else: + activation_func = MLP.sigmoid + print("Sorry. Only Sigmoid and ReLU are supported at the current time. Sigmoid will still be used.") + + sop_activ_mul = MLP.forward_path(x, w, b, activation_func) + prediction = sop_activ_mul[-1][1] + + return prediction \ No newline at end of file diff --git a/Ch11/Android/buildozer.spec b/Ch11/Android/buildozer.spec new file mode 100644 index 0000000..48ab2b1 --- /dev/null +++ b/Ch11/Android/buildozer.spec @@ -0,0 +1,306 @@ +[app] + +# (str) Title of your application +title = PyNeuNet + +# (str) Package name +package.name = myapp + +# (str) Package domain (needed for android/ios packaging) +package.domain = org.test + +# (str) Source code where the main.py live +source.dir = . + +# (list) Source files to include (let empty to include all the files) +source.include_exts = py,png,jpg,kv,atlas,txt,npy + +# (list) List of inclusions using pattern matching +#source.include_patterns = assets/*,images/*.png + +# (list) Source files to exclude (let empty to not exclude anything) +#source.exclude_exts = spec + +# (list) List of directory to exclude (let empty to not exclude anything) +#source.exclude_dirs = tests, bin + +# (list) List of exclusions using pattern matching +#source.exclude_patterns = license,images/*/*.jpg + +# (str) Application versioning (method 1) +version = 0.1 + +# (str) Application versioning (method 2) +# version.regex = __version__ = ['"](.*)['"] +# version.filename = %(source.dir)s/main.py + +# (list) Application requirements +# comma separated e.g. requirements = sqlite3,kivy +requirements = kivy,numpy,kivymd + +# (str) Custom source folders for requirements +# Sets custom source for any requirements with recipes +# requirements.source.kivy = ../../kivy + +# (list) Garden requirements +#garden_requirements = + +# (str) Presplash of the application +#presplash.filename = %(source.dir)s/data/presplash.png +presplash.filename = %(source.dir)s/icon.png + +# (str) Icon of the application +#icon.filename = %(source.dir)s/data/icon.png +icon.filename = %(source.dir)s/icon.png +# (str) Supported orientation (one of landscape, sensorLandscape, portrait or all) +orientation = portrait + +# (list) List of service to declare +#services = NAME:ENTRYPOINT_TO_PY,NAME2:ENTRYPOINT2_TO_PY + +# +# OSX Specific +# + +# +# author = © Copyright Info + +# change the major version of python used by the app +osx.python_version = 3 + +# Kivy version to use +osx.kivy_version = 1.9.1 + +# +# Android specific +# + +# (bool) Indicate if the application should be fullscreen or not +fullscreen = 0 + +# (string) Presplash background color (for new android toolchain) +# Supported formats are: #RRGGBB #AARRGGBB or one of the following names: +# red, blue, green, black, white, gray, cyan, magenta, yellow, lightgray, +# darkgray, grey, lightgrey, darkgrey, aqua, fuchsia, lime, maroon, navy, +# olive, purple, silver, teal. +#android.presplash_color = #FFFFFF + +# (list) Permissions +android.permissions = READ_EXTERNAL_STORAGE,WRITE_EXTERNAL_STORAGE + +# (int) Target Android API, should be as high as possible. +#android.api = 27 + +# (int) Minimum API your APK will support. +#android.minapi = 21 + +# (int) Android SDK version to use +#android.sdk = 20 + +# (str) Android NDK version to use +#android.ndk = 17c + +# (int) Android NDK API to use. This is the minimum API your app will support, it should usually match android.minapi. +#android.ndk_api = 21 + +# (bool) Use --private data storage (True) or --dir public storage (False) +#android.private_storage = True + +# (str) Android NDK directory (if empty, it will be automatically downloaded.) +#android.ndk_path = + +# (str) Android SDK directory (if empty, it will be automatically downloaded.) +#android.sdk_path = + +# (str) ANT directory (if empty, it will be automatically downloaded.) +#android.ant_path = + +# (bool) If True, then skip trying to update the Android sdk +# This can be useful to avoid excess Internet downloads or save time +# when an update is due and you just want to test/build your package +# android.skip_update = False + +# (bool) If True, then automatically accept SDK license +# agreements. This is intended for automation only. If set to False, +# the default, you will be shown the license when first running +# buildozer. +# android.accept_sdk_license = False + +# (str) Android entry point, default is ok for Kivy-based app +#android.entrypoint = org.renpy.android.PythonActivity + +# (list) Pattern to whitelist for the whole project +#android.whitelist = + +# (str) Path to a custom whitelist file +#android.whitelist_src = + +# (str) Path to a custom blacklist file +#android.blacklist_src = + +# (list) List of Java .jar files to add to the libs so that pyjnius can access +# their classes. Don't add jars that you do not need, since extra jars can slow +# down the build process. Allows wildcards matching, for example: +# OUYA-ODK/libs/*.jar +#android.add_jars = foo.jar,bar.jar,path/to/more/*.jar + +# (list) List of Java files to add to the android project (can be java or a +# directory containing the files) +#android.add_src = + +# (list) Android AAR archives to add (currently works only with sdl2_gradle +# bootstrap) +#android.add_aars = + +# (list) Gradle dependencies to add (currently works only with sdl2_gradle +# bootstrap) +#android.gradle_dependencies = + +# (list) Java classes to add as activities to the manifest. +#android.add_activites = com.example.ExampleActivity + +# (str) OUYA Console category. Should be one of GAME or APP +# If you leave this blank, OUYA support will not be enabled +#android.ouya.category = GAME + +# (str) Filename of OUYA Console icon. It must be a 732x412 png image. +#android.ouya.icon.filename = %(source.dir)s/data/ouya_icon.png + +# (str) XML file to include as an intent filters in tag +#android.manifest.intent_filters = + +# (str) launchMode to set for the main activity +#android.manifest.launch_mode = standard + +# (list) Android additional libraries to copy into libs/armeabi +#android.add_libs_armeabi = libs/android/*.so +#android.add_libs_armeabi_v7a = libs/android-v7/*.so +#android.add_libs_arm64_v8a = libs/android-v8/*.so +#android.add_libs_x86 = libs/android-x86/*.so +#android.add_libs_mips = libs/android-mips/*.so + +# (bool) Indicate whether the screen should stay on +# Don't forget to add the WAKE_LOCK permission if you set this to True +#android.wakelock = False + +# (list) Android application meta-data to set (key=value format) +#android.meta_data = + +# (list) Android library project to add (will be added in the +# project.properties automatically.) +#android.library_references = + +# (list) Android shared libraries which will be added to AndroidManifest.xml using tag +#android.uses_library = + +# (str) Android logcat filters to use +#android.logcat_filters = *:S python:D + +# (bool) Copy library instead of making a libpymodules.so +#android.copy_libs = 1 + +# (str) The Android arch to build for, choices: armeabi-v7a, arm64-v8a, x86, x86_64 +android.arch = armeabi-v7a + +# +# Python for android (p4a) specific +# + +# (str) python-for-android fork to use, defaults to upstream (kivy) +#p4a.fork = kivy + +# (str) python-for-android branch to use, defaults to master +#p4a.branch = master + +# (str) python-for-android git clone directory (if empty, it will be automatically cloned from github) +#p4a.source_dir = + +# (str) The directory in which python-for-android should look for your own build recipes (if any) +#p4a.local_recipes = + +# (str) Filename to the hook for p4a +#p4a.hook = + +# (str) Bootstrap to use for android builds +# p4a.bootstrap = sdl2 + +# (int) port number to specify an explicit --port= p4a argument (eg for bootstrap flask) +#p4a.port = + + +# +# iOS specific +# + +# (str) Path to a custom kivy-ios folder +#ios.kivy_ios_dir = ../kivy-ios +# Alternately, specify the URL and branch of a git checkout: +ios.kivy_ios_url = https://github.com/kivy/kivy-ios +ios.kivy_ios_branch = master + +# Another platform dependency: ios-deploy +# Uncomment to use a custom checkout +#ios.ios_deploy_dir = ../ios_deploy +# Or specify URL and branch +ios.ios_deploy_url = https://github.com/phonegap/ios-deploy +ios.ios_deploy_branch = 1.7.0 + +# (str) Name of the certificate to use for signing the debug version +# Get a list of available identities: buildozer ios list_identities +#ios.codesign.debug = "iPhone Developer: ()" + +# (str) Name of the certificate to use for signing the release version +#ios.codesign.release = %(ios.codesign.debug)s + + +[buildozer] + +# (int) Log level (0 = error only, 1 = info, 2 = debug (with command output)) +log_level = 2 + +# (int) Display warning if buildozer is run as root (0 = False, 1 = True) +warn_on_root = 1 + +# (str) Path to build artifact storage, absolute or relative to spec file +# build_dir = ./.buildozer + +# (str) Path to build output (i.e. .apk, .ipa) storage +# bin_dir = ./bin + +# ----------------------------------------------------------------------------- +# List as sections +# +# You can define all the "list" as [section:key]. +# Each line will be considered as a option to the list. +# Let's take [app] / source.exclude_patterns. +# Instead of doing: +# +#[app] +#source.exclude_patterns = license,data/audio/*.wav,data/images/original/* +# +# This can be translated into: +# +#[app:source.exclude_patterns] +#license +#data/audio/*.wav +#data/images/original/* +# + + +# ----------------------------------------------------------------------------- +# Profiles +# +# You can extend section / key with a profile +# For example, you want to deploy a demo version of your application without +# HD content. You could first change the title to add "(demo)" in the name +# and extend the excluded directories to remove the HD content. +# +#[app@demo] +#title = My Application (demo) +# +#[app:source.exclude_patterns@demo] +#images/hd/* +# +# Then, invoke the command line with the "demo" profile: +# +#buildozer --profile demo android debug diff --git a/Ch11/Android/data_inputs.pkl b/Ch11/Android/data_inputs.pkl new file mode 100644 index 0000000..5e64706 Binary files /dev/null and b/Ch11/Android/data_inputs.pkl differ diff --git a/Ch11/Android/data_outputs.pkl b/Ch11/Android/data_outputs.pkl new file mode 100644 index 0000000..5fef9c3 Binary files /dev/null and b/Ch11/Android/data_outputs.pkl differ diff --git a/Ch11/Android/icon.png b/Ch11/Android/icon.png new file mode 100644 index 0000000..abedf4c Binary files /dev/null and b/Ch11/Android/icon.png differ diff --git a/Ch11/Android/main.py b/Ch11/Android/main.py new file mode 100644 index 0000000..0cb9b20 --- /dev/null +++ b/Ch11/Android/main.py @@ -0,0 +1,250 @@ +import kivymd.app +import kivymd.uix.menu +import kivymd.uix.button +import threading +import MLP +import kivy.uix.screenmanager +import pickle + +class MainScreen(kivy.uix.screenmanager.Screen): + pass + +class InputFileChooserScreen(kivy.uix.screenmanager.Screen): + pass + +class OutputFileChooserScreen(kivy.uix.screenmanager.Screen): + pass + +class NeuralApp(kivymd.app.MDApp): + + def select_file(self, screen_num, *args): + path = self.root.screens[screen_num].ids.file_chooser.selection + if screen_num == 1: + self.input_file_select_path(path=path) + else: + self.output_file_select_path(path=path) + + def on_start(self): + +# with open('data_inputs.pkl', 'wb') as out: +# pickle.dump(numpy.array([[0.1, 0.4, 4.1, 4.3, 1.8, 2.0, 0.01, 0.9, 3.8, 1.6]]), out) + +# with open('data_outputs.pkl', 'wb') as out: +# pickle.dump(numpy.array([[0.2]]), out) + + self.x = None + self.y = None + self.net_arch = None + self.max_iter = None + self.tolerance = None + self.learning_rate = None + self.activation = None + self.GD_type = None + self.debug = False + + self.activation_menu = kivymd.uix.menu.MDDropdownMenu(caller=self.root.screens[0].ids.activation_menu, + items=[{"text": "sigmoid"}, {"text": "relu"}], + callback=self.activation_menu_callback, + width_mult=4) + + self.gdtype_menu = kivymd.uix.menu.MDDropdownMenu(caller=self.root.screens[0].ids.gdtype_menu, + items=[{"text": "stochastic"}, {"text": "batch"}], + callback=self.gdtype_menu_callback, + width_mult=4) + + def activation_menu_callback(self, activation_menu): + self.root.screens[0].ids.activation_menu.text = activation_menu.text + self.activation = activation_menu.text + self.activation_menu.dismiss() + + def gdtype_menu_callback(self, gdtype_menu): + self.root.screens[0].ids.gdtype_menu.text = gdtype_menu.text + self.GD_type = gdtype_menu.text + self.gdtype_menu.dismiss() + + def input_file_select_path(self, path): + if len(path) == 0: + self.root.screens[1].ids.select_file_label.text = "Error: No file selected." + self.root.screens[0].ids.input_data_file_button.text = "Input Data" + return + elif path[0][-4:] == ".pkl": + # kivymd.toast.toast(path) + with open(path[0], 'rb') as inp: + self.x = pickle.load(inp) + # self.x = numpy.load(path[0]) + if self.x.ndim != 2: + self.root.screens[1].ids.select_file_label.text = "Error: The input array must have 2 dimensions." + self.x = None + self.root.screens[0].ids.input_data_file_button.text = "Input Data" + return + else: + self.root.screens[1].ids.select_file_label.text = "Error: A pickle file must be selected." + self.x = None + self.root.screens[0].ids.input_data_file_button.text = "Input Data" + return + + self.root.screens[0].ids.input_data_file_button.text = path[0] + self.root.current = "main" + + def output_file_select_path(self, path): + if len(path) == 0: + self.root.screens[2].ids.select_file_label.text = "Error: No file selected." + self.root.screens[0].ids.output_data_file_button.text = "Output Data" + return + elif path[0][-4:] == ".pkl": + # kivymd.toast.toast(path) + with open(path[0], 'rb') as inp: + self.y = pickle.load(inp) + # self.y = numpy.load(path[0]) + if self.y.ndim != 2: + self.root.screens[2].ids.select_file_label.text = "Error: The output array must have 2 dimensions." + self.y = None + self.root.screens[0].ids.output_data_file_button.text = "Output Data" + return + else: + self.root.screens[2].ids.select_file_label.text = "Error: A pickle file must be selected." + self.y = None + self.root.screens[0].ids.output_data_file_button.text = "Output Data" + return + + self.root.screens[0].ids.output_data_file_button.text = path[0] + self.root.current = "main" + + def debug_switch(self, *args): + self.debug = self.root.screens[0].ids.debug.active + + def button_press(self, *args): + self.net_arch = None + self.max_iter = None + self.tolerance = None + self.learning_rate = None + + self.learning_rate = self.root.screens[0].ids.learning_rate.text + try: + self.learning_rate = float(self.learning_rate) + if self.learning_rate >= 0.0 and self.learning_rate <= 1.0: + self.root.screens[0].ids.label.text = "" + else: + self.root.screens[0].ids.label.text = "Wrong value for the learning rate." + self.learning_rate = None + return + except: + self.root.screens[0].ids.label.text = "Wrong value for the learning rate." + self.learning_rate = None + return + + self.tolerance = self.root.screens[0].ids.tolerance.text + try: + self.tolerance = float(self.tolerance) + self.root.screens[0].ids.label.text = "" + except: + self.root.screens[0].ids.label.text = "Wrong value for the tolerance." + self.tolerance = None + return + + self.max_iter = self.root.screens[0].ids.iterations.text + try: + if int(self.max_iter) < 0: + self.root.screens[0].ids.label.text = "Wrong value for the number of iterations." + self.max_iter = None + return + else: + self.max_iter = int(self.max_iter) + self.root.screens[0].ids.label.text = "" + except: + self.root.screens[0].ids.label.text = "Wrong value for the number of iterations." + self.max_iter = None + return + + net_arch = self.root.screens[0].ids.net_arch.text.split(",") + temp = [] + if len(net_arch) == 1 and net_arch[0].strip() == '': + self.net_arch = [] + else: + for idx in range(len(net_arch)): + try: + if int(net_arch[idx]) <= 0: + self.root.screens[0].ids.label.text = "Wrong network architecture." + self.net_arch = None + return + else: + temp.append(int(net_arch[idx])) + except: + self.root.screens[0].ids.label.text = "Wrong network architecture." + self.net_arch = None + return + self.net_arch = temp + + # all_params = [self.x, self.y, self.net_arch, self.max_iter, self.tolerance, self.learning_rate, self.activation, self.GD_type, self.debug] + all_params_type = [type(self.x), type(self.y), type(self.net_arch), type(self.max_iter), type(self.tolerance), type(self.learning_rate), type(self.activation), type(self.GD_type), type(self.debug)] + if type(None) in all_params_type: + self.root.screens[0].ids.label.text = "Something is wrong. Please check your inputs." + return + + if self.x.shape[0] != self.y.shape[0]: + self.root.screens[0].ids.label.text = "Error: Number of samples in the input and output files must match." + return + + self.root.screens[0].ids.label.text = "All inputs are correct." + self.root.screens[0].ids.loading.active = True + self.root.screens[0].ids.btn.disabled = True + + neural_thread = NeuralThread(self, + self.x, + self.y, + self.net_arch, + self.max_iter, + self.tolerance, + self.learning_rate, + self.activation, + self.GD_type, + self.debug) + neural_thread.start() + +class NeuralThread(threading.Thread): + + def __init__(self, app, + x, + y, + net_arch, + max_iter, + tolerance, + learning_rate, + activation, + GD_type, + debug): + + super().__init__() + + self.app = app + self.x = x + self.y = y + self.net_arch = net_arch + self.max_iter = max_iter + self.tolerance = tolerance + self.learning_rate = learning_rate + self.activation = activation + self.GD_type = GD_type + self.debug = debug + + def run(self): + # all_params = [self.x, self.y, self.net_arch, self.max_iter, self.tolerance, self.learning_rate, self.activation, self.GD_type, self.debug] + self.app.root.screens[0].ids.label.text = "Training started..." + + net = MLP.MLP.train(self.x, + self.y, + self.net_arch, + self.max_iter, + self.tolerance, + self.learning_rate, + self.activation, + self.GD_type, + self.debug) + + self.app.root.screens[0].ids.label.text = "Network is trained. \nTraining time (sec) : {train_time}\nNumber of elapsed iterations : {num_iters}\nNetwork Error : {net_err}".format(train_time=net["training_time_sec"], num_iters=net["elapsed_iter"], net_err=net["network_error"]) + self.app.root.screens[0].ids.loading.active = False + self.app.root.screens[0].ids.btn.disabled = False + +app = NeuralApp() +app.run() + diff --git a/Ch11/Android/neural.kv b/Ch11/Android/neural.kv new file mode 100644 index 0000000..670075b --- /dev/null +++ b/Ch11/Android/neural.kv @@ -0,0 +1,188 @@ +ScreenManager: + MainScreen: + InputFileChooserScreen: + OutputFileChooserScreen: + +: + name: "main" + MDBoxLayout: + padding: "5dp" + orientation: "vertical" + + MDGridLayout: + cols: 3 + rows: 3 + padding: "5dp" + size_hint_y: 0.4 + + MDTextField: + id: learning_rate + size_hint: [1.0, 1.0] + hint_text: "Learning Rate" + halign: "center" + text: "0.5" + helper_text: ">= 0 & <= 1.0" + helper_text_mode: "on_focus" + + MDTextField: + id: tolerance + size_hint: [1.0, 1.0] + hint_text: "Tolerance" + halign: "center" + text: "0.000001" + + MDTextField: + id: iterations + size_hint: [1.0, 1.0] + hint_text: "Iterations" + helper_text: "Integer" + helper_text_mode: "on_focus" + halign: "center" + text: "5000" + + MDTextField: + id: net_arch + size_hint: [1.0, 1.0] + hint_text: "Network Architecture" + helper_text: "Comma-separated integers" + helper_text_mode: "on_focus" + halign: "center" + text: "3, 2" + + MDBoxLayout: + padding: "5dp" + + MDLabel: + text: "Debug" + + MDCheckbox: + id: debug + active: False + on_press: app.debug_switch(self) + + MDBoxLayout: + padding: "5dp" + + MDRaisedButton: + id: activation_menu + size_hint: [1.0, 1.0] + text: "Activation Function" + on_press: app.activation_menu.open() + + MDBoxLayout: + padding: "5dp" + + MDRaisedButton: + id: gdtype_menu + size_hint: [1.0, 1.0] + text: "Gradient Descent Type" + on_press: app.gdtype_menu.open() + + MDBoxLayout: + padding: "5dp" + + MDRaisedButton: + id: input_data_file_button + size_hint: [1.0, 1.0] + text: "Input Data" + on_press: app.root.current="input_chooser" + + MDBoxLayout: + padding: "5dp" + + MDRaisedButton: + id: output_data_file_button + size_hint: [1.0, 1.0] + text: "Output Data" + on_press: app.root.current="output_chooser" + + MDSpinner: + id: loading + active: False + size_hint: [0.05, 0.05] + pos_hint: {'center_x': .5, 'center_y': .5} + + MDBoxLayout: + padding: "5dp" + size_hint_y: 0.1 + + MDRaisedButton: + id: btn + text: "Train Network" + font_size: "18dp" + size_hint: [1.0, 1.0] + on_press: app.button_press(self) + + MDLabel: + id: label + text: "" + size_hint: [1.0, 1.0] + halign: "center" + size_hint_y: 0.2 + +: + name: "input_chooser" + + BoxLayout: + orientation: "vertical" + padding: "5dp" + + FileChooserListView: + id: file_chooser + size_hint_y: 0.8 + rootpath: "/storage" + + MDLabel: + size_hint_x: 1.0 + size_hint_y: 0.1 + id: select_file_label + halign: "center" + text: "" + + BoxLayout: + size_hint_y: 0.1 + padding: "5dp" + + MDRaisedButton: + text: "Cancel" + on_press: app.root.current="main" + size_hint: [1.0, 1.0] + MDRaisedButton: + id: load_input_file + text: "Select" + on_press: app.select_file(1) + size_hint: [1.0, 1.0] + +: + name: "output_chooser" + + BoxLayout: + orientation: "vertical" + padding: "5dp" + + FileChooserListView: + id: file_chooser + size_hint_y: 0.8 + rootpath: "/storage" + + MDLabel: + size_hint_x: 1.0 + size_hint_y: 0.1 + id: select_file_label + halign: "center" + text: "" + + BoxLayout: + size_hint_y: 0.1 + padding: "5dp" + + MDRaisedButton: + text: "Cancel" + on_press: app.root.current="main" + size_hint: [1.0, 1.0] + MDRaisedButton: + id: load_input_file + text: "Select" + on_press: app.select_file(2) + size_hint: [1.0, 1.0] + diff --git a/Ch11/MLP.py b/Ch11/MLP.py new file mode 100644 index 0000000..e501bb9 --- /dev/null +++ b/Ch11/MLP.py @@ -0,0 +1,538 @@ +import numpy +import itertools +import sys +import time + +class MLP: + trained_ann = {} + + def train(x, y, net_arch, max_iter=5000, tolerance=0.0000001, learning_rate=0.001, activation="sigmoid", GD_type="stochastic", + debug=False): + """ + train(x, y, w, max_iter, tolerance, learning_rate, activation="sigmoid", debug=False) + Training artificial neural network (ANN) using gradient descent (GD). + + Inputs: + x: Training data inputs. + y: Training data outputs. + net_arch: Network architecture defining number of inputs/outputs and number of neurons per each hidden layer. + max_iter: Maximum number of iterations for training the network. + tolerance: The desired minimum difference between predicted and target values. The network tries to reach this value if possible. + learning_rate: Learning rate to adapt the network weights while learning. + activation: Activation function to be used for building the hidden layers. Only one activation function can be used for all hidden layers. + GD_type: Can be either stochastic or batch. + debug: If True, informational messages are printed while training. Note that this increases the training time. + + Outputs: + trained_ann: A dictionary representing the trained neural network. It holds all information necessary for restoring the trained network. + """ + + # Number of inputs, number of neurons per each hidden layer, number of output neurons + # The network architecture defined in the net_arch list does not include the number of inputs nor the number of outputs. The lines below updates the net_arch list to include these 2 numbers. + in_size = x.shape[1] + if y.ndim == 1: + out_size = 1 + else: + out_size = y.shape[1] + + net_arch = [in_size] + net_arch # Number of inputs is added at the beginning of the list. + net_arch = net_arch + [out_size] # Number of outputs is added at the end of the list. + net_arch = numpy.array(net_arch) # Converting net_arch from a list to a NumPy array. + + net_arch, w_initial, b_initial = MLP.initialize_weights(net_arch) + w = w_initial.copy() + b = b_initial.copy() + + start_time = time.time() + + network_error = sys.maxsize + if (debug == True): + print("Training Started") + + if activation == "sigmoid": + activation_func = MLP.sigmoid + elif activation == "relu": + activation_func = MLP.relu + else: + activation_func = MLP.sigmoid + print("Sorry. Only Sigmoid and ReLU are supported at the current time. Sigmoid will still be used.") + + # pred_target_diff holds the difference between the predicted output and the target output to be compared by the tolerance. + pred_target_diff = sys.maxsize + + w_temp = [] + b_temp = [] + for k1 in range(len(w)): + w_temp.append(numpy.zeros(w[k1].shape)) + b_temp.append(numpy.zeros(b[k1].shape)) + + iter_num = 0 + while (iter_num < max_iter and pred_target_diff > tolerance): + network_error = 0 # Declare a variable to hold the network error across all samples. + pred_target_diff = 0 # Declare a variable to hold the difference between the targets and predictions across all samples. + all_samples_predictions = [] + for sample_idx in range(x.shape[0]): + curr_sample_x = x[sample_idx] # inputs of current sample. + curr_sample_y = y[sample_idx] # Target of current sample. + + sop_activ_mul = MLP.forward_path(curr_sample_x, w, b, activation_func) + + prediction = sop_activ_mul[-1][-1] # Predictions for current sample. + all_samples_predictions.append(prediction) + + sample_pred_target_diff = numpy.sum(numpy.abs(curr_sample_y - prediction)) + pred_target_diff = pred_target_diff + sample_pred_target_diff + + sample_error = MLP.error(prediction, curr_sample_y) + network_error = network_error + sample_error + + if GD_type == "stochastic": # For stochastic gradient descent. + w, b = MLP.backward_pass(curr_sample_x, curr_sample_y, w, b, net_arch, sop_activ_mul, prediction, out_size, activation, learning_rate) + elif GD_type=="batch": # For batch gradient descent. + w_sample, b_sample = MLP.backward_pass(curr_sample_x, curr_sample_y, w, b, net_arch, sop_activ_mul, prediction, out_size, activation, learning_rate) + + for k1 in range(len(w)): + w_temp[k1] = w_temp[k1] + w_sample[k1] + b_temp[k1] = b_temp[k1] + b_sample[k1] + else: + print("Sorry. Only stochastic and batch gradient descent are supported. Stochastic gradient descent will be used.") + w, b = MLP.backward_pass(curr_sample_x, curr_sample_y, w, b, net_arch, sop_activ_mul, prediction, out_size, activation, learning_rate) + + if GD_type == "batch": + for k1 in range(len(w)): + w_temp[k1] = w_temp[k1]/x.shape[0] + b_temp[k1] = b_temp[k1]/x.shape[0] + + w = w_temp.copy() + b = b_temp.copy() + + w_temp = [] + b_temp = [] + for k1 in range(len(w)): + w_temp.append(numpy.zeros(w[k1].shape)) + b_temp.append(numpy.zeros(b[k1].shape)) + + if (debug == True): + print("\nIteration : ", iter_num, "\nError : ", network_error) + iter_num = iter_num + 1 + + if (debug == True): + print("Training Finished") + end_time = time.time() + + training_time = end_time - start_time + + MLP.trained_ann["w"] = w + MLP.trained_ann["b"] = b + MLP.trained_ann["max_iter"] = max_iter + MLP.trained_ann[ + "elapsed_iter"] = iter_num + MLP.trained_ann["tolerance"] = tolerance + MLP.trained_ann["activation"] = activation + MLP.trained_ann["learning_rate"] = learning_rate + MLP.trained_ann["initial_w"] = w_initial + MLP.trained_ann["net_arch"] = net_arch + MLP.trained_ann["num_hidden_layers"] = net_arch.shape[0] - 2 + MLP.trained_ann["training_time_sec"] = training_time + MLP.trained_ann["network_error"] = network_error + MLP.trained_ann["in_size"] = in_size + MLP.trained_ann["out_size"] = out_size + + return MLP.trained_ann + + def initialize_weights(net_arch): + """ + initialize_weights(net_arch) + Initializing the neural network weights. + + Inputs: + net_arch: Network architecture defining number of inputs/outputs and number of neurons per each hidden layer. The user input might need some refine. + + Outputs: + net_arch: The final refined network architecture [if neccessary]. + w: The initial weights. + b: Initial bias values. + """ + + w = [] + w_temp = [] + b = [] + for layer_counter in numpy.arange(net_arch.shape[0] - 1): + rand_array = numpy.random.uniform(low=0.0, high=0.1, size=net_arch[layer_counter + 1]) + b.append(rand_array) + for neuron_counter in numpy.arange(net_arch[layer_counter + 1]): + rand_array = numpy.random.uniform(low=0.0, high=0.1, size=net_arch[layer_counter]) + w_temp.append(rand_array) + w_temp = numpy.array(w_temp) + w.append(w_temp) + w_temp = [] + + return net_arch, w, b + + def forward_path(x, w, b, activation_func): + """ + forward_path(x, w, activation_func) + Implementation of the forward pass. + + Inputs: + x: training inputs + w: current set of weights. + b: current bias values. + activation_func: A string representing the activation function used. + Outputs: + sop_activ_mul: An array representing the outputs of each layer + The sop_activ_mul array has number of rows equal to number of layers. For each layer, 2 things are calculated: + 1) SOP between inputs and weights. + 2) Activation output for SOP. + For example, If there are 4 layers, then the shape of the sop_activ_mul array is (4, 2). + sop_activ_mul[x][0]: SOP in layer x. + sop_activ_mul[x][1]: Activation in layer x. + """ + sop_activ_mul = [] + + sop_activ_mul_temp = [] + curr_multiplicand = x + for layer_num in range(len(w)): + sop_temp = numpy.matmul(curr_multiplicand, w[layer_num].T) + b[layer_num] + activ_temp = activation_func(sop_temp) + + curr_multiplicand = activ_temp + + sop_activ_mul_temp.append([sop_temp, activ_temp]) + sop_activ_mul.extend(sop_activ_mul_temp) + sop_activ_mul_temp = [] + return sop_activ_mul + + def sigmoid(sop): + """ + sigmoid(sop) + Implementation of the sigmoid activation function. + + Inputs: + sop: A single value representing the sum of products between the layer inputs and its weights. + + Outputs: + A single value representing the output of the sigmoid. + """ + return 1.0 / (1 + numpy.exp(-1 * sop)) + + def relu(x): + """ + relu(sop) + Implementation of the rectified linear unit (ReLU) activation function. + + Inputs: + sop: A single value representing the sum of products between the layer inputs and its weights. + + Outputs: + A single value representing the output of the relu. + """ + return numpy.maximum(0, x) + + def error(predicted, target): + """ + error(predicted, target) + Preduction error in the current iteration. + + Inputs: + predicted: The predictions of the network using its current parameters. + target: The correct outputs that the network should predict. + Outputs: + Error. + """ + return numpy.power(predicted - target, 2) + + def backward_pass(x, y, w, b, net_arch, sop_activ_mul, prediction, num_outputs, activation, learning_rate): + """ + backward_pass(x, y, w, net_arch, sop_activ_mul, prediction, num_outputs, activation, learning_rate) + Implementation of the backward pass for training the neural network using gradient descent. + + Inputs: + x: Training inputs. Used for calcualting the derivative for the weights between the input layer and the hidden layer. + y: Training data outputs. + w: Current weights which are to be updated during the backward pass using the gradient descent. + b: Bias values to be updated during the backward pass using the gradient descent. + net_arch: A NumPy array defining the network archietcture defining the number of neurons in all layers. It is used here to know the number of neurons per each layer. + sop_activ_mul: An array holding all calculations during the forward pass. This includes the sum of products between the inputs and weights of each layer and the results of the activation functions. + prediction: The predicted outputs of the network using the current weights. + num_outputs: Number of outputs per sample. + learning_rate: Learning rate to adapt the network weights while learning. + + Outputs: + w: The updated weights using gradient descent. + """ + if activation == "sigmoid": + activation_sop_deriv = MLP.sigmoid_sop_deriv + elif activation == "relu": + activation_sop_deriv = MLP.relu_sop_deriv + else: + activation_sop_deriv = MLP.sigmoid_sop_deriv + print("Sorry. Only Sigmoid and ReLU are supported at the current time. Sigmoid will still be used.") + + g1 = MLP.error_predicted_deriv(prediction, y) + + g2 = activation_sop_deriv(sop_activ_mul[-1][0]) + + output_layer_derivs = g2 * g1 + + layer_weights_derivs = [] + layer_weights_grads = [] + + if net_arch.shape[0] == 2: + layer_weights_derivs = [MLP.sop_w_deriv(sop_activ_mul[-1][1])] + layer_weights_derivs + layer_weights_grads = [layer_weights_derivs[0] * output_layer_derivs] + layer_weights_grads + + for out_neuron_idx in range(num_outputs): + w[-1][out_neuron_idx] = w[-1][out_neuron_idx] - layer_weights_grads[0][out_neuron_idx] * learning_rate + b[-1][out_neuron_idx] = b[-1][out_neuron_idx] - layer_weights_derivs[0] * output_layer_derivs[out_neuron_idx] * learning_rate + + MLP.trained_ann["derivative_chain"] = output_layer_derivs + MLP.trained_ann["weights_derivatives"] = layer_weights_derivs + MLP.trained_ann["weights_gradients"] = layer_weights_grads + + return w, b + + w_old = w.copy() + + layer_weights_derivs = [MLP.sop_w_deriv(sop_activ_mul[-1][1])] + layer_weights_derivs + + layer_weights_grads = [layer_weights_derivs[0] * output_layer_derivs] + layer_weights_grads + + # Updating the weights between the last hidden layer and the output neurons. + for out_neuron_idx in range(num_outputs): + w[-1][out_neuron_idx] = w[-1][out_neuron_idx] - layer_weights_grads[0][out_neuron_idx] * learning_rate + b[-1][out_neuron_idx] = b[-1][out_neuron_idx] - layer_weights_derivs[0] * output_layer_derivs[out_neuron_idx] * learning_rate + + SOPs_ACTIVs_deriv_individual = [] + deriv_chain_final = [] + + # Derivatives of the neurons between the last hidden layer and the output neurons. + SOPs_ACTIVs_deriv_individual.append(output_layer_derivs) + deriv_chain_final.append(output_layer_derivs) + + ############# GENERIC GRADIENT DESCENT ############# + # The idea is to prepare the individual derivatives of all neurons in all layers. These derivatives include: + # -) Derivative of activation (output) to SOP (input) + # -) Derivative of SOP (input) to activation (output) + # These derivative not include the following: + # -) Derivative of SOP (output) to the weight. It will be calculated later. + # Using the chain rule, combinations are created from these individual derivatives. + # X) The total derivative at a given neuron is the mean of products of all these combinations. + # Y) For every neuron at a given layer, the derivative of the SOP (output) to the weight is calculated. + # The gradient to update a given weight is the product between the mean (step X) and the weight derivative (step Y). + + # Derivatives of all other layers + # If the network has no or just 1 hidden layer, this loop will not be executed. # It works when there are more than 1 hidden layer. + curr_idx = -1 + for curr_lay_idx in range(len(w)-2, -1, -1): + # ACTIVs_deriv are identical for all neurons in a given layer. If a layer has 5 neurons, then the length of ACTIVs_deriv is 5. + # But SOPs_deriv returns 5 values for every neuron. Thus, there will be an array of 5 rows, one row for one neuron. + SOPs_deriv = MLP.sop_w_deriv(w_old[curr_lay_idx + 1]) + ACTIVs_deriv = activation_sop_deriv(sop_activ_mul[curr_lay_idx][0]) + + temp = [] + for curr_neuron_idx in range(net_arch[curr_idx]): + temp.append(ACTIVs_deriv * SOPs_deriv[curr_neuron_idx]) + curr_idx = curr_idx - 1 + temp = numpy.array(temp) + # Individual Derivatives of the Network Except the Weights Derivatives + SOPs_ACTIVs_deriv_individual = [temp] + SOPs_ACTIVs_deriv_individual + + temp2 = MLP.deriv_chain_prod(temp[0], deriv_chain_final[-1]) + for neuron_calc_derivs_idx in range(1, temp.shape[0]): + # Only last element in the deriv_chain_final array is used for calculating the chain. + # This reuses the previous calculations because some chains were calculated previously. + temp2 = temp2 + MLP.deriv_chain_prod(temp[neuron_calc_derivs_idx], deriv_chain_final[-1]) + deriv_chain_final.append(temp2) + + #### Calculate WEIGHTS Derivatives and Gradients + # Index 1 means output of activation function for all neurons per layer. + # At layer with index i, the derivs (layer_weights_derivs) and grads (layer_weights_grads) of its weights are at index i. + # For example, layer indexed i=0 has its derivs (layer_weights_derivs) and grads (layer_weights_grads) in index 0. + if curr_lay_idx == 0: + layer_weights_derivs = [MLP.sop_w_deriv(x)] + layer_weights_derivs + else: + layer_weights_derivs = [MLP.sop_w_deriv(sop_activ_mul[curr_lay_idx - 1][1])] + layer_weights_derivs + layer_weights_grads = [layer_weights_derivs[0]] + layer_weights_grads + + # Derivatives of the Entire Network (Except Weights Derivatives) Chains Following the Chain Rule. + deriv_chain_final = numpy.array(deriv_chain_final) + # Derivatives of the Entire Nework including the Weights Derivatives + layer_weights_derivs = numpy.array(layer_weights_derivs) + # Gradients of the Entire Network Weights + layer_weights_grads = numpy.array(layer_weights_grads) + + deriv_chain_final = numpy.flip(deriv_chain_final) + + MLP.trained_ann["derivative_chain"] = deriv_chain_final + MLP.trained_ann["weights_derivatives"] = layer_weights_derivs + MLP.trained_ann["weights_gradients"] = layer_weights_grads + + #### Updating Weights of All Layers Except Last Layer Because it was Updated Previously + for layer_idx in range(len(w) - 1): + w[layer_idx] = MLP.update_weights(w[layer_idx], layer_weights_grads[layer_idx], + deriv_chain_final[layer_idx], learning_rate) + b[layer_idx] = MLP.update_bias(b[layer_idx], deriv_chain_final[layer_idx], learning_rate) + return w, b + + def error_predicted_deriv(predicted, target): + """ + error_predicted_deriv(predicted, target) + Derivative of the error to the predicted value. + + Inputs: + predicted: The predictions of the network using its current parameters. + target: The correct outputs that the network should predict. + + Outputs: + Derivative of the error to the predicted value. + """ + return 2 * (predicted - target) + + def sigmoid_sop_deriv(sop): + """ + sigmoid_sop_deriv(sop) + Calculating the derivative of the sigmoid to the sum of products. + + Inputs: + sop: Sum of products. + + Outputs: + Derivative of the sum of products to sigmoid. + """ + return MLP.sigmoid(sop) * (1.0 - MLP.sigmoid(sop)) + + def relu_sop_deriv(sop): + """ + relu_sop_deriv(sop) + Calculating the derivative of the relu to the sum of products. + + Inputs: + sop: Sum of products. + + Outputs: + Derivative of the sum of products to relu. + """ + sop[sop <= 0] = 0.0 + sop[sop > 0] = 1.0 + return sop + + def sop_w_deriv(x): + """ + sop_w_deriv(x) + Derivative of the sum of products to the weights. + + Inputs: + x: inputs to the current layer. + + Outputs: + Derivative of the sum of products to the weights. + """ + return x + + def deriv_chain_prod(layer_derivs, previous_lay_derivs_chains): + """ + deriv_chain_prod(derivs_arrs) + Derivative chains of a given layer. + + Inputs: + layer_derivs: Derivatives of the current layer. + previous_lay_derivs_chains: Derivatives chains of the previous layer. + Outputs: + deriv_chain_prod_sum: A NumPy array representing the sum of the derivatives products in all chains. + """ + derivs_arrs = [layer_derivs] + [[previous_lay_derivs_chains]] + deriv_combinations = numpy.array(list(itertools.product(*derivs_arrs))) + num_products_in_layer = deriv_combinations.shape[0] + num_neurons_in_layer = len(derivs_arrs[0]) + num_chains_for_neuron = numpy.uint32(num_products_in_layer / num_neurons_in_layer) + + # print("\n# of Products in Layer : ", num_products_in_layer, + # "\n# of Neurons in Layer : ", num_neurons_in_layer, + # "\n# of Products per Neuron : ", num_chains_for_neuron) + + deriv_chain_prod_sum = [] + for neuron_idx in range(num_neurons_in_layer): + start_idx = neuron_idx * num_chains_for_neuron + end_idx = (neuron_idx + 1) * num_chains_for_neuron + deriv_chain = deriv_combinations[start_idx:end_idx] + deriv_chain_prod = numpy.prod(deriv_chain, axis=1) + # When there are more than 1 chain to reach a neuron, the sum of all chains is calculated and returned as a single value. + deriv_chain_prod_sum.append(numpy.concatenate(deriv_chain_prod).sum()) + # Update weight according to chains product sum (deriv_chain_prod_sum) + deriv_chain_prod_sum = numpy.array(deriv_chain_prod_sum) + return deriv_chain_prod_sum + + def update_weights(weights, layer_weights_grads, deriv_chain_final, learning_rate): + """ + update_weights(weights, gradients, learning_rate) + Updating the weights based on the calcualted gradients. + + Inputs: + weights: Weights of a layer tbe updated. + layer_weights_grads: Gradient of the current layer for updating its weights. + deriv_chain_final: Chains of derivatives from the layers higher than the current layer. + learning_rate: Learnign rate. + + Outputs: + Updated weights. + """ + for neuron_idx in range(weights.shape[0]): + weights[neuron_idx] = weights[neuron_idx] - layer_weights_grads * deriv_chain_final[ + neuron_idx] * learning_rate + return weights + + def update_bias(b, deriv_chain_final, learning_rate): + """ + update_weights(weights, gradients, learning_rate) + Updating the weights based on the calcualted gradients. + + Inputs: + b: Bias. + deriv_chain_final: Chains of derivatives from the layers higher than the current layer. + learning_rate: Learnign rate. + + Outputs: + Updated b. + """ + for neuron_idx in range(b.shape[0]): + b[neuron_idx] = b[neuron_idx] - deriv_chain_final[neuron_idx] * learning_rate + return b + + def predict(trained_ann, x): + """ + predict(trained_ann, x) + Making prediction for a new sample. + + Inputs: + trained_ann: A dictionary representing trained MLP. + x: The new sample. + Outputs: + prediction: The predicted output for the current sample. + """ + in_size = trained_ann["in_size"] + if x.ndim == 1: + if x.shape[0] != in_size: + print("Input shape ", x.shape[0], " does not match the expected network input shape ", in_size) + return + else: + if x.shape[1] != in_size: + print("Input shape ", x.shape[1], " does not match the expected network input shape ", in_size) + return + + w = trained_ann["w"] + b = trained_ann["b"] + + activation = trained_ann["activation"] + if activation == "sigmoid": + activation_func = MLP.sigmoid + elif activation == "relu": + activation_func = MLP.relu + else: + activation_func = MLP.sigmoid + print("Sorry. Only Sigmoid and ReLU are supported at the current time. Sigmoid will still be used.") + + sop_activ_mul = MLP.forward_path(x, w, b, activation_func) + prediction = sop_activ_mul[-1][1] + + return prediction \ No newline at end of file diff --git a/Ch11/data_inputs.pkl b/Ch11/data_inputs.pkl new file mode 100644 index 0000000..5e64706 Binary files /dev/null and b/Ch11/data_inputs.pkl differ diff --git a/Ch11/data_outputs.pkl b/Ch11/data_outputs.pkl new file mode 100644 index 0000000..5fef9c3 Binary files /dev/null and b/Ch11/data_outputs.pkl differ diff --git a/Ch11/main.py b/Ch11/main.py new file mode 100644 index 0000000..5a698ef --- /dev/null +++ b/Ch11/main.py @@ -0,0 +1,249 @@ +import kivymd.app +import kivymd.uix.menu +import kivymd.uix.button +import threading +import MLP +import kivy.uix.screenmanager +import pickle + +class MainScreen(kivy.uix.screenmanager.Screen): + pass + +class InputFileChooserScreen(kivy.uix.screenmanager.Screen): + pass + +class OutputFileChooserScreen(kivy.uix.screenmanager.Screen): + pass + +class NeuralApp(kivymd.app.MDApp): + + def select_file(self, screen_num, *args): + path = self.root.screens[screen_num].ids.file_chooser.selection + if screen_num == 1: + self.input_file_select_path(path=path) + else: + self.output_file_select_path(path=path) + + def on_start(self): + +# with open('data_inputs.pkl', 'wb') as out: +# pickle.dump(numpy.array([[0.1, 0.4, 4.1, 4.3, 1.8, 2.0, 0.01, 0.9, 3.8, 1.6]]), out) + +# with open('data_outputs.pkl', 'wb') as out: +# pickle.dump(numpy.array([[0.2]]), out) + + self.x = None + self.y = None + self.net_arch = None + self.max_iter = None + self.tolerance = None + self.learning_rate = None + self.activation = None + self.GD_type = None + self.debug = False + + self.activation_menu = kivymd.uix.menu.MDDropdownMenu(caller=self.root.screens[0].ids.activation_menu, + items=[{"text": "sigmoid"}, {"text": "relu"}], + callback=self.activation_menu_callback, + width_mult=4) + + self.gdtype_menu = kivymd.uix.menu.MDDropdownMenu(caller=self.root.screens[0].ids.gdtype_menu, + items=[{"text": "stochastic"}, {"text": "batch"}], + callback=self.gdtype_menu_callback, + width_mult=4) + + def activation_menu_callback(self, activation_menu): + self.root.screens[0].ids.activation_menu.text = activation_menu.text + self.activation = activation_menu.text + self.activation_menu.dismiss() + + def gdtype_menu_callback(self, gdtype_menu): + self.root.screens[0].ids.gdtype_menu.text = gdtype_menu.text + self.GD_type = gdtype_menu.text + self.gdtype_menu.dismiss() + + def input_file_select_path(self, path): + if len(path) == 0: + self.root.screens[1].ids.select_file_label.text = "Error: No file selected." + self.root.screens[0].ids.input_data_file_button.text = "Input Data" + return + elif path[0][-4:] == ".pkl": + # kivymd.toast.toast(path) + with open(path[0], 'rb') as inp: + self.x = pickle.load(inp) + # self.x = numpy.load(path[0]) + if self.x.ndim != 2: + self.root.screens[1].ids.select_file_label.text = "Error: The input array must have 2 dimensions." + self.x = None + self.root.screens[0].ids.input_data_file_button.text = "Input Data" + return + else: + self.root.screens[1].ids.select_file_label.text = "Error: A pickle file must be selected." + self.x = None + self.root.screens[0].ids.input_data_file_button.text = "Input Data" + return + + self.root.screens[0].ids.input_data_file_button.text = path[0] + self.root.current = "main" + + def output_file_select_path(self, path): + if len(path) == 0: + self.root.screens[2].ids.select_file_label.text = "Error: No file selected." + self.root.screens[0].ids.output_data_file_button.text = "Output Data" + return + elif path[0][-4:] == ".pkl": + # kivymd.toast.toast(path) + with open(path[0], 'rb') as inp: + self.y = pickle.load(inp) + # self.y = numpy.load(path[0]) + if self.y.ndim != 2: + self.root.screens[2].ids.select_file_label.text = "Error: The output array must have 2 dimensions." + self.y = None + self.root.screens[0].ids.output_data_file_button.text = "Output Data" + return + else: + self.root.screens[2].ids.select_file_label.text = "Error: A pickle file must be selected." + self.y = None + self.root.screens[0].ids.output_data_file_button.text = "Output Data" + return + + self.root.screens[0].ids.output_data_file_button.text = path[0] + self.root.current = "main" + + def debug_switch(self, *args): + self.debug = self.root.screens[0].ids.debug.active + + def button_press(self, *args): + self.net_arch = None + self.max_iter = None + self.tolerance = None + self.learning_rate = None + + self.learning_rate = self.root.screens[0].ids.learning_rate.text + try: + self.learning_rate = float(self.learning_rate) + if self.learning_rate >= 0.0 and self.learning_rate <= 1.0: + self.root.screens[0].ids.label.text = "" + else: + self.root.screens[0].ids.label.text = "Wrong value for the learning rate." + self.learning_rate = None + return + except: + self.root.screens[0].ids.label.text = "Wrong value for the learning rate." + self.learning_rate = None + return + + self.tolerance = self.root.screens[0].ids.tolerance.text + try: + self.tolerance = float(self.tolerance) + self.root.screens[0].ids.label.text = "" + except: + self.root.screens[0].ids.label.text = "Wrong value for the tolerance." + self.tolerance = None + return + + self.max_iter = self.root.screens[0].ids.iterations.text + try: + if int(self.max_iter) < 0: + self.root.screens[0].ids.label.text = "Wrong value for the number of iterations." + self.max_iter = None + return + else: + self.max_iter = int(self.max_iter) + self.root.screens[0].ids.label.text = "" + except: + self.root.screens[0].ids.label.text = "Wrong value for the number of iterations." + self.max_iter = None + return + + net_arch = self.root.screens[0].ids.net_arch.text.split(",") + temp = [] + if len(net_arch) == 1 and net_arch[0].strip() == '': + self.net_arch = [] + else: + for idx in range(len(net_arch)): + try: + if int(net_arch[idx]) <= 0: + self.root.screens[0].ids.label.text = "Wrong network architecture." + self.net_arch = None + return + else: + temp.append(int(net_arch[idx])) + except: + self.root.screens[0].ids.label.text = "Wrong network architecture." + self.net_arch = None + return + self.net_arch = temp + + # all_params = [self.x, self.y, self.net_arch, self.max_iter, self.tolerance, self.learning_rate, self.activation, self.GD_type, self.debug] + all_params_type = [type(self.x), type(self.y), type(self.net_arch), type(self.max_iter), type(self.tolerance), type(self.learning_rate), type(self.activation), type(self.GD_type), type(self.debug)] + if type(None) in all_params_type: + self.root.screens[0].ids.label.text = "Something is wrong. Please check your inputs." + return + + if self.x.shape[0] != self.y.shape[0]: + self.root.screens[0].ids.label.text = "Error: Number of samples in the input and output files must match." + return + + self.root.screens[0].ids.label.text = "All inputs are correct." + self.root.screens[0].ids.loading.active = True + self.root.screens[0].ids.btn.disabled = True + + neural_thread = NeuralThread(self, + self.x, + self.y, + self.net_arch, + self.max_iter, + self.tolerance, + self.learning_rate, + self.activation, + self.GD_type, + self.debug) + neural_thread.start() + +class NeuralThread(threading.Thread): + + def __init__(self, app, + x, + y, + net_arch, + max_iter, + tolerance, + learning_rate, + activation, + GD_type, + debug): + + super().__init__() + + self.app = app + self.x = x + self.y = y + self.net_arch = net_arch + self.max_iter = max_iter + self.tolerance = tolerance + self.learning_rate = learning_rate + self.activation = activation + self.GD_type = GD_type + self.debug = debug + + def run(self): + # all_params = [self.x, self.y, self.net_arch, self.max_iter, self.tolerance, self.learning_rate, self.activation, self.GD_type, self.debug] + self.app.root.screens[0].ids.label.text = "Training started..." + + net = MLP.MLP.train(self.x, + self.y, + self.net_arch, + self.max_iter, + self.tolerance, + self.learning_rate, + self.activation, + self.GD_type, + self.debug) + + self.app.root.screens[0].ids.label.text = "Network is trained. \nTraining time (sec) : {train_time}\nNumber of elapsed iterations : {num_iters}\nNetwork Error : {net_err}".format(train_time=net["training_time_sec"], num_iters=net["elapsed_iter"], net_err=net["network_error"]) + self.app.root.screens[0].ids.loading.active = False + self.app.root.screens[0].ids.btn.disabled = False + +app = NeuralApp() +app.run() diff --git a/Ch11/neural.kv b/Ch11/neural.kv new file mode 100644 index 0000000..1644584 --- /dev/null +++ b/Ch11/neural.kv @@ -0,0 +1,187 @@ +ScreenManager: + MainScreen: + InputFileChooserScreen: + OutputFileChooserScreen: + +: + name: "main" + MDBoxLayout: + padding: "5dp" + orientation: "vertical" + + MDGridLayout: + cols: 3 + rows: 3 + padding: "5dp" + size_hint_y: 0.4 + + MDTextField: + id: learning_rate + size_hint: [1.0, 1.0] + hint_text: "Learning Rate" + halign: "center" + text: "0.5" + helper_text: ">= 0 & <= 1.0" + helper_text_mode: "on_focus" + + MDTextField: + id: tolerance + size_hint: [1.0, 1.0] + hint_text: "Tolerance" + halign: "center" + text: "0.000001" + + MDTextField: + id: iterations + size_hint: [1.0, 1.0] + hint_text: "Iterations" + helper_text: "Integer" + helper_text_mode: "on_focus" + halign: "center" + text: "5000" + + MDTextField: + id: net_arch + size_hint: [1.0, 1.0] + hint_text: "Network Architecture" + helper_text: "Comma-separated integers" + helper_text_mode: "on_focus" + halign: "center" + text: "3, 2" + + MDBoxLayout: + padding: "5dp" + + MDLabel: + text: "Debug" + + MDCheckbox: + id: debug + active: False + on_press: app.debug_switch(self) + + MDBoxLayout: + padding: "5dp" + + MDRaisedButton: + id: activation_menu + size_hint: [1.0, 1.0] + text: "Activation Function" + on_press: app.activation_menu.open() + + MDBoxLayout: + padding: "5dp" + + MDRaisedButton: + id: gdtype_menu + size_hint: [1.0, 1.0] + text: "Gradient Descent Type" + on_press: app.gdtype_menu.open() + + MDBoxLayout: + padding: "5dp" + + MDRaisedButton: + id: input_data_file_button + size_hint: [1.0, 1.0] + text: "Input Data" + on_press: app.root.current="input_chooser" + + MDBoxLayout: + padding: "5dp" + + MDRaisedButton: + id: output_data_file_button + size_hint: [1.0, 1.0] + text: "Output Data" + on_press: app.root.current="output_chooser" + + MDSpinner: + id: loading + active: False + size_hint: [0.05, 0.05] + pos_hint: {'center_x': .5, 'center_y': .5} + + MDBoxLayout: + padding: "5dp" + size_hint_y: 0.1 + + MDRaisedButton: + id: btn + text: "Train Network" + font_size: "18dp" + size_hint: [1.0, 1.0] + on_press: app.button_press(self) + + MDLabel: + id: label + text: "" + size_hint: [1.0, 1.0] + halign: "center" + size_hint_y: 0.2 + +: + name: "input_chooser" + + BoxLayout: + orientation: "vertical" + padding: "5dp" + + FileChooserListView: + id: file_chooser + size_hint_y: 0.8 + rootpath: "/" + + MDLabel: + size_hint_x: 1.0 + size_hint_y: 0.1 + id: select_file_label + halign: "center" + text: "" + + BoxLayout: + size_hint_y: 0.1 + padding: "5dp" + + MDRaisedButton: + text: "Cancel" + on_press: app.root.current="main" + size_hint: [1.0, 1.0] + MDRaisedButton: + id: load_input_file + text: "Select" + on_press: app.select_file(1) + size_hint: [1.0, 1.0] + +: + name: "output_chooser" + + BoxLayout: + orientation: "vertical" + padding: "5dp" + + FileChooserListView: + id: file_chooser + size_hint_y: 0.8 + rootpath: "/" + + MDLabel: + size_hint_x: 1.0 + size_hint_y: 0.1 + id: select_file_label + halign: "center" + text: "" + + BoxLayout: + size_hint_y: 0.1 + padding: "5dp" + + MDRaisedButton: + text: "Cancel" + on_press: app.root.current="main" + size_hint: [1.0, 1.0] + MDRaisedButton: + id: load_input_file + text: "Select" + on_press: app.select_file(2) + size_hint: [1.0, 1.0] diff --git a/README.md b/README.md index 630b30e..a8a406c 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,44 @@ # Introduction to Deep Learning with Python -This repository is updated by a number of introductory projects to deep learning with Python. +This repository is the GitHub project of this book: [Ahmed Fawzy Gad & Fatima Ezzahra Jarmouni, Introduction to Deep Learning and Neural Networks with Python™: A Practical Guide, 2020, 978-0323909334](https://www.elsevier.com/books/introduction-to-deep-learning-and-neural-networks-with-python/gad/978-0-323-90933-4). -The only project hosted in the repository builds a generic implementation of a neural network trained using the gradient descent algorithm. +[![Ahmed Fawzy Gad & Fatima Ezzahra Jarmouni, Introduction to Deep Learning and Neural Networks with Python™: A Practical Guide, 2020, 978-0323909334](https://secure-ecsd.elsevier.com/covers/80/Tango2/large/9780323909334.jpg)](https://www.elsevier.com/books/introduction-to-deep-learning-and-neural-networks-with-python/gad/978-0-323-90933-4) + +## Description + +*Introduction to Deep Learning and Neural Networks with Python™: A Practical Guide* is an intensive step-by-step guide for neuroscientists to fully understand, practice, and build neural networks. Providing math and Python™ code examples to clarify neural network calculations, by book’s end readers will fully understand how neural networks work starting from the simplest model Y=X and building from scratch. Details and explanations are provided on how a generic gradient descent algorithm works based on mathematical and Python™ examples, teaching you how to use the gradient descent algorithm to manually perform all calculations in both the forward and backward passes of training a neural network. + +## Key Features + +- Examines the practical side of deep learning and neural networks +- Provides a problem-based approach to building artificial neural networks using real data +- Describes Python™ functions and features for neuroscientists +- Uses a careful tutorial approach to describe implementation of neural networks in Python™ +- Features math and code examples (via companion website) with helpful instructions for easy implementation + +## Table of Contents + +1. Preparing the Development Environment + +2. Introduction to ANN + +3. ANN with 1 Input and 1 Output + +4. Working with Any Number of Inputs + +5. Working with Hidden Layers + +6. Using Any Number of Hidden Neurons + +7. ANN with 2 Hidden Layers + +8. ANN with 3 Hidden Layers + +9. Any Number of Hidden Layers + +10. Generic ANN + +11. Deploying Neural Network to Mobile Devices # Contact Us