feat: Examples (#43)

* export content from scheduler

* add binary classification example

* add multiple linear regression example

* lockfile

* fix log(0) and division by 0

* avoid positive log(y)

* prevent division by zero in bincrossentropy

* use proper bincrossentropy

* use log loss instead of MSE

* remove unused imports

* temp-fix: make softmax layer work like other activations

* add multiclass classification

* change method name

* add spam classifier

* add spam classifier task

* change comments

* remove unused imports

crates/core/src/cpu/cost.rs

@@ -1,6 +1,9 @@
-use std::ops::{Div, Mul, Sub};
+use std::{
+    f32::EPSILON,
+    ops::{Mul, Sub},
+};
 
-use ndarray::{s, ArrayD, ArrayViewD};
+use ndarray::{ArrayD, ArrayViewD};
 
 use crate::Cost;
 
@@ -43,27 +46,24 @@ fn mse_prime<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> ArrayD<f
 
 fn cross_entropy<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> f32 {
     let batches = y_hat.dim()[0];
-    let mut total = 0.0;
-    for b in 0..batches {
-        total -= y_hat.slice(s![b, ..]).mul(&y.slice(s![b, ..])).sum().ln()
-    }
+    let total = (-&y_hat * (y.map(|x| x.max(EPSILON).min(1f32 - EPSILON).ln()))).sum();
     return total / batches as f32;
 }
 
 fn cross_entropy_prime<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> ArrayD<f32> {
-    return -y_hat.div(&y);
+    return -&y_hat / (&y + EPSILON);
 }
 
 fn bin_cross_entropy<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> f32 {
     return -y_hat
-        .mul(y.map(|x| x.ln()))
-        .sub(((1.0).sub(&y_hat)).mul(y.map(|x| 1.0 - x.ln())))
+        .mul(y.map(|x| x.max(EPSILON).min(1f32 - EPSILON).ln()))
+        .sub(((1.0).sub(&y_hat)).mul(y.map(|x| 1.0 - x.max(EPSILON).min(1f32 - EPSILON).ln())))
         .sum()
         / y.len() as f32;
 }
 
 fn bin_cross_entropy_prime<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> ArrayD<f32> {
-    return y.sub(&y_hat).div(y.mul(1.0.sub(&y)));
+    return (-&y_hat / (&y + EPSILON)) + (1.0 - &y_hat) / (1.0 - &y + EPSILON);
 }
 
 fn hinge<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> f32 {

crates/core/src/cpu/layers/activation.rs

@@ -65,6 +65,7 @@ impl SoftmaxCPULayer {
     }
 
     pub fn forward_propagate(&mut self, inputs: ArrayD<f32>) -> ArrayD<f32> {
+        self.outputs = inputs.clone();
         let batches = self.outputs.dim()[0];
         for b in 0..batches {
             let exp = inputs.slice(s![b, ..]).map(|x| x.exp());
@@ -78,7 +79,7 @@ impl SoftmaxCPULayer {
     pub fn backward_propagate(&mut self, d_outputs: ArrayD<f32>) -> ArrayD<f32> {
         let batches = self.outputs.dim()[0];
         let array_size = self.outputs.dim().size() / batches;
-
+        
         let mut d_inputs = ArrayD::zeros(self.outputs.dim());
         for b in 0..batches {
             for y in 0..array_size {

deno.json

@@ -6,6 +6,10 @@
     "example:xor-gpu": "deno run -A --unstable ./examples/xor_gpu.ts",
     "example:xor-wasm": "deno run -A --unstable ./examples/xor_wasm.ts",
     "example:linear": "deno run -A --unstable ./examples/linear.ts",
+    "example:multiple-linear": "deno run -A --unstable ./examples/multiple-linear/student.ts",
+    "example:binary": "deno run -A --unstable ./examples/classification/binary_iris.ts",
+    "example:multiclass": "deno run -A --unstable ./examples/classification/iris.ts",
+    "example:text": "deno run -A --unstable ./examples/classification/spam.ts",
     "example:filters": "deno run -A --unstable examples/filters/conv.ts ",
     "example:train": "deno run -A --unstable examples/model/train.ts ",
     "example:run": "deno run -A --unstable examples/model/run.ts ",

examples/classification/README.md

@@ -0,0 +1,3 @@
+# Binary Classification
+This example showcases binary classification on the Iris dataset. 
+The `Iris Virginica` class is omitted for this example.

examples/classification/binary_iris.csv

@@ -0,0 +1,100 @@
+5.1,3.5,1.4,.2,"Setosa"
+4.9,3,1.4,.2,"Setosa"
+4.7,3.2,1.3,.2,"Setosa"
+4.6,3.1,1.5,.2,"Setosa"
+5,3.6,1.4,.2,"Setosa"
+5.4,3.9,1.7,.4,"Setosa"
+4.6,3.4,1.4,.3,"Setosa"
+5,3.4,1.5,.2,"Setosa"
+4.4,2.9,1.4,.2,"Setosa"
+4.9,3.1,1.5,.1,"Setosa"
+5.4,3.7,1.5,.2,"Setosa"
+4.8,3.4,1.6,.2,"Setosa"
+4.8,3,1.4,.1,"Setosa"
+4.3,3,1.1,.1,"Setosa"
+5.8,4,1.2,.2,"Setosa"
+5.7,4.4,1.5,.4,"Setosa"
+5.4,3.9,1.3,.4,"Setosa"
+5.1,3.5,1.4,.3,"Setosa"
+5.7,3.8,1.7,.3,"Setosa"
+5.1,3.8,1.5,.3,"Setosa"
+5.4,3.4,1.7,.2,"Setosa"
+5.1,3.7,1.5,.4,"Setosa"
+4.6,3.6,1,.2,"Setosa"
+5.1,3.3,1.7,.5,"Setosa"
+4.8,3.4,1.9,.2,"Setosa"
+5,3,1.6,.2,"Setosa"
+5,3.4,1.6,.4,"Setosa"
+5.2,3.5,1.5,.2,"Setosa"
+5.2,3.4,1.4,.2,"Setosa"
+4.7,3.2,1.6,.2,"Setosa"
+4.8,3.1,1.6,.2,"Setosa"
+5.4,3.4,1.5,.4,"Setosa"
+5.2,4.1,1.5,.1,"Setosa"
+5.5,4.2,1.4,.2,"Setosa"
+4.9,3.1,1.5,.2,"Setosa"
+5,3.2,1.2,.2,"Setosa"
+5.5,3.5,1.3,.2,"Setosa"
+4.9,3.6,1.4,.1,"Setosa"
+4.4,3,1.3,.2,"Setosa"
+5.1,3.4,1.5,.2,"Setosa"
+5,3.5,1.3,.3,"Setosa"
+4.5,2.3,1.3,.3,"Setosa"
+4.4,3.2,1.3,.2,"Setosa"
+5,3.5,1.6,.6,"Setosa"
+5.1,3.8,1.9,.4,"Setosa"
+4.8,3,1.4,.3,"Setosa"
+5.1,3.8,1.6,.2,"Setosa"
+4.6,3.2,1.4,.2,"Setosa"
+5.3,3.7,1.5,.2,"Setosa"
+5,3.3,1.4,.2,"Setosa"
+7,3.2,4.7,1.4,"Versicolor"
+6.4,3.2,4.5,1.5,"Versicolor"
+6.9,3.1,4.9,1.5,"Versicolor"
+5.5,2.3,4,1.3,"Versicolor"
+6.5,2.8,4.6,1.5,"Versicolor"
+5.7,2.8,4.5,1.3,"Versicolor"
+6.3,3.3,4.7,1.6,"Versicolor"
+4.9,2.4,3.3,1,"Versicolor"
+6.6,2.9,4.6,1.3,"Versicolor"
+5.2,2.7,3.9,1.4,"Versicolor"
+5,2,3.5,1,"Versicolor"
+5.9,3,4.2,1.5,"Versicolor"
+6,2.2,4,1,"Versicolor"
+6.1,2.9,4.7,1.4,"Versicolor"
+5.6,2.9,3.6,1.3,"Versicolor"
+6.7,3.1,4.4,1.4,"Versicolor"
+5.6,3,4.5,1.5,"Versicolor"
+5.8,2.7,4.1,1,"Versicolor"
+6.2,2.2,4.5,1.5,"Versicolor"
+5.6,2.5,3.9,1.1,"Versicolor"
+5.9,3.2,4.8,1.8,"Versicolor"
+6.1,2.8,4,1.3,"Versicolor"
+6.3,2.5,4.9,1.5,"Versicolor"
+6.1,2.8,4.7,1.2,"Versicolor"
+6.4,2.9,4.3,1.3,"Versicolor"
+6.6,3,4.4,1.4,"Versicolor"
+6.8,2.8,4.8,1.4,"Versicolor"
+6.7,3,5,1.7,"Versicolor"
+6,2.9,4.5,1.5,"Versicolor"
+5.7,2.6,3.5,1,"Versicolor"
+5.5,2.4,3.8,1.1,"Versicolor"
+5.5,2.4,3.7,1,"Versicolor"
+5.8,2.7,3.9,1.2,"Versicolor"
+6,2.7,5.1,1.6,"Versicolor"
+5.4,3,4.5,1.5,"Versicolor"
+6,3.4,4.5,1.6,"Versicolor"
+6.7,3.1,4.7,1.5,"Versicolor"
+6.3,2.3,4.4,1.3,"Versicolor"
+5.6,3,4.1,1.3,"Versicolor"
+5.5,2.5,4,1.3,"Versicolor"
+5.5,2.6,4.4,1.2,"Versicolor"
+6.1,3,4.6,1.4,"Versicolor"
+5.8,2.6,4,1.2,"Versicolor"
+5,2.3,3.3,1,"Versicolor"
+5.6,2.7,4.2,1.3,"Versicolor"
+5.7,3,4.2,1.2,"Versicolor"
+5.7,2.9,4.2,1.3,"Versicolor"
+6.2,2.9,4.3,1.3,"Versicolor"
+5.1,2.5,3,1.1,"Versicolor"
+5.7,2.8,4.1,1.3,"Versicolor"

examples/classification/binary_iris.ts

@@ -0,0 +1,89 @@
+import {
+  Cost,
+  CPU,
+  DenseLayer,
+  Sequential,
+  setupBackend,
+  SigmoidLayer,
+  tensor1D,
+  tensor2D,
+} from "../../mod.ts";
+
+import { parse } from "https://deno.land/std@0.204.0/csv/parse.ts";
+
+// Import helpers for metrics
+import {
+  ClassificationReport,
+  // Split the dataset
+  useSplit,
+} from "https://deno.land/x/vectorizer@v0.2.3/mod.ts";
+
+// Define classes
+const classes = ["Setosa", "Versicolor"];
+
+// Read the training dataset
+const _data = Deno.readTextFileSync("examples/classification/binary_iris.csv");
+const data = parse(_data);
+
+// Get the predictors (x) and targets (y)
+const x = data.map((fl) => fl.slice(0, 4).map(Number));
+const y = data.map((fl) => classes.indexOf(fl[4]));
+
+// Split the dataset for training and testing
+const [train, test] = useSplit({ ratio: [7, 3], shuffle: true }, x, y) as [
+  [typeof x, typeof y],
+  [typeof x, typeof y],
+];
+
+// Setup the CPU backend for Netsaur
+await setupBackend(CPU);
+
+// Create a sequential neural network
+const net = new Sequential({
+  // Set number of minibatches to 4
+  // Set size of output to 4
+  size: [4, 4],
+
+  // Disable logging during training
+  silent: false,
+
+  // Define each layer of the network
+  layers: [
+    // A dense layer with 4 neurons
+    DenseLayer({ size: [4] }),
+    // A sigmoid activation layer
+    SigmoidLayer(),
+    // A dense layer with 1 neuron
+    DenseLayer({ size: [1] }),
+    // Another sigmoid layer
+    SigmoidLayer(),
+  ],
+  // We are using Log Loss for finding cost
+  cost: Cost.BinCrossEntropy,
+});
+
+const time = performance.now();
+
+// Train the network
+net.train(
+  [
+    {
+      inputs: tensor2D(train[0]),
+      outputs: tensor2D(train[1].map((x) => [x])),
+    },
+  ],
+  // Train for 150 epochs
+  150,
+  1,
+  // Use a smaller learning rate
+  0.02
+);
+
+console.log(`training time: ${performance.now() - time}ms`);
+
+const res = await Promise.all(
+  test[0].map((input) => net.predict(tensor1D(input))),
+);
+const y1 = res.map((x) => x.data[0] < 0.5 ? 0 : 1);
+const cMatrix = new ClassificationReport(test[1], y1);
+console.log("Confusion Matrix: ", cMatrix);