修改导函数

lrenc · lrenc · commit 2f5c851a9633 · 2020-08-21T14:49:09.000+08:00
diff --git a/README.md b/README.md
@@ -1,5 +1,19 @@
 # 简单的全连接神经网络实现
 
-优化器使用批量梯度下降，损失函数使用交叉熵函数。
+优化器使用批量梯度下降，损失函数使用交叉熵函数，支持sigmoid与tanh两种激活函数
 
 <img src="https://raw.githubusercontent.com/lrenc/FoolNet/master/foolnet.png" width="50%">
+
+```javascript
+function fn(x1, x2) {
+  let y = 0;
+  if (x1 > 0.5 && x2 > 0.5) {
+    y = 1;
+  } else if (x1 < 0.5 && x2 < 0.5) {
+    y = 1;
+  }
+  return y;
+}
+```
+
+<img src="https://raw.githubusercontent.com/lrenc/FoolNet/master/result.png" width="50%">
diff --git a/index.html b/index.html
@@ -5,6 +5,8 @@
     <title>Foo Net</title>
   </head>
   <body>
-    <div id="root"></div>
+    <div id="root">
+      <canvas id="canvas" width="300" height="300"></canvas>
+    </div>
   </body>
 </html>
diff --git a/result.png b/result.png
diff --git a/src/activation.js b/src/activation.js
@@ -1,3 +1,8 @@
+
 export function sigmoid(x) {
   return 1 / (1 + Math.exp(-x));
 }
+
+export function tanh(x) {
+  return (Math.exp(x) - Math.exp(-x)) / (Math.exp(x) + Math.exp(-x))
+}
diff --git a/src/data.js b/src/data.js
@@ -1,14 +1,34 @@
 
+function fn(x1, x2) {
+  let y = 0;
+  if (x1 > 0.5 && x2 > 0.5) {
+    y = 1;
+  } else if (x1 < 0.5 && x2 < 0.5) {
+    y = 1;
+  }
+  return y;
+}
+
 export function getTrainData() {
   const inputs = [];
   const labels = [];
-  for (let i = 0; i < 300; i ++) {
+  for (let i = 0; i < 100; i ++) {
+    let x1 = Math.random();
+    let x2 = Math.random();
+    let y = fn(x1, x2);
+    inputs.push([x1, x2]);
+    labels.push([y]);
+  }
+  return [inputs, labels];
+}
+
+export function getTestData() {
+  const inputs = [];
+  const labels = [];
+  for (let i = 0; i < 8000; i ++) {
     let x1 = Math.random();
     let x2 = Math.random();
-    let y = 0;
-    if (x1 <= 0.5 && x2 <= 0.5) {
-      y = 1;
-    }
+    let y = fn(x1, x2);
     inputs.push([x1, x2]);
     labels.push([y]);
   }
diff --git a/src/derivative.js b/src/derivative.js
@@ -0,0 +1,8 @@
+// 激活函数的导函数
+export function sigmoid(a) {
+  return a * (1 - a);
+}
+
+export function tanh(a) {
+  return 1 - Math.pow(a, 2);
+}
diff --git a/src/index.css b/src/index.css
@@ -0,0 +1,4 @@
+canvas {
+  width: 150px;
+  height: 150px;
+}
diff --git a/src/index.js b/src/index.js
@@ -1,20 +1,19 @@
 import Layer from './layer';
 import Model from './Model';
-import { getTrainData } from './data';
-import './test';
+import { getTrainData, getTestData } from './data';
+import './index.css';
 
 const l0 = new Layer({
-  type: 'inputLayer',
   units: 2
 });
 
 const l1 = new Layer({
-  activation: 'sigmoid',
+  activation: 'tanh',
   units: 3
 });
 
 const l2 = new Layer({
-  activation: 'sigmoid',
+  activation: 'tanh',
   units: 3
 });
 
@@ -26,16 +25,42 @@ const l3 = new Layer({
 const model = new Model([l0, l1, l2, l3]);
 
 let step = 2;
-for (let i = 0; i < 5000; i ++) {
+for (let i = 0; i < 8000; i ++) {
   if (i % 1000 === 0) {
     step /= 2;
   }
   model.fit(...getTrainData(), { step });
 }
-console.log(model.predict([0.1, 0.1])); // 1
-console.log(model.predict([0.2, 0.2])); // 1
-console.log(model.predict([0.3, 0.3])); // 1
-console.log(model.predict([0.9, 0.1])); // 0
-console.log(model.predict([0.1, 0.9])); // 0
-console.log(model.predict([0.6, 0.6])); // 0
-console.log(model.predict([0.9, 0.9])); // 0
+
+const canvas = document.getElementById('canvas');
+const width = canvas.width;
+const height = canvas.height;
+const ctx = canvas.getContext('2d');
+
+function drawPoint(x, y, color) {
+  ctx.beginPath();
+  ctx.arc(x, y, 3, 0, Math.PI * 2, true);
+  ctx.fillStyle = color;
+  ctx.fill();
+}
+
+function test() {
+  const [inputs, labels] = getTestData();
+  for (let i = 0; i < inputs.length; i ++) {
+    const input = inputs[i];
+    const label = labels[i];
+    // const output = model.predict(input)[0];
+    const output = model.predict(input)[0];
+    const x1 = input[0] * width;
+    const x2 = input[1] * height;
+    let color;
+    if (output <= 0.5) {
+      color = `rgb(255, 0, 0)`;
+    } else {
+      color = `rgb(0, 0, 255)`;
+    }
+    drawPoint(x1, x2, color);
+  }
+}
+
+test();
diff --git a/src/layer.js b/src/layer.js
@@ -1,23 +1,17 @@
 import Neure from './neure';
-import * as fn from './activation';
-
-const INPUT_LAYER = 'inputLayer';
 
 export default class Layer {
-
   constructor(params) {
     const { units, activation, type } = params;
     this.units = units;
     this.neures = Array(units).fill(null);
-    this.activation = null;
-    if (fn[activation]) {
-      this.activation = fn[activation];
-    }
-    if (type === INPUT_LAYER) {
-      this.neures = this.neures.map(() => {
-        return new Neure(1, null, true);
-      });
-    }
+    this.activation = activation;
+  }
+
+  init() {
+    this.neures = this.neures.map(() => {
+      return new Neure(1, null, true);
+    });
   }
 
   apply(layer) {
diff --git a/src/model.js b/src/model.js
@@ -1,8 +1,12 @@
 
-import { dot, sub, sum, power, last, transpose, div, mul } from './util';
+import Layer from './layer';
+import * as derivate from './derivative';
+import { dot, sub, sum, power, transpose, div, mul, add } from './util';
 
 export default class Network {
   constructor(layers) {
+    const layer = layers[0];
+    layer.init();
     for (let i = 0; i < layers.length - 1; i ++) {
       const curr = layers[i];
       const next = layers[i + 1];
@@ -43,17 +47,28 @@ export default class Network {
     const size = inputs.length;
     let layers = this.layers.slice();
     let p = this.multiPredict(inputs);
-    let dz = sub(p, labels); // 二维数组
     let lastLayer = layers.pop();
+    let activation = lastLayer.activation;
+    let dz;
+    if (activation === 'sigmoid') {
+      dz = sub(p, labels); // 二维数组
+    } else {
+      dz = div(mul(sub(p, labels), add(1, p)), p);
+    }
     p = this.multiPredict(inputs, layers);
     let dw = div(dot(transpose(dz), p), size); // A1
     let db = sum(dz) / size;
     dws.unshift(dw);
     dbs.unshift(db);
     while (true) {
       const w = lastLayer.neures.map(neure => neure.weights);
-      dz = mul(dot(dz, w), sub(1, power(p, 2)));
       lastLayer = layers.pop();
+      activation = lastLayer.activation;
+      if (activation === 'sigmoid') {
+        dz = mul(dot(dz, w), sub(p, power(p, 2)));
+      } else {
+        dz = mul(dot(dz, w), sub(1, power(p, 2)));
+      }
       if (layers.length === 0) {
         break;
       }
diff --git a/src/neure.js b/src/neure.js
@@ -1,16 +1,17 @@
+import * as active from './activation';
 
 function random() {
   return Math.random() * 2 - 1;
 }
 
 export default class Neure {
-  constructor(length, activation, flag) {
+  constructor(length, activation, isFirstLayer = false) {
     this.length = length;
     this.activation = activation;
-    this.b = flag ? 0 : random();
+    this.b = 0;
     this.weights = [];
     for (let i = 0; i < length; i ++) { // 随机初始化权重
-      const weight = flag ? 1 : random();
+      const weight = isFirstLayer ? 1 : random();
       this.weights.push(weight);
     }
   }
@@ -27,8 +28,9 @@ export default class Neure {
     }
     result += this.b;
     // 激活函数处理
-    if (this.activation) {
-      result = this.activation(result);
+    const fn = this.activation && active[this.activation];
+    if (fn) {
+      result = active[this.activation](result);
     }
     return result;
   }
diff --git a/src/test.js b/src/test.js
@@ -1,6 +1,3 @@
-import { dot, sub, sum, power, last, transpose, div, mul } from './util';
+import { sigmoid } from './activation';
 
-const a = [[3, 3, 3], [6, 6, 6]];
-const b = [[3, 2, 1], [5, 5, 4]];
-// const b = 3
-console.log(sub(a, b));
+const input = [0.5, 0.5];
diff --git a/src/util.js b/src/util.js
@@ -38,7 +38,7 @@ export function dot(a, b) {
   return c;
 }
 
-// 二维数组相减
+// 减
 export function sub(a, b) {
   ([a, b] = format(a, b));
   const result = [];
@@ -51,6 +51,18 @@ export function sub(a, b) {
   return result;
 }
 
+export function add(a, b) {
+  ([a, b] = format(a, b));
+  const result = [];
+  for (let i = 0; i < a.length; i ++) {
+    result[i] = [];
+    for (let j = 0; j < a[i].length; j ++) {
+      result[i].push(a[i][j] + b[i][j]);
+    }
+  }
+  return result;
+}
+
 // 数组转置
 export function transpose(arr) {
   const result = [];
@@ -84,6 +96,7 @@ export function power(arr, n) {
   return result;
 }
 
+// 除
 export function div(a, b) {
   ([a, b] = format(a, b));
   const result = [];
@@ -96,6 +109,7 @@ export function div(a, b) {
   return result;
 }
 
+// 乘
 export function mul(a, b) {
   ([a, b] = format(a, b));
   const result = [];

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,8 @@`
	`1`	`+`
`1`	`2`	`export function sigmoid(x) {`
`2`	`3`	`return 1 / (1 + Math.exp(-x));`
`3`	`4`	`}`
	`5`	`+`
	`6`	`+export function tanh(x) {`
	`7`	`+ return (Math.exp(x) - Math.exp(-x)) / (Math.exp(x) + Math.exp(-x))`
	`8`	`+}`