Skip to content

Commit e127926

Browse files
jperljperl
authored
Merge pull request #5 from hedgehoglabs/numpy-linear
Add xor linear nn from scratch in numpy
2 parents b130a44 + 97d0c02 commit e127926

File tree

2 files changed

+194
-0
lines changed

2 files changed

+194
-0
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
.ipynb_checkpoints/
12
.vscode/
23
__pycache__/
34
data/

‎notebooks/Linear XOR.ipynb

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 1,
6+
"metadata": {},
7+
"outputs": [
8+
{
9+
"name": "stdout",
10+
"output_type": "stream",
11+
"text": [
12+
"bits [0 0] --> parity [0]\n",
13+
"bits [0 1] --> parity [1]\n",
14+
"bits [1 0] --> parity [1]\n",
15+
"bits [1 1] --> parity [0]\n"
16+
]
17+
}
18+
],
19+
"source": [
20+
"import numpy as np\n",
21+
"\n",
22+
"np.random.seed(0)\n",
23+
"\n",
24+
"# bits are our inputs\n",
25+
"X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])\n",
26+
"\n",
27+
"# parities are our labels\n",
28+
"Y = np.array([[0], [1], [1], [0]])\n",
29+
"\n",
30+
"for i, bits in enumerate(X):\n",
31+
" print(f'bits {bits} --> parity {Y[i]}')"
32+
]
33+
},
34+
{
35+
"cell_type": "code",
36+
"execution_count": 2,
37+
"metadata": {},
38+
"outputs": [
39+
{
40+
"name": "stdout",
41+
"output_type": "stream",
42+
"text": [
43+
"delta sigmoid [[0.25 ]\n",
44+
" [0.19661193]\n",
45+
" [0.19661193]\n",
46+
" [0.25 ]]\n",
47+
"delta sigmoid analytical [[0.25000002]\n",
48+
" [0.19661195]\n",
49+
" [0.19661195]\n",
50+
" [0.25000002]]\n"
51+
]
52+
}
53+
],
54+
"source": [
55+
"def sigmoid(x):\n",
56+
" return 1 / (1 + np.exp(-x))\n",
57+
"\n",
58+
"def delta_sigmoid(x):\n",
59+
" # to derive use the +1 trick from http://cs231n.github.io/optimization-2/\n",
60+
" return sigmoid(x) * (1 - sigmoid(x))\n",
61+
"\n",
62+
"def analytical_gradient(f, x):\n",
63+
" h = 1e-9\n",
64+
" return (f(x + h) - f(x)) / h\n",
65+
"\n",
66+
"print('delta sigmoid', delta_sigmoid(Y))\n",
67+
"print('delta sigmoid analytical', analytical_gradient(sigmoid, Y))"
68+
]
69+
},
70+
{
71+
"cell_type": "code",
72+
"execution_count": 3,
73+
"metadata": {},
74+
"outputs": [
75+
{
76+
"name": "stdout",
77+
"output_type": "stream",
78+
"text": [
79+
"loss 0.14451072667400197\n",
80+
"loss 0.007930633168167129\n",
81+
"loss 0.0031754754752917323\n",
82+
"loss 0.0021824385490060365\n"
83+
]
84+
}
85+
],
86+
"source": [
87+
"# X [4,2]\n",
88+
"input_dim = X.shape[-1]\n",
89+
"# Y [4,1]\n",
90+
"output_dim = Y.shape[-1]\n",
91+
"hidden_units = 2\n",
92+
"lr = 0.1\n",
93+
"\n",
94+
"# [2,2]\n",
95+
"Whidden = np.random.uniform(size=(input_dim, hidden_units)) # hidden layer\n",
96+
"\n",
97+
"# [2,1]\n",
98+
"Woutput = np.random.uniform(size=(hidden_units, output_dim)) # output layer\n",
99+
"\n",
100+
"for step in range(10000):\n",
101+
" # forward pass\n",
102+
" # loss = loss(output(activation(hidden(X))))\n",
103+
"\n",
104+
" # hidden(X) [4,2]\n",
105+
" hidden = X.dot(Whidden)\n",
106+
" \n",
107+
" # activation(hidden) [4,2]\n",
108+
" activation = sigmoid(hidden)\n",
109+
"\n",
110+
" # output(activation) [4,2]x[2,1] -> [4,1]\n",
111+
" output = activation.dot(Woutput)\n",
112+
"\n",
113+
" # loss(output) [4,1]\n",
114+
" loss = 0.5 * (output - Y)**2\n",
115+
" if step % 2500 == 0:\n",
116+
" print('loss', np.mean(loss))\n",
117+
" \n",
118+
" # backward pass\n",
119+
" # loss'(output) [4,1]\n",
120+
" dloss_output = output - Y\n",
121+
" \n",
122+
" # loss'(activation) = loss'(output) * output'(activation)\n",
123+
" # [4,1]x[1,2] -> [4,2]\n",
124+
" dloss_activation = dloss_output.dot(Woutput.T)\n",
125+
"\n",
126+
" # loss'(hidden) = loss'(activation) * activation'(hidden)\n",
127+
" # [4,2]*[4,2] -> [4,2]\n",
128+
" dloss_hidden = dloss_activation * delta_sigmoid(hidden)\n",
129+
"\n",
130+
" # Take a small step in the opposite direction of the gradient \n",
131+
" \n",
132+
" # loss'(Woutput) = loss'(output) * output'(Woutput)\n",
133+
" # [2,4]x[4,1] -> [2,1]\n",
134+
" dloss_woutput = activation.T.dot(dloss_output)\n",
135+
" Woutput -= dloss_woutput * lr\n",
136+
"\n",
137+
" # loss'(Whidden) = loss'(hidden) * hidden'(Whidden)\n",
138+
" # [2,4]x[4,2] -> [2,2]\n",
139+
" dloss_whidden = X.T.dot(dloss_hidden) \n",
140+
" Whidden -= dloss_whidden * lr"
141+
]
142+
},
143+
{
144+
"cell_type": "code",
145+
"execution_count": 4,
146+
"metadata": {},
147+
"outputs": [
148+
{
149+
"name": "stdout",
150+
"output_type": "stream",
151+
"text": [
152+
"prediction [-0.08500212] -> label [0]\n",
153+
"prediction [0.98169372] -> label [1]\n",
154+
"prediction [0.98169457] -> label [1]\n",
155+
"prediction [0.07744216] -> label [0]\n"
156+
]
157+
}
158+
],
159+
"source": [
160+
"for i, prediction in enumerate(output):\n",
161+
" print(f'prediction {prediction} -> label {Y[i]}')"
162+
]
163+
},
164+
{
165+
"cell_type": "code",
166+
"execution_count": null,
167+
"metadata": {},
168+
"outputs": [],
169+
"source": []
170+
}
171+
],
172+
"metadata": {
173+
"kernelspec": {
174+
"display_name": "Python 3",
175+
"language": "python",
176+
"name": "python3"
177+
},
178+
"language_info": {
179+
"codemirror_mode": {
180+
"name": "ipython",
181+
"version": 3
182+
},
183+
"file_extension": ".py",
184+
"mimetype": "text/x-python",
185+
"name": "python",
186+
"nbconvert_exporter": "python",
187+
"pygments_lexer": "ipython3",
188+
"version": "3.6.3"
189+
}
190+
},
191+
"nbformat": 4,
192+
"nbformat_minor": 2
193+
}

0 commit comments

Comments
 (0)