Skip to content

Commit ffce979

Browse files
mnielsenmnielsen
committed
Adding generate_gradient.py
1 parent 60d3f5a commit ffce979

34 files changed

+2336
-2306
lines changed

‎.gitignore

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
*~
2-
*.org
3-
*.pkl
1+
*~
2+
*.org
3+
*.pkl
44
*.pyc

‎README.md

+33-33
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,33 @@
1-
# Code samples for "Neural Networks and Deep Learning"
2-
3-
This repository contains code samples for my (forthcoming) book on
4-
"Neural Networks and Deep Learning".
5-
6-
As the code is written to accompany the book, I don't intend to add
7-
new features. However, bug reports are welcome, and you should feel
8-
free to fork and modify the code.
9-
10-
## License
11-
12-
MIT License
13-
14-
Copyright (c) 2012-2013 Michael Nielsen
15-
16-
Permission is hereby granted, free of charge, to any person obtaining
17-
a copy of this software and associated documentation files (the
18-
"Software"), to deal in the Software without restriction, including
19-
without limitation the rights to use, copy, modify, merge, publish,
20-
distribute, sublicense, and/or sell copies of the Software, and to
21-
permit persons to whom the Software is furnished to do so, subject to
22-
the following conditions:
23-
24-
The above copyright notice and this permission notice shall be
25-
included in all copies or substantial portions of the Software.
26-
27-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28-
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29-
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30-
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
31-
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
32-
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
33-
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
1+
# Code samples for "Neural Networks and Deep Learning"
2+
3+
This repository contains code samples for my (forthcoming) book on
4+
"Neural Networks and Deep Learning".
5+
6+
As the code is written to accompany the book, I don't intend to add
7+
new features. However, bug reports are welcome, and you should feel
8+
free to fork and modify the code.
9+
10+
## License
11+
12+
MIT License
13+
14+
Copyright (c) 2012-2013 Michael Nielsen
15+
16+
Permission is hereby granted, free of charge, to any person obtaining
17+
a copy of this software and associated documentation files (the
18+
"Software"), to deal in the Software without restriction, including
19+
without limitation the rights to use, copy, modify, merge, publish,
20+
distribute, sublicense, and/or sell copies of the Software, and to
21+
permit persons to whom the Software is furnished to do so, subject to
22+
the following conditions:
23+
24+
The above copyright notice and this permission notice shall be
25+
included in all copies or substantial portions of the Software.
26+
27+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
31+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
32+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
33+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

‎code/mnist_average_darkness.py

+64-64
Original file line numberDiff line numberDiff line change
@@ -1,64 +1,64 @@
1-
"""
2-
mnist_average_darkness
3-
~~~~~~~~~~~~~~~~~~~~~~
4-
5-
A naive classifier for recognizing handwritten digits from the MNIST
6-
data set. The program classifies digits based on how dark they are
7-
--- the idea is that digits like "1" tend to be less dark than digits
8-
like "8", simply because the latter has a more complex shape. When
9-
shown an image the classifier returns whichever digit in the training
10-
data had the closest average darkness.
11-
12-
The program works in two steps: first it trains the classifier, and
13-
then it applies the classifier to the MNIST test data to see how many
14-
digits are correctly classified.
15-
16-
Needless to say, this isn't a very good way of recognizing handwritten
17-
digits! Still, it's useful to show what sort of performance we get
18-
from naive ideas."""
19-
20-
#### Libraries
21-
# Standard library
22-
from collections import defaultdict
23-
24-
# My libraries
25-
import mnist_loader
26-
27-
def main():
28-
training_data, validation_data, test_data = mnist_loader.load_data()
29-
# training phase: compute the average darknesses for each digit,
30-
# based on the training data
31-
avgs = avg_darknesses(training_data)
32-
# testing phase: see how many of the test images are classified
33-
# correctly
34-
num_correct = sum(int(guess_digit(image, avgs) == digit)
35-
for image, digit in zip(test_data[0], test_data[1]))
36-
print "Baseline classifier using average darkness of image."
37-
print "%s of %s values correct." % (num_correct, len(test_data[1]))
38-
39-
def avg_darknesses(training_data):
40-
""" Return a defaultdict whose keys are the digits 0 through 9.
41-
For each digit we compute a value which is the average darkness of
42-
training images containing that digit. The darkness for any
43-
particular image is just the sum of the darknesses for each pixel."""
44-
digit_counts = defaultdict(int)
45-
darknesses = defaultdict(float)
46-
for image, digit in zip(training_data[0], training_data[1]):
47-
digit_counts[digit] += 1
48-
darknesses[digit] += sum(image)
49-
avgs = defaultdict(float)
50-
for digit, n in digit_counts.iteritems():
51-
avgs[digit] = darknesses[digit] / n
52-
return avgs
53-
54-
def guess_digit(image, avgs):
55-
"""Return the digit whose average darkness in the training data is
56-
closest to the darkness of ``image``. Note that ``avgs`` is
57-
assumed to be a defaultdict whose keys are 0...9, and whose values
58-
are the corresponding average darknesses across the training data."""
59-
darkness = sum(image)
60-
distances = {k: abs(v-darkness) for k, v in avgs.iteritems()}
61-
return min(distances, key=distances.get)
62-
63-
if __name__ == "__main__":
64-
main()
1+
"""
2+
mnist_average_darkness
3+
~~~~~~~~~~~~~~~~~~~~~~
4+
5+
A naive classifier for recognizing handwritten digits from the MNIST
6+
data set. The program classifies digits based on how dark they are
7+
--- the idea is that digits like "1" tend to be less dark than digits
8+
like "8", simply because the latter has a more complex shape. When
9+
shown an image the classifier returns whichever digit in the training
10+
data had the closest average darkness.
11+
12+
The program works in two steps: first it trains the classifier, and
13+
then it applies the classifier to the MNIST test data to see how many
14+
digits are correctly classified.
15+
16+
Needless to say, this isn't a very good way of recognizing handwritten
17+
digits! Still, it's useful to show what sort of performance we get
18+
from naive ideas."""
19+
20+
#### Libraries
21+
# Standard library
22+
from collections import defaultdict
23+
24+
# My libraries
25+
import mnist_loader
26+
27+
def main():
28+
training_data, validation_data, test_data = mnist_loader.load_data()
29+
# training phase: compute the average darknesses for each digit,
30+
# based on the training data
31+
avgs = avg_darknesses(training_data)
32+
# testing phase: see how many of the test images are classified
33+
# correctly
34+
num_correct = sum(int(guess_digit(image, avgs) == digit)
35+
for image, digit in zip(test_data[0], test_data[1]))
36+
print "Baseline classifier using average darkness of image."
37+
print "%s of %s values correct." % (num_correct, len(test_data[1]))
38+
39+
def avg_darknesses(training_data):
40+
""" Return a defaultdict whose keys are the digits 0 through 9.
41+
For each digit we compute a value which is the average darkness of
42+
training images containing that digit. The darkness for any
43+
particular image is just the sum of the darknesses for each pixel."""
44+
digit_counts = defaultdict(int)
45+
darknesses = defaultdict(float)
46+
for image, digit in zip(training_data[0], training_data[1]):
47+
digit_counts[digit] += 1
48+
darknesses[digit] += sum(image)
49+
avgs = defaultdict(float)
50+
for digit, n in digit_counts.iteritems():
51+
avgs[digit] = darknesses[digit] / n
52+
return avgs
53+
54+
def guess_digit(image, avgs):
55+
"""Return the digit whose average darkness in the training data is
56+
closest to the darkness of ``image``. Note that ``avgs`` is
57+
assumed to be a defaultdict whose keys are 0...9, and whose values
58+
are the corresponding average darknesses across the training data."""
59+
darkness = sum(image)
60+
distances = {k: abs(v-darkness) for k, v in avgs.iteritems()}
61+
return min(distances, key=distances.get)
62+
63+
if __name__ == "__main__":
64+
main()

‎code/mnist_loader.py

+85-85
Original file line numberDiff line numberDiff line change
@@ -1,85 +1,85 @@
1-
"""
2-
mnist_loader
3-
~~~~~~~~~~~~
4-
5-
A library to load the MNIST image data. For details of the data
6-
structures that are returned, see the doc strings for ``load_data``
7-
and ``load_data_wrapper``. In practice, ``load_data_wrapper`` is the
8-
function usually called by our neural network code.
9-
"""
10-
11-
#### Libraries
12-
# Standard library
13-
import cPickle
14-
import gzip
15-
16-
# Third-party libraries
17-
import numpy as np
18-
19-
def load_data():
20-
"""Return the MNIST data as a tuple containing the training data,
21-
the validation data, and the test data.
22-
23-
The ``training_data`` is returned as a tuple with two entries.
24-
The first entry contains the actual training images. This is a
25-
numpy ndarray with 50,000 entries. Each entry is, in turn, a
26-
numpy ndarray with 784 values, representing the 28 * 28 = 784
27-
pixels in a single MNIST image.
28-
29-
The second entry in the ``training_data`` tuple is a numpy ndarray
30-
containing 50,000 entries. Those entries are just the digit
31-
values (0...9) for the corresponding images contained in the first
32-
entry of the tuple.
33-
34-
The ``validation_data`` and ``test_data`` are similar, except
35-
each contains only 10,000 images.
36-
37-
This is a nice data format, but for use in neural networks it's
38-
helpful to modify the format of the ``training_data`` a little.
39-
That's done in the wrapper function ``load_data_wrapper()``, see
40-
below.
41-
"""
42-
f = gzip.open('../data/mnist.pkl.gz', 'rb')
43-
training_data, validation_data, test_data = cPickle.load(f)
44-
f.close()
45-
return (training_data, validation_data, test_data)
46-
47-
def load_data_wrapper():
48-
"""Return a tuple containing ``(training_data, validation_data,
49-
test_data)``. Based on ``load_data``, but the format is more
50-
convenient for use in our implementation of neural networks.
51-
52-
In particular, ``training_data`` is a list containing 50,000
53-
2-tuples ``(x, y)``. ``x`` is a 784-dimensional numpy.ndarray
54-
containing the input image. ``y`` is a 10-dimensional
55-
numpy.ndarray representing the unit vector corresponding to the
56-
correct digit for ``x``.
57-
58-
``validation_data`` and ``test_data`` are lists containing 10,000
59-
2-tuples ``(x, y)``. In each case, ``x`` is a 784-dimensional
60-
numpy.ndarry containing the input image, and ``y`` is the
61-
corresponding classification, i.e., the digit values (integers)
62-
corresponding to ``x``.
63-
64-
Obviously, this means we're using slightly different formats for
65-
the training data and the validation / test data. These formats
66-
turn out to be the most convenient for use in our neural network
67-
code."""
68-
tr_d, va_d, te_d = load_data()
69-
training_inputs = [np.reshape(x, (784, 1)) for x in tr_d[0]]
70-
training_results = [vectorized_result(y) for y in tr_d[1]]
71-
training_data = zip(training_inputs, training_results)
72-
validation_inputs = [np.reshape(x, (784, 1)) for x in va_d[0]]
73-
validation_data = zip(validation_inputs, va_d[1])
74-
test_inputs = [np.reshape(x, (784, 1)) for x in te_d[0]]
75-
test_data = zip(test_inputs, te_d[1])
76-
return (training_data, validation_data, test_data)
77-
78-
def vectorized_result(j):
79-
"""Return a 10-dimensional unit vector with a 1.0 in the jth
80-
position and zeroes elsewhere. This is used to convert a digit
81-
(0...9) into a corresponding desired output from the neural
82-
network."""
83-
e = np.zeros((10, 1))
84-
e[j] = 1.0
85-
return e
1+
"""
2+
mnist_loader
3+
~~~~~~~~~~~~
4+
5+
A library to load the MNIST image data. For details of the data
6+
structures that are returned, see the doc strings for ``load_data``
7+
and ``load_data_wrapper``. In practice, ``load_data_wrapper`` is the
8+
function usually called by our neural network code.
9+
"""
10+
11+
#### Libraries
12+
# Standard library
13+
import cPickle
14+
import gzip
15+
16+
# Third-party libraries
17+
import numpy as np
18+
19+
def load_data():
20+
"""Return the MNIST data as a tuple containing the training data,
21+
the validation data, and the test data.
22+
23+
The ``training_data`` is returned as a tuple with two entries.
24+
The first entry contains the actual training images. This is a
25+
numpy ndarray with 50,000 entries. Each entry is, in turn, a
26+
numpy ndarray with 784 values, representing the 28 * 28 = 784
27+
pixels in a single MNIST image.
28+
29+
The second entry in the ``training_data`` tuple is a numpy ndarray
30+
containing 50,000 entries. Those entries are just the digit
31+
values (0...9) for the corresponding images contained in the first
32+
entry of the tuple.
33+
34+
The ``validation_data`` and ``test_data`` are similar, except
35+
each contains only 10,000 images.
36+
37+
This is a nice data format, but for use in neural networks it's
38+
helpful to modify the format of the ``training_data`` a little.
39+
That's done in the wrapper function ``load_data_wrapper()``, see
40+
below.
41+
"""
42+
f = gzip.open('../data/mnist.pkl.gz', 'rb')
43+
training_data, validation_data, test_data = cPickle.load(f)
44+
f.close()
45+
return (training_data, validation_data, test_data)
46+
47+
def load_data_wrapper():
48+
"""Return a tuple containing ``(training_data, validation_data,
49+
test_data)``. Based on ``load_data``, but the format is more
50+
convenient for use in our implementation of neural networks.
51+
52+
In particular, ``training_data`` is a list containing 50,000
53+
2-tuples ``(x, y)``. ``x`` is a 784-dimensional numpy.ndarray
54+
containing the input image. ``y`` is a 10-dimensional
55+
numpy.ndarray representing the unit vector corresponding to the
56+
correct digit for ``x``.
57+
58+
``validation_data`` and ``test_data`` are lists containing 10,000
59+
2-tuples ``(x, y)``. In each case, ``x`` is a 784-dimensional
60+
numpy.ndarry containing the input image, and ``y`` is the
61+
corresponding classification, i.e., the digit values (integers)
62+
corresponding to ``x``.
63+
64+
Obviously, this means we're using slightly different formats for
65+
the training data and the validation / test data. These formats
66+
turn out to be the most convenient for use in our neural network
67+
code."""
68+
tr_d, va_d, te_d = load_data()
69+
training_inputs = [np.reshape(x, (784, 1)) for x in tr_d[0]]
70+
training_results = [vectorized_result(y) for y in tr_d[1]]
71+
training_data = zip(training_inputs, training_results)
72+
validation_inputs = [np.reshape(x, (784, 1)) for x in va_d[0]]
73+
validation_data = zip(validation_inputs, va_d[1])
74+
test_inputs = [np.reshape(x, (784, 1)) for x in te_d[0]]
75+
test_data = zip(test_inputs, te_d[1])
76+
return (training_data, validation_data, test_data)
77+
78+
def vectorized_result(j):
79+
"""Return a 10-dimensional unit vector with a 1.0 in the jth
80+
position and zeroes elsewhere. This is used to convert a digit
81+
(0...9) into a corresponding desired output from the neural
82+
network."""
83+
e = np.zeros((10, 1))
84+
e[j] = 1.0
85+
return e

0 commit comments

Comments
 (0)