Date:2018年9月29记
数据及彭亮给出的数据集和源码在下面网址:
https://github.com/mnielsen/neural-networks-and-deep-learning
在放上代码之前要先把踩过的坑补一下,不白费国庆前掉发找bug
[bug-1]
nabla_w = [nw + dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]
ValueError: operands could not be broadcast together with shapes (30,784) (30,)
Traceback (most recent call last):
File "/home/kongze/PycharmProjects/MNIST_Demo/main.py", line 6, in <module>
net.SGD(training_data, 1, 10, 3.0, test_data=test_data)
File "/home/kongze/PycharmProjects/MNIST_Demo/network.py", line 38, in SGD
self.update_mini_batch(mini_batch, eta)
File "/home/kongze/PycharmProjects/MNIST_Demo/network.py", line 50, in update_mini_batch
nabla_w = [nw + dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]
File "/home/kongze/PycharmProjects/MNIST_Demo/network.py", line 50, in <listcomp>
nabla_w = [nw + dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]
ValueError: operands could not be broadcast together with shapes (30,784) (30,)
这是百度,google都解决不了,花了好久才发现,这个bug在def backprop函数中的
#非输出层
for l in range(2, self.num_layers):
z = zs[-l]
sp = sigmoid_prime(z)
delta = np.dot(self.weights[-l+1].transpose(), delta) * sp
nabla_b[-l] = delta
nabla_w[-l] = np.dot(delta, activation[-l-1].transpose())
return (nabla_b,nabla_w)
activations,硬生生的写成了activation,绝情的少了一个s,花去了2个小时找这个失误,真是差之毫厘谬以千里
[bug-2] python2 与python3这种爹坑儿子的bug
TypeError: object of type 'zip' has no len()
Traceback (most recent call last):
training data
<class 'zip'>
File "/home/kongze/PycharmProjects/MNIST_Demo/demo.py", line 9, in <module>
print(len(training_data))
TypeError: object of type 'zip' has no len()
完美解决方案就是:在mnist_loader模块中
def load_data_wrapper():
tr_d, va_d, te_d = load_data()
#训练集
training_inputs = [np.reshape(x, (784, 1)) for x in tr_d[0]]
training_labels = [vectorized_label(y) for y in tr_d[1]]
training_data = zip(training_inputs, training_labels)
training_data那一行的zip前面加入list,改成 list(zip....).
[01-MNIST_loader.py]
import pickle
import gzip
import numpy as np
def load_data():
file = gzip.open('data/mnist.pkl.gz', 'rb')
training_data, validation_data, test_data = pickle.load(file, encoding='bytes')
file.close()
return (training_data, validation_data, test_data)
def load_data_wrapper():
tr_d, va_d, te_d = load_data()
#训练集
training_inputs = [np.reshape(x, (784, 1)) for x in tr_d[0]]
training_labels = [vectorized_label(y) for y in tr_d[1]]
training_data = list(zip(training_inputs, training_labels))
#验证集
validation_inputs = [np.reshape(x, (784, 1)) for x in va_d[0]]
validation_data = list(zip(validation_inputs, va_d[1]))
#测试集
test_inputs = [np.reshape(x, (784, 1)) for x in te_d[0]]
test_data = list(zip(test_inputs, te_d[1]))
return (training_data, validation_data, test_data)
def vectorized_label(j):
e = np.zeros((10, 1))
e[j] = 1.0
return e
[02-demo.py]
import mnist_loader
import network
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
print("training data")
print(type(training_data))
print(len(training_data))
print(training_data[0][0].shape)
print(training_data[0][1].shape)
[03-network.py]
import random
import numpy as np
class Network(object):
#构造初始化神经网络
def __init__(self, sizes):
self.num_layers = len(sizes)
self.sizes = sizes
self.biases = [np.random.randn(y, 1) for y in sizes[1:]]
self.weights = [np.random.randn(y, x) for x, y in zip(sizes[:-1], sizes[1:])]
#前向传播
def feedforward(self, a):
for b, w in zip(self.biases, self.weights):
a = sigmoid(np.dot(w, a)+b)
return a
#随机梯度下降(训练数据,迭代次数,小样本数量,学习率,是否有测试集)
def SGD(self, training_data, epochs, mini_batch_size, eta, test_data=None):
if test_data:
n_test = len(test_data)
n = len(training_data) #训练数据大小
#迭代过程
for j in range(epochs):
random.shuffle(training_data)
#mini_batchse是列表中存放切割后的列表
mini_batches = [training_data[k:k + mini_batch_size] for k in range(0, n, mini_batch_size)]
#每个mini_batch都更新一次,重复完整个数据集
for mini_batch in mini_batches:
#存储C对于各个参数的偏导
#格式和self.biases和self.weights是一模一样
self.update_mini_batch(mini_batch, eta)
if test_data:
print("Epoch {0}: {1} / {2}".format(j, self.evaluate(test_data), n_test))
else:
print("Epoch{0} complete".format(j))
def update_mini_batch(self, mini_batch, eta):
nabla_b = [np.zeros(b.shape) for b in self.biases]
nabla_w = [np.zeros(w.shape) for w in self.weights]
for x, y in mini_batch:
delta_nabla_b, delta_nabla_w = self.backprop(x, y)
nabla_b = [nb + dnb for nb, dnb in zip(nabla_b, delta_nabla_b)]
nabla_w = [nw + dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]
self.weights = [w-(eta/len(mini_batch))*nw for w, nw in zip(self.weights, nabla_w)]
self.biases = [b-(eta/len(mini_batch))*nb for b, nb in zip(self.biases, nabla_b)]
def backprop(self, x, y):
#存储C对于各个参数的偏导
#格式和self.biases和self.weight是一摸一样
nabla_b = [np.zeros(b.shape) for b in self.biases]
nabla_w = [np.zeros(w.shape) for w in self.weights]
#前向过程
activation = x
activations = [x]
zs = []
for b, w in zip(self.biases, self.weights):
z = np.dot(w, activation)+b
zs.append(z)
activation = sigmoid(z)
activations.append(activation)
#反向过程
#输出层error
delta = self.cost_derivative(activations[-1], y) * \
sigmoid_prime(zs[-1])
nabla_b[-1] = delta
nabla_w[-1] = np.dot(delta, activations[-2].transpose())
#非输出层
for l in range(2, self.num_layers):
z = zs[-l]
sp = sigmoid_prime(z)
delta = np.dot(self.weights[-l+1].transpose(), delta) * sp
nabla_b[-l] = delta
nabla_w[-l] = np.dot(delta, activations[-l-1].transpose())
return (nabla_b,nabla_w)
def evaluate(self, test_data):
test_results = [(np.argmax(self.feedforward(x)), y)
for (x, y) in test_data]
return sum(int(x == y) for (x, y) in test_results)
#输出层cost函数对于a的倒数
def cost_derivative(self, output_activations, y):
return (output_activations-y)
def sigmoid(z):
return 1.0 / (1.0 + np.exp(-z))
def sigmoid_prime(z):
return sigmoid(z) * (1 - sigmoid(z))
[04-main.py]
import mnist_loader
import network
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
net = network.Network([784, 30, 10])
net.SGD(training_data, 30, 10, 3.0, test_data=test_data)
另外可以查看,另一博客中代码解释:
https://blog.csdn.net/xierhacker/article/details/53282060/
[05-改进的network]
[network2.py]
import json
import random
import sys
import numpy as np
class QuadraticCost(object):
@staticmethod
def fn(a, y):
return 0.5*np.linalg.norm(a-y)**2
@staticmethod
def delta(z, a, y):
return(a-y)*sigmoid_prime(z)
class CrossEntropyCost(object):
@staticmethod
def fn(a, y):
return np.sum(np.nan_to_num(-y * np.log(a) - (1-y) * np.log(1-a)))
@staticmethod
def delta(z, a, y):
return (a-y)
class Network(object):
def __init__(self, sizes, cost=CrossEntropyCost):
self.num_layers = len(sizes)
self.sizes = sizes
self.default_weight_initializer()
self.cost = cost
def default_weight_initializer(self):
self.biases = [np.random.rand(y, 1) for y in self.sizes[1:]]
self.weights = [np.random.randn(y, x) / np.sqrt(x)
for x, y in zip(self.sizes[:-1], self.sizes[1:])]
def large_weight_initializer(self):
self.biases = [np.random.randn(y, 1) for y in self.sizes[1:]]
self.weight = [np.random.randn(y, x) for x, y in zip(self.sizes[:-1], self.sizes[1:])]
def feedforward(self, a):
for b, w in zip(self.biases, self.weights):
a = sigmoid(np.dot(w, a)+b)
return a
def SGD(self, training_data, epochs, mini_batch_size, eta,
lmbda=0.0,
evaluation_data=None,
monitor_evaluation_cost=False,
monitor_evaluation_accuracy=False,
monitor_training_cost=False,
monitor_training_accuracy=False):
if evaluation_data: n_data = len(evaluation_data)
n = len(training_data)
evaluation_cost, evaluation_accuracy = [], []
training_cost, training_accuracy = [], []
for j in range(epochs):
random.shuffle(training_data)
mini_batches = [
training_data[k:k+mini_batch_size]
for k in range(0, n, mini_batch_size)]
for mini_batch in mini_batches:
self.update_mini_batch(
mini_batch, eta, lmbda, len(training_data))
print("epoch %s training complete" % j)
if monitor_training_cost:
cost = self.total_cost(training_data, lmbda)
training_cost.append(cost)
print("cost on training data:{}".format(cost))
if monitor_training_accuracy:
accuracy = self.accuracy(training_data, convert=True)
training_accuracy.append(accuracy)
print("Accuracy on training data:{} / {}".format(accuracy, n))
if monitor_evaluation_cost:
cost = self.total_cost(evaluation_data, lmbda, convert=True)
evaluation_cost.append(cost)
print("Cost on evaluation data: {}".format(cost))
if monitor_evaluation_accuracy:
accuracy = self.accuracy(evaluation_data)
evaluation_accuracy.append(accuracy)
print("Accuracy on evaluation data:{} / {}".format(self.accuracy(evaluation_data), n_data))
print()
return evaluation_cost, evaluation_accuracy, training_cost, training_accuracy
def update_mini_batch(self, mini_batch, eta, lmbda, n):
nabla_b = [np.zeros(b.shape) for b in self.biases]
nabla_w = [np.zeros(w.shape) for w in self.weights]
for x, y in mini_batch:
delta_nabla_b, delta_nabla_w = self.backprop(x, y)
nabla_b = [nb+dnb for nb, dnb in zip(nabla_b, delta_nabla_b)]
nabla_w = [nw+dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]
self.weights = [(1-eta*(lmbda/n))*w - (eta/len(mini_batch))*nw
for w, nw in zip(self.weights, nabla_w)]
self.biases = [b-(eta/len(mini_batch))*nb
for b, nb in zip(self.biases, nabla_b)]
def backprop (self, x, y):
nabla_b = [np.zeros(b.shape) for b in self.biases]
nabla_w = [np.zeros(w.shape) for w in self.weights]
activation = x
activations = [x]
zs = []
for b, w in zip(self.biases, self.weights):
z = np.dot(w, activation)+b
zs.append(z)
activation = sigmoid(z)
activations.append(activation)
delta = (self.cost).delta(zs[-1], activations[-1], y)
nabla_b[-1] = delta
nabla_w[-1] = np.dot(delta, activations[-2].transpose())
for l in range(2, self.num_layers):
z = zs[-l]
sp = sigmoid_prime(z)
delta = np.dot(self.weights[-l+1].transpose(), delta)*sp
nabla_b[-l] = delta
nabla_w[-l] = np.dot(delta, activations[-l-1].transpose())
return (nabla_b, nabla_w)
def accuracy(self, data, convert=False):
if convert:
results = [(np.argmax(self.feedforward(x)), np.argmax(y)) for (x, y) in data]
else:
results = [(np.argmax(self.feedforward(x)), y) for (x, y) in data]
return sum(int(x == y) for (x, y) in results)
def total_cost(self, data, lmbda, convert=False):
cost = 0.0
for x, y in data:
a = self.feedforward(x)
if convert: y = vectorized_result(y)
cost += self.cost.fn(a, y)/len(data)
cost += 0.5*(lmbda/len(data))*sum(np.linalg.norm(w)**2 for w in self.weights)
return cost
def save(self, filename):
data = {"sizes": self.sizes,
"weights": [w.tolist() for w in self.weights],
"biases": [b.tolist() for b in self.biases],
"cost": str(self.cost.__name__)}
f = open(filename, "w")
json.dump(data, f)
f.close()
def load (filename):
f = open(filename, "r")
data = json.load(f)
f.close()
cost = getattr(sys.modules[__name__], data["cost"])
net = Network(data["sizes"], cost=cost)
net.weights = [np.array(w) for w in data["weights"]]
net.biases = [np.array(b) for b in data["biases"]]
return net
def vectorized_result(j):
e = np.zeros((10, 1))
e[j] = 1.0
return e
def sigmoid(z):
return 1.0/(1.0 + np.exp(-z))
def sigmoid_prime(z):
return sigmoid(z)*(1-sigmoid(z))
[06-main2.py]
import network2
import mnist_loader
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)
net.SGD(training_data, 30, 10, 0.5, 5.0,
evaluation_data=validation_data,
monitor_evaluation_accuracy=True,
monitor_evaluation_cost=True,
monitor_training_accuracy=True,
monitor_training_cost=True)