4.1 损失函数
import numpy as np
y1 = [ 0.1 , 0.05 , 0.6 , 0.0 , 0.05 , 0.1 , 0.0 , 0.1 , 0.0 , 0.0 ]
y2 = [ 0.1 , 0.05 , 0.1 , 0.0 , 0.05 , 0.1 , 0.0 , 0.6 , 0.0 , 0.0 ]
t = [ 0 , 0 , 1 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ]
def mean_squared_error ( y, t) :
return 0.5 * np. sum ( ( y- t) ** 2 )
mean_squared_error( np. array( y1) , np. array( t) )
mean_squared_error( np. array( y2) , np. array( t) )
def cross_entropy_error0 ( y, t) :
delta = 1e - 7
return - np. sum ( t* np. log( y+ delta) )
t = [ 0 , 0 , 1 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ]
y1 = [ 0.1 , 0.05 , 0.6 , 0.0 , 0.05 , 0.1 , 0.0 , 0.1 , 0.0 , 0.0 ]
y2 = [ 0.1 , 0.05 , 0.1 , 0.0 , 0.05 , 0.1 , 0.0 , 0.6 , 0.0 , 0.0 ]
cross_entropy_error0( np. array( y1) , np. array( t) )
cross_entropy_error0( np. array( y2) , np. array( t) )
import sys, os
sys. path. append( os. pardir)
import numpy as np
from res. mnist import load_mnist
( x_train, t_train) , ( x_test, t_test) = \
load_mnist( normalize= True , one_hot_label= True )
print ( x_train. shape)
print ( t_train. shape)
train_size = x_train. shape[ 0 ]
batch_size = 10
batch_mask = np. random. choice( train_size, batch_size)
x_batch = x_train[ batch_mask]
t_baych = t_train[ batch_mask]
def cross_entropy_error ( y, t) :
if y. ndim == 1 :
t = t. reshape( 1 , t. size)
y = y. reshape( 1 , y. size)
batch_size = y. shape[ 0 ]
return - np. sum ( t* np. log( y+ 1e - 7 ) ) / batch_size
def cross_entropy_label ( y, t) :
if y. ndim == 1 :
t = t. reshape( 1 , t. size)
y = y. reshape( 1 , y. size)
batch_size = y. shape[ 0 ]
return - np. sum ( np. log( y[ np. arange( batch_size) , t] + 1e - 7 ) ) / batch_size
4.2 数值微分
1、变化程度取无限小,但是计算机会省略掉一定小数点后面的值, 所以变化程度一般去10^(-4) 2、不是某一点的导数,一般去x-h,x+h之间的微分 3、不精确所以叫数值微分
def numerical_diff ( f, x) :
h = 1e - 4
return ( f( x+ h) - f( x- h) ) / ( 2 * h)
def func ( x) :
return 0.01 * x** 2 + 0.1 * x
numerical_diff( func, 10 )
def tangent_line ( f, x) :
'''切线'''
d = numerical_diff( f, x)
y = f( x) - d* x
return lambda t: d* t + y
import numpy as np
import matplotlib. pylab as plt
x = np. arange( 0.0 , 20.0 , 0.1 )
y1 = func( x)
tf = tangent_line( func, 10 )
y2 = tf( x)
plt. plot( x, y1)
plt. plot( x, y2)
plt. xlabel( 'x' )
plt. ylabel( 'f(x)' )
plt. show( )
def func_2 ( x) :
return np. sum ( x** 2 )
def numerical_gradient ( f, x) :
'''梯度'''
h = 1e - 4
grad = np. zeros_like( x)
it = np. nditer( x, flags= [ 'multi_index' ] , op_flags= [ 'readwrite' ] )
while not it. finished:
idx = it. multi_index
tmp_val = x[ idx]
x[ idx] = tmp_val + h
fxh1 = f( x)
x[ idx] = tmp_val - h
fxh2 = f( x)
grad[ idx] = ( fxh1- fxh2) / ( 2 * h)
x[ idx] = tmp_val
it. iternext( )
return grad
def gradient_descent ( f, init_x, lr= 0.01 , step_num= 1000 ) :
x = init_x
for i in range ( step_num) :
grad = numerical_gradient( f, x)
x -= lr* grad
return x
numerical_gradient( func_2, np. array( [ 3.0 , 4.0 ] ) )
x = gradient_descent( func_2, np. array( [ - 3.0 , 4.0 ] ) , lr= 0.1 )
func_2( x)
def softmax ( x) :
a = np. max ( x)
exp_x = np. exp( x- a)
exp_sum = np. sum ( exp_x)
y = exp_x/ exp_sum
return y
import numpy as np
class SimpleNet :
def __init__ ( self) :
self. W = np. random. randn( 2 , 3 )
def predict ( self, x) :
return np. dot( x, self. W)
def loss ( self, x, t) :
z = self. predict( x)
y = softmax( z)
loss = cross_entropy_error( y, t)
return loss
net = SimpleNet( )
print ( net. W)
x = np. array( [ 0.6 , 0.9 ] )
p = net. predict( x)
print ( p)
t = np. array( [ 0 , 0 , 1 ] )
f = lambda W: net. loss( x, t)
print ( f( 2 ) )
print ( f( 1 ) )
dW = numerical_gradient( f, net. W)
print ( dW)
4.3 两层神经网络的学习
随机选择min-batch计算损失函数 求损失函数的梯度 更新参数 直到梯度为0或者达到最大步数
def sigmoid ( x) :
return 1 / ( 1 + np. exp( - x) )
def sigmoid_grad ( x) :
return ( 1.0 - sigmoid( x) ) * sigmoid( x)
class TwoLayerNet :
def __init__ ( self, input_size, hidden_size, output_size, weight_std= 0.01 ) :
self. params = { }
self. params[ 'W1' ] = weight_std* np. random. randn( input_size, hidden_size)
self. params[ 'b1' ] = np. zeros( hidden_size)
self. params[ 'W2' ] = weight_std* np. random. randn( hidden_size, output_size)
self. params[ 'b2' ] = np. zeros( output_size)
def predict ( self, x) :
W1, W2 = self. params[ 'W1' ] , self. params[ 'W2' ]
b1, b2 = self. params[ 'b1' ] , self. params[ 'b2' ]
a1 = np. dot( x, W1) + b1
z1 = sigmoid( a1)
a2 = np. dot( z1, W2) + b2
y = softmax( a2)
return y
def loss ( self, x, t) :
y = self. predict( x)
return cross_entropy_error( y, t)
def accuracy ( self, x, t) :
y = self. predict( x)
y = np. argmax( y, axis= 1 )
t = np. argmax( t, axis= 1 )
accuracy = np. sum ( y== t) / float ( x. shape[ 0 ] )
return accuracy
def numerical_gradient ( self, x, t) :
loss_W = lambda W: self. loss( x, t)
grads = { }
grads[ 'W1' ] = numerical_gradient( loss_W, self. params[ 'W1' ] )
grads[ 'W2' ] = numerical_gradient( loss_W, self. params[ 'W2' ] )
grads[ 'b1' ] = numerical_gradient( loss_W, self. params[ 'b1' ] )
grads[ 'b2' ] = numerical_gradient( loss_W, self. params[ 'b2' ] )
return grads
def gradient ( self, x, t) :
W1, W2 = self. params[ 'W1' ] , self. params[ 'W2' ]
b1, b2 = self. params[ 'b1' ] , self. params[ 'b2' ]
grads = { }
batch_num = x. shape[ 0 ]
a1 = np. dot( x, W1) + b1
z1 = sigmoid( a1)
a2 = np. dot( z1, W2) + b2
y = softmax( a2)
dy = ( y - t) / batch_num
grads[ 'W2' ] = np. dot( z1. T, dy)
grads[ 'b2' ] = np. sum ( dy, axis= 0 )
da1 = np. dot( dy, W2. T)
dz1 = sigmoid_grad( a1) * da1
grads[ 'W1' ] = np. dot( x. T, dz1)
grads[ 'b1' ] = np. sum ( dz1, axis= 0 )
return grads
net = TwoLayerNet( 784 , 100 , 10 )
net. params[ 'W1' ] . shape
net. params[ 'b1' ] . shape
net. params[ 'W2' ] . shape
net. params[ 'b2' ] . shape
x = np. random. randn( 100 , 784 )
t = np. random. randn( 100 , 10 )
y = net. predict( x)
print ( y. shape)
grads = net. gradient( x, t)
grads
from res. mnist import load_mnist
( x_train, t_train) , ( x_test, t_test) = load_mnist( normalize= True , one_hot_label= True )
train_loss_list = [ ]
iters_num = 10000
train_size = x_train. shape[ 0 ]
batch_size = 100
learning_rate = 0.1
network = TwoLayerNet( 784 , 50 , 10 )
for i in range ( iters_num) :
batch_mask = np. random. choice( train_size, batch_size)
x_batch = x_train[ batch_mask]
t_batch = t_train[ batch_mask]
grad = network. gradient( x_batch, t_batch)
for key in ( 'W1' , 'b1' , 'W2' , 'b2' ) :
network. params[ key] -= learning_rate * grad[ key]
loss = network. loss( x_batch, t_batch)
train_loss_list. append( loss)
4.4 神经网络的评价
( x_train, t_train) , ( x_test, t_test) = load_mnist( normalize= True , one_hot_label= True )
train_loss_list = [ ]
train_acc_list = [ ]
test_acc_list = [ ]
iter_per_epoch = max ( train_size/ batch_size, 1 )
iters_num = 10000
batch_size = 100
learning_rate = 0.1
network = TwoLayerNet( 784 , 50 , 10 )
for i in range ( iters_num) :
batch_mask = np. random. choice( train_size, batch_size)
x_batch = x_train[ batch_mask]
t_batch = t_train[ batch_mask]
grad = network. gradient( x_batch, t_batch)
for key in ( 'W1' , 'b1' , 'W2' , 'b2' ) :
network. params[ key] -= grad[ key] * learning_rate
loss = network. loss( x_batch, t_batch)
train_loss_list. append( loss)
if i% iter_per_epoch== 0 :
train_acc = network. accuracy( x_train, t_train)
test_acc = network. accuracy( x_test, t_test)
train_acc_list. append( train_acc)
test_acc_list. append( test_acc)
print ( "train acc, test acc |" + str ( train_acc) + " , " + str ( test_acc) )
x = np. arange( len ( train_acc_list) )
plt. plot( x, train_acc_list, label= 'train_acc' )
plt. plot( x, test_acc_list, label= 'test_acc' , linestyle= "--" )
plt. xlabel( 'epochs' )
plt. ylabel( 'accuracy' )