immortal的PyTorch学习之路

本文完全参考以下网址,感谢‘’莫烦‘’！！！
莫烦PyTorch

1.torch基础运算

import torch
import numpy as np 
#torch、numpy的互换
np_data=np.arange(6).reshape((2,3))
array2tensor=torch.from_numpy(np_data)#tensor([[0, 1, 2],[3, 4, 5]], dtype=torch.int32)
tensor2array=array2tensor.numpy()
print(np_data)
print(array2tensor)
print(tensor2array)
#torch的数学运算(torch的tensor与numpy的array一致)
data=[-1,-3,1,6]
tensor=torch.FloatTensor(data)#转换成32位浮点tensor
print(np.abs(data))
print(torch.abs(tensor))#tensor([1., 3., 1., 6.])
print(np.sin(data))
print(torch.sin(tensor))
print(np.std(data))
print(torch.std(tensor))#tensor(3.8622)
#矩阵运算
data=[[1,2],[3,4]]
data=np.array(data)
tensor=torch.FloatTensor(data)
print(np.matmul(data,data))#矩阵乘法
print(data.dot(data))#同样是矩阵乘法

print(torch.mm(tensor,tensor))#矩阵乘法,tensor([[ 7., 10.],[15., 22.]])
print(tensor.dot(tensor)#错误！

2.torch变量

import torch
from torch.autograd import Variable
tensor=torch.FloatTensor([[1,2],[3,4]])
variable=Variable(tensor,requires_grad=True)#需要计算梯度，参与反向传播的变量
#若用一个variable进行计算，返回的是同类型的Variable
print(tensor)#tensor([[1., 2.],[3., 4.]])
print(variable)#tensor([[1., 2.],[3., 4.]], requires_grad=True)

t_out=torch.mean(tensor*tensor)
v_out=torch.mean(variable*variable)
print(t_out)#矩阵对应元素相乘,tensor(7.5000)
print(v_out)#tensor(7.5000, grad_fn=<MeanBackward0>)
#vairiable与tensor的不同：vairiable是计算图的一部分，
#最后反向传播时torch可根据计算图计算variable的梯度，而tensor不行

#v_out就是在计算图里添加的一个计算步骤
v_out.backward()
#d(v_out)/d(variable) = 1/4*2*variable = variable/2
print(variable.grad)#tensor([[0.5000, 1.0000],[1.5000, 2.0000]])

#用plt画图时用不了Variable形式的数据，所以要将Variable转成tensor或array
print(variable)
print(variable.data)#tensor形式
print(variable.data.numpy())#array形式

3.torch激励函数

当你的神经网络层只有两三层, 不是很多的时候, 对于隐藏层, 使用任意的激励函数, 随便掰弯是可以的, 不会有特别大的影响.
不过, 当你使用特别多层的神经网络, 在掰弯的时候, 玩玩不得随意选择利器. 因为这会涉及到梯度爆炸, 梯度消失的问题.
在卷积神经网络 Convolutional neural networks 的卷积层中, 推荐的激励函数是 relu. 在循环神经网络中 recurrent neural networks, 推荐的是 tanh 或者是 relu ——莫烦

import torch 
import torch.nn.functional as F #激励函数都在这里！！
from torch.autograd import Variable
import matplotlib.pyplot as plt 
x=torch.linspace(-5,5,200)#tensor类型
x=Variable(x)
x_np=x.data.numpy()#为了出图，转化为array类型
# y_relu=F.relu(torch.linspace(-5,5,200)).data.numpy()#函数里面用tensor也行，yongVariable也行
y_relu=F.relu(x).data.numpy()
y_sigmoid=F.sigmoid(x).data.numpy()
y_tanh=F.tanh(x).data.numpy()
y_softplus=F.softplus(x).data.numpy()
#y_softmax=F.softmax(x)#不能显示
plt.figure(1,figsize=(8,6))
plt.subplot(221)
plt.plot(x_np,y_relu,c='pink',label='relu')
plt.ylim((-1,5))#0-∞
plt.legend(loc="best")

plt.subplot(222)
plt.plot(x_np,y_sigmoid,c='gray',label='sigmoid')
plt.ylim(-0.2,1.2)#0-1
plt.legend(loc="best")

plt.subplot(223)
plt.plot(x_np,y_tanh,c='green',label='tanh')
plt.ylim(-1.2,1.2)#-1-1
plt.legend(loc="best")

plt.subplot(224)
plt.plot(x_np,y_softplus,c='orange',label='softplus')
plt.ylim(-0.2,6)#0-∞
plt.legend(loc="best")
plt.show()

4.回归

import torch 
import matplotlib.pyplot as plt 
import torch.nn.functional as F
x=torch.unsqueeze(torch.linspace(-1,1,100),dim=1)#tensor
# print(x.shape)#[100,1]
# print(torch.linspace(-1,1,100).shape)#[100]
y=x.pow(2)+0.2*torch.rand(x.size())#y=x^2+b,加点噪声
# plt.scatter(x.data.numpy(),y.data.numpy())

#建立神经网络
class Net(torch.nn.Module):#继承torch的Module
	#定义好网络
	def __init__(self,n_feature,n_hidden,n_output):
		super(Net,self).__init__()#继承__init__功能
		self.hidden=torch.nn.Linear(n_feature,n_hidden)
		self.predict=torch.nn.Linear(n_hidden,n_output)
	def forward(self,x):
		x=F.relu(self.hidden(x))
		x=self.predict(x)
		return x
net=Net(n_feature=1,n_hidden=10,n_output=1)
print(net)

#训练网络
optimizer=torch.optim.SGD(net.parameters(),lr=0.2)#传入net的所有参数，设置lr
loss_func=torch.nn.MSELoss()
#100个样本数据
# for i in range(100):
# 	prediction=net(x)
# 	loss=loss_func(prediction,y)
# 	optimizer.zero_grad()#清空上一步的残余更新参数值
# 	loss.backward()
# 	optimizer.step()#将参数更新值添加到net的parameters中

#可视化训练过程
plt.ion()#打开交互模式
for t in range(200):#200个epoch
	prediction=net(x)
	loss=loss_func(prediction,y)
	optimizer.zero_grad()#清空上一步的残余更新参数值
	loss.backward()
	optimizer.step()#将参数更新值添加到net的parameters中
	if t%5==0:
		plt.cla()#除当前图形中的当前活动轴，其他轴不受影响
		plt.scatter(x.data.numpy(),y.data.numpy())
		plt.plot(x.data.numpy(),prediction.data.numpy(),'r-',lw=5)
		plt.text(0.5,0,'Loss=%.4f'%loss.data.numpy(),fontdict={'size':20,'color':'pink'})
		plt.pause(0.1)
plt.show()

5.分类

import torch 
import matplotlib.pyplot as plt 
import torch.nn.functional as F
#生成假数据
n_data=torch.ones(100,2)
#类型0
x0=torch.normal(2*n_data,1)#均值为[2,2],所有抽取的样本共享标准差1
y0=torch.zeros(100)
#类型1
x1=torch.normal(-2*n_data,1)
y1=torch.ones(100)
x=torch.cat((x0,x1),0).type(torch.FloatTensor)#按维数0进行拼接
y=torch.cat((y0,y1),).type(torch.LongTensor)
print(x.shape,y.shape)#torch.Size([200, 2]) torch.Size([200])
# plt.scatter(x.data.numpy()[:, 0], x.data.numpy()[:, 1], c=y.data.numpy(), s=100, lw=0, cmap='RdYlGn')
# plt.show()
#定义网络结构
class Net(torch.nn.Module):
	def __init__(self,n_feature,n_hidden,n_output):
		super(Net,self).__init__()#继承__init__功能
		self.hidden=torch.nn.Linear(n_feature,n_hidden)
		self.out=torch.nn.Linear(n_hidden,n_output)
	def forward(self,x):
		x=F.relu(self.hidden(x))
		x=self.out(x)
		return x 
#训练网络
net=Net(n_feature=2,n_hidden=10,n_output=2)
print(net)
#Net(
#(hidden): Linear(in_features=2, out_features=10, bias=True)
#(out): Linear(in_features=10, out_features=2, bias=True))
optimizer=torch.optim.SGD(net.parameters(),lr=0.02)
loss_func=torch.nn.CrossEntropyLoss()
# for i in range(100):#训练100epoch，每个batch为200个样本？？
# 	out=net(x)
# 	loss=loss_func(out,y)
# 	optimizer.zero_grad()
# 	loss.backward()
# 	optimizer.step()
#可视化训练过程
plt.ion()
for t in range(100):
	out=net(x)
	loss=loss_func(out,y)
	optimizer.zero_grad()
	loss.backward()
	optimizer.step()

	if t%2==0:
		plt.cla()
		prediction=torch.max(F.softmax(out),1)[1]
		# print(torch.max(F.softmax(out),1))
		#torch.return_types.max(values=tensor([0.8619, ……0.9476, 0.9006], grad_fn=<MaxBackward0>) , indices=tensor([0,……1,1]))
		pred_y=prediction.data.numpy().squeeze()
		# print(pred_y.shape)#(200,)
		target_y=y.data.numpy()
		plt.scatter(x.data.numpy()[:,0],x.data.numpy()[:,1],c=pred_y,s=100,lw=0,cmap='RdYlGn')
		accuracy=sum(pred_y==target_y)/200
		plt.text(0.5,-4,'Accuracy=%.2f'%accuracy,fontdict={'size':20,'color':'red'})
		plt.pause(0.2)
plt.ioff()
plt.show()

6.保存/调用模型

import torch 
import matplotlib.pyplot as plt 
torch.manual_seed(1)
#产生假数据,100个样本，x、y都是一个scalar
x=torch.unsqueeze(torch.linspace(-1,1,100),dim=1)#[100,1]
y=x.pow(2)+0.2*torch.rand(x.size())#y=x^2+b，加点噪声
def save():
	#！快速！定义网络结构
	net1=torch.nn.Sequential(
		torch.nn.Linear(1,10),#hidden layer
		torch.nn.ReLU(),
		torch.nn.Linear(10,1)#output layer
		)
	optimizer=torch.optim.SGD(net1.parameters(),lr=0.2)
	loss_func=torch.nn.MSELoss()
	#训练网络
	for t in range(100):#100 epoch
		prediction=net1(x)
		loss=loss_func(prediction,y)
		optimizer.zero_grad()
		loss.backward()
		optimizer.step()
	plt.cla()
	plt.title('Net1')
	plt.plot(x.data.numpy(),y.data.numpy(),"b.",label='true')
	plt.plot(x.data.numpy(),prediction.data.numpy(),'r-',lw=5,label='prediction')
	plt.legend(loc="best")
		# plt.pause(0.05)
	#2种保存方式
	torch.save(net1,'net.pkl')#保存整个网络
	torch.save(net1.state_dict(),'net_params.pkl')#只保存网络中的参数(速度快，占内存少)

#2种重新调用网络的方式
def restore_net():#调用整个网络
	net2=torch.load('net.pkl')
	prediction=net2(x)
	plt.cla()
	plt.title('Net2')
	plt.plot(x.data.numpy(),y.data.numpy(),"b.",label='true')
	plt.plot(x.data.numpy(),prediction.data.numpy(),'r-',lw=5,label='prediction')
	plt.legend(loc="best")
def restore_params():#只调用网络参数
	net3=torch.nn.Sequential(
		torch.nn.Linear(1,10),
		torch.nn.ReLU(),
		torch.nn.Linear(10,1)
		)
	net3.load_state_dict(torch.load('net_params.pkl'))
	prediction=net3(x)
	plt.cla()
	plt.title('Net3')
	plt.plot(x.data.numpy(),y.data.numpy(),"b.",label='true')
	plt.plot(x.data.numpy(),prediction.data.numpy(),'r-',lw=5,label='prediction')
	plt.legend(loc="best")

plt.figure(figsize=(1,3))
plt.subplot(131)#1行3列——1
save()
plt.subplot(132)#1行3列——2
restore_net()
plt.subplot(133)#1行3列——3
restore_params()
plt.show()

7.DataLoader——批训练（minibatch）

import torch
import torch.utils.data as Data
torch.manual_seed(1)

BATCH_SIZE=8
x=torch.linspace(-1,1,10)#tensor([-1.0000, -0.7778, -0.5556, -0.3333, -0.1111,  0.1111,  0.3333,  0.5556,0.7778,  1.0000])
# print(x)
y=torch.linspace(10,1,10)#tensor([10.,  9.,  8.,  7.,  6.,  5.,  4.,  3.,  2.,  1.])
#转换成torch中的Dataset
torch_dataset=Data.TensorDataset(x,y)
#把dataset放入DataLoader中包装
loader=Data.DataLoader(
	dataset=torch_dataset,
	batch_size=BATCH_SIZE,
	shuffle=True,
	#多线程读取数据
	)
for epoch in range(3):#3epochs
	for step,(batch_x,batch_y) in enumerate(loader):#每一步为一个batch，step=batch
		#训练数据…………

		print('Epoch',epoch,'| Step',step,'| batch x',batch_x.numpy(),'| batch y',batch_y.numpy())

Epoch 0 | Step 0 | batch x [-0.1111111   0.33333337  1.         -0.5555556  -0.3333333  -0.7777778
 -1.          0.5555556 ] | batch y [ 6.  4.  1.  8.  7.  9. 10.  3.]
Epoch 0 | Step 1 | batch x [0.7777778  0.11111116] | batch y [2. 5.]
Epoch 1 | Step 0 | batch x [-0.3333333   0.11111116  0.33333337  1.          0.5555556  -0.1111111
 -0.5555556  -0.7777778 ] | batch y [7. 5. 4. 1. 3. 6. 8. 9.]
Epoch 1 | Step 1 | batch x [-1.         0.7777778] | batch y [10.  2.]
Epoch 2 | Step 0 | batch x [-0.3333333  -0.7777778  -0.1111111   0.11111116  1.         -0.5555556
  0.7777778  -1.        ] | batch y [ 7.  9.  6.  5.  1.  8.  2. 10.]
Epoch 2 | Step 1 | batch x [0.5555556  0.33333337] | batch y [3. 4.]

8.几种优化器（optimizer）

把数据拆分成小批小批的, 然后再分批不断放入 NN 中计算, 这就是我们常说的 SGD 的正确打开方式了. 每次使用批数据, 虽然不能反映整体数据的情况, 不过却很大程度上加速了 NN 的训练过程, 而且也不会丢失太多准确率.——莫烦

import torch 
import torch.utils.data as Data #DataLoader
import torch.nn.functional as F #激活函数
import matplotlib.pyplot as plt 

torch.manual_seed(1)
LR=0.01
BATCH_SIZE=32
EPOCH=12

x=torch.unsqueeze(torch.linspace(-1,1,1000),dim=1)
y=x.pow(2)+0.1*torch.normal(torch.zeros(x.size()))
# plt.scatter(x.numpy(),y.numpy())
# plt.show()
torch_dataset=Data.TensorDataset(x,y)
loader=Data.DataLoader(dataset=torch_dataset,batch_size=BATCH_SIZE,shuffle=True)

class Net(torch.nn.Module):
	def __init__(self):
		super(Net,self).__init__()
		self.hidden=torch.nn.Linear(1,20)
		self.predict=torch.nn.Linear(20,1)
	def forward(self,x):
		x=F.relu(self.hidden(x))
		x=self.predict(x)
		return x

#为每一个优化器创建一个net
net_SGD=Net()
net_Momentum=Net()
net_RMSprop=Net()
net_Adam=Net()
nets=[net_SGD,net_Momentum,net_RMSprop,net_Adam]

#设置优化器和损失函数
opt_SGD=torch.optim.SGD(net_SGD.parameters(),lr=LR)
opt_Momentum=torch.optim.SGD(net_Momentum.parameters(),lr=LR,momentum=0.8)#!!!
opt_RMSprop=torch.optim.RMSprop(net_RMSprop.parameters(),lr=LR,alpha=0.9)
opt_Adam=torch.optim.Adam(net_Adam.parameters(),lr=LR,betas=(0.9,0.99))
optimizers=[opt_SGD,opt_Momentum,opt_RMSprop,opt_Adam]

loss_func=torch.nn.MSELoss()
losses=[[],[],[],[]]#记录训练时不同神经网络的损失

#训练
for epoch in range(EPOCH):
	print('Epoch:',epoch)
	for step,(b_x,b_y) in enumerate(loader):
		for net,opt,l in zip(nets,optimizers,losses):
			output=net(b_x)
			loss=loss_func(output,b_y)
			opt.zero_grad()
			loss.backward()
			opt.step()
			l.append(loss.data.numpy())
plt.figure()
color=['r','g','b','k']
for l,c,opt in zip(losses,color,optimizers):
	plt.plot(l,str(c),label=str(opt))	
	plt.legend(loc="best")
	plt.ylim(0,0.2)
plt.show()

9.CNN

import torch 
import torch.nn as nn 
import torch.utils.data as Data 
import torchvision
import matplotlib.pyplot as plt 
import torch.nn.functional as F
from sklearn.manifold import TSNE
from matplotlib import cm
def plot_with_labels(lowDWeights,labels):
	plt.cla()
	X,Y=lowDWeights[:,0],lowDWeights[:,1]
	for x,y,s in zip(X,Y,labels):
		c=cm.rainbow(int(255*s/9))
		plt.text(x,y,s,backgroundcolor=c,fontsize=9)
		plt.xlim(X.min(),X.max())
		plt.ylim(Y.min(),Y.max())
		plt.title('Visualize last layer')
	plt.show()
	plt.pause(0.02)
torch.manual_seed(1)
EPOCH=1
BATCH_SIZE=50
LR=0.001
DOWNLOAD_MNIST=False 

train_data=torchvision.datasets.MNIST(root='/mnist_1',train=True,transform=torchvision.transforms.ToTensor(),download=DOWNLOAD_MNIST)
test_data=torchvision.datasets.MNIST(root='/mnist_1',train=False)
# print(test_data.test_data.shape)#torch.Size([10000, 28, 28])
#(50,1,28,28),1 channel
train_loader=Data.DataLoader(dataset=train_data,batch_size=BATCH_SIZE,shuffle=True)
test_x=torch.unsqueeze(test_data.test_data,dim=1).type(torch.FloatTensor)[:200]/255#torch.Size([2000, 1, 28, 28]),在index=1的维度扩充
# print(test_x.shape)
test_y=test_data.test_labels[:200]#torch.Size([2000])
# print(test_y.shape)

#定义网络结构
class CNN(nn.Module):
	def __init__(self):
		super(CNN,self).__init__()
		self.conv1=nn.Sequential(#input shape(1,28,28)
			nn.Conv2d(in_channels=1,out_channels=16,kernel_size=5,stride=1,padding=2),#长宽没有变化,(16,28,28)
			nn.ReLU(),#F.relu();nn.ReLU(),注意区别
			nn.MaxPool2d(kernel_size=2)#(16,14,14)
			)
		self.conv2=nn.Sequential(#(16,14,14)
			nn.Conv2d(in_channels=16,out_channels=32,kernel_size=5,stride=1,padding=2),#p=(k-1)/2,(32,14,14)
			nn.ReLU(),
			nn.MaxPool2d(kernel_size=2)#(32,7,7)
			)
		self.out=nn.Linear(32*7*7,10)#in_features=1568, out_features=10

	def forward(self,x):
		x=self.conv1(x)
		x=self.conv2(x)
		x=x.view(x.size(0),-1)#flatten,(batch_size,32*7*7)
		output=self.out(x)
		return output,x
cnn=CNN()
print(cnn)
"""
CNN(
  (conv1): Sequential(
    (0): Conv2d(1, 16, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))
    (1): ReLU()
    (2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
  )
  (conv2): Sequential(
    (0): Conv2d(16, 32, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))
    (1): ReLU()
    (2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
  )
  (out): Linear(in_features=1568, out_features=10, bias=True)
)
"""
#训练
optimizer=torch.optim.Adam(cnn.parameters(),lr=LR)
loss_func=nn.CrossEntropyLoss()
test_y=test_y.numpy()
plt.ion()
for epoch in range(EPOCH):
	for step,(b_x,b_y) in enumerate(train_loader):
		output,_=cnn(b_x)
		loss=loss_func(output,b_y)
		optimizer.zero_grad()
		loss.backward()
		optimizer.step()
		if step%1==0:
			test_output, last_layer = cnn(test_x)
			tsne=TSNE(perplexity=30,n_components=2,init='pca',n_iter=5000)
			low_dim_embs=tsne.fit_transform(last_layer.data.numpy())
			labels=test_y
			plot_with_labels(low_dim_embs,labels)
			# print(torch.max(F.softmax(cnn(test_x)),1).shape)#错误！！！
			# print(torch.max(F.softmax(cnn(test_x)),dim=1)[0])#概率
			a,b=cnn(test_x)
			predict=torch.max(F.softmax(a),dim=1)[1]#index,即类别
			predict=predict.data.numpy().squeeze()
			# test_y=test_y.numpy()#错误！！！
			# print(predict.shape,type(predict))#(200,) <class 'numpy.ndarray'>
			# print(test_y.shape,type(test_y))#torch.Size([200]) <class 'torch.Tensor'>
			accuracy=sum(predict==test_y)/200
			print('Epoch: ',epoch,'| train loss: ',loss.data.numpy(),'test accuracy: ',accuracy)
plt.ioff()
test_output,_=cnn(test_x[:10])
pred_y=torch.max(test_output,1)[1].data.numpy().squeeze()
print(pred_y)
print(test_y[:10].numpy())

10.RNN

import torch 
from torch import nn 
import torchvision.datasets as dsets 
import torchvision.transforms as transforms 
import matplotlib.pyplot as plt 
import torch.utils.data as Data
torch.manual_seed(1)
EPOCH=1
BATCH_SIZE=64
TIME_STEP=28#时间步/图片高度
INPUT_SIZE=28#每步输入数/图片每行像素
LR=0.01
DOWNLOAD_MNIST=False

train_data=dsets.MNIST(
	root='/mnist_1',
	train=True,
	transform=transforms.ToTensor(),#将图片转换为Tensor,归一化至[0,1]
	download=DOWNLOAD_MNIST
	)
test_data=dsets.MNIST(root='/mnist_1',train=False)
train_loader=Data.DataLoader(dataset=train_data,batch_size=BATCH_SIZE,shuffle=True)
test_x=torch.unsqueeze(test_data.test_data,dim=1).type(torch.FloatTensor)[:200]
test_y=test_data.test_labels[:200]
#定义网络结构
class RNN(nn.Module):
	def __init__(self):
		super(RNN,self).__init__()
		self.rnn=nn.LSTM(#nn.LSTM比nn.RNN效果更好
			input_size=28,
			hidden_size=64,#hidden unit,也就是时间步
			num_layers=1,#hidden layer
			batch_first=True#input/output是以batch size为第一维度的特征集(batch,time_step,input_size)
			)
		self.out=nn.Linear(64,10)
	def forward(self,x):
		#x shape (batch,time_step,input_size)
		#r_out shape (batch,time_step,output_size)
		#分线h_n shape (n_layers,batch,hidden_size)
		#主线h_c shape (n_layers,batch,hidden_size)
		r_out,(h_n,h_c)=self.rnn(x,None)#None表示hidden state会用全0的state
		out=self.out(r_out[:,-1,:])#选取最后一个时间步的r_out输出
		return out 
"""
RNN(
  (rnn): LSTM(28, 64, batch_first=True)
  (out): Linear(in_features=64, out_features=10, bias=True)
)
"""
rnn=RNN()
# print(rnn)

#训练
optimizer=torch.optim.Adam(rnn.parameters(),lr=LR)
loss_func=nn.CrossEntropyLoss()
for epoch in range(EPOCH):
	for step,(b_x,b_y) in enumerate(train_loader):
		# print(b_x.shape)#torch.Size([64, 1, 28, 28])
		b_x=b_x.view(-1,28,28)#reshape，x shape (batch,time_step,input_size)
		output=rnn(b_x)
		loss=loss_func(output,b_y)
		optimizer.zero_grad()
		loss.backward()
		optimizer.step()
		if step%50==0:
			print('Epoch: ',epoch,'| train loss: ',loss.data.numpy())
test_output=rnn(test_x[:10].view(-1,28,28))
pred_y=torch.max(test_output,1)[1].data.numpy().squeeze()
print(pred_y)
print(test_y[:10])

11.LSTM

import torch 
from torch import nn 
import numpy as np 
import matplotlib.pyplot as plt 
torch.manual_seed(1)

TIME_STEP=10#时间步/图像高度
INPUT_SIZE=1#输入尺寸/图像宽度
LR=0.02
DOWNLOAD_MNIST=False

class RNN(nn.Module):
	def __init__(self):
		super(RNN,self).__init__()
		self.rnn=nn.RNN(
			input_size=1,
			hidden_size=32,
			num_layers=1,
			batch_first=True
			)
		self.out=nn.Linear(32,1)
	def forward(self,x,h_state):
		#x (batch,time_step,input_size)
		#h_state (n_layers,batch,hidden_size)
		#r_out (batch,time_step,output_size)
		r_out,h_state=self.rnn(x,h_state)
		outs=[]#保留所有时间点的预测值
		for time_step in range(r_out.size(1)):#对每一个时间步计算output
			outs.append(self.out(r_out[:,time_step,:]))
		return torch.stack(outs,dim=1),h_state
	# def forward(self, x, h_state):
	# 	r_out, h_state = self.rnn(x, h_state)
	# 	r_out = r_out.view(-1, 32)
	# 	outs = self.out(r_out)
	# 	return outs.view(-1, 32, TIME_STEP), h_state

rnn=RNN()
print(rnn)
optimizer=torch.optim.Adam(rnn.parameters(),lr=LR)
loss_func=nn.MSELoss()
h_state=None#初始的hidden state=None
plt.ion()
for step in range(100):
	start,end=step*np.pi,(step+1)*np.pi
	#sin预测cos
	steps=np.linspace(start,end,10,dtype=np.float32)
	# print(steps)#每回都不一样
	x_np=np.sin(steps)
	# print(x_np.shape)#(10,)
	y_np=np.cos(steps)
	x=torch.from_numpy(x_np[np.newaxis,:,np.newaxis])#
	# print(x.numpy().shape)#(1,10,1)
	y=torch.from_numpy(y_np[np.newaxis,:,np.newaxis])
	prediction,h_state=rnn(x,h_state)
	h_state=h_state.data # 要把 h_state 重新包装一下才能放入下一个 iteration, 不然会报错
	print(type(prediction.data))#<class 'torch.Tensor'>
	print(prediction.data.numpy().shape)#(1, 10, 1)
	if step==0:
		plt.plot(np.array(steps),prediction.data.numpy()[0],'r',label='prediction')
		plt.plot(np.array(steps),y_np,'b',label='true')
		plt.legend(loc='upper right')
	else:
		plt.plot(np.array(steps),prediction.data.numpy()[0],'r',label='prediction')
		plt.plot(np.array(steps),y_np,'b',label='true')
		# plt.legend(loc="best")
	plt.pause(0.1)
	plt.show()
	loss=loss_func(prediction,y)
	optimizer.zero_grad()
	loss.backward()
	optimizer.step()
plt.ioff()

12.AutoEncoder

import torch 
import torch.nn as nn 
import torch.utils.data as Data 
import torchvision 
import numpy as np
from matplotlib import cm
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from torchvision.transforms import ToPILImage
EPOCH=1
BATCH_SIZE=64
LR=0.005
DOWNLOAD_MNIST=False
N_TEST_IMG=5

train_data=torchvision.datasets.MNIST(
	root='/mnist_1',
	train=True,
	transform=torchvision.transforms.ToTensor(),#仅仅做了格式转换，没有进行归一化处理
	download=DOWNLOAD_MNIST)
# print(type(train_data.train_data))#<class 'torch.Tensor'>
train_loader=Data.DataLoader(dataset=train_data,batch_size=BATCH_SIZE,shuffle=True)
class AutoEncoder(nn.Module):
	def __init__(self):
		super(AutoEncoder,self).__init__()

		#encoder
		self.encoder=nn.Sequential(
			nn.Linear(28*28,128),
			nn.Tanh(),
			nn.Linear(128,64),
			nn.Tanh(),
			nn.Linear(64,12),
			nn.Tanh(),
			nn.Linear(12,3)#压缩成3个特征，进行3D可视化
			)
		#decoder
		self.decoder=nn.Sequential(
			nn.Linear(3,12),
			nn.Tanh(),
			nn.Linear(12,64),
			nn.Tanh(),
			nn.Linear(64,128),
			nn.Tanh(),
			nn.Linear(128,28*28),
			nn.Sigmoid()#让输出值在(0,1)之间
			)
	def forward(self,x):
		encoder=self.encoder(x)
		decoder=self.decoder(encoder)
		return encoder,decoder
autoencoder=AutoEncoder()

optimizer=torch.optim.Adam(autoencoder.parameters(),lr=LR)
loss_func=nn.MSELoss()
plt.ion()
fig=plt.figure(1)
for i in range(1,N_TEST_IMG+1):
	plt.subplot(2,5,i)
	images=train_data.train_data[i].numpy()
	plt.imshow(images)
	plt.show()
data=train_data.train_data[1:6]
data=data.view(-1,28*28)
data=data.float()
# print(type(data))#<class 'torch.Tensor'>
for epoch in range(EPOCH):
	for step,(b_x,label) in enumerate(train_loader):
		b_x=b_x.view(-1,28*28)
		# print(type(b_x))#<class 'torch.Tensor'>
		b_y=b_x.view(-1,28*28)
		encoder,decoder=autoencoder(b_x)
		loss=loss_func(decoder,b_y)
		optimizer.zero_grad()
		loss.backward()
		optimizer.step()
		if step%50==0:
			print('epoch: ',epoch,'| loss: ',loss.data.numpy())
			_,decoder_images=autoencoder(data)
			decoder_images=decoder_images.view(5,28,28)
			for i in range(1,N_TEST_IMG+1):
				plt.subplot(2,5,i+5)
				images=decoder_images[i-1].data.numpy()
				plt.imshow(images)
			plt.pause(0.05)
plt.ioff()
view_data=train_data.train_data[:200].view(-1,28*28).type(torch.FloatTensor)/255
# print(np.unique(view_data))
encoded_data,_=autoencoder(view_data)
fig=plt.figure(2)
ax=Axes3D(fig)
X=encoded_data.data[:,0].numpy()
Y=encoded_data.data[:,1].numpy()
Z=encoded_data.data[:,2].numpy()
values=train_data.train_labels[:200].numpy()
for x,y,z,s in zip(X,Y,Z,values):
	c=cm.rainbow(int(255*s/9))
	ax.text(x,y,z,s,backgroundcolor=c)
ax.set_xlim(X.min(),X.max())
ax.set_ylim(Y.min(),Y.max())
ax.set_ylim(Z.min(),Z.max())
plt.show()

13.GAN

import torch 
import torch.nn as nn 
import numpy as np 
import matplotlib.pyplot as plt 

torch.manual_seed(1)
np.random.seed(1)

BATCH_SIZE=64
LR_G=0.0001#lr for generator
LR_D=0.0001#lr for discriminator
N_IDEAS=5
ART_COMPONENTS=15
PAINT_POINTS=np.vstack([np.linspace(-1,1,ART_COMPONENTS) for _ in range(BATCH_SIZE)])
# print(PAINT_POINTS.shape)#(64,15)
def artist_works():
	a=np.random.uniform(1,2,size=BATCH_SIZE)[:,np.newaxis]
	# print(a.shape)#(64,1)
	paintings=a*np.power(PAINT_POINTS,2)+(a-1)
	paintings=torch.from_numpy(paintings).float()
	return paintings
G=nn.Sequential(
	nn.Linear(N_IDEAS,128),
	nn.ReLU(),
	nn.Linear(128,ART_COMPONENTS)
	)
D=nn.Sequential(
	nn.Linear(ART_COMPONENTS,128),
	nn.ReLU(),
	nn.Linear(128,1),
	nn.Sigmoid()
	)
opt_D=torch.optim.Adam(D.parameters(),lr=LR_D)
opt_G=torch.optim.Adam(G.parameters(),lr=LR_G)
plt.ion()
for step in range(1000):
	artist_paintings=artist_works()
	G_ideas=torch.randn(BATCH_SIZE,N_IDEAS)#(64,5)
	G_paintings=G(G_ideas)
	prob_artist0=D(artist_paintings)
	prob_artist1=D(G_paintings)

	D_loss=-torch.mean(torch.log(prob_artist0)+torch.log(1-prob_artist1))
	G_loss=torch.mean(torch.log(1-prob_artist1))
	opt_D.zero_grad()
	D_loss.backward(retain_graph=True)#重用计算图
	opt_D.step()

	opt_G.zero_grad()
	G_loss.backward()
	opt_G.step()
	if step%50==9:
		plt.cla()
		plt.plot(PAINT_POINTS[0],G_paintings.data.numpy()[0],label="generated painting")
		plt.plot(PAINT_POINTS[0],2*np.power(PAINT_POINTS[0],2)+1,label="upper bound")
		plt.plot(PAINT_POINTS[0],1*np.power(PAINT_POINTS[0],2),label="lower bound")
		plt.text(-.5, 2.3, 'D accuracy=%.2f (0.5 for D to converge)' % prob_artist0.data.numpy().mean(), fontdict={'size': 13})
		plt.text(-.5, 2, 'D score= %.2f (-1.38 for G to converge)' % -D_loss.data.numpy(), fontdict={'size': 13})
		plt.ylim((0, 3));plt.legend(loc='upper right', fontsize=10);plt.draw();plt.pause(0.01)

plt.ioff()
plt.show()

14.overfitting

1.增加数据
2.正则化。L1, l2 regularization等等。简化机器学习的关键公式为 y=Wx ，在过拟合中, W 的值往往变化得特别大或特别小. 为了不让W变化太大, 我们在计算误差上做些手脚. 原始的 cost 误差是这样计算, cost = 预测值-真实值的平方. 如果 W 变得太大, 我们就让 cost 也跟着变大, 变成一种惩罚机制. 所以我们把 W 自己考虑进来. 这里 abs 是绝对值. 这一种形式的 正规化, 叫做 l1 正规化. L2 正规化和 l1 类似, 只是绝对值换成了平方. 其他的l3, l4 也都是换成了立方和4次方等等. 形式类似. 用这些方法,我们就能保证让学出来的线条不会过于扭曲.
3.dropout。我们都让每一次预测结果都不会依赖于其中某部分特定的神经元. 像l1, l2正规化一样, 过度依赖的 W , 也就是训练参数的数值会很大, l1, l2会惩罚这些大的 参数. Dropout 的做法是从根本上让神经网络没机会过度依赖——莫烦

15.Batch Normalization

x 换到了隐藏层当中, 我们能不能对隐藏层的输入结果进行像之前那样的normalization 处理呢? 答案是可以的, 因为大牛们发明了一种技术, 叫做 batch normalization, 正是处理这种情况.
把数据分成小批小批进行 stochastic gradient descent. 而且在每批数据进行前向传递 forward propagation 的时候, 对每一层都进行 normalization 的处理,
Batch Normalization (BN) 就被添加在每一个全连接和激励函数之间.
Batch normalization 不仅仅 normalize 了一下数据, 他还进行了反 normalize 的手续. 为什么要这样呢?
将 normalize 后的数据再扩展和平移. 原来这是为了让神经网络自己去学着使用和修改这个扩展参数 gamma, 和 平移参数 β, 这样神经网络就能自己慢慢琢磨出前面的 normalization 操作到底有没有起到优化的作用, 如果没有起到作用, 我就使用 gamma 和 belt 来抵消一些 normalization 的操作.
注意, 视频里面对于 momentum 的描述说错了, 不是用来更新缩减和平移参数的, 而是用来平滑化 batch mean and stddev 的