- 训练
#generic
class LayerLinearRegression(nn.Module):
def __init__(self):
super().__init__()
# Instead of our custom parameters, we use a Linear layer with single input and single output
self.linear = nn.Linear(1, 1)
def forward(self, x):
# Now it only takes a call to the layer to make predictions
return self.linear(x)
def make_train_step(model, loss_fn, optimizer):
# Builds function that performs a step in the train loop
def train_step(x, y):
# Sets model to TRAIN mode
model.train()
# Makes predictions
yhat = model(x)
# Computes loss
loss = loss_fn(y, yhat)
# Computes gradients
loss.backward()
# Updates parameters and zeroes gradients
optimizer.step()
optimizer.zero_grad()
# Returns the loss
return loss.item()
# Returns the function that will be called inside the train loop
return train_step
# Creates the train_step function for our model, loss function and optimizer
train_step = make_train_step(model, loss_fn, optimizer)
losses = []
# For each epoch...
for epoch in range(n_epochs):
# Performs one train step and returns the corresponding loss
loss = train_step(x_train_tensor, y_train_tensor)
losses.append(loss)
# Checks model's parameters
print(model.state_dict())
- Update parameters
# FIRST ATTEMPT
# AttributeError: 'NoneType' object has no attribute 'zero_'
# reason: we “lost” the gradient while reassigning the update results to our parameters
a = a - lr * a.grad
b = b - lr * b.grad
print(a)
# SECOND ATTEMPT
# RuntimeError: a leaf Variable that requires grad has been used in an in-place operation.
# reason: we build a unwanted dynamic computation graph
a -= lr * a.grad
b -= lr * b.grad
# THIRD ATTEMPT
# We need to use NO_GRAD to keep the update out of the gradient computation
with torch.no_grad():
a -= lr * a.grad
b -= lr * b.grad
# PyTorch is "clingy" to its computed gradients, we need to tell it to let it go...
a.grad.zero_()
b.grad.zero_()
- 第一种方法会失去requires_grad=True属性,第二种方法会建立不需要的dynamic computation graph,第三种方法使用with torch.no_grad()来阻止建立计算图,
- gradients会累积,需要a.grad.zero_()进行清除,注意在pytorch里以“_”结尾的functioin会在进行"inplace operation"
2.建立参数
device = 'cuda' if torch.cuda.is_available() else 'cpu'
数据:直接使用to(device)
x_train_tensor = torch.from_numpy(x_train).float().to(device)
参数
# 1
a = torch.randn(1, requires_grad=True, dtype=torch.float).to(device)
b = torch.randn(1, requires_grad=True, dtype=torch.float).to(device)
# 2
a = torch.randn(1, dtype=torch.float).to(device)
b = torch.randn(1, dtype=torch.float).to(device)
# and THEN set them as requiring gradients...
a.requires_grad_()
b.requires_grad_()
# 3
a = torch.randn(1, requires_grad=True, dtype=torch.float, device=device)
b = torch.randn(1, requires_grad=True, dtype=torch.float, device=device)
第一种方法不可行,因为会失去requires_grad=True属性(在使用gpu的情况下),第二第三种方法都可以
- gpu_tensor不能直接转numpy
np_array = cuda_tensor.cpu().numpy()
- optim.SGD([a, b], lr=lr)
if we use all training data at once for the update as we are actually doing in the code, the optimizer is performing a batch gradient descent, despite of its name.
- Dataloader
from torch.utils.data import Dataset, TensorDataset
from torch.utils.data import DataLoader
class CustomDataset(Dataset):
def __init__(self, x_tensor, y_tensor):
self.x = x_tensor
self.y = y_tensor
def __getitem__(self, index):
return (self.x[index], self.y[index])
def __len__(self):
return len(self.x)
# Wait, is this a CPU tensor now? Why? Where is .to(device)?
x_train_tensor = torch.from_numpy(x_train).float()
y_train_tensor = torch.from_numpy(y_train).float()
train_data = CustomDataset(x_train_tensor, y_train_tensor)
print(train_data[0])
train_data = TensorDataset(x_train_tensor, y_train_tensor)
print(train_data[0])
train_loader = DataLoader(dataset=train_data, batch_size=16, shuffle=True)
next(iter(train_loader))
在for 循环里, 总共有三点操作: Link
- 调用了dataloader 的__iter__() 方法, 产生了一个DataLoaderIter
- 反复调用DataLoaderIter 的__next__()来得到batch, 具体操作就是, 多次调用dataset的__getitem__()方法 (如果num_worker>0就多线程调用), 然后用collate_fn来把它们打包成batch. 中间还会涉及到shuffle , 以及sample 的方法等, 这里就不多说了.
- 当数据读完后, next()抛出一个StopIteration异常, for循环结束, dataloader 失效.
losses = []
train_step = make_train_step(model, loss_fn, optimizer)
for epoch in range(n_epochs):
for x_batch, y_batch in train_loader:
# the dataset "lives" in the CPU, so do our mini-batches
# therefore, we need to send those mini-batches to the
# device where the model "lives"
x_batch = x_batch.to(device)
y_batch = y_batch.to(device)
loss = train_step(x_batch, y_batch)
losses.append(loss)
print(model.state_dict())
- evaluation
with torch.no_grad():
for x_val, y_val in val_loader:
x_val = x_val.to(device)
y_val = y_val.to(device)
model.eval() #设置为evalate模式
yhat = model(x_val)
val_loss = loss_fn(y_val, yhat)
val_losses.append(val_loss.item())
参考:
Understanding PyTorch with an example: a step-by-step tutorial