Introduction to Deep Learning with PyTorch笔记
柴宝
Artificial Neural Networks
Preparing MNIST dataset
# Transform the data to torch tensors and normalize it
transform = transforms.Compose([transforms.ToTensor(),
transforms.Normalize((0.1307), ((0.3081)))])
# Prepare training set and testing set
trainset = torchvision.datasets.MNIST('mnist', train=True,
download=True, transform=transform)
testset = torchvision.datasets.MNIST('mnist', train=False,
download=True, transform=transform)
# Prepare training loader and testing loader
trainloader = torch.utils.data.DataLoader(trainset, batch_size=32,
shuffle=True, num_workers=0)
testloader = torch.utils.data.DataLoader(testset, batch_size=32,
shuffle=False, num_workers=0)
Inspecting the dataloaders
# Compute the shape of the training set and testing set
trainset_shape = trainset.train_data.shape
testset_shape = testset.test_data.shape
# Print the computed shapes
print(trainset_shape, testset_shape)
# Compute the size of the minibatch for training set and testing set
trainset_batchsize = trainloader.batch_size
testset_batchsize = testloader.batch_size
# Print sizes of the minibatch
print(trainset_batchsize, testset_batchsize)
Building a neural network
# Define the class Net
class Net(nn.Module):
def __init__(self):
# Define all the parameters of the net
super().__init__()
self.fc1 = nn.Linear(28 * 28 * 1, 200)
self.fc2 = nn.Linear(200, 10)
def forward(self, x):
# Do the forward pass
x = F.relu(self.fc1(x))
x = self.fc2(x)
return x
Training a neural network
# Instantiate the Adam optimizer and Cross-Entropy loss function
model = Net()
optimizer = optim.Adam(model.parameters(), lr=3e-4)
criterion = nn.CrossEntropyLoss()
for batch_idx, data_target in enumerate(train_loader):
data = data_target[0]
target = data_target[1]
data = data.view(-1, 28 * 28)
#Zero the parameter gradients
optimizer.zero_grad()
# Complete a forward pass
output = model(data)
# Compute the loss, gradients and change the weights
loss = criterion(output, target)
loss.backward()
optimizer.step()
Using the network to make predictions
# Set the model in eval mode
model.eval()
for i, data in enumerate(test_loader, 0):
inputs, labels = data
# Put each image into a vector
inputs = inputs.view(-1, 28*28)
# Do the forward pass and get the predictions
outputs = model(inputs)
_, outputs = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (outputs == labels).sum().item()
print('The testing set accuracy of the network is: %d %%' % (100 * correct / total))
Convolutional Neural Networks
Convolution operator - OOP way
# Create 10 random images of shape (1, 28, 28)
images = torch.rand(10, 1, 28, 28)
# Build 6 conv. filters
conv_filters = torch.nn.Conv2d(in_channels=1, out_channels=6, kernel_size=3, stride=1, padding=1)
# Convolve the image with the filters
output_feature = conv_filters(images)
print(output_feature.shape)
output:
torch.Size([10, 6, 28, 28])
Convolution operator - Functional way
import torch
import torch.nn.functional as F
# Create 10 random images
image = torch.rand(10, 1, 28, 28)
# Create 6 filters
filters = torch.rand(6, 1, 3, 3)
# Convolve the image with the filters
output_feature = F.conv2d(image, filters, stride=1, padding=1)
print(output_feature.shape)
output:
torch.Size([10, 6, 28, 28])
Max-pooling operator - 2 ways
# Build a pooling operator with size `2`.
max_pooling = torch.nn.MaxPool2d(2)
# 1 Apply the pooling operator
output_feature = max_pooling(im)
# 2 Use pooling operator in the image
output_feature_F = F.max_pool2d(im, 2)
# print the results of both cases
print(output_feature)
print(output_feature_F)
Average-pooling operator - 2 ways
# Build a pooling operator with size `2`.
avg_pooling = torch.nn.AvgPool2d(2)
# 1 Apply the pooling operator
output_feature = avg_pooling(im)
# 2 Use pooling operator in the image
output_feature_F = F.avg_pool2d(im, 2)
# print the results of both cases
print(output_feature)
print(output_feature_F)
Define your first CNN
class Net(nn.Module):
def __init__(self, num_classes):
super(Net, self).__init__()
# Instantiate the ReLU nonlinearity
self.relu = nn.ReLU()
# Instantiate two convolutional layers
self.conv1 = nn.Conv2d(in_channels=1, out_channels=5, kernel_size=3, padding=1)
self.conv2 = nn.Conv2d(in_channels=5, out_channels=10, kernel_size=3, padding=1)
# Instantiate a max pooling layer
self.pool = nn.MaxPool2d(2, 2)
# Instantiate a fully connected layer
self.fc = nn.Linear(7 * 7 * 10, 10)
def forward(self, x):
# Apply conv followd by relu, then in next line pool
x = self.relu(self.conv1(x))
x = self.pool(x)
# Apply conv followd by relu, then in next line pool
x = self.relu(self.conv2(x))
x = self.pool(x)
# Prepare the image for the fully connected layer
x = x.view(-1, 7 * 7 * 10)
# Apply the fully connected layer and return the result
return self.fc(x)
Training CNNs
for i, data in enumerate(train_loader, 0):
inputs, labels = data
optimizer.zero_grad()
# Compute the forward pass
outputs = net(inputs)
# Compute the loss function
loss = criterion(outputs, labels)
# Compute the gradients
loss.backward()
# Update the weights
optimizer.step()
Using CNNs to make predictions
# Iterate over the data in the test_loader
for i, data in enumerate(test_loader):
# Get the image and label from data
image, label = data
# Make a forward pass in the net with your image
output = net(image)
# Argmax the results of the net
_, predicted = torch.max(output.data, 1)
if predicted == label:
print("Yipes, your net made the right prediction " + str(predicted))
else:
print("Your net prediction was " + str(predicted) + ", but the correct label is: " + str(label))
Using Convolutional Neural Networks
Sequential module
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
# Declare all the layers for feature extraction
self.features = nn.Sequential(nn.Conv2d(in_channels=1, out_channels=5, kernel_size=3, padding=1),
nn.ReLU(inplace=True),
nn.Conv2d(in_channels=5, out_channels=10, kernel_size=3, padding=1),
nn.MaxPool2d(2, 2), nn.ReLU(inplace=True),
nn.Conv2d(in_channels=10, out_channels=20, kernel_size=3, padding=1),
nn.ReLU(inplace=True),
nn.Conv2d(in_channels=20, out_channels=40, kernel_size=3, padding=1),
nn.MaxPool2d(2, 2), nn.ReLU(inplace=True))
# Declare all the layers for classification
self.classifier = nn.Sequential(nn.Linear(7 * 7 * 40, 1024), nn.ReLU(inplace=True),
nn.Linear(1024, 2048), nn.ReLU(inplace=True),
nn.Linear(2048, 10))
def forward(self, x):
# Apply the feature extractor in the input
x = self.features(x)
# Squeeze the three spatial dimensions in one
x = x.view(-1, 7 * 7 * 40)
# Classify the images
x = self.classifier(x)
return x
Detecting overfitting
Overfitting的检测需要在验证集（而非测试集，测试集以用来测试最终模型性能），因此需要将模型划分为训练集、验证集和测试集（测试集仅用来最终测试一次）
以下代码用以划分验证集：
# Shuffle the indices
indices = np.arange(60000)
np.random.shuffle(indices)
# Build the train loader
train_loader = torch.utils.data.DataLoader(datasets.MNIST('mnist', download=True, train=True,
transform=transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])),
batch_size=64, shuffle=False, sampler=torch.utils.data.SubsetRandomSampler(indices[:55000]))
# Build the validation loader
val_loader = torch.utils.data.DataLoader(datasets.MNIST('mnist', download=True, train=True,
transform=transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])),
batch_size=64, shuffle=False, sampler=torch.utils.data.SubsetRandomSampler(indices[55000:60000]))
Regularization techniques
避免过拟合的一些方法
- L1、L2正则化
optimizer = optim.adam(net.parameters(),lr=3e-4,weight_decay=0.0001)
- Dropout
在每次迭代学习时，随机丢弃（不更新）每个单元里的一定比例的神经元
nn.Dropout(p=0.5)
- Batch-normalization
计算每一批次的每一个feature的均值和方差，然后进行正态化
self.bn = nn.BatchNorm2d(num_features=64,eps=1e-05,momentum=0.9)
- Early-stopping
每一个epoch结束后都在validation set里测试一下正确率，如果不再提升则早停
如何设置超参数？
先随机初始化多个超参数，在验证集中测试准确率，最后用最好的模型在测试集中进行最终测试。
L2-regularization
# Instantiate the network
model = Net()
# Instantiate the cross-entropy loss
criterion = nn.CrossEntropyLoss()
# Instantiate the Adam optimizer
optimizer = optim.Adam(model.parameters(), lr=3e-4, weight_decay=0.001)
Dropout
class Net(nn.Module):
def __init__(self):
# Define all the parameters of the net
self.classifier = nn.Sequential(
nn.Linear(28*28, 200),
nn.ReLU(inplace=True),
nn.Dropout(p=0.5),
nn.Linear(200, 500),
nn.ReLU(inplace=True),
nn.Linear(500, 10))
def forward(self, x):
# Do the forward pass
return self.classifier(x)
Batch-normalization
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
# Implement the sequential module for feature extraction
self.features = nn.Sequential(
nn.Conv2d(in_channels=1, out_channels=10, kernel_size=3, stride=1, padding=1),
nn.MaxPool2d(2, 2), nn.ReLU(inplace=True), nn.BatchNorm2d(10),
nn.Conv2d(in_channels=10, out_channels=20, kernel_size=3, stride=1, padding=1),
nn.MaxPool2d(2, 2), nn.ReLU(inplace=True), nn.BatchNorm2d(20))
# Implement the fully connected layer for classification
self.fc = nn.Linear(in_features=7*7*20, out_features=10)
Transfer learning
CNN的前几层用来检测基本特征（边缘、对角线），后几层得到更多抽象特征（轮子、眼睛）
迁移学习就是下载已经训练好的模型，应用于新的数据集，再次进行学习，有两种迁移学习的方式
冻结大多数层，不更新权重，仅微调最后几层
# Import the module
import torchvision
# Download resnet18
model = torchvision.models.resnet18(pretrained=True)
# Freeze all the layers bar the last one
for param in model.parameters():
param.requires_grad = False
# Change the number of output units
model.fc = nn.Linear(512, 7)
在已经训练好的模型中微调所有层
# Create a model using
model = Net()
# Load the parameters from the old model
model.load_state_dict(torch.load('my_net.pth'))
# Change the number of out channels
model.fc = nn.Linear(7 * 7 * 512, 26)
# Train and evaluate the model
model.train()
train_net(model, optimizer, criterion)
print("Accuracy of the net is: " + str(model.eval()))
如果使用第一种方法，可以避免过拟合的状况
Torchvision库中有很多已经训练好的模型，包括resnets
model = torchvision.models.resnet18(pretrained=True)
类似资料：