1. 优化器

① 损失函数调用backward方法，就可以调用损失函数的反向传播方法，就可以求出我们需要调节的梯度，我们就可以利用我们的优化器就可以根据梯度对参数进行调整，达到整体误差降低的目的。

② 梯度要清零，如果梯度不清零会导致梯度累加。

2. 神经网络优化一轮

import torch
import torchvision
from torch import nn 
from torch.nn import Conv2d, MaxPool2d, Flatten, Linear, Sequential
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

dataset = torchvision.datasets.CIFAR10("./dataset",train=False,transform=torchvision.transforms.ToTensor(),download=True)       
dataloader = DataLoader(dataset, batch_size=64,drop_last=True)

class Tudui(nn.Module):
    def __init__(self):
        super(Tudui, self).__init__()        
        self.model1 = Sequential(
            Conv2d(3,32,5,padding=2),
            MaxPool2d(2),
            Conv2d(32,32,5,padding=2),
            MaxPool2d(2),
            Conv2d(32,64,5,padding=2),
            MaxPool2d(2),
            Flatten(),
            Linear(1024,64),
            Linear(64,10)
        )
        
    def forward(self, x):
        x = self.model1(x)
        return x
    
loss = nn.CrossEntropyLoss() # 交叉熵    
tudui = Tudui()
optim = torch.optim.SGD(tudui.parameters(),lr=0.01)   # 随机梯度下降优化器
for data in dataloader:
    imgs, targets = data
    outputs = tudui(imgs)
    result_loss = loss(outputs, targets) # 计算实际输出与目标输出的差距
    optim.zero_grad()  # 梯度清零
    result_loss.backward() # 反向传播，计算损失函数的梯度
    optim.step()   # 根据梯度，对网络的参数进行调优
    print(result_loss) # 对数据只看了一遍，只看了一轮，所以loss下降不大

Files already downloaded and verified
tensor(2.2978, grad_fn=<NllLossBackward0>)
tensor(2.2988, grad_fn=<NllLossBackward0>)
tensor(2.3163, grad_fn=<NllLossBackward0>)
tensor(2.3253, grad_fn=<NllLossBackward0>)
tensor(2.2952, grad_fn=<NllLossBackward0>)
tensor(2.3066, grad_fn=<NllLossBackward0>)
tensor(2.3085, grad_fn=<NllLossBackward0>)
tensor(2.3106, grad_fn=<NllLossBackward0>)
tensor(2.2960, grad_fn=<NllLossBackward0>)
tensor(2.3053, grad_fn=<NllLossBackward0>)
tensor(2.2892, grad_fn=<NllLossBackward0>)
tensor(2.3090, grad_fn=<NllLossBackward0>)
tensor(2.2956, grad_fn=<NllLossBackward0>)
tensor(2.3041, grad_fn=<NllLossBackward0>)
tensor(2.3012, grad_fn=<NllLossBackward0>)
tensor(2.3043, grad_fn=<NllLossBackward0>)
tensor(2.2760, grad_fn=<NllLossBackward0>)
tensor(2.3051, grad_fn=<NllLossBackward0>)
tensor(2.2951, grad_fn=<NllLossBackward0>)
tensor(2.3168, grad_fn=<NllLossBackward0>)
tensor(2.3140, grad_fn=<NllLossBackward0>)
tensor(2.3096, grad_fn=<NllLossBackward0>)
tensor(2.2945, grad_fn=<NllLossBackward0>)
tensor(2.3115, grad_fn=<NllLossBackward0>)
tensor(2.2987, grad_fn=<NllLossBackward0>)
tensor(2.3029, grad_fn=<NllLossBackward0>)
tensor(2.3096, grad_fn=<NllLossBackward0>)
tensor(2.3064, grad_fn=<NllLossBackward0>)
tensor(2.3161, grad_fn=<NllLossBackward0>)
tensor(2.3129, grad_fn=<NllLossBackward0>)
tensor(2.2903, grad_fn=<NllLossBackward0>)
tensor(2.3043, grad_fn=<NllLossBackward0>)
tensor(2.3034, grad_fn=<NllLossBackward0>)
tensor(2.3169, grad_fn=<NllLossBackward0>)
tensor(2.3090, grad_fn=<NllLossBackward0>)
tensor(2.3039, grad_fn=<NllLossBackward0>)
tensor(2.3019, grad_fn=<NllLossBackward0>)
tensor(2.3071, grad_fn=<NllLossBackward0>)
tensor(2.3018, grad_fn=<NllLossBackward0>)
tensor(2.3083, grad_fn=<NllLossBackward0>)
tensor(2.2994, grad_fn=<NllLossBackward0>)
tensor(2.2909, grad_fn=<NllLossBackward0>)
tensor(2.3130, grad_fn=<NllLossBackward0>)
tensor(2.2993, grad_fn=<NllLossBackward0>)
tensor(2.2906, grad_fn=<NllLossBackward0>)
tensor(2.3084, grad_fn=<NllLossBackward0>)
tensor(2.3123, grad_fn=<NllLossBackward0>)
tensor(2.2931, grad_fn=<NllLossBackward0>)
tensor(2.3059, grad_fn=<NllLossBackward0>)
tensor(2.3117, grad_fn=<NllLossBackward0>)
tensor(2.2975, grad_fn=<NllLossBackward0>)
tensor(2.3109, grad_fn=<NllLossBackward0>)
tensor(2.3029, grad_fn=<NllLossBackward0>)
tensor(2.3020, grad_fn=<NllLossBackward0>)
tensor(2.3022, grad_fn=<NllLossBackward0>)
tensor(2.3005, grad_fn=<NllLossBackward0>)
tensor(2.2920, grad_fn=<NllLossBackward0>)
tensor(2.3016, grad_fn=<NllLossBackward0>)
tensor(2.3053, grad_fn=<NllLossBackward0>)
tensor(2.3082, grad_fn=<NllLossBackward0>)
tensor(2.3011, grad_fn=<NllLossBackward0>)
tensor(2.3040, grad_fn=<NllLossBackward0>)
tensor(2.3130, grad_fn=<NllLossBackward0>)
tensor(2.2981, grad_fn=<NllLossBackward0>)
tensor(2.2977, grad_fn=<NllLossBackward0>)
tensor(2.2994, grad_fn=<NllLossBackward0>)
tensor(2.3075, grad_fn=<NllLossBackward0>)
tensor(2.3016, grad_fn=<NllLossBackward0>)
tensor(2.2966, grad_fn=<NllLossBackward0>)
tensor(2.3015, grad_fn=<NllLossBackward0>)
tensor(2.3000, grad_fn=<NllLossBackward0>)
tensor(2.2953, grad_fn=<NllLossBackward0>)
tensor(2.2958, grad_fn=<NllLossBackward0>)
tensor(2.2977, grad_fn=<NllLossBackward0>)
tensor(2.2928, grad_fn=<NllLossBackward0>)
tensor(2.2989, grad_fn=<NllLossBackward0>)
tensor(2.2968, grad_fn=<NllLossBackward0>)
tensor(2.2982, grad_fn=<NllLossBackward0>)
tensor(2.2912, grad_fn=<NllLossBackward0>)
tensor(2.3005, grad_fn=<NllLossBackward0>)
tensor(2.2909, grad_fn=<NllLossBackward0>)
tensor(2.2940, grad_fn=<NllLossBackward0>)
tensor(2.2959, grad_fn=<NllLossBackward0>)
tensor(2.2993, grad_fn=<NllLossBackward0>)
tensor(2.2933, grad_fn=<NllLossBackward0>)
tensor(2.2951, grad_fn=<NllLossBackward0>)
tensor(2.2824, grad_fn=<NllLossBackward0>)
tensor(2.2987, grad_fn=<NllLossBackward0>)
tensor(2.2961, grad_fn=<NllLossBackward0>)
tensor(2.2914, grad_fn=<NllLossBackward0>)
tensor(2.3025, grad_fn=<NllLossBackward0>)
tensor(2.2895, grad_fn=<NllLossBackward0>)
tensor(2.2943, grad_fn=<NllLossBackward0>)
tensor(2.2974, grad_fn=<NllLossBackward0>)
tensor(2.2977, grad_fn=<NllLossBackward0>)
tensor(2.3069, grad_fn=<NllLossBackward0>)
tensor(2.2972, grad_fn=<NllLossBackward0>)
tensor(2.2979, grad_fn=<NllLossBackward0>)
tensor(2.2932, grad_fn=<NllLossBackward0>)
tensor(2.2940, grad_fn=<NllLossBackward0>)
tensor(2.3014, grad_fn=<NllLossBackward0>)
tensor(2.2958, grad_fn=<NllLossBackward0>)
tensor(2.3013, grad_fn=<NllLossBackward0>)
tensor(2.2953, grad_fn=<NllLossBackward0>)
tensor(2.2951, grad_fn=<NllLossBackward0>)
tensor(2.3116, grad_fn=<NllLossBackward0>)
tensor(2.2916, grad_fn=<NllLossBackward0>)
tensor(2.2871, grad_fn=<NllLossBackward0>)
tensor(2.2975, grad_fn=<NllLossBackward0>)
tensor(2.2950, grad_fn=<NllLossBackward0>)
tensor(2.3039, grad_fn=<NllLossBackward0>)
tensor(2.2901, grad_fn=<NllLossBackward0>)
tensor(2.2950, grad_fn=<NllLossBackward0>)
tensor(2.2958, grad_fn=<NllLossBackward0>)
tensor(2.2893, grad_fn=<NllLossBackward0>)
tensor(2.2917, grad_fn=<NllLossBackward0>)
tensor(2.3001, grad_fn=<NllLossBackward0>)
tensor(2.2988, grad_fn=<NllLossBackward0>)
tensor(2.3069, grad_fn=<NllLossBackward0>)
tensor(2.3083, grad_fn=<NllLossBackward0>)
tensor(2.2841, grad_fn=<NllLossBackward0>)
tensor(2.2932, grad_fn=<NllLossBackward0>)
tensor(2.2857, grad_fn=<NllLossBackward0>)
tensor(2.2971, grad_fn=<NllLossBackward0>)
tensor(2.2999, grad_fn=<NllLossBackward0>)
tensor(2.2911, grad_fn=<NllLossBackward0>)
tensor(2.2977, grad_fn=<NllLossBackward0>)
tensor(2.3027, grad_fn=<NllLossBackward0>)
tensor(2.2940, grad_fn=<NllLossBackward0>)
tensor(2.2939, grad_fn=<NllLossBackward0>)
tensor(2.2950, grad_fn=<NllLossBackward0>)
tensor(2.2951, grad_fn=<NllLossBackward0>)
tensor(2.3000, grad_fn=<NllLossBackward0>)
tensor(2.2935, grad_fn=<NllLossBackward0>)
tensor(2.2817, grad_fn=<NllLossBackward0>)
tensor(2.2977, grad_fn=<NllLossBackward0>)
tensor(2.3067, grad_fn=<NllLossBackward0>)
tensor(2.2742, grad_fn=<NllLossBackward0>)
tensor(2.2964, grad_fn=<NllLossBackward0>)
tensor(2.2927, grad_fn=<NllLossBackward0>)
tensor(2.2941, grad_fn=<NllLossBackward0>)
tensor(2.3003, grad_fn=<NllLossBackward0>)
tensor(2.2965, grad_fn=<NllLossBackward0>)
tensor(2.2908, grad_fn=<NllLossBackward0>)
tensor(2.2885, grad_fn=<NllLossBackward0>)
tensor(2.2984, grad_fn=<NllLossBackward0>)
tensor(2.3009, grad_fn=<NllLossBackward0>)
tensor(2.2931, grad_fn=<NllLossBackward0>)
tensor(2.2856, grad_fn=<NllLossBackward0>)
tensor(2.2907, grad_fn=<NllLossBackward0>)
tensor(2.2938, grad_fn=<NllLossBackward0>)
tensor(2.2880, grad_fn=<NllLossBackward0>)
tensor(2.2975, grad_fn=<NllLossBackward0>)
tensor(2.2922, grad_fn=<NllLossBackward0>)
tensor(2.2966, grad_fn=<NllLossBackward0>)
tensor(2.2804, grad_fn=<NllLossBackward0>)

3. 神经网络优化多轮

import torch
import torchvision
from torch import nn 
from torch.nn import Conv2d, MaxPool2d, Flatten, Linear, Sequential, Softmax
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

dataset = torchvision.datasets.CIFAR10("./dataset",train=False,transform=torchvision.transforms.ToTensor(),download=True)       
dataloader = DataLoader(dataset, batch_size=64,drop_last=True)

class Tudui(nn.Module):
    def __init__(self):
        super(Tudui, self).__init__()        
        self.model1 = Sequential(
            Conv2d(3,32,5,padding=2),
            MaxPool2d(2),
            Conv2d(32,32,5,padding=2),
            MaxPool2d(2),
            Conv2d(32,64,5,padding=2),
            MaxPool2d(2),
            Flatten(),
            Linear(1024,64),
            Linear(64,10),
            Softmax()
        )
        
    def forward(self, x):
        x = self.model1(x)
        return x
    
loss = nn.CrossEntropyLoss() # 交叉熵    
tudui = Tudui()
optim = torch.optim.SGD(tudui.parameters(),lr=0.01)   # 随机梯度下降优化器
for epoch in range(20):
    running_loss = 0.0
    for data in dataloader:
        imgs, targets = data
        outputs = tudui(imgs)
        result_loss = loss(outputs, targets) # 计算实际输出与目标输出的差距
        optim.zero_grad()  # 梯度清零
        result_loss.backward() # 反向传播，计算损失函数的梯度
        optim.step()   # 根据梯度，对网络的参数进行调优
        running_loss = running_loss + result_loss
    print(running_loss) # 对这一轮所有误差的总和

Files already downloaded and verified


F:\miniconda\envs\yolov5\lib\site-packages\torch\nn\modules\container.py:119: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.
  input = module(input)


tensor(359.2307, grad_fn=<AddBackward0>)
tensor(359.2193, grad_fn=<AddBackward0>)
tensor(359.2079, grad_fn=<AddBackward0>)
tensor(359.1968, grad_fn=<AddBackward0>)
tensor(359.1857, grad_fn=<AddBackward0>)
tensor(359.1743, grad_fn=<AddBackward0>)
tensor(359.1625, grad_fn=<AddBackward0>)
tensor(359.1497, grad_fn=<AddBackward0>)
tensor(359.1360, grad_fn=<AddBackward0>)
tensor(359.1210, grad_fn=<AddBackward0>)
tensor(359.1043, grad_fn=<AddBackward0>)
tensor(359.0850, grad_fn=<AddBackward0>)



---------------------------------------------------------------------------

KeyboardInterrupt                         Traceback (most recent call last)

Cell In[4], line 38
     36 for data in dataloader:
     37     imgs, targets = data
---> 38     outputs = tudui(imgs)
     39     result_loss = loss(outputs, targets) # 计算实际输出与目标输出的差距
     40     optim.zero_grad()  # 梯度清零


File F:\miniconda\envs\yolov5\lib\site-packages\torch\nn\modules\module.py:889, in Module._call_impl(self, *input, **kwargs)
    887     result = self._slow_forward(*input, **kwargs)
    888 else:
--> 889     result = self.forward(*input, **kwargs)
    890 for hook in itertools.chain(
    891         _global_forward_hooks.values(),
    892         self._forward_hooks.values()):
    893     hook_result = hook(self, input, result)


Cell In[4], line 28, in Tudui.forward(self, x)
     27 def forward(self, x):
---> 28     x = self.model1(x)
     29     return x


File F:\miniconda\envs\yolov5\lib\site-packages\torch\nn\modules\module.py:889, in Module._call_impl(self, *input, **kwargs)
    887     result = self._slow_forward(*input, **kwargs)
    888 else:
--> 889     result = self.forward(*input, **kwargs)
    890 for hook in itertools.chain(
    891         _global_forward_hooks.values(),
    892         self._forward_hooks.values()):
    893     hook_result = hook(self, input, result)


File F:\miniconda\envs\yolov5\lib\site-packages\torch\nn\modules\container.py:119, in Sequential.forward(self, input)
    117 def forward(self, input):
    118     for module in self:
--> 119         input = module(input)
    120     return input


File F:\miniconda\envs\yolov5\lib\site-packages\torch\nn\modules\module.py:889, in Module._call_impl(self, *input, **kwargs)
    887     result = self._slow_forward(*input, **kwargs)
    888 else:
--> 889     result = self.forward(*input, **kwargs)
    890 for hook in itertools.chain(
    891         _global_forward_hooks.values(),
    892         self._forward_hooks.values()):
    893     hook_result = hook(self, input, result)


File F:\miniconda\envs\yolov5\lib\site-packages\torch\nn\modules\pooling.py:162, in MaxPool2d.forward(self, input)
    161 def forward(self, input: Tensor) -> Tensor:
--> 162     return F.max_pool2d(input, self.kernel_size, self.stride,
    163                         self.padding, self.dilation, self.ceil_mode,
    164                         self.return_indices)


File F:\miniconda\envs\yolov5\lib\site-packages\torch\_jit_internal.py:365, in boolean_dispatch.<locals>.fn(*args, **kwargs)
    363     return if_true(*args, **kwargs)
    364 else:
--> 365     return if_false(*args, **kwargs)


File F:\miniconda\envs\yolov5\lib\site-packages\torch\nn\functional.py:659, in _max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode, return_indices)
    657 if stride is None:
    658     stride = torch.jit.annotate(List[int], [])
--> 659 return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)


KeyboardInterrupt: 

4. 神经网络学习率优化

import torch
from torch import nn
from torch.nn import MaxPool2d, Conv2d, Flatten, Linear, Sequential, Softmax
from torch.utils.data import DataLoader
import torchvision

dataset = torchvision.datasets.CIFAR10("./dataset",train=False,transform=torchvision.transforms.ToTensor(),download=True)
dataloader = DataLoader(dataset, batch_size=64,drop_last=True)

class Tudui(nn.Module):
    def __init__(self):
        super(Tudui, self).__init__()
        self.model1 = Sequential(
            Conv2d(3,32,5,padding=2),
            MaxPool2d(2),
            Conv2d(32,32,5,padding=2),
            MaxPool2d(2),
            Conv2d(32,64,5,padding=2),
            MaxPool2d(2),
            Flatten(),
            Linear(1024,64),
            Linear(64,10)
        )

    def forward(self, x):
        x = self.model1(x)
        return x

tudui = Tudui()
loss = nn.CrossEntropyLoss()
optim = torch.optim.Adam(tudui.parameters(), lr=0.01)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer=optim, step_size=5, gamma=0.1)
for epoch in range(20):
    running_loss = 0
    for data in dataloader:
        imgs, targets = data
        outputs = tudui(imgs)
        losses = loss(outputs, targets)
        optim.zero_grad()
        losses.backward()
        optim.step()
        scheduler.step()
        running_loss = running_loss + losses
    print("epoch {}".format(epoch), "final losses is {}".format(running_loss), end="\n")

Files already downloaded and verified
epoch 0 final losses is 1018.9046630859375
epoch 1 final losses is 826.07275390625
epoch 2 final losses is 826.07275390625
epoch 3 final losses is 826.07275390625
epoch 4 final losses is 826.07275390625
epoch 5 final losses is 826.07275390625
epoch 6 final losses is 826.07275390625
epoch 7 final losses is 826.07275390625
epoch 8 final losses is 826.07275390625
epoch 9 final losses is 826.07275390625
epoch 10 final losses is 826.07275390625
epoch 11 final losses is 826.07275390625
epoch 12 final losses is 826.07275390625
epoch 13 final losses is 826.07275390625
epoch 14 final losses is 826.07275390625
epoch 15 final losses is 826.07275390625
epoch 16 final losses is 826.07275390625
epoch 17 final losses is 826.07275390625
epoch 18 final losses is 826.07275390625
epoch 19 final losses is 826.07275390625

import torch
import torchvision
from torch import nn 
from torch.nn import Conv2d, MaxPool2d, Flatten, Linear, Sequential
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

dataset = torchvision.datasets.CIFAR10("./dataset",train=False,transform=torchvision.transforms.ToTensor(),download=True)       
dataloader = DataLoader(dataset, batch_size=64,drop_last=True)

class Tudui(nn.Module):
    def __init__(self):
        super(Tudui, self).__init__()        
        self.model1 = Sequential(
            Conv2d(3,32,5,padding=2),
            MaxPool2d(2),
            Conv2d(32,32,5,padding=2),
            MaxPool2d(2),
            Conv2d(32,64,5,padding=2),
            MaxPool2d(2),
            Flatten(),
            Linear(1024,64),
            Linear(64,10)
        )
        
    def forward(self, x):
        x = self.model1(x)
        return x
    
loss = nn.CrossEntropyLoss() # 交叉熵    
tudui = Tudui()
optim = torch.optim.SGD(tudui.parameters(),lr=0.01)   # 随机梯度下降优化器
scheduler = torch.optim.lr_scheduler.StepLR(optim, step_size=5, gamma=0.1) # 每过 step_size 更新一次优化器，更新是学习率为原来的学习率的的 0.1 倍    
for epoch in range(20):
    running_loss = 0.0
    for data in dataloader:
        imgs, targets = data
        outputs = tudui(imgs)
        result_loss = loss(outputs, targets) # 计算实际输出与目标输出的差距
        optim.zero_grad()  # 梯度清零
        result_loss.backward() # 反向传播，计算损失函数的梯度
        optim.step()   # 根据梯度，对网络的参数进行调优
        scheduler.step() # 学习率太小了，所以20个轮次后，相当于没走多少
        running_loss = running_loss + result_loss
    print(running_loss) # 对这一轮所有误差的总和

Files already downloaded and verified
tensor(359.4925, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)
tensor(359.4839, grad_fn=<AddBackward0>)