目录
一、为什么要调整学习率？
1.1 class _LRScheduler
二、pytorch的六种学习率调整策略
2.1 StepLR
代码实现：
import torch
import torch.optim as optim
import numpy as np
import matplotlib.pyplot as plt
torch.manual_seed(1)
LR = 0.1
iteration = 10
max_epoch = 200
# ------------------------------ fake data and optimizer ------------------------------
weights = torch.randn((1), requires_grad=True)
target = torch.zeros((1))
optimizer = optim.SGD([weights], lr=LR, momentum=0.9)
# ------------------------------ 1 Step LR ------------------------------
# flag = 0
flag = 1
if flag:
scheduler_lr = optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.1) # 设置学习率下降策略，50轮下降一次，每次下降10倍
lr_list, epoch_list = list(), list()
for epoch in range(max_epoch):
lr_list.append(scheduler_lr.get_lr())
epoch_list.append(epoch)
for i in range(iteration):
loss = torch.pow((weights - target), 2)
loss.backward()
optimizer.step()
optimizer.zero_grad()
scheduler_lr.step() # 学习率更新策略
plt.plot(epoch_list, lr_list, label="Step LR Scheduler")
plt.xlabel("Epoch")
plt.ylabel("Learning rate")
plt.legend()
plt.show()
输出结果：
2.2 MultiStepLR
代码实现
flag = 1
if flag:
milestones = [50, 125, 160] # 设置学习率下降的位置
scheduler_lr = optim.lr_scheduler.MultiStepLR(optimizer, milestones=milestones, gamma=0.1)
lr_list, epoch_list = list(), list()
for epoch in range(max_epoch):
lr_list.append(scheduler_lr.get_lr())
epoch_list.append(epoch)
for i in range(iteration):
loss = torch.pow((weights - target), 2)
loss.backward()
optimizer.step()
optimizer.zero_grad()
scheduler_lr.step()
plt.plot(epoch_list, lr_list, label="Multi Step LR Scheduler\nmilestones:{}".format(milestones))
plt.xlabel("Epoch")
plt.ylabel("Learning rate")
plt.legend()
plt.show()
输出结果
2.3 ExponentialLR
代码实现
flag = 1
if flag:
gamma = 0.95
scheduler_lr = optim.lr_scheduler.ExponentialLR(optimizer, gamma=gamma)
lr_list, epoch_list = list(), list()
for epoch in range(max_epoch):
lr_list.append(scheduler_lr.get_lr())
epoch_list.append(epoch)
for i in range(iteration):
loss = torch.pow((weights - target), 2)
loss.backward()
optimizer.step()
optimizer.zero_grad()
scheduler_lr.step()
plt.plot(epoch_list, lr_list, label="Exponential LR Scheduler\ngamma:{}".format(gamma))
plt.xlabel("Epoch")
plt.ylabel("Learning rate")
plt.legend()
plt.show()
输出结果
2.4 CosineAnnealingLR
代码实现
flag = 1
if flag:
t_max = 50
scheduler_lr = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=t_max, eta_min=0.)
lr_list, epoch_list = list(), list()
for epoch in range(max_epoch):
lr_list.append(scheduler_lr.get_lr())
epoch_list.append(epoch)
for i in range(iteration):
loss = torch.pow((weights - target), 2)
loss.backward()
optimizer.step()
optimizer.zero_grad()
scheduler_lr.step()
plt.plot(epoch_list, lr_list, label="CosineAnnealingLR Scheduler\nT_max:{}".format(t_max))
plt.xlabel("Epoch")
plt.ylabel("Learning rate")
plt.legend()
plt.show()
输出结果
2.5 ReduceLRonPlateau
代码实现
flag = 1
if flag:
loss_value = 0.5
accuray = 0.9
factor = 0.1 # 学习率变换参数
mode = "min"
patience = 10 # 能接受多少轮不变化
cooldown = 10 # 停止监控多少轮
min_lr = 1e-4 # 设置学习率下限
verbose = True # 打印更新日志
scheduler_lr = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=factor, mode=mode, patience=patience,
cooldown=cooldown, min_lr=min_lr, verbose=verbose)
for epoch in range(max_epoch):
for i in range(iteration):
# train(...)
optimizer.step()
optimizer.zero_grad()
#if epoch == 5:
# loss_value = 0.4
scheduler_lr.step(loss_value) #监控的标量是否下降
输出结果
2.6 LambdaLR
功能：自定义调整策略
主要参数：
• lr_lambda：function or list
代码实现
flag = 1
if flag:
lr_init = 0.1
weights_1 = torch.randn((6, 3, 5, 5))
weights_2 = torch.ones((5, 5))
optimizer = optim.SGD([
{'params': [weights_1]},
{'params': [weights_2]}], lr=lr_init)
# 设置两种不同的学习率调整方法
lambda1 = lambda epoch: 0.1 ** (epoch // 20) # 每到20轮的时候学习率变为原来的0.1倍
lambda2 = lambda epoch: 0.95 ** epoch # 将学习率进行指数下降
scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=[lambda1, lambda2])
lr_list, epoch_list = list(), list()
for epoch in range(max_epoch):
for i in range(iteration):
# train(...)
optimizer.step()
optimizer.zero_grad()
scheduler.step()
lr_list.append(scheduler.get_lr())
epoch_list.append(epoch)
print('epoch:{:5d}, lr:{}'.format(epoch, scheduler.get_lr()))
plt.plot(epoch_list, [i[0] for i in lr_list], label="lambda 1")
plt.plot(epoch_list, [i[1] for i in lr_list], label="lambda 2")
plt.xlabel("Epoch")
plt.ylabel("Learning Rate")
plt.title("LambdaLR")
plt.legend()
plt.show()
输出结果
三、学习率调整小结
- 有序调整：Step、MultiStep、Exponential 和 CosineAnnealing
- 自适应调整：ReduceLROnPleateau
- 自定义调整：Lambda
四、学习率初始化
1、设置较小数：0.01、0.001、0.0001
2、搜索最大学习率： 参考该篇《Cyclical Learning Rates for Training Neural Networks》
方法：我们可以设置学习率逐渐从小变大观察精确度的一个变化，下面这幅图，当学习率为0.055左右的时候模型精确度最高，当学习率大于0.055的时候精确度出现下降情况，所以在模型训练过程中我们可以设置学习率为0.055作为我们的初始学习率。
