optuna调参工具的使用

 导入包

!pip install optuna
import optuna

写一个object

 def objective(trial):

        参数

        实例化model

        损失函数

        优化器、调度器

        训练+测试

        return loss or accuracy;

 没有预训练的：

def objective(trial):
    param = {
        # 'metric':'CrossEntropyLoss',
        'img_size':trial.suggest_int('img_size',256,256),
        'patch_size' : trial.suggest_categorical('patch_size', [16]),
        'embed_dim':trial.suggest_int('embed_dim',768,768),
        'depth':trial.suggest_categorical('depth',[12]),
        'heads':trial.suggest_int('heads',12,12),
        'num_classes':trial.suggest_int('num_classes',5,5)
    }
    model = VisionTransformer(**param) # .to(device)
    # print("eff model = ",model_ft.buffers) #打印模型结构
    
    criterion = nn.CrossEntropyLoss()
    
    # optimizer = torch.optim.Adam(model.parameters())
    # scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', factor=0.94,verbose=True,patience = 1,min_lr = 0.000001) # 动态更新学习率

    lr = trial.suggest_float('lr', 1e-2, 1e-1, log=True)
    momentum = trial.suggest_float('momentum', 0.8, 0.9)
    epochs = trial.suggest_int('epochs',50, 50)

    optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=0.0004)
    
    if True:
        model = model.cuda()
        criterion = criterion.cuda()

    train_loss ,best_model  = train_model(model, criterion, optimizer, num_epochs=epochs)

    print('-' * 80)
    print('Test Accuracy:')

    test_loss = test_model(best_model, criterion)

    return test_loss

 加载预训练的模型

def objective(trial):
    
    # 训练模型并返回验证集的准确率或损失
    model_ft = torch.load('/root/saved_model/efficientnet-b703.pth')
    num_ftrs = model_ft._fc.in_features # 获取原来fc层的输入通道数
    model_ft._fc = nn.Linear(num_ftrs, 3)
    # print("eff model = ",model_ft.buffers) #打印模型结构

    criterion = nn.CrossEntropyLoss()

    lr = trial.suggest_float('lr', 1e-5, 1e-1, log=True)
    momentum = trial.suggest_float('momentum', 0.1, 0.9)

    optimizer = torch.optim.SGD(model_ft.parameters(), lr=lr, momentum=momentum, weight_decay=0.0004)
    
    if True:
        model_ft = model_ft.cuda()
        criterion = criterion.cuda()

    t_loss ,best_model  = train_model(model_ft, criterion, optimizer, exp_lr_scheduler,num_epochs=num_epochs)

    print('-' * 80)
    print('Test Accuracy:')

    test_loss = test_model(best_model , criterion)

    return test_loss

执行study

    study = optuna.create_study(direction='minimize',study_name='Optimizie hyperpaprameters')
    study.optimize(objective,n_trials=10)
    # 打印最优的参数组合
    # print('Best trail: ',study.best_trial.params)
    
    print("Best parameters:", study.best_params)

    # 打印最佳metric
    print("Best loss:", study.best_value)

 参考：optuna/optuna: A hyperparameter optimization framework (github.com)