Adalora使用教程
import loralib
layer = loralib.SVDlLinear(in_feature, outfeature, r=12)
model = BigModel()
loralib.mark_only_lora_as_trainable(model)
from loralib import RankAllocator
from loralib import compute_orth_regu
rankallocator = RankAllocator(model, lora_r=12, target_rank=8, init_warmup=500,final_warmup=1500,
mask_interval=10, total_step=3000, beta1=0.85, beta2=0.85)
(loss + compute_orth_regu(model,regu_weight=0.1)).batckward()
optimizer.step()
randallocator.update_and_mask(model, global_step)
glob_step += 1