Bert小黑初次尝试1:Bert_finetuning

1.参数定义

from importlib import import_module
import time
import torch
import numpy as np
import torch.nn as nn
from pytorch_pretrained import BertModel,BertTokenizer
# config类
class Config(object):
    '''配置参数'''
    def __init__(self,dataset):
        self.model_name = 'bert'
        self.train_path = dataset +'/data/dev.txt'    # 训练集
        self.dev_path = dataset + '/data/dev.txt'    # 验证集
        self.test_path = dataset + '/data/test.txt'    # 测试集
        self.class_list = [x.strip() for x in open(dataset+'/data/class.txt').readlines()]    # 类别标签
        self.save_path = dataset + '/saved_dict/' + self.model_name + '.ckpt'。  # 模型保存地址
        self.device = torch.device('cuda' if torch.cuda.is_available else 'cpu')
        self.require_improvement = 1000   # 若超过该迭代次数，模型准确率没有提升，就停止训练
        self.num_classes = len(self.class_list)。  # 类别数
        self.num_epochs = 3。  # 轮数
        self.batch_size = 128。  # 批次数量
        self.pad_size = 32。  # 每个句子长度
        self.learning_rate = 5e-5。  # 学习率
        self.bert_path = '../Bert-Chinese-Text-Classification-Pytorch-master/bert_pretrain/'    # bert地址
        self.tokenizer = BertTokenizer.from_pretrained(self.bert_path+'vocab.txt')
        self.hidden_size = 768。  # 隐层数量

2.模型建立

class Model(nn.Module):
    def __init__(self,config):
        super(Model,self).__init__()
        self.bert = BertModel.from_pretrained(config.bert_path)    # 加载预训练模型
        for param in self.bert.parameters():
            param.requires_grad = True
        self.fc = nn.Linear(config.hidden_size,config.num_classes)。  # 全连接层
    def forward(self,x):
        context = x[0]    # 输入的句子
        mask = x[2]     # 对padding进行mask
        _,pooled = self.bert(context,attention_mask = mask,output_all_encoded_layers = False)    # [batch_size,hidden_size]
        out = self.fc(pooled).   # [batch_size,num_classes]
        return out
#config = Config('.')
#model = Model(config)
#model.bert(torch.ones(4,8).long(),torch.ones(4,8).long())[1].shape
dataset = '.'
config = Config(dataset)
np.random.seed(1)
torch.manual_seed(1)
torch.cuda.manual_seed_all(1)
torch.backends.cudnn.deterministic = True    # 保证每次结果一样
start_time = time.time()
print('Loading data...')

3.数据集建立

from tqdm import tqdm
import time
from datetime import timedelta
PAD,CLS = '[PAD]','[CLS]'    
def build_dataset(config):
    def load_dataset(path,pad_size = 32):
        contents = []
        with open(path,'r',encoding = 'utf-8') as f:
            for line in tqdm(f):
                lin = line.strip()
                if not lin:
                    continue
                content,label = lin.split('\t')
                token = config.tokenizer.tokenize(content)
                token = [CLS] + token    # 开始标记
                seq_len = len(token)。  # 句子长度
                mask = []
                token_ids = config.tokenizer.convert_tokens_to_ids(token)
                if pad_size:    # 长截断，短pad补齐
                    if len(token) < pad_size:
                        mask = [1] * len(token_ids) + [0] * (pad_size - len(token))
                        token_ids += ([0] * (pad_size - len(token)))
                    else:
                        mask = [1] * pad_size
                        token_ids = token_ids[:pad_size]
                        seq_len = pad_size
                contents.append((token_ids,int(label),seq_len,mask))
        return contents
    train = load_dataset(config.train_path,config.pad_size)
    dev = load_dataset(config.dev_path,config.pad_size)
    test = load_dataset(config.test_path,config.pad_size)
    return train,dev,test
# 建立数据集迭代器
def build_iterator(dataset,config):
    iter_ = DatasetIterater(dataset,config.batch_size,config.device)
    return iter_
def get_time_dif(start_time):
    '''获取已经使用的时间'''
    end_time = time.time()
    time_dif = end_time - start_time
    return timedelta(seconds = int(round(time_dif)))
class DatasetIterater(object):
    def __init__(self,batches,batch_size,device):
        self.batch_size = batch_size
        self.batches = batches
        self.n_batches = len(batches) // batch_size
        self.residue = False
        if len(batches) % self.n_batches != 0:
            self.residue = True
        self.index = 0
        self.device = device
    def _to_tensor(self,datas):
        x = torch.LongTensor([_[0] for _ in datas]).to(self.device)
        y = torch.LongTensor([_[1] for _ in datas]).to(self.device)
        # pad前的长度(超过pad_size的设置为pad_size)
        seq_len = torch.LongTensor([_[2] for _ in datas]).to(self.device)
        mask = torch.LongTensor([_[3] for _ in datas]).to(self.device)
        return (x,seq_len,mask),y
    # 迭代下一个batch
    def __next__(self):
        if self.residue and self.index == self.n_batches:
            batches = self.batches[self.index * self.batch_size:len(self.batches)]
            self.index += 1
            batches = self._to_tensor(batches)
            return batches
        elif self.index >= self.n_batches:
            self.index = 0
            raise StopIteration
        else:
            batches = self.batches[self.index * self.batch_size: (self.index + 1) * self.batch_size]
            self.index += 1
            batches = self._to_tensor(batches)
            return batches
    def __iter__(self):
        return self
    def __len__(self):
        if self.residue:
            return self.n_batches + 1
        else:
            return self.n_batches
train_data,dev_data,test_data = build_dataset(config)
train_iter = build_iterator(train_data,config)
dev_iter = build_iterator(test_data,config)
test_iter = build_iterator(test_data,config)
time_dif = get_time_dif(start_time)
print('Time usage:',time_dif)

4.模型训练

# 建立模型
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from sklearn import metrics
import time
from pytorch_pretrained_bert.optimization import BertAdam
# 初始化参数
def init_network(model,method = 'xavier',exclude = 'embedding',seed = 123):
    for name,w in model.named_parameters():
        if exclude not in name:
            if len(w.size()) < 2:
                continue
            if 'weight' in name:
                if method == 'xavier':
                    nn.init.xavier_normal_(w)
                elif method == 'kaiming':
                    nn.init.kaiming_normal_(w)
                else:
                    nn.init.normal_(w)
            elif 'bias' in name:
                nn.init.constant_(w,0)
            else:
                pass
def train(config,model,train_iter,dev_iter,test_iter):
    start_time = time.time()
    model.train()
    param_optimizer = list(model.named_parameters())
    no_decay = ['bias','LayerNorm.bias','LayerNorm.weight']
    optimizer_grouped_parameters = [
        {'params':[p for n,p in param_optimizer if not any(nd in n for nd in no_decay)],'weight_decay':0.01},
        {'params':[p for n,p in param_optimizer if any(nd in n for nd in no_decay)],'weight_decay':0.0}
    ]
    optimizer = BertAdam(optimizer_grouped_parameters,lr = config.learning_rate,warmup = 0.05,t_total = len(train_iter) * config.num_epochs)
    total_batch = 0.   # 记录进行到多少batch
    dev_best_loss = float('inf')
    last_improve = 0    # 最后一次提升的batch序号
    flag = False
    model.train()
    for epoch in range(config.num_epochs):
        print('Epoch [{}/{}]'.format(epoch+1,config.num_epochs))
        for i,(trains,labels) in enumerate(train_iter):
            outputs = model(trains)
            model.zero_grad()
            loss = F.cross_entropy(outputs,labels)
            loss.backward()
            optimizer.step()
            if total_batch % 100 == 0:
                # 输出训练集和验证集效果
                true = labels.data.cpu()
                predic = torch.max(outputs.data,1)[1].cpu()
                train_acc = metrics.accuracy_score(true,predic)
                dev_acc,dev_loss = evaluate(config,model,dev_iter)
                if dev_loss < dev_best_loss:
                    dev_best_loss = dev_loss
                    torch.save(model.state_dict(),config.save_path)
                    improve = '*'
                    last_improve = total_batch
                else:
                    improve = ''
                time_dif = get_time_dif(start_time)
                msg = 'Iter:{0:>6},Train Loss:{1:>5.2},Train Acc:{2:>6.2%},Val Loss:{3:>5.2},Val Acc:{4:>6.2%},Time:{5} {6}'
                print(msg.format(total_batch,loss.item(),train_acc,dev_loss,dev_acc,time_dif,improve))
                model.train()
            total_batch += 1
            if total_batch - last_improve > config.require_improvement:
                print('No optimization for a long time,auto-stopping...')
                flag = True
                break
        if flag:
            break
    test(config,model,test_iter)
def test(config,model,test_iter):
    # test
    model.load_state_dict(torch.load(config.save_path))
    model.eval()
    start_time = time.time()
    test_acc,test_loss,test_report,test_confusion = evaluate(config,model,test_iter,test = True)
    msg = 'Test Loss:{0:>5.2},Test Acc:{1:>6.2%}'
    print(msg.format(test_loss,test_acc))
    print('Precision,Recall and F1-Score...')
    print(test_report)
    print('Confusion Matrix...')
    print(test_confusion)
    time_dif = get_time_dif(start_time)
    print('Time usage:',time_dif)
def evaluate(config,model,data_iter,test = False):
    model.eval()
    loss_total = 0
    predict_all = np.array([],dtype = int)
    labels_all = np.array([],dtype = int)
    with torch.no_grad():
        for texts,labels in data_iter:
            outputs = model(texts)
            loss = F.cross_entropy(outputs,labels)
            loss_total += loss
            labels = labels.data.cpu().numpy()
            predic = torch.max(outputs.data,1)[1].cpu().numpy()
            labels_all = np.append(labels_all,labels)
            predict_all = np.append(predict_all,predic)
    acc = metrics.accuracy_score(labels_all,predict_all)
    if test:
        report = metrics.classification_report(labels_all,predict_all,target_names = config.class_list,digits = 4)
        confusion = metrics.confusion_matrix(labels_all,predict_all)
        return acc,loss_total / len(data_iter),report,confusion
    return acc,loss_total / len(data_iter)

# 开始训练
model = Model(config).to(config.device)
train(config, model, train_iter, dev_iter, test_iter)

输出：
Epoch [1/3]
Iter: 0, Train Loss: 2.4, Train Acc: 13.28%, Val Loss: 2.4, Val Acc: 9.08%, Time: 0:05:45 *
Epoch [2/3]
Iter: 100, Train Loss: 0.43, Train Acc: 88.28%, Val Loss: 0.22, Val Acc: 93.70%, Time: 0:33:42 *
Epoch [3/3]
Iter: 200, Train Loss: 0.031, Train Acc: 100.00%, Val Loss: 0.091, Val Acc: 97.57%, Time: 0:59:47 *
Test Loss: 0.34, Test Acc: 90.52%
Precision, Recall and F1-Score…
precision recall f1-score support

  finance     0.9189    0.8720    0.8948      1000
   realty     0.9143    0.9170    0.9156      1000
   stocks     0.8579    0.8150    0.8359      1000
education     0.9551    0.9350    0.9449      1000
  science     0.8587    0.8690    0.8638      1000
  society     0.8780    0.9280    0.9023      1000
 politics     0.8807    0.9010    0.8908      1000
   sports     0.9732    0.9440    0.9584      1000
     game     0.9231    0.9240    0.9235      1000

entertainment 0.8968 0.9470 0.9212 1000

 accuracy                         0.9052     10000
macro avg     0.9057    0.9052    0.9051     10000

weighted avg 0.9057 0.9052 0.9051 10000

Confusion Matrix…
[[872 19 72 4 11 9 6 3 1 3]
[ 12 917 8 1 6 16 12 3 7 18]
[ 52 33 815 0 37 5 42 1 9 6]
[ 0 3 2 935 5 28 14 1 2 10]
[ 5 6 28 2 869 17 18 2 39 14]
[ 1 11 0 16 10 928 15 0 4 15]
[ 5 7 18 10 20 30 901 2 0 7]
[ 1 3 3 1 3 9 8 944 3 25]
[ 0 2 4 1 44 8 4 2 924 11]
[ 1 2 0 9 7 7 3 12 12 947]]
Time usage: 0:04:42