昇思25天学习打卡营第26天 | 基于MindSpore的GPT2文本摘要

基于MindSpore的GPT2文本摘要

数据集加载与处理

1. 数据集加载

   本次实验使用的是nlpcc2017摘要数据，内容为新闻正文及其摘要，总计50000个样本。

   from mindnlp.utils import http_get
   
   # download dataset
   url = 'https://download.mindspore.cn/toolkits/mindnlp/dataset/text_generation/nlpcc2017/train_with_summ.txt'
   path = http_get(url, './')
   
   from mindspore.dataset import TextFileDataset
   
   # load dataset
   dataset = TextFileDataset(str(path), shuffle=False)
   dataset.get_dataset_size()
   
   # split into training and testing dataset
   train_dataset, test_dataset = dataset.split([0.9, 0.1], randomize=False)
   

2. 数据预处理

   原始数据格式：

   article: [CLS] article_context [SEP]
   summary: [CLS] summary_context [SEP]
   

   预处理后的数据格式：

   [CLS] article_context [SEP] summary_context [SEP]

import json
import numpy as np

# preprocess dataset
def process_dataset(dataset, tokenizer, batch_size=6, max_seq_len=1024, shuffle=False):
    def read_map(text):
        data = json.loads(text.tobytes())
        return np.array(data['article']), np.array(data['summarization'])

    def merge_and_pad(article, summary):
        # tokenization
        # pad to max_seq_length, only truncate the article
        tokenized = tokenizer(text=article, text_pair=summary,
                              padding='max_length', truncation='only_first', max_length=max_seq_len)
        return tokenized['input_ids'], tokenized['input_ids']
    
    dataset = dataset.map(read_map, 'text', ['article', 'summary'])
    # change column names to input_ids and labels for the following training
    dataset = dataset.map(merge_and_pad, ['article', 'summary'], ['input_ids', 'labels'])

    dataset = dataset.batch(batch_size)
    if shuffle:
        dataset = dataset.shuffle(batch_size)

    return dataset

因GPT2无中文的tokenizer，我们使用BertTokenizer替代。

from mindnlp.transformers import BertTokenizer

# We use BertTokenizer for tokenizing chinese context.
tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')
len(tokenizer)

train_dataset = process_dataset(train_dataset, tokenizer, batch_size=4)

next(train_dataset.create_tuple_iterator())

模型构建

1. 构建GPT2ForSummarization模型，注意shift right的操作。

   from mindspore import ops
   from mindnlp.transformers import GPT2LMHeadModel
   
   class GPT2ForSummarization(GPT2LMHeadModel):
       def construct(
           self,
           input_ids = None,
           attention_mask = None,
           labels = None,
       ):
           outputs = super().construct(input_ids=input_ids, attention_mask=attention_mask)
           shift_logits = outputs.logits[..., :-1, :]
           shift_labels = labels[..., 1:]
           # Flatten the tokens
           loss = ops.cross_entropy(shift_logits.view(-1, shift_logits.shape[-1]), shift_labels.view(-1), ignore_index=tokenizer.pad_token_id)
           return loss
   from mindspore import ops
   from mindspore.nn.learning_rate_schedule import LearningRateSchedule
   
   class LinearWithWarmUp(LearningRateSchedule):
       """
       Warmup-decay learning rate.
       """
       def __init__(self, learning_rate, num_warmup_steps, num_training_steps):
           super().__init__()
           self.learning_rate = learning_rate
           self.num_warmup_steps = num_warmup_steps
           self.num_training_steps = num_training_steps
   
       def construct(self, global_step):
           if global_step < self.num_warmup_steps:
               return global_step / float(max(1, self.num_warmup_steps)) * self.learning_rate
           return ops.maximum(
               0.0, (self.num_training_steps - global_step) / (max(1, self.num_training_steps - self.num_warmup_steps))
           ) * self.learning_rate