Bert复现笔记

import pandas as pd
import os
import numpy as np
import re
import random
from tqdm import tqdm 

def read_data():
    
    #os.path.join自动拼接路径 不用写斜杠之类的
    all_data = pd.read_csv(os.path.join("data","test_data.csv"))
    #取content这一列文字数据，转成list类型数据，会比pandas的数据结构存取快很多
    all_text = all_data["content"].tolist()

    return all_text

def resplit_text(text_list):
    
    #句子短了就拼接
    result = []
    #初始化sentence为空
    sentence = ""
    for text in text_list:
        #加随机策略训练很短和很长的句子
        if sentence == "" :
            if random.random() < 0.2:
                result.append(text + "。")
        if len(sentence) < 30 or random.random() < 0.2:
            #拼接时加，
            sentence += text + ","
        else:
            result.append(sentence[:-1]+"。")
            #把句子里原来的分隔符去掉加句号
            sentence = ""

    return result

def split_text(text):

    #正则表达式  以patten符号进行切分
    #patten = r"。|？|！"
    patten = r"[，、：；。？]"
    sp_text = re.split(patten,text)
    new_sp_text = resplit_text(sp_text)

    return new_sp_text

def build_neg_pos_data(text_list):
    all_text1,all_text2 = [],[]
    all_label = []

    for tidx, text in enumerate(text_list):
        #枚举
        if tidx == len(text_list)-1:
            break
        #不break的话永远有下一句
        all_text1.append(text)
        all_text2.append(text_list[tidx+1])
        #下一句是上一句+1即相邻
        all_label.append(1)

        c_id = [i for i in range(len(text_list))if i != tidx and i != tidx+1]
        #不相邻 不为当前句和当前句的下一句
        other_idx = random.choice(c_id)
        
        other_text = text_list[other_idx]
        all_text1.append(text)
        all_text2.append(other_text)
        all_label.append(0)
    return all_text1,all_text2,all_label

def build_task2_dataset(text_list):

    #由于数据长度差很大，设计切割函数，按照句号切分
    #构建数据集
    all_text1 = []
    all_text2 = []
    all_label = []

    for text in tqdm(text_list):
        #tqdm可以看进度
        sp_text = split_text(text)

        #如果一篇小于两句话就没法执行正负标签
        if len(sp_text)<=2:
            continue
        text1,text2,label = build_neg_pos_data(sp_text)  
        
        all_text1.extend(text1)
        all_text2.extend(text2)
        all_label.extend(label)

    pd.DataFrame({"text1":all_text1,"text2":all_text2,"label":all_label}).to_csv("task2_hw.csv",index=False)

def build_word_2_index(all_text):
    #构建字到数字的映射
    word_2_index = {}

    for text in all_text:
        for w in text:
            if w not in word_2_index:
                word_2_index[w] = len(word_2_index)
            #else:           不需要
            #    print("")

    return word_2_index

#先写主函数
if __name__ == "__main__":
    all_text = read_data()
    #data_len = [len(i) for i in all_text]
    #np.mean(data_len)
    #统计数据长度 np.min(data_len)=14  np.max(data_len)=50523 长度差很大
    
    build_task2_dataset(all_text)
    word_2_index = build_word_2_index(all_text)
    print("")