第6章 实战之聊天语料处理—6.8 训练语料库的解压处理

def main(limit=20, x_limit=3, y_limit=6):
    from word_sequence import WordSequence

    print('extract lines')
    fp=open("dgk_shooter_min.conv","r",errors='ignore',encoding='utf-8')
    groups=[]
    group=[]

    for line in tqdm(fp):
        if line.startswith('M '):
            line=line.replace('\n','')

            if '/' in line:
                line=line[2:].split('/')
            else:
                line=list(line[2:])
            line=line[:-1]

            group.append(list(regular(''.join(line))))
        else:
            if group:
                groups.append(group)
                group=[]
    if group:
        groups.append(group)
        group=[]
    print('extract group')

if __name__=='__main__':
    main()