x_data=[]
y_data=[]
for group in tqdm(groups):
for i, line in enumerate(group):
last_line=None
if i>0:
last_line=group[i-1]
if not good_line(last_line):
last_line=None
next_line=None
if i<len(group)-1:
next_line=group[i+1]
if not good_line(next_line):
next_line=line
next_next_line=None
if i<len(group)-2:
next_next_line=group[i+2]
if not good_line(next_next_line):
next_next_line=None
if next_line:
x_data.append(line)
y_data.append(next_line)
if last_line and next_line:
x_data.append(last_line+make_split(last_line)+line)
y_data.append(next_line)
if next_line and next_next_line:
x_data.append(line)
y_data.append(next_line+make_split(next_line)+next_next_line)
print(len(x_data),len(y_data))
for ask, answer in zip(x_data[:20], y_data[:20]):
print(''.join(ask))
print(''.join(answer))
print('-'*20)
if __name__=='__main__':
main()
第6章实战之聊天语料处理—6.9,6.10训练语料问答对的处理
最新推荐文章于 2024-01-09 16:18:37 发布