data=list(zip(x_data, y_data))
data=[
(x,y)
for x,y in data
if len(x)<limit \
and len(y)<limit \
and len(y)>=y_limit \
and len(x)>=x_limit
]
x_data, y_data=zip(*data)
print('fit word_sequence')
ws_input=WordSequence()
ws_input.fit(x_data+y_data)
print('dump')
pickle.dump(
(x_data, y_data),
open('chatbot.pkl','wb')
)
pickle.dump(ws_input, open('ws.pkl','wb'))
print('done')
if __name__=='__main__':
main()
第6章实战之聊天语料处理—6.11数据模型打包处理
最新推荐文章于 2023-06-05 14:36:51 发布