创建数据集和数据字典
1.字典文件
2.训练数据
3.测试数据
注意:在字典中添加未知字符的编码
```
end_dict = {"<unk>": i}
dict_txt.update(end_dict)
```
创建数据读取器
#创建数据读取器train_reader和train_reader
#训练/测试数据的预处理
def data_mapper(sample):
data, label = sample
data = [int(data) for data in data.split(',')]
return data, int(label)
#创建数据读取器train_reader
def train_reader(train_list_path):
def reader():
with open(train_list_path,'r') as f:
lines = f.readlines()
#打乱顺序
np.random.shuffle(lines)
for line in lines:
data, label = line.split('\t')
yield data, label
return paddle.reader.xmap_readers(data_mapper, reader, cpu_count(),1024)
#创建数据读取器test_reader
def test_reader(test_list_path):
def reader():
with open(test_list_path, 'r'