# 文本处理,也就是样本生成过程
def text_processing(folder_path, test_size=0.2):
folder_list = os.listdir(folder_path)
data_list = []
class_list = []
# 遍历文件夹
for folder in folder_list:
new_folder_path = os.path.join(folder_path, folder)
files = os.listdir(new_folder_path)
# 读取文件
j = 1
for file in files:
if j > 100: # 怕内存爆掉,只取100个样本文件,你可以注释掉取完
break
with open(os.path.join(new_folder_path, file), 'r',encoding='UTF-8') as fp:
raw = fp.read()
## 是的,随处可见的jieba中文分词
#jieba.enable_parallel(4) # 开启并行分词模式,参数为并行进程数,不支持windows
word_cut = jieba.cut(raw, cut_all=False) # 精确模式,返回的结构是一个可迭代的genertor
word_list = list(word_cut) # genertor转化为list,每个词unicode格式
#jieba.disable_parallel() # 关闭并行分词模式
文本分类问题的训练集+测试集的生成
最新推荐文章于 2022-08-26 14:35:16 发布
![](https://img-home.csdnimg.cn/images/20240711042549.png)