自然语言处理项目之新闻主题分类Python实现

'''
#2018-06-10  June Sunday the 23 week, the 161 day SZ
数据来源：链接:https://pan.baidu.com/s/1_w7wOzNkUEaq3KAGco19EQ 密码:87o0
朴素贝叶斯与应用
文本分类问题
经典的新闻主题分类，用朴素贝叶斯做。
#还有点问题。无法正确读取数据。UnicodeDecodeError: 'charmap' codec can't decode byte 0x90 in position 41: character maps to <undefined>
folder_path = 'D:/自然语言处理/第2课/Lecture_2/Lecture_2/Naive-Bayes-Text-Classifier/Database/SogouC/Sample'
all_words_list, train_data_list, test_data_list, train_class_list, test_class_list = text_processing(folder_path,test_size=0.2)
'''


import os
import time
import random
import codecs
import jieba  #处理中文
#import nltk  #处理英文
import sklearn
from sklearn.naive_bayes import MultinomialNB
import numpy as np
import pylab as pl
import matplotlib.pyplot as plt

import sys
#reload(sys)
#sys.setdefaultencoding('utf8')
#粗暴的词去重
def make_word_set(words_file):
    words_set = set()
    with open(words_file, 'r') as fp:
        for line in fp.readlines():
            word = line.strip().decode("utf-8")
            if len(word)>0 and word not in words_set: # 去重
                words_set.add(word)
    return words_set
# 文本处理，也就是样本生成过程
def text_processing(folder_path, test_size=0.2):
    folder_list = os.listdir(folder_path)
    data_list = []
    class_list = []

    # 遍历文件夹
    for folder in fold