垃圾邮件分类

实验代码：

import numpy as np
import jieba
import re
import string
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import TfidfTransformer

def get_data():
    with open("ham_data.txt", encoding="utf8") as ham_f, open("spam_data.txt", encoding="utf8") as spam_f:
        ham_data = ham_f.readlines()
        spam_data = spam_f.readlines()
        ham_label = np.ones(len(ham_data)).tolist()
        spam_label = np.zeros(len(spam_data)).tolist()
        corpus = ham_data + spam_data
        labels = ham_label + spam_label
    return corpus, labels


def remove_empty_docs(corpus, labels):
    filtered_corpus =[]
    filtered_labels =[]
    for doc, label in zip(corpus, labels):
        if doc.strip():
            filtered_corpus.append(doc)
            filtered_labels.append(label)

    return filtered_corpus, filtered_labels


# 分词
def tokenize_text(text):
    tokens = jieba.cut(text)
    tokens = [token.strip() for token in tokens]
    return tokens


def remove_special_characters(text):
    tokens = tokenize_text(text)
    pattern = re.compile('[{}]'.format(re.escape(string.punctuation)))
    filtered_tokens = filter(None, [pattern.sub('', token) for token in tokens])
    filtered_text = ' '.join(filtered_tokens)
    return filtered_text


# 加载停用词
with open("stop_words.utf8", encoding="utf8") as f:
    stopword_list = f.readlines()


# 去除停用词
def remove_stopwords(text):
    tokens = tokenize_text(text)
    filtered_tokens = [token for token in tokens if token not in stopword_list]
    filtered_text = ''.join(filtered_tokens)
    return filtered_text


# 数据预处理
def normalize_corpus(corpus, tokenize=False):
    normalized_corpus = []
    for text in corpus:
        text = remove_special_characters(text)
        text = remove_stopwords(text)
        normalized_corpus.append(text)
        if tokenize:
            text = tokenize_text(text)
            normalized_corpus.append(text)

    return normalized_corpus


def bow_extractor(corpus, ngram_range=(1, 1)):
    vectorizer = CountVectorizer(min_df=1, ngram_range=ngram_range)
    features = vectorizer.fit_transform(corpus)
    return vectorizer, features

    
def tfidf_transformer(bow_matrix):
    transformer = TfidfTransformer(norm='l2',
                                   smooth_idf=True,
                                   use_idf=True)
    tfidf_matrix = transformer.fit_transform(bow_matrix)
    return transformer, tfidf_matrix


def tfidf_extractor(corpus, ngram_range=(1, 1)):
    vectorizer = TfidfVectorizer(min_df=1,
                                 norm='l2',
                                 smooth_idf=True,
                                 use_idf=True,
                                 ngram_range=ngram_range)
    features = vectorizer.fit_transform(corpus)
    return vectorizer, features

    corpus,labels=get_data()  # 获取数据集
print("总的数据量:", len(labels))
corpus, labels = remove_empty_docs(corpus, labels)
# 对数据进行划分
train_corpus, test_corpus, train_labels, test_labels = train_test_split(corpus, labels, test_size=0.3, random_state=42)

norm_train_corpus = normalize_corpus(train_corpus)
norm_test_corpus = normalize_corpus(test_corpus)

bow_vectorizer, bow_train_features = bow_extractor(norm_train_corpus)
bow_test_features = bow_vectorizer.transform(norm_test_corpus)
# tfidf 特征
tfidf_vectorizer, tfidf_train_features = tfidf_extractor(norm_train_corpus)
tfidf_test_features = tfidf_vectorizer.transform(norm_test_corpus)
mnb = MultinomialNB()
lr = LogisticRegression()

mnb.fit(bow_train_features,train_labels)
print("基于词袋模型的多项式朴素贝叶斯模型")
print("训练集得分：",mnb.score(bow_train_features,train_labels))
print("测试集得分：",mnb.score(bow_test_features,test_labels))


mnb.fit(bow_train_features, train_labels)
print("基于词袋模型的逻辑回归模型")
print("训练集得分：", mnb.score(bow_train_features, train_labels))
print("测试集得分：", mnb.score(bow_test_features, test_labels))

mnb.fit(tfidf_train_features, train_labels)
print("基于tfidf的多项式朴素贝叶斯模型")
print("训练集得分：", mnb.score(tfidf_train_features, train_labels))
print("测试集得分：", mnb.score(tfidf_test_features, test_labels))

lr.fit(tfidf_train_features, train_labels)
print("基于tfidf的逻辑回归模型")
print("训练集得分：", lr.score(tfidf_train_features, train_labels))
print("测试集得分：", lr.score(tfidf_test_features, test_labels))

实验结果：