nltk文本清洗

import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from gensim.models import Word2Vec
from nltk.tokenize import word_tokenize

# 读取处理后的数据
data = pd.read_excel('cleaned_Laos_news.xlsx')

# 检查并填充NaN值
data['cleaned_content'] = data['cleaned_content'].fillna('')

# 创建一个TF-IDF向量化器
tfidf_vectorizer = TfidfVectorizer(max_features=1000)  # 选择前1000个最重要的词汇

# 将清洗后的文本进行向量化
tfidf_matrix = tfidf_vectorizer.fit_transform(data['cleaned_content'])

# 将TF-IDF矩阵转换为DataFrame
tfidf_df = pd.DataFrame(tfidf_matrix.toarray(), columns=tfidf_vectorizer.get_feature_names_out())

# 创建Word2Vec模型
tokenized_data = [word_tokenize(text) for text in data['cleaned_content']]
word2vec_model = Word2Vec(tokenized_data, vector_size=100, window=5, min_count=1, sg=0)

# 将Word2Vec模型保存为独立文件
word2vec_model.save('word2vec_model.bin')

# 将TF-IDF特征和Word2Vec模型整合到原始数据中
data['tfidf_features'] = tfidf_df.values.tolist()  # 将TF-IDF特征列表添加到数据中
data['word2vec_model_path'] = 'word2vec_model.bin'  # 保存Word2Vec模型路径到数据中

# 保存整合后的数据到新的Excel文件
output_file_path = 'Laos_news_with_features.xlsx'
data.to_excel(output_file_path, index=False)