导包
import jieba
with open("NBA新闻.txt", "r", encoding="utf-8") as file_in:
with open("text_corpus.txt", "w", encoding="utf-8") as file_out:
row = file_in.readline()
while row:
jieba.load_userdict('dict.txt')
word = jieba.cut(row,cut_all=False)
words = ' '.join(word)+"\n"
file_out.write(words)
row = file_in.readline()
Word2Vec文本向量化
from gensim.models import Word2Vec
from gensim.models.word2vec import LineSentence
import logging
# 配置日志,避免训练过程中的警告信息
# logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
# 假设我们有一个名为'text_corpus.txt'的文本文件,每行包含一个句子,句子中的单词由空格分隔
# sentences = LineSentence('ext_corpus.txt') # 加载文本数据
# 训练Word2Vec模型
# 参数可以根据你的数据和需求进行调整
#参数如vector_size(向量维度)、window(上下文窗口大小)、min_count(最小词频)和workers(工作线程数)都可以根据你的数据集和需求进行调整
# model = Word2Vec(sentences, vector_size=100, window=15, min_count=1, workers=4)
# 保存模型,以便后续使用
# model.save("word2vec.model")
# 加载已保存的模型
model = Word2Vec.load("word2vec.model")
# 查找单词的向量表示
# vector = model.wv['丹佛掘金'] # 假设'apple'在我们的词汇表中
# print(len(vector))
# print(vector)
# 查找最相似的单词
similar_words = model.wv.most_similar('领导')
print(similar_words)
torch文本向量化
import torch
import torch.nn as nn
import torch.optim as optim
# 超参数
vocab_size = 10000 # 词汇表大小
embedding_dim = 50 # 嵌入维度
output_dim = 2 # 输出维度(例如,二分类任务)
num_epochs = 5 # 训练轮数
batch_size = 64 # 批处理大小
learning_rate = 0.001 # 学习率
# 模拟文本数据(实际中你会从文本数据集中获取这些索引)
# 这里我们随机生成一些单词索引作为示例
text_indices = torch.randint(0, vocab_size, (batch_size, 10)) # (batch_size, sequence_length)
# 标签数据(随机生成)
labels = torch.randint(0, output_dim, (batch_size,))
# 定义模型
class SimpleEmbeddingModel(nn.Module):
def __init__(self, vocab_size, embedding_dim, output_dim):
super(SimpleEmbeddingModel, self).__init__()
self.embedding = nn.Embedding(vocab_size, embedding_dim)
self.fc = nn.Linear(embedding_dim, output_dim)
def forward(self, text_indices):
# 取文本序列中最后一个单词的嵌入(简化的例子)
embedded = self.embedding(text_indices)[:, -1, :]
output = self.fc(embedded)
return output
# 实例化模型
model = SimpleEmbeddingModel(vocab_size, embedding_dim, output_dim)
# 定义损失函数和优化器
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=learning_rate)
# 训练模型
for epoch in range(num_epochs):
# 前向传播
outputs = model(text_indices)
loss = criterion(outputs, labels)
# 反向传播和优化
optimizer.zero_grad()
loss.backward()
optimizer.step()
# 打印统计信息
if (epoch+1) % 1 == 0:
print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {loss.item()}')
# 训练完成后,你可以通过调用model.embedding来获取嵌入层
# 例如,查询索引为5的单词的嵌入向量
embedding_vector = model.embedding(torch.tensor([5], dtype=torch.long))
print(embedding_vector)