#数据预处理,去除数字,只保留中文
def load_tencent_word_embedding(input_file, output_file):
n = 0
with open('tencent.txt', 'a', encoding='utf-8', errors='ignore') as w_f:
with open('Tencent_AILab_ChineseEmbedding.txt', 'r', encoding='utf-8', errors='ignore')as f:
for i in tqdm(range(8824330)):# 原始
data = f.readline()
a = data.split()#list
if len(a) == 201:
if not a[0].isdigit():
n = n + 1
w_f.write(data)
print('total lines:',n) # 8748463
#获取单个字词向量
def process_tencent_embedding_unigram(input_file, output_file):
n = 0
with open(output_file, 'a', encoding='utf-8') as g:
with open(input_file, 'r', encoding='utf-8') as f:
for i in tqdm(range(8748463)):
data = f.readline()
a = data.split()
if i == 0:
g.write(data)
else:
if len(a[0]) == 1:
n += 1
g.write(data)
print('total unigram:', n)
#获取一个词(两个字)的词向量
def process_tencent_embedding_bigrams(input_file, output_file):
n = 0
with open(output_file, 'a', encoding='utf-8') as g:
with open(input_file, 'r', encoding='utf-8') as f:
for i in tqdm(range(8748463)):
data = f.readline()
a = data.split()
if i == 0:
g.write(data)
else:
if len(a[0]) == 2:
n += 1
g.write(data)
print('total bigrams:',n)
if __name__ == '__main__':
input_file = 'Tencent_AILab_ChineseEmbedding'
output_file = 'tencent.txt'
load_tencent_word_embedding(input_file, output_file)
预处理tencent词向量
最新推荐文章于 2024-08-15 20:22:50 发布