python多表格提取特征_Python中extract_tags()怎么对多行文本提取特征词而不是一行一行计算...

满意答案

frank1228

2018.09.23

采纳率：46% 等级：7

已帮助：409人

[python] view plain copy

#coding:utf-8

import sys

reload(sys)

sys.setdefaultencoding("utf-8")

from multiprocessing import Pool,Queue,Process

import multiprocessing as mp

import time,random

import os

import codecs

import jieba.analyse

jieba.analyse.set_stop_words("yy_stop_words.txt")

def extract_keyword(input_string):

#print("Do task by process {proc}".format(proc=os.getpid()))

tags = jieba.analyse.extract_tags(input_string, topK=100)

#print("key words:{kw}".format(kw=" ".join(tags)))

return tags

#def parallel_extract_keyword(input_string,out_file):

def parallel_extract_keyword(input_string):

#print("Do task by process {proc}".format(proc=os.getpid()))

tags = jieba.analyse.extract_tags(input_string, topK=100)

#time.sleep(random.random())

#print("key words:{kw}".format(kw=" ".join(tags)))

#o_f = open(out_file,'w')

#o_f.write(" ".join(tags)+"\n")

return tags

if __name__ == "__main__":

data_file = sys.argv[1]

with codecs.open(data_file) as f:

lines = f.readlines()

f.close()

out_put = data_file.split('.')[0] +"_tags.txt"

t0 = time.time()

for line in lines:

parallel_extract_keyword(line)

#parallel_extract_keyword(line,out_put)

#extract_keyword(line)

print("串行处理花费时间{t}".format(t=time.time()-t0))

pool = Pool(processes=int(mp.cpu_count()*0.7))

t1 = time.time()

#for line in lines:

#pool.apply_async(parallel_extract_keyword,(line,out_put))

#保存处理的结果，可以方便输出到文件

res = pool.map(parallel_extract_keyword,lines)

#print("Print keywords:")

#for tag in res:

#print(" ".join(tag))

pool.close()

pool.join()

print("并行处理花费时间{t}s".format(t=time.time()-t1))

运行：

python data_process_by_multiprocess.py message.txt

message.txt是每行是一个文档，共581行，7M的数据

运行时间：

不使用sleep来挂起进程,也就是把time.sleep(random.random())注释掉，运行可以大大节省时间。

00分享举报