#!/usr/bin/env python
-- coding:utf-8 --
import pandas as pd
“”"
基于 TF-IDF 算法的关键词抽取¶
import jieba.analyse
jieba.analyse.extract_tags(sentence, topK=20, withWeight=False, allowPOS=())
sentence 为待提取的文本
topK 为返回几个 TF/IDF 权重最大的关键词,默认值为 20
withWeight 为是否一并返回关键词权重值,默认值为 False
allowPOS 仅包括指定词性的词,默认值为空,即不筛选
“”"
import jieba.analyse as analyse
import pandas as pd
df = pd.read_csv(“technology_news.csv”, encoding=‘utf-8’)
df = df.dropna()
lines=df.content.values.tolist()
content = “”.join(lines)
print(" ".join(analyse.extract_tags(content, topK=30, withWeight=False, allowPOS=())))
“”"
基于 TextRank 算法的关键词抽取
基本思想:
将待抽取关键词的文本进行分词
以固定窗口大小(默认为5,通过span属性调整),词之间的共现关系,构建图
计算图中节点的PageRank,注意是无向带权图
“”"
import jieba.analyse as analyse
import pandas as pd
df = pd.read_csv(“military_news.csv”, encoding=‘utf-8’)
df = df.dropna()
lines=df.content.values.tolist()
content = “”.join(lines)
print(" “.join(analyse.textrank(content, topK=20, withWeight=False, allowPOS=(‘ns’, ‘n’, ‘vn’, ‘v’))))
print(”---------------------我是分割线----------------")
print(" ".join(analyse.textrank(content, topK=20, withWeight=False, allowPOS=(‘ns’, ‘n’))))
“”"
用LDA主题模型建模;对新的文档进行主题分类
“”"
from gensim import corpora, models, similarities
import gensim
import pandas as pd
stopwords=pd.read_csv(“stopwords.txt”,index_col=False,quoting=3,sep="\t",names=[‘stopword’], encoding=‘utf-8’)
stopwords=stopwords[‘stopword’].values
import jieba
import pandas as pd
df = pd.read_csv(“technology_news.csv”, encoding=‘utf-8’)
df = df.dropna()
lines=df.content.values.tolist()
sentences=[]
for line in lines:
try:
segs=jieba.lcut(line)
segs = filter(lambda x:len(x)>1, segs)
segs = filter(lambda x:x not in stopwords, segs)
sentences.append(list(segs))
except Exception as e:
print(e)
continue
print(type(sentences[5]))
for ele in sentences[5]:
print(“ele:”,ele)
print("------------")
print(sentences[4])
print(type(sentences),type(sentences[4]))
for e in sentences[4]:
print(“e:”,e)
dictionary = corpora.Dictionary(sentences)
corpus = [dictionary.doc2bow(sentence) for sentence in sentences]
print(corpus[5])
print(type(corpus))
print(corpus)
print(type(dictionary))
print(“dictionary:”,dictionary)
lda = gensim.models.ldamodel.LdaModel(corpus=corpus, id2word=dictionary, num_topics=20)
#print(lda.print_topic(3, topn=5))
print(“topics :”)
for topic in lda.print_topics(num_topics=20, num_words=8):
print(topic[0],topic[1])
import re
def clean_email_text(text):
text = text.replace(’\n’, " “) # 去掉所有换行符
text = re.sub(r”-", " “, text) # 用空格替换掉所有- 将-连接的2个词分开
text = re.sub(r”\d+/\d+/\d+", " “, text) # 去掉所有的日期 26/2/2019类似这样的格式 +一次或多次
text = re.sub(r”[0-2]?[0-9]:[0-6][0-9]", " “, text) # 去除时间 12:40 ?的0次或者1次
text = re.sub(r”[\w]+@[.\w]+", " “, text) # 取出邮件地址
text = re.sub(r”/[a-zA-Z][😕/][A-Za-z0-9-]+.+[a-zA-Z0-9./%&=-?]+/i", “”, text) # 取出网址
pure_text = “”
for letter in text:
if letter.isalpha() or letter == " ":
pure_text += letter
text = ’ '.join(word for word in pure_text.split() if len(word) > 1)
return text
text1 = “百度 科技 人工智能 机器学习 自动驾驶 互联网 发展 城市 经济 时代”
预处理
text1 = clean_email_text(text1)
print(“before drop stopwords:”,text1)
去停用词
text1 = [word for word in text1.lower().split() if word not in [".",’&’]]
转成数字形式
print(“after drop stopwords:”,text1)
bow = dictionary.doc2bow(text1)
print(lda.get_document_topics(bow))