爬取新闻页面

import jieba
import os
import chardet
import matplotlib.pyplot as plt
from wordcloud import WordCloud,ImageColorGenerator
from imageio.v2 import imread

newsTextdir=r'd:\doc\sinenews'
img_path=r'.\background.jpg'
stop_word_path=r'.\stopword.txt'
my_word_path=r'.\myword.txt'

def add_stop_words(list):
    stop_words=set()
    for item in list:
        stop_words.add(item.strip())
    return stop_words

def getnewstext(newsdir):
    news_text=""
    sd=os.walk(newsdir)
    for d,s,fns in sd:
        for fn in fns:
            if fn[-3:]=='txt':
                file=d+os.sep+fn
                print(file)
                try:
                    f=open(file)
                    lines=f.readlines()
                except:
                    ft=open(file,"rb")
                    cs=chardet.detect(ft.read())
                    ft.close()
                    f=open(file,encoding=cs['encoding'])
                    lines=f.readlines()
                for i in range(len(lines)):
                    news_text+='.'.join(lines)
    return news_text

stopword_list=open(stop_word_path,encoding='utf-8').readlines()
myword_list=open(my_word_path,encoding='utf-8').readlines()
bg_img=imread(img_path)
news_text=getnewstext(newsTextdir)

stop_words=add_stop_words(stopword_list)
print('停用词共:',len(stop_words))

jieba.load_userdict(my_word_path)

seg_list=jieba.cut(news_text)
seg_space=''.join(seg_list)

wc=WordCloud(font_path='C:\Windows\Fonts\simfang.ttf',max_words=40,random_state=42,background_color='white',stopwords=stop_words,mask=bg_img,max_font_size=100,scale=5,collocations=False).generate(seg_space)
plt.imshow(wc)
plt.axis("off")
plt.show()
wc.to_file('.wordcloud_res.jpg')