python 爬带端口的网站_利用Python语言功能对网站电影短评的爬取：网络爬虫

在上一篇文章里介绍了利用find_all的方法来对网络上电影的排名，今天我们介绍一个利用select函数的方法。

方法二：利用select函数

#x1=soup.find_all("li")

x1 = soup.select("ol li") #子目录标签，各级标签之间利用空格进行分割

1.5影片短评的爬取

#循环获得短影评，电影编号 32659890

import requests as rtfrom bs4 import BeautifulSoup as bs
headers = {"User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'}
#循环获得短影评

k=range(0,1200,20)
my_txt=""
for i in k:
#my_url=my_list1[0]+r"comments?start="+str(i)+r"&limit=20&sort=new_score&status=P"
my_url =r"https://movie.douban.com/subject/32659890/" #电影编号32659890

+ r"comments?start=" + str(i)
+ r"&limit=20&sort=new_score&status=P"#print(my_url)
my_data=rt.get(my_url, headers=headers, timeout=30)
my_data.encoding="utf-8" #解决中文乱码问题

soup=bs(my_data.text,"html.parser")
#print(soup)
x1=soup.find_all("span",class_="short")for i in x1:
#print(i.text)
my_txt=my_txt+i.text
#词频统计

import jieba
x3=jieba.lcut(my_txt)#分词后的文章

x4=dict()
#屏蔽词

y1=["","一部"]for i in x3:if i not in y1 and len(i) > 1:
x4[i]=x4.get(i,1)+1
#排序

res = sorted(x4.items(),key=lambda d:d[1],reverse=True)
#print(type(res),len(res))for i in range(0,10):#词频的前10名

print(res[i])
#词云

from wordcloud import WordCloud
#图形

import cv2 #pip install opencv-python
mask =cv2.imread("000.jpg") #读取背景图片

my_txt2=" ".join(x3)
excludes=["","一部","电影","片子","没有","这部","一个"]
wordcloud = WordCloud(background_color="white",
width=800,
height=600,
font_path="msyh.ttc",
max_words=200,
max_font_size=200,
stopwords = excludes,
mask=mask
).generate(my_txt2)
wordcloud.to_file('111.jpg')
#显示图片

from PIL import Image
img=Image.open('111.jpg')
img.show()

显示图

1.6 爬取时光网排名

import requestsfrom bs4 import BeautifulSoup as bsx2=range(1,11)for my_n in x2:    if my_n==1:        my_url=r"http://www.mtime.com/top/movie/top100/"    else:        my_url=r"http://www.mtime.com/top/movie/top100/index-"+str(my_n)+r".html"    print(my_url)    my_data=requests.get(my_url)    my_data.encoding="utf-8" #解决中文乱码问题    soup=bs(my_data.text,"html.parser")    x1=soup.find_all('li')    for i in x1:        if i.find("div",class_="number")!=None:            print(i.find("em").text,end=" ")            try:                print(i.find("a",class_="c_fff").text,end=" ")            except:                print(i.find("a",class_="c_blue").text,end=" ")            print(i.find("span",class_="total").text,end="")            print(i.find("span", class_="total2").text)