网址:豆瓣电影 Top 250
目录
一.bs4和re正则爬取
源代码:
import urllib.request,urllib.error
import re
from bs4 import BeautifulSoup
import xlwt
baseurl = "https://movie.douban.com/top250?start="
head = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36"
}
title = re.compile('<span class="title">(.*?)</span>') #标题
link = re.compile('<a class="" href="(.*?)"') #电影链接
introduction = re.compile('<span class="inq">(.*?)</span>') #电影简介
appraise = re.compile('<span>(.*?)</span>') #评价人数
basedata = [] #最后写入到excel中,利用列表来写入
workbook = xlwt.Workbook(encoding='utf-8') #新建excel文档
worksheet = workbook.add_sheet('daoban') #excel文档中添加表格
for i in range(0,10):
url = url = baseurl+str(i*25)
#请求网页
request = urllib.request.Request(url=url,headers=head)
#得到网页回应并打开
response = urllib.request.urlopen(request)
#对打开的网页回应并解码到utf-8(有中文)
html = response.read().decode("utf-8")
# print(html) #字符类型的html
soup = BeautifulSoup(html,"html.parser") #后面是html。parser 解析器
for item in soup.find_all('div',class_="info"):
item = str(item)
data = [] #每一部电影有多个,每部电影放入一个列表中
ftitle = re.findall(title,item)
if len(ftitle) == 2:
etitle = ftitle[0]
data.append(etitle)
rtitle = ftitle[1].replace("/","")#去点无关符号
data.append(rtitle)
else:
data.append(ftitle[0])
data.append(' ')#外国名留空 #有的会没有外国电影,防止串行
flink = re.findall(link,item)[0]
data.append(flink)
# print(data)
# fintroduction = re.findall(introduction,item)[0]#有的电影没有概述
fintroduction = re.findall(introduction, item)
if len(fintroduction) != 0:
fintroduction = fintroduction[0].replace("。","")#取代最后面的句号
data.append(fintroduction)
else:
data.append(" ")
data.append(fintroduction)
fappraise = re.findall(appraise,item)[0]
data.append(fappraise)
basedata.append(data) #单个电影信息的列表加入到一个大列表中
for i in range(0,250):
print("第%d条记录"%i)
first = basedata[i]
for j in range(0,4):
second = first[j]
worksheet.write(i,j,second) #每个列表中保存数据
workbook.save('豆瓣.xls') #保存excel文档
print(basedata)
1.请求网页并且获取网页源代码的代码实现:
import urllib.request,urllib.error
baseurl = "https://movie.douban.com/top250?start="
head