#@Author:king
#@File: spider.py
#-*- coding = utf-8 -*-
#@Time: 2020/5/18 16:52
#@Software: PyCharm
from bs4 import BeautifulSoup
import re
import urllib.request,urllib.error
import xlwt
import sqlite3
def main():
baseurl="http://movie.douban.com/top250?start="
#1 爬取网页
datalist = getData(baseurl)
#2 逐一解析数据 在getData函数中实现
savepath = ".\\豆瓣电影top250.xls"
#3 保存数据
saveData(datalist,savepath)
#askURL(baseurl)
#影片链接
findLink = re.compile(r'<a href="(.*?)">')
#影片图片
findImgSrc = re.compile(r'<img.*src="(.*?)"',re.S)#让换行符包含在匹配
#影片名字
findTitle = re.compile(r'<span class="title">(.*)</span>')
#影片评分
findRating = re.compile(r'<span class="rating_num" property="v:average">(.*)</span>')
#评价人数
findJudge = re.compile(r'<span>(\d*)人评价</span>')
#找电影简介
findInq = re.compile(r'<span class="inq">(.*)</span>')
#找到电影相关内容,像演员导演等
findBd = re.compile(r'<p class="">(.*?)</p>',re.S)
def getData(baseurl):#爬取网页
datalist = []
for i in range(0,10):#调用获取信息函数十次
url = baseurl + str(i * 25)
html = askURL(url) #保存获取到的网页源码
# 2 逐一解析数据
soup = BeautifulSoup(html,"html.parser")#用html的解析器解析
for item in soup.find_all("div",class_="item"): #查找符合一定条件的字符串形成列表
#print(item) 测试查找所有电影信息
data = [] # 保存所要电影信息
item = str(item)
link = re.findall(findLink,item)[0]# 影片链接获取
data.append(link)
imgSrc = re.findall(findImgSrc,item)[0]# 图片
data.append(imgSrc)
titles = re.findall(findTitle,item)
if(len(titles)==2):
ctitle = titles[0]
data.append(ctitle) #添加中文名
otitle = titles[1].replace("/","") #去掉/
data.append(otitle) #添加外文名
else:
data.append(titles[0])
data.append(" ") #留空,外国名留空
rating = re.findall(findRating,item)[0]
data.append(rating)
judgeNum = re.findall(findJudge,item)[0]
data.append(judgeNum)
inq = re.findall(findInq,item)
if len(inq)!=0:
inq =inq[0].replace("。","")#去掉句号
data.append(inq)
else:
data.append(" ") # 留空
bd = re.findall(findBd,item)[0]
bd = re.sub('<br(\s+)?/>(\s+)',"",bd) #去掉<br/>
bd = re.sub('/',"",bd) #替换/
data.append(bd.strip()) #去除前后空格
datalist.append(data) #存入每部电影信息
print(datalist)
return datalist
#得到指定url的网页内容
def askURL(url):
#模拟浏览器头部信息,像豆瓣服务器发送消息
head = {"User-Agent":" Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36"}
#用户代理表示告诉豆瓣服务器我们是什么类型的机器(本质上是该苏浏览器我们可以接收什么样水平的文件内容)
request = urllib.request.Request(url=url,headers=head)
html = "" #作为字符串保存
try:
response = urllib.request.urlopen(request)
html = response.read().decode("utf-8")
#print(html)
except urllib.error.URLError as e:
if hasattr(e,"code"):
print(e.code)
if hasattr(e,"reason"):
print(e.reason)
return html
def saveData(datalist,savepath):#保存数据
print("save.......")
workbook = xlwt.Workbook(encoding='utf-8',style_compression=0)
worksheet = workbook.add_sheet("豆瓣电影top250",cell_overwrite_ok=True)
col = ("电影链接","图片详情","电影中文名","电影外文名","评分","评分人数","简介","电影信息")
for i in range(0,8):
worksheet.write(0,i,col[i])#列名
for i in range(0,250):
print("第%d条信息"%(i+1))
data = datalist[i]
for j in range(0,8):
worksheet.write(i+1,j,data[j]) #数据
workbook.save(savepath)
if __name__ == '__main__':
main()
print("爬取完毕")
Python爬虫技术5天速成(2020全新合集),跟该老师学习写的爬虫代码(1)用xlwt
最新推荐文章于 2024-03-20 00:35:48 发布