from bs4 import BeautifulSoup #网页解析·获取数据
import re #正则表达式·进行文室匹配
import urllib.request,urllib.error #制定URL:获取网页数据
import xlwt #进excel操作l
import sqlite3 #进-SOL
def main():
baseurl = "https://movie.douban.com/top250?start="#1.爬取网页
datalist = getData(baseurl)
savepath = ".\\豆瓣电影Top250.xls"
#3.保荐数椐
#saveData(savepath)
findLink = re.compile(r'<a href="(.*?)">‘)
#爬取网页
def getData(baseurl):
datalist = []
for i in range(0, 10):#调用获取页面信息的25次
url = baseurl + str(i*25)
html=askURL(url) #保存获取到的网页源码
#2.逐一解析数据
soup = BeautifulSoup(html, "html.parser")
for item in soup.find_all("div",class_="item"): #查找符合要求的字符串,形成列表
data=[] #保存一部电影的所有信息
item=str(item)
link=re.findall(findLink,item)[0] #通过正则表达式来查找字符串
print(link)
return datalist
#
def askURL(url):
head = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36"
}
request = urllib.request.Request(url, headers=head)
html = ""
try:
response = urllib.request.urlopen(request)
html = response.read().decode("utf-8")
except urllib.error.URLError as e:
if hasattr(e, "code"):
print(e.code)
if hasattr(e, "reason"):
print(e.reason)
return html
#保荐数据
def saveData(savepath):
print("save...")
if __name__ == "__main__":
main()
爬虫学习啊阿萨啊
最新推荐文章于 2024-06-14 16:31:33 发布