python和golang爬虫_python 的一个爬虫例子

#-*- codeing = utf-8 -*-

import io

import sys

import urllib.request

import urllib.error

import urllib.parse

from bs4 import BeautifulSoup

import re

import sqlite3

import xlwt

#改变标准输出的默认编码

sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='utf8')

def main():

baseurl = 'https://movie.douban.com/top250?start='

datalist = getData(baseurl)

savepath = ".\\豆瓣电影top250.xls"

saveData(datalist,savepath)

findLink = re.compile(r'') # 生成创建正则表达式对象，表示规则

findTitle = re.compile(r'(.*)') # 生成创建正则表达式对象，表示规则

findRating = re.compile(r'(.*)') # 生成创建正则表达式对象，表示规则

findJudge = re.compile(r'(\d*)人评价') # 生成创建正则表达式对象，表示规则

findBd = re.compile(r'

(.*?)

',re.S) # 生成创建正则表达式对象，表示规则

# 爬取网页

def getData(baseurl):

datalist = []

for i in range(0,10):

url = baseurl + str(i*25)

html = askURL(url)

# 逐一解析数据

soup = BeautifulSoup(html,"html.parser")

for item in soup.find_all('div',class_="item"): # 查找符合要求的字符串组成列表

# print(item)

data = []

item = str(item)

link = re.findall(findLink,item)[0] # 通过正则表达式查找指定的字符串

title = re.findall(findTitle,item)[0]

rating = re.findall(findRating,item)[0]

judge = re.findall(findJudge,item)[0]

bd = re.findall(findBd,item)[0]

bd = re.sub("
(\s+)?"," ",bd) # 去掉

bd = re.sub("/"," ",bd)

data.append(link)

data.append(title)

data.append(rating)

data.append(judge)

data.append(bd.strip()) # 去掉空格

datalist.append(data)

# print(datalist)

return datalist

def askURL(url):

# 用户代理，模拟浏览器头部信息

headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36'}

req = urllib.request.Request(url,headers=headers)

html = ""

try:

response = urllib.request.urlopen(req)

html = response.read().decode('utf-8')

# print(html)

except urllib.error.URLError as e:

print(response.status)

if hasattr(e,"code"):

print(e.code)

if hasattr(e,"reason"):

print(e.reason)

return html

# 保存数据

def saveData(datalist,savepath):

book = xlwt.Workbook(encoding = "utf-8",style_compression=0)

sheet = book.add_sheet('豆瓣电影TOP250',cell_overwrite_ok = True)

col = ("电影详情链接","影片名称","评分","评分人数","详情")

for i in range(0,5):

sheet.write(0,i,col[i])

for i in range(0,250):

print("第%d条" %(i+1))

data = datalist[i]

for j in range(0,5):

sheet.write(i+1,j,data[j])

book.save(savepath)

if __name__ == "__main__":

main()