错误分析
下面展示源代码`。
// An highlighted block
# 简易demo,爬取豆瓣电影的清单与得分
import json
import re
import requests
from requests import RequestException
# 获取网页的内容,以文本形式返回
def get_page(url):
headers = {
'Cookie':'OCSSID=4df0bjva6j7ejussu8al3eqo03',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
'(KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
}
try:
response = requests.get(url)#,headers=headers
print(response.status_code)
if response.status_code == 200:
return response.text
return None
except RequestException:
return None
# 对网页文本内容进行解析
def parse_page(html):
pattern = re.compile('<li.*?list-item.*?data-title="(.*?)".*?data-score="(.*?)".*?>.*?<img.*?src="(.*?)".*?/>', re.S)
items = re.findall(pattern, html.decode('utf-8'))
for item in items:#.encode('utf-8')
yield{
'title': item[0],
'score': item[1],
'image': item[2],}
# 爬虫主程序,同时print效果
def main():
url = "https://movie.douban.com/cinema/nowplaying/beijing/"#https://movie.douban.com/cinema/nowplaying/beijing/
html = get_page(url)
print(html)
for item in parse_page(html):
print(item)
if __name__ == '__main__':
main()
错误1:
修改如下
items = re.findall(pattern, html.encode('utf-8').decode('utf-8'))
错误2:
此处可以看到print出来none ,
print(response.status_code)
输出418
这是因为爬虫没有伪装成浏览器访问,所以要加上headers
headers = {
'Cookie':'OCSSID=4df0bjva6j7ejussu8al3eqo03',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
'(KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
}
try:
response = requests.get(url,headers=headers )
此时解决错误