这个程序主要是用来学习,requests.Session()是怎么使用的
import requests
import time
import parsel
session_get = requests.Session()
headers = {
'Referer':'https://accounts.douban.com/passport/login_popup?login_source=anony',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'
}
data = {
'ck': '',
'name': '',
'password': '',
'remember': 'false',
'ticket': ''
}
def login_douban():
url = 'https://accounts.douban.com/j/mobile/login/basic'
try:
response = session_get.post(url,headers=headers,data=data).content.decode('utf-8')
print(response)
except Exception as e:
print(e)
def get_artic(page=0):
pages = int(page * 20)
print(f'开始爬取第{int(pages)-20}-{int(pages)}条')
url = f'https://movie.douban.com/subject/26608230/comments?start={pages}&limit=20&sort=new_score&status=P'
try:
response = session_get.get(url,headers=headers)
except requests.HTTPError as e:
print(e)
print('错误码:',response.status_code)
return 0
except Exception as e:
print(e)
common_list = []
html_sel = parsel.Selector(response.content.decode('utf-8'))
common_tile = html_sel.xpath('//span[@class="comment-info"]')
common_artic = html_sel.xpath('//div[@class="comment"]/p')
for i, x in zip(common_tile, common_artic):
title = i.xpath('./a/text()').extract_first()
artic = x.xpath('./span/text()').extract_first()
common_list.append(title + ',' + artic)
print(common_list)
with open(f'E:/AA/{(pages-20)}-{pages}短评.txt', 'w', encoding='utf-8')as f:
f.write((str(common_list)).replace(',', '\n').replace('[', '').replace(']', '').replace("'", ''))
def batch_page():
page = int(22)
while 1:
pages = int(page*20)
print(pages)
url = f'https://movie.douban.com/subject/26608230/comments?start={(pages)+1}&limit=20&sort=new_score&status=P'
response = session_get.get(url,headers=headers)
html_sel = parsel.Selector(response.text)
common_page = html_sel.xpath('//div[@id="paginator"]/a[3]/text()').extract_first()
get_artic(page)
print(f'第{page}页',common_page)
if page == 0:
pass
else:
if not '后页' in str(common_page):
print('error')
return 0
time.sleep(3)
page += 1
if __name__ == "__main__":
login_douban()
batch_page()