#!/usr/bin/env python
#-*-coding:utf-8-*-
'''
requests 处理cookie的几种方法
'''
import requests
from lxml import etree
import random
import time
#将cookie值放在headers中
def get_page(url):
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36',
'Cookie':'替换cookie'
}
response = requests.get(url=url,headers=headers)
response.encoding='utf-8'
html = response.text
return html
def parse_html(html):
html_elem = etree.HTML(html)
day = html_elem.xpath('//span[@class="time"]/h1/text()')
month = html_elem.xpath('//span[@class="time"]/h2/text()')
title = html_elem.xpath('//span[@class="time"]/following::a[1]/@title')
data = zip(day,month,title)
return data
def openfile():
fd = open('banche01.txt','w',encoding='utf-8')
return fd
def savefile(fd,data):
for item in data:
fd.write('day:'+str(item[0])+'\n')
fd.write('month:'+str(item[1])+'\n')
fd.write('title:'+str(item[2])+'\n')
#爬取数据
def getInfo():
url = 'https://www.jit.edu.cn/xyzhfw/bcsk.htm'
fd = openfile()
html = get_page(url)
data = parse_html(html)
savefile(fd,data)
time.sleep(random.random())
if __name__ == "__main__":
getInfo()
pass
python request处理cookie方法1
最新推荐文章于 2024-05-14 13:53:37 发布