爬虫-古诗词网站（隐藏域-验证码的解决方案）

from lxml import etree
from urllib import request
import requests
url = 'https://www.gushiwen.cn/user/login.aspx?from=http%3a%2f%2fwww.gushiwen.cn%2fuser%2fcollect.aspx'
headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36'
}
# urllib来获取内容
# requester = request.Request(url, headers = headers)
# response = request.urlopen(requester)
# content = response.read().decode('utf-8')

# 上述代码也可用 requests.get()得到更加简便
response = requests.get(url,headers)
content = response.text

# xpath解析源码获取目的数据
tree = etree.HTML(content)
# 1.获取隐藏域的值
__VIEWSTATE = tree.xpath('//input[@name="__VIEWSTATE"]/@value')
__VIEWSTATEGENERATOR = tree.xpath('//input[@name="__VIEWSTATEGENERATOR"]/@value')
# 2.获取验证码图片
session = requests.session()  # 关键！！！
code_url = 'https://www.gushiwen.cn/RandCode.ashx'  # 这里也可以xpath获取；由于每个图片网址都一样，所以直接复制过来了
content_code = session.get(code_url, headers=headers).content  # ！！二进制格式  (跟request.get()用法一样只不过换成了session,并且读取不再是text)
with open('code.png','wb') as f:
    f.write(content_code)
code = input('输入验证码:')  # 找到下载的图片然后输入


# 3.传入获取的数据
data = {
'__VIEWSTATE': __VIEWSTATE,   # 隐藏域问题:查看源码Xpath定位到内容
'__VIEWSTATEGENERATOR': __VIEWSTATEGENERATOR,
'from': 'http://www.gushiwen.cn/user/collect.aspx',
'email': 'xxxxxxxxxxx',  # 换成自己的账号 
'pwd': 'xxxxxxxxxxxx',   # 换成自己的密码
'code': code,  # 看图后手动输入
'denglu': '登录',
}
# 4.访问--由于是用session下载的所以为了让再次访问的是同一个响应页面 ！！！必须用session
post_response = session.post(url,params=data,headers=headers)  # ！post请求
post_content = post_response.text  # 还是requests的读内容方法
with open('gushici.html', 'w', encoding='utf-8') as f:
    f.write(post_content)