利用requests模块中的session爬取登陆后的github页面

利用requests模块中的session爬取登陆后的github页面

1. 从登陆界面入手，找到登陆接口
   
2. 尝试爬取登陆后的主页，并保存session
3. 用session保持登陆状态爬取其他页面
   

import requests

login_url = 'https://github.com/session'

login_name = input("请输入用户名:")
login_pwd = input("请输入用户密码:")

login_headers = {
    'Cookie': '_ga=GA1.2.1665069827.1525761440; _octo=GH1.1.2035577381.1525761440; tz=Asia%2FShanghai; _gat=1; logged_in=no; _gh_sess=NlBrVWFYWHFDektVRFNEbFlxWi9QTG1oanlkTXRwYlA5b0c4cWRJUGlRdE5RR0t0VzQzU05FekZheEEzY0VPTWF1V3JjRWJxMTdXM2Q3VE5MYzYybnZIMDhtaFNjdGZYWUloS1JzbWp4RkhFT2xvWGFYWDE3OTM4dlZjUWJpZDM4eEE1eFF5eVRuYTh1NXIvU1NCM0RaVWFTNyszbWQ1a1IwTVV6YnYvbUMxMm9lN0dvOFQvVXg5QVpGcnZtZ0p3RGNhbXF0NDFDalFzQzFOOWdwTDVIdUlWOWZmU0xPQWZ6ajhXNHUvcmV5WDhtWHJGODNSUFBGRHh5Yk5YTEdNMmo2Y25GRkkwZVJMeVJJZGg3YnhydlNwb3FrUjl6N0M4R3JGUURmVjBmZGJWRi9XVENHTWpkWkU3L1dBd3FhQWtNWjhya3VsY25FRWs4ZVB1Z3hIbEtXNE1SN05JN3RUQnNhSXdNRnVBeTRrZHdRcUU1YW1ZcVdpQTVtdmtUV1dWdDA3ZUNvNVBIU2ZFVk1EcHhWSGIrcjJIQ3hWeWlLZTVlNWZYc2JYMzRvWT0tLUoybVJwNUhIZE5SZnVXcHh4Z0ZZV3c9PQ%3D%3D--eb113638d3e76ddd9e53ffd2c61290a88f0c22d3',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'
}

login_data = {
    "commit":"Sign in",
    "utf8":"✓",
    "authenticity_token":"kljtFT02m2qHHeuXuX0AgN/aOA0IFAhp3sopwPNf0OmRqws47vB8CKQ062pUBaggnHdaKX4CyW1PgTNUTp58xA==",
    "login":login_name,
    "password":login_pwd
}

session = requests.session()

# 1.从登陆接口入手获取到其session
login_response = session.post(url=login_url,headers=login_headers,data=login_data)
with open('sample14.html','wb') as f:
    f.write(login_response.content)

# 2.用session爬取登陆后的某页面
url = "https://github.com/settings/profile"

response = session.get(url=url)

with open('sample14-1.html','wb') as f:
    f.write(response.content)