python 从url中获取数据

import requests

#通过url获取数据
def get_page(url):
    #requests.get 自带 json.load
    page = requests.get(url)
    page = page.content
    #将bytes转换成字符串
    page = page.decode('utf-8')
    return  page

# print(get_page('https://www.duitang.com/napi/blog/list/by_search/?kw=%E6%A0%A1%E8%8A%B1&start=0&limit=1000'))

 

 

r= get_page('https://adventofcode.com/2020/day/2/input')

 

Puzzle inputs differ by user.  Please log in to get your puzzle input.\n'

 

如何模拟登陆呢？

 

封装成一个类：

import sys
import io
from urllib import request

class Data():
    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8')  # 改变标准输出的默认编码
    def __init__(self):

        #浏览器登录后得到的cookie，也就是刚才复制的字符串
        self.cookie_str = r'_ga=GA1.2.497555532.1608957411; _gid=GA1.2.648427415.1608957411; session=53616c7465645f5fec6238fdbf7a245cd4cc6ecfc3c8e7cbf293b6acc0ab9320cb23950d9c67b8346f3812daa8a7b417'

    def getData(self, url = r'https://adventofcode.com/2020/day/2/input'):
        # 登录后才能访问的网站
        req = request.Request(url)
        # 设置cookie
        req.add_header('cookie', self.cookie_str)
        # 设置请求头
        req.add_header('User-Agent',
                       'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36')
        resp = request.urlopen(req)
        return(resp.read().decode('utf-8'))

 

 

这样子调用：

from ChristmasGame.Data import *

Data = Data()
r = Data.getData()
print(r)