反爬虫抓取微信文章（前篇）

import requests
from urllib.parse import urlencode
from requests.exceptions import ConnectionError
from pyquery import PyQuery as pq

base_url = 'https://weixin.sogou.com/weixin?'

headers = {
    'Cookie': 'SUV=0050272B0E1788145AD0D1D0B5C42214; dt_ssuid=3835736327; pex=C864C03270DED3DD8A06887A372DA219231FFAC25A9D64AE09E82AED12E416AC; ssuid=8355709570; SUID=1488170E3665860A5B933B700007EAF7; SMYUV=1540625126607399; CXID=0E31D6148DFE7E3823CFFE36FA7A5EDF; pgv_pvi=389063680; GOTO=; toutiao_city_news=%E5%8C%97%E4%BA%AC; _ga=GA1.2.1568155290.1580978361; wuid=AAG/ghcULgAAAAqLMXVRzQkAGwY=; IPLOC=CN4401; ABTEST=0|1602147107|v1; SNUID=3CF06076777DC65C89455BFC78EA55FC; weixinIndexVisited=1; JSESSIONID=aaakEGIn8Y8Nx7fLl-atx',
    'Host': 'weixin.sogou.com',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
}

keyword = '风景'
proxy_pool_url = 'http://127.0.0.1:5555/random'

proxy = None
max_count =5

def get_proxy():
    try:
        response = requests.get(proxy_pool_url)
        if response.status_code == 200:
            return response.text
        return None
    except ConnectionError:
        return None

def get_html(url,count=1):
    print('Crawling',url)
    print('Trying Count',count)
    global proxy
    if count >= max_count:
        print('Tried Too Many Counts')
        return None
    try:
        if proxy:
            proxies = {
                'http':'http://'+proxy
            }
            response = requests.get(url, allow_redirects=False, headers=headers,proxies=proxies)
        else:
            response = requests.get(url, allow_redirects=False, headers=headers)
        if response.status_code == 200:
            return response.text
        if response.status_code == 302:
            #Need Proxy
            print('302')
            proxy = get_proxy()
            if proxy:
                print('Using Proxy',proxy)
                return get_html(url)
            else:
                print('Get Proxy Failed')
                return None

    except ConnectionError as e:
        print('Error occurred',e.args)
        proxy = get_proxy()
        count +=1
        return get_html(url,count)

def get_index(keyword,page):
    data = {
        'query':keyword,
        'type':2,
        'page':page
    }
    queries = urlencode(data)
    url = base_url + queries
    html = get_html(url)
    return html




def main():
    for page in range(1,101):
        html = get_index(keyword,page)
        print(html)



if __name__ == '__main__':
    main()