使用代理把搜索结果导入mongo

# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""
import pymongo
import requests
from requests.exceptions import ConnectionError
from pyquery import PyQuery as pq

client = pymongo.MongoClient('localhost')
db = client['weixin']
keyword = '微信'

PROXY_POOL_URL = 'http://localhost:5555/random'

proxy = None
max_count = 5

def get_proxy():
    try:
        response = requests.get(PROXY_POOL_URL)
        if response.status_code == 200:
            return response.text
    except ConnectionError:
        return None

base_url = 'https://weixin.sogou.com/weixin?query={}&s_from=input&type=2&page={}&ie=utf8'

headers = {
        'Cookie':'ABTEST=0|1571491299|v1; IPLOC=CN3205; SUID=DE7714703F18960A000000005DAB0DE3; SUID=DE7714702E18960A000000005DAB267B; weixinIndexVisited=1; SUV=002BF9B2701477DE5DAB267E8E012097; ppinf=5|1571497613|1572707213|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZToxODolRTclOUIlOVezh98Fz7eTGl0iiA2EuoTrHWAGZ6p6hqbcEo5MW-e88; sgid=17-33932773-AV2rJo1OvricXjyTmRKyxr3g; SNUID=CC660662121784570BDF41DE12B4BDB6; sct=2; ppmdig=1571530299000000774f0759922172293d5677ac996b6ece; JSESSIONID=aaas10rTvoJt06_aDbB1w',
        'Host':'weixin.sogou.com',
        'Upgrade-Insecure-Requests':'1',
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'
        }

#def get_proxy()

def get_html(url):
    global proxy
    try:
        if proxy:
            proxies = {
                    'htttp':'http//'+proxy
                    }
            response =requests.get(url,allow_redirects=False,headers=headers
                                   ,proxies=proxies
                                   )
        else:
            response =requests.get(url,allow_redirects=False,headers=headers)
            
            
        if response.status_code == 200:
            return response.text
            print('200')
        if response.status_code == 302:
            print('302')
            proxy = get_proxy()
            if proxy:
                print('Using proxyu', proxy)
                return get_html(url)
            else:
                print('Get porxy failed')
                return None
        if response.status_code == 404:
            print('404')
        
    except ConnectionError as e:
        print('Error Occurred:',e.args)
        proxy = get_proxy()
        return get_html(url)


def get_index(keyword,page):
    url = base_url.format(keyword,page)
    html = get_html(url)
    return html

    
def parse_index(html):
    doc = pq(html)
    items = doc('.news-list li .txt-box h3 a').items()
    for item in items:
        yield item.attr('data-share')

def get_detail(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        return None
    except ConnectionError:
        return None
    
def parse_detail(html):
    doc = pq(html)
    title = doc('.rich_media_title').text()
    date = doc('#publish_time').text()
    name = doc('.profile_nickname').text()
    detail = doc('.profile_meta_value').text()
    return {
            'title':title,
            'date':date,
            'name':name,
            'detail':detail}
    
def save_mongo(data):
    if db['articles'].update({'title':data['title']},{'$set':data},True):
        print('save to mongo',data['title'])
    else:
        print('failed',data['title'])
               
               
def main():
    for page in range(1,101):
        html = get_index(keyword,page)
        if html:
            article_urls = parse_index(html)
            for article_url in article_urls:
                article_html = get_detail(article_url)
                if article_html:
                    article_data =  parse_detail(article_html)
                    print(article_data)
                    save_mongo(article_data)
                    



if __name__ == '__main__':
    main()