python爬取微信文章

最新推荐文章于 2024-08-13 08:19:56 发布

Smart_cxr

最新推荐文章于 2024-08-13 08:19:56 发布

阅读量884

点赞数

分类专栏： python

本文链接：https://blog.csdn.net/smart_cxr/article/details/80378449

版权

python 专栏收录该内容

3 篇文章 0 订阅

订阅专栏

from urllib.parse import urlencode
from requests.exceptions import ConnectionError
from pyquery import PyQuery as pq
import pymongo
from config import *

client = pymongo.MongoClient(MONGO_URL)
db = client['weixin']

import requests

base_url = 'http://weixin.sogou.com/weixin?'

keyword = '风景'
proxy_pool_url = 'http://127.0.0.1:5000/get'

proxy = None
max_count = 5

headers = {
    'Cookie': 'ABTEST=0|1526734752|v1; SNUID=DB527D422B2940A28EB384222CAEB7F0; IPLOC=CN3505; SUID=F779516E721A910A000000005B001FA0; SUID=F779516E2513910A000000005B001FA0; weixinIndexVisited=1; SUV=005E51ED6E5179F75B001FA1C247B628; ppinf=5|1526734779|1527944379|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo1OlNtYXJ0fGNydDoxMDoxNTI2NzM0Nzc5fHJlZm5pY2s6NTpTbWFydHx1c2VyaWQ6NDQ6bzl0Mmx1R3F1QlQtWUhuNnUxLWxaLUl1WTZqOEB3ZWl4aW4uc29odS5jb218; pprdig=o6AdmRTOUUptZQw7iDlmIEuEHjPt_RboI7yG4c9_iIyCNyHNe-fgBehcKHdSsH-iWcmL1yySKckEhW-bzU8NXpOsz5QpJIkyYL7noWcEnTWlrYWlqJSW-coaEcZtQqWKMVogGewQOxatu1NKSv7r2T66SvaKezcelAmnVNqhTjs; sgid=13-35107991-AVsAH7sDP0MXQciayCKuuiaEE; ppmdig=1526734780000000dae1fe00f7f8221f5dcb1d034489fc95; sct=1; JSESSIONID=aaarQrSexCah38k9mmjnw',
    'Host': 'weixin.sogou.com',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'
}

def get_proxy():
    try:
        response = requests.get(proxy_pool_url)
        if response.status_code == 200:
            return response.text
        return None
    except ConnectionError:
        return None


def save_to_mongo(data):
    if db['articles'].update({'title':data['title']},{'$set': data},True):
        print('Saved to Monoge',data['title'])
    else:
        print('Save Faile')


def get_html(url, count = 1):
    global proxy
    if count >= max_count:
        print('Tried Too Many...')
        return None
    try:
        if proxy:
            proxies = {
                'http': 'http://' + proxy
            }
            response = requests.get(url, allow_redirects=False, headers=headers, proxies=proxies)
        else:
            response = requests.get(url, allow_redirects=False, headers=headers)
        if response.status_code == 200:
            return response.text
        if response.status_code == 302:
            # Proxy
            print('302')
            proxy = get_proxy()
            if proxy:
                print('Using Proxy...',proxy)
                return get_html(url)
            else:
                print('Get Faile')
                return None
    except ConnectionError as e:
        print('Error',e.args)
        proxy = get_proxy()
        count += 1
        return get_html(url)


def get_index(keyword, page):
    data = {
        'query': keyword,
        'type': 2,
        'page': page
    }
    queries = urlencode(data)
    url = base_url + queries
    html = get_html(url)
    return html

def parse_index(html):
    doc = pq(html)
    items = doc('.news-box .news-list li .txt-box h3 a').items()
    for item in items:
        yield item.attr('href')

def get_detail(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        return None
    except ConnectionError:
        return None

def parse_detail(html):
    doc = pq(html)
    title = doc('#activity-name').text()
    content = doc('.rich_media_content ').text()
    date = doc('#publish_time').text()
    nickname = doc('.rich_media_meta_list .rich_media_meta_nickname').text()
    wechat = doc('#profileBt').text()
    return {
        'title':title,
        'content':content,
        'date': date,
        'nickname':nickname,
        'wechat':wechat
    }



def main():
    for page in range(1, 100):
        html = get_index(keyword,page)
        if html:
            article_urls = parse_index(html)
            for article_url in article_urls:
                article_html = get_detail(article_url)
                if article_html:
                    article_data = parse_detail(article_html)
                    print(article_data)


if __name__ == '__main__':
    main()