简单爬取知乎热榜到mongodb

最新推荐文章于 2022-07-22 18:56:22 发布

weixin_43139613

最新推荐文章于 2022-07-22 18:56:22 发布

阅读量187

点赞数

分类专栏：笔记 python基础

本文链接：https://blog.csdn.net/weixin_43139613/article/details/102663068

版权

笔记同时被 2 个专栏收录

105 篇文章 1 订阅

订阅专栏

python基础

52 篇文章 0 订阅

订阅专栏

import pymongo
import requests
from requests.exceptions import ConnectionError
from pyquery import PyQuery as pq

client = pymongo.MongoClient('localhost')
db = client['zhihu']



base_url = 'https://www.zhihu.com/hot'

headers = {
        'Cookie':'_c10c',
        'Host':'www.zhihu.com',
        'Upgrade-Insecure-Requests':'1',
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'
        }


def get_html(url):
    try:
        response =requests.get(url,allow_redirects=False,headers=headers)
        if response.status_code == 200:
            return response.text
            print('200')
        if response.status_code == 302:
            print('302')
        if response.status_code == 404:
            print('404')
        
    except ConnectionError:
        return get_html(url)
        
def save_mongo(data):
    if db['hot'].update({'title':data['title']},{'$set':data},True):
        print('save to mongo',data['title'])
    else:
        print('failed',data['title'])

def get_index():
    url = base_url
    html = get_html(url)
    doc = pq(html)
    title = doc('.HotItem-title').items()
    hot = doc('.HotItem-metrics.HotItem-metrics--bottom').items()
    href = doc('.HotItem-content a').items()

    for o,i,e in zip(title,hot,href):
        dic1 = {}
        dic1['title'] = o.text(),
        dic1['hot'] = i.text(),
        dic1['href'] = e.attr('href')
        save_mongo(dic1)
#        print(dic1)

    
if __name__ == '__main__':
    get_index()