腾讯新闻和知乎

爬取腾讯新闻

import time
from  selenium import webdriver
from bs4 import BeautifulSoup

driever = webdriver.Chrome("F:/Download/Google/chromedriver")
driever.get('https://news.qq.com/')
html = driever.page_source
bs = BeautifulSoup(html, 'lxml')

driever = webdriver.Chrome("F:/Download/Google/chromedriver")
driever.get('https://news.qq.com/')
html = driever.page_source
bs = BeautifulSoup(html, 'lxml')

find_news(bs)

知乎爬虫

通过下面这个链接获得所需cookis
https://github.com/zkqiang/Zhihu-Login
在zhihu_login_master的文件夹下运行zhihu_login.py文件

import requests
from http import cookiejar
Session=requests.session()
Session.cookies = cookiejar.LWPCookieJar(filename='./cookies.txt')
Session.cookies.load(ignore_discard=True)
Session.headers={
            'Host': 'www.zhihu.com',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
                          '(KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'
        }
r=Session.get("https://www.zhihu.com/search?q=Datawhale&utm_content=search_history&type=content")
r.encoding="utf-8"

from bs4 import BeautifulSoup
import re
compiler=re.compile('"next":"(https:\\\\u002F\\\\u002Fapi.zhihu.com\\\\u002Fsearch_v3.*?)"')
r.text

bsObj=BeautifulSoup(r.text,"lxml")
url=compiler.findall(r.text)[0]

from urllib.parse import unquote
url=unquote(url,encoding="utf-8", errors='replace')
url=url.replace("\\u002F","/")
search_hash_id=re.search("search_hash_id=(.*?)&show_all_topics",url).group(1)
search_hash_id

offset=20
lc_idx=21
for i in range(5):
    r=Session.get("https://www.zhihu.com/api/v4/search_v3?t=general&q=Datawhale&correction=1&offset={offset}&limit=20&lc_idx={lc_idx}&show_all_topics=0&search_hash_id={search_hash_id}&vertical_info=0%2C0%2C1%2C0%2C0%2C0%2C0%2C0%2C0%2C0".format(**{"offset":offset+i*20,"lc_idx":lc_idx+i*20,"search_hash_id":search_hash_id}))
    r.encoding="utf-8"
    print(r.json())
    print("\n"*20)