09-27 python爬取36kr网站文章

准备：
from lxml import etree
import requests

#输入页数,得到网址
def wb_url(page_num):
for num in range(1, page_num+1):
url = ‘https://36kr.com/api/search-column/mainsite?per_page=20&page=’ + str(num)
yield url

#得到首页json数据
def rejson(url):
header = {‘User-Agent’: ‘Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36’}
wb_data = requests.get(url, headers=header)
return wb_data.json()

#根据首页,得到文章网址
def article_urls(rejson):
items_list = rejson.get(‘data’).get(‘items’)
for item_dict in items_list:
article_id = item_dict.get(‘id’)
article_url = ‘https://36kr.com/api/post/’ + str(article_id) + ‘/next’
yield article_url

#取出文章页数据,写入文件
def article_data(url):
wb_data = rejson(url)
title = str(wb_data.get(‘data’).get(‘title’)).repl