Python3 爬取今日头条(模拟ajax请求)
注:本文是跟据博主崔庆才的博客来写的,不单单是思路值得学习,还有代码规范更值得去学习。路漫漫其修远兮,吾将上下而求索。
新建一个config.py文件,如下
MONGO_URL = 'localhost'
MONGO_DB = 'toutiao'
MONGO_TABLE = 'toutiao'
GROUP_START = 1
GROUP_END = 20
KEYWORD = '街拍'
然后创建一个spider.py文件,内容如下:i
import requests
from urllib.parse import urlencode
from requests import RequestException
import json
from json import JSONDecodeError
from bs4 import BeautifulSoup
import re,os
import pymongo
from config import * #导入之前创建的config.py文件
from hashlib import md5
from multiprocessing import Pool
client = pymongo.MongoClient(MONGO_URL ,connect=False)
db = client[MONGO_DB]
#获取首页Json数据,记为1
def get_page_index(offset, keyword):
data = {
'offset': offset,
'format': 'json',
'keyword': keyword,
'autoload': 'true',
'count': '20',
'cur_tab': '3',
'from': 'gallery'
}
headers = {'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)'}
url = 'https://www.toutiao.com/search_content/?' + urlencode(data)
try:
response = requests.get(url, headers = headers)
if response.status_code == 200:
return response.text
return None
except RequestException:
print('请求出错')
return None
#解析1的Json数据,获取所需界面URL地址
def parse_page_index(html):
try:
data = json.loads(html)
if data and 'data' in data.keys(): #判断data是否存在
for i in data.get('data'):
# print(i.get('article_url'))
yield i.get('article_url')
except JSONDecodeError:
pass
#处理从1中解析得来的数据,记为2
def get_page_detail(url):
headers = {'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)'}
try:
response = requests.get(url, headers = headers)
if response.status_code == 200:
return response.text
return None
except RequestException:
print('请求详情页出错',url)
return None
#解析2
def parse_page_detail(html ,url):
headers = {'User-Agent': 'Mozilla / 5.0(X11;Ubuntu;Linuxx86_64;rv: 59.0) Gecko / 20100101Firefox / 59.0'}
soup = BeautifulSoup(html, 'lxml')
title = soup.select('title')[0].get_text()
print(title)
images_pattern = re.compile('var gallery = (.*?);',re.S)
result = re.search(images_pattern, html)
if result:
data = json.loads(result.group(1))
#print(result.group(1))
if data and 'sub_images' in data.keys():
sub_images = data.get('sub_images')
images = ['http:' + i.get('url') for i in sub_images]
print(images)
for images in images:
download_image(images,headers,title)
return {
'title':title,
'images': images,
'url':url,
}
#将解析的数据保存到Mongo数据库中
def save_to_mongo(result):
if db[MONGO_TABLE].insert(result):
print('存储到MongoDB成功',result)
return True
return False
#下载图片
def download_image(url, headers ,title):
print("正在下载:", url)
try:
response = requests.get(url, headers = headers)
if response.status_code == 200 :
save_image(response.content ,title)
return None
except RequestException:
print('请求图片出错', url)
return None
def save_image(content, title):
if not os.path.exists(title): #如果文件夹不存在就创建
os.mkdir(title)
file_path = '{0}/{1}.{2}'.format(title,md5(content).hexdigest(), 'jpg')
if not os.path.exists(file_path):
with open(file_path, 'wb') as f:
f.write(content)
f.close()
# offset 为要爬取的页码, KEYWORD为检索内容
def main(offset):
html = get_page_index(offset, KEYWORD)
for url in parse_page_index(html):
html = get_page_detail(url)
if html:
# print(html)
result = parse_page_detail(html,url)
#print(result)
if result: save_to_mongo(result)
if __name__ == '__main__':
groups = [x * 20 for x in range(GROUP_START, GROUP_END + 1)]
pool = Pool()
pool.map(main,groups)