Python --爬虫 头条街拍图片爬取

分析逻辑基本和前面的类似，只是这次将数据源存入mogodb中。具体实现代码如下：

####### config 文件

MONGO_URL = 'localhost'
MONGO_DB = 'toutiao'
MONGO_TABLE = 'toutiao'

group_start = 1
group_end = 10
key_word = '街拍'

import requests
from requests.exceptions import RequestException  #requests请求时错误类型
from urllib.parse import urlencode
import json  #json解析
from bs4 import BeautifulSoup
import re  # 正则表达式
from config import * # 自己定义的一些参数列表
import pymongo  # 使用MongoDB时引用的库
from hashlib import md5  # 存储文件时引用的库
import os  # 存储文件时引用当前路径存储的路劲
from multiprocessing import Pool  # 多线程运行

client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_DB]
headers = {'user-agent':
               'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36'}


def get_page_index(offset, keyword):
    data = {
        'offset': offset,
        'format': 'json',
        'keyword': keyword,
        'autoload': 'true',
        'count': '20',
        'cur_tab': 1
    }
    url = 'https://www.toutiao.com/search_content/?' + urlencode(data)
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        print('请求索引页出错')
        print(response.status_code)
        return None


# 获取最上层的url中article url的信息
def parse_page_index(html):
    data = json.loads(html)  # response 是json型的数据，因此用json.loads()处理
    if data and 'data' in data.keys():  # 判断data 是否在元数据中.data.keys()返回的是json文件中的key
        for item in data.get('data'):
            yield item.get('article_url')


def get_page_detail(url):
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        print('请求页面出错', url)
        return None


def parse_page_detail(html, url):
    soup = BeautifulSoup(html, 'lxml')
    title = str(soup.select('title'))
    print(title)
    images_pattern = re.compile('"(http:.*?)&quot', re.S)
    result = re.findall(images_pattern, html)
    images = [image for image in result]
    for image in images: download_image(image)
    if result:
        return {
            'title': title,
            'url': url,
            'images': images
        }


def save_image(content):
    file_path = '{0}/{1}.{2}'.format(os.getcwd(), md5(content).hexdigest(), 'jpg')
    if not os.path.exists(file_path):
        with open(file_path, 'wb') as f:
            f.write(content)
            f.close()


def download_image(url):
    print('正在下载：', url)
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            #            return response.text
            save_image(response.content)
        return None
    except RequestException:
        print('请求图片出错', url)
        return None


def save_to_mongo(result):
    if result:
        if db[MONGO_TABLE].insert(result):
            print('存储到MONGODB成功', result)
            return True
    return False


def main(offset):
    html = get_page_index(offset, key_word)
    for url in parse_page_index(html):
        html = get_page_detail(url)
        if html:
            result = parse_page_detail(html, url)
            #            print(result)
            save_to_mongo(result)


if __name__ == '__main__':
    # main()
    # groups = [i * 10 for i in range(group_start, group_end + 1)]
    # pool = Pool()
    # pool.map(main, groups)
    for i in range(group_start, group_end + 1):
        main(i)