抓取今日头条部分信息

2019独角兽企业重金招聘Python工程师标准>>>

import requests import re from urllib.parse import urlencode from requests.exceptions import RequestException import json from bs4 import BeautifulSoup import codecs from conm import *

header = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36', } #定义获取内容的方法 def get_page_index(offset,keyword): data = { 'offset': offset, 'format': 'json', 'keyword': keyword, 'autoload': 'true', 'count': 20, 'cur_tab': 3, } #合成url url = 'http://www.toutiao.com/search_content/?' + urlencode(data) #对请求做些异常处理 try: response = requests.get(url,headers = header) if response.status_code == 200: return response.text return None except RequestException: print('请求索引页出错') return None

def parse_page_index(html): data = json.loads(html) if data and 'data' in data.keys(): for item in data.get('data'): yield item.get('article_url')

#获取详情页面 def get_page_detail(url): res = requests.get(url,headers = header) try: if res.status_code == 200: return res.text return None except RequestException: print('请求失败') return None #解析详细页面 def parse_page_detail(html,url): soup = BeautifulSoup(html, 'lxml') title = soup.select('title')[0].get_text() images_pattern = re.compile(r'gallery: JSON.parse("(.*?)"),', re.S) result = re.search(images_pattern, html) if result: # group(0)是原始字符串, group(1)是第一个括号匹配到的字符串 # groups()以元组形式返回全部分组截获的字符串。相当于调用group(1,2,…last) # codecs: 使不具有转义的反斜杠具有转义功能,最难的一步骤 data_str = codecs.getdecoder('unicode_escape')(result.group(1))[0]

    json_data = json.loads(data_str)
    if json_data and 'sub_images' in json_data.keys():
        sub_images = json_data.get('sub_images')
        images = [item.get('url') for item in sub_images]
        for image in images:
            download_img(image)
        return {
            'title': title,
            'images': images,
            'url': url
        }
else:
    print('没有搜索到符合条件的gallery数据', end='\n\n')

import pymongo #定义数据库连接函数 client = pymongo.MongoClient(MONGO_URL) db = client[MONGO_DB]

#存储方法 def save_to_mongo(result): if db[MONG_TABLE].insert(result): print('存储成功') return True return False

#定义下在图片的方法 def download_img(url): print('正在下载',url) try: response = requests.get(url, headers=header) if response.status_code == 200: save_image(response.content) return None except RequestException: print('请求索引页出错') return None #定义存储图片的方法 import os from hashlib import md5 def save_image(content): file_path ='{0}/{1}.{2}'.format('mao',md5(content).hexdigest(),'jpg') if not os.path.exists(file_path): with open(file_path,'wb') as f: f.write(content)

#主测试函数 def main(offset): html = get_page_index(offset,'街拍') for url in parse_page_index(html): result = get_page_detail(url) ok = parse_page_detail(result,url) save_to_mongo(ok)

from multiprocessing import Pool if name == 'main': groups = [x * 20 for x in range(GROUP_START, GROUP_END)] pool = Pool() pool.map(main,groups)

转载于:https://my.oschina.net/u/2511906/blog/1933380