python3+scrapy 趣头条爬虫实例

项目简介

爬取趣头条新闻（http://home.qutoutiao.net/pages/home.html），具体内容：

1、列表页（json）：标题，简介、封面图、来源、发布时间

2、详情页（html）：详细内容和图片

目录结构

生成的数据文件-单条记录

主要代码说明

爬虫：

#爬取趣头条列表和详情页
qutoutiao.spiders.qutoutiaos.QutoutiaosSpider
管道文件：
#封面图片处理类
qutoutiao.imagepipelines.CoverImagePipeline
#内容图片处理类
qutoutiao.imagepipelines.ContentImagePipeline
#数据处理类
qutoutiao.pipelines.QutoutiaoPipeline
中间件：
#请求头设置类-这里只设置了user agent
qutoutiao.middlewares.RandomUserAgent
#代理设置类
qutoutiao.middlewares.RandomProxy

自定义：

#配置文件
qutoutiao.qttconfig.py  
#工具类
qutoutiao.qttutils.QttUtils

创建项目

cd /home/chaoge/mypython/crawler/
scrapy startproject qutoutiao

创建爬虫类即(qutoutiao.spiders.qutoutiaos.QutoutiaosSpider)

cd qutoutiao/qutoutiao/spiders

scrapy genspider qutoutiaos "api.1sapp.com"

执行

      scrapy crawl qutoutiaos

     

#scrapy crawl qutoutiaos --nolog#不显示log
#scrapy crawl qutoutiaos -o qutoutiaos_log.json #将log输出到qutoutiaos_log.json

代码实现

qutoutiao.qttconfig.py  

# 爬取域名（趣头条）
DOMAIN = 'http://home.qutoutiao.net/pages/home.html'
 
#数据存储路径
DATA_STORE = '/home/chaoge/mypython/crawler/qutoutiao/data'
 
#列表：http://api.1sapp.com/content/outList?cid=255&tn=1&page=1&limit=10
#列表API
LIST_API = 'http://api.1sapp.com/content/outList?'
#列表记录数
LIST_LIMIT = 10
#分类
CATEGORY_INFO = [
  {"cid":255,"name":"推荐"},
  {"cid":1,"name":"热点"},
  {"cid":6,"name":"娱乐"},
  {"cid":5,"name":"养生"},
  {"cid":2,"name":"搞笑"},
  {"cid":7,"name":"科技"},
  {"cid":8,"name":"生活"},
  {"cid":10,"name":"财经"},
  {"cid":9,"name":"汽车"},
]

qutoutiao.qttutils.py  

# -*- coding: utf-8 -*- 
# 趣头条工具类
 
import time
import os
import shutil
from qutoutiao import qttconfig as QttConfig
 
class QttUtils:
 
  # 获取存储路径
  #
  # @param  [string] action [remove删除目录，默认create]
  # @return [string] path/year/month/day/*
 
  @staticmethod
  def getStorePath(action='create'):
    localtimes = time.localtime()
    year = time.strftime("%Y", localtimes)
    month = time.strftime("%m", localtimes)
    day = time.strftime("%d", localtimes)
    store_path = QttConfig.DATA_STORE+"/%s/%s/%s"%(year,month,day)
 
    #删除目录
    if os.path.exists(store_path) and action == 'remove':
      #os.rmdir(store_path)  
      shutil.rmtree(store_path)
 
    #创建多级目录
    if not os.path.exists(store_path) and action == 'create':
      os.makedirs(store_path)
 
    return store_path

qutoutiao.settings.py

# -*- coding: utf-8 -*-
 
# Scrapy settings for qutoutiao project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html
 
BOT_NAME = 'qutoutiao'
 
SPIDER_MODULES = ['qutoutiao.spiders']
NEWSPIDER_MODULE = 'qutoutiao.spiders'
 
#日志
#LOG_FILE = "qutoutiao.log"
#日志等级
#LOG_LEVEL = "DEBUG"
 
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'qutoutiao (+http://www.yourdomain.com)'
 
# Obey robots.txt rules
ROBOTSTXT_OBEY = True
 
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
 
# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
 
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
 
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
 
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
#    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36',
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
#}
 
# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
SPIDER_MIDDLEWARES = {
#    'qutoutiao.middlewares.QutoutiaoSpiderMiddleware': 543,
     'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': None,#spider中的allowed_domains将不受限制
}
 
# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
   'qutoutiao.middlewares.RandomUserAgent': 100,
   'qutoutiao.middlewares.RandomProxy': 200,
}
#中间件中的UserAgent池
USER_AGENTS = [
        'User-Agent:Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
        'User-Agent:Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
        'User-Agent:Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0',
        'User-Agent:Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko',
        'User-Agent:Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;',
        'User-Agent:Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',
        'User-Agent:Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',
        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36',
        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
        'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16',
]
#中间件中的Proxy池
PROXIES = [
        {'ip_port':'121.42.140.113:16816','user_password':'username-xxxx:password-xxxx'},
        {'ip_port':'117.90.137.181:9000'},
        {'ip_port':'117.90.2.151:9000'},
        {'ip_port':'114.235.23.147:9000'},
]
 
 
# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}
 
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
    'qutoutiao.imagepipelines.CoverImagePipeline': 301,#封面图片下载
    'qutoutiao.imagepipelines.ContentImagePipeline': 302,#内容图片下载
    'qutoutiao.pipelines.QutoutiaoPipeline': 400,#数据处理
}
#图片存储路径
IMAGES_STORE = "/home/chaoge/mypython/crawler/qutoutiao/tmp/images"
#缩图设置
#IMAGES_THUMBS = {
#    'small':(50,50),
#    'big':(270,270),
#}
#图片宽和高在110*110以下忽略
IMAGE_MIN_HEIGHT = 110
IMAGE_MIN_WIDTH = 110
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
 
# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

qutoutiao.items.py

# -*- coding: utf-8 -*-
 
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
 
import scrapy
 
class QutoutiaoItem(scrapy.Item):
  # define the fields for your item here like:
 
  #文章id
  aid = scrapy.Field()
  #来源
  source_name = scrapy.Field()
  #标题
  title = scrapy.Field()
  #详细页url
  url = scrapy.Field()
  #简介
  introduction = scrapy.Field()
  #封面图
  cover = scrapy.Field()
  #发布时间
  publish_time = scrapy.Field()
  #分类ID
  cid = scrapy.Field()
  #内容
  content = scrapy.Field()
  #内容-中的图片
  content_images = scrapy.Field()

qutoutiao.middlewares.py  

# -*- coding: utf-8 -*- 
 
import random
import base64
from settings import USER_AGENTS
from settings import PROXIES
 
#随机User-Agent
class RandomUserAgent(object):
  def process_request(self,request,spider):
    useragent = random.choice(USER_AGENTS)
    request.headers.setdefault('User-Agent',useragent)
    #request.headers.setdefault('Host','html2.qktoutiao.com')
    #request.headers.setdefault('Referer','http://home.qutoutiao.net/pages/home.html')
 
#随机代理  
class RandomProxy(object):
  def process_request(self,request,spider):
    proxy = random.choice(PROXIES)
    request.meta['proxy'] = 'http://'+proxy['ip_port']
    #base64_user_password = base64.b64encode(bytes(proxy['user_password'], 'utf-8'))
    #decodebs64 = base64.b64decode(base64_user_password)
    #print(base64_user_password,decodebs64)
    if 'user_password' in proxy and proxy['user_password']:#需要用户名密码的代理
      base64_user_password = str(base64.b64encode(bytes(proxy['user_password'], 'utf-8')))
      request.headers['Proxy-Authorization'] = 'Basic '+base64_user_password

qutoutiao.imagepipelines.py

# -*- coding: utf-8 -*-
 
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
 
import scrapy
from scrapy.utils.project import get_project_settings
from scrapy.pipelines.images import ImagesPipeline
import os
from qutoutiao.qttutils import QttUtils
 
#封面图下载
class CoverImagePipeline(ImagesPipeline):
  #获取settings中的常量
  IMAGES_STORE = get_project_settings().get('IMAGES_STORE')
  #下载图片
  def get_media_requests(self, item, info):
    cover_images = item['cover']
    if cover_images:
      for image_url in cover_images:
        yield scrapy.Request(url=image_url)
 
 
  #下载完成
  def item_completed(self, results, item, info):
    #print('*'*20,results,item,info)
    image_path = [x['path'] for ok, x in results if ok] 
    #获取自定义存储路径
    store_path = QttUtils.getStorePath()
    coverImages = []
    #将图片移动到新的路径
    if image_path:
      for image_url in image_path:
        file_name = os.path.split(str(image_url))
        new_image = store_path+"/"+file_name[1]
        coverImages.append(new_image)
        os.rename(self.IMAGES_STORE+"/"+image_url,new_image)
    item['cover'] = coverImages
    return item
#内容图片下载
class ContentImagePipeline(ImagesPipeline):
  #获取settings中的常量
  IMAGES_STORE = get_project_settings().get('IMAGES_STORE')
  #下载图片
  def get_media_requests(self, item, info):
    content_images = item['content_images']
    if content_images:
      for image_url in content_images:
        yield scrapy.Request(image_url)
 
  #下载完成
  def item_completed(self, results, item, info):
    image_info = [(x['path'],x['url']) for ok, x in results if ok] 
    #获取自定义存储路径
    store_path = QttUtils.getStorePath()
    contentImages = []
    #将图片移动到新的路径
    if image_info:
      for value in image_info:
        image_url = value[0]
        image_source = value[1]
        file_name = os.path.split(str(image_url))
        new_image = store_path+"/"+file_name[1]
        contentImages.append((new_image,image_source))
        os.rename(self.IMAGES_STORE+"/"+image_url,new_image)
    item['content_images'] = contentImages
    return item

qutoutiao.pipelines.py 

# -*- coding: utf-8 -*-
 
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
 
import json
from qutoutiao.qttutils import QttUtils
 
class QutoutiaoPipeline(object):
  def __init__(self):
    #获取自定义存储路径
    store_path = QttUtils.getStorePath()
    json_path = store_path+"/"+"qutoutiao.json"
    self.filename = open(json_path,"wb")
 
  def process_item(self, item, spider):
    text = json.dumps(dict(item),ensure_ascii=False)+"\n"
    self.filename.write(text.encode("utf-8"))
    return item
 
  def close_spider(self,spider):
    self.filename.close()








qutoutiao.spiders.qutoutiaos.py
# -*- coding: utf-8 -*-
#web site:http://home.qutoutiao.net/pages/home.html
 
import scrapy
#通过CrawlSpider,Rule类爬取
#-*-from scrapy.spiders import CrawlSpider,Rule-*-
#-*-from scrapy.linkextractors import LinkExtractor-*-
from qutoutiao.items import QutoutiaoItem
import json
import re
from qutoutiao import qttconfig as QttConfig
 
#-*-class QutoutiaosSpider(CrawlSpider):-*-
 
class QutoutiaosSpider(scrapy.Spider):
  name = 'qutoutiaos'
  allowed_domains = ['api.1sapp.com']
 
  #爬取地址
  start_urls = []
  categoryInfo = QttConfig.CATEGORY_INFO
  limit = QttConfig.LIST_LIMIT
  for value in categoryInfo:
      url = QttConfig.LIST_API+"cid=%s&tn=1&page=1&limit=%s"%(str(value['cid']),str(limit))
      start_urls.append(url)
 
  #response里链接的提取规则
  # -*-pageLink = LinkExtractor(allow=("start=\d+"))-*-
  # -*-rules = [
  # -*-   #用pageLink提取规则跟进，通过parseQtt进行解析
  # -*-   Rule(pageLink,callback="parseQtt",follow=True)
  # -*-]
  def parse(self, response):
    response_url = response.url
    #分类id从url又获取了一次
    searchObj = re.search( r'(.*)cid=(\d+)', response_url)
    cid = searchObj and searchObj.group(2) or 0 
 
    data = json.loads(response.text)['data']['data']
  
    for value in data:
      #初始化模型对象
      item = QutoutiaoItem()
      #来源
      item['source_name'] = value['source_name']
      #标题
      item['title'] = value['title']
      #详细页url
      url = item['url'] = value['url']
      #url = url[0:url.find('?')]
      #简介
      item['introduction'] = value['introduction']
      #封面图
      item['cover'] = value['cover']
      #发布时间
      item['publish_time'] = value['publish_time']
      #分类
      item['cid'] = cid
 
      #爬取详情页
      yield scrapy.Request( url = item['url'], meta={'meta_item': item}, callback=self.detail_parse)
 
  #详情页
  def detail_parse(self, response):
    # 提取每次Response的meta数据
    meta_item = response.meta['meta_item']
    #取内容
    content_selector = response.xpath('//div[@class="content"]')
    meta_item['content_images'] = content_selector.xpath('//img/@src|//img/@src').extract()
    meta_item['content'] = content_selector.extract()[0]
    yield meta_item