腾讯视频电影信息爬取（scrapy框架下采用MySQL数据库）

一、准备

 1.cmd 命令行 pip install pymysql，pip install lxml,pip install requests

 2.创建scrapy项目并进行MySQL数据库配置

 具体配置过程可见转载博客：http://blog.csdn.net/qq_31518899/article/details/76576537 Scrapy连接MySQL数据库

 当然若有不太明白的可见本项目具体代码items.py中MysqlConnect类和settings.py代码比较后进行修改操作

二、代码块

 1.settings.py

# -*- coding: utf-8 -*-

# Scrapy settings for Tencent project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'Tencent'

SPIDER_MODULES = ['Tencent.spiders']
NEWSPIDER_MODULE = 'Tencent.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'Tencent (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY =False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  'Accept-Language': 'en',
  'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0',
}

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'Tencent.middlewares.TencentSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
#    'Tencent.middlewares.TencentDownloaderMiddleware': 543,
#}

# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'Tencent.pipelines.TencentPipeline': 300,
   #'Tencent.pipelines.MySQLConnectPipeline':305,
}

#Mysql数据库的配置信息
MYSQL_HOST='127.0.0.1'
MYSQL_USER='root'
#你自己数据库的密码
MYSQL_PASSWORD='root'
MYSQL_PORT =3306
#你自己数据库的名称
MYSQL_DB='test'
CHARSET='utf8'

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

 2.pipelines.py
 

# -*- coding: utf-8 -*-
import scrapy
import xlwt
import pymysql
from scrapy.conf import settings
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html


class TencentPipeline(object):


    def process_item(self, item, spider):
        item.insert_data(item)
        return item

 3.tencent_spider.py
 

# -*- coding: utf-8 -*-
import scrapy
from ..items import TencentItem,CommentItem
import re,requests,json


class TencentSpiderSpider(scrapy.Spider):
    name = 'tencent_spider'
    allowed_domains = ['v.qq.com']
    start_urls = ['https://v.qq.com/x/list/movie']

    def parse(self, response):
        category_part = response.xpath('//div[@class="mod_row_filter"]/ul/li/a/@href').extract()
        for href in category_part:
            detail_url='https://v.qq.com/x/list/movie{}'.format(href)
            yield scrapy.Request(url=detail_url,
                                 callback=self.detail_parse
            )
    def detail_parse(self,response):
        headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:53.0) Gecko/20100101 ' \
                          'Firefox/53.0'}
        #分类后的电影信息
        movie_links=response.xpath('//div[@class="mod_figures mod_figure_v"]/ul/li/a/@href').extract()
        movie_titles=response.xpath('//div[@class="figure_title_score"]/strong/a/text()').extract()
        movie_scores=response.xpath('//div[@class="figure_score"]//text()').extract()
        score_list=[]
        total_score=[]
        #得到处理后的评分列表
        for movie_score in movie_scores:
            if movie_score !='\n\t\t\t\t\t\t\t' and movie_score!='\n\t\t\t\t\t\t':
                score_list.append(movie_score)
        #print(score_list)
        j=0
        while j in range(0,len(score_list)-1):
            score=score_list[j]+score_list[j+1]
            j += 2
            total_score.append(score)
        #print(total_score)
        movie_playCounts=response.xpath('//div[@class="figure_count"]/span/text()').extract()#播放量
        movie_account=response.xpath('//span[@class="option_txt"]/em/text()').extract_first('')#个数
        #进入电影详情页
        for x in range(0,len(movie_links)):
            #获取电影链接中的cid例如中括号的内容https://ncgi.video.qq.com/fcgi-bin/video_comment_id?otype=json&callback=jQuery(19109829145422060698_1517407245638)&op=3&【cid=b5i4g9z3u5h31jy】
            #然后接合GET请求中的评论页的json链接获取json数据中的comment_id,然后拼接评论页url,获取评论内容
            cid=movie_links[x].split('/')[-1]#获取cid
            cid=cid.split('.')[0]
            #print(cid)
            #获取comment_id
            comment_id_url='https://ncgi.video.qq.com/fcgi-bin/video_comment_id?otype=json&callback=jQuery&op=3&cid={}'.format(cid)
            html=requests.get(comment_id_url).text
            pattern=re.compile(r'comment_id":"(.*?)"')
            comment_id=re.search(pattern,html).group(1)
            #print(comment_id)
            #获取评论页内容
            comment_url='http://coral.qq.com/article/{}/comment/'.format(comment_id)
            comment_html=requests.get(comment_url,headers=headers).text
            dict=json.loads(comment_html)#获得json数据，并通过解析取出需要数据
            data_dict = dict['data']
            commentid_list = data_dict['commentid']
            if commentid_list:#电影有评论
                for detail in commentid_list:
                    comment =CommentItem()
                    comment['movie_title'] = movie_titles[x]#电影名
                    comment['timeDifference'] = detail['timeDifference']# 发布时间
                    comment['content'] = detail['content']# 内容
                    comment['up'] = detail['up']# 点赞
                    comment['rep'] = detail['rep']# 踩
                    userinfo_dict = detail['userinfo']# 用户信息（字典）
                    userid = userinfo_dict['userid']
                    comment['userid']=userid# 用户id
                    comment['userLink']='http://video.coral.qq.com/review/user/{}'.format(userid)#用户链接
                    yield comment

            yield  scrapy.Request(url=movie_links[x],
                                callback=self.movie_parse,
                                 meta={'movie_link':movie_links[x],
                                     'movie_title':movie_titles[x],
                                       'score':total_score[x],
                                       'movie_playCount':movie_playCounts[x],
                                       'movie_account':movie_account}
            )
        # 下一页
        next_pg = response.xpath('//a[@class="page_next"]/@href').extract_first('')
        print(next_pg)
        if next_pg:
            next_url = 'https://v.qq.com/x/list/movie{}'.format(next_pg)
            yield scrapy.Request(url=next_url,
                                 callback=self.detail_parse
                                 )
    def movie_parse(self,response):

       #简介区
       abstract=response.xpath('//div[@class="mod_row_box"]/div[2]/ul/li/div[2]/p/text('
                                ')').extract_first('')
       directors=response.xpath('//div[@class="mod_row_box"]/div[2]/ul/li/div[1]/a//text()').extract()
       director_links = response.xpath('//div[@class="mod_row_box"]/div[2]/ul/li/div[1]/a//@href').extract()
       if directors:#存在导演信息
            director=directors[0]
            act=','.join(directors[1:])
            director_link=director_links[0]
            act_link=','.join(director_links[1:])
       else:
           director ='#'
           act = '#'
           director_link = '#'
           act_link = '#'
       #概览区
       movie_title=response.meta['movie_title']
       score=response.meta['score']
       movie_playCount=response.meta['movie_playCount']
       movie_account=response.meta['movie_account']
       movie_link=response.meta['movie_link']

       movie=TencentItem()
        #简介
       movie['abstract']=abstract
       movie['director']=director
       movie['act']=act
       movie['director_link']=director_link
       movie['act_link']=act_link
        #概览
       movie['movie_title']=movie_title
       movie['score']=score
       movie['movie_playCount']=movie_playCount
       movie['movie_link']=movie_link
       movie['movie_account']=movie_account
       yield movie

此为主要代码
 
4. items.py
 

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy
import pymysql
from scrapy.conf import settings

class MySQLConnect(scrapy.Item):
    @staticmethod
    def mysqlConnect(sql):
        host = settings['MYSQL_HOSTS']
        user = settings['MYSQL_USER']
        psd = settings['MYSQL_PASSWORD']
        db = settings['MYSQL_DB']
        charset = settings['CHARSET']
        port = settings['MYSQL_PORT']
        # 数据库连接
        con = pymysql.connect(host=host, user=user, passwd=psd, db=db, charset=charset, port=port)
    # 数据库游标
        cur = con.cursor()
        try:
            cur.execute(sql)
            print("insert success")  # 测试语句
        except Exception as e:
            print('Insert error:', e)
            con.rollback()
        else:
            con.commit()
        con.close()

class TencentItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    #简介区
    abstract=scrapy.Field()
    director=scrapy.Field()
    director_link=scrapy.Field()
    act=scrapy.Field()
    act_link=scrapy.Field()

    #概览区
    movie_title=scrapy.Field()
    score=scrapy.Field()
    movie_playCount=scrapy.Field()
    movie_account=scrapy.Field()
    movie_link=scrapy.Field()

    def insert_data(self,item):
        sql1 = "insert into abstract(movieTitle,director,directorLink,act,actLink,abstract)values('%s','%s','%s','%s','%s','%s');" % (item['movie_title'], item['director'], item['director_link'], item['act'], item['act_link'],item['abstract'])
        print('TencentItem insert.....................')
        MySQLConnect.mysqlConnect(sql1)
        sql3 = "insert into overview(movieTitle,score,playCount,link)values('%s','%s','%s','%s');" % (item['movie_title'], item['score'], item['movie_playCount'], item['movie_link'])
        MySQLConnect.mysqlConnect(sql3)
class CommentItem(scrapy.Item):
    # 评论区
    movie_title = scrapy.Field()
    timeDifference = scrapy.Field()
    content = scrapy.Field()
    up = scrapy.Field()
    rep = scrapy.Field()
    userLink = scrapy.Field()
    userid = scrapy.Field()
    def insert_data(self,item):
        sql2 = "insert into comment(userID,userLink,timeDiffrence,content,praise,ref,movieTitle)values('%s','%s','%s','%s','%s','%s','%s');" % (item['userid'], item['userLink'], item['timeDifference'], item['content'], item['up'],item['rep'], item['movie_title'])
        print('CommentItem insert................')
        MySQLConnect.mysqlConnect(sql2)

具体项目可见码云： https://gitee.com/YunZhiBiDuan3555/TengXunShiPinWangYeBanDianYingXinXiPaQu.git