猫眼测试代码

maoyan_spider.py

# -*- coding: utf-8 -*-
import scrapy

from maoyan.items import MaoyanItem


class MaoyanSpiderSpider(scrapy.Spider):
    name = 'maoyan_spider'
    # allowed_domains = ['www']

    # 请求分页
    start_urls = []
    for i in range(10):
        base_url = 'https://maoyan.com/board/4?offset={}'.format(i * 10)
        start_urls.append(base_url)

    def parse(self, response):
        item = MaoyanItem()
        namelist = response.xpath('//dl[@class="board-wrapper"]/dd')
        for t in namelist:
            title = t.xpath('.//a/@title').extract_first()
            actor = t.xpath('.//div/div/div/p[@class="star"]/text()').extract_first()
            datetime = t.xpath('.//div/div/div/p[3]').extract_first()
            base_url = t.xpath('.//div/div/div/p/a/@href').extract_first()
            item['title'] = title
            item['actor'] = actor.strip()
            item['datetime'] = datetime[23:-4]
            item['base_url'] = 'https://www.maoyan.com' + base_url
            # print(item)
            yield item

items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

import scrapy


class MaoyanItem(scrapy.Item):
    # define the fields for your item here like:
    title = scrapy.Field()
    actor = scrapy.Field()
    datetime = scrapy.Field()
    base_url = scrapy.Field()

settings.py

# -*- coding: utf-8 -*-

# Scrapy settings for maoyan project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'maoyan'

SPIDER_MODULES = ['maoyan.spiders']
NEWSPIDER_MODULE = 'maoyan.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
# USER_AGENT = 'maoyan (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
# CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
# DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
# CONCURRENT_REQUESTS_PER_DOMAIN = 16
# CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
# COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
# TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'en',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36',
    'Cookie': '__mta=213629483.1586196819456.1588166161560.1588168190272.65; _lxsdk_cuid=17150b2880dc8-0721b87c08fdb6-2393f61-144000-17150b2880ec8; uuid_n_v=v1; uuid=562DEE10783211EA9B1571464B4C3D1FB91D18E6FC164A64A66868EACD0A713E; _lxsdk=562DEE10783211EA9B1571464B4C3D1FB91D18E6FC164A64A66868EACD0A713E; mojo-uuid=8546698e3e630d3dd44a1969d6edd2ab; t_lxid=171866924aac8-09d87fb844275a-2393f61-144000-171866924aac8-tid; _csrf=7e6340a41994746c522c073f0fade60784e134a340d199051e0c3b1e9ec4e344; Hm_lvt_703e94591e87be68cc8da0da7cbd0be2=1587130079,1587382583,1587827732,1588166138; _lx_utm=utm_source%3DBaidu%26utm_medium%3Dorganic; __mta=213629483.1586196819456.1587829790886.1588166140421.63; mojo-session-id={"id":"cc9c9a0d836b0d7cc268d5fe555b60c6","time":1588168190116}; Hm_lpvt_703e94591e87be68cc8da0da7cbd0be2=1588168190; mojo-trace-id=2; _lxsdk_s=171c63366cb-a0e-fbf-059%7C%7C3',
}

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
# SPIDER_MIDDLEWARES = {
#    'maoyan.middlewares.MaoyanSpiderMiddleware': 543,
# }

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# DOWNLOADER_MIDDLEWARES = {
#    'maoyan.middlewares.MaoyanDownloaderMiddleware': 543,
# }

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
# EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
# }

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# ITEM_PIPELINES = {
#    'maoyan.pipelines.MaoyanPipeline': 300,
# }

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
# AUTOTHROTTLE_ENABLED = True
# The initial download delay
# AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
# AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
# AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
# HTTPCACHE_ENABLED = True
# HTTPCACHE_EXPIRATION_SECS = 0
# HTTPCACHE_DIR = 'httpcache'
# HTTPCACHE_IGNORE_HTTP_CODES = []
# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

movie2.csv

actor,base_url,datetime,title
"主演：葛优,巩俐,牛犇",https://www.maoyan.com/films/1375,上映时间：1994-05-17(法国),活着
"主演：艾德里安·布洛迪,艾米莉娅·福克斯,米哈乌·热布罗夫斯基",https://www.maoyan.com/films/4883,上映时间：2002-05-24(法国),钢琴家
"主演：梅尔·吉布森,苏菲·玛索,帕特里克·麦高汉",https://www.maoyan.com/films/3294,上映时间：1995-05-18(美国),勇敢的心
"主演：张国荣,张曼玉,刘德华",https://www.maoyan.com/films/11237,上映时间：2018-06-25,阿飞正传
"主演：张国荣,梁朝伟,张学友",https://www.maoyan.com/films/13824,上映时间：1993-02-05(中国香港),射雕英雄传之东成西就
"主演：俞承豪,金艺芬,童孝熙",https://www.maoyan.com/films/6796,上映时间：2002-04-05(韩国),爱·回家
"主演：马里奥·毛瑞尔,平采娜·乐维瑟派布恩,阿查拉那·阿瑞亚卫考",https://www.maoyan.com/films/263,上映时间：2012-06-05,初恋这件小事
"主演：莱昂纳多·迪卡普里奥,凯特·温丝莱特,比利·赞恩",https://www.maoyan.com/films/267,上映时间：1998-04-03,泰坦尼克号
"主演：雅克·贝汉,Philippe Labro",https://www.maoyan.com/films/1346,上映时间：2001-12-12(法国),迁徙的鸟
"主演：克里斯蒂安·贝尔,希斯·莱杰,阿伦·伊克哈特",https://www.maoyan.com/films/790,上映时间：2008-07-14(阿根廷),蝙蝠侠：黑暗骑士
"主演：张曼玉,梁家辉,甄子丹",https://www.maoyan.com/films/94,上映时间：2012-02-24,新龙门客栈
"主演：杰伊·巴鲁切尔,杰拉德·巴特勒,亚美莉卡·费雷拉",https://www.maoyan.com/films/330,上映时间：2010-05-14,驯龙高手
"主演：阿尔·帕西诺,罗伯特·德尼罗,黛安·基顿",https://www.maoyan.com/films/6641,上映时间：1974-12-12(美国),教父2
"主演：罗伯特·德尼罗,詹姆斯·伍兹,伊丽莎白·麦戈文",https://www.maoyan.com/films/1332,上映时间：2015-04-23,美国往事
"主演：费雯·丽,罗伯特·泰勒,露塞尔·沃特森",https://www.maoyan.com/films/2760,上映时间：1940-05-17(美国),魂断蓝桥
"主演：马修·布罗德里克,尼基塔·卡兰姆,詹姆斯·厄尔·琼斯",https://www.maoyan.com/films/592,上映时间：1995-07-15,狮子王
"主演：尼古拉斯·凯奇,艾玛·斯通,瑞安·雷诺兹",https://www.maoyan.com/films/78099,上映时间：2013-04-20,疯狂原始人
"主演：周星驰,巩俐,郑佩佩",https://www.maoyan.com/films/837,上映时间：1993-07-01(中国香港),唐伯虎点秋香
"主演：范·迪塞尔,保罗·沃克,道恩·强森",https://www.maoyan.com/films/364,上映时间：2011-05-12,速度与激情5
"主演：娜塔莉·波特曼,雨果·维文,斯蒂芬·瑞",https://www.maoyan.com/films/13636,上映时间：2005-12-11(美国),V字仇杀队
"主演：爱德华·阿斯纳,乔丹·长井,鲍勃·彼德森",https://www.maoyan.com/films/614,上映时间：2009-08-04,飞屋环游记
"主演：倍赏千惠子,木村拓哉,美轮明宏",https://www.maoyan.com/films/225980,上映时间：2004-09-05(意大利),哈尔的移动城堡
"主演：亚历桑德雷·罗德里格斯,艾莉丝·布拉加,莱安德鲁·菲尔米诺",https://www.maoyan.com/films/1309,上映时间：2002-05-18(戛纳电影节),上帝之城
"主演：罗素·克洛,詹妮弗·康纳利,艾德·哈里斯",https://www.maoyan.com/films/11473,上映时间：2001-12-13(美国),美丽心灵
"主演：蒂姆·罗宾斯,摩根·弗里曼,鲍勃·冈顿",https://www.maoyan.com/films/1297,上映时间：1994-09-10(加拿大),肖申克的救赎
"主演：罗伯托·贝尼尼,尼可莱塔·布拉斯基,乔治·坎塔里尼",https://www.maoyan.com/films/1303,上映时间：2020-01-03,美丽人生
"主演：张国荣,王祖贤,午马",https://www.maoyan.com/films/5430,上映时间：2011-04-30,倩女幽魂
"主演：爱德华·哈里森·诺顿,布拉德·皮特,海伦娜·伯翰·卡特",https://www.maoyan.com/films/1291,上映时间：1999-09-10(意大利),搏击俱乐部
"主演：张国荣,梁朝伟,张震",https://www.maoyan.com/films/1597,上映时间：1997-05-17(法国),春光乍泄
"主演：蒂姆·罗斯,比尔·努恩,克兰伦斯·威廉姆斯三世",https://www.maoyan.com/films/1292,上映时间：2019-11-15,海上钢琴师
"主演：河正宇,李璟荣,李大卫",https://www.maoyan.com/films/247791,上映时间：2013-07-31(韩国),恐怖直播
"主演：仲代达矢,春川真澄,井川比佐志",https://www.maoyan.com/films/25310,上映时间：1987-08-01(日本),忠犬八公物语
"主演：宋康昊,郭度沅,吴达洙",https://www.maoyan.com/films/246433,上映时间：2013-12-18(韩国),辩护人
"主演：周星驰,莫文蔚,张柏芝",https://www.maoyan.com/films/9025,上映时间：1999-02-13(中国香港),喜剧之王
"主演：克林特·伊斯特伍德,李·范·克里夫,埃里·瓦拉赫",https://www.maoyan.com/films/14124,上映时间：1966-12-23(意大利),黄金三镖客
"主演：让·雷诺,加里·奥德曼,娜塔莉·波特曼",https://www.maoyan.com/films/4055,上映时间：1994-09-14(法国),这个杀手不太冷
"主演：志田未来,神木隆之介,大竹忍",https://www.maoyan.com/films/57212,上映时间：2010-07-17(日本),借东西的小人阿莉埃蒂
"主演：周星驰,莫文蔚,吴孟达",https://www.maoyan.com/films/6823,上映时间：2014-10-24,大话西游之月光宝盒
"主演：吴念真,金燕玲,李凯莉",https://www.maoyan.com/films/1383,上映时间：2000-05-15(法国),一一
"主演：乌尔里希·穆埃,塞巴斯蒂安·科赫,马蒂娜·格德克",https://www.maoyan.com/films/23127,上映时间：2006-03-23(德国),窃听风暴
"主演：宋在浩,李顺才,尹秀晶",https://www.maoyan.com/films/67163,上映时间：2011-02-17(韩国),我爱你
"主演：邱岳峰,毕克,富润生",https://www.maoyan.com/films/14556,上映时间：1965-12-31,大闹天宫
"主演：约翰尼·德普,薇诺娜·瑞德,黛安娜·威斯特",https://www.maoyan.com/films/513,上映时间：1990-12-06(美国),剪刀手爱德华
"主演：黎明,张曼玉,曾志伟",https://www.maoyan.com/films/784,上映时间：2015-02-13,甜蜜蜜
"主演：阿尔·帕西诺,克里斯·奥唐纳,加布里埃尔·安瓦尔",https://www.maoyan.com/films/6289,上映时间：1992-12-23(美国),闻香识女人
"主演：狄龙,张国荣,周润发",https://www.maoyan.com/films/313,上映时间：2017-11-17,英雄本色
"主演：阿米尔·汗,黄渤,卡琳娜·卡普",https://www.maoyan.com/films/43,上映时间：2011-12-08,三傻大闹宝莱坞
"主演：基努·里维斯,雨果·维文,凯瑞-安·莫斯",https://www.maoyan.com/films/9053,上映时间：2003-11-05,黑客帝国3：矩阵革命
"主演：弗朗索瓦·克鲁塞,奥玛·希,安娜·勒尼",https://www.maoyan.com/films/70976,上映时间：2011-11-02(法国),触不可及
"主演：金·凯瑞,劳拉·琳妮,诺亚·艾默里奇 Noah Emmerich",https://www.maoyan.com/films/1304,上映时间：1998(罗马尼亚),楚门的世界
"主演：苏拉·沙玛,伊尔凡·可汗,塔布",https://www.maoyan.com/films/995,上映时间：2012-11-22,少年派的奇幻漂流
"主演：希斯·莱杰,杰克·吉伦哈尔,米歇尔·威廉姆斯",https://www.maoyan.com/films/14652,上映时间：2005-09-02(意大利),断背山
"主演：格利高里·派克,奥黛丽·赫本,埃迪·艾伯特",https://www.maoyan.com/films/2641,上映时间：1953-08-20(意大利),罗马假日
"主演：汤姆·克鲁斯,布拉德·皮特,克尔斯滕·邓斯特",https://www.maoyan.com/films/6771,上映时间：1994-11-11(美国),夜访吸血鬼
"主演：菲利浦·诺瓦雷,赛尔乔·卡斯特利托,蒂兹亚娜·罗达托",https://www.maoyan.com/films/718,上映时间：1988-11-17(意大利),天堂电影院
"主演：玛德琳·卡罗尔,卡兰·麦克奥利菲,艾丹·奎因",https://www.maoyan.com/films/46818,上映时间：2010-07-26(美国),怦然心动
"主演：费雯·丽,克拉克·盖博,奥利维娅·德哈维兰",https://www.maoyan.com/films/7431,上映时间：1939-12-15(美国),乱世佳人
"主演：凯文·科斯特纳,克林特·伊斯特伍德,T·J·劳瑟",https://www.maoyan.com/films/7971,上映时间：1993-11-24(美国),完美的世界
"主演：张国荣,张丰毅,巩俐",https://www.maoyan.com/films/1203,上映时间：1993-07-26,霸王别姬
"主演：三船敏郎,志村乔,千秋实",https://www.maoyan.com/films/3867,上映时间：1954-04-26(日本),七武士
"主演：亨利·方达,李·科布,马丁·鲍尔萨姆",https://www.maoyan.com/films/2049,上映时间：1957-04-13(美国),十二怒汉
"主演：李来,薛耿求,严志媛",https://www.maoyan.com/films/78538,上映时间：2013-10-02(韩国),素媛
"主演：周星驰,朱茵,莫文蔚",https://www.maoyan.com/films/158,上映时间：2014-10-24,大话西游之大圣娶亲
"主演：马龙·白兰度,阿尔·帕西诺,詹姆斯·肯恩",https://www.maoyan.com/films/1247,上映时间：2015-04-18,教父
"主演：雅克·贝汉,姜文,兰斯洛特·佩林",https://www.maoyan.com/films/168,上映时间：2011-08-12,海洋
"主演：基努·里维斯,凯瑞-安·莫斯,劳伦斯·菲什伯恩",https://www.maoyan.com/films/1249,上映时间：2000-01-14,黑客帝国
"主演：姜文,姜宏波,陈强",https://www.maoyan.com/films/1262,上映时间：2000-05-13(法国),鬼子来了
"主演：丹尼尔·雷德克里夫,鲁伯特·格林特,艾玛·沃特森",https://www.maoyan.com/films/916,上映时间：2011-08-04,哈利·波特与死亡圣器（下）
"主演：连姆·尼森,拉尔夫·费因斯,本·金斯利",https://www.maoyan.com/films/3667,上映时间：1993-11-30(美国),辛德勒的名单
"主演：伊莱贾·伍德,伊恩·麦克莱恩,丽芙·泰勒",https://www.maoyan.com/films/428,上映时间：2004-03-15,指环王3：王者无敌
"主演：Forest,理查·基尔,琼·艾伦",https://www.maoyan.com/films/38286,上映时间：2009-06-13(美国),忠犬八公的故事
"主演：里克·奥巴瑞,路易·西霍尤斯,哈迪·琼斯",https://www.maoyan.com/films/48465,上映时间：2009-07-31(美国),海豚湾
"主演：汤姆·汉克斯,马特·达蒙,汤姆·塞兹摩尔",https://www.maoyan.com/films/733,上映时间：1998-11-13,拯救大兵瑞恩
"主演：本·贝尔特,艾丽莎·奈特,杰夫·格尔林",https://www.maoyan.com/films/488,上映时间：2008-06-27(美国),机器人总动员
"主演：史蒂夫·卡瑞尔,杰森·席格尔,拉塞尔·布兰德",https://www.maoyan.com/films/491,上映时间：2010-06-20(俄罗斯),神偷奶爸
"主演：热拉尔·朱尼奥,让-巴蒂斯特·莫尼耶,玛丽·布奈尔",https://www.maoyan.com/films/1205,上映时间：2004-10-16,放牛班的春天
"主演：孔刘,郑有美,金智英",https://www.maoyan.com/films/67214,上映时间：2011-09-22(韩国),熔炉
"主演：萨姆·沃辛顿,佐伊·索尔达娜,米歇尔·罗德里格兹",https://www.maoyan.com/films/243,上映时间：2010-01-04,阿凡达
"主演：柊瑠美,周冬雨,入野自由",https://www.maoyan.com/films/1212,上映时间：2019-06-21,千与千寻
"主演：约翰·C·赖利,萨拉·西尔弗曼,简·林奇",https://www.maoyan.com/films/965,上映时间：2012-11-06,无敌破坏王
"主演：柳承龙,郑镇荣,朴信惠",https://www.maoyan.com/films/249294,上映时间：2013-01-23(韩国),7号房的礼物
"主演：莱昂纳多·迪卡普里奥,渡边谦,约瑟夫·高登-莱维特",https://www.maoyan.com/films/416,上映时间：2010-09-01,盗梦空间
"主演：约翰尼·德普,凯拉·奈特莉,奥兰多·布鲁姆",https://www.maoyan.com/films/5880,上映时间：2003-11-21,加勒比海盗
"主演：威尔·史密斯,贾登·史密斯,坦迪·牛顿",https://www.maoyan.com/films/899,上映时间：2008-01-17,当幸福来敲门
"主演：阿沙·巴特菲尔德,维拉·法梅加,大卫·休里斯",https://www.maoyan.com/films/38099,上映时间：2008-08-28(英国),穿条纹睡衣的男孩
"主演：朱莉·安德鲁斯,克里斯托弗·普卢默,埃琳诺·帕克",https://www.maoyan.com/films/3103,上映时间：1965-03-02(美国),音乐之声
"主演：刘德华,梁朝伟,黄秋生",https://www.maoyan.com/films/224973,上映时间：2003-09-05,无间道
"主演：休·杰克曼,克里斯蒂安·贝尔,迈克尔·凯恩",https://www.maoyan.com/films/20131,上映时间：2006-10-17(意大利),致命魔术
"主演：默罕默德·阿米尔·纳吉,Kamal Mirkarimi,Behzad Rafi",https://www.maoyan.com/films/9426,上映时间：1997(伊朗),小鞋子
"主演：内山昂辉,佐仓绫音,后藤弘树",https://www.maoyan.com/films/68213,上映时间：2011-09-17(日本),萤火之森
"主演：秦岚,糸井重里,岛本须美",https://www.maoyan.com/films/123,上映时间：2018-12-14,龙猫
"主演：布拉德·皮特,凯特·布兰切特,塔拉吉·P·汉森",https://www.maoyan.com/films/18318,上映时间：2008-12-25(美国),本杰明·巴顿奇事
"主演：伊莱贾·伍德,伊恩·麦克莱恩,丽芙·泰勒",https://www.maoyan.com/films/1221,上映时间：2003-04-25,指环王2：双塔奇兵
"主演：伊莱贾·伍德,伊恩·麦克莱恩,丽芙·泰勒",https://www.maoyan.com/films/1220,上映时间：2002-04-04,指环王1：护戒使者
"主演：瑞秋·麦克亚当斯,多姆纳尔·格里森,比尔·奈伊",https://www.maoyan.com/films/78312,上映时间：2013-09-04(英国),时空恋旅人
"主演：尊龙,陈冲,彼得·奥图尔",https://www.maoyan.com/films/2039,上映时间：1987-10-23,末代皇帝
"主演：寺田农,鹫尾真知子,龟山助清",https://www.maoyan.com/films/1228,上映时间：1992-05-01,天空之城
"主演：岛本须美,永井一郎,坂本千夏",https://www.maoyan.com/films/1230,上映时间：1992-05,风之谷
"主演：松田洋治,石田百合子,田中裕子",https://www.maoyan.com/films/5357,上映时间：1998-05-01,幽灵公主
"主演：克里斯蒂安·贝尔,迈克尔·凯恩,加里·奥德曼",https://www.maoyan.com/films/676,上映时间：2012-08-27,蝙蝠侠：黑暗骑士崛起