现在猫眼加了反爬虫了,所以加了headers。
试了一下只加User-Agent会出现验证码的问题,都拷过来加上,没事了。
试了一下注释掉cookie,发现也是可以的。
初次学习爬虫,太累了。
import json
import re
from multiprocessing import freeze_support
from multiprocessing.pool import Pool
import requests
from requests.exceptions import RequestException
def get_one_page(url,headers):
try:
res=requests.get(url,headers=headers)
if res.status_code==200:
return res.text
return None
except RequestException:
return None
def parse_one_page(html):
# patten=re.compile("<dd>.*?")
patten=re.compile('<dd>.*?<a href="(.*?)".*?title="(.*?)".*?<p class="star">(.*?)</p>.*?上映时间:(.*?)</p>.*?</dd>',re.S)
results=re.findall(patten,html)
for result in results:
href,title,star,time=result
star=re.sub('\s',"",star)
print(href,title,star,time)
yield {
'href':result[0],
'title':result[1],
'star':result[2].strip(),
'time':result[3]
}
def write_file(item):
with open('results.txt','a',encoding='utf-8') as f:
f.write(json.dumps(item,ensure_ascii=False)+"\n")
def main(offset):
url="https://maoyan.com/board/4?offset="+str(offset)
headers={
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36'
}
headers={
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Accept-Encoding':'gzip, deflate, br',
'Accept-Language':'zh-CN,zh;q=0.9',
'Cache-Control':'max-age=0',
'Connection':'keep-alive',
# 'Cookie':'__mta=53834519.1583065701340.1583108060211.1583108060676.20; uuid_n_v=v1; uuid=22053C605BB811EA9F0985C24ED856C3328A984601984D769ABA5F0CB37F9375; _csrf=ea175c7bf99c2857a14ab1f681c5b2ecda89ad7d289d75398ec75e3be3fced4e; Hm_lvt_703e94591e87be68cc8da0da7cbd0be2=1583065701; _lx_utm=utm_source%3DBaidu%26utm_medium%3Dorganic; _lxsdk_cuid=17096119979c8-002e7da7eaa60d-4313f6b-144000-1709611997ac8; mojo-uuid=dd0ebe64e78f7063615ea02b6a1e0aea; __mta=53834519.1583065701340.1583065822531.1583065825533.7; _lxsdk=22053C605BB811EA9F0985C24ED856C3328A984601984D769ABA5F0CB37F9375; Hm_lpvt_703e94591e87be68cc8da0da7cbd0be2=1583108061; _lxsdk_s=17098f889a5-f18-5b8-a84%7C%7C2',
'Host':'maoyan.com',
'Sec-Fetch-Dest':'document',
'Sec-Fetch-Mode':'navigate',
'Sec-Fetch-Site':'none',
'Sec-Fetch-User':'?1',
'Upgrade-Insecure-Requests':'1',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36',
}
html=get_one_page(url,headers)
# html=testRE.html
# print(html)
for item in parse_one_page(html):
write_file(item,)
if __name__ == '__main__':
freeze_support() #加上还是缺少三行
pool=Pool(3)
pool.map(main,[i*10 for i in range(10)]) #多进程缺少行,还有半行和乱码
pool.close()
pool.join()
'''
for i in range(0,100,10): #单进程执行100行,结果正确
main(i)
'''
***下面是爬出的结果:***斜体样式
{“href”: “/films/3103”, “title”: “音乐之声”, “star”: “主演:朱莉·安德鲁斯,克里斯托弗·普卢默,埃琳诺·帕克”, “time”: “1965-03-02(美国)”}
{“href”: “/films/1297”, “title”: “肖申克的救赎”, “star”: “主演:蒂姆·罗宾斯,摩根·弗里曼,鲍勃·冈顿”, “time”: “1994-09-10(加拿大)”}
{“href”: “/films/4055”, “title”: “这个杀手不太冷”, “star”: “主演:让·雷诺,加里·奥德曼,娜塔莉·波特曼”, “time”: “1994-09-14(法国)”}
{“href”: “/films/2641”, “title”: “罗马假日”, “star”: “主演:格利高里·派克,奥黛丽·赫本,埃迪·艾伯特”, “time”: “1953-08-20(意大利)”}
{“href”: “/films/267”, “title”: “泰坦尼克号”, “star”: “主演:莱昂纳多·迪卡普里奥,凯特·温丝莱特,比利·赞恩”, “time”: “1998-04-03”}
{“href”: “/films/1597”, “title”: “春光乍泄”, “star”: “主演:张国荣,梁朝伟,张震”, “time”: “1997-05-17(法国)”}
{“href”: “/films/7431”, “title”: “乱世佳人”, “star”: “主演:费雯·丽,克拉克·盖博,奥利维娅·德哈维兰”, “time”: “1939-12-15(美国)”}
{“href”: “/films/513”, “title”: “剪刀手爱德华”, “star”: “主演:约翰尼·德普,薇诺娜·瑞德,黛安·韦斯特”, “time”: “1990-12-06(美国)”}
{“href”: “/films/837”, “title”: “唐伯虎点秋香”, “star”: “主演:周星驰,巩俐,郑佩佩”, “time”: “1993-07-01(中国香港)”}
{“href”: “/films/2760”, “title”: “魂断蓝桥”, “star”: “主演:费雯·丽,罗伯特·泰勒,露塞尔·沃特森”, “time”: “1940-05-17(美国)”}
{“href”: “/films/1249”, “title”: “黑客帝国”, “star”: “主演:基努·里维斯,凯瑞-安·莫斯,劳伦斯·菲什伯恩”, “time”: “2000-01-14”}
{“href”: “/films/3667”, “title”: “辛德勒的名单”, “star”: “主演:连姆·尼森,拉尔夫·费因斯,本·金斯利”, “time”: “1993-11-30(美国)”}
{“href”: “/films/428”, “title”: “指环王3:王者无敌”, “star”: “主演:伊莱贾·伍德,伊恩·麦克莱恩,丽芙·泰勒”, “time”: “2004-03-15”}
{“href”: “/films/9025”, “title”: “喜剧之王”, “star”: “主演:周星驰,莫文蔚,张柏芝”, “time”: “1999-02-13(中国香港)”}
{“href”: “/films/3606”, “title”: “哈利·波特与魔法石”, “star”: “主演:丹尼尔·雷德克里夫,鲁伯特·格林特,艾玛·沃特森”, “time”: “2002-01-26”}
{“href”: “/films/5880”, “title”: “加勒比海盗”, “star”: “主演:约翰尼·德普,凯拉·奈特莉,奥兰多·布鲁姆”, “time”: “2003-11-21”}
{“href”: “/films/1304”, “title”: “楚门的世界”, “star”: “主演:金·凯瑞,劳拉·琳妮,诺亚·艾默里奇”, “time”: “1998(罗马尼亚)”}
{“href”: “/films/6641”, “title”: “教父2”, “star”: “主演:阿尔·帕西诺,罗伯特·德尼罗,黛安·基顿”, “time”: “1974-12-12(美国)”}
{“href”: “/films/224973”, “title”: “无间道”, “star”: “主演:刘德华,梁朝伟,黄秋生”, “time”: “2003-09-05”}
{“href”: “/films/790”, “title”: “蝙蝠侠:黑暗骑士”, “star”: “主演:克里斯蒂安·贝尔,希斯·莱杰,阿伦·伊克哈特”, “time”: “2008-07-14(阿根廷)”}
{“href”: “/films/13824”, “title”: “射雕英雄传之东成西就”, “star”: “主演:张国荣,梁朝伟,张学友”, “time”: “1993-02-05(中国香港)”}
{“href”: “/films/1220”, “title”: “指环王1:护戒使者”, “star”: “主演:伊莱贾·伍德,伊恩·麦克莱恩,丽芙·泰勒”, “time”: “2002-04-04”}
{“href”: “/films/1221”, “title”: “指环王2:双塔奇兵”, “star”: “主演:伊莱贾·伍德,伊恩·麦克莱恩,丽芙·泰勒”, “time”: “2003-04-25”}
{“href”: “/films/488”, “title”: “机器人总动员”, “star”: “主演:本·贝尔特,艾丽莎·奈特,杰夫·格尔林”, “time”: “2008-06-27(美国)”}
{“href”: “/films/718”, “title”: “天堂电影院”, “star”: “主演:菲利浦·诺瓦雷,赛尔乔·卡斯特利托,蒂兹亚娜·罗达托”, “time”: “1988-11-17(意大利)”}
{“href”: “/films/592”, “title”: “狮子王”, “star”: “主演:马修·布罗德里克,尼基塔·卡兰姆,詹姆斯·厄尔·琼斯”, “time”: “1995-07-15”}
{“href”: “/films/1375”, “title”: “活着”, “star”: “主演:葛优,巩俐,牛犇”, “time”: “1994-05-17(法国)”}
{“href”: “/films/899”, “title”: “当幸福来敲门”, “star”: “主演:威尔·史密斯,贾登·史密斯,坦迪·牛顿”, “time”: “2008-01-17”}
{“href”: “/films/1205”, “title”: “放牛班的春天”, “star”: “主演:热拉尔·朱尼奥,让-巴蒂斯特·莫尼耶,玛丽·布奈尔”, “time”: “2004-10-16”}
{“href”: “/films/6823”, “title”: “大话西游之月光宝盒”, “star”: “主演:周星驰,莫文蔚,吴孟达”, “time”: “2014-10-24”}
{“href”: “/films/1230”, “title”: “风之谷”, “star”: “主演:岛本须美,永井一郎,坂本千夏”, “time”: “1992-05”}
{“href”: “/films/2039”, “title”: “末代皇帝”, “star”: “主演:尊龙,陈冲,彼得·奥图尔”, “time”: “1987-10-23”}
{“href”: “/films/6289”, “title”: “闻香识女人”, “star”: “主演:阿尔·帕西诺,克里斯·奥唐纳,加布里埃尔·安瓦尔”, “time”: “1992-12-23(美国)”}
{“href”: “/films/43”, “title”: “三傻大闹宝莱坞”, “star”: “主演:阿米尔·汗,黄渤,卡琳娜·卡普”, “time”: “2011-12-08”}
{“href”: “/films/3294”, “title”: “勇敢的心”, “star”: “主演:梅尔·吉布森,苏菲·玛索,帕特里克·麦高汉”, “time”: “1995-05-18(美国)”}
{“href”: “/films/364”, “title”: “速度与激情5”, “star”: “主演:范·迪塞尔,保罗·沃克,道恩·强森”, “time”: “2011-05-12”}
{“href”: “/films/995”, “title”: “少年派的奇幻漂流”, “star”: “主演:苏拉·沙玛,伊尔凡·可汗,塔布”, “time”: “2012-11-22”}
{“href”: “/films/9053”, “title”: “黑客帝国3:矩阵革命”, “star”: “主演:基努·里维斯,雨果·维文,凯瑞-安·莫斯”, “time”: “2003-11-05”}
{“href”: “/films/614”, “title”: “飞屋环游记”, “star”: “主演:爱德华·阿斯纳,乔丹·长井,鲍勃·彼德森”, “time”: “2009-08-04”}
{“href”: “/films/330”, “title”: “驯龙高手”, “star”: “主演:杰伊·巴鲁切尔,杰拉德·巴特勒,亚美莉卡·费雷拉”, “time”: “2010-05-14”}
{“href”: “/films/46818”, “title”: “怦然心动”, “star”: “主演:玛德琳·卡罗尔,卡兰·麦克奥利菲,艾丹·奎因”, “time”: “2010-07-26(美国)”}
{“href”: “/films/14652”, “title”: “断背山”, “star”: “主演:希斯·莱杰,杰克·吉伦哈尔,米歇尔·威廉姆斯”, “time”: “2005-09-02(意大利)”}
{“href”: “/films/491”, “title”: “神偷奶爸”, “star”: “主演:史蒂夫·卡瑞尔,杰森·席格尔,拉塞尔·布兰德”, “time”: “2010-06-20(俄罗斯)”}
{“href”: “/films/20131”, “title”: “致命魔术”, “star”: “主演:休·杰克曼,克里斯蒂安·贝尔,迈克尔·凯恩”, “time”: “2006-10-17(意大利)”}
{“href”: “/films/965”, “title”: “无敌破坏王”, “star”: “主演:约翰·C·赖利,萨拉·西尔弗曼,简·林奇”, “time”: “2012-11-06”}
{“href”: “/films/1262”, “title”: “鬼子来了”, “star”: “主演:姜文,姜宏波,陈强”, “time”: “2000-05-13(法国)”}
{“href”: “/films/733”, “title”: “拯救大兵瑞恩”, “star”: “主演:汤姆·汉克斯,马特·达蒙,汤姆·塞兹摩尔”, “time”: “1998-11-13”}
{“href”: “/films/11473”, “title”: “美丽心灵”, “star”: “主演:罗素·克洛,詹妮弗·康纳利,艾德·哈里斯”, “time”: “2001-12-13(美国)”}
{“href”: “/films/416”, “title”: “盗梦空间”, “star”: “主演:莱昂纳多·迪卡普里奥,渡边谦,约瑟夫·高登-莱维特”, “time”: “2010-09-01”}
{“href”: “/films/225980”, “title”: “哈尔的移动城堡”, “star”: “主演:倍赏千惠子,木村拓哉,美轮明宏”, “time”: “2004-09-05(意大利)”}
{“href”: “/films/243”, “title”: “阿凡达”, “star”: “主演:萨姆·沃辛顿,佐伊·索尔达娜,米歇尔·罗德里格兹”, “time”: “2010-01-04”}
{“href”: “/films/78099”, “title”: “疯狂原始人”, “star”: “主演:尼古拉斯·凯奇,艾玛·斯通,瑞安·雷诺兹”, “time”: “2013-04-20”}
{“href”: “/films/1291”, “title”: “搏击俱乐部”, “star”: “主演:爱德华·哈里森·诺顿,布拉德·皮特,海伦娜·伯翰·卡特”, “time”: “1999-09-10(意大利)”}
{“href”: “/films/5357”, “title”: “幽灵公主”, “star”: “主演:松田洋治,石田百合子,田中裕子”, “time”: “1998-05-01”}
{“href”: “/films/2049”, “title”: “十二怒汉”, “star”: “主演:亨利·方达,李·科布,马丁·鲍尔萨姆”, “time”: “1957-04-13(美国)”}
{“href”: “/films/13636”, “title”: “V字仇杀队”, “star”: “主演:娜塔莉·波特曼,雨果·维文,斯蒂芬·瑞”, “time”: “2005-12-11(美国)”}
{“href”: “/films/676”, “title”: “蝙蝠侠:黑暗骑士崛起”, “star”: “主演:克里斯蒂安·贝尔,迈克尔·凯恩,加里·奥德曼”, “time”: “2012-08-27”}
{“href”: “/films/784”, “title”: “甜蜜蜜”, “star”: “主演:黎明,张曼玉,曾志伟”, “time”: “2015-02-13”}
{“href”: “/films/67214”, “title”: “熔炉”, “star”: “主演:孔刘,郑有美,金智英”, “time”: “2011-09-22(韩国)”}
{“href”: “/films/247791”, “title”: “恐怖直播”, “star”: “主演:河正宇,李璟荣,李大为”, “time”: “2013-07-31(韩国)”}
{“href”: “/films/916”, “title”: “哈利·波特与死亡圣器(下)”, “star”: “主演:丹尼尔·雷德克里夫,鲁伯特·格林特,艾玛·沃特森”, “time”: “2011-08-04”}
{“href”: “/films/9426”, “title”: “小鞋子”, “star”: “主演:默罕默德·阿米尔·纳吉,Kamal Mirkarimi,Behzad Rafi”, “time”: “1997(伊朗)”}
{“href”: “/films/313”, “title”: “英雄本色”, “star”: “主演:狄龙,张国荣,周润发”, “time”: “2017-11-17”}
{“href”: “/films/5430”, “title”: “倩女幽魂”, “star”: “主演:张国荣,王祖贤,午马”, “time”: “2011-04-30”}
{“href”: “/films/78538”, “title”: “素媛”, “star”: “主演:李来,薛耿求,严志媛”, “time”: “2013-10-02(韩国)”}
{“href”: “/films/48465”, “title”: “海豚湾”, “star”: “主演:里克·奥巴瑞,路易·西霍尤斯,哈迪·琼斯”, “time”: “2009-07-31(美国)”}
{“href”: “/films/18318”, “title”: “本杰明·巴顿奇事”, “star”: “主演:布拉德·皮特,凯特·布兰切特,塔拉吉·P·汉森”, “time”: “2008-12-25(美国)”}
{“href”: “/films/68213”, “title”: “萤火之森”, “star”: “主演:内山昂辉,佐仓绫音,后藤弘树”, “time”: “2011-09-17(日本)”}
{“href”: “/films/4883”, “title”: “钢琴家”, “star”: “主演:艾德里安·布洛迪,艾米莉娅·福克斯,米哈乌·热布罗夫斯基”, “time”: “2002-05-24(法国)”}
{“href”: “/films/25310”, “title”: “忠犬八公物语”, “star”: “主演:仲代达矢,春川真澄,井川比佐志”, “time”: “1987-08-01(日本)”}
{“href”: “/films/1247”, “title”: “教父”, “star”: “主演:马龙·白兰度,阿尔·帕西诺,詹姆斯·肯恩”, “time”: “2015-04-18”}
{“href”: “/films/70976”, “title”: “触不可及”, “star”: “主演:弗朗索瓦·克鲁塞,奥玛·希,安娜·勒尼”, “time”: “2011-11-02(法国)”}
{“href”: “/films/1309”, “title”: “上帝之城”, “star”: “主演:亚历桑德雷·罗德里格斯,艾莉丝·布拉加,莱安德鲁·菲尔米诺”, “time”: “2002(俄罗斯)”}
{“href”: “/films/94”, “title”: “新龙门客栈”, “star”: “主演:张曼玉,梁家辉,甄子丹”, “time”: “2012-02-24”}
{“href”: “/films/38099”, “title”: “穿条纹睡衣的男孩”, “star”: “主演:阿沙·巴特菲尔德,维拉·法梅加,大卫·休里斯”, “time”: “2008-08-28(英国)”}
{“href”: “/films/7971”, “title”: “完美的世界”, “star”: “主演:凯文·科斯特纳,克林特·伊斯特伍德,T·J·劳瑟”, “time”: “1993-11-24(美国)”}
{“href”: “/films/158”, “title”: “大话西游之大圣娶亲”, “star”: “主演:周星驰,朱茵,莫文蔚”, “time”: “2014-10-24”}
", “time”: “2013-09-04(英国)”}
{“href”: “/films/1332”, “title”: “美国往事”, “star”: “主演:罗伯特·德尼罗,詹姆斯·伍兹,伊丽莎白·麦戈文”, “time”: “2015-04-23”}
{“href”: “/films/23127”, “title”: “窃听风暴”, “star”: “主演:乌尔里希·穆埃,塞巴斯蒂安·科赫,马蒂娜·格德克”, “time”: “2006-03-23(德国)”}
{“href”: “/films/263”, “title”: “初恋这件小事”, “star”: “主演:马里奥·毛瑞尔,平采娜·乐维瑟派布恩,阿查拉那·阿瑞亚卫考”, “time”: “2012-06-05”}
{“href”: “/films/3867”, “title”: “七武士”, “star”: “主演:三船敏郎,志村乔,千秋实”, “time”: “1954-04-26(日本)”}
{“href”: “/films/249294”, “title”: “7号房的礼物”, “star”: “主演:柳承龙,郑镇荣,朴信惠”, “time”: “2013-01-23(韩国)”}
{“href”: “/films/1383”, “title”: “一一”, “star”: “主演:吴念真,金燕玲,李凯莉”, “time”: “2000-05-15(法国)”}
{“href”: “/films/57212”, “title”: “借东西的小人阿莉埃蒂”, “star”: “主演:志田未来,神木隆之介,大竹忍”, “time”: “2010-07-17(日本)”}
{“href”: “/films/123”, “title”: “龙猫”, “star”: “主演:秦岚,糸井重里,岛本须美”, “time”: “2018-12-14”}
{“href”: “/films/6796”, “title”: “爱·回家”, “star”: “主演:俞承豪,金艺芬,童孝熙”, “time”: “2002-04-05(韩国)”}
{“href”: “/films/11237”, “title”: “阿飞正传”, “star”: “主演:张国荣,张曼玉,刘德华”, “time”: “2018-06-25”}
{“href”: “/films/67163”, “title”: “我爱你”, “star”: “主演:宋在浩,李顺才,尹秀晶”, “time”: “2011-02-17(韩国)”}
{“href”: “/films/168”, “title”: “海洋”, “star”: “主演:雅克·贝汉,姜文,兰斯洛特·佩林”, “time”: “2011-08-12”}
{“href”: “/films/14124”, “title”: “黄金三镖客”, “star”: “主演:克林特·伊斯特伍德,李·范·克里夫,埃里·瓦拉赫”, “time”: “1966-12-23(意大利)”}
{“href”: “/films/1346”, “title”: “迁徙的鸟”, “star”: “主演:雅克·贝汉,Philippe Labro”, “time”: “2001-12-12(法国)”}
{“href”: “/films/1212”, “title”: “千与千寻”, “star”: “主演:柊瑠美,周冬雨,入野自由”, “time”: “2019-06-21”}
{“href”: “/films/1292”, “title”: “海上钢琴师”, “star”: “主演:蒂姆·罗斯,比尔·努恩,克兰伦斯·威廉姆斯三世”, “time”: “2019-11-15”}
{“href”: “/films/1303”, “title”: “美丽人生”, “star”: “主演:罗伯托·贝尼尼,尼可莱塔·布拉斯基,乔治·坎塔里尼”, “time”: “2020-01-03”}