Python爬虫入门:猫眼top100电影名,主演 和 上映时间
初次学习Python爬虫按照教程编写的爬虫脚本
运行后,输入"y",会继续爬取下一页.可以略作修改直接爬取10页100条.
仅供与学习参考
代码如下
# -*- coding: utf-8 -*-
"""
Created on Tue Sep 10 11:03:26 2019
@author: 123
"""
import urllib.request
import urllib.parse
import re
import csv
class MaoyanSpider:
def __init__(self):
self.baseurl = "https://maoyan.com/board/4?offset="
self.headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"}
self.page = 1
self.offset = 0
#下载页面
def loadPage(self,url):
req = urllib.request.Request(url,headers=self.headers)
res = urllib.request.urlopen(req)
html = res.read().decode("utf-8")
self.parsePage(html)
#解析页面
def parsePage(self,html):
p = re.compile('<div class="movie-item-info">.*?title="(.*?)".*?<p class="star">(.*?)</p>.*?上映时间:(.*?)</p>',re.S)
r_list = p.findall(html)
self.writePage(r_list)
#保存页面
def writePage(self,r_list):
print(r_list)
for r_tuple in r_list:
with open("猫眼电影.csv","a",newline="") as f:
#创建写入对象
writer = csv.writer(f)
# L = list(r_tuple)
L = [r_tuple[0].strip(),r_tuple[1].strip(),r_tuple[2].strip()]
writer.writerow(L)
def workOn(self):
with open("猫眼电影.csv","a",newline="") as f:
writer = csv.writer(f)
writer.writerow(["电影名称","主演","上映时间"])
while True:
c = input("爬取请按y(y/n):")
if c.strip().lower() == "y":
self.offset = (self.page-1)*10
url = self.baseurl+str(self.offset)
self.loadPage(url)
self.page += 1
else:
print("爬取结束")
break
if __name__ == "__main__":
spider = MaoyanSpider()
spider.workOn()