完成关于pipeline.py、setting.py以及middleware.py的设计后,便初步完成了spider.py的设计
主要分为三个部分
一.初始化数据
def __init__(self, keywords='vtuber', video_time_long="1000", video_time_short="0", task_id=2,
startDate=int(time.time()) - 3600 * 48 *7, endDate=int(time.time()), *args, **kwargs):
super(YoutubeSpider, self).__init__(*args, **kwargs)
self.keywords = keywords
self.task_id = task_id
self.video_time_long = video_time_long
self.video_time_short = video_time_short
self.upload_time_start_date = startDate
self.upload_time_end_date = endDate
self.allowed_domains = ['www.youtube.com']
self.url1 = 'http://www.youtube.com/results?sp=CAJCBAgBEgA%253D&search_query=' + self.keywords + '&p