scrapy---下载中国大学慕课课程视频及文件

scrapy—下载中国大学慕课课程视频及文件
1、本文很大程度参考了https://blog.csdn.net/qq_37244001/article/details/84780430,在此表示感谢
2、scrapy 应用如下：
settings 文件

COURSE_ID="PKU-1002536002"

#{（ａ）标清　　　（ｂ）高清　　　（ｃ）超清　}
VIDEO_TYPE="a"
#download.url
DOWNLOAD_UEL="F:\\python 爬虫\\mooc\\"

BOT_NAME = 'moocScrapy'

SPIDER_MODULES = ['moocScrapy.spiders']
NEWSPIDER_MODULE = 'moocScrapy.spiders'

ROBOTSTXT_OBEY = False
COOKIES_ENABLED = False
# URL不去重
DUPEFILTER_CLASS = 'scrapy.dupefilters.BaseDupeFilter'
DOWNLOAD_DELAY = 0.25    # 250 ms of delay
DEFAULT_REQUEST_HEADERS = {
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  'Accept-Language': 'en',
}
import datetime
Today=datetime.datetime.now()
Log_file_path='log/scrapy_{}_{}_{}.log'.format(Today.year,Today.month,Today.day)# 则在目标下增加log文件夹项目下  和scrapy.cfg同级

#Log_file_path='scrapy_{}_{}_{}.log'.format(Today.year,Today.month,Today.day)#时间为名字
LOG_LEVEL="WARNING"#级别，则高于或者等于该等级的信息就能输出到我的日志中，低于该级别的信息则输出不到我的日志信息中
LOG_FILE =Log_file_path
#mongodb.config
MONGO_URI = 'localhost'
MONGO_DB = 'mooc'


DOWNLOADER_MIDDLEWARES = {
   #'moocScrapy.middlewares.MoocscrapyDownloaderMiddleware': 543,
   'moocScrapy.middlewares.RandomUserAgent': 543,
}


ITEM_PIPELINES = {
   'moocScrapy.pipelines.MoocscrapyPipeline': 400,
   'moocScrapy.pipelines.MongoPipeline': 300,
   'moocScrapy.pipelines.CheckPipeline': 100,
}

在settings中
1、找出课件的id

COURSE_ID=“PKU-1002536002”
2、可以要求视频格式
#{（ａ）标清　　　（ｂ）高清　　　（ｃ）超清　}
VIDEO_TYPE=“a”

 3、存放地址
     DOWNLOAD_UEL="F:\\python 爬虫\\mooc\\"

items 文件

class MoocscrapyItem(scrapy.Item):
    # define the fields for your item here like:
    collection = 'course'
    course_title = scrapy.Field()
    course_collage = scrapy.Field()
    course_id = scrapy.Field()
    single_chaper = scrapy.Field()
    single_chaper_id = scrapy.Field()
    single_lesson = scrapy.Field()
    single_lesson_id = scrapy.Field()
    video_url = scrapy.Field()
    pdf_url = scrapy.Field()
    pass

pipelines 文件
1、 去重：

class CheckPipeline(object):
    """check item, and drop the duplicate one"""

    def __init__(self):
        self.names_seen = set()

    def process_item(self, item, spider):
        print(item)
        if item['single_lesson_id']:
            if item['single_lesson_id'] in self.names_seen:
                raise DropItem("Duplicate item found: %s" % item)
            else:
                self.names_seen.add(item['single_lesson_id'])
                return item
        else:
            raise DropItem("Missing price in %s" % item)

2 、储存到mongodb

class MongoPipeline(object):
    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(mongo_uri=crawler.settings.get('MONGO_URI'), mongo_db=crawler.settings.get('MONGO_DB'))

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def process_item(self, item, spider):
        self.db[item.collection].update({'single_lesson_id': item['single_lesson_id']}, dict(item), True)
        return item

3、下载pdf 和MP4文件

class MoocscrapyPipeline(object):
    def process_item(self, item, spider):
        if item["pdf_url"]:
            pdf_file = requests.get(item["pdf_url"])
            if not os.path.isdir(DOWNLOAD_UEL + 'PDFs'):
                os.mkdir(DOWNLOAD_UEL + r'PDFs')
            with open(DOWNLOAD_UEL + 'PDFs\\' + item["single_lesson"] + '.pdf', 'wb') as file:
                file.write(pdf_file.content)
        if item["video_url"]:
           try :
                mp4 = Url_mp4(url=item["video_url"],filename=item["single_lesson"]+".mp4",dir_name=DOWNLOAD_UEL)
                mp4.run()
           except Exception as e:
               print(e)
               url=replace(url=item["video_url"])
               mp4 = Url_mp4(url=url, filename=item["single_lesson"] + ".mp4", dir_name=DOWNLOAD_UEL)
               mp4.run()
        return item

spider 文件

class MoocSpider(scrapy.Spider):
    name = 'mooc'
    allowed_domains = []

    def __init__(self):
       self.course_page_url = 'http://www.icourse163.org/learn/'
       self.SOURCE_INFO_URL = 'http://www.icourse163.org/dwr/call/plaincall/CourseBean.getMocTermDto.dwr'
       self.SOURCE_RESOURCE_URL = 'http://www.icourse163.org/dwr/call/plaincall/CourseBean.getLessonUnitLearnVo.dwr'
       self.course=COURSE_ID
       function_init()




    def start_requests(self):
        yield scrapy.Request(url=self.course_page_url+self.course, callback=self.parse)

    def parse(self, response):
        '''
                  通过解析的course_id获取当前所有可下载的资源信息
        '''
        data=get_course_info(course_page=response)
        print(data)
        print(data.get("course_id"))
        # c0-param0：代表课程id
        # batchId：可以为任意时间戳
        # 其他字段为固定不变字段
        post_data = {
            'callCount': '1',
            'scriptSessionId': '${scriptSessionId}190',
            'c0-scriptName': 'CourseBean',
            'c0-methodName': 'getMocTermDto',
            'c0-id': '0',
            'c0-param0': 'number:' + data.get("course_id"),
            'c0-param1': 'number:1',
            'c0-param2': 'boolean:true',
            'batchId': '1492167717772'
        }
        yield scrapy.FormRequest( url=self.SOURCE_INFO_URL, formdata=post_data,meta=data,callback=self.get_course_all_source)


    def get_course_all_source(self,response):
        
        context=response.text.encode('utf-8').decode('unicode_escape')
        print(context)
        meta_data=response.meta
        chapter_pattern_compile = re.compile(
            r'homeworks=.*?;.+id=(\d+).*?name="(.*?)";')
        # 查找所有一级级目录id和name
        chapter_set = re.findall(chapter_pattern_compile, context)

        with open(DOWNLOAD_UEL+'TOC.txt', 'w', encoding='utf-8') as file:
            # 遍历所有一级目录id和name并写入目录
            for index, single_chaper in enumerate(chapter_set):
                file.write('%s    \n' % (single_chaper[1]))
                # 这里id为二级目录id
                lesson_pattern_compile = re.compile(
                    r'chapterId=' + single_chaper[0] +
                    r'.*?contentType=1.*?id=(\d+).+name="(.*?)".*?test')
                # 查找所有二级目录id和name
                lesson_set = re.findall(lesson_pattern_compile,
                                        context)
                # 遍历所有二级目录id和name并写入目录
                for sub_index, single_lesson in enumerate(lesson_set):
                    item=MoocscrapyItem()
                    item["course_title"]=meta_data["course_title"]
                    item["course_collage"]=meta_data["course_collage"]
                    item["course_id"]=meta_data["course_id"]
                    item["single_chaper"]=single_chaper[1]
                    item["single_chaper_id"]=single_chaper[0]
                    item["single_lesson"]=single_lesson[1]
                    item["single_lesson_id"]=single_lesson[0]
                    # item["video_url"]=
                    # item["pdf_url"]=
                    file.write('　%s    \n' % (single_lesson[1]))
                    # 查找二级目录下视频，并返回 [contentid,contenttype,id,name]<class 'list'>: [('1009008053', '1', '1006837824', '4.1-损失函数')]
                    video_pattern_compile = re.compile(
                        r'contentId=(\d+).+contentType=(1).*?id=(\d+).*?lessonId='
                        + single_lesson[0] + r'.*?name="(.+)"')
                    video_set = re.findall(video_pattern_compile,
                                           context)
                    # 查找二级目录下文档，并返回 [contentid,contenttype,id,name]
                    pdf_pattern_compile = re.compile(
                        r'contentId=(\d+).+contentType=(3).+id=(\d+).+lessonId=' +
                        single_lesson[0] + r'.+name="(.+)"')
                    pdf_set = re.findall(pdf_pattern_compile,
                                         context)
                    name_pattern_compile = re.compile(
                        r'^[第一二三四五六七八九十\d]+[\s\d\._章课节讲]*[\.\s、]\s*\d*')
                    # 遍历二级目录下视频集合，写入目录并下载
                    count_num = 0
                    param={
                        "type": 0,
                        "item":item

                    }
                    for video_index, single_video in enumerate(video_set):
                        rename = re.sub(name_pattern_compile, '', single_video[3])
                        file.write('　　[视频] %s \n' % (rename))
                        param["type"]=single_video[1]
                        post_data=self.get_content(single_video, '%d.%d.%d [视频] %s' %(index + 1, sub_index + 1, video_index + 1, rename),VIDEO_TYPE)
                        yield scrapy.FormRequest(self.SOURCE_RESOURCE_URL, formdata=post_data, meta=param,
                                                 callback=self.get_download_url)
                        count_num += 1
                    # 遍历二级目录下pdf集合，写入目录并下载
                    for pdf_index, single_pdf in enumerate(pdf_set):
                        rename = re.sub(name_pattern_compile, '', single_pdf[3])
                        file.write('　　[文档] %s \n' % (rename))
                        param["type"] = single_pdf[1]
                        post_data=self.get_content(single_pdf, '%d.%d.%d [文档] %s' %(index + 1, sub_index + 1, pdf_index + 1 + count_num, rename))
                        yield scrapy.FormRequest(self.SOURCE_RESOURCE_URL, formdata=post_data, meta=param,
                                                 callback=self.get_download_url)

   

    def get_content(self,single_content, name, *args):
        '''
        如果是文档，则直接下载
        如果是视频，则保存链接供第三方下载
        '''
        # 检查文件命名，防止网站资源有特殊字符本地无法保存
        file_pattern_compile = re.compile(r'[\\/:\*\?"<>\|]')
        name = re.sub(file_pattern_compile, '', name)
        # 检查是否有重名的（即已经下载过的）
        if os.path.exists(DOWNLOAD_UEL+'PDFs\\' + name + '.pdf'):
            print(name + "------------->已下载")
            return
        post_data = {
            'callCount': '1',
            'scriptSessionId': '${scriptSessionId}190',
            'httpSessionId': '5531d06316b34b9486a6891710115ebc',
            'c0-scriptName': 'CourseBean',
            'c0-methodName': 'getLessonUnitLearnVo',
            'c0-id': '0',
            'c0-param0': 'number:' + single_content[0],  # 二级目录id
            'c0-param1': 'number:' + single_content[1],  # 判定文件还是视频
            'c0-param2': 'number:0',
            'c0-param3': 'number:' + single_content[2],  # 具体资源id
            'batchId': '1492168138043'
        }
        return post_data
        



    def get_download_url(self,response):
        context = response.text.encode('utf-8').decode('unicode_escape')
        print(context)
        meta_data = response.meta
        item=meta_data.get("item")
        if meta_data["type"] == '1':
            try:
                if VIDEO_TYPE == 'a':
                    download_pattern_compile = re.compile(
                        r'mp4SdUrl="(.*?\.mp4).*?"')
                elif VIDEO_TYPE == "b":
                    download_pattern_compile = re.compile(
                        r'mp4HdUrl="(.*?\.mp4).*?"')
                else:
                    download_pattern_compile = re.compile(
                        r'mp4ShdUrl="(.*?\.mp4).*?"')
                video_down_url = re.search(download_pattern_compile,
                                           context).group(1)
            except AttributeError:
                print('－－－－－－－－－－－－－－－－－－－－－－－－')
                print(meta_data["item"].get("single_lesson") + '没有该清晰度格式，降级处理')
                print('－－－－－－－－－－－－－－－－－－－－－－－－')
                download_pattern_compile = re.compile(r'mp4SdUrl="(.*?\.mp4).*?"')
                video_down_url = re.search(download_pattern_compile,
                                           context).group(1)
            item["video_url"]=video_down_url
            item["pdf_url"]=None



            print('正在存储链接：' + meta_data["item"].get("single_lesson") + '.mp4')
            with open(DOWNLOAD_UEL+'Links.txt', 'a', encoding='utf-8') as file:
                file.write('%s \n' % (video_down_url))
            with open(DOWNLOAD_UEL+'Rename.bat', 'a', encoding='utf-8') as file:
                video_down_url = re.sub(r'/', '_', video_down_url)
                file.write('rename "' + re.search(
                    r'http:.*video_(.*.mp4)', video_down_url).group(1) + '" "' +
                           meta_data["item"].get("single_lesson") + '.mp4"' + '\n')
        # 如果是文档的话
        else:
            pdf_download_url = re.search(r'textOrigUrl:"(.*?)"', context).group(1)
            print('正在下载：' + meta_data["item"].get("single_lesson") + '.pdf')
            item["pdf_url"] = pdf_download_url
            item["video_url"] = None

            # pdf_file = requests.get(pdf_download_url)
            # if not os.path.isdir(DOWNLOAD_UEL+'PDFs'):
            #     os.mkdir(DOWNLOAD_UEL+r'PDFs')
            # with open(DOWNLOAD_UEL+'PDFs\\' +  meta_data["item"].get("single_lesson") + '.pdf', 'wb') as file:
            #     file.write(pdf_file.content)

        yield item

结果如下：

代码如下：
https://download.csdn.net/download/huangwencai123/11212620