Scrapy某网站歌曲下载代码

Spider代码

import re

import scrapy

from ..items import VocalItem


class tttSpider(scrapy.Spider):
    name = 'ttt'
    # allowed_domains = ['cc.com']
    start_urls = ['http://www.333ttt.com/up/top158.html']

    def access_mp3(self, response):
        mp3_url = response.url
        print('access_mp3', '------------------', mp3_url, '------------------')

        item = response.meta['item']
        item['mp3_url'] = mp3_url
        yield item

    def parse(self, response):
        li_list = response.xpath('/html/body/div[3]/div[1]/div[3]/div[1]/ul/li')
        for li in li_list:
            title = li.xpath('./@data-title').extract_first()
            href = li.xpath('./span[2]/a/@href').extract_first()
            href = response.urljoin(href)
            rid = re.findall('yy(.*?).html', href)[0]

            item = VocalItem()
            item['title'] = title
            # item['rid'] = rid
            # http://mp.111ttt.cn/mp3free/154145603.mp3
            detail_url = 'http://mp.111ttt.cn/mp3free/{}.mp3'.format(rid)
            print(title, rid, detail_url)
            yield scrapy.Request(url=detail_url, meta={'item': item}, callback=self.access_mp3)
            # break

管道代码

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


import scrapy
# useful for handling different item types with a single interface
from scrapy.pipelines.files import FilesPipeline


# class VocalPipeline:
#     def process_item(self, item, spider):
#         return item
class mp3Pipeline(FilesPipeline):
    def get_media_requests(self, item, info):
        # print(item['href'], '0000000000', item['title'])
        # http://mp.111ttt.cn/mp3music/151728901.mp3 0000000000 迷失幻境
        # http://gd.sycdn.kuwo.cn/0e434f4551351eacba465f4d3b9025b0/5fa76f14/resource/n1/35/58/1836475472.mp3
        yield scrapy.Request(item['mp3_url'], meta={'title': item['title']})

    def item_completed(self, results, item, info):
        return item

    def file_path(self, request, response=None, info=None):
        name = request.meta.get('title') + '.mp3'
        print('~~~~~~~~~~~~~~~~~~~~~~', name, '~~~~~~~~~~~~~~~~~~~~~~')
        return name