练习 淘宝信息

# -*- coding: utf-8 -*-
import scrapy
import re
from hw_project.items import TaobaoItem

class TaobaoSpider(scrapy.Spider):
    name = 'taobao'
    allowed_domains = ['taobao.com']
    start_urls = ['https://s.taobao.com/list?spm=a21fg.7938700.220511.15.34ae17ddjDg0o0&seller_type=taobao&seller_type=taobao&style=grid&style=grid&mid=869&mid=869&cat=35%2C50006004%2C50067081%2C50008165%2C54164002%2C50005998%2C56732005&q=%E5%AD%95%E5%A6%87%E4%B8%8A%E8%A3%85%E7%A7%8B%E5%86%AC&suggest=0_1&_input_charset=utf-8&wq=%E5%AD%95%E5%A6%87%E4%B8%8A%E8%A3%85&suggest_query=%E5%AD%95%E5%A6%87%E4%B8%8A%E8%A3%85&source=suggest&auction_tag%5B%5D=102978&bcoffset=12&s=60']

    def parse(self, response):
        base_url = 'https://s.taobao.com/list?spm=a21fg.7938700.220511.15.34ae17ddjDg0o0&seller_type=taobao&seller_type=taobao&style=grid&style=grid&mid=869&mid=869&cat=35%2C50006004%2C50067081%2C50008165%2C54164002%2C50005998%2C56732005&q=%E5%AD%95%E5%A6%87%E4%B8%8A%E8%A3%85%E7%A7%8B%E5%86%AC&suggest=0_1&_input_charset=utf-8&wq=%E5%AD%95%E5%A6%87%E4%B8%8A%E8%A3%85&suggest_query=%E5%AD%95%E5%A6%87%E4%B8%8A%E8%A3%85&source=suggest&auction_tag%5B%5D=102978&bcoffset=12&s={}'
        for i in range(1,2):
            full_url =  base_url.format(i*60)
            print(full_url)
            yield scrapy.Request(full_url,callback=self.detail)

    def detail(self,response):
        item = TaobaoItem()
        html = response.text
        title = re.findall(r'"raw_title":"(.*?)"', html)
        sales = re.findall(r'"view_sales":"(.*?)"', html)
        loc = re.findall(r',"item_loc":"(.*?)"', html)
        price = re.findall(r',"view_price":"(.*?)"', html)
        for i in range(0, len(title)):
            item['title'] = title[i]
            item['sales'] = sales[i]
            item['loc'] = loc[i]
            item['price'] = price[i]
            yield item