demoSpider.py
# !/usr/bin/env python
# -*- coding: utf-8 -*-
import scrapy
from tutorial.items import DmozItem
import os
import requests
class DmozSpider(scrapy.Spider):
name = "dmoz"
allowed_domains = ["doutula.com"]
start_urls = ["https://www.doutula.com/article/list/?page={}".format(i) for i in range(1,10)]
count = 0
def parse(self, response):
for sel in response.xpath('//*[@id="home"]/div/div[1]'):
item = DmozItem()
item['link'] = sel.xpath('a/@href').extract()
for i in range(len(item['link'])):
yield scrapy.Request(item['link'][i], callback=self.parse_img)
def parse_img(self, response):
for sell in response.xpath('//div[@class="col-sm-9"]/li'):
item = DmozItem()
item['title'] = sell.xpath('div[@class="pic-title"]/h1/a/text()').extract()[0]
item['img'] = sell.xpath('div[@class="pic-content"]//div[@class="artile_des"]/table/tbody/tr[1]/td/a/img/@alt').extract()
item['url'] = sell.xpath('div[@class="pic-content"]/div[@class="artile_des"]/table/tbody/tr[1]/td/a/img/@src').extract()
# print(item['title'])
if not os.path.exists("D:\JetBrains\PyCharm 2017.2\PycharmProjects\Scrapy\\tutorial\\tutorial\spiders\down\{}".format(item['title'])):
os.makedirs("D:\JetBrains\PyCharm 2017.2\PycharmProjects\Scrapy\\tutorial\\tutorial\spiders\down\{}".format(item['title']))
else:
print("存在重复文件夹 {}".format(item['title']))
for i in range(len(item['img'])):
filename = "down\{}".format(item['title'])+"\\{}".format(item['img'][i])+item['url'][i][-4:]
print(filename,"==============================")
if not os.path.exists(filename):
try:
img_req = requests.get(item['url'][i])
except requests.exceptions.MissingSchema:
new_url = 'http:'+item['url'][i]
img_req = requests.get(new_url)
with open(filename,'wb') as f:
f.write(img_req.content)
else:
print("图片 [%s] 已存在!"%item['img'][i],"==========================")
self.count += 1
print("共计完成%s次下载"%self.count)