1 手机APP抓包爬虫
(1)item.py
class DouyuspiderItem(scrapy.Item):
name = scrapy.Field()# 存储照片的名字
imagesUrls = scrapy.Field()# 照片的url路径
imagesPath = scrapy.Field()# 照片保存在本地的路径
(2)spiders/douyu.py
import scrapy
import json
from douyuSpider.items import DouyuspiderItem
class DouyuSpider(scrapy.Spider):
name = "douyu"
allowd_domains = ["http://capi.douyucdn.cn"]
offset = 0
url = "http://capi.douyucdn.cn/api/v1/getVerticalRoom?limit=20&offset="
start_urls = [url + str(offset)]
def parse(self, response):
# 返回从json里获取 data段数据集合
data = json.loads(response.text)["data"]
for each in data:
item = DouyuspiderItem()
item["name"] = each["nickname"]
item["imagesUrls"] = each["vertical_src"]
yield item
self.offset += 20
yield scrapy.Request(self.url + str(self.offset), callback = self.parse)
(3)设置setting.py
ITEM_PIPELINES = {
'douyuSpider.pipelines.ImagesPipeline': 1}
# Images 的存放位置,之后会在pipelines.py里调用
IMAGES_STORE = "/Users/Power/lesson_python/douyuSpider/Images"
# user-agent
USER_AGENT = 'DYZB/2.290 (iPhone; iOS 9.3.4; Scale/2.00)'
(4)pipelines.py
import scrapy
import os
from scrapy.pipelines.images import ImagesPipeline
from scrapy.utils.project import get_project_settings
class ImagesPipeline(ImagesPipeline):
IMAGES_STORE = get_project_settings().get("IMAGES_STORE")
def get_media_requests(self, item, info):
image_url