scrapy 智联 mysql_scrapy 爬取智联招聘

#-*- coding: utf-8 -*-

importosimportjsonfrom tinydb importTinyDB, Queryfrom furl importfurlimportscrapyclassZhaopinspiderSpider(scrapy.Spider):

name= 'ZhaopinSpider'allowed_domains= ['www.zhaopin.com', 'sou.zhaopin.com', 'fe-api.zhaopin.com']

start_urls= ['https://www.zhaopin.com/citymap']

cache_db= TinyDB('ZhaopinSpider-cache.json') #缓存数据库

allowed_cities = ['重庆', ]#'成都', '上海', '深圳', '昆明', '杭州', '贵阳', '宁波'] ## 允许的城市

F = furl('https://fe-api.zhaopin.com/c/i/sou?pageSize=90&kt=3') #URL母版

PAGE_SIZE = 90 #分页大小

defget_city_code(self, city_name):'''(根据城市名)获取城市代码'''Q=Query()

city=self.cache_db.get(Q.name.search(city_name))ifisinstance(city, dict):return city['code']else:print('