51job的scrapy爬虫

ps:遇到了几个坑，1。自己写的正则以为没问题，结果实际没匹配到。2。allowed_domains = [‘51job.com’] 。刚开始是search.51job.com。但是我的rule中的一个是另个一个域名的，所以都被过滤掉了。查看debug日志才发现。3。vim缩进问题，在vim中看不出来。nano下编辑发现了。

主要更改的文件如下：
1)pipeline.py

import json
class WyjobPipeline(object):
    def __init__(self):
        self.filename = open("hz.json", 'a+')

    def process_item(self, item, spider):
        text = json.dumps(dict(item), ensure_ascii=False) + ",\n"
        self.filename.write(text.encode("utf-8"))
        return item


    def close_spider(self, spider):
        self.filename.close()

2.) items.py

import scrapy


class DongguanItem(scrapy.Item):
    # define the fields for your item here like:
    title = scrapy.Field()
    content = scrapy.Field()
    url = scrapy.Field()
    number = scrapy.Field()

3)settings.py

BOT_NAME = 'dongguan'

SPIDER_MODULES = ['dongguan.spiders']
NEWSPIDER_MODULE = 'dongguan.spiders'
ITEM_PIPELINES = {
    'dongguan.pipelines.DongguanPipeline': 300,
}

LOG_FILE = "dg.log"
LOG_LEVEL = "DEBUG"

4.)hz.py

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from ..items import WyjobItem
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
class HzSpider(CrawlSpider):
    name = 'hz'
    allowed_domains = ['51job.com']
    start_urls = ['http://search.51job.com/list/080200,000000,0000,00,9,99,python,2,1.html?lang=c&stype=1&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=1&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=']

    rules = (
        Rule(LinkExtractor(allow=(r's=\d+&t=\d+')), callback='parse_item'),
        Rule(LxmlLinkExtractor(allow=(r'lang=c&stype'),restrict_xpaths=(r"/html/body/div[2]/div[4]/div[54]/div/div/div/ul/li")))
        ,
    )

    def parse_item(self, response):

         i = WyjobItem()
         i["position"] = response.xpath("//div//h1/text()").extract()[0]
         i["info"] = ",".join(response.xpath('//div[@class="t1"]//span/text()').extract())
         i["welfare"] = ",".join(response.xpath('//div[@class="jtag inbox"]//p/span/text()').extract())
         i["duty"] = " ".join(response.xpath('//div//div//div[@class="bmsg job_msg inbox"]/text()').extract()).replace(u"\t",'').replace(u"\n",'')
         i["address"] = " ".join(response.xpath('//div/div[@class="bmsg inbox"]/p/text()').extract()).replace(u"\t",'').replace(u"\n",'')
         yield  i