使用python scrapy爬虫框架 爬取科学网自然科学基金数据

使用python scrapy爬虫框架 爬取科学网自然科学基金数据

fundspider.py文件

# -*- coding: utf-8 -*-

from scrapy.selector import Selector
from fundsort.items import FundItem
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.http import Request
import re

class FundSpider(CrawlSpider):
    name = "fund"
    id = 0
    allowed_domains = ["fund.sciencenet.cn"]
    start_urls =["http://fund.sciencenet.cn/index.php/search/project?name=&person=&no=&company=%E5%8C%97%E4%BA%AC%E5%A4%96%E5%9B%BD%E8%AF%AD%E5%A4%A7%E5%AD%A6&subject=&money1=&money2=&startTime=2005&endTime=2015&subcategory=&redract_url=&submit.x=0&submit.y=0&page=1"
    ]#每换个大学换一次star_urls. url结尾一定是以“page=”结束，

    def parse_item(self, response):
        item = response.meta['item']
        sel = Selector(response)
        num=self.getid()
        num=str(num)
        item['id']=num
        item['school'] = sel.xpath('//tbody/tr[2]/td[@colspan="2"]/text()').extract()
        item['subcategory']=sel.xpath('//table[@class="tb no_print"]//tbody//tr[1]/td[@colspan="4"]/text()').extract()
        subcode=sel.xpath('//table[@class="tb no_print"]//tbody//tr[1]/td[@colspan="4"]/text()').extract()[0]
        #subcode=str(subcode)
        item['subcode']&