使用python scrapy爬虫框架 爬取科学网自然科学基金数据
fundspider.py文件
# -*- coding: utf-8 -*-
from scrapy.selector import Selector
from fundsort.items import FundItem
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.http import Request
import re
class FundSpider(CrawlSpider):
name = "fund"
id = 0
allowed_domains = ["fund.sciencenet.cn"]
start_urls =["http://fund.sciencenet.cn/index.php/search/project?name=&person=&no=&company=%E5%8C%97%E4%BA%AC%E5%A4%96%E5%9B%BD%E8%AF%AD%E5%A4%A7%E5%AD%A6&subject=&money1=&money2=&startTime=2005&endTime=2015&subcategory=&redract_url=&submit.x=0&submit.y=0&page=1"
]#每换个大学换一次star_urls. url结尾一定是以“page=”结束,
def parse_item(self, response):
item = response.meta['item']
sel = Selector(response)
num=self.getid()
num=str(num)
item['id']=num
item['school'] = sel.xpath('//tbody/tr[2]/td[@colspan="2"]/text()').extract()
item['subcategory']=sel.xpath('//table[@class="tb no_print"]//tbody//tr[1]/td[@colspan="4"]/text()').extract()
subcode=sel.xpath('//table[@class="tb no_print"]//tbody//tr[1]/td[@colspan="4"]/text()').extract()[0]
#subcode=str(subcode)
item['subcode']&