功能:设置起始URL和爬取范围,设置要提取的数据路径,返回item或下一个URL地址
# -*- coding: utf-8 -*-
import scrapy
from Tenxun.items import TenxunItem
class TencentSpider(scrapy.Spider):
name = 'tencent'
allowed_domains = ['tencent.com']
start_urls = ['http://hr.tencent.com/position.php?&start=0']
def parse(self, response):
nodelist = response.xpath("//tr[@class='even'] | //tr[@class='odd']")
for node in nodelist:
item = TenxunItem()
item['job_name'] = node.xpath("./td[1]/a/text()").extract()[0]
item['job_link'] = node.xpath("./td[1]/a/@href").extract()[0]
if len(node.xpath("./td[2]/text()")):
item['job_type'] = node.xpath("./td[2]/text()").extract()[0]
else:
item['job_type'] = "NULL"
item['job_people_num'] = node.xpath("./td[3]/text()").extract()[0]
item['job_site'] = node.xpath("./td[4]/text()").extract()[0]
item['publish_time'] = node.xpath("./td[5]/text()").extract()[0]
yield item
if len(response.xpath("//a[@class='noactive' and @id='next']")) == 0:
url = response.xpath("//a[@id='next']/@href").extract()[0]
yield scrapy.Request("http://hr.tencent.com/" + url, callback=self.parse)