理论知识详见:https://blog.csdn.net/apollo_miracle/article/details/85005155
版本一:tencent.py
# -*- coding: utf-8 -*-
from copy import deepcopy
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
class TencentSpider(CrawlSpider):
name = 'tencent'
allowed_domains = ['tencent.com']
start_urls = ['https://hr.tencent.com/position.php']
rules = (
# 提取列表页的url地址
Rule(LinkExtractor(allow=r'position\.php\?&start=\d*?#a'), callback='parse_list', follow=True)
)
def parse_list(self, response):
tr_list = response.xpath("//table[@class='tablelist']/tr")[1:11]
for tr in tr_list:
item = {}
item["name"] = tr.xpath("./td[1]/a/text()").extract_first()
item["href"] = tr.xpath("./td[1]/a/@href").extract_first()
item["cate"] = tr.xpath("./td[2]/text()").extract_first()
item["num"] = tr.xpath("./td[3]/text()").extract_first()
item["address"] = tr.xpath("./td[4]/text()").extract_first()
item["date"] = tr.xpath("./td[5]/text()").extract_first()
yield response.follow(
item["href"],
callback=self.parse_item,
meta={"item": deepcopy(item)}
)
def parse_item(self, response):
item = response.meta["item"]
item["duty"] = response.xpath("//table[@class='tablelist textl']/tr[3]//ul/li/text()").extract()
item["requests"] = response.xpath("//table[@class='tablelist textl']/tr[3]//ul/li/text()").extract()
print(item)
return item
版本二:tencent.py
# -*- coding: utf-8 -*-
from copy import deepcopy
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
class TencentSpider(CrawlSpider):
name = 'tencent1'
allowed_domains = ['tencent.com']
start_urls = ['https://hr.tencent.com/position.php']
rules = (
# 提取列表页的url地址
# Rule(LinkExtractor(allow=r'position\.php\?&start=\d*?#a'), callback='parse_item', follow=True),
Rule(LinkExtractor(allow=r'position\.php\?&start=\d*?#a'), follow=True),
# 提取详情页的url地址
Rule(LinkExtractor(allow=r'position_detail\.php\?id=\d*?&keywords=&tid=0&lid=0'), callback='parse_item')
)
def parse_item(self, response):
item = {}
item["name"] = response.xpath("//table[@class='tablelist textl']/tr[1]/td/text()").extract_first()
item["address"] = response.xpath("//table[@class='tablelist textl']/tr[2]/td[1]/text()").extract_first()
item["cate"] = response.xpath("//table[@class='tablelist textl']/tr[2]/td[2]/text()").extract_first()
item["num"] = response.xpath("//table[@class='tablelist textl']/tr[2]/td[3]/text()").extract_first()
item["duty"] = response.xpath("//table[@class='tablelist textl']/tr[3]//ul/li/text()").extract()
item["requests"] = response.xpath("//table[@class='tablelist textl']/tr[3]//ul/li/text()").extract()
item["url"] = response.url
print(item)
return item