使用crawlscrapy爬取腾讯的招聘信息

理论知识详见：https://blog.csdn.net/apollo_miracle/article/details/85005155

版本一：tencent.py

# -*- coding: utf-8 -*-
from copy import deepcopy

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule


class TencentSpider(CrawlSpider):
    name = 'tencent'
    allowed_domains = ['tencent.com']
    start_urls = ['https://hr.tencent.com/position.php']

    rules = (
        # 提取列表页的url地址
        Rule(LinkExtractor(allow=r'position\.php\?&start=\d*?#a'), callback='parse_list', follow=True)
    )

    def parse_list(self, response):
        tr_list = response.xpath("//table[@class='tablelist']/tr")[1:11]
        for tr in tr_list:
            item = {}
            item["name"] = tr.xpath("./td[1]/a/text()").extract_first()
            item["href"] = tr.xpath("./td[1]/a/@href").extract_first()
            item["cate"] = tr.xpath("./td[2]/text()").extract_first()
            item["num"] = tr.xpath("./td[3]/text()").extract_first()
            item["address"] = tr.xpath("./td[4]/text()").extract_first()
            item["date"] = tr.xpath("./td[5]/text()").extract_first()
            yield response.follow(
                item["href"],
                callback=self.parse_item,
                meta={"item": deepcopy(item)}
            )

    def parse_item(self, response):
        item = response.meta["item"]
        item["duty"] = response.xpath("//table[@class='tablelist textl']/tr[3]//ul/li/text()").extract()
        item["requests"] = response.xpath("//table[@class='tablelist textl']/tr[3]//ul/li/text()").extract()
        print(item)
        return item

版本二：tencent.py

# -*- coding: utf-8 -*-
from copy import deepcopy

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule


class TencentSpider(CrawlSpider):
    name = 'tencent1'
    allowed_domains = ['tencent.com']
    start_urls = ['https://hr.tencent.com/position.php']

    rules = (
        # 提取列表页的url地址
        # Rule(LinkExtractor(allow=r'position\.php\?&start=\d*?#a'), callback='parse_item', follow=True),
        Rule(LinkExtractor(allow=r'position\.php\?&start=\d*?#a'), follow=True),
        # 提取详情页的url地址
        Rule(LinkExtractor(allow=r'position_detail\.php\?id=\d*?&keywords=&tid=0&lid=0'), callback='parse_item')
    )

    def parse_item(self, response):
        item = {}
        item["name"] = response.xpath("//table[@class='tablelist textl']/tr[1]/td/text()").extract_first()
        item["address"] = response.xpath("//table[@class='tablelist textl']/tr[2]/td[1]/text()").extract_first()
        item["cate"] = response.xpath("//table[@class='tablelist textl']/tr[2]/td[2]/text()").extract_first()
        item["num"] = response.xpath("//table[@class='tablelist textl']/tr[2]/td[3]/text()").extract_first()
        item["duty"] = response.xpath("//table[@class='tablelist textl']/tr[3]//ul/li/text()").extract()
        item["requests"] = response.xpath("//table[@class='tablelist textl']/tr[3]//ul/li/text()").extract()
        item["url"] = response.url
        print(item)
        return item