python爬取数据搭建Hadoop、Flume用Hive做数据分析Sqoop存储到Mysql并实现可视化

加粗样式1.爬虫
爬取的是中华英才网和前程无忧两个网站。

spider代码：

# -*- coding: utf-8 -*-
import copy
import re
import pandas as pd
from scrapy import Spider, Request
from qiancheng.items import QianchengItem


class A51jobSpider(Spider):
    name = '51job'
    # allowed_domains = ['www.51job.com/']
    start_urls = ['https://search.51job.com/list/000000,000000,0000,00,9,99,%25E5%25A4%25A7%25E6%2595%25B0%25E6%258D%25AE%25E5%25BC%2580%25E5%258F%2591%25E5%25B7%25A5%25E7%25A8%258B%25E5%25B8%2588,2,{0}.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare='.format(num)for num in range(1, 2000)]

    def parse(self, response):
        a_list = response.xpath('//div[@class="dw_table"]//div[@class="el"]')
        # print(a_list)
        for a in a_list:
            item = QianchengItem()
            # print(a)
            item['work_name'] = a.xpath('//p/span/a/text()').extract_first()
            item['work_name'] = [str(i).strip() for i in item['work_name']]
            item['work_name'] = ''.join(item['work_name'])
            # print(item['work_name'])
            item['company'] = a.xpath('//span[@class="t2"]/a/text()').extract_first()
            item['place'] = a.xpath('./span[@class="t3"]/text()').extract_first()
            item['salary'] = a