Python爬虫练习

爬虫代码

import requests
from lxml import etree
import json
import time


# 发送请求获取响应
def spider_job(href):
    # 构造请求头
    header = {
        "user-agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) "
                      "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"
    }

    # 发送请求获取响应3
    response = requests.get(href, headers=header, timeout=5)

    # 返回响应内容
    return response.content


# 解析数据并保存
def analysis_content(content, filename):
    html = etree.HTML(content)
    # 匹配内容
    job = html.xpath('//script[@type="text/javascript"]/text()')
    # print(job)

    # 去除所有的空白字符以及转移符号
    job = job[0].replace('\r\n', '')
    job = job.replace('\\', '')

    job = job[job.find('=') + 1:]
    job = job[:job.rfind(',"jobid_count"')] + '}'
    job = '{"name":"' + filename + '","content":' + job + "}"

    job_1 = json.loads(job)
    # print(job_1['content']['engine_jds'])
    # 判断数据是否为空
    if not len(job_1['content']['engine_jds']):
        return False

    # 将数据保存到本地
    with open('job_file/{}.json'.format(filename), 'w', encoding='utf-8') as f:
        f.write(job)
        print('文件 {} 写入成功'.format(filename))

    return True


def run():
    search = ['web', 'java', '银行', '移动开发', '通信技术开发及应用', '国企', '事业单位', '政府机关', '上市公司']
    province = [
        {
            'name': '贵州',
            'id': '260200'
        }, {
            'name': '重庆',
            'id': '060100'
        }, {
            'name': '四川',
            'id': '090201'
        }, {
            'name': '云南',
            'id': '250200'
        }, {
            'name': '湖南',
            'id': '190200'
        }, {
            'name': '广西',
            'id': '140200'
        }]
    url = 'https://search.51job.com/list/{},000000,0000,00,9,99,{},2,{}.html?workyear=01'

    for i in search:
        for v in province:
            # 当前页
            page = 1
            # 错误次数
            err = 0
            while True:
                # 错误捕获
                try:
                    response = spider_job(url.format(v['id'], i, page))
                    flag = analysis_content(response, '{}_{}_{}'.format(i, v['name'], page))
                    time.sleep(5)
                    if flag:
                        page += 1
                    else:
                        break
                except:
                    err += 1
                    if err >= 3:
                        print('错误 {} 次, 已重新更换链接'.format(err))
                        break
                    print('错误 {} 次'.format(err))


if __name__ == "__main__":
    run()

数据筛选并写入Excel

import os
import json
import openpyxl


# 提起岗位关键信息
def jobCollect(content):
    obj_arr = []
    for obj in content["engine_jds"]:
        # 工作名称
        job_name = obj['job_name']
        # 工作链接
        job_href = obj['job_href']
        # 工资
        providesalary_text = obj['providesalary_text']
        # 公司名称
        company_name = obj['company_name']
        # 发布时间
        issuedate = obj['issuedate']
        # 工作地点
        workarea_text = obj["workarea_text"]
        # 公司类型
        companytype_text = obj['companytype_text']

        obj_arr.append({
            "job_name": job_name,
            "job_href": job_href,
            "providesalary_text": providesalary_text,
            "company_name": company_name,
            "issuedate": issuedate,
            'workarea_text': workarea_text,
            'companytype_text': companytype_text
        })
    return obj_arr


def run():
    # 获取文件夹下 所有的文件
    files = os.listdir('51job/job_file')

    # 循环读取所有文件
    for file in files:
        with open('51job/job_file/{}'.format(file), encoding='utf-8') as f:
            # 获取文件内容并转化为json
            job_json = json.loads(f.read())

            # 获取里面的岗位信息
            job_list = job_json['content']
            # 提取岗位信息
            job_linchpin = jobCollect(job_list)

            # 文件名处理 将最后的页码去掉 作为sheet 同一个方向的岗位写入同一个sheet
            sheet_name = file[:file.rfind('_')]

            # 定义表头
            title = ['岗位', '工作链接', '工资', '公司名称', '发布时间', '工作地点', '公司类型']

            wb = openpyxl.load_workbook('51job.xlsx')
            # 获取已存在的sheet
            sheet_exist = wb.sheetnames
            # 判断sheet是否存在 不存在创建 存在则追加到后面
            if sheet_name in sheet_exist:
                # 获取工作表
                sheet = wb[sheet_name]
                # 获取指定工作表列数
                row = len(list(sheet))
                # 将数据追加到后面
                for i in range(len(job_linchpin)):
                    col = 1
                    row += 1
                    for k in job_linchpin[i].keys():
                        # 跳过表头写入
                        sheet.cell(row, col, job_linchpin[i][k])
                        col += 1
                else:
                    print("追加成功")
            else:
                # 创建工作表
                sheet = wb.create_sheet(sheet_name)
                # 写入表头
                for i in range(len(title)):
                    sheet.cell(1, i + 1, title[i])

                # 将数据写入表格
                for i in range(len(job_linchpin)):
                    col = 1
                    for k in job_linchpin[i].keys():
                        # 跳过表头写入
                        sheet.cell(i + 2, col, job_linchpin[i][k])
                        col += 1
                else:
                    print("写入成功")
            wb.save('51job.xlsx')
            wb.close()


if __name__ == "__main__":
    run()