爬取tx招聘的小记录

# -*- ecoding: utf-8 -*-
# @ModuleName: SpiderTX
# @Function: 
# @Author: C
# @Time: 2021/10/28 19:46
import module.spider
import json
import jsonpath
import urllib.request
import xlwt
class Spidertx(object):
        base_url = "https://careers.tencent.com/tencentcareer/api/post/Query?timestamp=1635421641765&countryId=&cityId=&bgIds=&productId=&categoryId=&parentCategoryId=40001&attrId=&keyword=&pageIndex="
        use_url = "&pageSize=10&language=zh-cn&area=cn"
        def __init__(self):
            self.begin_page = int(input("请输入起始页:"))
            self.end_page = int(input("请输入终止页:"))




def load_tx(spidertx):

    page = 0
    headers = {
        "User-Agent": "Mozilla / 5.0(Windows NT 10.0;Win64;x64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 89.04389.114 Safari/537.36"
    }
    xlsx = xlwt.Workbook(encoding="utf-8")
    sheet = xlsx.add_sheet("清洗完成的数据", True)
    style = xlwt.XFStyle()
    font = xlwt.Font()
    font.name = 'Calibri'  # 设置字体
    # font.colour_index = 4  # 设置字体颜色
    font.height = 400  # 字体大小
    style.font = font
    sheet.write(0, 0, "城市", style)
    sheet.write(0, 1, "职业名称", style)
    sheet.write(0, 2, "职业描述", style)
    for i in range(spidertx.begin_page, spidertx.end_page):
        i = str(i)
        url = spidertx.base_url + i + spidertx.use_url
        req = urllib.request.Request(url, headers=headers)
        resp = urllib.request.urlopen(req)
        html = resp.read().decode("utf-8")
        json0 = json.loads(html)
        Posts = json0.get("Data").get("Posts")
        for i in range(0,len(Posts)):
            post = Posts[i]
            sheet.write(page+i+1, 0, post.get("LocationName"), style)
            sheet.write(page+i+1, 1, post.get("RecruitPostName"), style)
            sheet.write(page+i+1, 2, post.get("Responsibility"), style)

            if i == len(Posts)-1:
                page += len(Posts)

            # list.append(jsonpath.jsonpath(json0, "$.Data.Posts" + "[" + i + "]" + ".LocationName")[0])
            # list.append(jsonpath.jsonpath(json0, "$.Data.Posts"+"["+ i +"]"+".RecruitPostName")[0])
            # list.append(jsonpath.jsonpath(json0, "$.Data.Posts" + "[" + i + "]" + ".Responsibility")[0])
    xlsx.save("腾讯招聘.xls")


if __name__ == '__main__':

    spidertx = Spidertx()

    load_tx(spidertx)

小总结一下就是：

• 在读取到数据的时候就可以写入表格中！
• 动态获取数据的可以通过F12中查看XHR获取到接口
• json可以用字典类型处理