python设置代理IP来爬取拉勾网上的职位信息，

import requests
import json
import time

position = input('输入你要查询的职位：')

url = 'https://www.lagou.com/jobs/positionAjax.json?city=%E6%9D%AD%E5%B7%9E&needAddtionalResult=false'
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
}
prox = {        #设置代理服务器
    'http':'http://47.89.48.239:808',
    'https':'https://47.52.3.154:808'
}
def get_page():     #通过你输入的职位来获取每个职位的最大的页数，后期来进行遍历
    data = {
        'first': 'false',
        'pn': '1',
        'kd': position,
    }
    html = requests.post(url,headers=headers,data=data,proxies = prox)
    json_data = json.loads(html.text)
    job_messages = json_data['content']['positionResult']
    page = job_messages['totalCount']/job_messages['resultSize']        #总的工作职位除以每页显示的工作岗位，得出的页数，但是因为你会余数，所有做个判断
    num = str(job_messages['totalCount']/job_messages['resultSize'])[-1]
    if num != 0:        #如果是小数就加1，不是小数直接返回页数
        tole_page = int(page) + 1
        return tole_page
    else:
        tole_page = int(page)
        return tole_page

def get_job_message(data):    #获取每页的工作信息
    html = requests.post(url,headers=headers,data=data,proxies = prox)
    if html.status_code == 200:     #显示状态码
        json_data = json.loads(html.text)
        job_messages = json_data['content']['positionResult']['result']
        for job_message in job_messages:
            company = job_message['companyFullName']    #公司名称
            companySize = job_message['companySize']    #公司大小
            positionName = job_message['positionName']  #职位名称
            salary = job_message['salary']      #工资
            workYear = job_message['workYear']  #工作年限
            district = job_message['district']  #工作地址
            print(company,companySize,positionName,salary,workYear,district)
    else:
        print('没有获取网页数据')

for pn in range(1,get_page()+1):
    data = {
        'first': 'false',
        'pn': str(pn),
        'kd': position,
    }
    print(data)
    get_job_message(data)
    time.sleep(3)

 

转载于:https://www.cnblogs.com/114811yayi/p/6959629.html