爬取拉勾网招聘信息

标题爬取拉勾网招聘信息

本内容仅作个人研究使用，不作为商业用途
本人学识浅薄，写的不好还请见谅

#导入库以及创建表格
import requests
import xlwt
#url = 'https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false'#全国招聘
#urls='https://www.lagou.com/jobs/list_python?labelWords=sug&fromSearch=true&suginput=py'
#创建表格
w=xlwt.Workbook(encoding='utf-8')
ws=w.add_sheet('python',cell_overwrite_ok=True)
ws.write(0,0,'序号')
ws.write(0,1,'公司')
ws.write(0,2,'城市')
ws.write(0,3,'职位')
ws.write(0,4,'薪资')
ws.write(0,5,'学历要求')
ws.write(0,6,'工作经验')
ws.write(0,7,'职位优点')

通过解析网页可以知道为动态加载

解析网页得到了数据的动态地址


由此可以看到我们需要的内容都在这里
代码为：

urls='https://www.lagou.com/jobs/list_python?isSchoolJob=1'#校园招聘
url='https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false&isSchoolJob=1'
#实验headers包含user-agent后发现不能访问，出现访问频繁稍后再试等字样所以把headers全部内容复制
#headers = {
#	'Accept': 'application/json,text/javascript,*/*;q=0.01',
#	'Accept-Encoding': 'gzip,deflate,br',
#	'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
#	'Cache-Control': 'max-age=0',
#	'Connection': 'keep-alive',
#	'Content-Length': '25',
#	'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
#	'Cookie': 'JSESSIONID=ABAAABAAAGGABCB1B008A4CE726755A356D26C62CDF8682; _ga=GA1.2.2125033291.1529067719; user_trace_token=20180615210155-47c46dd3-709c-11e8-9642-5254005c3644; LGSID=20180615214357-26f030b8-70a2-11e8-9643-5254005c3644; LGRID=20180615214357-26f0325f-70a2-11e8-9643-5254005c3644; LGUID=20180615210155-47c4714d-709c-11e8-9642-5254005c3644; index_location_city=%E5%85%A8%E5%9B%BD; TG-TRACK-CODE=search_code; SEARCH_ID=e03b7df96b544c96ab10e0f5fa878e08; PRE_UTM=; PRE_HOST=www.baidu.com; PRE_SITE=https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3Dtlz5eF6VtDDmGVyeuPECp4PAP6-D80st6n12koGsSsm%26ck%3D5605.4.94.275.120.494.111.442%26shh%3Dwww.baidu.com%26sht%3Dmonline_dg%26wd%3D%26eqid%3Da25faec40002af21000000025b23b8bc; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2F',
#	'Host': 'www.lagou.com',
#	'Referer': 'https://www.lagou.com/jobs/list_python?labelWords=sug&fromSearch=true&suginput=p',
#	'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0',
#	'X-Anit-Forge-Code': '0',
#	'X-Anit-Forge-Token': 'None',
#	'X-Requested-With': 'XMLHttpRequest'
#}
#全部内容复制后发现还是显示访问频繁（拉钩的反扒机制）通过一次次实验得出包含下面三个参数即可访问
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36',
    'Referer': 'https://www.lagou.com/jobs/list_python?labelWords=sug&fromSearch=true&suginput=p',
    'Accept': 'application/json, text/javascript, */*; q=0.01'
}
#data为控制页数以及职位的参数
data = {
			'first': 'false',
			'kd': 'Python',
			'pn': 1
		}

#使用cookie模拟
cookie='复制cookie信息'
cookies={}
for line in cookie.split(';'):
    key,value = line.split('=',1)
    cookies[key]=value
#为了有效可以使用geturl来获取cookies    
s = requests.Session()
# 获取搜索页的cookies
s.get(urls, headers=headers, timeout=2)
# 为此次获取的cookies
cookie = s.cookies
# 获取此次文本（因为url访问方式为post所以使用post访问）
result = s.post(url, data=data, headers=headers, cookies=cookie, timeout=4)
#得到html内容
#info = result.text
#通过分析发现，转换成json比html更有层次感
info = result.json()

得到了info后，我们需要的数据也就有了，下面就是保存了

#取出需要的内容
info =info['content']['positionResult']['result']
row=1
for n in range(0, len(info)):
   ws.write(row,0,row)
   ws.write(row,1,info[n]['companyFullName'])
   ws.write(row,2,info[n]['city'])
   ws.write(row,3,info[n]['positionName'])
   ws.write(row,4,info[n]['salary'])
   ws.write(row,5,info[n]['education'])
   ws.write(row,6,info[n]['workYear'])
   ws.write(row,7,info[n]['positionAdvantage'])
   row+=1
   
w.save('拉钩.xls')
print('已完成')

下面给出整合后的完整代码：

import requests
import xlwt
def base(jos, page):
   
   urls='https://www.lagou.com/jobs/list_python?isSchoolJob=1'
   url='https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false&isSchoolJob=1'
   headers = {
   'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36',
   'Referer': 'https://www.lagou.com/jobs/list_python?labelWords=sug&fromSearch=true&suginput=p',
   'Accept': 'application/json, text/javascript, */*; q=0.01'
   }
   
   url_info(url,urls,headers,jos,page)
   return
   
def url_info(url,urls,headers,jos,page):
   info_list=[]
    
   for i in range(1,page+1):
       data = {
   		    'first': 'false',
   		    'kd': jos,
   		    'pn': i
   	       }
       s=requests.session()
       s.get(urls,headers=headers,timeout=2)
       cookie=s.cookies
       result=s.post(url,data=data,headers=headers,cookies=cookie,timeout=4)
       info=result.json()
       info_list.append(info['content']['positionResult']['result'])
       
       
   Download(info_list,jos)
   
      
  
     
def Download(infolist,jos):
   w=xlwt.Workbook(encoding='utf-8')
   ws=w.add_sheet(jos,cell_overwrite_ok=True)
   ws.write(0,0,'序号')
   ws.write(0,1,'公司')
   ws.write(0,2,'城市')
   ws.write(0,3,'职位')
   ws.write(0,4,'薪资')
   ws.write(0,5,'学历要求')
   ws.write(0,6,'工作经验')
   ws.write(0,7,'职位优点')
   row=1
   for info in infolist:
     for n in range(0,len(info)):
       ws.write(row,0,row)
       ws.write(row,1,info[n]['companyFullName'])
       ws.write(row,2,info[n]['city'])
       ws.write(row,3,info[n]['positionName'])
       ws.write(row,4,info[n]['salary'])
       ws.write(row,5,info[n]['education'])
       ws.write(row,6,info[n]['workYear'])
       ws.write(row,7,info[n]['positionAdvantage'])
       row+=1
   w.save('拉勾网.xls')
   print('已完成')
   return row
    

if __name__ == '__main__':
   position=input('请输入要查询的职位:')
   pg=input('请输入要爬取的页数<=30:')
   page=eval(pg)
   base(position,page)