#-*-coding:utf-8-*-
import json
import time
import multiprocessing
import requests
from lxml import etree
class HandelLaGou(object):
def __init__(self):
self.lagou_session = requests.session()
self.header = {
"User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.4.154.18 Safari/525.19"
}
self.city_list = ""
def handle_city(self):
city_url = "https://www.lagou.com/jobs/allCity.html"
city_result = self.handle_request(method="GET", url=city_url)
list = city_result.xpath("//ul[contains(@class, "city_list")]/li/a/text()")
self.city_list = [x for x in list]
self.lagou_session.cookies.clear()
def handle_city_job(self, city):
first_request_url = "https://www.lagou.com/jobs/list_python?&px=default&city=%s"%city
first_response = self.handle_request(method="GET",url=first_request_url)
try:
total_page = first_response.xpath("//span[contains(@class, "span totalNum")]/text()")
print(total_page)
except:
return
else:
for i in range(1, int(total_page[0])+1):
data = {
"pn":i,
"kd":"python"
}
print(i)
page_url = "https://www.lagou.com/jobs/positionAjax.json?px=default&city=%s&needAddtionalResult=false"%city
referer_url = "https://www.lagou.com/jobs/list_python?&px=default&city=%s"%city
self.header["Referer"] = referer_url.encode("utf-8")
response = self.handle_request(method="POST", url=page_url, data=data, info=city)
lagou_data = json.loads(response)
job_list = lagou_data["content"]["positionResult"]["result"]
for job in job_list:
print(job)
def handle_request(self, method, url , data=None, info=None):
while True:
if method == "GET":
response = self.lagou_session.get(url=url, headers=self.header)
item = etree.HTML(response.text)
elif method == "POST":
response = self.lagou_session.post(url=url, headers=self.header, data=data)
item = response.text
if "频繁" in response.text:
self.lagou_session.cookies.clear()
first_request_url = "https://www.lagou.com/jobs/list_python?&px=default&city=%s" %info
self.handle_request(method="GET", url=first_request_url)
time.sleep(15)
continue
return item
if __name__ == "__main__":
lagou = HandelLaGou()
lagou.handle_city()
pool = multiprocessing.Pool(2)
for city in lagou.city_list:
pool.apply_async(lagou.handle_city_job, args=(city,))
pool.close()
pool.join()