python爬虫实战数据可视化分析-Python爬虫实战数据可视化分析

#-*-coding:utf-8-*-

import json

import time

import multiprocessing

import requests

from lxml import etree

class HandelLaGou(object):

def __init__(self):

self.lagou_session = requests.session()

self.header = {

"User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.4.154.18 Safari/525.19"

}

self.city_list = ""

def handle_city(self):

city_url = "https://www.lagou.com/jobs/allCity.html"

city_result = self.handle_request(method="GET", url=city_url)

list = city_result.xpath("//ul[contains(@class, "city_list")]/li/a/text()")

self.city_list = [x for x in list]

self.lagou_session.cookies.clear()

def handle_city_job(self, city):

first_request_url = "https://www.lagou.com/jobs/list_python?&px=default&city=%s"%city

first_response = self.handle_request(method="GET",url=first_request_url)

try:

total_page = first_response.xpath("//span[contains(@class, "span totalNum")]/text()")

print(total_page)

except:

return

else:

for i in range(1, int(total_page[0])+1):

data = {

"pn":i,

"kd":"python"

}

print(i)

page_url = "https://www.lagou.com/jobs/positionAjax.json?px=default&city=%s&needAddtionalResult=false"%city

referer_url = "https://www.lagou.com/jobs/list_python?&px=default&city=%s"%city

self.header["Referer"] = referer_url.encode("utf-8")

response = self.handle_request(method="POST", url=page_url, data=data, info=city)

lagou_data = json.loads(response)

job_list = lagou_data["content"]["positionResult"]["result"]

for job in job_list:

print(job)

def handle_request(self, method, url , data=None, info=None):

while True:

if method == "GET":

response = self.lagou_session.get(url=url, headers=self.header)

item = etree.HTML(response.text)

elif method == "POST":

response = self.lagou_session.post(url=url, headers=self.header, data=data)

item = response.text

if "频繁" in response.text:

self.lagou_session.cookies.clear()

first_request_url = "https://www.lagou.com/jobs/list_python?&px=default&city=%s" %info

self.handle_request(method="GET", url=first_request_url)

time.sleep(15)

continue

return item

if __name__ == "__main__":

lagou = HandelLaGou()

lagou.handle_city()

pool = multiprocessing.Pool(2)

for city in lagou.city_list:

pool.apply_async(lagou.handle_city_job, args=(city,))

pool.close()

pool.join()