详细代码
import requests
import json
import math
import time
from fake_useragent import UserAgent
import pymongo
#from proxy_ip import get_one_proxy_ip
class LaGou:
def __init__(self):
self.start_url = 'https://www.lagou.com/jobs/positionAjax.json?'
self.params = {
'needAddtionalResult': 'false',
'first': 'true',
}
self.ua = UserAgent()
self.headers = {
'User-Agent': self.ua.random,
'Cookie': 'cookies',
'Referer': 'https://www.lagou.com/jobs/list_%E6%95%B0%E6%8D%AE%E5%88%86%E6%9E%90?labelWords=sug&fromSearch=true&suginput=%E6%95%B0%E6%8D%AE'
}
self.client = pymongo.MongoClient('localhost')
self.db = self.client.spider
# 如果有代理池,可用这个方法
# def get_page(self, page, kd, proxy=None):
## global proxy
# self.params['pn'] = page
# self.params['kd'] = kd
# try:
# if proxy:
# proxies = {
# 'http': 'http//{}'.format(proxy),
# 'https': 'https//{}'.format(proxy)
# }
# response = requests.get(self.start_url, params=self.params, headers=self.headers, proxies=proxies)
# else:
# response = requests.get(self.start_url, params=self.params, headers=self.headers)
# content = response.content
# content = json.loads(content)
# if 'success' in content.keys():
# return content
# else:
# print('Got Data Failed')
# proxy = self.get_proxy()
# if proxy:
# print('Using Proxy', proxy)
# return self.get_page(page, kd, proxy=proxy)
# else:
# print('Got Proxy Failed')
# return None
# except JSONDecodeError:
# proxy = self.get_proxy()
# if proxy:
# print('Using Proxy', proxy)
# return self.get_page(page, kd, proxy=proxy)
# def get_proxy(self):
# return get_one_proxy_ip()
def get_page(self, page, kd):
self.params['pn'] = page
self.params['kd'] = kd
response = requests.get(self.start_url, params=self.params, headers=self.headers)
content = response.content
content = json.loads(content)
if 'success' in content.keys():
return content
def get_data(self, json_content):
try:
total = json_content['content']['positionResult']['totalCount']
result_list = json_content['content']['positionResult']['result']
return result_list, total
except:
print('Got Data Failed')
def on_save(self, content):
if content:
self.db.lagou.update_one({'positionId': content['positionId']}, {'$set': content}, True)
def run(self, kd):
# 1.发送请求,接收响应
content = self.get_page(1, kd)
# 2.获取数据和数据量
result_list, total = self.get_data(content)
print(total)
for i in result_list:
self.on_save(i)
# time.sleep(5)
# 3.根据数据量构造后续请求,循环操作
page_num = math.ceil(total/15)
for i in range(2, page_num+1):
content = self.get_page(i, kd)
if content:
# time.sleep(5)
result_list, _ = self.get_data(content)
print('page',i,'='*50)
for j in result_list:
self.on_save(j)
else:
print('Got Data Failed')
break
# 4.存储数据
if __name__ == '__main__':
lagou = LaGou()
lagou.run('数据分析')