python爬虫之五 ——拉勾网数据

详细代码

import requests
import json
import math
import time
from fake_useragent import UserAgent
import pymongo
#from proxy_ip import get_one_proxy_ip


class LaGou:
    def __init__(self):
        self.start_url = 'https://www.lagou.com/jobs/positionAjax.json?'
        self.params = {
                'needAddtionalResult': 'false',
                'first': 'true',
                }
        self.ua = UserAgent()
        self.headers = {
                'User-Agent': self.ua.random,
                'Cookie': 'cookies',
                'Referer': 'https://www.lagou.com/jobs/list_%E6%95%B0%E6%8D%AE%E5%88%86%E6%9E%90?labelWords=sug&fromSearch=true&suginput=%E6%95%B0%E6%8D%AE'
                }
        self.client = pymongo.MongoClient('localhost')
        self.db = self.client.spider

#    如果有代理池，可用这个方法      
#    def get_page(self, page, kd, proxy=None):  
##        global proxy
#        self.params['pn'] = page
#        self.params['kd'] = kd
#        try:
#            if proxy:
#                proxies = {
#                        'http': 'http//{}'.format(proxy),
#                        'https': 'https//{}'.format(proxy)
#                        }
#                response = requests.get(self.start_url, params=self.params, headers=self.headers, proxies=proxies)
#            else:
#                response = requests.get(self.start_url, params=self.params, headers=self.headers)
#            content = response.content
#            content = json.loads(content)
#            if 'success' in content.keys():
#                return content
#            else:
#                print('Got Data Failed')
#                proxy = self.get_proxy()
#                if proxy:
#                    print('Using Proxy', proxy)
#                    return self.get_page(page, kd, proxy=proxy)
#                else:
#                    print('Got Proxy Failed')
#                    return None
#        except JSONDecodeError:
#            proxy = self.get_proxy()
#            if proxy:
#                print('Using Proxy', proxy)
#                return self.get_page(page, kd, proxy=proxy)
    
#    def get_proxy(self):
#        return get_one_proxy_ip()


    def get_page(self, page, kd):
        self.params['pn'] = page
        self.params['kd'] = kd
        
        response = requests.get(self.start_url, params=self.params, headers=self.headers)
        content = response.content
        content = json.loads(content)
        if 'success' in content.keys():
            return content    
    
    def get_data(self, json_content):
        try:
            total = json_content['content']['positionResult']['totalCount']
            result_list = json_content['content']['positionResult']['result']
            return result_list, total
        except:
            print('Got Data Failed')
        
    def on_save(self, content):
        if content:
            self.db.lagou.update_one({'positionId': content['positionId']}, {'$set': content}, True)
    
    def run(self, kd):
        # 1.发送请求，接收响应
        content = self.get_page(1, kd)        
        # 2.获取数据和数据量
        result_list, total = self.get_data(content)
        print(total)
        for i in result_list:
            self.on_save(i)
#        time.sleep(5)
        # 3.根据数据量构造后续请求，循环操作
        page_num = math.ceil(total/15)
        for i in range(2, page_num+1):
            content = self.get_page(i, kd)
            if content:
#                time.sleep(5)
                result_list, _ = self.get_data(content)   
                print('page',i,'='*50)
                for j in result_list:
                    self.on_save(j)
            else:
                print('Got Data Failed')
                break
            # 4.存储数据

if __name__ == '__main__':
    lagou = LaGou()
    lagou.run('数据分析')