自动爬取拉勾网

selectnium+requests爬取拉勾网招聘信息，包括一级和二级页面，无需人工干预，全程自动化

'''
scrapy 拉勾网
待完善：不够稳定，在爬取职位详情时，偶尔会出现302重定向
其它要点：
    post请求必须要携带cookie，cookie可以先get一个网页，然后session.cookies获取
    后面get可以不带cookies，先session取cookies，再requests请求（不能再用session否则容易302）
    Referer必须要准确
    爬取间隔时间要长一点，否则容易被拉入黑名单
'''
import  requests
from  selenium import webdriver
import re
import json
from lxml import etree
from requests import Session
import time
import urllib3
import pandas as pd
import numpy as np
import pymysql
import traceback

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

#sqlnet = pymysql.connect(host="localhost", user="test", password="123456", database="lagou",charset='utf8' )

url='https://www.lagou.com/jobs/positionAjax.json?'
url_start = "https://www.lagou.com/jobs/list_运维?city=%E6%88%90%E9%83%BD&cl=false&fromSearch=true&labelWords=&suginput="
baseurl='https://www.lagou.com/jobs/{}.html?show={}'
params={
'px': 'default',
'gm': '50-150人,150-500人',
'city': '成都',
'needAddtionalResult': 'false'
}

data={
'first': 'true',
'pn': 1,
'kd': 'C++'   ##职位
}

headers = {
'Connection': 'close',
'Host': 'www.lagou.com',
'Origin': 'https://www.lagou.com',
'Content-Type': 'application/json;charset=UTF-8',
'Referer': 'https://www.lagou.com/jobs/list_Python/p-city_252-gm_3_4?px=default',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.26 Safari/537.36 Edg/81.0.416.16',
}

##如果职业详情request失败，可以尝试使用chromedriver模拟
def GetJobDetailByChrome(job_url):
    item=''
    options = webdriver.ChromeOptions()
    options.add_argument("--headless")  ##无头模式
    driver = webdriver.Chrome(options=options)
    driver.get(job_url)
    time.sleep(2)
    print(driver.find_element_by_xpath('//*[@id="job_detail"]/dd[1]/span').text)
    print(driver.find_element_by_xpath('//*[@id="job_detail"]/dd[1]/p').text)
    print(driver.find_element_by_xpath('//*[@id="job_detail"]/dd[2]/h3').text)
    details=driver.find_elements_by_xpath('//*[@id="job_detail"]/dd[2]/div/p')
    for detail in details:
        print(detail.text)
        item=item+detail.text
    driver.close()

    return item




def GetJobDetail(job_url,cookie):
    item=''
    req=requests.get(url=job_url,headers=headers,verify=False,timeout=3)
    req.encoding = 'utf-8'
    if len(req.text)<10000:
        print('尝试用模拟浏览器获取')
        item=GetJobDetailByChrome(job_url)
        return item
    etree_html = etree.HTML(req.content)

    detail1=str(etree_html.xpath('string(//*[@id="job_detail"]/dd[1])')).lstrip()
    print("".join([s for s in detail1.splitlines(True) if s.strip()]))
    detail2 = str(etree_html.xpath('string(//*[@id="job_detail"]/dd[2])')).lstrip()
    print("".join([s for s in detail2.splitlines(True) if s.strip()]))
    detail3 = str(etree_html.xpath('string(//*[@id="container"]/div[1]/dl[2])')).lstrip()
    print("".join([s for s in detail3.splitlines(True) if s.strip()]))

    item="".join([s for s in detail1.splitlines(True) if s.strip()])+\
         "".join([s for s in detail2.splitlines(True) if s.strip()])+ \
         "".join([s for s in detail3.splitlines(True) if s.strip()])
    '''
    print(etree_html.xpath('string(//*[@id="job_detail"]/dd[1])').replace('\r', '').replace('\t', '').replace('\n\n', '\n').replace('\xa0', ' '))
    #print(etree_html.xpath('//*[@id="job_detail"]/dd[1]/p/text()'))
    print(etree_html.xpath('string(//*[@id="job_detail"]/dd[2])').replace('\r', '').replace('\t', '').replace('\n\n', '\n').replace('\xa0', ' '))
    #print(etree_html.xpath('//*[@id="job_detail"]/dd[2]/div/p/text()'))
    '''
    return item


def to_save(item,stage):
    print('stage=',stage)
    try:
        df=pd.DataFrame(item,index=[0])
    except Exception as e:
        df = pd.DataFrame(item)
    df.to_csv('拉勾网职位.csv',encoding='gb18030',header=stage, index=False, mode='a+')

'''
def to_mysql(item):
    mycursor = sqlnet.cursor()
    sql='insert into lagou_job() values()'
    sql.format()
    try:
        position_id=1
        company_name=1
        sql = "insert into lagou_job(position_id,company_name,job_name,salary,location,exprience,school,scale,state_date) \
                       values (%s,%s,%s,%s,%s,%s,%s,%s)"
        mycursor.execute(sql, (company_name))
        mycursor.commit()
    except Exception as e:
        print("出错", repr(e))
        print(' traceback.format_exc():{}'.format(traceback.format_exc()))
'''
def main(pagenum):
    session = requests.Session()

    response = session.get(url=url_start, headers=headers, verify=False, timeout=3)

    cookie = response.cookies  # 为此次获取的cookies
    data['pn']=pagenum
    #print(data)
    response = requests.post(url=url, headers=headers, data=data, cookies=cookie, params=params, verify=False).text
    response = json.loads(response)

    joblist = response.get('content').get('positionResult').get('result')
    showId = response.get('content').get('showId')
    item={'公司简称':'',
          '公司规模':'',
          '薪资':'',
          '公司领域': '',
          '融资阶段':'',
          '第一需求': '',
          '第二需求': '',
          '公司福利': '',
          '上班城市': '',
          '地区': '',
          '地点':'',
          '详情':''
          }
    global stage
    for job in joblist:
        print('公司:',job.get('companyFullName'))
        print('     公司简称:{},公司规模:{},薪资:{},公司领域:{},融资阶段:{},第一需求:{},第二需求:{},公司福利:{},上班城市:{},地区:{},地点:{}'\
              .format(job.get('companyShortName'),job.get('companySize'),job.get('salary'),job.get('industryField'),\
                      job.get('financeStage'),job.get('firstType'),job.get('secondType'),job.get('positionAdvantage'),\
                      job.get('city'),job.get('district'),job.get('businessZones')))
        item['公司简称']=job.get('companyShortName')
        item['公司规模'] = job.get('companySize')
        item['薪资'] = job.get('salary')
        item['公司领域'] = job.get('industryField')
        item['融资阶段'] = job.get('financeStage')
        item['第一需求'] = job.get('firstType')
        item['第二需求'] = job.get('secondType')
        item['公司福利'] = job.get('positionAdvantage')
        item['上班城市'] = job.get('city')
        item['地区'] = job.get('district')
        item['地点'] = job.get('businessZones')
        positionId=job.get('positionId')
        joburl=baseurl.format(positionId,showId)
        #print('joburl=',joburl)
        detail=GetJobDetail(joburl,cookie)
        item['详情'] = detail
        to_save(item,stage)
        if stage:
            stage=False
        time.sleep(10)
        print('\r')
        print('*'*100)
        print('\r')

if __name__ == '__main__':
    stage=True
    for pagenum in range(1,6): ##爬取前5页
        print('开始第{}页爬取'.format(pagenum))
        main(pagenum)