Tencent招聘

使用xpath

import requests
import pymysql
from lxml import etree

# 把数据存入数据库
class Mysql_db(object):
    def __init__(self):
        self.db = pymysql.connect(host='127.0.0.1',user='root',password='123456',port=3306,database='py10',charset='utf8')
        self.cursor = self.db.cursor()
    def execute_modify_sql(self,sql,data):
        self.cursor.execute(sql,data)
        self.db.commit()
    def __del__(self):
        self.cursor.close()
        self.db.close()
conn = Mysql_db()
sql = "insert into tencent(title,place,category,num,duty_str,requirement_str) values (%s,%s,%s,%s,%s,%s)"

def geturl(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
    }
    response = requests.get(url,headers=headers)
    html_ele = etree.HTML(response.text)
    title = html_ele.xpath('//*[@id="sharetitle"]/text()')[0]
    print(title)
    place = html_ele.xpath('//*[@id="position_detail"]/div/table/tr[2]/td[1]/text()')[0]
    print(place)
    category = html_ele.xpath('//*[@id="position_detail"]/div/table/tr[2]/td[2]/text()')[0]
    print(category)
    num = html_ele.xpath('//*[@id="position_detail"]/div/table/tr[2]/td[3]/text()')[0]
    print(num)
    duty = html_ele.xpath('//*[@id="position_detail"]/div/table/tr[3]/td/ul/li/text()')
    duty_str = ' '.join(duty)
    print(duty_str)
    requirement = html_ele.xpath('//*[@id="position_detail"]/div/table/tr[4]/td/ul/li/text()')
    requirement_str = ' '.join(requirement)
    print(requirement_str)
    data = (title,place,category,num,duty_str,requirement_str)
    conn.execute_modify_sql(sql,data)

def getdetail(page):
    base_url = 'https://hr.tencent.com/position.php?lid=&tid=&keywords=%E8%AF%B7%E8%BE%93%E5%85%A5%E5%85%B3%E9%94%AE%E8%AF%8D&start={}'
    for i in range(1,page+1):
        headers = {
            'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
        }
        start = (i-1)*10
        full_url = base_url.format(start)
        response = requests.get(full_url,headers=headers)
        html_ele = etree.HTML(response.text)
        tr_list = html_ele.xpath('//div[@id="position"]/div[1]/table/tr')[1:11]
        for i in tr_list:
            link_url = i.xpath('./td[@class="l square"]/a/@href')[0]
            url = 'https://hr.tencent.com/' + link_url
            geturl(url)




if __name__ == "__main__":
    getdetail(2)