腾讯招聘数据爬取、保存

# 在程序中打印无法显示内容，使用第一段程序保存至本地后再用第二段程序处理
from urllib import request
from lxml import etree
base_url = 'https://hr.tencent.com/position.php?lid=&tid=&keywords=%E8%AF%B7%E8%BE%93%E5%85%A5%E5%85%B3%E9%94%AE%E8%AF%8D&start={}#a'

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
}




for i in range(0,30,10):
    req = request.Request(base_url.format(i), headers=headers)
    response = request.urlopen(req)
    html_byte = response.read()
    # print(html_byte)
    with open('tencent{}.txt'.format(i),'wb') as f:
        f.write(html_byte)

import codecs
from lxml import etree
from pymysql_conn import Mysql_connect
# 从本地读取html内容

# 创建mysql对象及准备sql语句
mysql_object = Mysql_connect()
sql = 'insert into data_tencent (psname,pstype,psnum,psadd,pstime) value({},{},{},{},{})'

for i in range(1,4):
    with codecs.open('tencent{}.txt'.format(i),'r','utf-8') as f:
        content = f.read()
    # print(content)
    html_ele = etree.HTML(content)
    # print(html_ele)
    tr_list = html_ele.xpath('//table[@class="tablelist"]/tr')
    tr_list = tr_list[1:11]
    for tr_ele in tr_list:
        ps_name = tr_ele.xpath('./td')[0].xpath('./a')[0].text
        ps_type = tr_ele.xpath('./td')[1].text
        ps_num = tr_ele.xpath('./td')[2].text
        ps_add = tr_ele.xpath('./td')[3].text
        ps_time = tr_ele.xpath('./td')[4].text
        # print(ps_name,ps_type,ps_num,ps_add,ps_time)
        mysql_object.execute_sql(sql.format(repr(ps_name),repr(ps_type),repr(ps_num),repr(ps_add),repr(ps_time)))