基于python(xpath)的-爬取51job网信息(跳过User-Agent)

# -*- coding:utf-8 -*-

import requests
from fake_useragent import UserAgent
from lxml import etree

agent = UserAgent()
url = "http://search.51job.com/list/010000%252C020000%252C030200%252C040000%252C080200,000000,0000,00,9,99,python,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare="
response = requests.get(
    url,
    headers={"User-Agent":agent.random},
)
response.encoding = response.apparent_encoding
root = etree.HTML(response.text)
div_list = root.xpath('//div[@class="dw_table"]/div[@class="el"]')

for div in div_list:
    name = div.xpath('p/span/a/text()')[0]
    name = name.strip()
    company = div.xpath('span[@class="t2"]/a/text()')[0]
    place = div.xpath('span[@class="t3"]/text()')[0]
    money = div.xpath('span[@class="t4"]/text()')
    time = div.xpath('span[@class="t5"]/text()')
    # if not money:
    #     money = "面议"
    # else:
    #     money = money[0]
    money = money[0] if money else "面议"
    time = time[0] if time else "没有时间"
    print("职位名:%s" % name)
    print("公司名:%s" % company)
    print("工作地点:%s" % place)
    print("薪资:%s" % money)
    print("上传时间:%s" % time)
    print("----------------------------")
    # with open('job.csv', 'a', encoding='gb18030') as f:
    #     f.write(name+','+company+','+place+','+money+','+time)
    #     f.write('\n')
    with open('51job.csv', 'a', encoding='gb18030') as f:
        im_list = [name,company,place,money,time,'\n']
        f.write(','.join(im_list))