python使用lxml及request爬取-python+lxml 爬取网页信息及储存

用lxml来爬取招聘网站信息

用requests获取网页

解析网页内容

保存数据成csv格式

用requests获取网页

import requests

from lxml import etree

import time

import numpy as np

import pandas as pd

if __name__ == '__main__':

target = 'https://www.zhipin.com/c100010000/?query=数据分析&period=1&page='

targetpage = '&ka=page-'

page = 1

headers={'user-agent':'Mozilla/6.0'}

target targetpage 以及page用于定义翻页信息收集

网页内容将储存在定义的变量里

title = []

title_link = []

salary = []

location = []

date_update=[]

company_name = []

company_info = []

job_exp = []

解析网页内容

获取3页招聘网站基本信息

for n in range(1,4):

oburl = target+str(page)+targetpage+str(page)

page +=1

html = requests.get(url=oburl,headers=headers)

ab = etree.HTML(html.text)

title = np.concatenate((title,ab.xpath('//div[@class="job-title"]/text()')))

title_link = np.concatenate((title_link,ab.xpath('//div[@class="job-title"]/ancestor::a/@href')))

salary = np.concatenate((salary,ab.xpath('//span[@class="red"]/text()')))

location = np.concatenate((location,ab.xpath('//div[@class="info-primary"]/p/child::node()[1]')))

job_exp = np.concatenate((job_exp,ab.xpath('//div[@class="info-primary"]/p/child::node()[3]')))

date_update = np.concatenate((date_update,ab.xpath('//div[@class="info-publis"]/p/text()')))

company_name=np.concatenate((company_name,ab.xpath('//div[@class="company-text"]//a/text()')))

company_info=np.concatenate((company_info,ab.xpath('//div[@class="company-text"]/p/child::node()[1]')))

time.sleep(1)

组合变量形成dataframe格式，并且命名列名

data = pd.DataFrame([title,salary,location,job_exp,company_name,company_info,date_update,title_link]).T

data.rename(columns={0:'title',1:'salary',2:'location',3:'job_exp',4:'company name',5:'company info',6:'date_update',7:'link'},inplace=True)#

保存数据成csv格式

import csv

myfile = 'web_spyder_name.csv'

with open(myfile, 'wb') as csvfile:

data.to_csv(myfile, encoding='utf_8_sig')

储存成csv格式文件