代码:
import requests
from bs4 import BeautifulSoup
import time
def getHtml(url,code='gbk'):
try:
r = requests.get(url)
r.raise_for_status()
r.encoding = code
return r.text
except:
return ""
def ParserHtml(i,htmlText):
print("正在解析第{0}页".format(i))
RecuitInfos = []
soup = BeautifulSoup(htmlText,'lxml')
#获取职位信息
InfoPositions = soup.find_all('p',attrs={'class':'t1 '})
#获取公司名信息
InfoNames = soup.find_all('span',attrs={'class':'t2'})
#获取工作地点
InfoPlaces = soup.find_all('span', attrs={'class': 't3'})
#获取薪资信息
InfoSalarys = soup.find_all('span', attrs={'class': 't4'})
#获取招聘发布条件信息
InfoTimes = soup.find_all('span', attrs={'class': 't5'})
for m in range(1,len(InfoPositions)):
if (len(InfoPositions[m-1].text.split())==0)|(len(InfoNames[m].text.split())==0)|(len(InfoPlaces[m].text.split())==0)|(len(InfoSalarys[m].text.split())==0)|((InfoTimes[m].text.split())==0):
pass
else:
#print(len(InfoPositions),len(InfoNames),len(InfoPlaces),len(InfoSalarys),len(InfoTimes))
#print(InfoPositions[m-1].text.split()[0],InfoNames[m].text.split()[0],InfoPlaces[m].text.split()[0],InfoSalarys[m].text.split()[0],InfoTimes[m].text.split()[0])
RecuitInfos.append([InfoPositions[m-1].text.split()[0],InfoNames[m].text.split()[0],InfoPlaces[m].text.split()[0],InfoSalarys[m].text.split()[0],InfoTimes[m].text.split()[0]])
return RecuitInfos
def writeCSV(i,fw,Recruit_info):
for Info in Recruit_info:
print("正在写入第{0}页".format(i))
fw.write(",".join(Info)+'\n')
print("第{0}数据抓取完毕".format(i))
def main():
path = 'F:'
posttion = input("请输入要抓取的职位名称:")
fw = open(path +'\zhaopin_'+ posttion+'.csv', 'a+')
row = ["职位名","公司名","工作地点","薪资","发布时间"]
fw.write(",".join(row)+"\n")
star_url = 'http://search.51job.com/list/000000,000000,0000,00,9,99,'
mid_url = ',2,'
end_url = '.html?'
max = input("请输入最大抓取页数:")
for i in range(1,int(max)):
time.sleep(3)
url = star_url + posttion +mid_url + str(i) + end_url
htmlText = getHtml(url)
Recruit_info = ParserHtml(i,htmlText)
writeCSV(i,fw, Recruit_info)
if __name__ == '__main__':
main()