1 #-*- coding:utf-8 -*-
2
3 importurllib4 importurllib25 from bs4 importBeautifulSoup6 importre7 importxlwt8
9 #initUrl = 'http://www.lagou.com/zhaopin/Python/?labelWords=label'
10 defInit(skillName):11 totalPage = 30
12 initUrl = 'http://www.lagou.com/zhaopin/'
13 #skillName = 'Java'
14 userAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'
15 headers = {'User-Agent':userAgent}16
17 #create excel sheet
18 workBook = xlwt.Workbook(encoding='utf-8')19 sheetName = skillName + 'Sheet'
20 bookSheet =workBook.add_sheet(sheetName)21 rowStart =022 for page inrange(totalPage):23 page += 1
24 print '##################################################### Page',page,'#####################################################'
25 currPage = initUrl + skillName + '/' + str(page) + '/?filterOption=3'
26 #print currUrl
27 try:28 request = urllib2.Request(currPage,headers=headers)29 response =urllib2.urlopen(request)30 jobData =readPage(response)31 #rowLength = len(jobData)
32 for i,row inenumerate(jobData):33 for j,col inenumerate(row):34 bookSheet.write(rowStart +i,j,col)35 rowStart = rowStart + i +1
36 excepturllib2.URLError,e:37 if hasattr(e,"code"):38 printe.code39 if hasattr(e,"reason"):40 printe.reason41 xlsName = skillName + '.xls'
42 workBook.save(xlsName)43
44 defreadPage(response):45 btfsp =BeautifulSoup(response.read())46 webLinks = btfsp.body.find_all('div',{'class':'p_top'})47 #webLinks = btfsp.body.find_all('a',{'class':'position_link'})
48 #print weblinks.text
49 count = 1
50 jobData =[]51 for link inwebLinks:52 print 'No.',count,'==========================================================================================='
53 pageUrl = link.a['href']54 jobList =loadPage(pageUrl)55 #print jobList
56 jobData.append(jobList)57 count += 1
58 returnjobData59
60 defloadPage(pageUrl):61 currUrl = 'http:' +pageUrl62 userAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'
63 headers = {'User-Agent':userAgent}64 try:65 request = urllib2.Request(currUrl,headers=headers)66 response =urllib2.urlopen(request)67 content =loadContent(response.read())68 returncontent69 excepturllib2.URLError,e:70 if hasattr(e,"code"):71 printe.code72 if hasattr(e,"reason"):73 printe.reason74
75 defloadContent(pageContent):76 #print pageContent
77 btfsp =BeautifulSoup(pageContent)78 #job infomation
79 job_detail = btfsp.find('dl',{'id':'job_detail'})80 jobInfo =job_detail.h1.text81 tempInfo = re.split(r'(?:\s*)',jobInfo) #re.split is better than the Python's raw split function
82 jobTitle = tempInfo[1]83 jobName = tempInfo[2]84 job_request = job_detail.find('dd',{'class':'job_request'})85 reqList = job_request.find_all('p')86 jobAttract = reqList[1].text87 publishTime = reqList[2].text88 itemLists = job_request.find_all('span')89 salary =itemLists[0].text90 workplace = itemLists[1].text91 experience = itemLists[2].text92 education = itemLists[3].text93 worktime = itemLists[4].text94
95 #company's infomation
96 jobCompany = btfsp.find('dl',{'class':'job_company'})97 #companyName = jobCompany.h2
98 companyName = re.split(r'(?:\s*)',jobCompany.h2.text)[1]99 companyInfo = jobCompany.find_all('li')100 #workField = companyInfo[0].text.split(' ',1)
101 workField = re.split(r'(?:\s*)|(?:\n*)',companyInfo[0].text)[2]102 #companyScale = companyInfo[1].text
103 companyScale = re.split(r'(?:\s*)|(?:\n*)',companyInfo[1].text)[2]104 #homePage = companyInfo[2].text
105 homePage = re.split(r'(?:\s*)|(?:\n*)',companyInfo[2].text)[2]106 #currStage = companyInfo[3].text
107 currStage = re.split(r'(?:\s*)|(?:\n*)',companyInfo[3].text)[1]108 financeAgent = ''
109 if len(companyInfo) == 5:110 #financeAgent = companyInfo[4].text
111 financeAgent = re.split(r'(?:\s*)|(?:\n*)',companyInfo[4].text)[1]112 workAddress = ''
113 if jobCompany.find('div',{'class':'work_addr'}):114 workAddress = jobCompany.find('div',{'class':'work_addr'})115 workAddress = ''.join(workAddress.text.split()) #It's sooooo cool!
116
117 #workAddress = jobCompany.find('div',{'class':'work_addr'})
118 #workAddress = ''.join(workAddress.text.split()) # It's sooooo cool!
119
120 infoList =[companyName,jobTitle,jobName,salary,workplace,experience,education,worktime,jobAttract,publishTime,121 workField,companyScale,homePage,workAddress,currStage,financeAgent]122
123 returninfoList124
125 defSaveToExcel(pageContent):126 pass
127
128 if __name__ == '__main__':129 #Init(userAgent)
130 Init('Python')