# -*- coding:utf-8 -*-
import re
import chardet
import xlwt
import urllib2
import random
import json
#设计一个函数,只用来获取网页内容
count = 0
def getHtml(url):
USER_AGENTS = [
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10",
"Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/533.17.8 (KHTML, like Gecko) Version/5.0.1 Safari/533.17.8",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.1.17) Gecko/20110123 (like Firefox/3.x) SeaMonkey/2.0.12",
"Mozilla/5.0 (Windows NT 5.2; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 SeaMonkey/2.7.1",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.8 (KHTML, like Gecko) Chrome/4.0.302.2 Safari/532.8",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.464.0 Safari/534.3",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.15 Safari/534.13",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.54 Safari/535.2",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7",
"Mozilla/5.0 (Macintosh; U; Mac OS X Mach-O; en-US; rv:2.0a) Gecko/20040614 Firefox/3.0.0 ",
"Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10.5; en-US; rv:1.9.0.3) Gecko/2008092414 Firefox/3.0.3",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-US; rv:1.9.1) Gecko/20090624 Firefox/3.5",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2.14) Gecko/20110218 AlexaToolbar/alxf-2.0 Firefox/3.6.14",
"Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10.5; en-US; rv:1.9.2.15) Gecko/20110303 Firefox/3.6.15",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1"]
# 代理IP
proxies = ['116.213.98.6:8080','122.114.31.177:808','121.31.195.145:8123','125.118.147.0:808']
# 设置请求对象
req = urllib2.Request(url)
# 设置请求头部信息
req.add_header('User-Agent', random.choice(USER_AGENTS))
# 设置代理IP地址
proxy_support = urllib2.ProxyHandler({"http": random.choice(proxies)})
opener = urllib2.build_opener(proxy_support)
urllib2.install_opener(opener)
# 访问并获取服务端返回的对象
try:
res = urllib2.urlopen(req)
html = res.read()
return html
except:
global count
count+=1
if(count>20):
print count,json.dumps('请检查网络设置',ensure_ascii=False)
return
getHtml(url) #自己执行自己
#设计一个函数,返回正则处理后的对象
def getDataList(pageNum,jobName):
# 1.获取51job职位搜索网页内容
url = "https://search.51job.com/list/040000,000000,0000,00,9,99,"+str(jobName)+",2,"+str(pageNum)+".html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare="
html = getHtml(url)
print chardet.detect(html) #获取网页编码
#获取网页对象 编码,进行转码
code = chardet.detect(html)["encoding"] #'GB2312'中文编码
html = html.decode(code).encode('utf-8')
#使用 chardet模块
# 2.设置正则表达式 去掉的内容 .*? 需要获取的内容 (.*?)
regstr = '<p class="t1 ">.*?<a target="_blank" title="(.*?)".*?<span class="t2"><a target="_blank" title="(.*?)".*?<span class="t3">(.*?)</span>.*?<span class="t4">(.*?)</span>.*?<span class="t5">(.*?)</span>'
reg = re.compile(regstr,re.S) #多行匹配
# <p class="t1 ">.*?<a target="_blank" title="(.*?)" .*?</p>
# 获取数据结果
result = re.findall(reg,html)
return result
# 全局的数据列表datalist
datalist = []
# 向全局datalist内添加数据
def deal(pageNum,jobName):
global datalist
#根据设置的页数,用循环,在循环内多次执行获取数据
for k in range(pageNum):
data = getDataList(k+1,jobName)
for i in range(len(data)):
datalist.append(data[i])
# 设置存储的函数
def saveExcel(jobname,filename):
#存到本地
# 1.创建一个工作薄
book = xlwt.Workbook(encoding='utf-8')
# 2.创建一个工作表
sheet = book.add_sheet('51job-'+str(jobname)+'-职位信息')
# 3.存入第一行
col = ('职位名', '公司名', '工作地点', '薪资', '发布时间')
for i in range(len(col)):
sheet.write(0,i,col[i])
for i in range(len(datalist)):
for j in range(len(datalist[i])):
sheet.write(i+1,j,datalist[i][j])
# 07. 存储到文件
book.save(filename)
#保存为txt文本
def saveTxt(filename):
for i in range(0, len(datalist)):
data = datalist[i]
with open(filename, 'a') as f:
f.write(data[0] + '\t' + data[1] + '\t' + data[2] + '\t' + data[3] + '\t' + data[4] + '\n')
f.close()
#设置主函数
def main(jobName,pageNum,filename):
deal(pageNum, jobName)
if 'txt' in filename:
saveTxt(filename)
if 'xls' in filename:
saveExcel(jobName,filename)
#filename使用u''
main('人工智能',2,u'人工智能职位信息2.txt')
python27爬取51job网站数据
最新推荐文章于 2024-08-26 23:54:56 发布