爬取51job的爬虫（python）

最新推荐文章于 2024-08-26 23:54:56 发布

猎摘互联网软件测试业界技术文章专用博客

最新推荐文章于 2024-08-26 23:54:56 发布

阅读量763

点赞数

分类专栏： selenium功能自动化测试

本文链接：https://blog.csdn.net/cyjs1988/article/details/74740561

版权

selenium功能自动化测试专栏收录该内容

110 篇文章 9 订阅

订阅专栏

#coding=utf-8
__author__ = "carry(QQ:314375317)"


import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import urllib
import urllib2
import re


#获取原码
def get_content(page):
    headers = {'Host':'search.51job.com',
               'User-Agent':'Mozilla/5.0 (Windows NT 6.2; rv:16.0) Gecko/20100101 Firefox/16.0',
               'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
               'Connection':'keep-alive'
               }
    url ='http://search.51job.com/list/000000,000000,0000,00,9,99,python,2,'+ str(page)+'.html'
    req = urllib2.Request(url,headers=headers)
    r = urllib2.urlopen(req)
    response = r.read() #读取源代码并转为unicode
    html = response.decode('gbk').encode('utf-8')
    return html

def get(html):
    reg = re.compile(r'class="t1 ">.*? <a target="_blank" title="(.*?)".*? <span class="t2"><a target="_blank" title="(.*?)".*?<span class="t3">(.*?)</span>.*?<span class="t4">(.*?)</span>.*? <span class="t5">(.*?)</span>',re.S)#匹配换行符
    items=re.findall(reg,html)
    return items

#多页处理，下载到文件
for  j in range(1,10):
    print(u"正在爬取第"+str(j)+"页数据...")
    html = get_content(j) #调用获取网页原码
    for i in get(html):
        #print(i[0],i[1],i[2],i[3],i[4])
        with open ('51job.txt','a') as f:
            f.write(i[0]+'\t'+i[1]+'\t'+i[2]+'\t'+i[3]+'\t'+i[4]+'\n')
            f.close()