网络爬虫-python-爬取天涯求职贴

使用urllib请求页面,使用BeautifulSoup解析页面,使用xlwt3写入Excel

import urllib.request
from bs4 import BeautifulSoup
import time
import xlwt3
from xlrd import open_workbook
wExcel=xlwt3.Workbook()
sheet1=wExcel.add_sheet('my',cell_overwrite_ok=True)
num=0
fo=open(r'contents.txt','a',encoding='utf-8')
def getconten(url):
    opener = urllib.request.build_opener()
    try:
        content = opener.open(url).read()
        content2=content.decode('utf-8')
    except:
        try:
            content = opener.open(url).read()
            content2=content.decode('gbk')
        except:
            print('decode fail!')
            return None
        return None
    return content2
def getdetail(url):
    opener = urllib.request.build_opener()
    con=getconten(url)
##    print(url)
    if con:
        soup=BeautifulSoup(con)
        job=soup.find('div','bbs-content clearfix')
        if job:
            jobdetail=job.get_text()
            return jobdetail
    else:
        return None

def getonepage(url):
    global num
    opener = urllib.request.build_opener()
    content=getconten(url)
    if content:
        soup=BeautifulSoup(content)
    for tr in soup.find_all('tr','bg'):
        oneitem=[]
        j=0
        detailurl=tr.td.a['href']
        detailurl='http://bbs.tianya.cn'+detailurl
##        print(detailurl)
        detailcon=getdetail(detailurl)
##        print(detailcon)
        for item in tr.strings:
            item=item.strip()
            if item:
                oneitem.append(item)
                sheet1.write(num,j,item)
                j=j+1
##            print(item.strip())
        sheet1.write(num,j,detailcon)
        num=num+1
##        print('one is ok')



if __name__=='__main__':
    mainpage='http://bbs.tianya.cn/list.jsp?item=763&sub=2'
    getonepage(mainpage)
    wExcel.save('res0.xls')
    i=0
    soup=BeautifulSoup(getconten(mainpage))
    currentpage=soup.find('div','links').a.find_next_sibling('a')
    currentpage='http://bbs.tianya.cn'+currentpage['href']
    nextpage=currentpage
    while i<30:
        print(nextpage)
        getonepage(nextpage)
        print('one page finished!')
        con=getconten(nextpage)
        if con:
            soup=BeautifulSoup(con)
            currentpage=soup.find('div','links').a.find_next_sibling('a').find_next_sibling('a')
            nextpage='http://bbs.tianya.cn'+currentpage['href']
            i=i+1
        else:
            break
    wExcel.save('res.xls')
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值