python 爬虫 beautifulsoup example 例子

今天第一次用python的beautifulsoup，虽然比较生疏，但还是爬下来了。

爬的网站是： 网站排行

爬取的内容：包括网站的url, aleax排名，百度权重， PR等

import urllib2
import pandas as pd
from bs4 import BeautifulSoup

def urlprocess(url):
    u = url.replace('/Html/site_','').replace('.html','').replace('/site_www.','www.')
    return u

def baiduprs(baidu):
    return baidu.replace('/themes/default/images/baidu/','').replace('.gif','')

def PRprs(pr):
    return pr.replace('/themes/default/images/ranks/Rank_','').replace('.gif','')

url = 'http://top.chinaz.com/all/index_alexa.html'
res = pd.DataFrame(columns=['host','alex','baidu','pr','Inverse chain number','score','rank'])
for j in range(2,1157):
    print 'j:',j
    html = urllib2.urlopen(url).read()
    url = 'http://top.chinaz.com/all/index_alexa_' + str(j) + '.html'
    soup2 = BeautifulSoup(html)
    temp = soup2.findAll('ul', attrs = {'class':'listCentent'})[0]
    l = len(temp.contents)
    c = temp.contents
    for i in range(1,l-1):
        x = c[i]
        m = {#'introduction':[x.div.next_sibling.div.next_sibling.string],
        'host':[urlprocess(x.div.a.attrs['href']) ],
        'alex':[x.div.next_sibling.div.p.a.string ],
        'baidu':[baiduprs(x.div.next_sibling.div.p.next_sibling.next_sibling.a.img.attrs['src'])],
        'pr':[PRprs(x.div.next_sibling.div.p.next_sibling.next_sibling.next_sibling.next_sibling.img.attrs['src'])],
        'Inverse chain number':[x.div.next_sibling.div.p.next_sibling.next_sibling.next_sibling.next_sibling.\
                next_sibling.next_sibling.a.string],
        'rank':[x.div.next_sibling.next_sibling.div.strong.string],
        'score':[x.div.next_sibling.next_sibling.div.span.string[3:]]}
        res = res.append(pd.DataFrame(m))
res.to_csv('D://host_rank.csv')

结果如下：