基于beatifulsoup写一下简单的网络爬虫

1、目的,抓取mediawiki中 id="mw-content-text" 的内容,并再向下读取一层,最终可以把数据以文件或是数据库形式保存起来.

import urllib2
import lxml
import time
from bs4 import BeautifulSoup


searchroot = 'http://wiki.t-firefly.com/';
response = urllib2.urlopen("http://wiki.t-firefly.com")


def entrysecparse(url):
    "entry second parse func"
   
    print url
    time.sleep(3)
    soupitem = BeautifulSoup(urllib2.urlopen(url))
    print soupitem.prettify()
    test = soupitem.find_all("div", id="mw-content-text")
    print test

    #function_suite

    return 0

#print response.read()

#soup = BeautifulSoup(open('http://xxxx.t-firefly.com'))
soup = BeautifulSoup(response.read())
#print soup.prettify()
test = soup.find_all("div", id="mw-content-text")
print test
test = test[0];
#print test;
#new = BeautifulSoup(test)
#print new;
href = test.find_all('a')
#href = BeautifulSoup(test).find_all('a')
print href;
for item in href:        
    print 'item :',item
    itemurl = item.get('href')
    print itemurl;
    if itemurl[0] == '/':
        entrysecparse(searchroot + itemurl)