python 抓取网页--用BeautifulSoup

</pre><pre code_snippet_id="526653" snippet_file_name="blog_20141120_2_7668334" name="code" class="python"># -*- coding: utf8 -*-
import sys,urllib,re
from HTMLParser import HTMLParser
from bs4 import BeautifulSoup


class MyHTMLParser(HTMLParser):
    def __init__(self):
        HTMLParser.__init__(self)
        self.titleo=''
        self.divFlag=False


    def handle_data(self,data):
        if self.titleo=='':
            self.titleo+=data


url="http://www.putclub.com/html/radio/VOA/presidentspeech/index.html"
wp=urllib.urlopen(url)
content=wp.read()


pattern=re.compile('href="/html([\S\s]*?).html')
hrefs=pattern.findall(content)


for i in range(0,len(hrefs)):
    ##print "http://www.putclub.com/html"+hrefs[i]+".html"
    url="http://www.putclub.com/html"+hrefs[i]+".html"
    wp=urllib.urlopen(url)
    content=wp.read()
    pattern=re.compile('<div class="title">(\S*?)</div>')
    title=pattern.findall(content)


    if len(title)!=0:
        #确定编码
        soup=BeautifulSoup(content,from_encoding='gb2312')
        #查找结点
        body=soup.findAll('div',id='textbody')
        soup=BeautifulSoup(str(body[0]),from_encoding='gb2312')
        paras=soup.findAll('p')
        soupt=BeautifulSoup(str(title[0]),from_encoding='gb2312')
        
        demo=MyHTMLParser()
        demo.feed(title[0])


        #file=open("D:\\Program Files\\presidentspeech\\" + demo.titleo + ".txt",'w')
        file=open("D:\\Program Files\\presidentspeech\\" + soupt.get_text().encode('utf8').decode('utf8') + ".txt",'w')
        #获取标签内容
        file.write(soupt.get_text().encode('utf8')+'\n')
        file.write(url+'\n')
        #file.write(str(paras))
        file.write(soup.get_text().encode('utf8'))
        demo.close()
        file.close()


print "finish"