</pre><pre code_snippet_id="526653" snippet_file_name="blog_20141120_2_7668334" name="code" class="python"># -*- coding: utf8 -*-
import sys,urllib,re
from HTMLParser import HTMLParser
from bs4 import BeautifulSoup
class MyHTMLParser(HTMLParser):
def __init__(self):
HTMLParser.__init__(self)
self.titleo=''
self.divFlag=False
def handle_data(self,data):
if self.titleo=='':
self.titleo+=data
url="http://www.putclub.com/html/radio/VOA/presidentspeech/index.html"
wp=urllib.urlopen(url)
content=wp.read()
pattern=re.compile('href="/html([\S\s]*?).html')
hrefs=pattern.findall(content)
for i in range(0,len(hrefs)):
##print "http://www.putclub.com/html"+hrefs[i]+".html"
url="http://www.putclub.com/html"+hrefs[i]+".html"
wp=urllib.urlopen(url)
content=wp.read()
pattern=re.compile('<div class="title">(\S*?)</div>')
title=pattern.findall(content)
if len(title)!=0:
#确定编码
soup=BeautifulSoup(content,from_encoding='gb2312')
#查找结点
body=soup.findAll('div',id='textbody')
soup=BeautifulSoup(str(body[0]),from_encoding='gb2312')
paras=soup.findAll('p')
soupt=BeautifulSoup(str(title[0]),from_encoding='gb2312')
demo=MyHTMLParser()
demo.feed(title[0])
#file=open("D:\\Program Files\\presidentspeech\\" + demo.titleo + ".txt",'w')
file=open("D:\\Program Files\\presidentspeech\\" + soupt.get_text().encode('utf8').decode('utf8') + ".txt",'w')
#获取标签内容
file.write(soupt.get_text().encode('utf8')+'\n')
file.write(url+'\n')
#file.write(str(paras))
file.write(soup.get_text().encode('utf8'))
demo.close()
file.close()
print "finish"
python 抓取网页--用BeautifulSoup
最新推荐文章于 2024-03-29 17:51:22 发布