1.首先定义HTMLParser处理类MyHTMLParser ''' Created on 2011-1-11 @author: Administrator ''' from HTMLParser import HTMLParser class MyHTMLParser(HTMLParser): def __init__(self): HTMLParser.__init__(self) self.flag = 0 self.links = [] self.title="" self.img="" self.content="" def handle_starttag(self, tag, attrs): #print "Encountered the beginning of a %s tag" % tag if tag == "a": if len(attrs) == 0: pass else: for (variable, value) in attrs: if variable == "href": self.links.append(value) if tag == "ul": if len(attrs)==0:pass else: for(varviable,value) in attrs: if varviable=="class" and value=="info": self.flag=1 if tag=="meta": if len(attrs)==0:pass else: for (variable,value) in attrs: if variable=="content": self.title=value if tag=="img": if len(attrs)==0:pass else: img_host=u'http://pic.58.com/' for(variable,value) in attrs: if variable=="src" and img_host in value: if self.img=="": self.img=value if tag=="p": self.flag=2 if tag=="br": self.flag=2 def handle_data(self,data): if self.flag == 1: self.content=data self.flag==0 if self.flag==2: self.content+=data self.flag=0 2.控制类UrlParser ''' Created on 2011-1-11 @author: Administrator ''' import re import urllib from MyHTMLParser import MyHTMLParser class UrlParser: def __init__(self): self.detail_list=[] def get_urllist(self): url_dir=r='D:/Documents and Settings/Administrator/workspace/mysite/src/CategoryInfo/url.txt' f=open(url_dir) self.urllist=f.readlines() f.close() for path in self.urllist: print path data=urllib.urlopen(path).read().decode('utf-8') hp = MyHTMLParser() hp.feed(data) hp.close() for link in hp.links: if re.search('x/.shtml', link) is not None: self.addDetailUrl(link) print link def addDetailUrl(self,url): self.detail_list.append(url) def get_detail(self,url): patt=r'http://gy.58.com/(/S+)/(/S+)/.shtml' m=re.search(patt, url) if m is not None: self.id=m.group(1) detail_data=urllib.urlopen(url).read().decode('utf-8') hp = MyHTMLParser() hp.feed(detail_data) self.title=hp.title self.img=hp.img self.content=hp.content print '/nid:'+self.id+'/ntitle:'+self.title+"/nimg:"+self.img+'/ncontent:'+self.content if __name__ == "__main__": upr=UrlParser() #upr.get_urllist() upr.get_detail(r'http://gy.58.com/duanzu/3934202413954x.shtml') 3.初始化数据 http://gy.58.com/xiaoshouyuan http://gy.58.com/xiaoshouzhuli http://gy.58.com/xiaoshoujingli http://gy.58.com/xiaoshouzongjian http://gy.58.com/dianhuaxiaoshou http://gy.58.com/xiaoshougongchengshi http://gy.58.com/fangdichan http://gy.58.com/baoxian http://gy.58.com/qichexiaoshou http://gy.58.com/yiyaodaibiao