案例:使用XPath的爬虫
现在我们用XPath来做一个简单的爬虫,我们尝试爬取某个贴吧里的所有帖子,并且将该这个帖子里每个楼层发布的图片下载到本地。
# tieba_xpath.py #!/usr/bin/env python# -*- coding:utf-8 -*- import os import urllib import urllib2 from lxml import etree class tieba_xpath: def __init__(self): self.tiebaName = raw_input("请需要访问的贴吧:") self.beginPage = int(raw_input("请输入起始页:")) self.endPage = int(raw_input("请输入终止页:")) self.url = 'http://tieba.baidu.com/f' self.ua_header = {"User-Agent" : "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1 Trident/5.0;"} # 图片编号 self.userName = 1 def tiebaSpider(self): for page in range(self.beginPage, self.endPage + 1