python:xml爬取百度贴吧图片

使用xml爬取百度贴吧的图片

# -*- coding:utf-8 -*-

import urllib2
import urllib
from lxml import etree
import os

#定义贴吧爬虫类
class TiebaSpider:
    #爬虫的调度器
    def tiebaSpider(self , url , beginPage , endPage):

        for page in range(beginPage , endPge + 1):
            pn = (page - 1) * 50

            fullurll = url + "&pn=" + str(pn)

            #拿到每个帖子的链接
            self.loadPage(fullurl)

            print "谢谢使用"

    #拿到每一页帖子的集合
    def loadPage(self , url):
        headers = {"User-Agent" :"Mozilla/5.0(compatible;MSIE9.0;WindowsNT6.    1;Trident/5.0"}

        request = urllib2.Request(url , headers = headers)
        html = urllib2.urlopen(request).read()

        #解析为dom模型，并使用xml提取帖子链接
        content = etree.HTML(html)

        link_list = content.xpath('//div[@class="t_con cleafix"]/div/div/div/a/@href')

        for link in link_list:
            #拼接完整的帖子链接
            fulllink = "http://tieba.baidu.com" + link

            #下载图片
            self.loadImage(fulllink)

    def loadImage(self , link):

        headers =  {"User-Agent" :"Mozilla/5.0(compatible;MSIE9.0;WindowsNT6.    1;Trident/5.0"}

        request = urllib2.Request(link , headers = headers)

        response = urllib2.urlopen(request)

        html = response.read()

        content = etree.HTML(html)

        link_list = content.xpath('//div/img[@class="BDE_Image"]/@src')


        for link in link_list:
            self.writeImage(link)


    def writeImage(self , link):

        headers = {"User-Agent" :"Mozilla/5.0(compatible;MSIE9.0;WindowsNT6.    1;Trident/5.0"}

        request = urllib2.Request(link , headers = headers)

        image = urllib2.urlopen(request).read()

        #设定图片的名称
        filename = link[-10:]

        #指定文件夹
        dir = "pic"

        #判断文件夹是否存在，不存在就创建
        if not os.path.exists(dir):
            os.makedir(dir)

        #向硬盘写入图片
        with open("pic/" + filename , "wb") as f :
            f.write(image)

        print "已经成功下载" + filename


if __name__ == "__main__":

    #用户需要输入的接口数据
    kw = raw_input("请输入需要爬取的贴吧名：")

    beginPage = int(raw_input("请输入起始页："))

    endPage = int(raw_input("请输入结束页："))

    url = "http://tieba.baidu.com/f?"

    key = urllib.urlencode({"kw" : kw})

    fullurl = url + key

    #创建爬虫对象实例
    tiebaSpider = TiebaSpider()

    #开始爬取数据
    tiebaSpider.tiebaSpider(fullurl , beginPage , endPage)