使用QNetworkManager爬取反倒链网站的图片

#coding=utf-8

__author__ = 'ds'

from PyQt4.QtCore import *
from PyQt4.QtGui import *
from PyQt4.QtNetwork import *
from PyQt4.QtWebKit import *
import os,sys

DEFAULT_DIR = ur'F:\图片\美女'

#定制网络管理模块，从网络模块中筛选出定制的数据部分
class NetworkManager(QNetworkAccessManager):
    def __init__(self, parent = None):
        super(NetworkManager, self).__init__(parent)
        self._buffer = {}
        self._nextID = 1

    def createRequest(self, QNetworkAccessManager_Operation, request, QIODevice_device=None):
        if self.isSaveURL(request.url()):
            self._buffer[self._nextID] = QByteArray()
            request.setAttribute(QNetworkRequest.User, self._nextID)
        else:
            request.setAttribute(QNetworkRequest.User, 0)
        self._nextID += 1

        reply = super(NetworkManager, self).createRequest(QNetworkAccessManager_Operation, request, QIODevice_device)

        if reply.url().scheme() == 'http':
            reply.readyRead.connect(self.onReadyRead)
            reply.finished.connect(self.onFinish)

        return reply

    def isSaveURL(self, url):
        url = url.toString()
        rx = QRegExp(u'\\d+/\\d+\\d+\.jpg')
        if url.indexOf(rx) == -1:
            return False

        return True

    def onReadyRead(self):
        reply = self.sender()
        qvar = reply.request().attribute(QNetworkRequest.User, QVariant())
        id = qvar.toInt()[0]
        if not id:
            return

        #因为加载顺序是异步的，所以不能使用直接缓存，最后直接在finish里面使用
        #因为每次的readReady可能来自不能的文件，通过文件来源进行键值对的缓存
        size = reply.size()
        if id:
            self._buffer[id].append(reply.peek(size))

    def onFinish(self):
        reply = self.sender()
        req = reply.request()
        if not req:
            print('!!!!碰到了无效的请求 :%s' % (reply.url()))
            sys.exit(-10)
            return

        qvar = req.attribute(QNetworkRequest.User, QVariant())
        if qvar.isNull():
            print('无效数据:%s' % (reply.url()))
            return

        id = qvar.toInt()[0]
        #id不是0的时候为有效的数据
        if not id or id not in self._buffer:
            return

        url = reply.url().toString()
        segments = url.split('/')
        file_name = unicode(segments[-1])
        file_dirname = segments[-3] + os.path.sep + segments[-2]
        file_dir = os.path.join(DEFAULT_DIR, unicode(file_dirname))
        if not os.path.isdir(file_dir):
            os.makedirs(file_dir)

        file_fullname = os.path.join(file_dir, file_name)
        f = QFile(QString(file_fullname))
        ok = f.open(QFile.WriteOnly)
        if not ok:
            print('打开文件失败:%s' % (file_fullname))
            return

        f.write(self._buffer[id])
        f.close()
        self._buffer.pop(id)
        print('成功下载文件到:%s 来自:%s' % (file_fullname, url))

#定制QWebPage，在下载完成当前页面后，继续下一页
class WebPage(QWebPage):
    def __init__(self, parent = None):
        super(WebPage, self).__init__(parent)
        self.loadFinished.connect(self.onLoadFinish)
        self._albumn = []
        self._timer = QTimer()
        #3ms处理一次
        self._timer.setInterval(10000)
        self._timer.timeout.connect(self.onTimeout)
        #当前的URL地址
        self._current = QUrl()

    def start(self):
        if len(self._albumn) > 0:
            curAlbum = self._albumn[0]
            self._albumn = self._albumn[1:]
            self.loadURL(curAlbum)

    def loadURL(self,url):
        self._timer.stop()
        self._current = QUrl(url)
        self.mainFrame().load(self._current)
        self._timer.start()

    def setAlbumns(self, albumn):
        self._albumn = albumn

    def onTimeout(self):
        #重新加载
        self.loadURL(self._current)
        print('刷新页面:%s' % (self._current.toString()))

    def onLoadFinish(self, ok):
        frame = self.mainFrame()
        last_url = frame.url()
        if not ok:
            print('当前页面加载失败，尝试重新加载:%s' % (last_url,))
            #加载失败的话，重新加载
            self.onTimeout()
            return

        if self._timer.isActive():
            print('停止当前的定时器。')
            self._timer.stop()

        pages = frame.findFirstElement('#pages')
        if not pages:
            print('当前页面不存在下一页 %s' % (last_url,))
            sys.exit(-1)

        next_page = None
        pages = pages.findAll('a.a1')
        if len(pages) == 0:
            print('找不到上-下页标签%s' % (last_url,))
            sys.exit(-2)

        for p in pages:
            if p.toPlainText() == u'下一页':
                next_page = p
                break

        if not next_page:
            print('找不到下一页:%s' % (last_url,))
            sys.exit(-3)

        href = next_page.attribute(u'href')
        print(str(href))
        rx = QRegExp(u'/g/\\d+/\\d+')
        if href.indexOf(rx) == -1:
            print('已经加载到最后')
            #继续加载剩余的地址
            if len(self._albumn) > 0:
                self.start()
                return
            else:
                sys.exit(0)

        last_url.setPath(href)
        self.loadURL(last_url)
        print('加载下一页地址:%s' % (last_url.toString(),))

if __name__ == '__main__':
    reload(sys)
    sys.setdefaultencoding('utf-8')
    app = QApplication(sys.argv)
    #app.setQuitOnLastWindowClosed(True)

    #PAGES = [ '19019','18812','16751', '13207', '13206', '13205', '13148','11363','18214',]
    PAGES = [ '19019','18812',]
    pages = []
    for p in PAGES:
        url = QUrl(u'http://www.zngirls.com/g/%s' % (p,))
        pages.append(url)

    page = WebPage()
    page.setNetworkAccessManager(NetworkManager())
    page.setAlbumns(pages)

    #wv = QWebView()
    #wv.setWindowTitle(u'下载高清图片')
    #wv.setPage(page)
    #wv.show()
    page.start()

    sys.exit(app.exec_())