web测试常用python代码——爬虫程序

#coding=utf-8
#爬虫程序——起点
'''
Created on 2012-4-18

@author: xxx
'''

import urllib2
import Queue
import threading
import time
import socket
import sgmllib

urls = ['http://www.qidian.com/Book/%d.aspx/' % i for i in range(0, 1000000)]
threadsNum = 100
#设置线程栈大小
threading.stack_size(32768 * 16)
#设置连接超时
socket.setdefaulttimeout(10)

class BookSpider(sgmllib.SGMLParser):
    def __init__(self, threadsNum):
        self.opener = urllib2.build_opener(urllib2.HTTPHandler)
        self.lock = threading.Lock()
        #请求队列
        self.requestQueue = Queue.Queue()
        #完成队列
        self.completeQueue = Queue.Queue()
        self.runThreadsNum = 0
        for i in range(threadsNum):
            tmpThread = threading.Thread(target = self.threadRun)
            tmpThread.daemon = True
            tmpThread.start()

    def __del__(self):
        time.sleep(2)
        #等待两个队列结束
        self.requestQueue.join()
        self.completeQueue.join()

    def taskLeft(self):
        return self.requestQueue.qsize() + self.completeQueue.qsize() + self.runThreadsNum

    def push(self, request):
        self.requestQueue.put(request)

    def pop(self):
        return self.completeQueue.get()

    def threadRun(self):
        while True:
            request = self.requestQueue.get()
            with self.lock:
                self.runThreadsNum += 1
            try:
                result = self.opener.open(request).read()
                self.completeQueue.put((request, result))
            except Exception:
                time.sleep(0.1)
            with self.lock:
                self.runThreadsNum -= 1
            self.requestQueue.task_done()
            time.sleep(0.1)

class MyParser(sgmllib.SGMLParser):
    def __init__(self):
        sgmllib.SGMLParser.__init__(self)
        self.text_meta = []
        self.text_br = []
        self.is_b = 0

    def start_meta(self, attrs):
        for attr in attrs:
            if attr[0] == 'title':
                self.text_meta.append(attr[1])

    def start_b(self, attrs):
        for attr in attrs:
            if (attr[0] == 'style') and (attr[1] == 'color:Red; display:none'):
                self.is_b = 1

    def end_b(self):
        if self.is_b == 1:
            self.is_b = 2

    def unknown_starttag(self, tag, attrs):
        if (self.is_b == 2) and (tag != 'br'):
            self.is_b = 0

    def handle_data(self, text):
        if (self.is_b == 2):
            self.text_br.append(text.strip())

if __name__ == '__main__':
    spider = BookSpider(threadsNum)
    for url in urls:
        spider.push(url)
    while spider.taskLeft():
        url, contents = spider.pop()
        myParser = MyParser()
        myParser.feed(contents)
        writeFile = open('qidian.txt', 'a')
        for i in myParser.text_meta:
            writeFile.write(url + '\n')
            writeFile.write(i + '\n')
        for i in myParser.text_br:
            writeFile.write(i + '\n')
        writeFile.close()