python爬虫学习(下)——爬虫代码实现

上篇分析出了数据获取的完整路径，下面对应介绍具体的代码实现

注：代码说明、我的一些总结心得都放到了代码注释里

整个程序主要由以下几个类组成：

Class Car:汽车模型,存储每个车的信息

Class CarFactory:传入获取的网络数据，生产出汽车模型

Class CarSpider:爬虫的主体类，串联整个业务

Class DataSaver:负责数据库操作，数据存储

Class RequestThread:后期我把请求改成了多线程操作，引入了这个类

具体代码及注释：

#CarSpider对象负责抓取数据
spider = CarSpider()
cars = spider.getDatas()

#DataSaver对象负责入库
dataSaver = DataSaver()
dataSaver.updateCarsData(cars)
print "finish"
CarSpider:

class CarSpider:

    def __init__(self):
        #类属性要放到init中
        self.allCars = []

    #对外暴露函数
    def getDatas(self):
        # 根据首字母得到所有车系列表
        carList = self.__getCarSeriesListByInitialChar()
        # 从车型列表中提取出每个车系首页的url
        carUrlList = self.__getAllCarSeriesIndexUrl(carList)
        # 解析出车系配置详情页面url
        urlQueue = self.__getCarSeriesInfoUrls(carUrlList)
        # 获取车辆详情
        cars = self.__getCarsBycarSeriesInfoUrls(urlQueue)
        return cars


    #gzip解压(有的页面是经过gzip压缩的,gzip解压)
    #私有方法，方法名以__开头
    def __gzipDecode(self,response):
        if response.info().get('Content-Encoding') == 'gzip':
            compressedstream = StringIO.StringIO(response.read())
            gziper = gzip.GzipFile(fileobj=compressedstream)
            result = gziper.read()
            return result
        else:
            return response.read()

    #根据首字母得到所有车系列表
    def __getCarSeriesListByInitialChar(self):
        dataList = []
        for i in range(65,91):
            url = 'http://www.autohome.com.cn/grade/carhtml/%c.html'%(chr(i))
            try:
                response = urllib2.urlopen(url)
            except urllib2.HTTPError,e:
                print e.reason
            zipDecodeData = self.__gzipDecode(response)
            data = zipDecodeData.decode("gbk")#返回的数据有中文，需要经过编码转换
            dataList.append(data)
        return dataList

    #从车型列表中提取出每个车系首页的url
    def __getAllCarSeriesIndexUrl(self,dataList):
        carInfoUrlList = []
        #()分组的概念！！！
        pattern = re.compile(r'<h4.*><a href="(.*)">.*</a></h4>', re.M)
        for data in dataList:
            carList = re.findall(pattern, data)
            for carUrl in carList:
                if carUrl.find('greylink') == -1:#greylink意味着车系信息没有价值(不全或陈旧)，丢弃
                    carInfoUrlList.append(carUrl)
        return carInfoUrlList

    #解析出车系配置详情页面
    def __getCarSeriesInfoUrls(self,carUrlList):
        urlQueue = Queue()
        for carUrl in carUrlList:
            #从车系首页url中提取车系id
            pattern = re.compile(r'[0-9]+',re.M)
            carId = re.search(pattern,carUrl).group()
            #根绝总结的规律，拼出配置详情页url
            carSeriesInfoUrl = 'http://car.autohome.com.cn/config/series/%d.html'%(int(carId))
            #print carSeriesInfoUrl
            urlQueue.put(carSeriesInfoUrl)
        return urlQueue

    # 获取车辆详情
    def __getCarsBycarSeriesInfoUrls(self,urlQueue):
        #Queue，线程安全,有阻塞机制
        ouputDataQueue = Queue()
        threads = []
        for i in range(0,4):
            #查了一下，python的多线程是伪多线程，把计算放到线程里意义不大，只把请求放到线程中去做
            requestThread = RequestThread(i,urlQueue,ouputDataQueue)
            threads.append(requestThread)
            #不能在此处join,此处join后，循环是在当前线程中的，下次循环也会被阻塞住
            #requestThread.join()
            requestThread.start()

        for requestThread in threads:
            requestThread.join()

        while ouputDataQueue.qsize() != 0:
            outputData = ouputDataQueue.get()[0]
            seriesInfoUrl = ouputDataQueue.get()[1]
            data = outputData.decode("gbk")
            pattern = re.compile(r'var config = ({.*};)', re.M)
            result = re.findall(pattern, data)
            if len(result) > 0:
                infoJsonStr = result[0][0:-1]
                #json转换
                infoDict = json.loads(infoJsonStr)
                carFactory = CarFactory(infoDict, seriesInfoUrl)
                # carfactory依据infoDict生成car模型
                cars = carFactory.analysisData()
                for car in cars:
                    self.allCars.append(car)
            else:
                continue
            #从队列里get()数据后，数据使用完后要调用task_done(),让队列取消阻塞
            ouputDataQueue.task_done()
        return self.allCars

CarFactory:

class CarFactory:

    def __init__(self,carsData,carSeriesUrl):
        self.carsData = carsData
        self.carSeriesUrl = carSeriesUrl
        self.cars = []
        self.carsNum = 0

    def __setCarsParam(self,param,values):
        for i in range(self.carsNum):
            car = self.cars[i]
            value = values[i]["value"]
            # 反射机制
            setattr(car,param,value)

    def analysisData(self):
        #一个车系中会有多种车型的信息
        specsList = self.carsData["result"]["specsList"]
        for spec in specsList:
            car = Car();
            car.specid = spec["specid"]
            car.infoUrl = self.carSeriesUrl
            self.cars.append(car)
        self.carsNum = len(self.cars)

        paramTypeItems = self.carsData["result"]["paramtypeitems"]
        for paramTypeItem in paramTypeItems:
            paramTypeName = paramTypeItem["name"]
            paramItems = paramTypeItem["paramitems"]
            if paramTypeName == u"基本参数":
                for param in paramItems:
                    paramName = param["name"]
                    values = param["valueitems"]
                    if paramName == u"厂商":
                        self.__setCarsParam("family",values)
                    .......

            elif paramTypeName == u"发动机":
                for param in paramItems:
                    paramName = param["name"]
                    values = param["valueitems"]
                    if paramName == u"排量(L)":
                        self.__setCarsParam("sv", values)
                    elif paramName == u"最大马力(Ps)":
                        self.__setCarsParam("hpower", values)
                    ..........

                    ..........

                    ..........

        return self.cars

Car:

#python中的枚举实现方式之一
class FuelType:
    Gasoline = 0
    Diesel = 1

class GearType:
    MT = 0
    AUTO = 1
    DCT = 2
    CVT = 3
........

class Car:

    def __init__(self):
        #实例变量要定义在init中
        #self.xxx的调用也会触发__setattr__方法
        self.specid = 0L  # id,long型
        self.family = None  # 厂牌
        self.name = None  # 车型
        self.price = u"0万"  # 价格
        self.level = None  # 级别
        self.maxSpeed = 0  # 最高时速
	..........

    #因为CarFactory中用了反射进行属性赋值，而python中每个属性没有独立对应的默认set方法，所以只能把逻辑都写在__setattr__里了
    def __setattr__(self, key, value):
        if key == 'specid':
            self.__dict__[key] = int(value)
        elif key == 'isTurbo':
            if value.find(u'自然') != -1:
                # __dict__ 返回的是一个字典，它的键（key）是属性名，键值（value）是相应的属性对象的数据值
                self.__dict__[key] = False
            else:
                self.__dict__[key] = True
        elif key == 'price':
            #python里数量词默认是贪婪的（在少数语言里也可能是默认非贪婪），总是尝试匹配尽可能多的字符；非贪婪则相反，总是尝试匹配尽可能少的字符。在"*","?","+","{m,n}"后面加上？，使贪婪变成非贪婪。
            pattern = re.compile(ur'(.*?)万.*', re.M)
            result = re.findall(pattern,value)
            if len(result) > 0:
                self.__dict__[key] = float(result[0])
            else:
                self.__dict__[key] = float(0)
        elif key == 'fuleType':
            if value.find(u'汽油') != -1:
                self.__dict__[key] = FuelType.Gasoline
            else:
                self.__dict__[key] = FuelType.Diesel
        elif key == 'gearType':
            pattern = re.compile(ur'手动|MT', re.M)
            result = re.search(pattern,value)
            if result != None:
                self.__dict__[key] = GearType.MT
            else:
                pattern = re.compile(ur'双离合|DSG|DCT|PDK|tronic|MDKG|power shift', re.M)
                result = re.search(pattern, value)
                if result != None:
                    self.__dict__[key] = GearType.DCT
                else:
                    pattern = re.compile(ur'无极|CVT', re.M)
                    result = re.search(pattern, value)
                    if result != None:
                        self.__dict__[key] = GearType.CVT
                    else:
                        self.__dict__[key] = GearType.AUTO
        ..........
        else:
            try:
                number = float(value)
                self.__dict__[key] = number
            except StandardError:
                self.__dict__[key] = value

RequestThread:

class RequestThread(threading.Thread):
    
    def __init__(self,threadId,queue,outputQueue):
        threading.Thread.__init__(self)
        self.threadId = threadId
        self.queue = queue
        self.outputQueue = outputQueue


    def run(self):
	#从url队列中取出一个尚未处理的url
        while self.queue.qsize() != 0:
            url = self.queue.get()
            self.queue.task_done()#get配合task_done，告知queue，内容已取出，可以不阻塞了
            print "%d: url:%s %d\n" % (self.threadId, url, self.queue.qsize())
            try:
                response = urllib2.urlopen(url)
                #print id(response) #response会重复???
                #此处返回data外，还将车系url返回（url会存回数据库）
                data = self.gzipDecode(response)
                self.outputQueue.put((data,url))

            except urllib2.URLError,e:
                print e.reason

DataSaver:

class DataSaver:

    def __init__(self):
        self.db = sqlite3.connect("CarDB.sqlite")
        print self.db


    def updateCarsData(self,cars):
        #先删除之前所有数据，简单粗暴
        self.db.execute("delete from Cars")
        for car in cars:
            self.db.execute("insert into Cars (specid,name,family,price,level,maxSpeed,accelerate,sv,hpower,mpower) VALUES (?,?,?,?,?,?,?,?,?,?)",[car.specid,car.name,car.family,car.price,car.level,car.maxSpeed,car.accelerate,
                                                                                           car.sv,car.hpower,car.mpower])
            self.db.commit()

        self.db.close()

以上便是爬虫的主要代码，本人刚刚接触python,文中若有错误或不妥之处，望大家多多指教，谢谢。