基于宽度优先爬虫

# -*- coding: utf-8 -*-
from selenium import webdriver
import time
#from bs4 import BeautifulSoup
import urllib2
from selenium.webdriver.remote.webelement import WebElement
from selenium.common.exceptions import StaleElementReferenceException
import socket
import re
import zlib
import httplib,urllib,urlparse
import MySQLdb
import sys
reload(sys)
sys.setdefaultencoding('utf8')




#链接判断有效性
'''
def urlStat(urls):
    #errurl = []
    try:
        r = urllib.urlopen(url)
    except urllib.URLError as e:
        r = e
        return r.code


    for url in urls:
            up = urlparse.urlparse(url)
            #因为超链接有两种方式：一种是直接的http://...... 一种是相对路径，/.../sample.html
          #所以要分别处理


            if up.netloc =="" or up.scheme =="":
                #域名为空，直接抛弃掉，不作处理


                http = httplib.HTTP(uhost.netloc)
                http.putrequest("GET", "/"+up.path+"?"+up.params+up.query+up.fragment)
                http.putheader("Accept", "*/*")
                http.endheaders()


                continuee
            else:


                http = httplib.HTTP(up.netloc)
                http.putrequest("GET", up.path+"?"+up.params+up.query+up.fragment)
                http.putheader("Accept", "*/*")
                http.endheaders()


                try:
                    r = urllib.urlopen(url)
                except urllib.URLError as e:
                    r = e
            #errcode, errmsg, headers = http.getreply()


                if r.code in (200,401):
                    return r.code


                #rurl.append(url)
                    print url," : ok"
                else:
                    #errurl.append(errurl)
                    #print url," : ",r.code


 '''
class linkQueue:
    def __init__(self):
        #已访问的URL集合
        self.visited = []
        #待访问的URL集合
        self.unvisited = []


    #获取访问过的URL队列
    def getVisitedUrl(self):
        return self.visited
    #获取未访问过的队列
    def getUnvisitedUrl(self):
        return self.unvisited
    #添加URL到访问过的队列
    def addVisitedUrl(self,url):
        self.visited.append(url)
    #移除已访问队列中URL
    def removeVisitedUrl(self,url):
        self.visited.remove(url)
    #未访问过的URL出队列
    def unVisitedUrlDeQueue(self):
        try:
            return self.unvisited.pop()
        except:
            return None
    #保证每个URL只被访问一次
    def addUnvisitedUrl(self,url):
        if url!="" and url not in self.visited and url not in self.unvisited:#不为空并且不再已访问队列并且不再未访问队列
            self.unvisited.insert(0,url)


    #获取已访问的URL数量
    def getVisitedUrlCount(self):
        return len(self.visited)
    #获取未访问的URL数量
    def getUnvisitedUrlCount(self):
        return len(self.unvisited)
    #获取临时队列大小
    def getTempUrlCount(self):
        return len(self.temp)


    #判断未访问的URL队列是否为空
    def unVisitedUrlEmpty(self):
        return len(self.unvisited) == 0


class Crawler:
    def __init__(self,seeds):
        #self.phantomjs_path = r'D:\python\phantomjs-2.1.1-windows\bin\phantomjs.exe'
        #self.driver = webdriver.PhantomJS(self.phantomjs_path)
        #初始化爬虫深度，爬虫深度普遍17层
        self.current_deepth = 1
        self.temp = [] #临时队列
        self.linkQueue = linkQueue()#实例化队列


        if isinstance(seeds,str):  #字符串，传人单个URL
            self.linkQueue.addUnvisitedUrl(seeds)
        if isinstance(seeds,list): #列表，传入多个URL
            for l in seeds:
                self.linkQueue.addUnvisitedUrl(l)
        print "Add the seeds url \"%s\" to the unvisited url list"%str(self.linkQueue.unvisited)
    def Crawling(self,seeds,crawl_deepth):
        #循环条件：抓取深度不超过crawl_deepth
        while self.current_deepth <= crawl_deepth:
            #循环条件：待抓取的链接不为空
            while not self.linkQueue.unVisitedUrlEmpty():
                #队头URL出队列
                visitUrl = self.linkQueue.unVisitedUrlDeQueue()
                print "Pop out one url \"%s\" from unvisited url list"%visitUrl
                if visitUrl is None or visitUrl == "":
                    continue
                #获取超链接
                #links = self.getHref(visitUrl)
                links = self.getUrl(visitUrl)
                print "Get %d new links"%len(links)
                #将URL放入已访问的URL队列中
                self.linkQueue.addVisitedUrl(visitUrl)
                print "Visited url count: "+str(self.linkQueue.getVisitedUrlCount())
                print "Unvisited url count: "+str(self.linkQueue.getUnvisitedUrlCount())
                print "Visited deepth: "+str(self.current_deepth)
                num =self.linkQueue.getUnvisitedUrlCount()
                self.fileOper("Unvisited url count: %d"%num)
                try:
                    for link in links:
                        self.temp.append(link)
                        print "temp count :"+str(len(self.temp))
                except Exception,e:
                    self.fileOper(str(e))
                    print str(e)






            #未访问的URL入列
            for link in self.temp:
                self.linkQueue.addUnvisitedUrl(link)


            #写入数据库


            self.dataProcess = dataProcess()
            self.dataProcess.insertTable(self.temp)
            print "%d add to unvisited links:"%len(self.temp)
            self.fileOper("%d add to unvisited links:"%len(self.temp))
            self.temp = []
            self.fileOper("% end"%self.current_deepth)
            self.current_deepth += 1


        ISOTIMEFORMAT='%Y-%m-%d %X'
        print "Crawling end %s"%time.strftime(ISOTIMEFORMAT, time.localtime())
        self.fileOper("Crawling end %s"%time.strftime(ISOTIMEFORMAT, time.localtime()))
        #self.fileOper("Crawling end %s"%time.strftime(ISOTIMEFORMAT, time.localtime()))
        #self.driver.close()




    #获取源码中所有超链接
    def getHref1(self,url):


        urls = []
        Rurls = []
        data = self.getPageSource(url)
        links = re.findall(r'''<a(\s*)(.*?)(\s*)href(\s*)=(\s*)([\"\s]*)([^\"\']+?)([\"\s]+)(.*?)>''',data,re.S|re.I)
        for u in links:
            urls.append(u[6])
        Rurls = self.getDomianLinks(urls)  #过滤掉非域名列表的链接
        return Rurls


    #获取网页源码
    def getPageSource1(self,url):
        #phantomjs_path = r'C:\Users\joyking.xu\Desktop\phantomjs-2.1.1-windows\bin\phantomjs.exe'
        #driver = webdriver.PhantomJS(phantomjs_path)
        self.driver.get(url)


        #time.sleep(7)
        self.waitForLoad(self.driver)
        #print(driver.page_source)
        pageSource = self.driver.page_source
        #html = driver.page_source.find_element_by_xpath("//li")




        #return (driver.find_elements_by_xpath("//li").text)
        #driver.get_screenshot_as_file("2.jpg") #获取页面截图
        #self.driver.close()
        return pageSource


    def getPageSource(self,url,timeout=10,coding=None):
        try:
            socket.setdefaulttimeout(timeout)
            req = urllib2.Request(url)
            req.add_header('User-agent', 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)')
            response = urllib2.urlopen(req)
            page = ''




            if response.headers.get('Content-Encoding') == 'gzip':
                page = zlib.decompress(page, 16+zlib.MAX_WBITS)




            if coding is None:
                coding= response.headers.getparam("charset")


            #如果获取的网站编码为None
            if coding is None:
                page=response.read()
            #获取网站编码并转化为utf-8
            else:
                page=response.read()
                page=page.decode(coding).encode('utf-8')
            return page
        except Exception,e:
            print str(e)
            return [str(e),None]


    def getUrl(self,url):
        Rurls = []
        data=self.getPageSource(url)
        reg=r'"(http://.+?)"'
        regob=re.compile(reg,re.DOTALL)
        urllist=regob.findall(data)
        Rurls = self.getDomianLinks(urllist)
        return Rurls






    #通过域名列表过滤出所需要链接
    def getDomianLinks(self,urls):
        Links = []
        domainlist = ['act.vip.com','vip.com','trip.vip.com'] #爬取域列表


        for url in urls:
            up = urlparse.urlparse(url)
            for domain in domainlist:
                if up.netloc == domain:
                    Links.append(url)
                else:
                    continue
        return Links


    def waitForLoad(self,driver):  #隐式等待loading,页面加载成功
        elem = driver.find_element_by_tag_name("li")
        count = 0
        while True:
            count +=1
            if count > 20:
                print("Timing out after 10 seconds and returning")
                return
            time.sleep(.5)
            try:
                elem = driver.find_element_by_tag_name("li")
            except StaleElementReferenceException:
                return
    #写文件
    def fileOper(self,data):
        content = str(data)
        file = open('log.txt','ab+')
        file.write(content+"\r\n")
        file.close()


class dataProcess:
    def __init__(self):
        self.conn = MySQLdb.connect(host='10.199.243.34',user='root',passwd='vipshop-cms',db='vip_crawling',port=3306,charset='utf8')
        self.cur = self.conn.cursor()
        self.curtime = time.strftime('%Y-%m-%d',time.localtime(time.time()))
    def insertTable(self,values):
        data = []
        try:
            for i in values:
                status = self.urlStat(i)
                data = ['旅行PC',i,status,self.curtime]


                #data.append('旅行PC',i,status,self.curtime)
                #data.append(temp)
            #if len(data)<=100:
                self.cur.execute("insert into href_resource(platform,url,status,create_date) values(%s,%s,%s,%s)",data)
            print "%d add to DB:"%len(values)
            self.fileOper("%d add to DB:/r/n"%len(values))
            data = []
        except Exception,e:
            print str(e)
        self.conn.commit()
        self.cur.close()
        self.conn.close()
    #链接判断有效性
    def urlStat(self,url):


        try:
            r = urllib.urlopen(url)
            return r.code
        except urllib.URLError as e:
            r = e
            return r.code
    #写文件
    def fileOper(self,data):
        content = str(data)
        file = open('log.txt','ab+')
        file.write(content+"\r\n")
        file.close()




def main(seeds,crawl_deepth):
    ISOTIMEFORMAT='%Y-%m-%d %X'
    print "main start %s"%time.strftime(ISOTIMEFORMAT, time.localtime())
    craw = Crawler(seeds)
    craw.fileOper("main start %s"%time.strftime(ISOTIMEFORMAT, time.localtime()))
    craw.Crawling(seeds,crawl_deepth)






if __name__ == '__main__':


   main("http://trip.vip.com",3)