下载糗事百科的内容_python版

最新推荐文章于 2020-10-21 22:06:03 发布
jj_liuxin
最新推荐文章于 2020-10-21 22:06:03 发布
阅读量1.2k
点赞数
分类专栏： python 文章标签： python sqlite object url list class
本文链接：https://blog.csdn.net/jj_liuxin/article/details/3466701
版权
python 专栏收录该内容
22 篇文章 0 订阅
订阅专栏
 
  #coding:utf-8
import urllib.request
import xml.dom.minidom
import sqlite3
import threading
import time
class logger(object):
    def log(self,*msg):
        for i in msg:
            print(i)
Log = logger()
Log.log('测试下')
class downloader(object):
    
    def __init__(self,url):
        self.url = url
    
    def download(self):
        Log.log('开始下载',self.url)
        try:
            content = urllib.request.urlopen(self.url).read()
            #req = urllib.request.Request(url)
            #response = urllib.request.urlopen(req)
            #content = response.read()
            Log.log('下载完毕')
            return(content)   
        except:
            Log.log('下载出错')
            return(None)
       
            
class parser(object):
    
    def __init__(self,content):
        #获得根节点
        self.html = xml.dom.minidom.parseString(content)
    
    def parse(self):
        Log.log('开始提取数据')
        contents = {'content':'','url':[]}
        #获得div节点
        divs = self.html.getElementsByTagName('div')
        #获得content节点
        for div in divs:
            if div.hasAttribute('class') and /
               div.getAttribute('class') == 'content':
               #获得糗事百科的内容
               textNode = div.childNodes[0]
               qContent = textNode.data
               #数据填充
               contents['content'] = qContent
        
        #获得上一糗事、下一糗事节点
        spans = self.html.getElementsByTagName('span')
        for span in spans:
            pspan = span.parentNode
            if pspan.tagName == 'a':
                #pspan为对应的链接,此时需要将对应的地址加入数据库
                url = pspan.getAttribute('href')
                qid = url[10:][:-4]
                #数据填充
                contents['url'].append(qid)
        Log.log('提取数据完毕')
        return(contents)
def downloadPage(qid,db):
    url = 'http://www.qiushibaike.com/articles/'+str(qid)+'.htm'
    content = downloader(url).download()
    if content:
        contents = parser(content).parse()
        if contents['content']:
            db.updateContent(qid,contents['content'])
        for i in contents['url']:
            db.addQID(i)
        if len(contents['url']) == 2:
            db.updateStatus(qid,2)
#下载池，表示同时允许下载的链接个数
class downloaderPool(object):
    def __init__(self,maxLength=15):
        self.downloaders = [None]*maxLength
        self.downloadList = []
        self.db = None
    
    def setDownloadList(self,downloadList):
        self.downloadList = list(set(self.downloadList+downloadList))
    
    def setdb(self,db):
        self.db = db
    
    def daemon(self):
        #每隔一秒查询线程的状态，为非活动线程则设置为None
        Log.log('设置守护进程')
        for index,downloader in enumerate(self.downloaders):
            if downloader:
                if not downloader.isAlive():
                    Log.log('将下载器置空',index)
                    self.downloaders[index] = None
        
        #检查线程池状态
        for index,downloader in enumerate(self.downloaders):
            if not downloader:
                qid = self.getQID()
                if qid:
                    #创建线程
                    t = threading.Thread(target=downloadPage,args=(qid,self.db))   
                    self.downloaders[index] = t
                    t.start()
                    t.join()
                    Log.log('设置下载器',index)
        #间隔一秒执行一次
        time.sleep(1)            
    
    def getQID(self):
        try:
            tmp = self.downloadList[0]
            del self.downloadList[0]
            return(tmp)
        except:
            return(None)
        
    def beginDownload(self):
        #创建守护线程
        daemon = threading.Thread(target=self.daemon)
        daemon.setDaemon(True)
        daemon.start()
        daemon.join()          
                    
    def getDownloader(self):
        for index,downloader in enumerate(self.downloaders):
            if not downloader:
                return(index)
        return(None)
ADD_Q_ID = 'insert into qiushibaike(id,success) values(?,?)'
UPDATE_Q_CONTENT = 'update qiushibaike set content=? where id=?'
UPDATE_Q_STATUS = 'update qiushibaike set success=? where id=?'
Q_LIST = 'select id from qiushibaike where success=?'
Q_LIST_BY_ID = 'select count(*) from qiushibaike where id=?'
class dbConnect(object):
    """
    create table qiushibaike(
    id,Integer
    content,Varchar
    success,Interger
    )
    #id表示糗事的ID
    #content表示糗事的内容
    #success表示是否下载成功，当该糗事内容下载完成，且获得上一页、下一页ID时表示下载完成
            1表示未完成
            2表示完成
    """
    def __init__(self,dbpath='db.sqlite'):
        self.dbpath = dbpath
    
    def addQID(self,qid):
        Log.log('插入糗事百科',qid)
        #获得连接
        cn = sqlite3.connect(self.dbpath)
        c = cn.cursor()
        
        try:
            #添加内容并提交
            c.execute(ADD_Q_ID,(qid,1))
            cn.commit()
        except:
            Log.log('添加ID出错',qid)
        
        #关闭连接            
        c.close()
            
        cn.close()
        Log.log('插入成功')
    
    def updateContent(self,qid,content):
        Log.log('更新糗事百科',qid,content)
        #获得连接
        cn = sqlite3.connect(self.dbpath)
        c = cn.cursor()
        #添加内容并提交
        c.execute(UPDATE_Q_CONTENT,(content,qid))
        cn.commit()
        #关闭连接
        c.close()
        cn.close()
        Log.log('更新成功')
    
    def updateStatus(self,qid,flag):
        Log.log('更新状态',qid,flag)
        #获得连接
        cn = sqlite3.connect(self.dbpath)
        c = cn.cursor()
        #添加内容并提交
        c.execute(UPDATE_Q_STATUS,(flag,qid))
        cn.commit()
        #关闭连接
        c.close()
        cn.close()
        Log.log('更新状态成功')
    
    def getList(self,unDonloaded=1):
        Log.log('获得列表')
        l = []
        #获得连接
        cn = sqlite3.connect(self.dbpath)
        c = cn.cursor()
        #获得数据
        c.execute(Q_LIST,(unDonloaded,))
        rows = c.fetchall()
        
        for i in rows:
            l.append(i[0])
        #关闭连接
        c.close()
        cn.close()
        
        Log.log('获得列表成功')
        return(l)
class singleDownloader(object):
    def __init__(self):
        self.downloadList = []
    
    def setdb(self,db):
        self.db = db
    
    def setDownloadList(self,downloadList):
        self.downloadList = list(set(self.downloadList+downloadList))
        
    def beginDownload(self):
        for i in self.downloadList:
            downloadPage(i,self.db)
    
def main():
    db = dbConnect('db.sqlite')
    #dp = downloaderPool()
    #dp.setdb(db)
    sp = singleDownloader()
    sp.setdb(db)
    
    dp=sp
    
    unDownloadedList = db.getList()
    #当还有未下载的糗事时就要继续下载
    while(len(unDownloadedList)):
        #使用该列表填充下载池
        dp.setDownloadList(unDownloadedList)
        
        dp.beginDownload()
        
        time.sleep(1)
        #重置参数
        unDownloadedList = db.getList()
if __name__ == '__main__':
    main()
 