python爬虫练习5：博客阅读量助手

目标与环境

爬取csdn博客如“http://blog.csdn.net/nima1994”的文章列表，获取阅读量等，与上次进行比较
使用eclipse+pydev编写，win10 64位，python 3.6.2；用到requests，beautifulsoup4，pandas等库。

代码

'''
Created on 2017年9月30日

@author: Harry
'''
import requests
import re
from bs4 import BeautifulSoup
import pandas as pd
import datetime
import os

#content:文本内容  
def analysis(content):

    b=BeautifulSoup(content,"lxml").select("#article_list > div.article_item")
    for page in b:
        url=host+page.select_one("div.article_title > h1 > span > a").attrs['href']
        title=page.select_one("div.article_title > h1 > span").get_text().strip()
        time=page.select_one("div.article_manage > span.link_postdate").get_text()
        read_count=page.select_one("div.article_manage > span.link_view > a").next_sibling[1:-1]
        comment=page.select_one("div.article_manage > span.link_comments > a").next_sibling[1:-1]
        dt=[url,title,time,read_count,comment];
        #global datas
        datas.append(dt)
def getData():
    r=requests.get(mysite)
    page=int(re.findall(r"共(.*)页", BeautifulSoup(r.text,"lxml").select_one("#papelist > span").get_text())[0])
    analysis(r.text)
    for i in range(2,page+1):
        r=requests.get(mysite+"/article/list/"+str(i))
        analysis(r.text)  
def calculation():
    df=pd.DataFrame(datas)
    df_last=pd.read_csv("d:\\temp.csv", encoding = "utf-8")

    res=[]
    col=["标题","浏览量","上次浏览量","浏览量增加","新增评论"]
    s1=0
    s2=0
    s4=0
    for i in range(0,df[0].size):
        index=list(df_last["0"]).index(df[0][i])
        #单条增加
        s1+=int(df[3][index])
        s2+=int(df_last["3"][index])
        des=int(df[3][index])-int(df_last["3"][index]) 
        pinglu=int(df[4][index])-int(df_last["4"][index]) 
        s4+=pinglu
        if des>0 or pinglu>0:     
            r=[df[1][i],df[3][index],df_last["3"][index],des,pinglu]
            res.append(r)  


    spd=pd.DataFrame(res,columns=col)

    print(spd)
    s3=s1-s2
    print("合计")
    print("总浏览量：%d 上次总浏览量：%d 浏览量增加:%d 评论增加：%d" % (s1,s2,s3,s4))
    write2csv()
def write2csv():
    s=input("是否更新工作表:y?")
    if(s!="y"):
        print("未更新，程序已退出：",str(datetime.datetime.now()))
        return 
    df=pd.DataFrame(datas)
    df.to_csv("d:\\temp.csv",encoding = "utf-8") 
    print("写入成功：",str(datetime.datetime.now()))

datas=[]
host="http://blog.csdn.net"
mysite="http://blog.csdn.net/nima1994"  
if __name__ == '__main__':
    pd.set_option('display.max_rows',None)
    getData()    
    if(os.path.exists("d:\\temp.csv")==False):
        df=pd.DataFrame(datas)
        df.to_csv("d:\\temp.csv",encoding = "utf-8") 
        print("第一次写入，无运行结果，程序退出")
        exit()
    calculation()

运行结果

以下文档写于：2018年03月07日 17:32:00
由于csdn博客升级，爬虫部分变量更改，简略代码如下：

import requests
import re
from bs4 import BeautifulSoup

#content:文本内容  
def analysis(content):

    b=BeautifulSoup(content,"lxml").select("#article_list > div.article_item")
    for page in b:
        url=page.select_one(".article_title .link_title a").attrs['href']
        title=page.select_one(".article_title .link_title").get_text().strip()
        time=page.select_one(".link_postdate").get_text()
        read_count=str(page.select_one(".link_view").get_text()).strip()[3:-1]
        comment=str(page.select_one(".link_comments a").next_sibling).strip()[1:-1]
        dt=[url,title,time,read_count,comment];
        #global datas
        print(dt)
        datas.append(dt)
def getData():
    r=requests.get(mysite)
    analysis(r.text)
    while(True):
        rp = BeautifulSoup(r.text, 'lxml').select("#papelist > div > nav > ul > li")
        q = re.search(r'href="(.*?)".*?rel="(.*?)"', str(rp[len(rp) - 1]))
        if q==None:
            break
        r=requests.get(q.group(1))
        analysis(r.text)  
datas=[]
host="http://blog.csdn.net"
mysite="http://blog.csdn.net/nima1994"  
if __name__ == '__main__':
    getData()
    # for i in datas:
    #     print(i)