利用lxml爬取豆瓣小组内容文档并保存

# -*- coding: UTF-8 -*-
#参考资料：https://segmentfault.com/q/1010000004879947
import requests
from lxml import html
import time

def getTree(pageUrl):
    page_content = requests.get(pageUrl)
    page_content.encoding = 'utf-8'
    q = page_content.text.encode('utf-8')
    tree = html.fromstring(q)
    return tree

def getInsideUrl(m_tree):
    #找到总共有几页并获取URL
    insidePageUrl_arr = m_tree.xpath('//div[@class="paginator"]/a/@href')
    return insidePageUrl_arr

def getContent(n_tree):
    #内容
    content_arr = n_tree.xpath('//div[@class="article"]/ul[@class="topic-reply"]/li[@class="clearfix comment-item"]/div[@class="reply-doc content"]/p/text()')
    w = open(u'F:\\寂寞就读书\出处不明.txt','a')    
    for e in content_arr:
        w.write(e.encode('utf8'))
        w.write('\r\n\r\n')
    w.close()

if __name__ == '__main__':  
    mytree = getTree('https://www.douban.com/group/topic/14911241/')
    insideUrlArr = getInsideUrl(mytree)
    #取得第一页的内容
    getContent(mytree)
    #取得其他页的内容
    for f in insideUrlArr:
        k_tree = getTree(f)
        getContent(k_tree)
复制代码