练习1_小作文下载源码

练习1_小作文下载源码

import requests
from lxml import etree

# 1.网页源代码
def getlist():
    url = requests.get('https://www.chnlib.com/zuowenku/')  #.content.decode('utf-8')
    html = url.content.decode('utf-8')
#     print(str)
# getlist()

#2.获取每一个篇章的超链接
    doc = etree.HTML(html) #构造xpath解析对象@选取对象
    contents = doc.xpath('//*[@class="list-group"]/div')
    #print(contents)
    for content in contents:
        links = content.xpath('h4/a/@href') #获取超链接
        #print(links)
        for link in links: #每一个超链接
            url = 'https://www.chnlib.com/' + link #完整超链接
            html = requests.get(url).content.decode('utf-8') #发送请求，得到源代码
            doc = etree.HTML(html)
            content = doc.xpath('//*[@id="content"]/p/text()')#获取文本
#             # c1 = [c.replace("\r\n",'') for c in content]
#             # c2 = [c.strip() for c in c1]
#             # c3 = [c.replace('\xa0','') for c in c2]
            title = doc.xpath('/html/body/div[4]/div/div[1]/div/div[1]/h1/text()') #获取标题
            title1 = [t.replace('\r\n','') for t in title]
            print(title1,content)
# #3.存储数据
#             with open('download/%s.txt' %title1[0], 'w', encoding='utf-8') as f:
#                 for items in content:
#                     f.write(items)
getlist()