爬虫爬取小说

网络爬虫爬取小说：

#-*_coding:utf8-*
from bs4 import BeautifulSoup
import os
import urllib.request
import re

# 定义打开链接函数
def handlechapter(url) :
    response = urllib.request.urlopen(url)
    html = response.read().decode('gbk', 'ignore')
    return html
#下载资料
f=open('D:zongcai.txt','w',encoding='utf-8')
#爬虫网址
url ="http://www.246zw.com/html/18/18714/"
#打开链接
url_open=urllib.request.urlopen(url)
#获取html码
url_html=url_open.read().decode('gbk', 'ignore')
#赋值给beautifulSoup
soup = BeautifulSoup(url_html, 'html.parser')
#获取网站所有URl
print("获取所有链接")
links=soup.find(id='list').findAll("a", href=re.compile("^[^/html/]"))#截取字符串部分

for link in links:#循环遍历章节
    chattitle=link.string#获取章节标题
    t3 = link.get('href')#获取章节URL
   
    print('准备下载'+chattitle)
   
    f.write('\n'+chattitle+'\n')#写入文件
    htmlurl=url+str(link['href'])
   
    html = handlechapter( htmlurl)#获取构建的URL的html
    chapterhtml = BeautifulSoup(html, 'html.parser')
    for each in chapterhtml.find(id = 'content').strings:
        f.write('%s%s' % (each.replace('\xa0', ''), os.linesep))
    print('成功下载' +chattitle)
f.close()