Python爬虫利器之Beautiful Soup的用法

BeautifulSoup的帮助说明可以参考 http://cuiqingcai.com/1319.html

爬取网络小说

#-*- coding:utf-8 -*-

import urllib.request
import re
from bs4 import BeautifulSoup
import os

def getPage(url):
    page = urllib.request.urlopen(url).read().decode('utf-8')
    return page

def saveToFile(page,num):
    soup = BeautifulSoup(page)
    contents = soup.p.prettify()
    print(contents)
    contents = contents.replace("<br/>","").replace("/bk","").replace('<p>',"").replace('</p>',"")
    print("=======================")
    print(contents)
   
    title = '第'+str(num)+'章.txt'
    print(title)
    with open(title,'w') as file:
        file.writelines(contents)

if __name__=='__main__':
    if os.path.exists('死人经'):
        os.removedirs('死人经')
    os.mkdir("死人经")
    os.chdir("死人经")
    j = 98
    for i in range(620,653):
        url = "http://h6w.org.cn/post/" + str(i) + ".html"
        print(url)
        j = j+1
        saveToFile(getPage(url),j)