python3下BeautifulSoup练习一（爬取小说）

from bs4 import BeautifulSoup
import urllib.request
import re
import os,time

def getUrls(url):
    urls = []
    #url = 'http://www.ahzww.net/0/178/'
    req = urllib.request.Request(url)
    page = urllib.request.urlopen(req)

    html = page.read()

    soup = BeautifulSoup(html,'html.parser')

    i = 0
    for k in soup.find_all(href=re.compile('.html')):
        #print('www.qu.la'+k['href'],k.get_text())
        if i != 0:
            urls.append('http://www.ahzww.net'+k['href'])
        i = i+1
    return urls

def getContent(url):
    #url = 'http://www.ahzww.net/0/178/355185.html'
    headers = ('User-Agent','Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11')
    opener = urllib.request.build_opener()
    opener.addheaders = [headers]
    html = opener.open(url).read()
   
    soup = BeautifulSoup(html,'html.parser')
    content = soup.find('div',id='content')
    title = soup.find('h1')
    return title.get_text(),content.get_text()

if __name__ == '__main__':
    urls = getUrls('http://www.ahzww.net/0/178/')
    #print(urls)
    fp = open("不负娇宠.txt","w")
    for url in urls:
        print(url)
        title,content = getContent(url)
        fp.write(title+"\n")
        fp.write(content.replace('　　　　    ','\n')+"\n")
        time.sleep(2)
    fp.close()
    print("Done")

转载于:https://my.oschina.net/u/3695173/blog/1548900