NO1.爬取小说文字并下载至text文档

from urllib import request
from bs4 import BeautifulSoup
import webbrowser

if __name__ == "__main__":
　　url= 'http://www.biqukan.com/47_47404/17679470.html'
　　webbrowser.open(url)
　　head = {}
　　head['User-Agent'] = 'Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19'
　　requ = request.Request(url, headers = head)
　　download_response = request.urlopen(requ)
　　download_html = download_response.read().decode('gbk','ignore')
　　soup_texts = BeautifulSoup(download_html, 'lxml')
　　texts = soup_texts.find_all(id = 'content', class_ = 'showtxt')
　　soup_text = BeautifulSoup(str(texts), 'lxml')

　　#将\xa0无法解码的字符删除
　　download_text = soup_text.div.text.replace('\xa0','')

　　#将爬取结果写入文档中
　　with open("爬虫.txt","w") as filename:
　　　　filename.write(download_text)

转载于:https://www.cnblogs.com/kimi765096/p/8667197.html