爬取小说并存取为txt文件

代码：

#coding:utf-8
#author:Ericam_
import re
import sys
from bs4 import BeautifulSoup
import urllib.request
import time

headers = ('User-Agent', 'Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1')
opener = urllib.request.build_opener()
opener.addheaders = {headers}
urllib.request.install_opener(opener)

 
url = "http://www.xinshubao.net/66/66351/"

file = urllib.request.urlopen(url)
data = BeautifulSoup(file , from_encoding="utf8")  


# section_name = data.title.string
# print(section_name)
# print(type(str(section_name)))
# try:
#     section_text = data.select("#content p")[0].text
#     section_text=re.sub( '\s+', '\r\n\t', section_text).strip('\r\n\t')
#     section_text = section_text.replace('\t','')
#     section_text = section_text.replace('\n','')
#     section_text = section_text.replace('\r','')
# except IndexError:
#     section_text = ''
    

# print(section_text)

linklist = []
for x in data.find_all('a'):
    link = x.get('href')

    if link:
        
        if link[-1] == 'l':
           # link = 'https://www.yanqing-888.net/' + link
            linklist.append(link)

# print(linklist)


for i in linklist:
	time.sleep(2)
	
	file = urllib.request.urlopen(i)
	data = BeautifulSoup(file , from_encoding="utf8")  

	section_name = data.title.string.split('_')[1]
	print(section_name)
	sys.stdout.flush()
	try:
		section_text = data.select("#content")[0].text  
		section_text=re.sub( '\s+', '\r\n\t', section_text).strip('\r\n\t')
		section_text = section_text.replace('\t','')
		section_text = section_text.replace('\n','')
		section_text = section_text.replace('\r','')
	except IndexError:
		section_text = ''


	# fp = open('1230.txt','a')
	
	with open('1231.txt','a') as fp:

		fp.write(str(section_name))
		fp.write('\n')
		fp.write(section_text)
		fp.write('\n')

	
	sys.stdout.flush()