代码:
#coding:utf-8
#author:Ericam_
import re
import sys
from bs4 import BeautifulSoup
import urllib.request
import time
headers = ('User-Agent', 'Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1')
opener = urllib.request.build_opener()
opener.addheaders = {headers}
urllib.request.install_opener(opener)
url = "http://www.xinshubao.net/66/66351/"
file = urllib.request.urlopen(url)
data = BeautifulSoup(file , from_encoding="utf8")
# section_name = data.title.string
# print(section_name)
# print(type(str(section_name)))
# try:
# section_text = data.select("#content p")[0].text
# section_text=re.sub( '\s+', '\r\n\t', section_text).strip('\r\n\t')
# section_text = section_text.replace('\t','')
# section_text = section_text.replace('\n','')
# section_text = section_text.replace('\r','')
# except IndexError:
# section_text = ''
# print(section_text)
linklist = []
for x in data.find_all('a'):
link = x.get('href')
if link:
if link[-1] == 'l':
# link = 'https://www.yanqing-888.net/' + link
linklist.append(link)
# print(linklist)
for i in linklist:
time.sleep(2)
file = urllib.request.urlopen(i)
data = BeautifulSoup(file , from_encoding="utf8")
section_name = data.title.string.split('_')[1]
print(section_name)
sys.stdout.flush()
try:
section_text = data.select("#content")[0].text
section_text=re.sub( '\s+', '\r\n\t', section_text).strip('\r\n\t')
section_text = section_text.replace('\t','')
section_text = section_text.replace('\n','')
section_text = section_text.replace('\r','')
except IndexError:
section_text = ''
# fp = open('1230.txt','a')
with open('1231.txt','a') as fp:
fp.write(str(section_name))
fp.write('\n')
fp.write(section_text)
fp.write('\n')
sys.stdout.flush()