python爬虫小说系统_Python爬虫——小说

#encoding:utf8

import re

import urllib2

url = 'http://www.23us.com/html/55/55304/'

request = urllib2.Request(url)

response = urllib2.urlopen(request)

content = response.read().decode('gbk')

the_url = re.compile('

.*?',re.S) last_url = the_url.findall(content)

for i in last_url:

print i

url = 'http://www.23us.com/html/55/55304/'+i

request = urllib2.Request(url)

response = urllib2.urlopen(request)

zhi = response.read()

code = re.compile('.*?content="text.html; charset=(.*?)".*?',re.S)

last_code = code.findall(zhi)[0]

try:

content = zhi.decode(''+last_code)

except:

try:

content = zhi.decode('gb2312')

except:

continue

last_content = re.compile('

(.*?).*?(.*?)',re.S)

last_content = last_content.findall(content)

if last_content==[]:

print '采集失败'

print content

for I,J in last_content:

J = J.replace(' ','').replace('
','

')

file = open('小说.txt','a+')

t = '

' + I + '

' + '' + J

file.write(t.encode('utf-8'))

file.close()