起点小说下载工具 代码汇

起点现在的双域名分别是不同的页面架构技术:
cmfu是原来老的ASP,对于公开章节,起点采用的是js调用一个txt,相对原来直接页面显示,确实是一个明智的选择(对抓取的来说也方便:p).
qidian采用的是新的.Net技术,这个里面就需要采用正则去匹配了.

结合上一篇文章我的小程序,这里将社区内相关的代码全部show一下.
首先是我的:

#!/usr/bin/python
#filename:simpleCMFU
import re
import urllib

def alaynsis_id(url_given):
name=url_given[-11:-1]+url_given[-1]
return name


def read(url_given):
html=urllib.urlopen(url_given)
page=html.read()
html.close()
#rex = r'http:\/\/files\.qidian\.com\/\[a-zA-Z]{6}[0-9]\/d{6}\/\d{6}\.txt'
rex = r'http://files.qidian.com/author[0-9]/\d{6}/\d{7}.txt'
#http://files.qidian.com/author3/172602/4451850.txt
url_down=test(page,rex)
url=url_down[0]
#print rex
return url

def read2(url,name):
html=urllib.urlopen(url)
page=html.read()
html.close()
page=page[15:len(page)]
fl=file(name,'w')
fl.write(page)
fl.close()
return 'ok'

def test(html,rex):
#r = re.compile(rex)
matchs = re.findall(rex,html,re.DOTALL)
return matchs

def run():
url=raw_input('please send address you wanted:')
url_tmp=read(url)
name=alaynsis_id(url_tmp)
read2(url_tmp,name)

if __name__ == '__main__':
print 'this program is just for download text from qidian.com by duducai@msn.com\please visit http://duducai.iteye.com \n'
run()



其次是社区Ben Luo的大作,走的是sina读书频道:

#####################
#html2txt.py
#####################

from formatter import AbstractFormatter, NullWriter
from htmllib import HTMLParser

def _(str, in_encoder="gbk", out_encoder="utf8"):
return unicode(str, in_encoder).encode(out_encoder)


class myWriter(NullWriter):
def __init__(self):
NullWriter.__init__(self)
self._bodyText = []

def send_flowing_data(self, str):
self._bodyText.append(str)

def _get_bodyText(self):
return '\n'.join(self._bodyText)

bodyText = property(_get_bodyText, None, None, 'plain text from body')

class myHTMLParser(HTMLParser):
def do_meta(self, attrs):
self.metas = attrs

def convertFile(filename):
mywriter = myWriter()
absformatter = AbstractFormatter(mywriter)
parser = myHTMLParser(absformatter)
parser.feed(open(filename).read())
return ( _(parser.title), parser.formatter.writer.bodyText )

import os
import os.path

OUTPUTDIR = "./txt"
INPUTDIR = "."
if __name__ == "__main__":
if not os.path.exists(OUTPUTDIR):
os.mkdir(OUTPUTDIR)

for file in os.listdir(INPUTDIR):
if file[-4:] == '.htm' or file[-5:] == '.html':
print "Coverting", file,
outfilename = os.path.splitext(file)[0]
a, text = convertFile(file)
outfilename = outfilename + '.txt'
outfullname = os.path.join(OUTPUTDIR, outfilename)
open(outfullname, "wt").write(text)
print "Done!"




################################
#pickupcontent.py
################################

# -*- coding: utf-8 -*-

import sys
import glob
import os
import re

sys.argv[1:] = [item for arg in sys.argv[1:] for item in glob.glob(arg)]
startstr = u"^八十".encode("gb2312") # article title
endstr = u"^\[返回".encode("gb2312") #
tmp_start = re.compile(startstr)
tmp_end = re.compile(endstr)
for infile in sys.argv[1:]:
# print infile
f = open(infile,'r')
#print f
lines = f.readlines()
fout = ''
for index, line in enumerate(lines):
if tmp_start.match(line):
kstart = index
if tmp_end.match(line):
kend = index
break

f.close()
fout = fout.join(lines[kstart:kend])
tmp = open('tmp','w')
tmp.write(fout)
tmp.close()
os.remove(infile)
os.rename('tmp',infile)



最后是BIGZHU的:

#@+leo-ver=4-thin-encoding=gb2312,.
#@+node:BIGZHU.20070731160918:@thin d:/bigzhu/python/python_project/get_cmfu.py
#@+at
#@nonl
# 起点小说爬虫
#@-at
#@@c
#@@language python
#@+others
#@+node:BIGZHU.20070731161308:import
import httplib,urllib2,urllib,cookielib,re,threading
import os
#@nonl
#@-node:BIGZHU.20070731161308:import
#@+node:BIGZHU.20070731160928:getCookie
def getCookie():
cj = cookielib.CookieJar()#建立Cookie实例
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))#建立opener与Cookie关联
return opener
#@-node:BIGZHU.20070731160928:getCookie
#@-others
#@<<getBookIdList>>
#@+node:BIGZHU.20070731160918.1:<<getBookIdList>>
def getBookIdList(urlList):

BookIdList = []
for i in urlList:
url=i
#print url
request = urllib2.urlopen(url)
cmfu = request.read()
#cmfuURL = re.findall("<a href='showbook.asp\?bl_id=\d{1 ,}'",cmfu)
#BookIdListTemp = [re.sub("<a href='showbook.asp\?bl_id=",'',k) for k in cmfuURL]
#BookIdListTemp = [re.sub("'",'',k) for k in BookIdListTemp]
#起点的代码太不规范了,想一个更广泛性的匹配正则表达式
"""
cmfuURL = re.findall("showbook.asp\?bl_id=\d{1,}",cmfu)
BookIdListTemp = [re.sub("showbook.asp\?bl_id=",'',k) for k in cmfuURL]
"""
#更大众化一些
cmfuURL = re.findall("bl_id=\d{1,}",cmfu)
BookIdListTemp = [re.sub("bl_id=",'',k) for k in cmfuURL]
#BookIdListTemp = [ re.sub("'",'',k) for k in BookIdListTemp]
bookCount = len(BookIdList)
for listTemp in BookIdListTemp:
#检查该bookid是否在BookIdList中已有
if listTemp in BookIdList:
pass
else:
BookIdList.extend([listTemp])#加进去
print "取得书本数目:%i"%(len(BookIdList)-bookCount)
print "合计取得下载书本:%i"%len(BookIdList)
return BookIdList

#@-node:BIGZHU.20070731160918.1:<<getBookIdList>>
#@nl
#@<<getBookName>>
#@+node:BIGZHU.20070731164705:<<getBookName>>
def getBookName(opener,bookId=''):
if bookId == '':
print "传入BookIdList是空的"
bookURL = 'http://www.cmfu.com/readbook.asp?bl_id=%s'%bookId
request = urllib2.Request(bookURL)
bookPage = opener.open(request).read()
opener.close()
bookname = re.findall('bookname=\S{1,}',bookPage)

bookname = [re.sub("bookname=",'',k) for k in bookname]
bookname = [re.sub('"','',k) for k in bookname][0]

return bookname

#@-node:BIGZHU.20070731164705:<<getBookName>>
#@nl
#@<<getTextFile>>
#@+node: BIGZHU.20070731171721:<<getTextFile>>
def getTextFile(opener,bookId):
bookName = getBookName(opener,bookId)
#判断文件是否已经存在
if os.path.isfile(os.getcwd()+"\\起点\\%s.txt"%bookName):
print "%s 已经存在"%bookName
else:
url = 'http://download.cmfu.com/pda/%s.txt'%bookId
try:
bookData = opener.open(url).read()
except :
print "2 %s"%bookName
try:
bookData = opener.open(url).read()
except :
print "last try %s"%bookName
try:
bookData = opener.open(url).read()
except :
print "end try %s"%bookName

opener.close()

f=open(os.getcwd()+"\\起点\\%s.txt"%bookName,"wb")
f.write(bookData)
f.close()
print 'get book %s 完毕'%bookName
#@-node:BIGZHU.20070731171721:<<getTextFile>>
#@nl
#@<<class runGetFile>>
#@+node:BIGZHU.20070801172939:<<class runGetFile>>
class runGetFile(threading.Thread):
def __init__(self,bookId):
threading.Thread.__init__(self)
self.bookId = bookId
#self.opener = opener
def run(self):
opener = getCookie()
getTextFile(opener,self.bookId)
#@nonl
#@-node: BIGZHU.20070801172939:<<class runGetFile>>
#@nl
#@<<class ProcessURL>>
#@+node:BIGZHU.20070802171013:<<class ProcessURL>>
class ProcessURL:
"""对新输入url,save 到ini中
对已有url,忽视
每次使用,自动读取ini的url,提供使用"""
def __init__(self):
pass
#@ <<saveURL>>
#@+node:BIGZHU.20070802171013.1:<<saveURL>>
def saveURL(self,urlList=[]):
'''存储新的url到URL.ini中'''


try:
f=open(os.getcwd()+"\\起点\\URL.ini","wb")#追加内容
except IOError:
print "文件打开错误"
#格式化成字符串
s_urlList = ";".join(urlList)
f.write(s_urlList)
f.close()
#@-node:BIGZHU.20070802171013.1:<<saveURL>>
#@nl
#@ <<getURLIni>>
#@+node:BIGZHU.20070802171013.2:<<getURLIni>>
def getURLIni(self):
"""读取 URL.ini中的url
返回一个URL list"""
#判断目录是否存在
if os.path.exists (os.getcwd()+"\\起点"):
pass
else:
print "创建目录 \起点"
os.mkdir("起点")

iniData=''
if os.path.isfile(os.getcwd ()+"\\起点\\URL.ini"):
f=open(os.getcwd()+"\\起点\\URL.ini","rb")
iniData = f.read()
f.close()
else:
print "URL.txt不存在,创建之"
f=open(os.getcwd()+"\\起点\\URL.ini","wb")
#iniData = f.read()
f.close()
return iniData.split(";")#格式化成list
#@-node:BIGZHU.20070802171013.2: <<getURLIni>>
#@nl


#@-node:BIGZHU.20070802171013:<<class ProcessURL>>
#@nl
#@<<main>>
#@+node:BIGZHU.20070731164705.1:<<main>>
if __name__ == '__main__':
opener = getCookie()
#urlList =["http://www.cmfu.com/index.asp"," http://www.cmfu.com/listbookqb.asp?pageid=2007-8-1%2012:26&status=down","http://www.cmfu.com/listbookqb.asp?pageid=2007-7-31%2023:03&status=down ","http://www.cmfu.com/index_wxxx.asp"]
#存放和读取url
urlType = ProcessURL()
urlList = urlType.getURLIni()
saveIni = 0 # 标识是否有url 更新
while True:
url = raw_input("要截取的起点的某个页面: ")
if url=='':
break
if url in urlList:
print "%s 已有,忽视之"%url
else:
urlList.extend([url])
print "%s 是新的,添加之"%url
saveIni =1
#url = 'http://www.cmfu.com/index.asp'


bookIdList=getBookIdList(urlList)


for i in bookIdList:
thread = runGetFile(i)
thread.start()
#存储到ini中
if saveIni == 1:
urlType.saveURL(urlList)
#@-node:BIGZHU.20070731164705.1:<<main>>
#@nl
#@nonl
#@-node:BIGZHU.20070731160918:@thin d:/bigzhu/python/python_project/get_cmfu.py
#@-leo

  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值