python爬虫抽取武侠小说

这几天在手机上看《蜀山剑侠传》 看到一半 突然让付费，还是自己抓取文字嘛

就模仿着做了个简单的抽取,   最终结果存在文件里

__author__ = 'allen'

import urllib
import urllib2
import re
import chardet
import os

from bs4 import BeautifulSoup

import sys
print sys.getdefaultencoding()
reload(sys)
sys.setdefaultencoding('utf-8')

def get_page_str(page_num):
    if page_num < 10:
        return '0' + str(page_num)
    return  str(page_num)

def get_huanzhu_url(page_num):
    return 'http://www.my285.com/wuxia/hzlz/ssj3/' + get_page_str(page_num)+'.htm'

data_save = open('data.txt', 'wb+')

def spider_page(page_num):
    url = get_huanzhu_url(page_num)
    print(url)
    user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
    headers = { 'User-Agent' : user_agent }
    try:
        request = urllib2.Request(url,headers = headers)
        response = urllib2.urlopen(request)
        content = response.read()
        content = unicode(content, 'gbk')
        content = content.replace('<br>', '')
        soup = BeautifulSoup(content, 'html.parser')
        tmp = soup.find_all(lambda tag: tag.name=='td' and len(tag.attrs) == 1
                            and tag.has_key('colspan'))
        length = len(tmp)
        index = 0;
        for data in tmp:
            if index >= 1:
                break
            index = index + 1
            data_str = data.string
            data_str = data_str.replace('<td colspan="2">', '')
            data_str = data_str.replace('</td>', '')
            data_save.write(data_str)
        return True
    except urllib2.URLError, e:
        if hasattr(e,"code"):
            print e.code
            return False
        if hasattr(e,"reason"):
            print e.reason
            return False

page_num = 0
while True:
    page_num = page_num + 1
    res = spider_page(page_num)
    if res == False:
        break
    print(page_num)
data_save.close()