python 爬取电子书_看kindle网站电子书用Python爬取下载

一个下载看kindle(kankindle.com)的所有电子书的python脚本，程序会自动下载首页部分13页的所有电子书，下载到ebook目录下，程序会检测是否下载过。

#!/usr/bin/env python

# coding=utf-8

from bs4 import BeautifulSoup

import urllib2

import socket

import re

import unicodedata

import os

from urwid.text_layout import trim_line

def download(url):

print 'starting download %s' % url

response=urllib2.urlopen(url,timeout=30)

html_data=response.read()

soup=BeautifulSoup(html_data)

print 'start to analayse---------------'

title_soup=soup.find_all(class_='yanshi_xiazai')

name_soup = soup.find_all('h1')

tag_a = title_soup[0].a.attrs['href']

tag_name= title_soup[0].a.contents

link_name = name_soup[0]

link_name = str(link_name).replace("

","").replace("

","")

#print tag_name[0]

#print link_name

filename = link_name+".mobi"

filename = "ebook/"+filename

print 'filename is :%s' % filename

print "downloading with urllib2 %s" % tag_a

if os.path.exists(filename):

print 'already donwload ,ignore'

else:

try:

f = urllib2.urlopen(tag_a,timeout=60)

data = f.read()

#print 'the data is %s'% data

with open(filename, "wb") as code:

code.write(data)

except Exception,e:

print e

def get_all_link(url):

print 'Starting get all the list'

response=urllib2.urlopen(url,timeout=30)

html_data=response.read()

#print html_data

soup=BeautifulSoup(html_data)

link_soup = soup.find_all('a')

#print link_soup

for each_link in link_soup:

if re.search('view',str(each_link)):

#print each_link

print each_link

print each_link.attrs['href']

download(each_link.attrs['href'])

if __name__ == '__main__':

for page in range(1,13):

url = "http://kankindle.com/simple/page/3"+str(page)

url = url.strip()

print url

get_all_link(url)