Python 爬虫把美空的图片都扒下来了

[python] view plain copy print ?

1. #!/usr/bin/python 
2. # -*- coding: utf-8 -*- 
3.  
4. import re 
5. import os 
6. import urllib2 
7. def getHtml(url): 
8.     from urllib2 import HTTPError 
9.     try: 
10.       page = urllib2.urlopen(url) 
11.       html = page.read(); 
12.     except HTTPError:  
13.         return "http://www.moko.cc/channels/post/28/1.html" 
14.     # print html 
15.     return html 
16. def getJpg(url): 
17.     from urllib2 import HTTPError 
18.     try: 
19.        page = urllib2.urlopen(url) 
20.        html = page.read() 
21.     except HTTPError: 
22.         temp=[] 
23.         temp.append("http://img2.moko.cc/users/3/965/289681/post/35/img2_src_8862177.jpg") 
24.         return temp 
25.     pattern = re.compile(r'<p class="picBox"><img src2="(.*)\.jpg"') 
26.     m=re.findall(pattern,html) 
27.     #添加后缀 
28.     for i in range(len(m)): 
29.         m[i]=m[i]+".jpg" 
30.     print m 
31.     return m 
32. def  getDict(html): 
33.     #得到huml里面所有的 <a href="/post/ 
34.     pattern = re.compile(r'<a href="/post/(.*)\.html') 
35.     m=re.findall(pattern,html) 
36.     print "the length is: ",len(m) 
37.     print m 
38.     #得到html里面的tilte 
39.     pattern_titlename = re.compile(r'<div class="cover" cover-text="(.*)">') 
40.     titlename = re.findall(pattern_titlename,html) 
41.     print "the length of name list: ",len(titlename) 
42.     print titlename 
43.     print titlename[0] 
44.      
45.     url_name_dict={} 
46.     for url in m: 
47.         urltemp="http://www.moko.cc/post/"+str(url)+".html" 
48.         url_name_dict[urltemp]=titlename[m.index(url)] 
49.     print url_name_dict 
50.     #print the url and the title name 
51.     dict=url_name_dict 
52.     for key in dict.keys(): 
53.         print "url:",key,"titlename",dict[key] 
54.     return url_name_dict 
55.  
56. #得到每个title里面所有的jpg的url,放在一个字典里面,key 是名字 value是对应名字的所有jpg的url 
57. def getDict_title_jpgurl(dict_url_title): 
58.     dict_title_jpgurl={} 
59.     for url in dict_url_title.keys(): 
60.         print "the list of url is ",getJpg(url) 
61.         dict_title_jpgurl[dict_url_title[url]]=getJpg(url) 
62.     print "dict_title_jpgurl\n" 
63.     print dict_title_jpgurl 
64.     return dict_title_jpgurl 
65. #保存图片,建立父亲文件路径,分别建立相关的文件夹,保存图片 
66. def store_jpg(parentPath,dict_title_jpgurl): 
67.     # 去除首位空格 
68.     path=parentPath 
69.     path=path.strip() 
70.     #path=path.rstrip("\\") 
71.     # 判断路径是否存在 如果不存在则创建目录 
72.     isExists=os.path.exists(path) 
73.     if not isExists: 
74.        os.mkdir(path) 
75.     for title in dict_title_jpgurl: 
76.         childPath=path+"/"+str(title)         #这个是linux 下的文件路径,windows下要修改 
77.         if(os.path.exists(childPath)): 
78.            continue 
79.         print childPath; 
80.         try: 
81.           os.mkdir(childPath) 
82.         except OSError: 
83.           continue 
84.         '''''
85.         if((os.mkdir(childPath))==False):
86.             continue
87.         ''' 
88.         jpgurl=dict_title_jpgurl[title] 
89.         for url in jpgurl: 
90.             jpgname=(url.split("/"))[-1] 
91.             print "write the file",jpgname 
92.             data=urllib2.urlopen(url).read() 
93.             jpgPath=childPath+"/"+jpgname 
94.             f = file(jpgPath,"wb") 
95.             f.write(data) 
96.             f.close() 
97. def readConfigure(): 
98.     path="/home/chujiangke/Python/config.txt" 
99.     f=file(path,"r") 
100.     for line in f.readlines(): 
101.            print line 
102.            line=line.strip() 
103.            line=line.rstrip() 
104.            line=line.split("/") 
105.            del line[-1] 
106.            line="/".join(line) 
107.            print line 
108.            for i in range(1,11,1): 
109.                print i 
110.                temp = "http://www.moko.cc"+line+"/"+str(i)+".html" 
111.                print temp 
112.                html=getHtml(temp) 
113.                dict_url_title=getDict(html) 
114.                dict_title_jpgUrl=getDict_title_jpgurl(dict_url_title) 
115.                store_jpg("/home/chujiangke/Python/Spider",dict_title_jpgUrl); 
116. def run(): 
117.    for i in range(1,11,1): 
118.        print i 
119.        html=getHtml("http://www.moko.cc/channels/post/28/"+str(i)+".html") 
120.        dict_url_title=getDict(html) 
121.        dict_title_jpgUrl=getDict_title_jpgurl(dict_url_title) 
122.        store_jpg("/home/chujiangke/Python/Spider",dict_title_jpgUrl); 
123. #readConfigure() 
124. run() 
125. #getJpg("http://www.moko.cc/post/962832.html")