python之知乎的正确打开方式

学习链接：https://zhuanlan.zhihu.com/p/28680797

 

主要呢是用了chrome driver去模拟滚动和点击

同样是用了beautiSoup去解析html文件

解析出图片的url之后转码之后下载图片

 

代码：

 1 # -*- coding:utf-8 -*-
 2 
 3 import sys
 4 import time
 5 import urllib
 6 from bs4 import BeautifulSoup
 7 from HTMLParser import HTMLParser
 8 from selenium import webdriver
 9 
10 reload(sys)
11 sys.setdefaultencoding("utf-8")
12 
13 def main():
14 
15     #用chrome driver打开页面
16     driver = webdriver.Chrome()
17     driver.get("https://www.zhihu.com/question/54104076")
18 
19     #用driver进行点击操作并模拟点击查看更多
20     def execute_times(times):
21 
22         for i in range(times + 1):
23             driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
24             time.sleep(2)
25             try:
26                 driver.find_element_by_css_selector('button.QuestionMainAction').click()
27                 print "page"+str(i)
28                 time.sleep(1)
29             except :
30                 break
31     execute_times(3) #打开三个页面
32 
33     #结构化HTML文件并用beautifulSoup解析
34     result_raw=driver.page_source
35     result_soup = BeautifulSoup(result_raw, 'html.parser')
36     result_bf = result_soup.prettify()
37 
38     #找到所有图片
39     with open("D:\\imagine\\raw_result.txt", 'w') as girls:
40         girls.write(result_bf)
41     girls.close()
42     print "Store raw data successfully!!!"
43     
44     #找到所有<nonscript>标签并记录下来
45     with open("D:\\imagine\\noscript_meta.txt", 'w') as noscript_meta:
46         noscript_nodes = result_soup.find_all('noscript')
47         noscript_inner_all = ""
48         for noscript in noscript_nodes:
49             noscript_inner = noscript.get_text()
50             noscript_inner_all += noscript_inner + "\n"
51 
52         h = HTMLParser()
53         noscript_all = h.unescape(noscript_inner_all)
54         noscript_meta.write(noscript_all)
55     noscript_meta.close()
56     print "Store noscript meta data successfully!!!" #成功找到图片
57 
58     #开始下载图片
59     img_soup = BeautifulSoup(noscript_all, 'html.parser')
60     img_nodes = img_soup.find_all('img')
61     with open("D:\\imagine\\img_meta.txt", 'w') as img_meta:
62         count = 0
63         for img in img_nodes:
64             if img.get('src') is not None:
65                 img_url = img.get('src')
66 
67                 line = str(count) + "\t" + img_url + "\n"
68                 img_meta.write(line)
69                 urllib.urlretrieve(img_url, "D:\\imagine\\" + str(count) + ".jpg") #下载图片
70                 count += 1
71 
72     img_meta.close()
73     print "Store meta data and images successfully!!!" #成功存储图片
74 
75 if __name__ == '__main__':
76     main()

 

 

结果图：

 

转载于:https://www.cnblogs.com/general10/p/7530388.html