基本部署见:
selenium+chromedriver部署到云主机并爬取东方财富网的股票评论(一)
更新的内容:
- 增加了多线程,目前为六个线程。
- 将一部分用selenium完成的代码,用requests实现,提高了运行效率。
Windows下运行结果:
Linux运行结果
下载内容:
源码:
import re
import requests
import os
import time
from selenium import webdriver
import platform
from threading import Thread
class Win():
def __init__(self):
pass
#抓取评论的html页和评论页数
def get_general_page_html(self, url):
'''
option = webdriver.ChromeOptions()
option.add_argument('--headless')
driver = webdriver.Chrome(options=option)
driver.get(url)
'''
html = requests.get(url).text
return html
# 评论页数
def get_num(self, url):
option = webdriver.ChromeOptions()
option.add_argument('--headless')
driver = webdriver.Chrome(options=option)
driver.get(url)
html = driver.page_source
driver.quit()
page = re.findall('sumpage.*?>(\d+)</span>', html, re.S)
try:
num = int(page[0])
except:
print("num = int(page[0]) error 34 lines")
return num
#得到评论列表的url
def get_shrot_url_toConstruct(self,html):
pattern=re.compile('articleh.*?l3 a3.*?href="/(.*?)" title',re.S)
url_list=re.findall(pattern,html)
return url_list
#写入到txt文件
def writen_to_file(sefl,comment, path, filename):
totName = os.path.join(path,filename)
with open(totName + '.txt', 'w+') as f:
f.write(comment.strip())
print