python 爬虫源码 selenium_python爬虫——selenium爬取websocket的网站数据的简单实现

1、目标网址：https://eosflare.io/whales

2、通信机制：websocket

3、python代码

import time

import json

from selenium import webdriver

from selenium.webdriver.support.ui import WebDriverWait

from selenium.webdriver.support import expected_conditions as EC

from selenium.webdriver.common.by import By

def chrome_start():

#开启浏览器

chrome = webdriver.Chrome()

chrome.get('https://eosflare.io/whales')

wait = WebDriverWait(chrome, 10)

#采用xpath监测数据是否加载出来

wait.until(EC.presence_of_element_located((By.XPATH, '//div[@class="flex xs12 sm4 md3 lg2"]')))

#点击每页50按钮

chrome.find_element_by_xpath('//div[@class="input-group__selections__comma"]').click()

#点击每页500按钮

chrome.find_element_by_xpath('//div[@class="list__tile__title"][contains(text(),"500")]').click()

#延迟1秒，等待数据加载

time.sleep(1)

#继续监测数据是否加载

wait.until(EC.presence_of_element_located((By.XPATH, '//div[@class="flex xs12 sm4 md3 lg2"]')))

#处理数据

while True:

data_dispose(chrome)

chrome.find_element_by_xpath('//div[@class="data-iterator__actions__range-controls"]/button[2]').click()

time.sleep(1)

wait.until(EC.presence_of_element_located((By.XPATH, '//div[@class="flex xs12 sm4 md3 lg2"]')))

def data_dispose(chrome):

#提取整行数据

data = chrome.find_elements_by_xpath('//div[@class="container fluid grid-list-md"]/div[@class="layout row wrap"]')

olist = []

for d in data:

olist.append(d.text)

#打开文件，将数据写入文件中

with open("data.txt", "a", encoding="utf-8") as f:

f.write(json.dumps(olist, ensure_ascii=False))

f.write("\n")

print("保存成功")

if __name__ == '__main__':

# chrome_headless()

chrome_start()

4、运行结果