python chrome headless_[技巧] chrome headless 爬虫抓取websoket 数据

目录

源起

分析

实践

总结

源起

周末答应了一个朋友帮他看一下一个网站应该怎么爬，费话不说直接先上网站

https://datacenter.jin10.com/price

数据一直在不停的闪，直觉判断这种高频的显示应该不会用ajax 轮询的方式，至少也是websocket的方式

分析

老规矩，直接上chrome 的f12来分析看看

直接看到，右边正边疯狂的刷新数据，而使用的协议，正是websocket ，对于python下如何连接websocket ，网上有很多的文章，我这里就不用细说了，通常我们拿到这样的接口，都会本能去尝试直连看看，在进一步尝试之后，发现他们的api应该有一种特别的方式(又或者我的代码有写错的地方)

# coding:utf-8

from websocket import create_connection

from websocket import ABNF

api = "wss://sshibikfdn.jin10.com:9084/socket.io/?EIO=3&transport=websocket&sid=VsJvZikGdc8spBaPAAMO"

headers = {

'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',

'Cookie': 'UM_distinctid=16614315bf179b-0354a40c6714ff-34677908-232800-16614315bf2acb; XSRF-TOKEN=eyJpdiI6IkNZRU9uSmM1ZnY2M0VqNUttK1pxRGc9PSIsInZhbHVlIjoiRlJpNlRuekIxTDJZeVd3bHpvXC9OUEZGamw4VndZNEdXTEVsRjRMaFFyOEIxUHRtNDdTc1JaQ042eG4xdjlFeWJjWGlkcWFaeWl6NTRVUUlQMThaZmJ3PT0iLCJtYWMiOiJkYWU1MzQ2NjEyM2U3OTk0MzY5NWNjZTdhZmNlZjE0YTViMjc2YzBiYWM4YjhiMjNhZmRjMzU3YzliNDg3ZGIzIn0%3D; laravel_session=eyJpdiI6IjBRS3h0Y29XcGRBRlFIc0xIeWFiZGc9PSIsInZhbHVlIjoibVRLblpNTDJJa1JIN1ZJc0s5c2xrSkYzckNadDB6aGp0REd5SVJQTlkxNVAzajhvdXY5ZElSQ3VTcGVicjNiSXZ3NE9pZDZOdHJUM1d6WG1KQjZXNkE9PSIsIm1hYyI6Ijg3MWVkZDVlMDFjZDM2NDRjZmI2ZDhkNDJmZGI5MjNhMzk3MTViNmI1YTNmMDRmYWJjNzQ4ZGU2YWZhNzNhNzUifQ%3D%3D; io=VsJvZikGdc8spBaPAAMO',

'Host': 'sshibikfdn.jin10.com:9084',

'Connection': 'Upgrade',

'Origin': 'https://datacenter.jin10.com',

'Pragma': 'no-cache',

'Sec-WebSocket-Extensions': 'permessage-deflate; client_max_window_bits',

'Sec-WebSocket-Key': 'g4UA3smEJ0eGufMkyz7AOw==',

'Sec-WebSocket-Version': '13',

'Upgrade': 'websocket',

'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',

}

def get_web_socket():

start_message = "2probe"

ws = create_connection(

api,

header=headers,

cookie=headers['Cookie'],

#origin=headers['Origin'],

#host=headers['Host']

)

frame = ABNF.create_frame("2probe", ABNF.OPCODE_TEXT)

ws.send_frame(frame)

data = ws.recv_frame()

print(data)

if __name__ == '__main__':

get_web_socket()

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

# coding:utf-8

fromwebsocketimportcreate_connection

fromwebsocketimportABNF

api="wss://sshibikfdn.jin10.com:9084/socket.io/?EIO=3&transport=websocket&sid=VsJvZikGdc8spBaPAAMO"

headers={

'Accept-Language':'zh-CN,zh;q=0.9,en;q=0.8',

'Cookie':'UM_distinctid=16614315bf179b-0354a40c6714ff-34677908-232800-16614315bf2acb; XSRF-TOKEN=eyJpdiI6IkNZRU9uSmM1ZnY2M0VqNUttK1pxRGc9PSIsInZhbHVlIjoiRlJpNlRuekIxTDJZeVd3bHpvXC9OUEZGamw4VndZNEdXTEVsRjRMaFFyOEIxUHRtNDdTc1JaQ042eG4xdjlFeWJjWGlkcWFaeWl6NTRVUUlQMThaZmJ3PT0iLCJtYWMiOiJkYWU1MzQ2NjEyM2U3OTk0MzY5NWNjZTdhZmNlZjE0YTViMjc2YzBiYWM4YjhiMjNhZmRjMzU3YzliNDg3ZGIzIn0%3D; laravel_session=eyJpdiI6IjBRS3h0Y29XcGRBRlFIc0xIeWFiZGc9PSIsInZhbHVlIjoibVRLblpNTDJJa1JIN1ZJc0s5c2xrSkYzckNadDB6aGp0REd5SVJQTlkxNVAzajhvdXY5ZElSQ3VTcGVicjNiSXZ3NE9pZDZOdHJUM1d6WG1KQjZXNkE9PSIsIm1hYyI6Ijg3MWVkZDVlMDFjZDM2NDRjZmI2ZDhkNDJmZGI5MjNhMzk3MTViNmI1YTNmMDRmYWJjNzQ4ZGU2YWZhNzNhNzUifQ%3D%3D; io=VsJvZikGdc8spBaPAAMO',

'Host':'sshibikfdn.jin10.com:9084',

'Connection':'Upgrade',

'Origin':'https://datacenter.jin10.com',

'Pragma':'no-cache',

'Sec-WebSocket-Extensions':'permessage-deflate; client_max_window_bits',

'Sec-WebSocket-Key':'g4UA3smEJ0eGufMkyz7AOw==',

'Sec-WebSocket-Version':'13',

'Upgrade':'websocket',

'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',

}

defget_web_socket():

start_message="2probe"

ws=create_connection(

api,

header=headers,

cookie=headers['Cookie'],

#origin=headers['Origin'],

#host=headers['Host']

)

frame=ABNF.create_frame("2probe",ABNF.OPCODE_TEXT)

ws.send_frame(frame)

data=ws.recv_frame()

print(data)

if__name__=='__main__':

get_web_socket()

在运行程序之后毫无效果，另外端口的api会根据真实的请求变化 ，而且进一步的请求的cookie和key都会变化 ，看来直连的方式是行不通了，那没办法，只能走渲染的路了，selenium ? 可以是可以，不过我们要尝试一下新的路线和方法，那就直接上chrome-headless

实践

Headless Chrome指在headless模式下运行谷歌浏览器(以程序模式运行，没有界面)，自从这玩意儿出来之后， phantomjs的作者就宣布甩锅不维护了(人家也确实辛苦，没啥收益)，可以说是一个非常好的工具了，咱们说干就干

安装

直接使用docker 来安装chrome headless

docker run -d -p 9222:9222 --cap-add=SYS_ADMIN justinribeiro/chrome-headless

1

2

dockerrun-d-p9222:9222--cap-add=SYS_ADMINjustinribeiro/chrome-headless

环境

python 3.6

ubuntu16.04

pip install websocket-client

pip install requests

1

2

3

pipinstallwebsocket-client

pipinstallrequests

编码

这样我们已经启用了一个chrome headless的服务，那如何使用呢，我们使用websocket 和chrome header less进行交互,不多说了，直接上代码吧

import json

import time

import requests

import websocket

request_id = 0

target_url = 'https://datacenter.jin10.com/price'

def get_websocket_connection():

r = requests.get('http://10.10.2.42:9222/json') #这是开启docker chrome headless的机器地址

if r.status_code != 200:

raise ValueError("can not get the api ,please check if docker is ready")

conn_api = r.json()[0].get('webSocketDebuggerUrl')

return websocket.create_connection(conn_api)

def run_command(conn, method, **kwargs):

global request_id

request_id += 1

command = {'method': method,

'id': request_id,

'params': kwargs}

conn.send(json.dumps(command))

#while True:

msg = json.loads(conn.recv())

if msg.get('id') == request_id:

return msg

def get_element():

conn = get_websocket_connection()

msg = run_command(conn, 'Page.navigate', url=target_url)

time.sleep(5)

js = "var p = document.querySelector('.jin-pricewall_list-item_b').innerText ; p ;"

for _ in range(20):

time.sleep(1)

msg = run_command(conn, 'Runtime.evaluate', expression=js)

print(msg.get('result')['result']['value'])

if __name__ == '__main__':

get_element()

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

importjson

importtime

importrequests

importwebsocket

request_id=0

target_url='https://datacenter.jin10.com/price'

defget_websocket_connection():

r=requests.get('http://10.10.2.42:9222/json')#这是开启docker chrome headless的机器地址

ifr.status_code!=200:

raiseValueError("can not get the api ,please check if docker is ready")

conn_api=r.json()[0].get('webSocketDebuggerUrl')

returnwebsocket.create_connection(conn_api)

defrun_command(conn,method,**kwargs):

globalrequest_id

request_id+=1

command={'method':method,

'id':request_id,

'params':kwargs}

conn.send(json.dumps(command))

#while True:

msg=json.loads(conn.recv())

ifmsg.get('id')==request_id:

returnmsg

defget_element():

conn=get_websocket_connection()

msg=run_command(conn,'Page.navigate',url=target_url)

time.sleep(5)

js="var p = document.querySelector('.jin-pricewall_list-item_b').innerText ; p ;"

for_inrange(20):

time.sleep(1)

msg=run_command(conn,'Runtime.evaluate',expression=js)

print(msg.get('result')['result']['value'])

if__name__=='__main__':

get_element()

整体逻辑非常简单，打开指定页面，等待页面数据刷新，然后直接偷懒拿数据渲染之后的页面值，运行效果如下:

其中的0 是因为页面还在渲染之中，所以数据还没有正式的出现在前台界面上

总结

本次主要使用了chrome-headless的相关渲染环境来解决了我们抓取数据的问题，并且使用了websocket api 来进一步操作，其实google 官方有sdk进行操作，https://github.com/GoogleChrome/puppeteer ，渲染的终究不是高效的做法，但是对于这种单页面目的性很强的数据，可以尝试渲染大法