websocket爬虫

在爬取虚拟货币的交易信息时,为保证数据的及时性,网站通常会使用websocket协议.此时我们的爬虫也需要使用websocket来处理这类问题.

• websocket的优点是在于:
  1.可以节省每次请求的headers,一次请求可以节省几十个字节;
  2.服务器可以自主向客户端传递数据,不用像传统的轮询的方式一样.
• 爬取neotracker的货币的数据并存入数据库

import websocket
import json
import time
from mysql_link import Mysql_Link
if __name__ == '__main__':
    # 1544749512
    mysql = Mysql_Link()
    sql = 'select max(updateTime) from h_onchain where cid = %s'
    data = (3134)
    last_time = mysql.select(sql, data)[0][0]
    print(last_time)
    ws = websocket.create_connection('wss://neotracker.io/graphql')
    ws.send('{"type":"GQL_START","id":"6","query":{"id":"25","variables":{"hash":"0d821bd7b6d53f5c2b40e217c6defc8bbe896cf5","first":1000,"after":null}},"span":{}}')
    while True:
    	# 应为这个网站的刷新频率并没有那么高,所以
    	time.sleep(0.5)
	    content1 = ws.recv()
	    c1 = json.loads(content1)
	    print(type(c1),c1)
	    mysql = Mysql_Link()
	    all = c1['value']['data']['asset']['transfers']['edges']
	    for one in all:
	        txHash = one['node']['transaction_hash']
	        quarlity = one['node']['value']
	        From = one['node']['from_address_id']
	        To = one['node']['to_address_id']
	        updateTime = one['node']['block_time']
	        data = (txHash,quarlity,From,To)
	
	        #if txHash and quarlity and From and To and updateTime:
	
	        if updateTime > last_time:
	            value = (5, 3134, txHash, From, To, quarlity, updateTime, 2, 1)
	            sql = 'insert into h_onchain(chainId,cid,txHash,`from`,`to`,quantity,updateTime,tradeType,belongTo)values (%s,%s,%s,%s,%s,%s,%s,%s,%s)'
	            print('入库操作', value)
	
	            mysql.execute(sql, value)
    ws.close()