python版本:3.6
influxdb版本:1.5.2
kafka版本:2.11-1.1.0
1、基本环境
使用python消费kafka中的数据,然后插入influxdb
python安装 requests模块
python安装 kafka模块
kafka服务器:192.168.1.100:9092
influxdb服务器:192.168.1.100:8086
2、关键代码
# coding: utf-8
from kafka import KafkaConsumer
import requests
topic = 'host-resource-detector'
broker_list = ['192.168.1.100:9092'] ##kafka ip,也可以是集群
kafka = KafkaConsumer(topic, bootstrap_servers=broker_list, group_id='host-monitor', enable_auto_commit=True, auto_offset_reset='latest')
##带验证的写法,u 表示用户名,p 表示密码
influxdb_url = "http://10.25.172.88:8086/write?db=hostdb&u=monitor;&p=tiger1234"
for msg in kafka:
record = eval(str(msg.value,encoding = "utf-8"))
##从kafka消费数据并组装成influxdb sql语句
influxdb_data = "hostdb," + "hostname=" + record['hostname'] + " cpu_used=" + str(round(100-float(record['cpu_idle']),2)) + ",memory_used=" + str(record['memory_used']) + ",num_process=" + str(record['num_process']) + " " + str(record['timestamp']) + "000000000"
##通过http api插入数据到influxdb
response = requests.post(influxdb_url, data=influxdb_data)
##获取请求返回信息,特别是报错信息
print(response.content)