#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# here put the import lib
import argparse
from os import close
import time
from confluent_kafka import Consumer
import uuid
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--topic", type=str)
parser.add_argument("--host", type=str, default="127.0.0.1:9092")
parser.add_argument("-f", "--file", type=str, required=True)
args = parser.parse_args()
consumer = Consumer({
'bootstrap.servers': args.host,
'group.id': "downloader-{}".format(uuid.uuid4),
'auto.offset.reset': 'earliest',
})
consumer.subscribe([args.topic])
timeout_try_max = 3
timeout_try_count = 0
wf = open(args.file, "w", encoding="utf-8")
try:
while True:
msg = consumer.poll(2)
if msg is None:
print(None)
if timeout_try_count > timeout_try_max:
break
timeout_try_count += 1
time.sleep(1)
continue
elif msg.error():
buf = "Consumer error: {}".format(msg.error())
print(buf)
continue
wf.write(msg.value().decode("utf-8") + "\n")
timeout_try_count = 0
except KeyboardInterrupt:
pass
finally:
wf.close()
kafka 数据拉取到本地文件脚本
最新推荐文章于 2024-05-09 00:46:31 发布