from pykafka import KafkaClient
from pykafka.common import OffsetType
import datetime
def KafkaDownloader(host_, topic_, group_id_):
client = KafkaClient(hosts=host_)
_topic = client.topics[bytes(topic_.encode())]
consumer = _topic.get_simple_consumer(
consumer_group=bytes(group_id_.encode()),
auto_commit_enable=False,
auto_offset_reset=OffsetType.LATEST,
reset_offset_on_start=True
)
if consumer is not None:
for message in consumer:
if message is not None:
yield message.value
def get_kafka_data():
write_file = open("/data/itv_pm_data_1.txt", "w", encoding="utf8")
TOPIC = "_topic"
HOSTS = "IP1:9092,IP2:9092"
GROUP = "test"
dict_msg = {}
end_time = str(datetime.datetime.now() + datetime.timedelta(minutes=10))[0:len("201
python 在一定时间段获取kafka数据
最新推荐文章于 2024-08-12 08:37:55 发布
本文介绍了如何使用Python连接Kafka,并在指定时间范围内有效地获取和处理Kafka消息,详细阐述了相关代码实现与步骤。
摘要由CSDN通过智能技术生成