from pykafka import KafkaClient
from pykafka.common import OffsetType
import datetime
def KafkaDownloader(host_, topic_, group_id_):
client = KafkaClient(hosts=host_)
_topic = client.topics[bytes(topic_.encode())]
consumer = _topic.get_simple_consumer(
consumer_group=bytes(group_id_.encode()),
auto_commit_enable=False,
auto_offset_reset=OffsetType.LATEST,
reset_offset_on_start=True
)
if consumer is not None:
for message in consumer:
if message is not None:
yield message.value
def get_kafka_data():
write_file = open("/data/itv_pm_data_1.txt", "w", encoding="utf8")
TOPIC = "_topic"
HOSTS = "IP1:9092,IP2:9092"
GROUP = "test"
dict_msg = {}
end_time = str(datetime.datetime.now() + datetime.timedelta(minutes=10))[0:len("201
python 在一定时间段获取kafka数据
最新推荐文章于 2024-03-07 14:20:44 发布