用kafka-python 和tkinter 做一个kafka可视化管理小工具, 可以消费,生产数据, 查看topic和cosumer

用kafka-python 和tkinter 做一个kafka可视化管理小工具,  可以消费,生产数据, 查看topic和cosumer

 

import tkinter as tk
from tkinter import ttk
import socket
import json
import traceback
from kafka import KafkaConsumer, KafkaProducer, KafkaAdminClient
from kafka import BrokerConnection
from kafka.protocol.commit import *

from tkinter.scrolledtext import ScrolledText

def show_group_detail(event):
    kafka_admin = KafkaAdminClient(bootstrap_servers=['kafka01'])
    group_name = tab4_listbox_group.get(tab4_listbox_group.curselection())
    group_desc_list = kafka_admin.describe_consumer_groups([group_name])
    print(group_name)
    print(group_desc_list)
    tab4_detail_text.delete('1.0', 'end')

    for item in group_desc_list:
        #tab4_detail_text.insert("insert", json.dumps(item, indent=2))
        print(type(item))
        tab4_detail_text.insert("insert", "[group]: %s \n" % item.group)
        tab4_detail_text.insert("insert", "[authorized_operations]: %s \n" % item.authorized_operations)
        tab4_detail_text.insert("insert", "[error_code]: %s \n" % item.error_code)
        tab4_detail_text.insert("insert", "[protocol]: %s \n" % item.protocol)
        tab4_detail_text.insert("insert", "[protocol_type]: %s \n" % item.protocol_type)
        tab4_detail_text.insert("insert", "[state]: %s \n" % item.state)
        tab4_detail_text.insert("insert", "[members]: %s \n")
        members_list = item.members
        for member in members_list:
            print(type(member))
            tab4_detail_text.insert("insert", " |---[client_host]: %s \n" % member.client_host)
            tab4_detail_text.insert("insert", " |---[assignment]: %s \n" % member.member_assignment.assignment)
        tab4_detail_text.update()

    bc = BrokerConnection('kafka01', 9092, socket.AF_INET)
    bc.connect_blocking()
    fetch_offset_request = OffsetFetchRequest_v3(group_name, None)
    future = bc.send(fetch_offset_request)
    tab4_detail_text.insert("insert", 'offsets info fetching...\n')
    while not future.is_done:
        for resp, f in bc.recv():
            f.success(resp)
    for topic in future.value.topics:
        print('offsets for {0}'.format(topic[0]))
        tab4_detail_text.insert("insert", 'offsets for {0} \n'.format(topic[0]))
        for partition in topic[1]:
            print('- partition {0}, offset: {1}'.format(partition[0], partition[1]))
            tab4_detail_text.insert("insert", '|--- partition {0}, offset: {1}\n'.format(partition[0], partition[1]))
            tab4_detail_text.update()
    tab4_detail_text.insert("insert", '----------finish------------\n')
    tab4_detail_text.update()


def show_all_topics():
    consumer = KafkaConsumer(group_id='kafka-pyclient', bootstrap_servers=['kafka01'])
    alltopics = consumer.topics()
    #print(alltopics)
    alltopics_list = []
    for topic in alltopics:
        alltopics_list.append(topic)

    alltopics_list.sort()
    for topic in alltopics_list:
        tab3_listbox_topic.insert("end", topic)


def show_all_groups():
    kafka_admin = KafkaAdminClient(bootstrap_servers=['kafka01'])
    allgroup_list = kafka_admin.list_consumer_groups()
    allgroup_list.sort()
    allgroupname_list = []
    tab4_listbox_group.delete(0,tk.END)
    for group in allgroup_list:
        group_name = group[0] #("group_name", "consumer")
        allgroupname_list.append(group_name)
        tab4_listbox_group.insert("end", group[0])
        #group_desc = kafka_admin.describe_consumer_groups([group_name])
        #print(group_desc)

    allgroup_desc = kafka_admin.describe_consumer_groups(allgroupname_list)
    print(allgroup_desc)


def text_clear():
    text.delete('1.0','end')

def text_show():
    text.insert("insert", "consumer start...\n")
    text.update()

    kafka_server_list = [hostname.get(),]
    kafka_consumer_topic = topicname.get()
    kafka_group_id = groupidname.get()
    kafka_offset_reset  = offsetname.get()
    limit  = int(limitname.get())
    keyword  = keywordname.get()
    text.insert("insert", "host: %s, topic: %s, groupid: %s, offset: %s  \n" % (str(kafka_server_list), kafka_consumer_topic, kafka_group_id, kafka_offset_reset ))
    text.update()

    consumer = KafkaConsumer(kafka_consumer_topic, group_id=kafka_group_id, bootstrap_servers=kafka_server_list,
                             auto_offset_reset=kafka_offset_reset)

    text.insert("insert", "consumer init ok, consuming...\n")
    text.insert("insert", "==========================================================================\n")
    text.update()

    count = 0
    if limit>1000:
        limit = 1000

    for msg in consumer:
        if count<limit:
            message = msg.value.decode('utf-8')
            if keyword and keyword!='':
                if keyword not in message:
                    continue
            #topic = 'tracker_data', partition = 0, offset = 3904137805, timestamp = -1, timestamp_type = 0, key = None, value = b
            text.insert("insert", "-----------------------------------------------------------------------\n")
            text.insert("insert", "[topic]:%s [partition]:%s [offset]:%s [key]:%s \n" %  (msg.topic, msg.partition, msg.offset, msg.key) )
            text.insert("insert", "[value]: %s \n" %  message)
            text.update()

            count += 1
        else:
            break

def produce_message():
    try:
        kafka_server = tab2_hostname.get()
        kafka_produce_topic = tab2_topicname.get()
        kafka_produce_key = tab2_keyname.get()
        tab2_restext.insert("insert", "host: %s, topic: %s, key: %s  \n" % ( kafka_server, kafka_produce_topic, kafka_produce_key))
        text.update()

        producer = KafkaProducer(bootstrap_servers=kafka_server)
        tab2_restext.insert("insert", "producer init ok\n")
        text.update()

        msg = tab2_msgname.get()
        msg_dic = json.loads(msg)

        tab2_restext.insert("insert", "produce msg: %s \n" % msg)
        text.update()

        producer.send(kafka_produce_topic, value=bytes(msg, encoding = "utf8"), key=bytes(kafka_produce_key,encoding = "utf8"))

        tab2_restext.insert("insert", "produce msg finish\n")
        text.update()

        producer.close()
    except:
        tab2_restext.insert("insert", traceback.format_exc())
        text.update()



pc_hostname = socket.gethostname()

window = tk.Tk()
# 设置窗口大小
winWidth = 1500
winHeight = 800
# 获取屏幕分辨率
screenWidth = window.winfo_screenwidth()
screenHeight = window.winfo_screenheight()

x = int((screenWidth - winWidth) / 2)
y = int((screenHeight - winHeight) / 2)

# 设置主窗口标题
window.title("kafka client tools")
# 设置窗口初始位置在屏幕居中
window.geometry("%sx%s+%s+%s" % (winWidth, winHeight, x, y))
# 设置窗口图标
#window.iconbitmap("./image/icon.ico")
# 设置窗口宽高固定
window.resizable(0, 0)

# 定义列的名称
#Frame1---------------------------------------------------------------------------------------------------
tab = ttk.Notebook(window)
frame1 = tk.Frame(tab) #, bg="red")
tab1 = tab.add(frame1, text=" Kafka-Consumer ")

# label= tk.Label(frame1,text="Label",justify=tk.LEFT)
# label.pack(side=tk.LEFT)

tk.Label(frame1, text='host:').place(x=15, y=10)
hostname = tk.StringVar()
hostname.set("kafka01:9092")
entry_hostname = tk.Entry(frame1, textvariable=hostname)
entry_hostname.place(x=160, y=10)

tk.Label(frame1, text='topic:').place(x=15, y=40)
topicname = tk.StringVar()
topicname.set("topic_name")
entry_topic = tk.Entry(frame1, textvariable=topicname)
entry_topic.place(x=160, y=40)

tk.Label(frame1, text='groupid:').place(x=15, y=70)
groupidname = tk.StringVar()
groupidname.set(pc_hostname+"-test1")
entry_gourpid = tk.Entry(frame1, textvariable=groupidname)
entry_gourpid.place(x=160, y=70)

tk.Label(frame1, text='offset(earliest/latest):').place(x=15, y=100)
offsetname = tk.StringVar()
offsetname.set("earliest")
entry_offset = tk.Entry(frame1, textvariable=offsetname)
entry_offset.place(x=160, y=100)

tk.Label(frame1, text='limit:').place(x=15, y=130)
limitname = tk.StringVar()
limitname.set("10")
entry_limit = tk.Entry(frame1, textvariable=limitname, )
entry_limit.place(x=160, y=130)

tk.Label(frame1, text='msg str filter:').place(x=15, y=160)
keywordname = tk.StringVar()
keywordname.set("")
entry_keyword = tk.Entry(frame1, textvariable=keywordname, )
entry_keyword.place(x=160, y=160)

bt_start = tk.Button(frame1,text="Consumer Start", width=20, height=2, command=text_show) #
bt_start.place(x=360, y=10)

bt_clear = tk.Button(frame1,text="Clear", width=20, height=2, command=text_clear) #
bt_clear.place(x=360, y=80)

#text = tk.scrolledtext.ScrolledText(window, width=100, height=60,font=("隶书",18))
text = tk.scrolledtext.ScrolledText(frame1, width=200, height=40)
#text = tk.Text(window, width=200, height=60)
text.place(x=15, y=190)



#Frame2---------------------------------------------------------------------------------------------------
#frame2 = tk.Frame(tab, bg="yellow")
frame2 = tk.Frame(tab)
tab2 = tab.add(frame2, text=" Kafka-Producer ")

tab2_hostlabel = tk.Label(frame2, text = "host：").grid(row = 1, sticky = tk.W, padx=5, pady=5)
tab2_hostname = tk.StringVar()
tab2_hostname.set("kafka01")
tk.Entry(frame2, textvariable=tab2_hostname).grid(row = 1,  column = 1, sticky = tk.E , padx=5, pady=5)

tab2_topiclabel = tk.Label(frame2, text = "topic：")
tab2_topiclabel.grid(row = 2, sticky = tk.W, padx=5, pady=5)
tab2_topicname = tk.StringVar()
tab2_topicname.set("topic_name")
tk.Entry(frame2, textvariable=tab2_topicname).grid(row = 2,  column=1, sticky = tk.E, padx=5, pady=5)

tab2_keylabel = tk.Label(frame2, text = "key：").grid(row = 3, sticky = tk.W, padx=5, pady=5)
tab2_keyname = tk.StringVar()
tab2_keyname.set("")
tk.Entry(frame2, textvariable=tab2_keyname).grid(row = 3,  column = 1, sticky = tk.E , padx=5, pady=5)

tk.Label(frame2, text='msg:').place(x=1, y=150)
tab2_msgname = tk.StringVar()
tab2_msgentry = tk.Entry(frame2, textvariable=tab2_msgname, width=200)
tab2_msgentry.place(x=65, y=150)

tk.Label(frame2, text='result:').place(x=1, y=350)
tab2_restext = tk.scrolledtext.ScrolledText(frame2, width=200, height=20)
tab2_restext.place(x=65, y=350)

bt_start = tk.Button(frame2,text="Produce Message", width=20, height=2, command=produce_message)
bt_start.place(x=360, y=40)



#hi_there = tk.Button(frame2,text="say hi",command=say_hi)
#hi_there.pack()


#Frame3---------------------------------------------------------------------------------------------------
frame3 = tk.Frame(tab)
tab3 = tab.add(frame3, text=" topics ")

tab3_bt_start = tk.Button(frame3,text="Show Topics", width=15, height=1, command=show_all_topics) #
tab3_bt_start.pack(side='top', anchor='nw', padx=5, pady=5, expand='yes')

tab3_listbox_topic = tk.Listbox(frame3, width=50, height=80)
tab3_listbox_topic.pack(side='left')

tab3_bt_start2 = tk.Button(frame3,text="Show Topics-Groups", width=20, height=1, command=show_all_topics) #
tab3_bt_start2.pack(side='top', anchor='nw', padx=5, pady=5)

tab3_listbox_topic_group = tk.Listbox(frame3, width=50, height=80)
tab3_listbox_topic_group.pack(side='left')

#Frame4---------------------------------------------------------------------------------------------------
frame4 = tk.Frame(tab)
tab4 = tab.add(frame4, text=" groups ")

tab4_bt_start = tk.Button(frame4,text="Show Groups", width=15, height=1, command=show_all_groups) #
tab4_bt_start.pack(side='top', anchor='sw', padx=5, pady=5)

tab4_listbox_group = tk.Listbox(frame4, width=50, height=80)
tab4_listbox_group.bind('<Double-Button-1>',show_group_detail)
tab4_listbox_group.pack(side='left')

tab4_detail_text = tk.scrolledtext.ScrolledText(frame4, width=200, height=40)
tab4_detail_text.pack()


#---------------------------------------------------------------------------------------------------------------
tab.pack(expand=True, fill=tk.BOTH)
# 设置选中tab2
tab.select(frame1)

window.mainloop()

 

运行效果: