python读取kafka

需求： 

手动逐条提交偏移量

代码：

# -*- coding: utf-8 -*-
from kafka import KafkaConsumer, TopicPartition, OffsetAndMetadata
import ast
import pymysql
import time
import datetime





consumer = KafkaConsumer("order",bootstrap_servers=['服务'], auto_offset_reset='earliest',group_id='ordergroup3',enable_auto_commit=False)
'''
必要参数
                    1.主题（topic） 
                    2. 服务  (ip)
                    3. auto_offset_reset     earliest 从头开始消费    lates 从最新数据开始消费       两种参数在有提交偏移量的前提下均从上次提交的偏移量开始消费
                    4.消费组（group_id） 用来提交偏移量
                    5. enable_auto_commit  默认为True 即自动提交                                     
                    '''
# auto_offset_reset='earliest'
print("连接成功")

for msg in consumer:
    if msg is not None:
        value = msg.value.decode()
        #print(value)
        value_dict = ast.literal_eval(value)
        data = value_dict.get('original_data')
        data = eval(data)[:-3]
        #print(data)
        #print(type(data))
        
        
        db = pymysql.connect(host='0.0.0.0',user='root',port=3306,password='root',db='db')
        cur = db.cursor()
        sql = "insert ignore into work_main_info values({})".format(str(["%s" for i in range(0,len(data))]).strip('[]').replace("'",""))
        #print(sql)
        try:
            cur.execute(sql,data)
            db.commit()
            
        except Exception as e:
            print("入库失败",e)
            print(sql,data)
            print("====================回滚==================")
            db.rollback()
        else:
            print("==================================入库成功================================")
     
        cur.close()
        db.close()
        consumer.commit()