Kafka.net使用编程入门(三)

这个世界既不是有钱人的世界，也不是有权人的世界，它是有心人的世界。

---

一些有用的命令

1.列出主题：kafka-topics.bat --list  --zookeeper localhost:2181

---

2.描述主题：kafka-topics.bat --describe --zookeeper localhost:2181 --topic [Topic Name]

---

3.从头读取消息：kafka-console-consumer.bat --zookeeper localhost:2181 --topic [Topic Name] --from-beginning

---

4.删除主题：E:\WorkSoftWare\kafka2.11\bin\windows>kafka-run-class.bat kafka.admin.TopicCommand --delete --topic test2016 --zookeeper localhost:2181

---

实体类:

KafkaProducerMessage.cs代码:

  [Table("KafkaProducerMessage")]
    public partial class KafkaProducerMessage
    {
        public int KafkaProducerMessageID { get; set; }

        [Required]
        [StringLength(1000)]
        public string Topic { get; set; }

        [Required]
        public string MessageContent { get; set; }

        public DateTime CreatedAt { get; set; }
    }

---

KafkaConsumerMessage.cs代码:

    [Table("KafkaConsumerMessage")]
    public partial class KafkaConsumerMessage
    {
        public int KafkaConsumerMessageID { get; set; }

        [Required]
        [StringLength(1000)]
        public string Topic { get; set; }

        public int Partition { get; set; }

        public long Offset { get; set; }

        [Required]
        public string MessageContent { get; set; }

        public DateTime CreatedAt { get; set; }
    }

---

KafkaProducerMessageArchive.cs代码:

    [Table("KafkaProducerMessageArchive")]
    public partial class KafkaProducerMessageArchive
    {
        public int KafkaProducerMessageArchiveID { get; set; }

        public int KafkaProducerMessageID { get; set; }

        [Required]
        [StringLength(1000)]
        public string Topic { get; set; }

        [Required]
        public string MessageContent { get; set; }

        public DateTime CreatedAt { get; set; }

        public DateTime ArchivedAt { get; set; }
    }

vwMaxOffsetByPartitionAndTopic.cs代码:

    [Table("vwMaxOffsetByPartitionAndTopic")]
    public partial class vwMaxOffsetByPartitionAndTopic
    {
        [Key]
        [Column(Order = 0)]
        [StringLength(1000)]
        public string Topic { get; set; }

        [Key]
        [Column(Order = 1)]
        [DatabaseGenerated(DatabaseGeneratedOption.None)]
        public int Partition { get; set; }

        public long? MaxOffset { get; set; }
    }

---

KafkaConsumerRepository.cs代码:

    public class KafkaConsumerRepository
    {
        private KafkaModel context;

        public KafkaConsumerRepository ()
        {
            this.context = new KafkaModel();
        }

        public List<KafkaConsumerMessage> GetKafkaConsumerMessages()
        {
            return context.KafkaConsumerMessage.ToList();
        }

        public KafkaConsumerMessage GetKafkaConsumerMessageByID(int MessageID)
        {
            return context.KafkaConsumerMessage.Find(MessageID);
        }

        public List<KafkaConsumerMessage> GetKafkaConsumerMessageByTopic(string TopicName)
        {
            return context.KafkaConsumerMessage
                .Where(a => a.Topic == TopicName)
                .ToList();
        }

        public List<vwMaxOffsetByPartitionAndTopic> GetOffsetPositionByTopic(string TopicName)
        {
            return context.vwMaxOffsetByPartitionAndTopic
                .Where(a => a.Topic == TopicName)
                .ToList();
        }

        public void InsertKafkaConsumerMessage(KafkaConsumerMessage Message)
        {
            Console.WriteLine(String.Format("Insert {0}: {1}", Message.KafkaConsumerMessageID.ToString(), Message.MessageContent));
            context.KafkaConsumerMessage.Add(Message);
            context.SaveChanges();

            Console.WriteLine(String.Format("Saved {0}: {1}", Message.KafkaConsumerMessageID.ToString(), Message.MessageContent));
        }

        public void Save()
        {
            context.SaveChanges();
        }

        public void Dispose()
        {
            context.Dispose();
        }
    }

---

KafkaProducerRepository.cs代码:

  public class KafkaProducerRepository
    {
        private KafkaModel context;
        public KafkaProducerRepository()
        {
            this.context = new KafkaModel();
        }

        public List<KafkaProducerMessage> GetKafkaProducerMessages()
        {
            return context.KafkaProducerMessage.ToList();
        }

        public List<string> GetDistinctTopics()
        {
            return context.KafkaProducerMessage.Select(a => a.Topic)
                .Distinct()
                .ToList();
        }

        public KafkaProducerMessage GetKafkaProducerMessageByMessageID(int MessageID)
        {
            return context.KafkaProducerMessage.Find(MessageID);
        }

        public List<KafkaProducerMessage> GetKafkaProducerMessageByTopic(string TopicName)
        {
           return context.KafkaProducerMessage
                    .Where(a => a.Topic == TopicName)
                    .OrderBy(a => a.KafkaProducerMessageID)
                    .ToList<KafkaProducerMessage>();
        }

        public void InsertKafkaProducerMessage(KafkaProducerMessage Message)
        {
            context.KafkaProducerMessage.Add(Message);
            context.SaveChanges();
        }

        public void ArchiveKafkaProducerMessage(int MessageID)
        {
            KafkaProducerMessage m = context.KafkaProducerMessage.Find(MessageID);
            KafkaProducerMessageArchive archivedMessage = KafkaProducerMessageToKafkaProducerMessageArchive(m);

            using (var dbContextTransaction = context.Database.BeginTransaction())
            {
                try
                {
                    context.KafkaProducerMessageArchive.Add(archivedMessage);
                    context.KafkaProducerMessage.Remove(m);
                    dbContextTransaction.Commit();
                    context.SaveChanges();
                }
                catch (Exception)
                {
                    dbContextTransaction.Rollback();
                    context.SaveChanges();
                }
            }
        }

        public void ArchiveKafkaProducerMessageList(List<KafkaProducerMessage> Messages)
        {
            foreach (KafkaProducerMessage Message in Messages)
            {
                ArchiveKafkaProducerMessage(Message.KafkaProducerMessageID);
            }
        }

        public static KafkaProducerMessageArchive KafkaProducerMessageToKafkaProducerMessageArchive(KafkaProducerMessage Message)
        {
            return new KafkaProducerMessageArchive
            {
                KafkaProducerMessageID = Message.KafkaProducerMessageID,
                MessageContent = Message.MessageContent,
                Topic = Message.Topic,
                CreatedAt = Message.CreatedAt,
                ArchivedAt = DateTime.UtcNow
            };
        }

        public void Save()
        {
            context.SaveChanges();
        }

        public void Dispose()
        {
            context.Dispose();
        }
    }

---

数据库上下文KafkaModel.cs代码:

   public partial class KafkaModel : DbContext
    {
        public KafkaModel()
            : base("name=KafkaModel")
        {
        }

        public virtual DbSet<KafkaConsumerMessage> KafkaConsumerMessage { get; set; }
        public virtual DbSet<KafkaProducerMessage> KafkaProducerMessage { get; set; }
        public virtual DbSet<KafkaProducerMessageArchive> KafkaProducerMessageArchive { get; set; }
        public virtual DbSet<vwMaxOffsetByPartitionAndTopic> vwMaxOffsetByPartitionAndTopic { get; set; }

        protected override void OnModelCreating(DbModelBuilder modelBuilder)
        {
            modelBuilder.Entity<KafkaConsumerMessage>()
                .Property(e => e.Topic)
                .IsUnicode(false);

            modelBuilder.Entity<KafkaProducerMessage>()
                .Property(e => e.Topic)
                .IsUnicode(false);

            modelBuilder.Entity<KafkaProducerMessageArchive>()
                .Property(e => e.Topic)
                .IsUnicode(false);

            modelBuilder.Entity<vwMaxOffsetByPartitionAndTopic>()
                .Property(e => e.Topic)
                .IsUnicode(false);
        }
    }

---

Program.cs代码:

class Program
    {
        private static string Topic;
        static void Main(string[] args)
        {
            string invalidArgErrorMessage = "有效的args是：produce或consume";

            if (args.Length < 1)
            {
                throw (new Exception(invalidArgErrorMessage));
            }

            string intent = args[1];

            Topic = ConfigurationManager.AppSettings["Topic"];

            if (String.Equals(intent, "consume", StringComparison.OrdinalIgnoreCase))
            {
                Console.WriteLine("开始消费者服务");
                Consume();
            }
            else if (String.Equals(intent, "produce", StringComparison.OrdinalIgnoreCase))
            {
                Console.WriteLine("开始生产者服务");
                Produce();
            }
            else
            {
                throw (new Exception(invalidArgErrorMessage));
            }
        }

        private static BrokerRouter InitDefaultConfig()
        {
            List<Uri> ZKURIList = new List<Uri>();

            foreach (string s in ConfigurationManager.AppSettings["BrokerList"].Split(','))
            {
                ZKURIList.Add(new Uri(s));
            }

            var Options = new KafkaOptions(ZKURIList.ToArray());
            var Router = new BrokerRouter(Options);
            return Router;
        }

        private static void Consume()
        {
            KafkaConsumerRepository KafkaRepo = new KafkaConsumerRepository();
            bool FromBeginning = Boolean.Parse(ConfigurationManager.AppSettings["FromBeginning"]);

            var Router = InitDefaultConfig();
            var Consumer = new Consumer(new ConsumerOptions(Topic, Router)); 

            //如果我们不想从头开始，使用最新偏移量。
            if (!FromBeginning)
            {
                var MaxOffsetByPartition = KafkaRepo.GetOffsetPositionByTopic(Topic);
                //如果我们得到一个结果使用它，否则默认
                if (MaxOffsetByPartition.Count != 0)
                {
                    List<OffsetPosition> offsets = new List<OffsetPosition>();
                    foreach (var m in MaxOffsetByPartition)
                    {
                        OffsetPosition o = new OffsetPosition(m.Partition, (long)m.MaxOffset + 1);
                        offsets.Add(o);
                    }
                    Consumer.SetOffsetPosition(offsets.ToArray());
                }
                else
                {
                    Consumer.SetOffsetPosition(new OffsetPosition());
                }
            }

            //消耗返回一个阻塞IEnumerable（ie：从没有结束流）
            foreach (var message in Consumer.Consume())
            {
                string MessageContent = Encoding.UTF8.GetString(message.Value);

                Console.WriteLine(String.Format("处理带有内容的消息: {0}", MessageContent));

                KafkaRepo = new KafkaConsumerRepository();

                KafkaConsumerMessage ConsumerMessage = new KafkaConsumerMessage()
                {
                    Topic = Topic,
                    Offset = (int)message.Meta.Offset,
                    Partition = message.Meta.PartitionId,
                    MessageContent = MessageContent,
                    CreatedAt = DateTime.UtcNow
                };

                KafkaRepo.InsertKafkaConsumerMessage(ConsumerMessage);
                KafkaRepo.Dispose();
            }
        }

        private static void Produce()
        {
            KafkaProducerRepository KafkaRepo = new KafkaProducerRepository();

            var Router = InitDefaultConfig();
            var Client = new Producer(Router);

            List<Message> Messages = new List<Message>();

            foreach (KafkaProducerMessage message in KafkaRepo.GetKafkaProducerMessageByTopic(Topic))
            {
                Messages.Add(new Message(message.MessageContent));
                KafkaRepo.ArchiveKafkaProducerMessage(message.KafkaProducerMessageID);
            }

            Client.SendMessageAsync(Topic, Messages).Wait();

            KafkaRepo.Dispose();
        }
    }

---

App.config内容:

<?xml version="1.0" encoding="utf-8"?>
<configuration>
  <configSections>
    <section name="entityFramework" type="System.Data.Entity.Internal.ConfigFile.EntityFrameworkSection, EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" requirePermission="false"/>
  </configSections>
  <startup>
    <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5"/>
  </startup>
  <appSettings>
    <add key="BrokerList" value="http://localhost:9092"/>
    <add key="Topic" value="test2017"/>
    <add key="FromBeginning" value="true"/>
  </appSettings>
  <entityFramework>
    <defaultConnectionFactory type="System.Data.Entity.Infrastructure.LocalDbConnectionFactory, EntityFramework">
      <parameters>
        <parameter value="mssqllocaldb"/>
      </parameters>
    </defaultConnectionFactory>
    <providers>
      <provider invariantName="System.Data.SqlClient" type="System.Data.Entity.SqlServer.SqlProviderServices, EntityFramework.SqlServer"/>
    </providers>
  </entityFramework>
  <connectionStrings>
    <add name="KafkaModel" connectionString="Data Source=.;Initial Catalog=KafkaDemo;User ID=sa;Password=wu199010;multipleactiveresultsets=True;application name=EntityFramework" providerName="System.Data.SqlClient"/>
  </connectionStrings>
  <system.web>
    <membership defaultProvider="ClientAuthenticationMembershipProvider">
      <providers>
        <add name="ClientAuthenticationMembershipProvider" type="System.Web.ClientServices.Providers.ClientFormsAuthenticationMembershipProvider, System.Web.Extensions, Version=4.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35" serviceUri=""/>
      </providers>
    </membership>
    <roleManager defaultProvider="ClientRoleProvider" enabled="true">
      <providers>
        <add name="ClientRoleProvider" type="System.Web.ClientServices.Providers.ClientRoleProvider, System.Web.Extensions, Version=4.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35" serviceUri="" cacheTimeout="86400"/>
      </providers>
    </roleManager>
  </system.web>
</configuration>

---

sql脚本:

IF NOT EXISTS(SELECT 1 FROM sys.tables WHERE name = 'KafkaConsumerMessage')
BEGIN

    CREATE TABLE [dbo].[KafkaConsumerMessage](
        [KafkaConsumerMessageID] [int] IDENTITY(1,1) NOT NULL,
        [Topic] [varchar](1000) NOT NULL,
        [Partition] [int] NOT NULL,
        [Offset] [bigint] NOT NULL,
        [MessageContent] [nvarchar](max) NOT NULL,
        [CreatedAt] [datetime] NOT NULL,
        CONSTRAINT [PK_Kafka_Consumer_Message_KafkaConsumerMessageID] PRIMARY KEY CLUSTERED ([KafkaConsumerMessageID])
    )
END

GO

IF NOT EXISTS(SELECT 1 FROM sys.tables WHERE name = 'KafkaProducerMessage')
BEGIN
    CREATE TABLE [dbo].[KafkaProducerMessage](
        [KafkaProducerMessageID] [int] IDENTITY(1,1) NOT NULL,
        [Topic] [varchar](1000) NOT NULL,
        [MessageContent] [nvarchar](max) NOT NULL,
        [CreatedAt] [datetime] NOT NULL,
        CONSTRAINT [PK_KafkaProducerMessage_KafkaProducerMessageID] PRIMARY KEY CLUSTERED ([KafkaProducerMessageID])
    )
END

GO

IF NOT EXISTS(SELECT 1 FROM sys.tables WHERE name = 'KafkaProducerMessageArchive')
BEGIN
    CREATE TABLE [dbo].[KafkaProducerMessageArchive](
        [KafkaProducerMessageArchiveID] [int] IDENTITY(1,1) NOT NULL,
        [KafkaProducerMessageID] [int] NOT NULL,
        [Topic] [varchar](1000) NOT NULL,
        [MessageContent] [nvarchar](max) NOT NULL,
        [CreatedAt] [datetime] NOT NULL,
        [ArchivedAt] [datetime] NOT NULL,
        CONSTRAINT [PK_KafkaProducerMessageArchive_KafkaProducerMessageArchiveID] PRIMARY KEY CLUSTERED ([KafkaProducerMessageArchiveID])
    )
END

GO

---

引用.net kafka 组件：

---

启动zookeeper服务器，kafka服务器

打开一个DOS环境窗口

zkServer

打开第二个DOS窗口

kafka-server-start D:xxxxxxxxx\config\server.properties

---

结果如图:

手动添加生产者表三条记录:

---

---

消费者表

---