kafka学习 （三）csharp开发

.net 开发kafka，这今天绕了不少坑，好多人不负责任瞎写，网上dll都没有一个，自己编译了一个，结果发现一个好网站：nuget 查看开源地址和最新release包https://www.nuget.org ，现在比较好的就两个kafka-net和rdkafka。经过实际测试，还是rdfakfa更胜一筹，因为他是基于librdkafka做二次开发的。下面给出github链接地址:
- kafka-net: https://github.com/Jroland/kafka-net
- rdkafka: https://github.com/ah-/rdkafka-dotnet
dll包放到dll文件夹里面，地址在：https://github.com/bluetom520/kafka_for_net，下面分别用两种dll举例

1 kafka-net

kafka-net的消费者不能指定latest，只能从开头开始消费，没找到解决办法，无奈放弃，生产者还是不错的，消费有延迟

1.1全局配置

app.config

<?xml version="1.0" encoding="utf-8" ?>
<configuration>
    <startup> 
        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.1" />
    </startup>
  <appSettings>
    <add key="KafkaBroker" value="http://26.2.4.171:9092,http://26.2.4.172:9092,http://26.2.4.173:9092" />
    <add key="Topic" value="test_topic" />
  </appSettings>
</configuration>

1.2 producer

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using KafkaNet;
using KafkaNet.Common;
using KafkaNet.Model;
using KafkaNet.Protocol;
using System.Configuration;

namespace producer
{
    class Program
    {
        static void Main(string[] args)
        {
            do
            {
                Produce(GetKafkaBroker(), getTopicName());
                System.Threading.Thread.Sleep(3000);
            } while (true);
        }

        private static void Produce(string broker, string topic)
        {

            string[] temp = broker.Split(',');
            Uri[] url = new Uri[3];
            int index = 0;
            foreach (string item in temp)
            {
                url[index] = new Uri(item);
                index++;
            }
            var options = new KafkaOptions(url);
            var router = new BrokerRouter(options);
            var client = new Producer(router);

            var currentDatetime = DateTime.Now;
            var key = currentDatetime.Second.ToString();
            var events = new[] { new Message("Hello World " + currentDatetime, key) };
            client.SendMessageAsync(topic, events).Wait(2000);
            Console.WriteLine("Produced: Key: {0}. Message: {1}", key, events[0].Value.ToUtf8String());

            using (client) { }
        }

        private static string GetKafkaBroker()
        {
            string KafkaBroker = string.Empty;
            const string kafkaBrokerKeyName = "KafkaBroker";

            if (!ConfigurationManager.AppSettings.AllKeys.Contains(kafkaBrokerKeyName))
            {
                KafkaBroker = "http://localhost:9092";
            }
            else
            {
                KafkaBroker = ConfigurationManager.AppSettings[kafkaBrokerKeyName];
            }
            return KafkaBroker;
        }
        private static string getTopicName()
        {
            string TopicName = string.Empty;
            const string topicNameKeyName = "Topic";

            if (!ConfigurationManager.AppSettings.AllKeys.Contains(topicNameKeyName))
            {
                throw new Exception("Key \"" + topicNameKeyName + "\" not found in Config file -> configuration/AppSettings");
            }
            else
            {
                TopicName = ConfigurationManager.AppSettings[topicNameKeyName];
            }
            return TopicName;
        }
    }
}

1.3 consumer

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using KafkaNet;
using KafkaNet.Common;
using KafkaNet.Model;
using KafkaNet.Protocol;
using System.Configuration;

namespace comsumer
{
    class Program
    {
        static void Main(string[] args)
        {
            Consume(getKafkaBroker(), getTopicName());

        }

        private static void Consume(string broker, string topic)
        {
            string[] temp = broker.Split(',');
            Uri[] url =new Uri[3];
            int index = 0;
            foreach(string item in temp)
            {
                url[index] = new Uri(item);
                index++;
            }
            var options = new KafkaOptions(url);
            var router = new BrokerRouter(options);
            OffsetPosition[] off =new OffsetPosition[3];
            //off
            off[0] = new OffsetPosition(0, 9999);
            off[1] = new OffsetPosition(1, 9999);
            off[2] = new OffsetPosition(2, 9999);
            Consumer consumer = new Consumer(new ConsumerOptions(topic, router),off);
            //var tt2 = consumer.GetTopicOffsetAsync(topic, 2, -1);
            ////var tt = consumer.GetOffsetPosition();