.net 开发kafka,这今天绕了不少坑,好多人不负责任瞎写,网上dll都没有一个,自己编译了一个,结果发现一个好网站:nuget 查看开源地址和最新release包https://www.nuget.org ,现在比较好的就两个kafka-net和rdkafka。经过实际测试,还是rdfakfa更胜一筹,因为他是基于librdkafka做二次开发的。下面给出github链接地址:
- kafka-net: https://github.com/Jroland/kafka-net
- rdkafka: https://github.com/ah-/rdkafka-dotnet
dll包放到dll文件夹里面,地址在:https://github.com/bluetom520/kafka_for_net,下面分别用两种dll举例
1 kafka-net
kafka-net的消费者不能指定latest,只能从开头开始消费,没找到解决办法,无奈放弃,生产者还是不错的,消费有延迟
1.1全局配置
app.config
<?xml version="1.0" encoding="utf-8" ?>
<configuration>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.1" />
</startup>
<appSettings>
<add key="KafkaBroker" value="http://26.2.4.171:9092,http://26.2.4.172:9092,http://26.2.4.173:9092" />
<add key="Topic" value="test_topic" />
</appSettings>
</configuration>
1.2 producer
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using KafkaNet;
using KafkaNet.Common;
using KafkaNet.Model;
using KafkaNet.Protocol;
using System.Configuration;
namespace producer
{
class Program
{
static void Main(string[] args)
{
do
{
Produce(GetKafkaBroker(), getTopicName());
System.Threading.Thread.Sleep(3000);
} while (true);
}
private static void Produce(string broker, string topic)
{
string[] temp = broker.Split(',');
Uri[] url = new Uri[3];
int index = 0;
foreach (string item in temp)
{
url[index] = new Uri(item);
index++;
}
var options = new KafkaOptions(url);
var router = new BrokerRouter(options);
var client = new Producer(router);
var currentDatetime = DateTime.Now;
var key = currentDatetime.Second.ToString();
var events = new[] { new Message("Hello World " + currentDatetime, key) };
client.SendMessageAsync(topic, events).Wait(2000);
Console.WriteLine("Produced: Key: {0}. Message: {1}", key, events[0].Value.ToUtf8String());
using (client) { }
}
private static string GetKafkaBroker()
{
string KafkaBroker = string.Empty;
const string kafkaBrokerKeyName = "KafkaBroker";
if (!ConfigurationManager.AppSettings.AllKeys.Contains(kafkaBrokerKeyName))
{
KafkaBroker = "http://localhost:9092";
}
else
{
KafkaBroker = ConfigurationManager.AppSettings[kafkaBrokerKeyName];
}
return KafkaBroker;
}
private static string getTopicName()
{
string TopicName = string.Empty;
const string topicNameKeyName = "Topic";
if (!ConfigurationManager.AppSettings.AllKeys.Contains(topicNameKeyName))
{
throw new Exception("Key \"" + topicNameKeyName + "\" not found in Config file -> configuration/AppSettings");
}
else
{
TopicName = ConfigurationManager.AppSettings[topicNameKeyName];
}
return TopicName;
}
}
}
1.3 consumer
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using KafkaNet;
using KafkaNet.Common;
using KafkaNet.Model;
using KafkaNet.Protocol;
using System.Configuration;
namespace comsumer
{
class Program
{
static void Main(string[] args)
{
Consume(getKafkaBroker(), getTopicName());
}
private static void Consume(string broker, string topic)
{
string[] temp = broker.Split(',');
Uri[] url =new Uri[3];
int index = 0;
foreach(string item in temp)
{
url[index] = new Uri(item);
index++;
}
var options = new KafkaOptions(url);
var router = new BrokerRouter(options);
OffsetPosition[] off =new OffsetPosition[3];
//off
off[0] = new OffsetPosition(0, 9999);
off[1] = new OffsetPosition(1, 9999);
off[2] = new OffsetPosition(2, 9999);
Consumer consumer = new Consumer(new ConsumerOptions(topic, router),off);
//var tt2 = consumer.GetTopicOffsetAsync(topic, 2, -1);
////var tt = consumer.GetOffsetPosition();