kafka

ssh root@192.168.0.220

密码 123456

查看日志

tail -200f /data1/druidio/logs/jvm/overlord.log

查看topic列表

./bin/kafka-topics.sh --list --zookeeper dev221.sugo.net:2181,dev220.sugo.net:2181,dev222.sugo.net:2181/kafka 

删除topic

 ./kafka-topics.sh --delete --topic test1 --zookeeper dev221.sugo.net:2181,dev220.sugo.net:2181,dev222.sugo.net:2181/kafka 


消费
bin/kafka-console-consumer.sh  --zookeeper dev221.sugo.net:2181,dev220.sugo.net:2181,dev222.sugo.net:2181/kafka  --topic test1 --from-beginning


查看topic记录数

bin/kafka-run-class.sh kafka.tools.GetOffsetShell --topic test1  --time -1 --broker-list 192.168.0.220:9092,192.168.0.222:9092,192.168.0.221:9092 --partitions 0

从kafka导入到druid

{
  "type": "lucene_supervisor",
  "dataSchema": {
    "dataSource": "test1",
    "parser": {
      "type": "string",
      "parseSpec": {
        "format": "json",
        "timestampSpec": {
          "column": "d|actionTime",
          "format": "millis"
        },
        "dimensionsSpec": {
        "dynamicDimension": true
        }
      }
    },
    "metricsSpec": [],
    "granularitySpec": {
      "type": "uniform",
      "segmentGranularity": "DAY",
      "queryGranularity": "NONE"
    }
  },
  "tuningConfig": {
    "type": "kafka",
    "maxRowsInMemory": 500000,
    "maxRowsPerSegment": 10000000,
    "intermediatePersistPeriod": "PT10M",
    "reportParseExceptions": true,
    "buildV9Directly": true
  },
  "ioConfig": {
    "topic": "testData0616",
    "consumerProperties": {
      "bootstrap.servers": "192.168.0.220:9092,192.168.0.221:9092,192.168.0.222:9092"
    },
    "taskCount": 1,
    "replicas": 1,
    "taskDuration": "PT3600S",
    "useEarliestOffset": "true"
  }
}

查询datesource核对

目录：/tmp/sugo-plyql

指令：plyql -h 192.168.0.220 -q 'desc test1' -v

http://192.168.0.220:8090/supervisor.html 创建task

1: 列出已经创建的topic
./bin/kafka-topics.sh --list --zookeeper 192.168.0.215:2181/kafka


2: topic详细元数据描述
./bin/kafka-topics.sh --zookeeper 192.168.0.215:2181/kafka --describe --topic sugo


3: 删除topic
./bin/kafka-topics.sh --zookeeper 192.168.0.215:2181/kafka --delete --topic sugo


4: 消费数据
./bin/kafka-console-consumer.sh --zookeeper 192.168.0.215:2181/kafka --from-beginning --topic sugo


5: 生产数据
./bin/kafka-console-producer.sh --broker-list 192.168.0.215:9092 --topic sugo


6: 创建topic
./bin/kafka-topics.sh --create --zookeeper 192.168.0.215:2181/kafka --replication-factor 2 --partitions 2 --topic sugo


7: 检查是否有不可用的partition
./bin/kafka-topics.sh --describe --zookeeper 192.168.0.215:2181/kafka --unavailable-partitions


8: 查看topic的offset(time参数值 -1=latest, -2=earliest)
./bin/kafka-run-class.sh kafka.tools.GetOffsetShell --topic s1 --broker-list 192.168.0.215:9092 --time -1


9: 指定单个partition消费数据
./bin/kafka-run-class.sh kafka.tools.SimpleConsumerShell --broker-list 192.168.0.215:9092 --topic sugo --offset -1 --partition 0