Druid摄取kafka实时流json配置

{
	"type": "kafka",                              //摄取类型
	"dataSchema": {          
		"dataSource": "ea-test",                  //数据源名
		"parser": {
			"type": "string",                     
			"parseSpec": {
				"format": "json",                 //数据格式
				"timestampSpec": {                
					"column": "time",
					"format": "millis"
				},
				"dimensionsSpec": {               //维度列
					"dimensions": ["eventId", "appId"],
					"dimensionExclusions": ["time"]
				}
			}
		},
		"metricsSpec": [{
			"name": "eventCount",                  //指标列
			"fieldName": "eventId",
			"type": "count"                        //预聚合算法
		}],
		"granularitySpec": {
			"type": "uniform",
			"segmentGranularity": "HOUR",
			"queryGranularity": "MINUTE"
		},
	    "transformSpec": {                        
        "filter": {                                //数据过滤
          "type": "selector",
          "dimension": "appId",
          "value": "testAppId1"
        }
      }
	},
	"tuningConfig": {
		"type": "kafka",                           
		"maxRowsInMemory": "100000",
		"maxRowsPerSegment": 5000000,
		"workerThreads": 2,
		"reportParseExceptions": true
	},
	"ioConfig": {
		"topic": "ea_test_13",
		"consumerProperties": {
			"bootstrap.servers": "host1:port1,host2:port2,host3:post3",
			"group.id": "kafka_group1"
		},
		"useEarliestOffset": true,                 //kafka topic中数据从起始获取
		"taskCount": 1,
		"replicas": 1,
		"taskDuration": "PT1H"                    //task执行时长
	}
}

druid 摄取任务通过http请求overlord节点注册任务。

数据可以通过superset展示，注意在druid中的数据是预聚合之后的数据，而不是原始数据。

druid默认采用零时区。需要在middleManager配置文件中配置+8时区，同时superset dataSource中需要时区偏移8，新建查询任务时可以指定time的origin为16:00。这样聚合的数据才会准确。