Timeline Rest API与 TEZ
通过appId反向获取 queryId 和 sql等信息
可以通过timeline Rest API获取使用tez引擎的application的一些信息
获取dag_id
- /ws/v1/timeline/TEZ_DAG_ID?primaryFilter=applicationId%3A{$appId}
curl http://ip:port/ws/v1/timeline/TEZ_DAG_ID?primaryFilter=applicationId%3Aapplication_1689924389107_2946371
# 利用jq直接获取dag_id
curl http://ip:port/ws/v1/timeline/TEZ_DAG_ID?primaryFilter=applicationId%3Aapplication_1689924389107_2946371||jq -r .entities[0].entity
# 获取queryId、status等
jq -r .entities[0].primaryfilters.callerId[0]
jq -r .entities[0].primaryfilters.status[0]
输出主要包含以下信息,均可获取
- entity 该key的值即为dag_id
- starttime dag开始的时间戳
- events 包含dag初始化、提交、开始和结束的时间戳
- primaryfilters:包含dag的一些基本信息,包含hive的queryId、appId、app状态、提交的用户、app的队列
- otherinfo:其他的基本信息
{
"entities":[
{
"entitytype":"TEZ_DAG_ID",
"entity":"dag_1689924389107_2946371_1",
"starttime":1703046180693,
"events":[
{
"timestamp":1703046188967,
"eventtype":"DAG_FINISHED",
"eventinfo":{
}
},
{
"timestamp":1703046180944,
"eventtype":"DAG_STARTED",
"eventinfo":{
}
},
{
"timestamp":1703046180943,
"eventtype":"DAG_INITIALIZED",
"eventinfo":{
}
},
{
"timestamp":1703046180693,
"eventtype":"DAG_SUBMITTED",
"eventinfo":{
}
}
],
"domain":"Tez_ATS_application_1689924389107_2946371_1",
"primaryfilters":{
"dagName":[
"insert overwrite tabl...xxx,1,4),'00')\r\n)tmp (Stage-1)"
],
"queueName":[
"root.xxxxx"
],
"callerId":[
"hive_20231220122237_xxxxxxx"
],
"applicationId":[
"application_1689924389107_2946371"
],
"user":[
"xxx"
],
"status":[
"SUCCEEDED"
]
},
"relatedentities":{
},
"otherinfo":{
"completionApplicationAttemptId":"appattempt_1689924389107_2946371_000001",
"numFailedTaskAttempts":0,
"callerContext":"HIVE",
"numKilledTaskAttempts":0,
"numCompletedTasks":2,
"amWebServiceVersion":"2",
"inProgressLogsURL_1":"core-xxxx:8042/node/containerlogs/container_e02_1689924389107_2946371_01_000001/xxxxx",
"numSucceededTasks":2,
"vertexNameIdMapping":{
"Map 1":"vertex_1689924389107_2946371_1_00",
"Reducer 2":"vertex_1689924389107_2946371_1_01"
},
"numFailedTasks":0,
"timeTaken":8023,
"diagnostics":"",
"queueName":"root.xxxxx",
"applicationAttemptId":"appattempt_1689924389107_2946371_000001",
"callerType":"HIVE_QUERY_ID",
"startTime":1703046180944,
"callerId":"hive_20231220122237_xxxxxxx",
"endTime":1703046188967,
"applicationId":"application_1689924389107_2946371",
"numKilledTasks":0,
"initTime":1703046180943,
"user":"xxxxx",
"status":"SUCCEEDED"
}
}
]
}
获取sql
- /ws/v1/timeline/TEZ_DAG_EXTRA_INFO/{$dag_id}
curl http://ip:port/ws/v1/timeline/TEZ_DAG_EXTRA_INFO/{$dag_id}
输出主要包含以下信息
- entity:即dag_id
- starttime dag开始的时间戳
- events:dag提交和开始的时间戳
- otherinfo:一些统计指标及一些其他信息
curl http://ip:port/ws/v1/timeline/TEZ_DAG_EXTRA_INFO/dag_1689924389107_2946371_1
# 利用jq命令直接获取
curl http://ip:port/ws/v1/timeline/TEZ_DAG_EXTRA_INFO/dag_1689924389107_2946371_1|jq -r .otherinfo.dagPlan.dagContext.description
{
"entitytype": "TEZ_DAG_EXTRA_INFO",
"entity": "dag_1689924389107_2946371_1",
"starttime": 1703046180693,
"events": [{
"timestamp": 1703046188967,
"eventtype": "DAG_FINISHED",
"eventinfo": {}
}, {
"timestamp": 1703046180693,
"eventtype": "DAG_SUBMITTED",
"eventinfo": {}
}],
"domain": "Tez_ATS_application_1689924389107_2946371_1",
"primaryfilters": {},
"relatedentities": {},
"otherinfo": {
"counters": {
"counterGroups": [{
"counterGroupName": "org.apache.tez.common.counters.DAGCounter",
"counters": [{
"counterName": "NUM_SUCCEEDED_TASKS",
"counterValue": 2
}, {
"counterName": "TOTAL_LAUNCHED_TASKS",
"counterValue": 2
}, {
"counterName": "AM_CPU_MILLISECONDS",
"counterValue": 3060
}]
}, {
"counterGroupName": "org.apache.tez.common.counters.FileSystemCounter",
"counterGroupDisplayName": "File System Counters",
"counters": [{
"counterName": "xxx_BYTES_READ",
"counterValue": 65632
}, {
"counterName": "xxx_BYTES_WRITTEN",
"counterValue": 1306
}, {
"counterName": "FILE_BYTES_READ",
"counterValue": 164
}, {
"counterName": "FILE_BYTES_WRITTEN",
"counterValue": 100
}]
}, {
"counterGroupName": "org.apache.tez.common.counters.TaskCounter",
"counters": [{
"counterName": "REDUCE_INPUT_GROUPS",
"counterValue": 1
}, {
"counterName": "REDUCE_INPUT_RECORDS",
"counterValue": 1
}, {
"counterName": "SPILLED_RECORDS",
"counterValue": 2
}, {
"counterName": "NUM_SHUFFLED_INPUTS",
"counterValue": 1
}, {
"counterName": "MERGED_MAP_OUTPUTS",
"counterValue": 1
}, {
"counterName": "GC_TIME_MILLIS",
"counterValue": 82
}, {
"counterName": "CPU_MILLISECONDS",
"counterValue": 5710
}, {
"counterName": "PHYSICAL_MEMORY_BYTES",
"counterValue": 1538260992
}, {
"counterName": "VIRTUAL_MEMORY_BYTES",
"counterValue": 10990825472
}, {
"counterName": "COMMITTED_HEAP_BYTES",
"counterValue": 1538260992
}, {
"counterName": "INPUT_RECORDS_PROCESSED",
"counterValue": 3149
}, {
"counterName": "INPUT_SPLIT_LENGTH_BYTES",
"counterValue": 31679
}, {
"counterName": "OUTPUT_RECORDS",
"counterValue": 1
}, {
"counterName": "OUTPUT_BYTES",
"counterValue": 46
}, {
"counterName": "OUTPUT_BYTES_WITH_OVERHEAD",
"counterValue": 54
}, {
"counterName": "OUTPUT_BYTES_PHYSICAL",
"counterValue": 68
}, {
"counterName": "ADDITIONAL_SPILLS_BYTES_READ",
"counterValue": 68
}, {
"counterName": "SHUFFLE_CHUNK_COUNT",
"counterValue": 1
}, {
"counterName": "SHUFFLE_BYTES",
"counterValue": 68
}, {
"counterName": "SHUFFLE_BYTES_DECOMPRESSED",
"counterValue": 54
}, {
"counterName": "SHUFFLE_BYTES_DISK_DIRECT",
"counterValue": 68
}, {
"counterName": "SHUFFLE_PHASE_TIME",
"counterValue": 19
}, {
"counterName": "MERGE_PHASE_TIME",
"counterValue": 26
}, {
"counterName": "FIRST_EVENT_RECEIVED",
"counterValue": 13
}, {
"counterName": "LAST_EVENT_RECEIVED",
"counterValue": 13
}]
}, {
"counterGroupName": "HIVE",
"counters": [{
"counterName": "CREATED_FILES",
"counterValue": 1
}, {
"counterName": "RECORDS_IN_Map_1",
"counterValue": 3149
}, {
"counterName": "RECORDS_OUT_1_bds_db.bm_chenzhou_day_index1",
"counterValue": 1
}, {
"counterName": "RECORDS_OUT_INTERMEDIATE_Map_1",
"counterValue": 1
}, {
"counterName": "RECORDS_OUT_OPERATOR_FIL_10",
"counterValue": 3149
}, {
"counterName": "RECORDS_OUT_OPERATOR_FS_9",
"counterValue": 1
}, {
"counterName": "RECORDS_OUT_OPERATOR_GBY_4",
"counterValue": 1
}, {
"counterName": "RECORDS_OUT_OPERATOR_GBY_6",
"counterValue": 1
}, {
"counterName": "RECORDS_OUT_OPERATOR_RS_5",
"counterValue": 1
}, {
"counterName": "RECORDS_OUT_OPERATOR_SEL_2",
"counterValue": 3149
}, {
"counterName": "RECORDS_OUT_OPERATOR_SEL_7",
"counterValue": 1
}, {
"counterName": "RECORDS_OUT_OPERATOR_TS_0",
"counterValue": 3149
}]
}, {
"counterGroupName": "TaskCounter_Map_1_INPUT_t",
"counters": [{
"counterName": "INPUT_RECORDS_PROCESSED",
"counterValue": 3149
}, {
"counterName": "INPUT_SPLIT_LENGTH_BYTES",
"counterValue": 31679
}]
}, {
"counterGroupName": "TaskCounter_Map_1_OUTPUT_Reducer_2",
"counters": [{
"counterName": "OUTPUT_BYTES",
"counterValue": 46
}, {
"counterName": "OUTPUT_BYTES_PHYSICAL",
"counterValue": 68
}, {
"counterName": "OUTPUT_BYTES_WITH_OVERHEAD",
"counterValue": 54
}, {
"counterName": "OUTPUT_RECORDS",
"counterValue": 1
}, {
"counterName": "SHUFFLE_CHUNK_COUNT",
"counterValue": 1
}, {
"counterName": "SPILLED_RECORDS",
"counterValue": 1
}]
}, {
"counterGroupName": "TaskCounter_Reducer_2_INPUT_Map_1",
"counters": [{
"counterName": "ADDITIONAL_SPILLS_BYTES_READ",
"counterValue": 68
}, {
"counterName": "FIRST_EVENT_RECEIVED",
"counterValue": 13
}, {
"counterName": "LAST_EVENT_RECEIVED",
"counterValue": 13
}, {
"counterName": "MERGED_MAP_OUTPUTS",
"counterValue": 1
}, {
"counterName": "MERGE_PHASE_TIME",
"counterValue": 26
}, {
"counterName": "NUM_SHUFFLED_INPUTS",
"counterValue": 1
}, {
"counterName": "REDUCE_INPUT_GROUPS",
"counterValue": 1
}, {
"counterName": "REDUCE_INPUT_RECORDS",
"counterValue": 1
}, {
"counterName": "SHUFFLE_BYTES",
"counterValue": 68
}, {
"counterName": "SHUFFLE_BYTES_DECOMPRESSED",
"counterValue": 54
}, {
"counterName": "SHUFFLE_BYTES_DISK_DIRECT",
"counterValue": 68
}, {
"counterName": "SHUFFLE_PHASE_TIME",
"counterValue": 19
}, {
"counterName": "SPILLED_RECORDS",
"counterValue": 1
}]
}, {
"counterGroupName": "org.apache.hadoop.hive.ql.exec.tez.HiveInputCounters",
"counters": [{
"counterName": "GROUPED_INPUT_SPLITS_Map_1",
"counterValue": 1
}, {
"counterName": "INPUT_DIRECTORIES_Map_1",
"counterValue": 1
}, {
"counterName": "INPUT_FILES_Map_1",
"counterValue": 1
}, {
"counterName": "RAW_INPUT_SPLITS_Map_1",
"counterValue": 1
}]
}]
},
"dagPlan": {
"dagName": "insert overwrite tabl...xxx,1,4),'00')\r\n)tmp (Stage-1)",
"dagInfo": "{\"context\":\"Hive\",\"description\":\"insert overwrite table test_db.test_xxx partition(pt='2023-12-19')\\r\\nselect \\r\\n\\test.a1,\\r\\n\\test.a2,\\r\\n test.a3,\\r\\n test.a4,\\r\\n test.a5\\r\\nfrom\\r\\n(\\r\\n\\tselect\\r\\n\\t\\test1.a1,\\r\\n\\t\\tconcat(substr(test1.a2,1,4),'00') ,\\r\\n\\t\\tsum(test1.a3) ,\\r\\n\\t\\tsum(test1.a3),\\r\\n\\t\\tsum(t.a4) \\r\\n\\tfrom test_db.test_xxx test\\r\\n\\twhere test1.pt = '2023-12-19'\\r\\n\\tand test.a1 in ('1','2','3','4','5','6','7','8','9')\\r\\n\\tgroup by \\r\\n\\t\\ttest.a1,\\r\\n\\t\\tconcat(substr(test.a2,1,4),'00')\\r\\n)tmp\"}",
"dagContext": {
"callerId": "hive_20231220122237_xxxxx",
"callerType": "HIVE_QUERY_ID",
"context": "HIVE",
"description": "insert overwrite table test_db.test3 partition(pt='2023-12-19')\r\nselect \r\n\ttest.deal_day,\r\n\ttest.a1,\r\n test.a2\r\n test.a3\r\n test.a4\r\nfrom\r\n(\r\n\tselect\r\n\t\ttest2.a1,\r\n\t\tconcat(substr(t.country_id_new,1,4),'00') ,\r\n\t\tsum(test2.a3) ,\r\n\t\tsum(test2.a4) ,\r\n\t\tsum(test2.a5) \r\n\tfrom test_db.test3\r\n\twhere test2.pt = '2023-12-19'\r\n\tand test2.a3 in ('1','2','3','4','5','6','7','8','9')\r\n\tgroup by \r\n\t\ttest2.a1,\r\n\t\tconcat(substr(t.country_id_new,1,4),'00’)\r\n)test"
},
"version": 2,
"vertices": [{
"vertexName": "Map 1",
"processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
"outEdgeIds": ["720179762"],
"additionalInputs": [{
"name": "t",
"class": "org.apache.tez.mapreduce.input.MRInputLegacy",
"initializer": "org.apache.hadoop.hive.ql.exec.tez.HiveSplitGenerator"
}]
}, {
"vertexName": "Reducer 2",
"processorClass": "org.apache.hadoop.hive.ql.exec.tez.ReduceTezProcessor",
"inEdgeIds": ["720179762"],
"additionalOutputs": [{
"name": "out_Reducer 2",
"class": "org.apache.tez.mapreduce.output.MROutput"
}]
}],
"edges": [{
"edgeId": "720179762",
"inputVertexName": "Map 1",
"outputVertexName": "Reducer 2",
"dataMovementType": "SCATTER_GATHER",
"dataSourceType": "PERSISTED",
"schedulingType": "SEQUENTIAL",
"edgeSourceClass": "org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput",
"edgeDestinationClass": "org.apache.tez.runtime.library.input.OrderedGroupedKVInput"
}]
}
}
}