Timeline Rest API之Tez

Timeline Rest API与 TEZ

通过appId反向获取 queryId 和 sql等信息

可以通过timeline Rest API获取使用tez引擎的application的一些信息

获取dag_id

  • /ws/v1/timeline/TEZ_DAG_ID?primaryFilter=applicationId%3A{$appId}
curl http://ip:port/ws/v1/timeline/TEZ_DAG_ID?primaryFilter=applicationId%3Aapplication_1689924389107_2946371

# 利用jq直接获取dag_id
curl http://ip:port/ws/v1/timeline/TEZ_DAG_ID?primaryFilter=applicationId%3Aapplication_1689924389107_2946371||jq -r .entities[0].entity

# 获取queryId、status等
jq -r .entities[0].primaryfilters.callerId[0]
jq -r .entities[0].primaryfilters.status[0]

输出主要包含以下信息,均可获取

  • entity 该key的值即为dag_id
  • starttime dag开始的时间戳
  • events 包含dag初始化、提交、开始和结束的时间戳
  • primaryfilters:包含dag的一些基本信息,包含hive的queryId、appId、app状态、提交的用户、app的队列
  • otherinfo:其他的基本信息
{
    "entities":[
        {
            "entitytype":"TEZ_DAG_ID",
            "entity":"dag_1689924389107_2946371_1",
            "starttime":1703046180693,
            "events":[
                {
                    "timestamp":1703046188967,
                    "eventtype":"DAG_FINISHED",
                    "eventinfo":{

                    }
                },
                {
                    "timestamp":1703046180944,
                    "eventtype":"DAG_STARTED",
                    "eventinfo":{

                    }
                },
                {
                    "timestamp":1703046180943,
                    "eventtype":"DAG_INITIALIZED",
                    "eventinfo":{

                    }
                },
                {
                    "timestamp":1703046180693,
                    "eventtype":"DAG_SUBMITTED",
                    "eventinfo":{

                    }
                }
            ],
            "domain":"Tez_ATS_application_1689924389107_2946371_1",
            "primaryfilters":{
                "dagName":[
                    "insert overwrite tabl...xxx,1,4),'00')\r\n)tmp (Stage-1)"
                ],
                "queueName":[
                    "root.xxxxx"
                ],
                "callerId":[
                    "hive_20231220122237_xxxxxxx"
                ],
                "applicationId":[
                    "application_1689924389107_2946371"
                ],
                "user":[
                    "xxx"
                ],
                "status":[
                    "SUCCEEDED"
                ]
            },
            "relatedentities":{

            },
            "otherinfo":{
                "completionApplicationAttemptId":"appattempt_1689924389107_2946371_000001",
                "numFailedTaskAttempts":0,
                "callerContext":"HIVE",
                "numKilledTaskAttempts":0,
                "numCompletedTasks":2,
                "amWebServiceVersion":"2",
                "inProgressLogsURL_1":"core-xxxx:8042/node/containerlogs/container_e02_1689924389107_2946371_01_000001/xxxxx",
                "numSucceededTasks":2,
                "vertexNameIdMapping":{
                    "Map 1":"vertex_1689924389107_2946371_1_00",
                    "Reducer 2":"vertex_1689924389107_2946371_1_01"
                },
                "numFailedTasks":0,
                "timeTaken":8023,
                "diagnostics":"",
                "queueName":"root.xxxxx",
                "applicationAttemptId":"appattempt_1689924389107_2946371_000001",
                "callerType":"HIVE_QUERY_ID",
                "startTime":1703046180944,
                "callerId":"hive_20231220122237_xxxxxxx",
                "endTime":1703046188967,
                "applicationId":"application_1689924389107_2946371",
                "numKilledTasks":0,
                "initTime":1703046180943,
                "user":"xxxxx",
                "status":"SUCCEEDED"
            }
        }
    ]
}

获取sql

  • /ws/v1/timeline/TEZ_DAG_EXTRA_INFO/{$dag_id}
curl http://ip:port/ws/v1/timeline/TEZ_DAG_EXTRA_INFO/{$dag_id}

输出主要包含以下信息

  • entity:即dag_id
  • starttime dag开始的时间戳
  • events:dag提交和开始的时间戳
  • otherinfo:一些统计指标及一些其他信息
curl http://ip:port/ws/v1/timeline/TEZ_DAG_EXTRA_INFO/dag_1689924389107_2946371_1

# 利用jq命令直接获取
curl http://ip:port/ws/v1/timeline/TEZ_DAG_EXTRA_INFO/dag_1689924389107_2946371_1|jq -r .otherinfo.dagPlan.dagContext.description
{
	"entitytype": "TEZ_DAG_EXTRA_INFO",
	"entity": "dag_1689924389107_2946371_1",
	"starttime": 1703046180693,
	"events": [{
		"timestamp": 1703046188967,
		"eventtype": "DAG_FINISHED",
		"eventinfo": {}
	}, {
		"timestamp": 1703046180693,
		"eventtype": "DAG_SUBMITTED",
		"eventinfo": {}
	}],
	"domain": "Tez_ATS_application_1689924389107_2946371_1",
	"primaryfilters": {},
	"relatedentities": {},
	"otherinfo": {
		"counters": {
			"counterGroups": [{
				"counterGroupName": "org.apache.tez.common.counters.DAGCounter",
				"counters": [{
					"counterName": "NUM_SUCCEEDED_TASKS",
					"counterValue": 2
				}, {
					"counterName": "TOTAL_LAUNCHED_TASKS",
					"counterValue": 2
				}, {
					"counterName": "AM_CPU_MILLISECONDS",
					"counterValue": 3060
				}]
			}, {
				"counterGroupName": "org.apache.tez.common.counters.FileSystemCounter",
				"counterGroupDisplayName": "File System Counters",
				"counters": [{
					"counterName": "xxx_BYTES_READ",
					"counterValue": 65632
				}, {
					"counterName": "xxx_BYTES_WRITTEN",
					"counterValue": 1306
				}, {
					"counterName": "FILE_BYTES_READ",
					"counterValue": 164
				}, {
					"counterName": "FILE_BYTES_WRITTEN",
					"counterValue": 100
				}]
			}, {
				"counterGroupName": "org.apache.tez.common.counters.TaskCounter",
				"counters": [{
					"counterName": "REDUCE_INPUT_GROUPS",
					"counterValue": 1
				}, {
					"counterName": "REDUCE_INPUT_RECORDS",
					"counterValue": 1
				}, {
					"counterName": "SPILLED_RECORDS",
					"counterValue": 2
				}, {
					"counterName": "NUM_SHUFFLED_INPUTS",
					"counterValue": 1
				}, {
					"counterName": "MERGED_MAP_OUTPUTS",
					"counterValue": 1
				}, {
					"counterName": "GC_TIME_MILLIS",
					"counterValue": 82
				}, {
					"counterName": "CPU_MILLISECONDS",
					"counterValue": 5710
				}, {
					"counterName": "PHYSICAL_MEMORY_BYTES",
					"counterValue": 1538260992
				}, {
					"counterName": "VIRTUAL_MEMORY_BYTES",
					"counterValue": 10990825472
				}, {
					"counterName": "COMMITTED_HEAP_BYTES",
					"counterValue": 1538260992
				}, {
					"counterName": "INPUT_RECORDS_PROCESSED",
					"counterValue": 3149
				}, {
					"counterName": "INPUT_SPLIT_LENGTH_BYTES",
					"counterValue": 31679
				}, {
					"counterName": "OUTPUT_RECORDS",
					"counterValue": 1
				}, {
					"counterName": "OUTPUT_BYTES",
					"counterValue": 46
				}, {
					"counterName": "OUTPUT_BYTES_WITH_OVERHEAD",
					"counterValue": 54
				}, {
					"counterName": "OUTPUT_BYTES_PHYSICAL",
					"counterValue": 68
				}, {
					"counterName": "ADDITIONAL_SPILLS_BYTES_READ",
					"counterValue": 68
				}, {
					"counterName": "SHUFFLE_CHUNK_COUNT",
					"counterValue": 1
				}, {
					"counterName": "SHUFFLE_BYTES",
					"counterValue": 68
				}, {
					"counterName": "SHUFFLE_BYTES_DECOMPRESSED",
					"counterValue": 54
				}, {
					"counterName": "SHUFFLE_BYTES_DISK_DIRECT",
					"counterValue": 68
				}, {
					"counterName": "SHUFFLE_PHASE_TIME",
					"counterValue": 19
				}, {
					"counterName": "MERGE_PHASE_TIME",
					"counterValue": 26
				}, {
					"counterName": "FIRST_EVENT_RECEIVED",
					"counterValue": 13
				}, {
					"counterName": "LAST_EVENT_RECEIVED",
					"counterValue": 13
				}]
			}, {
				"counterGroupName": "HIVE",
				"counters": [{
					"counterName": "CREATED_FILES",
					"counterValue": 1
				}, {
					"counterName": "RECORDS_IN_Map_1",
					"counterValue": 3149
				}, {
					"counterName": "RECORDS_OUT_1_bds_db.bm_chenzhou_day_index1",
					"counterValue": 1
				}, {
					"counterName": "RECORDS_OUT_INTERMEDIATE_Map_1",
					"counterValue": 1
				}, {
					"counterName": "RECORDS_OUT_OPERATOR_FIL_10",
					"counterValue": 3149
				}, {
					"counterName": "RECORDS_OUT_OPERATOR_FS_9",
					"counterValue": 1
				}, {
					"counterName": "RECORDS_OUT_OPERATOR_GBY_4",
					"counterValue": 1
				}, {
					"counterName": "RECORDS_OUT_OPERATOR_GBY_6",
					"counterValue": 1
				}, {
					"counterName": "RECORDS_OUT_OPERATOR_RS_5",
					"counterValue": 1
				}, {
					"counterName": "RECORDS_OUT_OPERATOR_SEL_2",
					"counterValue": 3149
				}, {
					"counterName": "RECORDS_OUT_OPERATOR_SEL_7",
					"counterValue": 1
				}, {
					"counterName": "RECORDS_OUT_OPERATOR_TS_0",
					"counterValue": 3149
				}]
			}, {
				"counterGroupName": "TaskCounter_Map_1_INPUT_t",
				"counters": [{
					"counterName": "INPUT_RECORDS_PROCESSED",
					"counterValue": 3149
				}, {
					"counterName": "INPUT_SPLIT_LENGTH_BYTES",
					"counterValue": 31679
				}]
			}, {
				"counterGroupName": "TaskCounter_Map_1_OUTPUT_Reducer_2",
				"counters": [{
					"counterName": "OUTPUT_BYTES",
					"counterValue": 46
				}, {
					"counterName": "OUTPUT_BYTES_PHYSICAL",
					"counterValue": 68
				}, {
					"counterName": "OUTPUT_BYTES_WITH_OVERHEAD",
					"counterValue": 54
				}, {
					"counterName": "OUTPUT_RECORDS",
					"counterValue": 1
				}, {
					"counterName": "SHUFFLE_CHUNK_COUNT",
					"counterValue": 1
				}, {
					"counterName": "SPILLED_RECORDS",
					"counterValue": 1
				}]
			}, {
				"counterGroupName": "TaskCounter_Reducer_2_INPUT_Map_1",
				"counters": [{
					"counterName": "ADDITIONAL_SPILLS_BYTES_READ",
					"counterValue": 68
				}, {
					"counterName": "FIRST_EVENT_RECEIVED",
					"counterValue": 13
				}, {
					"counterName": "LAST_EVENT_RECEIVED",
					"counterValue": 13
				}, {
					"counterName": "MERGED_MAP_OUTPUTS",
					"counterValue": 1
				}, {
					"counterName": "MERGE_PHASE_TIME",
					"counterValue": 26
				}, {
					"counterName": "NUM_SHUFFLED_INPUTS",
					"counterValue": 1
				}, {
					"counterName": "REDUCE_INPUT_GROUPS",
					"counterValue": 1
				}, {
					"counterName": "REDUCE_INPUT_RECORDS",
					"counterValue": 1
				}, {
					"counterName": "SHUFFLE_BYTES",
					"counterValue": 68
				}, {
					"counterName": "SHUFFLE_BYTES_DECOMPRESSED",
					"counterValue": 54
				}, {
					"counterName": "SHUFFLE_BYTES_DISK_DIRECT",
					"counterValue": 68
				}, {
					"counterName": "SHUFFLE_PHASE_TIME",
					"counterValue": 19
				}, {
					"counterName": "SPILLED_RECORDS",
					"counterValue": 1
				}]
			}, {
				"counterGroupName": "org.apache.hadoop.hive.ql.exec.tez.HiveInputCounters",
				"counters": [{
					"counterName": "GROUPED_INPUT_SPLITS_Map_1",
					"counterValue": 1
				}, {
					"counterName": "INPUT_DIRECTORIES_Map_1",
					"counterValue": 1
				}, {
					"counterName": "INPUT_FILES_Map_1",
					"counterValue": 1
				}, {
					"counterName": "RAW_INPUT_SPLITS_Map_1",
					"counterValue": 1
				}]
			}]
		},
		"dagPlan": {
			"dagName": "insert overwrite tabl...xxx,1,4),'00')\r\n)tmp (Stage-1)",
			"dagInfo": "{\"context\":\"Hive\",\"description\":\"insert overwrite table test_db.test_xxx partition(pt='2023-12-19')\\r\\nselect \\r\\n\\test.a1,\\r\\n\\test.a2,\\r\\n    test.a3,\\r\\n    test.a4,\\r\\n    test.a5\\r\\nfrom\\r\\n(\\r\\n\\tselect\\r\\n\\t\\test1.a1,\\r\\n\\t\\tconcat(substr(test1.a2,1,4),'00') ,\\r\\n\\t\\tsum(test1.a3) ,\\r\\n\\t\\tsum(test1.a3),\\r\\n\\t\\tsum(t.a4) \\r\\n\\tfrom test_db.test_xxx test\\r\\n\\twhere test1.pt = '2023-12-19'\\r\\n\\tand  test.a1 in ('1','2','3','4','5','6','7','8','9')\\r\\n\\tgroup by \\r\\n\\t\\ttest.a1,\\r\\n\\t\\tconcat(substr(test.a2,1,4),'00')\\r\\n)tmp\"}",
			"dagContext": {
				"callerId": "hive_20231220122237_xxxxx",
				"callerType": "HIVE_QUERY_ID",
				"context": "HIVE",
				"description": "insert overwrite table test_db.test3 partition(pt='2023-12-19')\r\nselect \r\n\ttest.deal_day,\r\n\ttest.a1,\r\n    test.a2\r\n    test.a3\r\n    test.a4\r\nfrom\r\n(\r\n\tselect\r\n\t\ttest2.a1,\r\n\t\tconcat(substr(t.country_id_new,1,4),'00') ,\r\n\t\tsum(test2.a3) ,\r\n\t\tsum(test2.a4) ,\r\n\t\tsum(test2.a5) \r\n\tfrom test_db.test3\r\n\twhere test2.pt = '2023-12-19'\r\n\tand  test2.a3 in ('1','2','3','4','5','6','7','8','9')\r\n\tgroup by \r\n\t\ttest2.a1,\r\n\t\tconcat(substr(t.country_id_new,1,4),'00’)\r\n)test"
			},
			"version": 2,
			"vertices": [{
				"vertexName": "Map 1",
				"processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
				"outEdgeIds": ["720179762"],
				"additionalInputs": [{
					"name": "t",
					"class": "org.apache.tez.mapreduce.input.MRInputLegacy",
					"initializer": "org.apache.hadoop.hive.ql.exec.tez.HiveSplitGenerator"
				}]
			}, {
				"vertexName": "Reducer 2",
				"processorClass": "org.apache.hadoop.hive.ql.exec.tez.ReduceTezProcessor",
				"inEdgeIds": ["720179762"],
				"additionalOutputs": [{
					"name": "out_Reducer 2",
					"class": "org.apache.tez.mapreduce.output.MROutput"
				}]
			}],
			"edges": [{
				"edgeId": "720179762",
				"inputVertexName": "Map 1",
				"outputVertexName": "Reducer 2",
				"dataMovementType": "SCATTER_GATHER",
				"dataSourceType": "PERSISTED",
				"schedulingType": "SEQUENTIAL",
				"edgeSourceClass": "org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput",
				"edgeDestinationClass": "org.apache.tez.runtime.library.input.OrderedGroupedKVInput"
			}]
		}
	}
}
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值