将camus订阅的topics在hdfs上的某一天数据进行格式化并写为hudi表并同步到hive meatstore
引入相关环境
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# 将camus订阅的topics在hdfs上的某一天数据进行格式化并写为hudi表并同步到hive meatstore
from __future__ import print_function
from pyspark.sql import SparkSession
from pyspark.sql import Row
from pyspark.sql.types import *
import json
import datetime
import sys
定义json解析函数
# 将topic的json解析为一层平铺的k-v
def json_format(item):
new_item = {
}
for key,value in json.loads(item).items():
if key == 'data':
for k,v in json.loads(value).items():
new_item[k] = v
elif key == 'msg_data':
for k,v in value.items():
if k in ('vid