需求:将表中的两字段使用SQL存入新表中的一个字段,存储格式是KV键值对。使用自定义函数实现
import java.util.HashMap;
import java.util.Map;
/**
* @author liuhongbo
* @date 2023-02-10 15:00
*/
public class CollectSegmentAttrAcc implements java.io.Serializable {
private static final long serialVersionUID = 1521279572627987607L;
private Map<String, String> content = new HashMap<>();
public Map<String, String> getContent() {
return content;
}
public void setContent(Map<String, String> content) {
this.content = content;
}
}
import java.util.Iterator;
import java.util.Map;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.flink.table.functions.AggregateFunction;
/**
* @author liuhongbo
* @date 2023-02-10 15:00
*/
public class CollectSegmentAttrs extends AggregateFunction<String, CollectSegmentAttrAcc> {
@Override
public String getValue(CollectSegmentAttrAcc accumulator) {
String result = StringUtils.EMPTY;
ObjectMapper objectMapper = new ObjectMapper();
try {
Map<String, String> content = accumulator.getContent();
if (MapUtils.isNotEmpty(content)) {
result = objectMapper.writeValueAsString(content);
}
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
return result;
}
@Override
public CollectSegmentAttrAcc createAccumulator() {
return new CollectSegmentAttrAcc();
}
public void accumulate(CollectSegmentAttrAcc accumulator, String attrCode,String attrValue) {
Map<String, String> content = accumulator.getContent();
if (StringUtils.isNoneBlank(attrCode, attrValue)) {
content.put(attrCode, attrValue);
}
}
public void retract(CollectSegmentAttrAcc accumulator, String attrCode,String attrValue) {
accumulator.getContent().remove(attrCode);
}
public void merge(CollectSegmentAttrAcc acc, Iterable<CollectSegmentAttrAcc> it) {
Iterator<CollectSegmentAttrAcc> iter = it.iterator();
while (iter.hasNext()) {
CollectSegmentAttrAcc next = iter.next();
Map<String, String> content = next.getContent();
content.forEach((k, v) -> {
acc.getContent().put(k, v);
});
}
}
public void resetAccumulator(CollectSegmentAttrAcc acc) {
acc.getContent().clear();
}
}
将函数打包成jar,上传至服务器指定目录。
在SQL中的应用是,先引用jar,再创建自定义函数。
ADD JAR '/devops/flink/script/flink-quickstart-1.0-SNAPSHOT.jar';
CREATE TEMPORARY FUNCTION CollectSegmentAttrs AS 'com.airoc.flink.udf.CollectSegmentAttrs';
在SQL中的使用如下:
LEFT JOIN (SELECT SEGMENT_ID,CollectSegmentAttrs(ATTR_CODE, ATTR_VALUE) AS SEGMENT_ATTRS FROM table_name group by SEGMENT_ID )