文章目录
1.Flink的时间函数
时间戳字符串转化为时间戳
,ts as(
case
when receive_time is not null then TO_TIMESTAMP(cast(receive_time as BIGINT))
else CURRENT_TIMESTAMP
end
)
,ts2 as(
case
when receive_time is not null then TO_TIMESTAMP( FROM_UNIXTIME(cast(receive_time as BIGINT)/1000) ,'yyyy-MM-dd HH:mm:ss')
else CURRENT_TIMESTAMP
end
)
,WATERMARK wk FOR ts as withOffset(ts, 0)
2.Flink的Taskmanager与Slot的关系
Flink的Task与SubTask理解
Flink 架构浅析 & 并行度 Slot 的关系
3.Flink控制台输出的 +I,-U是什么意思?
– 是 撤回,+是操作后,I是插入,U是更新,D是删除
如 -U是撤回前的数据,+U是更新后的数据
踩坑记 | flink sql count 还有这种坑!
《[Flink-源码分析]Blink SQL 回撤解密》
《flink回撤流分析》
《flink retract 流》 为什么要有回撤流?
4.Flink去重
SELECT *
FROM (
SELECT *,
ROW_NUMBER() OVER ([PARTITION BY col1[, col2..]
ORDER BY timeAttributeCol [asc|desc]) AS rownum
FROM table_name)
WHERE rownum = 1
Flink SQL —Top-N ,Window Top-N
-- 原始SQL。
select a, b, c, window_start from (
select *,
row_number() over (partition by b, window_start, window_end order by c) as rk
from (
select a,
sum(b) as b,
max(c) as c,
window_start,
window_end
from table (tumble(table MyTable, descriptor(ts), interval '1' minute))
group by a, window_start, window_end)
) where rk < 3;
5.Flink的内存管理
6.Flink UDF
7.Flink UDAF
8.Flink UDTF
import com.alibaba.fastjson.JSONObject;
import org.apache.flink.table.functions.FunctionContext;
import org.apache.flink.table.functions.TableFunction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
public class JsonArrayUDTF extends TableFunction<String> {
private static final Logger logger = LoggerFactory.getLogger(JsonArrayUDTF.class);
// 可选,open方法可不编写。如果编写,则需要添加声明'import org.apache.flink.table.functions.FunctionContext;'。
@Override
public void open(FunctionContext context) {
// ... ...
}
/**
* 输入一条jsonArray,返回多条json对象
* @param jsonArray
*/
public void eval(String jsonArray) {
try {
List<JSONObject> jsonObjects = JSONObject.parseArray(jsonArray, JSONObject.class);
for (JSONObject object : jsonObjects) {
collect(object.toJSONString());
}
}catch(Exception e) {
logger.error("json数组解析异常 "+jsonArray+"\n"+e.getMessage());
}
}
// 可选,close方法可不编写。
@Override
public void close() {
// ... ...
}
}
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<flink.version>1.14.0</flink.version>
<scala.version>2.11</scala.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-common</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.48</version>
</dependency>
<!--日志打印-->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.30</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>8</source>
<target>8</target>
</configuration>
</plugin>
</plugins>
</build>