Slipstream 实验

最新推荐文章于 2021-11-26 15:30:43 发布

lijie2049

最新推荐文章于 2021-11-26 15:30:43 发布

阅读量238

点赞数

文章标签： kafka 分布式

本文链接：https://blog.csdn.net/lijie2049/article/details/121185658

版权

该博客演示了如何使用Transwarp StreamSQL创建基于Kafka Topic的输入流表，进行窗口操作，并将结果写入表。通过创建和启动StreamJob，实现从Kafka消费数据并进行流处理，最后展示了发送到Kafka的数据及查询结果表的过程。

摘要由CSDN通过智能技术生成

Slipstream

create application {application_user_name};

use application {application_user_name}

// 基于Kafka Topic创建输入流表logs

create stream logs (ip string, url string, time timestamp) row format delimited fields terminated by ',' tblproperties( "topic"="{topic_log_student_name}", "kafka.zookeeper"="{Zookeeper_List}", "kafka.broker.list"="{Broker_List}", "transwarp.consumer.security.protocol"="SASL_PLAINTEXT",

"transwarp.consumer.sasl.kerberos.service.name"="kafka",

"transwarp.consumer.sasl.jaas.config"="com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true keyTab=\"/etc/slipstream1/conf/student.keytab\" principal=\"admin@TDH\"");

// 基于输入流表创建衍生流表wlog，窗口Length和Slide为10秒

create stream wlog as select * from logs streamwindow sw as (length '10' second slide '10' second);

// 查看所有的流表

show streams;

//创建结果表

use {database_name}

create table result_table(url string, count int);

// 创建StreamJob，log_job为 Streamjob名，格式为job_账号名，如账号student1则为job_student1

CREATE STREAMJOB log_job AS (

" insert into result_table select url, count(*) from wlog group by url ")JOBPROPERTIES("morphling.result.auto.flush"="true");

// 查看已创建的StreamJob

show streamjobs;