pyflink 流处理自定义函数

scan724

已于 2023-06-08 22:04:15 修改

阅读量863

点赞数

分类专栏： Flink实时计算文章标签：数学建模 java spring

于 2023-06-08 19:49:12 首次发布

本文链接：https://blog.csdn.net/zhaoyangjian724/article/details/131114743

版权

Flink实时计算专栏收录该内容

102 篇文章 1 订阅

订阅专栏

[root@master pyflink]# cat elink_test.txt
ES047 625978 uuuuuuuuuuuuuuuuuuuuuuuuuuuuuuu
ES047 625978 094323 __________sys_app.c:78 [1573]<Esb><head><Serial>10402020230515094322018855</Serial><Channel>B05</Channel><DateTime>20230515094323</DateTime><HostCode>ES047</HostCode><TxnUsr>SJGY</TxnUsr><TxnOrg>999999999</TxnOrg><CONSUMER_ID>257010</CONSUMER_ID></head><body><DRAW_TYPE>0</DRAW_TYPE><PAYEE_ACCT_NO>xxxx</PAYEE_ACCT_NO><PUBLIC_PRIVATE_FLAG>1</PUBLIC_PRIVATE_FLAG><AMT>3000.00</AMT><FEE_COLLECT_ARRAY><row><FEE_DISCOUNT_CODE>TL010015</FEE_DISCOUNT_CODE><DDC_ACCT_NO>xxx</DDC_ACCT_NO><TRAN_AMT>3000.00</TRAN_AMT><DDC_ACCT_NO_SRC_CD>0</DDC_ACCT_NO_SRC_CD><SMMRY_CD>SXF002</SMMRY_CD></row></FEE_COLLECT_ARRAY><DISCOUNT_COMMISSION>0.00</DISCOUNT_COMMISSION><KEY_INDEX>IBANK.PWD-SM4.zpk</KEY_INDEX><CCY>156</CCY><OPTION_TYPE>0</OPTION_TYPE><CHANNEL_TYPE>1</CHANNEL_TYPE><PAYEE_BANK_CODE>402391000068</PAYEE_BANK_CODE><POSTSCRIPT></POSTSCRIPT><PAYER_ACCT_NAME>test</PAYER_ACCT_NAME><ABSTRACT></ABSTRACT><ACCT_CHK_MODE>0</ACCT_CHK_MODE><PAID_COMMISSION>0.00</PAID_]
ES047 625978 11111111111111111
ES047 625978 22222222222222222
ES047 625978 33333333333333333
ES047 625978 44444444444444444
ES047 625978 55555555555555555
6666666666666 66666666666666666
ES047 625978 77777777777777777
ES047 625978 88888888888888888
ES047 625978 99999999999999999

[root@master pyflink]# cat test.py
# -*- coding: utf-8 -*-
from pyflink.datastream import StreamExecutionEnvironment
from pyflink.datastream.functions import MapFunction, RuntimeContext, KeyedProcessFunction
from abc import ABC, abstractmethod
from pyflink.datastream import StreamExecutionEnvironment
from pyflink.datastream.functions import MapFunction, RuntimeContext, KeyedProcessFunction
from pyflink.datastream.state import MapStateDescriptor
from pyflink.datastream.connectors.kafka import FlinkKafkaConsumer
from pyflink.common.typeinfo import Types, TypeInformation
from pyflink.datastream.connectors.elasticsearch import Elasticsearch7SinkBuilder, ElasticsearchEmitter, FlushBackoffType
from pyflink.datastream.connectors import DeliveryGuarantee
from pyflink.common.serialization import SimpleStringSchema
import json
import re
from datetime import datetime
from elasticsearch import Elasticsearch
from pyflink.datastream.functions import RuntimeContext, FlatMapFunction

import re
import redis

# 创建 StreamExecutionEnvironment 对象
env = StreamExecutionEnvironment.get_execution_environment()
env.set_parallelism(1)

# 读取文件，创建 DataStream 对象
data_stream = env.read_text_file('/root/pyflink/elink_test.txt')
def my_map_func(value):
return value.split(' ')[1]
new_stream = data_stream.map(my_map_func)
#new_stream = data_stream.map(MyMapFunction(),output_type=Types.MAP(Types.STRING(),Types.STRING()))
#new_stream = data_stream.map(MyMapFunction(),output_type=Types.MAP(Types.STRING(),Types.STRING()))
# 输出到控制台
new_stream.print()

# 执行任务
env.execute('Add "bus_seq" to each line')

[root@master pyflink]# python test.py
625978
625978
625978
625978
625978
625978
625978
66666666666666666
625978
625978
625978
[root@master pyflink]#