1、自定义类
package cn.com.sgcc.jibei.event_warning; import java.math.BigDecimal; public class Event_warning { private BigDecimal merge_count; private BigDecimal id; private String code; private BigDecimal warningtype_code; private String warningtime; private BigDecimal warninglevel_code; private BigDecimal warninggen_code; private String warningdeviceip; private BigDecimal rule_id; private String scene_id; //clob private String attacktarget; private String attacksrc; private String attackaction; private BigDecimal testrunflag; private BigDecimal slogstart_id; private BigDecimal slogend_id; private String influenceentity; private BigDecimal warningstatus_code; private BigDecimal warningsubtype; private String warningname; private BigDecimal slogtype; private String entityname; private String resourcesystem; private BigDecimal entitytype_code; private String personincharge; //clob private String value1; private String value2; private String value3; private String value4; private String value5; private String warningsrc_code; private BigDecimal warning_count; private String createtime; private BigDecimal network_code; //clob private String handleadvice; private String warning_log; private String staff; private BigDecimal flag; private String handle_time; private String professional; private String operdep; private String attacktarget_sta; private String attacktarget_pro; private BigDecimal attacktarget_network_code; private String attacksrc_sta; private String attacksrc_pro; private BigDecimal attacksrc_network_code; private BigDecimal warningdevice_boundary; private BigDecimal ipv6flag; private String op_time; private String etl_date; public Event_warning() { } public Event_warning(BigDecimal merge_count, BigDecimal id, String code, BigDecimal warningtype_code, String warningtime, BigDecimal warninglevel_code, BigDecimal warninggen_code, String warningdeviceip, BigDecimal rule_id, String scene_id, String attacktarget, String attacksrc, String attackaction, BigDecimal testrunflag, BigDecimal slogstart_id, BigDecimal slogend_id, String influenceentity, BigDecimal warningstatus_code, BigDecimal warningsubtype, String warningname, BigDecimal slogtype, String entityname, String resourcesystem, BigDecimal entitytype_code, String personincharge, String value1, String value2, String value3, String value4, String value5, String warningsrc_code, BigDecimal warning_count, String createtime, BigDecimal network_code, String handleadvice, String warning_log, String staff, BigDecimal flag, String handle_time, String professional, String operdep, String attacktarget_sta, String attacktarget_pro, BigDecimal attacktarget_network_code, String attacksrc_sta, String attacksrc_pro, BigDecimal attacksrc_network_code, BigDecimal warningdevice_boundary, BigDecimal ipv6flag, String op_time, String etl_date) { this.merge_count = merge_count; this.id = id; this.code = code; this.warningtype_code = warningtype_code; this.warningtime = warningtime; this.warninglevel_code = warninglevel_code; this.warninggen_code = warninggen_code; this.warningdeviceip = warningdeviceip; this.rule_id = rule_id; this.scene_id = scene_id; this.attacktarget = attacktarget; this.attacksrc = attacksrc; this.attackaction = attackaction; this.testrunflag = testrunflag; this.slogstart_id = slogstart_id; this.slogend_id = slogend_id; this.influenceentity = influenceentity; this.warningstatus_code = warningstatus_code; this.warningsubtype = warningsubtype; this.warningname = warningname; this.slogtype = slogtype; this.entityname = entityname; this.resourcesystem = resourcesystem; this.entitytype_code = entitytype_code; this.personincharge = personincharge; this.value1 = value1; this.value2 = value2; this.value3 = value3; this.value4 = value4; this.value5 = value5; this.warningsrc_code = warningsrc_code; this.warning_count = warning_count; this.createtime = createtime; this.network_code = network_code; this.handleadvice = handleadvice; this.warning_log = warning_log; this.staff = staff; this.flag = flag; this.handle_time = handle_time; this.professional = professional; this.operdep = operdep; this.attacktarget_sta = attacktarget_sta; this.attacktarget_pro = attacktarget_pro; this.attacktarget_network_code = attacktarget_network_code; this.attacksrc_sta = attacksrc_sta; this.attacksrc_pro = attacksrc_pro; this.attacksrc_network_code = attacksrc_network_code; this.warningdevice_boundary = warningdevice_boundary; this.ipv6flag = ipv6flag; this.op_time = op_time; this.etl_date = etl_date; } public BigDecimal getMerge_count() { return merge_count; } public BigDecimal getId() { return id; } public String getCode() { return code; } public BigDecimal getWarningtype_code() { return warningtype_code; } public String getWarningtime() { return warningtime; } public BigDecimal getWarninglevel_code() { return warninglevel_code; } public BigDecimal getWarninggen_code() { return warninggen_code; } public String getWarningdeviceip() { return warningdeviceip; } public BigDecimal getRule_id() { return rule_id; } public String getScene_id() { return scene_id; } public String getAttacktarget() { return attacktarget; } public String getAttacksrc() { return attacksrc; } public String getAttackaction() { return attackaction; } public BigDecimal getTestrunflag() { return testrunflag; } public BigDecimal getSlogstart_id() { return slogstart_id; } public BigDecimal getSlogend_id() { return slogend_id; } public String getInfluenceentity() { return influenceentity; } public BigDecimal getWarningstatus_code() { return warningstatus_code; } public BigDecimal getWarningsubtype() { return warningsubtype; } public String getWarningname() { return warningname; } public BigDecimal getSlogtype() { return slogtype; } public String getEntityname() { return entityname; } public String getResourcesystem() { return resourcesystem; } public BigDecimal getEntitytype_code() { return entitytype_code; } public String getPersonincharge() { return personincharge; } public String getValue1() { return value1; } public String getValue2() { return value2; } public String getValue3() { return value3; } public String getValue4() { return value4; } public String getValue5() { return value5; } public String getWarningsrc_code() { return warningsrc_code; } public BigDecimal getWarning_count() { return warning_count; } public String getCreatetime() { return createtime; } public BigDecimal getNetwork_code() { return network_code; } public String getHandleadvice() { return handleadvice; } public String getWarning_log() { return warning_log; } public String getStaff() { return staff; } public BigDecimal getFlag() { return flag; } public String getHandle_time() { return handle_time; } public String getProfessional() { return professional; } public String getOperdep() { return operdep; } public String getAttacktarget_sta() { return attacktarget_sta; } public String getAttacktarget_pro() { return attacktarget_pro; } public BigDecimal getAttacktarget_network_code() { return attacktarget_network_code; } public String getAttacksrc_sta() { return attacksrc_sta; } public String getAttacksrc_pro() { return attacksrc_pro; } public BigDecimal getAttacksrc_network_code() { return attacksrc_network_code; } public BigDecimal getWarningdevice_boundary() { return warningdevice_boundary; } public BigDecimal getIpv6flag() { return ipv6flag; } public String getOp_time() { return op_time; } public String getEtl_date() { return etl_date; } @Override public String toString() { return "Event_warning{" + "merge_count=" + merge_count + ", id=" + id + ", code='" + code + '\'' + ", warningtype_code=" + warningtype_code + ", warningtime='" + warningtime + '\'' + ", warninglevel_code=" + warninglevel_code + ", warninggen_code=" + warninggen_code + ", warningdeviceip='" + warningdeviceip + '\'' + ", rule_id=" + rule_id + ", scene_id='" + scene_id + '\'' + ", attacktarget='" + attacktarget + '\'' + ", attacksrc='" + attacksrc + '\'' + ", attackaction='" + attackaction + '\'' + ", testrunflag=" + testrunflag + ", slogstart_id=" + slogstart_id + ", slogend_id=" + slogend_id + ", influenceentity='" + influenceentity + '\'' + ", warningstatus_code=" + warningstatus_code + ", warningsubtype=" + warningsubtype + ", warningname='" + warningname + '\'' + ", slogtype=" + slogtype + ", entityname='" + entityname + '\'' + ", resourcesystem='" + resourcesystem + '\'' + ", entitytype_code=" + entitytype_code + ", personincharge='" + personincharge + '\'' + ", value1='" + value1 + '\'' + ", value2='" + value2 + '\'' + ", value3='" + value3 + '\'' + ", value4='" + value4 + '\'' + ", value5='" + value5 + '\'' + ", warningsrc_code='" + warningsrc_code + '\'' + ", warning_count=" + warning_count + ", createtime='" + createtime + '\'' + ", network_code=" + network_code + ", handleadvice='" + handleadvice + '\'' + ", warning_log='" + warning_log + '\'' + ", staff='" + staff + '\'' + ", flag=" + flag + ", handle_time='" + handle_time + '\'' + ", professional='" + professional + '\'' + ", operdep='" + operdep + '\'' + ", attacktarget_sta='" + attacktarget_sta + '\'' + ", attacktarget_pro='" + attacktarget_pro + '\'' + ", attacktarget_network_code=" + attacktarget_network_code + ", attacksrc_sta='" + attacksrc_sta + '\'' + ", attacksrc_pro='" + attacksrc_pro + '\'' + ", attacksrc_network_code=" + attacksrc_network_code + ", warningdevice_boundary=" + warningdevice_boundary + ", ipv6flag=" + ipv6flag + ", op_time='" + op_time + '\'' + ", etl_date='" + etl_date + '\'' + '}'; } }
2、连接kafka
package cn.com.sgcc.jibei.event_warning; import cn.com.sgcc.jibei.util.PropertyUtil; import cn.com.sgcc.jibei.util.isBigDecimal; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.functions.FilterFunction; import org.apache.flink.api.common.functions.MapFunction; import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExtractor; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer08; import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.streaming.util.serialization.SimpleStringSchema; import java.math.BigDecimal; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Properties; import static org.apache.flink.util.Preconditions.checkNotNull; /** * @author 杨迎春 * 接收数据表:Event_warning */ public class KafkaToEvent_warning { public static void main(String[] args) { System.out.println("=====================kafka to Event_warning start====================="); Properties pro = new Properties(); pro.put("bootstrap.servers", PropertyUtil.get("kafka.hosts")); pro.put("zookeeper.connect", PropertyUtil.get("kafka.zookper")); pro.put("group.id", PropertyUtil.get("kafka.group")); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableSysoutLogging(); env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000)); env.enableCheckpointing(5000); final List<KafkaTopicPartition> partitions = new ArrayList<>(); String topic = PropertyUtil.get("kafka.topic_event_warning"); final DataStream<String> sourceStream = env.addSource( new FlinkKafkaConsumer08<String>(topic,new SimpleStringSchema(), pro) ); DataStream<Event_warning> sourceStreamTra = sourceStream .filter(new FilterFunction<String>() { public boolean filter(String value) throws Exception { return StringUtils.isNotBlank(value) &&value.split("\",\"").length==49; } }).map(new MapFunction<String, Event_warning>() { private static final long serialVersionUID = 1L; public Event_warning map(String value) throws Exception { String[] args = value.split("\",\""); //System.out.println("查看数据切割情况"); //System.out.println(args.length); SimpleDateFormat f=new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); return new Event_warning( isBigDecimal.isbigdecimal(args[0].trim().substring(1))?new BigDecimal(args[0].trim().substring(1)):null, isBigDecimal.isbigdecimal(args[1].trim())?new BigDecimal(args[1].trim()):null, args[2].trim().equals("NULL") ? null : args[2].trim(), isBigDecimal.isbigdecimal(args[3].trim())?new BigDecimal(args[3].trim()):null, args[4].trim().equals("NULL") ? null : args[4].trim(), isBigDecimal.isbigdecimal(args[5].trim())?new BigDecimal(args[5].trim()):null, isBigDecimal.isbigdecimal(args[6].trim())?new BigDecimal(args[6].trim()):null, args[7].trim().equals("NULL") ? null : args[7].trim(), isBigDecimal.isbigdecimal(args[8].trim())?new BigDecimal(args[8].trim()):null, args[9].trim().equals("NULL") ? null : args[9].trim(), args[10].trim().equals("NULL") ? null : args[10].trim(), args[11].trim().equals("NULL") ? null : args[11].trim(), args[12].trim().equals("NULL") ? null : args[12].trim(), isBigDecimal.isbigdecimal(args[13].trim())?new BigDecimal(args[13].trim()):null, isBigDecimal.isbigdecimal(args[14].trim())?new BigDecimal(args[14].trim()):null, isBigDecimal.isbigdecimal(args[15].trim())?new BigDecimal(args[15].trim()):null, args[16].trim().equals("NULL") ? null : args[16].trim(), isBigDecimal.isbigdecimal(args[17].trim())?new BigDecimal(args[17].trim()):null, isBigDecimal.isbigdecimal(args[18].trim())?new BigDecimal(args[18].trim()):null, args[19].trim().equals("NULL") ? null : args[19].trim(), isBigDecimal.isbigdecimal(args[20].trim())?new BigDecimal(args[20].trim()):null, args[21].trim().equals("NULL") ? null : args[21].trim(), args[22].trim().equals("NULL") ? null : args[22].trim(), isBigDecimal.isbigdecimal(args[23].trim())?new BigDecimal(args[23].trim()):null, args[24].trim().equals("NULL") ? null : args[24].trim(), args[25].trim().equals("NULL") ? null : args[25].trim(), args[26].trim().equals("NULL") ? null : args[26].trim(), args[27].trim().equals("NULL") ? null : args[27].trim(), args[28].trim().equals("NULL") ? null : args[28].trim(), args[29].trim().equals("NULL") ? null : args[29].trim(), args[30].trim().equals("NULL") ? null : args[30].trim(), isBigDecimal.isbigdecimal(args[31].trim())?new BigDecimal(args[31].trim()):null, args[32].trim().equals("NULL") ? null : args[32].trim(), isBigDecimal.isbigdecimal(args[33].trim())?new BigDecimal(args[33].trim()):null, args[34].trim().equals("NULL") ? null : args[34].trim(), args[35].trim().equals("NULL") ? null : args[35].trim(), args[36].trim().equals("NULL") ? null : args[36].trim(), isBigDecimal.isbigdecimal(args[37].trim())?new BigDecimal(args[37].trim()):null, args[38].trim().equals("NULL") ? null : args[38].trim(), args[39].trim().equals("NULL") ? null : args[39].trim(), args[40].trim().equals("NULL") ? null : args[40].trim(), args[41].trim().equals("NULL") ? null : args[41].trim(), args[42].trim().equals("NULL") ? null : args[42].trim(), isBigDecimal.isbigdecimal(args[43].trim())?new BigDecimal(args[43].trim()):null, args[44].trim().equals("NULL") ? null : args[44].trim(), args[45].trim().equals("NULL") ? null : args[45].trim(), isBigDecimal.isbigdecimal(args[46].trim())?new BigDecimal(args[46].trim()):null, isBigDecimal.isbigdecimal(args[47].trim())?new BigDecimal(args[47].trim()):null, args[48].replace("\"","").length()!=0 &&isBigDecimal.isbigdecimal(args[48].trim().substring(0,args[48].length()-1))? new BigDecimal(args[48].trim().substring(0,args[48].length()-1)):null, f.format(new Date()), f.format(new Date()) ); } }); System.out.println("===============!数据拉取成功标志!==============="); sourceStreamTra.addSink(new Event_warningSink()); try { env.execute("data to sync_E_mp_cur_curve start"); } catch (Exception e) { e.printStackTrace(); } } }
3、自定义oracle sink
package cn.com.sgcc.jibei.event_warning; import cn.com.sgcc.jibei.util.PropertyUtil; import cn.com.sgcc.jibei.util.isDateRight; import java.io.*; import java.sql.*; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; public class Event_warningSink implements org.apache.flink.streaming.api.functions.sink.SinkFunction<Event_warning> { private static final long serialVersionUID = 1L; private Connection connection; private PreparedStatement preparedStatement; private ResultSet rs; String username = PropertyUtil.get("oracle.user"); String password = PropertyUtil.get("oracle.pwd");; String drivername = PropertyUtil.get("oracle.driver"); String dburl = PropertyUtil.get("oracle.url"); private int rows =0; static SimpleDateFormat s1DateFormat=new SimpleDateFormat("yyyy-MM-dd"); @Override public void invoke(Event_warning event_warning) { try { //System.out.println("数据是否取到?"); //System.out.println(event_warning); Class.forName(drivername); connection = DriverManager.getConnection(dburl, username, password); if (event_warning.getId() != null && event_warning.getCode() != null && event_warning.getWarningtype_code() != null && event_warning.getWarningtime() != null && event_warning.getWarninglevel_code() != null && event_warning.getFlag() != null ) { String sql = "insert into jbbdkf.sync_s6000_event_warning(" + "merge_count,id,code,warningtype_code,warningtime,\n" + "warninglevel_code,warninggen_code,warningdeviceip,rule_id,scene_id,\n" + "attacktarget,attacksrc,attackaction,testrunflag,slogstart_id,\n" + "slogend_id,influenceentity,warningstatus_code,warningsubtype,warningname,\n" + "slogtype,entityname,resourcesystem,entitytype_code,personincharge,\n" + "value1,value2,value3,value4,value5,\n" + "warningsrc_code,warning_count,create_time,network_code,handleadvice,\n" + "warning_log,staff,flag,handle_time,professional,\n" + "operdep,attacktarget_sta,attacktarget_pro,attacktarget_network_code,attacksrc_sta,\n" + "attacksrc_pro,attacksrc_network_code,warningdevice_boundary,ipv6flag,op_time,\n" + "etl_date) " + "values(?,?,?,?,?,?,?,?,?,?,\n" + "?,?,?,?,?,?,?,?,?,?,\n" + "?,?,?,?,?,?,?,?,?,?,\n" + "?,?,?,?,?,?,?,?,?,?,\n" + "?,?,?,?,?,?,?,?,?,?,\n" + "?)"; SimpleDateFormat sDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); preparedStatement = connection.prepareStatement(sql); preparedStatement.setBigDecimal(1, event_warning.getMerge_count()); preparedStatement.setBigDecimal(2, event_warning.getId()); preparedStatement.setString(3, event_warning.getCode()); preparedStatement.setBigDecimal(4, event_warning.getWarningtype_code()); preparedStatement.setTimestamp(5, isDateRight.isDatetime(event_warning.getWarningtime()) ? new java.sql.Timestamp(sDateFormat.parse(event_warning.getWarningtime()).getTime()) : new java.sql.Timestamp(sDateFormat.parse(sDateFormat.format(new Date())).getTime())); preparedStatement.setBigDecimal(6, event_warning.getWarninglevel_code()); preparedStatement.setBigDecimal(7, event_warning.getWarninggen_code()); preparedStatement.setString(8, event_warning.getWarningdeviceip()); preparedStatement.setBigDecimal(9, event_warning.getRule_id()); preparedStatement.setString(10, event_warning.getScene_id()); preparedStatement.setClob(11, new BufferedReader(new CharArrayReader(event_warning.getAttacktarget().toCharArray()))); preparedStatement.setClob(12, new BufferedReader(new CharArrayReader(event_warning.getAttacksrc().toCharArray()))); preparedStatement.setClob(13, new BufferedReader(new CharArrayReader(event_warning.getAttackaction().toCharArray()))); preparedStatement.setBigDecimal(14, event_warning.getTestrunflag()); preparedStatement.setBigDecimal(15, event_warning.getSlogstart_id()); preparedStatement.setBigDecimal(16, event_warning.getSlogend_id()); preparedStatement.setString(17, event_warning.getInfluenceentity()); preparedStatement.setBigDecimal(18, event_warning.getWarningstatus_code()); preparedStatement.setBigDecimal(19, event_warning.getWarningsubtype()); preparedStatement.setString(20, event_warning.getWarningname()); preparedStatement.setBigDecimal(21, event_warning.getSlogtype()); preparedStatement.setString(22, event_warning.getEntityname()); preparedStatement.setString(23, event_warning.getResourcesystem()); preparedStatement.setBigDecimal(24, event_warning.getEntitytype_code()); preparedStatement.setString(25, event_warning.getPersonincharge()); preparedStatement.setClob(26, new BufferedReader(new CharArrayReader(event_warning.getValue1().toCharArray()))); preparedStatement.setClob(27, new BufferedReader(new CharArrayReader(event_warning.getValue2().toCharArray()))); preparedStatement.setString(28, event_warning.getValue3()); preparedStatement.setString(29, event_warning.getValue4()); preparedStatement.setString(30, event_warning.getValue5()); preparedStatement.setString(31, event_warning.getWarningsrc_code()); preparedStatement.setBigDecimal(32, event_warning.getWarning_count()); preparedStatement.setTimestamp(33, isDateRight.isDatetime(event_warning.getCreatetime()) ? new java.sql.Timestamp(sDateFormat.parse(event_warning.getCreatetime()).getTime()) : new java.sql.Timestamp(sDateFormat.parse(sDateFormat.format(new Date())).getTime())); preparedStatement.setBigDecimal(34, event_warning.getNetwork_code()); preparedStatement.setClob(35, new BufferedReader(new CharArrayReader(event_warning.getHandleadvice().toCharArray()))); preparedStatement.setString(36, event_warning.getWarning_log()); preparedStatement.setString(37, event_warning.getStaff()); preparedStatement.setBigDecimal(38, event_warning.getFlag()); preparedStatement.setTimestamp(39, isDateRight.isDatetime(event_warning.getHandle_time()) ? new java.sql.Timestamp(sDateFormat.parse(event_warning.getHandle_time()).getTime()) : new java.sql.Timestamp(sDateFormat.parse(sDateFormat.format(new Date())).getTime())); preparedStatement.setString(40, event_warning.getProfessional()); preparedStatement.setString(41, event_warning.getOperdep()); preparedStatement.setString(42, event_warning.getAttacktarget_sta()); preparedStatement.setString(43, event_warning.getAttacktarget_pro()); preparedStatement.setBigDecimal(44, event_warning.getAttacktarget_network_code()); preparedStatement.setString(45, event_warning.getAttacksrc_sta()); preparedStatement.setString(46, event_warning.getAttacksrc_pro()); preparedStatement.setBigDecimal(47, event_warning.getAttacksrc_network_code()); preparedStatement.setBigDecimal(48, event_warning.getWarningdevice_boundary()); preparedStatement.setBigDecimal(49, event_warning.getIpv6flag()); preparedStatement.setTimestamp(50, isDateRight.isDatetime(event_warning.getOp_time()) ? new java.sql.Timestamp(sDateFormat.parse(event_warning.getOp_time()).getTime()) : new java.sql.Timestamp(sDateFormat.parse(sDateFormat.format(new Date())).getTime())); preparedStatement.setTimestamp(51, isDateRight.isDatetime(event_warning.getEtl_date()) ? new java.sql.Timestamp(sDateFormat.parse(event_warning.getEtl_date()).getTime()) : new java.sql.Timestamp(sDateFormat.parse(sDateFormat.format(new Date())).getTime())); //preparedStatement.setClob(51,new BufferedReader(new CharArrayReader(event_warning.getAttacktarget().toCharArray()))); preparedStatement.executeUpdate(); } if (preparedStatement != null) { preparedStatement.close(); } if (connection != null) { connection.close(); } } catch (SQLException e) { e.printStackTrace(); System.out.println(event_warning); } catch (ClassNotFoundException e) { e.printStackTrace(); System.out.println(event_warning); } catch (ParseException e) { e.printStackTrace(); System.out.println(event_warning); } } }