操作ml_f

bin/flink run -class test com.fl.h.jar
bin/flink run -ynm test -m yarn-cluster -c test com.fl.ho.jar
ps -ef | grep java
jps -m
yarn application -list | grep dmp
yarn application -kill 
flink list
flink stop 
###############################################
bin/yarn-session.sh -n 4 -s 4 -jm 2048 -tm 4096  -nm  check_id   & 
./flink run --class HardDiskSerial_Check com.flink.honor.wwc-1.0-SNAPSHOT.jar
./flink run --class HardDiskSerial_Check com.flink.honor.wwc-1.0-SNAPSHOT.jar

查看运行进程：ps -ef|grep java
查看jps：jps -m
查看yarn任务：yarn application -list | grep dmp
通过命令来停止 yarn application -kill   application_id
查看flink任务信息：bin/flink list
取消flink任务：flink cansel  job的id

import org.apache.flink.streaming.api.scala.DataStream
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer
import org.apache.flink.streaming.api.functions.ProcessFunction
import org.apache.flink.streaming.api.scala._
import org.apache.flink.util.Collector
import redis.clients.jedis.{HostAndPort, JedisCluster, JedisPoolConfig}
import java.util.Properties
import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, ProducerRecord}
import org.apache.kafka.common.serialization.StringSerializer
import scala.util.matching.Regex
object Check_test {

//创建Kafka的Consumer连接：
  private val prop = new Properties()
  prop.setProperty("bootstrap.servers",":9092,:9092,:9092,:9092,1000.8.2.2:9092")
  prop.put(ConsumerConfig.GROUP_ID_CONFIG,"GGGGG")
  prop.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer].getName) //key的序列化;
  prop.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer].getName)//value的序列化;
  def getConsumer(topic:String ):FlinkKafkaConsumer[String]= {
    val myKafkaConsumer:FlinkKafkaConsumer[String] = new FlinkKafkaConsumer[String](topic, new SimpleStringSchema(), prop)
    //从最开始的位置消费：myKafkaConsumer.setStartFromEarliest()
    myKafkaConsumer.setStartFromEarliest()
    myKafkaConsumer
  }
//创建redis连接池：
  def redisConnect(): JedisCluster = {
    val jedisPoolConfig = new JedisPoolConfig;
    // 最大空闲连接数, 默认8个
    jedisPoolConfig.setMaxIdle(100);
    // 最大连接数, 默认8个
    jedisPoolConfig.setMaxTotal(500);
    //最小空闲连接数, 默认0
    jedisPoolConfig.setMinIdle(0);
    // 获取连接时的最大等待毫秒数(如果设置为阻塞时BlockWhenExhausted),如果超时就抛异常, 小于零:阻塞不确定的时间,  默认-1
    jedisPoolConfig.setMaxWaitMillis(2000); // 设置2秒
    //对拿到的connection进行validateObject校验
    jedisPoolConfig.setTestOnBorrow(true)
    val jedisClusterNodes = new java.util.HashSet[HostAndPort]()
    jedisClusterNodes.add(new HostAndPort("", 6379))
    jedisClusterNodes.add(new HostAndPort("", 6379))
    jedisClusterNodes.add(new HostAndPort("", 6379))
    jedisClusterNodes.add(new HostAndPort("", 6379))
    jedisClusterNodes.add(new HostAndPort("", 6379))
    jedisClusterNodes.add(new HostAndPort("", 6379))
    lazy val jedisCluster = new JedisCluster(jedisClusterNodes,100,100,50,"2",jedisPoolConfig)
    jedisCluster
  }

//连接redis
  val jedis = redisConnect()
  val hset_key_name = "hset_machineid_harddiskserial_check"

//连接生产者topic
  val kafkaConsumer  =getConsumer("acc_event_runlog")
  val environment: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment

//连接警报topic
  val topic = "acc_alarm"
  val properties = new Properties()
  properties.put("bootstrap.servers","")
  properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer].getName) //key的序列化;
  properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer].getName)//value的序列化;
  val producer = new KafkaProducer[String, String](properties)

//正则匹配规则
  val pattern_sex = new Regex(",\"machineid\":\".*?\",")
  val pattern_name = new Regex(",\"harddiskserial\":\".*?\",")

  var machineid = ""
  var harddiskserial = ""
  var h_get = ""
  var change_Kafka_data=""

  def main(args: Array[String]): Unit = {
    println("begin check")
    val dstream: DataStream[String] = environment.addSource(kafkaConsumer)
    dstream
      .process(new Check())
    environment.execute()}

  class Check() extends ProcessFunction[String, String] {
    override def processElement(data_str: String, p: ProcessFunction[String, String]#Context, out: Collector[String]): Unit = {
      machineid = (pattern_sex findAllIn data_str).mkString("").replaceAll(",\"machineid\":\"","").replaceAll("\",","")
      harddiskserial = (pattern_name findAllIn data_str).mkString("").replaceAll(",\"harddiskserial\":\"","").replaceAll("\",","")

//      Thread.sleep(2000)
//判断harddiskserial是否变化,如果变化redis记录变化
      if (harddiskserial != ""){
        h_get = jedis.hget(hset_key_name,machineid)
        if (h_get==null){
          println("first insert redis_field")
          jedis.hset(hset_key_name,machineid,harddiskserial)
        }else if(h_get==harddiskserial) {
          println("id not Change")
        }else{
          println("Disk Change")
//变化后更新redis键值,新增加old_id字段写入相应topic
        change_Kafka_data = data_str.replace("}",",")+"\"alarm_name\":\"Disk Change\","+"\"old_harddiskserial\""+":"+"\""+h_get+"\"}"
        jedis.hset(hset_key_name,machineid,harddiskserial)
        producer.send(new ProducerRecord(topic,change_Kafka_data))}

      }
    }
  }
}

######################################
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>honor_f</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <flink.version>1.7.2</flink.version>
        <scala.binary.version>2.11</scala.binary.version>
        <kafka.version>2.2.0</kafka.version>
        <jedis.version>2.9.0</jedis.version>
    </properties>

    <dependencies>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka_${scala.binary.version}</artifactId>
            <version>${kafka.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>redis.clients</groupId>
            <artifactId>jedis</artifactId>
            <version>${jedis.version}</version>
        </dependency>

    </dependencies>

</project>

###############################
创建redis连接方法:
//private val redisHost: String = ""
//private val redisPort: Int = 6379
//private val redisTimeOut: Int = 30000 // ms
//private val maxTotal = 300
//private val maxIdle = 100
//private val minIdle = 1
//private val passWord = "honerredis"
//@transient private var pool: JedisPool = _
//makePool(redisHost, redisPort, redisTimeOut, maxTotal, maxIdle, minIdle)
//def makePool(redisHost: String, redisPort: Int, redisTimeout: Int,
//             maxTotal: Int, maxIdle: Int, minIdle: Int): Unit = {
//  makePool(redisHost, redisPort, redisTimeout, maxTotal, maxIdle, minIdle, true, false, 10000)
//}
//def makePool(redisHost: String, redisPort: Int, redisTimeout: Int,
//             maxTotal: Int, maxIdle: Int, minIdle: Int, testOnBorrow: Boolean,
//             testOnReturn: Boolean, maxWaitMillis: Long): Unit = {
//  if (pool == null) {
//    val poolConfig = new GenericObjectPoolConfig()
//    poolConfig.setMaxTotal(maxTotal)
//    poolConfig.setMaxIdle(maxIdle)
//    poolConfig.setMinIdle(minIdle)
//    poolConfig.setTestOnBorrow(testOnBorrow)
//    poolConfig.setTestOnReturn(testOnReturn)
//    poolConfig.setMaxWaitMillis(maxWaitMillis)
//    pool = new JedisPool(poolConfig, redisHost, redisPort, redisTimeout)
//
//    val hook = new Thread {
//      override def run: Unit = pool.destroy()
//    }
//    sys.addShutdownHook(hook.run)
//  }
//}
//def getPool: JedisPool = {
//  assert(pool != null)
//  pool
//}
//调用：  val jedis = getPool.getResource

kafka输入：[Kafka/kafka/bin]$./kafka-console-producer.sh --broker-list 1.6.5.3:9092,1.8.4.3:9092,1.6.5.93:9092,1.6.5.5:9092,10.68.52.92:9092 --topic testout
topic所有信息：./kafka-console-consumer.sh   --topic testout  --bootstrap-server   --from-beginning
bin/kafka-console-consumer.sh   --topic acc_event_runlog  --bootstrap-server   --from-beginning

def getProduct(): KafkaProducer[String, String] = {
  val properties = new Properties()
  properties.put("bootstrap.servers",":9092,10.68:9092")
  properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer].getName) //key的序列化;
  properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer].getName)//value的序列化;
  lazy val myKafkaproducer = new KafkaProducer[String, String](properties)
  myKafkaproducer
}
//连接生产者topic
val kafkaConsumer  =getConsumer("nlog")
#######################################################
mysql:
val username ="test"
val password =""
val url ="jdbc:mysql://10.68.:3306/model
val connection = DriverManager.getConnection(url,username,password)
val statement=connection.createStatement()

val resultSet=staatement.executeQuery("select 'param' from 'test' where 'naame'='通用'" and ('param'='t' or 'param' = 't2'))

prop.put(ConsumerConfig.GROUP_ID_CONFIG,"GGGGG")