2、Flume多级连高可用

2019独角兽企业重金招聘Python工程师标准>>>

• 1、机器上的flume挂了如何启动起来：

    写一个脚本监听机器的Application进程，一旦没有了，快速启动它。
  

• 2、多级flume时，第二级会出现单点故障的问题。

  • 解决思路：

      第二级flume设置两个，让他们同时工作。
      由第一级的flume配置文件配置这两个flume的权重，从而只向其中的一个权重高的flume发
      送数据，只有当这个flume挂掉后，才往另一个flume发送。
    

• 多种方式：

  failover（配置权重） 、 random（随机的）

#第一级的配置文件

• [root@mini1 apache-flume-1.6.0-bin]# vi conf/agent.properties

    #agent1 name
    agent1.channels = c1
    agent1.sources = r1
    agent1.sinks = k1 k2
  
    #set gruop
    agent1.sinkgroups = g1
  
    #set channel
    agent1.channels.c1.type = memory
    agent1.channels.c1.capacity = 1000
    agent1.channels.c1.transactionCapacity = 100
  
    agent1.sources.r1.channels = c1
    agent1.sources.r1.type = exec
    agent1.sources.r1.command = tail -F /root/log/test.log
  
    agent1.sources.r1.interceptors = i1 i2
    agent1.sources.r1.interceptors.i1.type = static
    agent1.sources.r1.interceptors.i1.key = Type
    agent1.sources.r1.interceptors.i1.value = LOGIN
    agent1.sources.r1.interceptors.i2.type = timestamp
  
    # set sink1
    agent1.sinks.k1.channel = c1
    agent1.sinks.k1.type = avro
    agent1.sinks.k1.hostname = mini2
    agent1.sinks.k1.port = 52020
  
    # set sink2
    agent1.sinks.k2.channel = c1
    agent1.sinks.k2.type = avro
    agent1.sinks.k2.hostname = mini3
    agent1.sinks.k2.port = 52020
  
    #set sink group
    agent1.sinkgroups.g1.sinks = k1 k2
  
    #set failover
    agent1.sinkgroups.g1.processor.type = failover
    agent1.sinkgroups.g1.processor.priority.k1 = 10
    agent1.sinkgroups.g1.processor.priority.k2 = 1
    agent1.sinkgroups.g1.processor.maxpenalty = 10000
    "conf/client.properties" 43L, 1111C written
  

• 如何启动？

  [root@mini1 apache-flume-1.6.0-bin]# bin/flume-ng agent -n agent1 -c conf -f conf/client.properties -Dflume.root.logger=DEBUG,console

---

#第二级flume的配置文件

• [root@mini2 conf]# vi collector.properties

    #set Agent name
    a1.sources = r1
    a1.channels = c1
    a1.sinks = k1
  
    #set channel
    a1.channels.c1.type = memory
    a1.channels.c1.capacity = 1000
    a1.channels.c1.transactionCapacity = 100
  
    # other node,nna to nns
    a1.sources.r1.type = avro
    a1.sources.r1.bind = mini2
    a1.sources.r1.port = 52020
    a1.sources.r1.interceptors = i1
    a1.sources.r1.interceptors.i1.type = static
    a1.sources.r1.interceptors.i1.key = Collector
    a1.sources.r1.interceptors.i1.value = mini2
    a1.sources.r1.channels = c1
  
    #set sink to hdfs
    a1.sinks.k1.type=hdfs
    a1.sinks.k1.hdfs.path=/home/hdfs/flume/logdfs
    a1.sinks.k1.hdfs.fileType=DataStream
    a1.sinks.k1.hdfs.writeFormat=TEXT
    a1.sinks.k1.hdfs.rollInterval=1
    a1.sinks.k1.channel=c1
    a1.sinks.k1.hdfs.filePrefix=%Y-%m-%d
  

转载于:https://my.oschina.net/liufukin/blog/799168