Flume三个器

拦截器

自定义拦截器方法

java实现其interceptor接口
并实现其4个方法
1.初始化方法
2.单event方法
3.多event方法
4.close方法
5.创建一个静态的内部类 Builder

选择器

Replicating(默认)：会自动的发往全部的channel

Multiplexing：会选择发往对应的channel

监控器

ganglia 发现尝试提交的次数 远远大于最终提交成功次数； 说明flume性能不行；
自身；提高自己的内存 4-6g flume_env.sh （因为高于6G时性能并不会提升特别大所以最高设置为6G）
外援：增加flume台数

自定义拦截器代码及使用

导入相关依赖及打包依赖（因为我们最后是需要在Linux环境下运行我们的flume脚本）

<dependencies>
    <dependency>
        <groupId>org.apache.flume</groupId>
        <artifactId>flume-ng-core</artifactId>
        <version>1.9.0</version>
    </dependency>
</dependencies>

<build>
    <plugins>
        <plugin>
            <artifactId>maven-compiler-plugin</artifactId>
            <version>2.3.2</version>
            <configuration>
                <source>1.8</source>
                <target>1.8</target>
            </configuration>
        </plugin>
        <plugin>
            <artifactId>maven-assembly-plugin</artifactId>
            <configuration>
                <descriptorRefs>
                    <descriptorRef>jar-with-dependencies</descriptorRef>
                </descriptorRefs>
            </configuration>
            <executions>
                <execution>
                    <id>make-assembly</id>
                    <phase>package</phase>
                    <goals>
                        <goal>single</goal>
                    </goals>
                </execution>
            </executions>
        </plugin>
    </plugins>
</build>

package com.qqhru.flume.interceptor;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;

import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

public class FlumeInterceptor implements Interceptor{

	public void initialize() {

	}

	public Event intercept(Event event) {

		//1.获取数据
		byte[] body = event.getBody();
		String line = new String(body, Charset.forName("utf-8"));
		Map<String, String> headers = event.getHeaders();
		if (line.contains("start")) {
			headers.put("topic","topic_start");
		}else {
			headers.put("topic","topic_event");
		}

		return event;
	}

	public List<Event> intercept(List<Event> events) {
		ArrayList<Event> events1 = new ArrayList<Event>();
		for (Event event : events) {
			events1.add(intercept(event));

		}
		return events1;
	}

	public void close() {

	}
	
	public static class Buileder implements Interceptor.Builder{

		@Override
		public Interceptor build() {
			return new FlumeInterceptor();
		}

		@Override
		public void configure(Context context) {

		}
	}
}

a1.sources = r1
a1.channels = c1 c2
a1.sinks  = k1

#source
a1.sources.r1.type=TAILDIR
a1.sources.r1.positionFile = /opt/module/flume/test/log_position.json
a1.sources.r1.filegroups = f1 //此处体现多目录访问可以选择多个用户
a1.sources.r1.filegroups.f1 = /opt/module/hive//logs/*
a1.sources.r1.fileHeader = true
a1.sources.r1.channels = c1 c2


#interceptor
a1.sources.r1.interceptors =  i1
a1.sources.r1.interceptors.i1.type=com.atguigu.flume.interceptor.FlumeInterceptor$Builder

#selector
a1.sources.r1.selector.type = multiplexing
a1.sources.r1.selector.header = topic
a1.sources.r1.selector.mapping.topic_start = c1
a1.sources.r1.selector.mapping.topic_event = c2

# configure channel
a1.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel
a1.channels.c1.kafka.bootstrap.servers = localhost1:9092,localhost2:9092,localhost3:9092
a1.channels.c1.kafka.topic = topic_start
a1.channels.c1.parseAsFlumeEvent = false
a1.channels.c1.kafka.consumer.group.id = flume-consumer

a1.channels.c2.type = org.apache.flume.channel.kafka.KafkaChannel
a1.channels.c2.kafka.bootstrap.servers = localhost1:9092,localhost2:9092,localhost3:9092
a1.channels.c2.kafka.topic = topic_event
a1.channels.c2.parseAsFlumeEvent = false
a1.channels.c2.kafka.consumer.group.id = flume-consumer