添加过滤器 把不同的数据通过不同的通道进行传输 , 进而使用不同的sink接受
配置flume
a1.sources=s1
a1.channels=c1 c2
a1.sinks=k1 k2
#使用netcat作为客户端进行测试
a1.sources.s1.type=netcat
a1.sources.s1.bind=localhost
a1.sources.s1.port=5555
#设置拦截器
a1.sources.s1.interceptors=i1
#自己使用java写的连接器【详细请看下面的java代码】
a1.sources.s1.interceptors.i1.type=cn.kgc.InterceptorDemo$Builder #和java中对应的
a1.sources.s1.selector.type=multiplexing
#以下设置也需和java中对应
a1.sources.s1.selector.header=type
a1.sources.s1.selector.mapping.gree=c1
a1.sources.s1.selector.mapping.wq=c2
#设置两个通道 c1 c2
a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100
a1.channels.c2.type=memory
a1.channels.c2.capacity=1000
a1.channels.c2.transactionCapacity=100
#设置sink k1 k2 都写入hdfs中
a1.sinks.k1.type=hdfs
a1.sinks.k1.hdfs.fileType=DataStream
a1.sinks.k1.hdfs.filePrefix=gree
a1.sinks.k1.hdfs.fileSuffix=.csv
a1.sinks.k1.hdfs.path=hdfs://192.168.241.131:9000/flumegree/%Y-%m-%d
a1.sinks.k1.hdfs.useLocalTimeStamp=true
a1.sinks.k1.hdfs.batchSize=640
a1.sinks.k1.hdfs.rollCount= 0
a1.sinks.k1.hdfs.rollSize=100
a1.sinks.k1.hdfs.rollInterval =3
a1.sinks.k2.type=hdfs
a1.sinks.k2.hdfs.fileType=DataStream
a1.sinks.k2.hdfs.filePrefix=wq
a1.sinks.k2.hdfs.fileSuffix=.csv
a1.sinks.k2.hdfs.path=hdfs://192.168.241.131:9000/flumewq/%Y-%m-%d
a1.sinks.k2.hdfs.useLocalTimeStamp=true
a1.sinks.k2.hdfs.batchSize=640
a1.sinks.k2.hdfs.rollCount= 0
a1.sinks.k2.hdfs.rollSize=100
a1.sinks.k2.hdfs.rollInterval =3
#设置连接
a1.sources.s1.channels=c1 c2
a1.sinks.k1.channel=c1
a1.sinks.k2.channel=c2
ps:想在虚拟机中执行以上代码,需要把自定义的拦截器的jar包 放入到 flume的lib目录下。
自定义拦截器的java代码如下:
//flume shiyongjava 拦截器的构造
public class InterceptorDemo implements Interceptor {
private List<Event> addHeaderEvents;
@Override
public void initialize() {
addHeaderEvents=new ArrayList<>();
}
//以gree开头的进入一个channel 其他的进入另一个channel
@Override
public Event intercept(Event event) {
Map<String, String> headers = event.getHeaders();
String body= new String(event.getBody());
if(body.startsWith("gree")){
headers.put("type","gree");
}else{
headers.put("type","wq");
}
return event;
}
@Override
public List<Event> intercept(List<Event> list) {
addHeaderEvents.clear();
for (Event event : list) {
addHeaderEvents.add(intercept(event));
}
return addHeaderEvents;
}
@Override
public void close() {
}
public static class Builder implements Interceptor.Builder{
@Override
public Interceptor build() {
return new InterceptorDemo();
}
@Override
public void configure(Context context) {
}
}
}