Flume ——自定义Source采集源(Java代码)
在开始阅读之前让我们喊出口号:“要想人前显贵,就要人后受罪!”加油嘎嘣儿,加油所有想获得美好未来的你!
public class MyExecSource extends AbstractSource implements EventDrivenSource, Configurable{
private String posiFile;//偏移量记录的位置
private String filename;//所要读取的文件夹的位置
private String charset;//读取文件的编码格式
private Long interval;//监听数据间隔的时间
//以上的数据域需要从conf配置中读取出来
private ExecutorService executor;
private DealDateThread thread;
}
start方法:用来创建线程池,创建线程,并将线程放到线程池
@Override
public synchronized void start{
executor =Executors.newSingleThreadExecutor();
DealDateThread thread = new DealDateThread(filname,podiFile,interval,charset,getChannelProcessor);
executor.submit(thread);
super.start();
}
stop停止:关闭线程池,关闭线程
@Override
public synchronized void stop() {
thread.flag=false;
executor.shutdown();
while (!executor.isTerminated()) {
try {
executor.awaitTermination(500, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
super.stop();
}
DealDateThread线程类
public static class DealDateThread() implements Runnable{
private String posiFile;
private String filename;
private String charset;
private Long interval;
private ChannleProcdssor process;
private boolean flag=true;//做标记,为了一直让run方法进行运行
private Long offset=ol;
private RandomAccessFile raf ;//文件读取对象,可以按照偏移量进行读取
private File file;
public DealDateThread(String filename, String posiFile, Long interval, String charset, ChannelProcessor process) {
this.filename = filename;
this.posiFile = posiFile;
this.interval = interval;
this.charset = charset;
this.process = process;
this.file = new File(posiFile);
try {
if(!file.exists()) {
file.createNewFile();
}//判断记录偏移量的文件是否存在
String offsetStr = FileUtils.readFileToString(file);
//设置偏移量
if(offsetStr!=null&&!offsetStr.equals("")) {
offset = Long.parseLong(offsetStr);
}
raf = new RandomAccessFile(filename, "r");
//对读取的文件进行只读操作
raf.seek(offset);
}catch(Exception e) {
e.printStackTrace();
}
}
//ruan方法真正进行执行的
public void run(){
while (flag){
try {
//通过event对象,将数据发送到channel
String line = raf.readLine();
//读取数据以后将数据发给channel需要通过event对象
if(line!=null) {
Event e = EventBuilder.withBody(line.getBytes(charset));
process.processEvent(e);
//在获取当前的偏移量进行数据更新
this.offset=raf.getFilePointer();
FileUtils.writeStringToFile(file, offset+"");
}else {
Thread.sleep(interval);
}
}catch(Exception e) {
e.printStackTrace();
}
}
}
获取配置信息
public void configure(){
this.filename=context.getString("filename");
this.posiFile=context.getString("posiFile");
this.charset=context.getString("charset");
this.interval=context.getLong("interval");
}
若有写的不正确的地方,请指出。