一、配置文件taildir.conf
a1.sources = r1
a1.channels = c1
a1.sinks = k1
# Describe/configure the source
#source的类型为TAILDIR,这里的类型大小写都可以
a1.sources.r1.type = taildir
a1.sources.r1.channels = c1
#存储tial最后一个位置存储位置
a1.sources.r1.positionFile = /opt/hui/taildir_position.json
#设置tiail的组, 使用空格隔开
a1.sources.r1.filegroups = f1
#设置每个分组的绝对路径
#.匹配后缀为csv的文件,如hehe.csv
a1.sources.r1.filegroups.f1 = /opt/hui/files/.*.csv
a1.sources.r1.fileHeader = true
# Describe the sink
a1.sinks.k1.type = com.hui.yilianzhong.YwdataTask
a1.sinks.k1.username = hehe
a1.sinks.k1.password = haha@213
a1.sinks.k1.batchSize = 5
# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
二、maven的pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>cn.itcast.demo</groupId>
<artifactId>flume-gauss-sink</artifactId>
<version>1.0.0-SNAPSHOT</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<version.flume>1.8.0</version.flume>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-core</artifactId>
<version>${version.flume}</version>
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-configuration</artifactId>
<version>${version.flume}</version>
</dependency>
<dependency>
<groupId>com.huawei.gaussDb</groupId>
<artifactId>gsjdbc4</artifactId>
<version>1.0.0</version>
</dependency>
</dependencies>
<build>
<finalName>flume-gauss-sink</finalName>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
</plugin>
</plugins>
</build>
</project>
三、代码
package com.hui.yilianzhong;
import com.google.common.base.Preconditions;
import org.apache.flume.Channel;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.Transaction;
import org.apache.flume.conf.Configurable;
import org.apache.flume.sink.AbstractSink;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.SQLException;
/**
* Created by HuiQ on 2020-03-06.
*/
public class YwdataTask extends AbstractSink implements Configurable {
private static final Logger LOG = LoggerFactory.getLogger(YwdataTask.class);
private String username;
private String password;
private int batchSize;
private StringBuilder str = new StringBuilder(); // 要批量插入shbxhj_jcsj表中的数据
private String eventData;
//数据处理的逻辑都在process方法中实现
@Override
public Status process() {
Transaction tx = null ;
Status status=null;
Channel channel= getChannel();
try{
tx= channel.getTransaction();
tx.begin();
for(int i=0;i<batchSize;i++)
{
// 使用take方法尽可能的以批量的方式从Channel中读取事件,直到没有更多的事件
Event event = channel.take();
if(event == null)
{
break;
}
else{ // 也可以不需要else
byte[] body = event.getBody();
eventData = new String(body);
String[] yilianzhongData= new String(body).split("\\|");
String wyid = yilianzhongData[0]; // 唯一表示
String dwdm = yilianzhongData[1];
String dwmc = yilianzhongData[2];
String sqrxm = yilianzhongData[3];
String sqrhm = yilianzhongData[4];
String sqrsj = yilianzhongData[5];
String sqsj = yilianzhongData[6]; // 账单日期
String jfqx = yilianzhongData[7];
String shlx = yilianzhongData[8];
String jdyysr = yilianzhongData[9];
String yfyysr = yilianzhongData[10]; // 2020年内最近一个月营业收入
str.append("('" + wyid + "','" + dwdm + "','" + dwmc + "','" + sqrxm + "','" + sqrhm + "','" + sqrsj + "','" + sqsj + "','" + jfqx + "','" + shlx + "'," + jdyysr + "," + yfyysr + "),");
}
}
if (!"".equals(str.toString())) { // 判断str是否为空
//创建数据库连接。
Connection conn_gauss = GaussUttils.getConnection(username, password);
str.delete(str.length()-1, str.length()); // 去除最后一行末尾的逗号
String datas = str.toString();
// 将业务数据插入到库中的基础表
GaussUttils.insertAdd_shbxhj_jcsj(conn_gauss, datas);
str.delete(0, str.length());
// 执行业务SQL语句
GaussUttils.executeSqls(conn_gauss);
//关闭数据库连接。
try {
conn_gauss.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
tx.commit();
status=Status.READY;
}catch (Exception e) {
if (tx != null) {
System.out.println("这行插入gauss失败请排查-->" + eventData);
tx.commit();// commit to drop bad event, otherwise it will enter dead loop.
}
} finally {
if (tx != null) {
try{
tx.close();
}catch (Exception e){
tx.commit();
tx.close();
}
}
}
return status;
}
//该方法用于读取Flume中Sink的配置,在Sink初始化时调用
@Override
public void configure(Context context) {
username = context.getString("username");
Preconditions.checkNotNull(username, "username must be set");
password = context.getString("password");
Preconditions.checkNotNull("dateFormat must be set");
batchSize = context.getInteger("batchSize");
Preconditions.checkNotNull(batchSize > 0, "batchSize must be a positive number!!");
}
//该方法用于Sink启动时调用
@Override
public synchronized void start()
{
super.start();
LOG.info("GaussSink start...");
}
@Override
public void stop() {
super.stop();
LOG.info("GaussSink stop...");
}
}
然后在idea中执行mvn clean compile package打包并上传到flume安装目录下的lib目录下,执行flume命令即可运行
bin/flume-ng agent -c conf -f conf/taildir.conf -n a1 -Dflume.root.logger=INFO,console
注意:得把所需要的依赖包上传到flume的lib目录下,我这里的代码只用到了一个数据库的驱动包
备注:后来数据处理那块这样写比较好些,虽然batchSize参数没啥用了。因为上面的代码会导致进的数据行数只有batchSize的整数倍才会执行插入数据库的操作,没达到batchSize整数倍的数据一直在flume中实时性可能差点。
// for(int i=0;i<batchSize;i++)
// {
// 使用take方法尽可能的以批量的方式从Channel中读取事件,直到没有更多的事件
Event event = channel.take();
if(event == null)
{
if (!"".equals(str.toString())) { // 判断str是否为空
//创建数据库连接。
Connection conn_gauss = GaussUttils.getConnection(username, password);
str.delete(str.length()-1, str.length()); // 去除最后一行末尾的逗号
String datas = str.toString();
// 将业务数据插入到gauss库中的基础表
GaussUttils.insertAdd_shbxhj_jcsj(conn_gauss, datas);
str.delete(0, str.length());
// 执行业务SQL语句
GaussUttils.executeSqls(conn_gauss);
//关闭数据库连接。
try {
conn_gauss.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
} else{
byte[] body = event.getBody();
eventData = new String(body);
String[] yilianzhongData= new String(body).split("\\|");
String wyid = yilianzhongData[0];
String dwdm = yilianzhongData[1];
String dwmc = yilianzhongData[2];
String sqrxm = yilianzhongData[3];
String sqrhm = yilianzhongData[4];
String sqrsj = yilianzhongData[5];
String sqsj = yilianzhongData[6];
String jfqx = yilianzhongData[7];
String shlx = yilianzhongData[8];
String jdyysr = yilianzhongData[9];
String yfyysr = yilianzhongData[10];
str.append("('" + wyid + "','" + dwdm + "','" + dwmc + "','" + sqrxm + "','" + sqrhm + "','" + sqrsj + "','" + sqsj + "','" + jfqx + "','" + shlx + "'," + jdyysr + "," + yfyysr + "),");
}
// }