准备工作:
在MySQL的db_test数据库下创建表tb_user:
CREATE TABLE `tb_user` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`username` varchar(255) DEFAULT NULL,
`password` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=latin1;
具体实现
第一步:自定义Sink:
public class MySQLSink extends AbstractSink implements Configurable {
private Connection conn;
private Statement stmt;
private String columnNames;
private String url;
private String user;
private String password;
private String tableName;
// 在整个sink结束时执行一遍
@Override
public synchronized void stop() {
}
// 在整个sink开始时执行一遍
@Override
public synchronized void start() {
try {
Class.forName("com.mysql.jdbc.Driver");
conn = DriverManager.getConnection(url, user, password);
stmt = conn.createStatement();
} catch (ClassNotFoundException | SQLException e) {
e.printStackTrace();
}
}
// 不断循环调用
@Override
public Status process() throws EventDeliveryException {
Status res = Status.READY;
Channel channel = getChannel();
Transaction tx = channel.getTransaction();
Event event = null;
tx.begin();
while (true) {
event = channel.take();
if (event != null) {
break;
}
}
try {
String body = new String(event.getBody());
if (body.split(",").length == columnNames.split(",").length) {
String sql = "insert into " + tableName + "(" + columnNames + ") values(" + body + ")";
System.out.println("========sql======= " + sql);
stmt.executeUpdate(sql);
tx.commit();
} else {
System.out.println("input error!");
res = Status.BACKOFF;
tx.rollback();
}
} catch (Throwable th) {
tx.rollback();
} finally {
tx.close();
}
return res;
}
@Override
public void configure(Context context) {
columnNames = context.getString("columnNames");
url = context.getString("url");
user = context.getString("user");
password = context.getString("password");
tableName = context.getString("tableName");
}
}
第二步:在flume的job文件夹下自定义配置文件flume-to-mysql.conf:
agent.sources = s1
agent.channels = c1
agent.sinks = sk1
agent.sources.s1.type = netcat
agent.sources.s1.bind = localhost
agent.sources.s1.port = 5678
agent.sources.s1.channels = c1
agent.sinks.sk1.type = com.hc.tomysql.MySQLSink
agent.sinks.sk1.url=jdbc:mysql://hcmaster:3306/db_test?useSSL=false
agent.sinks.sk1.tableName= tb_user
agent.sinks.sk1.user=root
agent.sinks.sk1.password=root
agent.sinks.sk1.columnNames=id, username, password
agent.sinks.sk1.channel = c1
agent.channels.c1.type = memory
agent.channels.c1.capacity = 1000
agent.channels.c1.transactionCapacity = 100
第三步:项目打包并上传到flume的lib目录
第四步:执行代码:
- 启动flume:
- 启动telnet:
- 在telnet中输入数据:
- Flume中查看结果
- 数据库中结果: