1 自定义MysqlHdfsSource
package com.atguigu.custom;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.PollableSource;
import org.apache.flume.channel.ChannelProcessor;
import org.apache.flume.conf.Configurable;
import org.apache.flume.event.SimpleEvent;
import org.apache.flume.source.AbstractSource;
import java.util.ArrayList;
public class MysqlHdfsSource extends AbstractSource implements Configurable, PollableSource {
private Long sleep;
private String content;
@Override
public Status process() throws EventDeliveryException {
Status status = Status.READY;
ArrayList<Event> events = new ArrayList<>();
for (int i = 1; i <= 5; i++) {
SimpleEvent e = new SimpleEvent();
e.setBody((content + "--->第" + i + "个Event").getBytes());
events.add(e);
}
try {
ChannelProcessor channelProcessor = getChannelProcessor();
channelProcessor.processEventBatch(events);
Thread.sleep(sleep);
} catch (InterruptedException e) {
status = Status.BACKOFF;
e.printStackTrace();
}
return status;
}
@Override
public long getBackOffSleepIncrement() {
return 1000;
}
@Override
public long getMaxBackOffSleepInterval() {
return 5000;
}
@Override
public void configure(Context context) {
sleep = context.getLong("sleep");
content = context.getString("content", "无");
}
}
2 自定义MysqlHdfsSink
package com.atguigu.custom;
import org.apache.flume.*;
import org.apache.flume.conf.Configurable;
import org.apache.flume.sink.AbstractSink;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.Date;
public class MysqlHdfsSink extends AbstractSink implements Configurable {
private String mysqlurl = "";
private String username = "";
private String password = "";
private String tableName = "";
private String hdfsPath = "";
private int num = 0;
Connection con = null;
@Override
public Status process() throws EventDeliveryException {
Status status = null;
Channel ch = getChannel();
Transaction txn = ch.getTransaction();
txn.begin();
try
{
Event event = ch.take();
if (event != null)
{
String body = new String(event.getBody(), "UTF-8");
if(body.contains("delete") || body.contains("drop") || body.contains("alert")){
status = Status.BACKOFF;
}else{
Configuration conf = new Configuration();
conf.setBoolean("dfs.support.append",true);
Path filePath = new Path(hdfsPath);
FileSystem hdfs = filePath.getFileSystem(conf);
if (!hdfs.exists(filePath)) {
hdfs.createNewFile(filePath);
}
FSDataOutputStream outputStream = hdfs.append(filePath);
outputStream.write(body.getBytes("UTF-8"));
outputStream.write("\r\n".getBytes("UTF-8"));
outputStream.flush();
outputStream.close();
hdfs.close();
num++;
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String createtime = df.format(new Date());
PreparedStatement stmt = con.prepareStatement("insert into " + tableName + " (createtime, content, number) values (?, ?, ?)");
stmt.setString(1, createtime);
stmt.setString(2, body);
stmt.setInt(3, num);
stmt.execute();
stmt.close();
}
status = Status.READY;
}
else
{
status = Status.BACKOFF;
}
txn.commit();
}
catch (Throwable t)
{
txn.rollback();
t.getCause().printStackTrace();
status = Status.BACKOFF;
}
finally
{
txn.close();
}
return status;
}
@Override
public void configure(Context context) {
mysqlurl = context.getString("mysqlurl");
username = context.getString("username");
password = context.getString("password");
tableName = context.getString("tablename");
hdfsPath = context.getString("hdfspath");
}
@Override
public synchronized void stop()
{
try {
con.close();
} catch (SQLException e) {
e.printStackTrace();
}
super.stop();
}
@Override
public synchronized void start()
{
try
{
con = DriverManager.getConnection(mysqlurl, username, password);
super.start();
System.out.println("finish start");
}
catch (Exception ex)
{
ex.printStackTrace();
}
}
}