kafka sink mysql,kafka之七 sinkTask详解

使用kafka connector 功能实现一个数据从kafka到MySQL的sinkTask

一：实现JdbcSinkConnector类

public class JdbcSinkConnector extends SinkConnector{

private String url;

private String driven;

private String userName;

private String passwd;

public void start(Map props) {

this.url = PropertyVerify.getOrElse(props, Constant_Global.URL, "jdbc:mysql://localhost/test", "'URL' is null");

this.driven = PropertyVerify.getOrElse(props, Constant_Global.DRIVEN, "com.mysql.jdbc.Driver", "'DRIVEN' is null");

this.userName = PropertyVerify.getOrElse(props, Constant_Global.USERNAME, "root", "'USERNAME' is null");

this.passwd = PropertyVerify.getOrElse(props, Constant_Global.PASSED, "root", "'PASSED' is null");

}

public Class extends Task> taskClass() {

return JdbcSinkTask.class;

}

public List> taskConfigs(int maxTasks) {

ArrayList> configs = new ArrayList<>();

for(int i=0;i

Map conf = new HashMap();

conf.put("url", url);

conf.put("driven", driven);

conf.put("userName", userName);

conf.put("passwd", passwd);

configs.add(conf);

}

return configs;

}

@Override

public String version() {

return AppInfoParser.getVersion();

}

@Override

public void stop() {

// TODO Auto-generated method stub

}

}二：实现JdbcSinkConnector类

public class JdbcSinkTask extends SinkTask{

private static final Logger LOG = LoggerFactory.getLogger(JdbcSinkTask.class);

//private Connection conn = null;

public String shcema;

private JdbcDbWriter writer;

@Override

public String version() {

return new JdbcSinkConnector().version();

}

@Override

public void flush(Map map) {

LOG.info("================flush Map start................===========================================================");

}

@Override

public void put(Collection sinkRecords) {

if(sinkRecords.isEmpty()){

return;

}

try {

writer.write(sinkRecords,shcema,email);

} catch (SQLException | IOException e) {

try {

EmailUtil.init(Constant_Global.STMP, Constant_Global.EMAILUSER, Constant_Global.EMAILPASSWD, Constant_Global.EMAILTITAL, Constant_Global.EMAILADREE, email);

EmailUtil.send(" kafka sink 数据写入有问题 ");

} catch (MessagingException e1) {

e1.printStackTrace();

}

throw new JDBCConntorException("数据写入有问题");

}

}

@Override

public void start(Map pro) {

try {

DbPool.init(pro);

writer =new JdbcDbWriter();

} catch (PropertyVetoException e1) {

e1.printStackTrace();

LOG.info("数据库配置异常=====");

}

}

@Override

public void stop() {

}

}三 ：打包运行

3.1 单机版运行，配置文件在kafka/config目录下

a: cp   connect-file-sink.properties    connect-jdbc-sink.properties

b: vim connect-jdbc-sink.properties  配置如下

# kafka connector properties

name=canal-sink-connector #定义task名称

connector.class=com.trcloud.hamal.sink.jdbc.JdbcSinkConnector #定义自己打包中的类

tasks.max=1 #task个数

topics=words-out1 #消费的topic

url=jdbc:mysql://172.30.50.213/test #数据库参数

driven=com.mysql.jdbc.Driver #数据库驱动

userName=root #数据库用户名

passwd=123456 #数据库密码        c: 启动命令

./connect-standalone.sh   ../config/connect-standalone.properties  ../config/connect-jdbc-sink.properties

3.2 单机版用于测试，生产环境建议使用分布式

a: 配置文件 vim  jdbc-sink-distributed.properties

bootstrap.servers=node1:6667,node2:6667,node3:6667

group.id=test-consumer-group

key.converter=org.apache.kafka.connect.storage.StringConverter

value.converter=org.apache.kafka.connect.json.JsonConverter

key.converter.schemas.enable=false

value.converter.schemas.enable=false

internal.key.converter=org.apache.kafka.connect.json.JsonConverter

internal.value.converter=org.apache.kafka.connect.json.JsonConverter

internal.key.converter.schemas.enable=false

internal.value.converter.schemas.enable=false

offset.storage.topic=connect-offsets

offset.flush.interval.ms=10000

config.storage.topic=configs-topic

status.storage.topic=connect-status     3.2 启动命令 用rest接口启动

curl -X POST /connectors HTTP/1.1

Host: kafka.test.nd1

Content-Type: application/json

Accept: application/json

{

"name": "local-dw-sink",

"config": {

"connector.class":"com.trcloud.hamal.sink.jdbc.JdbcSinkConnector",

"tasks.max":"1",

"topics":"sql-log" ,

"url":"jdbc:mysql://node4:3306/DW",

"driven":"com.mysql.jdbc.Driver",

"userName":"root",

"passwd":"1234"

}

}