Flume 抓取日志文件存入MySQL中
- 代码如下:
- /**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- package org.flume.mysql.sink;
- /**
- * create by yong 2016-6-16
- */
- import com.google.common.base.Preconditions;
- import com.google.common.base.Throwables;
- import com.google.common.collect.Lists;
- import org.apache.flume.*;
- import org.apache.flume.conf.Configurable;
- import org.apache.flume.sink.AbstractSink;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
- import java.sql.Connection;
- import java.sql.DriverManager;
- import java.sql.PreparedStatement;
- import java.sql.SQLException;
- import java.util.List;
- public class MysqlSink extends AbstractSink implements Configurable {
- private Logger LOG = LoggerFactory.getLogger(MysqlSink.class);
- private String hostname;
- private String port;
- private String databaseName;
- private String tableName;
- private String user;
- private String password;
- private PreparedStatement preparedStatement;
- private Connection conn;
- private int batchSize;
- public MysqlSink() {
- LOG.info("MysqlSink start...");
- }
- public void configure(Context context) {
- hostname = context.getString("hostname");
- Preconditions.checkNotNull(hostname, "hostname must be set!!");
- port = context.getString("port");
- Preconditions.checkNotNull(port, "port must be set!!");
- databaseName = context.getString("databaseName");
- Preconditions.checkNotNull(databaseName, "databaseName must be set!!");
- tableName = context.getString("tableName");
- Preconditions.checkNotNull(tableName, "tableName must be set!!");
- user = context.getString("user");
- Preconditions.checkNotNull(user, "user must be set!!");
- password = context.getString("password");
- Preconditions.checkNotNull(password, "password must be set!!");
- batchSize = context.getInteger("batchSize", 100);
- Preconditions.checkNotNull(batchSize > 0, "batchSize must be a positive number!!");
- }
- @Override
- public void start() {
- super.start();
- try {
- //调用Class.forName()方法加载驱动程序
- Class.forName("com.mysql.jdbc.Driver");
- } catch (ClassNotFoundException e) {
- e.printStackTrace();
- }
- String url = "jdbc:mysql://" + hostname + ":" + port + "/" + databaseName;
- //调用DriverManager对象的getConnection()方法,获得一个Connection对象
- try {
- conn = DriverManager.getConnection(url, user, password);
- conn.setAutoCommit(false);
- //创建一个Statement对象
- preparedStatement = conn.prepareStatement("insert into " + tableName +
- " (content) values (?)");
- } catch (SQLException e) {
- e.printStackTrace();
- System.exit(1);
- }
- }
- @Override
- public void stop() {
- super.stop();
- if (preparedStatement != null) {
- try {
- preparedStatement.close();
- } catch (SQLException e) {
- e.printStackTrace();
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException e) {
- e.printStackTrace();
- }
- }
- }
- public Status process() throws EventDeliveryException {
- Status result = Status.READY;
- Channel channel = getChannel();
- Transaction transaction = channel.getTransaction();
- Event event;
- String content;
- List<String> actions = Lists.newArrayList();
- transaction.begin();
- try {
- for (int i = 0; i < batchSize; i++) {
- event = channel.take();
- if (event != null) {
- content = new String(event.getBody());
- actions.add(content);
- } else {
- result = Status.BACKOFF;
- break;
- }
- }
- if (actions.size() > 0) {
- preparedStatement.clearBatch();
- for (String temp : actions) {
- preparedStatement.setString(1, temp);
- preparedStatement.addBatch();
- }
- preparedStatement.executeBatch();
- conn.commit();
- }
- transaction.commit();
- } catch (Throwable e) {
- try {
- transaction.rollback();
- } catch (Exception e2) {
- LOG.error("Exception in rollback. Rollback might not have been" +
- "successful.", e2);
- }
- LOG.error("Failed to commit transaction." +
- "Transaction rolled back.", e);
- Throwables.propagate(e);
- } finally {
- transaction.close();
- }
- return result;
- }
- }
pom依赖如下:
- <?xml version="1.0"?>
- <!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
- <project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-parent</artifactId>
- <version>1.4.0</version>
- </parent>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-mysql-sink</artifactId>
- <version>1.4.0</version>
- <name>flume-mysql-sink</name>
- <url>http://maven.apache.org</url>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-configuration</artifactId>
- </dependency>
- <dependency>
- <groupId>mysql</groupId>
- <artifactId>mysql-connector-java</artifactId>
- <version>5.1.25</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </project>
讲代码打成jar包后,上传到flume安装目录下的lib文件夹中,同时需要上传MySQL的驱动jar包
=====================================================================================================================
给flume添加mysqlSink.conf文件:
agent1.sources = source1
agent1.sinks = mysqlSink
agent1.channels = channel1
# Describe/configure source1
agent1.sources.source1.
type
=
exec
agent1.sources.source1.
command
= tail -F /home/yong/Work/flum-1.6/tail_log_exec
agent1.sources.source1.channels = channel1
# Describe mysqlSink
agent1.sinks.mysqlSink.
type
= org.flume.mysql.sink.MysqlSink
agent1.sinks.mysqlSink.
hostname
=localhost
agent1.sinks.mysqlSink.port=3306
agent1.sinks.mysqlSink.databaseName=sinktest
agent1.sinks.mysqlSink.tableName=mysqltest
agent1.sinks.mysqlSink.user=root
agent1.sinks.mysqlSink.password=root
agent1.sinks.mysqlSink.channel = channel1
# Use a channel which buffers events in memory
agent1.channels.channel1.
type
=
memory
agent1.channels.channel1.capacity = 1000
agent1.channels.channel1.transactionCapactiy = 100
/home/yong/Work/flume-1.6/bin/flume-ng agent -c/home/yong/Work/flume-1.6/conf/ -f /home/yong/Work/flume-1.6/conf/mysqlSink.conf -n agent1 -Dflume.root.logger=INFO,console
============================================================================
运行程序时,先在Mysql中创建一个表:mysql> create table mysqltest(
-> id int(11) NOT NULL AUTO_INCREMENT,
-> content varchar(50000) NOT NULL,
-> PRIMARY KEY (`id`)
-> ) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8;
========================================================================================================================
生成数据到目标日志文件中:
for i in {1..1000};do echo "exec tail$i" >> /home/yong/Work/flume-1.6/tail_log_exec;done;
完成后,数据和预想中的一样,写入了数据库中。