flume篇4：flume把json数据写入elasticsearch(flume-elasticsearch-sink)
对应非json数据同样适用，可以把非json数据通过拦截器拼接成一个json，然后send出去，这样也是ok的
废话不多说，直接上干货
一、 自定义拦截器：
1 拦截器要求：新建一个新的工程，单独打包，保证每个flume的的拦截器都是单独的一个工程打的包，这样保证每次对拦截器修改的时候不影响其他flume业务
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.7</maven.compiler.source>
<maven.compiler.target>1.7</maven.compiler.target>
<scala.version>2.10.4</scala.version>
<flume.version>1.8.0</flume.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-core</artifactId>
<version>${flume.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>3.3</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.4</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<version>6.1.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-store-sdk</artifactId>
<version>1.5.3</version>
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-core</artifactId>
<version>1.5.3</version>
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-common</artifactId>
<version>1.5.3</version>
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-format</artifactId>
<version>1.5.3</version>
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-hadoop</artifactId>
<version>1.5.3</version>
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-processing</artifactId>
<version>1.5.3</version>
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbonata</artifactId>
<version>1.5.3</version>
<scope>system</scope>
<systemPath>${project.basedir}/lib/apache-carbondata-1.5.3-bin-spark2.3.2-hadoop2.6.0-cdh5.16.1.jar</systemPath>
</dependency>
<dependency>
<groupId>org.apache.mina</groupId>
<artifactId>mina-core</artifactId>
<version>2.0.9</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.9.5</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.sshd</groupId>
<artifactId>sshd-core</artifactId>
<version>0.14.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
<version>0.1.54</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.12</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.47</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.5</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>16.0.1</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>0.11.0.0</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.46</version>
<scope>compile</scope>
</dependency>
</dependencies>
2 拦截器代码如下：
（以下拦截器主要目的是：把一个嵌套2层的body Json中的各个字段取出来，并拼接成一个新的json send 出去）
package com.extracting.flume.interceptor.tc;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.text.SimpleDateFormat;
import java.util.List;
public class XyAccessJsonInterceptorTC implements Interceptor {
private static final Logger logger = LoggerFactory.getLogger(XyAccessJsonInterceptorTC.class);
private SimpleDateFormat dataFormat;
@Override
public void initialize() {
dataFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
}
@Override
public Event intercept(Event event) {
String body = new String(event.getBody());
try {
JSONObject jsonObject = JSON.parseObject(body);
JSONObject bodyObject1 = jsonObject.getJSONObject("body");
JSONObject bodyObject2 = bodyObject1.getJSONObject("body");
JSONObject resObject = new JSONObject();
getPut(resObject, bodyObject2, "id", "new_id");
getPut(resObject, bodyObject2, "name", "new_name");
getPutLong(resObject, bodyObject2, "age", "new_age");
getPutDate(resObject, bodyObject2, "time", "new_time", dataFormat);
logger.info("拦截器最hou输出结果为resObject:" + resObject.toString());
event.setBody(resObject.toString().getBytes());
return event;
} catch (Exception e) {
logger.info("ERROR格式数据" + body.toString());
return null;
}
}
@Override
public List<Event> intercept(List<Event> events) {
List<Event> resultList = Lists.newArrayList();
for (Event event : events) {
Event result = intercept(event);
if (result != null) {
resultList.add(result);
}
}
return resultList;
}
@Override
public void close() {
}
public static class Builder implements Interceptor.Builder {
@Override
public Interceptor build() {
return new XyAccessJsonInterceptorTC();
}
@Override
public void configure(Context context) {
}
}
public static void getPut(JSONObject resObject, JSONObject jsonObject, String oldName,String newName) {
Object value = jsonObject.get(oldName);
if (value !=null){
resObject.put(newName, value.toString());
}
}
public static void getPutDate(JSONObject resObject,JSONObject jsonObject,String oldName,String newName,SimpleDateFormat dataFormat) {
Object value = jsonObject.get(oldName);
if (value !=null){
Long valuelong= Long.parseLong(value.toString());
resObject.put(newName, dataFormat.format(valuelong).toString());
}
}
public static void getPutLong(JSONObject resObject,JSONObject jsonObject,String oldName,String newName) {
Object value = jsonObject.get(oldName);
if (value !=null){
Long valuelong= Long.parseLong(value.toString());
resObject.put(newName, valuelong);
}
}
}
3 打包上传，到flume的lib位置，cdh位置如下：/opt/cloudera/parcels/CDH/lib/flume-ng/lib/
二、自定义flume-elasticsearch-sink
1 flume-elasticsearch-sink目前官网还没有正式提供，主要原因是好像是es的版本迭代过快，es2和es5的api都有所变化，我这里是基于es5的，并且配置了动态索引，每月自动生成新的es索引,这是我自定义的一个通用的flume-elasticsearch-sink包，需要新建一个工程，在此声明，此es-sink是借鉴网上的，由于年代久远我一时半会已经找不到此es-sink的原作者了，如有侵权，我会立马删除的！
2 pom 如下：
<properties>
<elasticsearch-verison>5.4.1</elasticsearch-verison>
<flume-version>1.8.0</flume-version>
<hbase-version>1.2.1</hbase-version>
<log4j-version>2.8.2</log4j-version>
<junit-version>4.10</junit-version>
<avro-version>1.8.2</avro-version>
</properties>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.5.3</version>
<configuration>
<descriptors>
<descriptor>src/main/assembly/assembly.xml</descriptor>
</descriptors>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.47</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>${elasticsearch-verison}</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
<version>${elasticsearch-verison}</version>
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-sdk</artifactId>
<version>${flume-version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-core</artifactId>
<version>${flume-version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>${log4j-version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>${log4j-version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
<version>${junit-version}</version>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>${avro-version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>16.0.1</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.2.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.hbase/hbase-client -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
<version>1.2.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.hbase/hbase-client -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>1.2.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.7.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.7.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.zookeeper/zookeeper -->
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.10</version>
<type>pom</type>
</dependency>
</dependencies>
3 代码部分：代码有点多，给大家看一下整体的目录结构!
接下来大家就开始疯狂复制模式吧：
先是com.cognitree.flume.sink.elasticsearch.client
package com.cognitree.flume.sink.elasticsearch.client;
import com.cognitree.flume.sink.elasticsearch.Util;
import org.apache.flume.Context;
import org.elasticsearch.action.bulk.BackoffPolicy;
import org.elasticsearch.action.bulk.BulkProcessor;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.cognitree.flume.sink.elasticsearch.Constants.*;
public class BulkProcessorBuilder {
private static final Logger logger = LoggerFactory.getLogger(BulkProcessorBuilder.class);
private Integer bulkActions;
private String bulkProcessorName;
private ByteSizeValue bulkSize;
private Integer concurrentRequest;
private TimeValue flushIntervalTime;
private String backoffPolicyTimeInterval;
private Integer backoffPolicyRetries;
public BulkProcessor buildBulkProcessor(Context context, TransportClient client) {
bulkActions = context.getInteger(ES_BULK_ACTIONS,
DEFAULT_ES_BULK_ACTIONS);
bulkProcessorName = context.getString(ES_BULK_PROCESSOR_NAME,
DEFAULT_ES_BULK_PROCESSOR_NAME);
bulkSize = Util.getByteSizeValue(context.getInteger(ES_BULK_SIZE),
context.getString(ES_BULK_SIZE_UNIT));
concurrentRequest = context.getInteger(ES_CONCURRENT_REQUEST,
DEFAULT_ES_CONCURRENT_REQUEST);
flushIntervalTime = Util.getTimeValue(context.getString(ES_FLUSH_INTERVAL_TIME),
DEFAULT_ES_FLUSH_INTERVAL_TIME);
backoffPolicyTimeInterval = context.getString(ES_BACKOFF_POLICY_TIME_INTERVAL,
DEFAULT_ES_BACKOFF_POLICY_START_DELAY);
backoffPolicyRetries = context.getInteger(ES_BACKOFF_POLICY_RETRIES,
DEFAULT_ES_BACKOFF_POLICY_RETRIES);
return build(client);
}
private BulkProcessor build(TransportClient client) {
logger.trace("Bulk processor name: [{}] bulkActions: [{}], bulkSize: [{}], flush interval time: [{}]," +
" concurrent Request: [{}], backoffPolicyTimeInterval: [{}], backoffPolicyRetries: [{}] ",
new Object[]{bulkProcessorName, bulkActions, bulkSize, flushIntervalTime,
concurrentRequest, backoffPolicyTimeInterval, backoffPolicyRetries});
return BulkProcessor.builder(client, getListener())
.setName(bulkProcessorName)
.setBulkActions(bulkActions)
.setBulkSize(bulkSize)
.setFlushInterval(flushIntervalTime)
.setConcurrentRequests(concurrentRequest)
.setBackoffPolicy(BackoffPolicy.exponentialBackoff(
Util.getTimeValue(backoffPolicyTimeInterval,
DEFAULT_ES_BACKOFF_POLICY_START_DELAY),
backoffPolicyRetries))
.build();
}
private BulkProcessor.Listener getListener() {
return new BulkProcessor.Listener() {
public void beforeBulk(long executionId,
BulkRequest request) {
logger.trace("Bulk Execution [" + executionId + "]\n" +
"No of actions " + request.numberOfActions());
}
public void afterBulk(long executionId,
BulkRequest request,
BulkResponse response) {
logger.trace("Bulk execution completed [" + executionId + "]\n" +
"Took (ms): " + response.getTookInMillis() + "\n" +
"Failures: " + response.hasFailures() + "\n" +
"Failures Message: " + response.buildFailureMessage() + "\n" +
"Count: " + response.getItems().length);
}
public void afterBulk(long executionId,
BulkRequest request,
Throwable failure) {
logger.error("Bulk execution failed [" + executionId + "]" +
failure.toString());
}
};
}
}
package com.cognitree.flume.sink.elasticsearch.client;
import com.google.common.base.Throwables;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.List;
import static com.cognitree.flume.sink.elasticsearch.Constants.*;
public class ElasticsearchClientBuilder {
private static final Logger logger = LoggerFactory.getLogger(ElasticsearchClientBuilder.class);
private String clusterName;
private boolean transportSniff;
private boolean ignoreClusterName;
private TimeValue transportPingTimeout;
private TimeValue nodeSamplerInterval;
private List<InetSocketTransportAddress> transportAddresses;
public ElasticsearchClientBuilder(String clusterName, String[] hostnames) {
this.clusterName = clusterName;
setTransportAddresses(hostnames);
logger.debug("@@@@@@@@@@@@@@clusterName为 {}", clusterName);
}
public ElasticsearchClientBuilder setTransportSniff(boolean transportSniff) {
this.transportSniff = transportSniff;
return this;
}
public ElasticsearchClientBuilder setIgnoreClusterName(boolean ignoreClusterName) {
this.ignoreClusterName = ignoreClusterName;
return this;
}
public ElasticsearchClientBuilder setTransportPingTimeout(TimeValue transportPingTimeout) {
this.transportPingTimeout = transportPingTimeout;
return this;
}
public ElasticsearchClientBuilder setNodeSamplerInterval(TimeValue nodeSamplerInterval) {
this.nodeSamplerInterval = nodeSamplerInterval;
return this;
}
public TransportClient build() {
TransportClient client;
logger.trace("Cluster Name: [{}], Transport Sniff: [{}]" +
", Ignore Cluster Name: [{}], Transport Ping TimeOut: [{}], " +
"Node Sampler Interval: [{}], HostName: [{}], Port: [{}] ",
new Object[]{clusterName, transportSniff,
ignoreClusterName, transportPingTimeout,
nodeSamplerInterval, transportAddresses});
Settings settings = Settings.builder()
.put(ES_CLUSTER_NAME,
clusterName)
.put(ES_TRANSPORT_SNIFF,
transportSniff)
.put(ES_IGNORE_CLUSTER_NAME,
ignoreClusterName)
.put(ES_TRANSPORT_PING_TIMEOUT,
transportPingTimeout)
.put(ES_TRANSPORT_NODE_SAMPLER_INTERVAL,
nodeSamplerInterval)
.build();
client = new PreBuiltTransportClient(settings);
for (InetSocketTransportAddress inetSocketTransportAddress : transportAddresses) {
client.addTransportAddress(inetSocketTransportAddress);
}
return client;
}
private void setTransportAddresses(String[] transportAddresses) {
try {
this.transportAddresses = new ArrayList<InetSocketTransportAddress>(transportAddresses.length);
for (String transportAddress : transportAddresses) {
String hostName = transportAddress.split(":")[0];
Integer port = transportAddress.split(":").length > 1 ?
Integer.parseInt(transportAddress.split(":")[1]) : DEFAULT_ES_PORT;
this.transportAddresses.add(new InetSocketTransportAddress(InetAddress.getByName(hostName), port));
}
} catch (Exception e) {
logger.error("Error in creating the InetSocketTransportAddress for elastic search " + e.getMessage(), e);
Throwables.propagate(e);
}
}
}
接下来是com.cognitree.flume.sink.elasticsearch目录
/*
* Copyright 2017 Cognitree Technologies
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.cognitree.flume.sink.elasticsearch;
import com.alibaba.fastjson.JSONObject;
import com.google.common.base.Throwables;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.Decoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.elasticsearch.common.xcontent.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import static com.cognitree.flume.sink.elasticsearch.Constants.ES_AVRO_SCHEMA_FILE;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
/**
* Created by prashant
* <p>
* This Serializer assumes the event body to be in avro binary format
*/
public class AvroSerializer implements Serializer {
private static final Logger logger = LoggerFactory.getLogger(AvroSerializer.class);
private DatumReader<GenericRecord> datumReader;
/**
* Converts the avro binary data to the json format
*/
public XContentBuilder serialize(Event event) {
XContentBuilder builder = null;
try {
if (datumReader != null) {
Decoder decoder = new DecoderFactory().binaryDecoder(event.getBody(), null);
GenericRecord data = datumReader.read(null, decoder);
logger.trace("Record in event " + data);
XContentParser parser = XContentFactory
.xContent(XContentType.JSON)
.createParser(NamedXContentRegistry.EMPTY, data.toString());
builder = jsonBuilder().copyCurrentStructure(parser);
parser.close();
} else {
logger.error("Schema File is not configured");
}
} catch (IOException e) {
logger.error("Exception in parsing avro format data but continuing serialization to process further records",
e.getMessage(), e);
}
return builder;
}
public XContentBuilder serialize2(JSONObject event) {
XContentBuilder builder = null;
try {
if (datumReader != null) {
// logger.trace("Record in event " + data);
XContentParser parser = XContentFactory
.xContent(XContentType.JSON)
.createParser(NamedXContentRegistry.EMPTY, event.toString());
builder = jsonBuilder().copyCurrentStructure(parser);
parser.close();
} else {
logger.error("Schema File is not configured");
}
} catch (IOException e) {
logger.error("Exception in parsing avro format data but continuing serialization to process further records",
e.getMessage(), e);
}
return builder;
}
public void configure(Context context) {
String file = context.getString(ES_AVRO_SCHEMA_FILE);
if (file == null) {
Throwables.propagate(new Exception("Schema file is not configured, " +
"please configure the property " + ES_AVRO_SCHEMA_FILE));
}
try {
Schema schema = new Schema.Parser().parse(new File(file));
datumReader = new GenericDatumReader<GenericRecord>(schema);
} catch (IOException e) {
logger.error("Error in parsing schema file ", e.getMessage(), e);
Throwables.propagate(e);
}
}
}
/*
* Copyright 2017 Cognitree Technologies
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.cognitree.flume.sink.elasticsearch;
/**
* Created by prashant
*/
public class Constants {
public static final String COMMA = ",";
public static final String COLONS = ":";
public static final String PREFIX = "es.";
public static final String INDEX = "index";
public static final String TYPE = "type";
public static final String ID = "id";
public static final String ES_BULK_ACTIONS = "es.bulkActions";
public static final Integer DEFAULT_ES_BULK_ACTIONS = 1000;
public static final String ES_BULK_SIZE = "es.bulkSize";
public static final String ES_BULK_SIZE_UNIT = "es.bulkSize.unit";
public static final Integer DEFAULT_ES_BULK_SIZE = 5;
public static final String ES_BULK_PROCESSOR_NAME = "es.bulkProcessor.name";
public static final String DEFAULT_ES_BULK_PROCESSOR_NAME = "flume";
public static final String ES_CONCURRENT_REQUEST = "es.concurrent.request";
public static final Integer DEFAULT_ES_CONCURRENT_REQUEST = 1;
public static final String ES_FLUSH_INTERVAL_TIME = "es.flush.interval.time";
public static final String DEFAULT_ES_FLUSH_INTERVAL_TIME = "10s";
public static final String DEFAULT_ES_TIME = "5s";
public static final String ES_BACKOFF_POLICY_TIME_INTERVAL = "es.backoff.policy.time.interval";
public static final String DEFAULT_ES_BACKOFF_POLICY_START_DELAY = "50M";
public static final String ES_BACKOFF_POLICY_RETRIES = "es.backoff.policy.retries";
public static final Integer DEFAULT_ES_BACKOFF_POLICY_RETRIES = 8;
public static final String ES_INDEX = "es.index";
public static final String DEFAULT_ES_INDEX = "default";
public static final String ES_TYPE = "es.type";
public static final String DEFAULT_ES_TYPE = "default";
public static final String ES_INDEX_BUILDER = "es.index.builder";
public static final String DEFAULT_ES_INDEX_BUILDER = "com.cognitree.flume.sink.elasticsearch.StaticIndexBuilder";
public static final String ES_SERIALIZER = "es.serializer";
public static final String DEFAULT_ES_SERIALIZER = "com.cognitree.flume.sink.elasticsearch.SimpleSerializer";
// Mandatory Properties
public static final String ES_CLUSTER_NAME = "cluster.name";
public static final String DEFAULT_ES_CLUSTER_NAME = "elasticsearch";
public static final String ES_HOSTS = "es.client.hosts";
public static final Integer DEFAULT_ES_PORT = 9300;
public static final String ES_TRANSPORT_SNIFF = "client.transport.sniff";
public static final String ES_IGNORE_CLUSTER_NAME = "client.transport.ignore_cluster_name";
public static final String ES_TRANSPORT_PING_TIMEOUT = "client.transport.ping_timeout";
public static final String ES_TRANSPORT_NODE_SAMPLER_INTERVAL = "client.transport.nodes_sampler_interval";
public static final String ES_CSV_FIELDS = "es.serializer.csv.fields";
public static final String ES_CSV_DELIMITER = "es.serializer.csv.delimiter";
public static final String DEFAULT_ES_CSV_DELIMITER = ",";
public static final String ES_AVRO_SCHEMA_FILE = "es.serializer.avro.schema.file";
public static final String HBASE_ZK_QUORUM= "hbase.zookeeper.quorum";
public static final String HBASE_TABLE_NAME = "hbase.namespace.table";
public static final String HBASE_CF = "hbase.cf";
public static final String HBASE_ROWKEY= "hbase.rowkey";
public static final String HBASE_FIELDS = "hbase.fields";
public static final String ES_FIELDS = "hbase.fields";
/**
* This enum is used for the time unit
*
* Time unit can be in Second, Minute or Mili second
*/
public enum UnitEnum {
SECOND("s"),
MINUTE("m"),
MILI_SECOND("M"),
UNKNOWN("unknown");
private String unit;
UnitEnum(String unit) {
this.unit = unit;
}
@Override
public String toString() {
return unit;
}
public static UnitEnum fromString(String unit) {
for (UnitEnum unitEnum : UnitEnum.values()) {
if (unitEnum.unit.equals(unit)) {
return unitEnum;
}
}
return UNKNOWN;
}
}
/**
*
* This enum is used for unit of size of data
*
* Unit can be in Mega byte or kilo byte
*/
public enum ByteSizeEnum {
MB("mb"),
KB("kb");
private String byteSizeUnit;
ByteSizeEnum(String byteSizeUnit) {
this.byteSizeUnit = byteSizeUnit;
}
@Override
public String toString() {
return byteSizeUnit;
}
}
/**
* Enum for field type
*
*/
public enum FieldTypeEnum {
STRING("string"),
INT("int"),
FLOAT("float"),
LONG("long"),
BOOLEAN("boolean");
private String fieldType;
FieldTypeEnum(String fieldType) {
this.fieldType = fieldType;
}
@Override
public String toString() {
return fieldType;
}
}
}
package com.cognitree.flume.sink.elasticsearch;
import com.alibaba.fastjson.JSONObject;
import com.google.common.base.Charsets;
import com.google.common.base.Throwables;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import static com.cognitree.flume.sink.elasticsearch.Constants.*;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
public class CsvSerializer implements Serializer {
private static final Logger logger = LoggerFactory.getLogger(CsvSerializer.class);
private final List<String> names = new ArrayList<String>();
private final List<String> types = new ArrayList<String>();
private String delimiter;
/**
*
* Converts the csv data to the json format
*/
public XContentBuilder serialize(Event event) {
XContentBuilder xContentBuilder = null;
String body = new String(event.getBody(), Charsets.UTF_8);
try {
if (!names.isEmpty() && !types.isEmpty()) {
xContentBuilder = jsonBuilder().startObject();
List<String> values = Arrays.asList(body.split(delimiter));
for (int i = 0; i < names.size(); i++) {
Util.addField(xContentBuilder, names.get(i), values.get(i), types.get(i));
}
xContentBuilder.endObject();
} else {
logger.error("Fields for csv files are not configured, " +
"please configured the property " + ES_CSV_FIELDS);
}
} catch (Exception e) {
logger.error("Error in converting the body to the json format " + e.getMessage(), e);
}
return xContentBuilder;
}
public XContentBuilder serialize2(JSONObject event) {
XContentBuilder xContentBuilder = null;
// String body = new String(event.getBody(), Charsets.UTF_8);
try {
if (!names.isEmpty() && !types.isEmpty()) {
xContentBuilder = jsonBuilder().startObject();
List<String> values = Arrays.asList(event.toString().split(delimiter));
for (int i = 0; i < names.size(); i++) {
Util.addField(xContentBuilder, names.get(i), values.get(i), types.get(i));
}
xContentBuilder.endObject();
} else {
logger.error("Fields for csv files are not configured, " +
"please configured the property " + ES_CSV_FIELDS);
}
} catch (Exception e) {
logger.error("Error in converting the body to the json format " + e.getMessage(), e);
}
return xContentBuilder;
}
/**
*
* Returns name and value based on the index
*
*/
private String getValue(String fieldType, Integer index) {
String value = "";
if (fieldType.length() > index) {
value = fieldType.split(COLONS)[index];
}
return value;
}
/**
*
* Configure the field and its type with the custom delimiter
*/
public void configure(Context context) {
String fields = context.getString(ES_CSV_FIELDS);
if(fields == null) {
Throwables.propagate(new Exception("Fields for csv files are not configured," +
" please configured the property " + ES_CSV_FIELDS));
}
try {
delimiter = context.getString(ES_CSV_DELIMITER, DEFAULT_ES_CSV_DELIMITER);
String[] fieldTypes = fields.split(COMMA);
for (String fieldType : fieldTypes) {
names.add(getValue(fieldType, 0));
types.add(getValue(fieldType, 1));
}
} catch(Exception e) {
Throwables.propagate(e);
}
}
}
package com.cognitree.flume.sink.elasticsearch;
import org.apache.flume.Context;
import org.apache.flume.Event;
import java.util.Map;
public class HeaderBasedIndexBuilder extends StaticIndexBuilder
{
public String getIndex(Event event)
{
Map headers = event.getHeaders();
String index;
if (headers.get("index") != null)
index = (String)headers.get("index");
else {
index = super.getIndex(event);
}
return index;
}
public String getType(Event event)
{
Map headers = event.getHeaders();
String type;
if (headers.get("type") != null)
type = (String)headers.get("type");
else {
type = super.getType(event);
}
return type;
}
public String getId(Event event)
{
Map headers = event.getHeaders();
return (String)headers.get("id");
}
public void configure(Context context)
{
super.configure(context);
}
}
package com.cognitree.flume.sink.elasticsearch;
import org.apache.flume.Event;
import org.apache.flume.conf.Configurable;
public interface IndexBuilder extends Configurable
{
public String getIndex(Event paramEvent);
public String getType(Event paramEvent);
public String getId(Event paramEvent);
}
/*
* Copyright 2017 Cognitree Technologies
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.cognitree.flume.sink.elasticsearch;
import com.alibaba.fastjson.JSONObject;
import org.apache.flume.Event;
import org.apache.flume.conf.Configurable;
import org.elasticsearch.common.xcontent.XContentBuilder;
/**
* A serializer to convert the given Flume Event into a json document that will be indexed into Elasticsearch.
* A single instance of the class is created when the Sink initializes and is destroyed when the Sink is stopped.
*
*/
public interface Serializer extends Configurable {
/**
* Serialize the body of the event to
* XContentBuilder format
*/
XContentBuilder serialize(Event event);
}
/*
* Copyright 2017 Cognitree Technologies
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.cognitree.flume.sink.elasticsearch;
import com.google.common.base.Charsets;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.elasticsearch.common.xcontent.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
public class SimpleSerializer implements Serializer {
private static final Logger logger = LoggerFactory.getLogger(SimpleSerializer.class);
public XContentBuilder serialize(Event event) {
XContentBuilder builder = null;
try {
XContentParser parser = XContentFactory
.xContent(XContentType.JSON)
.createParser(NamedXContentRegistry.EMPTY,
new String(event.getBody(), Charsets.UTF_8));
builder = jsonBuilder().copyCurrentStructure(parser);
parser.close();
} catch (Exception e) {
logger.error("Error in Converting the body to json field " + e.getMessage(), e);
}
return builder;
}
public void configure(Context context) {
// No parameters needed from the configurations
}
}
package com.cognitree.flume.sink.elasticsearch;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class StaticIndexBuilder
implements IndexBuilder
{
private static final Logger logger = LoggerFactory.getLogger(StaticIndexBuilder.class);
private String index;
private String type;
public String getIndex(Event event)
{
String index;
if (this.index != null)
index = this.index;
else {
index = "default";
}
return index;
}
public String getType(Event event)
{
String type;
if (this.type != null)
type = this.type;
else {
type = "default";
}
return type;
}
public String getId(Event event)
{
return null;
}
public void configure(Context context)
{
this.index = Util.getContextValue(context, "es.index");
this.type = Util.getContextValue(context, "es.type");
logger.info("Simple Index builder, name [{}] type [{}] ", new Object[] { this.index, this.type });
}
}
/*
* Copyright 2017 Cognitree Technologies
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.cognitree.flume.sink.elasticsearch;
import org.apache.commons.lang.StringUtils;
import org.apache.flume.Context;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import static com.cognitree.flume.sink.elasticsearch.Constants.*;
/**
* Created by prashant
*/
public class Util {
private static final Logger logger = LoggerFactory.getLogger(Util.class);
/**
* Returns TimeValue based on the given interval
* Interval can be in minutes, seconds, mili seconds
*/
public static TimeValue getTimeValue(String interval, String defaultValue) {
TimeValue timeValue = null;
String timeInterval = interval != null ? interval : defaultValue;
logger.trace("Time interval is [{}] ", timeInterval);
if (timeInterval != null) {
Integer time = Integer.valueOf(timeInterval.substring(0, timeInterval.length() - 1));
String unit = timeInterval.substring(timeInterval.length() - 1);
UnitEnum unitEnum = UnitEnum.fromString(unit);
switch (unitEnum) {
case MINUTE:
timeValue = TimeValue.timeValueMinutes(time);
break;
case SECOND:
timeValue = TimeValue.timeValueSeconds(time);
break;
case MILI_SECOND:
timeValue = TimeValue.timeValueMillis(time);
break;
default:
logger.error("Unit is incorrect, please check the Time Value unit: " + unit);
}
}
return timeValue;
}
/**
* Returns ByteSizeValue of the given byteSize and unit
* byteSizeUnit can be in Mega bytes, Kilo Bytes
*/
public static ByteSizeValue getByteSizeValue(Integer byteSize, String unit) {
ByteSizeValue byteSizeValue = new ByteSizeValue(DEFAULT_ES_BULK_SIZE, ByteSizeUnit.MB);
logger.trace("Byte size value is [{}] ", byteSizeValue);
if (byteSize != null) {
ByteSizeEnum byteSizeEnum = ByteSizeEnum.valueOf(unit.toUpperCase());
switch (byteSizeEnum) {
case MB:
byteSizeValue = new ByteSizeValue(byteSize, ByteSizeUnit.MB);
break;
case KB:
byteSizeValue = new ByteSizeValue(byteSize, ByteSizeUnit.KB);
break;
default:
logger.error("Unit is incorrect, please check the Byte Size unit: " + unit);
}
}
return byteSizeValue;
}
/**
* Returns the context value for the contextId
*/
public static String getContextValue(Context context, String contextId) {
String contextValue = null;
if (StringUtils.isNotBlank(context.getString(contextId))) {
contextValue = context.getString(contextId);
}
return contextValue;
}
/**
* Add csv field to the XContentBuilder
*/
public static void addField(XContentBuilder xContentBuilder, String key, String value, String type) throws IOException{
if (type != null) {
FieldTypeEnum fieldTypeEnum = FieldTypeEnum.valueOf(type.toUpperCase());
switch (fieldTypeEnum) {
case STRING:
xContentBuilder.field(key, value);
break;
case FLOAT:
xContentBuilder.field(key, Float.valueOf(value));
break;
case INT:
xContentBuilder.field(key, Integer.parseInt(value));
break;
case BOOLEAN:
xContentBuilder.field(key, Boolean.valueOf(value));
break;
case LONG:
xContentBuilder.field(key, Long.parseLong(value));
break;
default:
logger.error("Type is incorrect, please check type: " + type);
}
}
}
}
package com.cognitree.flume.sink.elasticsearch;
import com.alibaba.fastjson.JSONObject;
import com.cognitree.flume.sink.elasticsearch.client.BulkProcessorBuilder;
import com.cognitree.flume.sink.elasticsearch.client.ElasticsearchClientBuilder;
import com.google.common.base.Charsets;
import com.google.common.base.Strings;
import com.google.common.base.Throwables;
import io.netty.util.internal.StringUtil;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.flume.*;
import org.apache.flume.conf.Configurable;
import org.apache.flume.sink.AbstractSink;
import org.elasticsearch.action.bulk.BulkProcessor;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.text.SimpleDateFormat;
import java.util.Date;
import static com.alibaba.fastjson.JSON.parseObject;
public class XyElasticSearchSink_DATE_UUID extends AbstractSink
implements Configurable
{
private static final Logger logger = LoggerFactory.getLogger(XyElasticSearchSink_DATE_UUID.class);
private BulkProcessor bulkProcessor;
private IndexBuilder indexBuilder;
private Serializer serializer;
private SimpleDateFormat dataFormat;
public void configure(Context context)
{
this.dataFormat = new SimpleDateFormat("yyyyMM");
String[] hosts = getHosts(context);
if (ArrayUtils.isNotEmpty(hosts))
{
TransportClient client = new ElasticsearchClientBuilder(context
.getString("es.cluster.name", "elasticsearch"),
hosts)
.setTransportSniff(context
.getBoolean("es.client.transport.sniff",
Boolean.valueOf(false))
.booleanValue())
.setIgnoreClusterName(context
.getBoolean("es.client.transport.ignore_cluster_name",
Boolean.valueOf(false))
.booleanValue())
.setTransportPingTimeout(Util.getTimeValue(context
.getString("es.client.transport.ping_timeout"),
"5s"))
.setNodeSamplerInterval(Util.getTimeValue(context
.getString("es.client.transport.nodes_sampler_interval"),
"5s"))
.build();
buildIndexBuilder(context);
buildSerializer(context);
this.bulkProcessor = new BulkProcessorBuilder().buildBulkProcessor(context, client);
} else {
logger.error("Could not create transport client, No host exist");
}
}
public Status process() throws EventDeliveryException
{
Date data = new Date();
String dateString = dataFormat.format(data);
Channel channel = getChannel();
Transaction txn = channel.getTransaction();
txn.begin();
try {
Event event = channel.take();
String body;
if (event != null) {
body = new String(event.getBody(), Charsets.UTF_8);
if (!Strings.isNullOrEmpty(body)) {
logger.debug("start to sink event [{}].", body);
String index = this.indexBuilder.getIndex(event)+"_"+dateString;
String type = this.indexBuilder.getType(event);
// JSONObject jsonObjct= parseObject(body);
// String id=jsonObjct.get("uuid").toString();
String id = this.indexBuilder.getId(event);
XContentBuilder xContentBuilder = this.serializer.serialize(event);
if (xContentBuilder != null) {
if (!StringUtil.isNullOrEmpty(id)) {
this.bulkProcessor.add(new IndexRequest(index, type, id)
.source(xContentBuilder));
}
else {
this.bulkProcessor.add(new IndexRequest(index, type)
.source(xContentBuilder));
}
}
else {
logger.error("Could not serialize the event body [{}] for index [{}], type[{}] and id [{}] ", new Object[] { body, index, type, id });
}
}
logger.debug("sink event [{}] successfully.", body);
}
txn.commit();
return Status.READY;
} catch (Throwable tx) {
try {
}
catch (Exception ex) {
logger.error("exception in rollback.", ex);
}
logger.error("transaction rolled back.", tx);
return Status.BACKOFF;
} finally {
txn.close();
}
}
public void stop()
{
if (this.bulkProcessor != null)
this.bulkProcessor.close();
}
private void buildIndexBuilder(Context context)
{
String indexBuilderClass = "com.cognitree.flume.sink.elasticsearch.StaticIndexBuilder";
if (StringUtils.isNotBlank(context.getString("es.index.builder"))) {
indexBuilderClass = context.getString("es.index.builder");
}
this.indexBuilder = ((IndexBuilder)instantiateClass(indexBuilderClass));
if (this.indexBuilder != null)
this.indexBuilder.configure(context);
}
private void buildSerializer(Context context)
{
String serializerClass = "com.cognitree.flume.sink.elasticsearch.SimpleSerializer";
if (StringUtils.isNotEmpty(context.getString("es.serializer"))) {
serializerClass = context.getString("es.serializer");
}
this.serializer = ((Serializer)instantiateClass(serializerClass));
if (this.serializer != null)
this.serializer.configure(context);
}
private <T> T instantiateClass(String className)
{
try
{
Class aClass = Class.forName(className);
return (T) aClass.newInstance();
} catch (Exception e) {
logger.error("Could not instantiate class " + className, e);
Throwables.propagate(e);
}return null;
}
private String[] getHosts(Context context)
{
String[] hosts = null;
if (StringUtils.isNotBlank(context.getString("es.client.hosts"))) {
hosts = context.getString("es.client.hosts").split(",");
}
return hosts;
}
}
3 打包上传flume-elasticserach-sink相关jar包
解释一下这个sink：主要是把json数据按月建立索引加载的es中
上传的jar包有：自定义的flume-elasticserach-sink包
以及pom里面涉及的elasticserach相关jar包：比如transport等，我这里不一一列举了
三、配置flume的conf
1 在 /opt/cloudera/parcels/CDH/lib/flume-ng/conf目录下，
vi elasticserach.conf
输入以下内容：
ng.sources = kafkaSource
ng.sources = kafkaSource
ng.channels = memorychannel
ng.sinks = es_sink
ng.sources.kafkaSource.type= org.apache.flume.source.kafka.KafkaSource
ng.sources.kafkaSource.kafka.bootstrap.servers=cdh01:9092,cdh02:9092,cdh03:9092
ng.sources.kafkaSource.kafka.consumer.group.id=xytest112
ng.sources.kafkaSource.kafka.topics=pd_ry_txjl
ng.sources.kafkaSource.batchSize=1000
ng.sources.kafkaSource.channels= memorychannel
ng.sources.kafkaSource.kafka.consumer.auto.offset.reset=latest
ng.sources.kafkaSource.interceptors= i1
ng.sources.kafkaSource.interceptors.i1.type=com.iflytek.extracting.flume.interceptor.XyAccessJsonInterceptorTC$Builder #自定义的拦截器
ng.channels.memorychannel.type = memory
ng.channels.memorychannel.keep-alive = 3
ng.channels.memorychannel.byteCapacityBufferPercentage = 20
ng.channels.memorychannel.transactionCapacity = 10000
ng.channels.memorychannel.capacity = 100000
ng.sinks.es_sink.type=com.cognitree.flume.sink.elasticsearch.XyElasticSearchSink_DATE_UUID#自定义的es-sink
ng.sinks.es_sink.es.bulkActions=5
ng.sinks.es_sink.es.bulkProcessor.name=bulkprocessor
ng.sinks.es_sink.es.bulkSize=5
ng.sinks.es_sink.es.bulkSize.unit=MB
ng.sinks.es_sink.es.concurrent.request=1
ng.sinks.es_sink.es.flush.interval.time=5m
ng.sinks.es_sink.es.backoff.policy.time.interval=50M
ng.sinks.es_sink.es.backoff.policy.retries=8
ng.sinks.es_sink.es.cluster.name=elasticsearch # es集群名称
ng.sinks.es_sink.es.client.hosts=cdh01:9300 #es ip和端口
ng.sinks.es_sink.es.index=rxjl_new #es索引,实际写入索引rxjl_new_201912,加了按月动态索引
ng.sinks.es_sink.es.type=capture #es type
ng.sinks.es_sink.es.index.builder=com.cognitree.flume.sink.elasticsearch.HeaderBasedIndexBuilder
ng.sinks.es_sink.es.serializer=com.cognitree.flume.sink.elasticsearch.SimpleSerializer
ng.sinks.es_sink.es.serializer.csv.fields=id:string,name:string,age:long,time:string
ng.sinks.es_sink.es.serializer.csv.delimiter=,
ng.sinks.es_sink.es.serializer.avro.schema.file=/usr/local/schme.avsc
ng.sinks.es_sink.channel=memorychannel
2 启动flume:
然后前台启动flume,进行调试:
bin/flume-ng agent -n ng -c conf -f conf/elasticserach.conf
cdh版本的flume默认的日志打在 /var/log/flume/flume.log里面
查看数据已经接入es，并确定没问题可以使用后台提交：
nohup bin/flume-ng agent -n ng -c conf -f conf/elasticserach.conf &
任务停止：
jcmd | grep elasticserach.conf # 找到含有 elasticserach.conf的任务
然后kill 任务id即可
1001
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
