flume + elasticsearch
flume-1.9.0 + elasticsearch-7.7.1 + kibana-7.7.1
不是在采坑就是在采坑的路上！！！
下载地址
flume-ng
http://flume.apache.org/download.html
elasticsearch
https://www.elastic.co/cn/downloads/past-releases#elasticsearch
kibana
https://www.elastic.co/cn/downloads/past-releases/kibana-7-7-1
默认flume支持0.x 1.x的elasticsearch
如使用高版本的elasticsearch 需要修改
The elasticsearch and lucene-core jars required for your environment must be placed in the lib directory of the Apache Flume installation. Elasticsearch requires that the major version of the client JAR match that of the server and that both are running the same minor version of the JVM. SerializationExceptions will appear if this is incorrect. To select the required version first determine the version of elasticsearch and the JVM version the target cluster is running. Then select an elasticsearch client library which matches the major version. A 0.19.x client can talk to a 0.19.x cluster; 0.20.x can talk to 0.20.x and 0.90.x can talk to 0.90.x. Once the elasticsearch version has been determined then read the pom.xml file to determine the correct lucene-core JAR version to use. The Flume agent which is running the ElasticSearchSink should also match the JVM the target cluster is running down to the minor version.
这个是网上找到的一个7.7.0版本的
https://github.com/bitilandu/flume-ng-elasticsearch-sink
配置参考
flume 配置参考 apache-flume-1.9.0/conf/flume-conf.properties
a1.sources = r1
a1.channels = c1
a1.sinks = k1
#a1.sources.r1.type = netcat
a1.sources.r1.type = thrift
a1.sources.r1.bind = 10.10.6.3
a1.sources.r1.port = 60001
a1.sources.r1.threads = 50
a1.sources.r1.channels = c1
#a1.sinks.k1.type = logger
#a1.sinks.k1.channel = c1
a1.sinks.k1.type=org.apache.flume.sink.elasticsearch.ElasticSearchSink
a1.sinks.k1.batchSize=100
a1.sinks.k1.hostNames=127.0.0.1:9200 # flume 默认使用9300 提交使用的http..
a1.sinks.k1.indexName=game_log
a1.sinks.k1.indexType=message
a1.sinks.k1.clusterName=log-es
a1.sinks.k1.serializer=org.apache.flume.sink.elasticsearch.ElasticSearchLogStashEventSerializer
#a1.sinks.k1.serializer = org.apache.flume.sink.elasticsearch.ElasticSearchDynamicSerializer
#a1.sinks.k1.ttl = 5d
a1.sinks.k1.channel = c1
a1.channels.c1.type = memory
a1.channels.c1.capacity = 10000000
a1.channels.c1.transactionCapacity = 100elasticsearch 配置参考 elasticsearch-7.7.1/config/elasticsearch.yml
cluster.name: log-es
node.name: "log-es01"
node.master: true
node.data: true
cluster.initial_master_nodes: ["log-es01"]
# Centos6不支持SecComp
bootstrap.memory_lock: false
bootstrap.system_call_filter: falsekibana 配置参考 kibana-7.7.1-linux-x86_64/config/kibana.yml
pid.file: /tmp/kibana.pid
server.port: 5601
server.host: "10.10.6.3"
elasticsearch.hosts: ["http://localhost:9200"]
kibana.index: ".kibana"
i18n.locale: "zh-CN"##一些报错
给出的建议是要删除index .kibana_2 然后重启 kibana
curl -XDELETE http://localhost:9200/.kibana_1
重启脚本参考
1. elasticsearch.sh 放到 elasticsearch-7.7.1 目录下
#!/bin/bash
#path=$(dirname -- $(readlink -f -- "$0"))
function func_start(){
#cd elasticsearch-7.7.1
./bin/elasticsearch -p /tmp/elasticsearch-pid -d
echo "start success"
}
function func_stop(){
pid=$(cat /tmp/elasticsearch-pid)
echo $pid
kill -SIGTERM $pid
}
function help(){
echo "start | stop"
}
case $1 in
start)
func_start;;
stop)
func_stop;;
*) help ;;
esac2. kibana.sh 放到 kibana-7.7.1-linux-x86_64 目录下
#!/bin/bash
HOME=$(dirname -- $(readlink -f -- "$0"))
LOG=$HOME/logs
BIN=$HOME/bin
PIDFILE=/tmp/kibana.pid
test -d $LOG || mkdir -p $LOG
CONSOLE=$LOG/kibana-console_`date '+%Y-%m-%d'`.log
function func_start(){
echo "starting ..."
nohup $BIN/kibana >$CONSOLE 2>&1 &
echo "start success"
}
function func_stop(){
if [[ -f "$PIDFILE" ]];then
echo "kibana stoping.."
pid=$(cat $PIDFILE)
echo $pid
kill -SIGTERM $pid
echo "stop kibana success"
else
echo "kibana is not running"
exit 0;
fi
}
function func_status(){
if [[ -f "$PIDFILE" ]];then
pid=$(cat $PIDFILE)
if [[ $pid -gt 0 ]]
then
echo "[$(date '+%Y-%m-%d %T')] kibana is running.( pid:$pid )"
else
echo "[$(date '+%Y-%m-%d %T')] kibana is not running"
fi
else
echo "kibana is not running"
fi
}
function func_resart(){
stop
start
}
function help(){
echo "start | stop | status | restart"
}
case $1 in
start)
func_start;;
stop)
func_stop;;
status)
func_status;;
restart)
func_restart;;
*) help ;;
esac3. flume.sh 放到 kibana-7.7.1-linux-x86_64 目录下
#!/bin/bash
#echo "begin start flume..."
#flume的安装根目录（根据自己情况，修改为自己的安装目录）
path=/usr/local/apache-flume-1.9.0
echo "flume home is :$path"
#flume的进程名称，固定值（不用修改）
JAR="flume"
#flume的配置文件名称（根据自己的情况，修改为自己的flume配置文件名称）
Flumeconf="flume-conf.properties"
#定义的soure名称
agentname="a1"
function start(){
echo "begin start flume process ...."
#查找flume运行的进程数
num=`ps -ef|grep java|grep $JAR|wc -l`
#判断是否有flume进程运行，如果有则运行执行nohup命令
if [ "$num" = "0" ] ;then
nohup $path/bin/flume-ng agent --conf $path/conf -f $path/conf/$Flumeconf --name $agentname -Dflume.root.logger=INFO,console &
echo "start success...."
echo "日志路径: $path/logs/flume.log"
else
echo "进程已经存在,启动失败,请检查....."
exit 0
fi
}
function stop(){
echo "begin stop flume process.."
num=`ps -ef|grep java|grep $JAR|wc -l`
#echo "$num...."
if [ "$num" != "0" ];then
#正常停止flume
ps -ef|grep java|grep $JAR|awk '{print $2;}'|xargs kill
echo "进程已经关闭..."
else
echo "服务未启动，无须停止..."
fi
}
function restart(){
#echo "begin stop flume process .."
#执行stop函数
stop
#判断程序是否彻底停止
num='ps -ef|grep java|grep $JAR|wc -l'
#stop完成之后，查找flume的进程数，判断进程数是否为0，如果不为0，则休眠5秒，再次查看，直到进程数为0
while [ $num -gt 0 ];do
sleep 5
num='ps -ef|grep java|grep $JAR|wc -l'
done
echo "flume process stoped,and starting..."
#执行start
start
echo "started...."
}
#case 命令获取输入的参数，如果参数为start,执行start函数，如果参数为stop执行stop函数，如果参数为restart，执行restart函数
case "$1" in
"start")
start
;;
"stop")
stop
;;
"restart")
restart
;;
*)
;;
esac
参考:
kibana：
https://www.cnblogs.com/chenqionghe/p/12503181.html?utm_source=tuicool&utm_medium=referral
flume参考:
https://blog.csdn.net/lijinqi1987/article/details/77449889
elasticsearch参考
https://www.cnblogs.com/zhi-leaf/p/6180492.html
2735
