ELK安装部署实践

ELK安装部署实践

环境准备

系统和SDK版本信息

系统版本信息

Linux version 5.4.0-65-generic (buildd@lcy01-amd64-018) (gcc version 9.3.0 (Ubuntu 9.3.0-17ubuntu1~20.04)) #73-Ubuntu SMP Mon Jan 18 17:25:17 UTC 2021

Java版本信息

root@# java -version
java version "1.8.0_161"
Java(TM) SE Runtime Environment (build 1.8.0_161-b12)
Java HotSpot(TM) 64-Bit Server VM (build 25.161-b12, mixed mode)

supervisor版本信息

root@# supervisord -v
4.2.2

Elasticsearch版本信息

root@ntm:/usr/local/supervisor# curl 'http://localhost:9200/?pretty' 
{
  "name" : "sLdEgyf",
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "bzf58qCpTUa-RAaRex8h2g",
  "version" : {
    "number" : "5.6.3",
    "build_hash" : "1a2f265",
    "build_date" : "2017-10-06T20:33:39.012Z",
    "build_snapshot" : false,
    "lucene_version" : "6.6.1"
  },
  "tagline" : "You Know, for Search"
}

Kibana版本信息

通过访问http://localhost:5601查看

防火墙配置

systemctl disable ufw
iptables -F

kibana配置

vim kibana/config/kibana.yml
server.host: "0.0.0.0"

SDK下载

百度网盘下载链接参考下面文章链接：

链接：https://download.csdn.net/download/Markwangyu/86500744

Linux下载链接

wget https://artifacts.elastic.co/downloads/logstash/logstash-5.6.3.zip

SDK部署

supervisor部署脚本

#! /bin/bash

echo -e ">> install supervisor\n"

if [ -f "/usr/local/supervisor/installed.log" ]; then
    echo -e "\033[31m  supervisor already exist, cannot install again, you need uninstall first and then install it if you want to reinstall it \033[0m \n"
else
    echo -e "\n"
    echo -e ">> start supervisord"
	if [ ! -d /var/log/ ];then
		mkdir /var/log/
	fi
    \cp -rf supervisor/usr/local/supervisor /usr/local/
    \cp -rf supervisor/usr/local/bin/*  /usr/local/bin/
    chmod +x /usr/local/bin/*
    \cp -rf supervisor/usr/lib/systemd/system/supervisord.service /etc/systemd/system/supervisord.service
    systemctl enable supervisord
    echo -e "\n"
fi

echo -e "\n"

JAVA部署脚本

#! /bin/bash

echo -e ">> install java\n"

if [ -f "/usr/local/java/java.log" ]; then
    echo -e "\033[31m  java already exist, cannot install again, you need uninstall first and then install it if you want to reinstall it \033[0m \n"
else
    echo -e "\n"
    echo -e ">> start java"
	if [ ! -d /var/log/java/ ];then
		mkdir /var/log/java/
	fi
    \cp -rf java/ /usr/local/
	echo -e "export JAVA_HOME=/usr/local/java/jdk1.8.0_161 " >> ~/.bashrc 
	echo  "export JRE_HOME=\${JAVA_HOME}/jre " >> ~/.bashrc 
	echo -e "export CLASSPATH=.:\${JAVA_HOME}/lib:\${JRE_HOME}/lib " >> ~/.bashrc 
	echo -e "export PATH=\${JAVA_HOME}/bin:\$PATH " >> ~/.bashrc 
	source ~/.bashrc
    echo -e "\n"
fi

echo -e "\n"

Elasticsearch部署脚本

#    systemctl stop ba
#! /bin/bash


echo -e ">> install elasticsearch\n"

if [ -f "/var/log/elasticsearch.log" ]; then
    echo -e "\033[31m  java already exist, cannot install again, you need uninstall first and then install it if you want to reinstall it \033[0m \n"
else
    echo -e "\n"
    echo -e ">> start elasticsearch"
        if [ ! -d /var/log/elasticsearch.log ];then
                touch /var/log/elasticsearch.log
        fi
    \cp -rf elasticsearch-5.6.3/ /usr/local/
        useradd esuser --password 123 -s /bin/bash
        chown -R esuser /usr/local/elasticsearch-5.6.3/
        chmod 755 /usr/local/elasticsearch-5.6.3/bin/elasticsearch
        echo -e "vm.max_map_count=262144" >> /etc/sysctl.conf
        sysctl -p
    echo -e "\n"
fi
echo -e "\n"

Kibana 部署脚本

#! /bin/bash

echo -e ">> install kibana\n"

if [ -f "/usr/local/kibana.log" ]; then
    echo -e "\033[31m  java already exist, cannot install again, you need uninstall first and then install it if you want to reinstall it \033[0m \n"
else
    echo -e "\n"
    echo -e ">> start kibana"
	if [ ! -d /var/log/kibana.log ];then
		touch /var/log/kibana.log
	fi
	\tar -xzvf kibana-5.6.3/kibana-5.6.3.tar.gz kibana-5.6.3/
    \cp -rf kibana-5.6.3/ /usr/local/
	\rm -rf /usr/local/kibana-5.6.3.tar.gz
	chmod 755 /usr/local/kibana-5.6.3/bin/kibana
	chmod 755 /usr/local/kibana-5.6.3/bin/kibana-plugin
    echo -e "\n"
fi
echo -e "\n"

elk部署脚本

#! /bin/bash

###############################################################
### start to install
###############################################################


echo -e ">> elk start to install "
echo -e ">> start to install elk ...\n"

./install_java.sh
./install_elasticsearch.sh
./install_kibana.sh

 \cp -rf elk/supervisor/conf.d/elasticsearch.ini /usr/local/supervisor/conf.d/
 \cp -rf elk/supervisor/conf.d/kibana.ini /usr/local/supervisor/conf.d/
 \cp -rf elk/supervisor/supervisord.conf /usr/local/supervisor/
 systemctl restart supervisord 


echo -e "\n"


echo -e "|---------------------------------|"
echo -e "|                                 |"
echo -e "| successfully! installing end!   |"
echo -e "|                                 |"
echo -e "|---------------------------------|\n\n"

数据接入

Python版本数据接入方法

class FileOperateUtil:

    @staticmethod
    def init_config_info(path, config_info):
        if not os.path.exists(path):  # 保证存在
            with open(path, "w", encoding="utf8") as f:
                f.write(json.dumps(config_info))

        if os.path.exists(path):  
            with open(path, "r", encoding="utf8") as f:
                config_info = f.read()
            config_info = json.loads(config_info)

        return config_info
        
def get_es_config():
    path = "/mnt/es_config.txt"
    config_info = get_es_host()
    es_config = FileOperateUtil().init_config_info(path, config_info)
    return es_config
    
URL = get_es_config()
es = Elasticsearch(URL)


class EsManage:
    def __init__(self, index_name, log_type):
        self.index_name = index_name
        self.log_type = log_type

    def creat_es_index(self, index_map_file):
        rtn = es.indices.exists(index=self.index_name)
        if rtn:
            msg = "200：索引已创建"
        else:
            try:
                with open(f"./{index_map_file}", "r", encoding="utf-8") as file_object:
                    mapping = json.load(file_object)
                es.indices.create(index=self.index_name, body=mapping)
                msg = "201：索引创建成功"
            except Exception as e:
                logger.error(e)
                msg = "400:创建索引失败"
        return msg

    def post_data_to_es(self, data):
        post_res = ""
        if isinstance(data, dict):
            post_res = es.index(index=self.index_name, doc_type=self.log_type, body=data)
        return post_res

    def bulk_data_to_es(self, alarm_data):
        msg = "400：批量插入数据失败"
        if isinstance(alarm_data, list):
            action = [{
                "_index": self.index_name,
                "_type": self.log_type,
                "_source": item
            } for item in alarm_data]
            helpers.bulk(es, action)
            msg = "200：批量插入数据成功"
        return msg