一、ELK简介
ELK是三个软件的统称,即Elasticsearch、Logstash和Kibana三个开源软件的缩写。这三款软件都是开源软件,通常配合使用,并且都先后归于Elastic.co企业名下,故被简称为ELK协议栈。ELK主要用于部署在企业架构中,收集多台设备上多个服务的日志信息,并将其统一整合后提供给用户。
二、ELK架构
在ELK架构中,Elasticsearch、Logstash和Kibana三款软件作用如下:
1、Elasticsearch
Elasticsearch是一个高度可扩展的全文搜索和分析引擎,基于Apache Lucence(事实上,Lucence也是百度所采用的搜索引擎)构建,能够对大容量的数据进行接近实时的存储、搜索和分析操作。
2、Logstash
Logstash是一个数据收集引擎,它可以动态的从各种数据源搜集数据,并对数据进行过滤、分析和统一格式等操作,并将输出结果存储到指定位置上。Logstash支持普通的日志文件和自定义Json格式的日志解析。
3、Kibana
Kibana是一个数据分析和可视化平台,通常与Elasticsearch配合使用,用于对其中的数据进行搜索、分析。
一、ELK简介
ELK是三个软件的统称,即Elasticsearch、Logstash和Kibana三个开源软件的缩写。这三款软件都是开源软件,通常配合使用,并且都先后归于Elastic.co企业名下,故被简称为ELK协议栈。ELK主要用于部署在企业架构中,收集多台设备上多个服务的日志信息,并将其统一整合后提供给用户。
ELK-Logstash(部署脚本)
#!/bin/bash
#####root判断#####
if
[ "$USER" != "root" ]
then
echo "错误:非root用户,权限不足!"
exit 0
fi
############防火墙与高级权限##########
systemctl stop firewalld && systemctl disable firewalld && echo "防火墙已经关闭"
sed -i 's/SELINUX=*/SELINUX=disabled/g' /etc/selinux/config && echo "关闭selinux"
###############检查网络连通是否正常############
ping -c 3 www.baidu.com
if
[ $? -eq 0 ]
then
echo -e "\n\033[32m-----------------------------------------------\033[0m"
echo -e "\033[32m网络畅通,即将安装服务\033[0m"
else
echo -e "\n\033[32m-----------------------------------------------\033[0m"
echo -e "\033[32m即将退出,请检查外网通讯 !\033[0m"
exit 1
fi
##############安装http#########
yum -y install httpd
if
[ $? = 0 ]
then
echo "http安装成功"
else
echo "http安装失败"
exit 1
fi
############启动服务############
systemctl start httpd
if
[ $? = 0 ]
then
echo "http服务已开启"
else
echo "http服务开启失败"
exit 1
fi
#############安装logstash##########
L="/opt/logstash-6.6.1.rpm"
if
[ ! -e $L ]
then
echo "安装包不存在,请上传安装文件到/opt/"
exit 1
fi
rpm -ivh /opt/logstash-6.6.1.rpm
chmod o+r /var/log/messages
##########在Apache主机上做对接配置#########
read -ep "请输入ELK-node1的IP:" ELK1
cat >> /etc/logstash/conf.d/system.conf << EOF
input {
file{
path => "/var/log/messages"
type => "system"
start_position => "beginning"
}
}
output {
elasticsearch {
hosts => ["$ELK1:9200"]
index => "system-%{+YYYY.MM.dd}"
}
}
EOF
###########启动logstash服务########
systemctl start logstash.service && systemctl enable logstash.service
if
[ $? = 0 ]
then
echo "logstash服务已开启"
else
echo "logstash服务开启失败"
exit 1
fi
###########测试##############
echo "Ctrl+c退出"
logstash -e 'input { stdin{} } output { stdout{} }'
sleep 2
logstash -e 'input { stdin{} } output { stdout{ codec=>rubydebug } }'
sleep 2
logstash -e 'input { stdin{} } output { elasticsearch { hosts=>["$ELK1:9200"] } }'
########对接Apache主机的Apache 日志文件(访问日志、错误日志)##########
cat >> /etc/logstash/conf.d/apache_log.conf << EOF
input {
file{
path => "/etc/httpd/logs/access_log"
type => "access"
start_position => "beginning"
}
file{
path => "/etc/httpd/logs/error_log"
type => "error"
start_position => "beginning"
}
}
output {
if [type] == "access" {
elasticsearch {
hosts => ["$ELK1:9200"]
index => "apache_access-%{+YYYY.MM.dd}"
}
}
if [type] == "error" {
elasticsearch {
hosts => ["$ELK1:9200"]
index => "apache_error-%{+YYYY.MM.dd}"
}
}
}
EOF
#################
cd /etc/logstash/conf.d/
logstash -f apache_log.conf
echo "logstash已安装完成"
ELK-node1(部署脚本)
#!/bin/bash
#####root判断#####
if
[ "$USER" != "root" ]
then
echo "错误:非root用户,权限不足!"
exit 0
fi
############防火墙与高级权限##########
systemctl stop firewalld && systemctl disable firewalld && echo "防火墙已经关闭"
sed -i 's/SELINUX=*/SELINUX=disabled/g' /etc/selinux/config && echo "关闭selinux"
###############检查网络连通是否正常############
ping -c 3 www.baidu.com
if
[ $? -eq 0 ]
then
echo -e "\n\033[32m-----------------------------------------------\033[0m"
echo -e "\033[32m网络畅通,即将安装服务\033[0m"
else
echo -e "\n\033[32m-----------------------------------------------\033[0m"
echo -e "\033[32m即将退出,请检查外网通讯 !\033[0m"
exit 1
fi
#############修改主机名##############
echo "请用hostnamectl set-hostname node1修改主机名"
echo "请用su切换shell生效主机名"
sleep 5
IP=$(hostname -I | awk '{print $1}' | awk -F '.' '{print $1"."$2"."$3"."$4}')
read -ep "请输入ELK-node2IP:" node2IP
cat >> /etc/hosts << EOF
$IP node1
$node2IP node2
EOF
yum -y install java
############安装elasticsearch##########
lcn="/opt/elasticsearch-6.6.2.rpm"
if
[ ! -e $lcn ]
then
echo "安装包不存在,请上传安装文件到/opt/"
exit 1
fi
rpm -ivh /opt/elasticsearch-6.6.2.rpm
sleep 3
###############创建数据存放路径并授权##########
mkdir -p /data/elk_data
chown elasticsearch:elasticsearch /data/elk_data/
cat >> /etc/security/limits.conf << EOF
* soft nofile 65536
* hard nofile 65536
EOF
##############修改elasticsearch主配置文件###############
cp /etc/elasticsearch/elasticsearch.yml /etc/elasticsearch/elasticsearch.yml.bak
cat > /etc/elasticsearch/elasticsearch.yml << EOF
cluster.name: my-application
node.name: node1
path.data: /data/elk_data
path.logs: /var/log/elasticsearch
bootstrap.memory_lock: false
network.host: 0.0.0.0
http.port: 9200
discovery.zen.ping.unicast.hosts: ["node1", "node2"]
EOF
##############编译安装node组件依赖包#############
cd /usr/local/src/
if
[ $? = 0 ]
then
echo "node依赖包安装完成"
else
echo "node依赖包安装失败"
exit 1
fi
###################下载node-v8.2.1包########################
wget https://npm.taobao.org/mirrors/node/v8.2.1/node-v8.2.1-linux-x64.tar.gz
if
[ $? = 0 ]
then
echo "node-v8.2.1包下载完成"
else
echo "node-v8.2.1包下载失败"
exit 1
fi
tar zxf node-v8.2.1-linux-x64.tar.gz
mv node-v8.2.1-linux-x64 node-v8.2.1
cat >> /etc/profile << EOH
export NODE_HOME=/usr/local/src/node-v8.2.1
export PATH=\$PATH:\$NODE_HOME/bin
export NODE_PATH=\$NODE_HOME/lib/node_modules
EOH
source /etc/profile
source /etc/profile
##############安装phantomjs############
nihao="/opt/phantomjs-2.1.1-linux-x86_64.tar.bz2"
if
[ ! -e $nihao ]
then
echo "安装包不存在,请上传安装文件到/opt/"
exit 1
fi
tar jxf /opt/phantomjs-2.1.1-linux-x86_64.tar.bz2 -C /usr/local/src/
cp /usr/local/src/phantomjs-2.1.1-linux-x86_64/bin/phantomjs /usr/local/bin/
##############安装elasticsearch-head(数据可视化工具)###########
head="/opt/elasticsearch-head-master.tar.gz"
if
[ ! -e $head ]
then
echo "安装包不存在,请上传安装文件到/opt/"
exit 1
fi
tar zxf /opt/elasticsearch-head-master.tar.gz -C /usr/local/src/
cd /usr/local/src/elasticsearch-head-master/
npm install
#############修改主配置文件########
cat >> /etc/elasticsearch/elasticsearch.yml << EOF
http.cors.enabled: true
http.cors.allow-origin: "*"
EOF
#############启动elasticsearch服务###########
systemctl daemon-reload && systemctl start elasticsearch && systemctl enable elasticsearch.service
if
[ $? = 0 ]
then
echo "elasticsearch服务已开启"
else
echo "elasticsearch服务开启失败"
exit 1
fi
sleep 3
###############启动elasticsearch-head-master############
cd /usr/local/src/elasticsearch-head-master/
npm run start &
sleep 3
if
[ $? = 0 ]
then
echo "elasticsearch-head已开启"
else
echo "elasticsearch-head开启失败"
exit 1
fi
#########安装kibana###########
ki="/opt/kibana-6.6.1-x86_64.rpm"
if
[ ! -e $ki ]
then
echo "安装包不存在,请上传安装文件到/opt/,上传完成再重新运行该脚本"
exit 1
fi
rpm -ivh /opt/kibana-6.6.1-x86_64.rpm
#############修改配置文件#############
cp /etc/kibana/kibana.yml /etc/kibana/kibana.yml.bak
cat >> /etc/kibana/kibana.yml << EOH
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.hosts: ["http://$IP:9200"]
kibana.index: ".kibana"
EOH
###########启动kibana服务#########
systemctl start kibana.service && systemctl enable kibana.service
if
[ $? = 0 ]
then
echo "kibana已开启"
else
echo "kibana开启失败"
exit 1
fi
#############################################################################
echo "ELK-node1已安装完成,请登录http://$IP:9100访问elasticsearch!"
echo "请登录http://$IP:5601查看管理日志!"
ELK-node2(部署脚本)
#!/bin/bash
#####root判断#####
if
[ "$USER" != "root" ]
then
echo "错误:非root用户,权限不足!"
exit 0
fi
############防火墙与高级权限##########
systemctl stop firewalld && systemctl disable firewalld && echo "防火墙已经关闭"
sed -i 's/SELINUX=*/SELINUX=disabled/g' /etc/selinux/config && echo "关闭selinux"
###############检查网络连通是否正常############
ping -c 3 www.baidu.com
if
[ $? -eq 0 ]
then
echo -e "\n\033[32m-----------------------------------------------\033[0m"
echo -e "\033[32m网络畅通,即将安装服务\033[0m"
else
echo -e "\n\033[32m-----------------------------------------------\033[0m"
echo -e "\033[32m即将退出,请检查外网通讯 !\033[0m"
exit 1
fi
#############修改主机名##############
echo "请用hostnamectl set-hostname node2修改主机名"
echo "请用su切换shell生效主机名"
sleep 5
IP=$(hostname -I | awk '{print $1}' | awk -F '.' '{print $1"."$2"."$3"."$4}')
read -ep "请输入ELK-node1IP:" node1IP
cat >> /etc/hosts << EOF
$node1IP node1
$IP node2
EOF
yum -y install java
############安装elasticsearch##########
e="/opt/elasticsearch-6.6.2.rpm"
if
[ ! -e $e ]
then
echo "安装包不存在,请上传安装文件到/opt/"
exit 1
fi
rpm -ivh /opt/elasticsearch-6.6.2.rpm
sleep 2
###############创建数据存放路径并授权##########
mkdir -p /data/elk_data
chown elasticsearch:elasticsearch /data/elk_data/
cat >> /etc/security/limits.conf << EOF
* soft nofile 65536
* hard nofile 65536
EOF
##############修改elasticsearch主配置文件###############
cp /etc/elasticsearch/elasticsearch.yml /etc/elasticsearch/elasticsearch.yml.bak
cat > /etc/elasticsearch/elasticsearch.yml << EOF
cluster.name: my-application
node.name: node2
path.data: /data/elk_data
path.logs: /var/log/elasticsearch
bootstrap.memory_lock: false
network.host: 0.0.0.0
http.port: 9200
discovery.zen.ping.unicast.hosts: ["node1", "node2"]
EOF
##############编译安装node组件依赖包#############
cd /usr/local/src/
if
[ $? = 0 ]
then
echo "依赖包安装完成"
else
echo "依赖包安装失败"
exit 1
fi
#############################node-v8.2.1#############################
wget https://npm.taobao.org/mirrors/node/v8.2.1/node-v8.2.1-linux-x64.tar.gz
if
[ $? = 0 ]
then
echo "node-v8.2.1包下载完成"
else
echo "node-v8.2.1包下载失败"
exit 1
fi
tar zxf node-v8.2.1-linux-x64.tar.gz
mv node-v8.2.1-linux-x64 node-v8.2.1
cat >> /etc/profile << EOH
export NODE_HOME=/usr/local/src/node-v8.2.1
export PATH=\$PATH:\$NODE_HOME/bin
export NODE_PATH=\$NODE_HOME/lib/node_modules
EOH
source /etc/profile
source /etc/profile
##############安装phantomjs(前端框架)############
p="/opt/phantomjs-2.1.1-linux-x86_64.tar.bz2"
if
[ ! -e $p ]
then
echo "安装包不存在,请上传安装文件到/opt/"
exit 1
fi
tar jxf /opt/phantomjs-2.1.1-linux-x86_64.tar.bz2 -C /usr/local/src/
cp /usr/local/src/phantomjs-2.1.1-linux-x86_64/bin/phantomjs /usr/local/bin/
##############安装elasticsearch-head(数据可视化工具)###########
head="/opt/elasticsearch-head-master.tar.gz"
if
[ ! -e $head ]
then
echo "安装包不存在,请上传安装文件到/opt/"
exit 1
fi
tar zxf /opt/elasticsearch-head-master.tar.gz -C /usr/local/src/
cd /usr/local/src/elasticsearch-head-master/
npm install
#############修改主配置文件########
cat >> /etc/elasticsearch/elasticsearch.yml << EOF
http.cors.enabled: true
http.cors.allow-origin: "*"
EOF
#############启动elasticsearch服务###########
systemctl daemon-reload && systemctl start elasticsearch && systemctl enable elasticsearch.service
if
[ $? = 0 ]
then
echo "elasticsearch服务已开启"
else
echo "elasticsearch服务开启失败"
exit 1
fi
sleep 3
###############启动elasticsearch-head-master############
cd /usr/local/src/elasticsearch-head-master/
npm run start &
sleep 3
if
[ $? = 0 ]
then
echo "elasticsearch-head已开启"
else
echo "elasticsearch-head开启失败"
exit 1
fi
#################################################################
echo "ELK-node2已安装完成,请登录http://$IP:9100访问elasticsearch!"