源码部署ELK

畅云客

已于 2024-05-22 09:01:16 修改

阅读量904

点赞数 11

分类专栏： ELK 文章标签： elk jenkins 运维云计算 es java 数据库

于 2024-05-22 08:59:56 首次发布

本文链接：https://blog.csdn.net/qq_33906471/article/details/139106981

版权

ELK 专栏收录该内容

3 篇文章 0 订阅

订阅专栏

ELK架构是最经典的一个日志收集平台，本文详细讲述了ELK的部署方式，其中包括单机es，集群es，单机kibana，集群kibana的部署流程。本文中涉及到的软件包如果有需要可以评论区找我要，无偿提供。

资源列表

操作系统	配置	主机名	IP
CentOS7.3.1611	2C4G	es01	192.168.207.131
CentOS7.3.1611	2C4G	kibana	192.168.207.165
CentOS7.3.1611	2C4G	logstash	192.168.207.166

基础环境

关闭防护墙

systemctl stop firewalld
systemctl disable firewalld

关闭内核安全机制

sed -i "s/.*SELINUX=.*/SELINUX=disabled/g" /etc/selinux/config
reboot

修改主机名

hostnamectl set-hostname es01
hostnamectl set-hostname kibana
hostnamectl set-hostname logstash

添加hosts映射

cat >> /etc/hosts << EOF
192.168.207.131 es01
192.168.207.165 kibana
192.168.207.166 logstash
EOF

一、部署elasticsearch

修改limit限制

cat > /etc/security/limits.d/es.conf << EOF
* soft nproc 655360
* hard nproc 655360
* soft nofile 655360
* hard nofile 655360
EOF

cat >> /etc/sysctl.conf << EOF
vm.max_map_count=655360
EOF
sysctl -p

部署elasticsearch

mkdir -p /data/elasticsearch
tar zxvf elasticsearch-7.14.0-linux-x86_64.tar.gz -C /data/elasticsearch

修改配置文件

单节点

mkdir /data/elasticsearch/{data,logs}

[root@es01 elasticsearch-7.14.0]# grep -v "^#" /data/elasticsearch/elasticsearch-7.14.0/config/elasticsearch.yml
cluster.name: my-application
node.name: es01
path.data: /data/elasticsearch/data
path.logs: /data/elasticsearch/logs
bootstrap.memory_lock: false
network.host: 0.0.0.0
http.port: 9200
cluster.initial_master_nodes: ["es01"]

集群(3台节点集群为例)

需要准备3台机器，主机名分别是es01，es02，es03

[root@es01 ~]# grep -v "^#" /data/elasticsearch/elasticsearch-7.14.0/config/elasticsearch.yml
cluster.name: es
node.name: es01
path.data: /data/elasticsearch/data
path.logs: /data/elasticsearch/logs
bootstrap.memory_lock: false
network.host: 0.0.0.0
http.port: 9200
discovery.seed_hosts: ["es01","es02","es03"]
cluster.initial_master_nodes: ["es01","es02","es03"]
node.master: true
node.data: true
http.cors.enabled: true
http.cors.allow-origin: "*"

[root@es02 ~]# grep -v "^#" /data/elasticsearch/elasticsearch-7.14.0/config/elasticsearch.yml
cluster.name: es
node.name: es02
path.data: /data/elasticsearch/data
path.logs: /data/elasticsearch/logs
bootstrap.memory_lock: false
network.host: 0.0.0.0
http.port: 9200
discovery.seed_hosts: ["es01","es02","es03"]
cluster.initial_master_nodes: ["es02", "es01", "es03"]
node.master: true
node.data: true
http.cors.enabled: true
http.cors.allow-origin: "*"


[root@es03 ~]# grep -v "^#" /data/elasticsearch/elasticsearch-7.14.0/config/elasticsearch.yml
cluster.name: es
node.name: es03
path.data: /data/elasticsearch/data
path.logs: /data/elasticsearch/logs
bootstrap.memory_lock: false
network.host: 0.0.0.0
http.port: 9200
discovery.seed_hosts: ["es01","es02","es03"]
cluster.initial_master_nodes: ["es01", "es02", "es03"]
node.master: true
node.data: true
http.cors.enabled: true
http.cors.allow-origin: "*"

启动

useradd es 
chown -R es:es /data/
su - es
/data/elasticsearch/elasticsearch-7.14.0/bin/elasticsearch -d

二、部署logstash

部署logstash

mkdir -p /data/logstash
tar zxvf logstash-7.14.0-linux-x86_64.tar.gz -C /data/logstash/

添加配置文件

mkdir /data/logstash/logstash-7.14.0/conf.d

cat > /data/logstash/logstash-7.14.0/conf.d/system.conf << 'EOF'
input { 
  file{ 
    path =>"/var/log/messages" 
    type =>"system" 
    start_position =>"beginning" 
  } 
}
output { 
  elasticsearch { 
    hosts => ["192.168.207.131:9200"] 
    index =>"system-%{+YYYY.MM.dd}" 
  } 
}
EOF

cat > /data/logstash/logstash-7.14.0/conf.d/apache.conf << 'EOF'
input {
  file{ 
    path =>"/var/log/httpd/access_log"  
    type =>"access" 
    start_position =>"beginning" 
  }
  file {
    path =>"/var/log/httpd/error_log" 
    type =>"error" 
    start_position =>"beginning" 
  }
}

output {
  if [type] == "access" { 
    elasticsearch { 
      hosts => ["192.168.207.131:9200"] 
      index =>"apache_access-%{+YYYY.MM.dd}" 
    }
  }
  if [type] == "error" { 
    elasticsearch { 
      hosts => ["192.168.207.131:9200"]
      index =>"apache_error-%{+YYYY.MM.dd}"
    }
  }
}
EOF

启动

/data/logstash/logstash-7.14.0/bin/logstash -f /data/logstash/logstash-7.14.0/conf.d/

三、部署kibana

单节点kibana

部署kibana

mkdir -p /data/kibana
tar zxvf kibana-7.14.0-linux-x86_64.tar.gz -C /data/kibana/

修改配置文件

grep -v "^#" /data/kibana/kibana-7.14.0-linux-x86_64/config/kibana.yml  | grep -v "^$"
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.hosts: ["http://192.168.207.131:9200"]
kibana.index: ".kibana"

启动

useradd kibana
chown -R kibana:kibana /data 
su - kibana
/data/kibana/kibana-7.14.0-linux-x86_64/bin/kibana

多节点kibana

每个节点配置相同

[root@es01 ~]# grep -v "^#" /data/kibana/kibana-7.14.0-linux-x86_64/config/kibana.yml  | grep -v "^$"
server.port: 5601
server.host: "0.0.0.0"
server.name: "your-hostname"
elasticsearch.hosts: ["http://es01:9200", "http://es02:9200", "http://es03:9200"]
kibana.index: ".kibana"