Elastic Stack 5.1 部署

最新推荐文章于 2023-08-30 21:41:46 发布

moonpure

最新推荐文章于 2023-08-30 21:41:46 发布

阅读量955

点赞数

Elastic Stack安装步骤

1、Elasticsearch
X-Pack for Elasticsearch
2、Kibana
X-Pack for Kibana
3、Logstash
4、Beats
5、Elasticsearch Hadoop
6、redis

参考文档

https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html

1、安装Elasticsearch & X-pack（yum安装）

X-Pack是一个Elastic Stack的扩展，将安全，警报，监视，报告和图形功能包含在一个易于安装的软件包中。x-Pack安装之后有一个超级用户elastic ，其默认的密码是changeme，拥有对所有索引和数据的控制权，可以使用该用户创建和修改其他用户。通过kibana的web界面进行用户和用户组的管理，也可以使用shell 终端进行管理。

[root@tzgsqvelk01 tzg]# rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

[root@tzgsqvelk01 tzg]# cat > /etc/yum.repos.d/elasticsearch.repo <<-EOF
[elasticsearch-5.x]
name=Elasticsearch repository for 5.x packages
baseurl=https://artifacts.elastic.co/packages/5.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
EOF

[root@tzgsqvelk01 tzg]# yum install elasticsearch -y

[root@tzgsqvelk01 tzg]# /bin/systemctl daemon-reload
[root@tzgsqvelk01 tzg]# /bin/systemctl enable elasticsearch.service
Created symlink from /etc/systemd/system/multi-user.target.wants/elasticsearch.service to /usr/lib/systemd/system/elasticsearch.service.
[root@tzgsqvelk01 tzg]# systemctl start elasticsearch.service
[root@tzgsqvelk01 tzg]# systemctl stop elasticsearch.service

[root@tzgsqvelk01 tzg]# grep -v "^#" /etc/elasticsearch/elasticsearch.yml
node.name: tzg-elk01
path.data: /tzgData/elk/elasticsearch/data
path.logs: /opt/elk/elasticsearch/logs
network.host: 0.0.0.0

[root@tzgsqvelk01 tzg]# mkdir -p /tzgData/elk/elasticsearch/data
[root@tzgsqvelk01 tzg]# mkdir -p /opt/elk/elasticsearch/logs
[root@tzgsqvelk01 tzg]# chown -R elasticsearch.elasticsearch /tzgData/elk/
[root@tzgsqvelk01 tzg]# chown -R elasticsearch.elasticsearch /opt/elk/
[root@tzgsqvelk01 tzg]#
[root@tzgsqvelk01 tzg]# systemctl start elasticsearch.service
[root@tzgsqvelk01 tzg]#
[root@tzgsqvelk01 tzg]#
[root@tzgsqvelk01 tzg]# curl http://localhost:9200
{
  "name" : "tzg-elk01",
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "K8bEym6OSpm9o1Y4CJrrEg",
  "version" : {
    "number" : "5.1.1",
    "build_hash" : "5395e21",
    "build_date" : "2016-12-06T12:36:15.409Z",
    "build_snapshot" : false,
    "lucene_version" : "6.3.0"
  },
  "tagline" : "You Know, for Search"
}


[root@tzgsqvelk01 tzg]# systemctl stop elasticsearch.service
[root@tzgsqvelk01 tzg]# /usr/share/elasticsearch/bin/elasticsearch-plugin install x-pack
[root@tzgsqvelk01 tzg]# systemctl start elasticsearch.service
[root@tzgsqvelk01 tzg]# curl -XPUT -u elastic 'localhost:9200/_xpack/security/user/elastic/_password' -d '{
"password" : "123456"
 }'
Enter host password for user 'elastic':
{}[root@tzgsqvelk01 tzg]#

[root@tzgsqvelk01 tzg]# systemctl stop elasticsearch.service
[root@tzgsqvelk01 tzg]# /usr/share/elasticsearch/bin/elasticsearch-plugin install x-pack
[root@tzgsqvelk01 tzg]# systemctl start elasticsearch.service

kibana & x-pack安装

[root@tzgsqvelk01 tzg]# cat > /etc/yum.repos.d/kibana.repo <<-EOF
[kibana-5.x]
name=Kibana repository for 5.x packages
baseurl=https://artifacts.elastic.co/packages/5.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
EOF
[root@tzgsqvelk01 tzg]#
[root@tzgsqvelk01 tzg]#
[root@tzgsqvelk01 tzg]# yum install kibana -y

[root@tzgsqvelk01 tzg]# grep "^[a-z]" /etc/kibana/kibana.yml
server.host: "172.16.5.37"
elasticsearch.username: "elastic"
elasticsearch.password: "123456"

[root@tzgsqvelk01 tzg]# systemctl stop kibana.service
[root@tzgsqvelk01 tzg]# /usr/share/kibana/bin/kibana-plugin install file:///home/x-pack-5.1.1.zip
[root@tzgsqvelk01 tzg]# systemctl start kibana.service
[root@tzgsqvelk01 tzg]# curl -XPUT -u elastic 'localhost:9200/_xpack/security/user/kibana/_password' -d '{
  "password" : "123456"
}'
Enter host password for user 'elastic':

logstash 安装

[root@tzgsqvelk01 tzg]# cat > /etc/yum.repos.d/logstash.repo <<-EOF
[logstash-5.x]
name=Elastic repository for 5.x packages
baseurl=https://artifacts.elastic.co/packages/5.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
EOF

[root@tzgsqvelk01 tzg]# yum install logstash -y

[root@tzgsqvelk01 tzg]# /usr/share/logstash/bin/logstash -e 'input { stdin{} } output { elasticsearch { hosts => "172.16.5.37:9200" user => elastic  password => 123456 } } '
hahahha
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs to console
The stdin plugin is now waiting for input:
10:47:39.397 [[main]-pipeline-manager] INFO  logstash.outputs.elasticsearch - Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>["http://~hidden~:~hidden~@172.16.5.37:9200"]}}
10:47:39.400 [[main]-pipeline-manager] INFO  logstash.outputs.elasticsearch - Running health check to see if an Elasticsearch connection is working {:url=>#<URI::HTTP:0x673e5894 URL:http://~hidden~:~hidden~@172.16.5.37:9200>, :healthcheck_path=>"/"}
10:47:39.617 [[main]-pipeline-manager] WARN  logstash.outputs.elasticsearch - Restored connection to ES instance {:url=>#<URI::HTTP:0x673e5894 URL:http://~hidden~:~hidden~@172.16.5.37:9200>}
10:47:39.617 [[main]-pipeline-manager] INFO  logstash.outputs.elasticsearch - Using mapping template from {:path=>nil}
10:47:39.658 [[main]-pipeline-manager] INFO  logstash.outputs.elasticsearch - Attempting to install template {:manage_template=>{"template"=>"logstash-*", "version"=>50001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"_all"=>{"enabled"=>true, "norms"=>false}, "dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword"}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date", "include_in_all"=>false}, "@version"=>{"type"=>"keyword", "include_in_all"=>false}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}
10:47:39.772 [[main]-pipeline-manager] INFO  logstash.outputs.elasticsearch - Installing elasticsearch template to _template/logstash
10:47:39.815 [[main]-pipeline-manager] INFO  logstash.outputs.elasticsearch - New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["172.16.5.37:9200"]}
10:47:39.817 [[main]-pipeline-manager] INFO  logstash.pipeline - Starting pipeline {"id"=>"main", "pipeline.workers"=>8, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>1000}
10:47:39.823 [[main]-pipeline-manager] INFO  logstash.pipeline - Pipeline main started
10:47:39.856 [Api Webserver] INFO  logstash.agent - Successfully started Logstash API endpoint {:port=>9600}
^C10:47:47.587 [SIGINT handler] WARN  logstash.runner - SIGINT received. Shutting down the agent.
10:47:47.592 [LogStash::Runner] WARN  logstash.agent - stopping pipeline {:id=>"main"}


[root@tzgsqvelk01 tzg]# /usr/share/logstash/bin/system-install
Successfully created system startup script for Logstash
[root@tzgsqvelk01 tzg]# chown -R logstash.logstash /usr/share/logstash/data/

redis安装

[root@ tzgsqvelk01 app]# wget http://download.redis.io/releases/redis-3.2.1.tar.gz
[root@ tzgsqvelk01 app]# yum install gcc -y
[root@ tzgsqvelk01 app]# tar zxvf redis-3.2.1.tar.gz
[root@ tzgsqvelk01 app]# cd redis-3.2.1
[root@tzgsqvelk01 redis-3.2.1]# make
[root@tzgsqvelk01 redis-3.2.1]# make install PREFIX=/opt/app/redis

[root@tzgsqvelk01 redis-3.2.1]# cat /opt/app/redis/conf/redis.conf
daemonize yes
pidfile /var/run/redis.pid
bind 172.16.5.37
port 6379
timeout 120
loglevel notice
logfile /opt/app/redis/logs/redis.log
databases 16
save 900 1
save 300 100
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /tzgData/redis/var
appendonly yes
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
slowlog-log-slower-than 10000
slowlog-max-len 128
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-entries 512
list-max-ziplist-value 64
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
maxmemory 4096000000
maxclients 10000
slave-serve-stale-data yes
slave-read-only yes
slave-priority 100
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no

[root@tzgsqvelk01 redis-3.2.1]# cp utils/redis_init_script /etc/init.d/redisd
[root@tzgsqvelk01 redis-3.2.1]# vim /etc/init.d/redisd

#!/bin/sh
# chkconfig:   2345 90 10
# description:  Redis is a persistent key-value database
#

REDISPORT=6379
EXEC=/opt/app/redis/bin/redis-server
CLIEXEC=/opt/app/redis/bin/redis-cli

PIDFILE=/var/run/redis_${REDISPORT}.pid
CONF="/opt/app/redis/conf/redis.conf"

[root@tzgsqvelk01 tzg]# chkconfig redisd on
[root@tzgsqvelk01 tzg]# service redisd start

[root@tzgsqvelk01 app]# rm -rf redis-3.2.1
[root@tzgsqvelk01 app]# rm -rf redis-3.2.1.tar.gz

logstash和elasticsearch 无需授权配置

参考官方文档
https://www.elastic.co/guide/en/x-pack/current/anonymous-access.html

[root@ tzgsqvelk01 elasticsearch]# vim elasticsearch.yml

xpack.security.authc:
  anonymous:
    username: anonymous_user
    roles: logstash_logs
    authz_exception: true

######添加 logstash_logs的权限######
curl -XPOST -u elastic 'localhost:9200/_xpack/security/role/logstash_logs' -d '{
  "cluster": ["manage_index_templates", "monitor"],
  "indices": [
    {
      "names": [ "logstash-*","business-index-*"], 
      "privileges": ["write","delete","create_index"]
    }
  ]
}'


[root@ tzgsqvelk01 elasticsearch]# systemctl restart elasticsearch.service
[root@ tzgsqvelk01 elasticsearch]# /usr/share/logstash/bin/logstash -e 'input { stdin{} } output { elasticsearch { hosts => "192.168.3.210:9200"} } '
tzghahaha
Sending Logstash's logs to /var/log/logstash/logs which is now configured via log4j2.properties
The stdin plugin is now waiting for input:
[2016-12-30T08:48:07,332][INFO ][logstash.outputs.elasticsearch] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>["http://192.168.3.210:9200"]}}
[2016-12-30T08:48:07,342][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:url=>#<URI::HTTP:0x52cfa9fa URL:http://192.168.3.210:9200>, :healthcheck_path=>"/"}
[2016-12-30T08:48:07,592][WARN ][logstash.outputs.elasticsearch] Restored connection to ES instance {:url=>#<URI::HTTP:0x52cfa9fa URL:http://192.168.3.210:9200>}
[2016-12-30T08:48:07,599][INFO ][logstash.outputs.elasticsearch] Using mapping template from {:path=>nil}
[2016-12-30T08:48:07,753][INFO ][logstash.outputs.elasticsearch] Attempting to install template {:manage_template=>{"template"=>"logstash-*", "version"=>50001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"_all"=>{"enabled"=>true, "norms"=>false}, "dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword"}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date", "include_in_all"=>false}, "@version"=>{"type"=>"keyword", "include_in_all"=>false}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}
[2016-12-30T08:48:07,773][INFO ][logstash.outputs.elasticsearch] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["192.168.3.210:9200"]}
[2016-12-30T08:48:07,784][INFO ][logstash.pipeline        ] Starting pipeline {"id"=>"main", "pipeline.workers"=>4, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>500}
[2016-12-30T08:48:07,814][INFO ][logstash.pipeline        ] Pipeline main started
[2016-12-30T08:48:07,989][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}