单机部署elk【7.3.2】

部署准备

wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.3.2-linux-x86_64.tar.gz
wget https://artifacts.elastic.co/downloads/logstash/logstash-7.3.2.tar.gz
wget https://artifacts.elastic.co/downloads/kibana/kibana-7.3.2-linux-x86_64.tar.gz
wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.3.2-linux-x86_64.tar.gz




# mkdir /home/elk

# ll /home/elk
-rw-r--r--. 1 root root 285050383 十一 23  2019 elasticsearch-7.3.2-linux-x86_64.tar.gz
-rw-r--r--. 1 root root  25274202 十一 23  2019 filebeat-7.3.2-linux-x86_64.tar.gz
-rw-r--r--. 1 root root 236654252 十一 23  2019 kibana-7.3.2-linux-x86_64.tar.gz
-rw-r--r--. 1 root root 171783584 十一 23  2019 logstash-7.3.2.tar.gz

elastic

# tar -zxvf  elasticsearch-7.3.2-linux-x86_64.tar.gz
修改配置文件
[root@dev-app-60 elasticsearch-7.3.2]# vim config/elasticsearch.yml 
——————————————————————————————————————————————————————————————————————————————————————————————————————————————————
#ES监听地址任意IP都可访问，也可以是自己服务器的IP
network.host: 0.0.0.0
http.port: 9200
——————————————————————————————————————————————————————————————————————————————————————————————————————————————————

优化类配置
vi /etc/sysctl.conf
fs.file-max=65536
vm.max_map_count=262144
# sysctl -p

vi /etc/security/limits.conf
*               soft    nofile          65536
*               hard    nofile          65536
*               soft    nproc           65536
*               hard    nproc           65536


#############################################添加用户和组#############################################################
添加组
groupadd elkgroup   
      
在elkgroup下添加elkuser用户，并设密码
useradd elkuser -g elkgroup -p elkuser


文件目录权限修改
[root@dev-app-60 elk]# chown  elkuser.  /home/elk  -R

# ll
total 0
drwxr-xr-x. 10 elkuser elkgroup 183 三月 10 18:05 elasticsearch-7.3.2
drwxr-xr-x.  5 elkuser elkgroup 212 三月 11 09:21 filebeat-7.3.2-linux-x86_64
drwxr-xr-x. 14 elkuser elkgroup 271 三月 11 09:21 kibana-7.3.2-linux-x86_64
drwxr-xr-x. 12 elkuser elkgroup 255 三月 10 18:08 logstash-7.3.2

 
定位到elasticsearch安装目录下为elkuser用户设置访问权限
# chown -R  elkuser  /home/elk/elasticsearch-7.3.2

启动
# su  elkuser 
$ cd /home/elk/elasticsearch-7.3.2
nohup  ./bin/elasticsearch &


检测是否启动
# netstat -tanp|grep  9200
tcp6       0      0 :::9200                 :::*                    LISTEN      9090/java     

$ curl 127.0.0.1:9200
{
  "name" : "dev-app-60",
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "_na_",
  "version" : {
    "number" : "7.3.2",
    "build_flavor" : "default",
    "build_type" : "tar",
    "build_hash" : "1c1faf1",
    "build_date" : "2019-09-06T14:40:30.409026Z",
    "build_snapshot" : false,
    "lucene_version" : "8.1.0",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "You Know, for Search"
}




elastic添加密码
[elkuser@dev-app-60 elasticsearch-7.3.2]$ vim config/elasticsearch.yml 
http.cors.enabled: true
http.cors.allow-origin: "*"
http.cors.allow-headers: Authorization
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.authc.accept_default_password: true




+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
指定密码比较复杂的时候，可以随机 密码 
[elkuser@dev-app-60 elasticsearch-7.3.2]$ ./bin/elasticsearch-setup-passwords auto
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++


执行设置用户名和密码的命令,这里需要为4个用户分别设置密码，elastic,apm_system, kibana, logstash_system,betas_system,remote_monitoring_user
[elkuser@dev-app-60 elasticsearch-7.3.2]$ ./bin/elasticsearch-setup-passwords interactive






 ./bin/elasticsearch-setup-passwords interactive
future versions of Elasticsearch will require Java 11; your Java version from [/home/jdk/jre] does not meet this requirement

Failed to determine the health of the cluster running at http://10.2.204.60:9200
Unexpected response code [503] from calling GET http://10.2.204.60:9200/_cluster/health?pretty
Cause: master_not_discovered_exception

It is recommended that you resolve the issues with your cluster before running elasticsearch-setup-passwords.
It is very likely that the password changes will fail when run against an unhealthy cluster.

Do you want to continue with the password setup process [y/N]y

Initiating the setup of passwords for reserved users elastic,apm_system,kibana,logstash_system,beats_system,remote_monitoring_user.
You will be prompted to enter passwords as the process progresses.
Please confirm that you would like to continue [y/N]y


Enter password for [elastic]: 
Reenter password for [elastic]: 
Enter password for [apm_system]: 
Reenter password for [apm_system]: 
Enter password for [kibana]: 
Reenter password for [kibana]: 
Enter password for [logstash_system]: 
Reenter password for [logstash_system]: 
Enter password for [beats_system]: 
Reenter password for [beats_system]: 
Enter password for [remote_monitoring_user]: 
Reenter password for [remote_monitoring_user]: 
Changed password for user [apm_system]
Changed password for user [kibana]
Changed password for user [logstash_system]
Changed password for user [beats_system]
Changed password for user [remote_monitoring_user]
Changed password for user [elastic]

logstash

#测试
curl -H "Content-Type:application/json" -XPOST -u elastic 'http://127.0.0.1:9200/_xpack/security/user/elastic/_password' -d '{ "password" : "123456" }'

# cd /home/elk
# tar -zxvf logstash-7.3.2.tar.gz
# cd logstash-7.3.2/

配置文件
vim  pipelines.yml
path.config: "/home/elk/logstash-7.3.2/config/conf.d/*.conf"
[root@dev-app-60 config]# mkdir  conf.d
[root@dev-app-60 config]# cd conf.d/


  
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
配置文件
[elkuser@dev-app-60 logstash-7.3.2]$ cat config/conf.d/app.conf 
input {
  beats {
    port => 5044
  }
}

filter {
  if "dev-app-allocation" in [tags] {
    grok {
      match => { "message" => "\[%{TIMESTAMP_ISO8601:log_timestamp}\]%{GREEDYDATA:log_info}" }
      remove_field => ["log_info","agent","ecs.version","log.flags","log.offset"]
      }
    mutate { 
      gsub => [ "log_info", "\\n", "\n\r" ] 
      }
    date { 
      match => [ "log_timestamp" , "yyyy-MM-dd HH:mm:ss,SSS" ]
     }
   }


output {
  if "dev-app" in [tags] {
    elasticsearch {
      hosts => ["http://10.2.204.60:9200"]
      index => "dev-app%{+YYYY.MM.dd}"
      user => "elastic"
      password => "elastic密码"
     }
   }
    #stdout { codec => rubydebug }
  }
}  
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#

启动
[elkuser@dev-app-60 logstash-7.3.2]$ nohup  ./bin/logstash &

logstash限制内存

[elkuser@dev-app-60 config]$ cat /home/elk/logstash-7.3.2/config/jvm.options 

排错记录

[elkuser@dev-app-60 elasticsearch-7.3.2]$ ./bin/elasticsearch-setup-passwords interactive
ERROR: Failed to set password for user [apm_system].

  
注释：
discovery.seed_hosts
cluster.initial_master_nodes
添加
discovery.type: single-node
重启elastic
  
  


[elkuser@dev-app-60 logstash-7.3.2]$ ./bin/logstash -e'input {stdin {}} output {stdout {}}'
Thread.exclusive is deprecated, use Thread::Mutex
Sending Logstash logs to /home/elk/logstash-7.3.2/logs which is now configured via log4j2.properties
[2021-03-11T11:57:21,225][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2021-03-11T11:57:21,240][INFO ][logstash.runner          ] Starting Logstash {"logstash.version"=>"7.3.2"}
[2021-03-11T11:57:21,267][INFO ][logstash.agent           ] No persistent UUID file found. Generating new UUID {:uuid=>"6426ad87-6c81-4c91-823e-1b44e06139a8", :path=>"/home/elk/logstash-7.3.2/data/uuid"}
[2021-03-11T11:57:22,581][INFO ][org.reflections.Reflections] Reflections took 110 ms to scan 1 urls, producing 19 keys and 39 values 
[2021-03-11T11:57:23,968][WARN ][org.logstash.instrument.metrics.gauge.LazyDelegatingGauge] A gauge metric of an unknown type (org.jruby.RubyArray) has been create for key: cluster_uuids. This may result in invalid serialization.  It is recommended to log an issue to the responsible developer/development team.
[2021-03-11T11:57:23,972][INFO ][logstash.javapipeline    ] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>8, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>1000, :thread=>"#<Thread:0x287ba869 run>"}
[2021-03-11T11:57:24,028][INFO ][logstash.javapipeline    ] Pipeline started {"pipeline.id"=>"main"}
The stdin plugin is now waiting for input:
[2021-03-11T11:57:24,161][INFO ][logstash.agent           ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
[2021-03-11T11:57:24,401][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}########出现此处则输入要输出的内容
xlxtest
/home/elk/logstash-7.3.2/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated
{
    "@timestamp" => 2021-03-11T03:57:45.394Z,
          "host" => "dev-app-60",
       "message" => "xlxtest",
      "@version" => "1"
}

{
    "@timestamp" => 2021-03-11T03:57:48.869Z,
          "host" => "dev-app-60",
       "message" => "",
      "@version" => "1"
}

根据配置文件调试

filebeat

# tar -zxvf   filebeat-7.3.2-linux-x86_64.tar.gz 
[root@dev-app-60 elk]# cd logstash-7.3.2/config/
[root@dev-app-60 config]# su elkuser
[elkuser@dev-app-60 filebeat-7.3.2-linux-x86_64]$ cp filebeat.yml   filebeat.yml_bak.0311 
[elkuser@dev-app-60 filebeat-7.3.2-linux-x86_64]$ vim filebeat.yml

  
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
[elkuser@dev-app-60 filebeat-7.3.2-linux-x86_64]$ cat filebeat.yml
filebeat.inputs:
 
- type: log
  enabled: true
  paths:
    - /home/output/logs/allocation/*.log
  tags: ["dev-app"]
  multiline.pattern: '^\[\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
  multiline.negate: true
  multiline.match: after

#============================= Filebeat modules ===============================
 
filebeat.config.modules:
  # Glob pattern for configuration loading
  path: ${path.config}/modules.d/*.yml
 
  # Set to true to enable config reloading
  reload.enabled: false
 
  # Period on which files under path should be checked for changes
  #reload.period: 10s
 
#==================== Elasticsearch template setting ==========================
 
setup.template.settings:
  index.number_of_shards: 1
  #index.codec: best_compression
  #_source.enabled: false
#================================ Outputs =====================================
 
 
#----------------------------- Logstash output --------------------------------
output.logstash:
  # The Logstash hosts
  hosts: ["10.2.204.60:5044"]  #  需要在 logstash 配置文件中配置
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

启动
# su  elkuser
[elkuser@dev-app-60 filebeat-7.3.2-linux-x86_64]$ nohup   /home/elk/filebeat-7.3.2-linux-x86_64/filebeat  &

kibana

# cd /home/elk/kibana-7.3.2-linux-x86_64/config
[elkuser@dev-app-60 config]$ cat kibana.yml|grep -v "#"|grep -v ^$
server.host: "0.0.0.0"
elasticsearch.username: "elastic"
elasticsearch.password: "elastic密码"

启动
[elkuser@dev-app-60 elk]$ nohup /home/elk/kibana-7.3.2-linux-x86_64/bin/kibana &

kibana查询通配符

通配符

? 匹配单个字符
* 匹配0到多个字符

示例：kiba?a, el*search
? * 不能用作第一个字符，例如：?text *text

kibana-转义特殊字符

转义特殊字符
+ - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /

以上字符当作值搜索的时候需要用\转义

1\+  用来查询1+

kibana–web界面

http://10.2.204.60:5601/login?next=%2F#?_g=()
elastic/elastic密码