elasticsearch+logstash并使用java代码实现日志检索

elasticsearch+logstash并使用java代码实现日志检索

为了项目日志不被泄露，数据展示不采用Kibana

1、环境准备

1.1 创建普通用户

#创建用户
useradd querylog

#设置密码
passwd queylog

#授权sudo权限
查找sudoers文件位置 
whereis sudoers
#修改文件为可编辑 
chmod -v u+w /etc/sudoers
#编辑文件
vi /etc/sudoers
#收回权限
chmod -v u-w /etc/sudoers
#第一次使用sudo会有提示

We trust you have received the usual lecture from the local System
Administrator. It usually boils down to these three things:

    #1) Respect the privacy of others.
    #2) Think before you type.
    #3) With great power comes great responsibility.

用户创建完成。

1.2 安装jdk

su queylog
cd /home/queylog
#解压jdk-8u191-linux-x64.tar.gz 
tar -zxvf jdk-8u191-linux-x64.tar.gz 
sudo mv jdk1.8.0_191 /opt/jdk1.8
#编辑/ect/profile
vi /ect/profile
export JAVA_HOME=/opt/jdk1.8
export JRE_HOME=$JAVA_HOME/jre
export CLASSPATH=.:$JAVA_HOME/lib:$JRE_HOME/lib:$CLASSPATH
export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$PATH
#刷新配置文件
source /ect/profile
#查看jdk版本
java -verion

1.3 防火墙设置

#放行指定IP
firewall-cmd --permanent --add-rich-rule="rule family="ipv4" source address="172.16.110.55"   accept" 
#重新载入
firewall-cmd --reload

2、安装elasticsearch

2.1 elasticsearch配置

注意：elasticsearch要使用普通用户启动要不然会报错

su queylog
cd /home/queylog
#解压elasticsearch-6.5.4.tar.gz
tar -zxvf elasticsearch-6.5.4.tar.gz
sudo mv elasticsearch-6.5.4 /opt/elasticsearch
#编辑es配置文件
vi /opt/elasticsearch/config/elasticsearch.yml 
# 配置es的集群名称
cluster.name: elastic
# 修改服务地址
network.host: 192.168.8.224
# 修改服务端口
http.port: 9200

#切换root用户
su root
#修改/etc/security/limits.conf 追加以下内容
vi /etc/security/limits.conf
* hard nofile 655360
* soft nofile 131072
* hard nproc 4096
* soft nproc 2048

#编辑 /etc/sysctl.conf，追加以下内容：
vi /etc/sysctl.conf
vm.max_map_count=655360
fs.file-max=655360

#保存后，重新加载：
sysctl -p

#切换回普通用户
su queylog
#启动elasticsearch
./opt/elasticsearch/bin/elasticsearch
#测试
curl http://192.168.8.224:9200
#控制台会打印
{
   
  "name" : "L_dA6oi",
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "eS7yP6fVTvC8KMhLutOz6w",
  "version" : {
   
    "number" : "6.5.4",
    "build_flavor" : "default",
    "build_type" : "tar",
    "build_hash" : "d2ef93d",
    "build_date" : "2018-12-17T21:17:40.758843Z",
    "build_snapshot" : false,
    "lucene_version" : "7.5.0",
    "minimum_wire_compatibility_version" : "5.6.0",
    "minimum_index_compatibility_version" : "5.0.0"
  },
  "tagline" : "You Know, for Search"
}

2.2 把elasticsearch作为服务进行管理

#切换root用户
su root
#编写服务配置文件
vi /usr/lib/systemd/system/elasticsearch.service
[unit]
Description=Elasticsearch
Documentation=http://www.elastic.co
Wants=network-online.target
After=network-online.target

[Service]
Environment=ES_HOME=/opt/elasticsearch
Environment=ES_PATH_CONF=/opt/elasticsearch/config
Environment=PID_DIR=/opt/elasticsearch/config
EnvironmentFile=/etc/sysconfig/elasticsearch
WorkingDirectory=/opt/elasticsearch
User=queylog
Group=queylog
ExecStart=/opt/elasticsearch/bin/elasticsearch -p ${PID_DIR}/elasticsearch.pid

# StandardOutput is configured to redirect to journalctl since
# some error messages may be logged in standard output before
# elasticsearch logging system is initialized. Elasticsearch
# stores its logs in /var/log/elasticsearch and does not use
# journalctl by default. If you also want to enable journalctl
# logging, you can simply remove the "quiet" option from ExecStart.
StandardOutput=journal
StandardError=inherit

# Specifies the maximum file descriptor number that can be opened by this process
LimitNOFILE=65536

# Specifies the maximum number of process
LimitNPROC=4096

# Specifies the maximum size of virtual memory
LimitAS=infinity

# Specifies the maximum file size
LimitFSIZE=infinity

# Disable timeout logic and wait until process is stopped
TimeoutStopSec=0

# SIGTERM signal is used to stop the Java process
KillSignal=SIGTERM

# Send the signal only to the JVM rather than its control group
KillMode=process

# Java process is never killed
SendSIGKILL=no

# When a JVM receives a SIGTERM signal it exits with code 143
SuccessExitStatus=143
 
[Install]
WantedBy=multi-user.target


vi /etc/sysconfig/elasticsearch

elasticsearch    #
#######################

# Elasticsearch home directory
ES_HOME=/opt/elasticsearch

# Elasticsearch Java path
JAVA_HOME=/home/liyijie/jdk1.8
CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JAVA_HOMR/jre/lib

# Elasticsearch configuration directory
ES_PATH_CONF=/opt/elasticsearch/config

# Elasticsearch PID directory
PID_DIR=/opt/elasticsearch/config

#############################
#   Elasticsearch Service   #
#############################

# SysV init.d
# The number of seconds to wait before checking if elasticsearch started successfully as a daemon process
ES_STARTUP_SLEEP_TIME=5

################################
#   Elasticsearch Properties   #
################################
# Specifies the maximum file descriptor number that can be opened by this process
# When using Systemd,this setting is ignored and the LimitNOFILE defined in
# /usr/lib/systemd/system/elasticsearch.service takes precedence
#MAX_OPEN_FILES=65536

# The maximum number of bytes of memory that may be locked into RAM
# Set to "unlimited" if you use the 'bootstrap.memory_lock: true' option
# in elasticsearch.yml.
# When using Systemd,LimitMEMLOCK must be set in a unit file such as
# /etc/systemd/system/elasticsearch.service.d/override.conf.
#MAX_LOCKED_MEMORY=unlimited

# Maximum number of VMA(Virtual Memory Areas) a process can own
# When using Systemd,this setting is ignored and the 'vm.max_map_count'
# property is set at boot time in /usr/lib/sysctl.d/elasticsearch.conf
#MAX_MAP_COUNT=262144

# 重新加载服务
systemctl daemon-reload
#切换普通用户
su queylog
#启动elasticsearch
sudo systemctl start elasticsearch
#设置开机自启动
sudo systemctl enable elasticsearch

3、安装logstash

3.1、logstash配置

su queylog