1 服务准备
redis安装部署
下载地址:wget http://download.redis.io/releases/redis-2.8.3.tar.gz
安装目录:/home/vkapp/ELK/redis
切换至安装目录下进行工具安装:
文件解压:tar –zxvf redis-2.8.3.tar.gz
文件重命名:mv redis-2.8.3 redis2
编译安装:
进入服务目录:/home/vkapp/ELK/redis/redis2
执行:make
建议将 src 目录下的下列几个文件取出来单独放置,方便进行启动与管理
启动服务:
执行:redis-server redis.conf &
添加&符号或 -d表示后台启动
注意:记得配置redis的连接密码
2 数据同步
部署完redis服务之后,就需要将logstash采集端采集到的数据存储到redis,接着logstash收集端从redis上将数据进行下载,将数据保存至logstash收集端的本地,经过elasticsearch与kibana处理后进行页面展示,实现跨主机日志数据实现同步
如下图所示:
主机A,数据采集端logstash配置:
input{
file{
path => "/home/vkapp/logs/current/admin.log"
type => "admin"
}
file{
path => "/home/vkapp/logs/current/wap.log"
type => "wap"
}
file{
path => "/home/vkapp/logs/current/web.log"
type => "web"
}
file{
path => "/home/vkapp/logs/current/mobile.log"
type => "mobile"
}
file{
path => "/home/vkapp/logs/current/org.log"
type => "org"
}
}
filter {
mutate {
replace => ["host", "主机A IP"]
}
multiline {
pattern => "^(\d{4})-(0\d{1}|1[0-2])-(0\d{1}|[12]\d{1}|3[01])"
negate => true
what => "previous"
}
if([message] =~ "INFO"){
drop{}
}
}
output{
#stdout{}
redis {
host => "主机B IP"
port => 6379
db => 10
data_type => "list"
key => "%{type}"
password => "password"
}
}
主机B,数据收集端logstash配置:
input{
redis {
host = "主机B IP"
port = 6379
db = 10
data_type = "list"
key = "admin"
password = "password"
}
redis {
host = "主机B IP"
port = 6379
db = 10
data_type = "list"
key = "wap"
password = "password"
}
redis {
host = "主机B IP"
port = 6379
db = 10
data_type = "list"
key = "web"
password = "password"
}
redis {
host = "主机B IP"
port = 6379
db = 10
data_type = "list"
key = "mobile"
password = "password"
}
}
output {
elasticsearch {
hosts = "主机B IP:9200"
index = "logstash-%{+YYYY.MM.dd}-主机A IP"
document_type = "%{host}"
user = elastic
password = "password"
}
}
此处最后会将收集到的数据保存elasticsearch中,然后将kibana进行启动,访问kibana的图形页面即可进行数据访问
附:
kibane配置:
server.port: 5601
server.host: "主机B IP"
elasticsearch.url: "http://主机B IP:9200"
kibana.index: ".kibana"
server.maxPayloadBytes: 20971520
logging.dest: "/home/vkapp/elk5/kibana/logs/kibana.log"
#xpack.security.enabled: true
elasticsearch.username: "ES用户名"
elasticsearch.password: "ES访问密码"
sentinl:
settings:
email:
active: true
user: "QQ邮箱"
password: "SMTP密码"
host: "smtp.qq.com"
ssl: true
report:
active: true
tmp_path: /tmp/
elasticsaerch配置:
cluster.name: cluster
node.name: node-104
network.host: 主机B IP
http.port: 9200
path.data: /app/elasticsearch/es/data
path.logs: /app/elasticsearch/es/logs
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
#bootstrap.mlockall: true
indices.fielddata.cache.size: 20%
discovery.zen.ping_timeout: 20s
node.master: true
node.data: true
http.cors.enabled: true
http.cors.allow-origin: "*"
indices.recovery.max_bytes_per_sec: 20mb
#discovery.zen.ping.multicast.enabled: false
discovery.zen.ping.unicast.hosts: ["主机B IP:9200"]
xpack.security.enabled: true
xpack.ml.enabled: false
action.auto_create_index: .security,.monitoring*,.watches,.triggered_watches,.watcher-history*,logstash-*