1 elk是什么
pass
2 使用docker-compose搭建elk系统的链接
github使用人数很多的连接地址:https://github.com/deviantony/docker-elk/
3 发送的日志在logstash接收到必须是json
发送时可以logger.info("hello world"), 但是接收必须是json
但是当初我日志利用python3+ 原生logging模块 + logstash模块(会把原生的logging的日志输出成json, 不用自己封装,也封装不好)
以下案例为向tcp xx.xxx.xxx.xx:5600发送message
# coding=utf-8
import connexion
import logging
import sys
import logstash
TCP_LOG_HOST = '192.168.11.187'
TCP_LOG_PORT = 5600
# 测试重写日志类
class Log(object):
"""
日志记录类
"""
# 基本日志等级
# logger.debug("this is debug")
# logger.info("this is info")
# logger.warning("this is warning")
# logging.error("this is error")
# logger.critical("this is critical")
# __instance = None
# def __new__(cls, *args, **kwargs):
# if not cls.__instance:
# cls.__instance = super().__new__(cls, *args, **kwargs)
# return cls.__instance
def __init__(self, host='192.168.11.187', port=5600):
# 初始化python默认类模块+logstash
self.host = host
self.port = port
self.extra = None
logger_obj = logging.getLogger('python-logstash-logger')
# 应该设置开关,测试的时候打开debug不用再写print, 直接打印到屏幕上
logger_obj.setLevel(logging.INFO)
stash_obj = logstash.TCPLogstashHandler(host=self.host, port=self.port, tags=[], version=1)
# print(stash_obj.__dict__)
console_obj = logging.StreamHandler()
logger_obj.addHandler(stash_obj)
logger_obj.addHandler(console_obj)
self.logger_obj = logger_obj
# 初始化日志信息, 要记录用户请求,用户返回;docker的位置信息,是哪个服务
# headers_info = connexion.request.headers
# request_info = connexion.request.__dict__
#
# print('headers_info:', headers_info)
# print('request_info:', request_info)
#
# user_id = connexion.request.headers.get('x-consumer-custom-id')
# remote_addr = request_info['environ'].get('REMOTE_ADDR')
# request_method = request_info['environ'].get('REQUEST_METHOD')
# http_host = request_info['environ'].get('HTTP_HOST')
# server_port = request_info['environ'].get('SERVER_PORT')
# self.extra = {
# "user_id": str(user_id),
# "remote_addr": str(remote_addr),
# "request_method": str(request_method),
# "http_host": str(http_host),
# "server_port": str(server_port)
# }
def get_extra(self):
# headers_info = connexion.request.headers
# request_info = connexion.request.__dict__
#
# print('headers_info:', headers_info)
# print('request_info:', request_info)
#
# user_id = connexion.request.headers.get('x-consumer-custom-id')
# remote_addr = request_info['environ'].get('REMOTE_ADDR')
# request_method = request_info['environ'].get('REQUEST_METHOD')
# http_host = request_info['environ'].get('HTTP_HOST')
# server_port = request_info['environ'].get('SERVER_PORT')
# self.extra = {
# "user_id": str(user_id),
# "remote_addr": str(remote_addr),
# "request_method": str(request_method),
# "http_host": str(http_host),
# "server_port": str(server_port)
# }
return self.extra
def set_extra(self, headers_info, request_info):
print('set_extra headers_info:', headers_info)
print('set_extra request_info:', request_info)
# 取user_id
user_id = headers_info.get('x-consumer-custom-id')
# 取请求信息
remote_addr = request_info['environ'].get('REMOTE_ADDR')
request_method = request_info['environ'].get('REQUEST_METHOD')
http_host = request_info['environ'].get('HTTP_HOST')
server_port = request_info['environ'].get('SERVER_PORT')
self.extra = {
"user_id": str(user_id),
"remote_addr": str(remote_addr),
"request_method": str(request_method),
"http_host": str(http_host),
"server_port": str(server_port)
}
return self.extra
def debug(self, msg=""):
# self.logger_obj.debug(msg, extra=self.extra)
self.logger_obj.debug(msg)
def info(self, msg=""):
print("************************")
print(self.extra)
print("************************")
self.logger_obj.info(msg, extra=self.extra)
# self.logger_obj.info(msg)
def warning(self, msg=""):
# self.logger_obj.warning(msg, extra=self.extra)
self.logger_obj.warning(msg)
def error(self, msg=""):
# self.logger_obj.error(msg, extra=self.extra)
self.logger_obj.error(msg)
def critical(self, msg=""):
# self.logger_obj.critical(msg, extra=self.extra)
self.logger_obj.critical(msg)
logger = Log(host=TCP_LOG_HOST, port=TCP_LOG_PORT)
# logger.debug("this is debug, 测试")
# logger.info("this is info, 测试, extra 0000")
# logger.warning("this is warning, 测试")
# logger.error("this is error, 测试")
# logger.critical("this is critical, 测试")
# 装饰器用于获取请求数据和docker环境微服务识别
def check_log(f):
def _wrap(*args, **kwargs):
log_dict = {"fun_name": f.__name__}
headers_info = connexion.request.headers
request_info = connexion.request.__dict__
logger.set_extra(headers_info, request_info)
# print('headers_info:', headers_info)
# print('----------------------------')
# print('request_info:', request_info)
# print('----------------------------')
#
# user_id = connexion.request.headers.get('x-consumer-custom-id')
# remote_addr = request_info['environ'].get('REMOTE_ADDR')
# request_method = request_info['environ'].get('REQUEST_METHOD')
# http_host = request_info['environ'].get('HTTP_HOST')
# server_port = request_info['environ'].get('SERVER_PORT')
# extra = {
# "user_id": str(user_id),
# "remote_addr": str(remote_addr),
# "request_method": str(request_method),
# "http_host": str(http_host),
# "server_port": str(server_port)
# }
func = f(*args, **kwargs)
# print('返回数据:', func.__dict__)
#
#
# result_data = func.__dict__['response']
# result_data = str(result_data)
# print('result_data:', result_data)
return func
return _wrap
5 docker-compose启动
当初下来以后我修改了配置文件,基本目录是/root/docker-elk
vi /root/docker-elk/logstash/pipeline/logstash.yml
input {
tcp {
port => 5600
codec => json { charset=>"ISO-8859-1" }
}
}
## Add your filters / logstash plugins configuration here
output {
elasticsearch {
hosts => "elasticsearch:9200"
codec => json
}
}
vi /root/docker-elk/logstash/pipeline/Dockerfile
ARG ELK_VERSION
# https://github.com/elastic/logstash-docker
FROM docker.elastic.co/logstash/logstash:${ELK_VERSION}
# Add your logstash plugins setup here
RUN logstash-plugin install logstash-filter-json
然后在docker-elk目录下:docker-compose up -d
但是需要注意es和logstash开始初始化比较慢,一定要等待或者看docker logs -f logstash, 查看logstash有没有报错
当时配的端口是:es: 9201, kibana:5601
5 logstash接收和es接收数据
{'_index': 'logstash-2019.06.18',
'_type': 'doc',
'_id': '8MQpamsBLZCnqbD6jmJF',
'_score': None,
'_source': {'user_id': '1032183',
'request_method': 'GET', 'path': './app/api/logs.py',
'@version': '1', 'stack_info': None, 'message': 'this is info, testing',
'tags': [], 'http_host': '192.168.11.187:5301',
'@timestamp': '2019-06-18T10:37:16.624Z',
'host': 'aifashion-app-test', 'level': 'INFO',
'server_port': '5301', 'port': 35824,
'logger_name': 'python-logstash-logger',
'type': 'logstash', 'remote_addr': '192.168.11.181'},
'sort': [1560854236624]}
查看message
6 kibana展示日志和查询
打开xxx.xxx.xx.xx:5601出现kibana界面,进入自己的,不用sample_data
点击右上角kibana图标==》add_data ==>use es data==>然后设置索引和时间戳==》首页discover就可以看到数据,如何快速查询数据?
左侧有列,点击加就会将该字段设置到右边列方便查看, 右边列有放大镜,点击直接可以相关搜索。
如图:
其他:error过程