docker:
sudo yum install -y yum-utils device-mapper-persistent-data lvm2
sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
sudo yum install docker-ce
sudo systemctl start docker
sudo docker run hello-world
# docker中-d参数是在后台运行, 若调试的话, 可以改为--rm
Install Elasticsearch with Docker
sudo docker pull docker.elastic.co/elasticsearch/elasticsearch:6.4.2
sys config
/etc/sysctl.conf中添加: vm.max_map_count=262144
sudo grep vm.max_map_count /etc/sysctl.conf
run:
docker run -p 9200:9200 -p 9300:9300 -d --name elasticsearch -d -e "discovery.type=single-node" -e network.publish_host=0.0.0.0 docker.elastic.co/elasticsearch/elasticsearch:6.4.2
inspect status of cluster:
curl http://127.0.0.1:9200/_cat/health
查看索引: curl -X GET 'http://localhost:9200/_cat/indices?v'
查看type: curl 'localhost:9200/_mapping?pretty=true'
新建 Index: curl -X PUT 'localhost:9200/weather'
查询记录: curl 'localhost:9200/accounts/person/_search' # /Index/Type/_search
Kibana
docker pull docker.elastic.co/kibana/kibana:6.4.2
docker run -p 5601:5601 --name kibana -d --link elasticsearch -e ELASTICSEARCH_URL=http://10.97.88.71:9200 -e elasticsearch.ssl.verify=false -e server.host=0.0.0.0 docker.elastic.co/kibana/kibana:6.4.2
# elasticsearch 为 Elasticsearch 的docker name
# 10.97.88.71, 使用docker的时候, 不能用localhost
http://10.97.88.71:5601
Logstash
docker pull docker.elastic.co/logstash/logstash:6.4.2
docker run --name logstash --rm -p 5144:5144 --link elasticsearch -e xpack.monitoring.enabled=true -e ELASTICSEARCH_URL=http://10.97.88.71:9200 -v ~/pipeline/:/usr/share/logstash/pipeline/ docker.elastic.co/logstash/logstash:6.4.2
配置文件 logstash.yml
input{
syslog {
type => "icc_rpc_log"
port => "5144"
}
}
filter {
if [type] == "icc_rpc_log"{
grok {
patterns_dir => "/usr/local/logstash/patterns" //设置自定义正则路径
# match => { "message" => "%{IP:client_id_address} %{LOGLEVEL:loglevel}" }
match => { "message" => "%{TIMESTAMP_ISO8601:log_date} %{LOGLEVEL:loglevel}\:index code %{NUMBER:index_code} is invoked by %{IP:client_id_address}\(%{DATA:user_cd}\-%{DATA:user_name}\) with parameter_list %{DATA:parameter_list} \+ default_parameter %{GREEDYDATA:parameter_default}" }
}
}
}
output {
if [type] == "icc_rpc_log" and [loglevel] == "INFO"{
stdout { codec => rubydebug }
}
if [type] == "icc_rpc_log" and [loglevel] == "ERROR"{
elasticsearch {
hosts => ["10.97.88.71:9200"]
index => "icc_calc_log"
# index => "system-syslog-log-%{+YYYY.MM.dd}"
}
}
}
测试数据:
2018-10-19 08:50:47 INFO:index code 000000001 is invoked by 1.2.2.2(-Anonymous) with parameter_list [{'asset_code': '000001', 'benm_code': '000002', 'yield_date_type': None, 'yield_type': None}] + default_parameter {'end_date': '2017-05-01', 'start_date': '2017-05-01', 'freq_code': 'D', 'riskfree_benm_code': '000003', 'annual_flag': False}
其他相关站点:
Logstash 最佳实践: https://doc.yonyoucloud.com/doc/logstash-best-practice-cn/output/elasticsearch.html
Kibana(一张图片胜过千万行日志): https://www.cnblogs.com/cjsblog/p/9476813.html
kibana visualize添加自定义查询: https://blog.csdn.net/xr568897472/article/details/71540937
全文搜索引擎 Elasticsearch 入门教程: http://www.ruanyifeng.com/blog/2017/08/elasticsearch.html
使用Docker搭建ELK日志系统: http://chenzhijun.me/2017/12/27/elk-docker/
logstash-patterns-core: https://github.com/logstash-plugins/logstash-patterns-core/blob/master/patterns/grok-patterns
1分钟搭建Elasticsearch可视化(没试过): https://blog.csdn.net/dounine/article/details/78887792
Elasticsearch 查询
curl -X GET 'localhost:9200/icc_calc_log/_search' -H 'Content-Type: application/json' -d '{"query":{"match":{"message":"invoked"}}}'
import calendar
import datetime
import requests
import json
"""
curl -X GET 'localhost:9200/icc_calc_log/_search' -H 'Content-Type: application/json'
-d '{"query":{"match":{"message":"invoked"}}}'
?size=3
"""
# json_param = {"query": {"match": {"message": "invoked"}}}
# json_param = {"query": {"range": {"log_date": {"gte": "2018-01-01", "lte": "2018-12-31"}}}}
# json_param = {"query": {"range": {"client_id_address": {"gte": "1.2.2.1", "lte": "1.2.2.2"}}}}
# json_param = {"query": {"regexp": {"log_date": "2018.*?"}}}
# def get_log_date_reg():
# start_month_end_date = calendar.monthrange(start_date.year, start_date.month)[1]
# end_month_end_date = calendar.monthrange(end_date.year, end_date.month)[1]
# year_list, month_list, day_list = [], [], []
#
# year_list = [str(i) for i in range(start_date.year + 1, end_date.year)]
#
# if start_date.year != end_date.year:
# for i in range(start_date.month + 1, 13):
# month_list.append(str(start_date.year) + "-" + str(i).rjust(2, "0"))
#
# for i in range(1, end_date.month):
# month_list.append(str(end_date.year) + "-" + str(i).rjust(2, "0"))
# else:
# for i in range(start_date.month + 1, end_date.month):
# month_list.append(str(start_date.year) + "-" + str(i).rjust(2, "0"))
#
# if start_date.day == 1:
# month_list.append(str(start_date.year) + "-" + str(start_date.month).rjust(2, "0"))
# start_date_is_month_begin_date_flag = True
# else:
# start_date_is_month_begin_date_flag = False
# if end_date.day == end_month_end_date:
# month_list.append(str(end_date.year) + "-" + str(end_date.month).rjust(2, "0"))
# end_date_is_month_end_date_flag = True
# else:
# end_date_is_month_end_date_flag = False
#
# if start_date_is_month_begin_date_flag is False:
# for i in range(start_date.day, start_month_end_date + 1):
# day_list.append(str(start_date.year) + "-" + str(start_date.month).rjust(2, "0") + "-" + str(i).rjust(2, "0"))
# if end_date_is_month_end_date_flag is False:
# for i in range(1, end_date.day + 1):
# day_list.append(str(end_date.year) + "-" + str(end_date.month).rjust(2, "0") + "-" + str(i).rjust(2, "0"))
#
# print(year_list)
# print(month_list)
# print(day_list)
# result = year_list + month_list + day_list
# result_str = "|".join(result)
# print(result_str)
# return result_str
# # "log_date": "2018-10-19 08:50:47",
ip = "10.97.88.71"
index = "icc_calc_log"
file_name = r"./qqqqqqqqqqqqqqqqqqqqq_view_log"
def save_result(result, f_handler):
# print(json.dumps(result, ensure_ascii=False, indent=4))
result_list = result.get("hits", {}).get("hits", [])
for result_now in result_list:
f_handler.write(json.dumps(result_now, ensure_ascii=False, indent=4))
f_handler.write("\n")
f_handler.flush()
return result["_scroll_id"]
def start():
# query_param = {"query": {"regexp": {"log_date": "{0}".format(get_log_date_reg())}}, "post_filter": {"regexp": {"message": "invoked"}}}
start_date, end_date = "2018-01-01", "2018-12-31"
query_param = {"query": {"range": {"@timestamp": {"gte": start_date, "lte": end_date}}}, "post_filter": {"regexp": {"message": "invoked"}}}
with open(file_name, 'w') as f:
result = requests.post(r"http://{0}:9200/{1}/_search?size=1&scroll=1m".format(ip, index), json=query_param).json()
_scroll_id = save_result(result, f)
while True:
result = requests.post(r"http://{0}:9200/_search/scroll?scroll=2m&scroll_id={1}".format(ip, _scroll_id), json={}).json()
_scroll_id = save_result(result, f)
if not result.get("hits", {}).get("hits"):
break
if __name__ == "__main__":
start()