logstash每天往es建好索引,按天生成,就目前的需求,需要清理不需要的数据,以保证最新日志的速度展示,哈哈,瞎搞了这个脚本,路过的大神批评。
#!/usr/bin/env python
# coding:utf8
#author: shantuwqk@163.com
import sys,datetime import os,time,commands HOST = '192.168.35.125' PORT = '9200' #排除不需要删除的索引列表 NotDEL = ['g1_sms_collect'] def Del_Indices(last4): TMP_LIST = [] DEL_DAY_LIST = [] file_info = '/tmp/es_indices_list.txt' CURLCMD = 'curl http://%s:%s/_cat/indices?v > %s'%(HOST,PORT,file_info) CURLINDEX_INFO = "cat /tmp/es_indices_list.txt | grep -v 'health status index'|egrep '*-%s'|awk '{print $3}'"%last4 #DEL_DAY_INFO = "%s|awk '{print $3}'"%CURLINDEX_INFO s,v = commands.getstatusoutput(CURLCMD) if s == 0: with open(file_info,'r') as f: st,vi = commands.getstatusoutput(CURLINDEX_INFO) TMP_LIST.append(vi) #DEL_DAY_LIST.append(vi) else: print "\033[31;1m curl cmd execute faild\033[0m" # print DEL_DAY_LIST for i in TMP_LIST[0].split('\n'): DEL_DAY_LIST.append(i) #sd,vd = commands.getstatusoutput('curl -XDELETE http://%s:%s/%s'%(HOST,PORT,i)) #print sd,vd, print DEL_DAY_LIST try: for t in NotDEL: DEL_DAY_LIST.remove("%s-%s"%(t,last4)) except Exception: pass for index in DEL_DAY_LIST: sd,vd = commands.getstatusoutput('curl -XDELETE http://%s:%s/%s'%(HOST,PORT,index)) if __name__ == '__main__': last4day = (datetime.date.today() - datetime.timedelta(days=3)).strftime('%Y.%m.%d') #print last4day Del_Indices(last4day)
最近由于es shard 老出问题, 在网上看有优化的功能,所以又扩展了以下脚本:做日常定时任务,主要保含清理和优化索引功能
#!/usr/bin/env python
# coding:utf8
#author:shantuwqk@163.com
import sys,datetime
import os,time,commands
import sys,os
HOST = 'localhost'
PORT = '9200'
#ES_ROOT = os.path.dirname(os.path.dirname(__file__))
ES_ROOT = '/'.join(os.getcwd().split('/')[:-1])
ES_BIN = ES_ROOT + '/bin'
ES_INDICES = ES_ROOT + '/data/elasticsearch/nodes/0/indices'
ES_OPTIMIZE = "_optimize"
ES_DEL_VAL = "_optimize?only_expunge_deletes=true"
print ES_ROOT
print ES_BIN
print ES_INDICES
def Indiceslist(date):
indiceslist = []
s,v = commands.getstatusoutput("ls %s -l| awk '{print $9}'|egrep \"*-%s\""%(ES_INDICES,date))
#print "#########indices",s,v
if s == 0:
for i in v.split('\n'):
indiceslist.append(i)
return indiceslist
#print indiceslist
def Del_Indices(date):
NotDEL = ['g1_sms_collect']
DEL_DAY_LIST = []
print date,
print Indiceslist(date)
for i in Indiceslist(date):
print i,
DEL_DAY_LIST.append(i)
try:
for t in NotDEL:
DEL_DAY_LIST.remove("%s-%s"%(t,date))
except Exception:
pass
print DEL_DAY_LIST
for index in DEL_DAY_LIST:
sd,vd = commands.getstatusoutput('curl -XDELETE http://%s:%s/%s'%(HOST,PORT,index))
if sd == 0:
print "DEL indices Sucess !!"
else:
print "DEL indices Failed !!"
ds,dv = commands.getstatusoutput('curl -XPOST http://%s:%s/%s/%s'%(HOST,PORT,index,ES_DEL_VAL))
if ds == 0:
print "Cel indices Sucess !!"
else:
print "Cel indices Failed !!"
def es_optimize(date):
#print Indiceslist(date)
for i in Indiceslist(date):
print "curl -XPOST \"http://%s:%s/%s/%s\""%(HOST,PORT,i,ES_OPTIMIZE)
s,v = commands.getstatusoutput("curl -XPOST \"http://%s:%s/%s/%s\""%(HOST,PORT,i,ES_OPTIMIZE))
print s,v,
if s == 0:
print "\033[32;1mES Optimize Success !!!\033[0m"
else:
print "\033[31;1m Es Optimze Failed!!!\033[0m"
def handle_cluster_status():
clustet_status = "curl -XGET \'http://localhost:9200/_cluster/health?pretty=true\'"
def help_prompt():
print """
This program prints files to the standard output.
Options include:
--version : Prints the version number
--help : Helpful tips
--task : To operate on missions [delindices|optimze]
sample : python es_manage.py --task delindices
: python es_manage.py --task optimze
"""
def error_prompt():
print '\033[31;1mUnknown option.Please refer to the help...\033[0m'
print "==============================================================="
help_prompt()
print "---------------------------------------------------------------"
sys.exit(1)
if __name__ == '__main__':
if len(sys.argv) == 1:
print "\033[31;1m No action specified.\033[0m"
help_prompt()
sys.exit(1)
if len(sys.argv) == 2 and sys.argv[1].startswith('--'):
if sys.argv[1][2:] == 'version':
print 'Version 1.0'
sys.exit(1)
elif sys.argv[1][2:] == 'help':
help_prompt()
if len(sys.argv) == 3 and sys.argv[1][2:] == 'task' and sys.argv[2] == "delindices":
DelDay = (datetime.date.today() - datetime.timedelta(days=3)).strftime('%Y.%m.%d')
Del_Indices(DelDay)
elif len(sys.argv) == 3 and sys.argv[1][2:] == 'task' and sys.argv[2] == "optimze":
yesterday = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y.%m.%d')
today = (datetime.date.today() - datetime.timedelta(days=0)).strftime('%Y.%m.%d')
print es_optimize(yesterday)
es_optimize(today)
else:
error_prompt()
exit(1)