ELK（分布式大数据搜索和日志挖掘及可视化）之实战（先不要审核只是保存一些记录）

logstash 配置

input{

 file{

     path =>"opt/logs/*.sql"

     type =>"logjson"

     start_position => "beginning"

     sincedb_path =>"/dev/null"

  }

}

filter{

  json{

  #将默认中的message内容转换成json内容，并删除message域

  source => "message"

  remove_field =>"message"

  }

}

output{

 elasticsearch{

  hosts =>["172.16.117.93:9200"]

index =>"query"

document_type=> "%{type}"

flush_size=>20000

idle_flush_time =>10

  }

}

es

public class ESsearch{

static Essearch ts=new ESearch();

static Client client =ts.TransportClientContect();

//返回client对象

public Client TransportClientContect(){

Settings settings =Setting.settingsBuilder().put("cluster.name","topic").build();

Client client=null;

try{

client =TransportClient.builder().setting(settings).build()

.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("172.16.117.93"),9300));

}catch(){}

return client;

}

}

}

class ESmain{

static ESearch ts=new ESearch();

static Client client=ts.TransportClientContect();

public static void main(String[] args) throws IOException{

Excel excel =new Excel();

Instant startClock=Instant.now();

System.out.println("开始时间"+startClock );

String index="logstash-sql---3p";

String type="loghson";

String value;

List<String> stringList=new ArrayList<String>();

//scroll模式启动 每次50000

SearchResponse scrollResponse=client.prepareSearch(index)

.setSearchType(SearchType.SCAN).setSize(10000)

.setQuery(QueryBuilders.matchAllQuery())

.setQuery(QueryBuilders.boolQuery()

   .must(QueryBuilders.matchQuery("q1","q1v"))

   .operator(prg.elasticsearch.index.query.MatchQueryBuilder.Operator.AND)

   .must(QueryBuilders.matchQuery("q2","q2v") )

setScroll(TimeVakue.timeValueMinutes（1）)

.execute().actionGet();

int count =(int)scrollResponse.getHits().getTotalHits();//第一次不返回数据

for(int i=0,sum=0;sum<count;i++){

scrollResponse=client.prepareSearchScroll(scrollResponse.getScrollId())

     .setScroll(Timevalue.timeValueMinutes(8))

.execute().actionGet();

sum+=scrollResponse.getHits().hits().length;

  for(SearchHit hit:srollResponse.getHits()){

value=hits.getSource.get("param").toString();

stringList.add(value);

      }

File destFile=new File("./output/"+i+".xls");

try{

excel.createStringExcelFile(stringList,destFile);

}catch(){}

StringList.clear();

System.out.println("总数："+count+"已查到："+sum);

    }

}

}