elasticsearch 是基于 lucene 的搜索引擎,我们在项目中 当做数据库使用,具有 查询快等 优点,下面是我在 项目 中 用到的 方法,以防忘记!
service 层 封装 QueryBuilder/FilterBuilder 语句,相当于 sql
代码如下:
package com.venustech.tsoc.cupid.botnet.service;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import com.venustech.tsoc.cupid.commons.mvc.Pager;
import org.nutz.ioc.loader.annotation.IocBean;
import com.venustech.tsoc.cupid.botnet.util.EsUtiOfBotnet;
import com.venustech.tsoc.cupid.botnet.util.SortUtil;
import com.venustech.tsoc.cupid.security.bean.BotNet;
import com.venustech.tsoc.cupid.customer.bean.Botnet;;
@IocBean
public class BotnetEsService {
/**
* 获取 首页的江苏Ck 个数
* @return
*/
public Map<String,Object> getStatisticNum(){
EsUtiOfBotnet esUti= EsUtiOfBotnet.getInstance();
QueryBuilder queryBuilder1=QueryBuilders.wildcardQuery("bigArea", "*电信");
QueryBuilder queryBuilderCkType=QueryBuilders.termQuery("type", "3");
QueryBuilder queryBuilderCCType=QueryBuilders.termQuery("type", "1");
QueryBuilder queryBuilderUrlType=QueryBuilders.termQuery("type", "2");
QueryBuilder queryBuilderCC=QueryBuilders.boolQuery().must(queryBuilder1).must(queryBuilderCCType);
QueryBuilder queryBuilderCk=QueryBuilders.boolQuery().must(queryBuilder1).must(queryBuilderCkType);
QueryBuilder queryBuilderUrl=QueryBuilders.boolQuery().must(queryBuilderUrlType);
long numberCC=esUti.getNumber(queryBuilderCC);
long numberCk=esUti.getNumber(queryBuilderCk);
long numberUrl=esUti.getNumber(queryBuilderUrl);
HashMap<String,Object> hashMap = new HashMap<String,Object>();
hashMap.put("cc", numberCC);
hashMap.put("ck", numberCk);
hashMap.put("url", numberUrl);
return hashMap;
}
/**
*
* @param isBig 是否显示省份还是省份之下
* @param isMap 是不是地图数据 还是 饼形图数据
* @param bigArea 所要查询的地区
* @param state 状态 1：疑似 2：确认
* @param type 1：CC 3：肉鸡
* @return
* 地图和饼形图都要 排序
*/
public LinkedList<BotNet> getChartAndMap(boolean isBig,boolean isMap,String bigArea,String state,String type,String ccIp){
EsUtiOfBotnet esUti= EsUtiOfBotnet.getInstance();
SortUtil sortUtil = SortUtil.getInstance();
LinkedList<BotNet> Data = new LinkedList<BotNet>();
if(bigArea==null) return new LinkedList<BotNet>();
String index="botnet";
//地图 ： 小地图的 情况就有一种，就是首页展示的右侧江苏地图，大地图的,其他的都是 大地图
if(isMap){
LinkedList<BotNet> mapData=new LinkedList<BotNet>();
if(isBig){
// 僵木蠕模块 大地图 疑似与 确认全取
QueryBuilder queryBuilderChina=QueryBuilders.wildcardQuery("bigArea", "*电信");
QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", type);
Data = esUti.getMapData(QueryBuilders.boolQuery().must(queryBuilderChina).must(queryBuilderType), "bigArea",false);
}else{
// 首页地图
QueryBuilder queryBuilderChina=QueryBuilders.termQuery("bigArea", "江苏电信");
QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", type);
mapData = esUti.getMapOrChartsData(index,QueryBuilders.boolQuery().must(queryBuilderChina).must(queryBuilderType), "smallArea");
Data=sortUtil.sort2(sortUtil.getJiangSuCityInfo(mapData));
}
}
// 饼形图：
else{
LinkedList<BotNet> chartsData=new LinkedList<BotNet>();
if(isBig){
// 省城，电信，运营商，其他。。。
if(bigArea.equals("运营商")){
QueryBuilder queryBuilderArea=QueryBuilders.termQuery("bigArea", null);
QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", type);
QueryBuilder queryBuilderState=QueryBuilders.termQuery("state", state);
if(ccIp!=null && !ccIp.equals("")){
QueryBuilder queryBuilderCCip=QueryBuilders.termQuery("ccIp", ccIp);
chartsData=esUti.getMapOrChartsData("botnetinfo",QueryBuilders.boolQuery().mustNot(queryBuilderArea).must(queryBuilderCCip).must(queryBuilderState), "bigArea");
}else{
chartsData=esUti.getMapOrChartsData(index,QueryBuilders.boolQuery().mustNot(queryBuilderArea).must(queryBuilderState).must(queryBuilderType), "bigArea");
}
}else{
QueryBuilder queryBuilderArea=QueryBuilders.wildcardQuery("bigArea", "*"+bigArea+"*");
QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", type);
QueryBuilder queryBuilderState=QueryBuilders.termQuery("state", state);
if(ccIp!=null && !ccIp.equals("")){
QueryBuilder queryBuilderCCip=QueryBuilders.termQuery("ccIp", ccIp);
chartsData=esUti.getMapOrChartsData("botnetinfo",QueryBuilders.boolQuery().must(queryBuilderArea).must(queryBuilderCCip).must(queryBuilderState), "bigArea");
}else{
chartsData=esUti.getMapOrChartsData(index,QueryBuilders.boolQuery().must(queryBuilderArea).must(queryBuilderState).must(queryBuilderType), "bigArea");
}
}
}else{
// 小地区
QueryBuilder queryBuilderArea=QueryBuilders.termQuery("bigArea", bigArea);
QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", type);
QueryBuilder queryBuilderState=QueryBuilders.termQuery("state", state);
if(ccIp!=null && !ccIp.equals("")){
QueryBuilder queryBuilderCCip=QueryBuilders.termQuery("ccIp", ccIp);
chartsData=esUti.getMapOrChartsData("botnetinfo",QueryBuilders.boolQuery().must(queryBuilderArea).must(queryBuilderCCip).must(queryBuilderState), "area");
}else{
chartsData=esUti.getMapOrChartsData(index,QueryBuilders.boolQuery().must(queryBuilderArea).must(queryBuilderState).must(queryBuilderType), "smallArea");
}
}
Data=sortUtil.sort(chartsData);
// 降序 排列
Collections.sort( Data, new Comparator<BotNet>(){//list
@Override
public int compare(BotNet b1, BotNet b2) {
if(b1.getNu()>b2.getNu())return -1;
else if(b1.getNu()<b2.getNu())return 1;
else return 0;
}
});
}
return Data;
}
/**
* 根据区域bigArea(全国电信,江苏电信,重庆电信等来显示肉鸡的地理分布),仅限定 电信
* 不区分确认与疑似,当为全国时 显示 bigArea ,当时某个省份时,则显示 二级地理信息 即为small,此方法可
* 继续扩展
* @param Area
* type : 1-CC ;3-Ck
* @return
*/
public LinkedList<BotNet> getMapDataByArea(String area,String type){
EsUtiOfBotnet esUti= EsUtiOfBotnet.getInstance();
SortUtil sortUtil = SortUtil.getInstance();
LinkedList<BotNet> Data = new LinkedList<BotNet>();
if(area==null) return new LinkedList<BotNet>();
String field="smallArea";
QueryBuilder queryBuilderArea=null;
if(area.equals("全国电信")){
field="bigArea";
// 僵木蠕模块 大地图 疑似与 确认全取
queryBuilderArea=QueryBuilders.wildcardQuery("bigArea", "*电信");
}else{
queryBuilderArea=QueryBuilders.termQuery("bigArea", area);
}
QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", type);
Data = esUti.getMapData(QueryBuilders.boolQuery().must(queryBuilderArea).must(queryBuilderType), field,true);
// 降序 排列
Collections.sort( Data, new Comparator<BotNet>(){//list
@Override
public int compare(BotNet b1, BotNet b2) {
if(b1.getNu()>b2.getNu())return -1;
else if(b1.getNu()<b2.getNu())return 1;
else return 0;
}
});
return Data;
}
/**
* 分页查询
* @param queryBuilder
* @param pager
* @return
*/
public Pager<Botnet> paging(QueryBuilder queryBuilder,Pager<Botnet> pager){
EsUtiOfBotnet esUti= EsUtiOfBotnet.getInstance();
return esUti.getList(queryBuilder, pager);
}
/**
* 增加僵木蠕到es
*/
public void addBotnet(Botnet bot){
EsUtiOfBotnet.getInstance().insertBotnetInfo(bot);
}
/**
* 删除 僵木蠕方法
* @param long[] lids
*/
public void deleteBotnet(long[] lids,long type){
EsUtiOfBotnet.getInstance().deleteInfoByLid(lids,type,"true");
}
/**
* 由 key 获取 botnet 信息，返回一个
* @param lid
* @return
*/
public Botnet find(String key,String value){
return EsUtiOfBotnet.getInstance().getBotnetInfoById(key,value);
}
/**
* 检验是否存在
* @param args
*/
public boolean checkInfo(String key,String value){
boolean flag=true;
Botnet botnet = EsUtiOfBotnet.getInstance().getBotnetInfoById(key,value);
if(botnet!=null){
flag=false;
}
return flag;
}
/**
* 更新数据
* @param bot
*/
public void updateBotnet(Botnet bot){
EsUtiOfBotnet.getInstance().editBotnetBykey("lid", bot.getLid()+"", bot);
}
/**
* 同步更新botnetinfo数据(肉鸡且状态改变)
* @param bot
*/
public void updateBotnetInfo(Botnet bot){
if(bot.getType()==3){
EsUtiOfBotnet.getInstance().updateBotnetMaping(bot.getIp(), bot);
}
}
public static void main(String[] args) {
BotnetEsService botnetEsService = new BotnetEsService();
//botnetEsService.getStatisticNum();
}
}
package com.venustech.tsoc.cupid.botnet.util;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.log4j.Logger;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.FilterBuilder;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
import org.elasticsearch.search.sort.SortOrder;
import com.venustech.tsoc.cupid.botnet.analysis.cache.BotnetCache;
import com.venustech.tsoc.cupid.botnet.analysis.cache.IpLibraryCache;
import com.venustech.tsoc.cupid.commons.mvc.Pager;
import com.venustech.tsoc.cupid.customer.bean.Botnet;
import com.venustech.tsoc.cupid.customer.bean.ccBotnetInfo;
import com.venustech.tsoc.cupid.knowledgebase.botnet.data.BotnetBean;
import com.venustech.tsoc.cupid.origins.port.IspInfo.cache.ISPCache;
import com.venustech.tsoc.cupid.security.bean.BotNet;
import com.venustech.tsoc.cupid.util.es.ESClient;
import com.venustech.tsoc.cupid.util.es.ESClient.Type;
import com.venustech.tsoc.cupid.util.es.ESClientFactory;
/**
* es 上面的 知识库 操作工具类
* @author me
* botnet
*/
public class EsUtiOfBotnet {
private Logger logger = Logger.getLogger(this.getClass());
private EsUtiOfBotnet(){}
private static EsUtiOfBotnet esUtil=null;
public static EsUtiOfBotnet getInstance(){
if (esUtil==null){
esUtil=new EsUtiOfBotnet();
}
return esUtil;
}
public static HashMap<String,Boolean> checkCCIPMap=new HashMap<String,Boolean>();
/**
* 是否为调试
*/
private static boolean Debug = false;
// 获取 client
//ESClient cupidClient = ESClientFactory.me().getDefaultDelegateClient();
private static ESClient buildClient(){
Settings settings = ImmutableSettings.settingsBuilder()//cupid-es-zhangli
.put("cluster.name","cupid-cdn-index")//cupid-cdn-index-> 132; cupid-es ->90
.put("client.transport.sniff", true).build();
@SuppressWarnings("resource")// 192.168.56.49
TransportClient client1 = new TransportClient(settings)//192.168.19.98 192.168.19.90 132.224.196.232
.addTransportAddress(new InetSocketTransportAddress("192.168.56.49", 8300));
ESClient client = new ESClient(client1, Type.cdn);
return client;
}
//cupidClient = ESClientFactory.me().getDefaultDelegateClient();
// 获取 首页江苏肉鸡的 个数
public ESClient getClient(){
ESClient cupidClient = null;
try {
cupidClient = ESClientFactory.me().getDefaultDelegateClient();
} catch (Exception e) {
}
return (!Debug)?cupidClient
:buildClient();
}
/**
* 首页 展示的 江苏 肉鸡与CC 个数
* @param queryBuilder
* @return
*/
public long getNumber(QueryBuilder queryBuilder){
long number=0;
ESClient client = this.getClient();
/*Settings settings = ImmutableSettings.settingsBuilder()//cupid-es-zhangli
.put("cluster.name","cupid-cdn-index")
.put("client.transport.sniff", true).build();
@SuppressWarnings("resource")
TransportClient client1 = new TransportClient(settings)//192.168.19.98
.addTransportAddress(new InetSocketTransportAddress("132.224.196.232", 8300));
ESClient client = new ESClient(client1, Type.cdn);*/
boolean existsIndex = isExistsIndex("botnet");
if(existsIndex){
number= client.prepareCount("botnet")//索引
//.setTypes("botnet")// 类型
.setQuery(queryBuilder)// ES 默认查询是10 条,想要改变的话 可以在此处 设置 size setSize(100)
.execute()
.actionGet().getCount();
}
return number;
}
/**
* 僵木蠕模块 地图数据
* @param queryBuilder
* @return
*/
public LinkedList<BotNet> getMapOrChartsData(String index,QueryBuilder queryBuilder,String field){
LinkedList<BotNet> mapData = new LinkedList<BotNet>();
ESClient client = this.getClient();
boolean existsIndex = isExistsIndex("botnet");
if(existsIndex){
//number= delegateClient.prepareCount("botnet")//索引
SearchRequestBuilder srb=client.prepareSearch(index);
// srb.setTypes(index);
srb.setSearchType(SearchType.COUNT);
AggregationBuilder<?> aggregation = AggregationBuilders.terms("na").field(field).size(Integer.MAX_VALUE);
srb.setQuery(queryBuilder);
srb.addAggregation(aggregation);
SearchResponse respose=srb.execute().actionGet();
//System.out.println("result:"+respose);
Map<String, Aggregation> asMap = respose.getAggregations().asMap();
StringTerms result =(StringTerms) asMap.get("na");
Iterator<Bucket> iterator = result.getBuckets().iterator();
while(iterator.hasNext()){
Bucket gradeBucket = iterator.next();
BotNet botNet = new BotNet();
String key = gradeBucket.getKey();
Long count = gradeBucket.getDocCount();
botNet.setNa(key);
botNet.setNu(count.intValue());
mapData.add(botNet);
}
}
return mapData;
}
/**
* 分页查询
* @param queryBuilderm
*/
public Pager<Botnet> getList(QueryBuilder queryBuilder,Pager<Botnet> pager){
ESClient client = this.getClient();
List<Botnet> list = new ArrayList<Botnet>();
long number=0;
int pageFrom=(pager.getPage()-1)*pager.getPageSize();
// 总页
boolean existsIndex = isExistsIndex("botnet");
if(existsIndex){
// 数据
SearchResponse respose = client
.prepareSearch("botnet")//索引
//.setTypes("botnet")// 类型
.setQuery(queryBuilder)// ES 默认查询是10 条,想要改变的话 可以在此处 设置 size setSize(100)
.addSort("createTime", SortOrder.DESC)// 将序
.setFrom(pageFrom).setSize(pager.getPageSize())
.execute()
.actionGet();
SearchHits hits = respose.getHits();// 命中数
number=hits.getTotalHits();
SearchHit[] searchHits = hits.getHits();
if(searchHits.length>0){
for (SearchHit searchHit : searchHits) {
Map<String, Object> source = searchHit.getSource();
Botnet bo = new Botnet();
long lid=(long)source.get("lid");
long type=Long.parseLong(source.get("type").toString());
long state=Long.parseLong(source.get("state").toString());
long source1=Long.parseLong(source.get("source").toString());
long create=Long.parseLong(source.get("createTime").toString());
long update=Long.parseLong(source.get("updateTime").toString());
long port =Long.parseLong(source.get("port").toString());
if(update==0){
update=create;
}
String ip=source.get("ip")==null ?"":source.get("ip").toString();
String url=source.get("url")==null ?"":source.get("url").toString();
String note=source.get("note")==null ?"":source.get("note").toString();
String createTime=getStrTime(create);
String updateTime=getStrTime(update);
bo.setIp(ip);
bo.setUrl(url);
bo.setLid(lid);
bo.setType(type);
bo.setSource(source1);
bo.setNote(note);
bo.setState(state);
bo.setAction(createTime);// 创建时间
bo.setProt(updateTime);// 更新时间
bo.setPort(port);
list.add(bo);
}
}
}
pager.setRecords(number);
pager.setData(list);
return pager;
}
/**
* 初始化 botnet 到缓存
*/
public void getAllBotnetbyType(String type1,int size){
//BotnetCache.botnets.put(key, value)
// QueryBuilders.
logger.info("初始化 --- botnet-"+type1+"-加载到缓存开始");
long start=System.currentTimeMillis();
ESClient client = this.getClient();
boolean existsIndex = isExistsIndex("botnet");
if(existsIndex){
// 数据
QueryBuilder queryBuilder=null;
QueryBuilder queryBuilder1=QueryBuilders.boolQuery().must(QueryBuilders.termQuery("type", type1));
if(type1.equals("3")){
//QueryBuilder queryBuilder2 = QueryBuilders.rangeQuery("createTime").from(System.currentTimeMillis()-(60*1000*60*24*5)).to(System.currentTimeMillis());
queryBuilder = QueryBuilders.boolQuery().must(queryBuilder1);//.must(queryBuilder2)
}else{
queryBuilder = QueryBuilders.boolQuery().must(queryBuilder1);
}
SearchResponse respose = client
.prepareSearch("botnet")//索引
.setTypes("botnet")// 类型
.setQuery(queryBuilder)
.setSize(size)
.execute()
.actionGet();
SearchHits hits = respose.getHits();// 命中数
logger.info("botnet-type-"+type1+" -个数："+hits.getTotalHits());
SearchHit[] searchHits = hits.getHits();
if(searchHits.length>0){
for (SearchHit searchHit : searchHits) {
Map<String, Object> source = searchHit.getSource();
BotnetBean bo = new BotnetBean();
long type=Long.parseLong(source.get("type").toString());
long state=Long.parseLong(source.get("state").toString());
String ip=source.get("ip")==null ?"":source.get("ip").toString();
String url=source.get("url")==null ?"":source.get("url").toString();
bo.setIp(ip);
bo.setUrl(url);
bo.setType(Integer.parseInt(type+""));
bo.setState(Integer.parseInt(state+""));
// 加入 初始化map
// url
if(type1.equals("2")){
BotnetCache.botnets.put(url, bo);
}
// cc + ck
else{
BotnetCache.botnets.put(ip, bo);
}
}
}
logger.info("botnet 加载进条数 ："+(searchHits.length) +"");
logger.info("botnet 加载进 缓存结束，共耗时："+(System.currentTimeMillis()-start) +"毫秒");
}
}
/**
* 初始化 botnet ck -- 到缓存 每次 24000
*/
public void getAllBotnetCkData(int size){
logger.info("开始加载肉鸡到内存----start botnet--2");
int allSize=0;
long start=System.currentTimeMillis();
ESClient client = this.getClient();//
boolean existsIndex = isExistsIndex("botnet");
if(existsIndex){
SearchResponse repose = client.prepareSearch("botnet")
//.setTypes("botnet")
.setQuery(QueryBuilders.termQuery("type", 3))
.setScroll(new TimeValue(600000))
.setSize(2000)
.setSearchType(SearchType.SCAN).execute()
.actionGet();
String scrollId = repose.getScrollId();
try {
while(true){
long start2=System.currentTimeMillis();
SearchResponse repose2 = client.prepareSearchScroll(scrollId)
.setScroll(new TimeValue(size*1000))
.execute().actionGet();
SearchHits hits = repose2.getHits();
if(hits.getHits().length==0){
break;
}
SearchHit[] hits2 = hits.getHits();
allSize+=hits2.length;
//System.out.println("size:"+hits2.length);
for (SearchHit searchHit : hits2) {
Map<String, Object> source = searchHit.getSource();
BotnetBean bo = new BotnetBean();
long type=Long.parseLong(source.get("type").toString());
long state=Long.parseLong(source.get("state").toString());
String ip=source.get("ip")==null ?"":source.get("ip").toString();
String url=source.get("url")==null ?"":source.get("url").toString();
bo.setIp(ip);
bo.setUrl(url);
bo.setType(Integer.parseInt(type+""));
bo.setState(Integer.parseInt(state+""));
BotnetCache.botnets.put(ip, bo);
}
//System.out.println("耗时："+(System.currentTimeMillis()-start2));
}
logger.info("botnet-肉鸡 导入缓存 结束..... 共耗时："+(System.currentTimeMillis()-start) +"毫秒 ，总条数" +allSize +"条");
BotnetCache.overFlag=true;
} catch (NumberFormatException e) {
// TODO Auto-generated catch block
//e.printStackTrace();
} catch (ElasticsearchException e) {
// TODO Auto-generated catch block
//e.printStackTrace();
}
}else{
/*
* 若是此时,botnet 不存在,则设置 默认读取完成
* */
BotnetCache.overFlag=true;
}
}
/**
* 验证 ip/url 是否在 botnet 中 true ： 含有
* @param key
* @param value
* @return
*/
public boolean checkBotnetkey_in(String key,String value,String type,String stateFlag){
logger.info(" --- botnet_检索开始 判断库中是否含有此Key:"+value);
long start=System.currentTimeMillis();
ESClient client = this.getClient();//
long number = 1;
boolean existsIndex = isExistsIndex("botnet");
if(existsIndex){
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
queryBuilder.must(QueryBuilders.termQuery("type", type)).must(QueryBuilders.termQuery(key, value));
if(stateFlag!=null && !stateFlag.equals("")){
long state=stateFlag.equals("true")?2:1;
queryBuilder.must(QueryBuilders.termQuery("state", state));
}
SearchResponse repose = client.prepareSearch("botnet")//索引
//.setTypes("botnet")// 类型
.setQuery(queryBuilder)// ES 默认查询是10 条,想要改变的话 可以在此处 设置 size setSize(100)
.setSize(1)
.execute()
.actionGet();
number=repose.getHits().getTotalHits();
if(number>0 && type.equals("3")){
int state=0;
if(!stateFlag.equals("")){
state=stateFlag.equals("true")?2:1;
}
BotnetCache.botnets.put(value, new BotnetBean(value,"",state,3));
}
logger.info("botnet 检索结束，共耗时："+(System.currentTimeMillis()-start) +"毫秒");
}
return number>0 ? true:false;
}
/**
* 僵木蠕模块 大地图数据
* @param queryBuilder
* @param isAll true:合并 疑似与确认 : false : 分开聚合
* @return
*/
public LinkedList<BotNet> getMapData(QueryBuilder queryBuilder,String field,boolean isAll){
LinkedList<BotNet> mapData = new LinkedList<BotNet>();
ESClient client = this.getClient();
boolean existsIndex = isExistsIndex("botnet");
if(existsIndex){
SearchRequestBuilder srb=client.prepareSearch("botnet");
//srb.setTypes("botnet");
srb.setSearchType(SearchType.COUNT);
srb.setQuery(queryBuilder);
AggregationBuilder<?> aggregationField = AggregationBuilders.terms("na").field(field).size(Integer.MAX_VALUE);
if(isAll==false){
AggregationBuilder<?> aggregationState = AggregationBuilders.terms("state").field("state").size(Integer.MAX_VALUE);
aggregationField.subAggregation(aggregationState);
}
srb.addAggregation(aggregationField);
SearchResponse respose=srb.execute().actionGet();
Map<String, Aggregation> asMap = respose.getAggregations().asMap();
StringTerms result =(StringTerms) asMap.get("na");
Iterator<Bucket> iterator = result.getBuckets().iterator();
// 合并疑似与确认
if(isAll){
while(iterator.hasNext()){
BotNet botNet = new BotNet();
Bucket fieldBucket = iterator.next();
String key = fieldBucket.getKey();
Long botnetNum= fieldBucket.getDocCount();
botNet.setNa(key);//区域
botNet.setNu(botnetNum.intValue());
mapData.add(botNet);
}
}
// 分开聚合
else{
while(iterator.hasNext()){
Bucket gradeBucket = iterator.next();
String key = gradeBucket.getKey();
LongTerms result2 =(LongTerms)gradeBucket.getAggregations().asMap().get("state");
Iterator<Bucket> iterator2 = result2.getBuckets().iterator();
while(iterator2.hasNext()){
Bucket next = iterator2.next();
BotNet botNet = new BotNet();
int keyState = Integer.parseInt(next.getKey()) ;
Long count = next.getDocCount();
botNet.setState(keyState);
botNet.setNa(key);
botNet.setNu(count.intValue());
mapData.add(botNet);
}
}
}
}
return mapData;
}
/**
* 增加一个 僵木蠕信息
* @param hitsArray
* @return
*/
public void insertBotnetInfo(Botnet bo){
// boolean existsIndex = isExistsIndex("botnet");
//if(existsIndex){
ESClient client = this.getClient();
if(client != null){
long lid=new Date().getTime()*10000+new Random().nextInt(10000);
bo.setLid(lid);
bo.setIsLive(1);
String jsonData=SortUtil.toJson(bo);
client.prepareIndex("botnet", "botnet").setId(new Date().getTime()*10000+new Random().nextInt(10000)+"")
.setSource(jsonData)
.setId(lid+"")
.execute()
.actionGet();
}
//}
}
/**
* 批量 添加 到 es botnet
* @param botnets
*/
public void insertBotnets(List<BotnetBean> botnets){
//boolean existsIndex = isExistsIndex("botnet");
//if(existsIndex){
if(botnets ==null || botnets.size()==0) return;
// 类型转化
List<Botnet> botnet_esList = convertTypeList(botnets);
//开启批量插入
ESClient client = this.getClient();
if(client != null){
BulkRequestBuilder bulkRequest = client.prepareBulk();
for (Botnet bo: botnet_esList) {
String jsonData=SortUtil.toJson(bo);
bulkRequest.add(client.prepareIndex("botnet", "botnet").setId(new Date().getTime()*10000+new Random().nextInt(10000)+"").setSource(jsonData));
}
bulkRequest.execute().actionGet();
logger.info("批量插入(botnet)数据："+botnet_esList.size());
}
//}
}
/**
* 批量 添加 到 es botnetinfo 关联表,
* 首次添加 还没有 ccIp
* @param botnets
* @param isQuartzDelete 是否为定时清理调度任务后的 insert 到 bontnet
*/
public void insertBotnetInfos(List<BotnetBean> botnets, boolean isQuartzDelete ){
// boolean existsIndex = isExistsIndex("botnetinfo");
// if(existsIndex){
if(botnets ==null || botnets.size()==0) return;
// 类型转化
List<ccBotnetInfo> botnet_esList = null;
if(isQuartzDelete==false){
botnet_esList=convertTypeList2(botnets,"frist");
}else{
botnet_esList=convertTypeList3(botnets,"frist");
}
//开启批量插入
ESClient client = this.getClient();
if(client != null){
BulkRequestBuilder bulkRequest = client.prepareBulk();
for (ccBotnetInfo bo: botnet_esList) {
checkCCIPMap.put(bo.getCkIp(), true);
String jsonData=SortUtil.toJson(bo);
bulkRequest.add(client.prepareIndex("botnetinfo", "botnetinfo").setId(new Date().getTime()*10000+new Random().nextInt(10000)+"").setSource(jsonData));
}
bulkRequest.execute().actionGet();
logger.info("批量插入(botnetinfo-ck)数据："+botnet_esList.size());
}
//}
}
/**
* update
* @param botnets
*/
public void updateBotnetInfos(List<ccBotnetInfo> botnets){
boolean existsIndex = isExistsIndex("botnetinfo");
if(existsIndex){
if(botnets ==null || botnets.size()==0) return;
//开启批量插入
ESClient client = this.getClient();
BulkRequestBuilder bulkRequest = client.prepareBulk();
for (ccBotnetInfo bo: botnets) {
String jsonData=SortUtil.toJson(bo);
bulkRequest.add(client.prepareIndex("botnetinfo", "botnetinfo").setId(new Date().getTime()*10000+new Random().nextInt(10000)+"").setSource(jsonData));
}
bulkRequest.execute().actionGet();
logger.info("首次批量update(botnetinfo-ck)数据："+botnets.size());
}
}
/**
* 验证关联表(botnetinfo)中 是否 含义ckIp +ccIp
* @return
*/
public boolean isContains(String ckIp,String ccIp){
long number = 1;
boolean existsIndex = isExistsIndex("botnetinfo");
if(existsIndex){
ESClient client = this.getClient();
BoolQueryBuilder query = QueryBuilders.boolQuery();
QueryBuilder queryBuilderCkIp=QueryBuilders.termQuery("ckIp", ckIp);
if(!ccIp.equals("afterDeleteccIp")){
QueryBuilder queryBuilderCcIp=QueryBuilders.termQuery("ccIp", ccIp);
// QueryBuilder queryBuilderPort=QueryBuilders.termQuery("port", port);
query.must(queryBuilderCkIp).must(queryBuilderCcIp);
}else{
query.must(queryBuilderCkIp);
}
number = client.prepareCount("botnetinfo")
//.setTypes("botnetinfo")
.setQuery(query)
.execute()
.actionGet().getCount();
}
return number>0?true:false;
}
/**
* 增加 映射关系 ， ccIp 储存在 note 属性中
*/
public void addMapping(List<BotnetBean> botnets){
if(botnets ==null || botnets.size()==0) return;
// 此时的 状态是 已经验证过 botnet ,含有此 ckIp ccIp
// 所以此时 需要 看是否 要更新 或者 添加一条 映射 关系，
// 先判断是否还有 空着 ccIp 的 ckIp
//1 .有的话 先 更新 含有 ccIp 2. 没有空着的 则 新增加一条
// 开启 批量添加
// 转化 类型
boolean existsIndex = isExistsIndex("botnetinfo");
if(existsIndex){
List<ccBotnetInfo> bot_info = convertTypeList2(botnets,"");
ESClient client = this.getClient();
BulkRequestBuilder bulkRequest = client.prepareBulk();
for (ccBotnetInfo b : bot_info) {
if(checkCCIPMap.get(b.getCkIp())!=null && checkCCIPMap.get(b.getCkIp())==true){
long lid=ccIpisNull(b.getCkIp());
// 更新
if(lid!=1){
// 先删除在 增加
deleteBotnetInfoByLid(lid);
checkCCIPMap.put(b.getCkIp(), false);
}
}
// 添加
String jsonData=SortUtil.toJson(b);
bulkRequest.add(client.prepareIndex("botnetinfo", "botnetinfo").setId(new Date().getTime()*10000+new Random().nextInt(10000)+"").setSource(jsonData));
}
if(bot_info.size()>0){
bulkRequest.execute().actionGet();
}
logger.info("批量插入(botnetinfo--mapping)数据："+bot_info.size());
}
}
/**
* 先判断是否还有 空着 ccIp 的 ckIp
* @return
*/
public long ccIpisNull(String ckIp){
long lid=1;
ESClient client = this.getClient();
boolean existsIndex = isExistsIndex("botnetinfo");
if(existsIndex){
SearchResponse respose = client.prepareSearch("botnetinfo").setTypes("botnetinfo")
.setQuery(QueryBuilders.termQuery("ckIp", ckIp))
.addSort("createTime", SortOrder.ASC)
.execute()
.actionGet();
SearchHits hits = respose.getHits();
SearchHit[] hits2 = hits.getHits();
if(hits2.length>0){
for (SearchHit searchHit : hits2) {
Map<String, Object> source = searchHit.getSource();
if(source.get("ccIp")==null || source.get("ccIp").equals("")){
lid=(long)source.get("lid");
break;
}
}
}
}
return lid;
}
/**
* 增加到 关联表 中
* @param bo
*/
public ccBotnetInfo insertBotnet_info(Botnet bo){
long lid=new Date().getTime()*10000+new Random().nextInt(10000);
ccBotnetInfo ccBotnetInfo = new ccBotnetInfo();
ccBotnetInfo.setArea(bo.getSmallArea());
//ccBotnetInfo.setCcIp(bo.get);
ccBotnetInfo.setCkIp(bo.getIp());
ccBotnetInfo.setLid(lid);
//ccBotnetInfo.set
ccBotnetInfo.setCreateTime(System.currentTimeMillis());
return ccBotnetInfo;
}
/**
* 更新 或增加
* @param botnets
*/
public void insertOrUpdateBotnetInfos(List<BotnetBean> botnets){
}
/**
* 判断 botnetinfo 中 含有 ckIp 的 是否
*/
/**
* 类型转换 过度
* @param bean
* @return
*/
public List<Botnet> convertTypeList(List<BotnetBean> botnets){
ArrayList<Botnet> botnet_es = new ArrayList<Botnet>();
for (BotnetBean bean : botnets) {
Botnet botnet = new Botnet();
long lid=new Date().getTime()*10000+new Random().nextInt(10000);
botnet.setLid(lid);
botnet.setType(bean.getType());
botnet.setBigArea(bean.getBigArea());
botnet.setSmallArea(bean.getSmallArea());
botnet.setIp( bean.getIp());
botnet.setUrl(bean.getUrl());
botnet.setSource(bean.getSource());
botnet.setFirstCreateTime(System.currentTimeMillis());
botnet.setCreateTime(System.currentTimeMillis());
botnet.setState(bean.getState());
botnet.setNote(bean.getNote());
botnet.setIsLive(1);
botnet.setPort(bean.getPort());
botnet.setUpdateTime(System.currentTimeMillis());
if(botnet.getIp().equals("")){
logger.info("将要插入的数据-URL:"+bean.getUrl() );
}else{
logger.info("将要插入的数据为-IP:"+ bean.getIp());
}
logger.info(" 备注:"+bean.getNote());
botnet_es.add(botnet);
}
return botnet_es;
}
/**
* 类型转换 过度 ccBotnetInfo 添加到 botnetinfo
*
* 里面 还没有 添加 ccIp
* @param bean
* @return
*/
public List<ccBotnetInfo> convertTypeList2(List<BotnetBean> botnets,String flag){
ArrayList<ccBotnetInfo> botnet_es = new ArrayList<ccBotnetInfo>();
for (BotnetBean bean : botnets) {
ccBotnetInfo botnet = new ccBotnetInfo();
long lid=new Date().getTime()*10000+new Random().nextInt(10000);
botnet.setLid(lid);
botnet.setBigArea(bean.getBigArea());
botnet.setArea(bean.getSmallArea());
botnet.setCkIp( bean.getIp());
botnet.setCreateTime(System.currentTimeMillis());
if(!flag.equals("frist")){
botnet.setCcIp(bean.getNote());
}
botnet.setState(bean.getState());
botnet.setPort(bean.getPort());
botnet.setIsLive(1);
botnet_es.add(botnet);
}
return botnet_es;
}
/**
* 类型转换 过度 ccBotnetInfo 添加到 botnetinfo
*
* 里面 还没有 添加 ccIp
* @param bean
* @return
*/
public List<ccBotnetInfo> convertTypeList3(List<BotnetBean> botnets,String flag){
ArrayList<ccBotnetInfo> botnet_es = new ArrayList<ccBotnetInfo>();
for (BotnetBean bean : botnets) {
ccBotnetInfo botnet = new ccBotnetInfo();
long lid=new Date().getTime()*10000+new Random().nextInt(10000);
botnet.setLid(lid);
botnet.setBigArea(bean.getBigArea());
botnet.setArea(bean.getSmallArea());
botnet.setCkIp( bean.getIp());
botnet.setCreateTime(bean.getCreateTime());
if(!flag.equals("frist")){
botnet.setCcIp(bean.getNote());
}
botnet.setState(bean.getState());
botnet.setPort(bean.getPort());
botnet.setIsLive(1);
botnet_es.add(botnet);
}
return botnet_es;
}
/**
* 改变知识库信息
* @param
* @return
*/
public void updateBotnetInfo(Botnet bo){
boolean existsIndex = isExistsIndex("botnet");
if(existsIndex){
ESClient client = this.getClient();
long[] lid=new long[1];
lid[0]=bo.getLid();
deleteInfoByLid(lid,bo.getType(),"false");
bo.setUpdateTime(System.currentTimeMillis());
String jsonData=SortUtil.toJson(bo);
client.prepareIndex("botnet", "botnet").setId(new Date().getTime()*10000+new Random().nextInt(10000)+"")
.setSource(jsonData)
.execute()
.actionGet();
}
//efge
}
/**
*
* @param key 表示键 比如 lid ip url等 具有唯一性的
* @param value 键的值
* @param botnet 要更新的信息储存在 其中
*/
public void editBotnetBykey(String key, String value, Botnet bean){
ESClient client = this.getClient();
boolean existsIndex = isExistsIndex("botnet");
if(existsIndex){
QueryBuilder queryBuilderKey=QueryBuilders.termQuery(key, value);
QueryBuilder queryBuilder=QueryBuilders.boolQuery().must(queryBuilderKey);
SearchResponse repose = client.prepareSearch("botnet")
.setTypes("botnet")
.setQuery(queryBuilder)
.setSize(Integer.MAX_VALUE)
.execute()
.actionGet();
SearchHits hits = repose.getHits();
SearchHit[] hits2 = hits.getHits();
long oldState = bean.getReliability();
if(hits2.length>0){
// 获取文档 ID
String docId = hits2[0].getId();
//更新 update
bean.setUpdateTime(System.currentTimeMillis());
bean.setReliability(0);
String jsonData=SortUtil.toJson(bean);
// 追加到 es 上
client.prepareIndex("botnet", "botnet")
.setId(docId)
.setSource(jsonData)
.execute()
.actionGet();
}
// 更新关联表的条件: 1: 为肉鸡ip ; 2: 状态值发生变化 state(oldState 放入了reliability )
if(bean.getType() == 3 && bean.getState() != oldState){
updateBotnetMaping(bean.getIp(), bean);
}
}
}
/**
* 更新关联信息
* @param ip
*/
public void updateBotnetMaping(String ckIp,Botnet bo){
boolean existsIndex = isExistsIndex("botnetinfo");
if(existsIndex){
ESClient client = this.getClient();
@SuppressWarnings("deprecation")
QueryBuilder queryBuilder=QueryBuilders.termQuery("ckIp", ckIp);
SearchResponse respose = client.prepareSearch("botnetinfo")
//.setTypes("botnetinfo")
.setQuery(queryBuilder)
.setSize(Integer.MAX_VALUE)
.execute()
.actionGet();
SearchHits hits = respose.getHits();
SearchHit[] hits2 = hits.getHits();
long state_new=bo.getState();
if(hits2.length>0){
for (SearchHit searchHit : hits2) {
ccBotnetInfo mapping = new ccBotnetInfo();
Map<String, Object> source = searchHit.getSource();
String docId = searchHit.getId();
long lid=Long.parseLong(source.get("lid").toString());
String ccIp=source.get("ccIp")==null ?"":source.get("ccIp").toString();
// String ck_Ip=bo.getIp();
String note=source.get("note")==null ?"":source.get("note").toString();
String countryCode=source.get("countryCode")==null ?"":source.get("countryCode").toString();
long createTime=Long.parseLong(source.get("createTime").toString());
long port=Long.parseLong(source.get("port").toString());
mapping.setLid(lid);
mapping.setBigArea(bo.getBigArea());
mapping.setArea(bo.getSmallArea());
mapping.setNote(note);
mapping.setCreateTime(createTime);
mapping.setIsLive(bo.getIsLive());
mapping.setPort(port);
mapping.setState(state_new);
mapping.setCountryCode(countryCode);
mapping.setCcIp(ccIp);
mapping.setCkIp(bo.getIp());
String jsonData=SortUtil.toJson(mapping);
// 追加到 es 上
client.prepareIndex("botnetinfo", "botnetinfo")
.setId(docId)
.setSource(jsonData)
.execute()
.actionGet();
}
}
}
}
/**
* 删除 僵木蠕信息 long [lids]
* @param lids
* @param type
* @param isDelete true: 表示连 关联表 botnetinfo 中的 数据也删: false :表示只删除botnet,不删 botnetinfo
*/
public void deleteInfoByLid(long[] lids,long type,String isDelete){
if(lids.length<=0) return;
boolean existsIndex = isExistsIndex("botnet");
if(existsIndex){
ESClient client = this.getClient();
@SuppressWarnings("deprecation")
QueryBuilder queryBuilder=QueryBuilders.inQuery("lid", lids);
SearchResponse respose = client.prepareSearch("botnet")
.setTypes("botnet")
.setQuery(queryBuilder)
.setSize(Integer.MAX_VALUE)
.execute()
.actionGet();
SearchHits hits = respose.getHits();
SearchHit[] hits2 = hits.getHits();
int arrSize=Integer.parseInt(hits.getTotalHits()+"");
// 储存要删除的ip ck
ArrayList<String> cKipList = new ArrayList<String>();
ArrayList<String> ccIpList = new ArrayList<String>();// cc
boolean flag= type==3?true:false;
BulkRequestBuilder bulkRefresh = client.prepareBulk().setRefresh(true);
int i=0;
for (SearchHit searchHit : hits2) {
// 1. 删除 botnet
DeleteRequest deleteRequest = new DeleteRequest("botnet","botnet",searchHit.getId());
bulkRefresh.add(deleteRequest);
Map<String, Object> obj = searchHit.getSource();
// 2. 为删除botnetinfo 做出准备，记录ips
if(type!=2){
String ip= obj.get("ip")==null ?"":obj.get("ip").toString();
if(flag){
cKipList.add(ip);
}
if(type==1){
ccIpList.add(ip);
}
BotnetCache.botnets.remove(ip);
}
// 去除url 内存
if(type==2){
String url= obj.get("url")==null ?"":obj.get("url").toString();
BotnetCache.botnets.remove(url);
}
i++;
}
bulkRefresh.execute().actionGet();
/*for (String ip : ips) {
System.out.println("要删除的ip:"+ip);
// 去内存
BotnetCache.botnets.remove(ip);
}*/
String[] array=new String[cKipList.size()];
String[] ckips = cKipList.toArray(array);
String[] array2=new String[ccIpList.size()];
String[] ccips = ccIpList.toArray(array2);
if(ckips.length>0 && flag && isDelete.equals("true")){
deleteBotnetInfo("ckIp",ckips);
}
if(ccips.length>0){
deleteBotnetInfo("ccIp",ccips);
}
}
}
/**
* inQuery() size<=1024 filter 则是释放的
* String field : ckIp /ccIp
* @param ips
*/
public void deleteBotnetInfo(String field,String[] ips){
if(ips==null || ips.length<=0) return;
ESClient client = this.getClient();
boolean existsIndex = isExistsIndex("botnetinfo");
if(existsIndex){
//QueryBuilder queryBuilder=QueryBuilders.inQuery("ckIp", ips);
@SuppressWarnings("deprecation")
FilterBuilder inFilter = FilterBuilders.inFilter(field, ips);
SearchResponse respose = client.prepareSearch("botnetinfo")
.setTypes("botnetinfo")
.setPostFilter(inFilter)
.setSize(Integer.MAX_VALUE)
.execute()
.actionGet();
SearchHits hits = respose.getHits();
SearchHit[] hits2 = hits.getHits();
BulkRequestBuilder bulkRefresh = client.prepareBulk().setRefresh(true);
// 储存要删除的ip ck
ArrayList<String> cKipList = new ArrayList<String>();
for (SearchHit searchHit : hits2) {
// 2. 为删除botnetinfo 做出准备，记录ips
DeleteRequest deleteRequest = new DeleteRequest("botnetinfo","botnetinfo",searchHit.getId());
bulkRefresh.add(deleteRequest);
// 如果删除的是CC 应该判断 把Ck的全部删完，若是全部删完则因该空部位即ccip=""
if(field.equals("ccIp")){
// 记录要删除的 ckIp
Map<String, Object> obj = searchHit.getSource();
String ckIp= obj.get("ckIp")==null ?"":obj.get("ckIp").toString();
if(ckIp!=null && !ckIp.equals("")){
if(!cKipList.contains(ckIp)){
cKipList.add(ckIp);
}
}
}
}
if(hits2.length>0){
bulkRefresh.execute().actionGet();
}
// 判断 info 中 ck--size
String[] array=new String[cKipList.size()];
String[] ckips = cKipList.toArray(array);
if(ckips!=null && ckips.length >0){
try {
//Thread.sleep(3000);
afterDeleteCC(ckips);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
//afterDeleteCC(ckips);
}
// inQuery() 里面若为 数组的话，是有限制的 大小为 1024，所以改为 filter
}
}
/**
* 判断botnetinfo 中 ckIp的size,
*/
public void afterDeleteCC(String[] ckips){
List<BotnetBean> BotnetInfos =new ArrayList<BotnetBean>();
for (String ckIp : ckips) {
Botnet oldInfo = getBotnetInfoById("ip",ckIp);// 检验botnet 根还存在
if( oldInfo!=null){
boolean flag = isContains(ckIp,"afterDeleteccIp");//检验botnetinfo
if(flag==true ){
// 表明 ckip 还存在，不用补位
}else{
// 现在 需要补位 List<BotnetBean> botnets
BotnetBean bin =new BotnetBean();
bin.setIp(ckIp);// 同事设定了 ipLong
/*if (bin.getIpLong() > 0) {
bin.setArea(ISPCache.getInstance().getArea(bin.getIpLong()));
}*/
bin.setBigArea(oldInfo.getBigArea());
bin.setSmallArea(oldInfo.getSmallArea());
bin.setState(Integer.parseInt(oldInfo.getState()+""));
// 设置 createTime
bin.setCreateTime(oldInfo.getCreateTime());
BotnetInfos.add(bin);
}
}
}
if(BotnetInfos.size()>0){
insertBotnetInfos(BotnetInfos,true);
}
}
public void deleteBotnetInfoByLids(long[] lids){
if(lids==null || lids.length<=0) return;
boolean existsIndex = isExistsIndex("botnetinfo");
if(existsIndex){
ESClient client = this.getClient();
@SuppressWarnings("deprecation")
QueryBuilder queryBuilder=QueryBuilders.inQuery("lid", lids);
SearchResponse respose = client.prepareSearch("botnetinfo")
//.setTypes("botnetinfo")
.setQuery(queryBuilder)
.setSize(Integer.MAX_VALUE)
.execute()
.actionGet();
SearchHits hits = respose.getHits();
SearchHit[] hits2 = hits.getHits();
BulkRequestBuilder bulkRefresh = client.prepareBulk().setRefresh(true);
for (SearchHit searchHit : hits2) {
DeleteRequest deleteRequest = new DeleteRequest("botnetinfo","botnetinfo",searchHit.getId());
bulkRefresh.add(deleteRequest);
}
if(hits2.length>0){
bulkRefresh.execute().actionGet();
}
}
}
/**
*
* @param lid
*/
public void deleteBotnetInfoByLid(long lid){
ESClient client = this.getClient();
boolean existsIndex = isExistsIndex("botnetinfo");
if(existsIndex){
@SuppressWarnings("deprecation")
QueryBuilder queryBuilder=QueryBuilders.termQuery("lid", lid);
SearchResponse respose = client.prepareSearch("botnetinfo")
//.setTypes("botnetinfo")
.setQuery(queryBuilder)
.setSize(Integer.MAX_VALUE)
.execute()
.actionGet();
SearchHits hits = respose.getHits();
SearchHit[] hits2 = hits.getHits();
BulkRequestBuilder bulkRefresh = client.prepareBulk().setRefresh(true);
for (SearchHit searchHit : hits2) {
DeleteRequest deleteRequest = new DeleteRequest("botnetinfo","botnetinfo",searchHit.getId());
bulkRefresh.add(deleteRequest);
}
bulkRefresh.execute().actionGet();
}
}
/**
* 由lid 获取
* @param lid
* @return
*/
public Botnet getBotnetInfoById(String key,String value){
Botnet botnet = null;
ESClient client = this.getClient();
QueryBuilder queryBuilder1=QueryBuilders.termQuery(key, value);
//QueryBuilder queryBuilder=QueryBuilders.boolQuery().must(queryBuilder1);
boolean existsIndex = isExistsIndex("botnet");
if(existsIndex){
SearchResponse respose = client.prepareSearch("botnet")
.setTypes("botnet")
.setQuery(queryBuilder1)
.execute()
.actionGet();
SearchHits hits = respose.getHits();
SearchHit[] hitsArr = hits.getHits();
if(hitsArr.length>0){
Map<String, Object> source = hitsArr[0].getSource();
// System.out.println("id:"+hitsArr[0].getId());
Botnet bo = new Botnet();
long lid2=(long)source.get("lid");
long type=Long.parseLong(source.get("type").toString());
long state=Long.parseLong(source.get("state").toString());
long source1=Long.parseLong(source.get("source").toString());
long createTime=Long.parseLong(source.get("createTime").toString());
long firstCreateTime=Long.parseLong(source.get("firstCreateTime").toString());
long create=createTime;
long update=Long.parseLong(source.get("updateTime").toString());
if(update==0){
update=create;
}
long port=Long.parseLong(source.get("port").toString());
String ip=source.get("ip")==null?"":source.get("ip").toString();
String url=source.get("url")==null?"":source.get("url").toString();
String note=source.get("note")==null?"":source.get("note").toString();
String bigArea=source.get("bigArea")==null?"":source.get("bigArea").toString();
String smallArea=source.get("smallArea")==null?"":source.get("smallArea").toString();
bo.setIp(ip);
bo.setUrl(url);
bo.setLid(lid2);
bo.setType(type);
bo.setSource(source1);
bo.setPort(port);
bo.setNote(note);
bo.setBigArea(bigArea);
bo.setSmallArea(smallArea);
bo.setState(state);
bo.setCreateTime(createTime);
bo.setFirstCreateTime(firstCreateTime);
bo.setAction(getStrTime(createTime));// 创建时间
bo.setProt(getStrTime(update));// 更新时间
botnet=bo;
}
}
return botnet;
}
/**
* 定时清理库中的 过期的 僵木蠕-肉鸡-信息
*/
public int deleteBotnetTimerTask(String field,long deleteInterval,String isOldData){
ESClient client = this.getClient();
int num = 0;
boolean existsIndex = isExistsIndex("botnet");
if(existsIndex){
// 小于 正常期的 都是即将要删除掉的 包括 肉鸡 与 cc+URL
// QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", "3");
long nowTime=System.currentTimeMillis();
long cycileTime=nowTime-deleteInterval;
QueryBuilder queryBuilderTime=QueryBuilders.rangeQuery(field).lt(cycileTime);
BoolQueryBuilder queryBuilder=QueryBuilders.boolQuery();
queryBuilder.must(queryBuilderTime);
// 老数据 中 updateTime 都是 0,所以 依 createTime 为准
QueryBuilder queryBuilderCreateTime=QueryBuilders.termQuery("updateTime", "0");
if(isOldData.equals("oldData")){
queryBuilder.must(queryBuilderCreateTime);
}else{
queryBuilder.mustNot(queryBuilderCreateTime);
}
// 1.选出你要 删除的 条目 ，获取 ip
SearchResponse repose = client.prepareSearch("botnet")
.setTypes("botnet")
.setQuery(queryBuilder)
.setSize(Integer.MAX_VALUE)
.execute()
.actionGet();
SearchHits hits = repose.getHits();
SearchHit[] hits2 = hits.getHits();
// 开启 批量删除
BulkRequestBuilder bulkRefresh = client.prepareBulk().setRefresh(true);
// 储存要删除的ip ck
ArrayList<String> ipList = new ArrayList<String>();
ArrayList<String> ccIpList = new ArrayList<String>();// cc
if(hits2.length>0){
for (SearchHit searchHit : hits2) {
DeleteRequest deleteRequest = new DeleteRequest("botnet", "botnet", searchHit.getId());
bulkRefresh.add(deleteRequest);
// get IP
Map<String, Object> obj = searchHit.getSource();
long type=obj.get("type")==null?0:Integer.parseInt(obj.get("type").toString());
if(type==2){
String url=obj.get("url")==null?"":obj.get("url").toString();
BotnetCache.botnets.remove(url);
}else{
String ip=obj.get("ip")==null?"":obj.get("ip").toString();
// 提高效率 去除 type=1 的情况
if(type==3){
ipList.add(ip);
}
if(type==1){
ccIpList.add(ip);
}
// 清除 缓存 信息
BotnetCache.botnets.remove(ip);
}
}
// 删除 botnet 中的 不合理 信息
bulkRefresh.execute().actionGet();
}
String[] array=new String[ipList.size()];
String[] ips = ipList.toArray(array);
String[] array2=new String[ccIpList.size()];
String[] ccips = ccIpList.toArray(array2);
// 同步清理 botnetinfo ckIp
if(ips.length>0){
deleteBotnetInfo("ckIp",ips);
}
// ccIp
if(ccips.length>0){
deleteBotnetInfo("ccIp",ccips);
}
num = ips.length;
}
return num;
}
/**
* 监测更新 ,update-->updateTime
*
* key： ip/url
*/
public void monitorUpdateInfo(String key,String value){
ESClient client = this.getClient();
// 小于 正常期的 都是即将要删除掉的
boolean existsIndex = isExistsIndex("botnet");
if(existsIndex){
QueryBuilder queryBuilder=null;
/*if(!key.equals("url")){
queryBuilder= QueryBuilders.termQuery(key, value);
}else{
queryBuilder= QueryBuilders.queryString(value).field(key);
}*/
queryBuilder= QueryBuilders.termQuery(key, value);
SearchResponse repose = client.prepareSearch("botnet")
//.setTypes("botnet")
.setQuery(queryBuilder)
.setSize(Integer.MAX_VALUE)
.execute()
.actionGet();
SearchHits hits = repose.getHits();
SearchHit[] hits2 = hits.getHits();
if(hits2.length>0){
// 获取文档 ID
String docId = hits2[0].getId();
Map<String, Object> source = hits2[0].getSource();
Botnet bo = new Botnet();
long lid2=(long)source.get("lid");
long type=Long.parseLong(source.get("type").toString());
long ipLong=Long.parseLong(source.get("ipLong").toString());
long port=Long.parseLong(source.get("port").toString());
long isLive=Long.parseLong(source.get("isLive").toString());
long reliability=Long.parseLong(source.get("reliability").toString());
long state=Long.parseLong(source.get("state").toString());
long source1=Long.parseLong(source.get("source").toString());
long createTime=Long.parseLong(source.get("createTime").toString());
long firstCreateTime=Long.parseLong(source.get("firstCreateTime").toString());
String ip=source.get("ip")==null?"":source.get("ip").toString();
String url=source.get("url")==null?"":source.get("url").toString();
String note=source.get("note")==null?"":source.get("note").toString();
String bigArea=source.get("bigArea")==null?"":source.get("bigArea").toString();
String smallArea=source.get("smallArea")==null?"":source.get("smallArea").toString();
bo.setIp(ip);
bo.setUrl(url);
bo.setLid(lid2);
bo.setType(type);
bo.setSource(source1);
bo.setNote(note);
bo.setBigArea(bigArea);
bo.setState(state);
bo.setCreateTime(createTime);
bo.setFirstCreateTime(firstCreateTime);
bo.setIsLive(isLive);
bo.setSmallArea(smallArea);
bo.setReliability(reliability);
bo.setIpLong(ipLong);
bo.setPort(port);
//更新 update
bo.setUpdateTime(System.currentTimeMillis());
String jsonData=SortUtil.toJson(bo);
// 追加到 es 上
client.prepareIndex("botnet", "botnet")
.setId(docId)
.setSource(jsonData)
.execute()
.actionGet();
}
}
}
/**
* update-->updateTime
* key： ip
*/
public void UpdateCCInfo(String key,String value,BotnetBean bean){
ESClient client = this.getClient();
// 小于 正常期的 都是即将要删除掉的
boolean existsIndex = isExistsIndex("botnet");
if(existsIndex){
QueryBuilder queryBuilderKey=QueryBuilders.termQuery(key, value);
QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", "1");
QueryBuilder queryBuilder=QueryBuilders.boolQuery().must(queryBuilderType).must(queryBuilderKey);
SearchResponse repose = client.prepareSearch("botnet")
.setTypes("botnet")
.setQuery(queryBuilder)
.setSize(Integer.MAX_VALUE)
.execute()
.actionGet();
SearchHits hits = repose.getHits();
SearchHit[] hits2 = hits.getHits();
if(hits2.length>0){
// 获取文档 ID
String docId = hits2[0].getId();
Map<String, Object> source = hits2[0].getSource();
Botnet bo = new Botnet();
long lid2=(long)source.get("lid");
long type=Long.parseLong(source.get("type").toString());
long ipLong=Long.parseLong(source.get("ipLong").toString());
long port=Long.parseLong(source.get("port").toString());
long isLive=Long.parseLong(source.get("isLive").toString());
long reliability=Long.parseLong(source.get("reliability").toString());
long state=bean.getState();
long source1=bean.getSource();
long createTime=Long.parseLong(source.get("createTime").toString());
long firstCreateTime=Long.parseLong(source.get("firstCreateTime").toString());
String ip=source.get("ip")==null?"":source.get("ip").toString();
String url=source.get("url")==null?"":source.get("url").toString();
String note=bean.getNote();
String bigArea=source.get("bigArea")==null?"":source.get("bigArea").toString();
String smallArea=source.get("smallArea")==null?"":source.get("smallArea").toString();
bo.setIp(ip);
bo.setUrl(url);
bo.setLid(lid2);
bo.setType(type);
bo.setSource(source1);
bo.setNote(note);
bo.setBigArea(bigArea);
bo.setState(state);
bo.setCreateTime(createTime);
bo.setFirstCreateTime(firstCreateTime);
bo.setIsLive(isLive);
bo.setSmallArea(smallArea);
bo.setReliability(reliability);
bo.setIpLong(ipLong);
bo.setPort(port);
//更新 update
bo.setUpdateTime(System.currentTimeMillis());
String jsonData=SortUtil.toJson(bo);
// 追加到 es 上
client.prepareIndex("botnet", "botnet")
.setId(docId)
.setSource(jsonData)
.execute()
.actionGet();
}
}
}
/**
* 时间格式化
* @return
*/
public String getStrTime(long time){
SimpleDateFormat sdf= new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Date date = new Date(time);
String format = sdf.format(date);
return format;
}
/*public List<BotNet> getList(SearchHit[] hitsArray){
List<BotNet> botList=new LinkedList<BotNet>();
for (SearchHit searchHit : hitsArray) {
BotNet botNet = new BotNet();
Map<String, Object> source = searchHit.getSource();
//botNet.setNa(source.get(""));
}
return botList;
}*/
/**
* 判断指定的索引名是否存在
* @param indexName 索引名
* @return 存在：true; 不存在：false;
*/
public boolean isExistsIndex(String indexName){
boolean isExist = false;
// 获取 client
ESClient client = this.getClient();
if(client != null){
try {
IndicesExistsResponse response =
client.admin().indices().exists(
new IndicesExistsRequest().indices(new String[]{indexName})).actionGet();
isExist = response.isExists();
} catch (ElasticsearchException e) {
logger.info("ES 发生异常:"+e);
e.printStackTrace();
isExist = false;
}
}
return isExist;
}
/**
* 判断指定的索引的类型是否存在
* @param indexName 索引名
* @param indexType 索引类型
* @return 存在：true; 不存在：false;
*/
public boolean isExistsType(String indexName,String indexType){
//ESClient client = this.getClient();
TypesExistsResponse response =
getClient().admin().indices()
.typesExists(new TypesExistsRequest(new String[]{indexName}, indexType)
).actionGet();
return response.isExists();
}
}
937
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
