java 操作 elasticsearch 杂记

原创 2017年01月17日 14:00:36

elasticsearch  是基于 lucene 的搜索引擎,我们在项目中 当做数据库使用,具有 查询快等 优点,下面是我在 项目 中 用到的 方法,以防忘记!

service 层  封装 QueryBuilder/FilterBuilder 语句,相当于 sql

代码如下:

package com.venustech.tsoc.cupid.botnet.service;

import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;

import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;

import com.venustech.tsoc.cupid.commons.mvc.Pager;

import org.nutz.ioc.loader.annotation.IocBean;

import com.venustech.tsoc.cupid.botnet.util.EsUtiOfBotnet;
import com.venustech.tsoc.cupid.botnet.util.SortUtil;
import com.venustech.tsoc.cupid.security.bean.BotNet;
import com.venustech.tsoc.cupid.customer.bean.Botnet;;

@IocBean
public class BotnetEsService {

	
	/**
	 *  获取 首页的江苏Ck 个数
	 * @return
	 */
	public Map<String,Object> getStatisticNum(){
		EsUtiOfBotnet	esUti= EsUtiOfBotnet.getInstance();
		QueryBuilder queryBuilder1=QueryBuilders.wildcardQuery("bigArea", "*电信");
		QueryBuilder queryBuilderCkType=QueryBuilders.termQuery("type", "3");
		QueryBuilder queryBuilderCCType=QueryBuilders.termQuery("type", "1");
		QueryBuilder queryBuilderUrlType=QueryBuilders.termQuery("type", "2");
		QueryBuilder queryBuilderCC=QueryBuilders.boolQuery().must(queryBuilder1).must(queryBuilderCCType);
		QueryBuilder queryBuilderCk=QueryBuilders.boolQuery().must(queryBuilder1).must(queryBuilderCkType);
		QueryBuilder queryBuilderUrl=QueryBuilders.boolQuery().must(queryBuilderUrlType);
		long	numberCC=esUti.getNumber(queryBuilderCC);
		long	numberCk=esUti.getNumber(queryBuilderCk);
		long	numberUrl=esUti.getNumber(queryBuilderUrl);
		HashMap<String,Object> hashMap = new HashMap<String,Object>();
		hashMap.put("cc", numberCC);
		hashMap.put("ck", numberCk);
		hashMap.put("url", numberUrl);
		return hashMap;
	}
	/**
	 * 
	 * @param isBig  是否显示省份还是省份之下
	 * @param isMap  是不是地图数据 还是 饼形图数据
	 * @param bigArea  所要查询的地区
	 * @param state   状态 1:疑似  2:确认
	 * @param type   1:CC   3:肉鸡 
	 * @return
	 * 地图和饼形图都要 排序
	 */
	public LinkedList<BotNet>  getChartAndMap(boolean isBig,boolean isMap,String bigArea,String state,String type,String ccIp){
		EsUtiOfBotnet	esUti= EsUtiOfBotnet.getInstance();
		SortUtil sortUtil = SortUtil.getInstance();
		LinkedList<BotNet> Data = new LinkedList<BotNet>();
		if(bigArea==null) return new LinkedList<BotNet>();
		String index="botnet";
		//地图 : 小地图的 情况就有一种,就是首页展示的右侧江苏地图,大地图的,其他的都是 大地图
		if(isMap){
			LinkedList<BotNet> mapData=new LinkedList<BotNet>();
			if(isBig){
				// 僵木蠕模块 大地图  疑似与 确认全取
				QueryBuilder queryBuilderChina=QueryBuilders.wildcardQuery("bigArea", "*电信");
				QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", type);
				
				Data = esUti.getMapData(QueryBuilders.boolQuery().must(queryBuilderChina).must(queryBuilderType), "bigArea",false);
				
			}else{
				// 首页地图
				QueryBuilder queryBuilderChina=QueryBuilders.termQuery("bigArea", "江苏电信");
				QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", type);
			
				
				 mapData = esUti.getMapOrChartsData(index,QueryBuilders.boolQuery().must(queryBuilderChina).must(queryBuilderType), "smallArea");
				
				 Data=sortUtil.sort2(sortUtil.getJiangSuCityInfo(mapData));
			}
		}
		// 饼形图: 
		else{
			LinkedList<BotNet> chartsData=new LinkedList<BotNet>();
			if(isBig){
				// 省城,电信,运营商,其他。。。
				if(bigArea.equals("运营商")){
					QueryBuilder queryBuilderArea=QueryBuilders.termQuery("bigArea", null);
					QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", type);
					QueryBuilder queryBuilderState=QueryBuilders.termQuery("state", state);
					if(ccIp!=null && !ccIp.equals("")){
						QueryBuilder queryBuilderCCip=QueryBuilders.termQuery("ccIp", ccIp);
						chartsData=esUti.getMapOrChartsData("botnetinfo",QueryBuilders.boolQuery().mustNot(queryBuilderArea).must(queryBuilderCCip).must(queryBuilderState), "bigArea");
					}else{
						
						chartsData=esUti.getMapOrChartsData(index,QueryBuilders.boolQuery().mustNot(queryBuilderArea).must(queryBuilderState).must(queryBuilderType), "bigArea");
					}
					
				}else{
					QueryBuilder queryBuilderArea=QueryBuilders.wildcardQuery("bigArea", "*"+bigArea+"*");
					QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", type);
					QueryBuilder queryBuilderState=QueryBuilders.termQuery("state", state);
					if(ccIp!=null && !ccIp.equals("")){
						QueryBuilder queryBuilderCCip=QueryBuilders.termQuery("ccIp", ccIp);
						chartsData=esUti.getMapOrChartsData("botnetinfo",QueryBuilders.boolQuery().must(queryBuilderArea).must(queryBuilderCCip).must(queryBuilderState), "bigArea");
					}else{
						
						chartsData=esUti.getMapOrChartsData(index,QueryBuilders.boolQuery().must(queryBuilderArea).must(queryBuilderState).must(queryBuilderType), "bigArea");
					}
				}
							
			}else{
				// 小地区
				QueryBuilder queryBuilderArea=QueryBuilders.termQuery("bigArea", bigArea);
				QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", type);
				QueryBuilder queryBuilderState=QueryBuilders.termQuery("state", state);
				if(ccIp!=null && !ccIp.equals("")){
					QueryBuilder queryBuilderCCip=QueryBuilders.termQuery("ccIp", ccIp);
					chartsData=esUti.getMapOrChartsData("botnetinfo",QueryBuilders.boolQuery().must(queryBuilderArea).must(queryBuilderCCip).must(queryBuilderState), "area");	
				}else{
					
					chartsData=esUti.getMapOrChartsData(index,QueryBuilders.boolQuery().must(queryBuilderArea).must(queryBuilderState).must(queryBuilderType), "smallArea");	
				}
			}
			Data=sortUtil.sort(chartsData);
			 // 降序 排列
	     	Collections.sort( Data, new Comparator<BotNet>(){//list
	     		@Override
	     		public int compare(BotNet b1, BotNet b2) {
	     			if(b1.getNu()>b2.getNu())return -1;
	     			else if(b1.getNu()<b2.getNu())return 1;
	     			else return 0;
	     		}
	     	});
		
		}
		
		
		
		return Data;
	}
	/**
	 * 根据区域bigArea(全国电信,江苏电信,重庆电信等来显示肉鸡的地理分布),仅限定 电信
	 * 不区分确认与疑似,当为全国时 显示 bigArea ,当时某个省份时,则显示 二级地理信息  即为small,此方法可
	 * 继续扩展
	 * @param Area  
	 * type : 1-CC ;3-Ck
	 * @return
	 */
	public LinkedList<BotNet>  getMapDataByArea(String area,String type){
		EsUtiOfBotnet	esUti= EsUtiOfBotnet.getInstance();
		SortUtil sortUtil = SortUtil.getInstance();
		LinkedList<BotNet> Data = new LinkedList<BotNet>();
		if(area==null) return new LinkedList<BotNet>();
		String field="smallArea";
		QueryBuilder queryBuilderArea=null;
		if(area.equals("全国电信")){
			field="bigArea";
			// 僵木蠕模块 大地图  疑似与 确认全取
			 queryBuilderArea=QueryBuilders.wildcardQuery("bigArea", "*电信");
		}else{
			 queryBuilderArea=QueryBuilders.termQuery("bigArea", area);
		}
		
		QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", type);
		
		Data = esUti.getMapData(QueryBuilders.boolQuery().must(queryBuilderArea).must(queryBuilderType), field,true);
		// 降序 排列
	 	Collections.sort( Data, new Comparator<BotNet>(){//list
	 		@Override
	 		public int compare(BotNet b1, BotNet b2) {
	 			if(b1.getNu()>b2.getNu())return -1;
	 			else if(b1.getNu()<b2.getNu())return 1;
	 			else return 0;
	 		}
	 	});
		return Data;
	}
	
	/**
	 * 分页查询
	 * @param queryBuilder
	 * @param pager
	 * @return
	 */
	public Pager<Botnet> paging(QueryBuilder queryBuilder,Pager<Botnet> pager){
		EsUtiOfBotnet	esUti= EsUtiOfBotnet.getInstance();
		
		return esUti.getList(queryBuilder, pager);
		
	}
	/**
	 * 增加僵木蠕到es
	 */
	public void addBotnet(Botnet bot){
		
		EsUtiOfBotnet.getInstance().insertBotnetInfo(bot);
	}
	/**
	 * 删除 僵木蠕方法 
	 * @param long[] lids
	 */
	public void deleteBotnet(long[] lids,long type){
		EsUtiOfBotnet.getInstance().deleteInfoByLid(lids,type,"true");
	}
	/**
	 * 由 key  获取 botnet 信息,返回一个
	 * @param lid
	 * @return
	 */
	public Botnet find(String key,String value){
		
		return EsUtiOfBotnet.getInstance().getBotnetInfoById(key,value);
		
	}
	/**
	 * 检验是否存在
	 * @param args
	 */
	public boolean checkInfo(String key,String value){
		boolean flag=true;
		Botnet botnet = EsUtiOfBotnet.getInstance().getBotnetInfoById(key,value);
		if(botnet!=null){
			flag=false;
		}
		return flag;
	}
	/**
	 * 更新数据
	 * @param bot
	 */
	public void updateBotnet(Botnet bot){
		EsUtiOfBotnet.getInstance().editBotnetBykey("lid", bot.getLid()+"", bot);
	}
	
	/**
	 * 同步更新botnetinfo数据(肉鸡且状态改变)
	 * @param bot
	 */
	public void updateBotnetInfo(Botnet bot){
		if(bot.getType()==3){
			
			EsUtiOfBotnet.getInstance().updateBotnetMaping(bot.getIp(), bot);
		}
	}
	
	public static void main(String[] args) {
		BotnetEsService botnetEsService = new BotnetEsService();
		//botnetEsService.getStatisticNum();
	}
}



具体执行类 如下:

package com.venustech.tsoc.cupid.botnet.util;

import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Random;

import org.apache.log4j.Logger;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.FilterBuilder;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
import org.elasticsearch.search.sort.SortOrder;

import com.venustech.tsoc.cupid.botnet.analysis.cache.BotnetCache;
import com.venustech.tsoc.cupid.botnet.analysis.cache.IpLibraryCache;
import com.venustech.tsoc.cupid.commons.mvc.Pager;
import com.venustech.tsoc.cupid.customer.bean.Botnet;
import com.venustech.tsoc.cupid.customer.bean.ccBotnetInfo;
import com.venustech.tsoc.cupid.knowledgebase.botnet.data.BotnetBean;
import com.venustech.tsoc.cupid.origins.port.IspInfo.cache.ISPCache;
import com.venustech.tsoc.cupid.security.bean.BotNet;
import com.venustech.tsoc.cupid.util.es.ESClient;
import com.venustech.tsoc.cupid.util.es.ESClient.Type;
import com.venustech.tsoc.cupid.util.es.ESClientFactory;

/**
 * es 上面的 知识库 操作工具类
 * @author me
 * botnet
 */
public class EsUtiOfBotnet {
	
	 private Logger logger = Logger.getLogger(this.getClass());
	 private EsUtiOfBotnet(){}
	private static EsUtiOfBotnet esUtil=null;
	
	public static EsUtiOfBotnet getInstance(){
		
		if (esUtil==null){
			esUtil=new EsUtiOfBotnet();
		}
		
		return esUtil;
	}
	
	public static  HashMap<String,Boolean> checkCCIPMap=new HashMap<String,Boolean>();
	/**
	 * 是否为调试
	 */
	private static boolean Debug = false;
	// 获取 client
	//ESClient cupidClient = ESClientFactory.me().getDefaultDelegateClient();
	private static ESClient buildClient(){
		Settings settings = ImmutableSettings.settingsBuilder()//cupid-es-zhangli
				.put("cluster.name","cupid-cdn-index")//cupid-cdn-index->  132; cupid-es ->90
				.put("client.transport.sniff", true).build();
		@SuppressWarnings("resource")// 192.168.56.49
		TransportClient client1 = new TransportClient(settings)//192.168.19.98  192.168.19.90 132.224.196.232
		.addTransportAddress(new InetSocketTransportAddress("192.168.56.49", 8300));
		 ESClient client = new ESClient(client1, Type.cdn);
		return client;
	}
	
	//cupidClient = ESClientFactory.me().getDefaultDelegateClient();
	// 获取 首页江苏肉鸡的 个数
	public  ESClient getClient(){
		ESClient cupidClient = null;
		try {
			cupidClient = ESClientFactory.me().getDefaultDelegateClient();
		} catch (Exception e) {
		}
		return (!Debug)?cupidClient
				:buildClient();
	}
	
	
	/**
	 * 首页 展示的 江苏 肉鸡与CC 个数
	 * @param queryBuilder
	 * @return
	 */
	public long getNumber(QueryBuilder queryBuilder){
		long number=0;
		
		ESClient client = this.getClient();
		/*Settings settings = ImmutableSettings.settingsBuilder()//cupid-es-zhangli
				.put("cluster.name","cupid-cdn-index")
				.put("client.transport.sniff", true).build();
		@SuppressWarnings("resource")
		TransportClient client1 = new TransportClient(settings)//192.168.19.98
		.addTransportAddress(new InetSocketTransportAddress("132.224.196.232", 8300));
		 ESClient client = new ESClient(client1, Type.cdn);*/
		boolean existsIndex = isExistsIndex("botnet");
		if(existsIndex){
			
			number= client.prepareCount("botnet")//索引
					//.setTypes("botnet")// 类型
					.setQuery(queryBuilder)// ES 默认查询是10 条,想要改变的话  可以在此处 设置 size setSize(100)
					.execute()
					.actionGet().getCount();
		}
		return number;
	}
	
	/**
	 * 僵木蠕模块 地图数据
	 * @param queryBuilder
	 * @return
	 */
	public LinkedList<BotNet> getMapOrChartsData(String index,QueryBuilder queryBuilder,String field){
		LinkedList<BotNet> mapData = new LinkedList<BotNet>();
		
		ESClient client = this.getClient();
		boolean existsIndex = isExistsIndex("botnet");
		if(existsIndex){
			//number= delegateClient.prepareCount("botnet")//索引
			 SearchRequestBuilder srb=client.prepareSearch(index);
			// srb.setTypes(index);
			 srb.setSearchType(SearchType.COUNT);
		
				AggregationBuilder<?> aggregation = AggregationBuilders.terms("na").field(field).size(Integer.MAX_VALUE);
		
				srb.setQuery(queryBuilder);
				srb.addAggregation(aggregation);
				SearchResponse respose=srb.execute().actionGet();
		
				//System.out.println("result:"+respose);
				Map<String, Aggregation> asMap = respose.getAggregations().asMap();
				StringTerms  result =(StringTerms)	asMap.get("na");
				Iterator<Bucket> iterator = result.getBuckets().iterator();
				while(iterator.hasNext()){
					 Bucket gradeBucket = iterator.next();
					 BotNet botNet = new BotNet();
					 String key = gradeBucket.getKey();
					 Long count = gradeBucket.getDocCount();
					 botNet.setNa(key);
					 botNet.setNu(count.intValue());
					 mapData.add(botNet);
				}
				
		}
		return mapData;
	}
	/**
	 * 分页查询
	 * @param queryBuilderm
	 */
	public Pager<Botnet>  getList(QueryBuilder queryBuilder,Pager<Botnet> pager){
		ESClient client = this.getClient();
		List<Botnet> list = new ArrayList<Botnet>();
		long number=0;
		int pageFrom=(pager.getPage()-1)*pager.getPageSize();
		// 总页
		boolean existsIndex = isExistsIndex("botnet");
		if(existsIndex){
			// 数据
			SearchResponse respose = client
					.prepareSearch("botnet")//索引
					//.setTypes("botnet")// 类型
					.setQuery(queryBuilder)// ES 默认查询是10 条,想要改变的话  可以在此处 设置 size setSize(100)
					.addSort("createTime", SortOrder.DESC)// 将序
					.setFrom(pageFrom).setSize(pager.getPageSize())
					.execute()
					.actionGet();
			SearchHits hits = respose.getHits();// 命中数
			number=hits.getTotalHits();
			SearchHit[] searchHits = hits.getHits();
			if(searchHits.length>0){
				for (SearchHit searchHit : searchHits) {
					Map<String, Object> source = searchHit.getSource();
					Botnet bo = new Botnet();
					long lid=(long)source.get("lid");
					long type=Long.parseLong(source.get("type").toString());
					long state=Long.parseLong(source.get("state").toString());
					long source1=Long.parseLong(source.get("source").toString());
					long create=Long.parseLong(source.get("createTime").toString());
					long update=Long.parseLong(source.get("updateTime").toString());
					long port =Long.parseLong(source.get("port").toString());
					if(update==0){
						update=create;
					}
					String ip=source.get("ip")==null ?"":source.get("ip").toString();
					String url=source.get("url")==null ?"":source.get("url").toString();
					String note=source.get("note")==null ?"":source.get("note").toString();
					
					String createTime=getStrTime(create);
					String updateTime=getStrTime(update);
					bo.setIp(ip);
					bo.setUrl(url);
					bo.setLid(lid);
					bo.setType(type);
					bo.setSource(source1);
					bo.setNote(note);
					bo.setState(state);
					bo.setAction(createTime);// 创建时间
					bo.setProt(updateTime);// 更新时间
					bo.setPort(port);
					list.add(bo);
				}
			}
			
		}
		pager.setRecords(number);
		pager.setData(list);
		return pager;
		
	}
	/**
	 * 初始化 botnet  到缓存
	 */
	public void getAllBotnetbyType(String type1,int size){
		//BotnetCache.botnets.put(key, value)
	//	QueryBuilders.
		logger.info("初始化 --- botnet-"+type1+"-加载到缓存开始");
		long start=System.currentTimeMillis();
		ESClient client = this.getClient();
		boolean existsIndex = isExistsIndex("botnet");
		if(existsIndex){
			
			// 数据
			QueryBuilder queryBuilder=null;
			QueryBuilder queryBuilder1=QueryBuilders.boolQuery().must(QueryBuilders.termQuery("type", type1));
			
			if(type1.equals("3")){
				
				//QueryBuilder queryBuilder2 = QueryBuilders.rangeQuery("createTime").from(System.currentTimeMillis()-(60*1000*60*24*5)).to(System.currentTimeMillis());
				queryBuilder = QueryBuilders.boolQuery().must(queryBuilder1);//.must(queryBuilder2)
			}else{
				queryBuilder = QueryBuilders.boolQuery().must(queryBuilder1);
			}
			
			SearchResponse respose = client
					.prepareSearch("botnet")//索引
					.setTypes("botnet")// 类型
					.setQuery(queryBuilder)
					.setSize(size)
					.execute()
					.actionGet();
			SearchHits hits = respose.getHits();// 命中数
			logger.info("botnet-type-"+type1+" -个数:"+hits.getTotalHits());
			SearchHit[] searchHits = hits.getHits();
			if(searchHits.length>0){
				for (SearchHit searchHit : searchHits) {
					Map<String, Object> source = searchHit.getSource();
					BotnetBean bo = new BotnetBean();
					long type=Long.parseLong(source.get("type").toString());
					long state=Long.parseLong(source.get("state").toString());
					String ip=source.get("ip")==null ?"":source.get("ip").toString();
					String url=source.get("url")==null ?"":source.get("url").toString();
					bo.setIp(ip);
					bo.setUrl(url);
					bo.setType(Integer.parseInt(type+""));
					bo.setState(Integer.parseInt(state+""));
					
					// 加入 初始化map
					// url
					if(type1.equals("2")){
						BotnetCache.botnets.put(url, bo);
					}
					// cc  + ck
					else{
						BotnetCache.botnets.put(ip, bo);
					}
				}
			}
			 logger.info("botnet 加载进条数 :"+(searchHits.length) +"");
			logger.info("botnet 加载进 缓存结束,共耗时:"+(System.currentTimeMillis()-start) +"毫秒");
		}
	

   
		
	}
	/**
	 * 初始化 botnet ck -- 到缓存  每次 24000
	 */
	public void getAllBotnetCkData(int size){
		logger.info("开始加载肉鸡到内存----start botnet--2");
		int allSize=0;
		 long start=System.currentTimeMillis();
		ESClient client = this.getClient();//
		boolean existsIndex = isExistsIndex("botnet");
		if(existsIndex){
			
			SearchResponse repose = client.prepareSearch("botnet")
					//.setTypes("botnet")
					.setQuery(QueryBuilders.termQuery("type", 3))
					.setScroll(new TimeValue(600000))
					.setSize(2000)
					.setSearchType(SearchType.SCAN).execute()
					.actionGet();
			String scrollId = repose.getScrollId();
			try {
				while(true){
					long start2=System.currentTimeMillis();
					SearchResponse repose2 = client.prepareSearchScroll(scrollId)
							.setScroll(new TimeValue(size*1000))
							.execute().actionGet();
					
					SearchHits hits = repose2.getHits();
					if(hits.getHits().length==0){
						break;
					}
					SearchHit[] hits2 = hits.getHits();
					allSize+=hits2.length;
					//System.out.println("size:"+hits2.length);
					for (SearchHit searchHit : hits2) {
						Map<String, Object> source = searchHit.getSource();
						BotnetBean bo = new BotnetBean();
						long type=Long.parseLong(source.get("type").toString());
						long state=Long.parseLong(source.get("state").toString());
						String ip=source.get("ip")==null ?"":source.get("ip").toString();
						String url=source.get("url")==null ?"":source.get("url").toString();
						bo.setIp(ip);
						bo.setUrl(url);
						bo.setType(Integer.parseInt(type+""));
						bo.setState(Integer.parseInt(state+""));
						
						BotnetCache.botnets.put(ip, bo);
					}	
					//System.out.println("耗时:"+(System.currentTimeMillis()-start2));
					
				}
				logger.info("botnet-肉鸡 导入缓存 结束..... 共耗时:"+(System.currentTimeMillis()-start) +"毫秒 ,总条数" +allSize +"条");
				BotnetCache.overFlag=true;
			} catch (NumberFormatException e) {
				// TODO Auto-generated catch block
				//e.printStackTrace();
			} catch (ElasticsearchException e) {
				// TODO Auto-generated catch block
				//e.printStackTrace();
			}
		}else{
			/*
			 * 若是此时,botnet 不存在,则设置 默认读取完成
			 * */
			BotnetCache.overFlag=true;
		}
		
		
	}
	/**
	 * 验证 ip/url 是否在 botnet 中  true : 含有
	 * @param key
	 * @param value
	 * @return
	 */
	public boolean checkBotnetkey_in(String key,String value,String type,String stateFlag){
		logger.info(" --- botnet_检索开始 判断库中是否含有此Key:"+value);
		long start=System.currentTimeMillis();
		ESClient client = this.getClient();//
		long number = 1;
		boolean existsIndex = isExistsIndex("botnet");
		if(existsIndex){
			
			BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
			queryBuilder.must(QueryBuilders.termQuery("type", type)).must(QueryBuilders.termQuery(key, value));
			if(stateFlag!=null && !stateFlag.equals("")){
				long state=stateFlag.equals("true")?2:1;
				queryBuilder.must(QueryBuilders.termQuery("state", state));
			}
			
			SearchResponse repose = client.prepareSearch("botnet")//索引
					//.setTypes("botnet")// 类型
					.setQuery(queryBuilder)// ES 默认查询是10 条,想要改变的话  可以在此处 设置 size setSize(100)
					.setSize(1)
					.execute()
					.actionGet();
			 number=repose.getHits().getTotalHits();
			
			if(number>0 && type.equals("3")){
				int state=0;
				if(!stateFlag.equals("")){
					
					state=stateFlag.equals("true")?2:1;
				}
				BotnetCache.botnets.put(value, new BotnetBean(value,"",state,3));
			}
			logger.info("botnet 检索结束,共耗时:"+(System.currentTimeMillis()-start) +"毫秒");
		}
	return number>0 ? true:false;
	}
	
	
	/**
	 * 僵木蠕模块 大地图数据
	 * @param queryBuilder
	 * @param isAll  true:合并 疑似与确认   : false : 分开聚合
	 * @return
	 */
	public LinkedList<BotNet> getMapData(QueryBuilder queryBuilder,String field,boolean isAll){
		LinkedList<BotNet> mapData = new LinkedList<BotNet>();
		ESClient client = this.getClient();
		boolean existsIndex = isExistsIndex("botnet");
		if(existsIndex){
			
			SearchRequestBuilder srb=client.prepareSearch("botnet");
			//srb.setTypes("botnet");
			srb.setSearchType(SearchType.COUNT);
			srb.setQuery(queryBuilder);
			AggregationBuilder<?> aggregationField = AggregationBuilders.terms("na").field(field).size(Integer.MAX_VALUE);
			if(isAll==false){
				
				AggregationBuilder<?> aggregationState = AggregationBuilders.terms("state").field("state").size(Integer.MAX_VALUE);
				aggregationField.subAggregation(aggregationState);
			}
			
			
			srb.addAggregation(aggregationField);
			
			SearchResponse respose=srb.execute().actionGet();
			Map<String, Aggregation> asMap = respose.getAggregations().asMap();
			StringTerms  result =(StringTerms)	asMap.get("na");
			Iterator<Bucket> iterator = result.getBuckets().iterator();
			// 合并疑似与确认
			if(isAll){
				while(iterator.hasNext()){
					BotNet botNet = new BotNet();
					Bucket fieldBucket = iterator.next();
					String key = fieldBucket.getKey();
					Long botnetNum=	fieldBucket.getDocCount();
					botNet.setNa(key);//区域
					botNet.setNu(botnetNum.intValue());
					mapData.add(botNet);
				}
			}
			// 分开聚合
			else{
				
				while(iterator.hasNext()){
					Bucket gradeBucket = iterator.next();
					String key = gradeBucket.getKey();
					LongTerms  result2 =(LongTerms)gradeBucket.getAggregations().asMap().get("state");
					Iterator<Bucket> iterator2 = result2.getBuckets().iterator();
					while(iterator2.hasNext()){
						Bucket next = iterator2.next();
						BotNet botNet = new BotNet();
						
						int keyState = Integer.parseInt(next.getKey()) ;
						Long count = next.getDocCount();
						botNet.setState(keyState);
						botNet.setNa(key);
						botNet.setNu(count.intValue());
						mapData.add(botNet);
					}
					
				}
			}
		}
		
		return mapData;
	}
	/**
	 * 增加一个 僵木蠕信息
	 * @param hitsArray
	 * @return
	 */
	public void insertBotnetInfo(Botnet  bo){
	//	boolean existsIndex = isExistsIndex("botnet");
		//if(existsIndex){
			ESClient client = this.getClient();
			if(client != null){
				
				long lid=new Date().getTime()*10000+new Random().nextInt(10000);
				bo.setLid(lid);
				bo.setIsLive(1);
				String  jsonData=SortUtil.toJson(bo);
				client.prepareIndex("botnet", "botnet").setId(new Date().getTime()*10000+new Random().nextInt(10000)+"")
				.setSource(jsonData)
				.setId(lid+"")
				.execute()
				.actionGet();
			}
		//}
		
		
	}
	/**
	 * 批量 添加 到 es  botnet
	 * @param botnets
	 */
	public void insertBotnets(List<BotnetBean> botnets){
		//boolean existsIndex = isExistsIndex("botnet");
		//if(existsIndex){
			
			if(botnets ==null || botnets.size()==0) return;
			// 类型转化
			List<Botnet> botnet_esList = convertTypeList(botnets);
			//开启批量插入
			ESClient client = this.getClient();
			if(client != null){
				
				BulkRequestBuilder bulkRequest = client.prepareBulk();
				for (Botnet bo: botnet_esList) {
					String  jsonData=SortUtil.toJson(bo);
					bulkRequest.add(client.prepareIndex("botnet", "botnet").setId(new Date().getTime()*10000+new Random().nextInt(10000)+"").setSource(jsonData));
				}
				bulkRequest.execute().actionGet();
				logger.info("批量插入(botnet)数据:"+botnet_esList.size());
			}
		//}
	}
	/**
	 * 批量 添加 到 es  botnetinfo 关联表,
	 * 首次添加 还没有 ccIp
	 * @param botnets
	 * @param isQuartzDelete  是否为定时清理调度任务后的 insert 到 bontnet
	 */
	public void insertBotnetInfos(List<BotnetBean> botnets, boolean isQuartzDelete ){
	//	boolean existsIndex = isExistsIndex("botnetinfo");
	//	if(existsIndex){
			
			if(botnets ==null || botnets.size()==0) return;
			// 类型转化
			List<ccBotnetInfo>	botnet_esList = null;
			if(isQuartzDelete==false){
				
				botnet_esList=convertTypeList2(botnets,"frist");
			}else{
				botnet_esList=convertTypeList3(botnets,"frist");
			}
			//开启批量插入
			ESClient client = this.getClient();
			if(client != null){
				
				BulkRequestBuilder bulkRequest = client.prepareBulk();
				for (ccBotnetInfo bo: botnet_esList) {
					checkCCIPMap.put(bo.getCkIp(), true);
					String  jsonData=SortUtil.toJson(bo);
					bulkRequest.add(client.prepareIndex("botnetinfo", "botnetinfo").setId(new Date().getTime()*10000+new Random().nextInt(10000)+"").setSource(jsonData));
				}
				
				bulkRequest.execute().actionGet();
				logger.info("批量插入(botnetinfo-ck)数据:"+botnet_esList.size());
			}
		//}
	}
	/**
	 * update
	 * @param botnets
	 */
	public void updateBotnetInfos(List<ccBotnetInfo> botnets){
		boolean existsIndex = isExistsIndex("botnetinfo");
		if(existsIndex){
			
			if(botnets ==null || botnets.size()==0) return;
			//开启批量插入
			ESClient client = this.getClient();
			BulkRequestBuilder bulkRequest = client.prepareBulk();
			for (ccBotnetInfo bo: botnets) {
				String  jsonData=SortUtil.toJson(bo);
				bulkRequest.add(client.prepareIndex("botnetinfo", "botnetinfo").setId(new Date().getTime()*10000+new Random().nextInt(10000)+"").setSource(jsonData));
			}
			
			bulkRequest.execute().actionGet();
			logger.info("首次批量update(botnetinfo-ck)数据:"+botnets.size());
		}
	}
	
	/**
	 * 验证关联表(botnetinfo)中 是否 含义ckIp +ccIp
	 * @return
	 */
	public boolean  isContains(String ckIp,String ccIp){
		long number = 1;
		boolean existsIndex = isExistsIndex("botnetinfo");
		if(existsIndex){
			
			ESClient client = this.getClient();
			BoolQueryBuilder query = QueryBuilders.boolQuery();
			QueryBuilder queryBuilderCkIp=QueryBuilders.termQuery("ckIp", ckIp);
			if(!ccIp.equals("afterDeleteccIp")){
				QueryBuilder queryBuilderCcIp=QueryBuilders.termQuery("ccIp", ccIp);
			//	QueryBuilder queryBuilderPort=QueryBuilders.termQuery("port", port);
				query.must(queryBuilderCkIp).must(queryBuilderCcIp);
			}else{
				query.must(queryBuilderCkIp);
			}
			 number = client.prepareCount("botnetinfo")
					//.setTypes("botnetinfo")
					.setQuery(query)
					.execute()
					.actionGet().getCount();
			
		}
		return number>0?true:false;
	}
	
	/**
	 * 增加 映射关系  , ccIp 储存在 note 属性中
	 */
	public void addMapping(List<BotnetBean> botnets){
		if(botnets ==null || botnets.size()==0) return;
		// 此时的 状态是 已经验证过 botnet ,含有此 ckIp ccIp
		// 所以此时 需要 看是否 要更新 或者 添加一条 映射 关系,
		// 先判断是否还有 空着 ccIp 的  ckIp
		//1 .有的话 先 更新 含有 ccIp 2. 没有空着的 则 新增加一条
		// 开启 批量添加
		// 转化 类型
		boolean existsIndex = isExistsIndex("botnetinfo");
		if(existsIndex){
			
			List<ccBotnetInfo> bot_info = convertTypeList2(botnets,"");
			ESClient client = this.getClient();
			BulkRequestBuilder bulkRequest = client.prepareBulk();
			for (ccBotnetInfo b : bot_info) {
				if(checkCCIPMap.get(b.getCkIp())!=null && checkCCIPMap.get(b.getCkIp())==true){
					
					long lid=ccIpisNull(b.getCkIp());
					// 更新
					if(lid!=1){
						// 先删除在 增加
						deleteBotnetInfoByLid(lid);
						checkCCIPMap.put(b.getCkIp(), false);
					}
				}
				// 添加
				String  jsonData=SortUtil.toJson(b);
				bulkRequest.add(client.prepareIndex("botnetinfo", "botnetinfo").setId(new Date().getTime()*10000+new Random().nextInt(10000)+"").setSource(jsonData));
				
			}
			if(bot_info.size()>0){
				
				bulkRequest.execute().actionGet();
			}
			logger.info("批量插入(botnetinfo--mapping)数据:"+bot_info.size());
		}
	}
	/**
	 * 先判断是否还有 空着 ccIp 的  ckIp
	 * @return
	 */
	public  long ccIpisNull(String ckIp){
		long lid=1;
		ESClient client = this.getClient();
		boolean existsIndex = isExistsIndex("botnetinfo");
		if(existsIndex){
			
			SearchResponse respose = client.prepareSearch("botnetinfo").setTypes("botnetinfo")
					.setQuery(QueryBuilders.termQuery("ckIp", ckIp))
					.addSort("createTime", SortOrder.ASC)
					.execute()
					.actionGet();
			SearchHits hits = respose.getHits();
			SearchHit[] hits2 = hits.getHits();
			if(hits2.length>0){
				for (SearchHit searchHit : hits2) {
					Map<String, Object> source = searchHit.getSource();
					
					
					if(source.get("ccIp")==null || source.get("ccIp").equals("")){
						lid=(long)source.get("lid");
						break;
					}
					
				}
			}
		}
		return lid;
		
	}
	/**
	 * 增加到 关联表 中 
	 * @param bo
	 */
	public ccBotnetInfo insertBotnet_info(Botnet  bo){
		long lid=new Date().getTime()*10000+new Random().nextInt(10000);
		ccBotnetInfo ccBotnetInfo = new ccBotnetInfo();
		ccBotnetInfo.setArea(bo.getSmallArea());
		//ccBotnetInfo.setCcIp(bo.get);
		ccBotnetInfo.setCkIp(bo.getIp());
		ccBotnetInfo.setLid(lid);
		//ccBotnetInfo.set
		ccBotnetInfo.setCreateTime(System.currentTimeMillis());
		return ccBotnetInfo;
	}
	
	/**
	 * 更新 或增加 
	 * @param botnets
	 */
	public void insertOrUpdateBotnetInfos(List<BotnetBean> botnets){
		
	}
	
	/**
	 * 判断 botnetinfo 中  含有 ckIp 的 是否
	 */
	
	 /**
	    * 类型转换  过度
	    * @param bean
	    * @return
	    */
	   public List<Botnet> convertTypeList(List<BotnetBean> botnets){
		   
		   ArrayList<Botnet> botnet_es = new ArrayList<Botnet>();
		   for (BotnetBean bean : botnets) {
			   
			   Botnet botnet = new Botnet();
			   long lid=new Date().getTime()*10000+new Random().nextInt(10000);
			   botnet.setLid(lid);
			   botnet.setType(bean.getType());
			   botnet.setBigArea(bean.getBigArea());
			   botnet.setSmallArea(bean.getSmallArea());
			   botnet.setIp( bean.getIp());
			   botnet.setUrl(bean.getUrl());
			   botnet.setSource(bean.getSource());
			   botnet.setFirstCreateTime(System.currentTimeMillis());
			   botnet.setCreateTime(System.currentTimeMillis());
			   botnet.setState(bean.getState());
			   botnet.setNote(bean.getNote());
			   botnet.setIsLive(1);
			   botnet.setPort(bean.getPort());
			   botnet.setUpdateTime(System.currentTimeMillis());
			   if(botnet.getIp().equals("")){
				   logger.info("将要插入的数据-URL:"+bean.getUrl() );
			   }else{
				   logger.info("将要插入的数据为-IP:"+ bean.getIp());
			   }
			   logger.info(" 备注:"+bean.getNote());
			   botnet_es.add(botnet);
		}
		   return botnet_es;
	   }
	
	   /**
	    * 类型转换  过度   ccBotnetInfo 添加到 botnetinfo
	    * 
	    * 里面 还没有 添加 ccIp
	    * @param bean
	    * @return
	    */
	   public List<ccBotnetInfo> convertTypeList2(List<BotnetBean> botnets,String flag){
		   
		   ArrayList<ccBotnetInfo> botnet_es = new ArrayList<ccBotnetInfo>();
		   for (BotnetBean bean : botnets) {
			   
			   ccBotnetInfo botnet = new ccBotnetInfo();
			   long lid=new Date().getTime()*10000+new Random().nextInt(10000);
			   botnet.setLid(lid);
			   botnet.setBigArea(bean.getBigArea());
			   botnet.setArea(bean.getSmallArea());
			   botnet.setCkIp( bean.getIp());
			   botnet.setCreateTime(System.currentTimeMillis());
			   if(!flag.equals("frist")){
				   botnet.setCcIp(bean.getNote());
			   }
			   botnet.setState(bean.getState());
			   botnet.setPort(bean.getPort());
			   botnet.setIsLive(1);
			   botnet_es.add(botnet);
		   }
		   return botnet_es;
	   }
	   /**
	    * 类型转换  过度   ccBotnetInfo 添加到 botnetinfo
	    * 
	    * 里面 还没有 添加 ccIp
	    * @param bean
	    * @return
	    */
	   public List<ccBotnetInfo> convertTypeList3(List<BotnetBean> botnets,String flag){
		   
		   ArrayList<ccBotnetInfo> botnet_es = new ArrayList<ccBotnetInfo>();
		   for (BotnetBean bean : botnets) {
			   
			   ccBotnetInfo botnet = new ccBotnetInfo();
			   long lid=new Date().getTime()*10000+new Random().nextInt(10000);
			   botnet.setLid(lid);
			   botnet.setBigArea(bean.getBigArea());
			   botnet.setArea(bean.getSmallArea());
			   botnet.setCkIp( bean.getIp());
			   botnet.setCreateTime(bean.getCreateTime());
			   if(!flag.equals("frist")){
				   botnet.setCcIp(bean.getNote());
			   }
			   botnet.setState(bean.getState());
			   botnet.setPort(bean.getPort());
			   botnet.setIsLive(1);
			   botnet_es.add(botnet);
		   }
		   return botnet_es;
	   }
	   
	   
	/**
	 * 改变知识库信息
	 * @param 
	 * @return
	 */
	public void updateBotnetInfo(Botnet bo){
		boolean existsIndex = isExistsIndex("botnet");
		if(existsIndex){
			
			ESClient client = this.getClient();
			long[] lid=new long[1];
			lid[0]=bo.getLid();
			deleteInfoByLid(lid,bo.getType(),"false");
			bo.setUpdateTime(System.currentTimeMillis());
			String  jsonData=SortUtil.toJson(bo);
			client.prepareIndex("botnet", "botnet").setId(new Date().getTime()*10000+new Random().nextInt(10000)+"")
			.setSource(jsonData)
			.execute()
			.actionGet();
		}
		//efge
	}
	/**
	 * 
	 * @param key  表示键  比如 lid  ip url等 具有唯一性的
	 * @param value  键的值
	 * @param botnet  要更新的信息储存在 其中
	 */
	public void editBotnetBykey(String key, String value, Botnet bean){
		ESClient client = this.getClient();
		boolean existsIndex = isExistsIndex("botnet");
		if(existsIndex){
			
			QueryBuilder queryBuilderKey=QueryBuilders.termQuery(key, value);
			QueryBuilder queryBuilder=QueryBuilders.boolQuery().must(queryBuilderKey);
			SearchResponse repose = client.prepareSearch("botnet")
					.setTypes("botnet")
					.setQuery(queryBuilder)
					.setSize(Integer.MAX_VALUE)
					.execute()
					.actionGet();
			SearchHits hits = repose.getHits();
			SearchHit[] hits2 = hits.getHits();
			long oldState = bean.getReliability();
			if(hits2.length>0){
				// 获取文档 ID
				String docId = hits2[0].getId();
				//更新 update
				bean.setUpdateTime(System.currentTimeMillis());
				bean.setReliability(0);
				String  jsonData=SortUtil.toJson(bean);
				// 追加到 es 上
				client.prepareIndex("botnet", "botnet")
				.setId(docId)
				.setSource(jsonData)
				.execute()
				.actionGet();
			}
			
			// 更新关联表的条件: 1: 为肉鸡ip ; 2: 状态值发生变化 state(oldState  放入了reliability )
			if(bean.getType() == 3 && bean.getState() != oldState){
				updateBotnetMaping(bean.getIp(), bean);
			}
		}
	}
	/**
	 * 更新关联信息
	 * @param ip
	 */
	public void updateBotnetMaping(String ckIp,Botnet bo){
		boolean existsIndex = isExistsIndex("botnetinfo");
		if(existsIndex){
			
			ESClient client = this.getClient();
			@SuppressWarnings("deprecation")
			QueryBuilder queryBuilder=QueryBuilders.termQuery("ckIp", ckIp);
			SearchResponse respose = client.prepareSearch("botnetinfo")
					//.setTypes("botnetinfo")
					.setQuery(queryBuilder)
					.setSize(Integer.MAX_VALUE)
					.execute()
					.actionGet();
			SearchHits hits = respose.getHits();
			SearchHit[] hits2 = hits.getHits();
			
			long state_new=bo.getState();
			if(hits2.length>0){
				for (SearchHit searchHit : hits2) {
					ccBotnetInfo mapping = new  ccBotnetInfo();
					
					Map<String, Object> source = searchHit.getSource();
					String docId = searchHit.getId();
					long lid=Long.parseLong(source.get("lid").toString());
					String ccIp=source.get("ccIp")==null ?"":source.get("ccIp").toString();
					//  String ck_Ip=bo.getIp();
					String note=source.get("note")==null ?"":source.get("note").toString();
					String countryCode=source.get("countryCode")==null ?"":source.get("countryCode").toString();
					long createTime=Long.parseLong(source.get("createTime").toString());
					long port=Long.parseLong(source.get("port").toString());
					mapping.setLid(lid);
					mapping.setBigArea(bo.getBigArea());
					mapping.setArea(bo.getSmallArea());
					mapping.setNote(note);
					mapping.setCreateTime(createTime);
					mapping.setIsLive(bo.getIsLive());
					mapping.setPort(port);
					mapping.setState(state_new);
					mapping.setCountryCode(countryCode);
					mapping.setCcIp(ccIp);
					mapping.setCkIp(bo.getIp());
					
					String  jsonData=SortUtil.toJson(mapping);
					// 追加到 es 上
					client.prepareIndex("botnetinfo", "botnetinfo")
					.setId(docId)
					.setSource(jsonData)
					.execute()
					.actionGet();
					
				}
			}
		}
		
		
	}
	
	/**
	 * 删除 僵木蠕信息  long [lids]
	 * @param lids
	 * @param type
	 * @param isDelete  true: 表示连 关联表 botnetinfo  中的 数据也删: false :表示只删除botnet,不删 botnetinfo
	 */
	public void deleteInfoByLid(long[] lids,long type,String isDelete){
		if(lids.length<=0)  return;
		boolean existsIndex = isExistsIndex("botnet");
		if(existsIndex){
			
			ESClient client = this.getClient();
			@SuppressWarnings("deprecation")
			QueryBuilder queryBuilder=QueryBuilders.inQuery("lid", lids);
			SearchResponse respose = client.prepareSearch("botnet")
					.setTypes("botnet")
					.setQuery(queryBuilder)
					.setSize(Integer.MAX_VALUE)
					.execute()
					.actionGet();
			SearchHits hits = respose.getHits();
			SearchHit[] hits2 = hits.getHits();
			int arrSize=Integer.parseInt(hits.getTotalHits()+"");
			// 储存要删除的ip  ck
			ArrayList<String> cKipList = new ArrayList<String>();
			
			
			ArrayList<String> ccIpList = new ArrayList<String>();// cc
			
			boolean flag= type==3?true:false;
			BulkRequestBuilder bulkRefresh = client.prepareBulk().setRefresh(true);
			int i=0;
			for (SearchHit searchHit : hits2) {
				// 1. 删除 botnet
				DeleteRequest deleteRequest = new DeleteRequest("botnet","botnet",searchHit.getId());
				bulkRefresh.add(deleteRequest);
				Map<String, Object> obj = searchHit.getSource();
				// 2. 为删除botnetinfo 做出准备,记录ips
				if(type!=2){
					
					String ip= obj.get("ip")==null ?"":obj.get("ip").toString();
					if(flag){
						
						cKipList.add(ip);
					}
					if(type==1){
						ccIpList.add(ip);
					}
					
					BotnetCache.botnets.remove(ip);
				}
				// 去除url 内存
				if(type==2){
					String url= obj.get("url")==null ?"":obj.get("url").toString();
					BotnetCache.botnets.remove(url); 
				}
				i++;
			}
			bulkRefresh.execute().actionGet();
			/*for (String ip : ips) {
			System.out.println("要删除的ip:"+ip);
			// 去内存
			BotnetCache.botnets.remove(ip);
		}*/
			String[] array=new String[cKipList.size()];
			String[] ckips = cKipList.toArray(array);
			
			String[] array2=new String[ccIpList.size()];
			String[] ccips = ccIpList.toArray(array2);
			if(ckips.length>0 && flag && isDelete.equals("true")){
				
				deleteBotnetInfo("ckIp",ckips);
			}
			if(ccips.length>0){
				deleteBotnetInfo("ccIp",ccips);
			}
		}
	}
	/**
	 * inQuery() size<=1024  filter  则是释放的
	 * String field  : ckIp /ccIp
	 * @param ips
	 */
	public void deleteBotnetInfo(String field,String[] ips){
		if(ips==null || ips.length<=0) return;
		ESClient client = this.getClient();
		boolean existsIndex = isExistsIndex("botnetinfo");
		if(existsIndex){
			
			//QueryBuilder queryBuilder=QueryBuilders.inQuery("ckIp", ips);
			@SuppressWarnings("deprecation")
			FilterBuilder inFilter = FilterBuilders.inFilter(field, ips);
			SearchResponse respose = client.prepareSearch("botnetinfo")
					.setTypes("botnetinfo")
					.setPostFilter(inFilter)
					.setSize(Integer.MAX_VALUE)
					.execute()
					.actionGet();
			SearchHits hits = respose.getHits();
			SearchHit[] hits2 = hits.getHits();
			BulkRequestBuilder bulkRefresh = client.prepareBulk().setRefresh(true);
			// 储存要删除的ip  ck
			ArrayList<String> cKipList = new ArrayList<String>();
			for (SearchHit searchHit : hits2) {
				// 2. 为删除botnetinfo 做出准备,记录ips
				DeleteRequest deleteRequest = new DeleteRequest("botnetinfo","botnetinfo",searchHit.getId());
				bulkRefresh.add(deleteRequest);
				// 如果删除的是CC 应该判断 把Ck的全部删完,若是全部删完则因该空部位即ccip=""
				if(field.equals("ccIp")){
					// 记录要删除的 ckIp
					Map<String, Object> obj = searchHit.getSource();
					String ckIp= obj.get("ckIp")==null ?"":obj.get("ckIp").toString();
					
					if(ckIp!=null && !ckIp.equals("")){
						if(!cKipList.contains(ckIp)){
							
							cKipList.add(ckIp);
						}
						
					}
				}
				
			}
			if(hits2.length>0){
				
				bulkRefresh.execute().actionGet();
			}
			// 判断 info 中 ck--size 
			String[] array=new String[cKipList.size()];
			String[] ckips = cKipList.toArray(array);
			
			if(ckips!=null && ckips.length >0){
				try {
					//Thread.sleep(3000);
					afterDeleteCC(ckips);
				} catch (Exception e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
				//afterDeleteCC(ckips);
			}
			// inQuery() 里面若为 数组的话,是有限制的 大小为 1024,所以改为 filter
		}
	}
	/**
	 * 判断botnetinfo 中 ckIp的size,
	 */
	public void afterDeleteCC(String[] ckips){
		List<BotnetBean> BotnetInfos =new ArrayList<BotnetBean>();
		for (String ckIp : ckips) {
			Botnet oldInfo = getBotnetInfoById("ip",ckIp);// 检验botnet  根还存在
			if( oldInfo!=null){
				
				boolean flag = isContains(ckIp,"afterDeleteccIp");//检验botnetinfo 
				if(flag==true ){
					// 表明 ckip  还存在,不用补位
				}else{
					// 现在 需要补位  List<BotnetBean> botnets
					BotnetBean bin =new BotnetBean();
					
					bin.setIp(ckIp);// 同事设定了  ipLong
					/*if (bin.getIpLong() > 0) {
						bin.setArea(ISPCache.getInstance().getArea(bin.getIpLong()));
					}*/
					bin.setBigArea(oldInfo.getBigArea());
					bin.setSmallArea(oldInfo.getSmallArea());
					bin.setState(Integer.parseInt(oldInfo.getState()+""));
					// 设置 createTime
					bin.setCreateTime(oldInfo.getCreateTime());
					BotnetInfos.add(bin);
					
				}
			}
			
		}
		if(BotnetInfos.size()>0){
			
			insertBotnetInfos(BotnetInfos,true);
		}
	}
	public void deleteBotnetInfoByLids(long[] lids){
		if(lids==null || lids.length<=0) return;
		boolean existsIndex = isExistsIndex("botnetinfo");
		if(existsIndex){
			
			ESClient client = this.getClient();
			@SuppressWarnings("deprecation")
			QueryBuilder queryBuilder=QueryBuilders.inQuery("lid", lids);
			SearchResponse respose = client.prepareSearch("botnetinfo")
					//.setTypes("botnetinfo")
					.setQuery(queryBuilder)
					.setSize(Integer.MAX_VALUE)
					.execute()
					.actionGet();
			SearchHits hits = respose.getHits();
			SearchHit[] hits2 = hits.getHits();
			BulkRequestBuilder bulkRefresh = client.prepareBulk().setRefresh(true);
			for (SearchHit searchHit : hits2) {
				DeleteRequest deleteRequest = new DeleteRequest("botnetinfo","botnetinfo",searchHit.getId());
				bulkRefresh.add(deleteRequest);
				
				
			}
			if(hits2.length>0){
				
				bulkRefresh.execute().actionGet();
			}
		}
		
	}
	/**
	 * 
	 * @param lid
	 */
	public void deleteBotnetInfoByLid(long lid){
		ESClient client = this.getClient();
		boolean existsIndex = isExistsIndex("botnetinfo");
		if(existsIndex){
			
			@SuppressWarnings("deprecation")
			QueryBuilder queryBuilder=QueryBuilders.termQuery("lid", lid);
			SearchResponse respose = client.prepareSearch("botnetinfo")
					//.setTypes("botnetinfo")
					.setQuery(queryBuilder)
					.setSize(Integer.MAX_VALUE)
					.execute()
					.actionGet();
			SearchHits hits = respose.getHits();
			SearchHit[] hits2 = hits.getHits();
			BulkRequestBuilder bulkRefresh = client.prepareBulk().setRefresh(true);
			for (SearchHit searchHit : hits2) {
				DeleteRequest deleteRequest = new DeleteRequest("botnetinfo","botnetinfo",searchHit.getId());
				bulkRefresh.add(deleteRequest);
				
				
			}
			bulkRefresh.execute().actionGet();
		}
		
	}
	
	
	/**
	 * 由lid 获取 
	 * @param lid
	 * @return
	 */
	public Botnet  getBotnetInfoById(String key,String value){
		Botnet botnet = null;
		ESClient client = this.getClient();
		QueryBuilder queryBuilder1=QueryBuilders.termQuery(key, value);
		//QueryBuilder queryBuilder=QueryBuilders.boolQuery().must(queryBuilder1);
		boolean existsIndex = isExistsIndex("botnet");
		if(existsIndex){
			
			SearchResponse respose = client.prepareSearch("botnet")
					.setTypes("botnet")
					.setQuery(queryBuilder1)
					.execute()
					.actionGet();
			SearchHits hits = respose.getHits();
			
			SearchHit[] hitsArr = hits.getHits();
			
			if(hitsArr.length>0){
				Map<String, Object> source = hitsArr[0].getSource();
				//	System.out.println("id:"+hitsArr[0].getId());
				Botnet bo = new Botnet();
				long lid2=(long)source.get("lid");
				long type=Long.parseLong(source.get("type").toString());
				long state=Long.parseLong(source.get("state").toString());
				long source1=Long.parseLong(source.get("source").toString());
				long createTime=Long.parseLong(source.get("createTime").toString());
				long firstCreateTime=Long.parseLong(source.get("firstCreateTime").toString());
				long create=createTime;
				long update=Long.parseLong(source.get("updateTime").toString());
				if(update==0){
					update=create;
				}
				long port=Long.parseLong(source.get("port").toString());
				String ip=source.get("ip")==null?"":source.get("ip").toString();
				String url=source.get("url")==null?"":source.get("url").toString();
				String note=source.get("note")==null?"":source.get("note").toString();
				String bigArea=source.get("bigArea")==null?"":source.get("bigArea").toString();
				String smallArea=source.get("smallArea")==null?"":source.get("smallArea").toString();
				bo.setIp(ip);
				bo.setUrl(url);
				bo.setLid(lid2);
				bo.setType(type);
				bo.setSource(source1);
				bo.setPort(port);
				bo.setNote(note);
				bo.setBigArea(bigArea);
				bo.setSmallArea(smallArea);
				bo.setState(state);
				bo.setCreateTime(createTime);
				bo.setFirstCreateTime(firstCreateTime);
				bo.setAction(getStrTime(createTime));// 创建时间
				bo.setProt(getStrTime(update));// 更新时间
				botnet=bo;
			}
		}
		
		return botnet;
	}
	/**
	 * 定时清理库中的 过期的 僵木蠕-肉鸡-信息
	 */
	public int deleteBotnetTimerTask(String field,long deleteInterval,String isOldData){
		ESClient client = this.getClient();
		int num = 0;
		boolean existsIndex = isExistsIndex("botnet");
		if(existsIndex){
			
			// 小于 正常期的 都是即将要删除掉的  包括 肉鸡 与 cc+URL
			//	QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", "3");
			long nowTime=System.currentTimeMillis();
			long cycileTime=nowTime-deleteInterval;
			QueryBuilder queryBuilderTime=QueryBuilders.rangeQuery(field).lt(cycileTime);
			BoolQueryBuilder queryBuilder=QueryBuilders.boolQuery();
			
			queryBuilder.must(queryBuilderTime);
			// 老数据 中 updateTime 都是 0,所以 依 createTime 为准
			QueryBuilder queryBuilderCreateTime=QueryBuilders.termQuery("updateTime", "0");
			if(isOldData.equals("oldData")){
				queryBuilder.must(queryBuilderCreateTime);
			}else{
				queryBuilder.mustNot(queryBuilderCreateTime);
			}
			// 1.选出你要 删除的 条目 ,获取 ip
			SearchResponse repose = client.prepareSearch("botnet")
					.setTypes("botnet")
					.setQuery(queryBuilder)
					.setSize(Integer.MAX_VALUE)
					.execute()
					.actionGet();
			SearchHits hits = repose.getHits();
			SearchHit[] hits2 = hits.getHits();
			// 开启 批量删除
			BulkRequestBuilder bulkRefresh = client.prepareBulk().setRefresh(true);
			// 储存要删除的ip  ck
			ArrayList<String> ipList = new ArrayList<String>();
			ArrayList<String> ccIpList = new ArrayList<String>();// cc
			if(hits2.length>0){
				for (SearchHit searchHit : hits2) {
					DeleteRequest deleteRequest = new DeleteRequest("botnet", "botnet", searchHit.getId());
					bulkRefresh.add(deleteRequest);
					
					// get IP
					Map<String, Object> obj = searchHit.getSource();
					long type=obj.get("type")==null?0:Integer.parseInt(obj.get("type").toString());
					if(type==2){
						String url=obj.get("url")==null?"":obj.get("url").toString();
						BotnetCache.botnets.remove(url);
					}else{
						
						String ip=obj.get("ip")==null?"":obj.get("ip").toString();
						// 提高效率 去除 type=1 的情况
						if(type==3){
							
							ipList.add(ip);
						}
						if(type==1){
							ccIpList.add(ip);
						}
						// 清除 缓存 信息
						BotnetCache.botnets.remove(ip);
					}
				}
				// 删除 botnet 中的  不合理 信息
				bulkRefresh.execute().actionGet();
			}
			String[] array=new String[ipList.size()];
			String[] ips = ipList.toArray(array);
			
			String[] array2=new String[ccIpList.size()];
			String[] ccips = ccIpList.toArray(array2);
			
			// 同步清理  botnetinfo  ckIp
			if(ips.length>0){
				deleteBotnetInfo("ckIp",ips);
			}
			// ccIp
			if(ccips.length>0){
				deleteBotnetInfo("ccIp",ccips);
			}
			num = ips.length;
		}
		return num;
	}

	/**
	 * 监测更新 ,update-->updateTime
	 * 
	 * key: ip/url
	 */
	public void monitorUpdateInfo(String key,String value){
		ESClient client = this.getClient();
		// 小于 正常期的 都是即将要删除掉的
		boolean existsIndex = isExistsIndex("botnet");
		if(existsIndex){
			
			QueryBuilder queryBuilder=null;
			/*if(!key.equals("url")){
				queryBuilder=	QueryBuilders.termQuery(key, value);
			}else{
				queryBuilder=	QueryBuilders.queryString(value).field(key);
			}*/
			queryBuilder=	QueryBuilders.termQuery(key, value);
			SearchResponse repose = client.prepareSearch("botnet")
					//.setTypes("botnet")
					.setQuery(queryBuilder)
					.setSize(Integer.MAX_VALUE)
					.execute()
					.actionGet();
			SearchHits hits = repose.getHits();
			SearchHit[] hits2 = hits.getHits();
			if(hits2.length>0){
				// 获取文档 ID
				String docId = hits2[0].getId();
				Map<String, Object> source = hits2[0].getSource();
				Botnet bo = new Botnet();
				long lid2=(long)source.get("lid");
				long type=Long.parseLong(source.get("type").toString());
				long ipLong=Long.parseLong(source.get("ipLong").toString());
				long port=Long.parseLong(source.get("port").toString());
				long isLive=Long.parseLong(source.get("isLive").toString());
				long reliability=Long.parseLong(source.get("reliability").toString());
				long state=Long.parseLong(source.get("state").toString());
				long source1=Long.parseLong(source.get("source").toString());
				long createTime=Long.parseLong(source.get("createTime").toString());
				long firstCreateTime=Long.parseLong(source.get("firstCreateTime").toString());
				String ip=source.get("ip")==null?"":source.get("ip").toString();
				String url=source.get("url")==null?"":source.get("url").toString();
				String note=source.get("note")==null?"":source.get("note").toString();
				String bigArea=source.get("bigArea")==null?"":source.get("bigArea").toString();
				String smallArea=source.get("smallArea")==null?"":source.get("smallArea").toString();
				bo.setIp(ip);
				bo.setUrl(url);
				bo.setLid(lid2);
				bo.setType(type);
				bo.setSource(source1);
				bo.setNote(note);
				bo.setBigArea(bigArea);
				bo.setState(state);
				bo.setCreateTime(createTime);
				bo.setFirstCreateTime(firstCreateTime);
				bo.setIsLive(isLive);
				bo.setSmallArea(smallArea);
				bo.setReliability(reliability);
				bo.setIpLong(ipLong);
				bo.setPort(port);
				
				//更新 update
				bo.setUpdateTime(System.currentTimeMillis());
				String  jsonData=SortUtil.toJson(bo);
				// 追加到 es 上
				client.prepareIndex("botnet", "botnet")
				.setId(docId)
				.setSource(jsonData)
				.execute()
				.actionGet();
			}
			
		}
		
	}
	/**
	 * update-->updateTime
	 * key: ip
	 */
	public void UpdateCCInfo(String key,String value,BotnetBean bean){
		ESClient client = this.getClient();
		// 小于 正常期的 都是即将要删除掉的
		boolean existsIndex = isExistsIndex("botnet");
		if(existsIndex){
			
			QueryBuilder queryBuilderKey=QueryBuilders.termQuery(key, value);
			QueryBuilder queryBuilderType=QueryBuilders.termQuery("type", "1");
			QueryBuilder queryBuilder=QueryBuilders.boolQuery().must(queryBuilderType).must(queryBuilderKey);
			SearchResponse repose = client.prepareSearch("botnet")
					.setTypes("botnet")
					.setQuery(queryBuilder)
					.setSize(Integer.MAX_VALUE)
					.execute()
					.actionGet();
			SearchHits hits = repose.getHits();
			SearchHit[] hits2 = hits.getHits();
			if(hits2.length>0){
				// 获取文档 ID
				String docId = hits2[0].getId();
				Map<String, Object> source = hits2[0].getSource();
				Botnet bo = new Botnet();
				long lid2=(long)source.get("lid");
				long type=Long.parseLong(source.get("type").toString());
				long ipLong=Long.parseLong(source.get("ipLong").toString());
				long port=Long.parseLong(source.get("port").toString());
				long isLive=Long.parseLong(source.get("isLive").toString());
				long reliability=Long.parseLong(source.get("reliability").toString());
				long state=bean.getState();
				long source1=bean.getSource();
				long createTime=Long.parseLong(source.get("createTime").toString());
				long firstCreateTime=Long.parseLong(source.get("firstCreateTime").toString());
				String ip=source.get("ip")==null?"":source.get("ip").toString();
				String url=source.get("url")==null?"":source.get("url").toString();
				String note=bean.getNote();
				String bigArea=source.get("bigArea")==null?"":source.get("bigArea").toString();
				String smallArea=source.get("smallArea")==null?"":source.get("smallArea").toString();
				bo.setIp(ip);
				bo.setUrl(url);
				bo.setLid(lid2);
				bo.setType(type);
				bo.setSource(source1);
				bo.setNote(note);
				bo.setBigArea(bigArea);
				bo.setState(state);
				bo.setCreateTime(createTime);
				bo.setFirstCreateTime(firstCreateTime);
				bo.setIsLive(isLive);
				bo.setSmallArea(smallArea);
				bo.setReliability(reliability);
				bo.setIpLong(ipLong);
				bo.setPort(port);
				
				//更新 update
				bo.setUpdateTime(System.currentTimeMillis());
				String  jsonData=SortUtil.toJson(bo);
				// 追加到 es 上
				client.prepareIndex("botnet", "botnet")
				.setId(docId)
				.setSource(jsonData)
				.execute()
				.actionGet();
			}
			
		}
		
	}
	/**
	 * 时间格式化
	 * @return
	 */
	public String getStrTime(long time){
		SimpleDateFormat sdf= new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    	Date date = new Date(time);
    	String format = sdf.format(date);
    	
    	return format;
	}
	/*public List<BotNet> getList(SearchHit[] hitsArray){
		List<BotNet> botList=new LinkedList<BotNet>();
		for (SearchHit searchHit : hitsArray) {
			BotNet botNet = new BotNet();
			Map<String, Object> source = searchHit.getSource();
			//botNet.setNa(source.get(""));
		}
		return botList;
	}*/
	
	/**
     * 判断指定的索引名是否存在
     * @param indexName 索引名
     * @return  存在:true; 不存在:false;
     */
    public boolean isExistsIndex(String indexName){
    	boolean isExist = false;
    	 // 获取 client
    	ESClient client = this.getClient();
    	if(client != null){
    		
    		try {
				IndicesExistsResponse  response = 
						client.admin().indices().exists( 
								new IndicesExistsRequest().indices(new String[]{indexName})).actionGet();
				isExist = response.isExists();
			} catch (ElasticsearchException e) {
				logger.info("ES 发生异常:"+e);
				e.printStackTrace();
				isExist = false;
			}
    	}
        return isExist;
}

    
	/**
	 * 判断指定的索引的类型是否存在
	 * @param indexName 索引名
	 * @param indexType 索引类型
	 * @return  存在:true; 不存在:false;
	 */
	public boolean isExistsType(String indexName,String indexType){
		//ESClient client = this.getClient();
	    TypesExistsResponse  response = 
	            getClient().admin().indices()
	            .typesExists(new TypesExistsRequest(new String[]{indexName}, indexType)
	            ).actionGet();
	    return response.isExists();
	}

	
}



版权声明:本文为博主原创文章,未经博主允许不得转载。

一周杂记,linux设置环境变量,java文件操作(append,修改文件名)

(1)关于对linux设置环境变量,我基本知道大概,但是没有进行过相对系统和完整的总结,所以经常忘记具体的操作和相关的系统配置文件名,所以在此总结一下加深印象和便于查找。 1.显示环境变量的命令: ...

java的io操作杂记

1.在各个操作系统中,实际上路径的分隔符是不一样的,例如:windows中使用反斜杠:”\“,Linux中使用正斜杠:”/“。如果想让java的可移植性继续保持,则最好根据所在的操作系统来自动使用分隔...

elasticsearch java操作的api实例

  • 2017年09月29日 16:39
  • 14.56MB
  • 下载

elasticsearch__1__java操作之连接es,创建Mapping,保存数据

package com.elasticsearch; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestB...

ELK研究(一):elasticsearch java api接口操作ES集群 ---TransportClient的使用介绍 bulk批量提交数据

Java client操作ES: 1:配置集群对象信息;2:创建客户端;3:查看集群信息 1:集群名称       默认集群名为elasticsearch,如果集群名称和指定的不一致则在使用节点...

java操作elasticsearch使用QueryBuilders进行数据查询

ElasticSearch是一个基于Lucene的搜索服务器。它提供了一个分布式多用户能力的全文搜索引擎,基于RESTful web接口。Elasticsearch是用Java开发的,并作为Apach...

elasticsearch JAVA客户端操作---搜索的过滤、分组高亮,elasticsearchjava

elasticsearch JAVA客户端操作---搜索的过滤、分组高亮,elasticsearchjava elasticsearch        提供的API比较多,看名字差不多可以明...

elasticsearch__4__java操作之QueryBuilders构建搜索Query

内容涉及代码GitHub地址:https://github.com/xiaohulu/util_xiaohulu/tree/master/src/com/elasticsearch 官方API:h...

Elasticsearch 2.3.2 java客户端操作

1         JAVA客户端连接 Eclipse测试时候 需要在在main方法的类中右键选择run configurations,在Arguments标签页下的vm arguments输...
内容举报
返回顶部
收藏助手
不良信息举报
您举报文章:java 操作 elasticsearch 杂记
举报原因:
原因补充:

(最多只允许输入30个字)