package com.geo.cn;
import com.alibaba.fastjson.JSONArray;
import com.geo.cn.pojo.DaModel;
import com.geo.cn.pojo.Score;
import com.geo.cn.util.*;
import org.apache.commons.lang.StringUtils;
import org.apache.flume.*;
import org.apache.flume.conf.Configurable;
import org.apache.flume.sink.AbstractSink;
import org.apache.log4j.Logger;
import org.elasticsearch.client.RestHighLevelClient;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisSentinelPool;
import redis.clients.jedis.exceptions.JedisException;
import java.util.*;
/**
* main class that process the da model log
*/
public class DASink extends AbstractSink implements Configurable {
private static final Logger log = Logger.getLogger(DASink.class);
//custom elasticsearch parameters
private static final String PROP_KEY_ES_USERNAME = "es_username";
private static final String PROP_KEY_ES_PASSWORD = "es_password";
private static final String PROP_KEY_ES_HOST = "es_host";
private static final String PROP_KEY_ES_PORT = "es_port";
private static final String PROP_KEY_ES_CLUSTER_NAME = "es_clusterName";
private static final String PROP_KEY_ES_INDEX = "es_index";
private static final String PROP_KEY_ES_BATCH_SIZE = "es_batch_size";
//redis check whther it has repeatable model id
private static final String PROP_KEY_ES_REDIS_IS_CLUSTER = "es_redis_isCluster";
private static final String PROP_KEY_ES_REDIS_CLUSTER_NAME = "es_redis_clusterName";
private static final String PROP_KEY_ES_REDIS_HOST_NAME = "es_redis_hostName";
private static final String PROP_KEY_ES_REDIS_PORT = "es_redis_port";
private static final String PROP_KEY_ES_REDIS_PASSWORD = "es_redis_passWord";
private static final String PROP_KEY_ES_REDIS_DATABASE = "es_redis_database";
//transform score into bucket , A value by mysql database
private static final String PROP_KEY_ES_MYSQL_HOST_NAME = "es_mysql_hostName";
private static final String PROP_KEY_ES_MYSQL_PORT = "es_mysql_port";
private static final String PROP_KEY_ES_MYSQL_USER = "es_mysql_user";
private static final String PROP_KEY_ES_MYSQL_PASSWORD = "es_mysql_password";
private static final String PROP_KEY_ES_MYSQL_DATABASE = "es_mysql_db";
private static final String PROP_KEY_ES_MYSQL_TABLE = "es_mysql_table";
private String es_username;
private String es_password;
private String es_clusterName;
private String es_host;
private String es_port;
private String es_index;
private String es_batch_size = "100";
private String es_redis_isCluster;
private String es_redis_clusterName;
private String es_redis_hostName;
private String es_redis_port;
private String es_redis_passWord;
private String es_redis_database;
private String es_mysql_hostName;
private String es_mysql_port;
private String es_mysql_user;
private String es_mysql_password;
private String es_mysql_db;
private String es_mysql_table;
/**
* get custom flume parameters value from flume-conf.properties
* @param context flume context that configure the parameters
*/
public void configure(Context context) {
//connect to es
es_clusterName = StringUtils.trim(context.getString(PROP_KEY_ES_CLUSTER_NAME));
es_username = StringUtils.trim(context.getString(PROP_KEY_ES_USERNAME));
es_password = StringUtils.trim(context.getString(PROP_KEY_ES_PASSWORD));
es_host = StringUtils.trim(context.getString(PROP_KEY_ES_HOST));
es_port = StringUtils.trim(context.getString(PROP_KEY_ES_PORT));
es_index = StringUtils.trim(context.getString(PROP_KEY_ES_INDEX));
es_batch_size = StringUtils.trim(context.getString(PROP_KEY_ES_BATCH_SIZE));
//connect to redis
es_redis_isCluster = StringUtils.trim(context.getString(PROP_KEY_ES_REDIS_IS_CLUSTER));
es_redis_clusterName = StringUtils.trim(context.getString(PROP_KEY_ES_REDIS_CLUSTER_NAME));
es_redis_hostName = StringUtils.trim(context.getString(PROP_KEY_ES_REDIS_HOST_NAME));
es_redis_port = StringUtils.trim(context.getString(PROP_KEY_ES_REDIS_PORT));
es_redis_passWord = StringUtils.trim(context.getString(PROP_KEY_ES_REDIS_PASSWORD));
es_redis_database = StringUtils.trim(context.getString(PROP_KEY_ES_REDIS_DATABASE));
//connect to mysql
es_mysql_hostName = StringUtils.trim(context.getString(PROP_KEY_ES_MYSQL_HOST_NAME));
es_mysql_port = StringUtils.trim(context.getString(PROP_KEY_ES_MYSQL_PORT));
es_mysql_user = StringUtils.trim(context.getString(PROP_KEY_ES_MYSQL_USER));
es_mysql_password = StringUtils.trim(context.getString(PROP_KEY_ES_MYSQL_PASSWORD));
es_mysql_db = StringUtils.trim(context.getString(PROP_KEY_ES_MYSQL_DATABASE));
es_mysql_table = StringUtils.trim(context.getString(PROP_KEY_ES_MYSQL_TABLE));
}
/**
* extract the data from flume
* @return Status process log status:READY, BACKOFF
* @throws EventDeliveryException Event delivery occurrs error
*/
public Status process() throws EventDeliveryException {
// data transformation and processing
//Status status = null;
Status status ;
Channel channel = getChannel();
//guarantee to reap complete event by getting transaction
Transaction txn = channel.getTransaction();
txn.begin();
ArrayList<Event> rawdataFromFlume = new ArrayList<>();
int count;
//process every event
for (count = 0; count < Integer.parseInt(es_batch_size); ++count) {
Event event = channel.take();
if (event == null) {
break;
}
rawdataFromFlume.add(event);
}
// iterate to process every body
try {
//transfer data from byte stream
//ES insert document by type
ArrayList<HashMap> saveToEsList = new ArrayList<>();
//ArrayList<HashMap> es = null;
ArrayList<HashMap> es ;
if (count > 0) {
if (rawdataFromFlume.size() > 0) {
for (Event e : rawdataFromFlume) {
String body = new String(e.getBody());
if (!"".equals(body.trim()) && body.contains("/dp_monitor_da.db/orig_rh_wide/#@!") && (body.split("\t", -1).length == 13)) {
es = insertToElasticSearch(body, es_redis_isCluster, es_redis_clusterName, es_index, es_redis_hostName, es_redis_port, es_redis_passWord, es_redis_database, es_mysql_hostName, es_mysql_port, es_mysql_user, es_mysql_password, es_mysql_db, es_mysql_table);
if (es.size() > 0) {
es.forEach(obj -> saveToEsList.add(obj));
es.clear();
}
}
}
}
}
//batch insert es
if (saveToEsList.size() > 0) {
RestHighLevelClient highLevelClient = EsUtil.getRestHighLevelClient(es_username, es_password, es_host, Integer.parseInt(es_port), FlumeConstant.ES_REQUEST_TYPE);
EsUtil.bulkInsertES(highLevelClient, saveToEsList);
saveToEsList.clear();
}
//if events were process completely, commit current transaction
txn.commit();
status = Status.READY;
} catch (Throwable throwable) {
log.info("es_redis_isCluster: " + es_redis_isCluster + ", es_redis_clusterName: " + es_redis_clusterName + ", es_redis_hostName: " + es_redis_hostName + ", es_redis_port: " + es_redis_port + ", es_redis_passWord: " + es_redis_passWord + ", es_redis_database: " + es_redis_database);
//if events were not process completely ,it will rollback transaction
txn.rollback();
//status = Status.BACKOFF;
if (throwable instanceof Error) {
throw (Error) throwable;
} else {
throw new EventDeliveryException(throwable);
}
} finally {
//finally,close transaction
txn.close();
}
//返回完成状态
return status;
}
/**
* insert converted data with special standard into Elasticsearch
*
* @param body:kafka log
* @param es_redis_isCluster:whether redis service is cluster or not
* @param es_redis_cluster_name:redis cluster name
* @param es_index :es index
* @param es_redis_hostName :es redis hostname
* @param es_redis_port :es redis port
* @param es_redis_passWord:redis passord that connect to redis
* @param es_redis_database:redis database that connect to redis
* @param es_mysql_hostName :MySQL hostname that connect to MySQL
* @param es_mysql_port : MySQL port that connect to MySQL
* @param es_mysql_user : MySQL user that connect to MySQL
* @param es_mysql_password : MySQL password that connect to MySQL
* @param es_mysql_db : MySQL database that connect to MySQL
* @param es_mysql_table :MySQL table that connect to MySQL
* @return :list that insert into es ,it's will convert some raw data into new data by model basic data
*/
private static ArrayList<HashMap> insertToElasticSearch(String body, String es_redis_isCluster, String es_redis_cluster_name, String es_index, String es_redis_hostName, String es_redis_port, String es_redis_passWord, String es_redis_database, String es_mysql_hostName, String es_mysql_port, String es_mysql_user, String es_mysql_password, String es_mysql_db, String es_mysql_table) {
ArrayList<HashMap> cleanList = new ArrayList<>();
//check whether mode_id exists in redis set
String[] modelArrary = body.split("\t", -1);
//String modelName = modelArrary[0].split("#@!")[1];
String modelName = modelArrary[1];
Jedis jedis = null;
JedisSentinelPool jedisSentinelPool = null;
JedisPool jedisPool = null;
try {
//Jedis jedis = RedisUtil.getJedis(es_redis_isCluster,es_redis_cluster_name,es_redis_hostName, es_redis_port, es_redis_passWord,es_redis_database);
if ("true".equals(es_redis_isCluster)) {
jedisSentinelPool = RedisUtil.getJediSentinelPool(es_redis_isCluster, es_redis_cluster_name, es_redis_hostName, es_redis_port, es_redis_passWord, es_redis_database);
jedis = jedisSentinelPool.getResource();
jedis.select(Integer.parseInt(es_redis_database));
} else {
jedisPool = RedisUtil.getJedisPool(es_redis_isCluster, es_redis_cluster_name, es_redis_hostName, es_redis_port, es_redis_passWord, es_redis_database);
jedis = jedisPool.getResource();
jedis.select(Integer.parseInt(es_redis_database));
}
} catch (JedisException e) {
log.info("es_redis_isCluster: " + es_redis_isCluster + ", es_redis_clusterName: " + es_redis_cluster_name + ", es_redis_hostName: " + es_redis_hostName + ", es_redis_port: " + es_redis_port + ", es_redis_passWord: " + es_redis_passWord + ", es_redis_database: " + es_redis_database);
log.info("数据传输发送错误： " + e.getMessage());
}
//新增多基准模型监控
Set<String> indexColsKeySet = jedis.keys(modelName + "*");
ArrayList<HashMap> subVersionList;
if (null != indexColsKeySet && indexColsKeySet.size() > 0) {
for (String indexColsNameJsonKey : indexColsKeySet) {
subVersionList = getDAScoreAndFeatureStatbilities(body, jedis.get(indexColsNameJsonKey), indexColsNameJsonKey, modelArrary, jedis, es_index, es_mysql_hostName, es_mysql_port, es_mysql_user, es_mysql_password, es_mysql_db, es_mysql_table);
if (subVersionList.size() > 0) {
cleanList.add(subVersionList.get(0));
cleanList.add(subVersionList.get(1));
cleanList.add(subVersionList.get(2));
subVersionList.clear();
}
}
}
//删除数据
if(null != indexColsKeySet)
indexColsKeySet.clear();
try {
if ("true".equals(es_redis_isCluster) && null != jedisSentinelPool) {
jedisSentinelPool.returnBrokenResource(jedis);
} else {
if (null != jedisPool) {
jedisPool.returnBrokenResource(jedis);
}
}
} catch (Exception e) {
log.info("回收redis连接发生错误： " + e.getMessage());
}
return cleanList;
}
/**
* convert raw data into standard data that will be insert into Elastic search according to score stability and feature statibility
* @param body kafka log
* @param indexColsNameJson relation between kafka log and column name
* @param modelName model name
* @param modelArrary kafka log array
* @param jedis redis client that connect to redis service
* @param es_index es index name
* @param es_mysql_hostName hostname that connect to MySQL
* @param es_mysql_port port that connect to MySQL
* @param es_mysql_user user that connect to MySQL
* @param es_mysql_password password that connect to MySQL
* @param es_mysql_db database that connect to MySQL
* @param es_mysql_table table that connect to MySQL
* @return ArrayList
*/
private static ArrayList<HashMap> getDAScoreAndFeatureStatbilities(String body, String indexColsNameJson, String modelName, String[] modelArrary, Jedis jedis, String es_index, String es_mysql_hostName, String es_mysql_port, String es_mysql_user, String es_mysql_password, String es_mysql_db, String es_mysql_table) {
ArrayList<HashMap> cleanList = new ArrayList<>();
String createTime = DateUtil.timestampToESDateStr(new Date().getTime());
if (null != indexColsNameJson && !"".equals(indexColsNameJson)) {
//bind kafka log with columnName from the redis
HashMap<String, Object> colsNameAndValues = DataUtil.tranferRawData(modelArrary, indexColsNameJson);
//获取DAmodeid 与mysql的映射关系 ，避免冲突；获取后删除
// score fault-tolerance
//String modeid = "";
String modeid ;
if (colsNameAndValues.size() > 0 && colsNameAndValues.containsKey(modelName)) {
modeid = colsNameAndValues.get(modelName).toString();
colsNameAndValues.remove(modelName);
//mode_id ,score(geo_score),colsName:values,createdTime,updatedTime
//get suitable data
//DaModel daModel = null;
DaModel daModel ;
if (null != colsNameAndValues.get("geo_score")) {
//constuctor Model object by model data
// Model(String query_id, Long log_id, String model_id, String query_type, String modelparameters, String score, Long code, Long startTime, Long endTime, Long lines)
if (null == colsNameAndValues.get("geo_score") || "".equals(colsNameAndValues.get("geo_score"))) {
daModel = new DaModel(modelName, null, colsNameAndValues, createTime);
} else {
daModel = new DaModel(modelName, colsNameAndValues.get("geo_score").toString(), colsNameAndValues, createTime);
}
//query mode score stability from mysql gai_model_params
Score score = null;
HashMap nameId = new HashMap();
//JSONArray scoreInfoAndNameIdjson = null;
JSONArray scoreInfoAndNameIdjson ;
//JSONArray featureNames = null;
String scoreInfoAndNameIdStr = jedis.get("modeid" + modelName);
if (null != scoreInfoAndNameIdStr) {
try {
score = DADButil.getDaGEOScore(scoreInfoAndNameIdStr, daModel);
} catch (Exception e) {
log.error("flume extract message： " + body);
log.error("CustomSink get score bucketId from redis： " + e.getMessage());
}
try {
nameId = DADButil.getDaGEONameId(scoreInfoAndNameIdStr, daModel.modelparameters, body);
//get featurename list
//featureNames = JSONArray.parseArray(scoreInfoAndNameIdStr);
} catch (Exception e) {
log.error("flume extract message： " + body);
log.error("CustomSink get feature stability of Model from redis： " + e.getMessage());
}
} else {
scoreInfoAndNameIdjson = DADButil.getModeInfoByModeidFromMysql(Integer.parseInt(modeid), jedis, es_mysql_hostName, es_mysql_port, es_mysql_user, es_mysql_password, es_mysql_db, es_mysql_table);
//featureNames = scoreInfoAndNameIdjson;
if (null != scoreInfoAndNameIdjson && scoreInfoAndNameIdjson.size() == 2) {
try {
score = DADButil.getDaGEOScore(scoreInfoAndNameIdjson.toString(), daModel);
} catch (Exception e) {
log.error("flume extract message： " + body);
log.error("CustomSink get score bucketId from MySQL： " + e.getMessage());
}
nameId = DADButil.getDaGEONameId(scoreInfoAndNameIdjson.toString(), daModel.modelparameters, body);
} else {
log.error("flume extract message： " + body);
log.error("Score domain and Feature stabilities query result from MySQL is not complete !");
}
}
///transform raw data to standard data and regular data
//1.计算出正确的调用评分稳定性和特征分布稳定性
if (null != score && nameId.size() > 0) {
//用过修改es index的index name,测试完毕删除
StringBuilder sb = new StringBuilder();
// 1.standard data
HashMap<Object, Object> resultToES = new HashMap<>(16);
String queryTime = DADateUtil.timeStrToDateStr(colsNameAndValues.get("query_time").toString());
resultToES.put("score", score.bucketid);
resultToES.put("score_section", score.section);
//feature stabilities
for (Object obj : nameId.entrySet()) {
Map.Entry entry = (Map.Entry) obj;
resultToES.put(entry.getKey(), entry.getValue());
}
resultToES.put("inner_if_type", colsNameAndValues.get("inner_if_type"));
//resultToES.put("user_other",colsNameAndValues.get("user_other"));
resultToES.put("user_id", colsNameAndValues.get("user_id"));
//resultToES.put("user_code",colsNameAndValues.get("user_code"));
resultToES.put("user_code", colsNameAndValues.get("user_code"));
resultToES.put("startTime", queryTime);
resultToES.put("endTime", queryTime);
//specify es_index es_type
//gai-monitor-bin-index.20190723
sb.append(es_index.split("\\.")[0]);
sb.append("-bin-index.");
sb.append(es_index.split("\\.")[1]);
resultToES.put("es_index", sb.toString());
resultToES.put("es_type", modeid);
sb.setLength(0);
// 2. old data
HashMap<String, Object> oldData = new HashMap<>(16);
//`queryid` string,
//`log_id` int,
// `modelid` int,
// `querytype` int,
// `modelparameters` string,
// `score` string,
// `code` int,
// `starttime` bigint,
// `endtime` bigint,
// `lines` int)
oldData.put("starttime", queryTime);
oldData.put("endtime", queryTime);
oldData.put("inner_if_type", colsNameAndValues.get("inner_if_type"));
//oldData.put("user_other",colsNameAndValues.get("user_other"));
oldData.put("user_id", colsNameAndValues.get("user_id"));
//oldData.put("user_code",colsNameAndValues.get("user_code"));
oldData.put("user_code", colsNameAndValues.get("user_code"));
//save model_params
//colsNameAndValues.values().forEach(val-> System.out.println(val));
colsNameAndValues.values().forEach(value -> sb.append(value).append(":"));
//score
//oldData
if (null != colsNameAndValues.get("geo_score") && !"null".equals(colsNameAndValues.get("geo_score")) && !"".equals(colsNameAndValues.get("geo_score"))) {
oldData.put("score", Double.parseDouble(colsNameAndValues.get("geo_score").toString()));
} else {
oldData.put("score", null);
}
//model 特征分布稳定性
oldData.put("modelparameters", sb.subSequence(0, sb.length() - 1).toString());
sb.setLength(0);
//specify es_index ,es_type
//gai-monitor-old-index.20190723
sb.append(es_index.split("\\.")[0]);
sb.append("-old-index.");
sb.append(es_index.split("\\.")[1]);
oldData.put("es_index", sb.toString());
oldData.put("es_type", modeid);
sb.setLength(0);
// 3. regurlar data: save splited log data with colName:colValue
HashMap<Object, Object> logSearchData = new HashMap<>(16);
logSearchData.put("starttime", queryTime);
logSearchData.put("endtime", queryTime);
logSearchData.put("responsetime", 0);
logSearchData.put("code", 1);
//splited log data
for (Object kv : colsNameAndValues.entrySet()) {
Map.Entry entry = (Map.Entry) kv;
try {
logSearchData.put(entry.getKey().toString(), Double.parseDouble(entry.getValue().toString()));
} catch (Exception e) {
logSearchData.put(entry.getKey().toString(), null);
}
}
logSearchData.put("inner_if_type", colsNameAndValues.get("inner_if_type"));
//logSearchData.put("user_other",colsNameAndValues.get("user_other"));
logSearchData.put("user_id", colsNameAndValues.get("user_id"));
//logSearchData.put("user_code",colsNameAndValues.get("user_code"));
logSearchData.put("user_code", colsNameAndValues.get("user_code"));
//specify es_index ,es_type
if (null != colsNameAndValues.get("geo_score") && !"null".equals(colsNameAndValues.get("geo_score")) && !"".equals(colsNameAndValues.get("geo_score"))) {
logSearchData.put("score", Double.parseDouble(colsNameAndValues.get("geo_score").toString()));
} else {
logSearchData.put("score", null);
}
logSearchData.remove("geo_score");
//gai-monitor-export-index.20190723
sb.append(es_index.split("\\.")[0]);
sb.append("-export-index.");
sb.append(es_index.split("\\.")[1]);
logSearchData.put("es_index", sb.toString());
logSearchData.put("es_type", modeid);
sb.setLength(0);
//new data
cleanList.add(resultToES);
//old data
cleanList.add(oldData);
//log search
cleanList.add(logSearchData);
//2.未计算出评分稳定性或特征稳定性
} else {
//用过修改es index的index name,测试完毕删除
StringBuilder sb = new StringBuilder();
// 1.standard data
HashMap<Object, Object> resultToES = new HashMap<>(16);
String queryTime = DADateUtil.timeStrToDateStr(colsNameAndValues.get("query_time").toString());
if (null != score) {
resultToES.put("score", score.bucketid);
resultToES.put("score_section", score.section);
} else {
resultToES.put("score", null);
resultToES.put("score_section", null);
}
//feature stabilities未正确计算，不予存储
// for (Object obj : nameId.entrySet()) {
// Map.Entry entry = (Map.Entry)obj;
// resultToES.put(entry.getKey(),entry.getValue() );
// }
resultToES.put("inner_if_type", colsNameAndValues.get("inner_if_type"));
//resultToES.put("user_other",colsNameAndValues.get("user_other"));
resultToES.put("user_id", colsNameAndValues.get("user_id"));
//resultToES.put("user_code",colsNameAndValues.get("user_code"));
resultToES.put("user_code", colsNameAndValues.get("user_code"));
resultToES.put("startTime", queryTime);
resultToES.put("endTime", queryTime);
//specify es_index es_type
//gai-monitor-bin-index.20190723
sb.append(es_index.split("\\.")[0]);
sb.append("-bin-index.");
sb.append(es_index.split("\\.")[1]);
resultToES.put("es_index", sb.toString());
resultToES.put("es_type", modeid);
sb.setLength(0);
// 2. old data
HashMap<String, Object> oldData = new HashMap<>(16);
//`queryid` string,
//`log_id` int,
// `modelid` int,
// `querytype` int,
// `modelparameters` string,
// `score` string,
// `code` int,
// `starttime` bigint,
// `endtime` bigint,
// `lines` int)
oldData.put("starttime", queryTime);
oldData.put("endtime", queryTime);
oldData.put("inner_if_type", colsNameAndValues.get("inner_if_type"));
//oldData.put("user_other",colsNameAndValues.get("user_other"));
oldData.put("user_id", colsNameAndValues.get("user_id"));
//oldData.put("user_code",colsNameAndValues.get("user_code"));
oldData.put("user_code", colsNameAndValues.get("user_code"));
//save model_params
//colsNameAndValues.values().forEach(val-> System.out.println(val));
colsNameAndValues.values().forEach(value -> sb.append(value).append(":"));
//score
//oldData
if (null != colsNameAndValues.get("geo_score") && !"null".equals(colsNameAndValues.get("geo_score")) && !"".equals(colsNameAndValues.get("geo_score"))) {
oldData.put("score", Double.parseDouble(colsNameAndValues.get("geo_score").toString()));
} else {
oldData.put("score", null);
}
//model 特征分布稳定性，未正确计算，存储原始数据
oldData.put("modelparameters", sb.subSequence(0, sb.length() - 1).toString());
sb.setLength(0);
//specify es_index ,es_type
//gai-monitor-old-index.20190723
sb.append(es_index.split("\\.")[0]);
sb.append("-old-index.");
sb.append(es_index.split("\\.")[1]);
oldData.put("es_index", sb.toString());
oldData.put("es_type", modeid);
sb.setLength(0);
// 3. regurlar data: save splited log data with colName:colValue
HashMap<Object, Object> logSearchData = new HashMap<>(16);
logSearchData.put("starttime", queryTime);
logSearchData.put("endtime", queryTime);
logSearchData.put("responsetime", 0);
logSearchData.put("code", 1);
//splited log data, feature stabilities未正确计算，不予存储
// for (Object kv:colsNameAndValues.entrySet()) {
// Map.Entry entry = (Map.Entry)kv;
// try {
// logSearchData.put(entry.getKey().toString(),Double.parseDouble(entry.getValue().toString()));
// } catch (Exception e) {
// logSearchData.put(entry.getKey().toString(),null);
// }
// }
logSearchData.put("inner_if_type", colsNameAndValues.get("inner_if_type"));
//logSearchData.put("user_other",colsNameAndValues.get("user_other"));
logSearchData.put("user_id", colsNameAndValues.get("user_id"));
//logSearchData.put("user_code",colsNameAndValues.get("user_code"));
logSearchData.put("user_code", colsNameAndValues.get("user_code"));
//specify es_index ,es_type
if (null != colsNameAndValues.get("geo_score") && !"null".equals(colsNameAndValues.get("geo_score")) && !"".equals(colsNameAndValues.get("geo_score"))) {
logSearchData.put("score", Double.parseDouble(colsNameAndValues.get("geo_score").toString()));
} else {
logSearchData.put("score", null);
}
logSearchData.remove("geo_score");
//gai-monitor-export-index.20190723
sb.append(es_index.split("\\.")[0]);
sb.append("-export-index.");
sb.append(es_index.split("\\.")[1]);
logSearchData.put("es_index", sb.toString());
logSearchData.put("es_type", modeid);
sb.setLength(0);
//new data
cleanList.add(resultToES);
//old data
cleanList.add(oldData);
//log search
cleanList.add(logSearchData);
}
} else {
log.error("================= There is not DA model Mapping Relation =================");
}
} else {
log.warn("There is suitable score value in kafka log data !! " + body);
}
if (colsNameAndValues.size() > 0) {
colsNameAndValues.clear();
}
} else {
log.warn("There is not related index_colNames data in redis! " + body);
}
return cleanList;
}
public static void main(String[] args) {
String body = "/dp_monitor_da.db/orig_rh_wide/#@!20191129#@!30392784\tKGH00002\t100040\ticbc\t20191129172224\t20191129\t17\t490\t\t\t{\"d\":\"0.07995524536550705\",\"yr\":\"0.06341848\",\"t_indcf03000\":\"0.591135081\",\"\":\"\",\"ym_indckf09000\":\"0.530127655\"}\t{\"reg_reg_max_time\":\"108\",\"reg_reg_max_time_180d\":\"108\",\"reg_reg_max_time_360d\":\"108\",\"reg_reg_max_time_b\":\"108\",\"reg_reg_max_time_b_360d\":\"108\",\"reg_reg_max_time_nb\":\"234\",\"reg_reg_max_time_nb_360d\":\"234\",\"reg_reg_min_time\":\"540\",\"reg_reg_min_time_180d\":\"108\",\"reg_reg_min_time_360d\":\"234\",\"reg_reg_min_time_b\":\"108\",\"reg_reg_min_time_b_360d\":\"108\",\"reg_reg_min_time_nb\":\"540\",\"reg_reg_min_time_nb_360d\":\"234\",\"reg_dt_mean\":\"216.00000000000000\",\"reg_dt_mean_180d\":\"-1\",\"reg_dt_mean_360d\":\"126.00000000000000\",\"reg_dt_mean_nb\":\"306.00000000000000\",\"reg_dt_mean_nb_360d\":\"-1\",\"reg_dt_std\":\"127.27922061357855\",\"reg_dt_std_180d\":\"-1\",\"reg_dt_std_360d\":\"-1\",\"reg_dt_std_nb\":\"-1\",\"reg_dt_std_nb_360d\":\"-1\",\"reg_dt_vi\":\"0.58925565098878958\",\"reg_dt_vi_180d\":\"-1\",\"reg_dt_vi_360d\":\"-1\",\"reg_dt_vi_nb\":\"-1\",\"reg_dt_vi_nb_360d\":\"-1\",\"reg_max_dt\":\"306.00000000000000\",\"reg_max_dt_180d\":\"-1\",\"reg_max_dt_360d\":\"126.00000000000000\",\"reg_max_dt_nb\":\"306.00000000000000\",\"reg_max_dt_nb_360d\":\"-1\",\"reg_max_dt_b\":\"-1\",\"reg_min_dt\":\"126.00000000000000\",\"reg_min_dt_180d\":\"-1\",\"reg_min_dt_360d\":\"126.00000000000000\",\"reg_min_dt_nb\":\"306.00000000000000\",\"reg_min_dt_nb_360d\":\"-1\",\"lo_to_reg_pn\":\"0\",\"ap_to_reg_pn\":\"0\",\"ap_to_reg_pn_180d\":\"0\",\"ap_to_reg_pn_360d\":\"0\",\"ap_to_reg_pn_nb\":\"0\",\"ap_to_reg_tn_nb\":\"0\",\"reg_reg_total_num\":\"3\",\"apply_reg_total_num\":\"0\",\"apply_reg_plat_num\":\"0\",\"reg_reg_plat_num\":\"3\",\"reg_reg_total_num_nb_360d\":\"1\",\"reg_reg_plat_num_nb_360d\":\"1\",\"apply_reg_total_num_nb_360d\":\"0\",\"apply_reg_plat_num_nb_360d\":\"0\",\"cdzc019_indca03000\":\"\"}\t1";
System.out.println(body.split("\t", -1).length);
System.out.println("是否包含前缀 " + body.contains("/dp_monitor_da.db/orig_rh_wide/#@!"));
// ArrayList<HashMap> saveToEsList = insertToElasticSearch(str,"false"
// , "redis-gai"
// , "gai"
// , "10.111.32.205"
// , "6379"
// , "Geotmt_123"
// , "1"
// , "10.111.32.205"
// , "3306"
// ,"root"
// , "Geotmt_123"
// , "gai"
// , "gai_model_params"
//);
// RestHighLevelClient highLevelClient = EsUtil.getRestHighLevelClient("elastic","","10.111.32.205", Integer.parseInt("9200"), FlumeConstant.ES_REQUEST_TYPE);
//
// EsUtil.bulkInsertES(highLevelClient, saveToEsList);
String es_index = "gai-monitor.20190723";
if (!"".equals(body.trim()) && body.contains("/dp_monitor_da.db/orig_rh_wide/#@!") && body.split("\t", -1).length == 13) {
System.out.println("=========================================== 开始插入数据");
ArrayList<HashMap> saveToEsList = insertToElasticSearch(body,
//"true"
"false"
, "mymaster"
, es_index
//, "10.111.32.187,10.111.32.183,10.111.32.248"
, "10.111.32.205"
//, "26371"
, "6379"
, "Geotmt_123"
, "1"
, "10.111.32.205"
, "3306"
, "root"
, "Geotmt_123"
, "gai"
, "gai_model_params");
RestHighLevelClient highLevelClient = EsUtil.getRestHighLevelClient("elastic", "", "10.111.32.205", Integer.parseInt("9200"), FlumeConstant.ES_REQUEST_TYPE);
System.out.println(saveToEsList);
EsUtil.bulkInsertES(highLevelClient, saveToEsList);
System.out.println("=========================================== 数据插入成功");
}
}
}
扫一扫
11-19
793
793
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
