java校验索引_ES全量索引校验-java

importcom.alibaba.fastjson.JSONArray;importcom.alibaba.fastjson.JSONObject;import com.qa.utils.*;importorg.testng.annotations.DataProvider;importorg.testng.annotations.Test;importjava.io.BufferedReader;importjava.io.FileInputStream;importjava.io.IOException;importjava.io.InputStreamReader;import java.sql.*;import java.util.*;public classGBIndexCheck3 {

@DataProvider(name= "mysqlData")public static Object[][] toBeDataProvider() throwsSQLException, ClassNotFoundException {returngetTestData();

}

@Test(dataProvider= "mysqlData")//@Test(dataProvider = "testData")

public void compareIndex(String goodsSn, String whcode) throwsException {//请求索引一

String goodsId = goodsSn + "#" +whcode;

String esURL= "http://xxxx/Full_Index_test_GB_old/_search";

String esSheetName= "es";

List list1 =indexGB(goodsId, esURL, esSheetName);//请求索引二

String aiURL = "http://xxxx/GB_daily_full/_search";

String aiSheetName= "ai";

List list2 =indexGB(goodsId, aiURL, aiSheetName);//判断两份索引数据的一致性

Boolean falg = false;

falg=isListEqual(list1, list2);if (!falg) {//System.out.println("【索引数据不一致】");//索引数据不一致sku写入txt

WriteTxtUtil.writerForString(goodsId);

}

}public List indexGB(String goodsId, String url, String sheetName) throwsException {//获取索引路径//String url = (String) PropertiesUtil.getProperties().get("GB.es.url");//String url = "http://10.4.4.80:9200/gearbest20191205141505/_search";//获取参数//String params = (String) PropertiesUtil.getProperties().get("GB.es.params");

String params = "{\"query\":{\"term\":{\"goodsId\":{\"value\":\"" + goodsId + "\"}}}}";//String params = "{\"query\":{\"term\":{\"goodsSn\":{\"value\":\"244297701\"}}}}";

System.out.println(params);//响应结果

JSONObject responseObject =RestClient.post(url, params);

System.out.println(sheetName+ " ---> " + "responce: " +responseObject);

JSONObject data= (JSONObject) responseObject.get("hits");

JSONArray jsonArray= data.getJSONArray("hits");//遍历jsonArray

Map map = new HashMap<>(); //索引字段数据存map

List list = new ArrayList<>(); //MD5加密后数据存list

for (int i = 0; i < jsonArray.size(); i++) {

JSONObject object=(JSONObject) jsonArray.get(i);

JSONObject source= (JSONObject) object.get("_source");//内嵌对象排序

JSONArray skuDescAttrs = source.getJSONArray("skuDescAttrs");

skuDescAttrs.sort(Comparator.comparing(e-> ((JSONObject) e).getInteger("attrValueKey")));

map.put("skuDescAttrs", skuDescAttrs.toString());// map.put("week2SalesVolume", source.getString("week2SalesVolume"));

map.put("payEndTime", source.getString("payEndTime"));

map.put("appSwellAmount", source.getString("appSwellAmount"));

map.put("appPriceType", source.getString("appPriceType"));

map.put("goodsId", source.getString("goodsId"));

map.put("youtube", source.getString("youtube"));//skuAttrs

JSONArray skuAttrs = source.getJSONArray("skuAttrs");

skuAttrs.sort(Comparator.comparing(e->((JSONObject)e).getInteger("attrType")));

map.put("skuAttrs",skuAttrs.toString());

map.put("week2Sales", source.getString("week2Sales"));

map.put("shopPrice", source.getString("shopPrice"));//grossMargin 先写死//map.put("grossMargin",source.getString("grossMargin"));

map.put("baseScore2", source.getString("baseScore2"));

map.put("subTitle", source.getString("subTitle"));

map.put("exposureFlag", source.getString("exposureFlag"));

map.put("originalUrl", source.getString("originalUrl"));

map.put("discount", source.getString("discount"));

map.put("whCode", source.getString("whCode"));

map.put("baseScore5", source.getString("baseScore5"));

map.put("payStartTime", source.getString("payStartTime"));

map.put("recommendedLevel", source.getString("recommendedLevel"));

map.put("thumbExtendUrl", source.getString("thumbExtendUrl"));

map.put("appStatus", source.getString("appStatus"));

map.put("passAvgScore", source.getString("passAvgScore"));

map.put("stockFlag", source.getString("stockFlag"));

map.put("brandName", source.getString("brandName"));

map.put("appDisplayPrice", source.getString("appDisplayPrice"));

map.put("centerWord", source.getString("centerWord"));

map.put("goodsTitle", source.getString("goodsTitle"));

map.put("brandCode", source.getString("brandCode"));

map.put("appDeposit", source.getString("appDeposit"));

map.put("vWhCode", source.getString("vWhCode"));

map.put("catId", source.getString("catId"));

map.put("priceRates", source.getString("priceRates"));

map.put("totalFavoriteCount", source.getString("totalFavoriteCount"));

map.put("baseScore4", source.getString("baseScore4"));

map.put("appPayEndTime", source.getString("appPayEndTime"));

map.put("urlTitle", source.getString("urlTitle"));//categories

JSONArray categories = source.getJSONArray("categories");

categories.sort(Comparator.comparing(e->((JSONObject)e).getInteger("catId")));

map.put("categories",categories.toString());

map.put("firstUpTime", source.getString("firstUpTime"));//labelFlags

JSONArray labelFlags = source.getJSONArray("labelFlags");

labelFlags.sort(Comparator.comparing(e->((JSONObject)e).getInteger("labelId")));

map.put("labelFlags",labelFlags.toString());

map.put("appPayStartTime", source.getString("appPayStartTime"));

map.put("expiredTime", source.getString("expiredTime"));

map.put("exposureSalesVolume", source.getString("exposureSalesVolume"));

map.put("appDiscount", source.getString("appDiscount"));

map.put("baseScore1", source.getString("baseScore1"));// JSONArray coupons = source.getJSONArray("coupons");

coupons.sort(Comparator.comparing(e->((JSONObject)e).getInteger("code")));

map.put("coupons",coupons.toString());

map.put("lang", source.getString("lang"));

map.put("isCod", source.getString("isCod"));

map.put("searchWords", source.getString("searchWords"));//defaultWh 先写死//map.put("defaultWh",source.getString("defaultWh"));

map.put("mStatus", source.getString("mStatus"));

map.put("dailyRate", source.getString("dailyRate"));

map.put("exposureSalesRate", source.getString("exposureSalesRate"));

map.put("sortOrder", source.getString("sortOrder"));

map.put("shopGroups", source.getString("shopGroups"));

map.put("passTotalNum", source.getString("passTotalNum"));

map.put("shopCode", source.getString("shopCode"));

map.put("appExpiredTime", source.getString("appExpiredTime"));

map.put("isTort", source.getString("isTort"));

map.put("baseScore3", source.getString("baseScore3"));

map.put("priceType", source.getString("priceType"));

map.put("createTime", source.getString("createTime"));

map.put("swellAmount", source.getString("swellAmount"));

map.put("saleMark", source.getString("saleMark"));

map.put("deposit", source.getString("deposit"));

map.put("goodsModelWord", source.getString("goodsModelWord"));//appDefaultWh 先写死//map.put("appDefaultWh",source.getString("appDefaultWh"));

map.put("goodsWebSku", source.getString("goodsWebSku"));

map.put("imgExtendUrl", source.getString("imgExtendUrl"));//activityIds 数组//map.put("activityIds",source.getString("activityIds"));

map.put("displayPrice", source.getString("displayPrice"));

map.put("goodsWebSpu", source.getString("goodsWebSpu"));

map.put("isPlatform", source.getString("isPlatform"));

map.put("webStatus", source.getString("webStatus"));

map.put("goodsSn", source.getString("goodsSn"));

map.put("imgUrl", source.getString("imgUrl"));//tags 数组//map.put("tags",source.getString("tags"));//map.put("yesterdaySales",source.getString("yesterdaySales"));

map.put("yesterdaySales", 0);

JSONArray activities= source.getJSONArray("activities");

activities.sort(Comparator.comparing(e->((JSONObject)e).getInteger("activityId")));

map.put("activities",activities.toString());//遍历map

for (Map.Entryentry : map.entrySet()) {//Log.info("索引方案"+sheetName);

Log.info(entry.getKey() + ":" +entry.getValue());

}//MD5处理

String str =MapToString.getMapToString(map);

System.out.println("map转string: " +str);

String str2= goodsId + "-" +MD5Util.md5(str);

System.out.println("MD5加密后：" +str2);

list.add(str2);

}//ESWriteExcelUtilsForList.doWrite(list,"D://索引测试数据.xlsx",sheetName,0,1);

returnlist;

}//内嵌对象排序

public voidarraySort(JSONArray jsonArray,String...values){

jsonArray.sort((a,b)->{int i = ((JSONObject)a).getString(values[0]).compareTo(((JSONObject)b).getString(values[0]));if(i==0){int j = ((JSONObject)a).getInteger(values[1]).compareTo(((JSONObject)b).getInteger(values[1]));returnj;

}returni;

});

}//获取Mysql数据库中的测试数据

public static Object[][] getTestData() throwsClassNotFoundException, SQLException {//设置mysql驱动

Class.forName("com.mysql.jdbc.Driver");//建立数据库的连接 10.60.46.206:3307

Connection conn = DriverManager.getConnection("jdbc:mysql://10.40.2.131:3306/gb_goods", "java-service", "java123456");//Connection conn = DriverManager.getConnection("jdbc:mysql:///test", "root", "root");//判断数据连接是否成功

if (!conn.isClosed()) {

System.out.println("连接成功");

}else{

System.out.println("连接失败");

}//创建Statement对象可以用对应的方法executeQuery(sql语句)获取测试数据

Statement statement =conn.createStatement();//创建一个结果集存放数据库执行完sql的数据//select DISTINCT good_sn from goods_info where good_sn in('237646510','258050019')

String sql= "SELECT gie.good_sn,gie.v_wh_code AS goodsSn FROM goods_info_extend_s_53 gie WHERE gie.goods_status IN (2,4,5) AND gie.platform IN (1,2,4) AND gie.site_code = 'GB' GROUP BY gie.good_sn, gie.v_wh_code;";//String sql ="SELECT goods_sn,1 from kp_goods;";

ResultSet resultSet =statement.executeQuery(sql);//声明存放泛型string数组的list对象

List list = new ArrayList();int colNum =resultSet.getMetaData().getColumnCount();while(resultSet.next()) {

String[] strings= newString[colNum];for (int i = 0; i < strings.length; i++) {

strings[i]= resultSet.getString(i + 1);

}

list.add(strings);

}

resultSet.close();

conn.close();//将list对象转换成二维数组

Object[][] results = newObject[list.size()][];for (int i = 0; i < list.size(); i++) {

results[i]=list.get(i);

}returnresults;

}//比较两个list内容是否一致

public static booleanisListEqual(List l0, List l1) {if (l0 ==l1)return true;if (l0 == null && l1 == null)return true;if (l0 == null || l1 == null)return false;if (l0.size() !=l1.size())return false;for(Object o : l0) {if (!l1.contains(o))return false;

}for(Object o : l1) {if (!l0.contains(o))return false;

}return true;

}//csv dataProvider

@DataProvider(name = "testData")public static Object[][] words() throwsIOException {return getTestData1("D://kp_goodsSn.csv");

}//读取CSV文件的方法

public static Object[][] getTestData1(String fileName) throwsIOException {//定义一个集合，存csv文件中的数据

List records = new ArrayList();

String record;

BufferedReader file= new BufferedReader(new InputStreamReader(new FileInputStream(fileName), "UTF-8"));//file.readLine();//跳过表头, 如果不需要表头的话，不要写这句

while ((record = file.readLine()) != null) {

String fields[]= record.split(",");

records.add(fields);

}

file.close();//定义方法的返回值，将list转换为Object二维数据

Object[][] results = newObject[records.size()][];//设置二维数每行的值，每行是一个Object对象

for (int i = 0; i < records.size(); i++) {

results[i]=records.get(i);

}returnresults;

}

}