//impala 一条一条插入影响性能,更换成另一个种方式,换成直接调用kuduClient之API,直接插入kudu,加油,小伴伙。 package com.gome.ghs.footfall.impala.service.impl; import com.gome.boot.common.annotation.aspect.LoggerAspect; import com.gome.boot.common.annotation.aspect.TimeAspect; import com.gome.boot.common.enums.DateStyleEnum; import com.gome.boot.common.logger.BaseLogger; import com.gome.boot.common.util.DateUtils; import com.gome.boot.common.util.JacksonUtils; import com.gome.ghs.footfall.impala.model.GhsOrder; import com.gome.ghs.footfall.impala.service.GhsOrderReceiveService; import org.apache.commons.lang.StringUtils; import org.apache.kudu.client.*; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import javax.validation.constraints.NotNull; import java.math.BigDecimal; /** * Created by Renzhanpo on 2019/6/11. */ @Service("ghsOrderReceiveService") @LoggerAspect @TimeAspect public class GhsOrderReceiveServiceImpl extends BaseLogger implements GhsOrderReceiveService { @Autowired KuduClient kuduInstance; @Override public void receiveOrder(@NotNull GhsOrder ghsOrder) { String orderId = ghsOrder.getOrderId(); String shippingGroupId = ghsOrder.getShippingGroupId(); int count = queryOrderByOrderId(orderId,shippingGroupId); if(count > 0){ updateOrderKudu(ghsOrder); }else{ insertOrderKudu(ghsOrder); } } private void insertOrderKudu(GhsOrder ghsOrder) { logger.info("GhsOrderReceiveServiceImpl.insertOrderKudu is begin json={}", JacksonUtils.convertObjectToJson(ghsOrder)); try { // 打开表 KuduTable table = kuduInstance.openTable("impala::t_homebiz.order_source_data"); // 创建写session,kudu必须通过session写入 KuduSession session = kuduInstance.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); session.setFlushInterval(500); session.setMutationBufferSpace(5000); Insert insert = table.newInsert(); // 设置字段内容 insert.getRow().addString("order_id", ghsOrder.getOrderId()); insert.getRow().addString("submitted_date", ghsOrder.getSubmittedDate()); insert.getRow().addString("shipping_groupid", ghsOrder.getShippingGroupId()); insert.getRow().addString("shop_code",ghsOrder.getShopCode()); insert.getRow().addString("shop_name",ghsOrder.getShopName()); insert.getRow().addString("order_state",ghsOrder.getOrderState()); insert.getRow().addString("shipping_state",ghsOrder.getShippingState()); insert.getRow().addString("user_id",ghsOrder.getUserId()); insert.getRow().addDecimal("amount",ghsOrder.getAmount()); insert.getRow().addDecimal("ship_amount",ghsOrder.getShipAmount()); insert.getRow().addString("sales_channel",ghsOrder.getSalesChannel()); insert.getRow().addString("create_time_day", DateUtils.dateToString(DateUtils.stringToDate(ghsOrder.getSubmittedDate(), DateStyleEnum.YYYYMMDDHHMMSS), DateStyleEnum.YYYYMMDD)); insert.getRow().addString("create_time_hour",DateUtils.dateToString(DateUtils.stringToDate(ghsOrder.getSubmittedDate(),DateStyleEnum.YYYYMMDDHHMMSS), "yyyyMMddHH")); session.apply(insert); session.close(); } catch (Exception e) { logger.error("GhsOrderReceiveServiceImpl.insertOrderKudu is error ",e); } } private void updateOrderKudu(GhsOrder ghsOrder) { logger.info("GhsOrderReceiveServiceImpl.updateOrderKudu is begin json={}",JacksonUtils.convertObjectToJson(ghsOrder)); try { //打开表 KuduTable table = kuduInstance.openTable("impala::t_homebiz.order_source_data"); KuduSession session = kuduInstance.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC); //更新数据 Update update = table.newUpdate(); PartialRow row1 = update.getRow(); row1.addString("order_id",ghsOrder.getOrderId()); row1.addString("shipping_groupid", ghsOrder.getShippingGroupId()); if(StringUtils.isNotEmpty(ghsOrder.getShopCode())) { row1.addString("order_state", ghsOrder.getOrderState()); } if(StringUtils.isNotEmpty(ghsOrder.getShopCode())) { row1.addString("shipping_state", ghsOrder.getShippingState()); } if(StringUtils.isNotEmpty(ghsOrder.getShopCode())) { row1.addDecimal("amount", ghsOrder.getAmount()); } if(StringUtils.isNotEmpty(ghsOrder.getShopCode())) { row1.addDecimal("ship_amount", ghsOrder.getShipAmount()); } session.apply(update); session.close(); } catch (Exception e) { logger.error("GhsOrderReceiveServiceImpl.updateOrderKudu is error",e); } } private int queryOrderByOrderId(String orderId,String shippingGroupId) { logger.info("GhsOrderReceiveServiceImpl.queryOrderByOrderId is begin orderId={}",orderId); int count = 0; try { //打开表 KuduTable table = kuduInstance.openTable("impala::t_homebiz.order_source_data"); KuduScanner.KuduScannerBuilder builder = kuduInstance.newScannerBuilder(table); /** * 设置搜索的条件 * 如果不设置,则全表扫描 */ KuduPredicate orderIdPredicate = KuduPredicate.newComparisonPredicate(table.getSchema().getColumn("order_id"), KuduPredicate.ComparisonOp.EQUAL, orderId); KuduPredicate shippingGroupIdPredicate = KuduPredicate.newComparisonPredicate(table.getSchema().getColumn("shipping_groupid"), KuduPredicate.ComparisonOp.EQUAL, shippingGroupId); builder.addPredicate(orderIdPredicate); builder.addPredicate(shippingGroupIdPredicate); // 开始扫描 KuduScanner scaner = builder.build(); while (scaner.hasMoreRows()) { RowResultIterator iterator = scaner.nextRows(); while (iterator.hasNext()) { count = 1; return count; } } scaner.close(); } catch (Exception e) { logger.error("GhsOrderReceiveServiceImpl.queryOrderByOrderId is error",e); } return count; } }
大数据Kudu使用方法
最新推荐文章于 2024-06-26 09:45:10 发布