一、阿里云消息队列 MQ
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
@Configuration
public class TaskConfiguration {
@Bean
public TaskExecutor getTaskExecutor() {
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setMaxPoolSize(10);
taskExecutor.setCorePoolSize(5);
taskExecutor.setQueueCapacity(20);
return taskExecutor;
}
}
import com.aliyun.mq.http.MQClient;
import com.aliyun.mq.http.MQConsumer;
import com.aliyun.mq.http.common.AckMessageException;
import com.aliyun.mq.http.model.Message;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.task.TaskExecutor;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@Service
@Slf4j
public class MessageUtil {
@Value(value = "${aliyun.endpoint}")
private String httpEndpoint;
@Value(value = "${aliyun.accesskey}")
private String accessKey;
@Value(value = "${aliyun.secretkey}")
private String secretKey;
@Value(value = "${aliyun.topic}")
private String topic;
@Value(value = "${aliyun.groupid}")
private String groupId;
@Value(value = "${aliyun.instanceid}")
private String instanceId;
@Resource
TaskExecutor taskExecutor;
public void receiveMsg() {
MQClient mqClient = new MQClient(
// 设置HTTP接入域名（此处以公共云生产环境为例）
httpEndpoint,
// AccessKey 阿里云身份验证，在阿里云服务器管理控制台创建
accessKey,
// SecretKey 阿里云身份验证，在阿里云服务器管理控制台创建
secretKey);
// 所属的 Topic
final String topic = this.topic;
// 您在控制台创建的 Group ID
final String groupId = this.groupId;
// Topic所属实例ID，默认实例为空
final String instanceId = this.instanceId;
final MQConsumer consumer;
if (Objects.nonNull(instanceId) && !Objects.equals(instanceId, "")) {
consumer = mqClient.getConsumer(instanceId, topic, groupId, null);
} else {
consumer = mqClient.getConsumer(topic, groupId);
}
// 在当前线程循环消费消息，建议是多开个几个线程并发消费消息
do {
List<Message> messages = null;
try {
// 长轮询消费消息
// 长轮询表示如果topic没有消息则请求会在服务端挂住3s，3s内如果有消息可以消费则立即返回
messages = consumer.consumeMessage(
16,
30
);
} catch (Throwable e) {
log.error("接收消息发生异常:{}", e);
try {
Thread.sleep(2000);
} catch (Exception ex) {
log.error("接收消息发生异常:{}", ex);
}
}
// 没有消息
if (messages == null || messages.isEmpty()) {
log.info(Thread.currentThread().getName() + ": no new message, continue!");
continue;
}
// 处理业务逻辑
messages.stream().forEach(s -> {
try {
log.info("messageId:{},messageBody:{}", s.getMessageId(), s.getMessageBodyString());
taskExecutor.execute(() -> {
//TODO 执行业务
});
} catch (Exception e) {
log.error("消费消息发生异常:{}", e);
}
});
messageCallBack(consumer, messages);
} while (true);
}
private void messageCallBack(MQConsumer consumer, List<Message> messages) {
// Message.nextConsumeTime前若不确认消息消费成功，则消息会重复消费
// 消息句柄有时间戳，同一条消息每次消费拿到的都不一样
List<String> handles = new ArrayList<>();
messages.stream().forEach(e -> handles.add(e.getReceiptHandle()));
try {
consumer.ackMessage(handles);
} catch (Throwable e) {
// 某些消息的句柄可能超时了会导致确认不成功
if (e instanceof AckMessageException) {
AckMessageException errors = (AckMessageException) e;
log.info("Ack message fail, requestId is:" + errors.getRequestId() + ", fail handles:");
if (errors.getErrorMessages() != null) {
errors.getErrorMessages().keySet().stream().forEach(s -> log.info("Handle:" + s + ", ErrorCode:" + errors.getErrorMessages().get(s).getErrorCode()
+ ", ErrorMsg:" + errors.getErrorMessages().get(s).getErrorMessage()));
}
return;
}
log.error("接收消息发生异常:{}", e);
}
}
}
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
@Service
@Slf4j
public class MessageClient {
static MessageUtil messageUtil;
@Resource
public void setMessageUtil(MessageUtil messageUtil) {
MessageClient.messageUtil = messageUtil;
}
public static void receiveMsg() {
messageUtil.receiveMsg();
}
}
二、数据同步 (canal rabbitmq 消息解析成 sql) 与 ( Maxwell rabbitmq 消息解析成 sql)
canal rabbitmq 消息解析成 sql
import com.alibaba.fastjson.JSON;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.util.*;
/**
* canal rabbitmq 消息解析成dml
* {
* "data":[
* {
* "id":"5",
* "name":"666"
* }
* ],
* "database":"test",
* "es":1609404510000,
* "id":4,
* "isDdl":false,
* "mysqlType":{
* "id":"int(11)",
* "name":"varchar(255)"
* },
* "old":null,
* "pkNames":[
* "id"
* ],
* "sql":"",
* "sqlType":{
* "id":4,
* "name":12
* },
* "table":"t_user",
* "ts":1609404510314,
* "type":"INSERT"
* }
*/
@Slf4j
@AllArgsConstructor
public class CanalData extends LinkedHashMap<String, Object> {
private static Gson gson = new GsonBuilder().create();
private CanalData(Map<String, Object> data) {
super();
super.putAll(data);
}
public static CanalData fromJsonString(String json) {
return new CanalData(gson.fromJson(json, new TypeToken<Map<String, Object>>() {}.getType()));
}
public String getTable() {
return super.containsKey("table") ? (String) super.get("table") : StringUtils.EMPTY;
}
public String getType() {
return super.containsKey("type") ? (String) super.get("type") : StringUtils.EMPTY;
}
public String getDmlSql() {
String type = this.getType();
switch (type) {
case "INSERT":
return getInsertSql();
case "UPDATE":
return getUpdateSql();
default:
log.warn("不支持该DML操作type:", type);
}
return "";
}
public String getInsertSql() {
return "insert into " + getTable() + insColAndVal();
}
public String getUpdateSql() {
return "update " + getTable() + " set " + updColAndVal();
}
public Map<String, String> getData() {
String json = gson.toJson(super.get("data"));
List list = gson.fromJson(json, List.class);
return gson.fromJson(JSON.toJSONString(list.get(0)), new TypeToken<Map<String, String>>() {}.getType());
}
public String updColAndVal() {
StringBuilder updColAndVal = new StringBuilder();
Map<String, String> dataMap = this.getData();
for (String key : dataMap.keySet()) {
if (Objects.equals(key, getPkNames())) {
continue;
}
updColAndVal.append(key).append("='").append(dataMap.get(key)).append("',");
}
updColAndVal = new StringBuilder(StringUtils.substringBeforeLast(updColAndVal.toString(), ","));
updColAndVal.append(" where ").append(getPkNames()).append("='").append(dataMap.get(getPkNames())).append("'");
return updColAndVal.toString();
}
public String insColAndVal() {
Map<String, String> sqlType = this.getData();
Set<String> set = sqlType.keySet();
String columns = StringUtils.join(set.iterator(), ",");
String values = StringUtils.join(sqlType.values(), "','");
return " (" + columns + ") values ('" + values + "')";
}
public String getPkNames() {
String json = gson.toJson(super.get("pkNames"));
List list = gson.fromJson(json, List.class);
return String.valueOf(list.get(0));
}
public String getMysqlType() {
return super.containsKey("mysqlType") ? (String) super.get("mysqlType") : StringUtils.EMPTY;
}
}
Maxwell rabbitmq 消息解析成 sql
import com.alibaba.fastjson.JSON;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
/**
* Maxwell rabbitmq 消息解析成 sql
* {
* "database": "test",
* "table": "maxwell",
* "type": "update",
* "ts": 1551862800,
* "xid": 450,
* "commit": true,
* "data": {
* "id": 1,
* "daemon": "firebus! firebus!"
* },
* "old": {
* "daemon": "Stanislaw Lem"
* }
* }
*
* {
* "database": "test",
* "table": "maxwell",
* "type": "insert",
* "ts": 1449786310,
* "xid": 940752,
* "commit": true,
* "data": { "id":1, "daemon": "Stanislaw Lem" }
* }
*
*/
@Data
@Slf4j
public class MaxwellData {
private static Gson gson = new GsonBuilder().create();
private String database;
private String table;
private String type;
private int ts;
private int xid;
private boolean commit;
private Map<String, String> data;
private Map<String, String> old;
/**
* 获取对象数据
* @param msg
* @return
*/
public static MaxwellData fromJsonString(String msg) {
return JSON.parseObject(msg, MaxwellData.class);
}
public String getDmlSql() {
String type = this.getType();
switch (type) {
case "insert":
return getInsertSql();
case "update":
return getUpdateSql();
default:
log.warn("不支持该DML操作type:", type);
}
return "";
}
public String getInsertSql() {
return "insert into " + getTable() + insColAndVal();
}
public String getUpdateSql() {
return "update " + getTable() + " set " + updColAndVal();
}
public String updColAndVal() {
StringBuilder updColAndVal = new StringBuilder();
Map<String, String> dataMap = this.getData();
for (String key : dataMap.keySet()) {
if (Objects.isNull(key) && Objects.isNull(dataMap.get(key))) {
continue;
}
updColAndVal.append(key).append("='").append(dataMap.get(key)).append("',");
}
updColAndVal = new StringBuilder(StringUtils.substringBeforeLast(updColAndVal.toString(), ","));
updColAndVal.append(" where ");
Map<String, String> oldMap = this.getOld();
for (String key : oldMap.keySet()) {
if (Objects.isNull(key) && Objects.isNull(dataMap.get(key))) {
continue;
}
updColAndVal.append(key).append("='").append(dataMap.get(key)).append("'");
}
return updColAndVal.toString();
}
public String insColAndVal() {
Map<String, String> sqlType = this.getData();
Set<String> set = sqlType.keySet();
String columns = StringUtils.join(set.iterator(), ",");
String values = StringUtils.join(sqlType.values(), "','");
return " (" + columns + ") values ('" + values + "')";
}
}
三、反射工具类. 提供调用getter/setter方法, 访问私有变量, 调用私有方法, 获取泛型类型Class, 被AOP过的真实类等工具函数.
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.poi.ss.usermodel.DateUtil;
import java.lang.reflect.*;
import java.util.Date;
/**
* 反射工具类. 提供调用getter/setter方法, 访问私有变量, 调用私有方法, 获取泛型类型Class, 被AOP过的真实类等工具函数.
*
*/
@SuppressWarnings("rawtypes")
@Slf4j
public class ReflectUtils
{
private static final String SETTER_PREFIX = "set";
private static final String GETTER_PREFIX = "get";
private static final String CGLIB_CLASS_SEPARATOR = "$$";
/**
* 调用Getter方法.
* 支持多级，如：对象名.对象名.方法
*/
@SuppressWarnings("unchecked")
public static <E> E invokeGetter(Object obj, String propertyName)
{
Object object = obj;
for (String name : StringUtils.split(propertyName, "."))
{
String getterMethodName = GETTER_PREFIX + StringUtils.capitalize(name);
object = invokeMethod(object, getterMethodName, new Class[] {}, new Object[] {});
}
return (E) object;
}
/**
* 调用Setter方法, 仅匹配方法名。
* 支持多级，如：对象名.对象名.方法
*/
public static <E> void invokeSetter(Object obj, String propertyName, E value)
{
Object object = obj;
String[] names = StringUtils.split(propertyName, ".");
for (int i = 0; i < names.length; i++)
{
if (i < names.length - 1)
{
String getterMethodName = GETTER_PREFIX + StringUtils.capitalize(names[i]);
object = invokeMethod(object, getterMethodName, new Class[] {}, new Object[] {});
}
else
{
String setterMethodName = SETTER_PREFIX + StringUtils.capitalize(names[i]);
invokeMethodByName(object, setterMethodName, new Object[] { value });
}
}
}
/**
* 直接读取对象属性值, 无视private/protected修饰符, 不经过getter函数.
*/
@SuppressWarnings("unchecked")
public static <E> E getFieldValue(final Object obj, final String fieldName)
{
Field field = getAccessibleField(obj, fieldName);
if (field == null)
{
log.debug("在 [" + obj.getClass() + "] 中，没有找到 [" + fieldName + "] 字段 ");
return null;
}
E result = null;
try
{
result = (E) field.get(obj);
}
catch (IllegalAccessException e)
{
log.error("不可能抛出的异常{}", e.getMessage());
}
return result;
}
/**
* 直接设置对象属性值, 无视private/protected修饰符, 不经过setter函数.
*/
public static <E> void setFieldValue(final Object obj, final String fieldName, final E value)
{
Field field = getAccessibleField(obj, fieldName);
if (field == null)
{
// throw new IllegalArgumentException("在 [" + obj.getClass() + "] 中，没有找到 [" + fieldName + "] 字段 ");
log.debug("在 [" + obj.getClass() + "] 中，没有找到 [" + fieldName + "] 字段 ");
return;
}
try
{
field.set(obj, value);
}
catch (IllegalAccessException e)
{
log.error("不可能抛出的异常: {}", e.getMessage());
}
}
/**
* 直接调用对象方法, 无视private/protected修饰符.
* 用于一次性调用的情况，否则应使用getAccessibleMethod()函数获得Method后反复调用.
* 同时匹配方法名+参数类型，
*/
@SuppressWarnings("unchecked")
public static <E> E invokeMethod(final Object obj, final String methodName, final Class<?>[] parameterTypes,
final Object[] args)
{
if (obj == null || methodName == null)
{
return null;
}
Method method = getAccessibleMethod(obj, methodName, parameterTypes);
if (method == null)
{
log.debug("在 [" + obj.getClass() + "] 中，没有找到 [" + methodName + "] 方法 ");
return null;
}
try
{
return (E) method.invoke(obj, args);
}
catch (Exception e)
{
String msg = "method: " + method + ", obj: " + obj + ", args: " + args + "";
throw convertReflectionExceptionToUnchecked(msg, e);
}
}
/**
* 直接调用对象方法, 无视private/protected修饰符，
* 用于一次性调用的情况，否则应使用getAccessibleMethodByName()函数获得Method后反复调用.
* 只匹配函数名，如果有多个同名函数调用第一个。
*/
@SuppressWarnings("unchecked")
public static <E> E invokeMethodByName(final Object obj, final String methodName, final Object[] args)
{
Method method = getAccessibleMethodByName(obj, methodName, args.length);
if (method == null)
{
// 如果为空不报错，直接返回空。
log.debug("在 [" + obj.getClass() + "] 中，没有找到 [" + methodName + "] 方法 ");
return null;
}
try
{
// 类型转换（将参数数据类型转换为目标方法参数类型）
Class<?>[] cs = method.getParameterTypes();
for (int i = 0; i < cs.length; i++)
{
if (args[i] != null && !args[i].getClass().equals(cs[i]))
{
if (cs[i] == String.class)
{
args[i] = Convert.toStr(args[i]);
if (StringUtils.endsWith((String) args[i], ".0"))
{
args[i] = StringUtils.substringBefore((String) args[i], ".0");
}
}
else if (cs[i] == Integer.class)
{
args[i] = Convert.toInt(args[i]);
}
else if (cs[i] == Long.class)
{
args[i] = Convert.toLong(args[i]);
}
else if (cs[i] == Double.class)
{
args[i] = Convert.toDouble(args[i]);
}
else if (cs[i] == Float.class)
{
args[i] = Convert.toFloat(args[i]);
}
else if (cs[i] == Date.class)
{
if (args[i] instanceof String)
{
args[i] = DateUtils.parseDate(String.valueOf(args[i]));
}
else
{
args[i] = DateUtil.getJavaDate((Double) args[i]);
}
}
else if (cs[i] == boolean.class || cs[i] == Boolean.class)
{
args[i] = Convert.toBool(args[i]);
}
}
}
return (E) method.invoke(obj, args);
}
catch (Exception e)
{
String msg = "method: " + method + ", obj: " + obj + ", args: " + args + "";
throw convertReflectionExceptionToUnchecked(msg, e);
}
}
/**
* 循环向上转型, 获取对象的DeclaredField, 并强制设置为可访问.
* 如向上转型到Object仍无法找到, 返回null.
*/
public static Field getAccessibleField(final Object obj, final String fieldName)
{
// 为空不报错。直接返回 null
if (obj == null)
{
return null;
}
Validate.notBlank(fieldName, "fieldName can't be blank");
for (Class<?> superClass = obj.getClass(); superClass != Object.class; superClass = superClass.getSuperclass())
{
try
{
Field field = superClass.getDeclaredField(fieldName);
makeAccessible(field);
return field;
}
catch (NoSuchFieldException e)
{
continue;
}
}
return null;
}
/**
* 循环向上转型, 获取对象的DeclaredMethod,并强制设置为可访问.
* 如向上转型到Object仍无法找到, 返回null.
* 匹配函数名+参数类型。
* 用于方法需要被多次调用的情况. 先使用本函数先取得Method,然后调用Method.invoke(Object obj, Object... args)
*/
public static Method getAccessibleMethod(final Object obj, final String methodName,
final Class<?>... parameterTypes)
{
// 为空不报错。直接返回 null
if (obj == null)
{
return null;
}
Validate.notBlank(methodName, "methodName can't be blank");
for (Class<?> searchType = obj.getClass(); searchType != Object.class; searchType = searchType.getSuperclass())
{
try
{
Method method = searchType.getDeclaredMethod(methodName, parameterTypes);
makeAccessible(method);
return method;
}
catch (NoSuchMethodException e)
{
continue;
}
}
return null;
}
/**
* 循环向上转型, 获取对象的DeclaredMethod,并强制设置为可访问.
* 如向上转型到Object仍无法找到, 返回null.
* 只匹配函数名。
* 用于方法需要被多次调用的情况. 先使用本函数先取得Method,然后调用Method.invoke(Object obj, Object... args)
*/
public static Method getAccessibleMethodByName(final Object obj, final String methodName, int argsNum)
{
// 为空不报错。直接返回 null
if (obj == null)
{
return null;
}
Validate.notBlank(methodName, "methodName can't be blank");
for (Class<?> searchType = obj.getClass(); searchType != Object.class; searchType = searchType.getSuperclass())
{
Method[] methods = searchType.getDeclaredMethods();
for (Method method : methods)
{
if (method.getName().equals(methodName) && method.getParameterTypes().length == argsNum)
{
makeAccessible(method);
return method;
}
}
}
return null;
}
/**
* 改变private/protected的方法为public，尽量不调用实际改动的语句，避免JDK的SecurityManager抱怨。
*/
public static void makeAccessible(Method method)
{
if ((!Modifier.isPublic(method.getModifiers()) || !Modifier.isPublic(method.getDeclaringClass().getModifiers()))
&& !method.isAccessible())
{
method.setAccessible(true);
}
}
/**
* 改变private/protected的成员变量为public，尽量不调用实际改动的语句，避免JDK的SecurityManager抱怨。
*/
public static void makeAccessible(Field field)
{
if ((!Modifier.isPublic(field.getModifiers()) || !Modifier.isPublic(field.getDeclaringClass().getModifiers())
|| Modifier.isFinal(field.getModifiers())) && !field.isAccessible())
{
field.setAccessible(true);
}
}
/**
* 通过反射, 获得Class定义中声明的泛型参数的类型, 注意泛型必须定义在父类处
* 如无法找到, 返回Object.class.
*/
@SuppressWarnings("unchecked")
public static <T> Class<T> getClassGenricType(final Class clazz)
{
return getClassGenricType(clazz, 0);
}
/**
* 通过反射, 获得Class定义中声明的父类的泛型参数的类型.
* 如无法找到, 返回Object.class.
*/
public static Class getClassGenricType(final Class clazz, final int index)
{
Type genType = clazz.getGenericSuperclass();
if (!(genType instanceof ParameterizedType))
{
log.debug(clazz.getSimpleName() + "'s superclass not ParameterizedType");
return Object.class;
}
Type[] params = ((ParameterizedType) genType).getActualTypeArguments();
if (index >= params.length || index < 0)
{
log.debug("Index: " + index + ", Size of " + clazz.getSimpleName() + "'s Parameterized Type: "
+ params.length);
return Object.class;
}
if (!(params[index] instanceof Class))
{
log.debug(clazz.getSimpleName() + " not set the actual class on superclass generic parameter");
return Object.class;
}
return (Class) params[index];
}
public static Class<?> getUserClass(Object instance)
{
if (instance == null)
{
throw new RuntimeException("Instance must not be null");
}
Class clazz = instance.getClass();
if (clazz != null && clazz.getName().contains(CGLIB_CLASS_SEPARATOR))
{
Class<?> superClass = clazz.getSuperclass();
if (superClass != null && !Object.class.equals(superClass))
{
return superClass;
}
}
return clazz;
}
/**
* 将反射时的checked exception转换为unchecked exception.
*/
public static RuntimeException convertReflectionExceptionToUnchecked(String msg, Exception e)
{
if (e instanceof IllegalAccessException || e instanceof IllegalArgumentException
|| e instanceof NoSuchMethodException)
{
return new IllegalArgumentException(msg, e);
}
else if (e instanceof InvocationTargetException)
{
return new RuntimeException(msg, ((InvocationTargetException) e).getTargetException());
}
return new RuntimeException(msg, e);
}
}
扫一扫
784
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
