java分布式消费策略

1：基于redis的quartz 分布式消费

2：通用实体
package com.qbao.baoy.utils;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;

import java.lang.reflect.Method;
import java.util.Date;
import java.util.List;

/**
 * mao.wang 2016/12/1
 */
public class BaseDistributedJob implements InitializingBean{

    /**
     * 日志记录对象
     */
    protected static final Logger LOG = LoggerFactory.getLogger(BaseDistributedJob.class);

    @Autowired
    private RedisUtil redisUtil;

    /**
     * redis标识key
     */
    private String jobName;
    /**
     * redis标识key
     */
    private String redisKey;
    /**
     * redis队列Key
     */
    private String redisListKey;

    /**
     * 锁失效时间
     */
    private int expire = 0;

    /**
     * redis消费主键
     */
    private List<Object> redisKeyList;

    /**
     * 实例对象
     */
    private Object classInvoke;

    @Override
    public void afterPropertiesSet()  {
        try {
            classInvoke=SpringContextHolder.getBean(jobName);
        }catch (Exception e) {
            LOG.error("BaseDistributedJob-afterPropertiesSet():JOB初始化异常:{}",jobName,e);
        }
    }

    public void execute() {
        Long startTime = System.currentTimeMillis();
        //创建队列
        if (getLock()) {
            try {
                Method m1 =classInvoke.getClass().getDeclaredMethod("getConsumeList");
                Object result=m1.invoke(classInvoke);
                List<Object> getSendFinishList = (List<Object>) result;
                queueCreate(getSendFinishList);
            } catch (NoSuchMethodException e) {
                LOG.error("BaseDistributedJob-execute():NoSuchMethodException:{}:{}", jobName, e);
            }catch (Exception e){
                LOG.error("BaseDistributedJob-execute():调用异常:{}:{}", jobName,e);
            }
        }
        int i = 0;
        //消费
        while (hasNextForLock()) {
            List<Object> ObjectList = popListElement();
            if (null != ObjectList) {
                for (Object objectId : ObjectList) {
                    if (null != objectId) {
                        try {
                            Class[] cArg = new Class[1];
                            cArg[0] = Object.class;
                            Method m1 =classInvoke.getClass().getDeclaredMethod("needConsume",cArg);
                            Object result=m1.invoke(classInvoke,objectId);
                            boolean flag = (boolean)result;
                            LOG.info("BaseDistributedJob-execute():定时任务消费数据:{}:{}", objectId, flag);
                            if (flag) {
                                i++;
                            }
                        } catch (Exception e) {
                            LOG.info("BaseDistributedJob-execute():定时任务消费数据异常:{}:{}", objectId, e);
                            LOG.error("BaseDistributedJob-execute():定时任务消费数据异常:{}:{}", objectId, e);
                        }
                    }
                }
            }
        }
        Long endTime = System.currentTimeMillis();
        LOG.info("BaseDistributedJob-execute():job定时任务消费数据:{}耗时{}:共处理数据{}", DateUtil.formatDate(new Date(), DateUtil.YYYY_MM_DD_HH_MI_SS), DateUtil.getTime((endTime - startTime) / 1000), i);
    }

    /**
     * 定义标识
     */
    private final String CONSTANTS_1 = "1";

    public boolean getLock() {
        Long isTrue = redisUtil.sadd(redisKey, "0");
        boolean flag = CONSTANTS_1.equals(isTrue.toString());
        //默认为当天结束
        if (flag) {
            if (0 == expire) {
                redisUtil.expire(redisKey, DateUtil.getRedisExpired().intValue());
            } else {
                redisUtil.expire(redisKey, expire);
            }
        }
        LOG.info("BaseDistributedJob-getLock():正在写入数据:{}:{}", isTrue, jobName);
        //单节点向redis队列存放数据
        LOG.info("BaseDistributedJob-getLock():实例redis队列:{}:{}:{}", jobName, IpUtil.getHostAddress(), isTrue);
        return flag;
    }

    /**
     * 定时任务将预处理数据缓存到redis
     */
    public void queueCreate(List<Object> ids) {
        setRedisKeyList(ids);
        if (null != redisKeyList) {
            int redisKeyListSize = redisUtil.lPipelinePush(redisListKey, redisKeyList);
            LOG.info("BaseDistributedJob-queueCreate()队列写入完成数据：{}：{}：{}：{}", jobName, redisKeyList, redisKeyList.size(), redisKeyListSize);
        }
        //标记数据处理完成
        redisUtil.sadd(redisKey, "1");
    }

    /**
     * 获取是否有下一个元素
     */
    public boolean hasNextForLock() {
        do {
            try {
                //lock 如果长度大于1则标识为队列以及实例化，进行消费
                long sLength = redisUtil.scard(redisKey);
                if (1 < sLength) {
                    //处理数据
                    Long redisListSize = redisUtil.getlen(redisListKey);
                    return 0 < redisListSize;
                } else {
                    //标识为定时任务没有完成，等待队列实例化
                    LOG.info("BaseDistributedJob-hasNextForLock():等待redis队列实例化完成{}:{}", jobName, IpUtil.getHostAddress());
                    RedisUtil.exceptionSleepThread();
                }
            } catch (Exception e) {
                LOG.error("BaseDistributedJob-hasNextForLock():等待队列异常暂停5S:{}", jobName);
                RedisUtil.exceptionSleepThread();
                return true;
            }
        } while (true);
    }

    /**
     * 获取是否有下一个元素
     */
    public boolean hasNext() {
        //处理数据
        Long redisListSize = redisUtil.getlen(redisListKey);
        return 0 < redisListSize;
    }

    public List<Object> popListElement() {
        return redisUtil.lPipelinePop(redisListKey, 10);
    }

    public String getRedisListKey() {
        return redisListKey;
    }

    public void setRedisListKey(String redisListKey) {
        this.redisListKey = redisListKey;
    }

    public List<Object> getRedisKeyList() {
        return redisKeyList;
    }

    public void setRedisKeyList(List<Object> redisKeyList) {
        this.redisKeyList = redisKeyList;
    }

    public String getRedisKey() {
        return redisKey;
    }

    public void setRedisKey(String redisKey) {
        this.redisKey = redisKey;
    }

    public String getJobName() {
        return jobName;
    }

    public void setJobName(String jobName) {
        this.jobName = jobName;
    }

    public int getExpire() {
        return expire;
    }

    public void setExpire(int expire) {
        this.expire = expire;
    }
}
3:ActivityTeamService 业务代码
    List<Object> getConsumeList();
    boolean needConsume(Object obj);
4:spring job xml配置

<bean id="activityTeamJobTrigger" class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
        <property name="jobDetail" ref="activityTeamJobDetail"/>
        <property name="cronExpression" value="0 0 */2 * * ?"/>
    </bean>
    <bean id="activityTeamJobDetail" class="org.springframework.scheduling.quartz.MethodInvokingJobDetailFactoryBean">
        <property name="targetObject" ref="activityTeamJob"/>
        <property name="targetMethod" value="execute"/>
        <property name="concurrent" value="false"/>
    </bean>
    <bean id="activityTeamJob" class="com.qbao.baoy.jobs.BaseDistributedJob">
        <property name="jobName" value="ActivityTeamService"/>
        <property name="redisKey" value="baoy:activity:team:needPromotion:lock"/>
        <property name="expire" value="3600"/>
        <property name="redisListKey" value="baoy:activity:team:needPromotion:teamList"/>
    </bean>

基于redis的 spring quartz分布式队列处理


使用场景
  1:定时任务批处理
  2:异步 订单等处理