定时数据集成--设计开发

 任务启动入口：

如果是定时调度任务，则交给 quartzManager

否则，通过TaskUtil 进行启动

@RequestMapping(value = "start", method = RequestMethod.POST)
    public @ResponseBody
    JSONResult<Object> start(@RequestParam(value = "param", required = true) String param) {
        JSONResult<Object> jsonResult = new JSONResult<>(true);
        try {
            //如果是定时任务，则加到定时调度框架中，不是则直接start
            TaskEntity.Task task = TaskDeserializer.deserializeTask(param, TaskUtil.NONE);
            logger.info("client try to start task:{}", task.getTaskId());

            if(SysConst.PERIOD_TASK_YES.equals(task.getIsPeriodic())) {
                //定时任务加到QuartzManager中
                if(quartzManager.addTask(task.getTaskId(), task.getCronExp(), param)){
                    String msg = Utils.join("",
                            "Dispatch [taskId::", task.getTaskId(),
                            "] with operation [action::", TaskEntity.Action.START, "] success.");
                    jsonResult.invokeSuccess(task.getTaskId(), msg);
                }else{
                    jsonResult.invokeFailed(Utils.join("","Dispatch [taskId::", task.getTaskId(), "] failed. Due to error: ", "定时任务失败"));
                }
            }else{//直接start
                TaskUtil.operate(TaskEntity.Action.START, param, jsonResult);
            }
        } catch (Exception e) {
//            logger.error("{},{}",e.getMessage(), e);
            jsonResult.invokeFailed(e.getMessage());
        }
        return jsonResult;
    }

TaskUtil中定义了保存异常信息的map

    /**
     * 任务异常信息，内部错误时放到此缓存中，后台线程处理
     */
    public static volatile ConcurrentHashMap<String, MyInternalException> TASK_EXCEPTION_CACHE = new ConcurrentHashMap<>();

TaskUtil.operator()方法调用TaskDispatcher的方法进行调度

TaskDispatcher.getInstance().dispatch(task.setAction(action));

public void dispatch(TaskEntity.Task task) throws TaskException {
		String taskId = task.getTaskId();
		try {
			if (task.isStartup()) {
				// 任务没有在运行
				if (!runningFlows.containsKey(taskId)) {
					startup(task);
				} else {
					logger.warn("task is already running " + taskId);
					throw new TaskException("task is already running " + taskId);
				}
			} else {
				// 先把cache中状态改变，让sink不再访问channel
				if (SysConst.TASK_CACHE.containsKey(taskId)) {
					SysConst.TASK_CACHE.get(taskId).setStatus(TaskStatusEnum.STOP.getValue());
				}
				shutdown(task);
				task.setStatus(TaskStatusEnum.STOP.getValue());
				SysConst.TASK_CACHE.remove(taskId);
			}
		}

其中runningFlows是当前的数据采集流向，source、channel、sink都是flume的基本组件，定义了数据采集的各个环节

private final Map<String, List<Flow>> runningFlows;

public class Flow {

    private static final Logger logger = LoggerFactory.getLogger(Flow.class);

    private String identity;

	private Source source;
	private List<Interceptor> interceptors;
	private Channel channel;
	private Sink sink;

    private EmbeddedAgent agent;
	private boolean disabled;

	private ReentrantLock operationLock = new ReentrantLock();
}

dispatch方法会调用startup方法，定义flow并添加到submitQueue中

private void startup(TaskEntity.Task task) throws Exception {
Flow.Interceptor[] interceptors = buildInterceptors(sourceDef, destDef, taskId);
			Flow.Sink sink = buildSink(destDef, finalTaskId);

			Flow flow = new Flow(finalTaskId).from(source).intercept(interceptors).channel(channel).to(sink);

			flows.add(flow);

			submitQueue.add(flow);
}

submitQueue中数据的处理是通过TaskDispatcher的构造方法来启动工作线程运行，主要是启动Flow类的start方法

private TaskDispatcher() {
submitPool = Executors.newSingleThreadScheduledExecutor(Utils.buildThreadFactory(TASK_SUBMIT_POOL_NAME));
		submitPool.scheduleAtFixedRate(() -> {
			logger.debug("Scanning tasks...");

			// Poll a flow out, and try start up
			Optional<Flow> flow = Optional.fromNullable(submitQueue.poll());
			if (flow.isPresent()) {
				try {
					logger.info("flow try to start:{}", flow.get().getIdentity());
					flow.get().start();
					// sleep 避免同时请求
					try {
						Thread.sleep(2000);
					} catch (Exception e) {
					}
				} catch (Exception e) {
					logger.error("Failed to running a task. Error: {},{}.", e.getMessage(), e);
					// If failed, put the flow back to the tail of the queue
					submitQueue.offer(flow.get());
				}
			}

			logger.debug("Scan finished.");
		}, NO_DELAY, SCHEDULE_PERIOD, TimeUnit.SECONDS);
	}

Flow类的start方法

    public void start() throws Exception {
	    logger.info("{} try to get start lock .", Thread.currentThread().getName());
        long startTime = System.currentTimeMillis();
        if (isDisabled()) {
            return;
        }
        operationLock.lock();
        agent.start();
    }

而agent的start方法如下，调用了channel、sinkRunner等start方法，这些都是flume的内部方法

  private void start() {
    boolean error = true;
    try {
      channel.start();
      sinkRunner.start();
      sourceRunner.start();

      supervisor.supervise(channel,
          new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START);
      supervisor.supervise(sinkRunner,
          new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START);
      supervisor.supervise(sourceRunner,
          new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START);
      error = false;
   }

第二部分，定时调度的实现

quartzManager的addTask方法，其中的任务Job其实是QuartzTask类的实例

public boolean addTask(String taskId, String cronExp, String param) {
JobDetail jobDetail = JobBuilder.newJob().withIdentity(new JobKey(taskId, JOB_DEFAULT_GROUP_NAME))
					.ofType((Class<Job>) Class.forName("com.iflytek.dp.service.quartz.QuartzTask")).build();
			jobDetail.getJobDataMap().put("param", param);
			jobDetail.getJobDataMap().put("cron", cronExp);
			Trigger trigger = TriggerBuilder.newTrigger().forJob(jobDetail)
					.withSchedule(CronScheduleBuilder.cronSchedule(cronExp))
					.withIdentity(new TriggerKey(taskId, TRIGGER_DEFAULT_GROUP_NAME)).build();
			scheduler.scheduleJob(jobDetail, trigger);
			result = true;
			PeriodicRunningCache.buildCacheModel(taskId, cronExp);
}

QuartzTask类主要定义了execute方法，作为工作方法。可以看到又调用到了TaskUtil的operate方法，后面的程序逻辑和非定时调度任务一致

@Override
public void execute(JobExecutionContext context) {
    TaskUtil.operate(TaskEntity.Action.START, param, jsonResult);
}