触发式azkaban

触发式azkaban

流程图

添加任务

• 添加任务
• ZSYMethodsInfoServiceImpl.java

  // 添加任务
        methodStatJobMapper.insert(methodStatJob);

        // 如果当前没有任务，该任务状态直接变为prepare,直接执行
        if (methodStatJob.getJobStatus() == ZSYMethodJobStatus.PREPARING.getValue()) {
            //执行Azkaban任务
            // 获取jobId
            JSONObject argsObj = JSONObject.parseObject(methodStatJob.getJobArgs());
            argsObj.put("start", (int) (methodsStatReqDTO.getStart() / 1000));
            argsObj.put("end", (int) (methodsStatReqDTO.getEnd() / 1000));
            argsObj.put("jobId", methodStatJob.getJobId());
            argsObj.remove("jobName");
            
            // 执行任务
            int execId = executeFlow(argsObj.toJSONString(), methodStatJob.getJobId(), false);
            if (execId < 0) {
                throw new ZSYServiceException("Azkaban工作流正在执行任务,请稍后再试.");
            }
            //更新
            MethodStatJob update = new MethodStatJob();
            update.setJobId(methodStatJob.getJobId());
            update.setJobExecId(execId);
            // 将执行id 更新到任务表中
            if (methodStatJobMapper.updateByJobId(update) == 0) {
                throw new ZSYServiceException(String.format("Azkaban工作流任务状态更新失败,参数jobId:%s,execId:%s.", methodStatJob.getJobId(), methodStatJob.getJobExecId()));
            }
        }

执行azkaban任务

• 执行azkaban任务
• ZSYMethodsInfoServiceImpl.java

 /**
     * 执行Azkaban流程
     *
     * @param args
     */
    private int executeFlow(String args, Integer jobId, boolean concurrent) {
        try {
            String encodeArgsJson = URLEncoder.encode(args, "UTF-8");
            // 获取会话id
            String sessionId = getSessionId();
            String formatUrl;

//            /**
//             * Azkaban任务执行地址
//             */
//            private static final String EXECUTE_FLOW_ARGS = "{0}/executor?ajax=executeFlow&session.id={1}&project={2}&flow={3}&flowOverride[args]={4}&flowOverride[job_id]={5}";
//
//            /**
//             * Azkaban任务执行地址(运行并行计算)
//             */
//            private static final String EXECUTE_FLOW_CONCURRENT = "{0}/executor?ajax=executeFlow&session.id={1}&project={2}&flow={3}&concurrentOption=ignore&flowOverride[args]={4}&flowOverride[job_id]={5}";

            if (concurrent) {
                formatUrl = MessageFormat.format(EXECUTE_FLOW_CONCURRENT, azkabanProperty.getUrl(), sessionId, azkabanProperty.getProjectName(), azkabanProperty.getFlowName(), encodeArgsJson, jobId);
            } else {
                formatUrl = MessageFormat.format(EXECUTE_FLOW_ARGS, azkabanProperty.getUrl(), sessionId, azkabanProperty.getProjectName(), azkabanProperty.getFlowName(), encodeArgsJson, jobId);
            }
            String result = OKHttpHelper.post(formatUrl);
            JSONObject resultObj = JSONObject.parseObject(result);
            String error = resultObj.getString("error");
            int execId = resultObj.getIntValue("execid");
            if (!Strings.isNullOrEmpty(error)) {
                if (execId < 0) {
                    return execId;
                }
                throw new ZSYServiceException("Azkaban流程执行失败");
            }
            return execId;
        } catch (Exception e) {
            throw new ZSYServiceException("Azkaban流程执行失败");
        }
    }

获取sessionId

• 获取sessionId
• ZSYMethodsInfoServiceImpl.java

 /**
     * 登录Azkaban获取session
     *
     * @return
     */
    private String getSessionId() {
        String params = String.format("action=login&username=%s&password=%s", azkabanProperty.getUserName(), azkabanProperty.getPassword());
        MediaType mediaType = MediaType.parse("application/x-www-form-urlencoded");
        RequestBody body = RequestBody.create(mediaType, params);
        Request request = new Request.Builder().url(azkabanProperty.getUrl()).post(body).build();
        String result = OKHttpHelper.handleResponse(request, OKHttpHelper.getClient());
        String status = JSONObject.parseObject(result).getString("status");
        if (status == null || !status.equals("success")) {
            throw new ZSYServiceException("Azkaban登录失败");
        }
        return JSONObject.parseObject(result).getString("session.id");
    }

shell脚本中的回调命令

#判断执行结果
if [[ $? -ne 0 ]]; then
   #执行失败
   echo `curl -H 'content-type: application/json' --header "Authorization: Basic ${basic_auth}" -X PUT ${report_url}/cgi/report/method-stat/job/callback/${job_id}/3`
else
   #执行成功
   echo `curl -H 'content-type: application/json' --header "Authorization: Basic ${basic_auth}" -X PUT ${report_url}/cgi/report/method-stat/job/callback/${job_id}/2`
fi

azkaban回调函数

• azkaban回调函数
• ZSYMethodsInfoServiceImpl.java

 @Override
    public void jobFinishedCallback(Integer jobId, Integer jobStatus) {
        MethodStatJob methodStatJob = validAndReturnJob(jobId);
        if (methodStatJob.getJobStatus() != ZSYMethodJobStatus.PREPARING.getValue()) {
            throw new ZSYServiceException(String.format("该任务状态异常,状态为:%s", methodStatJob.getJobStatus()));
        }
        MethodStatJob update = new MethodStatJob();
        update.setJobId(jobId);
        if (jobStatus == ZSYMethodJobStatus.SUCCESS.getValue()) {
            int methodCount = methodsInfoMapper.countMethodByJobId(jobId);
            int studentCount = methodsInfoMapper.countStudentByJobId(jobId);
            if (methodCount == 0 && studentCount == 0) {
                update.setJobStatus(ZSYMethodJobStatus.FAILED.getValue());
                update.setJobNote("考点或者学生数据统计失败");
            } else {
                update.setJobStatus(ZSYMethodJobStatus.SUCCESS.getValue());
            }
        } else {
            update.setJobStatus(ZSYMethodJobStatus.FAILED.getValue());
            update.setJobNote("统计任务Spark执行失败");
        }

        if (methodStatJobMapper.updateByJobId(update) == 0) {
            throw new ZSYServiceException("更新任务状态失败");
        }
        //查询等待中的任务
        MethodStatJob waitingJobs = methodStatJobMapper.selectWaitingJob();
        if (waitingJobs != null) {
            //将等待中的任务更新为执行中
            // 获取jobId
            JSONObject argsObj = JSONObject.parseObject(waitingJobs.getJobArgs());
            argsObj.put("start", (int) (argsObj.getLongValue("start") / 1000));
            argsObj.put("end", (int) (argsObj.getLongValue("emd") / 1000));
            argsObj.put("jobId", waitingJobs.getJobId());
            argsObj.remove("jobName");
            int execId = executeFlow(argsObj.toJSONString(), waitingJobs.getJobId(), true);
            if (execId < 0) {
                throw new ZSYServiceException("Azkaban工作流正在执行任务,请稍后再试.");
            }
            //更新
            MethodStatJob updateWaiting = new MethodStatJob();
            updateWaiting.setJobId(waitingJobs.getJobId());
            updateWaiting.setJobExecId(execId);
            updateWaiting.setJobStatus(ZSYMethodJobStatus.PREPARING.getValue());
            if (methodStatJobMapper.updateByJobId(updateWaiting) == 0) {
                throw new ZSYServiceException(String.format("Azkaban工作流任务状态更新失败,参数jobId:%s,execId:%s.", waitingJobs.getJobId(), waitingJobs.getJobExecId()));
            }
        }
    }