ICloudOperator 执行过程分析

Operator 执行过程分析

算子执行过程：

WorkflowExecutor执行到算子调用部分时，会调用executeOperator

operatorTaskId = operatorService.executeOperator(workflow.getWorkflowRunningId() + "-" + operator.getOperatorId(),
        operator.getOperatorType(), params);

executeOperator主要做的工作是：

1检查算子名称，算子参数，以及算子执行状态是否为NULL;

public String executeOperator(String operatorId, String operatorName, Map<String, Object> paras, Executor executor)
        throws OperatorException {
    if (operatorName == null || "".equals(operatorName.trim()) || paras == null || executor == null) {
        LOG.error("Unable to invoke operator： {}", operatorName);
        throw new OperatorException("Unable to invoke operator： " + operatorName);
    }

然后启动算子任务：

A：设置算子启动时间B：设置算子任务状态为RUNNING C:开始执行算子任务

public void operatorStarted(Workflow workflow, Operator operator) {
    operator.setStartTime(System.currentTimeMillis());
    operator.setState(TaskStatus.RUNNING);
    workFlowDatabaseOp.startOperator(operator.getId(), operator.getStartTime());
}

算子开始执行过程：

1检查传入算子参数，会调用OperatorParametersValidator方法；里面具体工作是：

A：根据算子定义文档判断算子是否存在B：检查输入参数是否为空C：输入输出口地址是否为空D：参数取值是否合法

final OperatorParameters operatorParas = new OperatorParametersValidator(operatorName, paras).validate();

2创建一个算子任务task，调用OperatorTask方法生成：

final OperatorTask task = new OperatorTask(connectorsFactory, operatorId, operatorName, operatorParas);

3 task主要干了这些事情：

1用isNeedStop作为任务是否执行的判断依据，在stop()方法和isStopTimeout()中会修改该值。具体是算子运行过程中出现什么状态就相应的调用并修改此值；

2设置当前线程名字为taskID；

3打印线程ID和taskID开始时间；

4同步更新OperatorReport的状态为RUNNING；

5初始化当前算子状态，传入参数；

6设置reporter.reportProgress(100)完成状态100%；

7 taskID运行结束，打印结束时间；

try {
        if (isNeedStop) {            return;        }

        //先申请资源，然后开始执行算子
        Thread.currentThread().setName(taskId);
        System.out.println("Thread Id" + Thread.currentThread().getId());
        now = new Date();
        LOG.info("--------" + taskId + " begin at " + dateFormat.format(now) + "--------");
        synchronized (operatorReport) {
            operatorReport.setState(OperatorReport.State.RUNNING);
        }
        OperatorTaskReporter reporter = new OperatorTaskReporter(operatorReport);
        operator.init(connectorsFactory, paras, reporter);
        operator.execute();
        reporter.reportProgress(100);
    } catch (Exception e) {
        String errorMessage = retainErrorMessage(e);
        operatorReport.setErrorMsg(errorMessage);
        LOG.error(e.getMessage(), e);
        finalStatus = OperatorReport.FinalStatus.FAILED;
    } finally {
        synchronized (operatorReport) {
            finish(finalStatus, errMessage);
        }
        Thread.currentThread().setName(defaultThreadName);
    }
    now = new Date();
    LOG.info("--------" + taskId + " end at " + dateFormat.format(now) + "--------");
}

OperatorReport方法

该方法主要是提供算子状态信息的获取和设置功能，算子信息主要包含：

private final String taskId;
private State state;//算子任务执行状态running、preparing、finished
private int progress;//线程完成率
private FinalStatus finalStatus; //算子最终状态标志success,stopped,failed
private long startedTime = System.currentTimeMillis();
private long endTime;

stopOperator部分

1 WorkflowExecutor开启算子任务之后，即OperatorRunner之后，进入while（true）循环，不断查询workflow的TaskStatus和OperatorReport的finalStatus的状态，当状态变为stopped和failure时，就设置以下三个的状态，并调用stopOperator。

workflowStateChanged(operator, TaskStatus.STOPPED);
executionListener.workflowFinished(workflow, TaskStatus.STOPPED);
operatorService.stopOperator(operatorTaskId);

 

workflowStateChanged(operator, TaskStatus.FAILURE);
executionListener.workflowFinished(workflow, TaskStatus.FAILURE);
operatorService.stopOperator(operatorTaskId);

 

 

具体的stopOperator方法

public void stopOperator(String taskId) throws Throwable {
    operatorTaskManager.stopTask(taskId);
}

调用stopTask之后，调用task.stop()

public void stop() {
    if (operator != null) {
        isNeedStop = true;
        stopTimeStamp = System.currentTimeMillis();
        operator.stop();
        try {
            operator.cleanup();
        } catch (OperatorException e) {
            LOG.info("clean work");
        }
        thread.stop();
    }
}

 

最后

附上OperatorRunner的run方法：

public void run() {
    String originThreadName = Thread.currentThread().getName();
    Thread.currentThread().setName(createThreadName(workflow, operator));
    String message = "Operator " + operator.getTitle() + " execution started.";
    LOG.info(message);
    executionListener.updateWorkflowLog(workflow, message);
    try {
        message = "Actually execute workflow operator: parameters: ." + params.toString();
    LOG.info(message);
    executionListener.updateWorkflowLog(workflow, message);
    String operatorTaskId = null;
    operatorTaskId = operatorService.executeOperator(workflow.getWorkflowRunningId() + "-" + operator.getOperatorId(),
    operator.getOperatorType(), params);
    executionListener.operatorStarted(workflow, operator);
        while (true) {
            Thread.sleep(2000);
       OperatorReport operatorReport =       operatorService.getOperatorReport(operatorTaskId);
 OperatorReport.FinalStatus finalStatus = operatorReport.getFinalStatus();
            if (workflow.getWorkflowStatus().equals(TaskStatus.STOPPING) ||
         workflow.getWorkflowStatus().equals(TaskStatus.STOPPED)) {
                finalStatus = OperatorReport.FinalStatus.STOPPED;
            }
            if (finalStatus == null) {
                continue;
            } else {
    if (finalStatus.equals(OperatorReport.FinalStatus.SUCCEEDED)) {
    List<OutputSource> outputs = operatorReport.getOutputSourceListFromOperator();
    handleNewOutputsFromOperator(outputs);
    workflowStateChanged(operator, TaskStatus.SUCCESS);
    } else if (finalStatus.equals(OperatorReport.FinalStatus.STOPPED)) {
   List<OutputSource> outputs = operatorReport.getOutputSourceListFromOperator();
    handleNewOutputsFromOperator(outputs);
    String error = operatorReport.getErrorMsg();
    operator.setErrorMessage(new StringContentBuffer(error));
    workflow.setErrorMessage(new StringContentBuffer(error));
    executionListener.updateWorkflowLog(workflow, error);
    workflowStateChanged(operator, TaskStatus.STOPPED);
    executionListener.workflowFinished(workflow, TaskStatus.STOPPED);
     operatorService.stopOperator(operatorTaskId);
    } else {
    List<OutputSource> outputs = operatorReport.getOutputSourceListFromOperator();
     handleNewOutputsFromOperator(outputs);
     String error = operatorReport.getErrorMsg();
     operator.setErrorMessage(new StringContentBuffer(error));
     workflow.setErrorMessage(new StringContentBuffer(error));
     executionListener.updateWorkflowLog(workflow, error);
     workflowStateChanged(operator, TaskStatus.FAILURE);
     executionListener.workflowFinished(workflow, TaskStatus.FAILURE);
       operatorService.stopOperator(operatorTaskId);
                }
                break;
            }
        }
    } catch (Throwable throwable) {
        workflowStateChanged(operator, TaskStatus.FAILURE);
        executionListener.updateWorkflowLog(workflow, handleOperatorExecuteException(throwable));
    }
    LOG.info("Leave operator execution.");
    Thread.currentThread().setName(originThreadName);
}