schedulerx 使用说明

---

schedulerx 使用说明

       

           

                                 

相关依赖

        

        <dependency>
            <groupId>com.aliyun.schedulerx</groupId>
            <artifactId>schedulerx2-spring-boot-starter</artifactId>
            <version>1.5.0.2</version>
        </dependency>

             

                 

                                 

自动配置类

    

                   

# META-INF/spring.factories
org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
com.alibaba.schedulerx.SchedulerxAutoConfigure

          

SchedulerxAutoConfigure

@Configuration
@ConditionalOnClass({SchedulerxWorker.class})
@ConditionalOnProperty(
    prefix = "spring.schedulerx2",
    name = {"enabled"},
    havingValue = "true",
    matchIfMissing = true
)
@EnableConfigurationProperties({SchedulerxProperties.class})  //配置类
public class SchedulerxAutoConfigure {
    @Autowired
    private SchedulerxProperties properties;
    @Autowired
    private JobSyncService jobSyncService;
    private static final Logger LOGGER = LogFactory.getLogger(SchedulerxAutoConfigure.class);

    public SchedulerxAutoConfigure() {
    }

    @Bean
    public void syncJobs() throws Exception {
        if (!this.properties.getJobs().isEmpty()) {
            LOGGER.info("spring.schedulerx2.jobs is not empty, start to sync jobs...");
            this.jobSyncService.syncJobs();
            LOGGER.info("sync jobs finished.");
        }

    }

    @Bean
    public SchedulerxWorker schedulerxWorker() {
        SchedulerxWorker schedulerxWorker = new SchedulerxWorker();
        schedulerxWorker.setDomainName(this.properties.getDomainName());
        schedulerxWorker.setGroupId(this.properties.getGroupId());
        schedulerxWorker.setEnableBatchWork(this.properties.isEnableBatchWork());
        schedulerxWorker.setDisableSites(this.properties.getDisableSites());
        schedulerxWorker.setEnableSites(this.properties.getEnableSites());
        schedulerxWorker.setDisableUnits(this.properties.getDisableUnits());
        schedulerxWorker.setEnableUnits(this.properties.getEnableUnits());
        schedulerxWorker.setAppKey(this.properties.getAppKey());
        schedulerxWorker.setAliyunAccessKey(this.properties.getAliyunAccessKey());
        schedulerxWorker.setAliyunSecretKey(this.properties.getAliyunSecretKey());
        schedulerxWorker.setNamespace(this.properties.getNamespace());
        schedulerxWorker.setHost(this.properties.getHost());
        schedulerxWorker.setPort(this.properties.getPort());
        schedulerxWorker.setEndpoint(this.properties.getEndpoint());
        schedulerxWorker.setNamespaceSource(this.properties.getNamespaceSource());
        schedulerxWorker.setMaxTaskBodySize(this.properties.getMaxTaskBodySize());
        schedulerxWorker.setBlockAppStart(this.properties.isBlockAppStart());
        schedulerxWorker.setSTSAccessKey(this.properties.getStsAccessKey());
        schedulerxWorker.setSTSSecretKey(this.properties.getStsSecretKey());
        schedulerxWorker.setSTSSecretToken(this.properties.getStsToken());
        schedulerxWorker.setSlsCollectorEnable(this.properties.isSlsCollectorEnable());
        schedulerxWorker.setShareContainerPool(this.properties.isShareContainerPool());
        schedulerxWorker.setLabel(this.properties.getLabel());
        if (this.properties.isShareContainerPool()) {
            schedulerxWorker.setSharePoolSize(this.properties.getSharePoolSize());
        }

        if (StringUtils.isNotEmpty(this.properties.getEndpointPort())) {
            schedulerxWorker.setEndpointPort(Integer.parseInt(this.properties.getEndpointPort()));
        }

        schedulerxWorker.setEnableCgroupMetrics(this.properties.isEnableCgroupMetrics());
        if (this.properties.isEnableCgroupMetrics()) {
            schedulerxWorker.setCgroupPathPrefix(this.properties.getCgroupPathPrefix());
        }

        if (StringUtils.isNotEmpty(this.properties.getNamespaceSource())) {
            schedulerxWorker.setNamespaceSource(this.properties.getNamespaceSource());
        }

        schedulerxWorker.setAkkaRemotingAutoRecover(this.properties.isAkkaRemotingAutoRecover());
        schedulerxWorker.setEnableHeartbeatLog(this.properties.isEnableHeartbeatLog());
        schedulerxWorker.setMapMasterStatusCheckInterval(this.properties.getMapMasterStatusCheckInterval());
        schedulerxWorker.setEnableSecondDelayCycleIntervalMs(this.properties.isEnableSecondDealyCycleIntervalMs());
        schedulerxWorker.setEnableMapMasterFailover(this.properties.isEnableMapMasterFailover());
        schedulerxWorker.setEnableSecondDelayStandaloneDispatch(this.properties.isEnableSecondDelayStandaloneDispatch());
        schedulerxWorker.setPageSize(this.properties.getPageSize());
        ConfigUtil.getWorkerConfig().setProperty("stater.mode", "springboot");
        return schedulerxWorker;
    }
}

            

SchedulerxProperties

@ConfigurationProperties(
    prefix = "spring.schedulerx2"
)
public class SchedulerxProperties {
    private String domainName;
    private String groupId;
    private String host;
    private int port = 0;
    private String enableUnits;
    private String disableUnits;
    private String enableSites;
    private String disableSites;
    private boolean enableBatchWork;
    private boolean enabled = true;
    private String appName;
    private String appKey;
    private String aliyunRamRole;
    private String aliyunAccessKey;
    private String aliyunSecretKey;
    private String stsAccessKey;
    private String stsSecretKey;
    private String stsToken;
    private String namespace;
    private String endpoint;
    private String endpointPort;
    private String namespaceName;
    private String namespaceSource;
    private int maxTaskBodySize = 65536;
    private boolean blockAppStart = true;
    private boolean slsCollectorEnable = true;
    private boolean shareContainerPool = false;
    private int sharePoolSize = 64;
    private String label;
    private boolean enableCgroupMetrics = false;
    private String cgroupPathPrefix = "/sys/fs/cgroup/cpu/";
    private boolean akkaRemotingAutoRecover = true;
    private boolean enableHeartbeatLog = true;
    private int mapMasterStatusCheckInterval = 3000;
    private boolean enableSecondDealyCycleIntervalMs = false;
    private boolean enableMapMasterFailover = true;
    private boolean enableSecondDelayStandaloneDispatch = false;
    private int pageSize = 1000;
    private String regionId;
    private Map<String, JobProperty> jobs = new LinkedHashMap();
    private String alarmChannel;
    private Map<String, ContactInfo> alarmUsers = new LinkedHashMap();

    public SchedulerxProperties() {
    }

           

JobProperty：任务属性

@ConfigurationProperties(
    prefix = "spring.schedulerx2"
)
public final class JobProperty {
    private String jobType;
    private String jobModel;
    private String className;
    private String cron;
    private String oneTime;
    private String jobParameter;
    private String description;
    private boolean overwrite;

    public JobProperty() {
        this.jobType = JobType.JAVA.getKey();
        this.jobModel = ExecuteMode.STANDALONE.getKey();
        this.overwrite = false;
    }

          

JobType：任务类型

public enum JobType {
    JAVA("java"),
    PYTHON("python"),
    SHELL("shell"),
    GO("go"),
    HTTP("http"),
    XXLJOB("xxljob"),
    DATAWORKS("dataworks");

    private String key;

    private JobType(String key) {
        this.key = key;
    }

    public String getKey() {
        return this.key;
    }

    public void setKey(String key) {
        this.key = key;
    }
}

             

ExecuteMode：任务执行模式

public enum ExecuteMode {
    STANDALONE("standalone", "单机运行"),
    BROADCAST("broadcast", "广播运行"),
    PARALLEL("parallel", "并行计算"),
    GRID("grid", "内存网格"),
    BATCH("batch", "网格计算"),
    SHARDING("sharding", "分片运行");

    private String key;
    private String desc;
    static Map<String, ExecuteMode> CACHE = Maps.newHashMap();

    private ExecuteMode(String key, String desc) {
        this.key = key;
        this.desc = desc;
    }

    public static ExecuteMode getByKey(String key) {


    public void setKey(String key) {
    public void setDesc(String desc) {

    public String getKey() {
    public String getDesc() {

         

ContactInfo：报警联系方式

public class ContactInfo {
    private String empId;
    private String userName;
    private String userPhone;
    private String userMail;
    private String ding;

    public ContactInfo() {
    }

    public ContactInfo(String empId, String userName) {
        this.empId = empId;
        this.userName = userName;
    }

         

                   

                                 

相关类与接口

   

JobProcessor

public interface JobProcessor extends BaseProcessor {
    ProcessResult process(JobContext var1) throws Exception;
}

                   

            

JobProcessorEx

public interface JobProcessorEx extends JobProcessor {

    void preProcess(JobContext var1) throws Exception;
    ProcessResult process(JobContext var1) throws Exception;
    ProcessResult postProcess(JobContext var1);

    void kill(JobContext var1);
}

JavaProcessor

public abstract class JavaProcessor implements JobProcessorEx {
    public JavaProcessor() {
    }

    public void preProcess(JobContext context) {
    }

    public ProcessResult postProcess(JobContext context) {
        return null;
    }

    public void kill(JobContext context) {
    }

    public ProcessResult process(JobContext context) throws Exception {
        return null;
    }
}

           

MapJobProcessor

public abstract class MapJobProcessor extends JavaProcessor {
    private LogCollector logCollector = LogCollectorFactory.get();
    private static final Logger LOGGER = LogFactory.getLogger(MapJobProcessor.class);
    private static final Integer MAX_RETRY_COUNT = 3;

    public MapJobProcessor() {
    }

    public ProcessResult map(List<? extends Object> taskList, String taskName) {

    private void checkTaskObject(Object taskObject) {
    private WorkerMapTaskResponse handleMapTask(TaskMaster taskMaster, WorkerMapTaskRequest request) throws Exception {


    protected boolean isRootTask(JobContext context) {
        return context.getTaskName().equals("MAP_TASK_ROOT");
    }
}

         

MapReduceJobProcessor

public abstract class MapReduceJobProcessor extends MapJobProcessor {
    private static final Logger LOGGER = LogFactory.getLogger(MapReduceJobProcessor.class);

    public MapReduceJobProcessor() {
    }

    public abstract ProcessResult reduce(JobContext var1) throws Exception;

    /** @deprecated */
    @Deprecated
    public ProcessResult postProcess(JobContext context) {
        LOGGER.warn("MapReduceJobProcessor not support postProcess, please use reduce to instead of");
        return null;
    }

    public boolean runReduceIfFail(JobContext context) {
        return true;
    }
}

            

ShellProcessor

public class ShellProcessor implements JobProcessorEx {
    private Process shellProcess = null;
    private LogCollector logCollector = LogCollectorFactory.get();
    protected static final Logger LOGGER = LogFactory.getLogger(ShellProcessor.class);
    private String uniqueId = null;

    public ShellProcessor() {
    }

    public ProcessResult process(JobContext context) {
        ProcessResult result = new ProcessResult(false);
        this.logCollector.collect(this.uniqueId, "Script process start to run, taskName=" + context.getTaskName());

        try {
            ProcessBuilder processBuilder = ShellUtil.createProcessBuilder(this.getContent(context));
            if (this.redirectStream()) {
                processBuilder.redirectErrorStream(true);
            }

            this.shellProcess = processBuilder.start();
            CountDownLatch countDownLatch = null;
            ShellStreamProcessor stderrStreamProcessor;
            if (this.redirectStream()) {
                countDownLatch = new CountDownLatch(1);
                stderrStreamProcessor = new ShellStreamProcessor(this, this.shellProcess.getInputStream(), StreamType.STD_ERR, countDownLatch);
                stderrStreamProcessor.start();
            } else {
                countDownLatch = new CountDownLatch(2);
                stderrStreamProcessor = new ShellStreamProcessor(this, this.shellProcess.getInputStream(), StreamType.STD_OUT, countDownLatch);
                stderrStreamProcessor.start();
                Thread stderrStreamProcessor = new ShellStreamProcessor(this, this.shellProcess.getErrorStream(), StreamType.STD_ERR, countDownLatch);
                stderrStreamProcessor.start();
            }

            countDownLatch.await();
            if (this.shellProcess.waitFor() == 0) {
                result.setStatus(true);
            }
        } catch (Throwable var10) {
            LOGGER.error("", var10);
            this.logCollector.collect(this.uniqueId, "script process errors", var10);
            result.setResult(ExceptionUtil.getMessage(var10));
        } finally {
            this.logCollector.collect(this.uniqueId, "Script process end, run status=" + result.getStatus().getEnDesc());
            return result;
        }
    }

    protected String[] getContent(JobContext context) {  //获取脚本内容
        String[] parameters;
        if (StringUtils.isNotEmpty(context.getShardingParameter())) {
            parameters = new String[]{String.valueOf(context.getShardingId()), context.getShardingParameter()};
        } else if (StringUtils.isNotEmpty(context.getInstanceParameters())) {
            parameters = context.getInstanceParameters().trim().split(" ");
        } else {
            parameters = context.getJobParameters().trim().split(" ");
        }

        String[] contents = new String[3 + parameters.length];
        contents[0] = "/bin/sh";
        contents[1] = "-c";
        contents[2] = context.getContent();

        for(int i = 0; i < parameters.length; ++i) {
            contents[3 + i] = parameters[i];
        }

        return contents;
    }

    protected void processStdOutputStream(InputStream inputStream) {
        String line = null;

        try {
            BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));

            while((line = br.readLine()) != null) {
                this.logCollector.collect(this.uniqueId, line);
            }
        } catch (Throwable var7) {
            LOGGER.error("error ShellJobProcessor stdout stream", var7);
            this.logCollector.collect(this.uniqueId, "error process stdout stream", var7);
        } finally {
            this.logCollector.collect(this.uniqueId, line, StreamType.STD_OUT, true);
        }

    }

    protected void processStdErrorStream(InputStream inputStream) {
        String line = null;

        try {
            BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));

            while((line = br.readLine()) != null) {
                this.logCollector.collect(this.uniqueId, line);
            }
        } catch (Throwable var7) {
            LOGGER.error("error ShellJobProcessor stderr stream", var7);
            this.logCollector.collect(this.uniqueId, "error process stderr stream", var7);
        } finally {
            this.logCollector.collect(this.uniqueId, line, StreamType.STD_OUT, true);
        }

    }

    protected boolean redirectStream() {
        return true;
    }

    public ProcessResult postProcess(JobContext context) {
        return null;
    }

    public void kill(JobContext context) {
        try {
            long pid = ShellUtil.getPidOfProcess(this.shellProcess);
            if (pid > 0L) {
                ShellUtil.killProcess(pid);
            }
        } catch (Throwable var5) {
            LOGGER.error("kill shell job jobInstanceId={} failed, {}", new Object[]{context.getJobInstanceId(), var5});
        }

        try {
            if (this.shellProcess != null) {
                this.shellProcess.destroy();
            }
        } catch (Throwable var4) {
            LOGGER.error("kill shell job jobInstanceId={} failed, {}", new Object[]{context.getJobInstanceId(), var4});
        }

    }

    public void preProcess(JobContext context) throws Exception {
        this.uniqueId = context.getUniqueId();
    }
}

        

JobCobtext

public final class JobContext {
    private long jobId;
    private long jobInstanceId;
    private Long wfInstanceId;
    private long taskId;
    private String jobName;
    private DateTime scheduleTime;
    private DateTime dataTime;
    private String executeMode;
    private String jobType;
    private String instanceMasterActorPath;
    private String taskName;
    private Object task;
    private String groupId;
    private String content;
    private String user;
    private int maxAttempt;
    private int attempt;
    private String jobParameters;
    private String instanceParameters;
    private List<JobInstanceData> upstreamData;
    private Map<Long, String> taskResults;
    private Map<Long, TaskStatus> taskStatuses;
    private int taskMaxAttempt;
    private int taskAttempt;
    private int taskAttemptInterval;
    private long serialNum;
    private LogCollector logCollector;
    private Long shardingId;
    private String shardingParameter;
    private int shardingNum;
    private List<String> allWorkerAddrs;
    private String workerAddr;
    private int timeType;
    private String timeExpression;

    private JobContext() {
        this.taskId = 0L;
        this.taskAttempt = 0;
        this.shardingNum = 0;
    }

        

ProcessResult

public class ProcessResult {
    private InstanceStatus status;
    private String result;

    public ProcessResult(InstanceStatus status) {
        this.status = status;
    }

    public ProcessResult(boolean status) {
        this.status = status ? InstanceStatus.SUCCESS : InstanceStatus.FAILED;
    }

    public ProcessResult(InstanceStatus status, String result) {
        this.status = status;
        this.result = result;
    }

    public ProcessResult(boolean status, String result) {
        if (result != null && result.getBytes().length > 1000) {
            throw new RuntimeException("result size more than 1000 bytes!");
        } else {
            this.status = status ? InstanceStatus.SUCCESS : InstanceStatus.FAILED;
            this.result = result;
        }
    }