schedulerx 使用说明
相关依赖
<dependency>
<groupId>com.aliyun.schedulerx</groupId>
<artifactId>schedulerx2-spring-boot-starter</artifactId>
<version>1.5.0.2</version>
</dependency>
自动配置类
# META-INF/spring.factories
org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
com.alibaba.schedulerx.SchedulerxAutoConfigure
SchedulerxAutoConfigure
@Configuration
@ConditionalOnClass({SchedulerxWorker.class})
@ConditionalOnProperty(
prefix = "spring.schedulerx2",
name = {"enabled"},
havingValue = "true",
matchIfMissing = true
)
@EnableConfigurationProperties({SchedulerxProperties.class}) //配置类
public class SchedulerxAutoConfigure {
@Autowired
private SchedulerxProperties properties;
@Autowired
private JobSyncService jobSyncService;
private static final Logger LOGGER = LogFactory.getLogger(SchedulerxAutoConfigure.class);
public SchedulerxAutoConfigure() {
}
@Bean
public void syncJobs() throws Exception {
if (!this.properties.getJobs().isEmpty()) {
LOGGER.info("spring.schedulerx2.jobs is not empty, start to sync jobs...");
this.jobSyncService.syncJobs();
LOGGER.info("sync jobs finished.");
}
}
@Bean
public SchedulerxWorker schedulerxWorker() {
SchedulerxWorker schedulerxWorker = new SchedulerxWorker();
schedulerxWorker.setDomainName(this.properties.getDomainName());
schedulerxWorker.setGroupId(this.properties.getGroupId());
schedulerxWorker.setEnableBatchWork(this.properties.isEnableBatchWork());
schedulerxWorker.setDisableSites(this.properties.getDisableSites());
schedulerxWorker.setEnableSites(this.properties.getEnableSites());
schedulerxWorker.setDisableUnits(this.properties.getDisableUnits());
schedulerxWorker.setEnableUnits(this.properties.getEnableUnits());
schedulerxWorker.setAppKey(this.properties.getAppKey());
schedulerxWorker.setAliyunAccessKey(this.properties.getAliyunAccessKey());
schedulerxWorker.setAliyunSecretKey(this.properties.getAliyunSecretKey());
schedulerxWorker.setNamespace(this.properties.getNamespace());
schedulerxWorker.setHost(this.properties.getHost());
schedulerxWorker.setPort(this.properties.getPort());
schedulerxWorker.setEndpoint(this.properties.getEndpoint());
schedulerxWorker.setNamespaceSource(this.properties.getNamespaceSource());
schedulerxWorker.setMaxTaskBodySize(this.properties.getMaxTaskBodySize());
schedulerxWorker.setBlockAppStart(this.properties.isBlockAppStart());
schedulerxWorker.setSTSAccessKey(this.properties.getStsAccessKey());
schedulerxWorker.setSTSSecretKey(this.properties.getStsSecretKey());
schedulerxWorker.setSTSSecretToken(this.properties.getStsToken());
schedulerxWorker.setSlsCollectorEnable(this.properties.isSlsCollectorEnable());
schedulerxWorker.setShareContainerPool(this.properties.isShareContainerPool());
schedulerxWorker.setLabel(this.properties.getLabel());
if (this.properties.isShareContainerPool()) {
schedulerxWorker.setSharePoolSize(this.properties.getSharePoolSize());
}
if (StringUtils.isNotEmpty(this.properties.getEndpointPort())) {
schedulerxWorker.setEndpointPort(Integer.parseInt(this.properties.getEndpointPort()));
}
schedulerxWorker.setEnableCgroupMetrics(this.properties.isEnableCgroupMetrics());
if (this.properties.isEnableCgroupMetrics()) {
schedulerxWorker.setCgroupPathPrefix(this.properties.getCgroupPathPrefix());
}
if (StringUtils.isNotEmpty(this.properties.getNamespaceSource())) {
schedulerxWorker.setNamespaceSource(this.properties.getNamespaceSource());
}
schedulerxWorker.setAkkaRemotingAutoRecover(this.properties.isAkkaRemotingAutoRecover());
schedulerxWorker.setEnableHeartbeatLog(this.properties.isEnableHeartbeatLog());
schedulerxWorker.setMapMasterStatusCheckInterval(this.properties.getMapMasterStatusCheckInterval());
schedulerxWorker.setEnableSecondDelayCycleIntervalMs(this.properties.isEnableSecondDealyCycleIntervalMs());
schedulerxWorker.setEnableMapMasterFailover(this.properties.isEnableMapMasterFailover());
schedulerxWorker.setEnableSecondDelayStandaloneDispatch(this.properties.isEnableSecondDelayStandaloneDispatch());
schedulerxWorker.setPageSize(this.properties.getPageSize());
ConfigUtil.getWorkerConfig().setProperty("stater.mode", "springboot");
return schedulerxWorker;
}
}
SchedulerxProperties
@ConfigurationProperties(
prefix = "spring.schedulerx2"
)
public class SchedulerxProperties {
private String domainName;
private String groupId;
private String host;
private int port = 0;
private String enableUnits;
private String disableUnits;
private String enableSites;
private String disableSites;
private boolean enableBatchWork;
private boolean enabled = true;
private String appName;
private String appKey;
private String aliyunRamRole;
private String aliyunAccessKey;
private String aliyunSecretKey;
private String stsAccessKey;
private String stsSecretKey;
private String stsToken;
private String namespace;
private String endpoint;
private String endpointPort;
private String namespaceName;
private String namespaceSource;
private int maxTaskBodySize = 65536;
private boolean blockAppStart = true;
private boolean slsCollectorEnable = true;
private boolean shareContainerPool = false;
private int sharePoolSize = 64;
private String label;
private boolean enableCgroupMetrics = false;
private String cgroupPathPrefix = "/sys/fs/cgroup/cpu/";
private boolean akkaRemotingAutoRecover = true;
private boolean enableHeartbeatLog = true;
private int mapMasterStatusCheckInterval = 3000;
private boolean enableSecondDealyCycleIntervalMs = false;
private boolean enableMapMasterFailover = true;
private boolean enableSecondDelayStandaloneDispatch = false;
private int pageSize = 1000;
private String regionId;
private Map<String, JobProperty> jobs = new LinkedHashMap();
private String alarmChannel;
private Map<String, ContactInfo> alarmUsers = new LinkedHashMap();
public SchedulerxProperties() {
}
JobProperty:任务属性
@ConfigurationProperties(
prefix = "spring.schedulerx2"
)
public final class JobProperty {
private String jobType;
private String jobModel;
private String className;
private String cron;
private String oneTime;
private String jobParameter;
private String description;
private boolean overwrite;
public JobProperty() {
this.jobType = JobType.JAVA.getKey();
this.jobModel = ExecuteMode.STANDALONE.getKey();
this.overwrite = false;
}
JobType:任务类型
public enum JobType {
JAVA("java"),
PYTHON("python"),
SHELL("shell"),
GO("go"),
HTTP("http"),
XXLJOB("xxljob"),
DATAWORKS("dataworks");
private String key;
private JobType(String key) {
this.key = key;
}
public String getKey() {
return this.key;
}
public void setKey(String key) {
this.key = key;
}
}
ExecuteMode:任务执行模式
public enum ExecuteMode {
STANDALONE("standalone", "单机运行"),
BROADCAST("broadcast", "广播运行"),
PARALLEL("parallel", "并行计算"),
GRID("grid", "内存网格"),
BATCH("batch", "网格计算"),
SHARDING("sharding", "分片运行");
private String key;
private String desc;
static Map<String, ExecuteMode> CACHE = Maps.newHashMap();
private ExecuteMode(String key, String desc) {
this.key = key;
this.desc = desc;
}
public static ExecuteMode getByKey(String key) {
public void setKey(String key) {
public void setDesc(String desc) {
public String getKey() {
public String getDesc() {
ContactInfo:报警联系方式
public class ContactInfo {
private String empId;
private String userName;
private String userPhone;
private String userMail;
private String ding;
public ContactInfo() {
}
public ContactInfo(String empId, String userName) {
this.empId = empId;
this.userName = userName;
}
相关类与接口
JobProcessor
public interface JobProcessor extends BaseProcessor {
ProcessResult process(JobContext var1) throws Exception;
}
JobProcessorEx
public interface JobProcessorEx extends JobProcessor {
void preProcess(JobContext var1) throws Exception;
ProcessResult process(JobContext var1) throws Exception;
ProcessResult postProcess(JobContext var1);
void kill(JobContext var1);
}
JavaProcessor
public abstract class JavaProcessor implements JobProcessorEx {
public JavaProcessor() {
}
public void preProcess(JobContext context) {
}
public ProcessResult postProcess(JobContext context) {
return null;
}
public void kill(JobContext context) {
}
public ProcessResult process(JobContext context) throws Exception {
return null;
}
}
MapJobProcessor
public abstract class MapJobProcessor extends JavaProcessor {
private LogCollector logCollector = LogCollectorFactory.get();
private static final Logger LOGGER = LogFactory.getLogger(MapJobProcessor.class);
private static final Integer MAX_RETRY_COUNT = 3;
public MapJobProcessor() {
}
public ProcessResult map(List<? extends Object> taskList, String taskName) {
private void checkTaskObject(Object taskObject) {
private WorkerMapTaskResponse handleMapTask(TaskMaster taskMaster, WorkerMapTaskRequest request) throws Exception {
protected boolean isRootTask(JobContext context) {
return context.getTaskName().equals("MAP_TASK_ROOT");
}
}
MapReduceJobProcessor
public abstract class MapReduceJobProcessor extends MapJobProcessor {
private static final Logger LOGGER = LogFactory.getLogger(MapReduceJobProcessor.class);
public MapReduceJobProcessor() {
}
public abstract ProcessResult reduce(JobContext var1) throws Exception;
/** @deprecated */
@Deprecated
public ProcessResult postProcess(JobContext context) {
LOGGER.warn("MapReduceJobProcessor not support postProcess, please use reduce to instead of");
return null;
}
public boolean runReduceIfFail(JobContext context) {
return true;
}
}
ShellProcessor
public class ShellProcessor implements JobProcessorEx {
private Process shellProcess = null;
private LogCollector logCollector = LogCollectorFactory.get();
protected static final Logger LOGGER = LogFactory.getLogger(ShellProcessor.class);
private String uniqueId = null;
public ShellProcessor() {
}
public ProcessResult process(JobContext context) {
ProcessResult result = new ProcessResult(false);
this.logCollector.collect(this.uniqueId, "Script process start to run, taskName=" + context.getTaskName());
try {
ProcessBuilder processBuilder = ShellUtil.createProcessBuilder(this.getContent(context));
if (this.redirectStream()) {
processBuilder.redirectErrorStream(true);
}
this.shellProcess = processBuilder.start();
CountDownLatch countDownLatch = null;
ShellStreamProcessor stderrStreamProcessor;
if (this.redirectStream()) {
countDownLatch = new CountDownLatch(1);
stderrStreamProcessor = new ShellStreamProcessor(this, this.shellProcess.getInputStream(), StreamType.STD_ERR, countDownLatch);
stderrStreamProcessor.start();
} else {
countDownLatch = new CountDownLatch(2);
stderrStreamProcessor = new ShellStreamProcessor(this, this.shellProcess.getInputStream(), StreamType.STD_OUT, countDownLatch);
stderrStreamProcessor.start();
Thread stderrStreamProcessor = new ShellStreamProcessor(this, this.shellProcess.getErrorStream(), StreamType.STD_ERR, countDownLatch);
stderrStreamProcessor.start();
}
countDownLatch.await();
if (this.shellProcess.waitFor() == 0) {
result.setStatus(true);
}
} catch (Throwable var10) {
LOGGER.error("", var10);
this.logCollector.collect(this.uniqueId, "script process errors", var10);
result.setResult(ExceptionUtil.getMessage(var10));
} finally {
this.logCollector.collect(this.uniqueId, "Script process end, run status=" + result.getStatus().getEnDesc());
return result;
}
}
protected String[] getContent(JobContext context) { //获取脚本内容
String[] parameters;
if (StringUtils.isNotEmpty(context.getShardingParameter())) {
parameters = new String[]{String.valueOf(context.getShardingId()), context.getShardingParameter()};
} else if (StringUtils.isNotEmpty(context.getInstanceParameters())) {
parameters = context.getInstanceParameters().trim().split(" ");
} else {
parameters = context.getJobParameters().trim().split(" ");
}
String[] contents = new String[3 + parameters.length];
contents[0] = "/bin/sh";
contents[1] = "-c";
contents[2] = context.getContent();
for(int i = 0; i < parameters.length; ++i) {
contents[3 + i] = parameters[i];
}
return contents;
}
protected void processStdOutputStream(InputStream inputStream) {
String line = null;
try {
BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
while((line = br.readLine()) != null) {
this.logCollector.collect(this.uniqueId, line);
}
} catch (Throwable var7) {
LOGGER.error("error ShellJobProcessor stdout stream", var7);
this.logCollector.collect(this.uniqueId, "error process stdout stream", var7);
} finally {
this.logCollector.collect(this.uniqueId, line, StreamType.STD_OUT, true);
}
}
protected void processStdErrorStream(InputStream inputStream) {
String line = null;
try {
BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
while((line = br.readLine()) != null) {
this.logCollector.collect(this.uniqueId, line);
}
} catch (Throwable var7) {
LOGGER.error("error ShellJobProcessor stderr stream", var7);
this.logCollector.collect(this.uniqueId, "error process stderr stream", var7);
} finally {
this.logCollector.collect(this.uniqueId, line, StreamType.STD_OUT, true);
}
}
protected boolean redirectStream() {
return true;
}
public ProcessResult postProcess(JobContext context) {
return null;
}
public void kill(JobContext context) {
try {
long pid = ShellUtil.getPidOfProcess(this.shellProcess);
if (pid > 0L) {
ShellUtil.killProcess(pid);
}
} catch (Throwable var5) {
LOGGER.error("kill shell job jobInstanceId={} failed, {}", new Object[]{context.getJobInstanceId(), var5});
}
try {
if (this.shellProcess != null) {
this.shellProcess.destroy();
}
} catch (Throwable var4) {
LOGGER.error("kill shell job jobInstanceId={} failed, {}", new Object[]{context.getJobInstanceId(), var4});
}
}
public void preProcess(JobContext context) throws Exception {
this.uniqueId = context.getUniqueId();
}
}
JobCobtext
public final class JobContext {
private long jobId;
private long jobInstanceId;
private Long wfInstanceId;
private long taskId;
private String jobName;
private DateTime scheduleTime;
private DateTime dataTime;
private String executeMode;
private String jobType;
private String instanceMasterActorPath;
private String taskName;
private Object task;
private String groupId;
private String content;
private String user;
private int maxAttempt;
private int attempt;
private String jobParameters;
private String instanceParameters;
private List<JobInstanceData> upstreamData;
private Map<Long, String> taskResults;
private Map<Long, TaskStatus> taskStatuses;
private int taskMaxAttempt;
private int taskAttempt;
private int taskAttemptInterval;
private long serialNum;
private LogCollector logCollector;
private Long shardingId;
private String shardingParameter;
private int shardingNum;
private List<String> allWorkerAddrs;
private String workerAddr;
private int timeType;
private String timeExpression;
private JobContext() {
this.taskId = 0L;
this.taskAttempt = 0;
this.shardingNum = 0;
}
ProcessResult
public class ProcessResult {
private InstanceStatus status;
private String result;
public ProcessResult(InstanceStatus status) {
this.status = status;
}
public ProcessResult(boolean status) {
this.status = status ? InstanceStatus.SUCCESS : InstanceStatus.FAILED;
}
public ProcessResult(InstanceStatus status, String result) {
this.status = status;
this.result = result;
}
public ProcessResult(boolean status, String result) {
if (result != null && result.getBytes().length > 1000) {
throw new RuntimeException("result size more than 1000 bytes!");
} else {
this.status = status ? InstanceStatus.SUCCESS : InstanceStatus.FAILED;
this.result = result;
}
}