Elastic-Job-Lite分布式调度（springboot）

最新推荐文章于 2024-07-24 18:44:23 发布

usernamehui

最新推荐文章于 2024-07-24 18:44:23 发布

阅读量1k

点赞数

本文链接：https://blog.csdn.net/usernamehui/article/details/121946563

版权

概述

Elastic-Job 是一个开源的分布式调度中间件，由两个相互独立的子项目 Elastic-Job-Lite 和 Elastic- Job-Cloud 组成。

Elastic-Job-Lite 为轻量级无中心化解决方案，使用 jar 包提供分布式任务的调度和治理。

Elastic-Job-Lite 定位为纯粹的作业中间件，仅关注分布式调度、协调以及分片等核心功能，将作业部署、资源分配等功能交于 Mesos 或 Kubernetes 处理；

项目架构

ElasticJob-Lite 定位为轻量级无中心化解决方案，使用 jar 的形式提供分布式任务的协调服务。

如何开发（SpringBoot项目为例）

POM配置

<dependency> 2.

<version>${elastic-job.latest.release.version}</version>

3. <artifactId>elasticjob-lite-core </artifactId> 4.

5. </dependency> 6.

<artifactId>elasticjob-lite-spring-boot-starter</artifactId>

7. <groupId>org.apache.shardingsphere.elasticjob </groupId> 8.

</dependency>

9. <version>${elastic-job.latest.release.version} </version> 10.

<groupId>org.apache.shardingsphere.elasticjob</groupId>

11. <dependency> 12.

<version>${latest.release.version}</version>

13. <artifactId>elasticjob-error-handler-dingtalk </artifactId> 14.

</dependency>

properties配置

elasticjob.reg-center.namespace=hsy-bz-elasticjob

# elastic 2.

3. elasticjob.reg-center.server-lists=192.168.0.6:2181, 192.168.0.7:2182,192.168.0.8:2183

elasticjob.reg-center.base-sleep-time-milliseconds=20000

elasticjob.reg-center.connection-timeout-milliseconds=60000

5. elasticjob .reg -center .max -sleep -time -milliseconds = 50000 6.

7. elasticjob.reg-center.max-retries=6 8.

elasticjob.jobs.UserChallenge.elastic-job-class=com.damei.hsydata.bz.datasync.job.UserCh

allenge

9. ### simple job 10.

elasticjob.jobs.UserChallenge.sharding-total-count=1

11. elasticjob .jobs . UserChallenge .cron = 55 * * * * ? 12.

13.

# dataflow job

14.

15. elasticjob.jobs.KolDayDataProcess.elastic-job-class=com.damei.hsydata.bz.datasync.job.Ko lDayDataProcess

elasticjob.jobs.KolDayDataProcess.cron=0 40 9 * * ?

16.

elasticjob.jobs.UserChallenge.sharding-item-parameters= 0=Beijing,1=Shanghai

17. elasticjob .jobs . KolDayDataProcess .sharding -total -count = 2 18.

elasticjob.jobs.KolDayDataProcess.overwrite=true

19. elasticjob .jobs . KolDayDataProcess .props .streaming .process = true 20.

21. elasticjob.dump.port=9111 22.

elasticjob.jobs.myScriptJob.elastic-job-type=SCRIPT

23. # scrip job 24.

elasticjob.jobs.myScriptJob.sharding-total-count=2

25. elasticjob .jobs .myScriptJob .cron = 0 / 10 * * * * ? 26.

elasticjob.jobs.myScriptJob.props.script.command.line= echo sharding execution context is $*

简单作业

1. package com.damei.hsydata.bz.datasync.job; 2.

import com.cms.cache.Cache;

3. import com .alibaba .fastjson . JSONObject ; 4.

import com.damei.hsydata.bz.datasync.constant.RedisKeys;

5. import com .damei .common .util . DateUtil ; 6.

import com.damei.hsydata.model.hsy.UserChallengeInfo;

7. import com .damei .hsydata .model .hsy . KolInfo ; 8.

import com.google.common.collect.Maps;

9. import com .damei .hsydata .service .hsy .*; 10.

import org.apache.shardingsphere.elasticjob.simple.job.SimpleJob;

11. import org .apache .shardingsphere .elasticjob .api . ShardingContext ; 12.

import org.slf4j.LoggerFactory;

13. import org .slf4j . Logger ; 14.

import org.springframework.stereotype.Component;

15. import org .springframework .beans .factory .annotation . Autowired ; 16.

17.

import java.util.Date;

18.

import java.util.Map;

19. import java .util . List ; 20.

21.

@Component

22.

private final Logger logger = LoggerFactory.getLogger(UserChallenge.class);

23. public class UserChallenge implements SimpleJob { 24.

private IUserChallengeInfoService userChallengeInfoService;

25. @Autowired 26.

private IKolInfoService kolInfoService;

27. @Autowired 28.

Cache cache;

29. @Autowired 30.

public void execute(ShardingContext context) {

31. @Override 32.

case 0:

33. switch (context .getShardingItem ()) { 34.

setUserChallengeCacheTask();

35. // do something by sharding item 0 36.

case 1:

37. break ; 38.

break;

39. // do something by sharding item 1 40.

// do something by sharding item 2

41. case 2 : 42.

// case n: ...

break ; 44.

}

45. } 46.

47. }

流式作业

1. package com.damei.hsydata.bz.datasync.job; 2.

import com.damei.hsydata.bz.datasync.constant.RedisKeys;

3. import com .cms .cache . Cache ; 4.

import com.damei.hsydata.bz.datasync.util.CommonUtil;

5. import com .damei .hsydata .bz .datasync .factory . KolDayDataProcessFactory ; 6.

import org.apache.shardingsphere.elasticjob.dataflow.job.DataflowJob;

7. import org .apache .shardingsphere .elasticjob .api . ShardingContext ; 8.

9. import org .slf4j . Logger ;

import org.slf4j.LoggerFactory;

10.

import org.springframework.stereotype.Component;

11. import org .springframework .beans .factory .annotation . Autowired ; 12.

13.

import java.util.*;

14.

15.

import java.util.concurrent.ThreadPoolExecutor;

16.

17.

@Component

18.

private final Logger logger = LoggerFactory.getLogger(KolDayDataProcess.class);

19. public class KolDayDataProcess implements DataflowJob { 20.

21.

@Autowired

22.

@Autowired

23. private KolDayDataProcessFactory kolDayDataProcessFactory ; 24.

private ThreadPoolExecutor threadPoolExecutor= CommonUtil.initDefaultThreadPool();

25. Cache cache ; 26.

27.

28.

public List fetchData(ShardingContext shardingContext) {

29. @Override 30.

31. try {

32.	return cache.mRpoplpush(RedisKeys.KolInfoIdCacheSourceKey,RedisKeys.KolInfoIdCacheDestinationKey
	,	RedisKeys.SRandomNumber);
33.
34.		} catch (Exception ex) {
35.		ex.printStackTrace();
36.		logger.error("执行设置KOL日数据缓存任务失败：", ex);
37.		}
38.		return null;
39.		}
40.
41.		@Override
42.		public void processData(ShardingContext shardingContext, List list) {
43.		for (Object id : list) {
44.		threadPoolExecutor.execute(new Runnable() {
45.		@Override
46.		public void run() {
47.		try {
48.		final Long kolId = Long.parseLong(id.toString());