分布式调度框架Elastic-Job
Elastic-Job介绍
Elastic-Job是当当⽹开源的⼀个分布式调度解决⽅案,基于Quartz⼆次开发的,由两个相互独⽴的⼦项⽬Elastic-Job-Lite和Elastic-Job-Cloud组成。我们要学习的是 Elastic-Job-Lite,它定位为轻量级⽆中⼼化解决⽅案,使⽤Jar包的形式提供分布式任务的协调服务,⽽Elastic-Job-Cloud⼦项⽬需要结合Mesos以及Docker在云环境下使⽤。
Elastic-Job的github地址: https://github.com/elasticjob
主要功能介绍
- 分布式调度协调
在分布式环境中,任务能够按指定的调度策略执⾏,并且能够避免同⼀任务多实例重复执⾏ - 丰富的调度策略 基于成熟的定时任务作业框架Quartz cron表达式执⾏定时任务
- 弹性扩容缩容 当集群中增加某⼀个实例,它应当也能够被选举并执⾏任务;当集群减少⼀个实例时,它所执⾏的任务能被转移到别的实例来执⾏。
- 失效转移 某实例在任务执⾏失败后,会被转移到其他实例执⾏
- 错过执⾏作业重触发 若因某种原因导致作业错过执⾏,⾃动记录错过执⾏的作业,并在上次作业完成后⾃动触发。
- ⽀持并⾏调度 ⽀持任务分⽚,任务分⽚是指将⼀个任务分为多个⼩任务项在多个实例同时执⾏。
- 作业分⽚⼀致性 当任务被分⽚后,保证同⼀分⽚在分布式环境中仅⼀个执⾏实例。
Elastic-Job-Lite应⽤
jar包(API) + 安装zk软件方式使用
Elastic-Job依赖于Zookeeper进⾏分布式协调,所以需要安装Zookeeper软件(3.4.6版本以上),Zookeeper的本质功能:
存储+通知。
Job类:
package elasticjob;
import com.dangdang.ddframe.job.api.ShardingContext;
import com.dangdang.ddframe.job.api.simple.SimpleJob;
import java.util.List;
import java.util.Map;
/**
* ElasticJobLite定时任务业务逻辑处理类
*/
public class ArchivieJob implements SimpleJob {
/**
* 需求:resume表中未归档的数据归档到resume_bak表中,每次归档1条记录
* execute方法中写我们的业务逻辑(execute方法每次定时任务执行都会执行一次)
* @param shardingContext
*/
@Override
public void execute(ShardingContext shardingContext) {
int shardingItem = shardingContext.getShardingItem();
System.out.println("=====>>>>当前分片:" + shardingItem);
// 获取分片参数
String shardingParameter = shardingContext.getShardingParameter(); // 0=bachelor,1=master,2=doctor
// 1 从resume表中查询出1条记录(未归档)
String selectSql = "select * from resume where state='未归档' and education='"+ shardingParameter +"' limit 1";
List<Map<String, Object>> list = JdbcUtil.executeQuery(selectSql);
if(list == null || list.size() ==0 ) {
System.out.println("数据已经处理完毕!!!!!!");
return;
}
// 2 "未归档"更改为"已归档"
Map<String, Object> stringObjectMap = list.get(0);
long id = (long) stringObjectMap.get("id");
String name = (String) stringObjectMap.get("name");
String education = (String) stringObjectMap.get("education");
System.out.println("=======>>>>id:" + id + " name:" + name + " education:" + education);
String updateSql = "update resume set state='已归档' where id=?";
JdbcUtil.executeUpdate(updateSql,id);
// 3 归档这条记录,把这条记录插入到resume_bak表
String insertSql = "insert into resume_bak select * from resume where id=?";
JdbcUtil.executeUpdate(insertSql,id);
}
}
工具类:
package elasticjob;
import java.sql.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class JdbcUtil {
//url
private static String url = "jdbc:mysql://localhost:3306/job?characterEncoding=utf8&useSSL=false";
//user
private static String user = "root";
//password
private static String password = "123456";
//驱动程序类
private static String driver = "com.mysql.jdbc.Driver";
static {
try {
Class.forName(driver);
} catch (ClassNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static Connection getConnection() {
try {
return DriverManager.getConnection(url, user, password);
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return null;
}
public static void close(ResultSet rs, PreparedStatement ps, Connection con) {
if (rs != null) {
try {
rs.close();
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
if (ps != null) {
try {
ps.close();
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
if (con != null) {
try {
con.close();
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}
}
}
}
/***
* DML操作(增删改)
* 1.获取连接数据库对象
* 2.预处理
* 3.执行更新操作
* @param sql
* @param obj
*/
//调用者只需传入一个sql语句,和一个Object数组。该数组存储的是SQL语句中的占位符
public static void executeUpdate(String sql,Object...obj) {
Connection con = getConnection();//调用getConnection()方法连接数据库
PreparedStatement ps = null;
try {
ps = con.prepareStatement(sql);//预处理
for (int i = 0; i < obj.length; i++) {//预处理声明占位符
ps.setObject(i + 1, obj[i]);
}
ps.executeUpdate();//执行更新操作
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
close(null, ps, con);//调用close()方法关闭资源
}
}
/***
* DQL查询
* Result获取数据集
*
* @param sql
* @param obj
* @return
*/
public static List<Map<String,Object>> executeQuery(String sql, Object...obj) {
Connection con = getConnection();
ResultSet rs = null;
PreparedStatement ps = null;
try {
ps = con.prepareStatement(sql);
for (int i = 0; i < obj.length; i++) {
ps.setObject(i + 1, obj[i]);
}
rs = ps.executeQuery();
//new 一个空的list集合用来存放查询结果
List<Map<String, Object>> list = new ArrayList<>();
//获取结果集的列数
int count = rs.getMetaData().getColumnCount();
//对结果集遍历每一条数据是一个Map集合,列是k,值是v
while (rs.next()) {
//一个空的map集合,用来存放每一行数据
Map<String, Object> map = new HashMap<String, Object>();
for (int i = 0; i < count; i++) {
Object ob = rs.getObject(i + 1);//获取值
String key = rs.getMetaData().getColumnName(i + 1);//获取k即列名
map.put(key, ob);
}
list.add(map);
}
return list;
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
close(rs, ps, con);
}
return null;
}
}
调度主类:
package elasticjob;
import com.dangdang.ddframe.job.config.JobCoreConfiguration;
import com.dangdang.ddframe.job.config.simple.SimpleJobConfiguration;
import com.dangdang.ddframe.job.lite.api.JobScheduler;
import com.dangdang.ddframe.job.lite.config.LiteJobConfiguration;
import com.dangdang.ddframe.job.reg.base.CoordinatorRegistryCenter;
import com.dangdang.ddframe.job.reg.zookeeper.ZookeeperConfiguration;
import com.dangdang.ddframe.job.reg.zookeeper.ZookeeperRegistryCenter;
public class ElasticJobMain {
public static void main(String[] args) {
// 配置分布式协调服务(注册中心)Zookeeper
ZookeeperConfiguration zookeeperConfiguration = new ZookeeperConfiguration("localhost:2181","data-archive-job");
CoordinatorRegistryCenter coordinatorRegistryCenter = new ZookeeperRegistryCenter(zookeeperConfiguration);
coordinatorRegistryCenter.init();
// 配置任务(时间事件、定时任务业务逻辑、调度器)
// 3 为分片数量
JobCoreConfiguration jobCoreConfiguration = JobCoreConfiguration
.newBuilder("archive-job", "*/2 * * * * ?", 3)
.shardingItemParameters("0=bachelor,1=master,2=doctor").build();
SimpleJobConfiguration simpleJobConfiguration = new SimpleJobConfiguration(jobCoreConfiguration,ArchivieJob.class.getName());
JobScheduler jobScheduler = new JobScheduler(coordinatorRegistryCenter, LiteJobConfiguration.newBuilder(simpleJobConfiguration).overwrite(true).build());
jobScheduler.init();
}
}