作业分片
分片概念:指任务的分布式执行,需要将一个任务拆分为多个独立的任务项,然后由分布式的应用实例分别执行某一个或几个分片项。
创建数据库
CREATE DATABASE `elastic_job_demo` CHARACTER SET 'utf8' COLLATE 'utf8_general_ci';
创建表
DROP TABLE IF EXISTS `t_file`;
CREATE TABLE `t_file` (
`id` varchar(11) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL,
`name` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
`type` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
`content` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
`backedUp` tinyint(1) NULL DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic;
创建maven项目,引入依赖
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.example</groupId>
<artifactId>springboot_lastic</artifactId>
<version>1.0-SNAPSHOT</version>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.1.5.RELEASE</version>
</parent>
<properties>
<project.build.sourceEncoding>UTF‐8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF‐8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
</dependency>
<dependency>
<groupId>com.dangdang</groupId>
<artifactId>elastic-job-lite-spring</artifactId>
<version>2.1.5</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<!---->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.48</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
</dependencies>
<build>
<finalName>${project.name}</finalName>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
<includes>
<include>**/*</include>
</includes>
</resource>
<resource>
<directory>src/main/java</directory>
<includes>
<include>**/*.xml</include>
</includes>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<configuration>
<encoding>utf‐8</encoding>
<useDefaultDelimiters>true</useDefaultDelimiters>
</configuration>
</plugin>
</plugins>
</build>
</project>
编写spring boot配置文件及启动类
springboot配置文件
server.port=56801
spring.application.name = task-scheduling-springboot
logging.level.root = info
#数据源定义
spring.datasource.driver-class-name = com.mysql.jdbc.Driver
spring.datasource.url = jdbc:mysql://172.xx.1.120:3306/elastic_job_demo?useUnicode=true
spring.datasource.username=root
spring.datasource.password=xxx
spring boot 启动类:
@SpringBootApplication
public class SchedulingBootstarap {
public static void main(String[] args) {
SpringApplication.run(SchedulingBootstarap.class,args);
}
}
文件实体类
@Data
public class FileCustom {
private String id;
/**
* 文件名
*/
private String name;
/**
* 文件类型,如text、image、radio、vedio
*/
private String type;
/**
* 文件内容
*/
private String content;
/**
* 是否已备份
*/
private Boolean backedUp = false;
public FileCustom(String id, String name, String type, String content){
this.id = id;
this.name = name;
this.type = type;
this.content = content;
}
public FileCustom() {
}
}
编写测试类,生成表数据
@SpringBootTest
@RunWith(SpringRunner.class)
public class test {
@Autowired
JdbcTemplate jdbcTemplate;
@Test
public void testGenerateData(){
clearTestFiles();
generateTestFiles();
}
/**
* 清除模拟数据
*/
public void clearTestFiles(){
jdbcTemplate.update("delete from t_file");
}
/**
* 创建模拟数据
*/
public void generateTestFiles(){
List<FileCustom> files =new ArrayList<>();
for(int i=1;i<11;i++){
files.add(new FileCustom(String.valueOf(i),"文件"+ i,"text","content"+ i));
files.add(new FileCustom(String.valueOf((i+10)),"文件"+(i+10),"image","content"+
(i+10)));
files.add(new FileCustom(String.valueOf((i+20)),"文件"+(i+20),"radio","content"+
(i+20)));
files.add(new FileCustom(String.valueOf((i+30)),"文件"+(i+30),"vedio","content"+
(i+30)));
}
for(FileCustom file : files){
jdbcTemplate.update("insert into t_file (id,name,type,content,backedUp) values (?,?,?,?,?)",
new Object[]{file.getId(),file.getName(),file.getType(),file.getContent(),file.getBackedUp()});
}
}
}
编写服务类
@Service
public class FileService {
@Autowired
private JdbcTemplate jdbcTemplate;
/**
* 获取某文件类型未备份文件
* @param fileType 文件类型
* @param count 获取条数
* @return
*/
public List<FileCustom> fetchUnBackupFiles(String fileType, Integer count) {
List<FileCustom> files = jdbcTemplate.query("select * from t_file t where t.type = ? and t.backedUp = 0 order by id limit 0,? "
, new Object[]{fileType, count}, new BeanPropertyRowMapper<>(FileCustom.class));
return files;
}
/**
* 备份文件
* @param files 要备份的文件
*/
public void backupFiles(List<FileCustom> files) {
for (FileCustom file : files) {
jdbcTemplate.update("update t_file set backedUp = 1 where id = ?",new Object[]{file.getId()});
System.out.println(String.format("线程%d | 已备份文件:%s 文件类型:%s" ,
Thread.currentThread().getId(),
file.getName(),
file.getType()));
}
}
}
编写elastic-job任务类
/**
* 文件备份任务类
* 每次任务执行时获取一定数目的文件 进行备份处理
*/
@Component
public class FileBackupJob implements SimpleJob {
@Autowired
FileService fileService;
//每次执行任务要备份的文件数量
private final int FETCH_SIZE = 1;
//文件列表(模拟
public static List<FileCustom> files = new ArrayList<FileCustom>();
/**
* 任务调度执行方法
* @param shardingContext
*/
@Override
public void execute(ShardingContext shardingContext) {
/* //作业切片信息
int shardingItem = shardingContext.getShardingItem();
System.out.println(String.format("作业分片:%d",shardingItem));
//获取未备份的文件
List<FileCustom> fileCustoms = fetchUnBackupFiles(FETCH_SIZE);
//备份文件
backupFiles(fileCustoms);*/
//获取未备份文件
List<FileCustom> unBackupFiles = fileService.fetchUnBackupFiles(shardingContext.getShardingParameter(),FETCH_SIZE);
System.out.println(String.format("Time:%s | 线程 %d | 分片 %s | 已获取文件数据 %d 条",
new SimpleDateFormat("HH:mm:ss").format(new Date()),
Thread.currentThread().getId(),
shardingContext.getShardingParameter(),
unBackupFiles.size()));
//执行文件备份
fileService.backupFiles(unBackupFiles);
}
//获取未备份的文件
public List<FileCustom> fetchUnBackupFiles(int count){
List<FileCustom> fetchList = new ArrayList<>();
int num = 0;
for(FileCustom fileCustom : files){
if (num >= count) {
break;
}
//未备份的文件则放入列表
if (!fileCustom.getBackedUp()){
fetchList.add(fileCustom);
num++;
}
}
//ManagementFactory.getRuntimeMXBean获取当前jvm进程的pid
System.out.println(String.format("%sTime:%s,已获取%d文件",
ManagementFactory.getRuntimeMXBean().getName(),new SimpleDateFormat("hh:mm:ss").format(new
Date()),num));
return fetchList;
}
/**
* 备份文件
* @param files
*/
public void backupFiles(List<FileCustom> files){
for (FileCustom file : files) {
//标记文件数据已备份
file.setBackedUp(Boolean.TRUE);
System.out.println(String.format("已备份文件:%s 文件类型:%s",file.getName(),file.getType()));
}
}
}
编写Elastic-Job配置类及任务类
Zookeeper配置类:
/**
* zookeeper配置类
*/
@Configuration
public class ElasticJobRegistryCenterConfig {
private String registerServerList = "localhost:2181";
private String registerNameSpace = "elastic-job-example-springboot";
/**
* 配置zookeeper
* @return
*/
@Bean(initMethod = "init")
public CoordinatorRegistryCenter createRegistryCenter(){
System.out.println("进入到zk配置类中");
ZookeeperConfiguration zkConfig = new ZookeeperConfiguration(registerServerList, registerNameSpace);
//设置超时时间
zkConfig.setSessionTimeoutMilliseconds(1000);
ZookeeperRegistryCenter registryCenter = new ZookeeperRegistryCenter(zkConfig);
System.out.println("返回了registryCenter="+registryCenter);
return registryCenter;
}
}
Elastic-Job配置类:
@Configuration
public class ElasticJobConfig {
@Autowired
private CoordinatorRegistryCenter registryCenter;
@Autowired
private FileBackupJob fileBackupJob;
@Autowired
private DataSource dataSource;
/**
* 配置任务详细信息
* @param jobClass 任务执行类
* @param corn 执行策略
* @param shardingTotalCount 分片数量
* @param shardingItemParams 分配个性化参数
* @return
*/
private LiteJobConfiguration createJobConfiguration(final Class<? extends SimpleJob> jobClass,
final String corn,
final int shardingTotalCount,
final String shardingItemParams){
System.out.println("进入方法createJobConfiguration中。。。");
//定义作业核心配置
JobCoreConfiguration.Builder builder = JobCoreConfiguration.newBuilder(jobClass.getName(), corn, shardingTotalCount);
if (!StringUtils.isEmpty(shardingItemParams)) {
builder.shardingItemParameters(shardingItemParams);
}
JobCoreConfiguration jobCoreConfiguration = builder.build();
//定义simple类型配置
SimpleJobConfiguration simpleJobConfig = new SimpleJobConfiguration(jobCoreConfiguration, jobClass.getCanonicalName());
//generateTestFiles();
//定义lite作业根配置
//JobRootConfiguration jobConfig = LiteJobConfiguration.newBuilder(simpleJobConfig).overwrite(true).build();
JobRootConfiguration jobConfig = LiteJobConfiguration.newBuilder(simpleJobConfig).monitorPort(9888).build();
return (LiteJobConfiguration) jobConfig;
}
/**
* 生成测试文件
*/
private static void generateTestFiles(){
for (int i=1; i<11; i++) {
FileBackupJob.files.add(new FileCustom(String.valueOf(i+10),"文件"+
(i+10),"text","content"+ (i+10)));
FileBackupJob.files.add(new FileCustom(String.valueOf(i+20),"文件"+
(i+20),"image","content"+ (i+20)));
FileBackupJob.files.add(new FileCustom(String.valueOf(i+30),"文件"+
(i+30),"radio","content"+ (i+30)));
FileBackupJob.files.add(new FileCustom(String.valueOf(i+40),"文件"+
(i+40),"vedio","content"+ (i+40)));
}
}
/**
* 任务启动
*/
@Bean(initMethod = "init")
public SpringJobScheduler initSimpleElasticJob(){
/* System.out.println("进入elastic-job配置类中");
SimpleJob job1 = this.fileBackupJob;
SpringJobScheduler jobScheduler = new SpringJobScheduler(job1,
registryCenter,
createJobConfiguration(job1.getClass(),"0/3 * * * * ?",1,null));*/
SimpleJob job1 = this.fileBackupJob;
//定义Lite作业根配置
JobEventRdbConfiguration jobEventConfig = new JobEventRdbConfiguration(dataSource);//增加任务事件追踪配置
SpringJobScheduler jobScheduler = new SpringJobScheduler(job1, registryCenter,
createJobConfiguration(job1.getClass(), "0/10 * * * * ?",
4, "0=text,1=image,2=radio,3=vedio")
, jobEventConfig);
return jobScheduler;
}
}