Elastic-Job分布式任务调度学习笔记3-springboot开发分布任务

作业分片

分片概念：指任务的分布式执行，需要将一个任务拆分为多个独立的任务项，然后由分布式的应用实例分别执行某一个或几个分片项。

创建数据库

CREATE DATABASE `elastic_job_demo` CHARACTER SET 'utf8' COLLATE 'utf8_general_ci';

创建表

DROP TABLE IF EXISTS `t_file`;
CREATE TABLE `t_file` (
`id` varchar(11) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL,
`name` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
`type` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
`content` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
`backedUp` tinyint(1) NULL DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic;

创建maven项目，引入依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>springboot_lastic</artifactId>
    <version>1.0-SNAPSHOT</version>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.1.5.RELEASE</version>
    </parent>

    <properties>
        <project.build.sourceEncoding>UTF‐8</project.build.sourceEncoding>
        <project.reporting.outputEncoding>UTF‐8</project.reporting.outputEncoding>
        <java.version>1.8</java.version>
    </properties>

    <dependencies>
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-web</artifactId>
    </dependency>
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-test</artifactId>
    </dependency>
    <dependency>
        <groupId>com.dangdang</groupId>
        <artifactId>elastic-job-lite-spring</artifactId>
        <version>2.1.5</version>
    </dependency>
    <dependency>
        <groupId>org.projectlombok</groupId>
        <artifactId>lombok</artifactId>
    </dependency>
        <!---->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-jdbc</artifactId>
    </dependency>
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-test</artifactId>
    </dependency>
    <dependency>
        <groupId>mysql</groupId>
        <artifactId>mysql-connector-java</artifactId>
        <version>5.1.48</version>
    </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-jdbc</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-jdbc</artifactId>
        </dependency>
    </dependencies>
    <build>
    <finalName>${project.name}</finalName>
    <resources>
        <resource>
            <directory>src/main/resources</directory>
            <filtering>true</filtering>
            <includes>
                <include>**/*</include>
            </includes>
        </resource>
        <resource>
            <directory>src/main/java</directory>
            <includes>
                <include>**/*.xml</include>
            </includes>
        </resource>
    </resources>
    <plugins>
        <plugin>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-maven-plugin</artifactId>
        </plugin>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-compiler-plugin</artifactId>
            <configuration>
                <source>1.8</source>
                <target>1.8</target>
            </configuration>
        </plugin>
        <plugin>
            <artifactId>maven-resources-plugin</artifactId>
            <configuration>
                <encoding>utf‐8</encoding>
                <useDefaultDelimiters>true</useDefaultDelimiters>
            </configuration>
        </plugin>
    </plugins>

    </build>

</project>

编写spring boot配置文件及启动类

springboot配置文件

server.port=56801
spring.application.name = task-scheduling-springboot
logging.level.root = info

#数据源定义
spring.datasource.driver-class-name = com.mysql.jdbc.Driver
spring.datasource.url = jdbc:mysql://172.xx.1.120:3306/elastic_job_demo?useUnicode=true
spring.datasource.username=root
spring.datasource.password=xxx

spring boot 启动类：

@SpringBootApplication
public class SchedulingBootstarap {

    public static void main(String[] args) {
        SpringApplication.run(SchedulingBootstarap.class,args);

    }

}

文件实体类

@Data
public class FileCustom {

    private String id;
    /**
     * 文件名
     */
    private String name;
    /**
     * 文件类型，如text、image、radio、vedio
     */
    private String type;
    /**
     * 文件内容
     */
    private String content;
    
    /**
     * 是否已备份
     */

    private Boolean backedUp = false;
    public FileCustom(String id, String name, String type, String content){
        this.id = id;
        this.name = name;
        this.type = type;
        this.content = content;
    }

    public FileCustom() {

    }
}

编写测试类，生成表数据

@SpringBootTest
@RunWith(SpringRunner.class)
public class test {

    @Autowired
    JdbcTemplate jdbcTemplate;

    @Test
    public void testGenerateData(){
        clearTestFiles();

        generateTestFiles();
    }

    /**
     * 清除模拟数据
     */
    public void clearTestFiles(){
        jdbcTemplate.update("delete from t_file");

    }

    /**
     * 创建模拟数据
     */
    public void generateTestFiles(){
        List<FileCustom> files =new ArrayList<>();
        for(int i=1;i<11;i++){
            files.add(new FileCustom(String.valueOf(i),"文件"+ i,"text","content"+ i));
            files.add(new FileCustom(String.valueOf((i+10)),"文件"+(i+10),"image","content"+
                    (i+10)));
            files.add(new FileCustom(String.valueOf((i+20)),"文件"+(i+20),"radio","content"+
                    (i+20)));
            files.add(new FileCustom(String.valueOf((i+30)),"文件"+(i+30),"vedio","content"+
                    (i+30)));
        }
        for(FileCustom file : files){
            jdbcTemplate.update("insert into t_file (id,name,type,content,backedUp) values (?,?,?,?,?)",
                    new Object[]{file.getId(),file.getName(),file.getType(),file.getContent(),file.getBackedUp()});
        }
    }
}

编写服务类

@Service
public class FileService {

    @Autowired
    private JdbcTemplate jdbcTemplate;

    /**
     * 获取某文件类型未备份文件
     * @param fileType 文件类型
     * @param count 获取条数
     * @return
     */
    public List<FileCustom> fetchUnBackupFiles(String fileType, Integer count) {
        List<FileCustom> files = jdbcTemplate.query("select * from t_file t where t.type = ? and t.backedUp = 0 order by id limit 0,? "
                , new Object[]{fileType, count}, new BeanPropertyRowMapper<>(FileCustom.class));

        return files;
    }

    /**
     * 备份文件
     * @param files 要备份的文件
     */
    public void backupFiles(List<FileCustom> files) {

        for (FileCustom file : files) {
            jdbcTemplate.update("update t_file set backedUp = 1 where id = ?",new Object[]{file.getId()});
            System.out.println(String.format("线程%d | 已备份文件：%s  文件类型：%s" ,
                    Thread.currentThread().getId(),
                    file.getName(),
                    file.getType()));
        }
    }
}

编写elastic-job任务类

/**
 * 文件备份任务类
 * 每次任务执行时获取一定数目的文件 进行备份处理
 */
@Component
public class FileBackupJob implements SimpleJob {

    @Autowired
    FileService fileService;

    //每次执行任务要备份的文件数量
    private final int FETCH_SIZE = 1;

    //文件列表（模拟
    public static List<FileCustom> files = new ArrayList<FileCustom>();

    /**
     * 任务调度执行方法
     * @param shardingContext
     */
    @Override
    public void execute(ShardingContext shardingContext) {
      /*  //作业切片信息
        int shardingItem = shardingContext.getShardingItem();
        System.out.println(String.format("作业分片：%d",shardingItem));
        //获取未备份的文件
        List<FileCustom> fileCustoms = fetchUnBackupFiles(FETCH_SIZE);
        //备份文件
        backupFiles(fileCustoms);*/

        //获取未备份文件
        List<FileCustom> unBackupFiles = fileService.fetchUnBackupFiles(shardingContext.getShardingParameter(),FETCH_SIZE);
        System.out.println(String.format("Time:%s | 线程 %d | 分片 %s | 已获取文件数据 %d 条",
                new SimpleDateFormat("HH:mm:ss").format(new Date()),
                Thread.currentThread().getId(),
                shardingContext.getShardingParameter(),
                unBackupFiles.size()));

        //执行文件备份
        fileService.backupFiles(unBackupFiles);

    }

    //获取未备份的文件
    public List<FileCustom> fetchUnBackupFiles(int count){
       List<FileCustom> fetchList = new ArrayList<>();
       int num = 0;
       for(FileCustom fileCustom : files){
           if (num >= count) {
               break;
           }
           //未备份的文件则放入列表
           if (!fileCustom.getBackedUp()){
               fetchList.add(fileCustom);
               num++;
           }
       }
       //ManagementFactory.getRuntimeMXBean获取当前jvm进程的pid
        System.out.println(String.format("%sTime:%s,已获取%d文件",
                ManagementFactory.getRuntimeMXBean().getName(),new SimpleDateFormat("hh:mm:ss").format(new
                        Date()),num));

        return  fetchList;
    }

    /**
     * 备份文件
     * @param files
     */
    public void backupFiles(List<FileCustom> files){

        for (FileCustom file : files) {
            //标记文件数据已备份
            file.setBackedUp(Boolean.TRUE);
            System.out.println(String.format("已备份文件:%s 文件类型:%s",file.getName(),file.getType()));
        }

    }

}

编写Elastic-Job配置类及任务类
Zookeeper配置类：

/**
 * zookeeper配置类
 */
@Configuration
public class ElasticJobRegistryCenterConfig {
    private String registerServerList = "localhost:2181";
    private String registerNameSpace = "elastic-job-example-springboot";

    /**
     * 配置zookeeper
     * @return
     */
    @Bean(initMethod = "init")
    public CoordinatorRegistryCenter createRegistryCenter(){
        System.out.println("进入到zk配置类中");
        ZookeeperConfiguration zkConfig = new ZookeeperConfiguration(registerServerList, registerNameSpace);
        //设置超时时间
        zkConfig.setSessionTimeoutMilliseconds(1000);
        ZookeeperRegistryCenter registryCenter = new ZookeeperRegistryCenter(zkConfig);
        System.out.println("返回了registryCenter="+registryCenter);
        return registryCenter;
    }
}

Elastic-Job配置类：

@Configuration
public class ElasticJobConfig {

    @Autowired
    private CoordinatorRegistryCenter registryCenter;

    @Autowired
    private FileBackupJob fileBackupJob;

    @Autowired
    private DataSource dataSource;

    /**
     * 配置任务详细信息
     * @param jobClass 任务执行类
     * @param corn 执行策略
     * @param shardingTotalCount 分片数量
     * @param shardingItemParams 分配个性化参数
     * @return
     */
    private LiteJobConfiguration createJobConfiguration(final Class<? extends SimpleJob> jobClass,
                                                        final String corn,
                                                        final int shardingTotalCount,
                                                        final String shardingItemParams){
        System.out.println("进入方法createJobConfiguration中。。。");


        //定义作业核心配置
        JobCoreConfiguration.Builder builder = JobCoreConfiguration.newBuilder(jobClass.getName(), corn, shardingTotalCount);
        if (!StringUtils.isEmpty(shardingItemParams)) {
            builder.shardingItemParameters(shardingItemParams);
        }
        JobCoreConfiguration jobCoreConfiguration = builder.build();
        //定义simple类型配置
        SimpleJobConfiguration simpleJobConfig = new SimpleJobConfiguration(jobCoreConfiguration, jobClass.getCanonicalName());
        //generateTestFiles();
        //定义lite作业根配置
        //JobRootConfiguration jobConfig = LiteJobConfiguration.newBuilder(simpleJobConfig).overwrite(true).build();
        JobRootConfiguration jobConfig = LiteJobConfiguration.newBuilder(simpleJobConfig).monitorPort(9888).build();

        return (LiteJobConfiguration) jobConfig;


    }
    /**
     * 生成测试文件
     */
    private static void generateTestFiles(){
        for (int i=1; i<11; i++) {
            FileBackupJob.files.add(new FileCustom(String.valueOf(i+10),"文件"+
                    (i+10),"text","content"+ (i+10)));
            FileBackupJob.files.add(new FileCustom(String.valueOf(i+20),"文件"+
                    (i+20),"image","content"+ (i+20)));
            FileBackupJob.files.add(new FileCustom(String.valueOf(i+30),"文件"+
                    (i+30),"radio","content"+ (i+30)));
            FileBackupJob.files.add(new FileCustom(String.valueOf(i+40),"文件"+
                    (i+40),"vedio","content"+ (i+40)));
        }

    }
    /**
     * 任务启动
     */
    @Bean(initMethod = "init")
    public SpringJobScheduler initSimpleElasticJob(){
       /* System.out.println("进入elastic-job配置类中");

        SimpleJob job1 = this.fileBackupJob;
        SpringJobScheduler jobScheduler = new SpringJobScheduler(job1,
                registryCenter,
                createJobConfiguration(job1.getClass(),"0/3 * * * * ?",1,null));*/
        SimpleJob job1 = this.fileBackupJob;
        //定义Lite作业根配置
        JobEventRdbConfiguration jobEventConfig = new JobEventRdbConfiguration(dataSource);//增加任务事件追踪配置
        SpringJobScheduler jobScheduler = new SpringJobScheduler(job1, registryCenter,
                createJobConfiguration(job1.getClass(), "0/10 * * * * ?",
                        4, "0=text,1=image,2=radio,3=vedio")
                , jobEventConfig);

        return jobScheduler;
    }

}