2021-10-30

最新推荐文章于 2024-10-02 15:19:42 发布

Pursue?????

最新推荐文章于 2024-10-02 15:19:42 发布

阅读量107

点赞数

文章标签： jar spring boot java

本文链接：https://blog.csdn.net/weixin_45884188/article/details/121055817

版权

本文档展示了如何在SpringBoot项目中整合Kettle进行大数据处理，包括需要导入的jar包配置，以及通过Kettle进行双表数据写入更新的作业实现。主要涉及了数据库连接、表输入步骤、更新表字段的步骤创建和转换执行等关键操作。

摘要由CSDN通过智能技术生成

springboot整合kettle

1、需要整合的jar包，部分需要手动导入

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <parent>
        <artifactId>cim</artifactId>
        <groupId>org.example</groupId>
        <version>1.0-SNAPSHOT</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

    <artifactId>kettle</artifactId>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
        </dependency>
        <!-- 集成kettle -->
        <dependency>
            <groupId>pentaho-kettle</groupId>
            <artifactId>kettle-core</artifactId>
            <scope>system</scope>
            <version>8.2.0.0-342</version>
            <systemPath>${project.basedir}/lib/kettle-core-8.2.0.0-342.jar</systemPath>
        </dependency>
        <dependency>
            <groupId>pentaho-kettle</groupId>
            <artifactId>kettle-engine</artifactId>
            <scope>system</scope>
            <version>8.2.0.0-342</version>
            <systemPath>${project.basedir}/lib/kettle-engine-8.2.0.0-342.jar</systemPath>
        </dependency>
        <dependency>
            <groupId>pentaho-kettle</groupId>
            <artifactId>metastore</artifactId>
            <version>8.2.0.0-342</version>
            <scope>system</scope>
            <systemPath>${project.basedir}/lib/metastore-8.2.0.0-342.jar</systemPath>
        </dependency>
       <!-- <dependency>
            <groupId>org.mozilla</groupId>
            <artifactId>javascript</artifactId>
            <version>1.7.2</version>
        </dependency>-->
        <!-- 以上四个需要自己手动添加 -->
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-vfs2</artifactId>
            <version>2.1</version>
        </dependency>
        <dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
            <version>27.1-jre</version>
        </dependency>
        <dependency>
            <groupId>commons-io</groupId>
            <artifactId>commons-io</artifactId>
            <version>2.2</version>
        </dependency>
        <dependency>
            <groupId>commons-lang</groupId>
            <artifactId>commons-lang</artifactId>
            <version>2.6</version>
        </dependency>
        <dependency>
            <groupId>cn.hutool</groupId>
            <artifactId>hutool-all</artifactId>
            <version>4.5.6</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/commons-codec/commons-codec -->
        <dependency>
            <groupId>commons-codec</groupId>
            <artifactId>commons-codec</artifactId>
            <version>1.14</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.scannotation/scannotation -->
        <dependency>
            <groupId>org.scannotation</groupId>
            <artifactId>scannotation</artifactId>
            <version>1.0.3</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/log4j/log4j -->
        <dependency>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
            <version>1.2.17</version>
        </dependency>
    </dependencies>
</project>

2、表数据转换作业双表数据写入更新操作

import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransHopMeta;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.steps.insertupdate.InsertUpdateMeta;
import org.pentaho.di.trans.steps.tableinput.TableInputMeta;
import org.springframework.stereotype.Component;

import java.util.Map;

/**
 * 大数据操作工具类
 */
@Component
public class KettleUtils {


    /**
     * @Author Hao
     * 转换作业 双数据库操作
     * 创建数据库连接
     * @param con1
     * @param con2
     * @return
     */
    public TransMeta createCon(Map<String,String> con1,Map<String,String> con2,String name){
        TransMeta transMeta = new TransMeta();
        // 设置转换名称
        transMeta.setName(name);
        // 添加数据库转换连接 1
        transMeta.addDatabase(new DatabaseMeta(con1.get("conName"),con1.get("conType"),con1.get("conWay"),con1.get("hostName"),
                con1.get("databaseName"),con1.get("port"),con1.get("username"),con1.get("password")));
        // 添加数据库转换连接 2
        transMeta.addDatabase(new DatabaseMeta(con2.get("conName"),con2.get("conType"),con2.get("conWay"),con2.get("hostName"),
                con2.get("databaseName"),con2.get("port"),con2.get("username"),con2.get("password")));
        return transMeta;
    }


    /**
     * 输入步骤 （输入: ->表输入）
     * @Author Hao
     * @return
     */
    public StepMeta tableInputStepMeta(TransMeta transMeta,String databaseConName,String handleStatement,String stepMetaName){
        TableInputMeta tableInputMeta = new TableInputMeta();
        // 设置数据库连接绑定
        tableInputMeta.setDatabaseMeta(transMeta.findDatabase(databaseConName));
        // 设置数据库操作语句
        tableInputMeta.setSQL(handleStatement);
        // 设置表输入步骤名称
        return new StepMeta(stepMetaName,tableInputMeta);
    }


    /**
     * @Author Hao
     * 输出步骤 （输出 ->更新表字段）
     * @param transMeta
     * @return
     */
    public StepMeta stepMetaInsUp(TransMeta transMeta,String databaseName,String tableName,
                                  Map<String,String[]>stringMap,Boolean[] isUpdate,String stepName){
        // 创建插入 更新步骤
        InsertUpdateMeta insertUpdateMeta = new InsertUpdateMeta();
        // 设置更新表 数据库绑定
        insertUpdateMeta.setDatabaseMeta(transMeta.findDatabase(databaseName));
        // 设置表绑定
        insertUpdateMeta.setTableName(tableName);
        // 设置查询关键字
        insertUpdateMeta.setKeyLookup(stringMap.get("keyLookup"));
        insertUpdateMeta.setKeyCondition(stringMap.get("condition"));
        insertUpdateMeta.setKeyStream(stringMap.get("keyStream"));
        insertUpdateMeta.setKeyStream2(stringMap.get("keyStream2"));

        /**
         * insertUpdateMeta.setKeyLookup(new String[] {"id2"});
         * insertUpdateMeta.setKeyCondition(new String[] {"="});
         * insertUpdateMeta.setKeyStream(new String[] {"id2"});
         * insertUpdateMeta.setKeyStream2(new String[] {""});// 一定要加上
         * // 设置要更新的字段
         * String[] updatelookup = {"id2", "name2"};
         * String[] updateStream = {"id2", "name2"};
         * Boolean[] updateOrNot = {false, true};
         * */        
        
        // 设置需要更新的字段
        insertUpdateMeta.setUpdateLookup(stringMap.get("lookup"));
        // 设置流字段
        insertUpdateMeta.setUpdateStream(stringMap.get("streams"));
        // 设置是否更新
        insertUpdateMeta.setUpdate(isUpdate);
        // 设置步骤2 的名称
        return new StepMeta(stepName,insertUpdateMeta);
    }

    /**
     * @Author Hao
     * 绑定两个连接点 步骤绑定 创建连接点
     * @param stepMeta1
     * @param stepMate2
     * @return
     */
    public TransHopMeta createHopMeta(StepMeta stepMeta1,StepMeta stepMate2){
        return new TransHopMeta(stepMeta1,stepMate2);
    }

    /**
     * @Author Hao
     * 执行运行转换
     * @param transMeta
     */
    public void runTrans(TransMeta transMeta) throws KettleException {
        Trans trans = new Trans(transMeta);
        trans.execute(null);  //执行转换
        trans.waitUntilFinished(); //等待转换结束
    }
}