Nifi开发之自动生成DataFolw

1.官方文档，封装Api

官方文档: Apache NiFi Documentation.

<!-- https://mvnrepository.com/artifact/org.apache.nifi/nifi-api -->
<dependency>
    <groupId>org.apache.nifi</groupId>
    <artifactId>nifi-api</artifactId>
    <version>1.15.0</version>
    <scope>provided</scope>
</dependency>

1.1封装taken

首先，F12找到请求的方法

在官方文档中找到对应的部分

封装部分

OkHttp3的封装,下面创建OkHttpClient需要用到：OkHttp3封装.

import java.io.IOException;
import java.util.Objects;
import okhttp3.MediaType;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import okhttp3.ResponseBody;
import okhttp3.FormBody.Builder;
import org.apache.commons.lang3.StringUtils;

public class AccessAPI {
    private static final String API_ACCESS_TOKEN = "/access/token";

    public AccessAPI() {
    }

    public String accessToken(String url, String username, String password) throws IOException {
        if (!StringUtils.isBlank(url) && !StringUtils.isBlank(username) && !StringUtils.isBlank(password)) {
            OkHttpClient okHttpClient = OKHttpConfiguration.getOkHttpClient();
            RequestBody formBody = (new Builder()).add("username", username).add("password", password).build();
            Request request = (new okhttp3.Request.Builder()).url(url + "/access/token").header("Content-Type", "application/x-www-form-urlencoded").post(formBody).build();
            Response response = okHttpClient.newCall(request).execute();
            Throwable var8 = null;

            String var9;
            try {
                if (!response.isSuccessful()) {
                    throw new IOException("Unexpected code " + response + "Reason is: " + (response.body() != null ? response.body().string() : null));
                }

                var9 = ((ResponseBody)Objects.requireNonNull(response.body())).string();
            } catch (Throwable var18) {
                var8 = var18;
                throw var18;
            } finally {
                if (response != null) {
                    if (var8 != null) {
                        try {
                            response.close();
                        } catch (Throwable var17) {
                            var8.addSuppressed(var17);
                        }
                    } else {
                        response.close();
                    }
                }

            }

            return var9;
        } else {
            throw new RuntimeException("url or username or password must not be null");
        }
    }
}

每一个方法执行，都要用到token，从配置文件拿到url，username，password

@Component
public class ETLServer {

    @Value("${nifi.url}")
    private String url;
    @Value("${nifi.user}")
    private String username;
    @Value("${nifi.password}")
    private String password;

    @Autowired
    private AccessAPI accessAPI;

    private static final Logger LOGGER = LoggerFactory.getLogger(ETLServer.class);

    public CommonEntity getCommonEntity() {

        LOGGER.info("begin login server URL：{}，username:{}", url, username);

        String token = null;
        try {
            token = accessAPI.accessToken(url, username, password);
        } catch (IOException e) {
            e.printStackTrace();
        }
        CommonEntity commonEntity = new CommonEntity();
        commonEntity.setUrl(url);
        commonEntity.setAccessToken(token);
        return commonEntity;
    }

}

@Data
public class CommonEntity {
    @NotEmpty(message = "url must not be empty")
    private String url;
    @NotEmpty(message = "accessToken must not be empty")
    private String accessToken;

1.2拉取模板

找到对应的官方Api，或者去画布上操作，F12可以找到每个动作调用的方法
没想到里面又引入了另一个类

查看apach提供的代码，只需要四个属性，
originX，originY，templateId，disconnectedNodeAcknowledged。

package org.apache.nifi.web.api.entity;

import io.swagger.annotations.ApiModelProperty;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.nifi.web.api.dto.FlowSnippetDTO;

@XmlRootElement(
    name = "instantiateTemplateRequestEntity"
)
public class InstantiateTemplateRequestEntity extends Entity {
    private Double originX;
    private Double originY;
    private String templateId;
    private String encodingVersion;
    private FlowSnippetDTO snippet;
    private Boolean disconnectedNodeAcknowledged;

    public InstantiateTemplateRequestEntity() {
    }

    @ApiModelProperty("The identifier of the template.")
    public String getTemplateId() {
        return this.templateId;
    }

    public void setTemplateId(String templateId) {
        this.templateId = templateId;
    }

    @ApiModelProperty("The x coordinate of the origin of the bounding box where the new components will be placed.")
    public Double getOriginX() {
        return this.originX;
    }

    public void setOriginX(Double originX) {
        this.originX = originX;
    }

    @ApiModelProperty("The y coordinate of the origin of the bounding box where the new components will be placed.")
    public Double getOriginY() {
        return this.originY;
    }

    public void setOriginY(Double originY) {
        this.originY = originY;
    }

    @ApiModelProperty("The encoding version of the flow snippet. If not specified, this is automatically populated by the node receiving the user request. If the snippet is specified, the version will be the latest. If the snippet is not specified, the version will come from the underlying template. These details need to be replicated throughout the cluster to ensure consistency.")
    public String getEncodingVersion() {
        return this.encodingVersion;
    }

    public void setEncodingVersion(String encodingVersion) {
        this.encodingVersion = encodingVersion;
    }

    @ApiModelProperty("A flow snippet of the template contents. If not specified, this is automatically populated by the node receiving the user request. These details need to be replicated throughout the cluster to ensure consistency.")
    public FlowSnippetDTO getSnippet() {
        return this.snippet;
    }

    public void setSnippet(FlowSnippetDTO snippet) {
        this.snippet = snippet;
    }

    @ApiModelProperty("Acknowledges that this node is disconnected to allow for mutable requests to proceed.")
    public Boolean isDisconnectedNodeAcknowledged() {
        return this.disconnectedNodeAcknowledged;
    }

    public void setDisconnectedNodeAcknowledged(Boolean disconnectedNodeAcknowledged) {
        this.disconnectedNodeAcknowledged = disconnectedNodeAcknowledged;
    }
}

开始干活啦

public class BuildParamUtil {

    public static InstantiateTemplateRequestEntity buildInstantiateTemplateRequestEntity(String templateId, Double originX, Double originY, Boolean disconnectedNodeAcknowledged) {
        InstantiateTemplateRequestEntity instantiateTemplateRequestEntity = new InstantiateTemplateRequestEntity();
        instantiateTemplateRequestEntity.setTemplateId(templateId);
        instantiateTemplateRequestEntity.setOriginX(originX);
        instantiateTemplateRequestEntity.setOriginY(originY);
        instantiateTemplateRequestEntity.setDisconnectedNodeAcknowledged(disconnectedNodeAcknowledged);
        return instantiateTemplateRequestEntity;
    }
}

然后封装instance Template这个方法

@Component
public class AtomicStrategy implements TemplateStrategy {
    @Autowired
    private ProcessGroupAPI processGroupAPI;
    @Autowired
    private ETLServer etlServer;

    public TemplateDTO getTemplateDto(String templateId,String templateGroupId) {
        TemplateDTO templateDTO = new TemplateDTO();
        templateDTO.setId(templateId);
        templateDTO.setGroupId(templateGroupId);
        return templateDTO;
    }

    @Override
    public FlowEntity generateTemplate(TemplateDTO templateDTO, PositionDTO positionDTO, String parentGroupId) throws IOException {

        CommonEntity commonEntity = etlServer.getCommonEntity();
        InstantiateTemplateRequestEntity instantiateTemplateRequestEntity = BuildParamUtil.buildInstantiateTemplateRequestEntity(templateDTO.getId(), positionDTO.getX(), positionDTO.getY(), false);
        FlowEntity flowEntity = processGroupAPI.instanceTemplate(commonEntity, parentGroupId, instantiateTemplateRequestEntity);
        return flowEntity;
    }
}

加载制作好的template

    @Value("${nifi.dbTemplate.template-id}")
    private String dbTemplateId;
    @Value("${nifi.dbTemplate.template-group-id}")
    @Autowired
    private AtomicStrategy atomicStrategy;
    
    private String dbTemplateGroupId;
    
    public void downloadTemplate(String tableName, int i, TemplateDTO templateDto, String parentGroupId, Double random, String category, TableInfo tableInfo) {
        CommonEntity commonEntity = etlServer.getCommonEntity();
        TemplateDTO templateDto = atomicStrategy.getTemplateDto(dbTemplateId, dbTemplateGroupId);

        PositionDTO positionDTO = new PositionDTO();
        positionDTO.setX(550 * i + 0.00);
        positionDTO.setY(random);

        try {
            FlowEntity flowEntity = atomicStrategy.generateTemplate(templateDto, positionDTO, parentGroupId);
            //下面的部分，海阔凭鱼跃，天高任鸟飞啦
            /*
            *1.修改每一个dataflow的全局变量
            *2.根据不同的需求，动态的添加，减少processor，connection
            *3.根据不同的需求，配置不同的script template
            *4.根据不同的需求，匹配相应的controller service（这个不是spring的，是nifi的）
            */
        }
}

1.3 随便封装一个方法

import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import okhttp3.Call;
import okhttp3.MediaType;
import okhttp3.RequestBody;
import okhttp3.Response;
import okhttp3.ResponseBody;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.web.api.entity.ProcessorEntity;
import org.apache.nifi.web.api.entity.ProcessorRunStatusEntity;

public class ProcessorsAPI {
    private static final String API_PROCESSOR_INFO = "/processors/%s";
    private static final String API_PROCESSOR_UPDATE = "/processors/%s";
    private static final String API_PROCESSOR_DELETE = "/processors/%s";
    private static final String API_PROCESSOR_RUN_STATUS = "/processors/%s/run-status";

    public ProcessorsAPI() {
    }

    public ProcessorEntity getProcessor(CommonEntity commonEntity, String id) throws IOException {
        ValidateUtil.validateObject(commonEntity);
        if (StringUtils.isBlank(id)) {
            throw new RuntimeException("processor's id must not be null");
        } else {
            Map<String, String> headParams = new HashMap();
            headParams.put("Authorization", " Bearer " + commonEntity.getAccessToken());
            Call call = OKHttpConfiguration.buildGetCall(String.format(commonEntity.getUrl() + "/processors/%s", id), Collections.emptyList(), headParams);
            Response response = call.execute();
            Throwable var6 = null;

            ProcessorEntity var7;
            try {
                if (!response.isSuccessful()) {
                    throw new IOException("Unexpected code " + response + "Reason is: " + (response.body() != null ? response.body().string() : null));
                }

                var7 = (ProcessorEntity)JsonUtils.toObject(response.body().string(), ProcessorEntity.class);
            } catch (Throwable var16) {
                var6 = var16;
                throw var16;
            } finally {
                if (response != null) {
                    if (var6 != null) {
                        try {
                            response.close();
                        } catch (Throwable var15) {
                            var6.addSuppressed(var15);
                        }
                    } else {
                        response.close();
                    }
                }

            }

            return var7;
        }
    }
    public ProcessorEntity updateProcessor(CommonEntity commonEntity, String id, ProcessorEntity processorEntity) throws IOException {
        ValidateUtil.validateObject(commonEntity);
        if (!StringUtils.isBlank(id) && !Objects.isNull(processorEntity)) {
            String jsonString = JsonUtils.toJson(processorEntity);
            RequestBody requestBody = RequestBody.create(OKHttpConfiguration.JSON, jsonString);
            Map<String, String> headParams = new HashMap();
            headParams.put("Content-Type", "application/json");
            headParams.put("Authorization", " Bearer " + commonEntity.getAccessToken());
            Call call = OKHttpConfiguration.buildPutCall(String.format(commonEntity.getUrl() + "/processors/%s", id), headParams, requestBody);
            Response response = call.execute();
            Throwable var9 = null;

            ProcessorEntity var10;
            try {
                if (!response.isSuccessful()) {
                    throw new IOException("Unexpected code " + response + "Reason is: " + (response.body() != null ? response.body().string() : null));
                }

                var10 = (ProcessorEntity)JsonUtils.toObject(((ResponseBody)Objects.requireNonNull(response.body())).string(), ProcessorEntity.class);
            } catch (Throwable var19) {
                var9 = var19;
                throw var19;
            } finally {
                if (response != null) {
                    if (var9 != null) {
                        try {
                            response.close();
                        } catch (Throwable var18) {
                            var9.addSuppressed(var18);
                        }
                    } else {
                        response.close();
                    }
                }

            }

            return var10;
        } else {
            throw new RuntimeException("processor's id or ProcessorEntity must not be null");
        }
    }
}

2.根据小帮手创建DataFlow

前面说了，我们下载完通用性，扩展性很OK的Template以后，通过读取Excel调整对应Table的DataFLow，将数据迁移到目标表。

-----------use EasyExcel---------

      <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>easyexcel</artifactId>
            <version>2.2.6</version>
        </dependency>

@RestController
@RequestMapping(value = "/dbTemplate")
public class DBController {

    @Autowired
    private DBService dbService;

    @Value("${nifi.parent-group-id}")
    private String parentGroupId;

    @RequestMapping(value = "/create")
    public void createDBTemplate(){

        dbService.simpleRead(parentGroupId);
    }

}

将excel中每一栏都映射到实体类中，DBMappingTableInfo.class

@Component
public class DBService {

    @Value("${nifi.templateFilesPath.db-template-file}")
    private String dbTemplateUrl;
    @Autowired
    private NifiService nifiService;
    @Autowired
    private AtomicStrategy atomicStrategy;

    private static final Logger LOGGER = LoggerFactory.getLogger(DBService.class);

    public void simpleRead(String parentGroupId) {

        String url = dbTemplateUrl;
        int startRow = 3;

        ExcelReader excelReader = null;
        try {
            excelReader = EasyExcel.read(url).build();

            ReadSheet readSheet1 =
                    EasyExcel.readSheet(0).headRowNumber(startRow).head(DBMappingTableInfo.class).registerReadListener(new DBMappingTableInfoListener()).build();
            ReadSheet readSheet2 =
                    EasyExcel.readSheet(1).headRowNumber(startRow).head(DBMappingColumnInfo.class).registerReadListener(new DBMappingColumnInfoListener()).build();

            excelReader.read(readSheet1, readSheet2);
        } finally {
            if (excelReader != null) {

                excelReader.finish();
            }
        }
    }
}

public class DBMappingTableInfoListener extends AnalysisEventListener<DBMappingTableInfo> {

    private static final Logger logger = LoggerFactory.getLogger(DBMappingTableInfoListener.class);

    List<DBMappingTableInfo> dbMappingTableInfoList = new ArrayList<>();

    @Override
    public void invoke(DBMappingTableInfo dbMappingTableInfo, AnalysisContext analysisContext) {
        logger.info("解析到一条数据:{}", JSON.toJSONString(dbMappingTableInfo));
        dbMappingTableInfoList.add(dbMappingTableInfo);

    }

    @Override
    public void doAfterAllAnalysed(AnalysisContext analysisContext) {

        Random random = new Random();
        Double number = random.nextInt(1000) + 0.00;
        for (int i = 0; i < dbMappingTableInfoList.size(); i++) {
            DBMappingTableInfo dbMappingTableInfo = dbMappingTableInfoList.get(i);
            String tableName = dbMappingTableInfo.getTableName();
            operationNifi(tableName, i, parentGroupId, number,dbMappingTableInfo);
        }
    }

    public void operationNifi(String tableName, int i, String parentGroupId, Double random, TableInfo tableInfo) {
        logger.info("dbTableInfo start work create template");
        nifiService.downloadTemplate(tableName, i, templateDto, parentGroupId, random,"table",tableInfo);
    }
}