Trino动态配置Catalog以及打包问题解决

修改 core\src\main\java\io\trino\metadata\StaticCatalogStore.java

/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.trino.metadata;

import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.io.Files;
import io.airlift.discovery.client.Announcer;
import io.airlift.discovery.client.ServiceAnnouncement;
import io.airlift.log.Logger;
import io.trino.connector.ConnectorManager;
import okhttp3.Headers;
import okhttp3.HttpUrl;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;

import javax.inject.Inject;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;

import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.nullToEmpty;
import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
import static io.airlift.discovery.client.ServiceAnnouncement.serviceAnnouncement;
import static java.util.Objects.requireNonNull;

public class StaticCatalogStore
{
    private static final Logger log = Logger.get(StaticCatalogStore.class);
    private final ConnectorManager connectorManager;
    private final File catalogConfigurationDir;
    private final Set<String> disabledCatalogs;
    private final AtomicBoolean catalogsLoading = new AtomicBoolean();
    private final AtomicBoolean catalogsLoaded = new AtomicBoolean();
    private Announcer announcer;
    private Map<String, String> catalogMap = new HashMap<>();
    public static final String c_split = "\1";
    public static final String f_split = "\2";

    private static final String URL = "http://xxx/xxx/catalog";

    @Inject
    public StaticCatalogStore(ConnectorManager connectorManager, StaticCatalogStoreConfig config, Announcer announcer)
    {
        this(connectorManager,
                config.getCatalogConfigurationDir(),
                firstNonNull(config.getDisabledCatalogs(), ImmutableList.of()), announcer);
    }

    public StaticCatalogStore(ConnectorManager connectorManager,
                               File catalogConfigurationDir,
                               List<String> disabledCatalogs,
                               Announcer announcer)
    {
        this.connectorManager = connectorManager;
        this.catalogConfigurationDir = catalogConfigurationDir;
        this.disabledCatalogs = ImmutableSet.copyOf(disabledCatalogs);
        this.announcer = requireNonNull(announcer, "announcer is null");
    }

    public boolean areCatalogsLoaded()
    {
        return catalogsLoaded.get();
    }

    public void loadCatalogs()
            throws Exception
    {
        if (!catalogsLoading.compareAndSet(false, true)) {
            return;
        }

        for (File file : listFiles(catalogConfigurationDir)) {
            if (file.isFile() && file.getName().endsWith(".properties")) {
                loadCatalog(file);
            }
        }

        catalogsLoaded.set(true);

        Timer timer = new Timer();
        timer.schedule(new TimerTask() {
            public void run()
            {
                log.info("catalogMap:" + catalogMap);
                Map<String, String> newCatalog = new HashMap<>();
                Map<String, String> updateCatalog = new HashMap<>();
                List<String> remoteCatalog = new ArrayList<>();

                Set<String> strings = null;

                try {
                    String catalogStr = catalogstr();
                    String[] catalogs = catalogStr.split(c_split);
                    for (String catalogLine : catalogs) {
                        String[] catalogInfo = catalogLine.split(f_split);
                        String catalogName = catalogInfo[0];
                        remoteCatalog.add(catalogName);
                        if (catalogMap.get(catalogName) == null) {
                            //add catalog
                            newCatalog.put(catalogName, catalogLine);
                            catalogMap.put(catalogName, catalogLine);
                        }
                        else if (catalogMap.get(catalogName) != null && !catalogMap.get(catalogName).equals(catalogLine)) {
                            //update catalog
                            updateCatalog.put(catalogName, catalogLine);
                        }
                    }

                    strings = newCatalog.keySet();
                    log.info("add catalog:" + strings);
                    for (String catalogname : strings) {
                        addCatalog(newCatalog.get(catalogname));
                    }

                    log.info("update catalog:" + updateCatalog);
                    for (String catalogname : updateCatalog.keySet()) {
                        updateCatalog(updateCatalog.get(catalogname));
                        catalogMap.put(catalogname, updateCatalog.get(catalogname));
                    }

                    List<String> deleteCatalog = catalogMap.keySet().stream().filter(x -> !remoteCatalog.contains(x)).collect(Collectors.toList());

                    log.info("delete catalog:" + deleteCatalog);
                    for (String catalogname : deleteCatalog) {
                        deleteCatalog(catalogMap.get(catalogname));
                        catalogMap.remove(catalogname);
                    }

                    newCatalog.clear();
                    updateCatalog.clear();
                    deleteCatalog.clear();
                }
                catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }, 1, 600_000);
    }

    private void deleteCatalog(String catalogLine)
    {
        log.info("deleteCatalog:" + catalogLine);
        if (catalogLine != null) {
            String[] catalogInfo = catalogLine.split(f_split);
            String catalogName = catalogInfo[0];
            connectorManager.dropConnection(catalogName);
            updateConnectorIds(announcer, catalogName, CataLogAction.DELETE);
        }
    }

    private void addCatalog(String catalogLine)
    {
        log.info("addCatalog:" + catalogLine);

        String[] catalogInfo = catalogLine.split(f_split);
        String catalogName = catalogInfo[0];
        String connectorName = catalogInfo[1];
        Map<String, String> properties = new HashMap<>();
        properties.put("connection-url", catalogInfo[2]);
        properties.put("connection-user", catalogInfo[3]);
        properties.put("connection-password", catalogInfo[4]);
        connectorManager.createCatalog(catalogName, connectorName, ImmutableMap.copyOf(properties));
        updateConnectorIds(announcer, catalogName, CataLogAction.ADD);
        log.info("--------------add or update catalog ------------------" + catalogLine);
    }

    private void updateCatalog(String catalogLine)
    {
        log.info("updateCatalog:" + catalogLine);
        deleteCatalog(catalogLine);
        addCatalog(catalogLine);
    }

    private String catalogstr() throws IOException
    {
        OkHttpClient client = new OkHttpClient();
        Request.Builder builder = new Request.Builder();
        Request request = builder.url(URL).build();
        HttpUrl.Builder urlBuilder = request.url().newBuilder();
        Headers.Builder headerBuilder = request.headers().newBuilder();

        builder.url(urlBuilder.build()).headers(headerBuilder.build());
        Response execute = client.newCall(builder.build()).execute();
        return execute.body().string();
    }

    private void loadCatalog(File file) throws Exception
    {
        String catalogName = Files.getNameWithoutExtension(file.getName());
        if (disabledCatalogs.contains(catalogName)) {
            log.info("Skipping disabled catalog %s", catalogName);
            return;
        }

        log.info("-- Loading catalog %s --", file);
        Map<String, String> properties = new HashMap<>(loadPropertiesFrom(file.getPath()));

        String connectorName = properties.remove("connector.name");
//        if("mysql".equals(connectorName)){
        StringBuilder sb = new StringBuilder();
        sb.append(catalogName).append(f_split)
                .append(properties.get("connection-url")).append(f_split)
                .append(properties.get("connection-user")).append(f_split)
                .append(properties.get("connection-password"));
        catalogMap.put(catalogName, sb.toString());
//        }

        checkState(connectorName != null, "Catalog configuration %s does not contain connector.name", file.getAbsoluteFile());

        connectorManager.createCatalog(catalogName, connectorName, ImmutableMap.copyOf(properties));
        log.info("-- Added catalog %s using connector %s --", catalogName, connectorName);
    }

    private static List<File> listFiles(File installedPluginsDir)
    {
        if (installedPluginsDir != null && installedPluginsDir.isDirectory()) {
            File[] files = installedPluginsDir.listFiles();
            if (files != null) {
                return ImmutableList.copyOf(files);
            }
        }
        return ImmutableList.of();
    }

    private void updateConnectorIds(Announcer announcer, String catalogName, CataLogAction action)
    {
        ServiceAnnouncement announcement = getTrinoAnnouncement(announcer.getServiceAnnouncements());
        String property = nullToEmpty(announcement.getProperties().get("connectorIds"));
        List<String> values = Splitter.on(',').trimResults().omitEmptyStrings().splitToList(property);
        Set<String> connectorIds = new LinkedHashSet<>(values);
        if (CataLogAction.DELETE.equals(action)) {
            connectorIds.remove(catalogName);
        }
        else {
            connectorIds.add(catalogName);
        }

        ServiceAnnouncement.ServiceAnnouncementBuilder builder = serviceAnnouncement(announcement.getType());
        for (Map.Entry<String, String> entry : announcement.getProperties().entrySet()) {
            if (!entry.getKey().equals("connectorIds")) {
                builder.addProperty(entry.getKey(), entry.getValue());
            }
        }
        builder.addProperty("connectorIds", Joiner.on(',').join(connectorIds));
        announcer.removeServiceAnnouncement(announcement.getId());
        announcer.addServiceAnnouncement(builder.build());
    }

    private static ServiceAnnouncement getTrinoAnnouncement(Set<ServiceAnnouncement> announcements)
    {
        for (ServiceAnnouncement announcement : announcements) {
            if (announcement.getType().equals("trino")) {
                return announcement;
            }
        }
        throw new IllegalArgumentException("Trinoi announcement not found: " + announcements);
    }

    public enum CataLogAction
    {
        ADD, DELETE, UPDATE
    }
}

另外需要在后端提供一个获取配置文件的接口：

    @RequestMapping(value= "catalog",method = RequestMethod.GET,produces="application/json;charset=UTF-8")
    @ApiOperation("trino 即席查询 获取源配置接口")
    public String getCataLogs() {
       return getCataLogs();
    }

//实现逻辑
 public static String getCatalogs(){
        StringBuilder sb = new StringBuilder();
        sb.append("mysql1")
                .append(f_split)
                .append("mysql")
                .append("f_split")
                .append("jdbc:mysql://127.0.0.1:3306")
                .append(f_split)
                .append("root")
                .append(f_split)
                .append("123456")
                .append(c_split)

                .append("oracle1")
                .append(f_split)
                .append("oracle")
                .append("f_split")
                .append("jdbc:oracle:thin:@://127.0.0.1:1521")
                .append(f_split)
                .append("root")
                .append(f_split)
                .append("123456")

                //...
        ;
        return sb.toString();
    }

修改完这些之后就要将Trino源码打包，这里说下打包遇到的一些问题和解决方案：

        1. 部分pom引用的包无法下载

                解决方案手动下载然后放到本地maven仓库中，地址https://mvnrepository.com

        2.碰到 trino-kafka 的test里面的有个类文件找不到的问题，我的解决方法是删除那个报错的文件

        3.修改源文件的时候语法校验严格，不然maven install 会报错，方法是按照报错提示来。

        4.由于 SaticCatalogStore 需要用到 Okhttp 的包，所在pom的文件是 trino-main 下面。pom已经引入了，但是<score>runtime</score>，需要注释掉。然后maven install还会报包顺序错误，请按照maven提示放到指定的位置

        5.maven 仓库好像要3.6.3以上的版本，jdk 需要11以上

上述的方法连接非关系型数据库可能存在问题，另外一些错误的配置信息可能导致Trino无法正常使用。因此修修补补弄了个新版本，请看：

https://blog.csdn.net/z275598733/article/details/119218079?spm=1001.2014.3001.5502

声明：本文借鉴和引用

https://blog.csdn.net/woloqun/article/details/107912076

https://www.freesion.com/article/9129949508/