Hadoop RBF API

对于Hadoop RBF API的使用，这个可能是全网独一份了,因为在我找之前，即便墙都翻懒了也都找不到案例,话不多,下面给出了两种方式来实现。

方式一:Hadoop 工具类实现

这种方式本质是命令行的方式实现的,代码示例如下

package rbfApi;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.tools.federation.RouterAdmin;
import org.apache.hadoop.security.AnnotatedSecurityInfo;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ToolRunner;


public class toolRunnerT {
    public static void main(String[] args) throws Exception {
        System.setProperty("HADOOP_USER_NAME","hadoop");
        Configuration conf = new Configuration();
        conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");


        SecurityUtil.setSecurityInfoProviders(new AnnotatedSecurityInfo());
        UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
        System.out.println("user="+currentUser);
        conf.addResource(new Path("F:\\tmp\\rbf\\client\\hdfs-site.xml"));
        conf.addResource(new Path("F:\\tmp\\rbf\\client\\core-site.xml"));
        conf.addResource(new Path("F:\\tmp\\rbf\\client\\hdfs-rbf-site.xml"));


        conf.set("dfs.federation.router.admin-address","192.168.242.191:8111");


        RouterAdmin routerAdmin = new RouterAdmin(conf);
        System.out.println(routerAdmin);
        //ToolRunner.run()方式 --命令行方式
        //1、列出挂载路径信息
        String[] ls = {"-ls","/hadoop1t/"};
        //2、获取挂载路径的nameservice名称
        String[] getDestination = {"-getDestination","/hadoop5t"};
        //3、设置配额
        String[] setQuota = {"-setQuota","/hadoop5t","-nsQuota","700","-ssQuota","700"};
        //4、清除配额(验证结果需等待一会才会生效)
        String[] clrQuota = {"-clrQuota","/hadoop5t"};
        //5、添加挂载信息
        String[] nns = {"nameservice1"};


        String[] add2 = {"-add","/router-h5","nameservice1,nameservice2","/routerh5","-readonly","-faulttolerant",
                "-order","RANDOM","-owner","hadoop","-group","hadoop","-mode","750"};


        String[] add = {"-add","/hadoop1apitest2","nameservice1","/hadoop1t-hadoop1apitest2"};
        //6、删除挂载信息(验证结果需等待一会才会生效,源路径)
        String[] rm = {"-rm","/hadoop1apitest2"};
        //7、刷新挂载点,即手动更新挂载点缓存
        String[] refresh = {"-refresh"};


        ToolRunner.run(routerAdmin, add);


    }
}

方式二:通过RouterClient类实现

代码示例如下

package rbfApi;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
import org.apache.hadoop.hdfs.server.federation.resolver.RouterGenericManager;
import org.apache.hadoop.hdfs.server.federation.router.NameserviceManager;
import org.apache.hadoop.hdfs.server.federation.router.RouterClient;
import org.apache.hadoop.hdfs.server.federation.router.RouterStateManager;
import org.apache.hadoop.hdfs.server.federation.store.protocol.*;
import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MountTablePBImpl;
import org.apache.hadoop.net.NetUtils;


import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;




public class routerClient {
    public static void main(String[] args) throws IOException, URISyntaxException {
        Configuration conf = new Configuration();
        System.setProperty("HADOOP_USER_NAME","hadoop");
        conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
        conf.addResource(new Path("F:\\tmp\\rbf\\client\\hdfs-site.xml"));
        conf.addResource(new Path("F:\\tmp\\rbf\\client\\core-site.xml"));
        conf.addResource(new Path("F:\\tmp\\rbf\\client\\hdfs-rbf-site.xml"));
//        conf.set("fs.router.default.router", "192.168.242.191:8888");


        InetSocketAddress socketAddr = NetUtils.createSocketAddr("192.168.242.191", 8111);
        RouterClient routerClient = new RouterClient(socketAddr,conf);


        //1、mountTableManager(用于管理挂载、删除、列出路径等等)
        MountTableManager mountTableManager = routerClient.getMountTableManager();


        GetMountTableEntriesRequest getRequest =
                GetMountTableEntriesRequest.newInstance("/hadoop1t");


        GetMountTableEntriesResponse tableEntries = mountTableManager.getMountTableEntries(getRequest);
        //1.1 得到挂载目录条目信息
        List<MountTable> entries = tableEntries.getEntries();
        System.out.println("entries集合长度="+entries.size());
        for (int i = 0; i < entries.size(); i++) {
            System.out.println(entries.get(i));
        }
        //1.2 挂载目录
        AddMountTableEntryRequest addMountTableEntryRequest = AddMountTableEntryRequest.newInstance();
        MountTablePBImpl mountTablePB = new MountTablePBImpl();
        ArrayList<RemoteLocation> list = new ArrayList<RemoteLocation>();
        //测试结果第三个参数没有意义 ,第一个参数是设置集群nameservice名称、第二个是设置源集群路径名称、理论第三个是设置rbf下的挂载路径但是没有效果(被setSourcePath方法取代)
        RemoteLocation routerLocationn1 = new RemoteLocation("nameservice1","/cccb","/bbb-router");
        RemoteLocation routerLocationn2 = new RemoteLocation("nameservice2","/hadoop5apitest","/hadoop5apitest-router");
       //可以将多个集群的路径挂载到rbf下同一个路径,即多对一(多是指不同集群一是指rbf下的路径)
        list.add(routerLocationn1);
        list.add(routerLocationn2);
        mountTablePB.setDestinations(list);
        mountTablePB.setSourcePath("/cccb-router4"); //设置rbf下的挂载路径名称,必须设置
        mountTablePB.setDateCreated(1); //创建时间必须设置,且不能为负数,目前看设置值的多少并没有什么效果
        mountTablePB.setDateModified(1);//修改时间必须设置,且不能为负数,目前看设置值的多少并没有什么效果
        addMountTableEntryRequest.setEntry(mountTablePB);
        //执行挂载操作
//        AddMountTableEntryResponse addMountTableEntryResponse = mountTableManager.addMountTableEntry(addMountTableEntryRequest);
        //设置true与false 暂无发现有什么区别
//        addMountTableEntryResponse.setStatus(true);
        //注:即便挂载的目录在原集群不存在,也可以挂载
        //1.3删除挂载路径
        RemoveMountTableEntryRequest removeMountTableEntryRequest = RemoveMountTableEntryRequest.newInstance();
        //设置state store下挂载的路径
        removeMountTableEntryRequest.setSrcPath("/cccb-router3");
//        mountTableManager.removeMountTableEntry(removeMountTableEntryRequest);


        //1.4 刷新节点
        RefreshMountTableEntriesRequest refreshMountTableEntriesRequest = RefreshMountTableEntriesRequest.newInstance();
        mountTableManager.refreshMountTableEntries(refreshMountTableEntriesRequest);


        //1.5 获取nameservice名称
        GetDestinationRequest getDestinationRequest = GetDestinationRequest.newInstance();
        getDestinationRequest.setSrcPath("/cccb-router2");
        GetDestinationResponse destination = mountTableManager.getDestination(getDestinationRequest);
        //获取rbf下的路径对应源集群的所有nameservicem名称
        System.out.println("获取rbf路径对应的源集群nameservice名称="+destination.getDestinations());


        //1.6 更新挂载信息
        UpdateMountTableEntryRequest updateMountTableEntryRequest = UpdateMountTableEntryRequest.newInstance();
        MountTablePBImpl refreTablePB = new MountTablePBImpl();
        //设置rbf下的路径名称
        refreTablePB.setSourcePath("/bbb-router2");
        refreTablePB.setDateModified(1);//设置设置时间
        refreTablePB.setDateCreated(1);//设置修改时间


        ArrayList<RemoteLocation> refrelist = new ArrayList<RemoteLocation>();
        RemoteLocation refrerouterLocationn = new RemoteLocation("nameservice1","/cccb-refresh","");
        refrelist.add(refrerouterLocationn);
        refreTablePB.setDestinations(refrelist);
        //在更新的同时,如果增加下面设置,还可以同时增加映射路径
        refreTablePB.addDestination("nameservice2","/cccb-ns2-refresh");
        updateMountTableEntryRequest.setEntry(refreTablePB);
        //执行更新节点操作,将 rbf下的/bbb-router2路径对应/bbb修改为  /bbb-router2路径/cccb-refresh路径
//        mountTableManager.updateMountTableEntry(updateMountTableEntryRequest);


//        2、nameserviceManager(用于管理nameservice)
        NameserviceManager nameserviceManager = routerClient.getNameserviceManager();


        //2.1 禁用指定的nameservice集群
        //在http://192.168.242.191:50071/页面Subcluster 对应的nameservice可以看到是否被禁用
        DisableNameserviceRequest disnameserviceRequest = DisableNameserviceRequest.newInstance("nameservice2");
        //执行禁用动作,当对应的nameservice集群被禁用之后,通过rbf将不能访问到有关路径及其内容,但是挂载信息还在
        //如 A挂载了/a路径在rbf下,A和B集群都挂载了一个目录在 rbf /b 路径下,当禁用A集群时,那么/a路径访问将提示不存在, /b路径也只会显示B集群下的目录信息
//        nameserviceManager.disableNameservice(disnameserviceRequest);


        //2.2 启动指定的nameservice集群
        EnableNameserviceRequest enableNameserviceRequest = EnableNameserviceRequest.newInstance();
        enableNameserviceRequest.setNameServiceId("nameservice2");
//        nameserviceManager.enableNameservice(enableNameserviceRequest);


        //2.3 获取被禁用集群的nameservice名称
        GetDisabledNameservicesRequest getDisabledNameservicesRequest = GetDisabledNameservicesRequest.newInstance();
        GetDisabledNameservicesResponse disabledNameservices = nameserviceManager.getDisabledNameservices(getDisabledNameservicesRequest);
        System.out.println("ns="+disabledNameservices.getNameservices());




        //3、管理Router的通用方法
        RouterGenericManager routerGenericManager = routerClient.getRouterGenericManager();
        //3.1执行刷新超级用户代理组映射(在RBF中使用) 暂未体会到效果,返回的boolean结果是是否执行成功
        boolean b = routerGenericManager.refreshSuperUserGroupsConfiguration();
        System.out.println("b="+b);


        //4、管理Router服务状态
        RouterStateManager routerStateManager = routerClient.getRouterStateManager();


        //4.1 设置Router进入安全模式(这里不是设置所有router服务,还是设置前面socketAddr指定的IP)
        EnterSafeModeRequest enterSafeModeRequest = EnterSafeModeRequest.newInstance();
        EnterSafeModeResponse enterSafeModeResponse = routerStateManager.enterSafeMode(enterSafeModeRequest);
        boolean enterSafeModestatus = enterSafeModeResponse.getStatus();
        System.out.println("enterSafeModestatus="+enterSafeModestatus);


        //4.2 设置Router离开安全模式
        LeaveSafeModeRequest leaveSafeModeRequest = LeaveSafeModeRequest.newInstance();
        LeaveSafeModeResponse leaveSafeModeResponse = routerStateManager.leaveSafeMode(leaveSafeModeRequest);
        boolean leaveSafeModeResponseStatus = leaveSafeModeResponse.getStatus();
        System.out.println("leaveSafeModeResponseStatus="+leaveSafeModeResponseStatus);




        //4.3 验证查看当前Router服务状态是否处于安全模式
        // 是否处于安全模式也可以从http://192.168.242.191:50071/federationhealth.html#tab-router页面的Router按钮中的Status状态可查看
        GetSafeModeRequest getSafeModeRequest = GetSafeModeRequest.newInstance();
        GetSafeModeResponse safeMode = routerStateManager.getSafeMode(getSafeModeRequest);
        boolean inSafeMode = safeMode.isInSafeMode();
        System.out.println("inSafeMode="+inSafeMode);




    }
}

本人旨在分享工作中一些用到的知识技能

如有感兴趣欢迎关注Wbigdata微信公众号，也可私信留下你的联系方式交流大数据相关以及之外的技术。

注:转载请标明出处。