使用Java程序来获得某个集群的NameNode或DataNode的相关信息

本文介绍如何配置Hadoop集群以获取NameNode和DataNode信息，并针对HA配置的集群提供获取Active NameNode的方法。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

http://www.aboutyun.com/thread-18336-1-1.html

软件版本：hadoop2.2，JDK1.7，Intellij idea14

0. 配置Configuration

如果需要使用Java程序来获得某个集群的NameNode或DataNode的相关信息，那么Configuration需要如下配置：

[Java] 纯文本查看复制代码

 
         /**  
        
         * 获得配置的Configuration  
        
         *   
        
         * @return  
        
         */  
        
         public 
          static 
          Configuration getConf() {   
        
         if 
          (conf ==  
         null 
         ) {   
        
         conf =  
         new 
          Configuration();   
        
         conf.setBoolean( 
         "mapreduce.app-submission.cross-platform" 
         ,  
         true 
         ); 
         // 配置使用跨平台提交任务   
        
         conf.set( 
         "fs.defaultFS" 
         ,  
         "hdfs://" 
          + namenode +  
         ":8020" 
         );// 指定namenode   
        
         conf.set( 
         "mapreduce.framework.name" 
         ,  
         "yarn" 
         );  
         // 指定使用yarn框架   
        
         conf.set( 
         "yarn.resourcemanager.address" 
         , resourcenode +  
         ":8032" 
         );  
         // 指定resourcemanager   
        
         conf.set( 
         "yarn.resourcemanager.scheduler.address" 
         , schedulernode +  
         ":8030" 
         ); 
         // 指定资源分配器   
        
         conf.set( 
         "mapreduce.jobhistory.address" 
         , jobhistorynode +  
         ":10020" 
         ); 
         // 指定historyserver   
        
         }   
        
         return 
          conf;   
        
         }

1. 针对Hadoop集群（不管有没有配置HA），获得DataNode相关信息；

[Java] 纯文本查看复制代码

 
         /** 
        
         * 获取子节点 
        
         * @return 
        
         */ 
        
         public 
          static 
          DatanodeInfo getDATANODE() { 
        
         if 
         (DATANODE== 
         null 
         ) { 
        
         try 
          { 
        
         DatanodeInfo[] live =  ((DistributedFileSystem) getFS()).getDataNodeStats(HdfsConstants.DatanodeReportType.LIVE); 
        
         if 
         (live.length< 
         1 
         ){ 
        
         log.error( 
         "获取live子节点失败!" 
         ); 
        
         } 
        
         DATANODE=live[ 
         0 
         ]; 
        
         } 
         catch 
          (IOException e){ 
        
         log.warn( 
         "获取live子节点异常！\n" 
         +e.getMessage()); 
        
         } 
        
         } 
        
         return 
          DATANODE; 
        
         }

2. 针对配置了HA的Hadoop集群，获得Active NameNode的代码：

[Java] 纯文本查看复制代码

 
         package 
          org.apache.hadoop.hdfs.tools; 
        
         import 
          org.apache.hadoop.conf.Configuration; 
        
         import 
          org.apache.hadoop.fs.CommonConfigurationKeys; 
        
         import 
          org.apache.hadoop.ha.HAServiceProtocol; 
        
         import 
          org.apache.hadoop.ha.HAServiceStatus; 
        
         import 
          org.apache.hadoop.ha.HAServiceTarget; 
        
         import 
          org.slf4j.Logger; 
        
         import 
          org.slf4j.LoggerFactory; 
        
         import 
          java.io.IOException; 
        
         /** 
        
         * 动态获得Active NameNode 
        
         * 针对HA集群 
        
         * Created by fansy on 2016/3/29. 
        
         */ 
        
         public 
          class 
          DFSHAAdminUtils  
         extends 
          DFSHAAdmin { 
        
         private 
          Logger log = LoggerFactory.getLogger(DFSHAAdminUtils. 
         class 
         ); 
        
         private 
          int 
          rpcTimeoutForChecks = - 
         1 
         ; 
        
         private  
          String nnidStr; 
        
         private 
          static 
          String activeNameNodeHostAndPort= 
         null 
         ; 
        
         public 
          DFSHAAdminUtils(){} 
        
         public 
          DFSHAAdminUtils(Configuration conf,String nnidStr){ 
        
         super 
         .setConf(conf); 
        
         if 
          (conf !=  
         null 
         ) { 
        
         rpcTimeoutForChecks = conf.getInt( 
        
         CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_KEY, 
        
         CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_DEFAULT); 
        
         } 
        
         this 
         .nnidStr= nnidStr; 
        
         } 
        
         public 
          String getActiveNameNode(){ 
        
         String[] nnids = nnidStr.split( 
         "," 
         ,- 
         1 
         ); 
        
         HAServiceProtocol proto =  
         null 
         ; 
        
         HAServiceTarget haServiceTarget= 
         null 
         ; 
        
         HAServiceStatus state=  
         null 
         ; 
        
         try 
          { 
        
         for 
          (String nnid : nnids){ 
        
         haServiceTarget = resolveTarget(nnid); 
        
         proto = haServiceTarget.getProxy( 
        
         getConf(), rpcTimeoutForChecks); 
        
         state = proto.getServiceStatus(); 
        
         if 
          (state.getState().ordinal()==HAServiceProtocol.HAServiceState.ACTIVE.ordinal()) { 
        
         log.info( 
         "Active NameNode:{}" 
         , 
        
         haServiceTarget.getAddress().getHostName() +  
         ":" 
          + haServiceTarget.getAddress().getPort()); 
        
         return 
          haServiceTarget.getAddress().getHostName()+ 
         ":" 
         +haServiceTarget.getAddress().getPort(); 
        
         } 
        
         } 
        
         } 
         catch 
         (IOException e){ 
        
         log.error( 
         "获取Active NameNode异常!" 
         ); 
        
         } 
        
         return 
          null 
         ; 
        
         } 
        
         }