我对HDFS的一些理解——NameNode启动过程。
public static void main(String argv[]) throws Exception {
if (DFSUtil.parseHelpArgument(argv, NameNode.USAGE, System.out, true)) {
System.exit(0);
}
try {
//输出namenode启动的一些信息
StringUtils.startupShutdownMessage(NameNode.class, argv, LOG);
//创建一个namenode节点
NameNode namenode = createNameNode(argv, null);
if (namenode != null) {
namenode.join();
}
} catch (Throwable e) {
LOG.error("Failed to start namenode.", e);
terminate(1, e);
}
}
上述main函数为namenode的启动入口,StringUtils.startupShutdownMessage主要输出启动该节点的一些系统信息,如classpath,jdk version等等,如下图所示。
![98025400238c64de7a0cf2922667e982.png](https://i-blog.csdnimg.cn/blog_migrate/478e0fc004ef64c13b4a825ad79a892e.jpeg)
接下来主要查看createNameNode(argv, null)做了哪些初始化操作。
public static NameNode createNameNode(String argv[], Configuration conf)
throws IOException {
LOG.info("createNameNode " + Arrays.asList(argv));
//创建conf配置类
if (conf == null)
conf = new HdfsConfiguration();
// Parse out some generic args into Configuration.
//加载配置
GenericOptionsParser hParser = new GenericOptionsParser(conf, argv);
argv = hParser.getRemainingArgs();
// Parse the rest, NN specific args.
//获取启动选项
StartupOption startOpt = parseArguments(argv);
if (startOpt == null) {
printUsage(System.err);
return null;
}
setStartupOption(conf, startOpt);
switch (startOpt) {
case FORMAT: {
boolean aborted = format(conf, startOpt.getForceFormat(),
startOpt.getInteractiveFormat());
terminate(aborted ? 1 : 0);
return null; // avoid javac warning
}
...
default: {
DefaultMetricsSystem.initialize("NameNode");
return new NameNode(conf);
}
}
}
以上主要是进行HDFS配置文件类HdfsConfiguration创建并加载hdfs-site.xml配置文件中定义的配置。并获取启动选项,根据不同的启动选项在switch分支中进行后续操作,这里是default,即后续代码是newNameNode(conf)。
/**
* Start NameNode.
* <p>
* The name-node can be started with one of the following startup options:
* <ul>
* <li>{@link StartupOption#REGULAR REGULAR} - normal name node startup</li>
* <li>{@link StartupOption#FORMAT FORMAT} - format name node</li>
* <li>{@link StartupOption#BACKUP BACKUP} - start backup node</li>
* <li>{@link StartupOption#CHECKPOINT CHECKPOINT} - start checkpoint node</li>
* <li>{@link StartupOption#UPGRADE UPGRADE} - start the cluster
* <li>{@link StartupOption#UPGRADEONLY UPGRADEONLY} - upgrade the cluster
* upgrade and create a snapshot of the current file system state</li>
* <li>{@link StartupOption#RECOVER RECOVERY} - recover name node
* metadata</li>
* <li>{@link StartupOption#ROLLBACK ROLLBACK} - roll the
* cluster back to the previous state</li>
* <li>{@link StartupOption#FINALIZE FINALIZE} - finalize
* previous upgrade</li>
* <li>{@link StartupOption#IMPORT IMPORT} - import checkpoint</li>
* </ul>
* The option is passed via configuration field:
* <tt>dfs.namenode.startup</tt>
*
* The conf will be modified to reflect the actual ports on which
* the NameNode is up and running if the user passes the port as
* <code>zero</code> in the conf.
*
* @param conf confirguration
* @throws IOException
*/
public NameNode(Configuration conf) throws IOException {
this(conf, NamenodeRole.NAMENODE);
}
主要是this(conf, NamenodeRole.NAMENODE),说明创建了namenode节点。这里也可以创建其他节点,如namenode的backup node。
rotected NameNode(Configuration conf, NamenodeRole role)
throws IOException {
this.conf = conf;
this.role = role;
//设置可供客户端访问的地址和端口信息。在hdfs-site.xml中配置。日志打印相关设置。
//NameNode.clientNamenodeAddress属性赋值为hdfsServer
//<property>
// <name>fs.default.name</name>
// <value>hdfs://hdfsServer</value>
//</property>
setClientNamenodeAddress(conf);
//nsId值为hdfsServer
String nsId = getNameServiceId(conf);
//namenodeId值为nn1
//<property>
// <name>dfs.ha.namenodes.hdfsServer</name>
// <value>nn1,nn2</value>
//</property>
String namenodeId = HAUtil.getNameNodeId(conf, nsId);
//true
this.haEnabled = HAUtil.isHAEnabled(conf, nsId);
state = createHAState(getStartupOption(conf));
this.allowStaleStandbyReads = HAUtil.shouldAllowStandbyReads(conf);
//创建高可用上下文
this.haContext = createHAContext();
try {
initializeGenericKeys(conf, nsId, namenodeId);
initialize(conf);
try {
haContext.writeLock();
state.prepareToEnterState(haContext);
state.enterState(haContext);
} finally {
haContext.writeUnlock();
}
} catch (IOException e) {
this.stop();
throw e;
} catch (HadoopIllegalArgumentException e) {
this.stop();
throw e;
}
this.started.set(true);
}
1、首先是setClientNamenodeAddress(conf),设置了server0:9000相关的配置。如下图所示。
![0e75771ac1a0f7beaa6f01566c4c605d.png](https://i-blog.csdnimg.cn/blog_migrate/dc380f46d0fa95cbf999083028b7d283.png)
protected void initialize(Configuration conf) throws IOException {
if (conf.get(HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS) == null) {
String intervals = conf.get(DFS_METRICS_PERCENTILES_INTERVALS_KEY);
if (intervals != null) {
conf.set(HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS,
intervals);
}
}
UserGroupInformation.setConfiguration(conf);
loginAsNameNodeUser(conf);
//指标监控
NameNode.initMetrics(conf, this.getRole());
StartupProgressMetrics.register(startupProgress);
if (NamenodeRole.NAMENODE == role) {
startHttpServer(conf);
}
this.spanReceiverHost =
SpanReceiverHost.get(conf, DFSConfigKeys.DFS_SERVER_HTRACE_PREFIX);
//加载文件系统
loadNamesystem(conf);
rpcServer = createRpcServer(conf);
if (clientNamenodeAddress == null) {
// This is expected for MiniDFSCluster. Set it now using
// the RPC server's bind address.
clientNamenodeAddress =
NetUtils.getHostPortString(rpcServer.getRpcAddress());
LOG.info("Clients are to use " + clientNamenodeAddress + " to access"
+ " this namenode/service.");
}
if (NamenodeRole.NAMENODE == role) {
httpServer.setNameNodeAddress(getNameNodeAddress());
httpServer.setFSImage(getFSImage());
}
pauseMonitor = new JvmPauseMonitor(conf);
pauseMonitor.start();
metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
startCommonServices(conf);
}