HDFS的NameNode节点信息管理（元数据）

绝域时空

已于 2022-02-25 14:22:34 修改

阅读量2.6k

点赞数

分类专栏：大数据组件文章标签： hdfs hadoop big data

于 2021-12-30 18:43:08 首次发布

本文链接：https://blog.csdn.net/m0_43405302/article/details/122243274

版权

大数据组件专栏收录该内容

24 篇文章 11 订阅

订阅专栏

本文详细介绍了HDFS中NameNode的元数据存储位置，包括fsimage和edits文件。fsimage文件存储了目录、文件的元数据，如命名空间ID、文件系统树结构等；edits文件则记录了尚未合并到fsimage的日志操作。通过hdfsoiv和hdfsoev命令可以查看这些文件的内容，以XML格式展示。

摘要由CSDN通过智能技术生成

文章目录

HDFS的NameNode信息

HDFS的NameNode信息

1、NameNode的信息存放地址

NameNode存储DataNode的元数据，NameNode主要是用于维护DataNode信息。它存储在hadoop文件夹下data/dfs/name/current/，其中以fsimage开头的文件主要存储目录、文件名称及其依赖关系，以edits开头的文件主要存储尚未来得及合并的日志记录

ls hadoop/data/dfs/name/current
#-----------------------------------------------------------------------------
edits_0000000000000000001-0000000000000000002  fsimage_0000000000000000005.md5
edits_0000000000000000003-0000000000000000003  fsimage_0000000000000000014
edits_0000000000000000004-0000000000000000005  fsimage_0000000000000000014.md5
edits_0000000000000000006-0000000000000000014  seen_txid
edits_inprogress_0000000000000000015           VERSION
fsimage_0000000000000000005
#-----------------------------------------------------------------------------

2、NameNode节点数据查看

#查看目录、文件名称及其依赖关系，将信息转化为xml格式
hdfs oiv -i hadoop_home/data/dfs/name/current/fsimage* -o save_path/filename -p XMl
#例程：
hdfs oiv -i /opt/software/hadoop313/data/dfs/name/current/fsimage_0000000000000000258 -o ~/fsimage_0000000000000000258 -p XML
#查看文件日志记录
hdfs oev -i hadoop_home/data/dfs/name/current/edits* -o save_path/filename -p XMl
#例程：
hdfs oev -i /opt/software/hadoop313/data/dfs/name/current/edits_inprogress_0000000000000000079 -o ~/edits_inprogress_0000000000000000079 -p XML

3、fsimage文件

<?xml version="1.0" encoding="utf-8"?>

<fsimage> 
  <!--版本信息-->
  <version> 
    <layoutVersion>-64</layoutVersion>  
    <onDiskVersion>1</onDiskVersion>  
    <oivRevision>ba631c436b806728f8ec2f54ab1e289526c90579</oivRevision> 
  </version>  
  <!--NameNode信息-->
  <NameSection> 
    <!--命名空间ID-->
    <namespaceId>1540392763</namespaceId>  
    <!--gen戳版本-->
    <genstampV1>1000</genstampV1>  
    <genstampV2>1006</genstampV2>  
    <genstampV1Limit>0</genstampV1Limit> 
    <!--最后追加的本地block块的ID-->
    <lastAllocatedBlockId>1073741830</lastAllocatedBlockId>  
    <!--进程编号-->
    <txid>44</txid> 
  </NameSection>
  <!--纠删码信息-->
  <ErasureCodingSection> 
    <!--第一个纠删码策略信息-->
    <erasureCodingPolicy> 
      <!--纠删码编号-->
      <policyId>1</policyId>  
      <!--里德-所罗门类纠删码-->
      <policyName>RS-6-3-1024k</policyName>  
      <!--单元大小为1MB-->
      <cellSize>1048576</cellSize>  
      <!--策略状态-->
      <policyState>DISABLED</policyState>  
      <!--策略的详细信息-->
      <ecSchema> 
        <!--策略名称-->
        <codecName>rs</codecName>  
        <!--数据单元-->
        <dataUnits>6</dataUnits>  
        <!--奇偶校验单元-->
        <parityUnits>3</parityUnits> 
      </ecSchema> 
    </erasureCodingPolicy>  
    <erasureCodingPolicy> 
      <policyId>2</policyId>  
      <policyName>RS-3-2-1024k</policyName>  
      <cellSize>1048576</cellSize>  
      <policyState>DISABLED</policyState>  
      <ecSchema> 
        <codecName>rs</codecName>  
        <dataUnits>3</dataUnits>  
        <parityUnits>2</parityUnits> 
      </ecSchema> 
    </erasureCodingPolicy>  
    <erasureCodingPolicy> 
      <policyId>3</policyId>
      <policyName>RS-LEGACY-6-3-1024k</policyName>
      <cellSize>1048576</cellSize>
      <policyState>DISABLED</policyState>
      <ecSchema> 
        <codecName>rs-legacy</codecName>
        <dataUnits>6</dataUnits>
        <parityUnits>3</parityUnits>
      </ecSchema> 
    </erasureCodingPolicy>  
    <erasureCodingPolicy> 
      <policyId>4</policyId>
      <policyName>XOR-2-1-1024k</policyName>
      <cellSize>1048576</cellSize>
      <policyState>DISABLED</policyState>
      <ecSchema> 
        <codecName>xor</codecName>
        <dataUnits>2</dataUnits>
        <parityUnits>1</parityUnits>
      </ecSchema> 
    </erasureCodingPolicy>  
    <erasureCodingPolicy> 
      <policyId>5</policyId>
      <policyName>RS-10-4-1024k</policyName>
      <cellSize>1048576</cellSize>
      <policyState>DISABLED</policyState>
      <ecSchema> 
        <codecName>rs</codecName>
        <dataUnits>10</dataUnits>
        <parityUnits>4</parityUnits>
      </ecSchema> 
    </erasureCodingPolicy> 
  </ErasureCodingSection>  
  <!--内部节点信息，用于维护文件系统-->
  <INodeSection>
    <!--最后一个内部节点ID-->
    <lastInodeId>16396</lastInodeId>
    <!--内部节点数量-->
    <numInodes>12</numInodes>
    <inode>
      <!--内部节点ID-->
      <id>16385</id>
      <!--节点类型-->
      <type>DIRECTORY</type>
      <!--自闭合的节点代表根节点-->
      <name/>
      <!--创建时间-->
      <mtime>1640743069439</mtime>
      <!--用户-->
      <permission>root:supergroup:0755</permission>
      <nsquota>9223372036854775807</nsquota>
      <dsquota>-1</dsquota>
    </inode>  
    <inode>
      <id>16386</id>
      <type>DIRECTORY</type>
      <name>tmp</name>
      <mtime>1640738001903</mtime>
      <permission>root:supergroup:0770</permission>
      <nsquota>-1</nsquota>
      <dsquota>-1</dsquota>
    </inode>  
    <inode>
      <id>16387</id>
      <type>DIRECTORY</type>
      <name>hadoop-yarn</name>
      <mtime>1640738001903</mtime>
      <permission>root:supergroup:0770</permission>
      <nsquota>-1</nsquota>
      <dsquota>-1</dsquota>
    </inode>  
    <inode>
      <id>16388</id>
      <type>DIRECTORY</type>
      <name>staging</name>
      <mtime>1640738001903</mtime>
      <permission>root:supergroup:0770</permission>
      <nsquota>-1</nsquota>
      <dsquota>-1</dsquota>
    </inode>  
    <inode>
      <id>16389</id>
      <type>DIRECTORY</type>
      <name>history</name>
      <mtime>1640738001940</mtime>
      <permission>root:supergroup:0770</permission>
      <nsquota>-1</nsquota>
      <dsquota>-1</dsquota>
    </inode>  
    <inode>
      <id>16390</id>
      <type>DIRECTORY</type>
      <name>done</name>
      <mtime>1640738001903</mtime>
      <permission>root:supergroup:0770</permission>
      <nsquota>-1</nsquota>
      <dsquota>-1</dsquota>
    </inode>  
    <inode>
      <id>16391</id>
      <type>DIRECTORY</type>
      <name>done_intermediate</name>
      <mtime>1640738001940</mtime>
      <permission>root:supergroup:1777</permission>
      <nsquota>-1</nsquota>
      <dsquota>-1</dsquota>
    </inode>  
    <inode>
      <id>16392</id>
      <type>DIRECTORY</type>
      <name>test</name>
      <mtime>1640743069439</mtime>
      <permission>root:supergroup:0755</permission>
      <nsquota>-1</nsquota>
      <dsquota>-1</dsquota>
    </inode>  
    <inode>
      <id>16393</id>
      <type>DIRECTORY</type>
      <name>kb16</name>
      <mtime>1640743124723</mtime>
      <permission>root:supergroup:0755</permission>
      <nsquota>-1</nsquota>
      <dsquota>-1</dsquota>
    </inode>  
    <inode>
      <id>16394</id>
      <type>DIRECTORY</type>
      <name>hadoop</name>
      <mtime>1640743625760</mtime>
      <permission>root:supergroup:0755</permission>
      <nsquota>-1</nsquota>
      <dsquota>-1</dsquota>
    </inode>  
    <inode>
      <id>16395</id>
      <type>DIRECTORY</type>
      <name>hive</name>
      <mtime>1640743124723</mtime>
      <permission>root:supergroup:0755</permission>
      <nsquota>-1</nsquota>
      <dsquota>-1</dsquota>
    </inode> 
    <!--数据节点-->
    <inode>
      <id>16396</id>
      <type>FILE</type>
      <!--数据文件名称-->
      <name>ratings.csv</name>
      <replication>1</replication>
      <mtime>1640743625739</mtime>
      <atime>1640743618694</atime>
      <preferredBlockSize>134217728</preferredBlockSize>
      <permission>root:supergroup:0644</permission>
      <!--数据块信息-->
      <blocks>
        <block>
          <id>1073741825</id>
          <genstamp>1001</genstamp>
          <numBytes>134217728</numBytes>
        </block>  
        <block>
          <id>1073741826</id>
          <genstamp>1002</genstamp>
          <numBytes>134217728</numBytes>
        </block>  
        <block>
          <id>1073741827</id>
          <genstamp>1003</genstamp>
          <numBytes>134217728</numBytes>
        </block>  
        <block>
          <id>1073741828</id>
          <genstamp>1004</genstamp>
          <numBytes>134217728</numBytes>
        </block>  
        <block>
          <id>1073741829</id>
          <genstamp>1005</genstamp>
          <numBytes>134217728</numBytes>
        </block>  
        <block>
          <id>1073741830</id>
          <genstamp>1006</genstamp>
          <numBytes>7172347</numBytes>
        </block> 
      </blocks> 
      <!--存储策略信息-->
      <storagePolicyId>0</storagePolicyId>
    </inode> 
  </INodeSection>  
  <INodeReferenceSection/>
  <!--数据快照信息-->
  <SnapshotSection>
    <snapshotCounter>0</snapshotCounter>
    <numSnapshots>0</numSnapshots>
  </SnapshotSection>  
  <!--文件的先驱后继信息-->
  <INodeDirectorySection>
    <directory>
      <parent>16385</parent>
      <child>16392</child>
      <child>16386</child>
    </directory>  
    <directory>
      <parent>16386</parent>
      <child>16387</child>
    </directory>  
    <directory>
      <parent>16387</parent>
      <child>16388</child>
    </directory>  
    <directory>
      <parent>16388</parent>
      <child>16389</child>
    </directory>  
    <directory>
      <parent>16389</parent>
      <child>16390</child>
      <child>16391</child>
    </directory>  
    <directory>
      <parent>16392</parent>
      <child>16393</child>
    </directory>  
    <directory>
      <parent>16393</parent>
      <child>16394</child>
      <child>16395</child>
    </directory>  
    <directory>
      <parent>16394</parent>
      <child>16396</child>
    </directory> 
  </INodeDirectorySection>  
  <FileUnderConstructionSection/> 
  <!--数据加密管理-->
  <SecretManagerSection>
    <currentId>0</currentId>
    <tokenSequenceNumber>0</tokenSequenceNumber>
    <numDelegationKeys>0</numDelegationKeys>
    <!--数据标记-->
    <numTokens>0</numTokens>
  </SecretManagerSection>
  <!--缓存管理信息-->
  <CacheManagerSection>
    <nextDirectiveId>1</nextDirectiveId>
    <numDirectives>0</numDirectives>
    <numPools>0</numPools>
  </CacheManagerSection> 
</fsimage>

4、edits文件

<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<!--代表edits文件-->
<EDITS>
  <!--edits版本信息-->
  <EDITS_VERSION>-64</EDITS_VERSION>
  <RECORD>
    <OPCODE>OP_START_LOG_SEGMENT</OPCODE>
    <DATA>
      <TXID>45</TXID>
    </DATA>
  </RECORD>
</EDITS>