[笔记]hdfs namenode FSImage分析1

[b]元数据文件fsimage的分析[/b]
fsimage为元数据镜像文件，存储的是某一时刻NameNode内存元数据信息，包括所有的INode信息、正在写入的文件信息以及其他的一些状态信息等。
[quote] /**
* Save current image and empty journal into {@code current} directory.
*/
protected void saveCurrent(StorageDirectory sd) throws IOException
[/quote]
当StorageDirectory的类型为NameNodeFile.IMAGE时，将FSImage持久化为../current/fsimage文件。
[i]fsimage的部分结构分析如下：[/i]
[table]
|int|int|long|long|...|
|LAYOUT_VERSION|namespaceID|numItemsInTree|timestamp|inodes|
[/table]
[i]inodes为目录时的结构[/i]
[table]
|short|binary|short|long|long|long|int|long|long|permission|
|namelen|name|0|modificationTime|0|0|-1|nsquota|dsquota|permission|
[/table]
[i]inodes为文件时的结构[/i]
[table]
|short|binary|short|long|long|long|int|...|permission|
|namelen|name|replication|modificationTime|accessTime|preferredBlockSize|numBlocksInFile|blocks|permission|
[/table]
[i]permission的结构[/i]
[table]
|Text|Text|short|
|username|groupname|permission|
[/table]
[i]blocks的结构[/i]
[table]
|long|long|long|
|blockId|numBytes|timestamp|
[/table]

[i]解析某fsimage的结果[/i]
LAYOUT_VERSION:-41
namespaceID:1167023747
numItemsInTree:19
generationStamp:1012

node:{name:, modtime:1377679793059, nsquota:2147483647, dsquota:-1, permission:hue supergroup 755}
node:{name:/test, modtime:1377679792720, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/tmp, modtime:1377679793059, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/test/input, modtime:1377679736766, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/test/output, modtime:1377679792720, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/test/input/chapter1.txt, copys:2, blksize:67108864, blk:[{3939378972164311396,8976,1002}], modtime:1377679736766, acctime:1377679736766, permission:hue supergroup 644}
node:{name:/test/output/_SUCCESS, copys:2, blksize:67108864, blk:[], modtime:1377679792720, acctime:1377679792720, permission:hue supergroup 644}
node:{name:/test/output/_logs, modtime:1377679773673, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/test/output/part-r-00000, copys:2, blksize:67108864, blk:[{5401467920515552367,6128,1011}], modtime:1377679787973, acctime:1377679787973, permission:hue supergroup 644}
node:{name:/test/output/_logs/history, modtime:1377679773673, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/test/output/_logs/history/job_201308281640_0001_1377679773095_hue_word+count, copys:2, blksize:67108864, blk:[{-7275418827867675308,13833,1012}], modtime:1377679773322, acctime:1377679773322, permission:hue supergroup 644}
node:{name:/test/output/_logs/history/job_201308281640_0001_conf.xml, copys:2, blksize:67108864, blk:[{-4116164172580388309,47105,1010}], modtime:1377679773673, acctime:1377679773673, permission:hue supergroup 644}
node:{name:/tmp/hadoop-hue, modtime:1377679793059, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/tmp/hadoop-hue/mapred, modtime:1377679793059, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/tmp/hadoop-hue/mapred/staging, modtime:1377679793059, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/tmp/hadoop-hue/mapred/system, modtime:1377679793049, nsquota:-1, dsquota:-1, permission:hue supergroup 700}
node:{name:/tmp/hadoop-hue/mapred/staging/hue, modtime:1377679793059, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/tmp/hadoop-hue/mapred/staging/hue/.staging, modtime:1377679793059, nsquota:-1, dsquota:-1, permission:hue supergroup 700}
node:{name:/tmp/hadoop-hue/mapred/system/jobtracker.info, copys:2, blksize:67108864, blk:[{-7415943897866965603,4,1001}], modtime:1377679253906, acctime:1377679253906, permission:hue supergroup 600}

[i]解析代码如下：[/i]

try (DataInputStream in = new DataInputStream(new BufferedInputStream(
				new FileInputStream(fileName)))) {
	System.out.printf("LAYOUT_VERSION:%s\n", in.readInt());
	System.out.printf("namespaceID:%s\n", in.readInt());
	long nodes = in.readLong();
	System.out.printf("numItemsInTree:%s\n", nodes);
	System.out.printf("generationStamp:%s\n\n", in.readLong());

	byte[] byteStore = new byte[4 * FSConstants.MAX_PATH_LENGTH];
	for (long i = 0; i < nodes; i++) {
		printINode(in, byteStore);
		Arrays.fill(byteStore, (byte) 0x00);
	}
}

	void printINode(DataInputStream in, byte[] byteStore)
			throws IOException {
		boolean isDir = true;
		short namelen = in.readShort();
		in.read(byteStore, 0, namelen);
		short replication = in.readShort();
		long modTime = in.readLong();
		long accessTime = in.readLong(); // access time
		long blockSize = in.readLong(); // preferred block size
		int blocks = in.readInt(); // # of blocks
		StringBuilder sb = new StringBuilder();
		long nsQuota = 0;
		long dsQuota = 0;
		if (blocks >= 0) {
			isDir = false;
			sb.append("[");
			for (int i = 0; i < blocks; i++) {
				sb.append("{").append(in.readLong()).append(",")
						.append(in.readLong()).append(",")
						.append(in.readLong()).append("}");
				if (i + 1 != blocks) {
					sb.append(",");
				}
			}
			sb.append("]");
		} else {
			nsQuota = in.readLong();
			dsQuota = in.readLong();
		}
		String userName = Text.readString(in);
		String groupName = Text.readString(in);
		short permit = in.readShort();
		if (isDir) {
			System.out
					.printf("node:{name:%s, modtime:%s, nsquota:%s, dsquota:%s, permission:%s %s %s}\n",
							new String(byteStore, 0, namelen), modTime,
							nsQuota, dsQuota, userName, groupName,
							Integer.toOctalString(permit));
		} else {
			System.out
					.printf("node:{name:%s, copys:%s, blksize:%s, blk:%s, modtime:%s, acctime:%s, permission:%s %s %s}\n",
							new String(byteStore, 0, namelen), replication,
							blockSize, sb.toString(), modTime, accessTime,
							userName, groupName, Integer.toOctalString(permit));
		}

	}