FileSystem fs;
DistributedFileSystem dfs;
得到FileSystem: fs = new Path("/").getFileSystem(conf);
RPC调用Namenode方法: fs.getClient().namenode.XXXXX();
得到某个文件corrupt的数据块:
LocatedBlocks locatedBlocks =
fs.getClient().namenode.getBlockLocations(uriPath, 0, stat.getLen());
for (LocatedBlock b: locatedBlocks.getLocatedBlocks()) {
if (b.isCorrupt() ||
(b.getLocations().length == 0 && b.getBlockSize() > 0)) {
// the block is corrupt;
}
}
final BlockLocation[] fileBlocks =
dfs.getFileBlockLocations(fileStatus, 0, fileLength);
for (BlockLocation fileBlock: fileBlocks) {
if (fileBlock.isCorrupt() ||
(fileBlock.getNames().length == 0 && fileBlock.getLength() > 0)) {
// the block is corrupt;
}
}
Path->String: String uri = path.toUri().getPath();
String->Path: Path path = new Path(uri);
java.net.URI dfsUri = DistributedFileSystem.getUri();
System.out.println(dfsUri) ------> hdfs://namenodehostname:9000
DistributedFileSystem.makeQualified(new Path("/foo"))); ---------> hdfs://namenodehostname:9000/foo
new Path(scheme+":"+"//"+authority + pathUri.getPath();
Namenode上记录了那些LOG:
static void turnOffNameNodeLogging() {
// change log level to ERROR: NameNode.LOG & NameNode.stateChangeLog
((Log4JLogger)NameNode.LOG).getLogger().setLevel(Level.ERROR);
((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ERROR);
((Log4JLogger)NetworkTopology.LOG).getLogger().setLevel(Level.ERROR);
((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.ERROR);
((Log4JLogger)FSNamesystem.auditLog).getLogger().setLevel(Level.ERROR);
((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ERROR);
}
切换HDFS的数据块位置选择策略:conf的dfs.block.replicator.classname
conf.set("dfs.block.replicator.classname", "org.apache.hadoop.hdfs.server.namenode.BlockPlacementPolicyRaid");
Path p = new Path("/raid"); LOG.info(p) ---> /raid
String xorPrefix = p.toUri().getPath(); LOG.info(xorPrefix); ----->/raid
Path p = new Path("hdfs://0.0.0.0:3000" + "/raid"); --->hdfs://0.0.0.0:3000/raid
String xorPrefix = p.toUri().getPath(); LOG.info(xorPrefix); ----->/raid