通过FileSystem API访问BlockLocations块位置信息
-----------------------------------------------------------------------------------------------
/**
* 通过FileSystem API访问BlockLocations块位置信息
* @throws IOException
*/
@Test
public void BlockLByFS() throws IOException{
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path file = new Path("/spaceQuota/hello.txt");
FileStatus fst = fs.getFileStatus(file);
BlockLocation[] blocklocations = fs.getFileBlockLocations(fst, 8, 30);
// BlockLocation[] blocklocations = fs.getFileBlockLocations(fst, 0, 30);
for (BlockLocation blockLocation : blocklocations) {
String[] hosts = blockLocation.getHosts();
for (String host : hosts) {
System.out.println(host);
}
String[] names = blockLocation.getNames();
for (String name : names) {
System.out.println(name);
}
String[] topologs = blockLocation.getTopologyPaths();
for (String topolog : topologs) {
System.out.println(topolog);
}
}
}
通过FileSystem API访问Datanode信息
----------------------------------------------------------------------------------------------
/**
* 通过FileSystem API访问Datanode信息
* @throws IOException
*/
@Test
public void datanodeLByFS() throws IOException{
Configuration conf = new Configuration();
System.setProperty("HADOOP_USER_NAME", "hyxy");
// conf.set("dfs.permissions.enabled", "false");
FileSystem fs = FileSystem.get(conf);
DistributedFileSystem dfs = (DistributedFileSystem)fs;
DatanodeInfo[] dni = dfs.getDataNodeStats();
for (DatanodeInfo datanodeInfo : dni) {
System.out.println("Capacity:"+datanodeInfo.getCapacity());
System.out.println("Remaining:"+datanodeInfo.getRemaining());
System.out.println("DatanodeReport:"+datanodeInfo.getDatanodeReport());
System.out.println("------------------------------------------");
}
}
从hadoop URL读取数据
------------------------------------------
static{
URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
}
/**
* 通过java.net.URL类访问读取HDFS数据
* @throws IOException
*/
public static void main(String[] args) throws Exception {
URL _url =new URL("hdfs://master1:9000/spaceQuota/text.txt");
InputStream in = _url.openStream();
IOUtils.copyBytes(in, System.out, 4096,false);
IOUtils.closeStream(in);
}
通过FileSystem API访问读取HDFS数据
---------------------------------------------
/**
* 通过FileSystem API访问读取HDFS数据
* $>hadoop fs -cat /spaceQuota/text.txt
* @throws IOException
*/
@Test
public void readByFS() throws IOException{
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
String _path = "hdfs://master1:9000/spaceQuota/text.txt";
FSDataInputStream in = fs.open(new Path(_path));
IOUtils.copyBytes(in, System.out, 4096, true);
}
通过FileSystem API创建文件夹
--------------------------------------------------------------
/**
* 通过FileSystem API创建文件夹
* $>hadoop fs -mkdir /mkdir_byAPI
* 注意:修改集群权限:$>hadoop fs -chmod -R a+w /
*/
@Test
public void mkdirByAPI()throws Exception{
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path _path = new Path("/mkdir_byAPI");
fs.mkdirs(_path);
fs.close();
}
FileStatus对象,获取文件的相关属性
-----------------------------------------------------------
/**
* 通过FileSystem API获取一些相关参数,有一些过期的方法,如何采用现有的
*/
@Test
public void getPar()throws Exception{
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
FileStatus fstu = fs.getFileStatus(new Path("/spaceQuota/text.txt"));
long _blocksize = fstu.getBlockSize();
System.out.println("getAccessTime="+fstu.getAccessTime());
System.out.println("blockszie="+_blocksize);
System.out.println("getGroup="+fstu.getGroup());
System.out.println("getLen="+fstu.getLen());
System.out.println("getModificationTime="+fstu.getModificationTime());
System.out.println("getOwner="+fstu.getOwner());
System.out.println("getReplication="+fstu.getReplication());
System.out.println("getPath="+fstu.getPath());
}