本文转载自:http://duanli.cublog.cn/, 转载请注明出处
以下列出了一些通过Java api来操作HDFS系统的方法,稍微改一改就能做工具类用了:)
package andy.hdfs.test;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.Date;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream;
import org.apache.hadoop.hdfs.DFSClient.DFSInputStream;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
public class ApiTest {
private static Configuration conf = new Configuration();
private static FileSystem fs;
private static DistributedFileSystem hdfs;
static {
try {
fs = FileSystem.get(conf);
hdfs = (DistributedFileSystem)fs;
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 列出所有DataNode的名字信息
*/
public void listDataNodeInfo() {
try {
DatanodeInfo[] dataNodeStats = hdfs.getDataNodeStats();
String[] names = new String[dataNodeStats.length];
System.out.println("List of all the datanode in the HDFS cluster:");
for (int i=0;i<names.length;i++) {
names[i] = dataNodeStats[i].getHostName();
System.out.println(names[i]);
}
System.out.println(hdfs.getUri().toString());
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 查看文件是否存在
*/
public void checkFileExist() {
try {
Path a= hdfs.getHomeDirectory();
System.out.println("main path:"+a.toString());
Path f = new Path("/user/xxx/input01/");
boolean exist = fs.exists(f);
System.out.println("Whether exist of this file:"+exist);
//删除文件
// if (exist) {
// boolean isDeleted = hdfs.delete(f, false);
// if(isDeleted) {
// System.out.println("Delete success");
// }
// }
} catch (Exception e) {
e.printStackTrace();
}
}
/**
*创建文件到HDFS系统上
*/
public void createFile() {
try {
Path f = new Path("/user/xxx/input02/file01");
System.out.println("Create and Write :"+f.getName()+" to hdfs");
FSDataOutputStream os = fs.create(f, true);
Writer out = new OutputStreamWriter(os, "utf-8");//以UTF-8格式写入文件,不乱码
out.write("你好 good job");
out.close();
os.close();
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 读取本地文件到HDFS系统<br>
* 请保证文件格式一直是UTF-8,从本地->HDFS
*/
public void copyFileToHDFS() {
try {
Path f = new Path("/user/xxx/input02/file01");
File file = new File("E:\\hadoopTest\\temporary.txt");
FileInputStream is = new FileInputStream(file);
InputStreamReader isr = new InputStreamReader(is, "utf-8");
BufferedReader br = new BufferedReader(isr);
FSDataOutputStream os = fs.create(f, true);
Writer out = new OutputStreamWriter(os, "utf-8");
String str = "";
while((str=br.readLine()) != null) {
out.write(str+"\n");
}
br.close();
isr.close();
is.close();
out.close();
os.close();
System.out.println("Write content of file "+file.getName()+" to hdfs file "+f.getName()+" success");
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 取得文件块所在的位置..
*/
public void getLocation() {
try {
Path f = new Path("/user/xxx/input02/file01");
FileStatus fileStatus = fs.getFileStatus(f);
BlockLocation[] blkLocations = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
for (BlockLocation currentLocation : blkLocations) {
String[] hosts = currentLocation.getHosts();
for (String host : hosts) {
System.out.println(host);
}
}
//取得最后修改时间
long modifyTime = fileStatus.getModificationTime();
Date d = new Date(modifyTime);
System.out.println(d);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 读取hdfs中的文件内容
*/
public void readFileFromHdfs() {
try {
Path f = new Path("/user/xxx/input02/file01");
FSDataInputStream dis = fs.open(f);
InputStreamReader isr = new InputStreamReader(dis, "utf-8");
BufferedReader br = new BufferedReader(isr);
String str = "";
while ((str = br.readLine()) !=null) {
System.out.println(str);
}
br.close();
isr.close();
dis.close();
} catch (Exception e) {
e.printStackTrace();
}
}
public static void main(String[] args) {
ApiTest a = new ApiTest();
// a.listDataNodeInfo();
// a.checkFileExist();
// a.createFile();
// a.copyFileToHDFS();
// a.getLocation();
a.readFileFromHdfs();
}
}