一、准备阶段
首先配置本机的环境变量,配置好HADOOP_HOME;在eclipse中安装好插件,以便在eclipse中操作HDFS集群,将hadoop-eclipse-plugin-2.6.0.jar包放入到eclipse安装文件夹下的dropins文件夹中的plugins下,此后重启eclipse即可在eclipse中操作hdfs集群。
注意: 编写的代码若有操作文件(例如修改、删除等功能)时,需要在本机配置权限的环境变量:HADOOP_USER_NAME=root,这样才会避免在eclipse中出现权限问题。
二、eclipse中的配置
环境变量配置好后,在eclipse中完成配置即可操作hdfs集群:
1)eclipse中的Hadoop控制平台
2)配置好所要连接的namenodeIP地址
此前要修改本机的hosts文件来确定IP地址对应的主机名
3)操作
4)替换工程对应的Hadoop包下的bin文件夹,使其方便操作
5)导入工程必需的jar包
三、JavaAPI代码
/**
* 1、查看文件
* 2、创建新文件夹
* 3、上传文件
* 4、下载文件
* 5、删除文件
* 6、内部移动
* 7、内部复制
* 8、重命名
* 9、创建新的文件
* 10、写文件
* 11、读文件内容
*
*/
//操作HDFS之前得先创建配置对象
Configuration conf = new Configuration(true);
//创建操作HDFS的对象
FileSystem fs = FileSystem.get(conf);
功能API代码:
//获取数据的位置
private static void getFileLocation(FileSystem fs, String string) throws IOException {
FileStatus fileStatus = fs.getFileStatus(new Path(string));
long len = fileStatus.getLen();
BlockLocation[] fileBlockLocations = fs.getFileBlockLocations(fileStatus, 0, len);
String[] hosts = fileBlockLocations[0].getHosts();
for (String string2 : hosts) {
System.out.println(string2);
}
HdfsBlockLocation blockLocation = (HdfsBlockLocation)fileBlockLocations[0];
long blockId = blockLocation.getLocatedBlock().getBlock().getBlockId();
System.out.println(blockId);
}
//读文件内容
private static void readFromHDFSFile(FileSystem fs, String string) throws IllegalArgumentException, IOException {
FSDataInputStream inputStream = fs.open(new Path(string));
FileStatus fileStatus = fs.getFileStatus(new Path(string));
long len = fileStatus.getLen();
byte[] b = new byte[(int)len];
int read = inputStream.read(b);
while(read != -1){
System.out.println(new String(b));
read = inputStream.read(b);
}
}
//写文件
private static void writeToHDFSFile(FileSystem fs, String filePath, String content) throws IllegalArgumentException, IOException {
FSDataOutputStream outputStream = fs.create(new Path(filePath));
outputStream.write(content.getBytes("UTF-8"));
outputStream.flush();
outputStream.close();
}
//创建一个新文件
private static void createNewFile(FileSystem fs, String string) throws IllegalArgumentException, IOException {
fs.createNewFile(new Path(string));
}
//内部移动 内部复制
private static void innerCopyAndMoveFile(FileSystem fs, Configuration conf,String src, String dest) throws IOException {
Path srcPath = new Path(src);
Path destPath = new Path(dest);
//内部拷贝
FileUtil.copy(srcPath.getFileSystem(conf), srcPath, destPath.getFileSystem(conf), destPath,false, conf);
//内部移动
FileUtil.copy(srcPath.getFileSystem(conf), srcPath, destPath.getFileSystem(conf), destPath,true, conf);
}
//重命名
private static void renameFile(FileSystem fs, String src, String dest) throws IOException {
Path srcPath = new Path(src);
Path destPath = new Path(dest);
fs.rename(srcPath, destPath);
}
//下载文件
private static void downLoadFileFromHDFS(FileSystem fs, String src, String dest) throws IOException {
Path srcPath = new Path(src);
Path destPath = new Path(dest);
//copyToLocal
fs.copyToLocalFile(srcPath, destPath);
//moveToLocal
fs.copyToLocalFile(true,srcPath, destPath);
}
//上传文件
private static void uploadFileToHDFS(FileSystem fs, String src, String dest) throws IOException {
Path srcPath = new Path(src);
Path destPath = new Path(dest);
//copyFromLocal
fs.copyFromLocalFile(srcPath, destPath);
//moveFromLocal
fs.copyFromLocalFile(true,srcPath, destPath);
}
//创建文件夹
private static void createDir(FileSystem fs, String string) throws IllegalArgumentException, IOException {
Path path = new Path(string);
if(fs.exists(path)){
fs.delete(path, true);
}
fs.mkdirs(path);
}
//查看文件系统的内容
private static List listFileSystem(FileSystem fs, String path) throws FileNotFoundException, IOException {
Path ppath = new Path(path);
FileStatus[] listStatus = fs.listStatus(ppath);
for (FileStatus fileStatus : listStatus) {
System.out.println(fileStatus.getPath());
}
return null;
}
//删除文件
private static String del(String path) throws IOException{
Configuration conf=new Configuration(true);
FileSystem fs = FileSystem.get(conf);
Path ppath=new Path(path);
try {
fs.delete(ppath);
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
System.out.println("删除文件:"+ppath);
}