使用HDFS提供的API来对分布式文件系统中的文件进行基本的操作,在linux下是不需要连接的,在windows下连接hdfs有三种方式:
Configuration conf = new Configuration();
//第一种方式
conf.set("fs.defaultFS", "hdfs://10.49.85.152:9000");
//第二种设置configuration方式
conf.addResource(new Path("D:\\hadoop\\1.xml"));
FileSystem hdfs = FileSystem.get(conf);
//第三种方式
FileSystem hdfs = FileSystem.get(new URI("hdfs://10.49.85.152:9000"), conf);
下面是对基本的API的调用,以供参考:
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class HDFSDemo2 {
static Configuration conf;
static FileSystem fs;
static{
conf = new Configuration();
try {
fs = FileSystem.get(new URI("hdfs://10.49.85.152:9000"), conf);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (URISyntaxException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
// public HDFSDemo2(){
// conf = new Configuration();
// try {
// fs = FileSystem.get(new URI("hdfs://10.49.85.152:9000"), conf);
// } catch (IOException e) {
// // TODO Auto-generated catch block
// e.printStackTrace();
// } catch (URISyntaxException e) {
// // TODO Auto-generated catch block
// e.printStackTrace();
// }
// }
//创建文件夹
public static void mkdir(String name){
boolean flag = false;
try {
if (!fs.exists(new Path(name+"1"))) {
flag = fs.mkdirs(new Path(name));
}
System.out.println(flag);
if(flag){
System.out.println("文件夹创建成功");
}else{
System.out.println("文件夹创建失败");
}
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
close();
}
}
//创建文件
public static void createFile(String file, String text){
FSDataOutputStream outputStream = null;
byte[] arg0 = text.getBytes();
try {
outputStream = fs.create(new Path(file));
outputStream.write(arg0, 0, arg0.length);
outputStream.close();
System.out.println("文件创建成功");
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
close();
}
}
//对文件重命名
public static void rename(String oldName, String newName){
boolean flag;
try {
flag = fs.rename(new Path(oldName), new Path(newName));
if(flag){
System.out.println("文件重命名成功");
}
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
close();
}
}
//判断文件是否存在
public static void existFile(String file){
boolean flag;
try {
flag = fs.exists(new Path(file));
if(flag){
System.out.println("存在");
}else
{
System.out.println("不存在");
}
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
close();
}
}
//删除文件
public static void deleteFile(String file){
boolean flag;
try {
flag = fs.deleteOnExit(new Path(file));
if(flag){
System.out.println("成功删除");
}
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
close();
}
}
public static void close(){
try {
fs.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
//上传文件
public static void uploadFile(String src, String dst){
try {
fs.copyFromLocalFile(false, new Path(src), new Path(dst));
System.out.println("文件上传成功");
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
close();
}
}
//下载文件
public static void downFile(String src, String dst){
try {
fs.copyToLocalFile(new Path(src), new Path(dst));
System.out.println("文件下载成功");
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
close();
}
}
//读取文件
public static String readFile(String file){
String str = "";
FSDataInputStream inputStream = null;
byte[] arg0 = new byte[1024];
try {
inputStream = fs.open(new Path(file));
inputStream.read(arg0);
str = new String(arg0);
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
close();
}
return str;
}
//返回指定文件夹的文件列表
public static void listName(String dir){
try {
FileStatus files[] = fs.listStatus(new Path(dir));
for(FileStatus file : files){
System.out.println(file.getPath().getName() + "\t" + file.getLen());
}
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
close();
}
}
public static void main(String[] args) throws IOException {
// TODO Auto-generated method stub
// HDFSDemo2 hdfs = new HDFSDemo2();
HDFSDemo2.mkdir("/test/t2");
// HDFSDemo2.createFile("/test/3.txt", "fdsf23432432违法所得");
// HDFSDemo2.createFile(args[0], args[1]);
// HDFSDemo2.rename("/test/test2", "/test/test1");
// HDFSDemo2.existFile("/test/3.txt");
// HDFSDemo2.deleteFile("/test/5.txt");
// HDFSDemo2.downFile(args[0], args[1]);
// String str = HDFSDemo2.readFile("/test/3.txt");
// System.out.println(str);
// HDFSDemo2.listName("/test");
}
}