Hadoop的API操作

版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。

本文链接： https://blog.csdn.net/qq_33202508/article/details/78690325

package www.hadoophdfs;

import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;

public class HDFSDemo {
// 文件上传
public void upload() throws IOException, InterruptedException, URISyntaxException {
// 获取配置文件 加载配置文件 core-site.xml core-default.xml
Configuration conf = new Configuration();
// conf.set(“dfs.replication”, “10”);
// 获取FileSystem对象
FileSystem fs = FileSystem.get(new URI(“hdfs://192.168.4.158:9000”), conf, “root”);
Path src = new Path(“D:/aa.txt”);
Path dst = new Path(“/aa”);
fs.copyFromLocalFile(src, dst);
fs.close();
System.out.println(“上传成功”);
}

// 文件下载
public void download() throws IOException, InterruptedException, URISyntaxException {
    // 获取配置文件 加载配置文件 core-site.xml core-default.xml
    Configuration conf = new Configuration();
    // 获取FileSystem对象
    FileSystem fs = FileSystem.get(new URI("hdfs://192.168.4.158:9000"), conf, "root");
    Path dst = new Path("/e");
    FSDataInputStream in = fs.open(dst);
    FileOutputStream fos = new FileOutputStream("E:\\data1.java");
    int n = -1;
    while ((n = in.read()) != -1) {
        System.out.println((char) n);
        fos.write(n);
    }
    // for(int i=0;i<3;i++){
    // System.out.println((char)in.read());
    // }
    fs.close();
    System.out.println("下载成功");
}

// 创建目录
public void mkdirs() throws IOException, InterruptedException, URISyntaxException {
    // 获取配置文件，加载配置文件core-size.xml core-default.xml
    Configuration conf = new Configuration();
    // 获取fs对象
    FileSystem fs = FileSystem.get(new URI("hdfs://192.168.4.158:9000"), conf, "root");
    fs.mkdirs(new Path("/c"));
    fs.close();
    System.out.println("创建成功！");
}

// 删除目录
public void delete() throws IllegalArgumentException, IOException, InterruptedException, URISyntaxException {
    // 获取配置文件
    Configuration conf = new Configuration();
    // 获取fs对象
    FileSystem fs = FileSystem.get(new URI("hdfs://192.168.4.158:9000"), conf, "root");
    fs.delete(new Path("/aa"), 1 > 0);// 1>0==true
    // fs.delete(new Path("/aa"));
    fs.close();
    System.out.println("删除成功");
}

// 查看目录
public void cat() throws IOException, InterruptedException, URISyntaxException {
    // 获取配置文件
    Configuration conf = new Configuration();
    // 获取对象
    FileSystem fs = FileSystem.get(new URI("hdfs://192.168.4.158:9000"), conf, "root");
    // RemoteIterator<LocatedFileStatus> rm=fs.listFiles(new Path("/"),
    // true);
    // while(rm.hasNext()){
    // LocatedFileStatus lfs=rm.next();
    // System.out.println(lfs);
    // }

    RemoteIterator<LocatedFileStatus> rm = fs.listLocatedStatus(new Path("/"));
    while (rm.hasNext()) {
        LocatedFileStatus lfs = rm.next();
        System.out.println(lfs);
    }
    fs.close();
    System.out.println("查看目录");
}

// 流的操作--上传文件
public void streamUp() throws IOException, InterruptedException, URISyntaxException {
    // 获取配置文件
    Configuration conf = new Configuration();
    // 获取fs对象
    FileSystem fs = FileSystem.get(new URI("hdfs://192.168.4.158:9000"), conf, "root");

    FileInputStream fis = new FileInputStream("E:\\java\\视频资源2\\9\\www\\1.java");
    FSDataOutputStream out = fs.create(new Path("/e"));
    IOUtils.copy(fis, out);
    System.out.println("上传成功--流的操作");
}

// 流的操作--下载文件
public void streanLoca() throws IOException, InterruptedException, URISyntaxException {
    // 获取配置文件
    Configuration conf = new Configuration();
    // 获取fs对象
    FileSystem fs = FileSystem.get(new URI("hdfs://192.168.4.158:9000"), conf, "root");
    // 创建输出流--写到本地磁盘
    FileOutputStream fos = new FileOutputStream("F:\\xiazai\\3.java");
    // 创建输入流--读取服务器上的内容
    FSDataInputStream in = fs.open(new Path("/e"));
    in.seek(2000);// 去除2000个字节（从前面开始去除）
    IOUtils.copy(in, fos);
    System.out.println("流的方式--下载成功");
}

public static void main(String[] args) throws IOException, InterruptedException, URISyntaxException {
    HDFSDemo hd = new HDFSDemo();
    // hd.upload();
    // hd.mkdirs();
    // hd.delete();
    // hd.cat();
    // hd.streamUp();
    // hd.streanLoca();
    hd.download();
}

 
 

 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104

}