一、 实验目的
1）理解 HDFS 在 Hadoop 体系结构中的角色。
2）熟练使用 HDFS 操作常用的 shell 命令。
3）熟悉 HDFS 操作常用的 Java API。
二、 实验平台
1）操作系统：Linux（Ubuntu18.04）；
2）Hadoop 版本：2.9.0；
3）JDK 版本：1.8；
4）Java IDE：Eclipse 3.8。
三、 实验内容
编程实现以下功能，并利用 Hadoop 提供的 Shell 命令完成相同任务：
1）向 HDFS 中上传任意文本文件，如果指定的文件在 HDFS 中已经存在，则由用户来指定是 追加到原有文件末尾还是覆盖原有的文件；
package lab1;
import java.io.*;
import java.util.*;
import java.net.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
public class test1 {
public static void test1(FileSystem hdfs, Path srcPath, Path desPath) {
try {
if (hdfs.exists(desPath)) {
System.out.println(desPath + "已存在");
System.out.println("覆盖原有文件(y)还是追加到原有文件末尾(n)？(y/n)");
if (new Scanner(System.in).next().equals("y")) {
hdfs.copyFromLocalFile(srcPath, desPath);
System.out.println("已覆盖");
} else {
FileInputStream inputStream = new FileInputStream(srcPath.toString());
FSDataOutputStream out = hdfs.append(desPath);
byte[] bytes = new byte[1024];
int read = -1;
while ((read = inputStream.read(bytes)) > 0) {
out.write(bytes, 0, read);
}
inputStream.close();
out.close();
System.out.println("已追加");
}
} else {
hdfs.copyFromLocalFile(srcPath, desPath);
System.out.println("已写入");
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws IOException {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://localhost:9000");
FileSystem fs = FileSystem.get(conf);
Path path1 = new Path("./src/lab1/test.txt");
Path path2 = new Path("hdfs://localhost:9000/test/test.txt");
test1(fs, path1, path2);
}
}
2）从 HDFS 中下载指定文件，如果本地文件与要下载的文件名称相同，则自动对下载的文件 重命名；
package lab1;
import java.io.*;
import java.util.*;
import java.net.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
public class test2 {
private static void test2(FileSystem hdfs, Path remotePath, Path localPath) {
try {
if (hdfs.exists(remotePath)) {
hdfs.copyToLocalFile(remotePath, localPath);
System.out.println("已下载");
} else {
System.out.println("HDFS中没有此文件");
}
} catch (FileAlreadyExistsException e) {
System.out.println("已存在");
try {
System.out.println(localPath.toString());
hdfs.copyToLocalFile(remotePath, new Path("./src/lab1/test2_" + new Random().nextInt() + ".txt"));
} catch (IOException e1) {
e1.printStackTrace();
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws IOException {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://localhost:9000");
FileSystem fs = FileSystem.get(conf);
Path path1 = new Path("hdfs://localhost:9000/test/test.txt");
Path path2 = new Path("./src/lab1/test2.txt");
test2(fs, path1, path2);
}
}
3）将 HDFS 中指定文件的内容输出到终端中；
package lab1;
import java.io.*;
import java.util.*;
import java.net.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
public class test3 {
public static void test3(FileSystem hdfs, Path srcPath) {
try {
if (!hdfs.exists(srcPath)) {
System.out.println(srcPath + "不存在");
} else {
FSDataInputStream in = hdfs.open(srcPath);
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(in));
String line;
while ((line = bufferedReader.readLine()) != null) {
System.out.println(line);
}
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws IOException {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://localhost:9000");
FileSystem fs = FileSystem.get(conf);
Path path = new Path("hdfs://localhost:9000/test/test.txt");
test3(fs, path);
}
}
4）显示 HDFS 中指定的文件的读写权限、大小、创建时间、路径等信息；
package lab1;
import java.io.*;
import java.util.*;
import java.net.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
import java.text.SimpleDateFormat;//用于格式化日期
public class test4 {
public static void test4(FileSystem hdfs, Path srcPath) {
try {
FileStatus[] fileStatus = hdfs.listStatus(srcPath);
for (FileStatus status : fileStatus) {
System.out.println("读写权限：" + status.getPermission().toString());
System.out.println("大小：" + status.getLen());
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
System.out.println("创建时间：" + format.format(status.getModificationTime()));
System.out.println("路径：" + status.getPath());
}
} catch (FileNotFoundException e) {
System.out.println(srcPath + "不存在");
} catch (IOException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws IOException {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://localhost:9000");
FileSystem fs = FileSystem.get(conf);
Path path = new Path("hdfs://localhost:9000/test/test.txt");
test4(fs, path);
}
}
5）给定 HDFS 中某一个目录，递归输出该目录下的所有文件的读写权限、大小、创建时间、 路径等信息;
package lab1;
import java.io.*;
import java.util.*;
import java.net.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
import java.text.SimpleDateFormat;//用于格式化日期
public class test5 {
public static void test5(FileSystem hdfs, Path srcPath) {
RemoteIterator<LocatedFileStatus> iterator;
try {
iterator = hdfs.listFiles(srcPath, true);
while (iterator.hasNext()) {
FileStatus status = iterator.next();
System.out.println("读写权限：" + status.getPermission().toString());
System.out.println("大小：" + status.getLen());
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
System.out.println("创建时间：" + format.format(status.getModificationTime()));
System.out.println("路径：" + status.getPath());
System.out.println();
}
} catch (FileNotFoundException e) {
System.out.println(srcPath + "不存在");
} catch (IOException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws IOException {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://localhost:9000");
FileSystem fs = FileSystem.get(conf);
Path path = new Path("hdfs://localhost:9000/ex1");
test5(fs, path);
}
}
6）提供一个 HDFS 内的文件的路径，对该文件进行创建和删除操作。如果文件所在目录不存在，则自动创建目录；
package lab1;
import java.io.*;
import java.util.*;
import java.net.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
public class test6 {
public static void test6(FileSystem hdfs, Path filePath) {
String filePath_s = filePath.toString();
String dirPath_s = filePath_s.substring(0, filePath_s.lastIndexOf('/'));
Path dirPath = new Path(dirPath_s);
try {
System.out.println("创建文件(y)还是删除文件(n)？(y/n)");
if (new Scanner(System.in).next().equals("y")) {
if (!hdfs.exists(dirPath)) {
System.out.println(dirPath + "不存在");
hdfs.mkdirs(dirPath);
System.out.println(dirPath + "已创建");
}
hdfs.create(filePath);
System.out.println(filePath + "已创建");
} else {
if (!hdfs.exists(filePath)) {
System.out.println(filePath + "不存在");
} else {
hdfs.delete(filePath, true);
System.out.println(filePath + "已删除");
}
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws IOException {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://localhost:9000");
FileSystem fs = FileSystem.get(conf);
Path path = new Path("hdfs://localhost:9000/test6/cre.txt");
test6(fs, path);
}
}
7）提供一个 HDFS 的目录的路径，对该目录进行创建和删除操作。创建目录时，如果目录文件所在目录不存在，则自动创建相应目录；删除目录时，当该目录为空时删除，当该目录不为空时不删除该目录；
package lab1;
import java.io.*;
import java.util.*;
import java.net.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
public class test7 {
public static void test7(FileSystem hdfs, Path dirPath) {
try {
System.out.println("创建目录(y)还是删除目录(n)？(y/n)");
if (new Scanner(System.in).next().equals("y")) {
if (hdfs.exists(dirPath)) {
System.out.println(dirPath + "已存在");
} else {
System.out.println(dirPath + "不存在");
hdfs.mkdirs(dirPath);
System.out.println(dirPath + "已创建");
}
} else {
if (!hdfs.exists(dirPath)) {
System.out.println(dirPath + "不存在");
} else {
FileStatus[] fileStatus = hdfs.listStatus(dirPath);
if (fileStatus.length == 0) {
hdfs.delete(dirPath, true);
System.out.println(dirPath + "为空，已删除");
} else {
System.out.println(dirPath + "非空，故不删除");
for (FileStatus status : fileStatus) {
System.out.println(status.getPath());
}
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws IOException {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://localhost:9000");
FileSystem fs = FileSystem.get(conf);
Path path = new Path("hdfs://localhost:9000/test");
test7(fs, path);
}
}
8）向 HDFS 中指定的文件追加内容，由用户指定内容追加到原有文件的开头或结尾；
package lab1;
import java.io.*;
import java.util.*;
import java.net.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
public class test8 {
private static void test8(FileSystem hdfs, Path remotePath, Path localPath) {
try {
if (!hdfs.exists(remotePath)) {
System.out.println("文件不存在");
return;
}
System.out.println("追加到开头(y)还是追加到结尾(n)？(y/n)");
if (new Scanner(System.in).next().equals("y")) {
Path path_tmp = new Path("./src/lab1/tmp.txt");
hdfs.moveToLocalFile(remotePath, path_tmp);
FSDataOutputStream fsDataOutputStream = hdfs.create(remotePath);
FileInputStream fileInputStream = new FileInputStream(localPath.toString());
FileInputStream fileInputStream1 = new FileInputStream("./src/lab1/tmp.txt");
byte[] bytes = new byte[1024];
int read = -1;
while ((read = fileInputStream.read(bytes)) > 0) {
fsDataOutputStream.write(bytes, 0, read);
}
while ((read = fileInputStream1.read(bytes)) > 0) {
fsDataOutputStream.write(bytes, 0, read);
}
fileInputStream.close();
fileInputStream1.close();
fsDataOutputStream.close();
System.out.println("已追加到开头");
} else {
FileInputStream inputStream = new FileInputStream(localPath.toString());
FSDataOutputStream outputStream = hdfs.append(remotePath);
byte[] bytes1 = new byte[1024];
int read1 = -1;
while ((read1 = inputStream.read(bytes1)) > 0) {
outputStream.write(bytes1, 0, read1);
}
inputStream.close();
outputStream.close();
System.out.println("已追加到结尾");
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws IOException {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://localhost:9000");
FileSystem fs = FileSystem.get(conf);
Path path1 = new Path("hdfs://localhost:9000/test/test.txt");
Path path2 = new Path("./src/lab1/test2.txt");
test8(fs, path1, path2);
}
}
9）删除 HDFS 中指定的文件；
package lab1;
import java.io.*;
import java.util.*;
import java.net.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
public class test9 {
private static void test9(FileSystem hdfs, Path remotePath) {
try {
if (hdfs.delete(remotePath, true)) {
System.out.println("删除成功");
} else {
System.out.println("删除失败");
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws IOException {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://localhost:9000");
FileSystem fs = FileSystem.get(conf);
Path path = new Path("hdfs://localhost:9000/test/aa.txt");
test9(fs, path);
}
}
10）在 HDFS 中，将文件从源路径移动到目的路径。
package lab1;
import java.io.*;
import java.util.*;
import java.net.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
public class test10 {
private static void test10(FileSystem hdfs, Path oldRemotePath, Path newRemotePath) {
try {
if (hdfs.rename(oldRemotePath, newRemotePath)) {
System.out.println("移动成功");
} else {
System.out.println("移动失败");
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws IOException {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://localhost:9000");
FileSystem fs = FileSystem.get(conf);
Path path1 = new Path("hdfs://localhost:9000/test/a.txt");
Path path2 = new Path("hdfs://localhost:9000/test/test2/a.txt");
test10(fs, path1, path2);
}
}
以任务8为例，通过编写Java程序，向 HDFS 中指定的文件追加内容，由用户指定内容追加到原有文件的开头或结尾，截图如下：
追加到开头前：
追加到开头后：
控制台输出：
1万+
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
