Java API、HDFS Shell常用的HDFS操作

•创建和删除文件夹

./bin/hdfs dfs -mkdir /folder
./bin/hdfs dfs -rm -r /folder

•向HDFS中上传任意文本文件，如果指定的文件在HDFS中已经存在，由用户指定是追加到原有文件末尾还是覆盖原有的文件；

#将本地文件test.txt上传到HDFS根目录中
./bin/hdfs dfs -put /home/hadoop/Downloads/test.txt /	
#将本地文件覆盖HDFS文件
./bin/hdfs dfs -copyFromLocal -f /home/hadoop/Downloads/test.txt /test.txt	
#将本地文件追加到HDFS文件末尾
./bin/hdfs dfs -appendToFile /home/hadoop/Downloads/test.txt /test.txt	

• 从HDFS中下载指定文件，如果本地文件与要下载的文件名称相同，则自动对下载的文件重命名；
• 将HDFS中指定文件的内容输出到终端中；

./bin/hdfs dfs -cat /test.txt	#显示test.txt文件的内容

• 显示HDFS中指定的文件的读写权限、大小、创建时间、路径等信息；

./bin/hdfs dfs -ls -h /test.txt	#显示test.txt文件的信息

• 给定HDFS中某一个目录，递归输出该目录下所有文件的读写权限、大小、创建时间、路径等信息；

./bin/hdfs dfs -ls -R -h /	#显示HDFS根目录下所有文件的信息

• 提供一个HDFS内的文件的路径，对该文件进行创建和删除操作。如果文件所在目录不存在，则自动创建目录，删除目录时，由用户指定当该目录不为空时是否还删除该目录；

#创建文件
if $(./bin/hdfs dfs -test -d /folder);
	then $(./bin/hdfs dfs -touchz /folder/test.txt);
else $(./bin/hdfs dfs -mkdir -p /folder &&./bin/hdfs dfs -touchz /folder/test.txt); 
fi

./bin/hadoop fs -rmdir /folder	#只删除非空目录
./bin/hdfs dfs -rm -r /folder	#递归删除目录

• 在HDFS中，将文件从源路径移动到目的路径。

./bin/hdfs dfs -mv /test.txt /folder	#将test文件移动到folder文件夹下

Java API：

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.OutputStreamWriter;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Scanner;

import javax.imageio.stream.FileImageOutputStream;
import javax.sound.sampled.AudioFormat.Encoding;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.RemoteIterator;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

public class test2 {
	/**
	 * 
	 * 创建文件夹
	 * 
	 * @throws IOException
	 */
	public static void createFolder(Configuration conf, String folderPath)
			throws IOException {
		FileSystem fs = FileSystem.get(conf);
		if (!fs.exists(new Path(folderPath))) {
			if (fs.mkdirs(new Path(folderPath)))
				System.out.println(folderPath + " 已创建");
		}
		fs.close();
	}

	/**
	 * 
	 * 删除文件(夹)
	 * 
	 * @throws IOException
	 */
	public static void deleteFile(Configuration conf, String filePath)
			throws IOException {

		FileSystem fs = FileSystem.get(conf);

		if (fs.isDirectory(new Path(filePath))) {
			RemoteIterator<LocatedFileStatus> list = fs.listFiles(new Path(
					filePath), true); // 递归查询
			if (list == null) {
				fs.deleteOnExit(new Path(filePath));
			} else {
				System.out.println("文件夹不为空是否删除(Y/N)");
				Scanner scan = new Scanner(System.in);
				String choice = scan.nextLine();
				scan.close();
				if (choice.equals("Y") || choice.equals("y")) {
					fs.delete(new Path(filePath), true);
					System.out.println("删除文件夹完成");
				} else {
					System.out.println("取消删除");
				}
			}

		} else {
			if (fs.exists(new Path(filePath))) {
				if (fs.deleteOnExit(new Path(filePath))) {
					System.out.println(filePath + " 已删除");
				} else {
					System.out.println("删除失败");
				}

			} else {
				System.out.println("文件不存在");
			}
		}

		fs.close();
	}

	/**
	 * 
	 * 判断文件存在
	 * 
	 * @throws IOException
	 */

	public static boolean HDFSFileExist(Configuration conf, String path)
			throws IOException {
		FileSystem fs = FileSystem.get(conf);
		if (fs.exists(new Path(path))) {
			System.out.println(path + " 存在.");
			return true;
		} else {
			System.out.println(path + " 不存在.");
			return false;
		}
	}

	/*
	 * 判断本地文件存在
	 */
	public static boolean localFileExist(String localPath) {
		File file = new File(localPath);
		if (file.exists()) {
			System.out.println(localPath + " 存在.");
			return true;
		} else {
			System.out.println(localPath + " 不存在.");
			return false;
		}
	}

	/**
	 * 
	 * 复制文件到指定路径
	 * 
	 * @throws IOException
	 */

	public static void CopyFromLocalFile(Configuration conf,

	String localFilePath, String hdfsFilePath) throws IOException {
		Path localPath = new Path(localFilePath);
		Path HDFSFilePath = new Path(hdfsFilePath);
		FileSystem fs = FileSystem.get(conf);
		if (!HDFSFileExist(conf, hdfsFilePath)) {
			fs.copyFromLocalFile(false, true, localPath, HDFSFilePath);
			System.out.println(localFilePath + " 已上传至 " + HDFSFilePath);
		} else {
			System.out.println("1.覆盖	2.追加	3.取消");
			Scanner scan = new Scanner(System.in);
			int choice = scan.nextInt();
			if (choice == 1) { // 覆盖
				fs.copyFromLocalFile(false, true, localPath, HDFSFilePath);
				System.out.println(localFilePath + " 已覆盖至 " + HDFSFilePath);
			} else if (choice == 2) { // 追加
				appendToFile(conf, localFilePath, hdfsFilePath);
				System.out.println(localFilePath + " 已追加至 " + HDFSFilePath);
			} else {
			}
			scan.close();
		}

	}

	/**
	 * 
	 * 追加文件
	 * 
	 * @throws IOException
	 */

	public static void appendToFile(Configuration conf, String localFilePath,

	String HDFSFilePath) throws IOException {

		FileSystem fs = FileSystem.get(conf);
		System.out.println("1.追加到文件末尾	2.追加到文件开头");
		Scanner scan = new Scanner(System.in);
		int choice = scan.nextInt();
		scan.close();
		if (choice == 2) {

			FSDataInputStream in = fs.open(new Path(HDFSFilePath));
			BufferedWriter out = new BufferedWriter(new OutputStreamWriter(
					new FileOutputStream(localFilePath, true)));
			File file = new File(localFilePath);
			byte[] data = new byte[1024];
			FileOutputStream fs2 = new FileOutputStream(file, true);// true->追加
			int len = 0;// 用来保存实际读到的字节数

			while ((len = in.read(data)) != -1)// 没有读到文件的末尾
			{
				fs2.write(data, 0, len);// 实际读到的字节数

			}
			fs.copyFromLocalFile(false, true, new Path(localFilePath),
					new Path(HDFSFilePath));
			System.out.println("已追加到文件开头");
			fs2.close();
			in.close();
			out.close();
		} else {
			FileInputStream in = new FileInputStream(localFilePath);

			FSDataOutputStream out = fs.append(new Path(HDFSFilePath));

			byte[] data = new byte[1024];

			int read = -1;

			while ((read = in.read(data)) > 0) {

				out.write(data, 0, read);

			}
			System.out.println("已追加到文件末尾");
			in.close();
			out.close();
		}
	}

	/*
	 * 从HDFS下载
	 */
	public static void CopyFromHdfs(Configuration conf, String localFilePath,
			String HDFSFilePath) throws IOException {

		FileSystem fs = FileSystem.get(conf);
		if (localFileExist(localFilePath)) { // 如果文件存在
			Date date = new Date();
			fs.copyToLocalFile(new Path(HDFSFilePath), new Path(localFilePath
					+ " " + date.getTime()));
			System.out.println("文件已下载并重命名！");
		} else {
			fs.copyToLocalFile(new Path(HDFSFilePath), new Path(localFilePath));
			if (localFileExist(localFilePath))
				System.out.println("文件已下载！");
		}

		fs.close();
	}
	/*
	 * 在HDFS上写入内容
	 */
	public static void HDFSWrite(Configuration conf,String HDFSFilePath) throws IOException{
		FileSystem fs = FileSystem.get(conf);
		FSDataOutputStream fsdataoutputstream = fs.create(new Path(HDFSFilePath));
		System.out.println("请输入内容(输入“!end!”结束)：");
		Scanner scan = new Scanner(System.in);
		while(true){
			if(scan.hasNextLine()){
				String s = scan.nextLine();
				if(s.equals("!end!")) break;
				fsdataoutputstream.write((s+"\n").getBytes());		
			}
		}
		System.out.println("写入完成");
		fsdataoutputstream.close();
		scan.close();
		fs.close();
	}

	// 读取文件至终端
	public static void ReadFile(Configuration conf, String HDFSFilePath)
			throws IOException {

		FileSystem fs = FileSystem.get(conf);
		FSDataInputStream hdfsInStream = fs.open(new Path(HDFSFilePath));
		byte[] ioBuffer = new byte[1024];
		int readLen = hdfsInStream.read(ioBuffer);
		while (readLen != -1) {
			System.out.write(ioBuffer, 0, readLen);
			readLen = hdfsInStream.read(ioBuffer);
		}
		hdfsInStream.close();
		fs.close();
	}

	/*
	 * 展示hdfs上的文件
	 */

	public static void showFilesInHdfs(Configuration conf, String path)

	throws IOException {

		FileSystem fs = FileSystem.get(conf);
		RemoteIterator<LocatedFileStatus> fls = fs.listFiles(new Path(path),
				true);
		if (!fls.hasNext()) {
			System.out.println("null");
		} else {
			while (fls.hasNext()) {
				LocatedFileStatus fl = fls.next();
				System.out.println("path: " + fl.getPath().toString());
				System.out.println("Permission: "
						+ fl.getPermission().toString());
				System.out.println("Group: " + fl.getGroup().toString());
				System.out.println("BlockSize: " + fl.getBlockSize());
				System.out.println("Owner: " + fl.getOwner().toString());
				System.out.println("Replication: " + fl.getReplication());

				System.out.println("ModificationTime: "
						+ fl.getModificationTime());

				BlockLocation[] bls = fl.getBlockLocations();
				for (BlockLocation bl : bls) {
					for (String loc : bl.getHosts())
						System.out.println("located in datanode :" + loc);
				}
				System.out.println();
			}
		}

		fs.close();
	}

	/*
	 * 移动HDFS文件
	 */
	public static void MoveHDFSFile(Configuration conf,

	String src, String dst) {

		try (FileSystem fs = FileSystem.get(conf)) {

			Path Src = new Path(src);

			Path Dst = new Path(dst);

			fs.rename(Src, Dst); // 文件移动
			System.out.println("文件已移动到" + dst);

		} catch (IOException e) {

			e.printStackTrace();

		}

	}

	/**
	 * 
	 * 主函数
	 * 
	 * @throws IOException
	 */

	public static void main(String[] args) throws IOException {
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://localhost:9000");
		conf.setBoolean(
				"dfs.client.block.write.replace-datanode-on-failure.enable",
				true);
		conf.set("dfs.client.block.write.replace-datanode-on-failure.policy",
				"NEVER");
		String localFilePath = "/home/hadoop/Downloads/word22.txt"; // 本地
		String HDFSFilePath = "/folder11/folder/test2.txt"; // HDFS

		// CopyFromLocalFile(conf,localFilePath,HDFSFilePath);
		// CopyFromHdfs(conf,localFilePath,HDFSFilePath);
		// ReadFile(conf,HDFSFilePath);
		// createFolder(conf,"/folder5/folder1");
		// deleteFile(conf, "/folder11/folder/test1.txt");
		// showFilesInHdfs(conf,HDFSFilePath);
		MoveHDFSFile(conf, "/folder11/folder/test.txt", "/test.txt");

	}
}