调用相关代码,实现相关hdfs操作
package hdfs;
import java.io.InputStream;
import java.net.URL;
import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.io.IOUtils;
public class App1 {
/**
* 异常:unknown host: chaoren 本机没有解析主机名chaoren
* 在C:\Windows\System32\drivers\etc\hosts文件中添加192.168.80.100
* chaoren(win10中要添加写入权限才能写入)
*/
static final String PATH = "hdfs://chaoren:9000/hello";
public static void main(String[] args) throws Exception {
// 让URL能够解析hdfs协议
URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
URL url = new URL(PATH);
InputStream in = url.openStream();
/**
* @param in
* 输入流
* @param out
* 输出流
* @param buffSize
* 缓冲大小
* @param close
* 在传输结束后是否关闭流
*/
IOUtils.copyBytes(in, System.out, 1024, true);// 读取文件hello中的内容
}
}
package hdfs;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
public class App2 {
static final String PATH = "hdfs://chaoren:9000/";
static final String DIR = "/d1";
static final String FILE = "/d1/hello";
public static void main(String[] args) throws Exception {
FileSystem fileSystem = getFileSystem();
// 创建文件夹 hadoop fs -mkdir /d1
mkDir(fileSystem);
// 上传文件 hadoop fs -put src des
putData(fileSystem);
// 下载文件 hadoop fs -get src des
getData(fileSystem);
// 浏览文件夹 hadoop fs -lsr path
list(fileSystem);
// 删除文件夹 hadoop fs -rmr /d1
remove(fileSystem);
}
private static void remove(FileSystem fileSystem) throws IOException {
fileSystem.delete(new Path(DIR), true);
}
private static void list(FileSystem fileSystem) throws IOException {
FileStatus[] listStatus = fileSystem.listStatus(new Path("/"));
for (FileStatus fileStatus : listStatus) {
String isDir = fileStatus.isDir() ? "文件夹" : "文件";
String permission = fileStatus.getPermission().toString();
int replication = fileStatus.getReplication();
long len = fileStatus.getLen();
String path = fileStatus.getPath().toString();
System.out.println(isDir + "\t" + permission + "\t" + replication
+ "\t" + len + "\t" + path);
}
}
private static void getData(FileSystem fileSystem) throws IOException {
FSDataInputStream inputStream = fileSystem.open(new Path(FILE));
IOUtils.copyBytes(inputStream, System.out, 1024, true);
}
private static void putData(FileSystem fileSystem) throws IOException,
FileNotFoundException {
FSDataOutputStream out = fileSystem.create(new Path(FILE));
FileInputStream in = new FileInputStream("C:/Users/ahu_lichang/cp.txt");// 斜杠方向跟Windows下是相反的
IOUtils.copyBytes(in, out, 1024, true);
}
private static void mkDir(FileSystem fileSystem) throws IOException {
fileSystem.mkdirs(new Path(DIR));
}
private static FileSystem getFileSystem() throws IOException,
URISyntaxException {
FileSystem fileSystem = FileSystem.get(new URI(PATH),
new Configuration());
return fileSystem;
}
}