package com.shijia.HDFS;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.IOException;
public class Demo01TestAPI {
public static void main(String[] args) throws IOException {
//创建一个配置文件
Configuration conf = new Configuration();
//设置NameNode的地址
conf.set("fs.defaultFS","hdfs://master:9000");
//与NameNode建立连接
FileSystem fs = FileSystem.get(conf);
System.out.println(fs.exists(new Path("/data")));
System.out.println(fs.exists(new Path("/data1")));
//关闭连接
fs.close();
}
}
package com.shijia.HDFS;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.*;
public class Demo02HDFSAPI {
FileSystem fs;
@Before
public void init() throws IOException {
//创建一个配置文件
Configuration conf = new Configuration();
//设置NameNode的地址
conf.set("fs.defaultFS", "hdfs://master:9000");
//设置副本数量
conf.set("dfs.replication","1");
//与NameNode建立连接
fs = FileSystem.get(conf);
}
@Test
//判断路径存在不存在
public void exists() throws IOException {
Path path = new Path("/data");
System.out.println(fs.exists(path));
}
@Test
//创建一个TestAPI/test1目录
public void mkdirs() throws IOException {
fs.mkdirs(new Path("/TestAPI/test1"));
fs.mkdirs(new Path("/TestAPI/test2"));
}
@Test
public void delete() throws IOException {
fs.delete(new Path("/TestAPI/test2"),true);
}
@Test
//上传文件到TestAPI/test1
public void copyFromLocalFile() throws IOException {
fs.copyFromLocalFile(new Path("data/students.txt"),new Path("/TestAPI/test1"));
}
@Test
//上传文件到/TestAPI/test1,并移除本地文件
public void moveFromLocalFile() throws IOException {
fs.moveFromLocalFile(new Path("data/students.txt"), new Path("/TestAPI/test1/students_new.txt"));
}
@Test
//将HDFS上的文件下载到本地,并移除HdFS的文件
public void moveToLocalFile() throws IOException {
fs.moveToLocalFile(new Path("/TestAPI/test1/students.txt"),new Path("data/"));
}
@Test
//读取HDFS文件
public void readHDFSFile() throws IOException {
FSDataInputStream fsDataInputStream = fs.open(new Path("/TestAPI/test1/students_new.txt"));
BufferedReader br = new BufferedReader(new InputStreamReader(fsDataInputStream));
String line;
while((line=br.readLine())!=null){
System.out.println(line);
}
br.close();
fsDataInputStream.close();
}
@Test
//将数据写入HDFS
public void writeHDFSFlie() throws IOException {
FSDataOutputStream fsDataOutputStream;
//如果文件存在,追加
//如果文件不存在,则创建
Path path = new Path("/TestAPI/test1/newFile.txt");
if(fs.exists(path)){
System.out.println("文件已存在,进行追加");
fsDataOutputStream= fs.append(path);
}else{
System.out.println("创建文件");
fsDataOutputStream=fs.create(path);
}
//写入文件
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream));
bw.write("hjm,2018110710");
bw.newLine();
bw.write("dxb,2018110702");
bw.newLine();
bw.write("zzj,2018110755");
bw.newLine();
bw.write("wyc,2018110734");
bw.newLine();
bw.write("yzw,2018110743");
bw.newLine();
bw.flush();
bw.close();
fsDataOutputStream.close();
}
@Test
//查看/TestAPI/test1目录下的文件
public void listStatus() throws IOException {
FileStatus[] fileStatuses = fs.listStatus(new Path("/TestAPI/test1"));
for (FileStatus fileStatus : fileStatuses) {
System.out.println(fileStatus.getPath());
}
}
@After
public void close() throws IOException {
fs.close();
}
}