在hadoop中,有三大法宝——HDFS,MapReduce,Hbase,但是无论是MapReduce,Hbase还是hadoop中的其他组件如:Hive等他们要处理的数据还是处理完了的数据都是存储在HDFS中。可见HDFS可以说是hadoop存储的基础和核心,因此对HDFS的文件读写操作显得十分重要。
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class Test1 {
/*
* 往hdfs中写数据
*/
public static void writeToHdfs(String filename,String text){
Configuration configuration=new Configuration();
FSDataOutputStream out=null;
String charset="UTF-8";
try {
FileSystem fSystem=FileSystem.get(URI.create(filename),configuration);
Path path=new Path(filename);
if(!fSystem.exists(path)){
//创建文件数据的输出流
out=fSystem.create(new Path(filename));
//通过输出流往hdfs中写入数据
out.write(text.getBytes(charset),0,text.getBytes(charset).length);
out.write("\n".getBytes(charset),0,"\n".getBytes(charset).length);
out.flush();
}else{
//往文件中追加数据
out=fSystem.append(path);
out.write(text.getBytes(charset),0,text.getBytes(charset).length);
out.write("\n".getBytes(charset),0,"\n".getBytes(charset).length);
out.flush();
}
} catch (IOException e) {
e.printStackTrace();
}finally{
//关闭输出流
if(out!=null){
try {
out.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
/*
* 从hdfs中读取数据
*/
public static void readFromHdfs(String fileName){
Configuration conf=new Configuration();
Path filePath=new Path(fileName);
try {
FileSystem fs=FileSystem.get(URI.create(fileName),conf);
if(fs.exists(filePath)){
String charset="UTF-8";
//打开文件数据输入流
FSDataInputStream fsDataInputStream=fs.open(filePath);
//创建文件输入
InputStreamReader inputStreamReader=new InputStreamReader(fsDataInputStream,charset);
String line=null;
//把数据读入到缓冲区中
BufferedReader reader=null;
reader=new BufferedReader(inputStreamReader);
//从缓冲区中读取数据
while((line=reader.readLine())!=null){
System.out.println("line="+line);
}
}
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* @param args
*/
public static void main(String[] args) {
//String filename="hdfs://119.29.167.178:9000/test";
String filename="/test/file1";
String text="我们很6666";
writeToHdfs(filename,text);
writeToHdfs(filename, "5555555");
readFromHdfs(filename);
}
}
运行结果图:
往hdfs中写数据:
读取hdfs上的数据: