前面已经搭建了hadoop的完全分布式,本文介绍分布式文件系统的常用操作
进入hadoop安装目录的bin目录下,我的路径是;/soft/hadoop/bin
1 创建文件夹
格式:hdfs dfs -mkdir -p 文件夹名称
hdfs dfs -mkdir -p /user/centos/hadoop
2 查看文件
格式:hdfs dfs -ls 查看路径
(/ :表示根路径)
hdfs dfs -ls /user
hdfs dfs -ls -R /user 递归查看
3 上传文件
格式:hdfs dfs -put 本地文件路径 要上传到的文件路径
hdfs dfs -put index.html /user/centos/hadoop
4 下载文件
格式:hdfs dfs -get 要下载的文件路径 下载后的文件路径
hdfs dfs -get /user/centos/hadoop/index.html a.html
这里也可以下载
5 删除文件
格式 :hdfs dfs -rm -r -f 要删除的文件路径
hdfs dfs -rm -r -f /user/centos/hadoop
JAVA 操作 HDFS
需要hadoop的jar包支持,楼主这里就不贴出来了
package hdfs;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.Test;
public class TestHDFS {
/**
* javazIO 读取hdfs文件
*/
@Test
public void readFile() throws Exception{
//注册url流处理器工厂(hdfs)
URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
URL url = new URL("hdfs://192.168.1.38:8020/user/centos/hadoop/index.html");
URLConnection conn = url.openConnection();
InputStream is = conn.getInputStream();
byte[] buf = new byte[is.available()];
is.read(buf);
is.close();
String str = new String(buf);
System.out.println(str);
}
/**
* 通过hadoop API访问文件
*/
@Test
public void readFileByAPI() throws Exception{
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://192.168.1.38:8020/");
FileSystem fs = FileSystem.get(conf) ;
Path p = new Path("/user/centos/hadoop/index.html");
FSDataInputStream fis = fs.open(p);
byte[] buf = new byte[1024];
int len = -1 ;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
while((len = fis.read(buf)) != -1){
baos.write(buf, 0, len);
}
fis.close();
baos.close();
System.out.println(new String(baos.toByteArray()));
}
/**
* 通过hadoop API访问文件2
*/
@Test
public void readFileByAPI2() throws Exception{
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://192.168.1.38:8020/");
FileSystem fs = FileSystem.get(conf) ;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
Path p = new Path("/user/centos/hadoop/index.html");
FSDataInputStream fis = fs.open(p);
IOUtils.copyBytes(fis, baos, 1024);
System.out.println(new String(baos.toByteArray()));
}
/**
* 创建文件夹
* 如果创建失败,给/user/centos 这个路径添加777 权限: hdfs dfs -chmod 777 /user/centos
*/
@Test
public void mkdir() throws Exception{
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://192.168.1.38:8020/");
FileSystem fs = FileSystem.get(conf) ;
fs.mkdirs(new Path("/user/centos/myhadoop"));
}
/**
* 创建文件,并添加内容
*/
@Test
public void putFile() throws Exception{
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://192.168.1.38:8020/");
FileSystem fs = FileSystem.get(conf) ;
FSDataOutputStream out = fs.create(new Path("/user/centos/myhadoop/a.txt"));
out.write("helloworld".getBytes());
out.close();
}
/**
* 删除文件
*/
@Test
public void removeFile() throws Exception{
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://192.168.1.38:8020/");
FileSystem fs = FileSystem.get(conf) ;
Path p = new Path("/user/centos/myhadoop");
fs.delete(p, true);
}
}