下面代码:封装了,实现创建目录,创建文件,增,删,改文件,获取节点,上传文件,下载文件,以流的方式上传视频
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
public class HdfsDemo {
//创建新文件
public static void CreateFile(String url,String path){
try {
//该类的对象封转了客户端或者服务器的配置。
Configuration config = new Configuration();
URI uri = new URI("hdfs://192.168.52.140:9000");
//该类的对象是一个文件系统对象,可以用该对象的一些方法来对文件进行操作。
FileSystem fs = FileSystem.get(uri, config);
//目标路径
Path p = new Path("hdfs://192.168.52.140:9000/dirs");
fs.create(p, new Progressable() {
@Override
public void progress() {
System.out.print(">>");
}
});
System.out.println("create dir successful!!!");
} catch (URISyntaxException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
//判断文件是否存在于Hdfs
public static boolean IfFileExits(String FileName)throws Exception
{
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(FileName),conf);
Path path = new Path(FileName);
boolean isExists = fs.exists(path);
return isExists;
}
//删除文件
public static void delete(String url,String filePath) throws IOException{
try {
Configuration conf = new Configuration();
URI uri = new URI(url);
FileSystem fs = FileSystem.get(uri,conf);
Path path = new Path(filePath);
boolean isok = fs.deleteOnExit(path);
if(isok){
System.out.println("delete ok!");
}else{
System.out.println("delete failure");
}
fs.close();
} catch (IllegalArgumentException e) {
e.printStackTrace();
} catch (URISyntaxException e) {
e.printStackTrace();
}
}
//上传本地文件
public static void uploadFile(String url,String src,String dst) throws IOException{
try {
Configuration conf = new Configuration();
URI uri = new URI(url);
FileSystem fs = FileSystem.get(uri,conf);
Path srcPath = new Path(src); //原路径
Path dstPath = new Path(dst); //目标路径
//调用文件系统的文件复制函数,前面参数是指是否删除原文件,true为删除,默认为false
fs.copyFromLocalFile(false,srcPath, dstPath);
//打印文件路径
System.out.println("Upload to "+conf.get("fs.default.name"));
System.out.println("------------list files------------"+"\n");
FileStatus [] fileStatus = fs.listStatus(dstPath);
for (FileStatus file : fileStatus)
{
System.out.println(file.getPath());
}
fs.close();
} catch (IllegalArgumentException e) {
e.printStackTrace();
} catch (URISyntaxException e) {
e.printStackTrace();
}
}
//从hdfs下载文件
public static void downFile(String url,String src,String dst) throws IOException{
try {
Configuration conf = new Configuration();
URI uri = new URI(url);
FileSystem fs = FileSystem.get(uri,conf);
Path srcPath = new Path(src); //原路径
Path dstPath = new Path(dst); //目标路径
//调用文件系统的文件复制函数,前面参数是指是否删除原文件,true为删除,默认为false
fs.copyToLocalFile(false,srcPath, dstPath);
//打印文件路径
System.out.println("Upload to "+conf.get("fs.default.name"));
System.out.println("------------list files------------"+"\n");
FileStatus [] fileStatus = fs.listStatus(dstPath);
for (FileStatus file : fileStatus)
{
System.out.println(file.getPath());
}
fs.close();
} catch (IllegalArgumentException e) {
} catch (URISyntaxException e) {
}
}
//文件重命名
public static void rename(String url,String oldName,String newName) throws IOException{
try {
Configuration conf = new Configuration();
URI uri = new URI(url);
FileSystem fs = FileSystem.get(uri,conf);
Path oldPath = new Path(oldName);
Path newPath = new Path(newName);
boolean isok = fs.rename(oldPath, newPath);
if(isok){
System.out.println("rename ok!");
}else{
System.out.println("rename failure");
}
fs.close();
} catch (IllegalArgumentException e) {
e.printStackTrace();
} catch (URISyntaxException e) {
e.printStackTrace();
}
}
//创建目录
public static void mkdir(String url,String path) throws IOException{
try {
Configuration conf = new Configuration();
URI uri = new URI(url);
FileSystem fs = FileSystem.get(uri,conf);
Path srcPath = new Path(path);
boolean isok = fs.mkdirs(srcPath);
if(isok){
System.out.println("create dir ok!");
}else{
System.out.println("create dir failure");
}
fs.close();
} catch (IllegalArgumentException e) {
e.printStackTrace();
} catch (URISyntaxException e) {
e.printStackTrace();
}
}
//上传视频
public static void shangchaungshiping(){
Configuration conf=new Configuration();
URI uri=new URI("hdfs://192.168.61.128:9000");
FileSystem hdfs=FileSystem.get(uri,conf);
FileSystem local=FileSystem.getLocal(conf);//获取windows下的文件管理系统
Path inputDir=new Path("F:\\1\\love.mkv");
Path hdfsFile=new Path("/vdio");
hdfs.mkdirs(hdfsFile);
FSDataOutputStream out;
FileStatus[] inputFiles=local.listStatus(inputDir);
//通过OutputStream.write()来将视频文件循环写入HDFS下的指定目录
for(int i=0;i<inputFiles.length;i++){
System.out.println("*******"+inputFiles[i].getPath().getName());
FSDataInputStream in=local.open(inputFiles[i].getPath());
out=hdfs.create(new Path("/vdio/"+inputFiles[i].getPath().getName()));
byte buffer[]=new byte[256];
int bytesRead=0;
while((bytesRead=in.read(buffer))>0){
out.write(buffer,0,bytesRead);
}
out.close();
in.close();
File file=new File(inputFiles[i].getPath().toString());
file.delete();
}
}
//读取文件的内容
public static void readFile(String url,String filePath) throws IOException{
try {
Configuration conf = new Configuration();
URI uri = new URI(url);
FileSystem fs = FileSystem.get(uri,conf);
Path srcPath = new Path(filePath);
InputStream in = null;
try {
in = fs.open(srcPath);
IOUtils.copyBytes(in, System.out, 4096, false); //复制到标准输出流
} finally {
IOUtils.closeStream(in);
}
} catch (IllegalArgumentException e) {
e.printStackTrace();
} catch (URISyntaxException e) {
e.printStackTrace();
}
}
public static void GetLocation(String url,String FileName)throws Exception//查看想看的文件存放在哪几个节点当中
{
Configuration conf = new Configuration();
URI uri = new URI(url);
FileSystem fs = FileSystem.get(uri,conf);
Path path = new Path(FileName);
FileStatus fileStatus = fs.getFileStatus(path);
BlockLocation[] blkLocations = fs.getFileBlockLocations(fileStatus , 0, fileStatus.getLen());
int blkCount = blkLocations.length;
for (int i=0; i < blkCount; i++) {
String[] hosts = blkLocations[i].getHosts();
System.out.print(blkLocations[i].getHosts());
System.out.print(hosts[i]);
// Do something with the block hosts
}
}
//显示Hdfs一组路径的文件信息
public static void ListStatus(String args[]) throws Exception
{
String uri = args[0];
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri),conf);
Path[] paths = new Path[args.length];
for(int i=0;i<paths.length;i++)
{
paths[i] = new Path(args[i]);
}
FileStatus[] status =fs.listStatus(paths);
Path[] listedPaths = FileUtil.stat2Paths(status);
for(Path p: listedPaths){
System.out.println(p);
}
}
public static void main(String[] args) throws Exception {
String url="hdfs://192.168.52.140:9000";
//CreateFile("hdfs://192.168.52.140:9000","hdfs://192.168.52.140:9000/dirs2");
//delete("hdfs://192.168.52.140:9000","hdfs://192.168.52.140:9000/dirs");
//rename(url,"hdfs://192.168.52.140:9000/dir","hdfs://192.168.52.140:9000/newdir");
// mkdir(url,"hdfs://192.168.52.140:9000/mydir");
String relativelyPath =System.getProperty("user.dir")+"/src/log4j.properties";
String downPath =System.getProperty("user.dir")+"/";
//uploadFile(url,relativelyPath,"hdfs://192.168.52.140:9000/mydir");
//readFile(url,"hdfs://192.168.52.140:9000/mydir/log4j.properties");
//downFile(url,"hdfs://192.168.52.140:9000/mydir/log4j.properties","D:/");
//System.out.println(IfFileExits("hdfs://192.168.52.140:9000/mydir/log4j.properties"));
GetLocation(url,"hdfs://192.168.52.140:9000/mydir/log4j.properties");
}
}