hadoop-hdfs编程

1、开发环境搭建

一、新建一个普通的java工程

二、引入hdfs相关的jar包

需要引入的jar包：

common下的jar

 

hdfs下的jar

 

2、编写HDFS相关的程序

package com.cvicse.ump.hadoop.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class FileOperation {
    
    //创建文件
    public static void createFile(String dst,byte[] contents) throws Exception{
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(conf);
        Path dstPath = new Path(dst);
        FSDataOutputStream outputStream = fs.create(dstPath);;
        outputStream.write(contents);
        outputStream.close();
        fs.close();
        System.out.println(dst+",文件创建成果");
    }
    
    //上传文件
    public static void uploadFile(String src,String dst) throws Exception{
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(conf);
        Path srcPath = new Path(src);
        Path dstPath = new Path(dst);
        fs.copyFromLocalFile(srcPath, dstPath);
        System.out.println("Upload to "+conf.get("fs.default.name"));
        System.out.println("------list files---------"+"\n");
        FileStatus[] fileStatus = fs.listStatus(dstPath);
        for(FileStatus file:fileStatus){
            System.out.println(file.getPath());
        }
        fs.close();
        
    }
    
    //删除目录
    public static void delete(String filePath)throws Exception{
        
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(conf);
        Path path = new Path(filePath);
        boolean isOk = fs.deleteOnExit(path);
        if(isOk){
            System.out.println("delete OK.");
        }else{
            System.out.println("delete failure.");
        }
        fs.close();
        
    }
    //创建目录
    public static void mkdir(String path)throws Exception{
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(conf);
        Path srcPath = new Path(path);
        boolean isOK = fs.mkdirs(srcPath);
        if(isOK){
            System.out.println("create dir ok!");
        }else{
            System.out.println("create dir failure!");
        }
        fs.close();
    }
    
    //下载文件
    public static void downFile(String src,String dst)throws Exception{
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(conf);
        Path srcPath = new Path(src);
        Path dstPath = new Path(dst);
        
        fs.copyToLocalFile(srcPath, dstPath);
        System.out.println("down load over");
        
    }
    
    public static void main(String[] args) throws Exception {
        /*String dst = args[0];
        byte[] contents = "hello,dyh".getBytes();
        createFile(dst, contents);*/
        
        /*String src = args[0];
        String dst = args[1];
        uploadFile(src, dst);*/
        
        /*String filePath = args[0];
        delete(filePath);*/
        
        /*String path = args[0];
        mkdir(path);*/
        
        String src = args[0];
        String dst = args[1];
        downFile(src, dst);
    }

}

 

导出jar包

 

上传jar到HADOOP运行环境，并执行

执行命令：hadoop jar jar包名字 main函数所在的类

 

 

转载于:https://www.cnblogs.com/dyh004/p/7865629.html