windows下操作HDFS

windows上部署hadoop包

1：部署win版本hadoop包
2：整和lib包（包括common,hdfs,mapreduce,yarn,tools（lib,.））下所有的jar包
3：将windows版本hadoop/bin/hadoop.dll 放到c:/windows/system32下

windows环境变量配置

hadoop的bin和sbin目录放PATH中
HADOOP_HOME+HADOOP_USER_NAME=root

HADOOP_HOME是自己hadoop包的位置

idea-java操作HDFS

将之前整合的jar包，添加到项目中图示lib文件夹下。将linux集群配置的core-site.xml和hdfs-site.xml文件拷贝到resources文件夹下。

配置pom.xml文件

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.hsd.counter</groupId>
    <artifactId>hdfs-api-exise</artifactId>
    <version>1.0-SNAPSHOT</version>
    <!-- 添加下述文件 -->
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <hadoop.version>2.6.5</hadoop.version>
    </properties>
    <dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>
    </dependencies>
</project>

package com.xpu.hdfs.test;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import sun.nio.ch.IOUtil;

import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;

public class HdfsTest {
    private Configuration conf;
    private FileSystem fileSystem;
    @Before
    public void before() throws Exception {
//        System.out.println("before");
        conf = new Configuration(true);
        //设置block大小为1M，上传文件使用
        conf.set("dfs.blocksize","1048576");
        fileSystem = FileSystem.get(conf);
    }
    @After
    public void after() throws Exception {
//        System.out.println("after");
        fileSystem.close();
    }
    @Test
    public void test(){
//        System.out.println("hello world");
    }
    //查看目录下有什么
    @Test
    public void testLs() throws Exception {
        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
        for(FileStatus fs : fileStatuses){
//            System.out.println(fs);
            System.out.println(fs.getPath());
        }
    }
    //创建目录
    @Test
    public void testAddDir() throws Exception {
        boolean mkdirs = fileSystem.mkdirs(new Path("/usr/local/hadoop"));
        System.out.println(mkdirs ? "创建成功" : "创建失败");
    }
    //上传文件
    @Test
    public void testAddFile() throws IOException {
        FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/usr/local/hadoop/log4j.properties"));
        FileInputStream fis = new FileInputStream("F:\\开发实战学习\\hdfspro\\src\\main\\resources\\log4j.properties");

        byte[] buf = new byte[1024];
        int len = -1 ;
        while((len =fis.read(buf))!=-1){
            fsDataOutputStream.write(buf,0,len);
        }
        fis.close();
        fsDataOutputStream.flush();
        fsDataOutputStream.close();
    }
	//工具类上传文件
    @Test
    public void testAddFile2() throws IOException {
        FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/usr/local/hadoop/dataPdf"));
        FileInputStream fis = new FileInputStream("F:\\参考资料\\数据生成\\生成模型理论与应用研究_淦艳.caj");
        IOUtils.copyBytes(fis,fsDataOutputStream,conf);
        fis.close();
        fsDataOutputStream.flush();
        fsDataOutputStream.close();
    }
    @Test
    public void testUpdate(){

    }
    @Test
    public void deleteFile(){

    }
    //删除目录
    @Test
    public void deleteDir() throws Exception {
        boolean delete = fileSystem.delete(new Path("/usr/local/hadoop"),false);//boolean表示目录是否有数据，迭代删除
        System.out.println(delete ? "ok" : "no ok");
    }
	//下载文件 fos是下载后文件保存的地址
    @Test
    public void download() throws IOException {
        FSDataInputStream open = fileSystem.open(new Path("/usr/local/hadoop/dataPdf"));
        FileOutputStream fos = new FileOutputStream("F:\\参考资料\\数据生成\\生成模型理论与应用研究_淦艳111.caj");

        byte[] buf = new byte[1024];
        int len =-1 ;
        while((len = open.read(buf))!=-1){
            fos.write(buf,0,len);
        }
        open.close();
        fos.flush();
        fos.close();
    }
	//工具类下载
    @Test
    public void download2() throws IOException {
       /* FSDataInputStream open = fileSystem.open(new Path("/usr/local/hadoop/dataPdf"));
        FileOutputStream fos = new FileOutputStream("F:\\参考资料\\数据生成\\生成模型理论与应用研究_淦艳2222.caj");

        IOUtils.copyBytes(open,fos,conf);
        open.close();
        fos.flush();
        fos.close();*/
        FSDataInputStream open = fileSystem.open(new Path("/home/root/hello1.txt"));
        FileOutputStream fos = new FileOutputStream("F:\\参考资料\\数据生成\\hello1.txt");
        IOUtils.copyBytes(open,fos,conf);
        open.close();
        fos.flush();
        fos.close();
    }
    //获取block块位置信息
     @Test
     public void getBlockLocations() throws IOException {
         FileStatus fileStatus = fileSystem.getFileStatus(new Path("/home/root/hello1.txt"));
         BlockLocation[] fileBlockLocations = fileSystem.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
         for (BlockLocation fileBlockLocation : fileBlockLocations) {
             System.out.println(fileBlockLocation);
         }
     }
     //读取文件
    @Test
    public void readFile() throws IOException {
        FSDataInputStream fsDataInputStream = fileSystem.open(new Path("/home/root/hello1.txt"));
        //seek是从什么某位置开始，读取字节文件
        fsDataInputStream.seek(0);
        System.out.println(fsDataInputStream.readLine());
    }
}