Hadoop_day04_HDFS的API操作

1、在 Windows 下配置 Hadoop 环境

1.1 未配置会产生的问题

缺少 winutils.exe

Could not locate executable null \bin\winutils.exe in the hadoop binaries 

缺少 hadoop.dll

Unable to load native-hadoop library for your platform… using builtin-Java classes where applicable  

1.2 配置的步骤

1. 将hadoop2.7.5文件夹拷贝到一个没有中文没有空格的路径下面
2. 在windows上面配置hadoop的环境变量： HADOOP_HOME，并将%HADOOP_HOME%\bin添加到path中
3. 把hadoop2.7.5文件夹中bin目录下的hadoop.dll文件放到系统盘: C:\Windows\System32 目录
4. 关闭windows重启

 

2、导入 Maven 依赖

建议：最好在网络较好的环境下下载，否则会出现下载不完全的情况

<dependencies>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-common</artifactId>
        <version>2.7.5</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>2.7.5</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-hdfs</artifactId>
        <version>2.7.5</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-mapreduce-client-core</artifactId>
        <version>2.7.5</version>
    </dependency>
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>RELEASE</version>
    </dependency>
</dependencies>
<build>
    <plugins>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-compiler-plugin</artifactId>
            <version>3.1</version>
            <configuration>
                <source>1.8</source>
                <target>1.8</target>
                <encoding>UTF-8</encoding>
                <!--    <verbal>true</verbal>-->
            </configuration>
        </plugin>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <version>2.4.3</version>
            <executions>
                <execution>
                    <phase>package</phase>
                    <goals>
                        <goal>shade</goal>
                    </goals>
                    <configuration>
                        <minimizeJar>true</minimizeJar>
                    </configuration>
                </execution>
            </executions>
        </plugin>
    </plugins>
</build>

3、使用 url 方式访问数据（不建议）

package com.cpz.hdfs_api_practice;


import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.junit.Test;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;

public class HDFS_API_Demo1 {

    @Test
    public void demo1() throws IOException {
        // 注册 URL
        URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());

        // 获取文件输入流
        InputStream inputStream = new URL("hdfs://node01:8020/a.txt").openStream();

        // 获取文件输出流
        FileOutputStream outputStream = new FileOutputStream(new File("D:\\1.txt"));

        // 实现文件的拷贝
        IOUtils.copy(inputStream,outputStream);

        // 关闭流
        IOUtils.closeQuietly(inputStream);
        IOUtils.closeQuietly(outputStream);
    }
}

4、使用文件系统方式访问数据

4.1 获取FileSystem（文件系统）的4种方式

注意：一定要确定导入的包是否正确

package com.cpz.hdfs_api_practice;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DFSClient;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;


public class HDFS_API_Demo2 {
    // 第一种方式
    @Test
    public void getFileSystem1() throws IOException {
        Configuration configuration = new Configuration();
        // 指定使用的文件系统类型
        configuration.set("fs.defaultFS","hdfs://node01:8020");
        // 获取指定的文件系统
        FileSystem fileSystem = FileSystem.get(configuration);
        System.out.println(fileSystem);
    }

    // 第二种方式
    @Test
    public void getFileSystem2() throws URISyntaxException, IOException {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"), new Configuration());
        System.out.println(fileSystem);
    }

    // 第三种方式
    @Test
    public void getFileSystem3() throws IOException {
        Configuration configuration = new Configuration();
        // 指定使用的文件系统类型
        configuration.set("fs.defaultFS","hdfs://node01:8020");
        // 获取指定的文件系统
        FileSystem fileSystem = FileSystem.newInstance(configuration);
        System.out.println(fileSystem);
    }

    // 第四种方式
    @Test
    public void getFileSystem4() throws URISyntaxException, IOException {
        FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://node01:8020"), new Configuration());
        System.out.println(fileSystem);
    }


}

4.2 常用操作

public class HDFS_API_Demo3 {

    // 遍历所有文件
    @Test
    public void listFiles() throws URISyntaxException, IOException {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"), new Configuration());
        RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator = fileSystem.listFiles(new Path("/"), true);
        while (locatedFileStatusRemoteIterator.hasNext()){
            LocatedFileStatus locatedFileStatus = locatedFileStatusRemoteIterator.next();
            System.out.println("绝对路径为：" + locatedFileStatus.getPath() + "  ，名称为：" + locatedFileStatus.getPath().getName());
        }
    }

    // 创建文件夹和文件
    @Test
    public void create() throws URISyntaxException, IOException {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"), new Configuration());
        fileSystem.mkdirs(new Path("/dir3"));
        fileSystem.create(new Path("/dir3/a.txt"));

        fileSystem.close();
    }

    // 下载文件
    @Test
    public void download() throws URISyntaxException, IOException {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"), new Configuration());
        fileSystem.copyToLocalFile(new Path("/a.txt"),new Path("d:\\test_download"));

        fileSystem.close();
    }

    // 文件上传
    @Test
    public void upload() throws URISyntaxException, IOException {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"), new Configuration());
        fileSystem.copyFromLocalFile(new Path("d:\\111.jpg"),new Path("/dir3/"));

        fileSystem.close();
    }

    // 删除操作
    @Test
    public void delete() throws URISyntaxException, IOException {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"), new Configuration());
        // 第二个参数表示是否递归删除
        fileSystem.delete(new Path("/dir3"),true);

        fileSystem.close();
    }

    // 其他操作
    @Test
    public void other() throws URISyntaxException, IOException {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"), new Configuration());
        RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator = fileSystem.listFiles(new Path("/"), true);
        while (locatedFileStatusRemoteIterator.hasNext()){
            LocatedFileStatus fileStatus = locatedFileStatusRemoteIterator.next();
            long blockSize = fileStatus.getBlockSize();
            System.out.println(blockSize);
            BlockLocation[] blockLocations = fileStatus.getBlockLocations();
            int length = blockLocations.length;
            System.out.println(length);
        }

    }

    // 小文件合并
    @Test
public void mergeFile() throws  Exception{
    //获取分布式文件系统
    FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"), new Configuration(),"root");
    FSDataOutputStream outputStream = fileSystem.create(new Path("/bigfile.txt"));
    //获取本地文件系统
    LocalFileSystem local = FileSystem.getLocal(new Configuration());
    //通过本地文件系统获取文件列表，为一个集合
    FileStatus[] fileStatuses = local.listStatus(new Path("file:///E:\\input"));
    for (FileStatus fileStatus : fileStatuses) {
        FSDataInputStream inputStream = local.open(fileStatus.getPath());
        IOUtils.copy(inputStream,outputStream);
        IOUtils.closeQuietly(inputStream);
    }
    IOUtils.closeQuietly(outputStream);
    local.close();
    fileSystem.close();
}
}