Hadoop-day05(java操作hadoop)

Java操作Hadoop

---

步骤一：创建父类maven项目

在父类项目中的 pom 文件中配置如下，同一版本号，然后子类供给子类使用

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>org.example</groupId>
    <artifactId>java-hadoop</artifactId>
    <packaging>pom</packaging>
    <version>1.0-SNAPSHOT</version>
    <modules>
        <module>hadoop-hdfs</module>
    </modules>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <hadoop-version>2.7.6</hadoop-version>
    </properties>
    <dependencyManagement>
        <dependencies>
            <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-common</artifactId>
                <version>${hadoop-version}</version>
            </dependency>
            <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-client</artifactId>
                <version>${hadoop-version}</version>
            </dependency>
            <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-hdfs</artifactId>
                <version>${hadoop-version}</version>
            </dependency>
        </dependencies>
    </dependencyManagement>
</project>

步骤二：创建子的maven项目

创建子类 hadoop-hdfs项目，用来操作hadoop中的文件，现在子类中的pom文件中配置

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <parent>
        <artifactId>java-hadoop</artifactId>
        <groupId>org.example</groupId>
        <version>1.0-SNAPSHOT</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

    <artifactId>hadoop-hdfs</artifactId>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
    </properties>
    <dependencies>
        <dependency>
        <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
        <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-hdfs</artifactId>
        </dependency>
    </dependencies>
</project>

步骤三：连接hadoop进行操作。

package com.shujia;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.yarn.webapp.hamlet.HamletSpec;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
/**
 * @author WangTao
 * @date 2022/5/25 19:53
 */
/*
    java操作HDFS来创建，上传，下载，删除，重命名，查看文件的描述信息

    进阶一下：使用 IO流的方式上传和下载
 */
public class JavaToHdfs {
    public static void main(String[] args)throws Exception {
        //1.获取HDFS的文件系统
        Configuration conf = new Configuration();
        //2.指定连接集群的主节点，和以及端口号 /usr/local/soft/hadoop-2.7.6/etc/hadoop/core-site.xml
        conf.set("fs.defaultFS","hdfs://master:9000");
        //如果想要操作HDFS,就必须要获取到HDFS提供抽象的父类实现：FileSystem
        FileSystem fs = FileSystem.get(conf);
        System.out.println(fs);

        //创建文件夹
        //boolean b = fs.mkdirs(new Path("/shujia/bigdata17"));
        //如果文件夹已经存在，则返回false
       // System.out.println(b);

        //文件的上传
        //第一个Path是本地的文件夹路径
        //第二个Path是HDFS上的路径
       fs.copyFromLocalFile(new Path("D:\\Softwore_java\\projects\\java-hadoop\\hadoop-hdfs\\src\\data\\students.txt"),
                            new Path("/shujia/bigdata17"));

        //文件的下载
        //fs.copyToLocalFile(new Path("/shujia/bigdata17/students.txt"),new Path("D:\\atao\\Documents"));

       //文件的删除
//        boolean delete = fs.delete(new Path("/shujia/bigdata17/students.txt"),true);
//        System.out.println(delete);

        //文件的重命名
        //fs.rename(new Path("/shujia/bigdata17/students.txt"),new Path("/shujia/bigdata17/students2.txt"));

        //查看文件的描述信息
        System.out.println("/shujia/bigdata17/目录下文件的相关信息：");
        RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/shujia/bigdata17"), true);
        while(listFiles.hasNext()){
            LocatedFileStatus status = listFiles.next();
            System.out.println("------------------------");
            //获取文件的名字
            String name = status.getPath().getName();
            System.out.println("文件的名字为："+name);
            //获取文件的长度
            long len = status.getLen();
            System.out.println("文件的字节数为："+len);
            //查看文件的所属权限
            FsPermission permission = status.getPermission();
            System.out.println("文件的所属权限："+permission);
            //文件的所属用户
            String owner = status.getOwner();
            System.out.println("文件的所属用户："+owner);
            //查看分组信息
            String group = status.getGroup();
            System.out.println("分组信息："+group);
            //修改时间-时间戳
            long modificationTime = status.getModificationTime();
            System.out.println(modificationTime);

            //查看存储块的信息
            BlockLocation[] blockLocations = status.getBlockLocations();
            for (BlockLocation blockLocation : blockLocations) {
                //获取块的对应的存储信息--结点信息
                String[] hosts = blockLocation.getHosts();
                for (String host : hosts) {
                    System.out.println("所在节点："+host);
                }
            }
        }
        //使用IO流的方式进行文件的上传
        //创建输入流读取本地的文件
        FileInputStream fis = new FileInputStream(new File("D:\\Softwore_java\\projects\\java-hadoop\\hadoop-hdfs\\src\\data\\students.txt"));
        //创建输出流 FSDataOutPutStream
        FSDataOutputStream fsDataOutputStream = fs.create(new Path("/haha/test.txt"));
        //使用流进行拷贝,流对拷贝
        IOUtils.copy(fis, fsDataOutputStream);
        //释放资源
        IOUtils.closeQuietly(fsDataOutputStream);
        IOUtils.closeQuietly(fis);


        //使用IO流的方式进行文件的下载
        FSDataInputStream open = fs.open(new Path("/haha/test.txt"));
        //创建输出流
        FileOutputStream fos = new FileOutputStream(new File("D:\\atao\\Documents\\wanLe.txt"));
        //使用流进行拷贝，流对拷贝
        IOUtils.copy(open,fos);
        //释放资源
        IOUtils.closeQuietly(open);
        IOUtils.closeQuietly(fos);

        //释放资源
        fs.close();


    }
}

目录结构如下