hdfs的ide远程管理

最新推荐文章于 2024-06-29 00:59:23 发布

moyue..

最新推荐文章于 2024-06-29 00:59:23 发布

阅读量1.7k

点赞数

文章标签： hdfs ide hadoop

本文链接：https://blog.csdn.net/weixin_52812666/article/details/121776143

版权

这篇博客详细介绍了如何在Windows环境下配置Hadoop2.7.6，并利用Maven项目配置相关依赖。通过Java代码展示了如何使用Hadoop API进行文件系统的操作，包括上传、下载、创建目录、删除目录、重命名文件等基本操作，以及利用IOUtil工具类进行文件读写。同时，还展示了获取文件信息的方法，包括文件大小、块位置等。

摘要由CSDN通过智能技术生成

windows下载hadoop2.x版本配置环境变量

配置pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>org.sw.hadoop.api</groupId>
    <artifactId>hadoop-api-m</artifactId>
    <version>0.0.1-SNAPSHOT</version>

    <properties>
        <hadoop.version>2.7.6</hadoop.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>jdk.tools</groupId>
            <artifactId>jdk.tools</artifactId>
            <version>1.8</version>
            <scope>system</scope>
            <systemPath>E:/ProgramFiles/Java/jdk1.8.0_291/lib/tools.jar</systemPath>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-core -->
        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-mapreduce-client-core -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-core</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-yarn-common -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-yarn-common</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-mapreduce-client-common -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-common</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.8.2</version>
            <scope>compile</scope>
        </dependency>
    </dependencies>


    <build>
        <plugins>
            <!-- 设置JDK版本 -->
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                    <encoding>UTF-8</encoding>
                </configuration>
            </plugin>
        </plugins>
    </build>
</project>

文件系统对象

#    将代码提取出来
FileSystem fileSystem=null;
Configuration configuration=null;
@Before
    public void before() throws IOException {
    System.setProperty("HADOOP_USER_NAME","root");
    configuration = new Configuration();
    configuration.set("fs.defaultFS","hdfs://192.168.10.101:8020");
    fileSystem = FileSystem.get(configuration);
    }
#    关闭流
@After
    public void testFileSystem() throws IOException {
        fileSystem.close();
    }

api之上传文件

@Test
    public void testDeleteFileOrDirectory() throws IOException {
        final FileSystem fileSystem = this.fileSystem.get(configuration);
        final Path path = new Path("D:\\input\\file.txt");
        final Path path1 = new Path("/input/file");
        fileSystem.copyFromLocalFile(path,path1);
    }

-rw-r--r--

root

supergroup

2.32 KB

2021/12/7 下午7:11:03

128 MB

file

api之下载文件

@Test
    public void testDeleteFileOrDirectory() throws IOException {
        final FileSystem fileSystem = this.fileSystem.get(configuration);
        final Path path = new Path("D:\\input");
        final Path path1 = new Path("/input/file");
        //fileSystem.copyFromLocalFile(path,path1);
        fileSystem.copyToLocalFile(path1,path);
    }

api之创建目录

@Test
    public void testDeleteFileOrDirectory() throws IOException {
        final FileSystem fileSystem = this.fileSystem.get(configuration);
        final Path path = new Path("D:\\input");
        final Path path1 = new Path("/input/mkdir.txt");
        //fileSystem.copyFromLocalFile(path,path1);
        //fileSystem.copyToLocalFile(path1,path);
        fileSystem.mkdirs(path1);
    }

drwxr-xr-x

root

supergroup

0 B

2021/12/7 下午7:21:53

0 B

mkdir.txt

api之删除目录

@Test
    public void testDeleteFileOrDirectory() throws IOException {
        final FileSystem fileSystem = this.fileSystem.get(configuration);
        final Path path = new Path("D:\\input");
        final Path path1 = new Path("/input/mkdir.txt");
        //fileSystem.copyFromLocalFile(path,path1);
        //fileSystem.copyToLocalFile(path1,path);
        fileSystem.delete(path1,true);
    }
#    true表示可以递归删除目录
#    false只能删除空文件

api之重命名

@Test
    public void testDeleteFileOrDirectory() throws IOException {
        final FileSystem fileSystem = this.fileSystem.get(configuration);
        final Path path = new Path("/input/mkdir.txt");
        final Path path1 = new Path("/input/mkdir");
        //fileSystem.copyFromLocalFile(path,path1);
        //fileSystem.copyToLocalFile(path1,path);
        //fileSystem.delete(path1,true);
        fileSystem.rename(path,path1);
    }

drwxr-xr-x

root

supergroup

0 B

2021/12/7 下午7:35:04

0 B

mkdir

IOUtil上传文件

@Test
    public void putFile() throws IOException {
    FileInputStream fileInputStream = new FileInputStream(new File("D://input//file.txt"));
    FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/input/file.txt"));
    org.apache.hadoop.io.IOUtils.copyBytes(fileInputStream,fsDataOutputStream,configuration);
    org.apache.hadoop.io.IOUtils.closeStream(fileInputStream);
    org.apache.hadoop.io.IOUtils.closeStream(fsDataOutputStream);
    }

-rw-r--r--

root

supergroup

2.32 KB

2021/12/7 下午8:01:08

128 MB

file.txt

IOUtil下载文件

@Test
      public void putFile() throws IOException, URISyntaxException {
          FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.10.101:8020"), configuration);
          FSDataInputStream open = fileSystem.open(new Path("/input/file.txt"));
          FileOutputStream fileOutputStream = new FileOutputStream(new File("D:\\output\\file.txt"));
          IOUtils.copyBytes(open,fileOutputStream,configuration);
          IOUtils.closeStream(open);
          IOUtils.closeStream(fileOutputStream);
      }
#    本地存储需要指定文件名

查看文件信息

@Test
        public void statusFile() throws IOException {
            final Path path = new Path("/input/file.txt");
            //获取文件状态
            final RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator = fileSystem.listLocatedStatus(path);
            while(locatedFileStatusRemoteIterator.hasNext()){
                //取出对象
                final LocatedFileStatus next = locatedFileStatusRemoteIterator.next();
                System.out.println("name:"+next.getPath());
                //获取位置
                final BlockLocation[] blockLocations = next.getBlockLocations();
                for (BlockLocation blockLocation : blockLocations) {
                    System.out.println("当前块的副本位置:"+ Arrays.toString(blockLocation.getHosts()));
                    System.out.println("当前块大小:"+blockLocation.getLength());
                    System.out.println("当前块的副本的IP地址信息:"+Arrays.toString(blockLocation.getNames()));
                    System.out.println("系统块大小:"+next.getBlockSize());
                    System.out.println("文件总长度:"+next.getLen());
                }
            }
        }

 log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
name:hdfs://192.168.10.101:8020/input/file.txt
当前块的副本位置:[qianfeng02, qianfeng01, qianfeng03]
当前块大小:2373
当前块的副本的IP地址信息:[192.168.10.102:50010, 192.168.10.101:50010, 192.168.10.103:50010]
系统块大小:134217728
文件总长度:2373