springboot JAVA API 操作hadoop伪分布式集群

虚拟机（192.168.74.114）中搭建了hadoop伪分布式集群，查看集群运行状态

• springboot maven依赖

		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-hdfs</artifactId>
			<version>2.7.4</version>
			<exclusions>
				<exclusion>
					<groupId>org.slf4j</groupId>
					<artifactId>slf4j-log4j12</artifactId>
				</exclusion>
				<exclusion>
					<groupId>log4j</groupId>
					<artifactId>log4j</artifactId>
				</exclusion>
				<exclusion>
					<groupId>javax.servlet</groupId>
					<artifactId>servlet-api</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
		<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-common</artifactId>
			<version>2.7.4</version>
			<exclusions>
				<exclusion>
					<groupId>org.slf4j</groupId>
					<artifactId>slf4j-log4j12</artifactId>
				</exclusion>
				<exclusion>
					<groupId>javax.servlet</groupId>
					<artifactId>servlet-api</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-client</artifactId>
			<version>2.7.4</version>
			<exclusions>
				<exclusion>
					<groupId>org.slf4j</groupId>
					<artifactId>slf4j-log4j12</artifactId>
				</exclusion>
				<exclusion>
					<groupId>log4j</groupId>
					<artifactId>log4j</artifactId>
				</exclusion>
				<exclusion>
					<groupId>javax.servlet</groupId>
					<artifactId>servlet-api</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
		</dependency>

• java代码

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.Before;
import org.junit.Test;

import java.net.URI;

public class HDFSUtils {
    public static final String HDFS_PATH = "hdfs://192.168.74.114:9000";
    FileSystem filesystem = null;
    Configuration configuration = null;

    @Before
    public void init() throws Exception{
        configuration = new Configuration();
        filesystem = FileSystem.get(new URI(HDFS_PATH),configuration,"root");
    }

//    上传文件
    @Test
    public void testUpload() throws Exception {
        Thread.sleep(2000);
        filesystem.copyFromLocalFile(new Path("D:/access.log"), new Path("/data/temp/access.log"));
        filesystem.close();
    }

//    新建文件路径
    @Test
    public void makeDir()throws Exception{
        filesystem.mkdirs(new Path("/data/temp"));
        filesystem.close();
    }

//    删除文件
    @Test
    public void delFile()throws Exception{
        filesystem.deleteOnExit(new Path("/data/temp/access.log"));
        filesystem.close();
    }

}

• hdfs shell命令查看（也可通过页面查看）

ls查看文件：
    hdfs dfs -ls 路径               
cat输出文件：
    hdfs dfs -cat 文件名