hadoop hdfs 开发环境

测试

1. 创建项目添加 pom 依赖

    <dependencies>
           <dependency>
               <groupId>org.apache.hadoop</groupId>
               <artifactId>hadoop-common</artifactId>
               <version>2.9.2</version>
           </dependency>
           <dependency>
               <groupId>org.apache.hadoop</groupId>
               <artifactId>hadoop-hdfs</artifactId>
               <version>2.9.2</version>
           </dependency>
           <dependency>
               <groupId>org.apache.hadoop</groupId>
               <artifactId>hadoop-hdfs-client</artifactId>
               <version>2.9.2</version>
           </dependency>
       </dependencies>
   

2. 测试代码连接 hdfs 上传文件

   public class HdfsClient {
       public static void main(String[] args) throws IOException, URISyntaxException, InterruptedException {
           ioStreamPutHdfs();
       }
   
       /**
        * @author zcz
        * @description io 流方式文件上传
        * @date 2020-03-28
        *
        * @return
        */
       public static void ioStreamPutHdfs() throws URISyntaxException, IOException, InterruptedException {
           Configuration conf = new Configuration();
           FileSystem fs = FileSystem.get(new URI("hdfs://172.16.235.162:8020"), conf, "hadoop");
   
           FileInputStream fileInputStream = new FileInputStream("/Users/zhangchenzhao/Desktop/local/zcz.xml");
   
           FSDataOutputStream fsDataOutputStream = fs.create(new Path("/user/zcz/io.xml"));
   
           IOUtils.copyBytes(fileInputStream, fsDataOutputStream, conf);
   
           fsDataOutputStream.close();
           fileInputStream.close();
           fs.close();
   
       }
   }
   

springboot hadoop

1. pom依赖

      <dependencies>
   <!--        spring boot-->
           <dependency>
               <groupId>org.springframework.boot</groupId>
               <artifactId>spring-boot-test</artifactId>
           </dependency>
   
           <dependency>
               <groupId>org.springframework.boot</groupId>
               <artifactId>spring-boot-starter-actuator</artifactId>
           </dependency>
           <dependency>
               <groupId>org.springframework.boot</groupId>
               <artifactId>spring-boot-starter-web</artifactId>
           </dependency>
   
   <!--        hadoop-hdfs-->
           <dependency>
               <groupId>org.apache.hadoop</groupId>
               <artifactId>hadoop-common</artifactId>
               <version>2.9.2</version>
           </dependency>
           <dependency>
               <groupId>org.apache.hadoop</groupId>
               <artifactId>hadoop-hdfs</artifactId>
               <version>2.9.2</version>
           </dependency>
           <dependency>
               <groupId>org.apache.hadoop</groupId>
               <artifactId>hadoop-hdfs-client</artifactId>
               <version>2.9.2</version>
           </dependency>
   
   
       </dependencies>
   

2. 配置文件

   spring.application.name=hadoop-service
   server.port=9100
   
   hdfs.user=hadoop
   hdfs.defaultFS=hdfs://172.16.235.162:8020
   

3. 配置类

   @Configuration
   public class HadoopConf {
   
   
       @Value("${hdfs.defaultFS}")
       private String defautFS;
   
       @Value("${hdfs.user}")
       private String user;
   
       @Bean
       public org.apache.hadoop.conf.Configuration configuration(){
           return new org.apache.hadoop.conf.Configuration();
       }
   
       @Bean
       public FileSystem fileSystem(org.apache.hadoop.conf.Configuration configuration) throws URISyntaxException, IOException, InterruptedException {
           FileSystem fs = FileSystem.get(new URI(defautFS), configuration, user);
           return fs;
       }
   }
   

4. log4j.properties

   log4j.rootLogger=INFO, stdout
   log4j.appender.stdout=org.apache.log4j.ConsoleAppender
   log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
   log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
   log4j.appender.logfile=org.apache.log4j.FileAppender
   log4j.appender.logfile.File=target/spring.log
   log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
   log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n
   

5. hadoop配置文件的 hdfs-site.xml,core-site.xml, mapred-site.xml, yarn-site.xml 可以放在 resource 下, 优先级高于 hadoop 集群配置

   <?xml version="1.0" encoding="UTF-8"?>
   <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
   
   <configuration>
       <!--文件备份数 默认3-->
       <property>
           <name>dfs.replication</name>
           <value>1</value>
       </property>
   </configuration>
   

6. 测试

   @RestController
   @RequestMapping("/hdfs")
   public class HdfsController {
   
   
       @Autowired
       private FileSystem fileSystem;
       @Autowired
       private Configuration configuration;
   
       @PutMapping("/upload")
       public void get(@RequestParam("filePath") String filePath) throws IOException {
           FileInputStream fileInputStream = new FileInputStream(filePath);
           FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/user/zcz/ion.xml"));
   
           IOUtils.copyBytes(fileInputStream, fsDataOutputStream, configuration);
   
           fsDataOutputStream.close();
           fileInputStream.close();
       }
   
   }
   

上传文件副本数为 1 了, 表示开发环境副本数优先级高于 hadoop 集群配置