Java操作HDFS

一、读取hdfs文件数据

方法一：使用hadoop URL读取数据

  URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
    	  InputStream in = null;
    	  
    	  try {
			in = new URL("hdfs://172.16.169.10:9000/WordCount.txt").openStream();
			IOUtils.copyBytes(in, System.out, 4096, false);
		} catch (IOException e) {
			
			e.printStackTrace();
		}finally {
			IOUtils.closeStream(in);
		}
    	  

方法二：使用FileSystem API读取数据

 String url = args[0];
    	   Configuration conf = new Configuration();
    	   conf.set("fs.hdfs.impl",org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
           FileSystem fs  = FileSystem.get(URI.create(url), conf);
           InputStream in = null;
          
			try {
				in = fs.open(new Path(url));
				IOUtils.copyBytes(in, System.out, 4096, false);
			} finally {
				IOUtils.closeStream(in);
			}
           

二、创建、删除文件（夹）

        Configuration conf = new Configuration();

     	FileSystem fs = FileSystem.get(new URI("hdfs://172.16.169.10:9000"), conf);
//    	crate a file floder
    	boolean boo = fs.mkdirs(new Path("/usr/fu/zi/Word.txt"));
    	System.out.println(boo);
//    	create a file
    	fs.create(new Path("/fu/zi/HelloWord.txt"));
//    	delete files
    	boolean boo1 = fs.delete(new Path("/fu"), true);
    	System.out.println(boo1);
    	boolean boo2 = fs.delete(new Path("/usr"), true);
    	System.out.println(boo2);
    	

三、向文件中写入数据

 

四、maven相关依赖

<build>
  
    <plugins>  
     
      <plugin>  
        <groupId>org.apache.maven.plugins</groupId>  
        <artifactId>maven-compiler-plugin</artifactId>  
        <configuration>  
          <source>1.8</source>  
          <target>1.8</target> 
          <encoding>UTF-8</encoding> 
        </configuration>  
      </plugin>
       
    </plugins>
    
    </build>
    
  <dependencies>
    <dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-common</artifactId>
    <version>2.7.4</version>
</dependency>


<!-- https://mvnrepository.com/artifact/com.google.guava/guava -->
<dependency>
    <groupId>com.google.guava</groupId>
    <artifactId>guava</artifactId>
    <version>25.0-jre</version>
</dependency>

 <dependency>
            <groupId>jdk.tools</groupId>
            <artifactId>jdk.tools</artifactId>
            <version>1.8</version>
            <scope>system</scope>
            <systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
    </dependency>
    
    <dependency>
    <groupId>commons-logging</groupId>
    <artifactId>commons-logging</artifactId>
    <version>1.1.3</version>
</dependency>
  
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-hdfs</artifactId>
    <version>2.7.4</version>
</dependency>

  
  </dependencies>