java上传本地文件到HDFS简单demo

本文整理了上传文件到hdfs的三种java代码，均已测试通过

1、创建maven项目 

2、pom依赖

<dependencies>
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>3.8.1</version>
        <scope>test</scope>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>2.7.3</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-common</artifactId>
        <version>2.7.3</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-hdfs</artifactId>
        <version>2.7.3</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hbase</groupId>
        <artifactId>hbase-client</artifactId>
        <version>0.96.1-hadoop2</version>
    </dependency>
</dependencies>

<build>
    <resources>
        <resource>
            <directory>src/main/resources</directory>
            <filtering>true</filtering>
        </resource>
        <resource>
            <directory>src/main/java</directory>
            <filtering>true</filtering>
            <excludes>
                <exclude>**/*.java</exclude>
            </excludes>
        </resource>
    </resources>

    <pluginManagement>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>1.6</source>
                    <target>1.6</target>
                    <encoding>UTF-8</encoding>
                </configuration>
            </plugin>
        </plugins>
    </pluginManagement>

    <plugins>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <executions>
                <execution>
                    <phase>package</phase>
                    <goals>
                        <goal>shade</goal>
                    </goals>
                    <configuration>
                        <filters>
                            <filter>
                                <artifact>*:*</artifact>
                            <excludes>
                                <exclude>META-INF/*.SF</exclude>
                                <exclude>META-INF/*.DSA</exclude>
                                <exclude>META-INF/*.RSA</exclude>
                            </excludes>
                            </filter>
                        </filters>
                        <transformers>
                            <transformer                                implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                <mainClass>${mainClass}</mainClass>
                            </transformer>
                            <transformer                                    implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                                <resource>META-INF/spring.handlers</resource>
                            </transformer>
                            <transformer                                implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                                <resource>META-INF/spring.schemas</resource>
                            </transformer>
                            <transformer                                implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                                <resource>META-INF/spring.tooling</resource>
                            </transformer>
                        </transformers>
                    </configuration>
                </execution>
            </executions>
        </plugin>
    </plugins>
</build>

3、java代码

代码一：

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class CopyFile {
public static void main(String[] args) {
    Configuration conf = new Configuration();
    String localDir = "/home/hdfs/files/test.txt";
    String hdfsDir = "hdfs://server:8020/bbbb";
    try{
            Path localPath = new Path(localDir);
            Path hdfsPath = new Path(hdfsDir);
            FileSystem hdfs = FileSystem.get(conf);
            if(!hdfs.exists(hdfsPath)){
                 hdfs.mkdirs(hdfsPath);
             }
             hdfs.copyFromLocalFile(localPath, hdfsPath);
         }catch(Exception e){
         e.printStackTrace();
         }
    }
}

代码二：

package my.test;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class UploadSingle {
 public static void main(String[] args) throws URISyntaxException, IOException{
     Configuration conf = new Configuration();
     URI uri = new URI("hdfs://server:8020");
     FileSystem fs = FileSystem.get(uri,conf);
     Path resP = new Path("/home/hdfs/files/test.txt");
     Path destP = new Path("/aaaaaa");
     if(!fs.exists(destP)){
          fs.mkdirs(destP);
     }
    fs.copyFromLocalFile(resP, destP);
    fs.close();
  }
}

代码三：

import java.io.IOException;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
//上传本地文件到HDFS
public class CopyFile {
  public static void main(String[] args) throws IOException {
      //获取配置文件信息
      Configuration conf = new Configuration();
      conf.addResource(new Path("conf/core-site.xml"));
      //获取文件系统
      FileSystem hdfs = FileSystem.get(conf);
      //文件名称
      Path src = new Path("/home/hdfs/files/test.txt");
      Path dst = new Path("hdfs://server:8020/cccc");
      if(!hdfs.exists(dst)){
          hdfs.mkdirs(dst);
      }
      hdfs.copyFromLocalFile(src, dst);
      System.out.println("Upload to " + conf.get("fs.default.name"));
      FileStatus files[] = hdfs.listStatus(dst);
      for(FileStatus file:files){
          System.out.println(file.getPath());
      }
  }
}

以上各段代码，其中，

if(!hdfs.exists(dst)){
hdfs.mkdirs(dst);
}

这段代码必须存在，否则通过 hdfs dfs -ls /cccc命令查看cccc等指定路径下的文件列表时显示不出text.txt文件，

而是将txt中的文字内容直接存入了此路径下，通过 hdfs dfs -cat /cccc命令可以查看文字内容。

4、在eclipse项目上右击，选择run as–》maven install，生成jar包，将jar包上传到服务器上。

5、运行jar包 

unix代码： 

su hdfs 
$ hadoop jar path/name.jar classname 

其中，path是jar包的路径，name是项目名称，classname是类名。