本文整理了上传文件到hdfs的三种java代码,均已测试通过
1、创建maven项目
2、pom依赖
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.7.3</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.7.3</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.7.3</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>0.96.1-hadoop2</version>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
<resource>
<directory>src/main/java</directory>
<filtering>true</filtering>
<excludes>
<exclude>**/*.java</exclude>
</excludes>
</resource>
</resources>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.6</source>
<target>1.6</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>${mainClass}</mainClass>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/spring.handlers</resource>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/spring.schemas</resource>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/spring.tooling</resource>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
3、java代码
代码一:
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class CopyFile {
public static void main(String[] args) {
Configuration conf = new Configuration();
String localDir = "/home/hdfs/files/test.txt";
String hdfsDir = "hdfs://server:8020/bbbb";
try{
Path localPath = new Path(localDir);
Path hdfsPath = new Path(hdfsDir);
FileSystem hdfs = FileSystem.get(conf);
if(!hdfs.exists(hdfsPath)){
hdfs.mkdirs(hdfsPath);
}
hdfs.copyFromLocalFile(localPath, hdfsPath);
}catch(Exception e){
e.printStackTrace();
}
}
}
代码二:
package my.test;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class UploadSingle {
public static void main(String[] args) throws URISyntaxException, IOException{
Configuration conf = new Configuration();
URI uri = new URI("hdfs://server:8020");
FileSystem fs = FileSystem.get(uri,conf);
Path resP = new Path("/home/hdfs/files/test.txt");
Path destP = new Path("/aaaaaa");
if(!fs.exists(destP)){
fs.mkdirs(destP);
}
fs.copyFromLocalFile(resP, destP);
fs.close();
}
}
代码三:
import java.io.IOException;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
//上传本地文件到HDFS
public class CopyFile {
public static void main(String[] args) throws IOException {
//获取配置文件信息
Configuration conf = new Configuration();
conf.addResource(new Path("conf/core-site.xml"));
//获取文件系统
FileSystem hdfs = FileSystem.get(conf);
//文件名称
Path src = new Path("/home/hdfs/files/test.txt");
Path dst = new Path("hdfs://server:8020/cccc");
if(!hdfs.exists(dst)){
hdfs.mkdirs(dst);
}
hdfs.copyFromLocalFile(src, dst);
System.out.println("Upload to " + conf.get("fs.default.name"));
FileStatus files[] = hdfs.listStatus(dst);
for(FileStatus file:files){
System.out.println(file.getPath());
}
}
}
以上各段代码,其中,
if(!hdfs.exists(dst)){
hdfs.mkdirs(dst);
}
这段代码必须存在,否则通过 hdfs dfs -ls /cccc命令查看cccc等指定路径下的文件列表时显示不出text.txt文件,
而是将txt中的文字内容直接存入了此路径下,通过 hdfs dfs -cat /cccc命令可以查看文字内容。
4、在eclipse项目上右击,选择run as–》maven install,生成jar包,将jar包上传到服务器上。
5、运行jar包
unix代码:
su hdfs
$ hadoop jar path/name.jar classname
其中,path是jar包的路径,name是项目名称,classname是类名。