<dependencies>
<dependency>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
<version>1.8</version>
<scope>system</scope>
<systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.8.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.8.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.8.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>2.8.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
<version>2.8.1</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>19.0</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.2</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>2.6</version>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>1.9</version>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>1.7.7</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.5</version>
</dependency>
<!-- https://mvnrepository.com/artifact/junit/junit -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<!--编译配置 -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<!--资源文件问题 -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>2.6</version>
<configuration>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<!-- 把依赖的jar包拷到lib目录下 -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
pom文件依赖如上图
编写测试类
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
/**
* 测试hadoop
*
* @author Administrator
*
*/
public class hadoopTest {
/**
* 查询所有的文件夹及文件
*
* @throws IOException
*/
@org.junit.Test
public void listFiles() throws IOException {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://192.168.10.110:9001");
FileSystem fs = FileSystem.newInstance(conf);
// true 表示递归查找 false 不进行递归查找
RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(new Path("/"), true);
while (iterator.hasNext()) {
LocatedFileStatus next = iterator.next();
System.out.println(next.getPath());
}
System.out.println("----------------------------------------------------------");
FileStatus[] fileStatuses = fs.listStatus(new Path("/"));
for (int i = 0; i < fileStatuses.length; i++) {
FileStatus fileStatus = fileStatuses[i];
System.out.println(fileStatus.getPath());
}
}
/**
* 上传文件到hdfs上
*/
@org.junit.Test
public void upload() throws IOException {
long currentTimeMillis = System.currentTimeMillis();
Configuration conf = new Configuration();
// conf.set("fs.defaultFS", "hdfs://192.168.10.110:9001");
conf.set("fs.default.name", "hdfs://192.168.10.110:9001");
FileSystem fs = FileSystem.get(conf);
fs.copyFromLocalFile(new Path("F:\\sso.jar"), new Path("/test1")); //第一个为本地文件路径,第二个为hadoop路径
long c = System.currentTimeMillis() - currentTimeMillis;
System.out.println(c);
}
/**
* 将hdfs上文件下载到本地
*/
@org.junit.Test
public void download() throws IOException {
long currentTimeMillis = System.currentTimeMillis();
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://192.168.10.110:9001");
conf.set("fs.default.name", "hdfs://192.168.10.110:9001");
// conf.set("hadoop.home.dir", "F:\\lys\\javatools\\hadoop/bin"); //设定你的hadoop的目录
FileSystem fs = FileSystem.newInstance(conf);
fs.copyToLocalFile(new Path("/test1/VMware-workstation-full-14.1.1.28517.exe"), new Path("D:\\home")); //
System.out.println("成功");
long c = System.currentTimeMillis() - currentTimeMillis;
System.out.println(c);
}
/**
* 在hdfs更目录下面创建test1文件夹
*
* @throws IOException
*/
@org.junit.Test
public void mkdir() throws IOException {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://192.168.10.110:9001");
FileSystem fs = FileSystem.newInstance(conf);
fs.mkdirs(new Path("/test1"));
System.out.println("成功");
}
}
fonfiguration为配置,里边要set一些hadoop的设置,
在文件下载操作时,要注意一些问题,我是在其他服务器上部署的hadoop,是一个单体的,但是我要实现下载文件操作,在本机上也要添加hadoop的安装,并且配置环境变量,并加入path文件,下载的hadoop的bin目录下还需要添加一些插件工具
此为bin目录下的所有文件,winutils.exe文件及其他的插件工具
如此以上配置,才可以使用文件下载操作,不然会报错,报错信息为 : hadoophome和一个.dir文件unset
以上为java+hadoop的一些操作,刚学习,欢迎补充