Maven
通俗上讲就是我们不在手动导入jar包,交给Maven去管理
Maven通过pom.xml中的坐标进行管理
- 新建Maven项目
导入依赖
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<hadoop.version>2.6.0-cdh5.7.0</hadoop.version>
</properties>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
<dependencies>
<!-- 添加hadoop依赖-->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
<!-- 添加junit依赖-->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.10</version>
<scope>test</scope>
</dependency>
</dependencies>
使用Java api在hdfs上新建文件夹
package hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.net.URI;
public class HDFSAPP {
Configuration configuration = null;
FileSystem fileSystem = null;
public static final String HDFS_PATH = "hdfs://192.168.2.4/:8020";
@Test
public void mkdir() throws Exception {
fileSystem.mkdirs(new Path("/ideahehe/test"));
}
// 测试之前要执行的代码
@Before
public void setUp() throws Exception {
System.out.println("开始建立与HDFS的连接");
configuration = new Configuration();
fileSystem = FileSystem.get(new URI(HDFS_PATH),configuration,"hadoop");
}
// 测试之后要执行的代码
@After
public void tearDown(){
configuration = null;
fileSystem = null;
System.out.println("关闭与HDFS的连接");
}
}
// 创建文件
@Test
public void create() throws Exception {
Path path = new Path("/ideahehe/test/hello.txt");
FSDataOutputStream outputStream = fileSystem.create(path);
outputStream.write("hello taiyuan gongye ".getBytes());
outputStream.flush();
outputStream.close();
}
// rename文件
@Test
public void rename() throws Exception {
Path oldPath = new Path("/ideahehe/test/hello.txt");
Path newPath = new Path("/ideahehe/test/xixi.txt");
fileSystem.rename(oldPath, newPath);
}
// 查看文件
@Test
public void cat() throws Exception {
Path newPath = new Path("/ideahehe/test/xixi.txt");
FSDataInputStream inputStream = fileSystem.open(newPath);
IOUtils.copyBytes(inputStream, System.out, 1024);
inputStream.close();
}
package com.neusoft.hadoop.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;
/**
* @author Eric Lee
* @date 2020/10/26 14:45
* Java 操作HDFS
*/
public class HDFSApp {
Configuration configuration = null;
FileSystem fileSystem = null;
public static final String HDFS_PATH = "hdfs://10.25.187.18:8020";
/**
* 在 hdfs中新建 文件夹 /你的名字/test
* @throws Exception
*/
@Test
public void mkdir() throws Exception {
fileSystem.mkdirs(new Path("/liyan33/test"));
}
/**
* 创建文件
* @throws Exception
*/
@Test
public void create() throws Exception {
FSDataOutputStream outputStream = fileSystem.create(new Path("/liyan33/test/haha.txt"));
outputStream.write("hello qilulihgong bigdata student".getBytes());
outputStream.flush();
outputStream.close();
}
/**
* 查看文件
* @throws Exception
* hdfs -fs -cat path
*/
@Test
public void cat() throws Exception {
FSDataInputStream in = fileSystem.open(new Path("/liyan22/test/hehe.txt"));
IOUtils.copyBytes(in, System.out, 1024);
in.close();
}
/**
* 重名文件
* @throws Exception
*
*/
@Test
public void rename() throws Exception {
Path oldPath = new Path("/liyan22/test/haha.txt");
Path newPath = new Path("/liyan22/test/hehe.txt");
fileSystem.rename(oldPath, newPath);
}
/**
* 上传文件到HDFS
* @throws Exception
*
*/
@Test
public void copyFromLocalFile() throws Exception {
Path loacalPath = new Path("hello.txt");
Path hdfsPath = new Path("/");
fileSystem.copyFromLocalFile(loacalPath, hdfsPath);
}
/**
* 上传文件到HDFS带进度信息
* @throws Exception
*
*/
@Test
public void copyFromLocalFileWithProgress() throws Exception {
InputStream in = new BufferedInputStream(
new FileInputStream(new File("cifar-10-python.tar.gz"))
);
FSDataOutputStream ouput = fileSystem.create(new Path("/liyan33/test/cifar-10-python.tar.gz"),
(Progressable) () -> {
System.out.print(".");
});
IOUtils.copyBytes(in, ouput, 4096);
}
/**
* 下载文件到HDFS
* @throws Exception
*
*/
// TODO
@Test
public void copyToLocalFile() throws Exception {
// Path hdfsPath = new Path("/liyan33/test/cifar-10-python.tar.gz");
// Path loacalPath = new Path("./dl/cifar-10-python.tar.gz");
Path hdfsPath = new Path("/liyan33/test/haha.txt");
Path loacalPath = new Path("./dl/haha.txt");
// useRawLocalFileSystem
fileSystem.copyToLocalFile(false, hdfsPath, loacalPath, true);
}
/**
* 查看某个目录下所有文件
* @throws Exception
*
*/
@Test
public void listFiles() throws Exception{
FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
for (FileStatus f :fileStatuses){
String isDir = f.isDirectory()? "文件夹" : "文件";
short replication = f.getReplication();
long len = f.getLen();
String path = f.getPath().toString();
System.out.println(isDir + "\t" + replication +"\t" + len + "\t" +path);
}
}
/**
* 删除文件
* @throws Exception
*
*/
@Test
public void delete() throws IOException {
fileSystem.delete(new Path("/heyueqian"), true);
}
//测试之前执行的代码
@Before
public void setUp() throws Exception {
System.out.println("开始建立与HDFS的连接");
configuration = new Configuration();
fileSystem = FileSystem.get(new URI(HDFS_PATH),configuration, "hadoop");
}
//测试之完执行的代码
@After
public void tearDown(){
configuration = null;
fileSystem = null;
System.out.println("关闭与HDFS的连接");
}
}