创建maven工程并导入jar包
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>org.apache.Hadoop</groupId>
<artifactId>Hadoop-client</artifactId>
<version>2.6.0-mr1-cdh5.14.0</version>
</dependency>
<dependency>
<groupId>org.apache.Hadoop</groupId>
<artifactId>Hadoop-common</artifactId>
<version>2.6.0-cdh5.14.0</version>
</dependency>
<dependency>
<groupId>org.apache.Hadoop</groupId>
<artifactId>Hadoop-hdfs</artifactId>
<version>2.6.0-cdh5.14.0</version>
</dependency>
<dependency>
<groupId>org.apache.Hadoop</groupId>
<artifactId>Hadoop-mapreduce-client-core</artifactId>
<version>2.6.0-cdh5.14.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/junit/junit -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<version>RELEASE</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
<!-- <verbal>true</verbal>-->
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.3</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<minimizeJar>true</minimizeJar>
</configuration>
</execution>
</executions>
</plugin>
<!-- <plugin>
<artifactId>maven-assembly-plugin </artifactId>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>cn.itcast.Hadoop.db.DBToHdfs2</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>-->
</plugins>
</build>
使用url的方式访问数据
@Test
public void demo1()throws Exception{
//第一步:注册hdfs 的url,让java代码能够识别hdfs的url形式
URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
InputStream inputStream = null;
FileOutputStream outputStream =null;
//定义文件访问的url地址
String url = "hdfs://192.168.52.100:8020/test/input/install.log";
//打开文件输入流
try {
inputStream = new URL(url).openStream();
outputStream = new FileOutputStream(new File("c:\\hello.txt"));
IOUtils.copy(inputStream, outputStream);
} catch (IOException e) {
e.printStackTrace();
}finally {
IOUtils.closeQuietly(inputStream);
IOUtils.closeQuietly(outputStream);
}
}
如果执行出现以下错误,可以参见资料如何解决,也可以不用理会,不会影响程序的执行。记得配置完成环境变量之后重启开发工具
在 java 中操作 HDFS,主要涉及以下 Class:
Configuration:该类的对象封转了客户端或者服务器的配置;
FileSystem:该类的对象是一个文件系统对象,可以用该对象的一些方法来对文件进行操作,通过 FileSystem 的静态方法 get 获得该对象。
FileSystem fs = FileSystem.get(conf)
get 方法从 conf 中的一个参数 fs.defaultFS 的配置值判断具体是什么类型的文件系统。如果我们的代码中没有指定 fs.defaultFS,并且工程 classpath下也没有给定相应的配置,conf中的默认值就来自于Hadoop的jar包中的core-default.xml , 默 认 值 为 : file:/// , 则 获 取 的 将 不 是 一 个DistributedFileSystem 的实例,而是一个本地文件系统的客户端对象
具体代码示例:
/**
* @author kismet
* @date 2019-11-07 13:26
*/
public class text02 {
static Configuration conf=new Configuration();
public static void main(String[] args) throws IOException, URISyntaxException {
// listStatus();
// rename();
// gettimr();
// delfile();
// mkdir();
// addfile();
// put();
check();
}
//展示目录下使有文件
public static void listStatus() throws URISyntaxException, IOException {
FileSystem hdfs = FileSystem.get(new URI("hdfs://192.168.100.88:8020"), conf);
FileStatus status[] = hdfs.listStatus(new Path("/abc"));
for (int i = 0; i <status.length ; ++i) {
System.out.println(status[i].getPath().toString());
hdfs.close();
}
}
//修改目录下指定文件名
public static void rename() throws URISyntaxException, IOException {
FileSystem hdfs = FileSystem.get(new URI("hdfs://192.168.100.88:8020"), conf);
Path path = new Path("/abc/a.txt");
Path path1 = new Path("/abc/aa.txt");
boolean rename = hdfs.rename(path, path1);
System.out.println(rename);
}
//获取文件日期
public static void gettimr() throws URISyntaxException, IOException {
FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.88:8020"), conf);
FileStatus fileStatus = fileSystem.getFileStatus(new Path("/abc/aa.txt"));
long modificationTime = fileStatus.getModificationTime();
System.out.println(modificationTime);
}
//删除文件
public static void delfile() throws URISyntaxException, IOException {
FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.88:8020"), conf);
boolean delete = fileSystem.delete(new Path("/abc/aa.txt"), true);
System.out.println(delete);
}
//创建文件夹
public static void mkdir() throws URISyntaxException, IOException{
FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.88:8020"), conf);
boolean mkdirs = fileSystem.mkdirs(new Path("/aaa"));
System.out.println(mkdirs);
}
//创建数据
public static void addfile() throws URISyntaxException, IOException{
FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.88:8020"), conf);
byte[] bytes = "hellowold".getBytes();
FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/aaa/a.txt"));
fsDataOutputStream.write(bytes,0,bytes.length);
fsDataOutputStream.close();
}
//上传数据
public static void put() throws URISyntaxException, IOException {
FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.88:8020"), conf);
Path path = new Path("F:/ttt.txt");
Path path1 = new Path("/");
fileSystem.copyFromLocalFile(path,path1);
}
//检查目录是否存在
public static void check() throws URISyntaxException, IOException {
FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.88:8020"), conf);
Path path = new Path("/abc");
boolean exists = fileSystem.exists(path);
System.out.println(exists);
}
}