hadoop java API实现文件上传

package com.zlf;

import java.io.FileInputStream;
import java.io.IOException;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;


public class Demo {

    public static void main(String[] args) throws IOException {
        // TODO Auto-generated method stub
        Configuration conf=new Configuration();

        conf.set("fs.defaultFS", "hdfs://localhost:9000");

        FileSystem fs=FileSystem.get(conf);

        FSDataOutputStream fsDataOutputStream=fs.create(new Path("/RoseHA.rar"));

        FileInputStream fileInputStream=new FileInputStream("d:/RoseHA.rar");

        IOUtils.copy(fileInputStream, fsDataOutputStream);


    }

}

下面是另一种实现方式,非常简单。

//一句话搞定文件的上传
fs.copyFromLocalFile(new Path("d:/PowerDesigner15_Evaluation.rar"), new Path("/PowerDesigner15_Evaluation.rar"));
已标记关键词 清除标记
现在通过java Api上传本地文件到hdfs中, 本地环境为win10,hdfs环境为centos7, hdfs通过kerberos来添加验证,上传过程中,小文件可以上传成功,大文件上传后,报错 ``` Exception in thread "main" java.io.IOException: 远程主机强迫关闭了一个现有的连接。 at sun.nio.ch.SocketDispatcher.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:43) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379) at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) at java.io.FilterInputStream.read(FilterInputStream.java:83) at java.io.FilterInputStream.read(FilterInputStream.java:83) at org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed(PBHelper.java:2280) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.createBlockOutputStream(DFSOutputStream.java:1347) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1266) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:449) ``` 无论大文件,小文件上传到hdfs中后,都为0字节 ![图片说明](https://img-ask.csdn.net/upload/201904/10/1554862487_90371.png) 以下为实现代码 ``` public static final String USER_KEY = "hdfs@TDH"; public static final String KEY_TAB_PATH = "C:\\Users\\xx\\Desktop\\keytab"; static Configuration conf = new Configuration(); static { System.setProperty("java.security.krb5.conf", "C:\\Users\\xx\\Desktop\\krb5.conf"); conf.set("hadoop.security.authentication", "kerberos"); conf.set("fs.defaultFS", "hdfs://xx.xx.xx.xx:8020"); //conf.set("dfs.client.use.datanode.hostname", "true"); try { UserGroupInformation.setConfiguration(conf); UserGroupInformation.loginUserFromKeytab(USER_KEY, KEY_TAB_PATH); } catch (IOException e) { e.printStackTrace(); } } public static void main(String[] args) throws IOException { FileSystem fs = FileSystem.get(conf); InputStream fis = new FileInputStream(new File("C:\\Users\\xx\Desktop\\a.zip")); OutputStream fos = fs.create(new Path("/tmp/b.exe")); IOUtils.copyBytes(fis,fos,1024); } ```
©️2020 CSDN 皮肤主题: 创作都市 设计师:CSDN官方博客 返回首页