编程完成输出HDFS中指定文件的文本到终端中

题目:查看Java帮助手册或其它资料,用“java.net.URL”和“org.apache.hadoop.fs.FsURLStreamHandlerFactory”编程完成输出HDFS中指定文件的文本到终端中。

java代码:

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;

public class MyFSDataInputStream extends FSDataInputStream {
    private static final int BUFFER_SIZE = 4096;
    private byte[] buffer;
    private int pos;
    private int end;
    private boolean eof;

    public MyFSDataInputStream(FileSystem fs, Path file) throws IOException {
        super(fs.open(file));
        buffer = new byte[BUFFER_SIZE];
        pos = 0;
        end = 0;
        eof = false;
    }

    public String readNextLine() throws IOException {
        StringBuilder sb = new StringBuilder();
        int b = -1;
        boolean found = false;

        while (!eof && !found) {
            if (pos >= end) {
                fillBuffer();
            }
            if (end == -1) {
                eof = true;
            }
            while (pos < end) {
                b = buffer[pos++];
                if (b == '\r' || b == '\n') {
                    found = true;
                    break;
                }
                sb.append((char) b);
            }
        }

        if (sb.length() == 0 && eof) {
            return "";
        } else {
            return sb.toString();
        }
    }

    private void fillBuffer() throws IOException {
        end = in.read(buffer);
        pos = 0;
    }

    public static void main(String[] args) throws IOException {
        if (args.length < 1) {
            System.err.println("Usage: MyFSDataInputStream <HDFS file path>");
            System.exit(1);
        }

        String filePath = args[0];

        Configuration conf = new Configuration();
        Path path = new Path(filePath);
        FileSystem fs = FileSystem.get(path.toUri(), conf);

        MyFSDataInputStream in = new MyFSDataInputStream(fs, path);
        String line;
        while ((line = in.readNextLine()) != null) {
            System.out.println(line);
        }
        in.close();
    }
}

命令:

1.	start-dfs.sh
2.	vim file.txt
3.	hadoop fs -mkdir /user/hadoop/input
4.	hadoop fs -put /root/file.txt /user/hadoop/input
5.	cd /usr/local/hadoop/share
6.	ls
7.	vim ReadHdfsFileFromURL.java
8.	ls
9.	javac -classpath /usr/local/hadoop/share/hadoop/common/hadoop-common-2.7.4.jar     ReadHdfsFileFromURL.java      
10.	ls
11.	HADOOP_CLASSPATH=. hadoop ReadHdfsFileFromURL 

结果:

  • 0
    点赞
  • 15
    收藏
    觉得还不错? 一键收藏
  • 2
    评论
评论 2
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值