JAVA 访问HDFS

最新推荐文章于 2024-09-27 22:18:42 发布

sunlggggg

最新推荐文章于 2024-09-27 22:18:42 发布

阅读量262

点赞数

分类专栏：算法 WebSocket Spark 文章标签： java HDFS Hadoop

本文链接：https://blog.csdn.net/sunlggggg/article/details/52122128

版权

WebSocket 同时被 3 个专栏收录

3 篇文章 0 订阅

订阅专栏

算法

2 篇文章 0 订阅

订阅专栏

Spark

2 篇文章 0 订阅

订阅专栏

package org.apache.spark.examples;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map.Entry;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
 * realize the wordcount by multi thread
 * 
 * @author sunlggggg
 *
 */
public class testNoSpark implements Runnable {

    private static HashMap<String, Integer> hashMap;
    BufferedReader bis;
    private static  BufferedWriter bw;

    public void init() throws IOException {
        hashMap = new HashMap<>();
        String path = "hdfs://master:8888/input/TEST2";
        String confFile = "/root/hadoop/etc/hadoop/core-site.xml";
        Configuration conf = new Configuration();
        FileInputStream fis = new FileInputStream(confFile);
        conf.addResource(fis);
        FileSystem fileSystem = FileSystem.get(conf);
        FSDataInputStream fs = fileSystem.open(new Path(path));
        bis = new BufferedReader(new InputStreamReader(fs, "GBK"));
        FileWriter fos = new FileWriter("/root/out");
        bw = new BufferedWriter(fos);
    }

    public void close() {
    }

    @Override
    public void run() {
        String line = null;
        String[] words = null;
        try {
            while ((line = bis.readLine()) != null) {
                words = line.split(" ");
                for (int i = 0; i < words.length; i++) {
                    Integer count = hashMap.get(words[i]);
                    hashMap.put(words[i], count == null ? 1 : count + 1);
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public void print() throws IOException {
        System.out.println("size" + hashMap.size());
        bw.write("size" + hashMap.size() +"\n");
        Iterator<Entry<String, Integer>> iterator = hashMap.entrySet().iterator();
        while (iterator.hasNext()) {
            Entry<String, Integer> en = iterator.next();
            System.out.println(en.getKey() + "------>" + en.getValue());
            bw.write(en.getKey() + "------>" + en.getValue() +"\n");
        }
    }

    public static void main(String[] args) throws IOException {
        long time1 = new Date().getTime();
        testNoSpark testNoSpark = new testNoSpark();
        testNoSpark.init();
        Thread t1 = new Thread(testNoSpark);
        Thread t2 = new Thread(testNoSpark);
        Thread t3 = new Thread(testNoSpark);
        t1.start();
        t2.start();
        t3.start();
        try {
            t1.join();
            t2.join();
            t3.join();
            long time2 = new Date().getTime();
            testNoSpark.bw.write(time2+"\n");
            System.out.println((time2 - time1)/1000.0);
            testNoSpark.print();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}