HDFS HA java API 实战

最新推荐文章于 2023-12-08 11:50:52 发布

笛香幽谷

最新推荐文章于 2023-12-08 11:50:52 发布

阅读量269

点赞数

分类专栏： hadoop

本文链接：https://blog.csdn.net/u011291159/article/details/88286109

版权

hadoop 专栏收录该内容

16 篇文章 0 订阅

订阅专栏

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import java.io.*;
import java.net.URI;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
public class HdfsHADataHouseTmpDeteDataCompute {
    public static void main(String[] args) throws Exception{

        Configuration conf = new Configuration();
        conf.set("fs.defaultFS", "hdfs://HWONE");
        conf.set("dfs.nameservices", "HWONE");
        conf.set("fs.hdfs.impl","org.apache.hadoop.hdfs.DistributedFileSystem");
        conf.set("dfs.ha.namenodes.HZWONE", "nn1,nn2");
        conf.set("dfs.namenode.rpc-address.HZWONE.nn1", "hadoop-master-11:8020");
        conf.set("dfs.namenode.rpc-address.HZWONE.nn2", "hadoop-master-21:8020");
        //conf.setBoolean(name, value);
        conf.set("dfs.client.failover.proxy.provider.HWONE", "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider");
        String path="/apps/hive/warehouse/"+args[0];
        //String path="/apps/hive/warehouse/fdm";
        FileSystem fs = FileSystem.get(new URI(path), conf, "hive");
        //FSDataInputStream hdfsInStream = fs.open(new Path("hdfs:apps/hive/warehouse/tmp"));
       // InputStreamReader isr = new InputStreamReader(hdfsInStream, "utf-8");
        RemoteIterator<LocatedFileStatus> fileIter= fs.listFiles(new Path(path),true);
        Map<String,Long> tableTimeStampMap = new HashMap<>();
        SimpleDateFormat format =  new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        String firstName="";
        int i=0;
        try {
        while (fileIter.hasNext()){
            LocatedFileStatus file = fileIter.next();
                if(fs.exists(file.getPath())){
                    String table= file.getPath().toString().split("/")[7];
                    long time=file.getModificationTime();
                    if(tableTimeStampMap.containsKey(table)){
                        if(time>tableTimeStampMap.get(table)){
                            tableTimeStampMap.put(table,time);
                        }
                    }else {
                        i++;
                        //System.out.println(i+":"+table+":"+time);
                        tableTimeStampMap.put(table,time);
                    }
                    if(!firstName.equals(table)){
                        System.out.println(i+":"+table+" time:"+format.format(tableTimeStampMap.get(table)));
                       // tableTimeStampMap.clear();
                    }
                    firstName=table;
                }
        }
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } finally {
        }

    }
}