Hadoop Replication Pipelining, Replica Placement Policy, Replication Rack Awareness

https://hadoopabcd.wordpress.com/2015/03/17/hdfs-file-blocks-distribution-in-datanodes/

Replication Rack Awareness

1. Default

    默认机架感知是基于脚本的。

        <property>
            <name>net.topology.node.switch.mapping.impl</name>                   
            <value>org.apache.hadoop.net.ScriptBasedMapping</value>
        </property>

        <!-- 配置脚本,默认为空，说明所有节点都在同一机架。 -->
        <property>
            <name>net.topology.script.file.name</name>
            <value></value>
        </property>

2.创建机架感知类

package com.it18zhang.hadoop.rackaware;

        import org.apache.hadoop.net.DNSToSwitchMapping;

        import java.io.FileOutputStream;
        import java.util.ArrayList;
        import java.util.List;

        /**
         * 自定义机架类
         */
        public class MyDNSwitchMapping implements DNSToSwitchMapping{

            public List<String> resolve(List<String> names) {
                saveNames(names);
                List<String> list = new ArrayList<String>();
                for(String s: names){
                    list.add(extractIp(s)) ;
                }
                return list;
            }

            public void reloadCachedMappings() {
            }

            public void reloadCachedMappings(List<String> names) {
            }

            private String extractIp(String name){
                if(name.startsWith("s")){
                    String ip = name.substring(1) ;
                    int intIp = Integer.parseInt(ip) ;
                    if(intIp < 103){
                        return "/rack1/" + ip ;
                    }
                    else{
                        return "/rack2/" + ip;
                    }
                }
                //
                else if(name.contains(".")){
                    String ip = name.split("\\.")[3] ;
                    int intIp = Integer.parseInt(ip);
                    if (intIp < 103) {
                        return "/rack1/" + ip;
                    } else {
                        return "/rack2/" + ip;
                    }
                }
                return "/rack1/" + name ;
            }

            /**
             * 保存名称到磁盘
             */
            private void saveNames(List<String> names){
                try {
                    FileOutputStream fos = new FileOutputStream("/home/centos/rack.log",true) ;
                    fos.write("===================\r\n".getBytes());
                    for(String s : names){
                        fos.write((s + "\r\n").getBytes());
                    }
                    fos.flush();
                    fos.close();
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }

3.导出jar包

    添加artifacts -> 删除依赖的jar

4.部署jar到hadoop集群/soft/hadoop/share/hadoop/common/lib

5.配置core-site.xnml

        <property>
            <name>net.topology.node.switch.mapping.impl</name>
            <value>com.it18zhang.hadoop.rackaware.MyDNSwitchMapping</value>
        </property>

6.重启集群，在client上传文件观察存放副本情况。