https://hadoopabcd.wordpress.com/2015/03/17/hdfs-file-blocks-distribution-in-datanodes/
Replication Rack Awareness
1. Default
默认机架感知是基于脚本的。
<property>
<name>net.topology.node.switch.mapping.impl</name>
<value>org.apache.hadoop.net.ScriptBasedMapping</value>
</property>
<!-- 配置脚本,默认为空,说明所有节点都在同一机架。 -->
<property>
<name>net.topology.script.file.name</name>
<value></value>
</property>
2.创建机架感知类
package com.it18zhang.hadoop.rackaware;
import org.apache.hadoop.net.DNSToSwitchMapping;
import java.io.FileOutputStream;
import java.util.ArrayList;
import java.util.List;
/**
* 自定义机架类
*/
public class MyDNSwitchMapping implements DNSToSwitchMapping{
public List<String> resolve(List<String> names) {
saveNames(names);
List<String> list = new ArrayList<String>();
for(String s: names){
list.add(extractIp(s)) ;
}
return list;
}
public void reloadCachedMappings() {
}
public void reloadCachedMappings(List<String> names) {
}
private String extractIp(String name){
if(name.startsWith("s")){
String ip = name.substring(1) ;
int intIp = Integer.parseInt(ip) ;
if(intIp < 103){
return "/rack1/" + ip ;
}
else{
return "/rack2/" + ip;
}
}
//
else if(name.contains(".")){
String ip = name.split("\\.")[3] ;
int intIp = Integer.parseInt(ip);
if (intIp < 103) {
return "/rack1/" + ip;
} else {
return "/rack2/" + ip;
}
}
return "/rack1/" + name ;
}
/**
* 保存名称到磁盘
*/
private void saveNames(List<String> names){
try {
FileOutputStream fos = new FileOutputStream("/home/centos/rack.log",true) ;
fos.write("===================\r\n".getBytes());
for(String s : names){
fos.write((s + "\r\n").getBytes());
}
fos.flush();
fos.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
3.导出jar包
添加artifacts -> 删除依赖的jar
4.部署jar到hadoop集群/soft/hadoop/share/hadoop/common/lib
5.配置core-site.xnml
<property>
<name>net.topology.node.switch.mapping.impl</name>
<value>com.it18zhang.hadoop.rackaware.MyDNSwitchMapping</value>
</property>