Update/genrate following files in ${HADOOP_CONF_DIR}:
- core-site.xml:
<!--Rack aware-->
<property>
<name>net.topology.script.file.name</name>
<value>${HADOOP_CONF_DIR}/rack-topology.sh</value>
</property> - Add file ${HADOOP_CONF_DIR}/rack-topology.sh and make it executable:
#!/bin/bash
# Supply appropriate rack prefix
RACK_PREFIX=default
# To test, supply a hostname as script input:
if [ $# -gt 0 ]; then
CTL_FILE=rack_topology.data
if [ ! -f ${HADOOP_CONF_DIR}/${CTL_FILE} ]; then
echo -n "/$RACK_PREFIX/rack "
exit 0
fi
while [ $# -gt 0 ] ; do
nodeArg=$1
exec< ${HADOOP_CONF_DIR}/${CTL_FILE}
result=""
while read line ; do
ar=( $line )
if [ "${ar[0]}" = "$nodeArg" ] ; then
result="${ar[1]}"
fi
done
shift
if [ -z "$result" ] ; then
echo -n "/$RACK_PREFIX/rack "
else
echo -n "/$RACK_PREFIX/rack_$result "
fi
done
else
echo -n "/$RACK_PREFIX/rack "
fi - Generate ${HADOOP_CONF_DIR}/rack_topology.data:
# Add Hostnames to this file. Format <host ip> <rack_location>
ip1 01
ip2 02 - Test to make sure could get rack information:
[hadoop@master01 hadoop]$ ./rack-topology.sh master01.dc.tj
/default/rack_01 - rsync the ${HADOOP_CONF_DIR} in all nodes od hadoop cluster