hadoop测试环境搭建（配置文件） 集群启动脚本

配置文件在/opt/hadoop/etc/hadoop

core-site.xml

<!-- Put site-specific property overrides in this file. -->

<configuration>
    <!-- 
		用于设置Hadoop的文件系统，由URI指定
		默认协议file:///属于本地路径，由于之后要用HDFS，因此需要修改为hdfs://主机名(ip):端口（内部的一个通讯）
	-->
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://node2:9000</value>
    </property>

    <!-- 配置Hadoop存储数据目录,默认/tmp/hadoop-${user.name} -->
    <property>
       <name>hadoop.tmp.dir</name>
       <value>/data/hadoop</value>
    </property>

    <!--  缓冲区大小，实际工作中根据服务器性能动态调整 -->
    <property>
       <name>io.file.buffer.size</name>
       <value>131072</value>
    </property>

    <property>
       <name>hadoop.http.staticuser.user</name>
       <value>hadoop</value>
    </property>
</configuration>

hdfs-site.xml

<!-- Put site-specific property overrides in this file. -->

<configuration>
       <!-- 设定SNN运行主机和端口。-->
        <property>
            <name>dfs.namenode.secondary.http-address</name>
            <value>node1:9868</value>
        </property>
        <!-- 指定HDFS保存数据副本的数量 -->
        <property>
                <name>dfs.replication</name>
                <value>3</value>
        </property>
        <!-- 设置一个block的大小：128M-->
        <property>
            <name>dfs.blocksize</name>
            <value>134217728</value>
        </property>
</configuration>

mapred-site.xml

<!-- Put site-specific property overrides in this file. -->

<configuration>
    	<!-- 指定 MapReduce 程序运行在 Yarn 上，表示MapReduce使用yarn框架 -->
        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>
</configuration>

yarn-site.xml
    

<configuration>
   <property>
      <name>yarn.resourcemanager.hostname</name>
      <value>node2</value>
   </property>
   <property>
      <name>yarn.nodemanager.aux-services</name>
      <value>mapreduce_shuffle</value>
   </property>
   <property>
       <name>yarn.application.classpath</name>
       <value>运行指令hadoop classpath获取值</value>
    </property>
</configuration>

启动脚本myhadoop

\#!/bin/bash
if [ $# -lt 1 ]
then
  echo "No Args Input..."
  exit ;
fi
case $1 in
"start")
     echo " =================== 启动 hadoop集群 ==================="
     echo " --------------- 启动 hdfs ---------------"
     ssh node2 "/opt/hadoop/sbin/start-dfs.sh"
     echo " --------------- 启动 yarn ---------------"
     ssh node2 "/opt/hadoop/sbin/start-yarn.sh"
;;
"stop")
     echo " =================== 关闭 hadoop集群 ==================="
     echo " --------------- 关闭 yarn ---------------"
     ssh node2 "/opt/hadoop/sbin/stop-yarn.sh"
     echo " --------------- 关闭 hdfs ---------------"
     ssh node2 "/opt/hadoop/sbin/stop-dfs.sh"
;;
*)
  echo "Input Args Error..."
;;
esac

查看进程脚本jspall

\#!/bin/bash
for host in node2 node1 node3
do
     echo =============== $host ===============
     ssh $host jps 
done