hadoop配置文件修改
先进入hadoop配置文件所在路径:
cd /usr/local/src/hadoop-2.6.1/etc/hadoop
修改slaves
vim slaves
修改core-site.xml
vim core-site.xml
<configuration>
<property>
<!--指定namenode的地址-->
<name>fs.defaultFS</name>
<value>hdfs://192.168.43.10:9000</value> #主机和端口
</property>
<property>
<!--用来指定使用hadoop时产生文件的存放目录-->
<name>hadoop.tmp.dir</name>
<value>file:/usr/local/src/hadoop-2.6.1/tmp</value>
</property>
</configuration>
修改mapred-site.xml
先拷贝一份:
cp mapred-site.xml.template mapred-site.xml
修改内容:
vim mapred-site.xml
<configuration>
<property>
<!--告诉hadoop以后MR(Map/Reduce)运行在YARN上-->
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>
修改hdfs-site.xml
vim hdfs-site.xml
<configuration>
<property>
<!--为secondary指定访问ip:port-->
<name>dfs.namenode.secondary.http-address</name>
<value>master:9001</value>
</property>
<property>
<!--指定hdfs中namenode的存储位置-->
<name>dfs.namenode.name.dir</name>
<value>file:/usr/local/src/hadoop-2.6.1/dfs/name</value>
</property>
<property>
<!--指定hdfs中datanode的存储位置-->
<name>dfs.datanode.data.dir</name>
<value>file:/usr/local/src/hadoop-2.6.1/dfs/data</value>
</property>
<property>
<!--指定hdfs保存数据的副本数量-->
<name>dfs.repliction</name>
<value>3</value>
</property>
</configuration>
修改hadoop-env.sh
导入JDK环境变量
vim hadoop-env.sh
export JAVA_HOME=/usr/local/src/jdk1.8.0_161
修改yarn-env.sh
导入JDK环境变量
vim yarn-env.sh
export JAVA_HOME=/usr/local/src/jdk1.8.0_161
修改PID存放路径(防止有时临时 目录被清空,启动后找不到节点)
export HADOOP_PID_DIR=/var/hadoop/pids
export HADOOP_SECURE_DN_PID_DIR=/var/hadoop/pids
修改yarn-site.xml
vim yarn-site.xml
<configuration>
<!-- Site specific YARN configuration properties -->
<property>
<!--nomenodeManager获取数据的方式是shuffle-->
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
<!--客户端对ResourceManager主机通过 host:port 提交作业-->
<name>yarn.resourcemanager.address</name>
<value>master:8032</value>
</property>
<property>
<!--ApplicationMasters 通过ResourceManager主机访问host:port跟踪调度程序获资源-->
<name>yarn.resourcemanager.scheduler.address</name>
<value>master:8030</value>
</property>
<property>
<!--NodeManagers通过ResourceManager主机访问host:port-->
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>master:8035</value>
</property>
<property>
<!--管理命令通过ResourceManager主机访问host:port-->
<name>yarn.resourcemanager.admin.address</name>
<value>master:8033</value>
</property>
<property>
<!--ResourceManager web页面host:port.-->
<name>yarn.resourcemanager.webapp.address</name>
<value>master:8088</value>
</property>
</configuration>