Hadoop单机版自动化部署脚本

大马猴写bug

于 2023-11-21 11:30:32 发布

阅读量165

点赞数

文章标签： hadoop 自动化

本文链接：https://blog.csdn.net/m0_57254949/article/details/134527821

版权

本文详细介绍了如何使用一个脚本来一键部署Hadoop单机伪分布环境，包括关闭防火墙、配置参数、创建文件夹、安装JDK、下载并解压Hadoop、配置环境变量、生成SSH秘钥、以及设置Hadoop配置文件和格式化Namenode。

摘要由CSDN通过智能技术生成

一、首先搞个小仪式

#!/bin/bash
echo "******欢迎使用Hadoop单机伪分布一键部署脚本******"
echo "****注意！在运行之前请先确认网络已经配置完毕****"
echo "*本机默认配置Hadoop-3.3.4版本,JDK版本为1.8_212**"
echo "**本机秘钥需要自己配置，详细请看文档注意事项！**"
echo "*************一切都在根目录下进行！*************"
echo "****************Hadoop部署开始！****************"

二、关闭防火墙和开机禁用

#关闭防火墙
systemctl stop firewalld.service
sleep 3
systemctl disable firewalld.service
sleep 3
echo "****************防火墙关闭完毕！****************"

三、配置参数

# 配置参数
hadoop_version="3.3.4"
java_home="opt/moudle/jdk"
hadoop_home="opt/moudle/hadoop"

四、创建需要的文件夹

#创建文件夹
sudo mkdir -p ./opt/moudle
sleep 3
jdk_tar=jdk-8u212-linux-x64.tar.gz
#配置jdk
# 检查文件是否存在
if [ -f "$jdk_tar" ]; then
  tar -zxvf "$jdk_tar" -C opt/moudle
else
  echo "***********************文件 $jdk_tar 不存在，请先上传！****************"
  exit 1
fi

五、解压上传的jdk和下载的hadoop

sleep 3
echo "****************JDK解压完成****************"
mv opt/moudle/jdk1.8.0_212 opt/moudle/jdk
echo "****************开始下载Hadoop!****************"
sleep 3
# 下载Hadoop
wget https://downloads.apache.org/hadoop/common/hadoop-$hadoop_version/hadoop-$hadoop_version.tar.gz
sleep 3
echo "****************下载完成开始解压！****************"
# 解压Hadoop
tar -xzf hadoop-$hadoop_version.tar.gz -C opt/moudle
sleep 3
# 文件重命名
sudo mv opt/moudle/hadoop-$hadoop_version opt/moudle/hadoop

六、配置环境变量

# 配置环境变量
#JAVA_HOME
echo "export JAVA_HOME=$java_home" >> ~/.bashrc
echo "export JRE_HOME=${JAVA_HOME}/jre" >> ~/.bashrc
echo "export CLASSPATH=${JAVA_HOME}/lib:${JRE_HOME}/lib" >> ~/.bashrc
echo "export PATH=${JAVA_HOME}/bin:$PATH" >> ~/.bashrc
#HADOOP_HOME
echo "export HADOOP_HOME=$hadoop_home" >> ~/.bashrc
echo "export PATH=\$PATH:\$HADOOP_HOME/bin:\$HADOOP_HOME/sbin" >> ~/.bashrc
sleep 3
source ~/.bashrc

七、配置秘钥

echo "****************环境变量配置完成！****************"
echo "****************开始配置秘钥，需要您手动回车四次!****************"
sleep 3
ssh_dir=~/.ssh
# 检查目录是否存在
if [ -d "$ssh_dir" ]; then
  cd "$ssh_dir"
  echo "****************请按回车键四次!****************"
  ssh-keygen -t rsa
else
  echo "文件夹 $ssh_dir 不存在"
  exit 1
fi
cat ./id_rsa.pub >> ./authorized_keys
cd
sleep 3

八、配置hadoop的配置文件

# 配置Hadoop
cd $hadoop_home/etc/hadoop
echo "****************配置core-site.xml****************"
# 设置core-site.xml
cat << EOF > core-site.xml
<configuration>
      <property>
        <name>hadoop.tmp.dir</name>
        <value>file:/usr/local/hadoop/tmp</value>
        <description>Abase for other temporary directories.</description>
    </property>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://localhost:9000</value>
    </property>
</configuration>
EOF
sleep 3
echo "****************配置hdfs-site.xml****************"
# 设置hdfs-site.xml
cat << EOF > hdfs-site.xml
<configuration>
<property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>file:/usr/local/hadoop/tmp/dfs/name</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>file:/usr/local/hadoop/tmp/dfs/data</value>
    </property>
	<property>
  <name>dfs.namenode.http-address</name>
  <value>hadoop:50070</value>
</property>
</configuration>
EOF
sleep 3
echo "****************配置yarn-site.xml****************"
# 设置yarn-site.xml
cat << EOF > yarn-site.xml
<configuration>
  <property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
  </property>
  <property>
    <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
  </property>
  <property>
    <name>yarn.resourcemanager.hostname</name>
    <value>localhost</value>
  </property>
</configuration>
EOF
cd 
sleep 3

九、格式化namenode

echo "****************文件格式化开始****************"
# 格式化Hadoop文件系统
hdfs namenode -format
sleep 3
echo "****************Hadoop部署完成！****************"
echo "************************************************"

十、最终完整代码

#!/bin/bash
echo "******欢迎使用Hadoop单机伪分布一键部署脚本******"
echo "****注意！在运行之前请先确认网络已经配置完毕****"
echo "*本机默认配置Hadoop-3.3.4版本,JDK版本为1.8_212**"
echo "**本机秘钥需要自己配置，详细请看文档注意事项！**"
echo "*************一切都在根目录下进行！*************"
echo "****************Hadoop部署开始！****************"

#关闭防火墙
systemctl stop firewalld.service
sleep 3
systemctl disable firewalld.service
sleep 3
echo "****************防火墙关闭完毕！****************"
# 配置参数
hadoop_version="3.3.4"
java_home="opt/moudle/jdk"
hadoop_home="opt/moudle/hadoop"


#创建文件夹
sudo mkdir -p ./opt/moudle
sleep 3
jdk_tar=jdk-8u212-linux-x64.tar.gz
#配置jdk
# 检查文件是否存在
if [ -f "$jdk_tar" ]; then
  tar -zxvf "$jdk_tar" -C opt/moudle
else
  echo "***********************文件 $jdk_tar 不存在，请先上传！****************"
  exit 1
fi

sleep 3
echo "****************JDK解压完成****************"
mv opt/moudle/jdk1.8.0_212 opt/moudle/jdk
echo "****************开始下载Hadoop!****************"
sleep 3
# 下载Hadoop
wget https://downloads.apache.org/hadoop/common/hadoop-$hadoop_version/hadoop-$hadoop_version.tar.gz
sleep 3
echo "****************下载完成开始解压！****************"
# 解压Hadoop
tar -xzf hadoop-$hadoop_version.tar.gz -C opt/moudle
sleep 3
# 文件重命名
sudo mv opt/moudle/hadoop-$hadoop_version opt/moudle/hadoop

# 配置环境变量
#JAVA_HOME
echo "export JAVA_HOME=$java_home" >> ~/.bashrc
echo "export JRE_HOME=${JAVA_HOME}/jre" >> ~/.bashrc
echo "export CLASSPATH=${JAVA_HOME}/lib:${JRE_HOME}/lib" >> ~/.bashrc
echo "export PATH=${JAVA_HOME}/bin:$PATH" >> ~/.bashrc
#HADOOP_HOME
echo "export HADOOP_HOME=$hadoop_home" >> ~/.bashrc
echo "export PATH=\$PATH:\$HADOOP_HOME/bin:\$HADOOP_HOME/sbin" >> ~/.bashrc
sleep 3
source ~/.bashrc
echo "****************环境变量配置完成！****************"
echo "****************开始配置秘钥，需要您手动回车四次!****************"
sleep 3
ssh_dir=~/.ssh
# 检查目录是否存在
if [ -d "$ssh_dir" ]; then
  cd "$ssh_dir"
  echo "****************请按回车键四次!****************"
  ssh-keygen -t rsa
else
  echo "文件夹 $ssh_dir 不存在"
  exit 1
fi
cat ./id_rsa.pub >> ./authorized_keys
cd
sleep 3
# 配置Hadoop
cd $hadoop_home/etc/hadoop
echo "****************配置core-site.xml****************"
# 设置core-site.xml
cat << EOF > core-site.xml
<configuration>
      <property>
        <name>hadoop.tmp.dir</name>
        <value>file:/usr/local/hadoop/tmp</value>
        <description>Abase for other temporary directories.</description>
    </property>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://localhost:9000</value>
    </property>
</configuration>
EOF
sleep 3
echo "****************配置hdfs-site.xml****************"
# 设置hdfs-site.xml
cat << EOF > hdfs-site.xml
<configuration>
<property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>file:/usr/local/hadoop/tmp/dfs/name</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>file:/usr/local/hadoop/tmp/dfs/data</value>
    </property>
	<property>
  <name>dfs.namenode.http-address</name>
  <value>hadoop:50070</value>
</property>
</configuration>
EOF
sleep 3
echo "****************配置yarn-site.xml****************"
# 设置yarn-site.xml
cat << EOF > yarn-site.xml
<configuration>
  <property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
  </property>
  <property>
    <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
  </property>
  <property>
    <name>yarn.resourcemanager.hostname</name>
    <value>localhost</value>
  </property>
</configuration>
EOF
cd 
sleep 3
echo "****************文件格式化开始****************"
# 格式化Hadoop文件系统
hdfs namenode -format
sleep 3
echo "****************Hadoop部署完成！****************"
echo "************************************************"