JDK1.8 + Hadoop3.2.2 + Spark3.1.2

JDK1.8

# 修改host
tar -zvxf jdk-8u251-linux-x64.tar.gz
​
mv jdk1.8.0_251 /usr/local/java
​
sudo nano /etc/profile
​
#JAVA环境变量
export JAVA_HOME=/usr/local/java
export PATH=$PATH:$JAVA_HOME/bin
​
source /etc/profile
​
java -version

---

Hadoop3.2.2

tar -zvxf hadoop-3.2.2.tar.gz
​
mv hadoop-3.2.2 /usr/local/hadoop
​
sudo nano /etc/profile
​
#HADOOP
export HADOOP_HOME=/usr/local/hadoop
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
​
source /etc/profile
​
hadoop version

---

Hadoop配置

stage1: 在 /usr/local/hadoop/etc/hadoop修改文件：

# Workers
Master

# core-site.xml
<configuration>
        <property>
                <name>fs.defaultFS</name>
                <value>hdfs://Master:9000</value>
        </property>
        <property>
                <name>hadoop.tmp.dir</name>
                <value>file:/usr/local/hadoop/tmp</value>
                <description>Abase for other temporary directories.</description>
        </property>
</configuration>

# hdfs-site.xml
<configuration>
        <property>
                <name>dfs.namenode.secondary.http-address</name>
                <value>Master:50090</value>
        </property>
        <property>
                <name>dfs.replication</name>
                <value>3</value>
        </property>
        <property>
                <name>dfs.namenode.name.dir</name>
                <value>file:/usr/local/hadoop/tmp/dfs/name</value>
        </property>
        <property>
                <name>dfs.datanode.data.dir</name>
                <value>file:/usr/local/hadoop/tmp/dfs/data</value>
        </property>
</configuration>

# mapred-site.xml
<configuration>
        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>
        <property>
                <name>mapreduce.jobhistory.address</name>
                <value>Master:10020</value>
        </property>
        <property>
                <name>mapreduce.jobhistory.webapp.address</name>
                <value>Master:19888</value>
        </property>
</configuration>

# yarn-site.xml
<configuration>
        <property>
                <name>yarn.resourcemanager.hostname</name>
                <value>Master</value>
        </property>
        <property>
                <name>yarn.nodemanager.aux-services</name>
                <value>mapreduce_shuffle</value>
        </property>
</configuration>

stage2: 在/usr/local/hadoop/sbin修改文件：

# start-dfs.sh、stop-dfs.sh
HDFS_DATANODE_USER=root
HDFS_DATANODE_SECURE_USER=hdfs
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root

# start-yarn.sh、stop-yarn.sh
YARN_RESOURCEMANAGER_USER=root
HDFS_DATANODE_SECURE_USER=yarn
YARN_NODEMANAGER_USER=root

source /etc/profile
​
hadoop version
​

Spark3.1.2

tar -zvxf spark-3.1.2-bin-hadoop3.2.tgz
​
mv spark-3.1.2-bin-hadoop3.2 /usr/local/spark
​

cp spark-env.sh.template spark-env.sh
# 在spark-env.sh中加入以下代码
export SPARK_DIST_CLASSPATH=$(/usr/local/hadoop/bin/hadoop classpath)
​

cd spark
./bin/spark-submit --master local[*] examples/src/main/python/pi.py