一.单机部署
系统版本:cat /etc/anolis-release
1.安装 JDK(JDK11)
TOP_PATH=$(pwd)
JAVA_PATH=/usr/local/java
FILE=`ls $TOP_PATH/ | grep jdk | grep .tar.gz | grep -v grep `
if [ -d $JAVA_PATH ];then
rm -rf $JAVA_PATH
fi
mkdir -p $JAVA_PATH
tar -zxvf $TOP_PATH/$FILE -C $JAVA_PATH
echo 'export JAVA_HOME=/usr/local/java/jdk-11.0.19' >> /etc/profile
echo 'export CLASSPATH=$JAVA_HOME/lib:$CLASSPATH' >> /etc/profile
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> /etc/profile
source /etc/profile
查看版本:java -version
2.安装 HADOOP
下载:Apache Hadoop
TOP_PATH=$(pwd)
HADOOP_PATH=/usr/local/hadoop
FILE=`ls $TOP_PATH/ | grep hadoop | grep .tar.gz | grep -v grep `
if [ -d $HADOOP_PATH ];then
rm -rf $HADOOP_PATH
fiHAD
mkdir -p $HADOOP_PATH
tar -zxvf $TOP_PATH/$FILE -C $HADOOP_PATH
echo 'export HADOOP_HOME=/usr/local/hadoop/hadoop-3.3.6' >> /etc/profile
echo 'export PATH=${PATH}:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin' >> /etc/profile
source /etc/profile
查看版本:hadoop version
3.测试
mkdir /home/input
cp $HADOOP_HOME/etc/hadoop/*.xml /home/input/
cd $HADOOP_HOME/
./bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.6.jar grep /home/input/ /home/output 'dfs[a-z]+'
查看输出结果:cat /home/output/part-r-00000
验证结果正确性:find /home/input -type f -name "*.xml" | xargs grep dfs
创建一个测试文件:touch /home/input/text.xml && echo dfstext >> /home/input/text.xml && rm -rf /home/output
重新统计:./bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.6.jar grep /home/input/ /home/output 'dfs[a-z]+'
查看结果:cat /home/output/part-r-00000