一步一个坑的在docker里安装hadoop

• 创建一个Dockerfile

FROM ubuntu RUN mkdir opt/fsd #RUN apt-get install ssh #RUN apt-get install pdsh RUN apt-get update RUN apt install vim --assume-yes ADD jdk-11.0.11_linux-x64_bin.tar.gz opt/fsd ENV JAVA_HOME=/opt/fsd/jdk-11.0.11 ENV PATH=$JAVA_HOME/bin:$PATH ENV CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

• 创建image

>docker build -t hado:latest .

 

• 运行container

>docker run -v /Users/Shared/dockerdir:/data --name hadoin  -it -p 9870:9870 hado /bin/bash

• 在docker中安装hadoop

>cp /data/hadoop-3.3.0.tar.gz .

>tar  -zxvf hadoop-3.3.0.tar.gz

#edit the file etc/hadoop/hadoop-env.sh
export JAVA_HOME=/opt/fsd/jdk-11.0.11

bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.0.jar grep input output 'dfs[a-z.]+'

• 安装ssh

>apt-get update
>apt-get install ssh
>apt-get install pdsh

/etc/profile里加入：
export PDSH_RCMD_TYPE=ssh

>source /etc/profile

 >service ssh status
>service ssh start

#Now check that you can ssh to the localhost without a passphrase:
 > ssh localhost

#If you cannot ssh to localhost without a passphrase, execute the following commands:
  > ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
  > cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
  > chmod 0600 ~/.ssh/authorized_keys

• 运行dfs

> bin/hdfs namenode -format
>sbin/start-dfs.sh

#Starting namenodes on [66cfa584a4d5]
#pdsh@66cfa584a4d5: 66cfa584a4d5: ssh exited with exit code 1
#check log find below error
#Invalid URI for NameNode address (check fs.defaultFS): file:/// has no authority.
 etc\hadoop\core-site.xml的<configuration>
<property>
    <name>fs.default.name</name>
    <value>hdfs://localhost:8070</value>
 </property>
 
 http://localhost:9870/