hadoop集群通用初始化详细步骤

分发免密：
把跳板机的（ambari-server）~/.ssh/id_rsa.pub 增加到要跳到的主机的authorized_keys文件中
通过分发authorized_keys实现批量免密

免密：
#!/bin/bash
#注：运行启动后，第一个(y/n)?需要手工输入 y
ssh-keygen -t rsa -P ‘’ -f /root/.ssh/id_rsa

cat ip.list|while read line
do
ip=echo $line|cut -d' ' -f1
#user=echo $line|cut -d' ' -f2
#passwd=echo $line|cut -d' ' -f3
user=“root”
passwd=“2D0snAY6GFCA6Pw2”

#将生成的公钥文件id_rsa.pub 拷贝至集群其他Ambari Agent主机，并加入Ambari Agent的授权列表
#“password:” {send “$passwd\r”;}
#“yes/no” {send “yes\r”;}
#“Enter file in which to save the key*” {send “\r”;}
#“Enter passphrase*” {send “\r”;}
#“Enter same passphrase again:” {send “\r”;}
#“Overwrite (y/n)” {send “\r”;}

expect -c "
spawn ssh-copy-id -i /root/.ssh/id_rsa.pub $user@$ip
expect {
“yes/no” { send “yes\r”; exp_continue}
“*assword:” { send “$passwd\r”}
“already exist” {send “\r”}
}
expect eof"
done

防火墙
systemctl stop iptables
systemctl stop firewalld
systemctl disable firewalld

yum源安装：
（rpm -qa|grep http）检查是否已经安装
（yum install yum-utils createrepo yum-plugin-priorities）
（yum install httpd wget）
修改HTTP默认配置
（vi /etc/httpd/conf/httpd.conf）
1）修改 DocumentRoot为"/var/www/html"
2）修改 <Directory “/var/www/html”>节：添加Options Indexes FollowSymLinks
3）修改“#ServerName www.example.com:80”去掉“#，改为ServerName localhost
4）删除默认页面：
（rm -f /etc/httpd/conf.d/welcome.conf /var/www/error/noindex.html）
设置HTTP开机自动重启
systemctl enable httpd
systemctl restart httpd

配置http yum源
（vi /etc/yum.repos.d/centos.repo）如下：
[Centos]
name=Centos7
baseurl=http://@host@/centos
gpgkey=http://@host@/centos/PM-GPG-KEY-CentOS-7
gpgcheck=1
enabled=1

（vi /etc/yum.repos.d/ambari.repo）如下：
[ambari-2.7.0.0-139]
name=ambari-2.7.0.0-139
baseurl=http://@host@/ambari/centos7/2.7.3.0-139/
path=/
gpgcheck=0
gpgkey= http://@host@/ambari/centos7/2.7.3.0-139/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
priority=1

（vi /etc/yum.repos.d/hdp.repo）如下：
[HDP-3.1]
name=HDP-3.1
baseurl=http://@host@/HDP/centos7/3.1.0.0-78/
path=/
enabled=1
gpgcheck=0

[HDP-UTILS-1.1.0.22]
name=HDP-UTILS-1.1.0.22
baseurl=http://@host@/HDP-UTILS/centos7/1.1.0.22/
path=/
enabled=1
gpgcheck=0

[HDP-3.1-GPL]
name=HDP-3.1-GPL
baseurl=http://@host@/HDP-GPL/centos7/3.1.0.0-78/
path=/
enabled=1
gpgcheck=0

yum clean all
yum repolist

./runRemoteCmd.sh “rm -rf /etc/yum.repos.d/*” bnew
./deploy.sh /etc/yum.repos.d /etc/ bnew

创建testuser用户：
groupadd -g 1027 testuser
useradd -u 1027 -g testuser -d /home/testuser testuser
echo ‘Aa12345’| passwd testuser --stdin

不允许开启sudo
cp /etc/sudoers /etc/sudoers_20201130
vi /etc/sudoers
增加 testuser ALL=(ALL) NOPASSWD:ALL

ntp时钟同步
yum install -y ntp
systemctl enable ntpd（设置开机自启）

server
（vi /etc/ntp.conf）

---

#Hosts on local network are less restricted.
restrict 10.19.0.0 mask 255.255.0.0 nomodify notrap # 允许内网其他机器同步时间

#Please consider joining the pool (http://www.pool.ntp.org/join.html).
#server 0.centos.pool.ntp.org iburst #注释掉原时钟服务器（）
server 127.127.1.0 # local clock
fudge 127.127.1.0 stratum 10

client
#(vi /etc/ntp.conf)注释原时钟服务器，增加2行
#Hosts on local network are less restricted
restrict @host@ nomodify notrap noquery
server @host@ #ntp server inner net
#server 1.centos.pool.ntp.org
#server 2.centos.pool.ntp.org

systemctl restart ntpd
systemctl status ntpd
ntpstat

getenforce
setenforce 0
sed -i ‘s/SELINUX=permissive/SELINUX=disabled/g’ /etc/selinux/config
cat /etc/selinux/config | grep SELINUX
rpm -qa | grep packagekitd

umask
umask 0022
echo umask 0022 >> /etc/profile
source /etc/profile

配置句柄数：
echo ‘’’

• soft nofile 65536
• hard nofile 65536
• soft nproc 65536
• hard nproc 65536
  ‘’’ >> /etc/security/limits.conf

echo ‘’’
session required pam_limits.so
‘’’ >> /etc/pam.d/login

ulimit -a

sudo ulimit -n 165536
sudo ulimit -u 165536

禁用THP:
sudo -i
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo “echo never > /sys/kernel/mm/transparent_hugepage/defrag” >> /etc/rc.d/rc.local
echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled " >> /etc/rc.d/rc.local
chmod +x /etc/rc.d/rc.local

禁用交换分区:
sudo -i
sysctl vm.swappiness=0
echo vm.swappiness=0 >>/etc/sysctl.conf

安装jdk：
yum install -y java-1.8.0-openjdk*
查看 jdk 安装路径
which java
ls -ltr /usr/bin/java
ls -ltr /etc/alternatives/java
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.181-7.b13.el7.x86_64/jre
echo export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.181-7.b13.el7.x86_64/jre >> /etc/profile
source /etc/profile

export JAVA_HOME=/usr/local/jdk1.8.0_271/
export PATH=$JAV{A}_{H}OME/bin:$PATH
echo export JAVA_HOME=/usr/local/jdk1.8.0_271/ >> /etc/profile
source /etc/profile

tar –zxvf jdk-8u91-linux-x64.tar.gz
chown -R testuser:testuser /usr/jdk1.8.0_131/
./runRemoteCmd.sh “chown -R testuser:testuser /usr/jdk1.8.0_131/” G
./runRemoteCmd.sh “chmod 755 -R /usr/jdk1.8.0_131/” G
使用java：
export PATH=$JAV{A}_{H}OME/bin:$PATH

环境检查：
python -V
rpm -qa | grep python-devel
yum -y install python-devel
$JAVA_HOME/bin/java -version
df -h
yum localinstall -y libtirpc-devel-0.2.4-0.16.el7.x86_64.rpm