使用ambari安装Hadoop集群

订阅专栏
一、环境准备
1.操作系统检查
操作系统版本检查

cat /etc/redhat-release

操作系统默认语言检查

echo $LANG
echo 'export LANG=en_US.UTF-8' >> ~/.bashrc

操作系统时区设置

    cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

设置主机名
vi /etc/sysconfig/network

NETWORKING=yes
HOSTNAME=node1

设置命令：（hostnamectl set-hostname {主机名}）

vi /etc/hosts

scp /etc/hosts root@{hosts}:/etc/hosts

验证：
hostname -i 正确返回主机ip
hostname -f 正确返回主机名称

创建普通用户

    groupadd username
    2)    useradd -g username -d /home/username username  //username分别对应用户目录、创建的用户
    3)    echo "root@123" | passwd --stdin username   //修改username密码
    4)    授sudo权限
    cp /etc/sudoers /etc/sudoers_bak
    echo "userName ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers

设置节点互信

ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
ssh-copy-id -i ~/.ssh/id_rsa.pub userName@{target_host}
chmod 700 ~/.ssh
chmod 600 ~/.ssh/authorized_keys

配置http

rpm -qa|grep http
yum install yum-utils createrepo yum-plugin-priorities httpd

vi /etc/httpd/conf/httpd.conf
修改“#ServerName www.example.com:80”去掉#，改为ServerName localhost

systemctl enable httpd
systemctl restart httpd

配置yum源
vi /etc/yum.repos.d/ambari.repo

[ambari-2.7.3]
name=ambari-2.7.3
baseurl=http://ip/xxx/ambari/centos7/2.7.3.0-139/
gpgkey=http://ip/xxx/ambari/centos7/2.7.3.0-139/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
gpgcheck=0
enabled=1

同步ambari.repo文件
清空repo缓存
yum clean all
验证
yum repolist

时钟同步
10.1 安装ntp服务：

rpm  -qa|grep ntp
yum install  -y ntp
systemctl is-enabled ntpd
systemctl enable ntpd

10.2 ntp server端配置文件
vi /etc/ntp.conf

restrict 192.168.6.3 nomodify notrap nopeer noquery          //当前节点IP地址
restrict 192.168.6.1 mask 255.255.255.0 nomodify notrap  //集群所在网段的网关（Gateway），子网掩码（Genmask）
server 127.127.1.0
Fudge 127.127.1.0 stratum 10

10.3 ntp client端配置文件
vi /etc/ntp.conf

restrict 192.168.6.4 nomodify notrap nopeer noquery          //当前节点IP地址
restrict 192.168.6.1 mask 255.255.255.0 nomodify notrap  //集群所在网段的网关（Gateway），子网掩码（Genmask）
server 192.168.6.3
Fudge 192.168.6.3 stratum 10

重启所有主机ntp服务
systemctl restart ntpd
client验证：ntpstat

关闭防火墙

systemctl disable firewalld
systemctl stop firewalld
systemctl disable iptables
systemctl stop iptables 

关闭SELinux
getenforce
临时关闭：setenforce 0

vi /etc/selinux/config

SELINUX=disabled

设置packagekit
vi /etc/yum/pluginconf.d/refresh-packagekit.conf

enabled=0

设置umask
umask 0022

echo umask 0022 >> /etc/profile

设置ulimit、nproc
vi /etc/security/limits.conf

userName  soft nofile  655350
userName  hard nofile  655350
userName  soft nproc  655350
userName  hard nproc  655350

禁用透明大叶THP
cat /sys/kernel/mm/transparent_hugepage/defrag
cat /sys/kernel/mm/transparent_hugepage/enabled
临时生效：
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled

vi /etc/rc.d/rc.local

if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
echo never > /sys/kernel/mm/transparent_hugepage/enabled
fi
if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
echo never > /sys/kernel/mm/transparent_hugepage/defrag
fi

chmod +x /etc/rc.d/rc.local

禁用交换分区
sysctl vm.swappiness=0

echo vm.swappiness=0 >> /etc/sysctl.conf

jdk
vi /etc/profile

export JAVA_HOME=/usr/local/jdk1.8.0_91
export PATH=$PATH:$JAVA_HOME/bin

二、安装ambari服务
MySQL-HA

查看MySQL
rpm -qa|grep mysql
rpm -qa|grep mariadb
卸载MySQL
rpm -e --nodeps mysql-5.1.73-7.el6.x86_64
rpm -e --nodeps mysql-connector-odbc-5.1.5r1144-7.el6.x86_64
rpm -e --nodeps mysql-libs-5.1.73-7.el6.x86_64
安装
依赖关系依次为common→libs→client→server
rpm -ivh mysql-community-common-5.7.**.el7.x86_64.rpm
rpm -ivh mysql-community-libs-5.7. **.el7.x86_64.rpm
rpm -ivh mysql-community-client-5.7. **-1.el7.x86_64.rpm
rpm -ivh mysql-community-server-5.7. **-1.el7.x86_64.rpm
yum install mysql-connector-java.noarch

vi /etc/my.cnf

init_connect='SET collation_connection = utf8_unicode_ci'
init_connect='SET NAMES utf8'
character-set-server=utf8
collation-server=utf8_unicode_ci
skip-character-set-client-handshake
#不区分大小写
lower_case_table_names=1

systemctl start mysqld
systemctl enable mysqld
找到临时登录密码：grep password /var/log/mysqld.log
mysql -u root -p

修改root密码
mysql> set global validate_password_policy=0;
mysql> set global validate_password_length=1;
mysql> ALTER USER 'root'@'localhost' IDENTIFIED BY 'root@123';

增加root远程登录
mysql> CREATE USER 'root'@'%' IDENTIFIED BY 'root@123';
mysql> grant all privileges on *.* to 'root'@'%' with grant option;
mysql> FLUSH PRIVILEGES;
mysql> grant all privileges on *.* to root@'%' identified by 'xxxxxx';
mysql> 
mysql> 
mysql> FLUSH PRIVILEGES;

CREATE USER 'ranger'@'%' IDENTIFIED BY 'ranger123';
GRANT ALL PRIVILEGES ON *.* TO 'ranger'@'%' WITH GRANT OPTION;

安装ambari服务

sudo yum -y install ambari-server
sudo ambari-server setup --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar
sudo ambari-server setup

创建ambari数据库：
mysql>create database ambari;
mysql> use ambari;
mysql>CREATE USER 'ambari'@'%' IDENTIFIED BY 'ambari123';
mysql>GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'%';
mysql>FLUSH PRIVILEGES;
mysql>source /var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql

三、部署Hadoop集群
安装基本组件服务

sudo ambari-server start

开启Kerberos

安装Kerberos服务
yum -y install krb5-libs krb5-server krb5-workstation krb5-auth-dialog

配置krb5.conf文件
vi /etc/krb5.conf

# Configuration snippets may be placed in this directory as well
includedir /etc/krb5.conf.d/

[logging]
 default = FILE:/var/log/krb5libs.log
 kdc = FILE:/var/log/krb5kdc.log
 admin_server = FILE:/var/log/kadmind.log

[libdefaults]
 dns_lookup_realm = false
 ticket_lifetime = 24h
 renew_lifetime = 7d
 forwardable = true
 rdns = false
 default_realm = HADOOP.COM
 default_ccache_name = KEYRING:persistent:%{uid}

[realms]
 HADOOP.COM = {
 kdc = node01
 admin_server = node01
 }

[domain_realm]
# .hadoop.com = HADOOP.COM
# hadoop.com = HADOOP.COM    

配置kdc.conf配置文件
vi /var/kerberos/krb5kdc/kdc.conf

[kdcdefaults]
 kdc_ports = 88
 kdc_tcp_ports = 88

[realms]
 # 修改域名称为HADOOP.COM
 HADOOP.COM = {
  #master_key_type = aes256‐cts
  acl_file = /var/kerberos/krb5kdc/kadm5.acl
  dict_file = /usr/share/dict/words
  admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab
  supported_enctypes = aes256‐cts:normal aes128‐cts:normal des3‐hmac‐sha1:normal arcfour‐hmac:normal camellia256‐cts:normal camellia128‐cts:normal des‐hmac‐sha1:normal des‐cbc‐md5:normal des‐cbc‐crc:normal
 }

配置kadm5.acl文件
vi /var/kerberos/krb5kdc/kadm5.acl
# 修改域名称为HADOOP.COM
*/admin@HADOOP.COM *

创建KDC数据库
kdb5_util create -r HADOOP.COM -s
输入key: kdc@123

systemctl start krb5kdc
systemctl start kadmin
systemctl enable krb5kdc
systemctl enable kadmin

创建Kerberos数据库的管理员账户
kadmin.local -q "addprinc admin/admin"
输入密码： kdc@123

测试：
查看所有principal，kadmin.local进入交互命令
kadmin.local: list_principals

认证验证：
kinit admin/admin
klist

创建一个principal:
kadmin.local ‐q "addprinc wus01/admin"

安装ldap

yum -y install openldap-servers openldap-clients
cp /usr/share/openldap-servers/DB_CONFIG.example /var/lib/ldap/DB_CONFIG
chown ldap:ldap /var/lib/ldap/DB_CONFIG
systemctl start slapd
systemctl enable slapd

四、问题
yum安装包HTTP Error 416问题

解决方式：

在配置的yum源安装包的路径下执行：createrepo ./
2.清理yum缓存：rm -rf /var/cache/yum
安装MySQL遇到冲突：

卸载：
rpm -ev postfix-2:2.10.1-6.el7.x86_64
rpm -ev mariadb-libs-1:5.5.56-2.el7.x86_64
在依次执行安装命令
rpm -ivh mysql-community-common-5.7.22-1.el7.x86_64.rpm
rpm -ivh mysql-community-libs-5.7.22-1.el7.x86_64.rpm
rpm -ivh mysql-community-client-5.7.22-1.el7.x86_64.rpm
rpm -ivh mysql-community-server-5.7.22-1.el7.x86_64.rpm
缺少mysqlclient.so
安装：rpm -ivh mysql-community-libs-compat-5.7.22-1.el7.x86_64.rpm
缺少 libtirpc-devel
手工下载程序包并安装：
libtirpc-0.2.4-0.16.el7.x86_64.rpm
libtirpc-devel-0.2.4-0.16.el7.x86_64.rpm