网络环境搭建CDH

安装文件准备：

https://archive.cloudera.com/cdh6/6.2.1/parcels/

https://archive.cloudera.com/cm6/6.2.1/redhat7/yum/RPMS/x86_64/
cloudera-manager-daemons-6.2.1-1426065.el7.x86_64.rpm

配置服务器固定ip

vim /etc/sysconfig/network-scripts/ifcfg-ens33

# 统一修改的参数
dhcp="static"

# hadoop01
UUID="821e9fe2-e759-4370-b66f-27a84dd1b377"
MACADDR="00:50:56:3A:E6:79"
IPADDR="192.168.5.130"
GATEWAY="192.168.5.2"
DNS1="114.114.114.114"
DNS2="221.138.24.66"

# hadoop02
UUID="821e9fe2-e759-4370-b66f-27a84dd1b388"
MACADDR="00:50:56:3E:8D:A5"
IPADDR="192.168.5.131"
GATEWAY="192.168.5.2"
DNS1="114.114.114.114"
DNS2="221.138.24.66"

# hadoop03
UUID="821e9fe2-e759-4370-b66f-27a84dd1b399"
MACADDR="00:50:56:3E:8D:AB"
IPADDR="192.168.5.132"
HWADDR=00:0c:29:b4:e3:22
GATEWAY="192.168.5.2"
DNS1="114.114.114.114"
DNS2="221.138.24.66"

安装基础环境

yum install -y cyrus-sasl-plain cyrus-sasl-gssapi portmap fuse-libs bind-utils libxslt fuse
yum install -y /lib/lsb/init-functions createrepo deltarpm python-deltarpm
yum install -y mod_ssl openssl-devel python-psycopg2 MySQL-python

修改hosts文件

vim /etc/hosts

192.168.5.130 hadoop01
192.168.5.131 hadoop02
192.168.5.132 hadoop03

添加centos7新用户cloudera-scm，并设置免密码登录

echo "cloudera-scm ALL=(root)NOPASSWD:ALL" >>/etc/sudoers
su - cloudera-scm
exit

mysql配置

设置密码 重设密码

set global validate_password_policy=0;
set global validate_password_length=1;
alter user 'root'@'localhost' identified by '123456';
flush privileges;

创建数据库并给权限

create database scm DEFAULT CHARACTER SET utf8;
grant all PRIVILEGES on *.* TO 'root'@'%' IDENTIFIED BY '123456' WITH GRANT OPTION;
grant all PRIVILEGES on *.* TO 'root'@'localhost' IDENTIFIED BY '123456' WITH GRANT OPTION;
grant all PRIVILEGES on *.* TO 'root'@'hadoop01' IDENTIFIED BY '123456' WITH GRANT OPTION;
flush privileges;

配置官方yum源 下载官方yum源

cd /etc/yum.repos.d/
wget https://archive.cloudera.com/cm6/6.2.1/redhat7/yum/cloudera-manager.repo
wget https://archive.cloudera.com/cdh6/6.2.1/redhat7/yum/RPM-GPG-KEY-cloudera

主节点安装server

yum install -y oracle-j2sdk1.8-1.8.0+update181-1.x86_64
yum install -y enterprise-debuginfo-6.2.1-1426065.el7.x86_64
yum install -y cloudera-manager-server-6.2.1-1426065.el7.x86_64

# 安装到一半停止后，将准备的cloudera-manager-agent-6.2.1-1426065.el7.x86_64文件放入到缓存路径中
yum install -y cloudera-manager-server-db-2-6.2.1-1426065.el7.x86_64

注意:如果在yum源中添加了新的包时:需要清除yum缓存yum clean all

设置元数据库为mysql

设置元数据库为mysql–前提是安装clouderaManager Server服务

/opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm -h192.168.5.130  -uroot -p'123456' --scm-host localhost scm scm scm
# 查看配置信息
cat /etc/cloudera-scm-server/db.properties

上传mysql驱动包jar到/opt/cloudera/cm/lib目录下文件重命名为 mysql-connector-java.jar

启动server服务进程(查看7180端口)

systemctl start cloudera-scm-server

查看启动日志:

tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log

查看启动状态:

systemctl status cloudera-scm-server

浏览器访问 ip:7180
账号密码都是admin
选择免费版
使用host搜索主节点
存储库默认
安装jdk，不安装java加密文件
输入ssh 密码

sysctl -w vm.swappiness=10
vim /etc/sysctl.conf
vm.swappiness=10

echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
vim /etc/rc.local
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled

每个节点

1.mysql-connector-java.jar放入/usr/share/java目录下

yum install -y oracle-j2sdk1.8-1.8.0+update181-1.x86_64
yum install -y cloudera-manager-daemons-6.2.1-1426065.el7.x86_64 cloudera-manager-agent-6.2.1-1426065.el7.x86_64
# 安装到一半停止后，将准备的cloudera-manager-agent-6.2.1-1426065.el7.x86_64文件放入到缓存路径中

配置agent

vim /etc/cloudera-scm-agent/config.ini
server_host=hadoop01

启动agent

systemctl start cloudera-scm-agent

浏览器访问 ip:7180

添加slave节点到集群1，步骤和添加master节点的设置一致

server

[root@hadoop01 ~]# mkdir -p /var/lib/cloudera-host-monitor
[root@hadoop01 ~]# mkdir /var/lib/cloudera-service-monitor
[root@hadoop01 ~]# chown -R cloudera-scm:cloudera-scm/var/lib/cloudera-service-monitor/
chown: missing operand after ‘cloudera-scm:cloudera-scm/var/lib/cloudera-service-monitor/’
Try 'chown --help' for more information.
chown -R cloudera-scm:cloudera-scm /var/lib/cloudera-service-monitor/
chown -R cloudera-scm:cloudera-scm /var/lib/cloudera-host-monitor

hdfs相关配置

1.hdfs安装之后 dfs.permissions该为False。否则不能创建目录
2.

# 查询safemode状态 确认为OFF
hdfs dfsadmin -safemode get
# 关闭safemode
bash hdfs dfsadmin -safemode leave

hive安装

hive服务所在主机需要在 /opt/cloudera/parcels/CDH/lib/hive/lib 目录下有 mysql-connector-java-8.0.22.jar