【大数据】CDH环境搭建

CDH安装

时钟对齐

systemctl enable ntpd
systemctl start ntpd 
systemctl status ntpd

免密登录

ssh-keygen -t rsa
						
ssh-copy-id node01
ssh-copy-id node02
ssh-copy-id node03

mysql安装

# mysql安装包下载
wget http://dev.mysql.com/get/Downloads/MySQL-5.7/mysql-5.7.16-1.el7.x86_64.rpm-bundle.tar


# 解压
tar -xvf mysql-5.7.16-1.el7.x86_64.rpm-bundle.tar 

# 安装
rpm -ivh mysql-community-common-5.7.16-1.el7.x86_64.rpm
rpm -ivh mysql-community-libs-5.7.16-1.el7.x86_64.rpm
rpm -ivh mysql-community-libs-compat-5.7.16-1.el7.x86_64.rpm
rpm -ivh mysql-community-client-5.7.16-1.el7.x86_64.rpm
rpm -ivh mysql-community-server-5.7.16-1.el7.x86_64.rpm

# 启动服务
systemctl start mysqld

# 查看初识密码
cat /var/log/mysqld.log | grep password

# mysql登录
mysql -uroot -p

# mysql数据初始化
create database cmf DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database amon DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
set global validate_password_length=4;
set global validate_password_policy=0;
grant all privileges on cmf.* to 'cmf'@'%' identified by 'P@ssw0rd';
grant all privileges on amon.* to 'amon'@'%' identified by 'P@ssw0rd';

# Hive数据库
create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

# grant #auth on #databaseName.#table to '#userName'@'#host';

# 给test账号赋值权限


# Oozie数据库
create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

# Hue数据库
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

flush privileges;
quit;

CM部署

安装依赖包

yum -y install chkconfig python bind-utils psmisc libxslt zlib sqlite cyrus-sasl-plain cyrus-sasl-gssapi fuse fuse-libs redhat-lsb portmap httpd mod_ssl openssl-devel python-psycopg2 MySQL-python libpq.so.5

主节点安装

rpm -ivh cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm 
rpm -ivh cloudera-manager-server-6.3.1-1466458.el7.x86_64.rpm  

修改db.properties

vim /etc/cloudera-scm-server/db.properties

# Copyright (c) 2012 Cloudera, Inc. All rights reserved.
#
# This file describes the database connection.
#

# The database type
# Currently 'mysql', 'postgresql' and 'oracle' are valid databases.
com.cloudera.cmf.db.type=mysql

# The database host
# If a non standard port is needed, use 'hostname:port'
#com.cloudera.cmf.db.host=node01:3306

# The database name
com.cloudera.cmf.db.name=cmf

# The database user
com.cloudera.cmf.db.user=cmf

# The database user's password
com.cloudera.cmf.db.password=P@ssw0rd

# The db setup type
# After fresh install it is set to INIT
# and will be changed post config.
# If scm-server uses Embedded DB then it is set to EMBEDDED
# If scm-server uses External DB then it is set to EXTERNAL
com.cloudera.cmf.db.setupType= EXTERNAL

安装agent

rpm -ivh cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm

vim /etc/cloudera-scm-agent/config.ini

[General]
# Hostname of the CM server.
server_host=node01

初始化

/opt/cloudera/cm/schema/scm_prepare_database.sh mysql cmf cmf

启动server和agent

主节点启动serve

systemctl start cloudera-scm-server

查看日志，出现7180说明启动成功,，有错误解决错误

tail -F /var/log/cloudera-scm-server/cloudera-scm-server.log

子节点安装daemons agent

rpm -ivh cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm 
rpm -ivh cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm

vi /etc/cloudera-scm-agent/config.ini

[General]
# Hostname of the CM server.
server_host=node01

所有节点启动 agent

systemctl start cloudera-scm-agent