1.配置 Cloudera Manager 仓库(所有节点)
wget https://archive.cloudera.com/cm6/6.2.0/redhat7/yum/cloudera-manager.repo -P /etc/yum.repos.d/
rpm --import https://archive.cloudera.com/cm6/6.2.0/redhat7/yum/RPM-GPG-KEY-cloudera
使用在线安装会比较慢,建议先把需要的rpm下载下来,进行离线安装或者建私有仓库,涉及下面三个软件包:
cloudera-manager-agent-6.2.0-968826.el7.x86_64.rpm
cloudera-manager-server-6.2.0-968826.el7.x86_64.rpm
cloudera-manager-daemons-6.2.0-968826.el7.x86_64.rpm
2.配置 JDK (所有节点) 如果已经安装过jdk那就跳过此步
//下载地址,https://repo.huaweicloud.com/java/jdk/8u202-b08/
rpm -ivh jdk-8u202-linux-x64.rpm
3.安装 CM Server 和 Agent
建议离线安装,把rpm包下载到服务器上面,传到其他节点一份,再本地安装,速度会快很多。
namenode01:
#yum localinstall cloudera-manager-daemons-6.2.0-968826.el7.x86_64.rpm -y
#yum localinstall cloudera-manager-agent-6.2.0-968826.el7.x86_64.rpm -y
#yum localinstall cloudera-manager-server-6.2.0-968826.el7.x86_64.rpm -y
datanode01:
#yum localinstall cloudera-manager-daemons-6.2.0-968826.el7.x86_64.rpm -y
#yum localinstall cloudera-manager-agent-6.2.0-968826.el7.x86_64.rpm -y
4.安装数据库(前提是mysql已经安装成功)
在namenode01节点,安装
yum install mariadb* -y
可选步骤。根据官方推荐的配置,编辑文件/etc/my.cnf,修改成如下内容:
# cp /etc/my.cnf{,.bak}
#vim /etc/my.cnf
[mysqld]
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
transaction-isolation = READ-COMMITTED
symbolic-links = 0
max_allowed_packet = 32M
thread_stack = 256K
thread_cache_size = 64
query_cache_limit = 8M
query_cache_size = 64M
query_cache_type = 1
max_connections = 550
log_bin=/var/lib/mysql/mysql_binary_log
server_id=1
binlog_format = mixed
read_buffer_size = 2M
read_rnd_buffer_size = 16M
sort_buffer_size = 8M
join_buffer_size = 8M
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit = 2
innodb_log_buffer_size = 64M
innodb_buffer_pool_size = 4G
innodb_thread_concurrency = 8
innodb_flush_method = O_DIRECT
innodb_log_file_size = 512M
[mysqld_safe]
log-error=/var/log/mariadb/mariadb.log
pid-file=/var/run/mariadb/mariadb.pid
!includedir /etc/my.cnf.d
启动服务
systemctl enable mariadb && systemctl restart mariadb
如果启动失败提示Job for mysqld.service failed because the control process exited with error code
解决方法 查看日志 tail -200f /var/log/mysqld.log
按提示删除ERROR的提示 rm -rf /var/lib/mysql/ 之前mysql配置相关信息全部失效 需重新设置
然后 [root@s100 mysqld]# systemctl restart mysqld.service
初始化数据库,按照相关提示操作即可。
/usr/bin/mysql_secure_installation (如果是mysql先关内容替换了 mariadb 则可以跳过此步)
按照下面提示输入。
[...]
Enter current password for root (enter for none):
OK, successfully used password, moving on...
[...]
Set root password? [Y/n] Y
New password:
Re-enter new password:
[...]
Remove anonymous users? [Y/n] Y
[...]
Disallow root login remotely? [Y/n] N
[...]
Remove test database and access to it [Y/n] Y
[...]
Reload privilege tables now? [Y/n] Y
[...]
All done! If you've completed all of the above steps, your MariaDB
installation should now be secure.
Thanks for using MariaDB!
5.安装 MySQL JDBC 驱动(所有节点)
用于各节点连接数据库。
wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz
tar xf mysql-connector-java-5.1.46.tar.gz
mkdir -p /usr/share/java/
cd mysql-connector-java-5.1.46
cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar
6.为 Cloudera 各软件创建数据库
将如下内容,写入到cdh.sql文件中。
版本 mysql5.7.xx
CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY '123456';
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON amon.* TO 'amon'@'%' IDENTIFIED BY '123456';
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON rman.* TO 'rman'@'%' IDENTIFIED BY '123456';
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON hue.* TO 'hue'@'%' IDENTIFIED BY '123456';
CREATE DATABASE metastore DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON metastore.* TO 'hive'@'%' IDENTIFIED BY '123456';
CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON sentry.* TO 'sentry'@'%' IDENTIFIED BY '123456';
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON nav.* TO 'nav'@'%' IDENTIFIED BY '123456';
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON navms.* TO 'navms'@'%' IDENTIFIED BY '123456';
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON oozie.* TO 'oozie'@'%' IDENTIFIED BY '123456';
flush privileges;
版本mysql8.0.xx 因为mysql8的权限分配有区别与mysql5 详细参看自己的博客 --CentOS7安装MySQL(完整版) 为了方便 所有库都分给了用户名root 和 密码123456
CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE metastore DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
flush privileges;
执行sql文件。
mysql -uroot -p<ROOT_PASSWORD> < ./cdh.sql
7.设置 Cloudera Manager 数据库
#/opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm
//接着,输入scm数据库密码
完整例子如
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm root 123456
CM数据库配置的位置在 /etc/cloudera-scm-server
8.安装 CDH
CM安装成功之后,接下来我们就可以通过CM安装CDH的方式构建企业大数据平台。所以首先需要把CDH的parcels包下载到CM主服务器上。同样的,我们为了加速我们的安装,我们可以把需要下载的软件包提前下载下来,也可以创建CDH私有仓库。
下载CDH的软件包 parcels(namenode01)
#cd /opt/cloudera/parcel-repo
#wget https://archive.cloudera.com/cdh6/6.2.0/parcels/CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel
#wget https://archive.cloudera.com/cdh6/6.2.0/parcels/CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel.sha1(最后改成.sha)
#wget https://archive.cloudera.com/cdh6/6.2.0/parcels/manifest.json
修改属主属组(查看权限如果属于cloudera-scm:cloudera-scm则不用更改)
chown -R cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo/*
9.启动 Cloudera Manager Server (此时最好保证server节点上agent服务关闭 为了后面CM ssh发现 可以找全集群机器)
#systemctl start cloudera-scm-server
systemctl stop cloudera-scm-server
systemctl restart cloudera-scm-server
systemctl status cloudera-scm-server
如果启动中有什么问题,可以查看日志。
tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log