大数据CDH6.3.1 集群搭建

集群CDH搭建

后续继续补充图片（软件链接失效-_-|| 如有问题欢迎留言）
本文参考众多CDH搭建过程，最靠谱的内容来自若泽大数据团队！点赞！

1. 需要yum安装的组件（不要先安装 遇到了 来找）

#hue 
解决方案：
1,进行软连接
ln /usr/local/mysql/lib/libmysqlclient.so.20 /usr/lib64/libmysqlclient_r.so.16
2，之后又报一个version的错误
yum -y install mysql-libs
3，错误解决，进行安装

yum -y install python-lxml #下载依赖

yum -y install httpd mod_ssl

yum install -y perl

yum install -y httpd

1. 配置服务器hosts和hostname(主、从)

配置hostname
vi /etc/hostname

配置hosts
vi /etc/hosts 

2. 机器的免密配置（主、从）

全部服务器
ssh-keygen -t rsa


分发给全部服务器
ssh-copy-id -i ~/.ssh/id_rsa.pub 192.168.9.110 

3. 在挂在盘上创建数据目录（主、从）

#软件包
mkdir -p /data/bd/cdh

mkdir -p /data/bd/dfs/dn
mkdir -p /data/bd/dfs/nn
mkdir -p /data/bd/dfs/snn

mkdir -p /data/bd/hive/warehouse
mkdir -p /data/bd/mysql/data

mkdir -p /data/bd/var/lib/oozie/data
mkdir -p /data/bd/var/lib/zookeeper


mkdir -p /data/bd/yarn/nm

4. 安装JDK（主、从）

4.1 由Tools把软件包移动到主节点

scp jdk-8u271-linux-x64.tar.gz  ***.***.***.74:/data/install/
scp jdk-8u271-linux-x64.tar.gz  ***.***.***.73:/data/install/
 

4.2 jdk安装

mkdir /usr/java
tar -zxvf jdk-8u271-linux-x64.tar.gz -C /usr/java
cd /usr/java/
ll
chown -R root:root /usr/java/jdk1.8.0_271
vi /etc/profile
添加
export JAVA_HOME=/usr/java/jdk1.8.0_271
export PATH=${JAVA_HOME}/bin:${PATH}
source /etc/profile
which java

5. 离线部署MySQL（主）

5.1 解压及创建my.cnf

cd /data/cdh/
tar xzvf mysql-5.7.11-linux-glibc2.5-x86_64.tar.gz -C /data/bd
ll
cd /data/bd
ll
mv mysql-5.7.11-linux-glibc2.5-x86_64 mysql
ll
mkdir mysql/arch mysql/data mysql/tmp
vi /etc/my.cnf
:.,$d 


[client]
port            = 3306
socket          = /data/bd/mysql/data/mysql.sock
default-character-set=utf8mb4

[mysqld]
port            = 3306
socket          = /data/bd/mysql/data/mysql.sock

skip-slave-start

skip-external-locking
key_buffer_size = 256M
sort_buffer_size = 2M
read_buffer_size = 2M
read_rnd_buffer_size = 4M
query_cache_size= 32M
max_allowed_packet = 16M
myisam_sort_buffer_size=128M
tmp_table_size=32M

table_open_cache = 512
thread_cache_size = 8
wait_timeout = 86400
interactive_timeout = 86400
max_connections = 600

# Try number of CPU's*2 for thread_concurrency
#thread_concurrency = 32

#isolation level and default engine
default-storage-engine = INNODB
transaction-isolation = READ-COMMITTED

server-id  = 1739
basedir     = /data/bd/mysql
datadir     = /data/bd/mysql/data
pid-file     = /data/bd/mysql/data/hostname.pid

#open performance schema
log-warnings
sysdate-is-now

binlog_format = ROW
log_bin_trust_function_creators=1
log-error  = /data/bd/mysql/data/hostname.err
log-bin = /data/bd/mysql/arch/mysql-bin
expire_logs_days = 7

innodb_write_io_threads=16

relay-log  = /data/bd/mysql/relay_log/relay-log
relay-log-index = /data/bd/mysql/relay_log/relay-log.index
relay_log_info_file= /data/bd/mysql/relay_log/relay-log.info

log_slave_updates=1
gtid_mode=OFF
enforce_gtid_consistency=OFF

# slave
slave-parallel-type=LOGICAL_CLOCK
slave-parallel-workers=4
master_info_repository=TABLE
relay_log_info_repository=TABLE
relay_log_recovery=ON

#other logs
#general_log =1
#general_log_file  = /data/bd/mysql/data/general_log.err
#slow_query_log=1
#slow_query_log_file=/data/bd/mysql/data/slow_log.err

#for replication slave
sync_binlog = 500

#for innodb options
innodb_data_home_dir = /data/bd/mysql/data
innodb_data_file_path = ibdata1:1G;ibdata2:1G:autoextend

innodb_log_group_home_dir = /data/bd/mysql/arch
innodb_log_files_in_group = 4
innodb_log_file_size = 1G
innodb_log_buffer_size = 200M

#根据生产需要，调整pool size
innodb_buffer_pool_size = 2G
#innodb_additional_mem_pool_size = 50M #deprecated in 5.6
tmpdir = /data/bd/mysql/tmp

innodb_lock_wait_timeout = 1000
#innodb_thread_concurrency = 0
innodb_flush_log_at_trx_commit = 2

innodb_locks_unsafe_for_binlog=1

#innodb io features: add for mysql5.5.8
performance_schema
innodb_read_io_threads=4
innodb-write-io-threads=4
innodb-io-capacity=200
#purge threads change default(0) to 1 for purge
innodb_purge_threads=1
innodb_use_native_aio=on

#case-sensitive file names and separate tablespace
innodb_file_per_table = 1
lower_case_table_names=1

[mysqldump]
quick
max_allowed_packet = 128M

[mysql]
no-auto-rehash
default-character-set=utf8mb4

[mysqlhotcopy]
interactive-timeout

[myisamchk]
key_buffer_size = 256M
sort_buffer_size = 256M
read_buffer = 2M
write_buffer = 2M

5.2 创建用户组及用户

[root@cdh-server01 local]# groupadd -g 101 dba
[root@cdh-server01 local]# useradd -u 514 -g dba -G root -d /data/bd/mysql mysqladmin
useradd: warning: the home directory already exists.
Not copying any file from skel directory into it.
[root@cdh-server01 local]# cp /etc/skel/.* /data/bd/mysql
cp: omitting directory ‘/etc/skel/.’
cp: omitting directory ‘/etc/skel/..’
[root@cdh-server01 local]# vi mysql/.bashrc
添加
    export MYSQL_BASE=/data/bd/mysql
    export PATH=${MYSQL_BASE}/bin:$PATH
[root@cdh-server01 local]# chown  mysqladmin:dba /etc/my.cnf
[root@cdh-server01 local]# chmod  640 /etc/my.cnf  
[root@cdh-server01 local]# chown -R mysqladmin:dba /data/bd/mysql
[root@cdh-server01 local]# chmod -R 755 /data/bd/mysql 
[root@cdh-server01 local]# cd /data/bd/mysql
[root@cdh-server01 mysql]# cp support-files/mysql.server /etc/rc.d/init.d/mysql
[root@cdh-server01 mysql]# chmod +x /etc/rc.d/init.d/mysql
[root@cdh-server01 mysql]# chkconfig --add mysql
[root@cdh-server01 mysql]# bin/mysqld \
--defaults-file=/etc/my.cnf \
--user=mysqladmin \
--basedir=/data/bd/mysql/ \
--datadir=/data/bd/mysql/data/ \
--initialize
  
############
  问题：bin/mysqld: error while loading shared libraries: libaio.so.1: cannot open shared object file: No such file or directory
    
  解决：yum install -y libaio
 ############
    
[root@cdh-server01 mysql]# cd data
[root@cdh-server01 data]# cat hostname.err |grep password
2019-12-06T14:24:59.548965Z 1 [Note] A temporary password is generated for root@localhost: 5=G#cpabuhXM
[root@cdh-server01 data]# su - mysqladmin
[mysqladmin@cdh-server01 ~]$ service mysql start
Starting MySQL... SUCCESS! 
[mysqladmin@cdh-server01 ~]$ mysql -uroot -p
Enter password: 
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 2
Server version: 5.7.11-log
 
Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
 
Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.
 
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
 
mysql> exit;
Bye
[mysqladmin@cdh-server01 ~]$ mysql -uroot -p
Enter password: 
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 3
Server version: 5.7.11-log
 
Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
 
Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.
 
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
 
mysql> alter user root@localhost identified by '###';
Query OK, 0 rows affected (0.00 sec)
 
mysql> GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '###' ;
Query OK, 0 rows affected, 1 warning (0.00 sec)
 
mysql> flush privileges;
Query OK, 0 rows affected (0.01 sec)
 
mysql> exit;
Bye
[mysqladmin@cdh-server01 ~]$ service mysql restart
Shutting down MySQL.. SUCCESS! 
Starting MySQL.. SUCCESS! 
[mysqladmin@cdh-server01 ~]$ mysql -uroot -p###
mysql: [Warning] Using a password on the command line interface can be insecure.
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 2
Server version: 5.7.11-log MySQL Community Server (GPL)
 
Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
 
Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.
 
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
 
mysql> create database cmf default character set utf8;
Query OK, 1 row affected (0.00 sec)
 
mysql> create database amon default character set utf8;
Query OK, 1 row affected (0.01 sec)
 
mysql> grant all privileges on cmf.*  to 'cmf'@'%' identified by 'golden';
Query OK, 0 rows affected, 1 warning (0.00 sec)
 
mysql> grant all privileges on amon.*  to 'amon'@'%' identified by 'golden';
Query OK, 0 rows affected, 1 warning (0.00 sec)

create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
CREATE DATABASE hive DEFAULT CHARACTER SET latin1;

grant all privileges on oozie.*  to 'oozie'@'%' identified by 'golden';
grant all privileges on hue.*  to 'hue'@'%' identified by 'golden';
grant all privileges on hive.*  to 'hive'@'%' identified by 'golden';

mysql> flush privileges;
Query OK, 0 rows affected (0.00 sec)
 
mysql> exit;
Bye
[mysqladmin@cdh-server01 ~]$ exit;
logout
[root@cdh-server01 data]# cd
[root@cdh-server01 ~]# cd CDH6.3.1/
[root@cdh-server01 CDH6.3.1]# ll
total 4118988
-rw-r--r--. 1 root root 2083878000 Dec  5 22:05 CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel
-rw-r--r--. 1 root root         40 Dec  5 22:06 CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.sha1
-rw-r--r--. 1 root root 1411444147 Dec  5 22:06 cm6.3.1-redhat7.tar.gz
-rw-r--r--. 1 root root  173271626 Dec  5 22:07 jdk-8u45-linux-x64.gz
-rw-r--r--. 1 root root      33887 Dec  5 22:07 manifest.json
-rw-r--r--. 1 root root  548193637 Dec  5 22:07 mysql-5.7.11-linux-glibc2.5-x86_64.tar.gz
-rw-r--r--. 1 root root    1007502 Dec  5 22:07 mysql-connector-java-5.1.47.jar
[root@cdh-server01 CDH6.3.1]# mkdir -p /usr/share/java
[root@cdh-server01 CDH6.3.1]# cp mysql-connector-java-5.1.47.jar  /usr/share/java/mysql-connector-java.jar
[root@cdh-server01 CDH6.3.1]# 

6. rpm包部署CM

6.1主节点

[root@cdh-server01 CDH6.3.1]# tar -xzvf cm6.3.1-redhat7.tar.gz
[root@cdh-server01 CDH6.3.1]# ll
total 4118988
-rw-r--r--. 1 root root 2083878000 Dec  5 22:05 CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel
-rw-r--r--. 1 root root         40 Dec  5 22:06 CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.sha1
drwxrwxr-x. 5 2001 2001         75 Sep 25 16:51 cm6.3.1
-rw-r--r--. 1 root root 1411444147 Dec  5 22:06 cm6.3.1-redhat7.tar.gz
-rw-r--r--. 1 root root  173271626 Dec  5 22:07 jdk-8u45-linux-x64.gz
-rw-r--r--. 1 root root      33887 Dec  5 22:07 manifest.json
-rw-r--r--. 1 root root  548193637 Dec  5 22:07 mysql-5.7.11-linux-glibc2.5-x86_64.tar.gz
-rw-r--r--. 1 root root    1007502 Dec  5 22:07 mysql-connector-java-5.1.47.jar
[root@cdh-server01 CDH6.3.1]# cd cm6.3.1
[root@cdh-server01 cm6.3.1]# ll
total 8
drwxrwxr-x. 2 2001 2001 4096 Sep 25 16:51 repodata
-rw-rw-r--. 1 2001 2001 1718 Sep 25 16:51 RPM-GPG-KEY-cloudera
drwxrwxr-x. 4 2001 2001   34 Sep 25 16:50 RPMS
drwxrwxr-x. 2 2001 2001    6 Sep 25 16:50 SRPMS
[root@cdh-server01 cm6.3.1]# cd RPMS/
[root@cdh-server01 RPMS]# ll
total 4
drwxrwxr-x. 2 2001 2001    6 Sep 25 16:50 noarch
drwxrwxr-x. 2 2001 2001 4096 Sep 25 16:51 x86_64
[root@cdh-server01 RPMS]# cd x86_64/
[root@cdh-server01 x86_64]# ll
total 1380420
-rw-r--r--. 1 2001 2001   10483568 Sep 25 16:51 cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm
-rw-r--r--. 1 2001 2001 1203832464 Sep 25 16:50 cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm
-rw-r--r--. 1 2001 2001      11488 Sep 25 16:51 cloudera-manager-server-6.3.1-1466458.el7.x86_64.rpm
-rw-r--r--. 1 2001 2001      10996 Sep 25 16:51 cloudera-manager-server-db-2-6.3.1-1466458.el7.x86_64.rpm
-rw-r--r--. 1 2001 2001   14209868 Sep 25 16:51 enterprise-debuginfo-6.3.1-1466458.el7.x86_64.rpm
-rw-rw-r--. 1 2001 2001  184988341 Sep 25 16:51 oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm
[root@cdh-server01 x86_64]# rpm -ivh cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm --nodeps --force
warning: cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID b0b19c9f: NOKEY
Preparing...                          ################################# [100%]
Updating / installing...
   1:cloudera-manager-daemons-6.3.1-14################################# [100%]
[root@cdh-server01 x86_64]# rpm -ivh cloudera-manager-server-6.3.1-1466458.el7.x86_64.rpm --nodeps --force
     warning: cloudera-manager-server-6.3.1-1466458.el7.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID b0b19c9f: NOKEY
Preparing...                          ################################# [100%]
Updating / installing...
   1:cloudera-manager-server-6.3.1-146################################# [100%]
Created symlink from /etc/systemd/system/multi-user.target.wants/cloudera-scm-server.service to /usr/lib/systemd/system/cloudera-scm-server.service.
[root@cdh-server01 x86_64]# cd
[root@cdh-server01 ~]# cd /etc/cloudera-scm-server/
[root@cdh-server01 cloudera-scm-server]# ll
total 20
-rw-r--r--. 1 root         root         8197 Sep 25 16:34 cmSubCaCert.profile
-rw-------. 1 cloudera-scm cloudera-scm  757 Sep 25 16:34 db.properties
-rw-r--r--. 1 root         root         3027 Sep 25 16:34 log4j.properties
[root@cdh-server01 cloudera-scm-server]# vi db.properties 
    # Copyright (c) 2012 Cloudera, Inc. All rights reserved.
    #
    # This file describes the database connection.
    #
    
    # The database type
    # Currently 'mysql', 'postgresql' and 'oracle' are valid databases.
    com.cloudera.cmf.db.type=mysql
    
    # The database host
    # If a non standard port is needed, use 'hostname:port'
    com.cloudera.cmf.db.host=jwl-data-prod-cdh-001
    
    # The database name
    com.cloudera.cmf.db.name=cmf
    
    # The database user
    com.cloudera.cmf.db.user=cmf
    
    # The database user's password
    com.cloudera.cmf.db.password=golden
    
    # The db setup type
    # After fresh install it is set to INIT
    # and will be changed post config.
    # If scm-server uses Embedded DB then it is set to EMBEDDED
    # If scm-server uses External DB then it is set to EXTERNAL
    com.cloudera.cmf.db.setupType=EXTERNAL
[root@cdh-server01 cloudera-scm-server]# service cloudera-scm-server start
Redirecting to /bin/systemctl start cloudera-scm-server.service
[root@cdh-server01 cloudera-scm-server]# service cloudera-scm-server status
Redirecting to /bin/systemctl status cloudera-scm-server.service
● cloudera-scm-server.service - Cloudera CM Server Service
   Loaded: loaded (/usr/lib/systemd/system/cloudera-scm-server.service; enabled; vendor preset: disabled)
   Active: active (running) since Sat 2019-12-07 14:59:23 CST; 5s ago
  Process: 5278 ExecStartPre=/opt/cloudera/cm/bin/cm-server-pre (code=exited, status=0/SUCCESS)
 Main PID: 5281 (java)
   CGroup: /system.slice/cloudera-scm-server.service
           └─5281 /usr/java/jdk1.8.0_45/bin/java -cp .:/usr/share/java/mysql-connector-java.jar:/usr/sh...
 
Dec 07 14:59:23 cdh-server01 cm-server[5281]: JAVA_HOME=/usr/java/jdk1.8.0_45
Dec 07 14:59:23 cdh-server01 cm-server[5281]: Java HotSpot(TM) 64-Bit Server VM warning: ignoring op...8.0
Dec 07 14:59:25 cdh-server01 cm-server[5281]: ERROR StatusLogger No log4j2 configuration file found....ng.
Dec 07 14:59:26 cdh-server01 cm-server[5281]: Sat Dec 07 14:59:26 CST 2019 WARN: Establishing SSL co...lic
Dec 07 14:59:26 cdh-server01 cm-server[5281]: Sat Dec 07 14:59:26 CST 2019 WARN: Establishing SSL co...lic
Dec 07 14:59:26 cdh-server01 cm-server[5281]: Sat Dec 07 14:59:26 CST 2019 WARN: Establishing SSL co...lic
Dec 07 14:59:26 cdh-server01 cm-server[5281]: Sat Dec 07 14:59:26 CST 2019 WARN: Establishing SSL co...lic
Dec 07 14:59:26 cdh-server01 cm-server[5281]: Sat Dec 07 14:59:26 CST 2019 WARN: Establishing SSL co...lic
Dec 07 14:59:29 cdh-server01 cm-server[5281]: 14:59:29.012 [main] ERROR org.hibernate.engine.jdbc.sp...ist
Dec 07 14:59:29 cdh-server01 cm-server[5281]: Sat Dec 07 14:59:29 CST 2019 WARN: Establishing SSL co...lic
Hint: Some lines were ellipsized, use -l to show in full.
[root@cdh-server01 cloudera-scm-server]# 
    # 此处启动一个新的主节点窗口，检查cloudera-scm-server服务启动日志
    # 出现7180，则证明成功，否则检查log文件，寻找问题
    # 网页打开192.168.1.31:7180,维持不动
    [root@cdh-server01 ~]# cd /var/log/cloudera-scm-server/
    [root@cdh-server01 cloudera-scm-server]# ll
    total 0
    [root@cdh-server01 cloudera-scm-server]# ll
    total 28
    -rw-r-----. 1 cloudera-scm cloudera-scm 28451 Dec  7 14:59 cloudera-scm-server.log
    -rw-r-----. 1 cloudera-scm cloudera-scm     0 Dec  7 14:59 cmf-server-nio.log
    -rw-r-----. 1 cloudera-scm cloudera-scm     0 Dec  7 14:59 cmf-server-perf.log
    [root@cdh-server01 cloudera-scm-server]# tail -F cloudera-scm-server.log 
    2019-12-07 15:00:39,657 INFO WebServerImpl:org.eclipse.jetty.server.AbstractConnector: Started ServerConnector@16a3e60{HTTP/1.1,[http/1.1]}{0.0.0.0:7180}
    2019-12-07 15:00:39,658 INFO WebServerImpl:org.eclipse.jetty.server.Server: Started @75687ms
    2019-12-07 15:00:39,658 INFO WebServerImpl:com.cloudera.server.cmf.WebServerImpl: Started Jetty server.
    2019-12-07 15:00:43,528 ERROR ParcelUpdateService:com.cloudera.parcel.components.ParcelDownloaderImpl: Failed to download manifest. Status code: 404 URI: https://archive.cloudera.com/sqoop-teradata-connector1/1.7.1c6/manifest.json/
[root@cdh-server01 cloudera-scm-server]#
[root@cdh-server01 cloudera-scm-server]# cd
[root@cdh-server01 CDH6.3.1]# cd CDH6.3.1/cm6.3.1/RPMS/x86_64/
[root@cdh-server01 x86_64]# rpm -ivh cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm --nodeps --force
warning: cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID b0b19c9f: NOKEY
Preparing...                          ################################# [100%]
Updating / installing...
   1:cloudera-manager-agent-6.3.1-1466################################# [100%]
Created symlink from /etc/systemd/system/multi-user.target.wants/cloudera-scm-agent.service to /usr/lib/systemd/system/cloudera-scm-agent.service.
Created symlink from /etc/systemd/system/multi-user.target.wants/supervisord.service to /usr/lib/systemd/system/supervisord.service.
[root@cdh-server01 x86_64]# vi /etc/cloudera-scm-agent/config.ini
# 修改
    server_host=jwl-data-prod-cdh-001
     
bmpo*gMC8PH%!WlK
[root@cdh-server01 x86_64]# scp cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm JWL-DATA-Prod-CDH-003:/data/bd/cdh/
[root@cdh-server01 x86_64]# scp cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm JWL-DATA-Prod-CDH-002:/data/bd/cdh/

 [root@cdh-server01 x86_64]# scp cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm JWL-DATA-Prod-CDH-003:/data/bd/cdh/
[root@cdh-server01 x86_64]# scp cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm ***-DATA-Prod-CDH-002:/data/bd/cdh/

6.2 次节点

[root@cdh-agent01 java]# cd /root/CDH6.3.1/
You have new mail in /var/spool/mail/root
[root@cdh-agent01 CDH6.3.1]# ll
total 1355072
-rw-r--r--. 1 root root   10483568 Dec  7 15:20 cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm
-rw-r--r--. 1 root root 1203832464 Dec  7 15:22 cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm
-rw-r--r--. 1 root root  173271626 Dec  5 23:01 jdk-8u45-linux-x64.gz
[root@cdh-agent01 CDH6.3.1]# rpm -ivh cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm --nodeps --force
warning: cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID b0b19c9f: NOKEY
Preparing...                          ################################# [100%]
Updating / installing...
  1:cloudera-manager-daemons-6.3.1-14################################# [100%]
[root@cdh-agent01 CDH6.3.1]# rpm -ivh cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm --nodeps --force warning: cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID b0b19c9f: NOKEY
Preparing...                          ################################# [100%]
Updating / installing...
  1:cloudera-manager-agent-6.3.1-1466################################# [100%]
Created symlink from /etc/systemd/system/multi-user.target.wants/cloudera-scm-agent.service to /usr/lib/systemd/system/cloudera-scm-agent.service.
Created symlink from /etc/systemd/system/multi-user.target.wants/supervisord.service to /usr/lib/systemd/system/supervisord.service.
[root@cdh-agent01 CDH6.3.1]# vi /etc/cloudera-scm-agent/config.ini
# 修改
   server_host=***-data-prod-cdh-001

6.3 启动agent服务(主、次节点)

[root@cdh-server01 x86_64]# service cloudera-scm-agent start
Redirecting to /bin/systemctl start cloudera-scm-agent.service
[root@cdh-agent01 CDH6.3.1]# service cloudera-scm-agent status
Redirecting to /bin/systemctl status cloudera-scm-agent.service
● cloudera-scm-agent.service - Cloudera Manager Agent Service
  Loaded: loaded (/usr/lib/systemd/system/cloudera-scm-agent.service; enabled; vendor preset: disabled)
  Active: active (running) since Sat 2019-12-07 15:42:30 CST; 1min 5s ago
Main PID: 3299 (cmagent)
  CGroup: /system.slice/cloudera-scm-agent.service
          └─3299 /usr/bin/python2 /opt/cloudera/cm-agent/bin/cm agent

Dec 07 15:42:32 cdh-agent01 cm[3299]: [07/Dec/2019 15:42:32 +0000] 3299 MainThread agent        INF...lood
Dec 07 15:42:32 cdh-agent01 cm[3299]: [07/Dec/2019 15:42:32 +0000] 3299 MainThread agent        INF...lude
Dec 07 15:42:32 cdh-agent01 cm[3299]: [07/Dec/2019 15:42:32 +0000] 3299 MainThread agent        INF...oups
Dec 07 15:42:32 cdh-agent01 cm[3299]: [07/Dec/2019 15:42:32 +0000] 3299 MainThread agent        INF...cess
Dec 07 15:42:32 cdh-agent01 cm[3299]: [07/Dec/2019 15:42:32 +0000] 3299 MainThread tmpfs        INF...cess
Dec 07 15:42:32 cdh-agent01 cm[3299]: [07/Dec/2019 15:42:32 +0000] 3299 MainThread logging      INF....log
Dec 07 15:42:38 cdh-agent01 cm[3299]: status_server: added process group
Dec 07 15:42:38 cdh-agent01 cm[3299]: flood: added process group
Dec 07 15:42:38 cdh-agent01 cm[3299]: /opt/cloudera/cm-agent/lib/python2.7/site-packages/psutil/_ps...to 0
Dec 07 15:42:38 cdh-agent01 cm[3299]: warnings.warn(msg, RuntimeWarning)
Hint: Some lines were ellipsized, use -l to show in full.
[root@cdh-agent01 CDH6.3.1]#yum install -y perl
 ...
 perl-Time-Local.noarch 0:1.2300-2.el7           perl-constant.noarch 0:1.27-2.el7     
 perl-libs.x86_64 4:5.16.3-294.el7_6             perl-macros.x86_64 4:5.16.3-294.el7_6 
 perl-parent.noarch 1:0.225-244.el7              perl-podlators.noarch 0:2.5.1-3.el7   
 perl-threads.x86_64 0:1.87-4.el7                perl-threads-shared.x86_64 0:1.43-6.el7 
Complete!
[root@cdh-server01 x86_64]# 

6.4 web页面配置信息

6.5 准备parcel（主）

[root@cdh-server01 x86_64]# cd
[root@cdh-server01 ~]# cd CDH6.3.1/
[root@cdh-server01 CDH6.3.1]# yum install -y httpd
Loaded plugins: fastestmirror
......
Dependency Installed:
 apr.x86_64 0:1.4.8-5.el7       apr-util.x86_64 0:1.5.2-6.el7  httpd-tools.x86_64 0:2.4.6-90.el7.centos 
 mailcap.noarch 0:2.1.41-2.el7 

Complete!
[root@cdh-server01 CDH6.3.1]# ll
total 4118988
-rw-r--r--. 1 root root 2083878000 Dec  5 22:05 CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel
-rw-r--r--. 1 root root         40 Dec  5 22:06 CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.sha1
drwxrwxr-x. 5 2001 2001         75 Sep 25 16:51 cm6.3.1
-rw-r--r--. 1 root root 1411444147 Dec  5 22:06 cm6.3.1-redhat7.tar.gz
-rw-r--r--. 1 root root  173271626 Dec  5 22:07 jdk-8u45-linux-x64.gz
-rw-r--r--. 1 root root      33887 Dec  5 22:07 manifest.json
-rw-r--r--. 1 root root  548193637 Dec  5 22:07 mysql-5.7.11-linux-glibc2.5-x86_64.tar.gz
-rw-r--r--. 1 root root    1007502 Dec  5 22:07 mysql-connector-java-5.1.47.jar
[root@cdh-server01 CDH6.3.1]# mkdir /var/www/html/cdh6_parcel
[root@cdh-server01 CDH6.3.1]# mv CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel /var/www/html/cdh6_parcel
[root@cdh-server01 CDH6.3.1]# mv CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.sha1 /var/www/html/cdh6_parcel/CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.sha
[root@cdh-server01 CDH6.3.1]# mv manifest.json /var/www/html/cdh6_parcel
[root@cdh-server01 CDH6.3.1]# ll /var/www/html/cdh6_parcel
total 2035080
-rw-r--r--. 1 root root 2083878000 Dec  5 22:05 CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel
-rw-r--r--. 1 root root         40 Dec  5 22:06 CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.sha
-rw-r--r--. 1 root root      33887 Dec  5 22:07 manifest.json
[root@cdh-server01 CDH6.3.1]# service httpd start
Redirecting to /bin/systemctl start httpd.service
[root@cdh-server01 html]# scp -r cdh6_parcel/ cdh6
[root@cdh-server01 html]# ll
total 0
drwxr-xr-x. 2 root root 131 Dec  7 16:08 cdh6
drwxr-xr-x. 2 root root 131 Dec  7 16:12 cdh6_parcel
[root@cdh-server01 html]# 

web打开***-001/cdh6_parcel/，验证httpd启动结果(一定要内网地址！！！！)

Bug:(激活不成功)

curl -u admin:admin -X POST http://:7180/api/v18/clusters/{clusterName}/parcels/products/{product}/versions/{version}/commands/deactivate

step 5. 选择存储库

step 6. 自动刷新页面，显示CDH

step 7. 安装Parcels

step 8. 检查群集

step 9. 服务选择

step 10. 角色分配(Zookeeper必须为奇数)

step 11. 数据库设置，测试连接

step 12. 首次运行命令详细信息

step 13. 汇总

5.5 问题修复

CM service 黄色报错

Zookeeper 黄色报错

解决方案:角色选择时，Zookeeper选择奇数个节点

HDFS 黄色报错

解决办法:Suppress

主机 黄色报错

解决办法:主机名不要有大写字

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-tGAu6Rw9-1648543142897)(image/image_11.png)]

修复出现的问题

1.修复透明大页面(主、次)

echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled

2.swappiness 设置(主、次)

echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
 
 
 
 
# oozie
mv ext-2.2.zip /var/lib/oozie/ 
cd /var/lib/oozie 
unzip ext-2.2.zip 
chown -R oozie:oozie ext-2.2

#hue 
解决方案：
1,进行软连接
ln /usr/local/mysql/lib/libmysqlclient.so.20 /usr/lib64/libmysqlclient_r.so.16
2，之后又报一个version的错误
yum install mysql-libs
3，错误解决，进行安装

yum -y install python-lxml #下载依赖
cd /usr/lib64
# 可以看到libxslt.so.1指向libxslt.so.1.1.28
lrwxrwxrwx.  1 root root      17 Oct 23 09:44 libxslt.so.1 -> libxslt.so.1.1.28
-rwxr-xr-x.  1 root root  258344 Jun 10  2014 libxslt.so.1.1.28
# 把这两个依赖拷贝到CDH的lib64路径下
cp libxslt.so.1.1.28 /opt/cloudera/parcels/CDH/lib64
cp libxslt.so.1 /opt/cloudera/parcels/CDH/lib64

yum -y install httpd mod_ssl

启动服务命令

主：
service cloudera-scm-server start
service cloudera-scm-agent start

从：
service cloudera-scm-agent start
  
10.100.xxx.xxx      xxx-DATA-Prod-CDH-001
10.100.xxx.xxx     xxx-DATA-Prod-CDH-002
10.100.xxx.xxx     xxx-DATA-Prod-CDH-003