先电大数据平台搭建

项目1

安装配置JDK

安装Tomcat

安装配置Eclipse

安装配置MySQL Server

安装Navicat

 

 

 

构建Hadoop开发平台

项目2

安装虚拟机（master和slave1）

 

 

启动虚拟机（master和slave1）

安装启动MobaXterm

 

 

配置主机名

设置ntpd自动启动

# master  
# yum -y install ntp  
# vi /etc/ntp.conf  
注释或者删除以下四行  
server 0.centos.pool.ntp.org iburst  
server 1.centos.pool.ntp.org iburst  
server 2.centos.pool.ntp.org iburst  
server 3.centos.pool.ntp.org iburst  
添加以下两行  
server 127.127.1.0   
fudge 127.127.1.0 stratum 10   
#systemctl enable ntpd  
#systemctl start  ntpd  
# slave1 （timedatectl查看是否同步）  
# yum -y install ntpdate  
# ntpdate master  
# systemctl enable ntpdate  

设置时间同步

（timedatectl查看是否同步）

设置免密登录

1.	检查2个节点是否可以通过无密钥相互访问，如果未配置，则进行SSH无密码公钥认证配置。如下：  
2.	# ssh-keygen  
3.	# ssh-copy-id root@master 
4.	# ssh-copy-id root@slave1
5.	# ssh-copy-id root@localhost
6.	ssh登录远程主机查看是否成功  
7.	# ssh master
8.	# exit  
9.	# ssh slave1  
10.	# exit  

安装配置JDK

1.	# master   
2.	# mkdir /usr/jdk64/    
3.	# 将JDK复制到 /opt/下 
4.	# cd /opt/
5.	# tar -zxvf jdk-8u191-linux-x64.tar.gz -C /usr/jdk64/ 
6.	# vi /etc/profile  
7.	export JAVA_HOME=/usr/jdk64/jdk1.8.0_191  
8.	export PATH=$JAVA_HOME/bin:$PATH  
9.	# source /etc/profile  
10.	#java –version  
11.	# slave1  
12.	#mkdir /usr/jdk64/  
13.	    #Master: scp – r jdk64 slave1:/usr/jdk64/  
14.	# vi /etc/profile  
15.	export JAVA_HOME=/usr/jdk64/jdk1.8.0_191
16.	export PATH=$JAVA_HOME/bin:$PATH  
17.	# source /etc/profile  
18.	#java –version  

安装Mysql

1.	# master  
2.	# yum install  mariadb mariadb-server mysql-connector-java -y 
3.	启动服务  
4.	# systemctl enable mariadb  
5.	# systemctl start mariadb  
6.	配置MySQL  
7.	#mysql_secure_installation  
8.	按enter确认后设置数据库root密码 
9.	如果报错， 请关闭防火墙  
10.	Remove anonymous users? [Y/n] y  
11.	Disallow root login remotely? [Y/n] n  
12.	Remove test database and access to it? [Y/n] y  
13.	Reload privilege tables now? [Y/n] y  
14.	创建ambari数据库  
15.	# mysql -uroot -pbigdata  
16.	MariaDB [(none)]> create database ambari;  
17.	MariaDB [(none)]> grant all privileges on ambari.* to 'ambari'@'localhost' identified by 'bigdata';  
18.	MariaDB [(none)]> grant all privileges on ambari.* to 'ambari'@'%' identified by 'bigdata';  
19.	MariaDB [(none)]> use ambari;  
20.	MariaDB [ambari]> quit

 

 

 

 

 

 

 

 

 

 

Ambari:

Hive:

Oozie:

配置本地下载站

 

 

 

 

 

 

 

 

 

 

Ambari:

Hive:

Oozie:

配置本地下载站

1.	# master  
2.	# yum -y install httpd  
3.	mkdir /var/www/html/ambari-hdp
4.	mkdir /var/www/html/ambari-hdp/HDP-UTILS
5.	将资源包ambari-2.6.0.0和HDP-2.6.1.0拖入/var/www/html/ambari-hdp目录，将资源包HDP-UTILS-1.1.0.21拖入/var/www/html/ambari-hdp/HDP-UTILS目录
6.	systemctl start httpd

配置Yum本地Repos资源池

由于yum源里面没有yum-plugin-priorities，则在网上下载，放入虚拟机手动安装

1.	rpm -ivh yum-plugin-priorities-1.1.31-519.fc30.noarch.rpm  
2.	vi /etc/yum/pluginconf.d/priorities.conf  
3.	[main]
4.	enabled = 1
5.	gpgcheck=0
6.	[root@master yum.repos.d]# touch ambari.repo
7.	[root@master yum.repos.d]# touch hdp.repo
8.	[root@master yum.repos.d]# touch hdp-util.repo
9.	[root@master yum.repos.d]# vi ambari.repo
10.	[ambari-2.6.0.0]
11.	name=ambari-2.6.0.0
12.	baseurl=http://192.168.100.10/ambari-hdp/ambari-2.6.0.0
13.	gpgcheck=0
14.	gpgkey=http://192.168.100.10/ambari-hdp/ambari-2.6.0.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
15.	enabled=1
16.	priority=1 
17.	[root@master yum.repos.d]# vi hdp.repo
18.	[HDP-2.6.1.0]
19.	name=Hortonworks Data Platform Version-HDP-2.6.1.0
20.	baseurl=http://192.168.100.10/ambari-hdp/HDP-2.6.1.0
21.	gpgcheck=0
22.	gpgkey=http://192.168.100.10/ambari-hdp/HDP-2.6.1.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
23.	enabled=1
24.	priority=1
25.	vi hdp-util.repo
26.	[HDP-UTILS-1.1.0.21]
27.	name=Hortonworks Data Platform Version-HDP-UTILS-1.1.0.21
28.	baseurl=http://192.168.100.10/ambari-hdp/HDP-UTILS/HDP-UTILS-1.1.0.21
29.	gpgcheck=0
30.	enabled=1
31.	priority=1
32.	priority=1
33.	 
34.	[root@master yum.repos.d]# yum clean all
35.	[root@master yum.repos.d]# yum update list
36.	[root@master yum.repos.d]# yum makecache
37.	[root@master yum.repos.d]# yum repolist

 

安装Ambari-Server

1.	yum install -y ambari-server  
2.	source /var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql;
3.	vi /etc/profile
4.	export buildNumber=2.6.0.0
5.	

配置Ambari-Server

[root@master ~]# ambari-server setup
Using python  /usr/bin/python
Setup ambari-server
Checking SELinux...
SELinux status is 'disabled'
Customize user account for ambari-server daemon [y/n] (n)? y
Enter user account for ambari-server daemon (root):root
Adjusting ambari-server permissions and ownership...
Checking firewall status...
Checking JDK...
[1] Oracle JDK 1.8 + Java Cryptography Extension (JCE) Policy Files 8
[2] Oracle JDK 1.7 + Java Cryptography Extension (JCE) Policy Files 7
[3] Custom JDK
==============================================================================
Enter choice (1): 3
WARNING: JDK must be installed on all hosts and JAVA_HOME must be valid on all hosts.
WARNING: JCE Policy files are required for configuring Kerberos security. If you plan to use Kerberos,please make sure JCE Unlimited Strength Jurisdiction Policy Files are valid on all hosts.
Path to JAVA_HOME: /usr/jdk64/jdk1.8.0_191
Validating JDK on Ambari Server...done.
Completing setup...
Configuring database...
Enter advanced database configuration [y/n] (n)? y
Configuring database...
==============================================================================
Choose one of the following options:
[1] - PostgreSQL (Embedded)
[2] - Oracle
[3] - MySQL / MariaDB
[4] - PostgreSQL
[5] - Microsoft SQL Server (Tech Preview)
[6] - SQL Anywhere
[7] - BDB
==============================================================================
Enter choice (1): 3
Hostname (localhost): 
Port (3306): 
Database name (ambari): 
Username (ambari): 
Enter Database Password (bigdata): 
Re-enter password: 
Configuring ambari database...
Configuring remote database connection properties...
WARNING: Before starting Ambari Server, you must run the following DDL against the database to create the schema: /var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql
Proceed with configuring remote database connection properties [y/n] (y)? y
Extracting system views...
ambari-admin-2.6.0.0.0.jar

Adjusting ambari-server permissions and ownership...
Ambari Server 'setup' completed successfully.

安装配置ambari-agent

1.	[root@slave1 jdk1.8.0_191]# yum clean  
2.	Error: clean requires an option: headers, packages, metadata, dbcache, plugins, expire-cache, rpmdb, all  
3.	[root@slave1 jdk1.8.0_191]# yum clean all  
4.	Cleaning repos: HDP-2.6.1.0 HDP-UTILS-1.1.0.21 ambari-2.6.0.0 centos  
5.	Cleaning up everything  
6.	[root@slave1 jdk1.8.0_191]# yum updata list  
7.	No such command: updata. Please use /usr/bin/yum --help  
8.	[root@slave1 jdk1.8.0_191]# yum update list  
9.	[root@master yum.repos.d]# yum install -y ambari-agent
10.	[root@master yum.repos.d]# chkconfig --add ambari-agent
11.	[root@master yum.repos.d]# vi /etc/ambari-agent/conf/ambari-agent.ini
12.	# ambari-agent restart