单机版hive

单机版hive部署

yum -y install lrzsz

1、安装jdk

yum list installed | grep java
cd /opt
wget https://repo.huaweicloud.com/java/jdk/8u192-b12/jdk-8u192-linux-x64.tar.gz
tar -zxvf jdk-8u192-linux-x64.tar.gz -C ./
vim /etc/profile

# java home
export JAVA_HOME=/opt/jdk1.8.0_192
export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$PATH:$JAVA_HOME/bin
source /etc/profile

java -version

2、安装MySQL

cd /opt

rpm -qa | grep mariadb
rpm -e mariadb-libs-5.5.60-1.el7_5.x86_64 --nodeps
rpm -qa | grep mysql

wget https://downloads.mysql.com/archives/get/p/23/file/mysql-8.0.31-el7-x86_64.tar.gz
tar -zxvf mysql-8.0.31-el7-x86_64.tar.gz
mv mysql-8.0.31-el7-x86_64 /usr/local/mysql

groupadd mysql
useradd -g mysql mysql
chown -R mysql:mysql /usr/local/mysql

vim /etc/my.cnf
[mysqld]
# 设置3306端口
port=3306
# 设置mysql的安装目录
basedir=/usr/local/mysql
# 设置mysql数据库的数据的存放目录
datadir=/usr/local/mysql/data
# 允许最大连接数
max_connections=200
# 允许连接失败的次数。这是为了防止有人从该主机试图攻击数据库系统
max_connect_errors=10
# 服务端使用的字符集默认为UTF8
character-set-server=utf8mb4
# 创建新表时将使用的默认存储引擎
default-storage-engine=INNODB
# 默认使用“mysql_native_password”插件认证
default_authentication_plugin=mysql_native_password

user=mysql
lower_case_table_names=1
default-time-zone='+8:00'
sql_mode=NO_AUTO_VALUE_ON_ZERO,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE

[client]
# 设置mysql客户端连接服务端时默认使用的端口
port=3306
default-character-set=utf8mb4

cd /usr/local/mysql

# bin/mysqld --initialize --console 的一些报错
bin/mysqld: error while loading shared libraries: libaio.so.1: cannot open shared object file: No such file or directory
# 解决方式
yum install -y libaio

bin/mysqld --initialize --console

xxxxxT04:26:25.683341Z 6 [Note] [MY-010454] [Server] A temporary password is generated for root@localhost: xxxxxxxx

ln -s /usr/local/mysql/bin/mysql /usr/bin
cp -a ./support-files/mysql.server /etc/init.d/mysql
chmod +x /etc/init.d/mysql

# 加入自启动
chkconfig --add mysql
# 检查服务是否生效
chkconfig --list mysql
# 启动
service mysql start
# 查看启动状态 
service mysql status
mysql -uroot -p
alter user 'root'@'localhost' identified by 'xxxxxx';
use mysql;
update user set host='%'  where user='root';
# 重新登录MySQL后
grant all privileges on *.* to 'root'@'%';
flush privileges;

# 域名和安全组
阿里云控制台添加入方向3306安全组
ecshost配置域名
放在前边
# 内网ip
xxxx ecs01

3、安装hadoop

wget https://archive.apache.org/dist/hadoop/common/hadoop-3.1.2/hadoop-3.1.2.tar.gz
tar -zxvf hadoop-3.1.2.tar.gz -C ./

vim /etc/profile

# hadoop home
export HADOOP_HOME=/opt/hadoop-3.1.2
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

source /etc/profile
hadoop version

cd /opt/hadoop-3.1.2/etc/hadoop/

vim hadoop-env.sh
export JAVA_HOME=/opt/jdk1.8.0_192

vim core-site.xml

<property>
    <name>fs.defaultFS</name>
    <value>hdfs://ecs01:9000</value>
</property>
<!-- 设置用户 -->
<property>
  <name>hadoop.http.staticuser.user</name>
  <value>hadoop</value>
</property>
<!-- 不开启权限检查 -->
<property>
  <name>dfs.permissions.enabled</name>
  <value>false</value>
</property>
<!-- beeline连接免密 -->
<property>
  <name>dfs.permissions.enabled</name>
  <value>false</value>
</property>
<property>
        <name>hadoop.proxyuser.hadoop.hosts</name>
        <value>*</value>
</property>
<property>
        <name>hadoop.proxyuser.hadoop.groups</name>
        <value>*</value>
</property>

vim hdfs-site.xml

<!-- HDFS副本数 -->
<property>
    <name>dfs.replication</name>
    <value>1</value>
</property>
<!-- NameNode日志文件目录 -->
<property>
    <name>dfs.namenode.name.dir</name>
    <value>/opt/hadoop-3.1.2/storage/dfs/name</value>
</property>
<!-- DataNode日志文件目录 -->
<property>
    <name>dfs.datanode.data.dir</name>
    <value>/opt/hadoop-3.1.2/storage/dfs/data</value>
</property>
<!-- secondaryNode启动服务的节点  -->
<property>
    <name>dfs.namenode.secondary.http-address</name>
    <value>ecs01:9868</value>
</property>
<!--secondaryNode日志文件目录  -->
<property>
    <name>dfs.namenode.checkpoint.dir</name>
    <value>/opt/hadoop-3.1.2/storage/dfs/secondary</value>
</property>

vim workers
ecs01

# 格式化hadoop文件格式
hadoop namenode -format
SHUTDOWN_MSG: Shutting down NameNode at ecs01/xxxxxxxIP

#创建用户名为hadoop的账号
useradd hadoop
#为hadoop账号设置密码
passwd hadoop
xiaoguan
#修改module、software文件夹的所有者和所属组均为hadoop用户 
chown -R hadoop:hadoop /opt/hadoop-3.1.2

#切换为hadoop账号
su - hadoop
# 免密登录 一路回车
ssh-keygen
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
#启动hadoop
start-dfs.sh

# jps
31877 DataNode
32071 SecondaryNameNode
32204 Jps
31727 NameNode

# 为了可以访问webui
阿里云控制台添加入方向9870安全组

# 本机配置域名 专有云内网ip 公有云公网ip
xxxxxx ecs01

http://ecs01:9870/

cd /home/hadoop
hdfs dfs -mkdir /bigdata
echo "hello hadoop" >> hadoop.txt

hdfs dfs -put hadoop.txt /bigdata/
为了可以下载和预览文件
阿里云控制台添加入方向9864安全组

cd /opt/hadoop-3.1.2/etc/hadoop/

vim mapred-site.xml

  <property>
	<name>mapreduce.framework.name</name>
    <value>yarn</value>
  </property>
  <!-- 历史服务器地址 -->
  <property>
	<name>mapreduce.jobhistory.address</name>
	<value>ecs01:10020</value>
   </property>
   <property>
	<name>mapreduce.jobhistory.webapp.address</name>
	<value>ecs01:19888</value>
   </property>

vim yarn-site.xml

  <property>
    <!--配置NodeManager上运行的附属服务。需要配置成mapreduce_shuffle后才可以在Yarn上运行MapReduce程序-->
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
  </property>
  <!-- resourcemanager配置 单机版可以在同一台服务器上可以不配置 -->
  <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>ecs01</value>
  </property>
  <!-- 白名单 -->
  <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
  </property>

  <!-- 以下配置要根据服务器情况进行配置 我的云服务器只有2G的内存-->
  <property>
        <name>yarn.scheduler.minimum-allocation-mb</name>
        <value>256</value>
  </property>
  <property>
        <name>yarn.scheduler.maximum-allocation-mb</name>
        <value>2048</value>
  </property>
  <property>
        <name>yarn.nodemanager.resource.memory-mb</name>
        <value>4096</value>
  </property>
  <property>
        <name>yarn.nodemanager.pmem-check-enabled</name>
        <value>false</value>
  </property>
  <property>
        <name>yarn.nodemanager.vmem-check-enabled</name>
        <value>false</value>
  </property>

  <!-- 开启日志聚集功能 -->
  <property>
    <name>yarn.log-aggregation-enable</name>
    <value>true</value>
  </property>

  <!-- 设置日志聚集服务器地址 -->
  <property>
    <name>yarn.log.server.url</name>
    <value>http://ecs01:19888/jobhistory/logs</value>
  </property>

  <!-- 设置日志保留时间为7天 -->
  <property>
    <name>yarn.log-aggregation.retain-seconds</name>
    <value>604800</value>
  </property>


# hadoop用户
start-yarn.sh

# 启动history server

# 过时
mr-jobhistory-daemon.sh start historyserver

mapred --daemon start historyserver

为了可以访问yarn ui
阿里云控制台添加入方向8088安全组

为了可以访问logs
阿里云控制台添加入方向8042安全组

为了可以访问log history
阿里云控制台添加入方向19888安全组

运行一个mr任务
hadoop jar /opt/hadoop-3.1.2/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.1.2.jar wordcount /bigdata/hadoop.txt /bigdata/output

hadoop jar /opt/hadoop-3.1.2/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.1.2.jar pi 10 100

4、安装hive

wget http://archive.apache.org/dist/hive/hive-3.1.2/apache-hive-3.1.2-bin.tar.gz

cd /opt
tar -zxvf apache-hive-3.1.2-bin.tar.gz
mv apache-hive-3.1.2-bin hive

vim /etc/profile

export HIVE_HOME=/opt/hive
export PATH=$PATH:$HIVE_HOME/bin

source /etc/profile

cd hive/conf/

cp hive-env.sh.template hive-env.sh
vim hive-env.sh
HADOOP_HOME=/opt/hadoop-3.1.2

vim hive-site.xml
# 注意jdbc连接串的&符号

<configuration>

<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:mysql://localhost:3306/hive?useUnicode=true&amp;characterEncoding=UTF-8&amp;createDatabaseIfNotExist=true&amp;useSSL=false&amp;serverTimezone=Asia/Shanghai</value>
<description>JDBC connect string for a JDBC metastore</description>
</property>
 
<property>
<name>javax.jdo.option.ConnectionDriverName</name>
<value>com.mysql.cj.jdbc.Driver</value>
<description>Driver class name for a JDBC metastore</description>
</property>
 
<property>
<name>javax.jdo.option.ConnectionUserName</name>
<value>root</value>
<description>username to use against metastore database</description>
</property>
 
<property>
<name>javax.jdo.option.ConnectionPassword</name>
<value>xxxxx</value>
</property>

<property>
   <name>hive.metastore.uris</name>
   <value>thrift://ecs01:9083</value>
</property>

<!--hiveserver2-->
<property>
<name>hive.server2.thrift.port</name>
<value>10000</value>
</property>

<property>
<name>hive.server2.thrift.bind.host</name>
<value>ecs01</value>
</property>

<!--WEB UI-->
<property>
<name>hive.server2.webui.host</name>
<value>ecs01</value>
</property>

<property>
<name>hive.server2.webui.port</name>
<value>10002</value>
</property>

<property>
    <name>hive.server2.active.passive.ha.enable</name>
    <value>true</value>
</property>

</configuration>


wget https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.31/mysql-connector-j-8.0.31.jar
到lib目录下
schematool -dbType mysql -initSchema --verbose
hive-schema-3.1.0.mysql.sql替换字符集

Error: Specified key was too long; max key length is 3072 bytes (state=42000,code=1071)
PART_COL_PRIVS
TBL_COL_PRIVS
TAB_COL_STATS
PART_COL_STATS
COMPLETED_TXN_COMPONENTS

chown -R hadoop:hadoop /opt/hive/
su hadoop
nohup hive --service metastore >> /tmp/hadoop/metastore.log 2>&1 &

nohup hive --service hiveserver2 >> /tmp/hadoop/hiveserver2.log 2>&1 &

netstat -nl | grep LISTEN

为了可以访问hiveserver2 ui
阿里云控制台添加入方向10002安全组

hadoop fs -chmod -R 777 /tmp