Linxu下的实时数仓demo环境搭建

主要组件

1、jdk

1、

uname -a
Linux bigData04 3.10.0-957.el7.x86_64 #1 SMP Thu Nov 8 23:39:32 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux

 rpm -qa | grep java
java-1.7.0-openjdk-1.7.0.191-2.6.15.5.el7.x86_64
java-1.8.0-openjdk-headless-1.8.0.181-7.b13.el7.x86_64
javamail-1.4.6-8.el7.noarch
pki-base-java-10.5.9-6.el7.noarch
java-11-openjdk-headless-11.0.ea.28-7.el7.x86_64
java-11-openjdk-11.0.ea.28-7.el7.x86_64
nuxwdog-client-java-1.0.3-8.el7.x86_64
javassist-3.16.1-10.el7.noarch
tzdata-java-2018e-3.el7.noarch
python-javapackages-3.4.1-11.el7.noarch
java-1.7.0-openjdk-headless-1.7.0.191-2.6.15.5.el7.x86_64
javapackages-tools-3.4.1-11.el7.noarch
 

rpm -ql java-1.8.0-openjdk-headless | more
/usr/lib/jvm-exports/java-1.8.0-openjdk-1.8.0.181-7.b13.el7.x86_64
/usr/lib/jvm-exports/java-1.8.0-openjdk-1.8.0.181-7.b13.el7.x86_64/jaas-1.8.0.181.jar
/usr/lib/jvm-exports/java-1.8.0-openjdk-1.8.0.181-7.b13.el7.x86_64/jaas-1.8.0.jar
/usr/lib/jvm-exports/java-1.8.0-openjdk-1.8.0.181-7.b13.el7.x86_64/jaas.jar
/usr/lib/jvm-exports/java-1.8.0-openjdk-1.8.0.181-7.b13.el7.x86_64/jce-1.8.0.181.jar
/usr/lib/jvm-exports/java-1.8.0-openjdk-1.8.0.181-7.b13.el7.x86_64/jce-1.8.0.jar
/usr/lib/jvm-exports/java-1.8.0-openjdk-1.8.0.181-7.b13.el7.x86_64/jce.jar
/usr/lib/jvm-exports/java-1.8.0-openjdk-1.8.0.181-7.b13.el7.x86_64/jdbc-stdext-1.8.0.181.jar
/usr/lib/jvm-exports/java-1.8.0-openjdk-1.8.0.181-7.b13.el7.x86_64/jdbc-stdext-1.8.0.jar
/usr/lib/jvm-exports/java-1.8.0-openjdk-1.8.0.181-7.b13.el7.x86_64/jdbc-stdext-3.0.jar

vim /etc/profile

# jdk config
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.181-7.b13.el7.x86_64
export JRE_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.181-7.b13.el7.x86_64/jre
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$PATH:$HOME/bin
 

source /etc/profile
[root@bigData04 jvm]# echo $JAVA_HOME
/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.181-7.b13.el7.x86_64

安装 yum install java-1.8.0-openjdk-devel.x86_64

openjdk安装完只有默认的运行环境,没有开发环境。

此时我们只要运行一下代码安装jdk开发环境

yum install java-1.8.0-openjdk-devel.x86_64


安装完成之后就能看到/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.292.b10-0.el8_3.x86_64/文件夹下lib、bin等文件都出现了
profile中的环境配置也修改了一下,不是以前那种**.jar了
 

2、mysql

卸载MySQL
查看MySQL安装包

rpm -qa mysql | grep -i mysql
perl-DBD-MySQL-4.023-6.el7.x86_64

使用yum卸载
yum remove perl-DBD-MySQL-4.023-6.el7.x86_64
后面需要手动输入y 同意卸载

查看卸载是否成功

rpm -qa mysql | grep -i mysql
查找MySQL相关目录并删除

find / -name mysql | xargs rm -rf
删除/etc/my.cnf

rm -rf /etc/my.cnf
删除/var/log/mysqld.log(如果不删除这个文件,会导致新安装的mysql无法生存新密码,导致无法登录)

rm -rf /var/log/mysqld.log

find / -name mysql | xargs rm -rf

[root@bigData04 jvm]# rm -rf /etc/my.cnf
[root@bigData04 jvm]# rm -rf /var/log/mysqld.log
 

安装

下载mysql 

MySQL :: Download MySQL Community Server (Archived Versions)

tar -xvf mysql-5.7.41-1.el7.x86_64.rpm-bundle.tar 
mysql-community-client-5.7.41-1.el7.x86_64.rpm
mysql-community-common-5.7.41-1.el7.x86_64.rpm
mysql-community-devel-5.7.41-1.el7.x86_64.rpm
mysql-community-embedded-5.7.41-1.el7.x86_64.rpm
mysql-community-embedded-compat-5.7.41-1.el7.x86_64.rpm
mysql-community-embedded-devel-5.7.41-1.el7.x86_64.rpm
mysql-community-libs-5.7.41-1.el7.x86_64.rpm
mysql-community-libs-compat-5.7.41-1.el7.x86_64.rpm
mysql-community-server-5.7.41-1.el7.x86_64.rpm
mysql-community-test-5.7.41-1.el7.x86_64.rpm
 

[root@bigData04 soft]# ls /opt/
rh
[root@bigData04 soft]# mv mysql-community-client-5.7.41-1.el7.x86_64.rpm /opt/
[root@bigData04 soft]# mv mysql-community-common-5.7.41-1.el7.x86_64.rpm /opt/
[root@bigData04 soft]# mv mysql-community-devel-5.7.41-1.el7.x86_64.rpm /opt/
[root@bigData04 soft]# mv mysql-community-libs-5.7.41-1.el7.x86_64.rpm /opt/
[root@bigData04 soft]# mv mysql-community-libs-compat-5.7.41-1.el7.x86_64.rpm /opt/
[root@bigData04 soft]# mv mysql-community-server-5.7.41-1.el7.x86_64.rpm /opt/
 

chmod -R 777 /tmp

 115  rpm -qa | grep libaio
  116  rpm -qa | grep net-tools
  117  cd /opt/

  119  yum remove mysql-libs
  120  rpm -ivh mysql-community-common-5.7.41-1.el7.x86_64.rpm 
  121  rpm -ivh mysql-community-libs-5.7.41-1.el7.x86_64.rpm 
  122  rpm -ivh mysql-community-libs-compat-5.7.41-1.el7.x86_64.rpm 
  123  rpm -ivh mysql-community-client-5.7.41-1.el7.x86_64.rpm 
  124  rpm -ivh mysql-community-server-5.7.41-1.el7.x86_64.rpm 
  125  rpm -ivh mysql-community-devel-5.7.41-1.el7.x86_64.rpm 

[root@bigData04 opt]# mysql -version
ERROR 2002 (HY000): Can't connect to local MySQL server through socket '/var/lib/mysql/mysql.sock' (2)
[root@bigData04 opt]# mysqladmin -version
mysqladmin: [ERROR] mysqladmin: unknown option '-e'
 

[root@bigData04 opt]# mysqld --initialize --user=mysql
[root@bigData04 opt]# 
[root@bigData04 opt]# 
[root@bigData04 opt]# cat /var/log/mysqld.log   

查看密码
2023-06-20T03:20:16.658650Z 0 [Warning] TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).
2023-06-20T03:20:17.375255Z 0 [Warning] InnoDB: New log files created, LSN=45790
2023-06-20T03:20:17.397789Z 0 [Warning] InnoDB: Creating foreign key constraint system tables.
2023-06-20T03:20:17.400862Z 0 [Warning] No existing UUID has been found, so we assume that this is the first time that this server has been started. Generating a new UUID: 612d18c6-0f19-11ee-8e5e-005056a7157d.
2023-06-20T03:20:17.401166Z 0 [Warning] Gtid table is not ready to be used. Table 'mysql.gtid_executed' cannot be opened.
2023-06-20T03:20:17.603447Z 0 [Warning] A deprecated TLS version TLSv1 is enabled. Please use TLSv1.2 or higher.
2023-06-20T03:20:17.603456Z 0 [Warning] A deprecated TLS version TLSv1.1 is enabled. Please use TLSv1.2 or higher.
2023-06-20T03:20:17.604017Z 0 [Warning] CA certificate ca.pem is self signed.
2023-06-20T03:20:17.638157Z 1 [Note] A temporary password is generated for root@localhost: .)SpDeu*f6&!
[root@bigData04 opt]# systemctl start mysqld.service
[root@bigData04 opt]# systemctl status mysqld.service
● mysqld.service - MySQL Server
   Loaded: loaded (/usr/lib/systemd/system/mysqld.service; enabled; vendor preset: disabled)
   Active: active (running) since Tue 2023-06-20 11:21:06 CST; 40s ago
     Docs: man:mysqld(8)
           http://dev.mysql.com/doc/refman/en/using-systemd.html
  Process: 16802 ExecStart=/usr/sbin/mysqld --daemonize --pid-file=/var/run/mysqld/mysqld.pid $MYSQLD_OPTS (code=exited, status=0/SUCCESS)
  Process: 16772 ExecStartPre=/usr/bin/mysqld_pre_systemd (code=exited, status=0/SUCCESS)
 Main PID: 16806 (mysqld)
    Tasks: 27
   CGroup: /system.slice/mysqld.service
           └─16806 /usr/sbin/mysqld --daemonize --pid-file=/var/run/mysqld/mysqld.pid

Jun 20 11:20:55 bigData04 systemd[1]: Starting MySQL Server...
Jun 20 11:21:06 bigData04 systemd[1]: Started MySQL Server.
 

其他命令

#启动:
systemctl start mysqld.service
#关闭:
systemctl stop mysqld.service
#重启:
systemctl restart mysqld.service
#查看状态:
systemctl status mysqld.service
#检查是否自启:
systemctl list-unit-files|grep mysqld.service
#开机自启:
systemctl enable mysqld.service
#关闭开机自启:
systemctl disable mysqld.service
#查看进程:
ps -ef | grep -i mysql

ALTER USER 'root'@'localhost' IDENTIFIED BY '123456';

  1. #将root的所在行的host设置成%,%是通配符,修改成这样任意ip都可以访问,生产环境中要根据ip修改

  2. update user set host='%' where user='root';

flush privileges;

 #登录mysql
mysql -uroot -proot@123
#创建数据库
mysql>
create database dinky;
#授权
mysql>
grant all privileges on dinky.* to 'dinky'@'%' identified by 'dinky' with grant option;
mysql>
flush privileges;
#此处用 dinky 用户登录
mysql -hlocalhost -udinky -pdinky

source /opt/module/dinky/sql/dinky.sql

修改配置文档

url: jdbc:mysql://192.168.1.244:3306/dinky?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&useSSL=false&zeroDateTimeBehavior=convertToNull&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true
    username: dinky
    password: dinky
    driver-class-name: com.mysql.cj.jdbc.Driver

3、doris

 202  cd /opt/
  203  ls
  204  cd module/
  205  ls
  206  mkdir doris
  207  cd doris/
  208  ls
  209  tar -zxvf ~/soft/apache-doris-fe-1.2.4.1-bin-x86_64.tar.xz .
  210  cd ..
  211  tar -xvf ~/soft/apache-doris-fe-1.2.4.1-bin-x86_64.tar.xz -C doris/
  212  ls doris/
  213  tar -xvf ~/soft/apache-doris-be-1.2.4.1-bin-x86_64.tar.xz -C doris/

配置 FE

我们进入到 apache-doris-x.x.x/fe 目录

cd apache-doris-x.x.x/fe

修改 FE 配置文件 conf/fe.conf ,这里我们主要修改两个参数:priority_networks 及 meta_dir ,如果你需要更多优化配置,请参考 FE 参数配置说明,进行调整。

  1. 添加 priority_networks 参数
priority_networks=172.23.16.0/24

ip a

 启动  /opt/module/doris/doris_fe/bin/start_fe.sh --daemon

curl http://127.0.0.1:18030/api/bootstrap
返回如下代表成功:
{"msg":"success","code":0,"data":{"replayedJournalId":0,"queryPort":0,"rpcPort":0,"version":""},"count":0}

[root@bigData04 doris_be]# /opt/module/doris/doris_be/bin/start_be.sh --daemon
Please set vm.max_map_count to be 2000000 under root using 'sysctl -w vm.max_map_count=2000000'.
[root@bigData04 doris_be]# sysctl -w vm.max_map_count=2000000
vm.max_map_count = 2000000
/opt/module/doris/doris_be/bin/start_be.sh --daemon

关闭防火墙

[root@bigData04 conf]# systemctl stop firewalld.service
[root@bigData04 conf]# systemctl disable firewalld.service
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
[root@bigData04 conf]# systemctl status firewalld.service
● firewalld.service - firewalld - dynamic firewall daemon
   Loaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor preset: enabled)
   Active: inactive (dead)
     Docs: man:firewalld(1)

Jun 19 17:34:01 bigData04 systemd[1]: Starting firewalld - dynamic firewall daemon...
Jun 19 17:34:04 bigData04 systemd[1]: Started firewalld - dynamic firewall daemon.
Jun 20 15:03:12 bigData04 systemd[1]: Stopping firewalld - dynamic firewall daemon...
Jun 20 15:03:12 bigData04 systemd[1]: Stopped firewalld - dynamic firewall daemon.
[root@bigData04 conf]# 
可以登录了

2.修改配置
priority_networks = 192.168.1.0/24
storage_root_path=/path/your/data_dir ##可以使用默认的,默认目录在 BE安装目录的 storage 目录下,BE 配置的存储目录必须先创建好


3.配置 JAVA_HOME 环境变量
以在start_be.sh启动脚本第一行添加export JAVA_HOME=your_java_home_path 来添加环境变量
如下:
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.372.b07-1.el7_9.x86_64

4.拷贝java udf jar包到be的lib目录
tar -xvf apache-doris-dependencies-1.2.2-bin-x86_64.tar.xz
cp java-udf-jar-with-dependencies.jar /opt/doris-be/lib/

tar apache-doris-dependencies-1.2.4.1-bin-x86_64.tar.xz 
 tar -xvf apache-doris-dependencies-1.2.4.1-bin-x86_64.tar.xz 

cp apache-doris-dependencies-1.2.4.1-bin-x86_64/java-udf-jar-with-dependencies.jar doris/doris-be/lib/

5.启动be
cd /opt/doris-be/bin
./start_be.sh --daemon

 
1.在fe上添加be并创建库和表
[root@host134 bin]# /opt/mysql5640/bin/mysql -uroot -P9030 -h127.0.0.1
mysql> ALTER SYSTEM ADD BACKEND "192.168.1.134:9050";
Query OK, 0 rows affected (0.13 sec)

4、dinky

上传 dinky的相关依赖包  

(46条消息) 【免费】Dinkyplugins的依赖jar包资源-CSDN文库

#启动
sh auto.sh start 1.14

#启动
sh auto.sh start
#停止
sh auto.sh stop
#重启
sh auto.sh restart
#查看状态
sh auto.sh status

链接sqlserver的异常

[root@bigData04 security]# pwd
/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.372.b07-1.el7_9.x86_64/jre/lib/security

 

 重启下dinky 即可

5、flink

tar xvf flink-1.14.6-bin-scala_2.12.tgz 
ls
vim flink-1.14.6/conf/flink-conf.yaml 
cd flink-1.14.6/bin/
 ls
./start-cluster.sh

进入conf目录下的flink-conf.yaml文件,修改rest.bind-address参数允许所有地址访问

6、dolphinscheduler  

 tar -vxf apache-dolphinscheduler-3.1.7-bin.tar.gz -C dolphinscheduler/
 ls
cd dolphinscheduler/

cd apache-dolphinscheduler-3.1.7-bin/

./bin/dolphinscheduler-daemon.sh start standalone-server

admin/dolphinscheduler123

需要初始化数据库信息  否则页面功能不全

 

 修改 {解压目录}/apache-dolphinscheduler-3.0.0-SNAPSHOT-bin/bin/env/dolphinscheduler_env.sh 文件中如下内容:

	export DATABASE=${DATABASE:-mysql}
	export SPRING_PROFILES_ACTIVE=${DATABASE}
	export SPRING_DATASOURCE_DRIVER_CLASS_NAME=com.mysql.cj.jdbc.Driver 
	export SPRING_DATASOURCE_URL={url}【mysql连接地址】
	export S PRING_DATASOURCE_USERNAME={username}【数据库账号】
	export SPRING_DATASOURCE_PASSWORD={password}【数据库密码】

dinky 的配置文件 要配置ds地址

 

  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值