1.下载
cd /root/apps
执行命令
git clone https://github.com/azkaban/azkaban.git
下载完成后可以看到有一个名为 azkaban 的文件夹
2.编译
执行如下命令生成一系列文件(如果不是第一次构建需要先clean,执行:./gradlew clean)
cd azkaban
./gradlew distTar
3.拷贝
mv azkaban azkaban1
新建文件夹:mkdir azkaban
执行命令如下命令拷贝到 azkaban目录下
cp /root/apps/azkaban1/azkaban-*/build/distributions/*.tar.gz /root/apps/azkaban
cd /root/apps/azkaban
tar zxvf azkaban-exec-server-3.62.0-7-g4f2f631.tar.gz
tar zxvf azkaban-web-server-3.62.0-7-g4f2f631.tar.gz
tar zxvf azkaban-db-3.62.0-7-g4f2f631.tar.gz
tar zxvf azkaban-solo-server-3.62.0-7-g4f2f631.tar.gz
mv azkaban-exec-server-3.62.0-7-g4f2f631.tar.gz azkaban-exec-server
mv azkaban-web-server-3.62.0-7-g4f2f631.tar.gz azkaban-web-server
mv azkaban-db-3.62.0-7-g4f2f631.tar.gz azkaban-db
mv azkaban-solo-server-3.62.0-7-g4f2f631.tar.gz azkaban-solo-server
4.数据库创建及数据结构导入
1) 以 root 用户登录mysql
2) CREATE DATABASE azkaban;
3) CREATE USER 'azkaban'@'%' IDENTIFIED BY 'azkaban';
4) GRANT SELECT,INSERT,UPDATE,DELETE ON azkaban.* to 'azkaban'@'%' WITH GRANT OPTION;
5) flush privileges;
6) use azkaban;
7) source /root/apps/create-all-sql-3.62.0-7-g4f2f631.sql
备注:这里使用的mysql(192.168.30.163)和azkaban不在同一台机器上(192.168.30.163)通过以下命令
scp /root/apps/azkaban/azkaban-db/create-all-sql-3.62.0-7-g4f2f631.sql root@h3:/root/apps (h3的地址就是192.168.30.163)
mysql数据结构导入时会报错【坑一】
ERROR 1071 (42000): Specified key was too long; max key length is 767 bytes
解决方案:删库,导入数据结构前做一下操作 :
set global innodb_file_per_table = ON ;
set global innodb_file_format = Barracuda;
set global innodb_large_prefix=on;
flush privileges;
修改create-all-sql-3.62.0-7-g4f2f631.sql 中表【execution_logs和execution_jobs】的建表语句
CREATE TABLE execution_jobs (
exec_id INT NOT NULL,
project_id INT NOT NULL,
version INT NOT NULL,
flow_id VARCHAR(128) NOT NULL,
job_id VARCHAR(512) NOT NULL,
attempt INT,
start_time BIGINT,
end_time BIGINT,
status TINYINT,
input_params LONGBLOB,
output_params LONGBLOB,
attachments LONGBLOB,
PRIMARY KEY (exec_id, job_id, flow_id, attempt)
) row_format=dynamic;
CREATE TABLE execution_logs (
exec_id INT NOT NULL,
name VARCHAR(640),
attempt INT,
enc_type TINYINT,
start_byte INT,
end_byte INT,
log LONGBLOB,
upload_time BIGINT,
PRIMARY KEY (exec_id, name, attempt, start_byte)
) row_format=dynamic;
5 配置azkaban-web-server
cd azkaban-web-server
keytool -keystore keystore -alias jetty -genkey -keyalg RSA
输入密钥库口令: azkaban
再次输入新口令: azkaban
您的名字与姓氏是什么?
[Unknown]: myfutech
您的组织单位名称是什么?
[Unknown]: myfutech
您的组织名称是什么?
[Unknown]: myfutech
您所在的城市或区域名称是什么?
[Unknown]: beijing
您所在的省/市/自治区名称是什么?
[Unknown]: beijing
该单位的双字母国家/地区代码是什么?
[Unknown]: CN
CN=myfutech, OU=myfutech, O=myfutech, L=beijing, ST=beijing, C=CN是否正确?
[否]: Y
输入 <jetty> 的密钥口令
(如果和密钥库口令相同, 按回车): azkaban
6.azkaban-exec-server中配置azkaban.properties
将azkaban-solo-server 的sql文件夹拷贝到azkaban-exec-server
$ cd azkaban-solo-server
$ cp -a sql/ /root/apps/azkaban/azkaban-exec-server
$ cd ../azkaban-exec-server/conf
配置conf/azkaban.properties
# Azkaban Personalization Settings
azkaban.name=Azkaban
azkaban.label=My Local Azkaban
azkaban.color=#FF3601
azkaban.default.servlet.path=/index
web.resource.dir=web/
default.timezone.id=Asia/Shanghai
# Azkaban UserManager class
user.manager.class=azkaban.user.XmlUserManager
user.manager.xml.file=conf/azkaban-users.xml
# Loader for projects
executor.global.properties=/root/apps/azkaban/azkaban-exec-server/conf/global.properties
azkaban.project.dir=projects
# Velocity dev mode
velocity.dev.mode=false
# Azkaban Jetty server properties.
jetty.use.ssl=false
jetty.maxThreads=25
jetty.port=8081
# Where the Azkaban web server is located
azkaban.webserver.url=http://localhost:8081
# mail settings
mail.sender=
mail.host=
# User facing web server configurations used to construct the user facing server URLs. They are useful when there is a reverse proxy between Azkaban web servers and users.
# enduser -> myazkabanhost:443 -> proxy -> localhost:8081
# when this parameters set then these parameters are used to generate email links.
# if these parameters are not set then jetty.hostname, and jetty.port(if ssl configured jetty.ssl.port) are used.
# azkaban.webserver.external_hostname=myazkabanhost.com
# azkaban.webserver.external_ssl_port=443
# azkaban.webserver.external_port=8081
job.failure.email=
job.success.email=
lockdown.create.projects=false
cache.directory=cache
# JMX stats
jetty.connector.stats=true
executor.connector.stats=true
# Azkaban plugin settings
azkaban.jobtype.plugin.dir=/root/apps/azkaban/azkaban-exec-server/plugins/jobtypes
# Azkaban mysql settings by default. Users should configure their own username and password.
database.type=mysql
mysql.port=3306
mysql.host=192.168.30.163
mysql.database=azkaban
mysql.user=azkaban
mysql.password=azkaban
mysql.numconnections=100
# Azkaban Executor settings
executor.maxThreads=50
executor.flow.threads=30
配置log4j.properties
log4j.rootLogger=INFO,C
log4j.appender.C=org.apache.log4j.ConsoleAppender
log4j.appender.C.Target=System.err
log4j.appender.C.layout=org.apache.log4j.PatternLayout
log4j.appender.C.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
添加MySQL驱动
将lib目录下的mysql驱动复制到extlib目录下
$ cd ../
$ mkdir extlib
$ cp lib/mysql-connector-java-5.1.28.jar extlib/
拷贝azkaban-exec-server目录下的conf和extlib到azkaban-web-server目录下
$ cp -a conf/ extlib/ ../azkaban-web-server
修改azkaban-exec-server/conf/azkaban.properties
# Azkaban Personalization Settings
azkaban.name=Azkaban
azkaban.label=My Local Azkaban
azkaban.color=#FF3601
azkaban.default.servlet.path=/index
web.resource.dir=/root/apps/azkaban/azkaban-web-server/web/
default.timezone.id=Asia/Shanghai
# Azkaban UserManager class
user.manager.class=azkaban.user.XmlUserManager
user.manager.xml.file=/root/apps/azkaban/azkaban-web-server/conf/azkaban-users.xml
# Loader for projects
executor.global.properties=conf/global.properties
azkaban.project.dir=projects
# Velocity dev mode
velocity.dev.mode=false
# Azkaban Jetty server properties.
jetty.use.ssl=false
jetty.maxThreads=25
jetty.port=8443
# Where the Azkaban web server is located
azkaban.webserver.url=http://localhost:8443
# mail settings
mail.sender=
mail.host=
# User facing web server configurations used to construct the user facing server URLs. They are useful when there is a reverse proxy between Azkaban web servers and users.
# enduser -> myazkabanhost:443 -> proxy -> localhost:8081
# when this parameters set then these parameters are used to generate email links.
# if these parameters are not set then jetty.hostname, and jetty.port(if ssl configured jetty.ssl.port) are used.
# azkaban.webserver.external_hostname=myazkabanhost.com
# azkaban.webserver.external_ssl_port=443
# azkaban.webserver.external_port=8081
job.failure.email=
job.success.email=
lockdown.create.projects=false
cache.directory=cache
# JMX stats
jetty.connector.stats=true
executor.connector.stats=true
# Azkaban plugin settings
azkaban.jobtype.plugin.dir=/root/apps/azkaban/azkaban-exec-server/plugins/jobtypes
# Azkaban mysql settings by default. Users should configure their own username and password.
database.type=mysql
mysql.port=3306
mysql.host=192.168.30.163
mysql.database=azkaban
mysql.user=azkaban
mysql.password=azkaban
mysql.numconnections=100
# Azkaban Executor settings
executor.maxThreads=50
executor.flow.threads=30
azkaban.use.multiple.executors=true
azkaban.executorselector.filters=StaticRemainingFlowSize,MinimumFreeMemory,CpuStatus
azkaban.executorselector.comparator.NumberOfAssignedFlowComparator=1
azkaban.executorselector.comparator.Memory=1
azkaban.executorselector.comparator.LastDispatched=1
azkaban.executorselector.comparator.CpuUsage=1
7 启动
启动web-server和exec-server,执行bin/azkaban-executor-start.sh 和 bin/azkaban-web-start.sh 命令。
创建Project,两种方式:web界面创建和通过API创建(方法)。
登录192.168.30.163:8443
输入用户名密码azkaban/azkaban登录
参考【填坑的参考网址】:
https://blog.csdn.net/liu16659/article/details/81367312
https://www.jianshu.com/p/701329d3ccd1
https://blog.csdn.net/hg_harvey/article/details/80342396
https://yq.aliyun.com/ziliao/306566
https://blog.csdn.net/songmaolin_csdn/article/details/72621357
https://blog.csdn.net/C_thin/article/details/73252709