服务器集群服务搭建

集群免密配置

# 新建hadoop_pro用户
useradd hadoop_pro
# 设置一个密码
passwd  hadoop_pro
# 添加进 root组
usermod -a -G root hadoop_pro
# 创建目录并移交权限
mkdir /data01/bigdata
mkdir /data01/bigdata/data
mkdir /data01/bigdata/server
chown -R hadoop_pro:hadoop_pro /data01/bigdata/
# 修改主机名
hostname athadoopxxx
vim /etc/hostname
# 安装sshpass
yum -y install sshpass
# 密钥生成
su hadoop_pro
ssh-keygen -t rsa
# 密钥传送给其他机器
sshpass -p 'devOps2023@' ssh-copy-id -o StrictHostKeyChecking=no  -i ~/.ssh/id_rsa.pub athadoopxx
# 远程其他机器 成功连接则设置完成
ssh hadoopxx

spark搭建

# 下载spark包
wget https://archive.apache.org/dist/spark/spark-3.1.3/spark-3.1.3-bin-hadoop3.2.tgz

# 解压
tar -zxvf spark-3.1.3-bin-hadoop3.2.tgz

# 更名
mv spark-3.1.3-bin-hadoop3.2 spark-3.1.3

# 修改配置文件-master节点主机名/ip需要手动修改
cd spark-3.1.3/conf
cp spark-env.sh.template spark-env.sh
echo "export JAVA_HOME=${JAVA_HOME}" >> spark-env.sh
# Standalone模式填写 该教程使用的是zookeeper主备切换高可用方案 故注销  若需要使用该方案 则将SPARK_DAEMON_JAVA_OPTS 这一行注销
# echo "export SPARK_MASTER_HOST=主节点主机名/ip" >> spark-env.sh
echo "export SPARK_MASTER_PORT=7077" >> spark-env.sh
echo 'export SPARK_DAEMON_JAVA_OPTS="-Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=qcp-zk.vivo.xyz:2181 -Dspark.deploy.zookeeper.dir=/spark"' >> spark-env.sh

# 增加slave节点
cp slaves.template slaves
echo "从节点主机名/ip" >> slaves

# 配置日志记录信息，日志上传HDFS
cp spark-defaults.conf.template spark-defaults.conf
echo "spark.eventLog.enabled  true" >> spark-defaults.conf
echo "spark.eventLog.dir      hdfs://HDFS主机名/ip:8020/spark_log" >> spark-defaults.conf
echo "spark.eventLog.compress true" >> spark-defaults.conf
echo "export SPARK_HISTORY_OPTS="-Dspark.history.ui.port=4000 -Dspark.history.retainedApplications=3 -Dspark.history.fs.logDirectory=hdfs://node01:8020/spark_log" >> spark-env.sh

# 启动 停止
cd spark-3.1.3/sbin
./start-all.sh
./start-history-server.sh
./start-master.sh

# 访问对应master节点 8080端口 验证

gitlab卸载

# 卸载相关服务
sudo systemctl stop gitlab-runsvdir
sudo systemctl disable gitlab-runsvdir
sudo rm /usr/lib/systemd/system/gitlab-runsvdir.service
sudo systemctl daemon-reload
sudo systemctl reset-failed
sudo gitlab-ctl uninstall

# 删除数据
sudo gitlab-ctl cleanse && sudo rm -r /opt/gitlab

# 卸载
sudo yum remove gitlab-ce

gitlab搭建

# gitlab官方地址： https://docs.gitlab.com/16.7/omnibus/installation/
# 创建gitlab源
touch /etc/yum.repos.d/gitlab-ce.repo
echo "[gitlab-ce]" >> /etc/yum.repos.d/gitlab-ce.repo
echo "name=gitlab-ce" >> /etc/yum.repos.d/gitlab-ce.repo
echo "baseurl=https://mirrors.tuna.tsinghua.edu.cn/gitlab-ce/yum/el7/" >> /etc/yum.repos.d/gitlab-ce.repo
echo "gpgcheck=0" >> /etc/yum.repos.d/gitlab-ce.repo
echo "enabled=1" >> /etc/yum.repos.d/gitlab-ce.repo
# 刷新源
yum clean all && yum makecache

# 安装gitlab， gitlab-ce-xxx  可以指定版本
yum install -y  gitlab-ce-16.7.0

# 修改配置文件, external_url '需要替换的访问地址:端口'
vim /etc/gitlab/gitlab.rb

# 重载配置，若是报错则先执行systemctl restart gitlab-runsvdir
gitlab-ctl reconfigure

# 启动 external_url指定的端口以及8080端口不能被占用 否则502
gitlab-ctl start

# 查看状态
gitlab-ctl status

# 初始化管理员密码
cd /opt/gitlab/bin/
gitlab-rails console -e production

u=User.where(id:1).first
u.password='test@1234'
u.password_confirmation='test@1234'
u.save!