#!/bin/bash
echo “##### Update /etc/hosts #####”
cat >> /etc/hosts <<EOF
192.168.100.83 es83
192.168.100.86 es86
192.168.100.87 es87
EOF
echo “##### Stop firewalld #####”
systemctl stop firewalld
systemctl disable firewalld
echo “##### Close selinux #####”
setenforce 0
sed -i “s/SELINUX=enforcing/SELINUX=disabled/g” /etc/selinux/config
echo “##### Close swap #####”
swapoff -a
提示:修改完该文件后,需重新登录终端才可生效,可通过ulimit -a查看。
echo “##### Modify /etc/security/limits.conf #####”
cat > /etc/security/limits.conf <<EOF
* soft nofile 65536
* hard nofile 131072
* soft nproc 65536
* hard nproc 65536
* soft memlock unlimited
* hard memlock unlimited
EOF
echo “##### Modify /etc/sysctl.conf #####”
cat >> /etc/sysctl.conf <<EOF
vm.max_map_count=562144
EOF
sysctl -p
echo “##### Create user(密码随意) #####”
useradd elkuser
echo 123456 | passwd --stdin elkuser
echo “##### 配置SSH免密通信 #####”
ssh-keygen # 一路回车即可
ssh-copy-id 192.168.100.83
ssh-copy-id 192.168.100.86
ssh-copy-id 192.168.100.87
04Elasticsearch集群部署
Elasticsearch 是一个分布式、RESTful风格的搜索和数据分析引擎;它实现了用于全文检索的倒排索引,而且为每个数据都编入了索引,搜索速度非常快;它具有可扩展性和弹性,每秒钟能处理海量事件,并且它适用于所有数据类型,例如结构化数据、非结构化数据、地理位置等。
笔者在生产环境上,为Elasticsearch分配了30G内存(最大不要超过32G),6块446G的SSD磁盘,并使用G1的垃圾回收策略,关于硬件配置大家根据实际情况来分配使用!
提示:笔者已事先下载好了所有软件包到服务器上;本文的三个es节点默认都做主节点和数据节点,当使用xpack加密时,主节点也必须做数据节点,否则加密配置写入不进es存储!
在本文中,笔者直接在83节点上完成了es集群的部署,请仔细阅读下方的命令!
下载方式:wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.2.0-linux-x86_64.tar.gz
echo “##### 解压Elasticsearch #####”
[root@es83 ~]# cd /home/elkuser/
[root@es83 elkuser]# tar -xvf elasticsearch-7.2.0-linux-x86_64.tar.gz
echo “##### 修改jvm文件 #####”
[root@es83 elkuser]# cd ./elasticsearch-7.2.0/
[root@es83 elasticsearch-7.2.0]# sed -i -e ‘s/1g/30g/g’ -e ‘36,38s/^-/#&/g’ ./config/jvm.options
[root@es83 elasticsearch-7.2.0]# sed -i -e ‘N;38 a -XX:+UseG1GC \n-XX:MaxGCPauseMillis=50’ ./config/jvm.options
echo “##### 生成关键证书文件 #####”
[root@es83 elasticsearch-7.2.0]# ./bin/elasticsearch-certutil ca
…
Please enter the desired output file [elastic-stack-ca.p12]: 回车Enter
Enter password for elastic-stack-ca.p12 : 回车Enter
echo “##### 利用关键证书生成所有es节点证书文件 #####”
[root@es83 elasticsearch-7.2.0]# ./bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12 --ip 192.168.100.83
…
Enter password for CA (elastic-stack-ca.p12) : 回车Enter
Please enter the desired output file [elastic-certificates.p12]: es83.p12
Enter password for es83.p12 : 回车Enter
[root@es83 elasticsearch-7.2.0]# ./bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12 --ip 192.168.100.86
…
Enter password for CA (elastic-stack-ca.p12) : 回车Enter
Please enter the desired output file [elastic-certificates.p12]: es86.p12
Enter password for es86.p12 : 回车Enter
[root@es83 elasticsearch-7.2.0]# ./bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12 --ip 192.168.100.87
…
Enter password for CA (elastic-stack-ca.p12) : 回车Enter
Please enter the desired output file [elastic-certificates.p12]: es87.p12
Enter password for es87.p12 : 回车Enter
echo “##### 利用关键证书生成后续logstash所需证书 #####”
[root@es83 elasticsearch-7.2.0]# openssl pkcs12 -in elastic-stack-ca.p12 -clcerts -nokeys > root.cer
[root@es83 elasticsearch-7.2.0]# openssl x509 -in root.cer -out root.pem
echo “##### 利用关键证书生成后续kibana所需证书 #####”
[root@es83 elasticsearch-7.2.0]# ./bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12 -name “CN=something,OU=Consulting Team,DC=mydomain,DC=com”
…
Enter password for CA (elastic-stack-ca.p12) : 回车Enter
Please enter the desired output file [CN=something,OU=Consulting Team,DC=mydomain,DC=com.p12]: client.p12
Enter password for client.p12 : 回车Enter
echo “##### 移动所生成的证书文件到指定目录下 #####”
[root@es83 elasticsearch-7.2.0]# cp *.p12 ./config/
echo “##### 修改es配置文件 #####”
[root@es83 elasticsearch-7.2.0]# cat > ./config/elasticsearch.yml <<EOF
cluster.name: chilu_elk
node.name: es83
node.master: true
node.data: true
path.data: /logdata/data1,/logdata/data2,/logdata/data3,/logdata/data4,/logdata/data5,/logdata/data6
bootstrap.memory_lock: true
bootstrap.system_call_filter: false
network.host: 192.168.100.83
http.port: 9200
transport.tcp.port: 9300
discovery.zen.ping.unicast.hosts: [“192.168.100.83:9300”,“192.168.100.86:9300”,“192.168.100.87:9300”]
cluster.initial_master_nodes: [“192.168.100.83:9300”,“192.168.100.86:9300”,“192.168.100.87:9300”]
node.max_local_storage_nodes: 256
indices.fielddata.cache.size: 50%
http.cors.enabled: true
http.cors.allow-origin: “*”
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: es83.p12
xpack.security.transport.ssl.truststore.path: elastic-stack-ca.p12
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.keystore.path: es83.p12
xpack.security.http.ssl.truststore.path: elastic-stack-ca.p12
xpack.security.http.ssl.client_authentication: optional
EOF
echo “##### scp目录到其他节点上,并修改配置 #####”
[root@es83 elasticsearch-7.2.0]# cd …/
[root@es83 elkuser]# scp -r ./elasticsearch-7.2.0 192.168.100.86:/home/elkuser/
[root@es83 elkuser]# scp -r ./elasticsearch-7.2.0 192.168.100.87:/home/elkuser/
[root@es83 elkuser]# ssh 192.168.100.86 “sed -i -e ‘s/es83/es86/g’ -e ‘8s/192.168.100.83/192.168.100.86/’ /home/elkuser/elasticsearch-7.2.0/config/elasticsearch.yml”
[root@es83 elkuser]# ssh 192.168.100.87 “sed -i -e ‘s/es83/es87/g’ -e ‘8s/192.168.100.83/192.168.100.87/’ /home/elkuser/elasticsearch-7.2.0/config/elasticsearch.yml”
echo “##### 修改各目录的属主和组 #####”
[root@es83 elkuser]# chown -R elkuser:elkuser /logdata ./elasticsearch-7.2.0
[root@es83 elkuser]# ssh 192.168.100.86 “chown -R elkuser:elkuser /logdata /home/elkuser/elasticsearch-7.2.0”
[root@es83 elkuser]# ssh 192.168.100.87 “chown -R elkuser:elkuser /logdata /home/elkuser/elasticsearch-7.2.0”
echo “##### 切换普通用户,后台运行elasticsearch服务 #####”
[root@es83 elasticsearch-7.2.0]# su elkuser
[elkuser@es83 elasticsearch-7.2.0]$ ./bin/elasticsearch -d
[elkuser@es83 elasticsearch-7.2.0]$ ssh elkuser@192.168.100.86 “/home/elkuser/elasticsearch-7.2.0/bin/elasticsearch -d”
[elkuser@es83 elasticsearch-7.2.0]$ ssh elkuser@192.168.100.87 “/home/elkuser/elasticsearch-7.2.0/bin/elasticsearch -d”
echo “##### 自动生成用户密码(记得保存好用户密码) #####”
[elkuser@es83 elasticsearch-7.2.0]$ echo y | ./bin/elasticsearch-setup-passwords auto | tee elk_pwd.log
Initiating the setup of passwords for reserved users elastic,apm_system,kibana,logstash_system,beats_system,remote_monitoring_user.
The passwords will be randomly generated and printed to the console.
Changed password for user apm_system
PASSWORD apm_system = HojN4w88Nwgl51Oe7o12
Changed password for user kibana
PASSWORD kibana = JPYDvJYn2CDmls5gIlNG
Changed password for user logstash_system
PASSWORD logstash_system = kXxmVCX34PGpUluBXABX
Changed password for user beats_system
PASSWORD beats_system = rY90aBHjAdidQPwgX87u
Changed password for user remote_monitoring_user
PASSWORD remote_monitoring_user = 0VxaGROqo255y60P1kBV
Changed password for user elastic
PASSWORD elastic = NvOBRGpUE3DoaSbYaUp3
echo “##### 测试es加密,查看集群状态是否为green #####”
[elkuser@es83 elasticsearch-7.2.0]$ curl --tlsv1 -XGET “https://192.168.100.83:9200/_cluster/health?pretty” --user elastic:NvOBRGpUE3DoaSbYaUp3 -k
05Kafka集群部署
Kafka 是最初由Linkedin公司开发,是一个分布式、分区的、多副本的、多订阅者,基于zookeeper协调的分布式消息系统;它具有高吞吐量、低延迟、可扩展性、持久性、可靠性、容错性和高并发等特点,可以处理几十万条消息,延迟只有几毫秒,集群式部署支持热扩展,消息可被持久化到本地磁盘,防止数据丢失,而且支持数千个客户端同时读写。
在本文的架构中,kafka是用作缓存消息队列,用来实时接收日志和发送日志到logstash,实现解耦和流量削峰,解决logstash消费能力跟不上导致的数据丢失问题;笔者采用的是kafka内置的zookeeper,也是以集群方式部署,无需再单独搭建zookeeper集群服务。
注意:kafka的集群配置信息,状态维护是存储在zookeeper这个进程里的,所以kafka在启动前需要先配置启动zookeeper!
笔者为zookeeper服务分配了4G内存,为kafka服务分配了31G内存和5块SSD磁盘,关于硬件配置大家根据实际情况来分配使用!
下载方式:wget https://archive.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz
echo “##### 解压Kafka #####”
[root@es83 ~]# cd /opt/
[root@es83 opt]# tar -xvf ./kafka_2.12-2.3.0.tgz
echo “##### 修改zookeeper配置文件 #####”
[root@es83 opt]# cd ./kafka_2.12-2.3.0/
[root@es83 kafka_2.12-2.3.0]# cat > ./config/zookeeper.properties <<EOF
dataDir=/opt/zookeeper
clientPort=2181
maxClientCnxns=0
tickTime=2000
initLimit=10
syncLimit=5
server.1=192.168.100.83:2888:3888
server.2=192.168.100.86:2888:3888
server.3=192.168.100.87:2888:3888
authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
requireClientAuthScheme=sasl
jaasLoginRenew=3600000
4lw.commands.whitelist=
EOF
echo “##### 创建zookeeper数据目录和对应的myid文件 #####”
[root@es83 kafka_2.12-2.3.0]# mkdir /opt/zookeeper
[root@es83 kafka_2.12-2.3.0]# echo 1 > /opt/zookeeper/myid
echo “##### 修改kafka配置文件 #####”
[root@es83 kafka_2.12-2.3.0]# cat > ./config/server.properties <<EOF
broker.id=83
listeners=SASL_PLAINTEXT://192.168.100.83:9092
advertised.listeners=SASL_PLAINTEXT://192.168.100.83:9092
num.network.threads=5
num.io.threads=8
socket.send.buffer.bytes=1024000
socket.receive.buffer.bytes=1024000
socket.request.max.bytes=1048576000
log.dirs=/logdata/kfkdata1,/logdata/kfkdata2,/logdata/kfkdata3,/logdata/kfkdata4,/logdata/kfkdata5
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=72
log.segment.delete.delay.ms=1000
log.cleaner.enable=true
log.cleanup.policy=delete
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.100.83:2181,192.168.100.86:2181,192.168.100.87:2181
zookeeper.connection.timeout.ms=60000
group.initial.rebalance.delay.ms=0
delete.topic.enable=true
security.inter.broker.protocol=SASL_PLAINTEXT
sasl.enabled.mechanisms=PLAIN
sasl.mechanism.inter.broker.protocol=PLAIN
authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer
allow.everyone.if.no.acl.found=true
super.users=User:admin;User:kafka
EOF
echo “##### 创建zk和kafka的sasl jaas文件 #####”
[root@es83 kafka_2.12-2.3.0]# cat > ./config/zk_server_jaas.conf <<EOF
Server {
org.apache.kafka.common.security.plain.PlainLoginModule required
username=“admin”
password=“chilu@rljie”
user_kafka=“chilu@rljie”
user_producer=“chilu@rljie”;
};
EOF
[root@es83 kafka_2.12-2.3.0]# cat > ./config/kafka_server_jaas.conf <<EOF
KafkaServer {
org.apache.kafka.common.security.plain.PlainLoginModule required
username=“admin”
password=“chilu@rljie”
user_admin=“chilu@rljie”
user_producer=“chilu@rljie”
user_consumer=“chilu@rljie”;
};
KafkaClient {
org.apache.kafka.common.security.plain.PlainLoginModule required
username=“kafka”
password=“chilu@rljie”;
};
Client {
org.apache.kafka.common.security.plain.PlainLoginModule required
username=“kafka”
password=“chilu@rljie”;
};
EOF
echo “##### 修改zk和kafka的启动文件(增加SASL的环境配置) #####”
[root@es83 kafka_2.12-2.3.0]# sed -i -e ‘s/512M/4G/g’ -e ‘s#Xms4G#Xms4G -Djava.security.auth.login.config=/opt/kafka_2.12-2.3.0/config/zk_server_jaas.conf#’ ./bin/zookeeper-server-start.sh
[root@es83 kafka_2.12-2.3.0]# sed -i -e ‘s/1G/31G/g’ -e ‘s#Xms31G#Xms31G -Djava.security.auth.login.config=/opt/kafka_2.12-2.3.0/config/kafka_server_jaas.conf#’ ./bin/kafka-server-start.sh
echo “##### 将相关目录复制到其他两台节点上,并进行修改 #####”
[root@es83 kafka_2.12-2.3.0]# cd …/
[root@es83 opt]# scp -r ./zookeeper ./kafka_2.12-2.3.0 192.168.100.86:/opt/
[root@es83 opt]# scp -r ./zookeeper ./kafka_2.12-2.3.0 192.168.100.87:/opt/
[root@es83 opt]# ssh 192.168.100.86 “echo 2 > /opt/zookeeper/myid ; sed -i ‘1,3s/83/86/’ /opt/kafka_2.12-2.3.0/config/server.properties”
[root@es83 opt]# ssh 192.168.100.87 “echo 3 > /opt/zookeeper/myid ; sed -i ‘1,3s/83/87/’ /opt/kafka_2.12-2.3.0/config/server.properties”
echo “##### 后台启动zookeeper服务 #####”
[root@es83 opt]# cd ./kafka_2.12-2.3.0/
[root@es83 kafka_2.12-2.3.0]# ./bin/zookeeper-server-start.sh -daemon ./config/zookeeper.properties
[root@es83 kafka_2.12-2.3.0]# ssh 192.168.100.86 “/opt/kafka_2.12-2.3.0/bin/zookeeper-server-start.sh -daemon /opt/kafka_2.12-2.3.0/config/zookeeper.properties”
[root@es83 kafka_2.12-2.3.0]# ssh 192.168.100.87 “/opt/kafka_2.12-2.3.0/bin/zookeeper-server-start.sh -daemon /opt/kafka_2.12-2.3.0/config/zookeeper.properties”
echo “##### 后台启动kafka服务 #####”
[root@es83 kafka_2.12-2.3.0]# ./bin/kafka-server-start.sh -daemon ./config/server.properties
[root@es83 kafka_2.12-2.3.0]# ssh 192.168.100.86 “/opt/kafka_2.12-2.3.0/bin/kafka-server-start.sh -daemon /opt/kafka_2.12-2.3.0/config/server.properties”
[root@es83 kafka_2.12-2.3.0]# ssh 192.168.100.87 “/opt/kafka_2.12-2.3.0/bin/kafka-server-start.sh -daemon /opt/kafka_2.12-2.3.0/config/server.properties”
当zk和kafka服务都启动后,可以先检查下相关端口状态是否正常
[root@es83 kafka_2.12-2.3.0]# netstat -antlp | grep -E “2888|3888|2181|9092”
当集群服务一切正常后,即可在其中一台kafka节点上配置ACL访问控制权限,对生产者producer和消费者consumer的主题topic和组group设置访问权限,可以限制只允许指定的机器访问。
提示:下面的mykafka是通过/etc/hosts自定义一个IP的域名,例如:192.168.100.83 mykafka;如果写成localhost可能没有权限,执行命令后会报NoAuth;如果写成IP地址会报CONNECT !!!
echo “##### 编写配置ACL访问权限脚本 #####”
[root@es83 kafka_2.12-2.3.0]# cat > ./kfkacls.sh <<EOF
#!/bin/bash
/opt/kafka_2.12-2.3.0/bin/kafka-acls.sh --authorizer-properties zookeeper.connect=mykafka:2181 --add --allow-principal User:producer --allow-host 0.0.0.0 --operation Read --operation Write --topic elk
/opt/kafka_2.12-2.3.0/bin/kafka-acls.sh --authorizer-properties zookeeper.connect=mykafka:2181 --add --allow-principal User:producer --topic elk --producer --group chilu
/opt/kafka_2.12-2.3.0/bin/kafka-acls.sh --authorizer-properties zookeeper.connect=mykafka:2181 --add --allow-principal User:consumer --allow-host 0.0.0.0 --operation Read --operation Write --topic elk
/opt/kafka_2.12-2.3.0/bin/kafka-acls.sh --authorizer-properties zookeeper.connect=mykafka:2181 --add --allow-principal User:consumer --topic elk --consumer --group chilu
EOF
echo “##### 执行脚本 #####”
[root@es83 kafka_2.12-2.3.0]# bash ./kfkacls.sh
echo “##### 查看ACL权限列表 #####”
[root@es83 kafka_2.12-2.3.0]# ./bin/kafka-acls.sh --authorizer-properties zookeeper.connect=mykafka:2181 --list
提示:下面是交互式的命令配置
echo “##### 增加ACL访问权限 #####”
[root@es83 kafka_2.12-2.3.0]# ./bin/zookeeper-shell.sh mykafka:2181
Welcome to ZooKeeper!
JLine support is disabled
WATCHER::
WatchedEvent state:SyncConnected type:None path:null
此时可以直接在这个控制台输入命令
如ls / 查看ZK的目录
检查默认权限
getAcl /
默认所有人可以查看
添加权限命令为:(仅添加kafka主机的IP)
setAcl / ip:192.168.100.83:cdrwa,ip:192.168.100.86:cdrwa,ip:192.168.100.87:cdrwa
setAcl /kafka-acl ip:192.168.100.83:cdrwa,ip:192.168.100.86:cdrwa,ip:192.168.100.87:cdrwa
检查是否生效
getAcl /
输出:
'ip,'192.168.100.83
: cdrwa
自我介绍一下,小编13年上海交大毕业,曾经在小公司待过,也去过华为、OPPO等大厂,18年进入阿里一直到现在。
深知大多数软件测试工程师,想要提升技能,往往是自己摸索成长或者是报班学习,但对于培训机构动则几千的学费,着实压力不小。自己不成体系的自学效果低效又漫长,而且极易碰到天花板技术停滞不前!
因此收集整理了一份《2024年软件测试全套学习资料》,初衷也很简单,就是希望能够帮助到想自学提升又不知道该从何学起的朋友,同时减轻大家的负担。
既有适合小白学习的零基础资料,也有适合3年以上经验的小伙伴深入学习提升的进阶课程,基本涵盖了95%以上软件测试开发知识点,真正体系化!
由于文件比较大,这里只是将部分目录大纲截图出来,每个节点里面都包含大厂面经、学习笔记、源码讲义、实战项目、讲解视频,并且后续会持续更新
如果你觉得这些内容对你有帮助,可以添加V获取:vip1024b (备注软件测试)
一个人可以走的很快,但一群人才能走的更远。不论你是正从事IT行业的老鸟或是对IT行业感兴趣的新人,都欢迎扫码加入我们的的圈子(技术交流、学习资源、职场吐槽、大厂内推、面试辅导),让我们一起学习成长!
qB8-1712956277900)]
[外链图片转存中…(img-URNoNgEp-1712956277900)]
既有适合小白学习的零基础资料,也有适合3年以上经验的小伙伴深入学习提升的进阶课程,基本涵盖了95%以上软件测试开发知识点,真正体系化!
由于文件比较大,这里只是将部分目录大纲截图出来,每个节点里面都包含大厂面经、学习笔记、源码讲义、实战项目、讲解视频,并且后续会持续更新
如果你觉得这些内容对你有帮助,可以添加V获取:vip1024b (备注软件测试)
[外链图片转存中…(img-DwMz2IPK-1712956277901)]
一个人可以走的很快,但一群人才能走的更远。不论你是正从事IT行业的老鸟或是对IT行业感兴趣的新人,都欢迎扫码加入我们的的圈子(技术交流、学习资源、职场吐槽、大厂内推、面试辅导),让我们一起学习成长!