SkyWalking实现基于k8环境环境Dubbo微服务链路跟踪

构建基础镜像

FROM centos:7.9.2009 
ADD filebeat-7.12.1-x86_64.rpm /tmp
RUN  yum makecache fast 
RUN yum install -y /tmp/filebeat-7.12.1-x86_64.rpm vim wget tree  lrzsz gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel openssl openssl-devel iproute net-tools iotop &&  rm -rf /etc/localtime /tmp/filebeat-7.12.1-x86_64.rpm && ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime  && useradd nginx -u 2022
上传到私有仓库
nerdctl build -t  harbor.luohw.net/baseimages/centos-base:7.9.2009 . 
nerdctl push harbor.luohw.net/baseimages/centos-base:7.9.2009 

构建jdk镜像

profile文件添加java环境变量

export JAVA_HOME=/usr/local/jdk
export TOMCAT_HOME=/apps/tomcat
export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$TOMCAT_HOME/bin:$PATH
export CLASSPATH=.$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib:$JAVA_HOME/lib/tools.jar

oot@k8s-node3:~/skwalking-yml/web/pub-images/jdk-1.8.212# cat Dockerfile 
#JDK Base Image
FROM  harbor.luohw.net/baseimages/centos-base:7.9.2009 
ADD jdk-8u212-linux-x64.tar.gz /usr/local/src/
RUN ln -sv /usr/local/src/jdk1.8.0_212 /usr/local/jdk 
ADD profile /etc/profile
ENV JAVA_HOME /usr/local/jdk
ENV JRE_HOME $JAVA_HOME/jre
ENV CLASSPATH $JAVA_HOME/lib/:$JRE_HOME/lib/
ENV PATH $PATH:$JAVA_HOME/bin

运行构建

/web/pub-images/jdk-1.8.212# sh  build-command.sh 
nerdctl build -t harbor.luohw.net/baseimages/jdk-base:v8.212  .
sleep 1
nerdctl push  harbor.luohw.net/baseimages/jdk-base:v8.21

harbor.luohw.net/baseimages/jdk-base:v8.212 #镜像名

构建zookeeper镜像

上传到阿里云，镜像名registry.cn-beijing.aliyuncs.com/luohuiwen20/zookeeper:v3.4.14

创建pv，注意nfs的地址
oot@k8s-node3:~/skwalking-yml/yaml/myserver/zookeeper# vi pv/zookeeper-persistentvolume.yaml

root@k8s-node3:~/skwalking-yml/yaml/myserver/zookeeper# kubectl apply -f  pv/zookeeper-persistentvolume.yaml 
persistentvolume/zookeeper-datadir-pv-1 created
persistentvolume/zookeeper-datadir-pv-2 created
persistentvolume/zookeeper-datadir-pv-3 created

创建pvc

root@k8s-node3:~/skwalking-yml/yaml/myserver/zookeeper# kubectl apply -f  pv/zookeeper-persistentvolumeclaim.yaml 
persistentvolumeclaim/zookeeper-datadir-pvc-1 created
persistentvolumeclaim/zookeeper-datadir-pvc-2 created
persistentvolumeclaim/zookeeper-datadir-pvc-3 created

查看pvc绑定状态

root@k8s-node3:~/skwalking-yml/yaml/myserver/zookeeper# kubectl get pvc -A

构建provider 镜像

修改skywalkiing agent 指定server为 1.80:11800，修改服务名

root@k8s-node3:~/skwalking-yml/web/myserver/dubbo/consumer# vi skywalking-agent/config/agent.config 
agent.namespace=${SW_AGENT_NAMESPACE:myserver}
 The service name in UI
agent.service_name=${SW_AGENT_NAME:application-provider}

collector.backend_service=${SW_AGENT_COLLECTOR_BACKEND_SERVICES:192.168.1.80:11800}

修改dubbo-server.jar中config.properties指定k8s中zk集群地址

dubbo.registry=zookeeper://zookeeper1.myserver.svc.cluster.local:2181?backup=zookeeper2.myserver.svc.cluster.local:2181,zookeeper3.myserver.svc.cluster.local:2181
dubbo.port=20880

cat run_java.sh

su - user1 -c "java -javaagent:/skywalking-agent/skywalking-agent.jar -jar /apps/dubbo/provider/dubbo-server.jar"

构建镜像
root@k8s-node3:~/skwalking-yml/web/myserver/dubbo/provider# cat Dockerfile

#Dubbo provider
FROM harbor.luohw.net/baseimages/jdk-base:v8.212
RUN yum install file nc -y && useradd user1 -u 2000
RUN mkdir -p /apps/dubbo/provider
ADD dubbo-server.jar  /apps/dubbo/provider/
ADD run_java.sh /apps/dubbo/provider/bin/
ADD skywalking-agent/ /skywalking-agent/
RUN chown user1.user1 /apps /skywalking-agent -R
RUN chmod a+x /apps/dubbo/provider/bin/*.sh
CMD ["/apps/dubbo/provider/bin/run_java.sh"]

harbor.luohw.net/luohw/dubbo-provider:v1-2022092301-zookeeper1 

运行provider 容器
root@k8s-node3:~/skwalking-yml/yaml/myserver/dubbo/provider# kubectl apply -f provider.yaml

查看日志
kubectl describe pod -n myserver myserver-provider-deployment-5459b467cf-btqk2

root@k8s-node3:~/skwalking-yml/yaml/myserver/dubbo/provider# kubectl get svc -n myserver 
NAME                         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                                        AGE
magedu-tomcat-app1-service   NodePort    10.68.147.139   <none>        80:40080/TCP                                   29d
myserver-myapp-service       ClusterIP   None            <none>        80/TCP                                         35d
myserver-provider-spec       NodePort    10.68.20.190    <none>        8080:38800/TCP                                 13m
zookeeper                    ClusterIP   10.68.62.193    <none>        2181/TCP                                       46m
zookeeper1                   NodePort    10.68.225.111   <none>        2181:32181/TCP,2888:60008/TCP,3888:44177/TCP   46m
zookeeper2                   NodePort    10.68.102.93    <none>        2181:32182/TCP,2888:49837/TCP,3888:36502/TCP   46m
zookeeper3                   NodePort    10.68.200.61    <none>        2181:32183/TCP,2888:34218/TCP,3888:37057/TCP   46m

通过node:32181 连接查看

构建consumer

skywalking-agent目录是SkyWalking Java Agent解压后的目录，需要修改其中的配置文件指定服务名称和SkyWalking Server的地址

dubbo-client.jar是项目代码，里面的配置文件config.properties已经指定了zookerper的地址
dubbo.registry=zookeeper://zookeeper1.myserver.svc.cluster.local:2181?backup=zookeeper2.myserver.svc.cluster.local:2181,zookeeper3.myserver.svc.cluster.local:2181

root@k8s-node3:~/skwalking-yml/web/myserver/dubbo/consumer# ls 
Dockerfile  apache-skywalking-java-agent-8.8.0.tgz  build-command.sh  dubbo-client.jar  dubbo-client.jar2  run_java.sh  skywalking-agent

root@k8s-node3:~/skwalking-yml/web/myserver/dubbo/consumer# cat Dockerfile 
#Dubbo consumer
FROM harbor.luohw.net/baseimages/jdk-base:v8.212 
RUN yum install file -y
RUN mkdir -p /apps/dubbo/consumer &&  useradd user1 -u 2000
ADD run_java.sh /apps/dubbo/consumer/bin/
ADD skywalking-agent/ /skywalking-agent/
ADD dubbo-client.jar   /apps/dubbo/consumer/dubbo-client.jar
RUN chown user1.user1 /apps /skywalking-agent -R
RUN chmod a+x /apps/dubbo/consumer/bin/*.sh
CMD ["/apps/dubbo/consumer/bin/run_java.sh"]

cat run_java.sh
u - user1 -c "java -javaagent:/skywalking-agent/skywalking-agent.jar -jar /apps/dubbo/consumer/dubbo-client.jar"

运行构建镜像并上传阿里云命令

nerdctl build -t  registry.cn-beijing.aliyuncs.com/luohw/dubbo-consumer:v2.5.3-20230525-skwalking-zookeeper-k8s .
root@k8s-node3:~/skwalking-yml/web/myserver/dubbo/consumer# nerdctl push registry.cn-beijing.aliyuncs.com/luohw/dubbo-consumer:v2.5.3-20230525-skwalking-zookeeper-k8s

访问

root@k8s-node3:~/skwalking-yml/web/myserver/dubbo/consumer# kubectl get svc -n myserver  myserver-consumer-server 
NAME                       TYPE       CLUSTER-IP    EXTERNAL-IP   PORT(S)        AGE
myserver-consumer-server   NodePort   10.68.35.91   <none>        80:30001/TCP   18h

http://192.168.1.227:30001/hello?name=888

root@k8s-node3:~/skwalking-yml/web/myserver/dubbo/provider# kubectl get svc -n myserver 
NAME                         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                                        AGE
magedu-tomcat-app1-service   NodePort    10.68.147.139   <none>        80:40080/TCP                                   29d
myserver-consumer-server     NodePort    10.68.35.91     <none>        80:30001/TCP                                   18h
myserver-myapp-service       ClusterIP   None            <none>        80/TCP                                         36d
myserver-provider-spec       NodePort    10.68.20.190    <none>        8080:38800/TCP                                 19h
zookeeper                    ClusterIP   10.68.62.193    <none>        2181/TCP                                       20h
zookeeper1                   NodePort    10.68.225.111   <none>        2181:32181/TCP,2888:60008/TCP,3888:44177/TCP   20h
zookeeper2                   NodePort    10.68.102.93    <none>        2181:32182/TCP,2888:49837/TCP,3888:36502/TCP   20h
zookeeper3                   NodePort    10.68.200.61    <none>        2181:32183/TCP,2888:34218/TCP,3888:37057/TCP   20h

查看日志

访问skywalking server 端 http://192.168.1.80:8080/general 可用看到新增的application-consumer和application-provider