项目随手记
初始原因
由于项目的一次迭代,导致代码差异很大。可前端(支付宝,微信小程序)发布还需要很久的审核,而且支付宝审核是人工审核,功能上不可用是大概率不给过的。至此衍生出了需要服务需要多版本支持。
项目现状
技术栈:
1. 服务端主要技术栈springcloud+dubbo+nacos
2.部署平台为k8s集群
3.CI/CD平台为jenkins
4.网关为springcloud gateway
需要解决的问题
这里的多版本支持需要调整如下
1.duubo服务多版本
2.springcloud 的服务多版本
3.服务上线老版本不下线
4.gateway支持多版本调度
5.客户端需携带版本号(如无版本号直接在已存在的服务中进行轮训负载)
落地
直接上代码
gateway
1.自定义LoadBalance类
package com.dataspace.xxxxx.gateway.loadBalance;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.cloud.client.ServiceInstance;
import org.springframework.cloud.client.loadbalancer.*;
import org.springframework.cloud.loadbalancer.core.*;
import org.springframework.http.HttpHeaders;
import org.springframework.util.ObjectUtils;
import reactor.core.publisher.Mono;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
/**********************************************************
* @title:EnhancedRoundRobinLoadBalancer<br>
*
* @date:2024/6/5 15:25<br>
* @description:<br>
**********************************************************
*/
@Slf4j
public class EnhancedRoundRobinLoadBalancer implements ReactorServiceInstanceLoadBalancer {
private final String serviceId;
private final AtomicInteger position;
private final ObjectProvider<ServiceInstanceListSupplier> serviceInstanceListSupplierProvider;
public EnhancedRoundRobinLoadBalancer(String serviceId,
ObjectProvider<ServiceInstanceListSupplier> serviceInstanceListSupplierProvider) {
this.serviceId = serviceId;
this.position = new AtomicInteger(new Random().nextInt(1000));
this.serviceInstanceListSupplierProvider = serviceInstanceListSupplierProvider;
}
@Override
public Mono<Response<ServiceInstance>> choose(Request request) {
ServiceInstanceListSupplier supplier = this.serviceInstanceListSupplierProvider.getIfAvailable(NoopServiceInstanceListSupplier::new);
return supplier.get(request).next().map((serviceInstances) -> {
return this.processInstanceResponse(request, supplier, serviceInstances);
});
}
/**
* copy自 RoundRobinLoadBalancer
*
* @param request
* @param supplier
* @param serviceInstances {@link RoundRobinLoadBalancer}
* @return
*/
private Response<ServiceInstance> processInstanceResponse(Request request,
ServiceInstanceListSupplier supplier,
List<ServiceInstance> serviceInstances) {
String headerVersion = getRequestHeader(request, "version");
if (!ObjectUtils.isEmpty(headerVersion)){
ArrayList<ServiceInstance> versionServiceInstances = new ArrayList<>();
for (ServiceInstance serviceInstance : serviceInstances) {
Map<String, String> metadata = serviceInstance.getMetadata();
String version = metadata.get("version");
if (!ObjectUtils.isEmpty(version) && headerVersion.equals(version)){
versionServiceInstances.add(serviceInstance);
}
}
if (!ObjectUtils.isEmpty(versionServiceInstances)){
Response<ServiceInstance> serviceInstanceResponse = getRoundRobinInstance(versionServiceInstances);
if (supplier instanceof SelectedInstanceCallback && serviceInstanceResponse.hasServer()) {
((SelectedInstanceCallback) supplier).selectedServiceInstance(serviceInstanceResponse.getServer());
}
return serviceInstanceResponse;
}
}else{
//如果header版本为空的话,需要当做一个空版本,查看是否实例中存在不存在version的实例
for (ServiceInstance serviceInstance : serviceInstances) {
Map<String, String> metadata = serviceInstance.getMetadata();
ArrayList<ServiceInstance> noVersionServiceInstances = new ArrayList<>();
if(!metadata.containsKey("version")){
noVersionServiceInstances.add(serviceInstance);
}
if (!ObjectUtils.isEmpty(noVersionServiceInstances)){
Response<ServiceInstance> serviceInstanceResponse = getRoundRobinInstance(noVersionServiceInstances);
if (supplier instanceof SelectedInstanceCallback && serviceInstanceResponse.hasServer()) {
((SelectedInstanceCallback) supplier).selectedServiceInstance(serviceInstanceResponse.getServer());
}
return serviceInstanceResponse;
}
}
}
Response<ServiceInstance> serviceInstanceResponse = getRoundRobinInstance(serviceInstances);
if (supplier instanceof SelectedInstanceCallback && serviceInstanceResponse.hasServer()) {
((SelectedInstanceCallback) supplier).selectedServiceInstance(serviceInstanceResponse.getServer());
}
return serviceInstanceResponse;
}
/**
* 使用RoundRobin机制获取节点
*
* @param instances 实例
* @return {@link Response }<{@link ServiceInstance }>
* @author : baohui
*/
private Response<ServiceInstance> getRoundRobinInstance(List<ServiceInstance> instances) {
if (instances.isEmpty()) {
if (log.isWarnEnabled()) {
log.warn("不存在可用的服务: " + serviceId);
}
return new EmptyResponse();
} else if (instances.size() == 1) {
return new DefaultResponse(instances.get(0));
} else {
// 每一次计数器都自动+1,实现轮询的效果
int pos = this.position.incrementAndGet() & Integer.MAX_VALUE;
ServiceInstance instance = instances.get(pos % instances.size());
return new DefaultResponse(instance);
}
}
private String getRequestHeader(Request request, String headerField) {
HttpHeaders headers = ((RequestDataContext) request.getContext()).getClientRequest().getHeaders();
//log.info("headers:{}", headers);
return headers.getFirst(headerField);
}
}
2.创建配置文件
package com.dataspace.xxx.gateway.config;
import com.dataspace.xxxx.gateway.loadBalance.EnhancedRoundRobinLoadBalancer;
import lombok.extern.slf4j.Slf4j;
import org.springframework.cloud.client.ServiceInstance;
import org.springframework.cloud.loadbalancer.core.ReactorLoadBalancer;
import org.springframework.cloud.loadbalancer.core.ServiceInstanceListSupplier;
import org.springframework.cloud.loadbalancer.support.LoadBalancerClientFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.env.Environment;
/**********************************************************
* @title:LoadBalanceConfig<br>
*>
* @date:2024/6/5 20:21<br>
* @description:<br>
**********************************************************
*/
@Configuration(proxyBeanMethods = false)
@Slf4j
public class LoadBalanceConfig {
@Bean
public ReactorLoadBalancer<ServiceInstance> customLoadBalancer(Environment environment, LoadBalancerClientFactory loadBalancerClientFactory) {
String name = environment.getProperty(LoadBalancerClientFactory.PROPERTY_NAME);
return new EnhancedRoundRobinLoadBalancer(name, loadBalancerClientFactory.getLazyProvider(name, ServiceInstanceListSupplier.class));
}
}
3启动项指定配置类
@LoadBalancerClients(defaultConfiguration = LoadBalanceConfig.class)
public class Bootstrap {
public static void main(String[] args) {
//System.setProperty("csp.sentinel.app.type", "1");
SpringApplication.run(Bootstrap.class, args);
log.info("Server path:" + System.getProperty("user.dir"));
log.info("maxMemory:" + Runtime.getRuntime().maxMemory());
log.info("totalMemory:" + Runtime.getRuntime().totalMemory());
log.info("freeMemory:" + Runtime.getRuntime().freeMemory());
log.info("------------------------Server startup------------------------");
}
}
springcloud改造
1.nacos增加version相关配置,这里就直接使用version.yaml
增加的vesion配置文件中springcloud 和dubbo分别指定version
xxx:
xxx-iot:
xxx-iot-setting-api:
metadata:
version: 1.0.0
xxx-iot-default:
metadata:
version: 1.0.0
dubbo:
version: 1.0.0
在bootstrap.yaml中指定metadata中的version来保证在nacos注册元数据的时候携带到nacos【携带到nacos上之后,gateway才可以在自定义调度策略中拿得到】 ,由于项目中使用的是dubbo服务,并且提供了健康检查的api入口,所以metadata中的配置也需要在dubbo服务中的bootstrap.yaml同样配置一份。这也就是为什么veresion.yaml中的xxx-iot-default也会有metadata的配置项。
dubbo改造
dubbo服务中需要在实现类添加@DubboService注解,并指定version配置项
构建流程改造
pipeline {
agent any
environment
{
GROUP = "xxx-park"
MODULE = "xxx-rpc"
DWR = "10.1.4.1:180"
TAG = "latest"
VERSION= "1.0.0"
}
stages {
stage('Docker Build') {
steps {
echo "1.Build Docker Image Stage"
sh "cd ${JENKINS_HOME}/workspace/${GROUP}/${MODULE}/${JOB_BASE_NAME}/target/build/ && docker build -t ${JOB_BASE_NAME}:${TAG} ."
}
}
stage('Docker Push') {
steps {
echo "2.Push Docker Image Stage"
sh "docker login -u admin -p xxx ${DWR}"
sh "docker tag ${JOB_BASE_NAME}:${TAG} ${DWR}/xxx-parking/${GROUP}/${JOB_BASE_NAME}:${TAG}"
sh "docker push ${DWR}/xxx-parking/${GROUP}/${JOB_BASE_NAME}:${TAG}"
sh "docker rmi ${DWR}/xxx-parking/${GROUP}/${JOB_BASE_NAME}:${TAG}"
sh "docker rmi ${JOB_BASE_NAME}:${TAG}"
}
}
stage('k8s deployment') {
steps {
echo "3. k8s generate deployment yaml"
sh """
cd /opt/k8sDeployment/
cp -f deployment.yaml ${JOB_BASE_NAME}.${GROUP}.deployment.yaml
sed -i 's/{JOB-BASE-NAME}/${JOB_BASE_NAME}/g' ${JOB_BASE_NAME}.${GROUP}.deployment.yaml
sed -i 's/{GROUP-NAME}/${GROUP}/g' ${JOB_BASE_NAME}.${GROUP}.deployment.yaml
sed -i 's/{NAME-SPACE}/${GROUP}/g' ${JOB_BASE_NAME}.${GROUP}.deployment.yaml
sed -i 's/{VERSION}/${VERSION}/g' ${JOB_BASE_NAME}.${GROUP}.deployment.yaml
"""
}
}
stage('k8s Push') {
steps {
echo "4.k8s Stage"
sh "cd /opt/k8sDeployment/ && kubectl delete -f ${JOB_BASE_NAME}.${GROUP}.deployment.yaml && kubectl apply -f ${JOB_BASE_NAME}.${GROUP}.deployment.yaml --record"
}
}
}
}
deployment.yaml模板改造
kind: Deployment
apiVersion: apps/v1
metadata:
name: {JOB-BASE-NAME}-{VERSION}
namespace: {NAME-SPACE}
labels:
app: {JOB-BASE-NAME}
spec:
replicas: 1
selector:
matchLabels:
app: {JOB-BASE-NAME}
version: {VERSION}
template:
metadata:
labels:
app: {JOB-BASE-NAME}
version: {VERSION}
spec:
volumes:
- name: cat
hostPath:
path: /data/appdatas/cat/
type: ''
- name: elknfs
nfs:
path: /opt/elknfs/
server: 10.1.4.5
containers:
- name: {JOB-BASE-NAME}
image: >-
10.1.4.1:180/xxxxx{GROUP-NAME}/{JOB-BASE-NAME}:latest
lifecycle:
preStop:
exec:
command: ["/bin/sh","-c","sleep 70"] ##延迟重启的钩子
resources:
limits:
# cpu: 800m
memory: 800Mi
requests:
# cpu: 300m
memory: 800Mi
imagePullPolicy: Always
volumeMounts:
- name: cat
mountPath: /data/appdatas/cat/
- name: elknfs
mountPath: /opt/logs/
restartPolicy: Always
terminationGracePeriodSeconds: 70 ##延迟重启的钩子
dnsPolicy: ClusterFirst
securityContext: {}
imagePullSecrets:
- name: kuboard-harbor
schedulerName: default-scheduler
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 25%
maxSurge: 25%
revisionHistoryLimit: 10
progressDeadlineSeconds: 600
至此所有的相关多版本支持的告一段落