kubesphere+kubernetes搭建生产环境高可用集群(四-1)

k8s常用中间件的部署

Centos7安装部署nginx

参考链接

nginx官网地址:https://nginx.org/en/download.html

rpm包操作系统版本选择地址:http://nginx.org/packages/

rpm包centos7操作系统官方下载地址:http://nginx.org/packages/centos/7/x86_64/RPMS/

这里根据官网提供的主线版本自行到rpm包连接地址中挑选下载

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-IsSMJWM9-1656494597434)(./images/nginx_1.jpg)]

部署步骤

#下载rpm离线包,这里选择官方的最新稳定版1.20.2
[root@nginx-0 ~]# wget http://nginx.org/packages/centos/7/x86_64/RPMS/nginx-1.20.2-1.el7.ngx.x86_64.rpm
#安装
[root@nginx-0 ~]# rpm -ivh nginx-1.20.2-1.el7.ngx.x86_64.rpm
warning: nginx-1.20.2-1.el7.ngx.x86_64.rpm: Header V4 RSA/SHA1 Signature, key ID 7bd9bf62: NOKEY
Preparing...                          ################################# [100%]
Updating / installing...
   1:nginx-1:1.20.2-1.el7.ngx         ################################# [100%]
#查看已安装的nginx
[root@nginx-0 ~]# rpm -qa | grep nginx
nginx-1.20.2-1.el7.ngx.x86_64
#更据需要修改/etc/nginx/nginx.conf(主要用于自定义nginx服务器的各种配置等)
[root@nginx-0 ~]# vi /etc/nginx/nginx.conf
示例配置见附录
#更据需要修改/etc/nginx/conf.d/***.conf(主要用于自定义nginx的各种转发规则等)
[root@nginx-0 ~]# vi /etc/nginx/conf.d/***.conf
示例配置见附录
[root@nginx-0 ~]# 
#检测nginx配置文件
[root@nginx-0 ~]# nginx -t
#启动nginx,并配置开机自启
[root@nginx-0 ~]# systemctl start nginx && systemctl enable nginx
#热加载配置文件(请确保事先使用nginx -t检测配置文件无误)
[root@nginx-0 ~]# nginx -s reload

k8s部署mysql5.7

部署步骤

创建mysql相关部署文件
  1. 创建mysql存储pvc yaml文件
[root@k8s-master-0 ~]# vi mysql-pvc.yaml
  • mysql-pv.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: mysql-pvc
  namespace: test											#注意修改命名空间
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
  storageClassName: glusterfs
  1. 创建mysql配置configmap yaml文件
[root@k8s-master-0 ~]# vi mysql-config.yaml
  • mysql-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mysql-cm
  namespace: test											#注意修改命名空间
data:
  mysqld.cnf: |
    [mysqld]
    pid-file        = /var/run/mysqld/mysqld.pid
    socket          = /var/run/mysqld/mysqld.sock
    datadir         = /var/lib/mysql
    bind-address    = 0.0.0.0
    port = 3306
    log-bin = mysql-bin
    server-id = 1
  1. 创建mysql服务service yaml文件
[root@k8s-master-0 ~]# vi mysql-service.yaml
  • mysql-service.yaml
apiVersion: v1
kind: Service
metadata:
  name: mysql
  namespace: test											#注意修改命名空间
spec:
  type: NodePort
  ports:
  - port: 3306
    targetPort: 3306
    nodePort: 30850
  selector:
    app: mysql
  1. 创建mysql配置副本Deployment yaml文件
[root@k8s-master-0 ~]# vi mysql.yaml
  • mysql.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: mysql
  namespace: test													#注意修改命名空间
spec:
  selector:
    matchLabels:
      app: mysql
  template:
    metadata:
      labels:
        app: mysql
    spec:
      containers:
      - name: mysql
        image: mysql:5.7.32
        env:
        - name: MYSQL_ROOT_PASSWORD
          value: "root@my.123" 										#数据库root的密码
        ports:
        - containerPort: 3306
        volumeMounts:
        - name: mysql-persistent-storage
          mountPath: /var/lib/mysql
        volumeMounts:
        - name: mysql-config
          mountPath: /etc/mysql/mysql.conf.d/mysqld.cnf
          subPath: mysqld.cnf
      volumes:
      - name: mysql-persistent-storage
        persistentVolumeClaim:
          claimName: mysql-pvc
      - name: mysql-config
        configMap:
          name: mysql-cm
部署mysql
[root@k8s-master01 ~]# kubectl apply -f mysql-pvc.yaml
[root@k8s-master01 ~]# kubectl apply -f mysql-config.yaml
[root@k8s-master01 ~]# kubectl apply -f mysql-service.yaml
[root@k8s-master01 ~]# kubectl apply -f mysql.yaml
验证mysql
NAME                     READY   STATUS    RESTARTS   AGE
mysql-589dcf6597-5ps6x   1/1     Running   0          8m3s
#进入pod登录验证
[root@k8s-master01 ~]# kubectl exec -it mysql-589dcf6597-5ps6x /bin/bash -n test
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
root@mysql-589dcf6597-5ps6x:/# mysql -u root -p
Enter password:

mysql> ALTER USER 'root'@'localhost' IDENTIFIED BY 'P@ssword-123';		#修改root密码
Query OK, 0 rows affected (0.00 sec)

k8s部署nacos集群

参考链接

部署参考:https://github.com/nacos-group/nacos-k8s

sql文件参考:https://github.com/alibaba/nacos/blob/develop/distribution/conf/nacos-mysql.sql

部署步骤

拉取项目代码
#拉取nacos代码,见附录
[root@k8s-master-0 ~]# git clone https://github.com/nacos-group/nacos-k8s.git
#拉取nacos的初始化数据库sql文件,见附录
[root@k8s-master-0 ~]# https://github.com/alibaba/nacos/blob/develop/distribution/conf/nacos-mysql.sql
创建nacos所需数据库
#登录数据库
[root@k8s-master01 mysql]# kubectl exec -it mysql-589dcf6597-5ps6x /bin/bash -n test
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
root@mysql-589dcf6597-5ps6x:/# mysql -u root -p
Enter password:
#创建数据库
mysql> CREATE DATABASE  IF NOT EXISTS `nacos_dev` DEFAULT CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_unicode_ci;
#创建nacos用户
mysql> GRANT ALL PRIVILEGES ON  nacos.* to nacos@'%' IDENTIFIED BY 'nacos';
#刷新权限
mysql> FLUSH PRIVILEGES;
#导入nacos数据库
mysql> use nacos;
Database changed

#查看mysql的连接端口
[root@k8s-master01 mysql]# kubectl get svc -n test
NAME                                                     TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
mysql                                                    NodePort    10.233.20.110   <none>        3306:30850/TCP   47m
#使用mysql连接工具连接数据库并导入上方sql,因为mysql service的类型为nodeport,所以连接地址为任一k8s集群的节点ip地址,端口为30850

#回到命令行查看nacos数据表
mysql> show tables;
+----------------------+
| Tables_in_nacos      |
+----------------------+
| config_info          |
| config_info_aggr     |
| config_info_beta     |
| config_info_tag      |
| config_tags_relation |
| group_capacity       |
| his_config_info      |
| permissions          |
| roles                |
| tenant_capacity      |
| tenant_info          |
| users                |
+----------------------+
12 rows in set (0.00 sec)
修改nacos部署yaml文件
#进入项目目录
[root@k8s-master-0 ~]# cd nacos-k8s/deploy/nacos
[root@k8s-master-0 ~]# mv nacos-pvc-nfs.yaml nacos.yaml
#修改部署文件
[root@k8s-master-0 nacos]# vi nacos.yaml
  • nacos.yaml
---											#修改8848端口为nodeport形式,以便集群外部访问
apiVersion: v1
kind: Service
metadata:
  name: nacos-server
  namespace: test							#增加命名空间
  labels:
    app: nacos
  annotations:
    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
spec:
  type: NodePort
  ports:
    - port: 8848
      name: server
      targetPort: 8848
      nodePort: 30848
  selector:
    app: nacos
---
apiVersion: v1
kind: Service
metadata:
  name: nacos-headless
  namespace: test							#增加命名空间
  labels:
    app: nacos
  annotations:
    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
spec:
  ports:
    - port: 8848
      name: server
      targetPort: 8848
    - port: 9848
      name: client-rpc
      targetPort: 9848
    - port: 9849
      name: raft-rpc
      targetPort: 9849
    ## 兼容1.4.x版本的选举端口
    - port: 7848
      name: old-raft-rpc
      targetPort: 7848
  clusterIP: None
  selector:
    app: nacos
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: nacos-cm
  namespace: test							#增加命名空间
data:
  mysql.host: "mysql.test.svc.cluster.local"						#增加mysql连接地址
  mysql.db.name: "nacos_devtest"
  mysql.port: "3306"
  mysql.user: "nacos"
  mysql.password: "nacos"
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: nacos
  namespace: test							#增加命名空间
spec:
  serviceName: nacos-headless
  replicas: 3
  template:
    metadata:
      labels:
        app: nacos
      annotations:
        pod.alpha.kubernetes.io/initialized: "true"
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: "app"
                    operator: In
                    values:
                      - nacos
              topologyKey: "kubernetes.io/hostname"
      serviceAccountName: nfs-client-provisioner
      initContainers:
        - name: peer-finder-plugin-install
          image: nacos/nacos-peer-finder-plugin:1.1
          imagePullPolicy: Always
          volumeMounts:
            - mountPath: /home/nacos/plugins/peer-finder
              name: data
              subPath: peer-finder
      containers:
        - name: nacos
          imagePullPolicy: Always
          image: nacos/nacos-server:latest
          resources:
            requests:
              memory: "2Gi"
              cpu: "500m"
          ports:
            - containerPort: 8848
              name: client-port
            - containerPort: 9848
              name: client-rpc
            - containerPort: 9849
              name: raft-rpc
            - containerPort: 7848
              name: old-raft-rpc
          env:
            - name: NACOS_REPLICAS
              value: "3"
            - name: SERVICE_NAME
              value: "nacos-headless"
            - name: DOMAIN_NAME
              value: "cluster.local"						#修改k8s集群地址,可通过cat /etc/kubernetes/kubelet.conf查看对应字段:contexts/- context/cluster
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.namespace
            - name: MYSQL_SERVICE_HOST						#增加获取mysql连接地址的环境变量
              valueFrom:
                configMapKeyRef:
                  name: nacos-cm
                  key: mysql.host
            - name: MYSQL_SERVICE_DB_NAME
              valueFrom:
                configMapKeyRef:
                  name: nacos-cm
                  key: mysql.db.name
            - name: MYSQL_SERVICE_PORT
              valueFrom:
                configMapKeyRef:
                  name: nacos-cm
                  key: mysql.port
            - name: MYSQL_SERVICE_USER
              valueFrom:
                configMapKeyRef:
                  name: nacos-cm
                  key: mysql.user
            - name: MYSQL_SERVICE_PASSWORD
              valueFrom:
                configMapKeyRef:
                  name: nacos-cm
                  key: mysql.password
            - name: NACOS_SERVER_PORT
              value: "8848"
            - name: NACOS_APPLICATION_PORT
              value: "8848"
            - name: PREFER_HOST_MODE
              value: "hostname"
          volumeMounts:
            - name: data
              mountPath: /home/nacos/plugins/peer-finder
              subPath: peer-finder
            - name: data
              mountPath: /home/nacos/data
              subPath: data
            - name: data
              mountPath: /home/nacos/logs
              subPath: logs
  volumeClaimTemplates:
    - metadata:
        name: data
        namespace: test								#增加命名空间
        annotations:
          volume.beta.kubernetes.io/storage-class: "managed-nfs-storage"
      spec:
        accessModes: [ "ReadWriteMany" ]
        storageClassName: "glusterfs"					#增加storageClass选择器,以便自动创建pvc
        resources:
          requests:
            storage: 20Gi
  selector:
    matchLabels:
      app: nacos
部署集群
[root@k8s-master-0 ~]# kubectl apply  -f  nacos.yaml
验证集群
#查看nacos pod
[root@k8s-master-0 ~]# kubectl get pod -n test | grep nacos
nacos-0                                               1/1     Running   0          120d
nacos-1                                               1/1     Running   0          120d
nacos-2                                               1/1     Running   0          120d
#查看nacos svc
[root@k8s-master01 ~]# kubectl get svc -n test | grep nacos
NAME                               TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                               AGE
nacos-headless                     ClusterIP   None            <none>        8848/TCP,9848/TCP,9849/TCP,7848/TCP   216d
nacos-server                       NodePort    10.233.20.35    <none>        8848:30848/TCP                        213d
#访问任意节点ip+30848(nodeport端口)进行验证

k8s安装redis三主三从集群

部署步骤

创建redis副本文件
[root@k8s-master-0 ~]# vi redis-statefulset.yaml
  • redis-statefulset.yaml
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: redis-cluster-cm
  namespace: test
data:
  redis-conf: |
    appendonly yes														#开启AOF模式
    protected-mode no													#关闭protected-mode模式,此时外部网络可以直接访问
    cluster-enabled yes													#开启集群模式
    cluster-config-file /data/nodes.conf								#Redis集群节点的集群配置文件
    cluster-node-timeout 5000											#指节点在失败状态下必须不可到达的毫秒数。大多数其他内部时间限制是节点超时的倍数
    dir /data															#数据存储目录
    port 6379
    requirepass redis@123.com											#redis密码,自定义
    masterauth redis@123.com											#如果master是密码保护的,在启动复制同步进程之前,可以告诉奴隶进行身份验证,否则主人将拒绝奴隶请求。
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
  namespace: szxc
  labels:
    app: redis
spec:
  serviceName: redis-headless
  replicas: 6
  selector:
    matchLabels:
      app: redis
  template:
    metadata:
      labels:
        app: redis
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - redis
              topologyKey: kubernetes.io/hostname
      containers:
      - name: redis
        image: redis:6.2.5
        command:
          - "redis-server"
        args:
          - "/etc/redis/redis.conf"
          - "--protected-mode"
          - "no"
          - "--cluster-announce-ip"
          - "$(POD_IP)"
        env:
          - name: POD_IP
            valueFrom:
              fieldRef:
                fieldPath: status.podIP
        ports:
            - name: redis
              containerPort: 6379
              protocol: "TCP"
            - name: cluster
              containerPort: 16379
              protocol: "TCP"
        volumeMounts:
          - name: redis-conf
            mountPath: /etc/redis        
          - name: redis-data
            mountPath: /data
      volumes:
      - name: redis-conf
        configMap:
          name: redis-cluster-cm
          items:
            - key: redis-conf
              path: redis.conf
  volumeClaimTemplates:
    - metadata:
        name: redis-data
        namespace: test
      spec:
        accessModes: [ "ReadWriteOnce" ]
        storageClassName: "glusterfs"										#注意修改为自己的storageClass
        resources:
          requests:
            storage: 10Gi													#pvc容量,自定义

---
apiVersion: v1
kind: Service
metadata:
  name: redis-headless
  namespace: test
  labels:
    app: redis
spec:
  type: NodePort
  ports:
    - name: redis-port
      port: 6379
      targetPort: 6379
      nodePort: 30849														#nodeport端口自定义
  selector:
    app: redis
部署并配置集群
#部署
[root@k8s-master-0 ~]# kubectl apply -f redis-statefulset.yaml
#配置集群
#自动配置3个master,3个slave节点的集群,-a指定密码
[root@k8s-master-0 ~]# kubectl exec -it redis-0 -n test -- redis-cli -a redis@123.com --cluster create --cluster-replicas 1 $(kubectl get pods -n test -l app=redis -o jsonpath='{range.items[*]}{.status.podIP}:6379 {end}')
验证集群
#对redis集群进行验证
[root@k8s-master-0 ~]# kubectl exec -it redis-0 -n test -- redis-cli -a redis@123.com --cluster check $(kubectl get pods -n test -l app=redis -o jsonpath='{range.items[0]}{.status.podIP}:6379{end}')
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.233.67.76:6379 (b8e966ed...) -> 286 keys | 5461 slots | 1 slaves.
10.233.94.207:6379 (31d925a7...) -> 263 keys | 5462 slots | 1 slaves.
10.233.98.106:6379 (11b42330...) -> 275 keys | 5461 slots | 1 slaves.
[OK] 824 keys in 3 masters.
0.05 keys per slot on average.
>>> Performing Cluster Check (using node 10.233.67.76:6379)
M: b8e966ed2e00d2c9fb24ebdd409fd7eef90cbb11 10.233.67.76:6379
   slots:[0-5460] (5461 slots) master
   1 additional replica(s)
S: 35d46c8a708f234b73d647d1a800e52a620f2fbd 10.233.82.103:6379
   slots: (0 slots) slave
   replicates 31d925a76d8276ec6b1735a65b3e8d238ca5b63f
S: 8bbb3e01a5d9087327ff5ea2ee57e87c772c5ba9 10.233.94.205:6379
   slots: (0 slots) slave
   replicates 11b42330416a1fe3da01ff696b573e35f95be0c6
M: 31d925a76d8276ec6b1735a65b3e8d238ca5b63f 10.233.94.207:6379
   slots:[5461-10922] (5462 slots) master
   1 additional replica(s)
M: 11b42330416a1fe3da01ff696b573e35f95be0c6 10.233.98.106:6379
   slots:[10923-16383] (5461 slots) master
   1 additional replica(s)
S: 121cdf362ae961cb18df31588df56e2e0cd42f10 10.233.123.249:6379
   slots: (0 slots) slave
   replicates b8e966ed2e00d2c9fb24ebdd409fd7eef90cbb11
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.

[root@k8s-master-0 ~]# kubectl get pod -n szxc | grep redis
redis-0                                               1/1     Running   0          130d
redis-1                                               1/1     Running   0          130d
redis-2                                               1/1     Running   0          130d
redis-3                                               1/1     Running   0          130d
redis-4                                               1/1     Running   0          130d
redis-5                                               1/1     Running   0          130d

k8s部署rokcetmq集群

参考链接

部署参考地址:https://github.com/apache/rocketmq-operator

部署步骤

部署rockermq相关组件的crd资源
#拉取rocketmq部署文件
[root@k8s-master-0 ~]# git clone -b 0.2.1 https://github.com/apache/rocketmq-operator.git
[root@k8s-master-0 ~]# cd rocketmq-operator
#查看crd部署脚本
[root@k8s-master-0 rocketmq-operator]# cat install-operator.sh
#!/bin/bash

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

kubectl create -f deploy/crds/rocketmq_v1alpha1_broker_crd.yaml
kubectl create -f deploy/crds/rocketmq_v1alpha1_nameservice_crd.yaml
kubectl create -f deploy/crds/rocketmq_v1alpha1_topictransfer_crd.yaml
kubectl create -f deploy/service_account.yaml
kubectl create -f deploy/role.yaml
kubectl create -f deploy/role_binding.yaml
kubectl create -f deploy/operator.yaml

#kubectl create -f example/rocketmq_v1alpha1_rocketmq_cluster.yaml

#为所有的yaml文件增加命名空间
[root@k8s-master-0 rocketmq-operator]# vi deploy/crds/rocketmq_v1alpha1_broker_crd.yaml
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: brokers.rocketmq.apache.org
  namespace: test
  
[root@k8s-master-0 rocketmq-operator]# vi deploy/crds/rocketmq_v1alpha1_nameservice_crd.yaml
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: nameservices.rocketmq.apache.org
  namespace: test
  
[root@k8s-master-0 rocketmq-operator]# deploy/crds/rocketmq_v1alpha1_consoles_crd.yaml
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: consoles.rocketmq.apache.org
  namespace: test

[root@k8s-master-0 rocketmq-operator]# vi deploy/crds/rocketmq_v1alpha1_topictransfer_crd.yaml
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: topictransfers.rocketmq.apache.org
  namespace: test

[root@k8s-master-0 rocketmq-operator]# vi deploy/service_account.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: rocketmq-operator
  namespace: test

[root@k8s-master-0 rocketmq-operator]# vi deploy/role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  creationTimestamp: null
  name: rocketmq-operator
  namespace: test

[root@k8s-master-0 rocketmq-operator]# vi deploy/role_binding.yaml
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rocketmq-operator
  namespace: test
  
[root@k8s-master-0 rocketmq-operator]# vi deploy/operator.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: rocketmq-operator
  namespace: test

#创建rockermq Operator
[root@k8s-master-0 rocketmq-operator]# sh install-operator.sh
Warning: apiextensions.k8s.io/v1beta1 CustomResourceDefinition is deprecated in v1.16+, unavailable in v1.22+; use apiextensions.k8s.io/v1 CustomResourceDefinition
customresourcedefinition.apiextensions.k8s.io/brokers.rocketmq.apache.org created
Warning: apiextensions.k8s.io/v1beta1 CustomResourceDefinition is deprecated in v1.16+, unavailable in v1.22+; use apiextensions.k8s.io/v1 CustomResourceDefinition
customresourcedefinition.apiextensions.k8s.io/nameservices.rocketmq.apache.org created
Warning: apiextensions.k8s.io/v1beta1 CustomResourceDefinition is deprecated in v1.16+, unavailable in v1.22+; use apiextensions.k8s.io/v1 CustomResourceDefinition
customresourcedefinition.apiextensions.k8s.io/topictransfers.rocketmq.apache.org created
serviceaccount/rocketmq-operator created
role.rbac.authorization.k8s.io/rocketmq-operator created
rolebinding.rbac.authorization.k8s.io/rocketmq-operator created
deployment.apps/rocketmq-operator created
#查看rockermq Operator
[root@k8s-master01 rocketmq-operator]# kubectl get pod -n test
NAME                               READY   STATUS    RESTARTS   AGE
rocketmq-operator-867c4955-dhgzh   1/1     Running   0          7m40s
配置rockermq集群部署 yaml文件
[root@k8s-master-0 rocketmq-operator]# vi example/rocketmq_v1alpha1_rocketmq_cluster.yaml
  • rocketmq_v1alpha1_rocketmq_cluster.yaml
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: v1
kind: ConfigMap
metadata:
  name: broker-config
  namespace: test											#添加命名空间
data:
  # BROKER_MEM sets the broker JVM, if set to "" then Xms = Xmx = max(min(1/2 ram, 1024MB), min(1/4 ram, 8GB))
  BROKER_MEM: " -Xms2g -Xmx2g -Xmn1g "
  broker-common.conf: |
    # brokerClusterName, brokerName, brokerId are automatically generated by the operator and do not set it manually!!!
    deleteWhen=04
    fileReservedTime=48
    flushDiskType=ASYNC_FLUSH
    # set brokerRole to ASYNC_MASTER or SYNC_MASTER. DO NOT set to SLAVE because the replica instance will automatically be set!!!
    brokerRole=ASYNC_MASTER

---
apiVersion: rocketmq.apache.org/v1alpha1
kind: Broker
metadata:
  # name of broker cluster
  name: broker
  namespace: test											#添加命名空间
spec:
  # size is the number of the broker cluster, each broker cluster contains a master broker and [replicaPerGroup] replica brokers.
  size: 1
  # nameServers is the [ip:port] list of name service
  nameServers: ""											#无需填写自动获取
  # replicaPerGroup is the number of each broker cluster
  replicaPerGroup: 1
  # brokerImage is the customized docker image repo of the RocketMQ broker
  brokerImage: apacherocketmq/rocketmq-broker:4.5.0-alpine-operator-0.3.0
  # imagePullPolicy is the image pull policy
  imagePullPolicy: Always
  # resources describes the compute resource requirements and limits
  resources:
    requests:
      memory: "2048Mi"
      cpu: "250m"
    limits:
      memory: "12288Mi"
      cpu: "500m"
  # allowRestart defines whether allow pod restart
  allowRestart: true
  # storageMode can be EmptyDir, HostPath, StorageClass
  storageMode: StorageClass
  # hostPath is the local path to store data
  hostPath: /data/rocketmq/broker
  # scalePodName is [Broker name]-[broker group number]-master-0
  scalePodName: broker-0-master-0
  # env defines custom env, e.g. BROKER_MEM
  env:
    - name: BROKER_MEM
      valueFrom:
        configMapKeyRef:
          name: broker-config
          key: BROKER_MEM
  # volumes defines the broker.conf
  volumes:
    - name: broker-config
      configMap:
        name: broker-config
        items:
          - key: broker-common.conf
            path: broker-common.conf
  # volumeClaimTemplates defines the storageClass
  volumeClaimTemplates:
    - metadata:
        name: broker-storage
      spec:
        accessModes:
          - ReadWriteOnce
        storageClassName: "glusterfs"							#修改为自己的storageClass
        resources:
          requests:
            storage: 8Gi										#可自定义pvc容量
---
apiVersion: rocketmq.apache.org/v1alpha1
kind: NameService
metadata:
  name: name-service
  namespace: test												#添加命名空间
spec:
  # size is the the name service instance number of the name service cluster
  size: 1
  # nameServiceImage is the customized docker image repo of the RocketMQ name service
  nameServiceImage: apacherocketmq/rocketmq-nameserver:4.5.0-alpine-operator-0.3.0
  # imagePullPolicy is the image pull policy
  imagePullPolicy: Always
  # hostNetwork can be true or false
  hostNetwork: true
  #  Set DNS policy for the pod.
  #  Defaults to "ClusterFirst".
  #  Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'.
  #  DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy.
  #  To have DNS options set along with hostNetwork, you have to specify DNS policy
  #  explicitly to 'ClusterFirstWithHostNet'.
  dnsPolicy: ClusterFirstWithHostNet
  # resources describes the compute resource requirements and limits
  resources:
    requests:
      memory: "512Mi"
      cpu: "250m"
    limits:
      memory: "1024Mi"
      cpu: "500m"
  # storageMode can be EmptyDir, HostPath, StorageClass
  storageMode: StorageClass
  # hostPath is the local path to store data
  hostPath: /data/rocketmq/nameserver
  # volumeClaimTemplates defines the storageClass
  volumeClaimTemplates:
    - metadata:
        name: namesrv-storage
      spec:
        accessModes:
          - ReadWriteOnce
        storageClassName: rocketmq-storage
        storageClassName: "glusterfs"								#修改为自己的storageClass
        resources:
          requests:
            storage: 1Gi											#可自定义pvc容量

---
apiVersion: rocketmq.apache.org/v1alpha1
kind: Console
metadata:
  name: console
  namespace: test													#添加命名空间
spec:
  # nameServers is the [ip:port] list of name service
  nameServers: ""
  # consoleDeployment define the console deployment
  consoleDeployment:
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      labels:
        app: rocketmq-console
    spec:
      replicas: 1
      selector:
        matchLabels:
          app: rocketmq-console
      template:
        metadata:
          labels:
            app: rocketmq-console
        spec:
          containers:
            - name: console
              image: apacherocketmq/rocketmq-console:2.0.0
              ports:
                - containerPort: 8080
配置rockermq集群service yaml文件
[root@k8s-master-0 rocketmq-operator]# vi example/rocketmq_v1alpha1_cluster_service.yaml

  • rocketmq_v1alpha1_cluster_service.yaml
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: v1
kind: Service
metadata:
  name: console-service
  namespace: test									#修改命名空间
  labels:
    app: rocketmq-console
spec:
  type: NodePort
  selector:
    app: rocketmq-console
  ports:
    - port: 8080
      targetPort: 8080
      protocol: TCP
      nodePort: 30849								#注意修改nodeport端口
#---
#apiVersion: v1										#如果集群外的服务需要使用rockermq可以取消此service注释
#kind: Service
#metadata:
#  name: name-server-service
#  namespace: test
#spec:
#  type: NodePort
#  selector:
#    name_service_cr: name-service
#  ports:
#    - port: 9876
#      targetPort: 9876
#      # use this port to access the name server cluster
#      nodePort: 30001
#---

安装集群
#安装rockermq集群
[root@k8s-master01 rocketmq-operator]# kubectl apply -f example/rocketmq_v1alpha1_rocketmq_cluster.yaml
#安装rockermq集群service
[root@k8s-master01 rocketmq-operator]# kubectl apply -f example/rocketmq_v1alpha1_cluster_service.yaml

验证集群
#查看rockermq集群pod
[root@k8s-master01 rocketmq-operator]# kubectl get pod -n test
NAME                               READY   STATUS    RESTARTS   AGE
broker-0-master-0                  1/1     Running   0          13m
broker-0-replica-1-0               1/1     Running   0          13m
console-fd66cc958-t7twh            1/1     Running   0          13m
name-service-0                     1/1     Running   0          13m
rocketmq-operator-867c4955-dhgzh   1/1     Running   0          13m
#查看rockermq集群svc
[root@k8s-master01 rocketmq-operator]# kubectl get svc -n test
NAME                                                     TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
console-service                                          NodePort    10.233.46.31    <none>        8080:30849/TCP   12m
glusterfs-dynamic-0fc569c2-e2fe-4ef1-be6a-b3d56f1058d1   ClusterIP   10.233.36.32    <none>        1/TCP            15h
glusterfs-dynamic-ceed9eef-7264-45c5-b727-4afc64ab34ab   ClusterIP   10.233.60.200   <none>        1/TCP            16h
glusterfs-dynamic-e7c85113-e164-4423-a5ad-99f7c3f8b0f1   ClusterIP   10.233.24.21    <none>        1/TCP            15h
rocketmq-operator                                        ClusterIP   10.233.61.255   <none>        8383/TCP         13m
#查看rockermq集群pvc
[root@k8s-master01 rocketmq-operator]# kubectl get pvc -n test
NAME                                  STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
broker-storage-broker-0-master-0      Bound    pvc-e7c85113-e164-4423-a5ad-99f7c3f8b0f1   8Gi        RWO            glusterfs      15h
broker-storage-broker-0-replica-1-0   Bound    pvc-0fc569c2-e2fe-4ef1-be6a-b3d56f1058d1   8Gi        RWO            glusterfs      15h
namesrv-storage-name-service-0        Bound    pvc-ceed9eef-7264-45c5-b727-4afc64ab34ab   10Gi       RWO            glusterfs      16h
#浏览器访问nodeip+console-service的nodeport端口进行验证

k8s部署xxl-job

参考链接

官方地址:https://www.xuxueli.com/xxl-job/

xxl-job sql文件:https://github.com/xuxueli/xxl-job/blob/master/doc/db/tables_xxl_job.sql

xxl-job github源码仓库地址:https://github.com/xuxueli/xxl-job

部署步骤

创建xxl-job所需数据库
#数据库sql参考地址包含创建数据库步骤,见附录
https://github.com/xuxueli/xxl-job/blob/master/doc/db/tables_xxl_job.sql
#查看mysql的连接端口
[root@k8s-master-1 ~]# kubectl get svc -n test
NAME                                                     TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
mysql                                                    NodePort    10.233.20.110   <none>        3306:30850/TCP   47m
#使用mysql连接工具连接数据库并导入上方sql,因为mysql service的类型为nodeport,所以连接地址为任一k8s集群的节点ip地址,端口为30850
#以下创建用户操作也可以再连接工具中操作
#登录数据库
[root@k8s-master-1 ~]# kubectl exec -it mysql-589dcf6597-5ps6x /bin/bash -n test
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
root@mysql-589dcf6597-5ps6x:/# mysql -u root -p
Enter password:

mysql> create user 'xxl'@'%' identified by 'P@ssw0rd@xxl';				#创建xxl用户用于xxl-job服务连接
Query OK, 0 rows affected (0.00 sec)

mysql> GRANT ALL PRIVILEGES ON  xxl_job.* TO 'xxl'@'%' IDENTIFIED BY 'P@ssw0rd@xxl'; 	#对xxl用户授权
Query OK, 0 rows affected (0.00 sec)

mysql> FLUSH PRIVILEGES;									#刷新权限
Query OK, 0 rows affected (0.00 sec)

创建xxl-job的副本 yaml文件
[root@k8s-master-0 ~]# vi xxl-job.yaml

  • xxl-job.yaml
kind: Deployment
apiVersion: apps/v1
metadata:
  name: xxl-job
  namespace: test
  labels:
    app: xxl-job
  annotations:
    deployment.kubernetes.io/revision: '6'
spec:
  replicas: 1
  selector:
    matchLabels:
      app: xxl-job
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: xxl-job
      annotations:
        cni.projectcalico.org/ipv4pools: '["default-ipv4-ippool"]'
    spec:
      containers:
        - name: container-e0tn05
          image: 'xuxueli/xxl-job-admin:2.3.0'
          ports:
            - name: tcp-8080
              containerPort: 8080
              protocol: TCP
          env:
            - name: PARAMS
              value: >-
                --spring.datasource.url=jdbc:mysql://mysql.test:3306/xxl_job?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=Asia/Shanghai
                --spring.datasource.username=xxl
                --spring.datasource.password=P@ssw0rd@xxl
                --xxl.job.accessToken=RfCwgzKLuRGbrqqN9Tg9WT3t				#注意修改数据库连接地址端口及用户密码
          resources:
            limits:
              cpu: '4'
              memory: 8000Mi
            requests:
              cpu: 500m
              memory: 500Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          imagePullPolicy: IfNotPresent
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      dnsPolicy: ClusterFirst
      serviceAccountName: default
      serviceAccount: default
      securityContext: {}
      affinity: {}
      schedulerName: default-scheduler
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 25%
      maxSurge: 25%
  revisionHistoryLimit: 10
  progressDeadlineSeconds: 600

创建xxl-job的service yaml文件
[root@k8s-master-0 ~]# vi xxl-job-service.yaml

  • xxl-job-service.yaml
kind: Service
apiVersion: v1
metadata:
  name: xxl-job-service
  namespace: test
  labels:
    app: xxl-job-service
spec:
  ports:
    - name: http-8080
      protocol: TCP
      port: 8080
      targetPort: 8080
      nodePort: 30850
  selector:
    app: xxl-job
  type: NodePort
  sessionAffinity: None
  externalTrafficPolicy: Cluster

创建xxl-job
#部署xxl_job副本
[root@k8s-master-1 ~]# kubectl apply -f  xxl-job.yaml
#部署xxl_job服务svc
[root@k8s-master-1 ~]# kubectl apply -f  xxl-job-service.yaml

验证xxl-job
#查看xxl-job的pod
[root@k8s-master-1 ~]# kubectl get pod -n test
#查看xxl-job的svc
[root@k8s-master-1 ~]# kubectl get svc -n test
#浏览器访问任一节点ip+30850(nodeport端口)进行验证
http://ip:30850/xxl-job-admin/
用户名/密码 :admin/123456

k8s部署skywalking

部署步骤

创建skywalking的配置 yaml文件
[root@k8s-master-1 ~]# vi skywalking-cm.yaml

  • skywalking-cm.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: skywalking-cm
  namespace: test
data:
  STORAGE: 'elasticsearch7'
  STORAGE_ES_CLUSTER_NODES: '192.168.0.23:9200'				#es的连接地址
  ES_USER: '****'											#es的用户名
  ES_PASSWORD: '********'									#es的密码
  CORE_GRPC_PORT: '11800'
  CORE_REST_PORT: '12800'

创建skywalking的副本 yaml文件
[root@k8s-master-1 ~]# vi skywalking-deployment.yaml

  • skywalking-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: skywalking
  name: szxc-skywalk-skywalking-oap
  namespace: test
spec:
  replicas: 2
  selector:
    matchLabels:
      app: skywalking
  template:
    metadata:
      labels:
        app: skywalking
    spec:
      containers:
        - envFrom:
          - prefix: SW_
            configMapRef:
              name: skywalking-cm
          image: apache/skywalking-oap-server:8.7.0-es7
          imagePullPolicy: IfNotPresent
          name: skywalking
          ports:
            - containerPort: 12800
              name: http
              protocol: TCP
            - containerPort: 11800
              name: grpc
              protocol: TCP
          resources:
            limits:
              cpu: '2'
              memory: 2Gi
            requests:
              cpu: '1'
              memory: 2Gi
          volumeMounts:
            - mountPath: /etc/localtime
              name: volume-localtime
      volumes:
        - hostPath:
            path: /etc/localtime
            type: ''
          name: volume-localtime

创建skywalking的service yaml文件
[root@k8s-master-1 ~]# vi skywalking-deployment.yaml
  • skywalking-service.yaml
apiVersion: v1
kind: Service
metadata:
  name: skywalk-skywalking-oap
  namespace: test
  labels:
    app: skywalking
spec:
  type: ClusterIP
  ports:
    - name: http
      port: 12800
      protocol: TCP
      targetPort: 12800
    - name: grpc
      port: 11800
      protocol: TCP
      targetPort: 11800
  selector:
    app: skywalking
创建skywalking ui副本的service yaml文件
[root@k8s-master-1 ~]# vi skywalking-ui-service.yaml

  • skywalking-ui-service.yaml
apiVersion: v1
kind: Service
metadata:
  name: skywalking-ui
  labels:
    app: skywalking-ui
  namespace: test
spec:
  type: ClusterIP
  ports:
    - name: http
      port: 8080
      protocol: TCP
      targetPort: 8080
  selector:
    app: skywalking-ui

创建skywalking ui的副本 yaml文件
[root@k8s-master-1 ~]# vi skywalking-ui.yaml
  • skywalking-ui.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: skywalking-ui
  name: skywalking-ui
  namespace: test
spec:
  replicas: 1
  selector:
    matchLabels:
      app: skywalking-ui
  template:
    metadata:
      labels:
        app: skywalking-ui
    spec:
      containers:
        - env:
            - name: SW_OAP_ADDRESS
              value: "skywalking:12800"
          image: apache/skywalking-ui:8.9.1
          imagePullPolicy: IfNotPresent
          name: skywalking-ui
          ports:
            - containerPort: 8080
              name: http
              protocol: TCP
          resources:
            limits:
              cpu: '2'
              memory: 1Gi
            requests:
              cpu: '1'
              memory: 1Gi
          volumeMounts:
            - mountPath: /etc/localtime
              name: volume-localtime
      volumes:
        - hostPath:
            path: /etc/localtime
            type: ''
          name: volume-localtime

部署安装 skywalking
[root@k8s-master01 ~]# kubectl apply -f skywalking-cm.yaml
[root@k8s-master01 ~]# kubectl apply -f skywalking-deployment.yaml
[root@k8s-master01 ~]# kubectl apply -f skywalking-service.yaml
[root@k8s-master01 ~]# kubectl apply -f skywalking-ui-service.yaml
[root@k8s-master01 ~]# kubectl apply -f skywalking-ui.yaml

验证 skywalking
#查看skywalking副本
[root@k8s-master01 ~]# kubectl get pod -n test
NAME                                                  READY   STATUS    RESTARTS   AGE
skywalking-ui-7cb7f68686-4sgtq                        1/1     Running   0          22d
skywalk-skywalking-oap-67f6cd45fd-dm7d4          1/1     Running   0          22d
skywalk-skywalking-oap-67f6cd45fd-ggg4z          1/1     Running   0          22d
#查看skywalking service
[root@k8s-master01 ~]# kubectl get svc -n test 
NAME                                    TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                               AGE
skywalking-ui                           NodePort    10.233.16.121   <none>        8080:30849/TCP                        22d
skywalk-skywalking-oap             ClusterIP   10.233.12.1     <none>        12800/TCP,11800/TCP                   22d
#使用浏览器访问任意节点ip+30849(nodeport端口)进行验证

附录

  • /etc/nginx/nginx.conf
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;

# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic.
include /usr/share/nginx/modules/*.conf;

events {
    worker_connections 10240;
}

http {
    server_tokens off;												#隐藏版本号
    client_header_timeout 60;										#客户端向服务端发送一个完整的 request header 的超时时间。如果客户端在指定时间内没有发送一个完整的 request header,Nginx 返回 HTTP 408(“Request timed out”)
    client_body_timeout 60;											#该指令设置请求正文即请求体(request body)的读超时时间。超时仅设置为两个连续读取操作之间的时间段,而不是整个请求主体的传输。如果客户端在此时间内未传输任何内容,请求将以408(请求超时)错误终止
    limit_conn_zone $binary_remote_addr zone=one:10m;				#限制可以存储多少个并发连接数(1m 可以储存 32000 个并发会话)
    limit_conn one 50;												#限制每个IP只能发起50个并发连接
    limit_rate 2000k;												#控制下载速度
    send_timeout 10;												#服务端向客户端传输数据的超时时间,单位(s)
    keepalive_timeout   65;											#每个TCP连接的超时时间
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    log_format  debug  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for" '
                      '"debug-cors" $cors_origin "debug-origin" $http_origin ';

    access_log  /var/log/nginx/access.log  main;

    sendfile            on;											#sendfile是个比 read 和 write 更高性能的系统接口, 不过需要注意的是,sendfile 是将 in_fd 的内容发送到 out_fd 。而 in_fd 不能是 socket , 也就是只能文件句柄。 所以当 Nginx 是一个静态文件服务器的时候,开启 SENDFILE 配置项能大大提高 Nginx 的性能.
    tcp_nopush          on;											#可以配置一次发送数据包的大小。也就是说,数据包累积到一定大小后就发送,tcp_nopush必须和sendfile配合使用.
    tcp_nodelay         on;											#会增加小包的数量,但是可以提高响应速度。在及时性高的通信场景中应该会有不错的效果
    types_hash_max_size 4096;										#nginx使用了一个散列表来保存MIME type与文件扩展名之间的映射,该参数就是指定该散列表桶的大小的

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;
    error_page  400 404 413 502 504  /index.html;

    #开启gzip压缩
    gzip  on;
    #不压缩临界值,大于1K的才压缩
    gzip_min_length 1k;
    #buffer
    gzip_buffers 4 16k;
    #用了反向代理的话,末端通信是HTTP/1.0,默认是HTTP/1.1
    #gzip_http_version 1.0;
    #压缩级别,1-10,数字越大压缩的越好,时间也越长
    gzip_comp_level 2;
    #进行压缩的文件类型,缺啥补啥
    gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascriptapplication/x-httpd-php image/jpeg image/gif image/png;
    #跟Squid等缓存服务有关,on的话会在Header里增加"Vary: Accept-Encoding"
    gzip_vary off;
    #IE6对Gzip不友好,不进行Gzip压缩
    gzip_disable "MSIE [1-6]\.";

    client_max_body_size 100m;									#nginx对上传文件大小的限制
    proxy_buffer_size  128k;									#Nginx使用该大小申请read_buf,即大小指定了 upstream header 最大长度,如果响应头超过了这个长度,Nginx会报upstream sent too big header错误,然后client收到的是502
    proxy_buffers   32 32k;										#设置存储被代理服务器响应的body所占用的buffer个数和每个buffer大小
    proxy_busy_buffers_size 128k;								#proxy_busy_buffers_size不是独立的空间,他是proxy_buffers和proxy_buffer_size的一部分。nginx会在没有完全读完后端响应就开始向客户端传送数据,所以它会划出一部分busy状态的buffer来专门向客户端传送数据(建议为proxy_buffers中单个缓冲区的2倍),然后它继续从后端取数据。proxy_busy_buffer_size参数用来设置处于busy状态的buffer有多大

    fastcgi_buffers 16 256k;									#设定用来读取从FastCGI服务器端收到的响应信息的缓冲区大小和缓冲区数量
    fastcgi_buffer_size 128k;									#Nginx FastCGI 的缓冲区大小,用来读取从FastCGI服务器端收到的第一部分响应信息的缓冲区大小
    fastcgi_busy_buffers_size 256k;								#用于设置系统很忙时可以使用的 proxy_buffers 大小

    map $http_upgrade $connection_upgrade {  #开启websocket升级代理功能,可选
        default upgrade;
        '' close;
    }

    include /etc/nginx/conf.d/*.conf;

    server {													#http(80)转https(443)配置				
        listen       80;
        listen       [::]:80;
        server_name  www.abc.com;
        rewrite ^(.*)$ https://${server_name}$1 permanent;
        #root         /usr/share/nginx/html;

        # Load configuration files for the default server block.
        include /etc/nginx/default.d/*.conf;

        error_page 404 /404.html;
        location = /404.html {
        }

        error_page 500 502 503 504 /50x.html;
        location = /50x.html {
        }
    }
}

  • /etc/nginx/conf.d/default.conf
server {
listen 80;
  server_name www.abc.com;
   access_log  /var/log/nginx/abc.access.log  main;				#自定义专属日志文件
  
  location / {
    root /usr/share/nginx/dist;
    index  index.html index.htm;
  }

    location ^~ /con/ {											#转发示例
        proxy_pass http://192.168.1.1:8003/;
        proxy_redirect off;
        proxy_set_header Host $host;
        proxy_set_header REMOTE-HOST $remote_addr;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    }
}
  • nacos-mysql.sql
/*
 * Copyright 1999-2018 Alibaba Group Holding Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/******************************************/
/*   数据库全名 = nacos_config   */
/*   表名称 = config_info   */
/******************************************/
CREATE TABLE `config_info` (
  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id',
  `data_id` varchar(255) NOT NULL COMMENT 'data_id',
  `group_id` varchar(255) DEFAULT NULL,
  `content` longtext NOT NULL COMMENT 'content',
  `md5` varchar(32) DEFAULT NULL COMMENT 'md5',
  `gmt_create` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
  `gmt_modified` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间',
  `src_user` text COMMENT 'source user',
  `src_ip` varchar(50) DEFAULT NULL COMMENT 'source ip',
  `app_name` varchar(128) DEFAULT NULL,
  `tenant_id` varchar(128) DEFAULT '' COMMENT '租户字段',
  `c_desc` varchar(256) DEFAULT NULL,
  `c_use` varchar(64) DEFAULT NULL,
  `effect` varchar(64) DEFAULT NULL,
  `type` varchar(64) DEFAULT NULL,
  `c_schema` text,
  PRIMARY KEY (`id`),
  UNIQUE KEY `uk_configinfo_datagrouptenant` (`data_id`,`group_id`,`tenant_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='config_info';

/******************************************/
/*   数据库全名 = nacos_config   */
/*   表名称 = config_info_aggr   */
/******************************************/
CREATE TABLE `config_info_aggr` (
  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id',
  `data_id` varchar(255) NOT NULL COMMENT 'data_id',
  `group_id` varchar(255) NOT NULL COMMENT 'group_id',
  `datum_id` varchar(255) NOT NULL COMMENT 'datum_id',
  `content` longtext NOT NULL COMMENT '内容',
  `gmt_modified` datetime NOT NULL COMMENT '修改时间',
  `app_name` varchar(128) DEFAULT NULL,
  `tenant_id` varchar(128) DEFAULT '' COMMENT '租户字段',
  PRIMARY KEY (`id`),
  UNIQUE KEY `uk_configinfoaggr_datagrouptenantdatum` (`data_id`,`group_id`,`tenant_id`,`datum_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='增加租户字段';


/******************************************/
/*   数据库全名 = nacos_config   */
/*   表名称 = config_info_beta   */
/******************************************/
CREATE TABLE `config_info_beta` (
  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id',
  `data_id` varchar(255) NOT NULL COMMENT 'data_id',
  `group_id` varchar(128) NOT NULL COMMENT 'group_id',
  `app_name` varchar(128) DEFAULT NULL COMMENT 'app_name',
  `content` longtext NOT NULL COMMENT 'content',
  `beta_ips` varchar(1024) DEFAULT NULL COMMENT 'betaIps',
  `md5` varchar(32) DEFAULT NULL COMMENT 'md5',
  `gmt_create` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
  `gmt_modified` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间',
  `src_user` text COMMENT 'source user',
  `src_ip` varchar(50) DEFAULT NULL COMMENT 'source ip',
  `tenant_id` varchar(128) DEFAULT '' COMMENT '租户字段',
  PRIMARY KEY (`id`),
  UNIQUE KEY `uk_configinfobeta_datagrouptenant` (`data_id`,`group_id`,`tenant_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='config_info_beta';

/******************************************/
/*   数据库全名 = nacos_config   */
/*   表名称 = config_info_tag   */
/******************************************/
CREATE TABLE `config_info_tag` (
  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id',
  `data_id` varchar(255) NOT NULL COMMENT 'data_id',
  `group_id` varchar(128) NOT NULL COMMENT 'group_id',
  `tenant_id` varchar(128) DEFAULT '' COMMENT 'tenant_id',
  `tag_id` varchar(128) NOT NULL COMMENT 'tag_id',
  `app_name` varchar(128) DEFAULT NULL COMMENT 'app_name',
  `content` longtext NOT NULL COMMENT 'content',
  `md5` varchar(32) DEFAULT NULL COMMENT 'md5',
  `gmt_create` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
  `gmt_modified` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间',
  `src_user` text COMMENT 'source user',
  `src_ip` varchar(50) DEFAULT NULL COMMENT 'source ip',
  PRIMARY KEY (`id`),
  UNIQUE KEY `uk_configinfotag_datagrouptenanttag` (`data_id`,`group_id`,`tenant_id`,`tag_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='config_info_tag';

/******************************************/
/*   数据库全名 = nacos_config   */
/*   表名称 = config_tags_relation   */
/******************************************/
CREATE TABLE `config_tags_relation` (
  `id` bigint(20) NOT NULL COMMENT 'id',
  `tag_name` varchar(128) NOT NULL COMMENT 'tag_name',
  `tag_type` varchar(64) DEFAULT NULL COMMENT 'tag_type',
  `data_id` varchar(255) NOT NULL COMMENT 'data_id',
  `group_id` varchar(128) NOT NULL COMMENT 'group_id',
  `tenant_id` varchar(128) DEFAULT '' COMMENT 'tenant_id',
  `nid` bigint(20) NOT NULL AUTO_INCREMENT,
  PRIMARY KEY (`nid`),
  UNIQUE KEY `uk_configtagrelation_configidtag` (`id`,`tag_name`,`tag_type`),
  KEY `idx_tenant_id` (`tenant_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='config_tag_relation';

/******************************************/
/*   数据库全名 = nacos_config   */
/*   表名称 = group_capacity   */
/******************************************/
CREATE TABLE `group_capacity` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键ID',
  `group_id` varchar(128) NOT NULL DEFAULT '' COMMENT 'Group ID,空字符表示整个集群',
  `quota` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '配额,0表示使用默认值',
  `usage` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '使用量',
  `max_size` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '单个配置大小上限,单位为字节,0表示使用默认值',
  `max_aggr_count` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '聚合子配置最大个数,,0表示使用默认值',
  `max_aggr_size` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '单个聚合数据的子配置大小上限,单位为字节,0表示使用默认值',
  `max_history_count` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '最大变更历史数量',
  `gmt_create` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
  `gmt_modified` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间',
  PRIMARY KEY (`id`),
  UNIQUE KEY `uk_group_id` (`group_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='集群、各Group容量信息表';

/******************************************/
/*   数据库全名 = nacos_config   */
/*   表名称 = his_config_info   */
/******************************************/
CREATE TABLE `his_config_info` (
  `id` bigint(64) unsigned NOT NULL,
  `nid` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
  `data_id` varchar(255) NOT NULL,
  `group_id` varchar(128) NOT NULL,
  `app_name` varchar(128) DEFAULT NULL COMMENT 'app_name',
  `content` longtext NOT NULL,
  `md5` varchar(32) DEFAULT NULL,
  `gmt_create` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
  `gmt_modified` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
  `src_user` text,
  `src_ip` varchar(50) DEFAULT NULL,
  `op_type` char(10) DEFAULT NULL,
  `tenant_id` varchar(128) DEFAULT '' COMMENT '租户字段',
  PRIMARY KEY (`nid`),
  KEY `idx_gmt_create` (`gmt_create`),
  KEY `idx_gmt_modified` (`gmt_modified`),
  KEY `idx_did` (`data_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='多租户改造';


/******************************************/
/*   数据库全名 = nacos_config   */
/*   表名称 = tenant_capacity   */
/******************************************/
CREATE TABLE `tenant_capacity` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键ID',
  `tenant_id` varchar(128) NOT NULL DEFAULT '' COMMENT 'Tenant ID',
  `quota` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '配额,0表示使用默认值',
  `usage` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '使用量',
  `max_size` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '单个配置大小上限,单位为字节,0表示使用默认值',
  `max_aggr_count` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '聚合子配置最大个数',
  `max_aggr_size` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '单个聚合数据的子配置大小上限,单位为字节,0表示使用默认值',
  `max_history_count` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '最大变更历史数量',
  `gmt_create` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
  `gmt_modified` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间',
  PRIMARY KEY (`id`),
  UNIQUE KEY `uk_tenant_id` (`tenant_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='租户容量信息表';


CREATE TABLE `tenant_info` (
  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id',
  `kp` varchar(128) NOT NULL COMMENT 'kp',
  `tenant_id` varchar(128) default '' COMMENT 'tenant_id',
  `tenant_name` varchar(128) default '' COMMENT 'tenant_name',
  `tenant_desc` varchar(256) DEFAULT NULL COMMENT 'tenant_desc',
  `create_source` varchar(32) DEFAULT NULL COMMENT 'create_source',
  `gmt_create` bigint(20) NOT NULL COMMENT '创建时间',
  `gmt_modified` bigint(20) NOT NULL COMMENT '修改时间',
  PRIMARY KEY (`id`),
  UNIQUE KEY `uk_tenant_info_kptenantid` (`kp`,`tenant_id`),
  KEY `idx_tenant_id` (`tenant_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='tenant_info';

CREATE TABLE `users` (
	`username` varchar(50) NOT NULL PRIMARY KEY,
	`password` varchar(500) NOT NULL,
	`enabled` boolean NOT NULL
);

CREATE TABLE `roles` (
	`username` varchar(50) NOT NULL,
	`role` varchar(50) NOT NULL,
	UNIQUE INDEX `idx_user_role` (`username` ASC, `role` ASC) USING BTREE
);

CREATE TABLE `permissions` (
    `role` varchar(50) NOT NULL,
    `resource` varchar(255) NOT NULL,
    `action` varchar(8) NOT NULL,
    UNIQUE INDEX `uk_role_permission` (`role`,`resource`,`action`) USING BTREE
);

INSERT INTO users (username, password, enabled) VALUES ('nacos', '$2a$10$EuWPZHzz32dJN7jexM34MOeYirDdFAZm2kuWj7VEOJhhZkDrxfvUu', TRUE);

INSERT INTO roles (username, role) VALUES ('nacos', 'ROLE_ADMIN');
  • tables_xxl_job.sql
#
# XXL-JOB v2.3.1-SNAPSHOT
# Copyright (c) 2015-present, xuxueli.

CREATE database if NOT EXISTS `xxl_job` default character set utf8mb4 collate utf8mb4_unicode_ci;
use `xxl_job`;

SET NAMES utf8mb4;

CREATE TABLE `xxl_job_info` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `job_group` int(11) NOT NULL COMMENT '执行器主键ID',
  `job_desc` varchar(255) NOT NULL,
  `add_time` datetime DEFAULT NULL,
  `update_time` datetime DEFAULT NULL,
  `author` varchar(64) DEFAULT NULL COMMENT '作者',
  `alarm_email` varchar(255) DEFAULT NULL COMMENT '报警邮件',
  `schedule_type` varchar(50) NOT NULL DEFAULT 'NONE' COMMENT '调度类型',
  `schedule_conf` varchar(128) DEFAULT NULL COMMENT '调度配置,值含义取决于调度类型',
  `misfire_strategy` varchar(50) NOT NULL DEFAULT 'DO_NOTHING' COMMENT '调度过期策略',
  `executor_route_strategy` varchar(50) DEFAULT NULL COMMENT '执行器路由策略',
  `executor_handler` varchar(255) DEFAULT NULL COMMENT '执行器任务handler',
  `executor_param` varchar(512) DEFAULT NULL COMMENT '执行器任务参数',
  `executor_block_strategy` varchar(50) DEFAULT NULL COMMENT '阻塞处理策略',
  `executor_timeout` int(11) NOT NULL DEFAULT '0' COMMENT '任务执行超时时间,单位秒',
  `executor_fail_retry_count` int(11) NOT NULL DEFAULT '0' COMMENT '失败重试次数',
  `glue_type` varchar(50) NOT NULL COMMENT 'GLUE类型',
  `glue_source` mediumtext COMMENT 'GLUE源代码',
  `glue_remark` varchar(128) DEFAULT NULL COMMENT 'GLUE备注',
  `glue_updatetime` datetime DEFAULT NULL COMMENT 'GLUE更新时间',
  `child_jobid` varchar(255) DEFAULT NULL COMMENT '子任务ID,多个逗号分隔',
  `trigger_status` tinyint(4) NOT NULL DEFAULT '0' COMMENT '调度状态:0-停止,1-运行',
  `trigger_last_time` bigint(13) NOT NULL DEFAULT '0' COMMENT '上次调度时间',
  `trigger_next_time` bigint(13) NOT NULL DEFAULT '0' COMMENT '下次调度时间',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE `xxl_job_log` (
  `id` bigint(20) NOT NULL AUTO_INCREMENT,
  `job_group` int(11) NOT NULL COMMENT '执行器主键ID',
  `job_id` int(11) NOT NULL COMMENT '任务,主键ID',
  `executor_address` varchar(255) DEFAULT NULL COMMENT '执行器地址,本次执行的地址',
  `executor_handler` varchar(255) DEFAULT NULL COMMENT '执行器任务handler',
  `executor_param` varchar(512) DEFAULT NULL COMMENT '执行器任务参数',
  `executor_sharding_param` varchar(20) DEFAULT NULL COMMENT '执行器任务分片参数,格式如 1/2',
  `executor_fail_retry_count` int(11) NOT NULL DEFAULT '0' COMMENT '失败重试次数',
  `trigger_time` datetime DEFAULT NULL COMMENT '调度-时间',
  `trigger_code` int(11) NOT NULL COMMENT '调度-结果',
  `trigger_msg` text COMMENT '调度-日志',
  `handle_time` datetime DEFAULT NULL COMMENT '执行-时间',
  `handle_code` int(11) NOT NULL COMMENT '执行-状态',
  `handle_msg` text COMMENT '执行-日志',
  `alarm_status` tinyint(4) NOT NULL DEFAULT '0' COMMENT '告警状态:0-默认、1-无需告警、2-告警成功、3-告警失败',
  PRIMARY KEY (`id`),
  KEY `I_trigger_time` (`trigger_time`),
  KEY `I_handle_code` (`handle_code`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE `xxl_job_log_report` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `trigger_day` datetime DEFAULT NULL COMMENT '调度-时间',
  `running_count` int(11) NOT NULL DEFAULT '0' COMMENT '运行中-日志数量',
  `suc_count` int(11) NOT NULL DEFAULT '0' COMMENT '执行成功-日志数量',
  `fail_count` int(11) NOT NULL DEFAULT '0' COMMENT '执行失败-日志数量',
  `update_time` datetime DEFAULT NULL,
  PRIMARY KEY (`id`),
  UNIQUE KEY `i_trigger_day` (`trigger_day`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE `xxl_job_logglue` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `job_id` int(11) NOT NULL COMMENT '任务,主键ID',
  `glue_type` varchar(50) DEFAULT NULL COMMENT 'GLUE类型',
  `glue_source` mediumtext COMMENT 'GLUE源代码',
  `glue_remark` varchar(128) NOT NULL COMMENT 'GLUE备注',
  `add_time` datetime DEFAULT NULL,
  `update_time` datetime DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE `xxl_job_registry` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `registry_group` varchar(50) NOT NULL,
  `registry_key` varchar(255) NOT NULL,
  `registry_value` varchar(255) NOT NULL,
  `update_time` datetime DEFAULT NULL,
  PRIMARY KEY (`id`),
  KEY `i_g_k_v` (`registry_group`,`registry_key`,`registry_value`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE `xxl_job_group` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `app_name` varchar(64) NOT NULL COMMENT '执行器AppName',
  `title` varchar(12) NOT NULL COMMENT '执行器名称',
  `address_type` tinyint(4) NOT NULL DEFAULT '0' COMMENT '执行器地址类型:0=自动注册、1=手动录入',
  `address_list` text COMMENT '执行器地址列表,多地址逗号分隔',
  `update_time` datetime DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE `xxl_job_user` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `username` varchar(50) NOT NULL COMMENT '账号',
  `password` varchar(50) NOT NULL COMMENT '密码',
  `role` tinyint(4) NOT NULL COMMENT '角色:0-普通用户、1-管理员',
  `permission` varchar(255) DEFAULT NULL COMMENT '权限:执行器ID列表,多个逗号分割',
  PRIMARY KEY (`id`),
  UNIQUE KEY `i_username` (`username`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE `xxl_job_lock` (
  `lock_name` varchar(50) NOT NULL COMMENT '锁名称',
  PRIMARY KEY (`lock_name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

INSERT INTO `xxl_job_group`(`id`, `app_name`, `title`, `address_type`, `address_list`, `update_time`) VALUES (1, 'xxl-job-executor-sample', '示例执行器', 0, NULL, '2018-11-03 22:21:31' );
INSERT INTO `xxl_job_info`(`id`, `job_group`, `job_desc`, `add_time`, `update_time`, `author`, `alarm_email`, `schedule_type`, `schedule_conf`, `misfire_strategy`, `executor_route_strategy`, `executor_handler`, `executor_param`, `executor_block_strategy`, `executor_timeout`, `executor_fail_retry_count`, `glue_type`, `glue_source`, `glue_remark`, `glue_updatetime`, `child_jobid`) VALUES (1, 1, '测试任务1', '2018-11-03 22:21:31', '2018-11-03 22:21:31', 'XXL', '', 'CRON', '0 0 0 * * ? *', 'DO_NOTHING', 'FIRST', 'demoJobHandler', '', 'SERIAL_EXECUTION', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2018-11-03 22:21:31', '');
INSERT INTO `xxl_job_user`(`id`, `username`, `password`, `role`, `permission`) VALUES (1, 'admin', 'e10adc3949ba59abbe56e057f20f883e', 1, NULL);
INSERT INTO `xxl_job_lock` ( `lock_name`) VALUES ( 'schedule_lock');

commit;

  • 0
    点赞
  • 1
    收藏
    觉得还不错? 一键收藏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值