k8s部署frp，并配置80端口

一开始我的frp是用docker运行的，但是端口不是80端口，测试微信支付只能内网穿透回调80端口，因为服务器上跑了nginx-ingress-controller和一些其他的服务，服务器上安装nginx反射代理80端口的话，提示80端口占用，下面使用k8s部署frp，并用ingress解决80端口穿透

1. 编写 frp-config-pvc.yaml

挂载一个配置文件 frps.ini

[root@master frp-k8s]# cat frp-config-pvc.yaml 
apiVersion: v1
kind: PersistentVolume
metadata:
  name: frp-config-pv
spec:
  capacity:
    storage: 50Mi
  accessModes:
  - ReadWriteMany
  persistentVolumeReclaimPolicy: Delete
  nfs:
    server: 172.17.0.1
    path: /data/k8s
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: frp-config-pvc
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 50Mi

执行

kubectl apply -f frp-config-pvc.yaml

因为装了Rancher，将local-path设置成了默认的storageclass，上面声明pvc，会优先使用local-path，不使用下面的pv，pvc会一直Pending
取消local-path设置成了默认的storageclass
kubectl patch storageclass local-path -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"false"}}}'

2. 编写 frp-k8s.yaml

[root@master frp-k8s]# cat <<EOF> frp-k8s.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: frp-k8s
spec:
  replicas: 1
  selector:
    matchLabels:
      app: frp-k8s
  template:
    metadata:
      labels:
        app: frp-k8s
    spec:
      containers:
      - name: frps
        image: snowdreamtech/frps
        imagePullPolicy: IfNotPresent
        ports:
        - name: http
          containerPort: 8080
          protocol: TCP
        - name: https
          containerPort: 8081
          protocol: TCP
        - name: dashboard
          containerPort: 7500
          protocol: TCP
        - name: frps
          containerPort: 7000
          protocol: TCP
        volumeMounts:
        - name: frp-conf
          subPath: frp-conf
          mountPath: /etc/frp
      volumes:
      - name: frp-conf
        persistentVolumeClaim:
          claimName: frp-config-pvc
---
apiVersion: v1
kind: Service
metadata:
  name: frp-k8s-1
spec:
  type: ClusterIP
  ports:
    - name: dashboard
      protocol: TCP
      port: 7500
      targetPort: 7500
    - name: http
      protocol: TCP
      port: 8080
      targetPort: 8080
    - name: https
      protocol: TCP
      port: 8081
      targetPort: 8081
  selector:
    app: frp-k8s
---
# 这个 Service 客户端frpc.ini配置需要一个server_port，所有使用 NodePort暴露端口，名字不能和上面 Service 名字一样，否则会覆盖
apiVersion: v1
kind: Service
metadata:
  name: frp-k8s-2
spec:
  type: NodePort
  ports:
    - name: dashboard
      protocol: TCP
      port: 7000
      targetPort: 7000
      nodePort: 32456
  selector:
    app: frp-k8s
---
# frp查看界面，可以不添加
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: frp-dashboard
  annotations:
    kubernetes.io/ingress.class: "nginx"    
    kubernetes.io/tls-acme: "true"
spec:
  tls:
  - hosts:
    - frp-dashboard.xxx.com
    secretName: frp-dashboard-tls
  rules:
  - host: frp-dashboard.xxx.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: frp-k8s-1
            port:
              number: 7500
EOF

执行

kubectl apply -f frp-k8s.yaml
# 此时pod会报错，必须下面配置配置文件后重新启动才会成功

[root@master frp-k8s]# ls /data/k8s/
frp-conf

[root@master frp-k8s]# cat <<EOF> /data/k8s/frp-conf/frps.ini 
[common]
bind_port = 7000
vhost_http_port = 8080
vhost_https_port = 8081
dashboard_addr = 0.0.0.0
dashboard_port = 7500
dashboard_user = admin
dashboard_pwd = admin
EOF

重启 frp-k8s.yaml

kubectl delete -f frp-k8s.yaml

kubectl apply -f frp-k8s.yaml

[root@master frp-k8s]# kubectl get pods | grep frp
frp-k8s-6fc4996cfb-k9bzl                 1/1     Running   0          36m

[root@master frp-k8s]# kubectl get svc | grep frp
frp-k8s-1     ClusterIP   10.106.198.197   <none>        7500/TCP,8080/TCP,8081/TCP   36m
frp-k8s-2     NodePort    10.105.158.251   <none>        7000:32456/TCP               36m

7500: dashboard界面端口，现在可以直接使用 https://frp-dashboar.xxxxxx.com 访问
7000: 客户端 frpc连接需要server_port = 32456

3. 编写 http ingress

[root@master frp-k8s]# cat http-frp-ingress.yaml 
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: frp-http
  annotations:
    kubernetes.io/ingress.class: "nginx"
spec:
  rules:
  - host: frp.xxxxxx.com
    http:
      paths:
      - path: /
        backend:
          serviceName: frp-k8s-1
          servicePort: 8080

执行

kubectl apply -f http-frp-ingress.yaml

如果有多个web客户端

root@master:~/i/docker-app/frpc# cat frpc.ini 
[common]
server_addr = 122.51.103.44
server_port = 32456

[web01]
type = http
local_port = 30999
custom_domains = frp.wanfei.wang

[ssh]
type = tcp
local_ip = 127.0.0.1
local_port = 22
remote_port = 22

[web02]
type = http
local_port = 30005
custom_domains = jenkins.wanfei.wang

[web03]
type = http
local_port = 30032
custom_domains = harbor.wanfei.wang

web02和web03在frps端再添加两个ingress，和http-frp-ingress.yaml 一样，修改下名字和域名

配置ssh，需要添加service指定一个NodePort端口指向remote_port

4. 客户端 http 连接

# http访问
[common]
server_addr = 公网ip
server_port = 32456
[web]
type = http
local_port = 62160
custom_domains = frp.xxxxxx.com

5. 测试http 连接

本地服务启动端口62160

此时通过 ingress 直接访问frp-k8s-1的8080端口

6. 客户端 https 连接

客户端和服务端ingress保持证书一致，没有测试了

7. 客户端k8s 连接

ConfigMap挂载配置

cat <<EOF> frpc-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: frpc
  namespace: default
data:
  frpc.ini: |-
    [common]
    server_addr = 122.51.103.44
    server_port = 32456

    [web-tai]
    type = http
    local_ip = 192.168.4.27
    local_port = 30880
    custom_domains = frp-tai.wanfei.wang
EOF

注意：一定要配置local_ip = 192.168.4.27，否则是127.0.0.1，无法正确连接，报错
[web-tai] connect to local service [127.0.0.1:30880] error: dial tcp 127.0.0.1:30880: connect: connection refused

ip + port 同理

frpc-k8s.yaml

cat <<EOF> frpc-k8s.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: frpc-k8s
spec:
  replicas: 1
  selector:
    matchLabels:
      app: frpc-k8s
  template:
    metadata:
      labels:
        app: frpc-k8s
    spec:
      containers:
      - name: frpc
        image: snowdreamtech/frpc
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - name: frpc-conf
          mountPath: /etc/frp
      volumes:
      - configMap:
          items:
          - key: frpc.ini
            path: frpc.ini
          name: frpc
        name: frpc-conf
EOF

成功

8. 配置win10远程桌面

8.1 frps服务端配置

[root@master tcp-port]# cat <<EOF> remote-desk.yaml 
apiVersion: v1
kind: Service
metadata:
  name: remote-desk-tcp
spec:
  type: NodePort
  ports:
    - name: http
      protocol: TCP
      port: 32389
      targetPort: 32389
      nodePort: 32389
  selector:
    app: frp-k8s
EOF

32389是提供连接的远程端口

8.2 frpc客户端配置

frpc.ini

[common]
server_addr = 122.51.103.44
server_port = 32456

[rdp]
type = tcp
local_ip = 127.0.0.1
#远程桌面的默认端口
local_port = 3389
# 服务端开启的端口，外网访问
remote_port = 32389

可以配置一个bat脚本启动

cd /d %~dp0
frpc

运行客户端

frpc客户端电脑允许远程桌面连接
此电脑 —> 属性 —> 远程桌面

可以在下面用户账户添加允许连接的用户，不配置的话要用当前的win10用户登录账号密码连接

8.3 远程连接

使用另外一台电脑
win+R 输入 mstsc

点击连接，会弹出一个弹窗，

点击是就能远程连接了

9. IP + Port远程连接

9.1 server端开放远程端口32384

cat <<EOF> ip-port-remote-32384.yaml 
apiVersion: v1
kind: Service
metadata:
  name: ip-port-remote-tcp
spec:
  type: NodePort
  ports:
    - name: http
      protocol: TCP
      port: 32384
      targetPort: 32384
      nodePort: 32384
  selector:
    app: frp-k8s
EOF

执行

kubectl apply -f ip-port-remote-32384.yaml

9.2 客户端连接

本地客户端是vm虚拟机安装的centos7服务器

注意：此时公网IP不一定是域名绑定的服务器公网IP，也可以是k8s集群其他节点速度更快的公网IP

参考 使用docker部署部署frpc 查看 2.3

9.3 测试连接mongodb

本地安装mongodb

• 本地连接
  
• 远程 IP + Port连接