#!/bin/bash
POSITIONAL=()
while [[ $# -gt 0 ]]; do
if [[ $1 == -* || $1 == --* ]]; then
key="$1"
if [key ]
echo "key:"$key"--------------------"
fi
if [[ $2 == -* || $2 == --* ]]; then
shift
echo "continue:--------------------"
continue
fi
case $key in
-k|--skip)
skip+=("$2")
shift # past argument
;;
-o|--orign)
orgin+=("$2")
shift # past argument
;;
-h|--help)
help="true"
shift
;;
*)
echo "Error! invalid flag: ${key}"
help="true"
break
;;
esac
done
# echo "${params[@]}"
# for i in ${params[*]};do echo $i; done
usage () {
echo "USAGE: $0 [--image-list longhorn-images.txt] [--images longhorn-images.tar.gz]"
echo " [-l|--images-list path] text file with list of images. 1 per line."
echo " [-i|--images path] tar.gz generated by docker save. If this flag is empty, the script does not export images to a tar.gz file"
echo " [-h|--help] Usage message"
}
if [[ $help ]]; then
usage
exit 0
fi
另一段处理yaml的kustomize过程:
#!/bin/bash
globalKey=""
POSITIONAL=()
while [[ $# -gt 0 ]]; do
key="$1"
case $key in
-k|--skip)
skip+="$2"
shift # past argument
shift # past value
;;
-l|--image-list)
list="$2"
shift # past argument
shift # past value
;;
-h|--help)
help="true"
shift
;;
*)
echo "Error! invalid flag: ${key}"
help="true"
break
;;
esac
done
usage () {
echo "USAGE: $0 [--image-list longhorn-images.txt] [--images longhorn-images.tar.gz]"
echo " [-l|--images-list path] text file with list of images. 1 per line."
echo " [-i|--images path] tar.gz generated by docker save. If this flag is empty, the script does not export images to a tar.gz file"
echo " [-h|--help] Usage message"
}
if [[ $help ]]; then
usage
exit 0
fi
set -e -x
function baseFile(){
basePath=$2
find $1 -type f -name "*.yaml" |awk -v basePath="$basePath" '{n=split($1,aaa,"\/");print "cp",$1,basePath"/tmp/"aaa[n]}' | tee ./graph/script/cp-base.sh
chmod 755 ./graph/script/cp-base.sh
bash ./graph/script/cp-base.sh
yq --null-input '.apiVersion ="kustomize.config.k8s.io/v1beta1"
|.kind="Kustomization"
|.commonLabels.k8s-app="p-expoter",.commonLabels.k8s-app line_comment="会在所有的资源对象上都会加上该label"
|.namespace="stage",.namespace line_comment="会在所有的资源对象上都指定该ns"
| .resources = []' > $basePath"/kustomization.yaml"
splitFiles $basePath
renameFiles
yq '.resources = (.resources | unique)' -i $basePath"/kustomization.yaml"
rm -rf $basePath/tmp
}
function splitFiles(){
rm -rf /tmp/yaml/*
for file in $basePath/tmp/*.yaml
do
if [ -f "$file" ]; then
orignFileShortName=`echo $(basename $file)|cut -d "." -f1 `
# awk 'BEGIN{FS="";RS="---";ORS=""}{path="/tmp/yaml/"NR".yaml";if($0!="")print $0 > path;}' $file
ofs=$orignFileShortName yq -s '"/tmp/yaml/" + env(ofs) + "."+ $index + ".yaml" ' $file
rm -rf $file
fi
##处理namespace
done
}
function renameFiles(){
for file in `find /tmp/yaml/ -name "*.yaml" `
do
echo $file"-----------------------------------------------------------------"
type=`yq '.kind' $file `;
orignFileShortName=`echo $(basename $file)|cut -d "." -f1 `
lowerType=${type,,};
fileShortName=`yq '.metadata.name' $file `;
if [[ "$lowerType" == "namespace" ]]; then
fileName=("$basePath/$fileShortName.yaml")
relativePath="ns-$fileShortName.yaml";
elif [[ "$lowerType" == "job" ]]; then
fileName="$basePath/$lowerType/$fileShortName.yaml";
relativePath="$lowerType/$fileShortName.yaml";
elif [[ "$lowerType" == "service" ]]; then
fileName="$basePath/$orignFileShortName/deployment/$fileShortName.yaml";
relativePath="$orignFileShortName/deployment/$fileShortName.yaml";
else
fileName="$basePath/$orignFileShortName/$lowerType/$fileShortName.yaml";
relativePath="$orignFileShortName/$lowerType/$fileShortName.yaml";
fi;
mkdir -p `dirname $fileName`;
if [ -f $fileName ] && [ "$lowerType" != "namespace" ]; then
cat $file >> $fileName;
else
cat $file > $fileName;
fi
if [[ "$lowerType" == "ingress" || "$lowerType" == "namespace" ]]; then
continue ;
fi;
rp=$relativePath yq '.resources += env(rp)' -i $basePath"/kustomization.yaml"
done
}
function initDir(){
rm -rf ./data/* ./graph/* ./image/* /tmp/yaml/*
mkdir -p ./data/files ./data/script ./graph/files/base/tmp ./graph/files/overlay/test/config ./graph/script ./image/files ./image/script /tmp/yaml
cp ./attachment/save-images.sh ./image/script/
cat /dev/null > ./image/images.txt && cat /dev/null > ./data/data.txt && cat /dev/null > ./data/script/data.sh && cat /dev/null > ./graph/script/cp-overlay.sh && cat /dev/null > ./graph/script/cp-base.sh
}
function image(){
#构建本地拉取资源文件
nerdctl build -t images.knowdee.com/library/rsync:v1 -f ./attachment/rsync/rsync.dockerfile ./attachment/rsync
find $1 -type f -name "*.yaml" |xargs grep "image:"| awk -F '[ ]+' '!a[$3]++{sub(/nexus.knowdee.com\/utils\/mysql-client:v1.0.0/,"harbor.knowdee.com/chatbot/mysql-client:v1.0.0");print $3}'|tee ./image/images.txt
echo "images.knowdee.com/library/rsync:v1" >> ./image/images.txt
bash ./image/script/save-images.sh --image-list ./image/images.txt --images ./image/files/app-images.tar.gz
}
function resource(){
#find $1 -type f -name "*.yaml" |xargs egrep 's3cmd\s+get'| awk -F '[ ]+' '!a[$4,$5]++ {cmd="echo -n "$4" | md5sum|cut -d \" \" -f1"; cmd|getline md5; close(cmd);split($1,mm,":"); print mm[1], $4 ,$5,md5}' | tee ./data/data.txt
find $1 -type f -name "*.yaml" |xargs egrep 's3cmd\s+get'| awk -F '[ ]+' '!a[$4,$5]++ {split($1,fileArr,":"); match($4,/s3:\/\/(.*)/,p);print fileArr[1], $4 ,$5,p[1]}' | tee ./data/data.txt
#下载s3上面的资源文件
awk '{cmd="dirname "$2""; cmd|getline dir; close(cmd);sub(/s3:\/\//,"",$2);print "mkdir -p ./data/files/"dir,"\ns3cmd get","s3://"$2,"./data/files/"$2}' ./data/data.txt| tee ./data/script/data.sh
chmod 755 ./data/script/data.sh
bash ./data/script/data.sh
}
function overlayFile(){
overlayPath="./graph/files/overlay/test"
rm -rf $overlayPath/* && mkdir -p $overlayPath/config
cat /dev/null > $overlayPath"/kustomization.yaml"
yq '.apiVersion ="kustomize.config.k8s.io/v1beta1"
|.kind="Kustomization"
|.commonLabels.k8s-app="p-expoter"
|.commonLabels.k8s-app line_comment="会在所有的资源对象上都会加上该label"
|.namespace="stage"
|.namespace line_comment="会在所有的资源对象上都指定该ns"
|.generatorOptions.disableNameSuffixHash=true
|.nameSuffix="canary"
|.resources[0]="../../base"
|.patches=[]
|.configMapGenerator=[]
|.vars=[]' -i $overlayPath"/kustomization.yaml"
tmpDir="/tmp/yaml"
rm -rf $tmpDir/*
for file in `find $basePath/**/ -name "*.yaml" `
do
if [ -f "$file" ]; then
echo $file
rsyncFile
#提取configmap内容到临时文件
yq 'select(.kind == "ConfigMap") |.data' $file >> $tmpDir/configmap.yaml
yq 'select(.kind == "Deployment") | .spec.template.spec.volumes[].persistentVolumeClaim.claimName|select(. != null)' $file >> $tmpDir/pvc.yaml
fi
done
addRsyncCfg
addGlobalCfg
addPvcCfg
}
function addPvcCfg(){
pvcs=`awk '!a[$1]++{print $0}' $tmpDir/pvc.yaml `
for pvcName in $pvcs
do
pvcFile="$overlayPath/config/$pvcName-pvc.yaml"
name=$pvcName yq '.metadata.name = env(name)' "./attachment/fragment/pvc.yaml" > $pvcFile
pvcFile="./config/$pvcName-pvc.yaml" yq '.resources += env(pvcFile)' -i $overlayPath"/kustomization.yaml"
done
}
function addGlobalCfg(){
configmap=`awk '!a[$1]++&&(/^DB/||/^REDIS/||/^NACOS/){print $0}' $tmpDir/configmap.yaml |sort`
cmap="$configmap" yq -n '.apiVersion="v1"|.kind="ConfigMap"|.metadata.name="replace-cm"|.data = env(cmap)|.$patch = "replace" ' > $overlayPath/config/config.yaml
yq '.patches += {"path":"./config/config.yaml","target":{"kind":"ConfigMap"} } ' -i $overlayPath"/kustomization.yaml"
}
function addRsyncCfg(){
cp ./attachment/rsync/rsync-cm.yaml $overlayPath/config/rsync-cm.yaml
yq '.resources += "./config/rsync-cm.yaml" ' -i $overlayPath"/kustomization.yaml"
vars=`cat ./attachment/rsync/rsync.vars` yq '.vars += env(vars)' -i $overlayPath"/kustomization.yaml"
}
function rsyncFile(){
s3cmd_count=`yq 'select(.kind == "Deployment") | .spec.template.spec.initContainers[].args[0] | select(.=="s3cmd*")' $file | wc -l`
if [ $s3cmd_count -gt 0 ]; then
orignFileShortName=`echo $(basename $file)|cut -d "." -f1 `
patchFile="$overlayPath/$orignFileShortName/$(basename $file)"
initTargetFile
dir=$tmpDir/$orignFileShortName
mkdir -p $dir
yq 'select(.kind == "Deployment") | .spec.template.spec.initContainers[] | select(.args[0] =="s3cmd*")|split_doc ' $file | dir=$dir yq -s 'env(dir) + "/"+ $index + ".yaml" '
# find $dir -type f |xargs -i bash -c 'localRsync=`yq '.args[0]' {}`'
index=0
for splitFile in `find $dir -type f `
do
localRsync=`yq '.args[0]' $splitFile | awk 'FNR==NR {dict[$2]=$0; next}{if($3 in dict) {split(dict[$3],arr," ");$1="pull -u $(pullUser) -p $(pullPwd) -h $(pullHost) -s $(basePath)/resource/$(project)/data/files/"arr[4];$2="-t"; $3=arr[3];$4="";$5="";} if($0!="")print $1,$2,$3}' ./data/data.txt -`
printf -v rsyncV "$localRsync" ; rsyncV="$rsyncV" name=`yq '.name' $splitFile ` rsyncImage="images.knowdee.com/library/rsync:v1" idx=$index yq 'with(.spec.template.spec.initContainers[env(idx)];.$patch = "replace" | .name=env(name)| .image=env(rsyncImage) |.args[0]=strenv(rsyncV) )' -i $patchFile
let index+=1
done
releativePath="./$orignFileShortName/$(basename $file)"
targetFile="$releativePath" yq '.patches += {"path":env(targetFile)} ' -i $overlayPath"/kustomization.yaml"
fi
}
function initTargetFile(){
mkdir -p $(dirname $patchFile)
if [ -z $patchFile ] ;then
apiVersion=`yq 'select(.kind == "Deployment") | .apiVersion' $file ` kind=`yq 'select(.kind == "Deployment") | .kind' $file ` name=`yq 'select(.kind == "Deployment") | .metadata.name' $file ` yq '.kind=env(kind)|.apiVersion=env(apiVersion)|.metadata.name=env(name)' $patchFile
else
yq 'select(.kind == "Deployment") | pick (["apiVersion","kind","metadata"]) | del(.metadata.labels) ' $file > $patchFile
fi
}
#初始化目录
initDir
#抽取镜像名称
image $1
#抽取资源文件
resource $1
basePath=./graph/files/base
#抽取全部编排文件
baseFile $1 $basePath
overlayFile $1 $basePath