zeaslity/ProjectOctopus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[ Cmii ] [ Octopus ] - image sync refresh

Browse Source
This commit is contained in:
zeaslity
2024-03-18 16:38:54 +08:00
parent 5e80d7baad
commit df44d6eef9
31 changed files with 360 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmii_operator/image/CmiiDependencyImageConfig.go
View File

@@ -25,6 +25,7 @@ var MiddlewareAmd64 = []string{
"nginx:1.21.3",
"redis:6.0.20-alpine",
"dyrnq/nfs-subdir-external-provisioner:v4.0.2",
"jerrychina2020/rke-tools:v0.175-linux",
"busybox:latest",
}
@@ -85,7 +86,6 @@ var Rancher1204Amd64 = []string{
"rancher/rancher-webhook:v0.1.0-beta9",
"rancher/rancher:v2.5.7",
"rancher/rke-tools:v0.1.72",
"jerrychina2020/rke-tools:v0.175-linux",
"rancher/security-scan:v0.1.14",
"rancher/security-scan:v0.2.2",
"rancher/shell:v0.1.6",

79
cmii_operator/image/CmiiImageSync.go
View File

@@ -17,6 +17,7 @@ import (
"strings"
"wdd.io/agent-go/executor"
"wdd.io/agent-go/logger"
"wdd.io/agent-go/utils"
)
var apiClient = newClient()
@@ -109,7 +110,7 @@ func Delete(imageName string) []types.ImageDeleteResponseItem {
func PruneAllCmiiImages() (errorRemoveImageNameList []string) {
apiClient.ImagesPrune(context.TODO(), filters.Args{})
_, _ = apiClient.ImagesPrune(context.TODO(), filters.Args{})
imageGetAll := GetAll()
@@ -134,6 +135,8 @@ func PruneAllCmiiImages() (errorRemoveImageNameList []string) {
}
}
_, _ = apiClient.ImagesPrune(context.TODO(), filters.Args{})
return errorRemoveImageNameList
}
@@ -175,11 +178,15 @@ func PushToOctopusKindHarbor(targetImageName string) (pushResult io.ReadCloser)
return pushResult
}
// TagFromListAndPushToCHarbor 需要支持 harbor.cdcyy.cn ip:8033 rancher/rancher:v2.5.7 nginx:latest
func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost string) (errorPushImageNameList []string) {
for _, imageName := range referenceImageList {
// check image
// harbor.cdcyy.cn
cmiiImageFullName := imageName
if strings.HasPrefix(imageName, "cmii") {
cmiiImageFullName = CmiiHarborPrefix + imageName
}
@@ -190,7 +197,10 @@ func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost s
}
if strings.Contains(imageName, "/") {
//
imageName = strings.Split(imageName, "/")[1]
} else {
}
targetImageName := targetHarborHost + ":8033/" + targetProject + "/" + imageName
@@ -232,14 +242,14 @@ func PullFromCmiiHarbor(imageName string) (pullResult io.ReadCloser) {
return pullResult
}
func ImagePullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode bool) (errorPullImageList []string) {
func PullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode bool) (fullImageNameList, errorPullImageList []string) {
fullImageNameList := ConvertCMiiImageMapToList(imageVersionMap)
return ImagePullFromFullNameList(fullImageNameList)
fullImageNameList = ConvertCMiiImageMapToList(imageVersionMap)
return fullImageNameList, PullFromFullNameList(fullImageNameList)
}
func ImagePullCMiiFromFileJson(filePathName string) {
func PullCmiiFromFileJson(filePathName string) {
readFile, err := os.ReadFile(filePathName)
if err != nil {
@@ -274,7 +284,8 @@ func ImagePullCMiiFromFileJson(filePathName string) {
}
func ImagePullFromFullNameList(fullImageNameList []string) (errorPullImageList []string) {
// PullFromFullNameList 根据镜像名列表拉取全部的镜像
func PullFromFullNameList(fullImageNameList []string) (errorPullImageList []string) {
for _, dep := range fullImageNameList {
@@ -301,7 +312,7 @@ func ImagePullFromFullNameList(fullImageNameList []string) (errorPullImageList [
func PullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string) (errorPullImageList, errorGzipImageList []string) {
errorPullImageList = ImagePullFromFullNameList(fullImageNameList)
errorPullImageList = PullFromFullNameList(fullImageNameList)
// generate a project folder
err := os.MkdirAll(gzipFolder, os.ModeDir)
@@ -311,16 +322,22 @@ func PullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string)
}
}
var tarGzipFileNameList []string
for _, image := range fullImageNameList {
if !SaveToTarGZ(image, gzipFolder) {
errorGzipImageList = append(errorGzipImageList, image)
continue
}
tarGzipFileNameList = append(tarGzipFileNameList, convertImageGzipFileName(image))
}
utils.BeautifulPrintListWithTitle(tarGzipFileNameList, "image gzip name list")
return errorPullImageList, errorGzipImageList
}
func ImageLoadFromFile(gzipFullPath string) bool {
func LoadFromGzipFilePath(gzipFullPath string) bool {
openFile, err := os.OpenFile(gzipFullPath, 0, fs.ModePerm)
if err != nil {
log.ErrorF("[ImageLoadFromFile] - failed to open file %s, error is %s", gzipFullPath, err.Error())
@@ -344,7 +361,7 @@ func ImageLoadFromFile(gzipFullPath string) bool {
return true
}
func ImageLoadFromFolderPath(folderPath string) (errorLoadImageNameList []string) {
func LoadFromFolderPath(folderPath string) (errorLoadImageNameList []string) {
if !strings.HasSuffix(folderPath, "/") {
folderPath += "/"
}
@@ -358,7 +375,7 @@ func ImageLoadFromFolderPath(folderPath string) (errorLoadImageNameList []string
// load gzip file
for _, dirEntry := range dirEntries {
if strings.HasSuffix(dirEntry.Name(), ".tar.gz") {
if !ImageLoadFromFile(folderPath + dirEntry.Name()) {
if !LoadFromGzipFilePath(folderPath + dirEntry.Name()) {
errorLoadImageNameList = append(errorLoadImageNameList, folderPath+dirEntry.Name())
}
}
@@ -385,6 +402,7 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool {
for _, repoTag := range imageGetByName.RepoTags {
if !strings.Contains(repoTag, "8033") {
realImageTag = repoTag
break
}
}
@@ -393,9 +411,9 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool {
folderPathPrefix += "/"
}
_ = os.MkdirAll(folderPathPrefix, os.ModeDir)
gzipImageFile = folderPathPrefix + gzipImageFile
log.InfoF("[ImageSaveToTarGZ] - start to save [%s] to [%s]", realImageTag, gzipImageFile)
_ = os.Remove(gzipImageFile)
tarFile, err := os.Create(gzipImageFile)
if err != nil {
@@ -420,9 +438,19 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool {
return true
}
// convertImageGzipFileName 必须输出长度完全一致的内容
// convertImageGzipFileName 必须输出长度为4的内容 =出现得次数为3
func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) {
// harbor.cdcyy.cn/cmii/cmii-uav-platform:5.4.0 ==> cmlc=cmii=cmii-uav-platform=5.4.0.tar.gz
// rancher/fleet:v0.3.4
// ossr/srs:v5.0.1 ==> docker=cmii=srs=v5.0.1.tar.gz
// nginx:latest
// bitnami/minio:2022.5.4
// simonrupf/chronyd:0.4.3
// 10.1.1.1:8033/cmii/ok:1.2 不支持 不允许存在
split := strings.Split(imageRepoTag, ":")
//log.DebugF(" %s to %s", imageRepoTag, split)
if len(split) == 1 {
@@ -433,28 +461,23 @@ func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) {
first := strings.Split(split[0], "/")
//log.DebugF(" split[0] %s to %s", split[0], first)
if len(first) == 3 {
if strings.Contains(first[0], "cdcyy") {
gzipFileName += "cmlc="
// harbor.cdcyy.cn/cmii/cmii-uav-platform:5.4.0
// docker.io/ossr/srs:v5.0.1
if strings.HasPrefix(split[0], CmiiHarborPrefix) {
gzipFileName += "cmlc=cmii="
} else {
gzipFileName += "docker="
gzipFileName += "docker=cmii="
}
gzipFileName += first[1]
gzipFileName += "="
gzipFileName += first[2]
gzipFileName += "="
} else if len(first) == 4 {
if strings.Contains(first[0], "cdcyy") {
gzipFileName += "cmlc="
} else {
gzipFileName += "docker="
// harbor.cdcyy.cn/cmii/ossr/srs:v5.0.1
if !strings.HasPrefix(split[0], CmiiHarborPrefix) {
return imageRepoTag
}
gzipFileName += first[1]
gzipFileName += "="
gzipFileName += first[2]
gzipFileName += "cmlc=cmii="
gzipFileName += first[3]
gzipFileName += "="
} else if len(first) == 2 {
// bitnami/redis
@@ -465,7 +488,7 @@ func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) {
gzipFileName += first[1]
gzipFileName += "="
} else if len(first) == 1 {
// nginx
// nginx:latest
return "docker=library=" + split[0] + "=" + split[1] + ".tar.gz"
}

10
cmii_operator/image/CmiiImageSync_test.go
View File

@@ -100,7 +100,7 @@ func TestImagePushToOctopusKindHarbor(t *testing.T) {
}
func TestImageLoadFromFile(t *testing.T) {
loadFromFile := ImageLoadFromFile("/root/octopus_image/cqga/cqga/cmlc=cmii=cmii-uav-user=5.2.0.tar.gz")
loadFromFile := LoadFromGzipFilePath("/root/octopus_image/cqga/cqga/cmlc=cmii=cmii-uav-user=5.2.0.tar.gz")
assert.Equal(t, loadFromFile, true, "image load error !")
}
@@ -117,20 +117,12 @@ func TestConvertImageGzipFileName(t *testing.T) {
test := []string{
"bitnami/redis:6.2.6-debian-10-r0",
"bitnami/mysql:8.1.0-debian-11-r42",
"simonrupf/chronyd:0.4.3",
"bitnami/bitnami-shell:11-debian-11-r136",
"harbor.cdcyy.com.cn/cmii/cmii-rtsp-operator:v4.1.0",
"harbor.cdcyy.com.cn/cmii/ossrs/srs:v4.0.136",
"bitnami/rabbitmq:3.9.12-debian-10-r3",
"ossrs/srs:v4.0.136",
"emqx/emqx:4.2.12",
"nacos/nacos-server:v2.1.2",
"mongo:5.0",
"bitnami/minio:2023.5.4",
"kubernetesui/metrics-scraper:v1.0.4",
"redis:6.0.20-alpine",
"dyrnq/nfs-subdir-external-provisioner:v4.0.2",
"busybox:latest",
"busybox",
}

49
cmii_operator/image/image_sync.sh
View File

@@ -6,29 +6,29 @@ cmii_image_list=(
docker=ossrs=srs=v4.0.136.tar.gz
)
middle_image_list=(
cmlc=cmii=nfs-subdir-external-provisioner=v4.0.2.tar.gz
docker=busybox=latest.tar.gz
cmlc=cmii=srs=v4.0-r3.tar.gz
docker=emqx=emqx=4.2.12.tar.gz
docker=bitnami=bitnami-shell=10-debian-10-r140.tar.gz
docker=kubernetesui=dashboard=v2.0.1.tar.gz
docker=bitnami=bitnami-shell=11-debian-11-r136.tar.gz
docker=kubernetesui=metrics-scraper=v1.0.4.tar.gz
docker=bitnami=minio=2022.5.4.tar.gz
docker=mongo=5.0.tar.gz
docker=library=busybox=latest.tar.gz
docker=bitnami=minio=2023.5.4.tar.gz
docker=nacos=nacos-server=v2.1.2-slim.tar.gz
docker=library=mongo=5.0.tar.gz
docker=bitnami=mysql=8.0.35-debian-11-r1.tar.gz
docker=nginx=1.21.3.tar.gz
docker=library=nginx=1.21.3.tar.gz
docker=bitnami=mysql=8.1.0-debian-11-r42.tar.gz
docker=ossrs=srs=v4.0.136.tar.gz
docker=library=rabbitmq=3.9-management.tar.gz
docker=bitnami=rabbitmq=3.11.26-debian-11-r2.tar.gz
docker=ossrs=srs=v5.0.195.tar.gz
docker=library=redis=6.0.20-alpine.tar.gz
docker=bitnami=rabbitmq=3.9.12-debian-10-r3.tar.gz
docker=rabbitmq=3.9-management.tar.gz
docker=nacos=nacos-server=v2.1.2-slim.tar.gz
docker=bitnami=redis=6.2.14-debian-11-r1.tar.gz
docker=redis=6.0.20-alpine.tar.gz
docker=ossrs=srs=v4.0.136.tar.gz
docker=bitnami=redis=6.2.6-debian-10-r0.tar.gz
docker=ossrs=srs=v4.0-r3.tar.gz
docker=dyrnq=nfs-subdir-external-provisioner=v4.0.2.tar.gz
docker=ossrs=srs=v5.0.195.tar.gz
docker=emqx=emqx=4.2.12.tar.gz
docker=simonrupf=chronyd=0.4.3.tar.gz
)
rke_image_list=(
@@ -161,19 +161,20 @@ image_load_to_harbor() {
echo
done
# for image in "${middle_image_list[@]}"; do
# echo "start to load => $image"
# docker load <"$image"
# echo ""
# local app_name_prefix=$(echo $image | cut -d "=" -f2)
# local app_name=$(echo $image | cut -d "=" -f3)
# local ccc=$(echo $image | cut -d "=" -f4)
# local app_tag="${ccc%.tar.gz}"
# echo "from $app_name_prefix/$app_name:$app_tag ==> $target_harbor_host/rancher/$app_name:$app_tag"
# echo "start to push => $target_harbor_host/rancher/$app_name:$app_tag"
# docker login -u admin -p V2ryStr@ngPss $target_harbor_host
# docker push "$target_harbor_host/rancher/$app_name:$app_tag"
# done
for image in "${middle_image_list[@]}"; do
echo "start to load => $image"
docker load <"$image"
echo ""
# docker=bitnami=redis=6.2.14-debian-11-r1.tar.gz
local app_name_prefix=$(echo $image | cut -d "=" -f2)
local app_name=$(echo $image | cut -d "=" -f3)
local ccc=$(echo $image | cut -d "=" -f4)
local app_tag="${ccc%.tar.gz}"
echo "from $app_name_prefix/$app_name:$app_tag ==> $target_harbor_host/cmii/$app_name:$app_tag"
echo "start to push => $target_harbor_host/cmii/$app_name:$app_tag"
docker login -u admin -p V2ryStr@ngPss $target_harbor_host
docker push "$target_harbor_host/cmii/$app_name:$app_tag"
done
}