[Agent][Deploy] - 修改ImageSync模块 DCU的核心内容

agent-operator/ImageSyncOperator.go

@@ -3,6 +3,7 @@ package main
 import (
 	"errors"
 	"os"
+	"path/filepath"
 	"slices"
 	"strings"
 	image2 "wdd.io/agent-common/image"
@@ -11,20 +12,19 @@ import (
 	"wdd.io/agent-operator/image"
 )
 
-const OfflineDeployHarborHost = "harbor.wdd.io"
+const (
-const PublicDeployHarborHost = "42.192.52.227"
+	OfflineDeployHarborHost    = "harbor.wdd.io"
-const DirectPushDeployHarborHost = "chongqingcis-9b4a3da9.ecis.chongqing-1.cmecloud.cn"
+	PublicDeployHarborHost     = "42.192.52.227"
+	DirectPushDeployHarborHost = "chongqingcis-9b4a3da9.ecis.chongqing-1.cmecloud.cn"
+
+	AllCmiiImageListLocalFileName = "all-cmii-image-list.txt"
+	AllGzipImageLocalFileName     = "all-gzip-image-list.txt"
+)
 
 type ImageSyncEntity struct {
-	ProjectName    string // 优先级3 优先级最低 从DEMO拉取镜像
+	DownloadCondition *DownloadEntity // D的条件
-	ProjectVersion string // 优先级2 高于ProjectName 优先拉取特定版本的镜像
+	CompressCondition *CompressEntity // C的条件
-
+	UploadCondition   *UploadEntity   // U的条件
-	CmiiNameTagList   []string // 优先级1 appName:tag 会被转换为FullNameImageList
-	FullNameImageList []string // 优先级1 优先下载此类型
-
-	ShouldDownloadImage       bool // 下载镜像 DCU中的D
-	ShouldCompressImageToGzip bool // 压缩镜像 DCU中的C
-	ShouldUploadToDemoMinio   bool // 上传镜像 DCU中的U
 
 	ShouldDownloadFromOss bool // 下载镜像 DLTU中的D
 	ShouldUpdateImageTag  bool // 更新镜像 DLTU中的U
@@ -33,75 +33,99 @@ type ImageSyncEntity struct {
 	DirectHarborHost         string // IP:Port or 域名:PORT 不带http前缀
 }
 
+// DownloadEntity DCU中的D的条件
+type DownloadEntity struct {
+	ShouldDownloadImage bool // 下载镜像 DCU中的D 实际无用
+
+	ProjectName    string // 优先级3 优先级最低 从DEMO拉取镜像
+	ProjectVersion string // 优先级2 高于ProjectName 优先拉取特定版本的镜像
+
+	CmiiNameTagList   []string // 优先级1 appName:tag 会被转换为FullNameImageList
+	FullNameImageList []string // 优先级1 优先下载此类型
+
+	DownloadAuthUserName string // 下载需要认证的用户名
+	DownloadAuthPassword string // 下载需要认证的密码
+}
+
+// CompressEntity DCU中的C的条件
+type CompressEntity struct {
+	ShouldCompressImageToGzip bool   // 压缩镜像 DCU中的C
+	ShouldGzipSplit           bool   // 压缩镜像 是否应该分割存储  true=独立存储 false=整个存储
+	GzipLocalFolder           string // 压缩镜像 保存压缩镜像文件的本地目录
+}
+
+// UploadEntity DCU中的U的条件
+type UploadEntity struct {
+	ShouldUploadToDemoMinio bool // 上传镜像 DCU中的U
+}
+
 type ImageSyncResult struct {
-	ErrorPullImageList     []string
+	ProcedureSuccessImageList []string // 经过特定步骤之后成功的镜像
-	ErrorGzipImageList     []string
+
-	ErrorPushImageNameList []string
+	DownloadResult *DownloadResultEntity
-	RealImageNameList      []string
+	CompressResult *CompressResultEntity
-	RealGzipFileNameList   []string
+	UploadResult   *UploadResultEntity
-	AllCmiiImageNameList   []string
+}
+
+type DownloadResultEntity struct {
+	ErrorPullImageList              []string // 下载镜像 DCU中的D 下载失败的镜像
+	SuccessPullImageList            []string // 下载镜像 DCU中的D 下载成功的镜像
+	SuccessPullTxtFileLocalFullPath string   // 下载镜像 DCU中的D 下载成功的镜像保存的文件地址 GzipLocalFolder + all
+}
+
+type CompressResultEntity struct {
+	ErrorGzipImageList       []string // 压缩镜像 DCU中的C 压缩失败的镜像
+	SuccessGzipImageList     []string // 压缩镜像 DCU中的C 压缩成功的镜像
+	GzipTxtFileLocalFullPath string   // 压缩镜像 DCU中的C 压缩镜像保存的目录
+}
+
+type UploadResultEntity struct {
+	ErrorUploadImageList []string // 上传镜像 DCU中的U 上传失败的镜像
+	AllDownloadUrl       []string // 上传镜像 DCU中的U 正式的下载地址列表i
 }
 
 // PullFromEntityAndSyncConditionally 根据ImageSyncEntity拉取特定的镜像，然后上传到特定的目标机器(或者上传的minio中)
-func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResult ImageSyncResult) {
+func (syncCondition *ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResult *ImageSyncResult) {
 
-	var realCmiiImageList []string
+	imageSyncResult = &ImageSyncResult{}
-	var allCmiiImageNameList []string
 
-	var errorPullImageList []string
+	if (syncCondition.DownloadCondition.CmiiNameTagList == nil && syncCondition.DownloadCondition.FullNameImageList == nil) || (len(syncCondition.DownloadCondition.CmiiNameTagList) == 0 && len(syncCondition.DownloadCondition.FullNameImageList) == 0) {
-	var allGzipFileNameList []string
-	var errorGzipImageList []string
-	var errorPushImageNameList []string
-	var gzipFolderFullPath string
-
-	if (sync.CmiiNameTagList == nil && sync.FullNameImageList == nil) || (len(sync.CmiiNameTagList) == 0 && len(sync.FullNameImageList) == 0) {
 		// 没有指定特定的镜像，那么根据 ProjectVersion 或者从DEMO拉取镜像
 		// pull images
 		// compress
-		if sync.ProjectVersion != "" {
+		if syncCondition.DownloadCondition.ProjectVersion != "" {
 
 			// 获取特定版本的镜像
-			errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = C_DownloadCompressUploadFromVersion(sync.ProjectVersion, sync.ShouldCompressImageToGzip, sync.ShouldUploadToDemoMinio)
+			C_DownloadCompressUploadFromVersion(syncCondition, imageSyncResult)
-
-			gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectVersion
 
 		} else {
-
 			// 获取DEMO的镜像
-			errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = C_DownloadCompressUploadFromDemo(sync.ProjectName, sync.ShouldCompressImageToGzip, sync.ShouldUploadToDemoMinio)
+			C_DownloadCompressUploadFromDemo(syncCondition, imageSyncResult)
-			gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectName
 
 		}
 	} else {
-		// 拉取特定的镜像
+		// 根据列表拉取镜像
 
-		gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + "tmp"
 		// 组装镜像名称
-		allCmiiImageNameList = concatAndUniformCmiiImage(sync.FullNameImageList, sync.CmiiNameTagList)
+		syncCondition.DownloadCondition.FullNameImageList = concatAndUniformCmiiImage(syncCondition.DownloadCondition.FullNameImageList, syncCondition.DownloadCondition.CmiiNameTagList)
+
+		// gzip file folder path
+		syncCondition.CompressCondition.GzipLocalFolder = filepath.Join(image.OfflineImageGzipFolderPrefix, "tmp")
 
 		// DCU
-		errorPullImageList, errorGzipImageList, realCmiiImageList, allGzipFileNameList = A_DownloadCompressUpload(true, allCmiiImageNameList, sync.ShouldCompressImageToGzip, gzipFolderFullPath, sync.ShouldUploadToDemoMinio)
+		A_DownloadCompressUpload(syncCondition, imageSyncResult)
 	}
 
 	// 直接传输到目标Harbor仓库
-	if sync.ShouldDirectPushToHarbor {
+	if syncCondition.ShouldDirectPushToHarbor {
-		if sync.DirectHarborHost == "" {
+		if syncCondition.DirectHarborHost == "" {
 			log.ErrorF("DirectHarborHost is null ! can't push to target harbor !")
 		}
 		// push to
-		errorPushImageNameList = image.TagFromListAndPushToCHarbor(allCmiiImageNameList, sync.DirectHarborHost)
+		//errorPushImageNameList = image.TagFromListAndPushToCHarbor(allCmiiImageNameList, syncCondition.DirectHarborHost)
 	}
 
 	// build result
-	imageSyncResult.AllCmiiImageNameList = allCmiiImageNameList
-
-	imageSyncResult.RealImageNameList = realCmiiImageList
-	imageSyncResult.ErrorPullImageList = errorPullImageList
-
-	imageSyncResult.RealGzipFileNameList = allGzipFileNameList
-	imageSyncResult.ErrorGzipImageList = errorGzipImageList
-
-	imageSyncResult.ErrorPushImageNameList = errorPushImageNameList
 
 	return imageSyncResult
 }
@@ -123,66 +147,75 @@ func concatAndUniformCmiiImage(fullImageList []string, cmiiImageList []string) [
 }
 
 // A_DownloadCompressUpload DCU 镜像同步的前半部分，通常在35.71  LapPro执行，无需Bastion Mode
-func A_DownloadCompressUpload(downloadImage bool, fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileFullNameList []string) {
+func A_DownloadCompressUpload(syncEntity *ImageSyncEntity, syncResult *ImageSyncResult) {
 
-	// write to file
+	// all image full name list need to download
-	localGzipFileListTxt := gzipFolderFullPath + "all-gzip-image-file-name.txt"
+	fullNameList := syncEntity.DownloadCondition.FullNameImageList
 
 	// Download
-	log.Info("DOWNLOAD START !")
+	log.Info("[DCU] - DOWNLOAD START !")
-	if downloadImage {
+	if syncEntity.DownloadCondition.ShouldDownloadImage && fullNameList != nil && len(fullNameList) > 0 {
-		if fullNameList == nil || len(fullNameList) == 0 {
-			log.InfoF("no image name list !")
-		} else {
-			errorPullImageList = image.PullFromFullNameList(fullNameList)
-		}
-	}
 
-	// remove failed
+		syncResult.DownloadResult.ErrorPullImageList = image.PullFromFullNameList(fullNameList)
+
+		// remove failed download image from full name list
 		fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool {
-		return slices.Contains(errorPullImageList, imageName)
+			return slices.Contains(syncResult.DownloadResult.ErrorPullImageList, imageName)
 		})
 
+	} else {
+		log.Info("[DCU] - No Need To Download !")
+	}
+
+	syncResult.ProcedureSuccessImageList = fullNameList
+
+	gzipLocalFolderPath := syncEntity.CompressCondition.GzipLocalFolder
+
 	// Compress
-	if shouldGzip {
+	if syncEntity.CompressCondition.ShouldCompressImageToGzip {
 
 		// remove file
+		localGzipFileListTxt := filepath.Join(gzipLocalFolderPath, AllGzipImageLocalFileName)
 		_ = os.Remove(localGzipFileListTxt)
 
+		// 找到已经存在的压缩文件，跳过
 		gzipFileAlready := make(map[string]bool)
-		if utils.FileOrFolderExists(gzipFolderFullPath) {
+		if utils.FileOrFolderExists(gzipLocalFolderPath) {
-			dir, _ := os.ReadDir(gzipFolderFullPath)
+			dir, _ := os.ReadDir(gzipLocalFolderPath)
 			for _, entry := range dir {
 				if entry.IsDir() {
 					continue
 				}
-				gzipFileAlready[strings.TrimPrefix(entry.Name(), gzipFolderFullPath)] = true
+				gzipFileAlready[strings.TrimPrefix(entry.Name(), gzipLocalFolderPath)] = true
 			}
 		}
 
 		// mkdir folder
-		err := os.MkdirAll(gzipFolderFullPath, os.ModeDir)
+		err := os.MkdirAll(gzipLocalFolderPath, os.ModeDir)
 		if err != nil {
 			if !errors.Is(err, os.ErrExist) {
-				log.ErrorF("create folder error of %s", gzipFolderFullPath)
+				log.ErrorF("create folder error of %s", gzipLocalFolderPath)
 				panic(err)
 			}
 		}
 
 		// 循环遍历压缩
-		log.Info("COMPRESS START")
+		log.Info("[DCU] - COMPRESS START")
+		var errorGzipImageList []string
+		var allGzipFileFullNameList []string
 
+		if syncEntity.CompressCondition.ShouldGzipSplit {
+			// 独立压缩
 			for _, imageFullName := range fullNameList {
 
 				// gzip image file already exists
-
 				gzipFileName := image2.ImageFullNameToGzipFileName(imageFullName)
-			gzipImageFileFullPath := gzipFolderFullPath + gzipFileName
+				gzipImageFileFullPath := gzipLocalFolderPath + gzipFileName
 				_, ok := gzipFileAlready[gzipFileName]
 				if len(gzipFileAlready) > 0 && ok {
 					log.DebugF("gzip file %s already exists !", gzipFileName)
 				} else {
-				ok, gzipImageFileFullPath = image.SaveToGzipFile(imageFullName, gzipFolderFullPath)
+					ok, gzipImageFileFullPath = image.SaveToGzipFile(imageFullName, gzipLocalFolderPath)
 					if !ok {
 						errorGzipImageList = append(errorGzipImageList, imageFullName)
 						continue
@@ -191,51 +224,102 @@ func A_DownloadCompressUpload(downloadImage bool, fullNameList []string, shouldG
 
 				// 压缩成功
 				allGzipFileFullNameList = append(allGzipFileFullNameList, gzipImageFileFullPath)
-		}
+
+				syncResult.CompressResult.SuccessGzipImageList = allGzipFileFullNameList
+				syncResult.CompressResult.ErrorGzipImageList = errorGzipImageList
+
 				// remove failed
 				fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool {
 					return slices.Contains(errorGzipImageList, imageName)
 				})
 
+				// write all gzipped file name to file
 				for _, gzipFileFullName := range allGzipFileFullNameList {
 					utils.AppendContentToFile(
-				strings.TrimPrefix(gzipFileFullName, gzipFolderFullPath)+"\n",
+						strings.TrimPrefix(gzipFileFullName, gzipLocalFolderPath)+"\n",
 						localGzipFileListTxt,
 					)
 				}
+			}
+		} else {
+			// 压缩为一个大的压缩包
+			gzipFileName := generateMonolithicGzipFileName(syncEntity)
 
-		log.InfoF("all gzip file name list is %s", localGzipFileListTxt)
+			ok, gzipFileFullPath, errorGzipImageListTmp := image.SaveImageListToGzipFile(fullNameList, gzipLocalFolderPath, gzipFileName)
+			if !ok {
+				panic("[DCU] - gzip error to a monolithic file !")
 			}
 
+			// write all gzipped file name to file
+			utils.AppendContentToFile(
+				utils.BeautifulPrintToString(fullNameList),
+				localGzipFileListTxt,
+			)
+
+			// remove failed
+			fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool {
+				return slices.Contains(errorGzipImageListTmp, imageName)
+			})
+
+			syncResult.CompressResult.SuccessGzipImageList = fullNameList
+			syncResult.CompressResult.ErrorGzipImageList = errorGzipImageListTmp
+
+			log.InfoF("[DCU] - gzip all image from list to monolithic file %s", gzipFileFullPath)
+		}
+
+		syncResult.CompressResult.GzipTxtFileLocalFullPath = localGzipFileListTxt
+
+		log.InfoF("[DCU] - all gzip file name list is %s", localGzipFileListTxt)
+	}
+
+	syncResult.ProcedureSuccessImageList = fullNameList
+
 	// Upload
-	if shouldOss {
+	if syncEntity.UploadCondition.ShouldUploadToDemoMinio {
 		//uploadGzipFileToDemoMinio()
 		// get gzip file name list
-		log.Info("UPLOAD OSS START !")
+		log.Info("[DCU] - UPLOAD OSS START !")
+
+		var errorUploadOssGzipNameList []string
+		var allDownloadUrl []string
 
 		// start to upload
 		// extract demo oss location suffix from gzipFolderFullPath
-		trimPrefix := strings.TrimPrefix(gzipFolderFullPath, image.OfflineImageGzipFolderPrefix)
+		// 根据本地保存Gzip的目录路径提取到 相应的后缀  项目代码
-		bucketNameWithPrefix := "cmlc-installation/" + trimPrefix
+		// projectName / projectVersion
+		projectUniqueName := strings.TrimPrefix(gzipLocalFolderPath, image.OfflineImageGzipFolderPrefix)
+
+		bucketNameWithPrefix := "cmlc-installation/" + projectUniqueName
 		log.InfoF("gzip file location in demo oss is %s", DefaultDemoEndpoint+"/"+bucketNameWithPrefix)
 
 		// upload gzip file list txt to demo
-		if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipFolderFullPath, strings.TrimPrefix(localGzipFileListTxt, gzipFolderFullPath)) {
+		localGzipFileListTxt := syncResult.CompressResult.GzipTxtFileLocalFullPath
+		if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipLocalFolderPath, strings.TrimPrefix(localGzipFileListTxt, gzipLocalFolderPath)) {
 			log.ErrorF("upload of %s to demo oss error !", localGzipFileListTxt)
 		}
 
 		log.InfoF("upload all gzip file to demo oss !")
-		for _, gzipFileFullName := range allGzipFileFullNameList {
+		for _, gzipFileFullName := range syncResult.CompressResult.SuccessGzipImageList {
 			// SaveToGzipFile 返回的是全路径 归一化处理 gzip file name
-			gzipFileFullName = strings.TrimPrefix(gzipFileFullName, gzipFolderFullPath)
+			gzipFileName := strings.TrimPrefix(gzipFileFullName, gzipLocalFolderPath)
-			if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipFolderFullPath, gzipFileFullName) {
+			if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipLocalFolderPath, gzipFileName) {
-				log.ErrorF("upload of %s to demo oss error !", gzipFileFullName)
+				log.ErrorF("upload of %s to demo oss error !", gzipFileName)
+				errorUploadOssGzipNameList = append(errorUploadOssGzipNameList, gzipFileName)
+			} else {
+				allDownloadUrl = append(allDownloadUrl, DefaultDemoEndpoint+"/"+bucketNameWithPrefix+"/"+gzipFileName)
 			}
 		}
 
+		syncResult.UploadResult.AllDownloadUrl = allDownloadUrl
+		syncResult.UploadResult.ErrorUploadImageList = errorUploadOssGzipNameList
+
 	}
 
-	return errorPullImageList, errorGzipImageList, fullNameList, allGzipFileFullNameList
+}
+
+func generateMonolithicGzipFileName(syncEntity *ImageSyncEntity) string {
+
+	return strings.TrimPrefix(syncEntity.CompressCondition.GzipLocalFolder, image.OfflineImageGzipFolderPrefix)
 }
 
 // A_DownloadLoadTagUpload DLTU procedure ImageSync的另外一般流程，需要支持 堡垒机（纯离线）的模式
@@ -344,23 +428,35 @@ func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string)
 }
 
 // C_DownloadCompressUploadFromDemo 获取DEMO环境的全部镜像
-func C_DownloadCompressUploadFromDemo(projectName string, shouldGzip, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
+func C_DownloadCompressUploadFromDemo(syncEntity *ImageSyncEntity, syncResult *ImageSyncResult) {
 
 	// generate a project folder
-	err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+projectName, os.ModeDir)
+	projectName := syncEntity.DownloadCondition.ProjectName
+	gzipFolderLocalPath := filepath.Join(image.OfflineImageGzipFolderPrefix, projectName)
+
+	err := os.MkdirAll(gzipFolderLocalPath, os.ModeDir)
 	if err != nil {
 		if !errors.Is(err, os.ErrExist) {
-			log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", image.OfflineImageGzipFolderPrefix+projectName, err.Error())
+			log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", gzipFolderLocalPath, err.Error())
-			return errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList
+
 		}
 	}
 
+	syncEntity.CompressCondition.GzipLocalFolder = gzipFolderLocalPath
+
 	// get demo image version map
-	allCmiiImageNameListFromDemo := buildAllCmiiImageNameListFromDemo(projectName)
+	allCmiiImageFullNameList := buildAllCmiiImageNameListFromDemo(projectName)
+
+	// save all cmii image to file
+	allPullImageNameTxtFileName := filepath.Join(gzipFolderLocalPath, AllCmiiImageListLocalFileName)
+	utils.AppendOverwriteContentToFile(utils.BeautifulPrintToString(allCmiiImageFullNameList), allPullImageNameTxtFileName)
+
+	// save to result
+	syncResult.DownloadResult.SuccessPullTxtFileLocalFullPath = allPullImageNameTxtFileName
 
 	// do work
 	// DCU
-	return A_DownloadCompressUpload(true, allCmiiImageNameListFromDemo, shouldGzip, image.OfflineImageGzipFolderPrefix+projectName, shouldOss)
+	A_DownloadCompressUpload(syncEntity, syncResult)
 }
 
 func buildAllCmiiImageNameListFromDemo(projectName string) []string {
@@ -400,26 +496,39 @@ func buildAllCmiiImageNameListFromDemo(projectName string) []string {
 }
 
 // C_DownloadCompressUploadFromVersion 根据版本下载全部的CMII镜像
-func C_DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
+func C_DownloadCompressUploadFromVersion(syncEntity *ImageSyncEntity, syncResult *ImageSyncResult) {
 
 	// generate a project folder
-	err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+cmiiVersion, os.ModeDir)
+	projectCmiiVersion := syncEntity.DownloadCondition.ProjectVersion
+
+	// gzip local path
+	gzipFolderLocalPath := filepath.Join(image.OfflineImageGzipFolderPrefix, projectCmiiVersion)
+
+	err := os.MkdirAll(gzipFolderLocalPath, os.ModeDir)
 	if err != nil {
 		if !errors.Is(err, os.ErrExist) {
-			log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", image.OfflineImageGzipFolderPrefix+cmiiVersion, err.Error())
+			log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", gzipFolderLocalPath, err.Error())
-			return errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList
 		}
 	}
 
+	syncEntity.CompressCondition.GzipLocalFolder = gzipFolderLocalPath
+
 	// build all cmii image name list
-	realCmiiImageName = buildAllCmiiImageNameListFromVersion(cmiiVersion)
+	allCmiiImageFullNameList := buildAllCmiiImageNameListFromVersion(projectCmiiVersion)
+
+	// save all cmii image to file
+	allImageListTxtFileFullName := filepath.Join(gzipFolderLocalPath, AllCmiiImageListLocalFileName)
+	utils.AppendOverwriteContentToFile(utils.BeautifulPrintToString(allCmiiImageFullNameList), allImageListTxtFileFullName)
+
+	// save to result
+	syncResult.DownloadResult.SuccessPullTxtFileLocalFullPath = allImageListTxtFileFullName
 
 	// do work
 	// DCU procedure
-	return A_DownloadCompressUpload(true, realCmiiImageName, shouldGzip, image.OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss)
+	A_DownloadCompressUpload(syncEntity, syncResult)
-
 }
 
+// buildAllCmiiImageNameListFromVersion 根据VersionTag构建完整的应用名称列表
 func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string {
 
 	var realCmiiImageName []string
@@ -458,7 +567,7 @@ func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string {
 }
 
 // C_DownloadCompressUploadDependency DCU所有的依赖镜像
-func C_DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, shouldDownload bool, isRKE bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
+func C_DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, isRKE bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
 
 	log.Info("DCU for middle and rke!")
 	err := os.MkdirAll(image.OfflineImageGzipFolderPrefix, os.ModeDir)
@@ -468,21 +577,47 @@ func C_DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, shouldD
 		}
 	}
 
-	var fulleImageNameList []string
+	var fullImageNameList []string
 	var gzipFolderPrefix string
 
 	if isRKE {
 		log.Info("DCU for rke!")
-		fulleImageNameList = d_app.Rancher1204Amd64
+		fullImageNameList = d_app.Rancher1204Amd64
 		gzipFolderPrefix = image.OfflineImageGzipFolderPrefix + "rke/"
 	} else {
 		log.Info("DCU for middle!")
 
-		fulleImageNameList = d_app.MiddlewareAmd64
+		fullImageNameList = d_app.MiddlewareAmd64
 		gzipFolderPrefix = image.OfflineImageGzipFolderPrefix + "middle/"
 	}
 
-	return A_DownloadCompressUpload(shouldDownload, fulleImageNameList, shouldGzip, gzipFolderPrefix, shouldOss)
+	syncEntity := &ImageSyncEntity{
+		DownloadCondition: &DownloadEntity{
+			ShouldDownloadImage:  false,
+			ProjectName:          "",
+			ProjectVersion:       "",
+			CmiiNameTagList:      nil,
+			FullNameImageList:    fullImageNameList,
+			DownloadAuthUserName: "",
+			DownloadAuthPassword: "",
+		},
+		CompressCondition: &CompressEntity{
+			ShouldCompressImageToGzip: shouldGzip,
+			ShouldGzipSplit:           true,
+			GzipLocalFolder:           gzipFolderPrefix,
+		},
+		UploadCondition:          &UploadEntity{ShouldUploadToDemoMinio: shouldOss},
+		ShouldDownloadFromOss:    false,
+		ShouldUpdateImageTag:     false,
+		ShouldDirectPushToHarbor: false,
+		DirectHarborHost:         "",
+	}
+
+	syncResult := &ImageSyncResult{}
+
+	A_DownloadCompressUpload(syncEntity, syncResult)
+
+	return syncResult.DownloadResult.ErrorPullImageList, syncResult.CompressResult.ErrorGzipImageList, syncResult.ProcedureSuccessImageList, syncResult.CompressResult.SuccessGzipImageList
 }
 
 func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) {

agent-operator/ImageSyncOperator_test.go

@@ -11,7 +11,7 @@ import (
 
 func TestFetchDependencyRepos_Middle(t *testing.T) {
 
-	errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := C_DownloadCompressUploadDependency(true, true, false, false)
+	errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := C_DownloadCompressUploadDependency(true, false, false)
 
 	utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList")
 	utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList")
@@ -22,7 +22,7 @@ func TestFetchDependencyRepos_Middle(t *testing.T) {
 
 func TestFetchDependencyRepos_RKE(t *testing.T) {
 
-	errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := C_DownloadCompressUploadDependency(true, true, false, true)
+	errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := C_DownloadCompressUploadDependency(true, false, true)
 
 	utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList")
 	utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList")

agent-operator/image/CmiiImageOperator.go

@@ -13,6 +13,7 @@ import (
 	"io"
 	"io/fs"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
@@ -445,10 +446,11 @@ func SaveToGzipFile(imageFullName, folderPathPrefix string) (gzipOK bool, gzipIm
 	}
 
 	gzipImageFileFullPath = image2.ImageFullNameToGzipFileName(realImageTag)
-	if !strings.HasSuffix(folderPathPrefix, "/") {
+
-		folderPathPrefix += "/"
+	if err := os.MkdirAll(filepath.Dir(gzipImageFileFullPath), os.ModePerm); err != nil {
+		log.ErrorF("[ImageSaveToTarGZ] - failed to create directory: %s", err)
+		return false, ""
 	}
-	_ = os.MkdirAll(folderPathPrefix, os.ModeDir)
 
 	// 生成gzip压缩文件的全路径名称
 	gzipImageFileFullPath = folderPathPrefix + gzipImageFileFullPath
@@ -456,7 +458,10 @@ func SaveToGzipFile(imageFullName, folderPathPrefix string) (gzipOK bool, gzipIm
 	log.InfoF("[ImageSaveToTarGZ] - start to save [%s] to [%s]", realImageTag, gzipImageFileFullPath)
 
 	// 删除掉旧的Gzip文件
-	_ = os.Remove(gzipImageFileFullPath)
+	if err := os.Remove(gzipImageFileFullPath); err != nil && !os.IsNotExist(err) {
+		log.ErrorF("[ImageSaveToTarGZ] - failed to remove old gzip file: %s", err)
+		return false, ""
+	}
 
 	// 创建
 	tarFile, err := os.Create(gzipImageFileFullPath)
@@ -483,6 +488,76 @@ func SaveToGzipFile(imageFullName, folderPathPrefix string) (gzipOK bool, gzipIm
 	return true, gzipImageFileFullPath
 }
 
+// SaveImageListToGzipFile 将一个列表内的镜像全部压缩为一个tar.gz文件
+func SaveImageListToGzipFile(imageFullNames []string, folderPathPrefix string, outputFileName string) (gzipOK bool, gzipFileFullPath string, errorGzipImageList []string) {
+
+	if len(imageFullNames) == 0 {
+		log.Error("[SaveImagesToGzipFile] - no images provided")
+		return false, "", errorGzipImageList
+	}
+
+	// 确保输出文件路径
+	if err := os.MkdirAll(filepath.Dir(folderPathPrefix), os.ModePerm); err != nil {
+		log.ErrorF("[SaveImagesToGzipFile] - failed to create directory: %s", err)
+		return false, "", errorGzipImageList
+	}
+
+	gzipFileFullPath = filepath.Join(folderPathPrefix, outputFileName)
+	log.InfoF("[SaveImagesToGzipFile] - start saving images to [%s]", gzipFileFullPath)
+
+	// 删除旧的Gzip文件
+	if err := os.Remove(gzipFileFullPath); err != nil && !os.IsNotExist(err) {
+		log.ErrorF("[SaveImagesToGzipFile] - failed to remove old gzip file: %s", err)
+		return false, "", errorGzipImageList
+	}
+
+	tarFile, err := os.Create(gzipFileFullPath)
+	if err != nil {
+		log.ErrorF("[SaveImagesToGzipFile] - error creating gzip file: %s", err)
+		return false, "", errorGzipImageList
+	}
+	defer tarFile.Close()
+
+	gw, err := pgzip.NewWriterLevel(tarFile, pgzip.DefaultCompression)
+	if err != nil {
+		log.ErrorF("[SaveImagesToGzipFile] - pgzip writer creation error: %s", err)
+		return false, "", errorGzipImageList
+	}
+	defer gw.Close()
+
+	errorGzipImageList = []string{}
+
+	for _, imageFullName := range imageFullNames {
+		imageGetByName := GetByName(imageFullName)
+		if imageGetByName == nil {
+			log.WarnF("[SaveImagesToGzipFile] - %s not exists, skipping", imageFullName)
+			errorGzipImageList = append(errorGzipImageList, imageFullName)
+			continue
+		}
+
+		imageSaveTarStream, err := apiClient.ImageSave(context.TODO(), imageGetByName.RepoTags)
+		if err != nil {
+			log.ErrorF("[SaveImagesToGzipFile] - image save error for %s: %s", imageFullName, err)
+			errorGzipImageList = append(errorGzipImageList, imageFullName)
+			continue
+		}
+
+		if _, err := io.Copy(gw, imageSaveTarStream); err != nil {
+			log.ErrorF("[SaveImagesToGzipFile] - failed to copy tar archive for %s to gzip writer: %s", imageFullName, err)
+			errorGzipImageList = append(errorGzipImageList, imageFullName)
+			continue
+		}
+	}
+
+	if err := gw.Close(); err != nil {
+		log.ErrorF("[SaveImagesToGzipFile] - error closing gzip writer: %s", err)
+		return false, "", errorGzipImageList
+	}
+
+	log.InfoF("[SaveImagesToGzipFile] - successfully saved images to [%s]", gzipFileFullPath)
+	return true, gzipFileFullPath, errorGzipImageList
+}
+
 func CmiiImageMapToFullNameList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) {
 
 	for image, tag := range cmiiImageVersionMap {
@@ -523,10 +598,10 @@ func GenerateCmiiTagVersionImageMap(specificTag string) (backendMap, frontendMap
 	frontendMap = make(map[string]string, len(d_app.CmiiFrontendAppMap))
 	srsMap = make(map[string]string, len(d_app.CmiiSrsAppMap))
 
-	for imageName, _ := range d_app.CmiiBackendAppMap {
+	for imageName := range d_app.CmiiBackendAppMap {
 		backendMap[imageName] = specificTag
 	}
-	for imageName, _ := range d_app.CmiiFrontendAppMap {
+	for imageName := range d_app.CmiiFrontendAppMap {
 		frontendMap[imageName] = specificTag
 	}
 	for imageName, imageTag := range d_app.CmiiSrsAppMap {

agent-operator/main.go

@@ -15,8 +15,6 @@ var LocalKubeConfigFile = "/root/.kube/config"
 // C:\Users\wddsh\go\bin\gox.exe -osarch="linux/amd64" -output "build/agent-operator_{{.OS}}_{{.Arch}}"
 // C:\Users\wddsh\go\bin\gox.exe -osarch="linux/amd64 linux/arm64" -output "build/agent-operator_{{.OS}}_{{.Arch}}"
 
-//
-
 func BuildDefaultK8sOperator() {
 
 	// build from local LocalKubeConfigFile