From e26b7a7a008f9940d1b7ec9425e871d7a7154fbc Mon Sep 17 00:00:00 2001 From: zeaslity Date: Tue, 20 Aug 2024 15:32:38 +0800 Subject: [PATCH] =?UTF-8?q?[Agent][Deploy]=20-=20=E4=BF=AE=E6=94=B9ImageSy?= =?UTF-8?q?nc=E6=A8=A1=E5=9D=97=20DCU=E7=9A=84=E6=A0=B8=E5=BF=83=E5=86=85?= =?UTF-8?q?=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent-operator/ImageSyncOperator.go | 379 +++++++++----- agent-operator/ImageSyncOperator.svg | 582 ++++++++++++++++++++++ agent-operator/ImageSyncOperator_test.go | 4 +- agent-operator/image/CmiiImageOperator.go | 87 +++- agent-operator/main.go | 2 - 5 files changed, 922 insertions(+), 132 deletions(-) create mode 100644 agent-operator/ImageSyncOperator.svg diff --git a/agent-operator/ImageSyncOperator.go b/agent-operator/ImageSyncOperator.go index 5f38c2f..f3d263f 100755 --- a/agent-operator/ImageSyncOperator.go +++ b/agent-operator/ImageSyncOperator.go @@ -3,6 +3,7 @@ package main import ( "errors" "os" + "path/filepath" "slices" "strings" image2 "wdd.io/agent-common/image" @@ -11,20 +12,19 @@ import ( "wdd.io/agent-operator/image" ) -const OfflineDeployHarborHost = "harbor.wdd.io" -const PublicDeployHarborHost = "42.192.52.227" -const DirectPushDeployHarborHost = "chongqingcis-9b4a3da9.ecis.chongqing-1.cmecloud.cn" +const ( + OfflineDeployHarborHost = "harbor.wdd.io" + PublicDeployHarborHost = "42.192.52.227" + DirectPushDeployHarborHost = "chongqingcis-9b4a3da9.ecis.chongqing-1.cmecloud.cn" + + AllCmiiImageListLocalFileName = "all-cmii-image-list.txt" + AllGzipImageLocalFileName = "all-gzip-image-list.txt" +) type ImageSyncEntity struct { - ProjectName string // 优先级3 优先级最低 从DEMO拉取镜像 - ProjectVersion string // 优先级2 高于ProjectName 优先拉取特定版本的镜像 - - CmiiNameTagList []string // 优先级1 appName:tag 会被转换为FullNameImageList - FullNameImageList []string // 优先级1 优先下载此类型 - - ShouldDownloadImage bool // 下载镜像 DCU中的D - ShouldCompressImageToGzip bool // 压缩镜像 DCU中的C - ShouldUploadToDemoMinio bool // 上传镜像 DCU中的U + DownloadCondition *DownloadEntity // D的条件 + CompressCondition *CompressEntity // C的条件 + UploadCondition *UploadEntity // U的条件 ShouldDownloadFromOss bool // 下载镜像 DLTU中的D ShouldUpdateImageTag bool // 更新镜像 DLTU中的U @@ -33,75 +33,99 @@ type ImageSyncEntity struct { DirectHarborHost string // IP:Port or 域名:PORT 不带http前缀 } +// DownloadEntity DCU中的D的条件 +type DownloadEntity struct { + ShouldDownloadImage bool // 下载镜像 DCU中的D 实际无用 + + ProjectName string // 优先级3 优先级最低 从DEMO拉取镜像 + ProjectVersion string // 优先级2 高于ProjectName 优先拉取特定版本的镜像 + + CmiiNameTagList []string // 优先级1 appName:tag 会被转换为FullNameImageList + FullNameImageList []string // 优先级1 优先下载此类型 + + DownloadAuthUserName string // 下载需要认证的用户名 + DownloadAuthPassword string // 下载需要认证的密码 +} + +// CompressEntity DCU中的C的条件 +type CompressEntity struct { + ShouldCompressImageToGzip bool // 压缩镜像 DCU中的C + ShouldGzipSplit bool // 压缩镜像 是否应该分割存储 true=独立存储 false=整个存储 + GzipLocalFolder string // 压缩镜像 保存压缩镜像文件的本地目录 +} + +// UploadEntity DCU中的U的条件 +type UploadEntity struct { + ShouldUploadToDemoMinio bool // 上传镜像 DCU中的U +} + type ImageSyncResult struct { - ErrorPullImageList []string - ErrorGzipImageList []string - ErrorPushImageNameList []string - RealImageNameList []string - RealGzipFileNameList []string - AllCmiiImageNameList []string + ProcedureSuccessImageList []string // 经过特定步骤之后成功的镜像 + + DownloadResult *DownloadResultEntity + CompressResult *CompressResultEntity + UploadResult *UploadResultEntity +} + +type DownloadResultEntity struct { + ErrorPullImageList []string // 下载镜像 DCU中的D 下载失败的镜像 + SuccessPullImageList []string // 下载镜像 DCU中的D 下载成功的镜像 + SuccessPullTxtFileLocalFullPath string // 下载镜像 DCU中的D 下载成功的镜像保存的文件地址 GzipLocalFolder + all +} + +type CompressResultEntity struct { + ErrorGzipImageList []string // 压缩镜像 DCU中的C 压缩失败的镜像 + SuccessGzipImageList []string // 压缩镜像 DCU中的C 压缩成功的镜像 + GzipTxtFileLocalFullPath string // 压缩镜像 DCU中的C 压缩镜像保存的目录 +} + +type UploadResultEntity struct { + ErrorUploadImageList []string // 上传镜像 DCU中的U 上传失败的镜像 + AllDownloadUrl []string // 上传镜像 DCU中的U 正式的下载地址列表i } // PullFromEntityAndSyncConditionally 根据ImageSyncEntity拉取特定的镜像，然后上传到特定的目标机器(或者上传的minio中) -func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResult ImageSyncResult) { +func (syncCondition *ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResult *ImageSyncResult) { - var realCmiiImageList []string - var allCmiiImageNameList []string + imageSyncResult = &ImageSyncResult{} - var errorPullImageList []string - var allGzipFileNameList []string - var errorGzipImageList []string - var errorPushImageNameList []string - var gzipFolderFullPath string - - if (sync.CmiiNameTagList == nil && sync.FullNameImageList == nil) || (len(sync.CmiiNameTagList) == 0 && len(sync.FullNameImageList) == 0) { + if (syncCondition.DownloadCondition.CmiiNameTagList == nil && syncCondition.DownloadCondition.FullNameImageList == nil) || (len(syncCondition.DownloadCondition.CmiiNameTagList) == 0 && len(syncCondition.DownloadCondition.FullNameImageList) == 0) { // 没有指定特定的镜像，那么根据 ProjectVersion 或者从DEMO拉取镜像 // pull images // compress - if sync.ProjectVersion != "" { + if syncCondition.DownloadCondition.ProjectVersion != "" { // 获取特定版本的镜像 - errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = C_DownloadCompressUploadFromVersion(sync.ProjectVersion, sync.ShouldCompressImageToGzip, sync.ShouldUploadToDemoMinio) - - gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectVersion + C_DownloadCompressUploadFromVersion(syncCondition, imageSyncResult) } else { - // 获取DEMO的镜像 - errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = C_DownloadCompressUploadFromDemo(sync.ProjectName, sync.ShouldCompressImageToGzip, sync.ShouldUploadToDemoMinio) - gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectName + C_DownloadCompressUploadFromDemo(syncCondition, imageSyncResult) } } else { - // 拉取特定的镜像 + // 根据列表拉取镜像 - gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + "tmp" // 组装镜像名称 - allCmiiImageNameList = concatAndUniformCmiiImage(sync.FullNameImageList, sync.CmiiNameTagList) + syncCondition.DownloadCondition.FullNameImageList = concatAndUniformCmiiImage(syncCondition.DownloadCondition.FullNameImageList, syncCondition.DownloadCondition.CmiiNameTagList) + + // gzip file folder path + syncCondition.CompressCondition.GzipLocalFolder = filepath.Join(image.OfflineImageGzipFolderPrefix, "tmp") // DCU - errorPullImageList, errorGzipImageList, realCmiiImageList, allGzipFileNameList = A_DownloadCompressUpload(true, allCmiiImageNameList, sync.ShouldCompressImageToGzip, gzipFolderFullPath, sync.ShouldUploadToDemoMinio) + A_DownloadCompressUpload(syncCondition, imageSyncResult) } // 直接传输到目标Harbor仓库 - if sync.ShouldDirectPushToHarbor { - if sync.DirectHarborHost == "" { + if syncCondition.ShouldDirectPushToHarbor { + if syncCondition.DirectHarborHost == "" { log.ErrorF("DirectHarborHost is null ! can't push to target harbor !") } // push to - errorPushImageNameList = image.TagFromListAndPushToCHarbor(allCmiiImageNameList, sync.DirectHarborHost) + //errorPushImageNameList = image.TagFromListAndPushToCHarbor(allCmiiImageNameList, syncCondition.DirectHarborHost) } // build result - imageSyncResult.AllCmiiImageNameList = allCmiiImageNameList - - imageSyncResult.RealImageNameList = realCmiiImageList - imageSyncResult.ErrorPullImageList = errorPullImageList - - imageSyncResult.RealGzipFileNameList = allGzipFileNameList - imageSyncResult.ErrorGzipImageList = errorGzipImageList - - imageSyncResult.ErrorPushImageNameList = errorPushImageNameList return imageSyncResult } @@ -123,119 +147,179 @@ func concatAndUniformCmiiImage(fullImageList []string, cmiiImageList []string) [ } // A_DownloadCompressUpload DCU 镜像同步的前半部分，通常在35.71 LapPro执行，无需Bastion Mode -func A_DownloadCompressUpload(downloadImage bool, fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileFullNameList []string) { +func A_DownloadCompressUpload(syncEntity *ImageSyncEntity, syncResult *ImageSyncResult) { - // write to file - localGzipFileListTxt := gzipFolderFullPath + "all-gzip-image-file-name.txt" + // all image full name list need to download + fullNameList := syncEntity.DownloadCondition.FullNameImageList // Download - log.Info("DOWNLOAD START !") - if downloadImage { - if fullNameList == nil || len(fullNameList) == 0 { - log.InfoF("no image name list !") - } else { - errorPullImageList = image.PullFromFullNameList(fullNameList) - } + log.Info("[DCU] - DOWNLOAD START !") + if syncEntity.DownloadCondition.ShouldDownloadImage && fullNameList != nil && len(fullNameList) > 0 { + + syncResult.DownloadResult.ErrorPullImageList = image.PullFromFullNameList(fullNameList) + + // remove failed download image from full name list + fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool { + return slices.Contains(syncResult.DownloadResult.ErrorPullImageList, imageName) + }) + + } else { + log.Info("[DCU] - No Need To Download !") } - // remove failed - fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool { - return slices.Contains(errorPullImageList, imageName) - }) + syncResult.ProcedureSuccessImageList = fullNameList + + gzipLocalFolderPath := syncEntity.CompressCondition.GzipLocalFolder // Compress - if shouldGzip { + if syncEntity.CompressCondition.ShouldCompressImageToGzip { // remove file + localGzipFileListTxt := filepath.Join(gzipLocalFolderPath, AllGzipImageLocalFileName) _ = os.Remove(localGzipFileListTxt) + // 找到已经存在的压缩文件，跳过 gzipFileAlready := make(map[string]bool) - if utils.FileOrFolderExists(gzipFolderFullPath) { - dir, _ := os.ReadDir(gzipFolderFullPath) + if utils.FileOrFolderExists(gzipLocalFolderPath) { + dir, _ := os.ReadDir(gzipLocalFolderPath) for _, entry := range dir { if entry.IsDir() { continue } - gzipFileAlready[strings.TrimPrefix(entry.Name(), gzipFolderFullPath)] = true + gzipFileAlready[strings.TrimPrefix(entry.Name(), gzipLocalFolderPath)] = true } } // mkdir folder - err := os.MkdirAll(gzipFolderFullPath, os.ModeDir) + err := os.MkdirAll(gzipLocalFolderPath, os.ModeDir) if err != nil { if !errors.Is(err, os.ErrExist) { - log.ErrorF("create folder error of %s", gzipFolderFullPath) + log.ErrorF("create folder error of %s", gzipLocalFolderPath) panic(err) } } // 循环遍历压缩 - log.Info("COMPRESS START") + log.Info("[DCU] - COMPRESS START") + var errorGzipImageList []string + var allGzipFileFullNameList []string - for _, imageFullName := range fullNameList { + if syncEntity.CompressCondition.ShouldGzipSplit { + // 独立压缩 + for _, imageFullName := range fullNameList { - // gzip image file already exists + // gzip image file already exists + gzipFileName := image2.ImageFullNameToGzipFileName(imageFullName) + gzipImageFileFullPath := gzipLocalFolderPath + gzipFileName + _, ok := gzipFileAlready[gzipFileName] + if len(gzipFileAlready) > 0 && ok { + log.DebugF("gzip file %s already exists !", gzipFileName) + } else { + ok, gzipImageFileFullPath = image.SaveToGzipFile(imageFullName, gzipLocalFolderPath) + if !ok { + errorGzipImageList = append(errorGzipImageList, imageFullName) + continue + } + } - gzipFileName := image2.ImageFullNameToGzipFileName(imageFullName) - gzipImageFileFullPath := gzipFolderFullPath + gzipFileName - _, ok := gzipFileAlready[gzipFileName] - if len(gzipFileAlready) > 0 && ok { - log.DebugF("gzip file %s already exists !", gzipFileName) - } else { - ok, gzipImageFileFullPath = image.SaveToGzipFile(imageFullName, gzipFolderFullPath) - if !ok { - errorGzipImageList = append(errorGzipImageList, imageFullName) - continue + // 压缩成功 + allGzipFileFullNameList = append(allGzipFileFullNameList, gzipImageFileFullPath) + + syncResult.CompressResult.SuccessGzipImageList = allGzipFileFullNameList + syncResult.CompressResult.ErrorGzipImageList = errorGzipImageList + + // remove failed + fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool { + return slices.Contains(errorGzipImageList, imageName) + }) + + // write all gzipped file name to file + for _, gzipFileFullName := range allGzipFileFullNameList { + utils.AppendContentToFile( + strings.TrimPrefix(gzipFileFullName, gzipLocalFolderPath)+"\n", + localGzipFileListTxt, + ) } } + } else { + // 压缩为一个大的压缩包 + gzipFileName := generateMonolithicGzipFileName(syncEntity) - // 压缩成功 - allGzipFileFullNameList = append(allGzipFileFullNameList, gzipImageFileFullPath) - } - // remove failed - fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool { - return slices.Contains(errorGzipImageList, imageName) - }) + ok, gzipFileFullPath, errorGzipImageListTmp := image.SaveImageListToGzipFile(fullNameList, gzipLocalFolderPath, gzipFileName) + if !ok { + panic("[DCU] - gzip error to a monolithic file !") + } - for _, gzipFileFullName := range allGzipFileFullNameList { + // write all gzipped file name to file utils.AppendContentToFile( - strings.TrimPrefix(gzipFileFullName, gzipFolderFullPath)+"\n", + utils.BeautifulPrintToString(fullNameList), localGzipFileListTxt, ) + + // remove failed + fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool { + return slices.Contains(errorGzipImageListTmp, imageName) + }) + + syncResult.CompressResult.SuccessGzipImageList = fullNameList + syncResult.CompressResult.ErrorGzipImageList = errorGzipImageListTmp + + log.InfoF("[DCU] - gzip all image from list to monolithic file %s", gzipFileFullPath) } - log.InfoF("all gzip file name list is %s", localGzipFileListTxt) + syncResult.CompressResult.GzipTxtFileLocalFullPath = localGzipFileListTxt + + log.InfoF("[DCU] - all gzip file name list is %s", localGzipFileListTxt) } + syncResult.ProcedureSuccessImageList = fullNameList + // Upload - if shouldOss { + if syncEntity.UploadCondition.ShouldUploadToDemoMinio { //uploadGzipFileToDemoMinio() // get gzip file name list - log.Info("UPLOAD OSS START !") + log.Info("[DCU] - UPLOAD OSS START !") + + var errorUploadOssGzipNameList []string + var allDownloadUrl []string // start to upload // extract demo oss location suffix from gzipFolderFullPath - trimPrefix := strings.TrimPrefix(gzipFolderFullPath, image.OfflineImageGzipFolderPrefix) - bucketNameWithPrefix := "cmlc-installation/" + trimPrefix + // 根据本地保存Gzip的目录路径提取到 相应的后缀 项目代码 + // projectName / projectVersion + projectUniqueName := strings.TrimPrefix(gzipLocalFolderPath, image.OfflineImageGzipFolderPrefix) + + bucketNameWithPrefix := "cmlc-installation/" + projectUniqueName log.InfoF("gzip file location in demo oss is %s", DefaultDemoEndpoint+"/"+bucketNameWithPrefix) // upload gzip file list txt to demo - if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipFolderFullPath, strings.TrimPrefix(localGzipFileListTxt, gzipFolderFullPath)) { + localGzipFileListTxt := syncResult.CompressResult.GzipTxtFileLocalFullPath + if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipLocalFolderPath, strings.TrimPrefix(localGzipFileListTxt, gzipLocalFolderPath)) { log.ErrorF("upload of %s to demo oss error !", localGzipFileListTxt) } log.InfoF("upload all gzip file to demo oss !") - for _, gzipFileFullName := range allGzipFileFullNameList { + for _, gzipFileFullName := range syncResult.CompressResult.SuccessGzipImageList { // SaveToGzipFile 返回的是全路径 归一化处理 gzip file name - gzipFileFullName = strings.TrimPrefix(gzipFileFullName, gzipFolderFullPath) - if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipFolderFullPath, gzipFileFullName) { - log.ErrorF("upload of %s to demo oss error !", gzipFileFullName) + gzipFileName := strings.TrimPrefix(gzipFileFullName, gzipLocalFolderPath) + if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipLocalFolderPath, gzipFileName) { + log.ErrorF("upload of %s to demo oss error !", gzipFileName) + errorUploadOssGzipNameList = append(errorUploadOssGzipNameList, gzipFileName) + } else { + allDownloadUrl = append(allDownloadUrl, DefaultDemoEndpoint+"/"+bucketNameWithPrefix+"/"+gzipFileName) } } + syncResult.UploadResult.AllDownloadUrl = allDownloadUrl + syncResult.UploadResult.ErrorUploadImageList = errorUploadOssGzipNameList + } - return errorPullImageList, errorGzipImageList, fullNameList, allGzipFileFullNameList +} + +func generateMonolithicGzipFileName(syncEntity *ImageSyncEntity) string { + + return strings.TrimPrefix(syncEntity.CompressCondition.GzipLocalFolder, image.OfflineImageGzipFolderPrefix) } // A_DownloadLoadTagUpload DLTU procedure ImageSync的另外一般流程，需要支持 堡垒机（纯离线）的模式 @@ -344,23 +428,35 @@ func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string) } // C_DownloadCompressUploadFromDemo 获取DEMO环境的全部镜像 -func C_DownloadCompressUploadFromDemo(projectName string, shouldGzip, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { +func C_DownloadCompressUploadFromDemo(syncEntity *ImageSyncEntity, syncResult *ImageSyncResult) { // generate a project folder - err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+projectName, os.ModeDir) + projectName := syncEntity.DownloadCondition.ProjectName + gzipFolderLocalPath := filepath.Join(image.OfflineImageGzipFolderPrefix, projectName) + + err := os.MkdirAll(gzipFolderLocalPath, os.ModeDir) if err != nil { if !errors.Is(err, os.ErrExist) { - log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", image.OfflineImageGzipFolderPrefix+projectName, err.Error()) - return errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList + log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", gzipFolderLocalPath, err.Error()) + } } + syncEntity.CompressCondition.GzipLocalFolder = gzipFolderLocalPath + // get demo image version map - allCmiiImageNameListFromDemo := buildAllCmiiImageNameListFromDemo(projectName) + allCmiiImageFullNameList := buildAllCmiiImageNameListFromDemo(projectName) + + // save all cmii image to file + allPullImageNameTxtFileName := filepath.Join(gzipFolderLocalPath, AllCmiiImageListLocalFileName) + utils.AppendOverwriteContentToFile(utils.BeautifulPrintToString(allCmiiImageFullNameList), allPullImageNameTxtFileName) + + // save to result + syncResult.DownloadResult.SuccessPullTxtFileLocalFullPath = allPullImageNameTxtFileName // do work // DCU - return A_DownloadCompressUpload(true, allCmiiImageNameListFromDemo, shouldGzip, image.OfflineImageGzipFolderPrefix+projectName, shouldOss) + A_DownloadCompressUpload(syncEntity, syncResult) } func buildAllCmiiImageNameListFromDemo(projectName string) []string { @@ -400,26 +496,39 @@ func buildAllCmiiImageNameListFromDemo(projectName string) []string { } // C_DownloadCompressUploadFromVersion 根据版本下载全部的CMII镜像 -func C_DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { +func C_DownloadCompressUploadFromVersion(syncEntity *ImageSyncEntity, syncResult *ImageSyncResult) { // generate a project folder - err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+cmiiVersion, os.ModeDir) + projectCmiiVersion := syncEntity.DownloadCondition.ProjectVersion + + // gzip local path + gzipFolderLocalPath := filepath.Join(image.OfflineImageGzipFolderPrefix, projectCmiiVersion) + + err := os.MkdirAll(gzipFolderLocalPath, os.ModeDir) if err != nil { if !errors.Is(err, os.ErrExist) { - log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", image.OfflineImageGzipFolderPrefix+cmiiVersion, err.Error()) - return errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList + log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", gzipFolderLocalPath, err.Error()) } } + syncEntity.CompressCondition.GzipLocalFolder = gzipFolderLocalPath + // build all cmii image name list - realCmiiImageName = buildAllCmiiImageNameListFromVersion(cmiiVersion) + allCmiiImageFullNameList := buildAllCmiiImageNameListFromVersion(projectCmiiVersion) + + // save all cmii image to file + allImageListTxtFileFullName := filepath.Join(gzipFolderLocalPath, AllCmiiImageListLocalFileName) + utils.AppendOverwriteContentToFile(utils.BeautifulPrintToString(allCmiiImageFullNameList), allImageListTxtFileFullName) + + // save to result + syncResult.DownloadResult.SuccessPullTxtFileLocalFullPath = allImageListTxtFileFullName // do work // DCU procedure - return A_DownloadCompressUpload(true, realCmiiImageName, shouldGzip, image.OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss) - + A_DownloadCompressUpload(syncEntity, syncResult) } +// buildAllCmiiImageNameListFromVersion 根据VersionTag构建完整的应用名称列表 func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string { var realCmiiImageName []string @@ -458,7 +567,7 @@ func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string { } // C_DownloadCompressUploadDependency DCU所有的依赖镜像 -func C_DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, shouldDownload bool, isRKE bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { +func C_DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, isRKE bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { log.Info("DCU for middle and rke!") err := os.MkdirAll(image.OfflineImageGzipFolderPrefix, os.ModeDir) @@ -468,21 +577,47 @@ func C_DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, shouldD } } - var fulleImageNameList []string + var fullImageNameList []string var gzipFolderPrefix string if isRKE { log.Info("DCU for rke!") - fulleImageNameList = d_app.Rancher1204Amd64 + fullImageNameList = d_app.Rancher1204Amd64 gzipFolderPrefix = image.OfflineImageGzipFolderPrefix + "rke/" } else { log.Info("DCU for middle!") - fulleImageNameList = d_app.MiddlewareAmd64 + fullImageNameList = d_app.MiddlewareAmd64 gzipFolderPrefix = image.OfflineImageGzipFolderPrefix + "middle/" } - return A_DownloadCompressUpload(shouldDownload, fulleImageNameList, shouldGzip, gzipFolderPrefix, shouldOss) + syncEntity := &ImageSyncEntity{ + DownloadCondition: &DownloadEntity{ + ShouldDownloadImage: false, + ProjectName: "", + ProjectVersion: "", + CmiiNameTagList: nil, + FullNameImageList: fullImageNameList, + DownloadAuthUserName: "", + DownloadAuthPassword: "", + }, + CompressCondition: &CompressEntity{ + ShouldCompressImageToGzip: shouldGzip, + ShouldGzipSplit: true, + GzipLocalFolder: gzipFolderPrefix, + }, + UploadCondition: &UploadEntity{ShouldUploadToDemoMinio: shouldOss}, + ShouldDownloadFromOss: false, + ShouldUpdateImageTag: false, + ShouldDirectPushToHarbor: false, + DirectHarborHost: "", + } + + syncResult := &ImageSyncResult{} + + A_DownloadCompressUpload(syncEntity, syncResult) + + return syncResult.DownloadResult.ErrorPullImageList, syncResult.CompressResult.ErrorGzipImageList, syncResult.ProcedureSuccessImageList, syncResult.CompressResult.SuccessGzipImageList } func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) { diff --git a/agent-operator/ImageSyncOperator.svg b/agent-operator/ImageSyncOperator.svg new file mode 100644 index 0000000..12f6d29 --- /dev/null +++ b/agent-operator/ImageSyncOperator.svg @@ -0,0 +1,582 @@ + + + + + + + + + + + + + + + + + + + + + +
+
+
+ Download +
+
+
+ + Download + + + + + + + + + + + + + +
+
+
+ Compress +
+
+
+ + Compress + + + + + + + + + + + + + +
+
+
+ Upload +
+
+
+ + Upload + + + + + + + + + + + + + +
+
+
+ DEMO +
+
+
+ + DEMO + + + + + + + + + + + + + +
+
+
+ VersionTag +
+
+
+ + VersionTag + + + + + + + + + + + + + +
+
+
+ Split +
+
+
+ + Split + + + + + + + + + + + + + +
+
+
+ monolithic +
+
+
+ + monolithic + + + + + + + + + + + + + +
+
+
+ Path +
+
+
+ + Path + + + + + + + + + + + + + + + + + + + + +
+
+
+ should +
+
+
+ + should + + + + + + + + + + + + + + +
+
+
+ AUTH +
+
+
+ + AUTH + + + + + + + + + + + + + +
+
+
+ Local Path +
+
+
+ + Local Path + + + + + + + + + + + + + + + + + + +
+
+
+
+ 
+                                                        
+                                                            ErrorPullImageList
+                                                            
+                                                                

+                                                            
+                                                        
+
+
+
+
+
+ + ErrorPullImageList + + + + + + + + + + + + + +
+
+
+
+ 
+                                                        
+                                                            
+                                                                ErrorGzipImageList
+                                                            
+                                                            
+                                                                

+                                                            
+                                                        
+
+
+
+
+
+ + ErrorGzipImageList + + + + + + + + + + + + + +
+
+
+ GzipLocalFolder +
+
+
+ + GzipLocalFolder + + + + + + + + + + + + + +
+
+
+ 
+                                                    ProcedureSuccessImageList
+
+
+
+
+ + ProcedureSuccessImageList + + + + + + + + + + + + + +
+
+
+
+ SuccessPulledImagList + +
+
+
+ +
+ +
+
[FILE]- GzipLocalFolder+name + +
+ +
+
+
+
+ + SuccessPulledImagList... + + + + + + + + + + + + + +
+
+
+ AllDownloadUrl +
+
+
+ + AllDownloadUrl + + + + + + + + + + + + + +
+
+
+ ErrorUploadImageList +
+
+
+ + ErrorUploadImageList + + + + + + + + + + + + Text is not SVG - cannot display + + + \ No newline at end of file diff --git a/agent-operator/ImageSyncOperator_test.go b/agent-operator/ImageSyncOperator_test.go index eeab20b..f7455c5 100755 --- a/agent-operator/ImageSyncOperator_test.go +++ b/agent-operator/ImageSyncOperator_test.go @@ -11,7 +11,7 @@ import ( func TestFetchDependencyRepos_Middle(t *testing.T) { - errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := C_DownloadCompressUploadDependency(true, true, false, false) + errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := C_DownloadCompressUploadDependency(true, false, false) utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList") utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList") @@ -22,7 +22,7 @@ func TestFetchDependencyRepos_Middle(t *testing.T) { func TestFetchDependencyRepos_RKE(t *testing.T) { - errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := C_DownloadCompressUploadDependency(true, true, false, true) + errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := C_DownloadCompressUploadDependency(true, false, true) utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList") utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList") diff --git a/agent-operator/image/CmiiImageOperator.go b/agent-operator/image/CmiiImageOperator.go index a30c4fc..d414def 100755 --- a/agent-operator/image/CmiiImageOperator.go +++ b/agent-operator/image/CmiiImageOperator.go @@ -13,6 +13,7 @@ import ( "io" "io/fs" "os" + "path/filepath" "regexp" "strconv" "strings" @@ -445,10 +446,11 @@ func SaveToGzipFile(imageFullName, folderPathPrefix string) (gzipOK bool, gzipIm } gzipImageFileFullPath = image2.ImageFullNameToGzipFileName(realImageTag) - if !strings.HasSuffix(folderPathPrefix, "/") { - folderPathPrefix += "/" + + if err := os.MkdirAll(filepath.Dir(gzipImageFileFullPath), os.ModePerm); err != nil { + log.ErrorF("[ImageSaveToTarGZ] - failed to create directory: %s", err) + return false, "" } - _ = os.MkdirAll(folderPathPrefix, os.ModeDir) // 生成gzip压缩文件的全路径名称 gzipImageFileFullPath = folderPathPrefix + gzipImageFileFullPath @@ -456,7 +458,10 @@ func SaveToGzipFile(imageFullName, folderPathPrefix string) (gzipOK bool, gzipIm log.InfoF("[ImageSaveToTarGZ] - start to save [%s] to [%s]", realImageTag, gzipImageFileFullPath) // 删除掉旧的Gzip文件 - _ = os.Remove(gzipImageFileFullPath) + if err := os.Remove(gzipImageFileFullPath); err != nil && !os.IsNotExist(err) { + log.ErrorF("[ImageSaveToTarGZ] - failed to remove old gzip file: %s", err) + return false, "" + } // 创建 tarFile, err := os.Create(gzipImageFileFullPath) @@ -483,6 +488,76 @@ func SaveToGzipFile(imageFullName, folderPathPrefix string) (gzipOK bool, gzipIm return true, gzipImageFileFullPath } +// SaveImageListToGzipFile 将一个列表内的镜像全部压缩为一个tar.gz文件 +func SaveImageListToGzipFile(imageFullNames []string, folderPathPrefix string, outputFileName string) (gzipOK bool, gzipFileFullPath string, errorGzipImageList []string) { + + if len(imageFullNames) == 0 { + log.Error("[SaveImagesToGzipFile] - no images provided") + return false, "", errorGzipImageList + } + + // 确保输出文件路径 + if err := os.MkdirAll(filepath.Dir(folderPathPrefix), os.ModePerm); err != nil { + log.ErrorF("[SaveImagesToGzipFile] - failed to create directory: %s", err) + return false, "", errorGzipImageList + } + + gzipFileFullPath = filepath.Join(folderPathPrefix, outputFileName) + log.InfoF("[SaveImagesToGzipFile] - start saving images to [%s]", gzipFileFullPath) + + // 删除旧的Gzip文件 + if err := os.Remove(gzipFileFullPath); err != nil && !os.IsNotExist(err) { + log.ErrorF("[SaveImagesToGzipFile] - failed to remove old gzip file: %s", err) + return false, "", errorGzipImageList + } + + tarFile, err := os.Create(gzipFileFullPath) + if err != nil { + log.ErrorF("[SaveImagesToGzipFile] - error creating gzip file: %s", err) + return false, "", errorGzipImageList + } + defer tarFile.Close() + + gw, err := pgzip.NewWriterLevel(tarFile, pgzip.DefaultCompression) + if err != nil { + log.ErrorF("[SaveImagesToGzipFile] - pgzip writer creation error: %s", err) + return false, "", errorGzipImageList + } + defer gw.Close() + + errorGzipImageList = []string{} + + for _, imageFullName := range imageFullNames { + imageGetByName := GetByName(imageFullName) + if imageGetByName == nil { + log.WarnF("[SaveImagesToGzipFile] - %s not exists, skipping", imageFullName) + errorGzipImageList = append(errorGzipImageList, imageFullName) + continue + } + + imageSaveTarStream, err := apiClient.ImageSave(context.TODO(), imageGetByName.RepoTags) + if err != nil { + log.ErrorF("[SaveImagesToGzipFile] - image save error for %s: %s", imageFullName, err) + errorGzipImageList = append(errorGzipImageList, imageFullName) + continue + } + + if _, err := io.Copy(gw, imageSaveTarStream); err != nil { + log.ErrorF("[SaveImagesToGzipFile] - failed to copy tar archive for %s to gzip writer: %s", imageFullName, err) + errorGzipImageList = append(errorGzipImageList, imageFullName) + continue + } + } + + if err := gw.Close(); err != nil { + log.ErrorF("[SaveImagesToGzipFile] - error closing gzip writer: %s", err) + return false, "", errorGzipImageList + } + + log.InfoF("[SaveImagesToGzipFile] - successfully saved images to [%s]", gzipFileFullPath) + return true, gzipFileFullPath, errorGzipImageList +} + func CmiiImageMapToFullNameList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) { for image, tag := range cmiiImageVersionMap { @@ -523,10 +598,10 @@ func GenerateCmiiTagVersionImageMap(specificTag string) (backendMap, frontendMap frontendMap = make(map[string]string, len(d_app.CmiiFrontendAppMap)) srsMap = make(map[string]string, len(d_app.CmiiSrsAppMap)) - for imageName, _ := range d_app.CmiiBackendAppMap { + for imageName := range d_app.CmiiBackendAppMap { backendMap[imageName] = specificTag } - for imageName, _ := range d_app.CmiiFrontendAppMap { + for imageName := range d_app.CmiiFrontendAppMap { frontendMap[imageName] = specificTag } for imageName, imageTag := range d_app.CmiiSrsAppMap { diff --git a/agent-operator/main.go b/agent-operator/main.go index bbd9499..838d9c7 100755 --- a/agent-operator/main.go +++ b/agent-operator/main.go @@ -15,8 +15,6 @@ var LocalKubeConfigFile = "/root/.kube/config" // C:\Users\wddsh\go\bin\gox.exe -osarch="linux/amd64" -output "build/agent-operator_{{.OS}}_{{.Arch}}" // C:\Users\wddsh\go\bin\gox.exe -osarch="linux/amd64 linux/arm64" -output "build/agent-operator_{{.OS}}_{{.Arch}}" -// - func BuildDefaultK8sOperator() { // build from local LocalKubeConfigFile