From f1b432fc3af8d83cb4b8668f7d24dc0aaa2c3129 Mon Sep 17 00:00:00 2001 From: zeaslity Date: Wed, 8 May 2024 17:03:08 +0800 Subject: [PATCH] [Agent][Operator] - DLTU bug fix --- agent-common/utils/FileUtils.go | 9 +++ agent-operator/CmiiMinioOperator.go | 2 +- agent-operator/CmiiOperator.go | 93 +++++++++++++++++++---------- agent-operator/CmiiOperator_test.go | 15 ++++- 4 files changed, 86 insertions(+), 33 deletions(-) diff --git a/agent-common/utils/FileUtils.go b/agent-common/utils/FileUtils.go index 59ceb20..4079d2c 100644 --- a/agent-common/utils/FileUtils.go +++ b/agent-common/utils/FileUtils.go @@ -131,6 +131,15 @@ func IsDirOrFile(path string) bool { // FileExists 文件存在返回true，不存在返回false，如果文件是一个目录，也返回false func FileExists(fileFullPath string) bool { + info, err := os.Stat(fileFullPath) + if err != nil { + return false + } + return !info.IsDir() +} + +// FileOrFolderExists 文件或者目录是否返回true，不存在返回false +func FileOrFolderExists(fileFullPath string) bool { _, err := os.Stat(fileFullPath) return err == nil } diff --git a/agent-operator/CmiiMinioOperator.go b/agent-operator/CmiiMinioOperator.go index 5292db9..5da10c2 100644 --- a/agent-operator/CmiiMinioOperator.go +++ b/agent-operator/CmiiMinioOperator.go @@ -28,7 +28,7 @@ func newInstance() *CmiiMinioOperator { const ( DefaultLocalEndpoint = "10.250.0.100:9000" DefaultPublicEndpoint = "42.192.52.227:9000" - DefaultDemoEndpoint = "oss.demo.uavcmlc.com:18000" + DefaultDemoEndpoint = "oss.demo.uavcmlc.com" DefaultAccessKeyID = "cmii" DefaultSecretAccessKey = "B#923fC7mk" DefaultOctopusBucketName = "octopus" diff --git a/agent-operator/CmiiOperator.go b/agent-operator/CmiiOperator.go index d3df561..03e3bd9 100644 --- a/agent-operator/CmiiOperator.go +++ b/agent-operator/CmiiOperator.go @@ -21,6 +21,7 @@ type ImageSyncEntity struct { ProjectVersion string // 优先级2 CmiiNameTagList []string // 优先级1 appName:tag的形式 FullNameImageList []string // 优先级1 + DownloadImage bool // 下载镜像 DownloadFromOss bool // 下载镜像 CompressImageToGzip bool // 压缩镜像 UploadToDemoMinio bool // 上传镜像 @@ -76,7 +77,7 @@ func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResul allCmiiImageNameList = concatAndUniformCmiiImage(sync.FullNameImageList, sync.CmiiNameTagList) // DCU - errorPullImageList, errorGzipImageList, realCmiiImageList, allGzipFileNameList = DownloadCompressUpload(allCmiiImageNameList, sync.CompressImageToGzip, gzipFolderFullPath, sync.UploadToDemoMinio) + errorPullImageList, errorGzipImageList, realCmiiImageList, allGzipFileNameList = DownloadCompressUpload(true, allCmiiImageNameList, sync.CompressImageToGzip, gzipFolderFullPath, sync.UploadToDemoMinio) } // 直接传输到目标Harbor仓库 @@ -119,20 +120,43 @@ func concatAndUniformCmiiImage(fullImageList []string, cmiiImageList []string) [ } // DownloadCompressUpload DCU 镜像同步的前半部分，通常在35.71 LapPro执行，无需Bastion Mode -func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileFullNameList []string) { +func DownloadCompressUpload(downloadImage bool, fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileFullNameList []string) { + + // write to file + localGzipFileListTxt := gzipFolderFullPath + "all-gzip-image-file-name.txt" // Download log.Info("DOWNLOAD START !") - errorPullImageList = image.PullFromFullNameList(fullNameList) + if downloadImage { + if fullNameList == nil || len(fullNameList) == 0 { + log.InfoF("no image name list !") + } else { + errorPullImageList = image.PullFromFullNameList(fullNameList) + } + } + // remove failed fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool { return slices.Contains(errorPullImageList, imageName) }) - var localGzipFileListTxt string - // Compress if shouldGzip { + + // remove file + _ = os.Remove(localGzipFileListTxt) + + gzipFileAlready := make(map[string]bool) + if utils.FileOrFolderExists(gzipFolderFullPath) { + dir, _ := os.ReadDir(gzipFolderFullPath) + for _, entry := range dir { + if entry.IsDir() { + continue + } + gzipFileAlready[strings.TrimPrefix(entry.Name(), gzipFolderFullPath)] = true + } + } + // mkdir folder err := os.MkdirAll(gzipFolderFullPath, os.ModeDir) if err != nil { @@ -146,11 +170,22 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu log.Info("COMPRESS START") for _, imageFullName := range fullNameList { - ok, gzipImageFileFullPath := image.SaveToGzipFile(imageFullName, gzipFolderFullPath) - if !ok { - errorGzipImageList = append(errorGzipImageList, imageFullName) - continue + + // gzip image file already exists + + gzipFileName := image2.ImageFullNameToGzipFileName(imageFullName) + gzipImageFileFullPath := gzipFolderFullPath + gzipFileName + _, ok := gzipFileAlready[gzipFileName] + if len(gzipFileAlready) > 0 && ok { + log.DebugF("gzip file %s already exists !", gzipFileName) + } else { + ok, gzipImageFileFullPath = image.SaveToGzipFile(imageFullName, gzipFolderFullPath) + if !ok { + errorGzipImageList = append(errorGzipImageList, imageFullName) + continue + } } + // 压缩成功 allGzipFileFullNameList = append(allGzipFileFullNameList, gzipImageFileFullPath) } @@ -159,12 +194,9 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu return slices.Contains(errorGzipImageList, imageName) }) - // write to file - localGzipFileListTxt = gzipFolderFullPath + string(os.PathSeparator) + "all-gzip-image-file-name.txt" - for _, gzipFileFullName := range allGzipFileFullNameList { utils.AppendContentToFile( - strings.TrimPrefix(gzipFileFullName, gzipFolderFullPath), + strings.TrimPrefix(gzipFileFullName, gzipFolderFullPath)+"\n", localGzipFileListTxt, ) } @@ -185,7 +217,7 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu log.InfoF("gzip file location in demo oss is %s", DefaultDemoEndpoint+"/"+bucketNameWithPrefix) // upload gzip file list txt to demo - if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipFolderFullPath, localGzipFileListTxt) { + if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipFolderFullPath, strings.TrimPrefix(localGzipFileListTxt, gzipFolderFullPath)) { log.ErrorF("upload of %s to demo oss error !", localGzipFileListTxt) } @@ -309,7 +341,7 @@ func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string) } // DownloadCompressUploadFromDemo 获取DEMO环境的全部镜像 -func DownloadCompressUploadFromDemo(projectName string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { +func DownloadCompressUploadFromDemo(projectName string, shouldGzip, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { // generate a project folder err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+projectName, os.ModeDir) @@ -325,7 +357,7 @@ func DownloadCompressUploadFromDemo(projectName string, shouldGzip bool, shouldO // do work // DCU - return DownloadCompressUpload(allCmiiImageNameListFromDemo, shouldGzip, image.OfflineImageGzipFolderPrefix+projectName, shouldOss) + return DownloadCompressUpload(true, allCmiiImageNameListFromDemo, shouldGzip, image.OfflineImageGzipFolderPrefix+projectName, shouldOss) } func buildAllCmiiImageNameListFromDemo(projectName string) []string { @@ -381,7 +413,7 @@ func DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shou // do work // DCU procedure - return DownloadCompressUpload(realCmiiImageName, shouldGzip, image.OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss) + return DownloadCompressUpload(true, realCmiiImageName, shouldGzip, image.OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss) } @@ -422,7 +454,9 @@ func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string { return realCmiiImageName } -func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, downloadMiddle bool, downloadRke bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { +func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, shouldDownload bool, isRKE bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { + + log.Info("DCU for middle and rke!") err := os.MkdirAll(image.OfflineImageGzipFolderPrefix, os.ModeDir) if err != nil { if !errors.Is(err, os.ErrExist) { @@ -430,22 +464,21 @@ func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, downloadM } } - if downloadMiddle { + var fulleImageNameList []string + var gzipFolderPrefix string - gzipFolderPrefix := image.OfflineImageGzipFolderPrefix + "middle/" + if isRKE { + log.Info("DCU for rke!") + fulleImageNameList = image.Rancher1204Amd64 + gzipFolderPrefix = image.OfflineImageGzipFolderPrefix + "rke/" + } else { + log.Info("DCU for middle!") - // remove folder first - utils.RemoveFolderComplete(gzipFolderPrefix) - - return DownloadCompressUpload(image.MiddlewareAmd64, shouldGzip, gzipFolderPrefix, shouldOss) + fulleImageNameList = image.MiddlewareAmd64 + gzipFolderPrefix = image.OfflineImageGzipFolderPrefix + "middle/" } - if downloadRke { - gzipFolderPrefix := image.OfflineImageGzipFolderPrefix + "rke/" - return DownloadCompressUpload(image.MiddlewareAmd64, shouldGzip, gzipFolderPrefix, shouldOss) - } - - return errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList + return DownloadCompressUpload(shouldDownload, fulleImageNameList, shouldGzip, gzipFolderPrefix, shouldOss) } func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) { diff --git a/agent-operator/CmiiOperator_test.go b/agent-operator/CmiiOperator_test.go index 68f0834..ea4054f 100644 --- a/agent-operator/CmiiOperator_test.go +++ b/agent-operator/CmiiOperator_test.go @@ -9,9 +9,20 @@ import ( /* 拉取 */ -func TestFetchDependencyRepos(t *testing.T) { +func TestFetchDependencyRepos_Middle(t *testing.T) { - errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := DownloadCompressUploadDependency(true, false, true, false) + errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := DownloadCompressUploadDependency(true, true, false, false) + + utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList") + utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList") + utils.BeautifulPrintListWithTitle(realCmiiImageName, "realCmiiImageName") + utils.BeautifulPrintListWithTitle(allGzipFileNameList, "allGzipFileNameList") + +} + +func TestFetchDependencyRepos_RKE(t *testing.T) { + + errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := DownloadCompressUploadDependency(true, true, false, true) utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList") utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList")