diff --git a/agent-common/utils/FileUtils.go b/agent-common/utils/FileUtils.go index 8e0f794..59ceb20 100644 --- a/agent-common/utils/FileUtils.go +++ b/agent-common/utils/FileUtils.go @@ -120,8 +120,8 @@ func WordSpaceCompletion(source string, totalLength int) string { return source } -// IsFileOrDir 如果是目录则返回true，是文件则返回false -func IsFileOrDir(path string) bool { +// IsDirOrFile 如果是目录则返回true，是文件则返回false +func IsDirOrFile(path string) bool { info, err := os.Stat(path) if err != nil { return false diff --git a/agent-operator/CmiiOperator.go b/agent-operator/CmiiOperator.go index 2ceb356..d3df561 100644 --- a/agent-operator/CmiiOperator.go +++ b/agent-operator/CmiiOperator.go @@ -119,7 +119,7 @@ func concatAndUniformCmiiImage(fullImageList []string, cmiiImageList []string) [ } // DownloadCompressUpload DCU 镜像同步的前半部分，通常在35.71 LapPro执行，无需Bastion Mode -func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { +func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileFullNameList []string) { // Download log.Info("DOWNLOAD START !") @@ -129,6 +129,8 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu return slices.Contains(errorPullImageList, imageName) }) + var localGzipFileListTxt string + // Compress if shouldGzip { // mkdir folder @@ -150,7 +152,7 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu continue } // 压缩成功 - allGzipFileNameList = append(allGzipFileNameList, gzipImageFileFullPath) + allGzipFileFullNameList = append(allGzipFileFullNameList, gzipImageFileFullPath) } // remove failed fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool { @@ -158,16 +160,16 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu }) // write to file - targetFile := gzipFolderFullPath + string(os.PathSeparator) + "all-gzip-image-file-name.txt" + localGzipFileListTxt = gzipFolderFullPath + string(os.PathSeparator) + "all-gzip-image-file-name.txt" - for _, gzipFileFullName := range allGzipFileNameList { + for _, gzipFileFullName := range allGzipFileFullNameList { utils.AppendContentToFile( strings.TrimPrefix(gzipFileFullName, gzipFolderFullPath), - targetFile, + localGzipFileListTxt, ) } - log.InfoF("all gzip file name list is %s", targetFile) + log.InfoF("all gzip file name list is %s", localGzipFileListTxt) } // Upload @@ -182,17 +184,23 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu bucketNameWithPrefix := "cmlc-installation/" + trimPrefix log.InfoF("gzip file location in demo oss is %s", DefaultDemoEndpoint+"/"+bucketNameWithPrefix) - for _, gzipFileName := range allGzipFileNameList { + // upload gzip file list txt to demo + if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipFolderFullPath, localGzipFileListTxt) { + log.ErrorF("upload of %s to demo oss error !", localGzipFileListTxt) + } + + log.InfoF("upload all gzip file to demo oss !") + for _, gzipFileFullName := range allGzipFileFullNameList { // SaveToGzipFile 返回的是全路径 归一化处理 gzip file name - gzipFileName = strings.TrimPrefix(gzipFileName, gzipFolderFullPath) - if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipFolderFullPath, gzipFileName) { - log.ErrorF("upload of %s to demo oss error !", gzipFolderFullPath+gzipFileName) + gzipFileFullName = strings.TrimPrefix(gzipFileFullName, gzipFolderFullPath) + if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipFolderFullPath, gzipFileFullName) { + log.ErrorF("upload of %s to demo oss error !", gzipFileFullName) } } } - return errorPullImageList, errorGzipImageList, fullNameList, allGzipFileNameList + return errorPullImageList, errorGzipImageList, fullNameList, allGzipFileFullNameList } // DownloadLoadTagUpload DLTU procedure ImageSync的另外一般流程，需要支持 堡垒机（纯离线）的模式 @@ -201,10 +209,10 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu // 3. 读取本机的IP地址 - 参数传递 // 4. OSS地址 - ossUrlPrefix传空 则使用默认值 // 5. ossFileName - 如果结尾为txt，则为文件的形式，如果为tar.gz，则为gzip文件夹的形式 -func DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile string, targetHarborFullName string) (targetImageFullNameList []string) { +func DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileNameOrGzipFileListTxt, localGzipFolderOrGzipFile string, targetHarborFullName string) (targetImageFullNameList []string) { // 支持单文件的形式 - if !utils.IsFileOrDir(localGzipFolderOrGzipFile) { + if !utils.IsDirOrFile(localGzipFolderOrGzipFile) { // 单个压缩文件 肯定是离线的形式 if !strings.HasSuffix(localGzipFolderOrGzipFile, ".tar.gz") { log.ErrorF("local gzip file %s is not a .tar.gz file !", localGzipFolderOrGzipFile) @@ -221,7 +229,7 @@ func DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileName, loca // download if downloadFromOss { - if !parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile) { + if !parseAndDownloadFromOss(ossUrlPrefix, ossFileNameOrGzipFileListTxt, localGzipFolderOrGzipFile) { log.ErrorF("download from oss error !") return nil } @@ -276,22 +284,19 @@ func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string) if ossUrlPrefix == "" { ossUrlPrefix = DefaultOssUrlPrefix } - log.InfoF("prepare to download from %s%s", ossUrlPrefix, ossFileName) - - // get oss endpoint - // mc login - - if !strings.HasPrefix(ossUrlPrefix, "/") { + if !strings.HasSuffix(ossUrlPrefix, "/") { ossUrlPrefix += "/" } + log.InfoF("prepare to download from %s%s", ossUrlPrefix, ossFileName) + if !DefaultCmiiMinioOperator.DemoMinioOperator.DownloadFileFromOssFullUrl(ossUrlPrefix+ossFileName, localGzipFolder) { log.ErrorF("download %s from oss error !", ossUrlPrefix+ossFileName) return false } if strings.HasSuffix(ossFileName, ".txt") { - // a list of files + // download from gzip file list txt // download all files in the txt file result := utils.ReadLineFromFile(localGzipFolder + ossFileName) for _, gzipFileName := range result { diff --git a/agent-operator/main.go b/agent-operator/main.go index 0173f05..e4fd622 100644 --- a/agent-operator/main.go +++ b/agent-operator/main.go @@ -78,7 +78,7 @@ func CmiiRunner() { var DLTUHelp = ` DLTUHelp - dltu [ossUrlPrefix] [ossFileName] [localGzipFolder] [harborHostFullName] [namespace] + dltu [ossUrlPrefix] [ossFileName] [localGzipFolderOrGzipFile] [harborHostFullName] [namespace] ` func main() { @@ -124,13 +124,13 @@ func main() { ossUrlPrefix := result[1] ossFileName := result[2] - localGzipFolder := result[3] + localGzipFolderOrGzipFile := result[3] harborHostFullName := result[4] namespace := result[5] fmt.Println("ossUrlPrefix: ", ossUrlPrefix) fmt.Println("ossFileName: ", ossFileName) - fmt.Println("localGzipFolder: ", localGzipFolder) + fmt.Println("localGzipFolderOrGzipFile: ", localGzipFolderOrGzipFile) fmt.Println("harborHostFullName: ", harborHostFullName) fmt.Println("namespace: ", namespace) fmt.Println() @@ -141,7 +141,7 @@ func main() { } // DLTU - targetImageFullNameList := DownloadLoadTagUpload(downloadFromOss, ossUrlPrefix, ossFileName, localGzipFolder, harborHostFullName) + targetImageFullNameList := DownloadLoadTagUpload(downloadFromOss, ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile, harborHostFullName) // 是否需要更新 if namespace != "" {