[Agent][Operator] - bug fix

2024-05-08 11:31:24 +08:00
parent e56b9bd65c
commit bd2a50bbae
3 changed files with 32 additions and 27 deletions
--- a/agent-common/utils/FileUtils.go
+++ b/agent-common/utils/FileUtils.go
@@ -120,8 +120,8 @@ func WordSpaceCompletion(source string, totalLength int) string {
 	return source
 }

-// IsFileOrDir 如果是目录则返回true，是文件则返回false
-func IsFileOrDir(path string) bool {
+// IsDirOrFile 如果是目录则返回true，是文件则返回false
+func IsDirOrFile(path string) bool {
 	info, err := os.Stat(path)
 	if err != nil {
 		return false
--- a/agent-operator/CmiiOperator.go
+++ b/agent-operator/CmiiOperator.go
@@ -119,7 +119,7 @@ func concatAndUniformCmiiImage(fullImageList []string, cmiiImageList []string) [
 }

 // DownloadCompressUpload DCU 镜像同步的前半部分，通常在35.71  LapPro执行，无需Bastion Mode
-func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
+func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileFullNameList []string) {

 	// Download
 	log.Info("DOWNLOAD START !")
@@ -129,6 +129,8 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
 		return slices.Contains(errorPullImageList, imageName)
 	})

+	var localGzipFileListTxt string
+
 	// Compress
 	if shouldGzip {
 		// mkdir folder
@@ -150,7 +152,7 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
 				continue
 			}
 			// 压缩成功
-			allGzipFileNameList = append(allGzipFileNameList, gzipImageFileFullPath)
+			allGzipFileFullNameList = append(allGzipFileFullNameList, gzipImageFileFullPath)
 		}
 		// remove failed
 		fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool {
@@ -158,16 +160,16 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
 		})

 		// write to file
-		targetFile := gzipFolderFullPath + string(os.PathSeparator) + "all-gzip-image-file-name.txt"
+		localGzipFileListTxt = gzipFolderFullPath + string(os.PathSeparator) + "all-gzip-image-file-name.txt"

-		for _, gzipFileFullName := range allGzipFileNameList {
+		for _, gzipFileFullName := range allGzipFileFullNameList {
 			utils.AppendContentToFile(
 				strings.TrimPrefix(gzipFileFullName, gzipFolderFullPath),
-				targetFile,
+				localGzipFileListTxt,
 			)
 		}

-		log.InfoF("all gzip file name list is %s", targetFile)
+		log.InfoF("all gzip file name list is %s", localGzipFileListTxt)
 	}

 	// Upload
@@ -182,17 +184,23 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
 		bucketNameWithPrefix := "cmlc-installation/" + trimPrefix
 		log.InfoF("gzip file location in demo oss is %s", DefaultDemoEndpoint+"/"+bucketNameWithPrefix)

-		for _, gzipFileName := range allGzipFileNameList {
+		// upload gzip file list txt to demo
+		if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipFolderFullPath, localGzipFileListTxt) {
+			log.ErrorF("upload of %s to demo oss error !", localGzipFileListTxt)
+		}
+
+		log.InfoF("upload all gzip file to demo oss !")
+		for _, gzipFileFullName := range allGzipFileFullNameList {
 			// SaveToGzipFile 返回的是全路径 归一化处理 gzip file name
-			gzipFileName = strings.TrimPrefix(gzipFileName, gzipFolderFullPath)
-			if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipFolderFullPath, gzipFileName) {
-				log.ErrorF("upload of %s to demo oss error !", gzipFolderFullPath+gzipFileName)
+			gzipFileFullName = strings.TrimPrefix(gzipFileFullName, gzipFolderFullPath)
+			if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipFolderFullPath, gzipFileFullName) {
+				log.ErrorF("upload of %s to demo oss error !", gzipFileFullName)
 			}
 		}

 	}

-	return errorPullImageList, errorGzipImageList, fullNameList, allGzipFileNameList
+	return errorPullImageList, errorGzipImageList, fullNameList, allGzipFileFullNameList
 }

 // DownloadLoadTagUpload DLTU procedure ImageSync的另外一般流程，需要支持 堡垒机（纯离线）的模式
@@ -201,10 +209,10 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
 //  3. 读取本机的IP地址 - 参数传递
 //  4. OSS地址 - ossUrlPrefix传空  则使用默认值
 //  5. ossFileName - 如果结尾为txt，则为文件的形式，如果为tar.gz，则为gzip文件夹的形式
-func DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile string, targetHarborFullName string) (targetImageFullNameList []string) {
+func DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileNameOrGzipFileListTxt, localGzipFolderOrGzipFile string, targetHarborFullName string) (targetImageFullNameList []string) {

 	// 支持单文件的形式
-	if !utils.IsFileOrDir(localGzipFolderOrGzipFile) {
+	if !utils.IsDirOrFile(localGzipFolderOrGzipFile) {
 		// 单个压缩文件 肯定是离线的形式
 		if !strings.HasSuffix(localGzipFolderOrGzipFile, ".tar.gz") {
 			log.ErrorF("local gzip file %s is not a .tar.gz file !", localGzipFolderOrGzipFile)
@@ -221,7 +229,7 @@ func DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileName, loca

 		// download
 		if downloadFromOss {
-			if !parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile) {
+			if !parseAndDownloadFromOss(ossUrlPrefix, ossFileNameOrGzipFileListTxt, localGzipFolderOrGzipFile) {
 				log.ErrorF("download from oss error !")
 				return nil
 			}
@@ -276,22 +284,19 @@ func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string)
 	if ossUrlPrefix == "" {
 		ossUrlPrefix = DefaultOssUrlPrefix
 	}
-	log.InfoF("prepare to download from %s%s", ossUrlPrefix, ossFileName)
-
-	// get oss endpoint
-	// mc login
-
-	if !strings.HasPrefix(ossUrlPrefix, "/") {
+	if !strings.HasSuffix(ossUrlPrefix, "/") {
 		ossUrlPrefix += "/"
 	}

+	log.InfoF("prepare to download from %s%s", ossUrlPrefix, ossFileName)
+
 	if !DefaultCmiiMinioOperator.DemoMinioOperator.DownloadFileFromOssFullUrl(ossUrlPrefix+ossFileName, localGzipFolder) {
 		log.ErrorF("download %s from oss error !", ossUrlPrefix+ossFileName)
 		return false
 	}

 	if strings.HasSuffix(ossFileName, ".txt") {
-		// a list of files
+		// download from gzip file list txt
 		// download all files in the txt file
 		result := utils.ReadLineFromFile(localGzipFolder + ossFileName)
 		for _, gzipFileName := range result {
--- a/agent-operator/main.go
+++ b/agent-operator/main.go
@@ -78,7 +78,7 @@ func CmiiRunner() {

 var DLTUHelp = `
 DLTUHelp
-  dltu [ossUrlPrefix] [ossFileName] [localGzipFolder] [harborHostFullName] [namespace]
+  dltu [ossUrlPrefix] [ossFileName] [localGzipFolderOrGzipFile] [harborHostFullName] [namespace]
 `

 func main() {
@@ -124,13 +124,13 @@ func main() {

 				ossUrlPrefix := result[1]
 				ossFileName := result[2]
-				localGzipFolder := result[3]
+				localGzipFolderOrGzipFile := result[3]
 				harborHostFullName := result[4]
 				namespace := result[5]

 				fmt.Println("ossUrlPrefix: ", ossUrlPrefix)
 				fmt.Println("ossFileName: ", ossFileName)
-				fmt.Println("localGzipFolder: ", localGzipFolder)
+				fmt.Println("localGzipFolderOrGzipFile: ", localGzipFolderOrGzipFile)
 				fmt.Println("harborHostFullName: ", harborHostFullName)
 				fmt.Println("namespace: ", namespace)
 				fmt.Println()
@@ -141,7 +141,7 @@ func main() {
 				}

 				// DLTU
-				targetImageFullNameList := DownloadLoadTagUpload(downloadFromOss, ossUrlPrefix, ossFileName, localGzipFolder, harborHostFullName)
+				targetImageFullNameList := DownloadLoadTagUpload(downloadFromOss, ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile, harborHostFullName)

 				// 是否需要更新
 				if namespace != "" {