[agent-operator] - DLTU download from oss part

agent-common/utils/FileUtils.go

@@ -1,6 +1,8 @@
 package utils
 
 import (
+	"bufio"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -160,3 +162,28 @@ func RemoveFolderComplete(folderName string) bool {
 	}
 	return true
 }
+
+func ReadLineFromFile(fileFullPath string) (result []string) {
+
+	f, err := os.Open(fileFullPath)
+	if err != nil {
+		fmt.Println(err)
+		return result
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if len(line) > 0 { // ignore empty lines
+			result = append(result, line)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		fmt.Println(err)
+		return result
+	}
+
+	return result
+}

agent-operator/CmiiMinioOperator.go

@@ -143,15 +143,16 @@ func (op *MinioOperator) DownloadFileFromOssFullUrl(ossFullUrl string, filePath
 		return false
 	}
 
-	bucketName := split[0]
+	ossFullUrl = strings.TrimPrefix(ossFullUrl, split[0]+"/")
-	fileName := strings.Join(split[1:], "/")
+	bucketName := split[1]
-	//realFileName := split[len(split)-1]
+
+	fileName := strings.TrimPrefix(ossFullUrl, bucketName)
 
 	return op.DownloadFile(bucketName, filePath, fileName)
 
 }
 
-// DownloadFile objectName格式为 cmlc-installation/tmp/123/123.txt
+// DownloadFile fileNameWithPrefix格式为 tmp/123/123.txt
 func (op *MinioOperator) DownloadFile(bucketName, filePath, fileNameWithPrefix string) bool {
 
 	separator := os.PathSeparator

agent-operator/CmiiMinioOperator_test.go

@@ -17,3 +17,7 @@ func TestMinioOperator_UploadFile(t *testing.T) {
 	DefaultCmiiMinioOperator.DemoMinioOperator.UploadFile("cmlc-installation/tmp", "C:\\Users\\wddsh\\Documents\\Visual Studio 2022", "cmii-srs-oss-adaptor=2023-SA=2024-04-11=847.tar.gz")
 
 }
+
+func TestMinioOperator_DownloadFileFromOssFullUrl(t *testing.T) {
+	DefaultCmiiMinioOperator.DemoMinioOperator.DownloadFileFromOssFullUrl("https://oss.demo.uavcmlc.com:18000/cmlc-installation/tmp/cmlc=cmii=cmii-srs-oss-adaptor=2023-SA.tar.gz", "C:\\Users\\wddsh\\Documents\\Visual Studio 2022")
+}

agent-operator/CmiiOperator.go

@@ -203,9 +203,17 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
 //  5. ossFileName - 如果结尾为txt，则为文件的形式，如果为tar.gz，则为gzip文件夹的形式
 func DownloadLoadTagPush(downloadFromOss bool, ossUrlPrefix, ossFileName, localGzipFolder string, targetHarborFullName string) []string {
 
+	separator := os.PathSeparator
+	if !strings.HasSuffix(localGzipFolder, string(separator)) {
+		localGzipFolder += string(separator)
+	}
+
 	// download
 	if downloadFromOss {
-		parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder)
+		if !parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder) {
+			log.ErrorF("download from oss error !")
+			return nil
+		}
 	}
 	// load
 	loadAllGzipImageFromLocalFolder(localGzipFolder)
@@ -247,7 +255,7 @@ func loadAllGzipImageFromLocalFolder(localGzipFolder string) {
 
 }
 
-func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string) {
+func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string) bool {
 
 	if ossUrlPrefix == "" {
 		ossUrlPrefix = DefaultOssUrlPrefix
@@ -257,20 +265,26 @@ func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string)
 	// get oss endpoint
 	// mc login
 
+	if !strings.HasPrefix(ossUrlPrefix, "/") {
+		ossUrlPrefix += "/"
+	}
+
+	if !DefaultCmiiMinioOperator.DemoMinioOperator.DownloadFileFromOssFullUrl(ossUrlPrefix+ossFileName, localGzipFolder) {
+		log.ErrorF("download %s from oss error !", ossUrlPrefix+ossFileName)
+		return false
+	}
+
 	if strings.HasSuffix(ossFileName, ".txt") {
 		// a list of files
-
+		// download all files in the txt file
+		result := utils.ReadLineFromFile(localGzipFolder + ossFileName)
+		for _, gzipFileName := range result {
+			DefaultCmiiMinioOperator.DemoMinioOperator.DownloadFileFromOssFullUrl(ossUrlPrefix+gzipFileName, localGzipFolder)
+		}
 	}
 
-	if strings.HasSuffix(ossFileName, ".tar.gz") {
-		// single gzip file
-	}
-
-	// mv list all gzip file name
-
-	// download all gzip files to local folder
-
 	// 解析
+	return true
 }
 
 // DownloadCompressUploadFromDemo 获取DEMO环境的全部镜像