diff --git a/agent-common/utils/FileUtils.go b/agent-common/utils/FileUtils.go index b9b9b19..4c0778a 100644 --- a/agent-common/utils/FileUtils.go +++ b/agent-common/utils/FileUtils.go @@ -1,6 +1,8 @@ package utils import ( + "bufio" + "fmt" "io" "os" "path/filepath" @@ -160,3 +162,28 @@ func RemoveFolderComplete(folderName string) bool { } return true } + +func ReadLineFromFile(fileFullPath string) (result []string) { + + f, err := os.Open(fileFullPath) + if err != nil { + fmt.Println(err) + return result + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + if len(line) > 0 { // ignore empty lines + result = append(result, line) + } + } + + if err := scanner.Err(); err != nil { + fmt.Println(err) + return result + } + + return result +} diff --git a/agent-operator/CmiiMinioOperator.go b/agent-operator/CmiiMinioOperator.go index cc963c0..7bd2abe 100644 --- a/agent-operator/CmiiMinioOperator.go +++ b/agent-operator/CmiiMinioOperator.go @@ -143,15 +143,16 @@ func (op *MinioOperator) DownloadFileFromOssFullUrl(ossFullUrl string, filePath return false } - bucketName := split[0] - fileName := strings.Join(split[1:], "/") - //realFileName := split[len(split)-1] + ossFullUrl = strings.TrimPrefix(ossFullUrl, split[0]+"/") + bucketName := split[1] + + fileName := strings.TrimPrefix(ossFullUrl, bucketName) return op.DownloadFile(bucketName, filePath, fileName) } -// DownloadFile objectName格式为 cmlc-installation/tmp/123/123.txt +// DownloadFile fileNameWithPrefix格式为 tmp/123/123.txt func (op *MinioOperator) DownloadFile(bucketName, filePath, fileNameWithPrefix string) bool { separator := os.PathSeparator diff --git a/agent-operator/CmiiMinioOperator_test.go b/agent-operator/CmiiMinioOperator_test.go index dc63609..eaec317 100644 --- a/agent-operator/CmiiMinioOperator_test.go +++ b/agent-operator/CmiiMinioOperator_test.go @@ -17,3 +17,7 @@ func TestMinioOperator_UploadFile(t *testing.T) { DefaultCmiiMinioOperator.DemoMinioOperator.UploadFile("cmlc-installation/tmp", "C:\\Users\\wddsh\\Documents\\Visual Studio 2022", "cmii-srs-oss-adaptor=2023-SA=2024-04-11=847.tar.gz") } + +func TestMinioOperator_DownloadFileFromOssFullUrl(t *testing.T) { + DefaultCmiiMinioOperator.DemoMinioOperator.DownloadFileFromOssFullUrl("https://oss.demo.uavcmlc.com:18000/cmlc-installation/tmp/cmlc=cmii=cmii-srs-oss-adaptor=2023-SA.tar.gz", "C:\\Users\\wddsh\\Documents\\Visual Studio 2022") +} diff --git a/agent-operator/CmiiOperator.go b/agent-operator/CmiiOperator.go index 1fcf638..d85cd25 100644 --- a/agent-operator/CmiiOperator.go +++ b/agent-operator/CmiiOperator.go @@ -203,9 +203,17 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu // 5. ossFileName - 如果结尾为txt,则为文件的形式,如果为tar.gz,则为gzip文件夹的形式 func DownloadLoadTagPush(downloadFromOss bool, ossUrlPrefix, ossFileName, localGzipFolder string, targetHarborFullName string) []string { + separator := os.PathSeparator + if !strings.HasSuffix(localGzipFolder, string(separator)) { + localGzipFolder += string(separator) + } + // download if downloadFromOss { - parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder) + if !parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder) { + log.ErrorF("download from oss error !") + return nil + } } // load loadAllGzipImageFromLocalFolder(localGzipFolder) @@ -247,7 +255,7 @@ func loadAllGzipImageFromLocalFolder(localGzipFolder string) { } -func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string) { +func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string) bool { if ossUrlPrefix == "" { ossUrlPrefix = DefaultOssUrlPrefix @@ -257,20 +265,26 @@ func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string) // get oss endpoint // mc login + if !strings.HasPrefix(ossUrlPrefix, "/") { + ossUrlPrefix += "/" + } + + if !DefaultCmiiMinioOperator.DemoMinioOperator.DownloadFileFromOssFullUrl(ossUrlPrefix+ossFileName, localGzipFolder) { + log.ErrorF("download %s from oss error !", ossUrlPrefix+ossFileName) + return false + } + if strings.HasSuffix(ossFileName, ".txt") { // a list of files - + // download all files in the txt file + result := utils.ReadLineFromFile(localGzipFolder + ossFileName) + for _, gzipFileName := range result { + DefaultCmiiMinioOperator.DemoMinioOperator.DownloadFileFromOssFullUrl(ossUrlPrefix+gzipFileName, localGzipFolder) + } } - if strings.HasSuffix(ossFileName, ".tar.gz") { - // single gzip file - } - - // mv list all gzip file name - - // download all gzip files to local folder - // 解析 + return true } // DownloadCompressUploadFromDemo 获取DEMO环境的全部镜像