diff --git a/agent-common/image/ImageNameConvert_test.go b/agent-common/image/ImageNameConvert_test.go index 0befdd3..4cf44ca 100644 --- a/agent-common/image/ImageNameConvert_test.go +++ b/agent-common/image/ImageNameConvert_test.go @@ -155,29 +155,33 @@ func TestGzipFileNameToImageFullName(t *testing.T) { } test3 := []string{ - "docker=bitnami=bitnami-shell=10-debian-10-r140.tar.gz", - "docker=kubernetesui=dashboard=v2.0.1.tar.gz", - "docker=bitnami=bitnami-shell=11-debian-11-r136.tar.gz", - "docker=kubernetesui=metrics-scraper=v1.0.4.tar.gz", - "docker=bitnami=minio=2022.5.4.tar.gz", - "docker=mongo=5.0.tar.gz", - "docker=bitnami=minio=2023.5.4.tar.gz", - "docker=nacos=nacos-server=v2.1.2-slim.tar.gz", - "docker=bitnami=mysql=8.0.35-debian-11-r1.tar.gz", - "docker=nginx=1.21.3.tar.gz", - "docker=bitnami=mysql=8.1.0-debian-11-r42.tar.gz", - "docker=ossrs=srs=v4.0.136.tar.gz", - "docker=bitnami=rabbitmq=3.11.26-debian-11-r2.tar.gz", - "docker=ossrs=srs=v4.0-r3.tar.gz", - "docker=bitnami=rabbitmq=3.9.12-debian-10-r3.tar.gz", - "docker=rabbitmq=3.9-management.tar.gz", - "docker=bitnami=redis=6.2.14-debian-11-r1.tar.gz", - "docker=redis=6.0.20-alpine.tar.gz", "docker=bitnami=redis=6.2.6-debian-10-r0.tar.gz", + "docker=bitnami=redis=6.2.14-debian-11-r1.tar.gz", + "docker=bitnami=mysql=8.0.35-debian-11-r1.tar.gz", + "docker=bitnami=mysql=8.1.0-debian-11-r42.tar.gz", "docker=simonrupf=chronyd=0.4.3.tar.gz", - "docker=busybox=latest.tar.gz", - "docker=dyrnq=nfs-subdir-external-provisioner=v4.0.2.tar.gz", + "docker=bitnami=bitnami-shell=10-debian-10-r140.tar.gz", + "docker=bitnami=bitnami-shell=11-debian-11-r136.tar.gz", + "docker=bitnami=rabbitmq=3.9.12-debian-10-r3.tar.gz", + "docker=bitnami=rabbitmq=3.11.26-debian-11-r2.tar.gz", + "docker=ossrs=srs=v4.0.136.tar.gz", + "docker=ossrs=srs=v5.0.195.tar.gz", + "docker=ossrs=srs=v4.0-r3.tar.gz", "docker=emqx=emqx=4.2.12.tar.gz", + "docker=emqx=emqx=5.5.1.tar.gz", + "docker=nacos=nacos-server=v2.1.2.tar.gz", + "docker=nacos=nacos-server=v2.1.2-slim.tar.gz", + "docker=library=mongo=5.0.tar.gz", + "docker=library=rabbitmq=3.9-management.tar.gz", + "docker=bitnami=minio=2022.5.4.tar.gz", + "docker=bitnami=minio=2023.5.4.tar.gz", + "docker=kubernetesui=dashboard=v2.0.1.tar.gz", + "docker=kubernetesui=metrics-scraper=v1.0.4.tar.gz", + "docker=library=nginx=1.21.3.tar.gz", + "docker=library=redis=6.0.20-alpine.tar.gz", + "docker=dyrnq=nfs-subdir-external-provisioner=v4.0.2.tar.gz", + "docker=jerrychina2020=rke-tools=v0.175-linux.tar.gz", + "docker=library=busybox=latest.tar.gz", } test = append(test, test2...) diff --git a/agent-operator/CmiiMinioOperator.go b/agent-operator/CmiiMinioOperator.go index d3f4ede..cc963c0 100644 --- a/agent-operator/CmiiMinioOperator.go +++ b/agent-operator/CmiiMinioOperator.go @@ -1,27 +1,23 @@ package main import ( - "github.com/minio/minio-go" + "os" "strings" + + "github.com/minio/minio-go" + "wdd.io/agent-common/utils" ) type CmiiMinioOperator struct { - LocalEndpoint string - PublicEndpoint string - - AccessKeyID string - SecretAccessKey string - - LocalClient *minio.Client - PublicClient *minio.Client - DemoClient *minio.Client - - OctopusBucketName string + LocalMinioOperator *MinioOperator + PublicMinioOperator *MinioOperator + DemoMinioOperator *MinioOperator } var DefaultCmiiMinioOperator = newInstance() func newInstance() *CmiiMinioOperator { + op := &CmiiMinioOperator{} op.buildCmiiMinioOperator() @@ -32,87 +28,182 @@ func newInstance() *CmiiMinioOperator { const ( DefaultLocalEndpoint = "10.250.0.100:9000" DefaultPublicEndpoint = "42.192.52.227:9000" - DemoEndpoint = "oss.demo.uavcmlc.com:18000" + DefaultDemoEndpoint = "oss.demo.uavcmlc.com:18000" DefaultAccessKeyID = "cmii" DefaultSecretAccessKey = "B#923fC7mk" DefaultOctopusBucketName = "octopus" - DefaultOssUrlPrefix = "https://oss.demo.uavcmlc.com/cmlc-installation/tmp/" + DefaultOssUrlPrefix = "https://oss.demo.uavcmlc.com:18000/cmlc-installation/tmp/" ) func (op *CmiiMinioOperator) buildCmiiMinioOperator() { - var err error - if op.LocalClient == nil { - // 初始化Minio客户端对象。 - op.LocalClient, err = minio.New(DefaultLocalEndpoint, DefaultAccessKeyID, DefaultSecretAccessKey, false) - if err != nil { - log.ErrorF("[buildCmiiMinioOperator] - build for LocalClient error !=> %s", err.Error()) + if op.LocalMinioOperator == nil { + op.LocalMinioOperator = &MinioOperator{ + MinioEndpoint: DefaultLocalEndpoint, + EndpointSecure: false, + AccessKeyID: DefaultAccessKeyID, + SecretAccessKey: DefaultSecretAccessKey, + Client: nil, } + + op.LocalMinioOperator.BuildMinioOperator() } - if op.PublicClient == nil { - // 初始化Minio客户端对象。 - op.PublicClient, err = minio.New(DefaultPublicEndpoint, DefaultAccessKeyID, DefaultSecretAccessKey, false) - if err != nil { - log.ErrorF("[buildCmiiMinioOperator] - build for PublicClient error ! => %s", err.Error()) + if op.PublicMinioOperator == nil { + op.PublicMinioOperator = &MinioOperator{ + MinioEndpoint: DefaultPublicEndpoint, + EndpointSecure: false, + AccessKeyID: DefaultAccessKeyID, + SecretAccessKey: DefaultSecretAccessKey, + Client: nil, } + + op.PublicMinioOperator.BuildMinioOperator() } - if op.DemoClient == nil { - // 初始化Minio客户端对象。 - op.DemoClient, err = minio.New(DemoEndpoint, DefaultAccessKeyID, DefaultSecretAccessKey, true) - if err != nil { - log.ErrorF("[buildCmiiMinioOperator] - build for DemoClient error ! => %s", err.Error()) + if op.DemoMinioOperator == nil { + op.DemoMinioOperator = &MinioOperator{ + MinioEndpoint: DefaultDemoEndpoint, + EndpointSecure: true, + AccessKeyID: DefaultAccessKeyID, + SecretAccessKey: DefaultSecretAccessKey, + Client: nil, } + + op.DemoMinioOperator.BuildMinioOperator() } log.DebugF("[buildCmiiMinioOperator] - build client success !") } func (op *CmiiMinioOperator) UploadToLocalOctopus(filePath, fileName string) bool { - return op.uploadToOctopus(op.LocalClient, filePath, fileName) + return op.LocalMinioOperator.UploadFile(DefaultOctopusBucketName, filePath, fileName) } func (op *CmiiMinioOperator) UploadToPublicOctopus(filePath, fileName string) bool { - return op.uploadToOctopus(op.PublicClient, filePath, fileName) + return op.PublicMinioOperator.UploadFile(DefaultOctopusBucketName, filePath, fileName) } func (op *CmiiMinioOperator) UploadToDemo(bucketName, filePath, fileName string) bool { - return op.uploadToOss(op.DemoClient, bucketName, filePath, fileName) + return op.DemoMinioOperator.UploadFile(bucketName, filePath, fileName) } -func (op *CmiiMinioOperator) uploadToOctopus(client *minio.Client, filePath, fileName string) bool { +type MinioOperator struct { + MinioEndpoint string + EndpointSecure bool - return op.uploadToOss(client, DefaultOctopusBucketName, filePath, fileName) + AccessKeyID string + SecretAccessKey string + + Client *minio.Client } -func (op *CmiiMinioOperator) uploadToOss(client *minio.Client, bucketName, filePath, fileName string) bool { +func (op *MinioOperator) BuildMinioOperator() { - if !strings.HasSuffix(filePath, "/") { - filePath += "/" + op.MinioEndpoint = strings.TrimPrefix(op.MinioEndpoint, "http://") + op.MinioEndpoint = strings.TrimPrefix(op.MinioEndpoint, "https://") + if strings.Contains(op.MinioEndpoint, "/") { + op.MinioEndpoint = strings.Split(op.MinioEndpoint, "/")[0] } - realFileName := fileName - // 解析bucket上传的真实名称 - if strings.Contains(bucketName, "/") { - splitN := strings.SplitN(bucketName, "/", 2) - bucketName = splitN[0] - if !strings.HasSuffix(splitN[1], "/") { - splitN[1] = splitN[1] + "/" - } - realFileName = splitN[1] + fileName - } - - // 使用PutObject上传文件 - // realFileName ==> tmp/123/123.txt - n, err := client.FPutObject(bucketName, realFileName, filePath+fileName, minio.PutObjectOptions{}) + client, err := minio.New(op.MinioEndpoint, op.AccessKeyID, op.SecretAccessKey, op.EndpointSecure) if err != nil { - log.ErrorF("[uploadToOss] - upload %s error %s", filePath+fileName, err.Error()) + log.ErrorF("Failed to create client: %v", err) + } + log.DebugF("endpoint: %s accessKeyId: %s secretAccessKey: %s", op.MinioEndpoint, op.AccessKeyID, op.SecretAccessKey) + + op.Client = client +} + +func (op *MinioOperator) ListFileInBucket(bucketName string) { + // 使用ListObjectsV2获取bucket中的所有文件名。 + doneChan := make(chan struct{}) + + objectInfo := op.Client.ListObjectsV2(bucketName, "", false, doneChan) + + for object := range objectInfo { + utils.BeautifulPrint(object) + } + +} + +func (op *MinioOperator) DownloadFileFromOssFullUrl(ossFullUrl string, filePath string) bool { + log.InfoF("[DownloadFileFromOssFullUrl] - download from %s", ossFullUrl) + + // https://oss.demo.uavcmlc.com:18000/cmlc-installation/tmp/docker.tar.gz + ossFullUrl = strings.TrimPrefix(ossFullUrl, "http://") + ossFullUrl = strings.TrimPrefix(ossFullUrl, "https://") + // oss.demo.uavcmlc.com:18000/cmlc-installation/tmp/docker.tar.gz + + split := strings.Split(ossFullUrl, "/") + if len(split) < 2 { + log.ErrorF("DownloadFileFromOssFullUrl - split error len < 2 !") + return false + } + + bucketName := split[0] + fileName := strings.Join(split[1:], "/") + //realFileName := split[len(split)-1] + + return op.DownloadFile(bucketName, filePath, fileName) + +} + +// DownloadFile objectName格式为 cmlc-installation/tmp/123/123.txt +func (op *MinioOperator) DownloadFile(bucketName, filePath, fileNameWithPrefix string) bool { + + separator := os.PathSeparator + if !strings.HasSuffix(filePath, string(separator)) { + filePath += string(separator) + } + + fileNameWithPrefix = strings.TrimPrefix(fileNameWithPrefix, "/") + + // 获取下载的真实文件名称 123.txt + realFileName := fileNameWithPrefix + if strings.Contains(fileNameWithPrefix, "/") { + realFileName = strings.Split(fileNameWithPrefix, "/")[len(strings.Split(fileNameWithPrefix, "/"))-1] + } + + log.InfoF("[DownloadFile] - download from [%s] to [%s]", op.MinioEndpoint+"/"+bucketName+"/"+fileNameWithPrefix, filePath+realFileName) + err := op.Client.FGetObject(bucketName, fileNameWithPrefix, filePath+realFileName, minio.GetObjectOptions{}) + if err != nil { + log.ErrorF("[DownloadFile] - download %s error %s", filePath+realFileName, err.Error()) return false } - log.InfoF("[uploadToOss] - uploaded %s of size %d", filePath+fileName, n) + return true +} + +// UploadFile bucketNameWithSuffix格式为 cmlc-installation/tmp/123 的格式 +func (op *MinioOperator) UploadFile(bucketNameWithSuffix, filePath, fileName string) bool { + + separator := os.PathSeparator + if !strings.HasSuffix(filePath, string(separator)) { + filePath += string(separator) + } + + bucketNameWithSuffix = strings.TrimPrefix(bucketNameWithSuffix, "/") + oldBucketName := bucketNameWithSuffix + realFileName := fileName + // 解析bucket上传的真实名称 + if strings.Contains(bucketNameWithSuffix, "/") { + bucketNameWithSuffix = strings.Split(bucketNameWithSuffix, "/")[0] + fileName = strings.TrimPrefix(oldBucketName, bucketNameWithSuffix) + "/" + realFileName + } + + // 使用PutObject上传文件 + // fileName ==> tmp/123/123.txt + // realFileName ==> 123.txt + log.InfoF("[UploadFile] - upload from [%s] to [%s]", filePath+realFileName, op.MinioEndpoint+"/"+bucketNameWithSuffix+"/"+fileName) + n, err := op.Client.FPutObject(bucketNameWithSuffix, fileName, filePath+realFileName, minio.PutObjectOptions{}) + if err != nil { + log.ErrorF("[UploadFile] - upload [%s] to [%s] error %s", filePath+realFileName, op.MinioEndpoint+"/"+bucketNameWithSuffix+"/"+fileName, err.Error()) + return false + } + + log.InfoF("[UploadFile] - uploaded %s of size %d", filePath+fileName, n) return true } diff --git a/agent-operator/CmiiMinioOperator_test.go b/agent-operator/CmiiMinioOperator_test.go index edec949..dc63609 100644 --- a/agent-operator/CmiiMinioOperator_test.go +++ b/agent-operator/CmiiMinioOperator_test.go @@ -7,3 +7,13 @@ func TestCmiiMinioOperator_UploadToDemo(t *testing.T) { DefaultCmiiMinioOperator.UploadToDemo("cmlc-installation", "/home/wdd/Downloads/", "go1.22.1.linux-amd64.tar.gz") } + +func TestMinioOperator_DownloadFile(t *testing.T) { + DefaultCmiiMinioOperator.DemoMinioOperator.DownloadFile("cmlc-installation", "C:\\Users\\wddsh\\Documents\\Visual Studio 2022", "tmp/cmii-srs-oss-adaptor=2023-SA=2024-04-11=847.tar.gz") +} + +func TestMinioOperator_UploadFile(t *testing.T) { + + DefaultCmiiMinioOperator.DemoMinioOperator.UploadFile("cmlc-installation/tmp", "C:\\Users\\wddsh\\Documents\\Visual Studio 2022", "cmii-srs-oss-adaptor=2023-SA=2024-04-11=847.tar.gz") + +} diff --git a/agent-operator/CmiiOperator.go b/agent-operator/CmiiOperator.go index 04bb9ee..1fcf638 100644 --- a/agent-operator/CmiiOperator.go +++ b/agent-operator/CmiiOperator.go @@ -9,6 +9,7 @@ import ( "path/filepath" "slices" "strings" + image2 "wdd.io/agent-common/image" "wdd.io/agent-common/utils" "wdd.io/agent-operator/image" @@ -179,7 +180,7 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu // extract demo oss location suffix from gzipFolderFullPath trimPrefix := strings.TrimPrefix(gzipFolderFullPath, OfflineImageGzipFolderPrefix) bucketName := "cmlc-installation/" + trimPrefix - log.InfoF("gzip file location in demo oss is %s", DemoEndpoint+"/"+bucketName) + log.InfoF("gzip file location in demo oss is %s", DefaultDemoEndpoint+"/"+bucketName) minioOperator := CmiiMinioOperator{} for _, gzipFileName := range allGzipFileNameList { @@ -195,16 +196,16 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu } // DownloadLoadTagPush DLTU procedure ImageSync的另外一般流程，需要支持 堡垒机（纯离线）的模式 -// 1. 文件模式，从文件中读取全部的镜像全名称-写死到文件中 -// 2. Gzip文件目录，RKE MIDDLE CMII三个文件目录 - 是否需要约定目录 +// 2. Gzip文件目录，RKE MIDDLE CMII三个文件目录 - 约定目录 // 约定目录 /root/wdd/image/rke/ /root/wdd/image/middle/ /root/wdd/image/cmii/ -// 3. 读取本机的IP地址 - 或者进行参数传递 -// 4. OSS地址 - 参数传递gizpFileName -func DownloadLoadTagPush(downloadFromOss bool, ossUrlPrefix, localGzipFolder string, targetHarborFullName string) []string { +// 3. 读取本机的IP地址 - 参数传递 +// 4. OSS地址 - ossUrlPrefix传空 则使用默认值 +// 5. ossFileName - 如果结尾为txt，则为文件的形式，如果为tar.gz，则为gzip文件夹的形式 +func DownloadLoadTagPush(downloadFromOss bool, ossUrlPrefix, ossFileName, localGzipFolder string, targetHarborFullName string) []string { // download if downloadFromOss { - parseAndDownloadFromOssUrlPrefix(ossUrlPrefix, localGzipFolder) + parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder) } // load loadAllGzipImageFromLocalFolder(localGzipFolder) @@ -246,8 +247,28 @@ func loadAllGzipImageFromLocalFolder(localGzipFolder string) { } -// todo -func parseAndDownloadFromOssUrlPrefix(ossUrlPrefix string, localGzipFolder string) { +func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string) { + + if ossUrlPrefix == "" { + ossUrlPrefix = DefaultOssUrlPrefix + } + log.InfoF("prepare to download from %s%s", ossUrlPrefix, ossFileName) + + // get oss endpoint + // mc login + + if strings.HasSuffix(ossFileName, ".txt") { + // a list of files + + } + + if strings.HasSuffix(ossFileName, ".tar.gz") { + // single gzip file + } + + // mv list all gzip file name + + // download all gzip files to local folder // 解析 } diff --git a/agent-operator/image/CmiiImageSync.go b/agent-operator/image/CmiiImageSync.go index e1ade20..60db9a9 100644 --- a/agent-operator/image/CmiiImageSync.go +++ b/agent-operator/image/CmiiImageSync.go @@ -6,15 +6,16 @@ import ( "encoding/json" "errors" "fmt" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/filters" - "github.com/docker/docker/client" - "github.com/klauspost/pgzip" "io" "io/fs" "os" "strconv" "strings" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/client" + "github.com/klauspost/pgzip" image2 "wdd.io/agent-common/image" "wdd.io/agent-common/logger" "wdd.io/agent-common/utils" diff --git a/agent-operator/main.go b/agent-operator/main.go index d22c6b6..6af27cb 100644 --- a/agent-operator/main.go +++ b/agent-operator/main.go @@ -77,11 +77,13 @@ func CmiiRunner() { var DLTUHelp = ` DLTUHelp - dltu [ossUrl] [localGzipFolder] [harborHostFullName] + dltu [ossUrlPrefix] [ossFileName] [localGzipFolder] [harborHostFullName] ` func main() { + // C:\Users\wddsh\go\bin\gox.exe -osarch="linux/amd66" -output "build/operator_{{.OS}}_{{.Arch}}" + //RealProjectRunner() // 解析命令行参数 @@ -111,22 +113,29 @@ func main() { result = append(result, text) } - if len(result) != 4 { + if len(result) != 5 { fmt.Println("input error!") fmt.Printf(DLTUHelp) return } - ossUrl := result[1] - localGzipFolder := result[2] - harborHostFullName := result[3] + ossUrlPrefix := result[1] + ossFileName := result[2] + localGzipFolder := result[3] + harborHostFullName := result[4] - fmt.Println("ossUrl: ", ossUrl) + fmt.Println("ossUrlPrefix: ", ossUrlPrefix) + fmt.Println("ossFileName: ", ossFileName) fmt.Println("localGzipFolder: ", localGzipFolder) fmt.Println("harborHostFullName: ", harborHostFullName) fmt.Println() - DownloadLoadTagPush(false, ossUrl, localGzipFolder, harborHostFullName) + var downloadFromOss bool + if ossFileName != "" { + downloadFromOss = true + } + + DownloadLoadTagPush(downloadFromOss, ossUrlPrefix, ossFileName, localGzipFolder, harborHostFullName) // 下载 diff --git a/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java b/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java index 158ec00..6fe0f60 100644 --- a/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java +++ b/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java @@ -38,13 +38,13 @@ public class TestImageSyncScheduler { public void runImageSync() { ArrayList CmiiAppNameList = new ArrayList<>(List.of( - "cmii-uav-process:5.4.0-041901" +// "cmii-uav-process:5.4.0-041901" )); ArrayList ImageFullNameList = new ArrayList<>(List.of( // "harbor.cdcyy.com.cn/cmii/cmii-live-operator:5.2.0", // "harbor.cdcyy.com.cn/cmii/cmii/srs:v5.0.195" -// "harbor.cdcyy.com.cn/cmii/cmii-uav-industrial-portfolio:5.54.0-cqly-041601" + "harbor.cdcyy.com.cn/cmii/cmii-uav-platform:5.3.0-cqly-042202" )); Boolean downloadAndCompressOnly = false;