[agent-operator] - minio refresh

2024-04-22 16:20:55 +08:00
parent c6ee9ed859
commit c8bfb2b133
7 changed files with 232 additions and 96 deletions
--- a/agent-common/image/ImageNameConvert_test.go
+++ b/agent-common/image/ImageNameConvert_test.go
@@ -155,29 +155,33 @@ func TestGzipFileNameToImageFullName(t *testing.T) {
 	}

 	test3 := []string{
-		"docker=bitnami=bitnami-shell=10-debian-10-r140.tar.gz",
-		"docker=kubernetesui=dashboard=v2.0.1.tar.gz",
-		"docker=bitnami=bitnami-shell=11-debian-11-r136.tar.gz",
-		"docker=kubernetesui=metrics-scraper=v1.0.4.tar.gz",
-		"docker=bitnami=minio=2022.5.4.tar.gz",
-		"docker=mongo=5.0.tar.gz",
-		"docker=bitnami=minio=2023.5.4.tar.gz",
-		"docker=nacos=nacos-server=v2.1.2-slim.tar.gz",
-		"docker=bitnami=mysql=8.0.35-debian-11-r1.tar.gz",
-		"docker=nginx=1.21.3.tar.gz",
-		"docker=bitnami=mysql=8.1.0-debian-11-r42.tar.gz",
-		"docker=ossrs=srs=v4.0.136.tar.gz",
-		"docker=bitnami=rabbitmq=3.11.26-debian-11-r2.tar.gz",
-		"docker=ossrs=srs=v4.0-r3.tar.gz",
-		"docker=bitnami=rabbitmq=3.9.12-debian-10-r3.tar.gz",
-		"docker=rabbitmq=3.9-management.tar.gz",
-		"docker=bitnami=redis=6.2.14-debian-11-r1.tar.gz",
-		"docker=redis=6.0.20-alpine.tar.gz",
 		"docker=bitnami=redis=6.2.6-debian-10-r0.tar.gz",
+		"docker=bitnami=redis=6.2.14-debian-11-r1.tar.gz",
+		"docker=bitnami=mysql=8.0.35-debian-11-r1.tar.gz",
+		"docker=bitnami=mysql=8.1.0-debian-11-r42.tar.gz",
 		"docker=simonrupf=chronyd=0.4.3.tar.gz",
-		"docker=busybox=latest.tar.gz",
-		"docker=dyrnq=nfs-subdir-external-provisioner=v4.0.2.tar.gz",
+		"docker=bitnami=bitnami-shell=10-debian-10-r140.tar.gz",
+		"docker=bitnami=bitnami-shell=11-debian-11-r136.tar.gz",
+		"docker=bitnami=rabbitmq=3.9.12-debian-10-r3.tar.gz",
+		"docker=bitnami=rabbitmq=3.11.26-debian-11-r2.tar.gz",
+		"docker=ossrs=srs=v4.0.136.tar.gz",
+		"docker=ossrs=srs=v5.0.195.tar.gz",
+		"docker=ossrs=srs=v4.0-r3.tar.gz",
 		"docker=emqx=emqx=4.2.12.tar.gz",
+		"docker=emqx=emqx=5.5.1.tar.gz",
+		"docker=nacos=nacos-server=v2.1.2.tar.gz",
+		"docker=nacos=nacos-server=v2.1.2-slim.tar.gz",
+		"docker=library=mongo=5.0.tar.gz",
+		"docker=library=rabbitmq=3.9-management.tar.gz",
+		"docker=bitnami=minio=2022.5.4.tar.gz",
+		"docker=bitnami=minio=2023.5.4.tar.gz",
+		"docker=kubernetesui=dashboard=v2.0.1.tar.gz",
+		"docker=kubernetesui=metrics-scraper=v1.0.4.tar.gz",
+		"docker=library=nginx=1.21.3.tar.gz",
+		"docker=library=redis=6.0.20-alpine.tar.gz",
+		"docker=dyrnq=nfs-subdir-external-provisioner=v4.0.2.tar.gz",
+		"docker=jerrychina2020=rke-tools=v0.175-linux.tar.gz",
+		"docker=library=busybox=latest.tar.gz",
 	}

 	test = append(test, test2...)
--- a/agent-operator/CmiiMinioOperator.go
+++ b/agent-operator/CmiiMinioOperator.go
@@ -1,27 +1,23 @@
 package main

 import (
-	"github.com/minio/minio-go"
+	"os"
 	"strings"
+
+	"github.com/minio/minio-go"
+	"wdd.io/agent-common/utils"
 )

 type CmiiMinioOperator struct {
-	LocalEndpoint  string
-	PublicEndpoint string
-
-	AccessKeyID     string
-	SecretAccessKey string
-
-	LocalClient  *minio.Client
-	PublicClient *minio.Client
-	DemoClient   *minio.Client
-
-	OctopusBucketName string
+	LocalMinioOperator  *MinioOperator
+	PublicMinioOperator *MinioOperator
+	DemoMinioOperator   *MinioOperator
 }

 var DefaultCmiiMinioOperator = newInstance()

 func newInstance() *CmiiMinioOperator {
+
 	op := &CmiiMinioOperator{}

 	op.buildCmiiMinioOperator()
@@ -32,87 +28,182 @@ func newInstance() *CmiiMinioOperator {
 const (
 	DefaultLocalEndpoint     = "10.250.0.100:9000"
 	DefaultPublicEndpoint    = "42.192.52.227:9000"
-	DemoEndpoint             = "oss.demo.uavcmlc.com:18000"
+	DefaultDemoEndpoint      = "oss.demo.uavcmlc.com:18000"
 	DefaultAccessKeyID       = "cmii"
 	DefaultSecretAccessKey   = "B#923fC7mk"
 	DefaultOctopusBucketName = "octopus"
-	DefaultOssUrlPrefix      = "https://oss.demo.uavcmlc.com/cmlc-installation/tmp/"
+	DefaultOssUrlPrefix      = "https://oss.demo.uavcmlc.com:18000/cmlc-installation/tmp/"
 )

 func (op *CmiiMinioOperator) buildCmiiMinioOperator() {

-	var err error
-	if op.LocalClient == nil {
-		// 初始化Minio客户端对象。
-		op.LocalClient, err = minio.New(DefaultLocalEndpoint, DefaultAccessKeyID, DefaultSecretAccessKey, false)
-		if err != nil {
-			log.ErrorF("[buildCmiiMinioOperator] - build for LocalClient error !=> %s", err.Error())
+	if op.LocalMinioOperator == nil {
+		op.LocalMinioOperator = &MinioOperator{
+			MinioEndpoint:   DefaultLocalEndpoint,
+			EndpointSecure:  false,
+			AccessKeyID:     DefaultAccessKeyID,
+			SecretAccessKey: DefaultSecretAccessKey,
+			Client:          nil,
 		}
+
+		op.LocalMinioOperator.BuildMinioOperator()
 	}

-	if op.PublicClient == nil {
-		// 初始化Minio客户端对象。
-		op.PublicClient, err = minio.New(DefaultPublicEndpoint, DefaultAccessKeyID, DefaultSecretAccessKey, false)
-		if err != nil {
-			log.ErrorF("[buildCmiiMinioOperator] - build for PublicClient error ! => %s", err.Error())
+	if op.PublicMinioOperator == nil {
+		op.PublicMinioOperator = &MinioOperator{
+			MinioEndpoint:   DefaultPublicEndpoint,
+			EndpointSecure:  false,
+			AccessKeyID:     DefaultAccessKeyID,
+			SecretAccessKey: DefaultSecretAccessKey,
+			Client:          nil,
 		}
+
+		op.PublicMinioOperator.BuildMinioOperator()
 	}

-	if op.DemoClient == nil {
-		// 初始化Minio客户端对象。
-		op.DemoClient, err = minio.New(DemoEndpoint, DefaultAccessKeyID, DefaultSecretAccessKey, true)
-		if err != nil {
-			log.ErrorF("[buildCmiiMinioOperator] - build for DemoClient error ! => %s", err.Error())
+	if op.DemoMinioOperator == nil {
+		op.DemoMinioOperator = &MinioOperator{
+			MinioEndpoint:   DefaultDemoEndpoint,
+			EndpointSecure:  true,
+			AccessKeyID:     DefaultAccessKeyID,
+			SecretAccessKey: DefaultSecretAccessKey,
+			Client:          nil,
 		}
+
+		op.DemoMinioOperator.BuildMinioOperator()
 	}

 	log.DebugF("[buildCmiiMinioOperator] - build client success !")
 }

 func (op *CmiiMinioOperator) UploadToLocalOctopus(filePath, fileName string) bool {
-	return op.uploadToOctopus(op.LocalClient, filePath, fileName)
+	return op.LocalMinioOperator.UploadFile(DefaultOctopusBucketName, filePath, fileName)
 }

 func (op *CmiiMinioOperator) UploadToPublicOctopus(filePath, fileName string) bool {

-	return op.uploadToOctopus(op.PublicClient, filePath, fileName)
+	return op.PublicMinioOperator.UploadFile(DefaultOctopusBucketName, filePath, fileName)
 }

 func (op *CmiiMinioOperator) UploadToDemo(bucketName, filePath, fileName string) bool {

-	return op.uploadToOss(op.DemoClient, bucketName, filePath, fileName)
+	return op.DemoMinioOperator.UploadFile(bucketName, filePath, fileName)
 }

-func (op *CmiiMinioOperator) uploadToOctopus(client *minio.Client, filePath, fileName string) bool {
+type MinioOperator struct {
+	MinioEndpoint  string
+	EndpointSecure bool

-	return op.uploadToOss(client, DefaultOctopusBucketName, filePath, fileName)
+	AccessKeyID     string
+	SecretAccessKey string
+
+	Client *minio.Client
 }

-func (op *CmiiMinioOperator) uploadToOss(client *minio.Client, bucketName, filePath, fileName string) bool {
+func (op *MinioOperator) BuildMinioOperator() {

-	if !strings.HasSuffix(filePath, "/") {
-		filePath += "/"
+	op.MinioEndpoint = strings.TrimPrefix(op.MinioEndpoint, "http://")
+	op.MinioEndpoint = strings.TrimPrefix(op.MinioEndpoint, "https://")
+	if strings.Contains(op.MinioEndpoint, "/") {
+		op.MinioEndpoint = strings.Split(op.MinioEndpoint, "/")[0]
 	}

-	realFileName := fileName
-	// 解析bucket上传的真实名称
-	if strings.Contains(bucketName, "/") {
-		splitN := strings.SplitN(bucketName, "/", 2)
-		bucketName = splitN[0]
-		if !strings.HasSuffix(splitN[1], "/") {
-			splitN[1] = splitN[1] + "/"
-		}
-		realFileName = splitN[1] + fileName
-	}
-
-	// 使用PutObject上传文件
-	// realFileName  ==> tmp/123/123.txt
-	n, err := client.FPutObject(bucketName, realFileName, filePath+fileName, minio.PutObjectOptions{})
+	client, err := minio.New(op.MinioEndpoint, op.AccessKeyID, op.SecretAccessKey, op.EndpointSecure)
 	if err != nil {
-		log.ErrorF("[uploadToOss] - upload %s error %s", filePath+fileName, err.Error())
+		log.ErrorF("Failed to create client: %v", err)
+	}
+	log.DebugF("endpoint: %s accessKeyId: %s secretAccessKey: %s", op.MinioEndpoint, op.AccessKeyID, op.SecretAccessKey)
+
+	op.Client = client
+}
+
+func (op *MinioOperator) ListFileInBucket(bucketName string) {
+	// 使用ListObjectsV2获取bucket中的所有文件名。
+	doneChan := make(chan struct{})
+
+	objectInfo := op.Client.ListObjectsV2(bucketName, "", false, doneChan)
+
+	for object := range objectInfo {
+		utils.BeautifulPrint(object)
+	}
+
+}
+
+func (op *MinioOperator) DownloadFileFromOssFullUrl(ossFullUrl string, filePath string) bool {
+	log.InfoF("[DownloadFileFromOssFullUrl] - download  from %s", ossFullUrl)
+
+	// https://oss.demo.uavcmlc.com:18000/cmlc-installation/tmp/docker.tar.gz
+	ossFullUrl = strings.TrimPrefix(ossFullUrl, "http://")
+	ossFullUrl = strings.TrimPrefix(ossFullUrl, "https://")
+	// oss.demo.uavcmlc.com:18000/cmlc-installation/tmp/docker.tar.gz
+
+	split := strings.Split(ossFullUrl, "/")
+	if len(split) < 2 {
+		log.ErrorF("DownloadFileFromOssFullUrl - split error len < 2 !")
+		return false
+	}
+
+	bucketName := split[0]
+	fileName := strings.Join(split[1:], "/")
+	//realFileName := split[len(split)-1]
+
+	return op.DownloadFile(bucketName, filePath, fileName)
+
+}
+
+// DownloadFile objectName格式为 cmlc-installation/tmp/123/123.txt
+func (op *MinioOperator) DownloadFile(bucketName, filePath, fileNameWithPrefix string) bool {
+
+	separator := os.PathSeparator
+	if !strings.HasSuffix(filePath, string(separator)) {
+		filePath += string(separator)
+	}
+
+	fileNameWithPrefix = strings.TrimPrefix(fileNameWithPrefix, "/")
+
+	// 获取下载的真实文件名称 123.txt
+	realFileName := fileNameWithPrefix
+	if strings.Contains(fileNameWithPrefix, "/") {
+		realFileName = strings.Split(fileNameWithPrefix, "/")[len(strings.Split(fileNameWithPrefix, "/"))-1]
+	}
+
+	log.InfoF("[DownloadFile] - download from [%s] to [%s]", op.MinioEndpoint+"/"+bucketName+"/"+fileNameWithPrefix, filePath+realFileName)
+	err := op.Client.FGetObject(bucketName, fileNameWithPrefix, filePath+realFileName, minio.GetObjectOptions{})
+	if err != nil {
+		log.ErrorF("[DownloadFile] - download %s error %s", filePath+realFileName, err.Error())
 		return false
 	}

-	log.InfoF("[uploadToOss] - uploaded %s of size %d", filePath+fileName, n)
+	return true
+}
+
+// UploadFile bucketNameWithSuffix格式为 cmlc-installation/tmp/123 的格式
+func (op *MinioOperator) UploadFile(bucketNameWithSuffix, filePath, fileName string) bool {
+
+	separator := os.PathSeparator
+	if !strings.HasSuffix(filePath, string(separator)) {
+		filePath += string(separator)
+	}
+
+	bucketNameWithSuffix = strings.TrimPrefix(bucketNameWithSuffix, "/")
+	oldBucketName := bucketNameWithSuffix
+	realFileName := fileName
+	// 解析bucket上传的真实名称
+	if strings.Contains(bucketNameWithSuffix, "/") {
+		bucketNameWithSuffix = strings.Split(bucketNameWithSuffix, "/")[0]
+		fileName = strings.TrimPrefix(oldBucketName, bucketNameWithSuffix) + "/" + realFileName
+	}
+
+	// 使用PutObject上传文件
+	// fileName  ==> tmp/123/123.txt
+	// realFileName ==> 123.txt
+	log.InfoF("[UploadFile] - upload from [%s] to [%s]", filePath+realFileName, op.MinioEndpoint+"/"+bucketNameWithSuffix+"/"+fileName)
+	n, err := op.Client.FPutObject(bucketNameWithSuffix, fileName, filePath+realFileName, minio.PutObjectOptions{})
+	if err != nil {
+		log.ErrorF("[UploadFile] - upload [%s] to [%s] error %s", filePath+realFileName, op.MinioEndpoint+"/"+bucketNameWithSuffix+"/"+fileName, err.Error())
+		return false
+	}
+
+	log.InfoF("[UploadFile] - uploaded %s of size %d", filePath+fileName, n)
 	return true
 }
--- a/agent-operator/CmiiMinioOperator_test.go
+++ b/agent-operator/CmiiMinioOperator_test.go
@@ -7,3 +7,13 @@ func TestCmiiMinioOperator_UploadToDemo(t *testing.T) {
 	DefaultCmiiMinioOperator.UploadToDemo("cmlc-installation", "/home/wdd/Downloads/", "go1.22.1.linux-amd64.tar.gz")

 }
+
+func TestMinioOperator_DownloadFile(t *testing.T) {
+	DefaultCmiiMinioOperator.DemoMinioOperator.DownloadFile("cmlc-installation", "C:\\Users\\wddsh\\Documents\\Visual Studio 2022", "tmp/cmii-srs-oss-adaptor=2023-SA=2024-04-11=847.tar.gz")
+}
+
+func TestMinioOperator_UploadFile(t *testing.T) {
+
+	DefaultCmiiMinioOperator.DemoMinioOperator.UploadFile("cmlc-installation/tmp", "C:\\Users\\wddsh\\Documents\\Visual Studio 2022", "cmii-srs-oss-adaptor=2023-SA=2024-04-11=847.tar.gz")
+
+}
--- a/agent-operator/CmiiOperator.go
+++ b/agent-operator/CmiiOperator.go
@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"slices"
 	"strings"
+
 	image2 "wdd.io/agent-common/image"
 	"wdd.io/agent-common/utils"
 	"wdd.io/agent-operator/image"
@@ -179,7 +180,7 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
 		// extract demo oss location suffix from gzipFolderFullPath
 		trimPrefix := strings.TrimPrefix(gzipFolderFullPath, OfflineImageGzipFolderPrefix)
 		bucketName := "cmlc-installation/" + trimPrefix
-		log.InfoF("gzip file location in demo oss is %s", DemoEndpoint+"/"+bucketName)
+		log.InfoF("gzip file location in demo oss is %s", DefaultDemoEndpoint+"/"+bucketName)

 		minioOperator := CmiiMinioOperator{}
 		for _, gzipFileName := range allGzipFileNameList {
@@ -195,16 +196,16 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
 }

 // DownloadLoadTagPush DLTU procedure ImageSync的另外一般流程，需要支持 堡垒机（纯离线）的模式
-//  1. 文件模式，从文件中读取全部的镜像全名称-写死到文件中
-//  2. Gzip文件目录，RKE MIDDLE CMII三个文件目录 - 是否需要约定目录
+//  2. Gzip文件目录，RKE MIDDLE CMII三个文件目录 - 约定目录
 //     约定目录 /root/wdd/image/rke/  /root/wdd/image/middle/  /root/wdd/image/cmii/
-//  3. 读取本机的IP地址 - 或者进行参数传递
-//  4. OSS地址 - 参数传递gizpFileName
-func DownloadLoadTagPush(downloadFromOss bool, ossUrlPrefix, localGzipFolder string, targetHarborFullName string) []string {
+//  3. 读取本机的IP地址 - 参数传递
+//  4. OSS地址 - ossUrlPrefix传空  则使用默认值
+//  5. ossFileName - 如果结尾为txt，则为文件的形式，如果为tar.gz，则为gzip文件夹的形式
+func DownloadLoadTagPush(downloadFromOss bool, ossUrlPrefix, ossFileName, localGzipFolder string, targetHarborFullName string) []string {

 	// download
 	if downloadFromOss {
-		parseAndDownloadFromOssUrlPrefix(ossUrlPrefix, localGzipFolder)
+		parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder)
 	}
 	// load
 	loadAllGzipImageFromLocalFolder(localGzipFolder)
@@ -246,8 +247,28 @@ func loadAllGzipImageFromLocalFolder(localGzipFolder string) {

 }

-// todo
-func parseAndDownloadFromOssUrlPrefix(ossUrlPrefix string, localGzipFolder string) {
+func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string) {
+
+	if ossUrlPrefix == "" {
+		ossUrlPrefix = DefaultOssUrlPrefix
+	}
+	log.InfoF("prepare to download from %s%s", ossUrlPrefix, ossFileName)
+
+	// get oss endpoint
+	// mc login
+
+	if strings.HasSuffix(ossFileName, ".txt") {
+		// a list of files
+
+	}
+
+	if strings.HasSuffix(ossFileName, ".tar.gz") {
+		// single gzip file
+	}
+
+	// mv list all gzip file name
+
+	// download all gzip files to local folder

 	// 解析
 }
--- a/agent-operator/image/CmiiImageSync.go
+++ b/agent-operator/image/CmiiImageSync.go
@@ -6,15 +6,16 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	"github.com/docker/docker/client"
-	"github.com/klauspost/pgzip"
 	"io"
 	"io/fs"
 	"os"
 	"strconv"
 	"strings"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
+	"github.com/klauspost/pgzip"
 	image2 "wdd.io/agent-common/image"
 	"wdd.io/agent-common/logger"
 	"wdd.io/agent-common/utils"
--- a/agent-operator/main.go
+++ b/agent-operator/main.go
@@ -77,11 +77,13 @@ func CmiiRunner() {

 var DLTUHelp = `
 DLTUHelp
-  dltu [ossUrl] [localGzipFolder] [harborHostFullName]
+  dltu [ossUrlPrefix] [ossFileName] [localGzipFolder] [harborHostFullName]
 `

 func main() {

+	// C:\Users\wddsh\go\bin\gox.exe -osarch="linux/amd66" -output "build/operator_{{.OS}}_{{.Arch}}"
+
 	//RealProjectRunner()

 	// 解析命令行参数
@@ -111,22 +113,29 @@ func main() {
 					result = append(result, text)
 				}

-				if len(result) != 4 {
+				if len(result) != 5 {
 					fmt.Println("input error!")
 					fmt.Printf(DLTUHelp)
 					return
 				}

-				ossUrl := result[1]
-				localGzipFolder := result[2]
-				harborHostFullName := result[3]
+				ossUrlPrefix := result[1]
+				ossFileName := result[2]
+				localGzipFolder := result[3]
+				harborHostFullName := result[4]

-				fmt.Println("ossUrl: ", ossUrl)
+				fmt.Println("ossUrlPrefix: ", ossUrlPrefix)
+				fmt.Println("ossFileName: ", ossFileName)
 				fmt.Println("localGzipFolder: ", localGzipFolder)
 				fmt.Println("harborHostFullName: ", harborHostFullName)
 				fmt.Println()

-				DownloadLoadTagPush(false, ossUrl, localGzipFolder, harborHostFullName)
+				var downloadFromOss bool
+				if ossFileName != "" {
+					downloadFromOss = true
+				}
+
+				DownloadLoadTagPush(downloadFromOss, ossUrlPrefix, ossFileName, localGzipFolder, harborHostFullName)

 				// 下载

--- a/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java
+++ b/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java
@@ -38,13 +38,13 @@ public class TestImageSyncScheduler {
    public void runImageSync() {

        ArrayList<String> CmiiAppNameList = new ArrayList<>(List.of(
-                "cmii-uav-process:5.4.0-041901"
+//                "cmii-uav-process:5.4.0-041901"
        ));

        ArrayList<String> ImageFullNameList = new ArrayList<>(List.of(
 //                "harbor.cdcyy.com.cn/cmii/cmii-live-operator:5.2.0",
 //                "harbor.cdcyy.com.cn/cmii/cmii/srs:v5.0.195"
-//                "harbor.cdcyy.com.cn/cmii/cmii-uav-industrial-portfolio:5.54.0-cqly-041601"
+                "harbor.cdcyy.com.cn/cmii/cmii-uav-platform:5.3.0-cqly-042202"
        ));

        Boolean downloadAndCompressOnly = false;