[agent-operator] - add tag from gzip folder

agent-common/image/ImageNameConvert.go

@@ -148,3 +148,27 @@ func GzipFileNameToImageFullName(gzipFileName string) (imageFullName string) {
 
 	return gzipFileName
 }
+
+func GzipFileNameToImageNameAndTag(gzipFileName string) (imageName, imageTag string) {
+	if !strings.HasSuffix(gzipFileName, ".tar.gz") {
+		log.ErrorF(" %s is not end with .tar.gz", gzipFileName)
+		return "", ""
+	}
+	gzipFileName = strings.TrimSuffix(gzipFileName, ".tar.gz")
+
+	if strings.HasPrefix(gzipFileName, "docker=library") {
+		// docker=library=busybox=latest.tar.gz
+		return strings.Split(gzipFileName, "=")[2], strings.Split(gzipFileName, "=")[3]
+	}
+
+	if strings.HasPrefix(gzipFileName, "docker") {
+		// docker=kubernetes=kubernetes-dashboard=v2.4.0.tar.gz
+		return strings.Split(gzipFileName, "=")[1] + "/" + strings.Split(gzipFileName, "=")[2], strings.Split(gzipFileName, "=")[3]
+	}
+
+	if strings.HasPrefix(gzipFileName, "cmlc=cmii=") {
+		return strings.Split(gzipFileName, "=")[2], strings.Split(gzipFileName, "=")[3]
+	}
+
+	return "", ""
+}

agent-common/utils/FileUtils.go

@@ -150,12 +150,21 @@ func FileExistAndNotNull(filename string) bool {
 	return size > 0
 }
 
+// ListAllFileInFolder 列出一个目录中的所有文件，返回文件名，忽略folder，不带全路径
 func ListAllFileInFolder(folderName string) ([]string, error) {
+	return listAllFileInFolderWithFullPath(folderName, false)
+}
+
+func listAllFileInFolderWithFullPath(folderName string, fullPath bool) ([]string, error) {
 	files := make([]string, 0)
 	err := filepath.Walk(folderName, func(path string, info os.FileInfo, err error) error {
 		if !info.IsDir() {
+			if fullPath {
+				files = append(files, path)
+			} else {
 				files = append(files, info.Name())
 			}
+		}
 		return nil
 	})
 	if err != nil {

agent-operator/CmiiOperator.go

@@ -15,7 +15,6 @@ import (
 	"wdd.io/agent-operator/image"
 )
 
-const OfflineImageGzipFolderPrefix = "/root/octopus_image/"
 const OfflineDeployHarborHost = "harbor.wdd.io"
 const PublicDeployHarborHost = "42.192.52.227"
 const DirectPushDeployHarborHost = "36.134.71.138"
@@ -63,19 +62,19 @@ func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResul
 			// get version images
 			errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = DownloadCompressUploadFromVersion(sync.ProjectVersion, sync.CompressImageToGzip, sync.UploadToDemoMinio)
 
-			gzipFolderFullPath = OfflineImageGzipFolderPrefix + sync.ProjectVersion
+			gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectVersion
 
 		} else {
 			// get demo images
 
 			errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = DownloadCompressUploadFromDemo(sync.ProjectName, sync.CompressImageToGzip, sync.UploadToDemoMinio)
-			gzipFolderFullPath = OfflineImageGzipFolderPrefix + sync.ProjectName
+			gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectName
 
 		}
 	} else {
 		// 拉取特定的镜像
 
-		gzipFolderFullPath = OfflineImageGzipFolderPrefix + "tmp"
+		gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + "tmp"
 		// 组装镜像名称
 		allCmiiImageNameList = concatAndUniformCmiiImage(sync.FullNameImageList, sync.CmiiNameTagList)
 
@@ -178,7 +177,7 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
 
 		// start to upload
 		// extract demo oss location suffix from gzipFolderFullPath
-		trimPrefix := strings.TrimPrefix(gzipFolderFullPath, OfflineImageGzipFolderPrefix)
+		trimPrefix := strings.TrimPrefix(gzipFolderFullPath, image.OfflineImageGzipFolderPrefix)
 		bucketName := "cmlc-installation/" + trimPrefix
 		log.InfoF("gzip file location in demo oss is %s", DefaultDemoEndpoint+"/"+bucketName)
 
@@ -290,10 +289,10 @@ func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string)
 func DownloadCompressUploadFromDemo(projectName string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
 
 	// generate a project folder
-	err := os.MkdirAll(OfflineImageGzipFolderPrefix+projectName, os.ModeDir)
+	err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+projectName, os.ModeDir)
 	if err != nil {
 		if !errors.Is(err, os.ErrExist) {
-			log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", OfflineImageGzipFolderPrefix+projectName, err.Error())
+			log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", image.OfflineImageGzipFolderPrefix+projectName, err.Error())
 			return errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList
 		}
 	}
@@ -303,7 +302,7 @@ func DownloadCompressUploadFromDemo(projectName string, shouldGzip bool, shouldO
 
 	// do work
 	// DCU
-	return DownloadCompressUpload(allCmiiImageNameListFromDemo, shouldGzip, OfflineImageGzipFolderPrefix+projectName, shouldOss)
+	return DownloadCompressUpload(allCmiiImageNameListFromDemo, shouldGzip, image.OfflineImageGzipFolderPrefix+projectName, shouldOss)
 }
 
 func buildAllCmiiImageNameListFromDemo(projectName string) []string {
@@ -313,9 +312,9 @@ func buildAllCmiiImageNameListFromDemo(projectName string) []string {
 	backendMap, frontendMap, srsMap := BackupAllCmiiDeploymentToMap(demo)
 
 	// save map to file
-	backendMapFile := OfflineImageGzipFolderPrefix + projectName + "-backend-app.json"
+	backendMapFile := image.OfflineImageGzipFolderPrefix + projectName + "-backend-app.json"
-	frontendMapFile := OfflineImageGzipFolderPrefix + projectName + "-frontend-app.json"
+	frontendMapFile := image.OfflineImageGzipFolderPrefix + projectName + "-frontend-app.json"
-	srsMapFile := OfflineImageGzipFolderPrefix + projectName + "-srs-app.json"
+	srsMapFile := image.OfflineImageGzipFolderPrefix + projectName + "-srs-app.json"
 	_ = os.Remove(backendMapFile)
 	_ = os.Remove(frontendMapFile)
 	_ = os.Remove(srsMapFile)
@@ -333,9 +332,9 @@ func buildAllCmiiImageNameListFromDemo(projectName string) []string {
 	//	srsMapFile,
 	//)
 
-	realCmiiImageName = append(realCmiiImageName, image.ConvertCMiiImageMapToList(backendMap)...)
+	realCmiiImageName = append(realCmiiImageName, image.ConvertCmiiImageMapToList(backendMap)...)
-	realCmiiImageName = append(realCmiiImageName, image.ConvertCMiiImageMapToList(frontendMap)...)
+	realCmiiImageName = append(realCmiiImageName, image.ConvertCmiiImageMapToList(frontendMap)...)
-	realCmiiImageName = append(realCmiiImageName, image.ConvertCMiiImageMapToList(srsMap)...)
+	realCmiiImageName = append(realCmiiImageName, image.ConvertCmiiImageMapToList(srsMap)...)
 
 	utils.BeautifulPrintListWithTitle(realCmiiImageName, "Cmii Project Image => "+projectName)
 
@@ -346,10 +345,10 @@ func buildAllCmiiImageNameListFromDemo(projectName string) []string {
 func DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
 
 	// generate a project folder
-	err := os.MkdirAll(OfflineImageGzipFolderPrefix+cmiiVersion, os.ModeDir)
+	err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+cmiiVersion, os.ModeDir)
 	if err != nil {
 		if !errors.Is(err, os.ErrExist) {
-			log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", OfflineImageGzipFolderPrefix+cmiiVersion, err.Error())
+			log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", image.OfflineImageGzipFolderPrefix+cmiiVersion, err.Error())
 			return errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList
 		}
 	}
@@ -359,7 +358,7 @@ func DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shou
 
 	// do work
 	// DCU procedure
-	return DownloadCompressUpload(realCmiiImageName, shouldGzip, OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss)
+	return DownloadCompressUpload(realCmiiImageName, shouldGzip, image.OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss)
 
 }
 
@@ -377,8 +376,8 @@ func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string {
 		frontendMap[app] = cmiiVersion
 	}
 
-	realCmiiImageName = append(realCmiiImageName, image.ConvertCMiiImageMapToList(backendMap)...)
+	realCmiiImageName = append(realCmiiImageName, image.ConvertCmiiImageMapToList(backendMap)...)
-	realCmiiImageName = append(realCmiiImageName, image.ConvertCMiiImageMapToList(frontendMap)...)
+	realCmiiImageName = append(realCmiiImageName, image.ConvertCmiiImageMapToList(frontendMap)...)
 
 	for key, value := range CmiiSrsAppMap {
 		var app *CmiiDeploymentInterface
@@ -401,16 +400,16 @@ func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string {
 }
 
 func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, downloadMiddle bool, downloadRke bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
-	err := os.MkdirAll(OfflineImageGzipFolderPrefix, os.ModeDir)
+	err := os.MkdirAll(image.OfflineImageGzipFolderPrefix, os.ModeDir)
 	if err != nil {
 		if !errors.Is(err, os.ErrExist) {
-			log.ErrorF("[FetchDependencyRepos] - create folder of %s error %s", OfflineImageGzipFolderPrefix, err.Error())
+			log.ErrorF("[FetchDependencyRepos] - create folder of %s error %s", image.OfflineImageGzipFolderPrefix, err.Error())
 		}
 	}
 
 	if downloadMiddle {
 
-		gzipFolderPrefix := OfflineImageGzipFolderPrefix + "middle/"
+		gzipFolderPrefix := image.OfflineImageGzipFolderPrefix + "middle/"
 
 		// remove folder first
 		utils.RemoveFolderComplete(gzipFolderPrefix)
@@ -419,7 +418,7 @@ func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, downloadM
 	}
 
 	if downloadRke {
-		gzipFolderPrefix := OfflineImageGzipFolderPrefix + "rke/"
+		gzipFolderPrefix := image.OfflineImageGzipFolderPrefix + "rke/"
 		return DownloadCompressUpload(image.MiddlewareAmd64, shouldGzip, gzipFolderPrefix, shouldOss)
 	}
 
@@ -429,7 +428,7 @@ func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, downloadM
 func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) {
 
 	// list folder
-	projectGzipFolder := OfflineImageGzipFolderPrefix + projectName
+	projectGzipFolder := image.OfflineImageGzipFolderPrefix + projectName
 	errorLoadImageNameList = append(errorLoadImageNameList, image.LoadFromFolderPath(projectGzipFolder)...)
 	// read from json
 	errorPushImageNameList = append(errorPushImageNameList, image.TagFromListAndPushToCHarbor(image.Cmii520DemoImageList, targetHarborHost)...)
@@ -445,8 +444,8 @@ func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string)
 
 func LoadSplitDepGzipImageToTargetHarbor(targetHarborHost string) (errorLoadImageNameList []string, errorPushImageNameList []string) {
 
-	//middle := OfflineImageGzipFolderPrefix + "middle/"
+	//middle := image.OfflineImageGzipFolderPrefix + "middle/"
-	//rke := OfflineImageGzipFolderPrefix + "rke/"
+	//rke := image.OfflineImageGzipFolderPrefix + "rke/"
 	//errorLoadImageNameList = append(errorLoadImageNameList, ImageLoadFromFolderPath(middle)...)
 	//errorLoadImageNameList = append(errorLoadImageNameList, ImageLoadFromFolderPath(rke)...)
 

agent-operator/deploy/c_app/DeployCmiiApp.go

@@ -102,9 +102,18 @@ func (frontend *CmiiFrontendConfig) FrontendDeploy(common *z_dep.CommonEnvironme
 	return true
 }
 
-func SrsDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool {
+func (frontend *CmiiFrontendConfig) ConfigMapDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool {
 
-	return commonEnv.ParseCommonEnvToApplyFile(CmiiSrsTemplate, SRSApplyFilePath)
+	// copy
+	z_dep.CopySameFields(commonEnv, frontend)
+
+	// manual validate
+	if frontend.ShortName == "" || frontend.ClientId == "" {
+		log.ErrorF("short name or client id is empty !")
+		return false
+	}
+
+	return z_dep.ParseEnvToApplyFile(frontend, CmiiFrontendConfigMapTemplate, ConfigMapApplyFilePath)
 }
 
 func IngressDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool {
@@ -123,16 +132,7 @@ func IngressDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool {
 	return true
 }
 
-func (frontend *CmiiFrontendConfig) ConfigMapDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool {
+func SrsDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool {
 
-	// copy
+	return commonEnv.ParseCommonEnvToApplyFile(CmiiSrsTemplate, SRSApplyFilePath)
-	z_dep.CopySameFields(commonEnv, frontend)
-
-	// manual validate
-	if frontend.ShortName == "" || frontend.ClientId == "" {
-		log.ErrorF("short name or client id is empty !")
-		return false
-	}
-
-	return z_dep.ParseEnvToApplyFile(frontend, CmiiFrontendConfigMapTemplate, ConfigMapApplyFilePath)
 }

agent-operator/go.mod

@@ -14,7 +14,6 @@ require (
 	k8s.io/api v0.29.1
 	k8s.io/apimachinery v0.29.1
 	k8s.io/client-go v0.29.1
-	sigs.k8s.io/yaml v1.4.0
 	wdd.io/agent-common v0.0.0
 )
 
@@ -96,4 +95,5 @@ require (
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
 )

agent-operator/image/CmiiImageSync.go

@@ -24,6 +24,8 @@ import (
 var apiClient = newClient()
 var log = logger.Log
 
+const OfflineImageGzipFolderPrefix = "/root/octopus_image/"
+
 func newClient() *client.Client {
 	apiClient, err := client.NewClientWithOpts(client.FromEnv)
 
@@ -276,7 +278,7 @@ func PullFromCmiiHarbor(imageName string) (pullResult io.ReadCloser) {
 
 func PullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode bool) (fullImageNameList, errorPullImageList []string) {
 
-	fullImageNameList = ConvertCMiiImageMapToList(imageVersionMap)
+	fullImageNameList = ConvertCmiiImageMapToList(imageVersionMap)
 	return fullImageNameList, PullFromFullNameList(fullImageNameList)
 
 }
@@ -472,7 +474,7 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool {
 	return true
 }
 
-func ConvertCMiiImageMapToList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) {
+func ConvertCmiiImageMapToList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) {
 
 	for image, tag := range cmiiImageVersionMap {
 		s := image2.CmiiHarborPrefix + image + ":" + tag
@@ -482,6 +484,23 @@ func ConvertCMiiImageMapToList(cmiiImageVersionMap map[string]string) (fullImage
 	return fullImageNameList
 }
 
+func ConvertCmiiImageMapFromGzipFolder(gzipFileFolder string) (cmiiImageVersionMap map[string]string) {
+
+	allFileInFolder, err := utils.ListAllFileInFolder(gzipFileFolder)
+	if err != nil {
+		return nil
+	}
+
+	cmiiImageVersionMap = make(map[string]string, len(allFileInFolder))
+	for _, gzipFileName := range allFileInFolder {
+		log.DebugF("gzip file name is %s", gzipFileName)
+		imageName, imageTag := image2.GzipFileNameToImageNameAndTag(gzipFileName)
+		cmiiImageVersionMap[imageName] = imageTag
+	}
+
+	return cmiiImageVersionMap
+}
+
 func loginToDockerHub(HarborFullHost string) {
 
 	if HarborFullHost == "" {

agent-operator/image/CmiiImageSync_test.go

@@ -195,3 +195,10 @@ func TestSaveSpecificImageToGzipFile(t *testing.T) {
 	}
 
 }
+
+func TestConvertCmiiImageMapFromGzipFolder(t *testing.T) {
+
+	versionMap := ConvertCmiiImageMapFromGzipFolder(OfflineImageGzipFolderPrefix)
+
+	utils.BeautifulPrint(versionMap)
+}