diff --git a/agent-common/image/ImageNameConvert.go b/agent-common/image/ImageNameConvert.go index ca4eedc..b517fa5 100644 --- a/agent-common/image/ImageNameConvert.go +++ b/agent-common/image/ImageNameConvert.go @@ -148,3 +148,27 @@ func GzipFileNameToImageFullName(gzipFileName string) (imageFullName string) { return gzipFileName } + +func GzipFileNameToImageNameAndTag(gzipFileName string) (imageName, imageTag string) { + if !strings.HasSuffix(gzipFileName, ".tar.gz") { + log.ErrorF(" %s is not end with .tar.gz", gzipFileName) + return "", "" + } + gzipFileName = strings.TrimSuffix(gzipFileName, ".tar.gz") + + if strings.HasPrefix(gzipFileName, "docker=library") { + // docker=library=busybox=latest.tar.gz + return strings.Split(gzipFileName, "=")[2], strings.Split(gzipFileName, "=")[3] + } + + if strings.HasPrefix(gzipFileName, "docker") { + // docker=kubernetes=kubernetes-dashboard=v2.4.0.tar.gz + return strings.Split(gzipFileName, "=")[1] + "/" + strings.Split(gzipFileName, "=")[2], strings.Split(gzipFileName, "=")[3] + } + + if strings.HasPrefix(gzipFileName, "cmlc=cmii=") { + return strings.Split(gzipFileName, "=")[2], strings.Split(gzipFileName, "=")[3] + } + + return "", "" +} diff --git a/agent-common/utils/FileUtils.go b/agent-common/utils/FileUtils.go index b9b5739..9a3c6d3 100644 --- a/agent-common/utils/FileUtils.go +++ b/agent-common/utils/FileUtils.go @@ -150,11 +150,20 @@ func FileExistAndNotNull(filename string) bool { return size > 0 } +// ListAllFileInFolder 列出一个目录中的所有文件，返回文件名，忽略folder，不带全路径 func ListAllFileInFolder(folderName string) ([]string, error) { + return listAllFileInFolderWithFullPath(folderName, false) +} + +func listAllFileInFolderWithFullPath(folderName string, fullPath bool) ([]string, error) { files := make([]string, 0) err := filepath.Walk(folderName, func(path string, info os.FileInfo, err error) error { if !info.IsDir() { - files = append(files, info.Name()) + if fullPath { + files = append(files, path) + } else { + files = append(files, info.Name()) + } } return nil }) diff --git a/agent-operator/CmiiOperator.go b/agent-operator/CmiiOperator.go index beec49a..64121d6 100644 --- a/agent-operator/CmiiOperator.go +++ b/agent-operator/CmiiOperator.go @@ -15,7 +15,6 @@ import ( "wdd.io/agent-operator/image" ) -const OfflineImageGzipFolderPrefix = "/root/octopus_image/" const OfflineDeployHarborHost = "harbor.wdd.io" const PublicDeployHarborHost = "42.192.52.227" const DirectPushDeployHarborHost = "36.134.71.138" @@ -63,19 +62,19 @@ func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResul // get version images errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = DownloadCompressUploadFromVersion(sync.ProjectVersion, sync.CompressImageToGzip, sync.UploadToDemoMinio) - gzipFolderFullPath = OfflineImageGzipFolderPrefix + sync.ProjectVersion + gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectVersion } else { // get demo images errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = DownloadCompressUploadFromDemo(sync.ProjectName, sync.CompressImageToGzip, sync.UploadToDemoMinio) - gzipFolderFullPath = OfflineImageGzipFolderPrefix + sync.ProjectName + gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectName } } else { // 拉取特定的镜像 - gzipFolderFullPath = OfflineImageGzipFolderPrefix + "tmp" + gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + "tmp" // 组装镜像名称 allCmiiImageNameList = concatAndUniformCmiiImage(sync.FullNameImageList, sync.CmiiNameTagList) @@ -178,7 +177,7 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu // start to upload // extract demo oss location suffix from gzipFolderFullPath - trimPrefix := strings.TrimPrefix(gzipFolderFullPath, OfflineImageGzipFolderPrefix) + trimPrefix := strings.TrimPrefix(gzipFolderFullPath, image.OfflineImageGzipFolderPrefix) bucketName := "cmlc-installation/" + trimPrefix log.InfoF("gzip file location in demo oss is %s", DefaultDemoEndpoint+"/"+bucketName) @@ -290,10 +289,10 @@ func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string) func DownloadCompressUploadFromDemo(projectName string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { // generate a project folder - err := os.MkdirAll(OfflineImageGzipFolderPrefix+projectName, os.ModeDir) + err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+projectName, os.ModeDir) if err != nil { if !errors.Is(err, os.ErrExist) { - log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", OfflineImageGzipFolderPrefix+projectName, err.Error()) + log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", image.OfflineImageGzipFolderPrefix+projectName, err.Error()) return errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList } } @@ -303,7 +302,7 @@ func DownloadCompressUploadFromDemo(projectName string, shouldGzip bool, shouldO // do work // DCU - return DownloadCompressUpload(allCmiiImageNameListFromDemo, shouldGzip, OfflineImageGzipFolderPrefix+projectName, shouldOss) + return DownloadCompressUpload(allCmiiImageNameListFromDemo, shouldGzip, image.OfflineImageGzipFolderPrefix+projectName, shouldOss) } func buildAllCmiiImageNameListFromDemo(projectName string) []string { @@ -313,9 +312,9 @@ func buildAllCmiiImageNameListFromDemo(projectName string) []string { backendMap, frontendMap, srsMap := BackupAllCmiiDeploymentToMap(demo) // save map to file - backendMapFile := OfflineImageGzipFolderPrefix + projectName + "-backend-app.json" - frontendMapFile := OfflineImageGzipFolderPrefix + projectName + "-frontend-app.json" - srsMapFile := OfflineImageGzipFolderPrefix + projectName + "-srs-app.json" + backendMapFile := image.OfflineImageGzipFolderPrefix + projectName + "-backend-app.json" + frontendMapFile := image.OfflineImageGzipFolderPrefix + projectName + "-frontend-app.json" + srsMapFile := image.OfflineImageGzipFolderPrefix + projectName + "-srs-app.json" _ = os.Remove(backendMapFile) _ = os.Remove(frontendMapFile) _ = os.Remove(srsMapFile) @@ -333,9 +332,9 @@ func buildAllCmiiImageNameListFromDemo(projectName string) []string { // srsMapFile, //) - realCmiiImageName = append(realCmiiImageName, image.ConvertCMiiImageMapToList(backendMap)...) - realCmiiImageName = append(realCmiiImageName, image.ConvertCMiiImageMapToList(frontendMap)...) - realCmiiImageName = append(realCmiiImageName, image.ConvertCMiiImageMapToList(srsMap)...) + realCmiiImageName = append(realCmiiImageName, image.ConvertCmiiImageMapToList(backendMap)...) + realCmiiImageName = append(realCmiiImageName, image.ConvertCmiiImageMapToList(frontendMap)...) + realCmiiImageName = append(realCmiiImageName, image.ConvertCmiiImageMapToList(srsMap)...) utils.BeautifulPrintListWithTitle(realCmiiImageName, "Cmii Project Image => "+projectName) @@ -346,10 +345,10 @@ func buildAllCmiiImageNameListFromDemo(projectName string) []string { func DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { // generate a project folder - err := os.MkdirAll(OfflineImageGzipFolderPrefix+cmiiVersion, os.ModeDir) + err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+cmiiVersion, os.ModeDir) if err != nil { if !errors.Is(err, os.ErrExist) { - log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", OfflineImageGzipFolderPrefix+cmiiVersion, err.Error()) + log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", image.OfflineImageGzipFolderPrefix+cmiiVersion, err.Error()) return errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList } } @@ -359,7 +358,7 @@ func DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shou // do work // DCU procedure - return DownloadCompressUpload(realCmiiImageName, shouldGzip, OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss) + return DownloadCompressUpload(realCmiiImageName, shouldGzip, image.OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss) } @@ -377,8 +376,8 @@ func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string { frontendMap[app] = cmiiVersion } - realCmiiImageName = append(realCmiiImageName, image.ConvertCMiiImageMapToList(backendMap)...) - realCmiiImageName = append(realCmiiImageName, image.ConvertCMiiImageMapToList(frontendMap)...) + realCmiiImageName = append(realCmiiImageName, image.ConvertCmiiImageMapToList(backendMap)...) + realCmiiImageName = append(realCmiiImageName, image.ConvertCmiiImageMapToList(frontendMap)...) for key, value := range CmiiSrsAppMap { var app *CmiiDeploymentInterface @@ -401,16 +400,16 @@ func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string { } func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, downloadMiddle bool, downloadRke bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { - err := os.MkdirAll(OfflineImageGzipFolderPrefix, os.ModeDir) + err := os.MkdirAll(image.OfflineImageGzipFolderPrefix, os.ModeDir) if err != nil { if !errors.Is(err, os.ErrExist) { - log.ErrorF("[FetchDependencyRepos] - create folder of %s error %s", OfflineImageGzipFolderPrefix, err.Error()) + log.ErrorF("[FetchDependencyRepos] - create folder of %s error %s", image.OfflineImageGzipFolderPrefix, err.Error()) } } if downloadMiddle { - gzipFolderPrefix := OfflineImageGzipFolderPrefix + "middle/" + gzipFolderPrefix := image.OfflineImageGzipFolderPrefix + "middle/" // remove folder first utils.RemoveFolderComplete(gzipFolderPrefix) @@ -419,7 +418,7 @@ func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, downloadM } if downloadRke { - gzipFolderPrefix := OfflineImageGzipFolderPrefix + "rke/" + gzipFolderPrefix := image.OfflineImageGzipFolderPrefix + "rke/" return DownloadCompressUpload(image.MiddlewareAmd64, shouldGzip, gzipFolderPrefix, shouldOss) } @@ -429,7 +428,7 @@ func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, downloadM func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) { // list folder - projectGzipFolder := OfflineImageGzipFolderPrefix + projectName + projectGzipFolder := image.OfflineImageGzipFolderPrefix + projectName errorLoadImageNameList = append(errorLoadImageNameList, image.LoadFromFolderPath(projectGzipFolder)...) // read from json errorPushImageNameList = append(errorPushImageNameList, image.TagFromListAndPushToCHarbor(image.Cmii520DemoImageList, targetHarborHost)...) @@ -445,8 +444,8 @@ func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) func LoadSplitDepGzipImageToTargetHarbor(targetHarborHost string) (errorLoadImageNameList []string, errorPushImageNameList []string) { - //middle := OfflineImageGzipFolderPrefix + "middle/" - //rke := OfflineImageGzipFolderPrefix + "rke/" + //middle := image.OfflineImageGzipFolderPrefix + "middle/" + //rke := image.OfflineImageGzipFolderPrefix + "rke/" //errorLoadImageNameList = append(errorLoadImageNameList, ImageLoadFromFolderPath(middle)...) //errorLoadImageNameList = append(errorLoadImageNameList, ImageLoadFromFolderPath(rke)...) diff --git a/agent-operator/deploy/c_app/DeployCmiiApp.go b/agent-operator/deploy/c_app/DeployCmiiApp.go index 12959f4..5bd0108 100644 --- a/agent-operator/deploy/c_app/DeployCmiiApp.go +++ b/agent-operator/deploy/c_app/DeployCmiiApp.go @@ -102,9 +102,18 @@ func (frontend *CmiiFrontendConfig) FrontendDeploy(common *z_dep.CommonEnvironme return true } -func SrsDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool { +func (frontend *CmiiFrontendConfig) ConfigMapDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool { - return commonEnv.ParseCommonEnvToApplyFile(CmiiSrsTemplate, SRSApplyFilePath) + // copy + z_dep.CopySameFields(commonEnv, frontend) + + // manual validate + if frontend.ShortName == "" || frontend.ClientId == "" { + log.ErrorF("short name or client id is empty !") + return false + } + + return z_dep.ParseEnvToApplyFile(frontend, CmiiFrontendConfigMapTemplate, ConfigMapApplyFilePath) } func IngressDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool { @@ -123,16 +132,7 @@ func IngressDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool { return true } -func (frontend *CmiiFrontendConfig) ConfigMapDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool { +func SrsDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool { - // copy - z_dep.CopySameFields(commonEnv, frontend) - - // manual validate - if frontend.ShortName == "" || frontend.ClientId == "" { - log.ErrorF("short name or client id is empty !") - return false - } - - return z_dep.ParseEnvToApplyFile(frontend, CmiiFrontendConfigMapTemplate, ConfigMapApplyFilePath) + return commonEnv.ParseCommonEnvToApplyFile(CmiiSrsTemplate, SRSApplyFilePath) } diff --git a/agent-operator/go.mod b/agent-operator/go.mod index df4fc01..e0968ab 100644 --- a/agent-operator/go.mod +++ b/agent-operator/go.mod @@ -14,7 +14,6 @@ require ( k8s.io/api v0.29.1 k8s.io/apimachinery v0.29.1 k8s.io/client-go v0.29.1 - sigs.k8s.io/yaml v1.4.0 wdd.io/agent-common v0.0.0 ) @@ -96,4 +95,5 @@ require ( k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/agent-operator/image/CmiiImageSync.go b/agent-operator/image/CmiiImageSync.go index 60db9a9..5fba63b 100644 --- a/agent-operator/image/CmiiImageSync.go +++ b/agent-operator/image/CmiiImageSync.go @@ -24,6 +24,8 @@ import ( var apiClient = newClient() var log = logger.Log +const OfflineImageGzipFolderPrefix = "/root/octopus_image/" + func newClient() *client.Client { apiClient, err := client.NewClientWithOpts(client.FromEnv) @@ -276,7 +278,7 @@ func PullFromCmiiHarbor(imageName string) (pullResult io.ReadCloser) { func PullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode bool) (fullImageNameList, errorPullImageList []string) { - fullImageNameList = ConvertCMiiImageMapToList(imageVersionMap) + fullImageNameList = ConvertCmiiImageMapToList(imageVersionMap) return fullImageNameList, PullFromFullNameList(fullImageNameList) } @@ -472,7 +474,7 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool { return true } -func ConvertCMiiImageMapToList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) { +func ConvertCmiiImageMapToList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) { for image, tag := range cmiiImageVersionMap { s := image2.CmiiHarborPrefix + image + ":" + tag @@ -482,6 +484,23 @@ func ConvertCMiiImageMapToList(cmiiImageVersionMap map[string]string) (fullImage return fullImageNameList } +func ConvertCmiiImageMapFromGzipFolder(gzipFileFolder string) (cmiiImageVersionMap map[string]string) { + + allFileInFolder, err := utils.ListAllFileInFolder(gzipFileFolder) + if err != nil { + return nil + } + + cmiiImageVersionMap = make(map[string]string, len(allFileInFolder)) + for _, gzipFileName := range allFileInFolder { + log.DebugF("gzip file name is %s", gzipFileName) + imageName, imageTag := image2.GzipFileNameToImageNameAndTag(gzipFileName) + cmiiImageVersionMap[imageName] = imageTag + } + + return cmiiImageVersionMap +} + func loginToDockerHub(HarborFullHost string) { if HarborFullHost == "" { diff --git a/agent-operator/image/CmiiImageSync_test.go b/agent-operator/image/CmiiImageSync_test.go index f96f1b7..d9efe37 100644 --- a/agent-operator/image/CmiiImageSync_test.go +++ b/agent-operator/image/CmiiImageSync_test.go @@ -195,3 +195,10 @@ func TestSaveSpecificImageToGzipFile(t *testing.T) { } } + +func TestConvertCmiiImageMapFromGzipFolder(t *testing.T) { + + versionMap := ConvertCmiiImageMapFromGzipFolder(OfflineImageGzipFolderPrefix) + + utils.BeautifulPrint(versionMap) +}