package main import ( "errors" "io/fs" "os" "path/filepath" "slices" "strings" "wdd.io/agent-common/utils" "wdd.io/agent-operator/image" ) const OfflineImageGzipFolderPrefix = "/root/octopus_image/" const OfflineDeployHarborHost = "harbor.wdd.io" const PublicDeployHarborHost = "42.192.52.227" const DirectPushDeployHarborHost = "36.134.71.138" type ImageSyncEntity struct { ProjectName string ProjectVersion string DirectHarborHost string //此参数决定是否能够直连目标主机,如果有则代表直连,可以直接推送景象 PushToDemoMinio bool } type ImageSyncResult struct { ErrorPullImageList []string ErrorGzipImageList []string ErrorPushImageNameList []string RealImageNameList []string RealGzipFileNameList []string AllCmiiImageNameList []string } // PullFromEntityAndSyncConditionally 根据ImageSyncEntity拉取特定的镜像,然后上传到特定的目标机器 func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResult ImageSyncResult) { var realCmiiImageList []string var errorPullImageList []string var errorGzipImageList []string var allCmiiImageNameList []string var allGzipFileNameList []string var errorPushImageNameList []string var gzipFolderFullPath string // get all image name by Name or Version // pull images // compress if sync.ProjectVersion != "" { // get version if sync.DirectHarborHost == "" { errorPullImageList, errorGzipImageList, allCmiiImageNameList = FetchVersionImages(sync.ProjectVersion, true) gzipFolderFullPath = OfflineImageGzipFolderPrefix + sync.ProjectVersion } else { errorPullImageList, errorGzipImageList, allCmiiImageNameList = FetchVersionImages(sync.ProjectVersion, false) } } else { // get demo images if sync.DirectHarborHost == "" { errorPullImageList, errorGzipImageList, allCmiiImageNameList = FetchDemoImages(sync.ProjectName, true) gzipFolderFullPath = OfflineImageGzipFolderPrefix + sync.ProjectName } else { errorPullImageList, errorGzipImageList, allCmiiImageNameList = FetchDemoImages(sync.ProjectName, false) } } backAllImageNameList := allCmiiImageNameList[:] realCmiiImageList = slices.DeleteFunc(backAllImageNameList, func(imageName string) bool { return slices.Contains(errorPullImageList, imageName) }) realCmiiImageList = slices.DeleteFunc(backAllImageNameList, func(imageName string) bool { return slices.Contains(errorGzipImageList, imageName) }) // direct push if can if sync.DirectHarborHost != "" { // push to errorPushImageNameList = image.TagFromListAndPushToCHarbor(realCmiiImageList, sync.DirectHarborHost) // build imageSyncResult.AllCmiiImageNameList = allCmiiImageNameList imageSyncResult.ErrorPullImageList = errorPullImageList imageSyncResult.ErrorGzipImageList = errorGzipImageList imageSyncResult.ErrorPushImageNameList = errorPushImageNameList imageSyncResult.RealImageNameList = realCmiiImageList // no gzip file return imageSyncResult } // get gzip file name list err := filepath.WalkDir(gzipFolderFullPath, func(path string, d fs.DirEntry, err error) error { if err != nil { log.ErrorF("error getting gzip file name list 1! %s", err.Error()) } if !d.IsDir() { allGzipFileNameList = append(allGzipFileNameList, d.Name()) } return nil }) if err != nil { log.ErrorF("error getting gzip file name list 2! %s", err.Error()) } // push to demo minio if sync.PushToDemoMinio { log.InfoF("pretend to push to minio !") // create path // push } // build imageSyncResult.AllCmiiImageNameList = allCmiiImageNameList imageSyncResult.ErrorPullImageList = errorPullImageList imageSyncResult.ErrorGzipImageList = errorGzipImageList imageSyncResult.ErrorPushImageNameList = errorPushImageNameList imageSyncResult.RealGzipFileNameList = allGzipFileNameList imageSyncResult.RealImageNameList = realCmiiImageList // no gzip file return imageSyncResult } func remove(s1, s2 []string) []string { m := make(map[string]struct{}, len(s2)) for _, v := range s2 { m[v] = struct{}{} } res := make([]string, 0, len(s1)) for _, v := range s1 { if _, ok := m[v]; !ok { res = append(res, v) } } return res } func FetchDemoImages(projectName string, gzipSplit bool) (errorPullImageList, errorGzipImageList, allCmiiImageName []string) { // generate a project folder err := os.MkdirAll(OfflineImageGzipFolderPrefix+projectName, os.ModeDir) if err != nil { if !errors.Is(err, os.ErrExist) { log.ErrorF("[FetchDemoImages] - create folder of %s error %s", OfflineImageGzipFolderPrefix+projectName, err.Error()) return errorPullImageList, errorGzipImageList, allCmiiImageName } } // get demo image version map backendMap, frontendMap, srsMap := BackupAllCmiiDeploymentToMap(demo) utils.BeautifulPrint(backendMap) utils.BeautifulPrint(frontendMap) utils.BeautifulPrint(srsMap) // save map to file backendMapFile := OfflineImageGzipFolderPrefix + projectName + "-backend-app.json" frontendMapFile := OfflineImageGzipFolderPrefix + projectName + "-frontend-app.json" srsMapFile := OfflineImageGzipFolderPrefix + projectName + "-srs-app.json" _ = os.Remove(backendMapFile) _ = os.Remove(frontendMapFile) _ = os.Remove(srsMapFile) utils.AppendContentToFile( utils.BeautifulPrintToString(backendMap), backendMapFile, ) utils.AppendContentToFile( utils.BeautifulPrintToString(frontendMap), frontendMapFile, ) utils.AppendContentToFile( utils.BeautifulPrintToString(srsMapFile), srsMapFile, ) // download image backendFullNameList, backendPull := image.PullFromCmiiHarborByMap(backendMap, true) frontendFullNameList, frontendPull := image.PullFromCmiiHarborByMap(frontendMap, true) srsFullNameList, srsPull := image.PullFromCmiiHarborByMap(srsMap, true) allCmiiImageName = append(allCmiiImageName, backendFullNameList...) allCmiiImageName = append(allCmiiImageName, frontendFullNameList...) allCmiiImageName = append(allCmiiImageName, srsFullNameList...) // compress image if gzipSplit { for image_name, tag := range backendMap { if !image.SaveToTarGZ(image_name+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") { errorGzipImageList = append(errorGzipImageList, image.CmiiHarborPrefix+image_name+":"+tag) } } for image_name, tag := range frontendMap { if !image.SaveToTarGZ(image_name+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") { errorGzipImageList = append(errorGzipImageList, image.CmiiHarborPrefix+image_name+":"+tag) } } for image_name, tag := range srsMap { if !image.SaveToTarGZ(image_name+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") { errorGzipImageList = append(errorGzipImageList, image.CmiiHarborPrefix+image_name+":"+tag) } } } // upload to harbor // clean up images errorPullImageList = append(errorPullImageList, backendPull...) errorPullImageList = append(errorPullImageList, frontendPull...) errorPullImageList = append(errorPullImageList, srsPull...) return errorPullImageList, errorGzipImageList, allCmiiImageName } func FetchVersionImages(cmiiVersion string, shouldGzip bool) (errorPullImageList, errorGzipImageList, allCmiiImageName []string) { // generate a project folder err := os.MkdirAll(OfflineImageGzipFolderPrefix+cmiiVersion, os.ModeDir) if err != nil { if !errors.Is(err, os.ErrExist) { log.ErrorF("[FetchDemoImages] - create folder of %s error %s", OfflineImageGzipFolderPrefix+cmiiVersion, err.Error()) return errorPullImageList, errorGzipImageList, allCmiiImageName } } backendMap := CmiiBackendAppMap frontendMap := CmiiFrontendAppMap for app := range backendMap { backendMap[app] = cmiiVersion } for app := range frontendMap { frontendMap[app] = cmiiVersion } allCmiiImageName = append(allCmiiImageName, image.ConvertCMiiImageMapToList(backendMap)...) allCmiiImageName = append(allCmiiImageName, image.ConvertCMiiImageMapToList(frontendMap)...) for key, value := range CmiiSrsAppMap { var app *CmiiDeploymentInterface if strings.Contains(value, "deployment") { app = CmiiOperator.DeploymentOneInterface(demo, key) if app != nil { allCmiiImageName = append(allCmiiImageName, app.Image) } } else if strings.Contains(value, "state") { app = CmiiOperator.StatefulSetOneInterface(demo, key) if app != nil { for _, imageName := range app.ContainerImageMap { allCmiiImageName = append(allCmiiImageName, imageName) } } } } utils.BeautifulPrintListWithTitle(allCmiiImageName, "Cmii Version Image => "+cmiiVersion) // do work if shouldGzip { errorPullImageList, errorGzipImageList = image.PullFromListAndCompressSplit(allCmiiImageName, OfflineImageGzipFolderPrefix+cmiiVersion) } else { errorPullImageList = image.PullFromFullNameList(allCmiiImageName) } return errorPullImageList, errorGzipImageList, allCmiiImageName } func FetchDependencyRepos(gzipSplit bool) (errorPullImageList, errorGzipImageList []string) { err := os.MkdirAll(OfflineImageGzipFolderPrefix, os.ModeDir) if err != nil { if !errors.Is(err, os.ErrExist) { log.ErrorF("[FetchDependencyRepos] - create folder of %s error %s", OfflineImageGzipFolderPrefix, err.Error()) } } errorPullImageList, errorGzipImageList = image.PullFromListAndCompressSplit(image.MiddlewareAmd64, OfflineImageGzipFolderPrefix+"middle/") pull, gzipImageList := image.PullFromListAndCompressSplit(image.Rancher1204Amd64, OfflineImageGzipFolderPrefix+"rke/") return append(errorPullImageList, pull...), append(errorGzipImageList, gzipImageList...) } func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) { // list folder projectGzipFolder := OfflineImageGzipFolderPrefix + projectName errorLoadImageNameList = append(errorLoadImageNameList, image.LoadFromFolderPath(projectGzipFolder)...) // read from json errorPushImageNameList = append(errorPushImageNameList, image.TagFromListAndPushToCHarbor(image.Cmii520DemoImageList, targetHarborHost)...) // re-tag // push // todo clean host and harbor // check harbor exits return errorLoadImageNameList, errorPushImageNameList } func LoadSplitDepGzipImageToTargetHarbor(targetHarborHost string) (errorLoadImageNameList []string, errorPushImageNameList []string) { //middle := OfflineImageGzipFolderPrefix + "middle/" //rke := OfflineImageGzipFolderPrefix + "rke/" //errorLoadImageNameList = append(errorLoadImageNameList, ImageLoadFromFolderPath(middle)...) //errorLoadImageNameList = append(errorLoadImageNameList, ImageLoadFromFolderPath(rke)...) errorPushImageNameList = append(errorPushImageNameList, image.TagFromListAndPushToCHarbor(image.MiddlewareAmd64, targetHarborHost)...) errorPushImageNameList = append(errorPushImageNameList, image.TagFromListAndPushToCHarbor(image.Rancher1204Amd64, targetHarborHost)...) return errorLoadImageNameList, errorPushImageNameList }