diff --git a/agent-deploy/d_app/CmiiImageConfig.go b/agent-deploy/d_app/CmiiImageConfig.go index e032125..d1fac67 100755 --- a/agent-deploy/d_app/CmiiImageConfig.go +++ b/agent-deploy/d_app/CmiiImageConfig.go @@ -122,6 +122,7 @@ var MiddlewareAmd64 = []string{ "redis:6.0.20-alpine", "dyrnq/nfs-subdir-external-provisioner:v4.0.2", "jerrychina2020/rke-tools:v0.175-linux", + "jerrychina2020/rke-tools:v0.175", "busybox:latest", } diff --git a/agent-operator/CmiiOperator.go b/agent-operator/CmiiOperator.go index 3119b0f..5dfbb30 100755 --- a/agent-operator/CmiiOperator.go +++ b/agent-operator/CmiiOperator.go @@ -16,17 +16,21 @@ const PublicDeployHarborHost = "42.192.52.227" const DirectPushDeployHarborHost = "36.134.71.138" type ImageSyncEntity struct { - ProjectName string // 优先级3 - ProjectFromDemoNameNotNull bool - ProjectVersion string // 优先级2 - CmiiNameTagList []string // 优先级1 appName:tag的形式 - FullNameImageList []string // 优先级1 - DownloadImage bool // 下载镜像 - DownloadFromOss bool // 下载镜像 - CompressImageToGzip bool // 压缩镜像 - UploadToDemoMinio bool // 上传镜像 - ShouldDirectPushToHarbor bool // 直接推送到对方的主机 || 离线部署机 使用此部门 - DirectHarborHost string // 目标Harbor仓库的Host，全名称带端口 不带http前缀 + ProjectName string // 优先级3 优先级最低 从DEMO拉取镜像 + ProjectVersion string // 优先级2 高于ProjectName 优先拉取特定版本的镜像 + + CmiiNameTagList []string // 优先级1 appName:tag 会被转换为FullNameImageList + FullNameImageList []string // 优先级1 优先下载此类型 + + ShouldDownloadImage bool // 下载镜像 DCU中的D + ShouldCompressImageToGzip bool // 压缩镜像 DCU中的C + ShouldUploadToDemoMinio bool // 上传镜像 DCU中的U + + ShouldDownloadFromOss bool // 下载镜像 DLTU中的D + ShouldUpdateImageTag bool // 更新镜像 DLTU中的U + + ShouldDirectPushToHarbor bool // 直接推送到对方的主机 || 离线部署机 + DirectHarborHost string // IP:Port or 域名:PORT 不带http前缀 } type ImageSyncResult struct { @@ -57,15 +61,14 @@ func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResul if sync.ProjectVersion != "" { // 获取特定版本的镜像 - - errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = DownloadCompressUploadFromVersion(sync.ProjectVersion, sync.CompressImageToGzip, sync.UploadToDemoMinio) + errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = C_DownloadCompressUploadFromVersion(sync.ProjectVersion, sync.ShouldCompressImageToGzip, sync.ShouldUploadToDemoMinio) gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectVersion } else { - // 获取DEMO的镜像 - errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = DownloadCompressUploadFromDemo(sync.ProjectName, sync.CompressImageToGzip, sync.UploadToDemoMinio) + // 获取DEMO的镜像 + errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = C_DownloadCompressUploadFromDemo(sync.ProjectName, sync.ShouldCompressImageToGzip, sync.ShouldUploadToDemoMinio) gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectName } @@ -77,7 +80,7 @@ func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResul allCmiiImageNameList = concatAndUniformCmiiImage(sync.FullNameImageList, sync.CmiiNameTagList) // DCU - errorPullImageList, errorGzipImageList, realCmiiImageList, allGzipFileNameList = DownloadCompressUpload(true, allCmiiImageNameList, sync.CompressImageToGzip, gzipFolderFullPath, sync.UploadToDemoMinio) + errorPullImageList, errorGzipImageList, realCmiiImageList, allGzipFileNameList = A_DownloadCompressUpload(true, allCmiiImageNameList, sync.ShouldCompressImageToGzip, gzipFolderFullPath, sync.ShouldUploadToDemoMinio) } // 直接传输到目标Harbor仓库 @@ -119,8 +122,8 @@ func concatAndUniformCmiiImage(fullImageList []string, cmiiImageList []string) [ return fullImageList } -// DownloadCompressUpload DCU 镜像同步的前半部分，通常在35.71 LapPro执行，无需Bastion Mode -func DownloadCompressUpload(downloadImage bool, fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileFullNameList []string) { +// A_DownloadCompressUpload DCU 镜像同步的前半部分，通常在35.71 LapPro执行，无需Bastion Mode +func A_DownloadCompressUpload(downloadImage bool, fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileFullNameList []string) { // write to file localGzipFileListTxt := gzipFolderFullPath + "all-gzip-image-file-name.txt" @@ -235,13 +238,13 @@ func DownloadCompressUpload(downloadImage bool, fullNameList []string, shouldGzi return errorPullImageList, errorGzipImageList, fullNameList, allGzipFileFullNameList } -// DownloadLoadTagUpload DLTU procedure ImageSync的另外一般流程，需要支持 堡垒机（纯离线）的模式 +// A_DownloadLoadTagUpload DLTU procedure ImageSync的另外一般流程，需要支持 堡垒机（纯离线）的模式 // 2. Gzip文件目录，RKE MIDDLE CMII三个文件目录 - 约定目录 // 约定目录 /root/wdd/image/rke/ /root/wdd/image/middle/ /root/wdd/image/cmii/ // 3. 读取本机的IP地址 - 参数传递 // 4. OSS地址 - ossUrlPrefix传空 则使用默认值 // 5. ossFileName - 如果结尾为txt，则为文件的形式，如果为tar.gz，则为gzip文件夹的形式 -func DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileNameOrGzipFileListTxt, localGzipFolderOrGzipFile string, targetHarborFullName string) (targetImageFullNameList []string) { +func A_DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileNameOrGzipFileListTxt, localGzipFolderOrGzipFile string, targetHarborFullName string) (targetImageFullNameList []string) { // 支持单文件的形式 if !utils.IsDirOrFile(localGzipFolderOrGzipFile) { @@ -340,8 +343,8 @@ func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string) return true } -// DownloadCompressUploadFromDemo 获取DEMO环境的全部镜像 -func DownloadCompressUploadFromDemo(projectName string, shouldGzip, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { +// C_DownloadCompressUploadFromDemo 获取DEMO环境的全部镜像 +func C_DownloadCompressUploadFromDemo(projectName string, shouldGzip, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { // generate a project folder err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+projectName, os.ModeDir) @@ -357,7 +360,7 @@ func DownloadCompressUploadFromDemo(projectName string, shouldGzip, shouldOss bo // do work // DCU - return DownloadCompressUpload(true, allCmiiImageNameListFromDemo, shouldGzip, image.OfflineImageGzipFolderPrefix+projectName, shouldOss) + return A_DownloadCompressUpload(true, allCmiiImageNameListFromDemo, shouldGzip, image.OfflineImageGzipFolderPrefix+projectName, shouldOss) } func buildAllCmiiImageNameListFromDemo(projectName string) []string { @@ -396,8 +399,8 @@ func buildAllCmiiImageNameListFromDemo(projectName string) []string { return realCmiiImageName } -// DownloadCompressUploadFromVersion 根据版本下载全部的CMII镜像 -func DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { +// C_DownloadCompressUploadFromVersion 根据版本下载全部的CMII镜像 +func C_DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { // generate a project folder err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+cmiiVersion, os.ModeDir) @@ -413,7 +416,7 @@ func DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shou // do work // DCU procedure - return DownloadCompressUpload(true, realCmiiImageName, shouldGzip, image.OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss) + return A_DownloadCompressUpload(true, realCmiiImageName, shouldGzip, image.OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss) } @@ -454,7 +457,8 @@ func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string { return realCmiiImageName } -func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, shouldDownload bool, isRKE bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { +// C_DownloadCompressUploadDependency DCU所有的依赖镜像 +func C_DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, shouldDownload bool, isRKE bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { log.Info("DCU for middle and rke!") err := os.MkdirAll(image.OfflineImageGzipFolderPrefix, os.ModeDir) @@ -478,7 +482,7 @@ func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, shouldDow gzipFolderPrefix = image.OfflineImageGzipFolderPrefix + "middle/" } - return DownloadCompressUpload(shouldDownload, fulleImageNameList, shouldGzip, gzipFolderPrefix, shouldOss) + return A_DownloadCompressUpload(shouldDownload, fulleImageNameList, shouldGzip, gzipFolderPrefix, shouldOss) } func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) { diff --git a/agent-operator/CmiiOperator_test.go b/agent-operator/CmiiOperator_test.go index 0aa3fa1..8e9856c 100755 --- a/agent-operator/CmiiOperator_test.go +++ b/agent-operator/CmiiOperator_test.go @@ -11,7 +11,7 @@ import ( func TestFetchDependencyRepos_Middle(t *testing.T) { - errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := DownloadCompressUploadDependency(true, true, false, false) + errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := C_DownloadCompressUploadDependency(true, true, false, false) utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList") utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList") @@ -22,7 +22,7 @@ func TestFetchDependencyRepos_Middle(t *testing.T) { func TestFetchDependencyRepos_RKE(t *testing.T) { - errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := DownloadCompressUploadDependency(true, true, false, true) + errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := C_DownloadCompressUploadDependency(true, true, false, true) utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList") utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList") @@ -50,18 +50,15 @@ func TestPullFromEntityAndSyncConditionally(t *testing.T) { sync := ImageSyncEntity{ CmiiNameTagList: []string{ "cmii-uav-mqtthandler:5.4.0-bjdyt-052102", - "cmii-uav-industrial-portfolio:5.4.0-bjdyt-052801", - "cmii-uav-platform-armypeople:5.4.0-bjdyt-052802", - "cmii-uav-platform:5.4.0-bjdyt-052901", }, - FullNameImageList: nil, - ProjectVersion: "", - ProjectName: "bjtg", - DirectHarborHost: "10.250.0.110", - DownloadImage: true, - CompressImageToGzip: false, - UploadToDemoMinio: false, - ShouldDirectPushToHarbor: true, + FullNameImageList: nil, + ProjectVersion: "", + ProjectName: "cqsh", + DirectHarborHost: "chongqingcis-9b4a3da9.ecis.chongqing-1.cmecloud.cn", + ShouldDownloadImage: true, + ShouldCompressImageToGzip: false, + ShouldUploadToDemoMinio: false, + ShouldDirectPushToHarbor: true, } // 调用函数并获取结果。这里需要根据你的实际需求来验证返回的结果是否符合预期。 @@ -87,7 +84,7 @@ func TestDownloadLoadTagUpload(t *testing.T) { for _, s := range localGzipFileList { log.InfoF("start to sync => %s", s) - targetImageFullNameList := DownloadLoadTagUpload(false, "", "", s, "172.28.0.251:8033") + targetImageFullNameList := A_DownloadLoadTagUpload(false, "", "", s, "172.28.0.251:8033") utils.BeautifulPrintListWithTitle(targetImageFullNameList, "targetImageFullNameList") diff --git a/agent-operator/image/CmiiImageOperator.go b/agent-operator/image/CmiiImageOperator.go index 7993747..c5a2015 100755 --- a/agent-operator/image/CmiiImageOperator.go +++ b/agent-operator/image/CmiiImageOperator.go @@ -204,20 +204,28 @@ func UploadToOctopusKindHarbor(targetImageName string) (pushResult io.ReadCloser } func UploadToHarbor(targetImageName string) (uploadOK bool) { + pushResult := UploadToOctopusKindHarbor(targetImageName) + defer pushResult.Close() + scanner := bufio.NewScanner(pushResult) for scanner.Scan() { } + fmt.Println() log.InfoF("[UploadToHarbor] - upload %s success!", targetImageName) fmt.Println() + return true } // TagFromListAndPushToCHarbor 需要支持 harbor.cdcyy.cn ip:8033 rancher/rancher:v2.5.7 nginx:latest func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost string) (errorPushImageNameList []string) { + targetHarborHost = strings.TrimPrefix(targetHarborHost, "http://") + targetHarborHost = strings.TrimPrefix(targetHarborHost, "https://") + for _, imageName := range referenceImageList { // check image @@ -247,7 +255,7 @@ func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost s } } - targetImageName := targetHarborHost + ":8033/" + targetProject + "/" + imageName + targetImageName := targetHarborHost + targetProject + "/" + imageName //fmt.Println(cmiiImageFullName) //fmt.Println(targetImageName) diff --git a/agent-operator/main.go b/agent-operator/main.go index 7eab9ef..bbd9499 100755 --- a/agent-operator/main.go +++ b/agent-operator/main.go @@ -150,7 +150,7 @@ func main() { } // DLTU - targetImageFullNameList := DownloadLoadTagUpload(downloadFromOss, ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile, harborHostFullName) + targetImageFullNameList := A_DownloadLoadTagUpload(downloadFromOss, ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile, harborHostFullName) // 是否需要更新 if namespace != "" { diff --git a/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java b/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java index abb287d..8d6fa37 100755 --- a/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java +++ b/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java @@ -38,7 +38,7 @@ public class TestImageSyncScheduler { public void runImageSync() { ArrayList CmiiAppNameList = new ArrayList<>(List.of( - "cmii-uav-platform:5.3.0-061901" + "cmii-uav-integration:5.4.0-070301" )); ArrayList ImageFullNameList = new ArrayList<>(List.of( @@ -48,10 +48,10 @@ public class TestImageSyncScheduler { Boolean downloadAndCompressOnly = false; // String projectNamespace = "wdd"; // wdd - String projectNamespace = "cqlyj"; // 重庆林业局 +// String projectNamespace = "cqlyj"; // 重庆林业局 // String projectNamespace = "jlyd"; // 吉林移动 // String projectNamespace = "jxyd"; // 吉林移动 -// String projectNamespace = "xmyd"; // 厦门移动 + String projectNamespace = "xmyd"; // 厦门移动 // String innerWorkerAgentName = "Chengdu-amd64-65-lapwdd"; //wdd String innerWorkerAgentName = "Chengdu-amd64-071-ed6e2b7f"; //prod