From 20ba7a9f38203d59f9ba664920f38f6238c6aadf Mon Sep 17 00:00:00 2001 From: zeaslity Date: Tue, 26 Mar 2024 14:07:51 +0800 Subject: [PATCH] [ Cmii ] [ Octopus ] - add Image Function - 1 --- .../image/ImageNameConvert.go | 11 +- agent-go/executor/BasicFunction.go | 40 +++ agent-go/executor/ImageFunction.go | 249 ++++++++++++++++++ agent-go/tmp/test.go | 50 ---- agent-operator/CmiiOperator.go | 23 +- agent-operator/image/CmiiImageSync.go | 17 +- agent-operator/image/CmiiImageSync_test.go | 3 +- .../func/auto/beans/ImageFunctionEnum.java | 62 +++++ 8 files changed, 376 insertions(+), 79 deletions(-) rename {agent-operator => agent-common}/image/ImageNameConvert.go (93%) create mode 100644 agent-go/executor/ImageFunction.go create mode 100644 server/src/main/java/io/wdd/func/auto/beans/ImageFunctionEnum.java diff --git a/agent-operator/image/ImageNameConvert.go b/agent-common/image/ImageNameConvert.go similarity index 93% rename from agent-operator/image/ImageNameConvert.go rename to agent-common/image/ImageNameConvert.go index da3314d..8e7bf4c 100644 --- a/agent-operator/image/ImageNameConvert.go +++ b/agent-common/image/ImageNameConvert.go @@ -1,9 +1,13 @@ package image -import "strings" +import ( + "strings" +) var unSupported = "UN-SUPPORT-APP-NAME" +const CmiiHarborPrefix = "harbor.cdcyy.com.cn/cmii/" + func ImageFullNameToAppName(imageFullName string) (appName string) { // harbor.cdcyy.cn/cmii/cmii-uav-platform:5.4.0 ==> cmii-uav-platform @@ -95,6 +99,11 @@ func ImageFullNameToGzipFileName(imageFullName string) (gzipFileName string) { return gzipFileName } +func ImageFullNameToTargetImageFullName(imageFullName, targetHost string) string { + + return imageFullName +} + func GzipFileNameToImageFullName(gzipFileName string) (imageFullName string) { return gzipFileName diff --git a/agent-go/executor/BasicFunction.go b/agent-go/executor/BasicFunction.go index 5198409..2dfa812 100644 --- a/agent-go/executor/BasicFunction.go +++ b/agent-go/executor/BasicFunction.go @@ -522,6 +522,46 @@ func BasicDownloadFile(downloadUrl, desFile string) (downloadOk bool, resultLog return true, nil } +func BasicDownloadFileWithProxy(downloadUrl, proxyUrl, desFile string) (downloadOk bool, resultLog []string) { + + var ok bool + if !BasicCommandExistByPath("curl") { + return false, []string{ + "curl not exits!", + } + } + + ok, resultLog = AllCommandExecutor([]string{ + "curl", + "-x", + proxyUrl, + "-o", + desFile, + "--insecure", + "--max-time", + "5", + downloadUrl, + }) + + errLog := fmt.Sprintf("[BasicDownloadFileWithProxy] - download file [ %s ] with proxy [ %s ] from [ %s ] failed !", desFile, proxyUrl, downloadUrl) + if !ok { + log.Error(errLog) + return false, []string{ + errLog, + } + } + // check file exists + existAndNotNull := BasicFileExistAndNotNull(desFile) + if !existAndNotNull { + return false, []string{ + errLog, + "[BasicDownloadFile] - file not exist !", + } + } + + return true, nil +} + // BasicAppendSourceToFile 将源文件的内容添加到目标文件，使用golang标准库完成，跨平台、安全性更强 func BasicAppendSourceToFile(sourceFile, targetFile string) bool { diff --git a/agent-go/executor/ImageFunction.go b/agent-go/executor/ImageFunction.go new file mode 100644 index 0000000..6b4f2c1 --- /dev/null +++ b/agent-go/executor/ImageFunction.go @@ -0,0 +1,249 @@ +package executor + +import ( + "strings" + "wdd.io/agent-common/image" +) + +var LocalGzipImageFolderPrefix = "/var/lib/docker/image_sync/" + +func (op *AgentOsOperator) Sync(baseFuncName string, funcArgs ...string) (bool, []string) { + resultOk := false + var errorLog []string + + switch baseFuncName { + + case "DOWNLOAD_DOCKER_IMAGE": + resultOk, errorLog = op.downloadDockerImage(funcArgs) + break + case "COMPRESS_IMAGE_TO_GZIP": + resultOk, errorLog = op.compressImageToGzip(funcArgs) + break + case "UPLOAD_GZIP_TO_OSS": + resultOk, errorLog = op.uploadGzipFileToOss(funcArgs) + break + case "DOWNLOAD_GZIP_IMAGE_FILE": + resultOk, errorLog = op.downloadGzipImageFile(funcArgs) + break + case "LOAD_DOCKER_IMAGE_FROM_GZIP": + resultOk, errorLog = op.loadDockerImageFromGzip(funcArgs) + break + case "PUSH_IMAGE_TO_TARGET_HARBOR": + resultOk, errorLog = op.pushImageToTargetHarbor(funcArgs) + break + case "UPDATE_IMAGE_TAG": + resultOk, errorLog = op.updateImageTag(funcArgs) + break + default: + resultOk, errorLog = op.okExec(funcArgs) + } + + return resultOk, errorLog +} + +func (op *AgentOsOperator) downloadDockerImage(funcArgs []string) (bool, []string) { + // funcArgs are imageFullName gzipFolderPrefix gzipFileName ossUrlPrefix namespace newImageTag + + if !BasicCommandExistByPath("docker") { + return false, []string{ + "docker not exits !", + } + } + imageFullName := funcArgs[0] + log.InfoF("[downloadDockerImage]- start to pull docker image %s", imageFullName) + if !PureResultSingleExecute([]string{ + "docker", + "pull", + imageFullName, + }) { + return false, []string{ + "docker pull failed of " + imageFullName, + } + } + + if !JudgeDockerImageExists(funcArgs[0]) { + return false, []string{ + "image not exits ! unknown error happened!", + } + } + return true, []string{ + imageFullName, + } +} + +func (op *AgentOsOperator) compressImageToGzip(funcArgs []string) (bool, []string) { + if !BasicCommandExistByPath("docker") { + return false, []string{ + "docker not exits !", + } + } + if !BasicCommandExistByPath("gzip") { + return false, []string{ + "gzip not exits !", + } + } + + gzipFolderPrefix := funcArgs[1] + if !BasicFolderExists(gzipFolderPrefix) { + BasicCreateFolder(gzipFolderPrefix) + } + + imageFullName := funcArgs[0] + if JudgeDockerImageExists(imageFullName) { + return false, []string{ + "image not exits !", + } + } + + if !strings.HasSuffix(gzipFolderPrefix, "/") { + gzipFolderPrefix += "/" + } + + gzipImageFromFullName := image.ImageFullNameToGzipFileName(imageFullName) + dockerSaveCommand := "docker save " + imageFullName + " | gzip > " + gzipFolderPrefix + gzipImageFromFullName + + executor, i := HardCodeCommandExecutor(dockerSaveCommand) + if !executor { + return false, i + } + + if !BasicFileExistAndNotNull(gzipFolderPrefix + gzipImageFromFullName) { + return false, []string{ + "gzip of ile error ", + } + } + + return true, []string{ + gzipImageFromFullName, + } +} + +func (op *AgentOsOperator) uploadGzipFileToOss(funcArgs []string) (bool, []string) { + + if !BasicCommandExistByPath("mc") { + return false, []string{ + "mc not exits!", + } + } + + gzipFolderPrefix := funcArgs[1] + gzipImageFromFullName := funcArgs[2] + + ok, resultLog := AllCommandExecutor([]string{ + "mc", + "cp", + gzipFolderPrefix + gzipImageFromFullName, + "demo/cmlc-installation/tmp/", + }) + + if !ok { + return false, resultLog + } + + return true, nil + +} + +func (op *AgentOsOperator) downloadGzipImageFile(funcArgs []string) (bool, []string) { + + ossUrlPrefix := funcArgs[3] + gzipImageFromFullName := funcArgs[2] + proxyUrl := funcArgs[4] + + if proxyUrl == "" { + BasicDownloadFile(ossUrlPrefix+gzipImageFromFullName, LocalGzipImageFolderPrefix+gzipImageFromFullName) + } else { + BasicDownloadFileWithProxy(ossUrlPrefix+gzipImageFromFullName, proxyUrl, LocalGzipImageFolderPrefix+gzipImageFromFullName) + } + + return true, nil +} + +func (op *AgentOsOperator) loadDockerImageFromGzip(funcArgs []string) (bool, []string) { + gzipImageFromFullName := funcArgs[2] + + if !BasicFileExistAndNotNull(LocalGzipImageFolderPrefix + gzipImageFromFullName) { + return false, []string{ + LocalGzipImageFolderPrefix + gzipImageFromFullName, + "local gzip file not exits!", + } + } + + hardCodeCommand := "docker load < " + LocalGzipImageFolderPrefix + gzipImageFromFullName + executor, i := HardCodeCommandExecutor(hardCodeCommand) + if !executor { + return false, i + } + + return true, nil +} +func (op *AgentOsOperator) pushImageToTargetHarbor(funcArgs []string) (bool, []string) { + + targetHarborHost := funcArgs[5] + imageFullName := funcArgs[0] + + if !strings.Contains(targetHarborHost, "8033") { + targetHarborHost += ":8033" + } + + targetImageFullName := image.ImageFullNameToTargetImageFullName(imageFullName, targetHarborHost) + + if !PureResultSingleExecute([]string{ + "docker", + "tag", + imageFullName, + targetImageFullName, + }) { + return false, []string{ + "docker tag error!", + } + } + + HardCodeCommandExecutor("docker login -u admin -p V2ryStr@ngPss " + targetHarborHost) + + ok, resultLog := AllCommandExecutor([]string{ + "docker", + "push", + targetImageFullName, + }) + if !ok { + return false, resultLog + } + + return true, []string{ + targetImageFullName, + } +} +func (op *AgentOsOperator) updateImageTag(funcArgs []string) (bool, []string) { + namespace := funcArgs[6] + targetImageFullName := funcArgs[7] + if !BasicCommandExistByPath("kubectl") { + return false, []string{ + "kubectl not exits !", + } + } + imageFullName := funcArgs[0] + if !strings.HasPrefix(imageFullName, image.CmiiHarborPrefix) { + return false, []string{ + "cant update this image !", + } + } + + appName := image.ImageFullNameToAppName(imageFullName) + + updateCommand := "kubectl -n " + namespace + " patch deployment " + appName + "-p \"{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\": " + appName + ",\"image\": " + targetImageFullName + "}]}}}}" + executor, i := HardCodeCommandExecutor(updateCommand) + if !executor { + return false, i + } + + return true, nil +} + +func JudgeDockerImageExists(imageFullName string) bool { + return true +} + +//func BuildGzipImageFromFullName(imageFullName string) string { +// +//} diff --git a/agent-go/tmp/test.go b/agent-go/tmp/test.go index e887992..06ab7d0 100644 --- a/agent-go/tmp/test.go +++ b/agent-go/tmp/test.go @@ -1,51 +1 @@ package main - -import "fmt" - -// quic sort s a slice of ints using quicksort algorithm. -// The function returns nothing. It modifies the slice in place. -func QuickSort(a []int) { // <-- pass by value, so we can't modify it - quick_sort(0, len(a)-1, a) -} - -/* - * The following function implements the quicksort algorithm. It requires three parameters: - * left: an index indicating the left bound of the subarray to be sorted. - * right: an index indicating the right bound of the subarray to be sorted. - * a: the slice of integers to sort. The function modifies the slice in place. - */ -func quick_sort(left int, right int, a []int) { - if left < right { // only continue if there is more than one element in the subarray - pivot := partition(left, right, a) - quick_sort(left, pivot-1, a) - quick_sort(pivot+1, right, a) // <-- call this function again on the right half of the subarray - } else { // if there is only one element in the subarray, we can return immediately - return - } -} - -func partition(left int, right int, a []int) int { - // find a pivot element. This is just one of the examples in the book - pivot := a[right] - - // index i will keep track of where to put the pivot element - i := left - 1 - - // swap the pivot element with a[right] - a[right], a[left] = a[left], a[right] - - for j := left; j < right; j++ { - if a[j] <= pivot { - i += 1 // increment i - a[i], a[j] = a[j], a[i] - } - } - - return i + 1 // put the pivot element at its final position -} - -func main() { - arr := []int{2, 45, 36, -78, 90, 1000, -1, 0, -76} - QuickSort(arr) - fmt.Println("Sorted array:", arr) -} diff --git a/agent-operator/CmiiOperator.go b/agent-operator/CmiiOperator.go index a201ae4..8fcb052 100644 --- a/agent-operator/CmiiOperator.go +++ b/agent-operator/CmiiOperator.go @@ -17,9 +17,10 @@ const PublicDeployHarborHost = "42.192.52.227" const DirectPushDeployHarborHost = "36.134.71.138" type ImageSyncEntity struct { - ProjectName string - ProjectVersion string - DirectHarborHost string //此参数决定是否能够直连目标主机，如果有则代表直连，可以直接推送景象 + ProjectName string // 3 + ProjectVersion string // 2 + CmiiImageList map[string]string // 1 + DirectHarborHost string //此参数决定是否能够直连目标主机，如果有则代表直连，可以直接推送景象 PushToDemoMinio bool } @@ -32,7 +33,7 @@ type ImageSyncResult struct { AllCmiiImageNameList []string } -// PullFromEntityAndSyncConditionally 根据ImageSyncEntity拉取特定的镜像，然后上传到特定的目标机器 +// PullFromEntityAndSyncConditionally 根据ImageSyncEntity拉取特定的镜像，然后上传到特定的目标机器(或者上传的minio中) func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResult ImageSyncResult) { var realCmiiImageList []string @@ -122,20 +123,6 @@ func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResul return imageSyncResult } -func remove(s1, s2 []string) []string { - m := make(map[string]struct{}, len(s2)) - for _, v := range s2 { - m[v] = struct{}{} - } - res := make([]string, 0, len(s1)) - for _, v := range s1 { - if _, ok := m[v]; !ok { - res = append(res, v) - } - } - return res -} - func FetchDemoImages(projectName string, gzipSplit bool) (errorPullImageList, errorGzipImageList, allCmiiImageName []string) { // generate a project folder diff --git a/agent-operator/image/CmiiImageSync.go b/agent-operator/image/CmiiImageSync.go index 521f98c..636daa3 100644 --- a/agent-operator/image/CmiiImageSync.go +++ b/agent-operator/image/CmiiImageSync.go @@ -15,6 +15,7 @@ import ( "os" "strconv" "strings" + image2 "wdd.io/agent-common/image" "wdd.io/agent-common/logger" "wdd.io/agent-common/utils" ) @@ -22,8 +23,6 @@ import ( var apiClient = newClient() var log = logger.Log -const CmiiHarborPrefix = "harbor.cdcyy.com.cn/cmii/" - func newClient() *client.Client { apiClient, err := client.NewClientWithOpts(client.FromEnv) @@ -140,7 +139,7 @@ func PruneAllCmiiImages() (errorRemoveImageNameList []string) { for _, imageSummary := range imageGetAll { for _, repoTag := range imageSummary.RepoTags { - if strings.HasPrefix(repoTag, CmiiHarborPrefix) { + if strings.HasPrefix(repoTag, image2.CmiiHarborPrefix) { for _, tag := range imageSummary.RepoTags { _, err := apiClient.ImageRemove(context.TODO(), imageSummary.ID, types.ImageRemoveOptions{ Force: true, @@ -209,7 +208,7 @@ func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost s cmiiImageFullName := imageName if strings.HasPrefix(imageName, "cmii") { - cmiiImageFullName = CmiiHarborPrefix + imageName + cmiiImageFullName = image2.CmiiHarborPrefix + imageName } targetProject := "cmii" @@ -217,9 +216,9 @@ func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost s targetProject = "rancher" } - if strings.HasPrefix(imageName, CmiiHarborPrefix) { + if strings.HasPrefix(imageName, image2.CmiiHarborPrefix) { // - imageName = strings.TrimPrefix(imageName, CmiiHarborPrefix) + imageName = strings.TrimPrefix(imageName, image2.CmiiHarborPrefix) } else { // todo if strings.Contains(imageName, "/") { @@ -355,7 +354,7 @@ func PullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string) errorGzipImageList = append(errorGzipImageList, image) continue } - tarGzipFileNameList = append(tarGzipFileNameList, ImageFullNameToGzipFileName(image)) + tarGzipFileNameList = append(tarGzipFileNameList, image2.ImageFullNameToGzipFileName(image)) } utils.BeautifulPrintListWithTitle(tarGzipFileNameList, "image gzip name list") @@ -431,7 +430,7 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool { } } - gzipImageFile := ImageFullNameToGzipFileName(realImageTag) + gzipImageFile := image2.ImageFullNameToGzipFileName(realImageTag) if !strings.HasSuffix(folderPathPrefix, "/") { folderPathPrefix += "/" } @@ -466,7 +465,7 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool { func ConvertCMiiImageMapToList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) { for image, tag := range cmiiImageVersionMap { - s := CmiiHarborPrefix + image + ":" + tag + s := image2.CmiiHarborPrefix + image + ":" + tag fullImageNameList = append(fullImageNameList, s) } diff --git a/agent-operator/image/CmiiImageSync_test.go b/agent-operator/image/CmiiImageSync_test.go index 39ffd10..5d8ec70 100644 --- a/agent-operator/image/CmiiImageSync_test.go +++ b/agent-operator/image/CmiiImageSync_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" "wdd.io/agent-common/assert" + "wdd.io/agent-common/image" "wdd.io/agent-common/utils" ) @@ -130,7 +131,7 @@ func TestImageFullNameToGzipFileName(t *testing.T) { } for _, s := range test { - gzipFileName := ImageFullNameToGzipFileName(s) + gzipFileName := image.ImageFullNameToGzipFileName(s) fmt.Printf(" %s to %s \n", s, gzipFileName) } } diff --git a/server/src/main/java/io/wdd/func/auto/beans/ImageFunctionEnum.java b/server/src/main/java/io/wdd/func/auto/beans/ImageFunctionEnum.java new file mode 100644 index 0000000..51a6820 --- /dev/null +++ b/server/src/main/java/io/wdd/func/auto/beans/ImageFunctionEnum.java @@ -0,0 +1,62 @@ +package io.wdd.func.auto.beans; + + +/** + * 参数固定顺序为 + * imageFullName gzipFolderPrefix gzipFileName ossUrlPrefix proxyUrl targetHarborHost namespace targetImageFullName + */ +public enum ImageFunctionEnum { + + DOWNLOAD_DOCKER_IMAGE( + "DOWNLOAD_DOCKER_IMAGE", + "args imageFullName return imageFullName" + ), + + COMPRESS_IMAGE_TO_GZIP( + "COMPRESS_IMAGE_TO_GZIP", + "args imageFullName return gzipFileName" + ), + + UPLOAD_GZIP_TO_OSS( + "UPLOAD_GZIP_TO_OSS", + "关闭防火墙" + ), + + DOWNLOAD_GZIP_IMAGE_FILE( + "DOWNLOAD_GZIP_IMAGE_FILE", + "关闭防火墙" + ), + + LOAD_DOCKER_IMAGE_FROM_GZIP( + "LOAD_DOCKER_IMAGE_FROM_GZIP", + "关闭防火墙" + ), + + PUSH_IMAGE_TO_TARGET_HARBOR( + "PUSH_IMAGE_TO_TARGET_HARBOR", + "关闭防火墙" + ), + + UPDATE_IMAGE_TAG( + "UPDATE_IMAGE_TAG", + "关闭防火墙" + ); + String funcName; + + String desc; + + + ImageFunctionEnum(String funcName, String desc) { + this.funcName = funcName; + this.desc = desc; + } + + public String getFuncName() { + return funcName; + } + + public String getDesc() { + return desc; + } + +}