diff --git a/agent-common/image/ImageNameConvert.go b/agent-common/image/ImageNameConvert.go index 8e7bf4c..6b606e6 100644 --- a/agent-common/image/ImageNameConvert.go +++ b/agent-common/image/ImageNameConvert.go @@ -101,7 +101,23 @@ func ImageFullNameToGzipFileName(imageFullName string) (gzipFileName string) { func ImageFullNameToTargetImageFullName(imageFullName, targetHost string) string { - return imageFullName + if strings.HasPrefix(imageFullName, CmiiHarborPrefix) { + imageFullName = strings.TrimPrefix(imageFullName, CmiiHarborPrefix) + } else if strings.HasPrefix(imageFullName, "docker.io") { + imageFullName = strings.TrimPrefix(imageFullName, "docker.io") + } + // rancher/123:v123 + if strings.HasPrefix(imageFullName, "rancher") { + return targetHost + "/" + imageFullName + } + // ossr/srs:v4.0.5 + if strings.Contains(imageFullName, "/") { + imageFullName = strings.Split(imageFullName, "/")[1] + } + + // srs:v4.0.5 + // cmii-uav-platform:5.4.0 + return targetHost + "/cmii/" + imageFullName } func GzipFileNameToImageFullName(gzipFileName string) (imageFullName string) { diff --git a/agent-go/executor/BaseFunction.go b/agent-go/executor/BaseFunction.go index cfbb819..4da1343 100644 --- a/agent-go/executor/BaseFunction.go +++ b/agent-go/executor/BaseFunction.go @@ -336,9 +336,9 @@ func (op *AgentOsOperator) disableSELinuxExec() (bool, []string) { } // delete contend - BasicFindDeleteContendLineInFile("SELINUX=enforcing", "/etc/selinux/config") - BasicFindDeleteContendLineInFile("SELINUX=permissive", "/etc/selinux/config") - BasicFindDeleteContendLineInFile("SELINUX=disabled", "/etc/selinux/config") + BasicFindAndDeleteContendLineInFile("SELINUX=enforcing", "/etc/selinux/config") + BasicFindAndDeleteContendLineInFile("SELINUX=permissive", "/etc/selinux/config") + BasicFindAndDeleteContendLineInFile("SELINUX=disabled", "/etc/selinux/config") BasicAppendContentToFile("SELINUX=disabled", "/etc/selinux/config") diff --git a/agent-go/executor/BasicFunction.go b/agent-go/executor/BasicFunction.go index 2dfa812..3259f85 100644 --- a/agent-go/executor/BasicFunction.go +++ b/agent-go/executor/BasicFunction.go @@ -150,7 +150,7 @@ func BasicFindContentInFile(content string, fileName string) bool { return false } -func BasicFindDeleteContendLineInFile(content string, fileName string) bool { +func BasicFindAndDeleteContendLineInFile(content string, fileName string) bool { // Open the file file, err := os.Open(fileName) @@ -196,6 +196,38 @@ func BasicFindDeleteContendLineInFile(content string, fileName string) bool { return true } +func BasicFindContentInCommandOutput(hardCodeCommand string, content string) (bool, []string) { + ok, resultLog := HardCodeCommandExecutor(hardCodeCommand) + if !ok { + log.ErrorF("[BasicFindContentInCommandOutput] - command error ! => %v", resultLog) + return false, nil + } + + return BasicFindContentInList(content, resultLog), resultLog +} + +func BasicFindContentInList(content string, list []string) bool { + for _, item := range list { + if strings.Contains(item, content) { + return true + } + } + return false +} + +func BasicDockerImageExistByFullName(imageFullName string) bool { + + var imageVersion string + + split := strings.Split(imageFullName, ":") + imageVersion = split[len(split)-1] + + imageFullName = strings.TrimSuffix(imageFullName, ":"+imageVersion) + + //log.DebugF("[BasicDockerImageExistByFullName] -- %s %s", imageFullName, imageVersion) + return BasicDockerImageExists(imageFullName, imageVersion) +} + func BasicDockerImageExists(imageName, imageVersion string) bool { if !BasicCommandExistByPath("docker") { @@ -324,22 +356,33 @@ func BasicFolderExists(folderName string) bool { } func BasicCreateFolder(folderName string) bool { - cmd := exec.Command("test", "-d", folderName) - err := cmd.Run() - if err == nil { - log.DebugF("folder %s already exists !", folderName) - return true + // 检查目录是否存在 + if _, err := os.Stat(folderName); os.IsNotExist(err) { + // 目录不存在,创建目录 + err := os.MkdirAll(folderName, os.ModePerm) + if err != nil { + // 如果创建目录失败,返回false + return false + } + } + // 目录存在或者成功创建,返回true + return true +} + +func BasicRemoveFileOrFolder(folderName string) bool { + // Check if the folder exists + if _, err := os.Stat(folderName); os.IsNotExist(err) { + return true // folder does not exist } - cmd2 := exec.Command("mkdir", "-p", folderName) - err = cmd2.Run() + // Folder exists, remove it + err := os.Remove(folderName) if err != nil { - log.ErrorF("folder %s create error!", folderName) + // Handle error return false - } else { - log.DebugF("folder %s create success!", folderName) - return true } + + return true // folder was removed } // BasicPrettyPrint 打印执行结果 diff --git a/agent-go/executor/BasicFunction_test.go b/agent-go/executor/BasicFunction_test.go index 802653d..0002b8b 100644 --- a/agent-go/executor/BasicFunction_test.go +++ b/agent-go/executor/BasicFunction_test.go @@ -66,3 +66,43 @@ func TestBasicCommandExists(t *testing.T) { t.Logf("command exists is => %s", strconv.FormatBool(exists)) } + +func TestBasicDockerImageExistByFullName(t *testing.T) { + + // Test cases + testCases := []struct { + name string + imageFullName string + expectedResult bool + }{ + { + name: "Image exists with full name and version", + imageFullName: "harbor.cdcyy.com.cn/cmii/busybox:0326", + expectedResult: true, + }, + { + name: "Image does not exist with full name and version", + imageFullName: "nonexistentImage:latest", + expectedResult: false, + }, + { + name: "Image exists with full name but no version", + imageFullName: "harbor.cdcyy.com.cn/cmii/busybox", + expectedResult: true, + }, + { + name: "Image does not exist with full name but no version", + imageFullName: "nonexistentImage", + expectedResult: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := BasicDockerImageExistByFullName(tc.imageFullName) + if result != tc.expectedResult { + t.Errorf("Expected %t for imageFullName %s, but got %t", tc.expectedResult, tc.imageFullName, result) + } + }) + } +} diff --git a/agent-go/executor/FunctionalExecutor.go b/agent-go/executor/FunctionalExecutor.go index c3fc508..8d28bd6 100644 --- a/agent-go/executor/FunctionalExecutor.go +++ b/agent-go/executor/FunctionalExecutor.go @@ -232,7 +232,7 @@ func collectOutput(r io.Reader, resultSlice []string) []string { } // HardCodeCommandExecutor 执行硬编码的shell命令,如 echo sda > sdasd ; rpcinfo -p localhost 等 -func HardCodeCommandExecutor(hardCodeCommand string) (bool, []string) { +func HardCodeCommandExecutor(hardCodeCommand string) (ok bool, resultLog []string) { // result var resultSlice []string @@ -264,7 +264,7 @@ func HardCodeCommandExecutor(hardCodeCommand string) (bool, []string) { resultOk = false } - log.DebugF("hard code command of [ %s ] result are => %v", hardCodeCommand, resultSlice) + //log.DebugF("hard code command of [ %s ] result are => %v", hardCodeCommand, resultSlice) return resultOk, resultSlice } diff --git a/agent-go/executor/FunctionalExecutor_test.go b/agent-go/executor/FunctionalExecutor_test.go index 7a92be1..d869743 100644 --- a/agent-go/executor/FunctionalExecutor_test.go +++ b/agent-go/executor/FunctionalExecutor_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "os/exec" + "reflect" "strings" "testing" ) @@ -133,6 +134,47 @@ func TestPipelineCommandExecutor(t *testing.T) { BasicPrettyPrint(ok, resultLog) } +func TestPipelineCommandExecutor2(t *testing.T) { + // Test case 1: Empty pipeline + pipelineCommand := [][]string{} + ok, log := PipelineCommandExecutor(pipelineCommand) + if !ok { + t.Errorf("Expected success, got failure") + } + if len(log) != 0 { + t.Errorf("Expected no log, got %v", log) + } + + // Test case 2: Single command in pipeline + pipelineCommand = [][]string{ + {"echo", "Hello, world!"}, + } + ok, log = PipelineCommandExecutor(pipelineCommand) + if !ok { + t.Errorf("Expected success, got failure") + } + expectedLog := []string{"Hello, world!"} + if !reflect.DeepEqual(log, expectedLog) { + t.Errorf("Expected log %v, got %v", expectedLog, log) + } + + // Test case 3: Multiple commands in pipeline + pipelineCommand = [][]string{ + {"echo", "Hello"}, + {"echo", "world!"}, + } + ok, log = PipelineCommandExecutor(pipelineCommand) + if !ok { + t.Errorf("Expected success, got failure") + } + expectedLog = []string{"Hello", "world!"} + if !reflect.DeepEqual(log, expectedLog) { + t.Errorf("Expected log %v, got %v", expectedLog, log) + } + + // Add more test cases as needed... +} + func TestSimple(t *testing.T) { for i := 0; i < 10; i++ { diff --git a/agent-go/executor/ImageFunction.go b/agent-go/executor/ImageFunction.go index 6b4f2c1..0619163 100644 --- a/agent-go/executor/ImageFunction.go +++ b/agent-go/executor/ImageFunction.go @@ -51,6 +51,12 @@ func (op *AgentOsOperator) downloadDockerImage(funcArgs []string) (bool, []strin } imageFullName := funcArgs[0] log.InfoF("[downloadDockerImage]- start to pull docker image %s", imageFullName) + + // login + if strings.HasPrefix(imageFullName, image.CmiiHarborPrefix) { + HardCodeCommandExecutor("docker login -u rad02_drone -p Drone@1234 harbor.cdcyy.com.cn") + } + if !PureResultSingleExecute([]string{ "docker", "pull", @@ -61,7 +67,7 @@ func (op *AgentOsOperator) downloadDockerImage(funcArgs []string) (bool, []strin } } - if !JudgeDockerImageExists(funcArgs[0]) { + if !BasicDockerImageExistByFullName(funcArgs[0]) { return false, []string{ "image not exits ! unknown error happened!", } @@ -89,7 +95,7 @@ func (op *AgentOsOperator) compressImageToGzip(funcArgs []string) (bool, []strin } imageFullName := funcArgs[0] - if JudgeDockerImageExists(imageFullName) { + if !BasicDockerImageExistByFullName(imageFullName) { return false, []string{ "image not exits !", } @@ -129,18 +135,35 @@ func (op *AgentOsOperator) uploadGzipFileToOss(funcArgs []string) (bool, []strin gzipFolderPrefix := funcArgs[1] gzipImageFromFullName := funcArgs[2] - ok, resultLog := AllCommandExecutor([]string{ + ok, resultLog := HardCodeCommandExecutor("mc --insecure alias set demo https://oss.ig-demo.uavcmlc.com cmii B#923fC7mk") + //ok, resultLog = HardCodeCommandExecutor("mc alias list") + + PureResultSingleExecute([]string{ + "mc", + "rm", + "demo/cmlc-installation/tmp/" + gzipImageFromFullName, + }) + + ok, resultLog = AllCommandExecutor([]string{ "mc", "cp", gzipFolderPrefix + gzipImageFromFullName, - "demo/cmlc-installation/tmp/", + "demo/cmlc-installation/tmp/" + gzipImageFromFullName, }) - if !ok { return false, resultLog } - return true, nil + find, _ := BasicFindContentInCommandOutput("mc ls demo/cmlc-installation/tmp/", gzipImageFromFullName) + if !find { + return false, []string{ + "demo oss can't find gzip file !", + } + } + + return true, []string{ + gzipImageFromFullName, + } } @@ -150,13 +173,33 @@ func (op *AgentOsOperator) downloadGzipImageFile(funcArgs []string) (bool, []str gzipImageFromFullName := funcArgs[2] proxyUrl := funcArgs[4] - if proxyUrl == "" { - BasicDownloadFile(ossUrlPrefix+gzipImageFromFullName, LocalGzipImageFolderPrefix+gzipImageFromFullName) - } else { - BasicDownloadFileWithProxy(ossUrlPrefix+gzipImageFromFullName, proxyUrl, LocalGzipImageFolderPrefix+gzipImageFromFullName) + // create folder + BasicCreateFolder(LocalGzipImageFolderPrefix) + + // remove file + desFile := LocalGzipImageFolderPrefix + gzipImageFromFullName + + if !BasicRemoveFileOrFolder(desFile) { + return false, []string{ + "file already exits ! can't remove it!", + } } - return true, nil + var download bool + var downloadLog []string + if proxyUrl == "" { + download, downloadLog = BasicDownloadFile(ossUrlPrefix+gzipImageFromFullName, desFile) + } else { + download, downloadLog = BasicDownloadFileWithProxy(ossUrlPrefix+gzipImageFromFullName, proxyUrl, desFile) + } + + if !download { + return false, downloadLog + } + + return true, []string{ + desFile, + } } func (op *AgentOsOperator) loadDockerImageFromGzip(funcArgs []string) (bool, []string) { @@ -175,6 +218,13 @@ func (op *AgentOsOperator) loadDockerImageFromGzip(funcArgs []string) (bool, []s return false, i } + if !BasicDockerImageExistByFullName(funcArgs[0]) { + return false, []string{ + "docker load from gzip file error ! image not exits!", + funcArgs[0], + } + } + return true, nil } func (op *AgentOsOperator) pushImageToTargetHarbor(funcArgs []string) (bool, []string) { @@ -198,8 +248,11 @@ func (op *AgentOsOperator) pushImageToTargetHarbor(funcArgs []string) (bool, []s "docker tag error!", } } - - HardCodeCommandExecutor("docker login -u admin -p V2ryStr@ngPss " + targetHarborHost) + if strings.HasPrefix(targetImageFullName, image.CmiiHarborPrefix) { + HardCodeCommandExecutor("docker login -u rad02_drone -p Drone@1234 harbor.cdcyy.com.cn") + } else { + HardCodeCommandExecutor("docker login -u admin -p V2ryStr@ngPss " + targetHarborHost) + } ok, resultLog := AllCommandExecutor([]string{ "docker", @@ -240,10 +293,6 @@ func (op *AgentOsOperator) updateImageTag(funcArgs []string) (bool, []string) { return true, nil } -func JudgeDockerImageExists(imageFullName string) bool { - return true -} - //func BuildGzipImageFromFullName(imageFullName string) string { // //} diff --git a/agent-go/executor/ImageFunction_test.go b/agent-go/executor/ImageFunction_test.go index 03feb87..e2c8c54 100644 --- a/agent-go/executor/ImageFunction_test.go +++ b/agent-go/executor/ImageFunction_test.go @@ -12,14 +12,14 @@ func TestAgentOsOperator_Sync(t *testing.T) { // imageFullName gzipFolderPrefix gzipFileName ossUrlPrefix proxyUrl targetHarborHost namespace targetImageFullName funcArgs := []string{ "harbor.cdcyy.com.cn/cmii/busybox:0326", - "/root/octopus_image/", - "", - "https://oss.demo.uavcmlc.com/cmlc-installation/tmp/", - "", + "/var/lib/docker/octopus_image/", + "cmlc=cmii=busybox=0326.tar.gz", + "https://oss.demo.uavcmlc.com:18000/cmlc-installation/tmp/", "", + "10.250.0.100", "", } - sync, strings := agentOP.Sync("DOWNLOAD_DOCKER_IMAGE", funcArgs...) + sync, strings := agentOP.Sync("PUSH_IMAGE_TO_TARGET_HARBOR", funcArgs...) fmt.Println("func result are => " + strconv.FormatBool(sync)) utils.BeautifulPrint(strings) } diff --git a/agent-go/executor/demo/cmlc-installation/tmp/cmlc=cmii=busybox=0326.tar.gz b/agent-go/executor/demo/cmlc-installation/tmp/cmlc=cmii=busybox=0326.tar.gz new file mode 100644 index 0000000..35dcc13 Binary files /dev/null and b/agent-go/executor/demo/cmlc-installation/tmp/cmlc=cmii=busybox=0326.tar.gz differ diff --git a/agent-operator/CmiiK8sOperator.go b/agent-operator/CmiiK8sOperator.go index 1547538..3d807c2 100644 --- a/agent-operator/CmiiK8sOperator.go +++ b/agent-operator/CmiiK8sOperator.go @@ -4,8 +4,8 @@ import ( "bufio" "strings" "time" + image2 "wdd.io/agent-common/image" "wdd.io/agent-common/utils" - "wdd.io/agent-operator/image" ) var CmiiOperator = CmiiK8sOperator{} @@ -470,8 +470,8 @@ func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap, srsM if app != nil { for _, imageName := range app.ContainerImageMap { split := strings.Split(imageName, ":") - if strings.Contains(split[0], image.CmiiHarborPrefix) { - split[0] = strings.Split(split[0], image.CmiiHarborPrefix)[1] + if strings.Contains(split[0], image2.CmiiHarborPrefix) { + split[0] = strings.Split(split[0], image2.CmiiHarborPrefix)[1] } srsMap[split[0]] = split[1] } @@ -481,7 +481,7 @@ func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap, srsM if app != nil { for _, imageName := range app.ContainerImageMap { split := strings.Split(imageName, ":") - split[0], _ = strings.CutPrefix(split[0], image.CmiiHarborPrefix) + split[0], _ = strings.CutPrefix(split[0], image2.CmiiHarborPrefix) srsMap[split[0]] = split[1] } } diff --git a/agent-operator/CmiiK8sOperator_test.go b/agent-operator/CmiiK8sOperator_test.go index cd5b366..5e3369f 100644 --- a/agent-operator/CmiiK8sOperator_test.go +++ b/agent-operator/CmiiK8sOperator_test.go @@ -243,9 +243,9 @@ func TestRestartCmiiDeployment(t *testing.T) { func TestUpdateCmiiDeploymentImageTag(t *testing.T) { - cmiiEnv := demo - appName := "cmii-uav-process" - newTag := "5.4.0-032201" + cmiiEnv := integration + appName := "cmii-uav-platform" + newTag := "5.4.0-032601" tag := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag) assert.Equal(t, tag, true, "update image tag failed !") diff --git a/agent-operator/CmiiOperator.go b/agent-operator/CmiiOperator.go index 8fcb052..fd63c94 100644 --- a/agent-operator/CmiiOperator.go +++ b/agent-operator/CmiiOperator.go @@ -7,6 +7,7 @@ import ( "path/filepath" "slices" "strings" + image2 "wdd.io/agent-common/image" "wdd.io/agent-common/utils" "wdd.io/agent-operator/image" ) @@ -175,17 +176,17 @@ func FetchDemoImages(projectName string, gzipSplit bool) (errorPullImageList, er if gzipSplit { for image_name, tag := range backendMap { if !image.SaveToTarGZ(image_name+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") { - errorGzipImageList = append(errorGzipImageList, image.CmiiHarborPrefix+image_name+":"+tag) + errorGzipImageList = append(errorGzipImageList, image2.CmiiHarborPrefix+image_name+":"+tag) } } for image_name, tag := range frontendMap { if !image.SaveToTarGZ(image_name+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") { - errorGzipImageList = append(errorGzipImageList, image.CmiiHarborPrefix+image_name+":"+tag) + errorGzipImageList = append(errorGzipImageList, image2.CmiiHarborPrefix+image_name+":"+tag) } } for image_name, tag := range srsMap { if !image.SaveToTarGZ(image_name+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") { - errorGzipImageList = append(errorGzipImageList, image.CmiiHarborPrefix+image_name+":"+tag) + errorGzipImageList = append(errorGzipImageList, image2.CmiiHarborPrefix+image_name+":"+tag) } } } diff --git a/agent-operator/K8sOperator.go b/agent-operator/K8sOperator.go index 2a64a46..add3dc4 100644 --- a/agent-operator/K8sOperator.go +++ b/agent-operator/K8sOperator.go @@ -15,9 +15,9 @@ import ( "strings" "sync" "time" + image2 "wdd.io/agent-common/image" "wdd.io/agent-common/logger" "wdd.io/agent-common/utils" - "wdd.io/agent-operator/image" ) var log = logger.Log @@ -407,7 +407,7 @@ func (op *CmiiK8sOperator) DeploymentUpdateTag(cmiiEnv, appName, newTag string) oldName := container.Image split := strings.Split(container.Image, ":") - if strings.HasPrefix(container.Image, image.CmiiHarborPrefix) { + if strings.HasPrefix(container.Image, image2.CmiiHarborPrefix) { // harbor container.Image = split[0] + ":" + newTag } else if strings.Contains(container.Image, "8033") { diff --git a/cmii_operator/log/cmii-update-log.txt b/cmii_operator/log/cmii-update-log.txt index 1517543..34e20fe 100644 --- a/cmii_operator/log/cmii-update-log.txt +++ b/cmii_operator/log/cmii-update-log.txt @@ -15,3 +15,5 @@ 2024-03-22-14-02-05 uavcloud-demo cmii-uav-platform 5.4.0-25263-032011 5.4.0-25263-emerg-ai-032201 2024-03-22-17-14-43 uavcloud-demo cmii-uav-platform-armypeople 5.4.0 5.4.0-26860-032201 2024-03-22-17-17-15 uavcloud-demo cmii-uav-process 5.4.0 5.4.0-032201 +2024-03-26-16-27-58 uavcloud-demo cmii-uav-multilink 5.4.0 5.4.0-032602 +2024-03-26-17-11-29 uavcloud-test cmii-uav-platform 5.4.0-032002 5.4.0-032601