[Cmii][ImageSync] -Image Function accomplish -1

agent-common/image/ImageNameConvert.go

@@ -101,7 +101,23 @@ func ImageFullNameToGzipFileName(imageFullName string) (gzipFileName string) {
 
 func ImageFullNameToTargetImageFullName(imageFullName, targetHost string) string {
 
-	return imageFullName
+	if strings.HasPrefix(imageFullName, CmiiHarborPrefix) {
+		imageFullName = strings.TrimPrefix(imageFullName, CmiiHarborPrefix)
+	} else if strings.HasPrefix(imageFullName, "docker.io") {
+		imageFullName = strings.TrimPrefix(imageFullName, "docker.io")
+	}
+	// rancher/123:v123
+	if strings.HasPrefix(imageFullName, "rancher") {
+		return targetHost + "/" + imageFullName
+	}
+	// ossr/srs:v4.0.5
+	if strings.Contains(imageFullName, "/") {
+		imageFullName = strings.Split(imageFullName, "/")[1]
+	}
+
+	// srs:v4.0.5
+	// cmii-uav-platform:5.4.0
+	return targetHost + "/cmii/" + imageFullName
 }
 
 func GzipFileNameToImageFullName(gzipFileName string) (imageFullName string) {

agent-go/executor/BaseFunction.go

@@ -336,9 +336,9 @@ func (op *AgentOsOperator) disableSELinuxExec() (bool, []string) {
 	}
 
 	// delete contend
-	BasicFindDeleteContendLineInFile("SELINUX=enforcing", "/etc/selinux/config")
-	BasicFindDeleteContendLineInFile("SELINUX=permissive", "/etc/selinux/config")
-	BasicFindDeleteContendLineInFile("SELINUX=disabled", "/etc/selinux/config")
+	BasicFindAndDeleteContendLineInFile("SELINUX=enforcing", "/etc/selinux/config")
+	BasicFindAndDeleteContendLineInFile("SELINUX=permissive", "/etc/selinux/config")
+	BasicFindAndDeleteContendLineInFile("SELINUX=disabled", "/etc/selinux/config")
 
 	BasicAppendContentToFile("SELINUX=disabled", "/etc/selinux/config")
 

agent-go/executor/BasicFunction.go

@@ -150,7 +150,7 @@ func BasicFindContentInFile(content string, fileName string) bool {
 	return false
 }
 
-func BasicFindDeleteContendLineInFile(content string, fileName string) bool {
+func BasicFindAndDeleteContendLineInFile(content string, fileName string) bool {
 
 	// Open the file
 	file, err := os.Open(fileName)
@@ -196,6 +196,38 @@ func BasicFindDeleteContendLineInFile(content string, fileName string) bool {
 	return true
 }
 
+func BasicFindContentInCommandOutput(hardCodeCommand string, content string) (bool, []string) {
+	ok, resultLog := HardCodeCommandExecutor(hardCodeCommand)
+	if !ok {
+		log.ErrorF("[BasicFindContentInCommandOutput] - command error ! => %v", resultLog)
+		return false, nil
+	}
+
+	return BasicFindContentInList(content, resultLog), resultLog
+}
+
+func BasicFindContentInList(content string, list []string) bool {
+	for _, item := range list {
+		if strings.Contains(item, content) {
+			return true
+		}
+	}
+	return false
+}
+
+func BasicDockerImageExistByFullName(imageFullName string) bool {
+
+	var imageVersion string
+
+	split := strings.Split(imageFullName, ":")
+	imageVersion = split[len(split)-1]
+
+	imageFullName = strings.TrimSuffix(imageFullName, ":"+imageVersion)
+
+	//log.DebugF("[BasicDockerImageExistByFullName] -- %s  %s", imageFullName, imageVersion)
+	return BasicDockerImageExists(imageFullName, imageVersion)
+}
+
 func BasicDockerImageExists(imageName, imageVersion string) bool {
 
 	if !BasicCommandExistByPath("docker") {
@@ -324,22 +356,33 @@ func BasicFolderExists(folderName string) bool {
 }
 
 func BasicCreateFolder(folderName string) bool {
-	cmd := exec.Command("test", "-d", folderName)
-	err := cmd.Run()
-	if err == nil {
-		log.DebugF("folder %s already exists !", folderName)
+	// 检查目录是否存在
+	if _, err := os.Stat(folderName); os.IsNotExist(err) {
+		// 目录不存在，创建目录
+		err := os.MkdirAll(folderName, os.ModePerm)
+		if err != nil {
+			// 如果创建目录失败，返回false
+			return false
+		}
+	}
+	// 目录存在或者成功创建，返回true
 	return true
+}
+
+func BasicRemoveFileOrFolder(folderName string) bool {
+	// Check if the folder exists
+	if _, err := os.Stat(folderName); os.IsNotExist(err) {
+		return true // folder does not exist
 	}
 
-	cmd2 := exec.Command("mkdir", "-p", folderName)
-	err = cmd2.Run()
+	// Folder exists, remove it
+	err := os.Remove(folderName)
 	if err != nil {
-		log.ErrorF("folder %s create error!", folderName)
+		// Handle error
 		return false
-	} else {
-		log.DebugF("folder %s create success!", folderName)
-		return true
 	}
+
+	return true // folder was removed
 }
 
 // BasicPrettyPrint 打印执行结果

agent-go/executor/BasicFunction_test.go

@@ -66,3 +66,43 @@ func TestBasicCommandExists(t *testing.T) {
 	t.Logf("command exists is => %s", strconv.FormatBool(exists))
 
 }
+
+func TestBasicDockerImageExistByFullName(t *testing.T) {
+
+	// Test cases
+	testCases := []struct {
+		name           string
+		imageFullName  string
+		expectedResult bool
+	}{
+		{
+			name:           "Image exists with full name and version",
+			imageFullName:  "harbor.cdcyy.com.cn/cmii/busybox:0326",
+			expectedResult: true,
+		},
+		{
+			name:           "Image does not exist with full name and version",
+			imageFullName:  "nonexistentImage:latest",
+			expectedResult: false,
+		},
+		{
+			name:           "Image exists with full name but no version",
+			imageFullName:  "harbor.cdcyy.com.cn/cmii/busybox",
+			expectedResult: true,
+		},
+		{
+			name:           "Image does not exist with full name but no version",
+			imageFullName:  "nonexistentImage",
+			expectedResult: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := BasicDockerImageExistByFullName(tc.imageFullName)
+			if result != tc.expectedResult {
+				t.Errorf("Expected %t for imageFullName %s, but got %t", tc.expectedResult, tc.imageFullName, result)
+			}
+		})
+	}
+}

agent-go/executor/FunctionalExecutor.go

@@ -232,7 +232,7 @@ func collectOutput(r io.Reader, resultSlice []string) []string {
 }
 
 // HardCodeCommandExecutor 执行硬编码的shell命令，如 echo sda > sdasd ;  rpcinfo -p localhost 等
-func HardCodeCommandExecutor(hardCodeCommand string) (bool, []string) {
+func HardCodeCommandExecutor(hardCodeCommand string) (ok bool, resultLog []string) {
 
 	// result
 	var resultSlice []string
@@ -264,7 +264,7 @@ func HardCodeCommandExecutor(hardCodeCommand string) (bool, []string) {
 		resultOk = false
 	}
 
-	log.DebugF("hard code command of [ %s ] result are =>  %v", hardCodeCommand, resultSlice)
+	//log.DebugF("hard code command of [ %s ] result are =>  %v", hardCodeCommand, resultSlice)
 
 	return resultOk, resultSlice
 }

agent-go/executor/FunctionalExecutor_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"reflect"
 	"strings"
 	"testing"
 )
@@ -133,6 +134,47 @@ func TestPipelineCommandExecutor(t *testing.T) {
 	BasicPrettyPrint(ok, resultLog)
 }
 
+func TestPipelineCommandExecutor2(t *testing.T) {
+	// Test case 1: Empty pipeline
+	pipelineCommand := [][]string{}
+	ok, log := PipelineCommandExecutor(pipelineCommand)
+	if !ok {
+		t.Errorf("Expected success, got failure")
+	}
+	if len(log) != 0 {
+		t.Errorf("Expected no log, got %v", log)
+	}
+
+	// Test case 2: Single command in pipeline
+	pipelineCommand = [][]string{
+		{"echo", "Hello, world!"},
+	}
+	ok, log = PipelineCommandExecutor(pipelineCommand)
+	if !ok {
+		t.Errorf("Expected success, got failure")
+	}
+	expectedLog := []string{"Hello, world!"}
+	if !reflect.DeepEqual(log, expectedLog) {
+		t.Errorf("Expected log %v, got %v", expectedLog, log)
+	}
+
+	// Test case 3: Multiple commands in pipeline
+	pipelineCommand = [][]string{
+		{"echo", "Hello"},
+		{"echo", "world!"},
+	}
+	ok, log = PipelineCommandExecutor(pipelineCommand)
+	if !ok {
+		t.Errorf("Expected success, got failure")
+	}
+	expectedLog = []string{"Hello", "world!"}
+	if !reflect.DeepEqual(log, expectedLog) {
+		t.Errorf("Expected log %v, got %v", expectedLog, log)
+	}
+
+	// Add more test cases as needed...
+}
+
 func TestSimple(t *testing.T) {
 
 	for i := 0; i < 10; i++ {

agent-go/executor/ImageFunction.go

@@ -51,6 +51,12 @@ func (op *AgentOsOperator) downloadDockerImage(funcArgs []string) (bool, []strin
 	}
 	imageFullName := funcArgs[0]
 	log.InfoF("[downloadDockerImage]- start to pull docker image %s", imageFullName)
+
+	// login
+	if strings.HasPrefix(imageFullName, image.CmiiHarborPrefix) {
+		HardCodeCommandExecutor("docker login -u rad02_drone -p Drone@1234 harbor.cdcyy.com.cn")
+	}
+
 	if !PureResultSingleExecute([]string{
 		"docker",
 		"pull",
@@ -61,7 +67,7 @@ func (op *AgentOsOperator) downloadDockerImage(funcArgs []string) (bool, []strin
 		}
 	}
 
-	if !JudgeDockerImageExists(funcArgs[0]) {
+	if !BasicDockerImageExistByFullName(funcArgs[0]) {
 		return false, []string{
 			"image not exits ! unknown error happened!",
 		}
@@ -89,7 +95,7 @@ func (op *AgentOsOperator) compressImageToGzip(funcArgs []string) (bool, []strin
 	}
 
 	imageFullName := funcArgs[0]
-	if JudgeDockerImageExists(imageFullName) {
+	if !BasicDockerImageExistByFullName(imageFullName) {
 		return false, []string{
 			"image not exits !",
 		}
@@ -129,18 +135,35 @@ func (op *AgentOsOperator) uploadGzipFileToOss(funcArgs []string) (bool, []strin
 	gzipFolderPrefix := funcArgs[1]
 	gzipImageFromFullName := funcArgs[2]
 
-	ok, resultLog := AllCommandExecutor([]string{
+	ok, resultLog := HardCodeCommandExecutor("mc --insecure alias set demo https://oss.ig-demo.uavcmlc.com cmii B#923fC7mk")
+	//ok, resultLog = HardCodeCommandExecutor("mc alias list")
+
+	PureResultSingleExecute([]string{
+		"mc",
+		"rm",
+		"demo/cmlc-installation/tmp/" + gzipImageFromFullName,
+	})
+
+	ok, resultLog = AllCommandExecutor([]string{
 		"mc",
 		"cp",
 		gzipFolderPrefix + gzipImageFromFullName,
-		"demo/cmlc-installation/tmp/",
+		"demo/cmlc-installation/tmp/" + gzipImageFromFullName,
 	})
-
 	if !ok {
 		return false, resultLog
 	}
 
-	return true, nil
+	find, _ := BasicFindContentInCommandOutput("mc ls demo/cmlc-installation/tmp/", gzipImageFromFullName)
+	if !find {
+		return false, []string{
+			"demo oss can't find gzip file !",
+		}
+	}
+
+	return true, []string{
+		gzipImageFromFullName,
+	}
 
 }
 
@@ -150,13 +173,33 @@ func (op *AgentOsOperator) downloadGzipImageFile(funcArgs []string) (bool, []str
 	gzipImageFromFullName := funcArgs[2]
 	proxyUrl := funcArgs[4]
 
-	if proxyUrl == "" {
-		BasicDownloadFile(ossUrlPrefix+gzipImageFromFullName, LocalGzipImageFolderPrefix+gzipImageFromFullName)
-	} else {
-		BasicDownloadFileWithProxy(ossUrlPrefix+gzipImageFromFullName, proxyUrl, LocalGzipImageFolderPrefix+gzipImageFromFullName)
+	// create folder
+	BasicCreateFolder(LocalGzipImageFolderPrefix)
+
+	// remove file
+	desFile := LocalGzipImageFolderPrefix + gzipImageFromFullName
+
+	if !BasicRemoveFileOrFolder(desFile) {
+		return false, []string{
+			"file already exits ! can't remove it!",
+		}
 	}
 
-	return true, nil
+	var download bool
+	var downloadLog []string
+	if proxyUrl == "" {
+		download, downloadLog = BasicDownloadFile(ossUrlPrefix+gzipImageFromFullName, desFile)
+	} else {
+		download, downloadLog = BasicDownloadFileWithProxy(ossUrlPrefix+gzipImageFromFullName, proxyUrl, desFile)
+	}
+
+	if !download {
+		return false, downloadLog
+	}
+
+	return true, []string{
+		desFile,
+	}
 }
 
 func (op *AgentOsOperator) loadDockerImageFromGzip(funcArgs []string) (bool, []string) {
@@ -175,6 +218,13 @@ func (op *AgentOsOperator) loadDockerImageFromGzip(funcArgs []string) (bool, []s
 		return false, i
 	}
 
+	if !BasicDockerImageExistByFullName(funcArgs[0]) {
+		return false, []string{
+			"docker load from gzip file error ! image not exits!",
+			funcArgs[0],
+		}
+	}
+
 	return true, nil
 }
 func (op *AgentOsOperator) pushImageToTargetHarbor(funcArgs []string) (bool, []string) {
@@ -198,8 +248,11 @@ func (op *AgentOsOperator) pushImageToTargetHarbor(funcArgs []string) (bool, []s
 			"docker tag error!",
 		}
 	}
-
+	if strings.HasPrefix(targetImageFullName, image.CmiiHarborPrefix) {
+		HardCodeCommandExecutor("docker login -u rad02_drone -p Drone@1234 harbor.cdcyy.com.cn")
+	} else {
 		HardCodeCommandExecutor("docker login -u admin -p V2ryStr@ngPss " + targetHarborHost)
+	}
 
 	ok, resultLog := AllCommandExecutor([]string{
 		"docker",
@@ -240,10 +293,6 @@ func (op *AgentOsOperator) updateImageTag(funcArgs []string) (bool, []string) {
 	return true, nil
 }
 
-func JudgeDockerImageExists(imageFullName string) bool {
-	return true
-}
-
 //func BuildGzipImageFromFullName(imageFullName string) string {
 //
 //}

agent-go/executor/ImageFunction_test.go

@@ -12,14 +12,14 @@ func TestAgentOsOperator_Sync(t *testing.T) {
 	// imageFullName gzipFolderPrefix gzipFileName ossUrlPrefix proxyUrl targetHarborHost namespace targetImageFullName
 	funcArgs := []string{
 		"harbor.cdcyy.com.cn/cmii/busybox:0326",
-		"/root/octopus_image/",
-		"",
-		"https://oss.demo.uavcmlc.com/cmlc-installation/tmp/",
-		"",
+		"/var/lib/docker/octopus_image/",
+		"cmlc=cmii=busybox=0326.tar.gz",
+		"https://oss.demo.uavcmlc.com:18000/cmlc-installation/tmp/",
 		"",
+		"10.250.0.100",
 		"",
 	}
-	sync, strings := agentOP.Sync("DOWNLOAD_DOCKER_IMAGE", funcArgs...)
+	sync, strings := agentOP.Sync("PUSH_IMAGE_TO_TARGET_HARBOR", funcArgs...)
 	fmt.Println("func result are => " + strconv.FormatBool(sync))
 	utils.BeautifulPrint(strings)
 }

agent-operator/CmiiK8sOperator.go

@@ -4,8 +4,8 @@ import (
 	"bufio"
 	"strings"
 	"time"
+	image2 "wdd.io/agent-common/image"
 	"wdd.io/agent-common/utils"
-	"wdd.io/agent-operator/image"
 )
 
 var CmiiOperator = CmiiK8sOperator{}
@@ -470,8 +470,8 @@ func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap, srsM
 			if app != nil {
 				for _, imageName := range app.ContainerImageMap {
 					split := strings.Split(imageName, ":")
-					if strings.Contains(split[0], image.CmiiHarborPrefix) {
-						split[0] = strings.Split(split[0], image.CmiiHarborPrefix)[1]
+					if strings.Contains(split[0], image2.CmiiHarborPrefix) {
+						split[0] = strings.Split(split[0], image2.CmiiHarborPrefix)[1]
 					}
 					srsMap[split[0]] = split[1]
 				}
@@ -481,7 +481,7 @@ func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap, srsM
 			if app != nil {
 				for _, imageName := range app.ContainerImageMap {
 					split := strings.Split(imageName, ":")
-					split[0], _ = strings.CutPrefix(split[0], image.CmiiHarborPrefix)
+					split[0], _ = strings.CutPrefix(split[0], image2.CmiiHarborPrefix)
 					srsMap[split[0]] = split[1]
 				}
 			}

agent-operator/CmiiK8sOperator_test.go

@@ -243,9 +243,9 @@ func TestRestartCmiiDeployment(t *testing.T) {
 
 func TestUpdateCmiiDeploymentImageTag(t *testing.T) {
 
-	cmiiEnv := demo
-	appName := "cmii-uav-process"
-	newTag := "5.4.0-032201"
+	cmiiEnv := integration
+	appName := "cmii-uav-platform"
+	newTag := "5.4.0-032601"
 
 	tag := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag)
 	assert.Equal(t, tag, true, "update image tag failed !")

agent-operator/CmiiOperator.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"slices"
 	"strings"
+	image2 "wdd.io/agent-common/image"
 	"wdd.io/agent-common/utils"
 	"wdd.io/agent-operator/image"
 )
@@ -175,17 +176,17 @@ func FetchDemoImages(projectName string, gzipSplit bool) (errorPullImageList, er
 	if gzipSplit {
 		for image_name, tag := range backendMap {
 			if !image.SaveToTarGZ(image_name+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") {
-				errorGzipImageList = append(errorGzipImageList, image.CmiiHarborPrefix+image_name+":"+tag)
+				errorGzipImageList = append(errorGzipImageList, image2.CmiiHarborPrefix+image_name+":"+tag)
 			}
 		}
 		for image_name, tag := range frontendMap {
 			if !image.SaveToTarGZ(image_name+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") {
-				errorGzipImageList = append(errorGzipImageList, image.CmiiHarborPrefix+image_name+":"+tag)
+				errorGzipImageList = append(errorGzipImageList, image2.CmiiHarborPrefix+image_name+":"+tag)
 			}
 		}
 		for image_name, tag := range srsMap {
 			if !image.SaveToTarGZ(image_name+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") {
-				errorGzipImageList = append(errorGzipImageList, image.CmiiHarborPrefix+image_name+":"+tag)
+				errorGzipImageList = append(errorGzipImageList, image2.CmiiHarborPrefix+image_name+":"+tag)
 			}
 		}
 	}

agent-operator/K8sOperator.go

@@ -15,9 +15,9 @@ import (
 	"strings"
 	"sync"
 	"time"
+	image2 "wdd.io/agent-common/image"
 	"wdd.io/agent-common/logger"
 	"wdd.io/agent-common/utils"
-	"wdd.io/agent-operator/image"
 )
 
 var log = logger.Log
@@ -407,7 +407,7 @@ func (op *CmiiK8sOperator) DeploymentUpdateTag(cmiiEnv, appName, newTag string)
 		oldName := container.Image
 
 		split := strings.Split(container.Image, ":")
-		if strings.HasPrefix(container.Image, image.CmiiHarborPrefix) {
+		if strings.HasPrefix(container.Image, image2.CmiiHarborPrefix) {
 			// harbor
 			container.Image = split[0] + ":" + newTag
 		} else if strings.Contains(container.Image, "8033") {

cmii_operator/log/cmii-update-log.txt

@@ -15,3 +15,5 @@
 2024-03-22-14-02-05 uavcloud-demo  cmii-uav-platform                       5.4.0-25263-032011            5.4.0-25263-emerg-ai-032201
 2024-03-22-17-14-43 uavcloud-demo  cmii-uav-platform-armypeople            5.4.0                         5.4.0-26860-032201
 2024-03-22-17-17-15 uavcloud-demo  cmii-uav-process                        5.4.0                         5.4.0-032201
+2024-03-26-16-27-58 uavcloud-demo  cmii-uav-multilink                      5.4.0                         5.4.0-032602
+2024-03-26-17-11-29 uavcloud-test  cmii-uav-platform                       5.4.0-032002                  5.4.0-032601