[ Cmii ] [ Octopus ] - add Image Function - 1

agent-common/image/ImageNameConvert.go

@@ -1,9 +1,13 @@
 package image
 
-import "strings"
+import (
+	"strings"
+)
 
 var unSupported = "UN-SUPPORT-APP-NAME"
 
+const CmiiHarborPrefix = "harbor.cdcyy.com.cn/cmii/"
+
 func ImageFullNameToAppName(imageFullName string) (appName string) {
 
 	// harbor.cdcyy.cn/cmii/cmii-uav-platform:5.4.0 ==> cmii-uav-platform
@@ -95,6 +99,11 @@ func ImageFullNameToGzipFileName(imageFullName string) (gzipFileName string) {
 	return gzipFileName
 }
 
+func ImageFullNameToTargetImageFullName(imageFullName, targetHost string) string {
+
+	return imageFullName
+}
+
 func GzipFileNameToImageFullName(gzipFileName string) (imageFullName string) {
 
 	return gzipFileName

agent-go/executor/BasicFunction.go

@@ -522,6 +522,46 @@ func BasicDownloadFile(downloadUrl, desFile string) (downloadOk bool, resultLog
 	return true, nil
 }
 
+func BasicDownloadFileWithProxy(downloadUrl, proxyUrl, desFile string) (downloadOk bool, resultLog []string) {
+
+	var ok bool
+	if !BasicCommandExistByPath("curl") {
+		return false, []string{
+			"curl not exits!",
+		}
+	}
+
+	ok, resultLog = AllCommandExecutor([]string{
+		"curl",
+		"-x",
+		proxyUrl,
+		"-o",
+		desFile,
+		"--insecure",
+		"--max-time",
+		"5",
+		downloadUrl,
+	})
+
+	errLog := fmt.Sprintf("[BasicDownloadFileWithProxy] - download file [ %s ] with proxy [ %s ] from [ %s ] failed !", desFile, proxyUrl, downloadUrl)
+	if !ok {
+		log.Error(errLog)
+		return false, []string{
+			errLog,
+		}
+	}
+	// check file exists
+	existAndNotNull := BasicFileExistAndNotNull(desFile)
+	if !existAndNotNull {
+		return false, []string{
+			errLog,
+			"[BasicDownloadFile] - file not exist !",
+		}
+	}
+
+	return true, nil
+}
+
 // BasicAppendSourceToFile 将源文件的内容添加到目标文件，使用golang标准库完成，跨平台、安全性更强
 func BasicAppendSourceToFile(sourceFile, targetFile string) bool {
 

agent-go/executor/ImageFunction.go

@@ -0,0 +1,249 @@
+package executor
+
+import (
+	"strings"
+	"wdd.io/agent-common/image"
+)
+
+var LocalGzipImageFolderPrefix = "/var/lib/docker/image_sync/"
+
+func (op *AgentOsOperator) Sync(baseFuncName string, funcArgs ...string) (bool, []string) {
+	resultOk := false
+	var errorLog []string
+
+	switch baseFuncName {
+
+	case "DOWNLOAD_DOCKER_IMAGE":
+		resultOk, errorLog = op.downloadDockerImage(funcArgs)
+		break
+	case "COMPRESS_IMAGE_TO_GZIP":
+		resultOk, errorLog = op.compressImageToGzip(funcArgs)
+		break
+	case "UPLOAD_GZIP_TO_OSS":
+		resultOk, errorLog = op.uploadGzipFileToOss(funcArgs)
+		break
+	case "DOWNLOAD_GZIP_IMAGE_FILE":
+		resultOk, errorLog = op.downloadGzipImageFile(funcArgs)
+		break
+	case "LOAD_DOCKER_IMAGE_FROM_GZIP":
+		resultOk, errorLog = op.loadDockerImageFromGzip(funcArgs)
+		break
+	case "PUSH_IMAGE_TO_TARGET_HARBOR":
+		resultOk, errorLog = op.pushImageToTargetHarbor(funcArgs)
+		break
+	case "UPDATE_IMAGE_TAG":
+		resultOk, errorLog = op.updateImageTag(funcArgs)
+		break
+	default:
+		resultOk, errorLog = op.okExec(funcArgs)
+	}
+
+	return resultOk, errorLog
+}
+
+func (op *AgentOsOperator) downloadDockerImage(funcArgs []string) (bool, []string) {
+	// funcArgs are imageFullName gzipFolderPrefix gzipFileName ossUrlPrefix namespace newImageTag
+
+	if !BasicCommandExistByPath("docker") {
+		return false, []string{
+			"docker not exits !",
+		}
+	}
+	imageFullName := funcArgs[0]
+	log.InfoF("[downloadDockerImage]- start to pull docker image %s", imageFullName)
+	if !PureResultSingleExecute([]string{
+		"docker",
+		"pull",
+		imageFullName,
+	}) {
+		return false, []string{
+			"docker pull failed of " + imageFullName,
+		}
+	}
+
+	if !JudgeDockerImageExists(funcArgs[0]) {
+		return false, []string{
+			"image not exits ! unknown error happened!",
+		}
+	}
+	return true, []string{
+		imageFullName,
+	}
+}
+
+func (op *AgentOsOperator) compressImageToGzip(funcArgs []string) (bool, []string) {
+	if !BasicCommandExistByPath("docker") {
+		return false, []string{
+			"docker not exits !",
+		}
+	}
+	if !BasicCommandExistByPath("gzip") {
+		return false, []string{
+			"gzip not exits !",
+		}
+	}
+
+	gzipFolderPrefix := funcArgs[1]
+	if !BasicFolderExists(gzipFolderPrefix) {
+		BasicCreateFolder(gzipFolderPrefix)
+	}
+
+	imageFullName := funcArgs[0]
+	if JudgeDockerImageExists(imageFullName) {
+		return false, []string{
+			"image not exits !",
+		}
+	}
+
+	if !strings.HasSuffix(gzipFolderPrefix, "/") {
+		gzipFolderPrefix += "/"
+	}
+
+	gzipImageFromFullName := image.ImageFullNameToGzipFileName(imageFullName)
+	dockerSaveCommand := "docker save " + imageFullName + " | gzip > " + gzipFolderPrefix + gzipImageFromFullName
+
+	executor, i := HardCodeCommandExecutor(dockerSaveCommand)
+	if !executor {
+		return false, i
+	}
+
+	if !BasicFileExistAndNotNull(gzipFolderPrefix + gzipImageFromFullName) {
+		return false, []string{
+			"gzip of ile error ",
+		}
+	}
+
+	return true, []string{
+		gzipImageFromFullName,
+	}
+}
+
+func (op *AgentOsOperator) uploadGzipFileToOss(funcArgs []string) (bool, []string) {
+
+	if !BasicCommandExistByPath("mc") {
+		return false, []string{
+			"mc not exits!",
+		}
+	}
+
+	gzipFolderPrefix := funcArgs[1]
+	gzipImageFromFullName := funcArgs[2]
+
+	ok, resultLog := AllCommandExecutor([]string{
+		"mc",
+		"cp",
+		gzipFolderPrefix + gzipImageFromFullName,
+		"demo/cmlc-installation/tmp/",
+	})
+
+	if !ok {
+		return false, resultLog
+	}
+
+	return true, nil
+
+}
+
+func (op *AgentOsOperator) downloadGzipImageFile(funcArgs []string) (bool, []string) {
+
+	ossUrlPrefix := funcArgs[3]
+	gzipImageFromFullName := funcArgs[2]
+	proxyUrl := funcArgs[4]
+
+	if proxyUrl == "" {
+		BasicDownloadFile(ossUrlPrefix+gzipImageFromFullName, LocalGzipImageFolderPrefix+gzipImageFromFullName)
+	} else {
+		BasicDownloadFileWithProxy(ossUrlPrefix+gzipImageFromFullName, proxyUrl, LocalGzipImageFolderPrefix+gzipImageFromFullName)
+	}
+
+	return true, nil
+}
+
+func (op *AgentOsOperator) loadDockerImageFromGzip(funcArgs []string) (bool, []string) {
+	gzipImageFromFullName := funcArgs[2]
+
+	if !BasicFileExistAndNotNull(LocalGzipImageFolderPrefix + gzipImageFromFullName) {
+		return false, []string{
+			LocalGzipImageFolderPrefix + gzipImageFromFullName,
+			"local gzip file not exits!",
+		}
+	}
+
+	hardCodeCommand := "docker load < " + LocalGzipImageFolderPrefix + gzipImageFromFullName
+	executor, i := HardCodeCommandExecutor(hardCodeCommand)
+	if !executor {
+		return false, i
+	}
+
+	return true, nil
+}
+func (op *AgentOsOperator) pushImageToTargetHarbor(funcArgs []string) (bool, []string) {
+
+	targetHarborHost := funcArgs[5]
+	imageFullName := funcArgs[0]
+
+	if !strings.Contains(targetHarborHost, "8033") {
+		targetHarborHost += ":8033"
+	}
+
+	targetImageFullName := image.ImageFullNameToTargetImageFullName(imageFullName, targetHarborHost)
+
+	if !PureResultSingleExecute([]string{
+		"docker",
+		"tag",
+		imageFullName,
+		targetImageFullName,
+	}) {
+		return false, []string{
+			"docker tag error!",
+		}
+	}
+
+	HardCodeCommandExecutor("docker login -u admin -p V2ryStr@ngPss " + targetHarborHost)
+
+	ok, resultLog := AllCommandExecutor([]string{
+		"docker",
+		"push",
+		targetImageFullName,
+	})
+	if !ok {
+		return false, resultLog
+	}
+
+	return true, []string{
+		targetImageFullName,
+	}
+}
+func (op *AgentOsOperator) updateImageTag(funcArgs []string) (bool, []string) {
+	namespace := funcArgs[6]
+	targetImageFullName := funcArgs[7]
+	if !BasicCommandExistByPath("kubectl") {
+		return false, []string{
+			"kubectl not exits !",
+		}
+	}
+	imageFullName := funcArgs[0]
+	if !strings.HasPrefix(imageFullName, image.CmiiHarborPrefix) {
+		return false, []string{
+			"cant update this image !",
+		}
+	}
+
+	appName := image.ImageFullNameToAppName(imageFullName)
+
+	updateCommand := "kubectl -n " + namespace + " patch deployment " + appName + "-p \"{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\": " + appName + ",\"image\": " + targetImageFullName + "}]}}}}"
+	executor, i := HardCodeCommandExecutor(updateCommand)
+	if !executor {
+		return false, i
+	}
+
+	return true, nil
+}
+
+func JudgeDockerImageExists(imageFullName string) bool {
+	return true
+}
+
+//func BuildGzipImageFromFullName(imageFullName string) string {
+//
+//}

agent-go/tmp/test.go

@@ -1,51 +1 @@
 package main
-
-import "fmt"
-
-// quic sort s a slice of ints using quicksort algorithm.
-// The function returns nothing. It modifies the slice in place.
-func QuickSort(a []int) { // <-- pass by value, so we can't modify it
-	quick_sort(0, len(a)-1, a)
-}
-
-/*
- * The following function implements the quicksort algorithm. It requires three parameters:
- *   left: an index indicating   the left bound of the subarray to be sorted.
- *   right: an index indicating the right bound of the subarray to be sorted.
- *   a: the slice of integers to sort. The function modifies the slice in place.
- */
-func quick_sort(left int, right int, a []int) {
-	if left < right { // only continue if there is more than one element in the subarray
-		pivot := partition(left, right, a)
-		quick_sort(left, pivot-1, a)
-		quick_sort(pivot+1, right, a) // <-- call this function again on the right half of the subarray
-	} else { // if there is only one element in the subarray, we can return immediately
-		return
-	}
-}
-
-func partition(left int, right int, a []int) int {
-	// find a pivot element. This is just one of the examples in the book
-	pivot := a[right]
-
-	// index i will keep track of where to put the pivot element
-	i := left - 1
-
-	// swap the pivot element with a[right]
-	a[right], a[left] = a[left], a[right]
-
-	for j := left; j < right; j++ {
-		if a[j] <= pivot {
-			i += 1 // increment i
-			a[i], a[j] = a[j], a[i]
-		}
-	}
-
-	return i + 1 // put the pivot element at its final position
-}
-
-func main() {
-	arr := []int{2, 45, 36, -78, 90, 1000, -1, 0, -76}
-	QuickSort(arr)
-	fmt.Println("Sorted array:", arr)
-}

agent-operator/CmiiOperator.go

@@ -17,8 +17,9 @@ const PublicDeployHarborHost = "42.192.52.227"
 const DirectPushDeployHarborHost = "36.134.71.138"
 
 type ImageSyncEntity struct {
-	ProjectName      string
+	ProjectName      string            // 3
-	ProjectVersion   string
+	ProjectVersion   string            // 2
+	CmiiImageList    map[string]string // 1
 	DirectHarborHost string            //此参数决定是否能够直连目标主机，如果有则代表直连，可以直接推送景象
 	PushToDemoMinio  bool
 }
@@ -32,7 +33,7 @@ type ImageSyncResult struct {
 	AllCmiiImageNameList   []string
 }
 
-// PullFromEntityAndSyncConditionally 根据ImageSyncEntity拉取特定的镜像，然后上传到特定的目标机器
+// PullFromEntityAndSyncConditionally 根据ImageSyncEntity拉取特定的镜像，然后上传到特定的目标机器(或者上传的minio中)
 func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResult ImageSyncResult) {
 
 	var realCmiiImageList []string
@@ -122,20 +123,6 @@ func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResul
 	return imageSyncResult
 }
 
-func remove(s1, s2 []string) []string {
-	m := make(map[string]struct{}, len(s2))
-	for _, v := range s2 {
-		m[v] = struct{}{}
-	}
-	res := make([]string, 0, len(s1))
-	for _, v := range s1 {
-		if _, ok := m[v]; !ok {
-			res = append(res, v)
-		}
-	}
-	return res
-}
-
 func FetchDemoImages(projectName string, gzipSplit bool) (errorPullImageList, errorGzipImageList, allCmiiImageName []string) {
 
 	// generate a project folder

agent-operator/image/CmiiImageSync.go

@@ -15,6 +15,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	image2 "wdd.io/agent-common/image"
 	"wdd.io/agent-common/logger"
 	"wdd.io/agent-common/utils"
 )
@@ -22,8 +23,6 @@ import (
 var apiClient = newClient()
 var log = logger.Log
 
-const CmiiHarborPrefix = "harbor.cdcyy.com.cn/cmii/"
-
 func newClient() *client.Client {
 	apiClient, err := client.NewClientWithOpts(client.FromEnv)
 
@@ -140,7 +139,7 @@ func PruneAllCmiiImages() (errorRemoveImageNameList []string) {
 
 	for _, imageSummary := range imageGetAll {
 		for _, repoTag := range imageSummary.RepoTags {
-			if strings.HasPrefix(repoTag, CmiiHarborPrefix) {
+			if strings.HasPrefix(repoTag, image2.CmiiHarborPrefix) {
 				for _, tag := range imageSummary.RepoTags {
 					_, err := apiClient.ImageRemove(context.TODO(), imageSummary.ID, types.ImageRemoveOptions{
 						Force:         true,
@@ -209,7 +208,7 @@ func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost s
 		cmiiImageFullName := imageName
 
 		if strings.HasPrefix(imageName, "cmii") {
-			cmiiImageFullName = CmiiHarborPrefix + imageName
+			cmiiImageFullName = image2.CmiiHarborPrefix + imageName
 		}
 
 		targetProject := "cmii"
@@ -217,9 +216,9 @@ func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost s
 			targetProject = "rancher"
 		}
 
-		if strings.HasPrefix(imageName, CmiiHarborPrefix) {
+		if strings.HasPrefix(imageName, image2.CmiiHarborPrefix) {
 			//
-			imageName = strings.TrimPrefix(imageName, CmiiHarborPrefix)
+			imageName = strings.TrimPrefix(imageName, image2.CmiiHarborPrefix)
 		} else {
 			// todo
 			if strings.Contains(imageName, "/") {
@@ -355,7 +354,7 @@ func PullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string)
 			errorGzipImageList = append(errorGzipImageList, image)
 			continue
 		}
-		tarGzipFileNameList = append(tarGzipFileNameList, ImageFullNameToGzipFileName(image))
+		tarGzipFileNameList = append(tarGzipFileNameList, image2.ImageFullNameToGzipFileName(image))
 	}
 
 	utils.BeautifulPrintListWithTitle(tarGzipFileNameList, "image gzip name list")
@@ -431,7 +430,7 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool {
 		}
 	}
 
-	gzipImageFile := ImageFullNameToGzipFileName(realImageTag)
+	gzipImageFile := image2.ImageFullNameToGzipFileName(realImageTag)
 	if !strings.HasSuffix(folderPathPrefix, "/") {
 		folderPathPrefix += "/"
 	}
@@ -466,7 +465,7 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool {
 func ConvertCMiiImageMapToList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) {
 
 	for image, tag := range cmiiImageVersionMap {
-		s := CmiiHarborPrefix + image + ":" + tag
+		s := image2.CmiiHarborPrefix + image + ":" + tag
 		fullImageNameList = append(fullImageNameList, s)
 	}
 

agent-operator/image/CmiiImageSync_test.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"testing"
 	"wdd.io/agent-common/assert"
+	"wdd.io/agent-common/image"
 	"wdd.io/agent-common/utils"
 )
 
@@ -130,7 +131,7 @@ func TestImageFullNameToGzipFileName(t *testing.T) {
 	}
 
 	for _, s := range test {
-		gzipFileName := ImageFullNameToGzipFileName(s)
+		gzipFileName := image.ImageFullNameToGzipFileName(s)
 		fmt.Printf(" %s to %s \n", s, gzipFileName)
 	}
 }

server/src/main/java/io/wdd/func/auto/beans/ImageFunctionEnum.java

@@ -0,0 +1,62 @@
+package io.wdd.func.auto.beans;
+
+
+/**
+ * 参数固定顺序为
+ * imageFullName gzipFolderPrefix gzipFileName ossUrlPrefix proxyUrl targetHarborHost namespace targetImageFullName
+ */
+public enum ImageFunctionEnum {
+
+    DOWNLOAD_DOCKER_IMAGE(
+            "DOWNLOAD_DOCKER_IMAGE",
+            "args imageFullName return  imageFullName"
+    ),
+
+    COMPRESS_IMAGE_TO_GZIP(
+            "COMPRESS_IMAGE_TO_GZIP",
+            "args imageFullName return  gzipFileName"
+    ),
+
+    UPLOAD_GZIP_TO_OSS(
+            "UPLOAD_GZIP_TO_OSS",
+            "关闭防火墙"
+    ),
+
+    DOWNLOAD_GZIP_IMAGE_FILE(
+            "DOWNLOAD_GZIP_IMAGE_FILE",
+            "关闭防火墙"
+    ),
+
+    LOAD_DOCKER_IMAGE_FROM_GZIP(
+            "LOAD_DOCKER_IMAGE_FROM_GZIP",
+            "关闭防火墙"
+    ),
+
+    PUSH_IMAGE_TO_TARGET_HARBOR(
+            "PUSH_IMAGE_TO_TARGET_HARBOR",
+            "关闭防火墙"
+    ),
+
+    UPDATE_IMAGE_TAG(
+            "UPDATE_IMAGE_TAG",
+            "关闭防火墙"
+    );
+    String funcName;
+
+    String desc;
+
+
+    ImageFunctionEnum(String funcName, String desc) {
+        this.funcName = funcName;
+        this.desc = desc;
+    }
+
+    public String getFuncName() {
+        return funcName;
+    }
+
+    public String getDesc() {
+        return desc;
+    }
+
+}