zeaslity/ProjectOctopus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[ Cmii ] [ Octopus ] - add Image Function - 1

Browse Source
This commit is contained in:
zeaslity
2024-03-26 14:07:51 +08:00
parent 2af05c903c
commit 20ba7a9f38
8 changed files with 376 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
agent-operator/image/ImageNameConvert.go → agent-common/image/ImageNameConvert.go
View File

@@ -1,9 +1,13 @@
package image
import "strings"
import (
"strings"
)
var unSupported = "UN-SUPPORT-APP-NAME"
const CmiiHarborPrefix = "harbor.cdcyy.com.cn/cmii/"
func ImageFullNameToAppName(imageFullName string) (appName string) {
// harbor.cdcyy.cn/cmii/cmii-uav-platform:5.4.0 ==> cmii-uav-platform
@@ -95,6 +99,11 @@ func ImageFullNameToGzipFileName(imageFullName string) (gzipFileName string) {
return gzipFileName
}
func ImageFullNameToTargetImageFullName(imageFullName, targetHost string) string {
return imageFullName
}
func GzipFileNameToImageFullName(gzipFileName string) (imageFullName string) {
return gzipFileName

40
agent-go/executor/BasicFunction.go
View File

@@ -522,6 +522,46 @@ func BasicDownloadFile(downloadUrl, desFile string) (downloadOk bool, resultLog
return true, nil
}
func BasicDownloadFileWithProxy(downloadUrl, proxyUrl, desFile string) (downloadOk bool, resultLog []string) {
var ok bool
if !BasicCommandExistByPath("curl") {
return false, []string{
"curl not exits!",
}
}
ok, resultLog = AllCommandExecutor([]string{
"curl",
"-x",
proxyUrl,
"-o",
desFile,
"--insecure",
"--max-time",
"5",
downloadUrl,
})
errLog := fmt.Sprintf("[BasicDownloadFileWithProxy] - download file [ %s ] with proxy [ %s ] from [ %s ] failed !", desFile, proxyUrl, downloadUrl)
if !ok {
log.Error(errLog)
return false, []string{
errLog,
}
}
// check file exists
existAndNotNull := BasicFileExistAndNotNull(desFile)
if !existAndNotNull {
return false, []string{
errLog,
"[BasicDownloadFile] - file not exist !",
}
}
return true, nil
}
// BasicAppendSourceToFile 将源文件的内容添加到目标文件，使用golang标准库完成，跨平台、安全性更强
func BasicAppendSourceToFile(sourceFile, targetFile string) bool {

249
agent-go/executor/ImageFunction.go Normal file
View File

@@ -0,0 +1,249 @@
package executor
import (
"strings"
"wdd.io/agent-common/image"
)
var LocalGzipImageFolderPrefix = "/var/lib/docker/image_sync/"
func (op *AgentOsOperator) Sync(baseFuncName string, funcArgs ...string) (bool, []string) {
resultOk := false
var errorLog []string
switch baseFuncName {
case "DOWNLOAD_DOCKER_IMAGE":
resultOk, errorLog = op.downloadDockerImage(funcArgs)
break
case "COMPRESS_IMAGE_TO_GZIP":
resultOk, errorLog = op.compressImageToGzip(funcArgs)
break
case "UPLOAD_GZIP_TO_OSS":
resultOk, errorLog = op.uploadGzipFileToOss(funcArgs)
break
case "DOWNLOAD_GZIP_IMAGE_FILE":
resultOk, errorLog = op.downloadGzipImageFile(funcArgs)
break
case "LOAD_DOCKER_IMAGE_FROM_GZIP":
resultOk, errorLog = op.loadDockerImageFromGzip(funcArgs)
break
case "PUSH_IMAGE_TO_TARGET_HARBOR":
resultOk, errorLog = op.pushImageToTargetHarbor(funcArgs)
break
case "UPDATE_IMAGE_TAG":
resultOk, errorLog = op.updateImageTag(funcArgs)
break
default:
resultOk, errorLog = op.okExec(funcArgs)
}
return resultOk, errorLog
}
func (op *AgentOsOperator) downloadDockerImage(funcArgs []string) (bool, []string) {
// funcArgs are imageFullName gzipFolderPrefix gzipFileName ossUrlPrefix namespace newImageTag
if !BasicCommandExistByPath("docker") {
return false, []string{
"docker not exits !",
}
}
imageFullName := funcArgs[0]
log.InfoF("[downloadDockerImage]- start to pull docker image %s", imageFullName)
if !PureResultSingleExecute([]string{
"docker",
"pull",
imageFullName,
}) {
return false, []string{
"docker pull failed of " + imageFullName,
}
}
if !JudgeDockerImageExists(funcArgs[0]) {
return false, []string{
"image not exits ! unknown error happened!",
}
}
return true, []string{
imageFullName,
}
}
func (op *AgentOsOperator) compressImageToGzip(funcArgs []string) (bool, []string) {
if !BasicCommandExistByPath("docker") {
return false, []string{
"docker not exits !",
}
}
if !BasicCommandExistByPath("gzip") {
return false, []string{
"gzip not exits !",
}
}
gzipFolderPrefix := funcArgs[1]
if !BasicFolderExists(gzipFolderPrefix) {
BasicCreateFolder(gzipFolderPrefix)
}
imageFullName := funcArgs[0]
if JudgeDockerImageExists(imageFullName) {
return false, []string{
"image not exits !",
}
}
if !strings.HasSuffix(gzipFolderPrefix, "/") {
gzipFolderPrefix += "/"
}
gzipImageFromFullName := image.ImageFullNameToGzipFileName(imageFullName)
dockerSaveCommand := "docker save " + imageFullName + " | gzip > " + gzipFolderPrefix + gzipImageFromFullName
executor, i := HardCodeCommandExecutor(dockerSaveCommand)
if !executor {
return false, i
}
if !BasicFileExistAndNotNull(gzipFolderPrefix + gzipImageFromFullName) {
return false, []string{
"gzip of ile error ",
}
}
return true, []string{
gzipImageFromFullName,
}
}
func (op *AgentOsOperator) uploadGzipFileToOss(funcArgs []string) (bool, []string) {
if !BasicCommandExistByPath("mc") {
return false, []string{
"mc not exits!",
}
}
gzipFolderPrefix := funcArgs[1]
gzipImageFromFullName := funcArgs[2]
ok, resultLog := AllCommandExecutor([]string{
"mc",
"cp",
gzipFolderPrefix + gzipImageFromFullName,
"demo/cmlc-installation/tmp/",
})
if !ok {
return false, resultLog
}
return true, nil
}
func (op *AgentOsOperator) downloadGzipImageFile(funcArgs []string) (bool, []string) {
ossUrlPrefix := funcArgs[3]
gzipImageFromFullName := funcArgs[2]
proxyUrl := funcArgs[4]
if proxyUrl == "" {
BasicDownloadFile(ossUrlPrefix+gzipImageFromFullName, LocalGzipImageFolderPrefix+gzipImageFromFullName)
} else {
BasicDownloadFileWithProxy(ossUrlPrefix+gzipImageFromFullName, proxyUrl, LocalGzipImageFolderPrefix+gzipImageFromFullName)
}
return true, nil
}
func (op *AgentOsOperator) loadDockerImageFromGzip(funcArgs []string) (bool, []string) {
gzipImageFromFullName := funcArgs[2]
if !BasicFileExistAndNotNull(LocalGzipImageFolderPrefix + gzipImageFromFullName) {
return false, []string{
LocalGzipImageFolderPrefix + gzipImageFromFullName,
"local gzip file not exits!",
}
}
hardCodeCommand := "docker load < " + LocalGzipImageFolderPrefix + gzipImageFromFullName
executor, i := HardCodeCommandExecutor(hardCodeCommand)
if !executor {
return false, i
}
return true, nil
}
func (op *AgentOsOperator) pushImageToTargetHarbor(funcArgs []string) (bool, []string) {
targetHarborHost := funcArgs[5]
imageFullName := funcArgs[0]
if !strings.Contains(targetHarborHost, "8033") {
targetHarborHost += ":8033"
}
targetImageFullName := image.ImageFullNameToTargetImageFullName(imageFullName, targetHarborHost)
if !PureResultSingleExecute([]string{
"docker",
"tag",
imageFullName,
targetImageFullName,
}) {
return false, []string{
"docker tag error!",
}
}
HardCodeCommandExecutor("docker login -u admin -p V2ryStr@ngPss " + targetHarborHost)
ok, resultLog := AllCommandExecutor([]string{
"docker",
"push",
targetImageFullName,
})
if !ok {
return false, resultLog
}
return true, []string{
targetImageFullName,
}
}
func (op *AgentOsOperator) updateImageTag(funcArgs []string) (bool, []string) {
namespace := funcArgs[6]
targetImageFullName := funcArgs[7]
if !BasicCommandExistByPath("kubectl") {
return false, []string{
"kubectl not exits !",
}
}
imageFullName := funcArgs[0]
if !strings.HasPrefix(imageFullName, image.CmiiHarborPrefix) {
return false, []string{
"cant update this image !",
}
}
appName := image.ImageFullNameToAppName(imageFullName)
updateCommand := "kubectl -n " + namespace + " patch deployment " + appName + "-p \"{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\": " + appName + ",\"image\": " + targetImageFullName + "}]}}}}"
executor, i := HardCodeCommandExecutor(updateCommand)
if !executor {
return false, i
}
return true, nil
}
func JudgeDockerImageExists(imageFullName string) bool {
return true
}
//func BuildGzipImageFromFullName(imageFullName string) string {
//
//}

50
agent-go/tmp/test.go
View File

@@ -1,51 +1 @@
package main
import "fmt"
// quic sort s a slice of ints using quicksort algorithm.
// The function returns nothing. It modifies the slice in place.
func QuickSort(a []int) { // <-- pass by value, so we can't modify it
quick_sort(0, len(a)-1, a)
}
/*
* The following function implements the quicksort algorithm. It requires three parameters:
* left: an index indicating the left bound of the subarray to be sorted.
* right: an index indicating the right bound of the subarray to be sorted.
* a: the slice of integers to sort. The function modifies the slice in place.
*/
func quick_sort(left int, right int, a []int) {
if left < right { // only continue if there is more than one element in the subarray
pivot := partition(left, right, a)
quick_sort(left, pivot-1, a)
quick_sort(pivot+1, right, a) // <-- call this function again on the right half of the subarray
} else { // if there is only one element in the subarray, we can return immediately
return
}
}
func partition(left int, right int, a []int) int {
// find a pivot element. This is just one of the examples in the book
pivot := a[right]
// index i will keep track of where to put the pivot element
i := left - 1
// swap the pivot element with a[right]
a[right], a[left] = a[left], a[right]
for j := left; j < right; j++ {
if a[j] <= pivot {
i += 1 // increment i
a[i], a[j] = a[j], a[i]
}
}
return i + 1 // put the pivot element at its final position
}
func main() {
arr := []int{2, 45, 36, -78, 90, 1000, -1, 0, -76}
QuickSort(arr)
fmt.Println("Sorted array:", arr)
}

21
agent-operator/CmiiOperator.go
View File

@@ -17,8 +17,9 @@ const PublicDeployHarborHost = "42.192.52.227"
const DirectPushDeployHarborHost = "36.134.71.138"
type ImageSyncEntity struct {
ProjectName string
ProjectVersion string
ProjectName string // 3
ProjectVersion string // 2
CmiiImageList map[string]string // 1
DirectHarborHost string //此参数决定是否能够直连目标主机，如果有则代表直连，可以直接推送景象
PushToDemoMinio bool
}
@@ -32,7 +33,7 @@ type ImageSyncResult struct {
AllCmiiImageNameList []string
}
// PullFromEntityAndSyncConditionally 根据ImageSyncEntity拉取特定的镜像，然后上传到特定的目标机器
// PullFromEntityAndSyncConditionally 根据ImageSyncEntity拉取特定的镜像，然后上传到特定的目标机器(或者上传的minio中)
func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResult ImageSyncResult) {
var realCmiiImageList []string
@@ -122,20 +123,6 @@ func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResul
return imageSyncResult
}
func remove(s1, s2 []string) []string {
m := make(map[string]struct{}, len(s2))
for _, v := range s2 {
m[v] = struct{}{}
}
res := make([]string, 0, len(s1))
for _, v := range s1 {
if _, ok := m[v]; !ok {
res = append(res, v)
}
}
return res
}
func FetchDemoImages(projectName string, gzipSplit bool) (errorPullImageList, errorGzipImageList, allCmiiImageName []string) {
// generate a project folder

17
agent-operator/image/CmiiImageSync.go
View File

@@ -15,6 +15,7 @@ import (
"os"
"strconv"
"strings"
image2 "wdd.io/agent-common/image"
"wdd.io/agent-common/logger"
"wdd.io/agent-common/utils"
)
@@ -22,8 +23,6 @@ import (
var apiClient = newClient()
var log = logger.Log
const CmiiHarborPrefix = "harbor.cdcyy.com.cn/cmii/"
func newClient() *client.Client {
apiClient, err := client.NewClientWithOpts(client.FromEnv)
@@ -140,7 +139,7 @@ func PruneAllCmiiImages() (errorRemoveImageNameList []string) {
for _, imageSummary := range imageGetAll {
for _, repoTag := range imageSummary.RepoTags {
if strings.HasPrefix(repoTag, CmiiHarborPrefix) {
if strings.HasPrefix(repoTag, image2.CmiiHarborPrefix) {
for _, tag := range imageSummary.RepoTags {
_, err := apiClient.ImageRemove(context.TODO(), imageSummary.ID, types.ImageRemoveOptions{
Force: true,
@@ -209,7 +208,7 @@ func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost s
cmiiImageFullName := imageName
if strings.HasPrefix(imageName, "cmii") {
cmiiImageFullName = CmiiHarborPrefix + imageName
cmiiImageFullName = image2.CmiiHarborPrefix + imageName
}
targetProject := "cmii"
@@ -217,9 +216,9 @@ func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost s
targetProject = "rancher"
}
if strings.HasPrefix(imageName, CmiiHarborPrefix) {
if strings.HasPrefix(imageName, image2.CmiiHarborPrefix) {
//
imageName = strings.TrimPrefix(imageName, CmiiHarborPrefix)
imageName = strings.TrimPrefix(imageName, image2.CmiiHarborPrefix)
} else {
// todo
if strings.Contains(imageName, "/") {
@@ -355,7 +354,7 @@ func PullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string)
errorGzipImageList = append(errorGzipImageList, image)
continue
}
tarGzipFileNameList = append(tarGzipFileNameList, ImageFullNameToGzipFileName(image))
tarGzipFileNameList = append(tarGzipFileNameList, image2.ImageFullNameToGzipFileName(image))
}
utils.BeautifulPrintListWithTitle(tarGzipFileNameList, "image gzip name list")
@@ -431,7 +430,7 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool {
}
}
gzipImageFile := ImageFullNameToGzipFileName(realImageTag)
gzipImageFile := image2.ImageFullNameToGzipFileName(realImageTag)
if !strings.HasSuffix(folderPathPrefix, "/") {
folderPathPrefix += "/"
}
@@ -466,7 +465,7 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool {
func ConvertCMiiImageMapToList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) {
for image, tag := range cmiiImageVersionMap {
s := CmiiHarborPrefix + image + ":" + tag
s := image2.CmiiHarborPrefix + image + ":" + tag
fullImageNameList = append(fullImageNameList, s)
}

3
agent-operator/image/CmiiImageSync_test.go
View File

@@ -6,6 +6,7 @@ import (
"strings"
"testing"
"wdd.io/agent-common/assert"
"wdd.io/agent-common/image"
"wdd.io/agent-common/utils"
)
@@ -130,7 +131,7 @@ func TestImageFullNameToGzipFileName(t *testing.T) {
}
for _, s := range test {
gzipFileName := ImageFullNameToGzipFileName(s)
gzipFileName := image.ImageFullNameToGzipFileName(s)
fmt.Printf(" %s to %s \n", s, gzipFileName)
}
}

62
server/src/main/java/io/wdd/func/auto/beans/ImageFunctionEnum.java Normal file
View File

@@ -0,0 +1,62 @@
package io.wdd.func.auto.beans;
/**
* 参数固定顺序为
* imageFullName gzipFolderPrefix gzipFileName ossUrlPrefix proxyUrl targetHarborHost namespace targetImageFullName
*/
public enum ImageFunctionEnum {
DOWNLOAD_DOCKER_IMAGE(
"DOWNLOAD_DOCKER_IMAGE",
"args imageFullName return imageFullName"
),
COMPRESS_IMAGE_TO_GZIP(
"COMPRESS_IMAGE_TO_GZIP",
"args imageFullName return gzipFileName"
),
UPLOAD_GZIP_TO_OSS(
"UPLOAD_GZIP_TO_OSS",
"关闭防火墙"
),
DOWNLOAD_GZIP_IMAGE_FILE(
"DOWNLOAD_GZIP_IMAGE_FILE",
"关闭防火墙"
),
LOAD_DOCKER_IMAGE_FROM_GZIP(
"LOAD_DOCKER_IMAGE_FROM_GZIP",
"关闭防火墙"
),
PUSH_IMAGE_TO_TARGET_HARBOR(
"PUSH_IMAGE_TO_TARGET_HARBOR",
"关闭防火墙"
),
UPDATE_IMAGE_TAG(
"UPDATE_IMAGE_TAG",
"关闭防火墙"
);
String funcName;
String desc;
ImageFunctionEnum(String funcName, String desc) {
this.funcName = funcName;
this.desc = desc;
}
public String getFuncName() {
return funcName;
}
public String getDesc() {
return desc;
}
}