Merge branch 'refs/heads/local-ss' into main

2024-04-28 16:55:35 +08:00
parent e7771a2153 5d80d187a5
commit ec1aed5aa7
15 changed files with 156 additions and 56 deletions
--- a/.run/agent-go
+++ b/.run/agent-go
@@ -2,9 +2,8 @@
    <configuration default="false" name="agent-go main" type="GoApplicationRunConfiguration"
                   factoryName="Go Application">
        <module name="ProjectOctopus"/>
-        <target name="LapPro-10.250.0.100"/>
        <working_directory value="$PROJECT_DIR$/agent-go"/>
-        <parameters value="-age"/>
+        <parameters value="-version=dev"/>
        <kind value="PACKAGE"/>
        <package value="wdd.io/agent-go"/>
        <directory value="$PROJECT_DIR$"/>
--- a/agent-common/utils/ReflectUtils.go
+++ b/agent-common/utils/ReflectUtils.go
@@ -0,0 +1,21 @@
+package utils
+
+import "reflect"
+
+// CopySameFields 利用反射，将a中的所有同名字段的值 复制到b中的对应字段
+func CopySameFields(source, target interface{}) {
+	va := reflect.ValueOf(source).Elem()
+	vb := reflect.ValueOf(target).Elem()
+
+	for i := 0; i < va.NumField(); i++ {
+		// 忽略source中 空值的部分
+		if va.Field(i).IsZero() {
+			continue
+		}
+
+		fieldName := va.Type().Field(i).Name
+		if vb.FieldByName(fieldName).IsValid() {
+			vb.FieldByName(fieldName).Set(va.Field(i))
+		}
+	}
+}
--- a/agent-common/utils/StringUtils.go
+++ b/agent-common/utils/StringUtils.go
@@ -1,6 +1,7 @@
 package utils

 import (
+	"fmt"
 	"math/rand"
 	"time"
 )
@@ -15,3 +16,36 @@ func GenerateRandomString(length int) string {
 	}
 	return string(b)
 }
+
+func ByteSizeToString(size uint64) string {
+	const (
+		B  = 1
+		KB = 1024 * B
+		MB = 1024 * KB
+		GB = 1024 * MB
+		TB = 1024 * GB
+	)
+
+	var unit string
+	var value float64
+
+	switch {
+	case size >= TB:
+		value = float64(size) / TB
+		unit = "TB"
+	case size >= GB:
+		value = float64(size) / GB
+		unit = "GB"
+	case size >= MB:
+		value = float64(size) / MB
+		unit = "MB"
+	case size >= KB:
+		value = float64(size) / KB
+		unit = "KB"
+	default:
+		value = float64(size)
+		unit = "B"
+	}
+
+	return fmt.Sprintf("%.2f %s", value, unit)
+}
--- a/agent-go/a_agent/AgentServerInfo.go
+++ b/agent-go/a_agent/AgentServerInfo.go
@@ -28,6 +28,7 @@ type AgentServerInfo struct {
 	KernelArch      string `json:"kernelArch"`      // native cpu architecture queried at runtime, as returned by `uname -m` or empty string in case of error
 	IoSpeed         string `json:"ioSpeed" yaml:"ioSpeed"`
 	MemoryTotal     string `json:"memoryTotal" yaml:"memoryTotal"`
+	SwapTotal       string `json:"swapTotal" yaml:"swapTotal"`
 	DiskTotal       string `json:"diskTotal" yaml:"diskTotal"`
 	DiskUsage       string `json:"diskUsage" yaml:"diskUsage"`
 	Comment         string `json:"comment" yaml:"comment"`
--- a/agent-go/a_init/AgentInitialization.go
+++ b/agent-go/a_init/AgentInitialization.go
@@ -5,13 +5,15 @@ import (
 	"fmt"
 	"github.com/spf13/viper"
 	"gopkg.in/yaml.v3"
-	"io/ioutil"
 	"math/rand"
+	"os"
 	"reflect"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 	"wdd.io/agent-common/logger"
+	"wdd.io/agent-common/utils"
 	"wdd.io/agent-go/a_agent"
 	"wdd.io/agent-go/a_executor"
 	"wdd.io/agent-go/a_status"
@@ -19,25 +21,35 @@ import (
 	"wdd.io/agent-go/rabbitmq"
 )

+const AgentServerInfoLocalFilePath = "/usr/local/etc/octopus-agent/octopus-agent.conf"
+
 var initOmType = g.InitOmType
 var P = g.G.P

 var log = logger.Log

-func INIT(octopusAgentConfigFileName string, agentServerInfoConf string) chan bool {
+func INIT(octopusAgentConfigFileName string) chan bool {

-	// 获取系统的环境变量
-	agentServerInfo := parseAgentServerInfo(agentServerInfoConf)
-	// todo totally get from a_status module
+	// 初始化变量
+	agentServerInfo := &a_agent.AgentServerInfo{}

-	// 初始化Nacos的连接配置
+	// 初始化Agent的 RabbitMQ连接信息
 	agentConfig := parseOctopusAgentConf(octopusAgentConfigFileName)
 	a_agent.AgentConfig = agentConfig

-	// re-get agentInfo from status module
+	// 使用a_status模块，自身获取到运行的环境信息
 	agentInfo := a_status.ReportAgentInfo()
 	refreshAgentInfoByStatusInfo(agentInfo, agentServerInfo)

+	if utils.FileExistAndNotNull(AgentServerInfoLocalFilePath) {
+
+		// 获取系统的环境变量
+		agentServerInfoFromLocalFile := parseAgentServerInfo(AgentServerInfoLocalFilePath)
+
+		// 合并系统环境变量，手动输入的部分会覆盖自身获得内容
+		utils.CopySameFields(agentServerInfoFromLocalFile, agentServerInfo)
+	}
+
 	// build operator cache
 	buildAgentOsOperator(agentInfo, agentServerInfo)

@@ -235,12 +247,44 @@ func buildOctopusTCPConnect(agentConfig *viper.Viper) *rabbitmq.RabbitTCPConnect
 func refreshAgentInfoByStatusInfo(agentInfo *a_status.AgentInfo, agentServerInfo *a_agent.AgentServerInfo) {

 	// host info
+	agentServerInfo.ServerName = agentInfo.HostInfo.Hostname
+	agentServerInfo.MachineID = agentInfo.HostInfo.HostID
+
+	// cpu part
+	agentServerInfo.CPUCore = strconv.FormatInt(int64(agentInfo.CPUInfo.NumCores), 10)
+	if len(agentInfo.CPUInfo.CPUInfo) > 0 {
+		marshal, _ := json.Marshal(agentInfo.CPUInfo.CPUInfo[0].ModelName)
+		agentServerInfo.CPUBrand = string(marshal)
+	}
+
+	// os info
+	agentServerInfo.OSInfo = agentInfo.HostInfo.PlatformFamily + " " + agentInfo.HostInfo.Platform + " " + agentInfo.HostInfo.PlatformVersion
+	agentServerInfo.OSKernelInfo = agentInfo.HostInfo.KernelVersion
+	agentServerInfo.Virtualization = agentInfo.HostInfo.VirtualizationSystem + " " + agentInfo.HostInfo.VirtualizationRole
 	agentServerInfo.Platform = agentInfo.HostInfo.Platform
 	agentServerInfo.PlatformFamily = agentInfo.HostInfo.PlatformFamily
 	agentServerInfo.PlatformVersion = agentInfo.HostInfo.PlatformVersion
 	agentServerInfo.KernelVersion = agentInfo.HostInfo.KernelVersion
 	agentServerInfo.KernelArch = agentInfo.HostInfo.KernelArch

+	// memory part
+	agentServerInfo.MemoryTotal = utils.ByteSizeToString(agentInfo.MemoryInfo.TotalMemory)
+	agentServerInfo.SwapTotal = utils.ByteSizeToString(agentInfo.MemoryInfo.SwapTotal)
+
+	// disk part
+	info := agentInfo.DiskInfo
+	diskTotal := uint64(0)
+	diskUsage := uint64(0)
+
+	for _, diskInfo := range info {
+		if diskInfo.Total > diskTotal {
+			diskTotal = diskInfo.Total
+			diskUsage = diskInfo.Used
+		}
+	}
+	agentServerInfo.DiskTotal = utils.ByteSizeToString(diskTotal)
+	agentServerInfo.DiskUsage = utils.ByteSizeToString(diskUsage)
+
 	// network part
 	refreshAgentNetworkInfo(agentInfo, agentServerInfo)

@@ -357,7 +401,7 @@ func parseAgentServerInfo(agentServerInfoConf string) *a_agent.AgentServerInfo {

 	// 约定文件地址为 /octopus-agent/octopus-agent.conf
 	var agentServerInfo *a_agent.AgentServerInfo
-	yamlFile, err := ioutil.ReadFile(agentServerInfoConf)
+	yamlFile, err := os.ReadFile(agentServerInfoConf)

 	if err != nil {
 		panic(fmt.Errorf("failed to read YAML file: %v", err))
--- a/agent-go/main.go
+++ b/agent-go/main.go
@@ -13,11 +13,11 @@ func main() {

 	// 解析命令行参数
 	var version string
-	var agentServerInfoConfFile string
+	//var agentServerInfoConfFile string
 	var mode string
 	flag.StringVar(&version, "version", "", "config file version")
 	flag.StringVar(&mode, "mode", "agent", "agent run mode")
-	flag.StringVar(&agentServerInfoConfFile, "agentServerInfoConfFile", "", "agent server info conf file")
+	//flag.StringVar(&agentServerInfoConfFile, "agentServerInfoConfFile", "", "agent server info conf file")
 	flag.Parse()

 	if mode == "bastion" {
@@ -30,10 +30,10 @@ func main() {
 	// 读取对应版本的配置文件
 	octopusAgentConfigFileName := fmt.Sprintf("octopus-agent-%s.yaml", version)
 	fmt.Println("config file name is => " + octopusAgentConfigFileName)
-	fmt.Println("agent server info file is => " + agentServerInfoConfFile)
+	//fmt.Println("agent server info file is => " + agentServerInfoConfFile)

 	// 执行初始化之策工作
-	businessForeverChan := a_init.INIT(octopusAgentConfigFileName, agentServerInfoConfFile)
+	businessForeverChan := a_init.INIT(octopusAgentConfigFileName)

 	// 永远等待 runtime的队列消息
 	<-businessForeverChan
--- a/agent-operator/CmiiK8sOperator_test.go
+++ b/agent-operator/CmiiK8sOperator_test.go
@@ -252,7 +252,7 @@ func TestUpdateCmiiDeploymentImageTag(t *testing.T) {

 	// 计算20:00的时间
 	now := time.Now()
-	targetTime := time.Date(now.Year(), now.Month(), now.Day(), 17, 45, 00, 0, now.Location())
+	targetTime := time.Date(now.Year(), now.Month(), now.Day(), 12, 03, 00, 0, now.Location())

 	duration := time.Duration(0)

@@ -275,7 +275,10 @@ func TestUpdateCmiiDeploymentImageTag(t *testing.T) {

 	appNameTagMap := map[string]string{
 		//"cmii-uav-multilink": "5.5.0",
-		"cmii-uav-data-post-process": "5.5.0-042501",
+		"cmii-uav-platform-cms-portal": "5.5.0-042801",
+		"cmii-uav-platform":            "5.5.0-042801",
+		"cmii-uav-platform-oms":        "5.5.0-042801",
+		"cmii-uav-user":                "5.5.0-042801",
 	}

 	for appName, newTag := range appNameTagMap {
--- a/agent-operator/CmiiOperator.go
+++ b/agent-operator/CmiiOperator.go
@@ -193,7 +193,7 @@ func DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileName, loca

 	// 支持单文件的形式
 	if !utils.IsFileOrDir(localGzipFolderOrGzipFile) {
-		// 单个压缩文件
+		// 单个压缩文件 肯定是离线的形式
 		if !strings.HasSuffix(localGzipFolderOrGzipFile, ".tar.gz") {
 			log.ErrorF("local gzip file %s is not a .tar.gz file !", localGzipFolderOrGzipFile)
 			return nil
@@ -201,22 +201,23 @@ func DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileName, loca

 		// load
 		image.LoadFromGzipFilePath(localGzipFolderOrGzipFile)
-	}
-
-	separator := os.PathSeparator
-	if !strings.HasSuffix(localGzipFolderOrGzipFile, string(separator)) {
-		localGzipFolderOrGzipFile += string(separator)
-	}
-
-	// download
-	if downloadFromOss {
-		if !parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile) {
-			log.ErrorF("download from oss error !")
-			return nil
+	} else {
+		separator := os.PathSeparator
+		if !strings.HasSuffix(localGzipFolderOrGzipFile, string(separator)) {
+			localGzipFolderOrGzipFile += string(separator)
 		}
+
+		// download
+		if downloadFromOss {
+			if !parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile) {
+				log.ErrorF("download from oss error !")
+				return nil
+			}
+		}
+
+		// load
+		loadAllGzipImageFromLocalFolder(localGzipFolderOrGzipFile)
 	}
-	// load
-	loadAllGzipImageFromLocalFolder(localGzipFolderOrGzipFile)

 	// tag
 	// push
--- a/agent-operator/K8sOperator.go
+++ b/agent-operator/K8sOperator.go
@@ -387,6 +387,7 @@ func (op *CmiiK8sOperator) DeploymentScale(cmiiEnv, appName string, scaleCount i
 	return true
 }
 func (op *CmiiK8sOperator) DeploymentUpdateTagByImageFullName(cmiiEnv, imageFullName string) bool {
+	// todo

 	split := strings.Split(imageFullName, ":")
 	// harbor
--- a/agent-operator/deploy/c_app/DeployCmiiApp.go
+++ b/agent-operator/deploy/c_app/DeployCmiiApp.go
@@ -4,6 +4,7 @@ import (
 	"github.com/go-playground/validator/v10"
 	"os"
 	"wdd.io/agent-common/logger"
+	"wdd.io/agent-common/utils"
 	"wdd.io/agent-operator/deploy/z_dep"
 )

@@ -55,7 +56,7 @@ func init() {
 func (backend *CmiiBackendConfig) BackendDeploy(common *z_dep.CommonEnvironmentConfig) bool {

 	// copy
-	z_dep.CopySameFields(common, backend)
+	utils.CopySameFields(common, backend)

 	validate := validator.New()
 	err := validate.Struct(backend)
@@ -84,7 +85,7 @@ func (backend *CmiiBackendConfig) BackendDeploy(common *z_dep.CommonEnvironmentC
 func (frontend *CmiiFrontendConfig) FrontendDeploy(common *z_dep.CommonEnvironmentConfig) bool {

 	// copy
-	z_dep.CopySameFields(common, frontend)
+	utils.CopySameFields(common, frontend)

 	validate := validator.New()
 	err := validate.Struct(frontend)
@@ -106,7 +107,7 @@ func (frontend *CmiiFrontendConfig) FrontendDeploy(common *z_dep.CommonEnvironme
 func (frontend *CmiiFrontendConfig) ConfigMapDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool {

 	// copy
-	z_dep.CopySameFields(commonEnv, frontend)
+	utils.CopySameFields(commonEnv, frontend)

 	// manual validate
 	if frontend.ShortName == "" || frontend.ClientId == "" {
--- a/agent-operator/deploy/z_dep/G.go
+++ b/agent-operator/deploy/z_dep/G.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/go-playground/validator/v10"
-	"reflect"
 	"runtime"
 	"text/template"
 	"wdd.io/agent-common/assert"
@@ -104,16 +103,3 @@ func ParseEnvToApplyFile(environment any, applyTemplate string, applyFilePath st

 	return true
 }
-
-// CopySameFields 利用反射，将a中的所有同名字段的值 复制到b中的对应字段
-func CopySameFields(a, b interface{}) {
-	va := reflect.ValueOf(a).Elem()
-	vb := reflect.ValueOf(b).Elem()
-
-	for i := 0; i < va.NumField(); i++ {
-		fieldName := va.Type().Field(i).Name
-		if vb.FieldByName(fieldName).IsValid() {
-			vb.FieldByName(fieldName).Set(va.Field(i))
-		}
-	}
-}
--- a/agent-operator/image/CmiiImageSync.go
+++ b/agent-operator/image/CmiiImageSync.go
@@ -211,6 +211,7 @@ func UploadToHarbor(targetImageName string) (uploadOK bool) {
 	fmt.Println()
 	log.InfoF("[UploadToHarbor] - upload %s success!", targetImageName)
 	fmt.Println()
+	return true
 }

 // TagFromListAndPushToCHarbor 需要支持 harbor.cdcyy.cn  ip:8033  rancher/rancher:v2.5.7  nginx:latest
--- a/agent-operator/main.go
+++ b/agent-operator/main.go
@@ -7,7 +7,6 @@ import (
 	"os"
 	"strings"
 	"wdd.io/agent-common/utils"
-	"wdd.io/agent-operator/real_project/bgtg"
 )

 var LocalKubeConfigFile = "/root/.kube/config"
@@ -16,7 +15,8 @@ var LocalKubeConfigFile = "/root/.kube/config"
 // C:\Users\wddsh\go\bin\gox.exe -osarch="linux/amd64" -output "build/agent-operator_{{.OS}}_{{.Arch}}"
 // C:\Users\wddsh\go\bin\gox.exe -osarch="linux/amd64 linux/arm64" -output "build/agent-operator_{{.OS}}_{{.Arch}}"

-func RealProjectRunner() {
+func BuildDefaultK8sOperator() {
+
 	// build from local LocalKubeConfigFile
 	if !utils.FileExists(LocalKubeConfigFile) {
 		log.ErrorF("%s not exits! error!", LocalKubeConfigFile)
@@ -29,18 +29,19 @@ func RealProjectRunner() {
 		return
 	}

-	realNamespace := "bjtg"
-
 	op := CmiiK8sOperator{}
 	op.BuildCurrentClientFromConfig(readFile)
 	DefaultCmiiOperator = op
+}
+
+func RealProjectRunner() {

 	// ops

 	// update
-	result := UpdateCmiiImageTagFromNameTagList(realNamespace, bgtg.AllCmiiImageTagList)
+	//result := UpdateCmiiImageTagFromNameTagList(realNamespace, bgtg.AllCmiiImageTagList)
 	//result := UpdateCmiiImageTagFromNameTagMap(realNamespace, xmyd.Real540ImageTagMap)
-	utils.BeautifulPrint(result)
+	//utils.BeautifulPrint(result)

 	//for _, imageFullName := range xmyd.AllCmiiImageTagList {
 	//	appName := image.ImageFullNameToAppName(imageFullName)
@@ -92,6 +93,8 @@ func main() {
 	flag.Parse()

 	if mode == "image" {
+
+		BuildDefaultK8sOperator()
 		// 堡垒机模式 的 镜像美容
 		// 初始化堡垒机模式
 		for {
--- a/cmii_operator/log/cmii-update-log.txt
+++ b/cmii_operator/log/cmii-update-log.txt
@@ -21,3 +21,8 @@
 2024-04-25-17-42-00 uavcloud-demo  cmii-uav-platform                       5.5.0-042501                  5.5.0-042503
 2024-04-25-17-42-06 uavcloud-demo  cmii-uav-platform-splice                5.5.0                         5.5.0-042501
 2024-04-25-17-45-00 uavcloud-demo  cmii-uav-data-post-process              5.5.0                         5.5.0-042501
+2024-04-26-17-55-00 uavcloud-demo  cmii-uav-platform-splice                5.5.0-042501                  5.5.0-042601
+2024-04-28-12-03-00 uavcloud-demo  cmii-uav-platform-cms-portal            5.5.0                         5.5.0-042801
+2024-04-28-12-03-05 uavcloud-demo  cmii-uav-platform                       5.5.0-042503                  5.5.0-042801
+2024-04-28-12-03-10 uavcloud-demo  cmii-uav-platform-oms                   5.5.0                         5.5.0-042801
+2024-04-28-12-03-13 uavcloud-demo  cmii-uav-user                           5.5.0                         5.5.0-042801
--- a/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java
+++ b/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java
@@ -44,7 +44,7 @@ public class TestImageSyncScheduler {
        ArrayList<String> ImageFullNameList = new ArrayList<>(List.of(
 //                "harbor.cdcyy.com.cn/cmii/cmii-live-operator:5.2.0",
 //                "harbor.cdcyy.com.cn/cmii/cmii/srs:v5.0.195"
-                "harbor.cdcyy.com.cn/cmii/cmii-uav-platform:5.3.0-cqly-042601"
+                "harbor.cdcyy.com.cn/cmii/cmii-uav-industrial-portfolio:5.4.0-cqly-042802"
        ));

        Boolean downloadAndCompressOnly = false;
@@ -102,7 +102,7 @@ public class TestImageSyncScheduler {
        ));

        ArrayList<String> ImageFullNameList = new ArrayList<>(List.of(
-                "harbor.cdcyy.com.cn/cmii/cmii-uav-industrial-portfolio:5.4.0-cqly-032802"
+                "harbor.cdcyy.com.cn/cmii/cmii-uav-industrial-portfolio:5.4.0-cqly-042801"
        ));

        Boolean downloadAndCompressOnly = true;