diff --git a/.run/agent-go main.run.xml b/.run/agent-go main.run.xml index 6c14cc7..78a18cf 100644 --- a/.run/agent-go main.run.xml +++ b/.run/agent-go main.run.xml @@ -2,9 +2,8 @@ - - + diff --git a/agent-common/utils/ReflectUtils.go b/agent-common/utils/ReflectUtils.go new file mode 100644 index 0000000..fe637a4 --- /dev/null +++ b/agent-common/utils/ReflectUtils.go @@ -0,0 +1,21 @@ +package utils + +import "reflect" + +// CopySameFields 利用反射，将a中的所有同名字段的值 复制到b中的对应字段 +func CopySameFields(source, target interface{}) { + va := reflect.ValueOf(source).Elem() + vb := reflect.ValueOf(target).Elem() + + for i := 0; i < va.NumField(); i++ { + // 忽略source中 空值的部分 + if va.Field(i).IsZero() { + continue + } + + fieldName := va.Type().Field(i).Name + if vb.FieldByName(fieldName).IsValid() { + vb.FieldByName(fieldName).Set(va.Field(i)) + } + } +} diff --git a/agent-common/utils/StringUtils.go b/agent-common/utils/StringUtils.go index ad85839..54f9f98 100644 --- a/agent-common/utils/StringUtils.go +++ b/agent-common/utils/StringUtils.go @@ -1,6 +1,7 @@ package utils import ( + "fmt" "math/rand" "time" ) @@ -15,3 +16,36 @@ func GenerateRandomString(length int) string { } return string(b) } + +func ByteSizeToString(size uint64) string { + const ( + B = 1 + KB = 1024 * B + MB = 1024 * KB + GB = 1024 * MB + TB = 1024 * GB + ) + + var unit string + var value float64 + + switch { + case size >= TB: + value = float64(size) / TB + unit = "TB" + case size >= GB: + value = float64(size) / GB + unit = "GB" + case size >= MB: + value = float64(size) / MB + unit = "MB" + case size >= KB: + value = float64(size) / KB + unit = "KB" + default: + value = float64(size) + unit = "B" + } + + return fmt.Sprintf("%.2f %s", value, unit) +} diff --git a/agent-go/a_agent/AgentServerInfo.go b/agent-go/a_agent/AgentServerInfo.go index 3f64541..cf92421 100644 --- a/agent-go/a_agent/AgentServerInfo.go +++ b/agent-go/a_agent/AgentServerInfo.go @@ -28,6 +28,7 @@ type AgentServerInfo struct { KernelArch string `json:"kernelArch"` // native cpu architecture queried at runtime, as returned by `uname -m` or empty string in case of error IoSpeed string `json:"ioSpeed" yaml:"ioSpeed"` MemoryTotal string `json:"memoryTotal" yaml:"memoryTotal"` + SwapTotal string `json:"swapTotal" yaml:"swapTotal"` DiskTotal string `json:"diskTotal" yaml:"diskTotal"` DiskUsage string `json:"diskUsage" yaml:"diskUsage"` Comment string `json:"comment" yaml:"comment"` diff --git a/agent-go/a_init/AgentInitialization.go b/agent-go/a_init/AgentInitialization.go index 286aba2..29febd5 100644 --- a/agent-go/a_init/AgentInitialization.go +++ b/agent-go/a_init/AgentInitialization.go @@ -5,13 +5,15 @@ import ( "fmt" "github.com/spf13/viper" "gopkg.in/yaml.v3" - "io/ioutil" "math/rand" + "os" "reflect" "regexp" + "strconv" "strings" "time" "wdd.io/agent-common/logger" + "wdd.io/agent-common/utils" "wdd.io/agent-go/a_agent" "wdd.io/agent-go/a_executor" "wdd.io/agent-go/a_status" @@ -19,25 +21,35 @@ import ( "wdd.io/agent-go/rabbitmq" ) +const AgentServerInfoLocalFilePath = "/usr/local/etc/octopus-agent/octopus-agent.conf" + var initOmType = g.InitOmType var P = g.G.P var log = logger.Log -func INIT(octopusAgentConfigFileName string, agentServerInfoConf string) chan bool { +func INIT(octopusAgentConfigFileName string) chan bool { - // 获取系统的环境变量 - agentServerInfo := parseAgentServerInfo(agentServerInfoConf) - // todo totally get from a_status module + // 初始化变量 + agentServerInfo := &a_agent.AgentServerInfo{} - // 初始化Nacos的连接配置 + // 初始化Agent的 RabbitMQ连接信息 agentConfig := parseOctopusAgentConf(octopusAgentConfigFileName) a_agent.AgentConfig = agentConfig - // re-get agentInfo from status module + // 使用a_status模块，自身获取到运行的环境信息 agentInfo := a_status.ReportAgentInfo() refreshAgentInfoByStatusInfo(agentInfo, agentServerInfo) + if utils.FileExistAndNotNull(AgentServerInfoLocalFilePath) { + + // 获取系统的环境变量 + agentServerInfoFromLocalFile := parseAgentServerInfo(AgentServerInfoLocalFilePath) + + // 合并系统环境变量，手动输入的部分会覆盖自身获得内容 + utils.CopySameFields(agentServerInfoFromLocalFile, agentServerInfo) + } + // build operator cache buildAgentOsOperator(agentInfo, agentServerInfo) @@ -235,12 +247,44 @@ func buildOctopusTCPConnect(agentConfig *viper.Viper) *rabbitmq.RabbitTCPConnect func refreshAgentInfoByStatusInfo(agentInfo *a_status.AgentInfo, agentServerInfo *a_agent.AgentServerInfo) { // host info + agentServerInfo.ServerName = agentInfo.HostInfo.Hostname + agentServerInfo.MachineID = agentInfo.HostInfo.HostID + + // cpu part + agentServerInfo.CPUCore = strconv.FormatInt(int64(agentInfo.CPUInfo.NumCores), 10) + if len(agentInfo.CPUInfo.CPUInfo) > 0 { + marshal, _ := json.Marshal(agentInfo.CPUInfo.CPUInfo[0].ModelName) + agentServerInfo.CPUBrand = string(marshal) + } + + // os info + agentServerInfo.OSInfo = agentInfo.HostInfo.PlatformFamily + " " + agentInfo.HostInfo.Platform + " " + agentInfo.HostInfo.PlatformVersion + agentServerInfo.OSKernelInfo = agentInfo.HostInfo.KernelVersion + agentServerInfo.Virtualization = agentInfo.HostInfo.VirtualizationSystem + " " + agentInfo.HostInfo.VirtualizationRole agentServerInfo.Platform = agentInfo.HostInfo.Platform agentServerInfo.PlatformFamily = agentInfo.HostInfo.PlatformFamily agentServerInfo.PlatformVersion = agentInfo.HostInfo.PlatformVersion agentServerInfo.KernelVersion = agentInfo.HostInfo.KernelVersion agentServerInfo.KernelArch = agentInfo.HostInfo.KernelArch + // memory part + agentServerInfo.MemoryTotal = utils.ByteSizeToString(agentInfo.MemoryInfo.TotalMemory) + agentServerInfo.SwapTotal = utils.ByteSizeToString(agentInfo.MemoryInfo.SwapTotal) + + // disk part + info := agentInfo.DiskInfo + diskTotal := uint64(0) + diskUsage := uint64(0) + + for _, diskInfo := range info { + if diskInfo.Total > diskTotal { + diskTotal = diskInfo.Total + diskUsage = diskInfo.Used + } + } + agentServerInfo.DiskTotal = utils.ByteSizeToString(diskTotal) + agentServerInfo.DiskUsage = utils.ByteSizeToString(diskUsage) + // network part refreshAgentNetworkInfo(agentInfo, agentServerInfo) @@ -357,7 +401,7 @@ func parseAgentServerInfo(agentServerInfoConf string) *a_agent.AgentServerInfo { // 约定文件地址为 /octopus-agent/octopus-agent.conf var agentServerInfo *a_agent.AgentServerInfo - yamlFile, err := ioutil.ReadFile(agentServerInfoConf) + yamlFile, err := os.ReadFile(agentServerInfoConf) if err != nil { panic(fmt.Errorf("failed to read YAML file: %v", err)) diff --git a/agent-go/main.go b/agent-go/main.go index 52b35a8..f7a5f60 100644 --- a/agent-go/main.go +++ b/agent-go/main.go @@ -13,11 +13,11 @@ func main() { // 解析命令行参数 var version string - var agentServerInfoConfFile string + //var agentServerInfoConfFile string var mode string flag.StringVar(&version, "version", "", "config file version") flag.StringVar(&mode, "mode", "agent", "agent run mode") - flag.StringVar(&agentServerInfoConfFile, "agentServerInfoConfFile", "", "agent server info conf file") + //flag.StringVar(&agentServerInfoConfFile, "agentServerInfoConfFile", "", "agent server info conf file") flag.Parse() if mode == "bastion" { @@ -30,10 +30,10 @@ func main() { // 读取对应版本的配置文件 octopusAgentConfigFileName := fmt.Sprintf("octopus-agent-%s.yaml", version) fmt.Println("config file name is => " + octopusAgentConfigFileName) - fmt.Println("agent server info file is => " + agentServerInfoConfFile) + //fmt.Println("agent server info file is => " + agentServerInfoConfFile) // 执行初始化之策工作 - businessForeverChan := a_init.INIT(octopusAgentConfigFileName, agentServerInfoConfFile) + businessForeverChan := a_init.INIT(octopusAgentConfigFileName) // 永远等待 runtime的队列消息 <-businessForeverChan diff --git a/agent-operator/CmiiK8sOperator_test.go b/agent-operator/CmiiK8sOperator_test.go index 5f55246..1a1e26f 100644 --- a/agent-operator/CmiiK8sOperator_test.go +++ b/agent-operator/CmiiK8sOperator_test.go @@ -252,7 +252,7 @@ func TestUpdateCmiiDeploymentImageTag(t *testing.T) { // 计算20:00的时间 now := time.Now() - targetTime := time.Date(now.Year(), now.Month(), now.Day(), 17, 45, 00, 0, now.Location()) + targetTime := time.Date(now.Year(), now.Month(), now.Day(), 12, 03, 00, 0, now.Location()) duration := time.Duration(0) @@ -275,7 +275,10 @@ func TestUpdateCmiiDeploymentImageTag(t *testing.T) { appNameTagMap := map[string]string{ //"cmii-uav-multilink": "5.5.0", - "cmii-uav-data-post-process": "5.5.0-042501", + "cmii-uav-platform-cms-portal": "5.5.0-042801", + "cmii-uav-platform": "5.5.0-042801", + "cmii-uav-platform-oms": "5.5.0-042801", + "cmii-uav-user": "5.5.0-042801", } for appName, newTag := range appNameTagMap { diff --git a/agent-operator/CmiiOperator.go b/agent-operator/CmiiOperator.go index b7bd3f0..a441005 100644 --- a/agent-operator/CmiiOperator.go +++ b/agent-operator/CmiiOperator.go @@ -193,7 +193,7 @@ func DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileName, loca // 支持单文件的形式 if !utils.IsFileOrDir(localGzipFolderOrGzipFile) { - // 单个压缩文件 + // 单个压缩文件 肯定是离线的形式 if !strings.HasSuffix(localGzipFolderOrGzipFile, ".tar.gz") { log.ErrorF("local gzip file %s is not a .tar.gz file !", localGzipFolderOrGzipFile) return nil @@ -201,22 +201,23 @@ func DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileName, loca // load image.LoadFromGzipFilePath(localGzipFolderOrGzipFile) - } - - separator := os.PathSeparator - if !strings.HasSuffix(localGzipFolderOrGzipFile, string(separator)) { - localGzipFolderOrGzipFile += string(separator) - } - - // download - if downloadFromOss { - if !parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile) { - log.ErrorF("download from oss error !") - return nil + } else { + separator := os.PathSeparator + if !strings.HasSuffix(localGzipFolderOrGzipFile, string(separator)) { + localGzipFolderOrGzipFile += string(separator) } + + // download + if downloadFromOss { + if !parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile) { + log.ErrorF("download from oss error !") + return nil + } + } + + // load + loadAllGzipImageFromLocalFolder(localGzipFolderOrGzipFile) } - // load - loadAllGzipImageFromLocalFolder(localGzipFolderOrGzipFile) // tag // push diff --git a/agent-operator/K8sOperator.go b/agent-operator/K8sOperator.go index 2522ada..d051626 100644 --- a/agent-operator/K8sOperator.go +++ b/agent-operator/K8sOperator.go @@ -387,6 +387,7 @@ func (op *CmiiK8sOperator) DeploymentScale(cmiiEnv, appName string, scaleCount i return true } func (op *CmiiK8sOperator) DeploymentUpdateTagByImageFullName(cmiiEnv, imageFullName string) bool { + // todo split := strings.Split(imageFullName, ":") // harbor diff --git a/agent-operator/deploy/c_app/DeployCmiiApp.go b/agent-operator/deploy/c_app/DeployCmiiApp.go index b2a68a0..4cda66a 100644 --- a/agent-operator/deploy/c_app/DeployCmiiApp.go +++ b/agent-operator/deploy/c_app/DeployCmiiApp.go @@ -4,6 +4,7 @@ import ( "github.com/go-playground/validator/v10" "os" "wdd.io/agent-common/logger" + "wdd.io/agent-common/utils" "wdd.io/agent-operator/deploy/z_dep" ) @@ -55,7 +56,7 @@ func init() { func (backend *CmiiBackendConfig) BackendDeploy(common *z_dep.CommonEnvironmentConfig) bool { // copy - z_dep.CopySameFields(common, backend) + utils.CopySameFields(common, backend) validate := validator.New() err := validate.Struct(backend) @@ -84,7 +85,7 @@ func (backend *CmiiBackendConfig) BackendDeploy(common *z_dep.CommonEnvironmentC func (frontend *CmiiFrontendConfig) FrontendDeploy(common *z_dep.CommonEnvironmentConfig) bool { // copy - z_dep.CopySameFields(common, frontend) + utils.CopySameFields(common, frontend) validate := validator.New() err := validate.Struct(frontend) @@ -106,7 +107,7 @@ func (frontend *CmiiFrontendConfig) FrontendDeploy(common *z_dep.CommonEnvironme func (frontend *CmiiFrontendConfig) ConfigMapDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool { // copy - z_dep.CopySameFields(commonEnv, frontend) + utils.CopySameFields(commonEnv, frontend) // manual validate if frontend.ShortName == "" || frontend.ClientId == "" { diff --git a/agent-operator/deploy/z_dep/G.go b/agent-operator/deploy/z_dep/G.go index d5cdf16..68f0af1 100644 --- a/agent-operator/deploy/z_dep/G.go +++ b/agent-operator/deploy/z_dep/G.go @@ -4,7 +4,6 @@ import ( "bytes" "fmt" "github.com/go-playground/validator/v10" - "reflect" "runtime" "text/template" "wdd.io/agent-common/assert" @@ -104,16 +103,3 @@ func ParseEnvToApplyFile(environment any, applyTemplate string, applyFilePath st return true } - -// CopySameFields 利用反射，将a中的所有同名字段的值 复制到b中的对应字段 -func CopySameFields(a, b interface{}) { - va := reflect.ValueOf(a).Elem() - vb := reflect.ValueOf(b).Elem() - - for i := 0; i < va.NumField(); i++ { - fieldName := va.Type().Field(i).Name - if vb.FieldByName(fieldName).IsValid() { - vb.FieldByName(fieldName).Set(va.Field(i)) - } - } -} diff --git a/agent-operator/image/CmiiImageSync.go b/agent-operator/image/CmiiImageSync.go index ed62097..40bcc6b 100644 --- a/agent-operator/image/CmiiImageSync.go +++ b/agent-operator/image/CmiiImageSync.go @@ -211,6 +211,7 @@ func UploadToHarbor(targetImageName string) (uploadOK bool) { fmt.Println() log.InfoF("[UploadToHarbor] - upload %s success!", targetImageName) fmt.Println() + return true } // TagFromListAndPushToCHarbor 需要支持 harbor.cdcyy.cn ip:8033 rancher/rancher:v2.5.7 nginx:latest diff --git a/agent-operator/main.go b/agent-operator/main.go index fc57af4..0173f05 100644 --- a/agent-operator/main.go +++ b/agent-operator/main.go @@ -7,7 +7,6 @@ import ( "os" "strings" "wdd.io/agent-common/utils" - "wdd.io/agent-operator/real_project/bgtg" ) var LocalKubeConfigFile = "/root/.kube/config" @@ -16,7 +15,8 @@ var LocalKubeConfigFile = "/root/.kube/config" // C:\Users\wddsh\go\bin\gox.exe -osarch="linux/amd64" -output "build/agent-operator_{{.OS}}_{{.Arch}}" // C:\Users\wddsh\go\bin\gox.exe -osarch="linux/amd64 linux/arm64" -output "build/agent-operator_{{.OS}}_{{.Arch}}" -func RealProjectRunner() { +func BuildDefaultK8sOperator() { + // build from local LocalKubeConfigFile if !utils.FileExists(LocalKubeConfigFile) { log.ErrorF("%s not exits! error!", LocalKubeConfigFile) @@ -29,18 +29,19 @@ func RealProjectRunner() { return } - realNamespace := "bjtg" - op := CmiiK8sOperator{} op.BuildCurrentClientFromConfig(readFile) DefaultCmiiOperator = op +} + +func RealProjectRunner() { // ops // update - result := UpdateCmiiImageTagFromNameTagList(realNamespace, bgtg.AllCmiiImageTagList) + //result := UpdateCmiiImageTagFromNameTagList(realNamespace, bgtg.AllCmiiImageTagList) //result := UpdateCmiiImageTagFromNameTagMap(realNamespace, xmyd.Real540ImageTagMap) - utils.BeautifulPrint(result) + //utils.BeautifulPrint(result) //for _, imageFullName := range xmyd.AllCmiiImageTagList { // appName := image.ImageFullNameToAppName(imageFullName) @@ -92,6 +93,8 @@ func main() { flag.Parse() if mode == "image" { + + BuildDefaultK8sOperator() // 堡垒机模式 的 镜像美容 // 初始化堡垒机模式 for { diff --git a/cmii_operator/log/cmii-update-log.txt b/cmii_operator/log/cmii-update-log.txt index c2b0845..4aa8948 100644 --- a/cmii_operator/log/cmii-update-log.txt +++ b/cmii_operator/log/cmii-update-log.txt @@ -21,3 +21,8 @@ 2024-04-25-17-42-00 uavcloud-demo cmii-uav-platform 5.5.0-042501 5.5.0-042503 2024-04-25-17-42-06 uavcloud-demo cmii-uav-platform-splice 5.5.0 5.5.0-042501 2024-04-25-17-45-00 uavcloud-demo cmii-uav-data-post-process 5.5.0 5.5.0-042501 +2024-04-26-17-55-00 uavcloud-demo cmii-uav-platform-splice 5.5.0-042501 5.5.0-042601 +2024-04-28-12-03-00 uavcloud-demo cmii-uav-platform-cms-portal 5.5.0 5.5.0-042801 +2024-04-28-12-03-05 uavcloud-demo cmii-uav-platform 5.5.0-042503 5.5.0-042801 +2024-04-28-12-03-10 uavcloud-demo cmii-uav-platform-oms 5.5.0 5.5.0-042801 +2024-04-28-12-03-13 uavcloud-demo cmii-uav-user 5.5.0 5.5.0-042801 diff --git a/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java b/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java index 83442a6..5c54eb7 100644 --- a/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java +++ b/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java @@ -44,7 +44,7 @@ public class TestImageSyncScheduler { ArrayList ImageFullNameList = new ArrayList<>(List.of( // "harbor.cdcyy.com.cn/cmii/cmii-live-operator:5.2.0", // "harbor.cdcyy.com.cn/cmii/cmii/srs:v5.0.195" - "harbor.cdcyy.com.cn/cmii/cmii-uav-platform:5.3.0-cqly-042601" + "harbor.cdcyy.com.cn/cmii/cmii-uav-industrial-portfolio:5.4.0-cqly-042802" )); Boolean downloadAndCompressOnly = false; @@ -102,7 +102,7 @@ public class TestImageSyncScheduler { )); ArrayList ImageFullNameList = new ArrayList<>(List.of( - "harbor.cdcyy.com.cn/cmii/cmii-uav-industrial-portfolio:5.4.0-cqly-032802" + "harbor.cdcyy.com.cn/cmii/cmii-uav-industrial-portfolio:5.4.0-cqly-042801" )); Boolean downloadAndCompressOnly = true;