zeaslity/ProjectOctopus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'refs/heads/local-ss'

Browse Source
This commit is contained in:
zeaslity
2024-04-29 10:24:07 +08:00
parent e7771a2153 b58ccd6dbe
commit 74dba43a96
19 changed files with 247 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.run/agent-go main.run.xml
View File

@@ -2,9 +2,8 @@
<configuration default="false" name="agent-go main" type="GoApplicationRunConfiguration"
factoryName="Go Application">
<module name="ProjectOctopus"/>
<target name="LapPro-10.250.0.100"/>
<working_directory value="$PROJECT_DIR$/agent-go"/>
<parameters value="-age"/>
<parameters value="-version=dev"/>
<kind value="PACKAGE"/>
<package value="wdd.io/agent-go"/>
<directory value="$PROJECT_DIR$"/>

21
agent-common/utils/ReflectUtils.go Normal file
View File

@@ -0,0 +1,21 @@
package utils
import "reflect"
// CopySameFields 利用反射，将a中的所有同名字段的值 复制到b中的对应字段
func CopySameFields(source, target interface{}) {
va := reflect.ValueOf(source).Elem()
vb := reflect.ValueOf(target).Elem()
for i := 0; i < va.NumField(); i++ {
// 忽略source中 空值的部分
if va.Field(i).IsZero() {
continue
}
fieldName := va.Type().Field(i).Name
if vb.FieldByName(fieldName).IsValid() {
vb.FieldByName(fieldName).Set(va.Field(i))
}
}
}

34
agent-common/utils/StringUtils.go
View File

@@ -1,6 +1,7 @@
package utils
import (
"fmt"
"math/rand"
"time"
)
@@ -15,3 +16,36 @@ func GenerateRandomString(length int) string {
}
return string(b)
}
func ByteSizeToString(size uint64) string {
const (
B = 1
KB = 1024 * B
MB = 1024 * KB
GB = 1024 * MB
TB = 1024 * GB
)
var unit string
var value float64
switch {
case size >= TB:
value = float64(size) / TB
unit = "TB"
case size >= GB:
value = float64(size) / GB
unit = "GB"
case size >= MB:
value = float64(size) / MB
unit = "MB"
case size >= KB:
value = float64(size) / KB
unit = "KB"
default:
value = float64(size)
unit = "B"
}
return fmt.Sprintf("%.2f %s", value, unit)
}

6
agent-go/a_agent/AgentServerInfo.go
View File

@@ -12,6 +12,11 @@ type AgentServerInfo struct {
ServerIPInV4 string `json:"serverIpInV4" yaml:"serverIpInV4"`
ServerIPPbV6 string `json:"serverIpPbV6" yaml:"serverIpPbV6"`
ServerIPInV6 string `json:"serverIpInV6" yaml:"serverIpInV6"`
City string `json:"city" yaml:"city"`
Region string `json:"region" yaml:"region"`
Country string `json:"country" yaml:"country"`
Organization string `json:"organization" yaml:"organization"`
TimeZone string `json:"timeZone" yaml:"timeZone"`
Location string `json:"location" yaml:"location"`
Provider string `json:"provider" yaml:"provider"`
ManagePort string `json:"managePort" yaml:"managePort"`
@@ -28,6 +33,7 @@ type AgentServerInfo struct {
KernelArch string `json:"kernelArch"` // native cpu architecture queried at runtime, as returned by `uname -m` or empty string in case of error
IoSpeed string `json:"ioSpeed" yaml:"ioSpeed"`
MemoryTotal string `json:"memoryTotal" yaml:"memoryTotal"`
SwapTotal string `json:"swapTotal" yaml:"swapTotal"`
DiskTotal string `json:"diskTotal" yaml:"diskTotal"`
DiskUsage string `json:"diskUsage" yaml:"diskUsage"`
Comment string `json:"comment" yaml:"comment"`

82
agent-go/a_init/AgentInitialization.go
View File

@@ -5,13 +5,15 @@ import (
"fmt"
"github.com/spf13/viper"
"gopkg.in/yaml.v3"
"io/ioutil"
"math/rand"
"os"
"reflect"
"regexp"
"strconv"
"strings"
"time"
"wdd.io/agent-common/logger"
"wdd.io/agent-common/utils"
"wdd.io/agent-go/a_agent"
"wdd.io/agent-go/a_executor"
"wdd.io/agent-go/a_status"
@@ -19,25 +21,36 @@ import (
"wdd.io/agent-go/rabbitmq"
)
const AgentServerInfoLocalFilePath = "/usr/local/etc/octopus-agent/octopus-agent.conf"
var initOmType = g.InitOmType
var P = g.G.P
var log = logger.Log
func INIT(octopusAgentConfigFileName string, agentServerInfoConf string) chan bool {
func INIT(octopusAgentConfigFileName string) chan bool {
// 获取系统的环境变量
agentServerInfo := parseAgentServerInfo(agentServerInfoConf)
// todo totally get from a_status module
// 初始化变量
agentServerInfo := &a_agent.AgentServerInfo{}
// 初始化Nacos的连接配置
// 初始化Agent的 RabbitMQ连接信息
agentConfig := parseOctopusAgentConf(octopusAgentConfigFileName)
a_agent.AgentConfig = agentConfig
// re-get agentInfo from status module
// 使用a_status模块，自身获取到运行的环境信息
agentInfo := a_status.ReportAgentInfo()
refreshAgentInfoByStatusInfo(agentInfo, agentServerInfo)
// 如果从在本机文件，那么会使用手动写入的环境变量 进行覆盖
if utils.FileExistAndNotNull(AgentServerInfoLocalFilePath) {
// 获取系统的环境变量
agentServerInfoFromLocalFile := parseAgentServerInfo(AgentServerInfoLocalFilePath)
// 合并系统环境变量，手动输入的部分会覆盖自身获得内容
utils.CopySameFields(agentServerInfoFromLocalFile, agentServerInfo)
}
// build operator cache
buildAgentOsOperator(agentInfo, agentServerInfo)
@@ -235,12 +248,44 @@ func buildOctopusTCPConnect(agentConfig *viper.Viper) *rabbitmq.RabbitTCPConnect
func refreshAgentInfoByStatusInfo(agentInfo *a_status.AgentInfo, agentServerInfo *a_agent.AgentServerInfo) {
// host info
agentServerInfo.ServerName = agentInfo.HostInfo.Hostname
agentServerInfo.MachineID = agentInfo.HostInfo.HostID
// cpu part
agentServerInfo.CPUCore = strconv.FormatInt(int64(agentInfo.CPUInfo.NumCores), 10)
if len(agentInfo.CPUInfo.CPUInfo) > 0 {
marshal, _ := json.Marshal(agentInfo.CPUInfo.CPUInfo[0].ModelName)
agentServerInfo.CPUBrand = string(marshal)
}
// os info
agentServerInfo.OSInfo = agentInfo.HostInfo.PlatformFamily + " " + agentInfo.HostInfo.Platform + " " + agentInfo.HostInfo.PlatformVersion
agentServerInfo.OSKernelInfo = agentInfo.HostInfo.KernelVersion
agentServerInfo.Virtualization = agentInfo.HostInfo.VirtualizationSystem + " " + agentInfo.HostInfo.VirtualizationRole
agentServerInfo.Platform = agentInfo.HostInfo.Platform
agentServerInfo.PlatformFamily = agentInfo.HostInfo.PlatformFamily
agentServerInfo.PlatformVersion = agentInfo.HostInfo.PlatformVersion
agentServerInfo.KernelVersion = agentInfo.HostInfo.KernelVersion
agentServerInfo.KernelArch = agentInfo.HostInfo.KernelArch
// memory part
agentServerInfo.MemoryTotal = utils.ByteSizeToString(agentInfo.MemoryInfo.TotalMemory)
agentServerInfo.SwapTotal = utils.ByteSizeToString(agentInfo.MemoryInfo.SwapTotal)
// disk part
info := agentInfo.DiskInfo
diskTotal := uint64(0)
diskUsage := uint64(0)
for _, diskInfo := range info {
if diskInfo.Total > diskTotal {
diskTotal = diskInfo.Total
diskUsage = diskInfo.Used
}
}
agentServerInfo.DiskTotal = utils.ByteSizeToString(diskTotal)
agentServerInfo.DiskUsage = utils.ByteSizeToString(diskUsage)
// network part
refreshAgentNetworkInfo(agentInfo, agentServerInfo)
@@ -249,11 +294,22 @@ func refreshAgentInfoByStatusInfo(agentInfo *a_status.AgentInfo, agentServerInfo
func refreshAgentNetworkInfo(agentInfo *a_status.AgentInfo, agentServerInfo *a_agent.AgentServerInfo) {
// 测试网卡名称
//testCases := []string{"ens33", "eno1", "enp0s3", "enp1s2", "eth0", "enp2s5", "enx1234567890ab", "ens1234567890ab", "enp1234567890ab", "enp1234567890ab", "enp1", "lo","","docker0", "virbr0", "veth0",}
//for _, tc := range testCases {
// fmt.Printf("Network interface '%s' is %s\n", tc, fmt.Sprintf("%v", isNetworkInterface(tc)))
//}
// 获取Agent的公网服务信息
publicNetworkInfo := a_status.PublicNetworkInfo{}
publicNetworkInfo.GetPublicNetworkInfo()
marshal, _ := json.Marshal(publicNetworkInfo)
log.InfoF("refreshAgentNetworkInfo - public network info is %s", marshal)
if publicNetworkInfo.IP != "" {
agentServerInfo.ServerIPPbV4 = publicNetworkInfo.IP
agentServerInfo.Region = publicNetworkInfo.Region
agentServerInfo.City = publicNetworkInfo.City
agentServerInfo.Country = publicNetworkInfo.Country
agentServerInfo.Location = publicNetworkInfo.Loc
agentServerInfo.Organization = publicNetworkInfo.Org
agentServerInfo.TimeZone = publicNetworkInfo.Timezone
}
// inner ip v4 v6
for _, networkInfo := range agentInfo.NetworkInfo {
@@ -357,7 +413,7 @@ func parseAgentServerInfo(agentServerInfoConf string) *a_agent.AgentServerInfo {
// 约定文件地址为 /octopus-agent/octopus-agent.conf
var agentServerInfo *a_agent.AgentServerInfo
yamlFile, err := ioutil.ReadFile(agentServerInfoConf)
yamlFile, err := os.ReadFile(agentServerInfoConf)
if err != nil {
panic(fmt.Errorf("failed to read YAML file: %v", err))

49
agent-go/a_status/Network.go
View File

@@ -5,7 +5,9 @@ import (
"errors"
"fmt"
"github.com/shirou/gopsutil/v3/net"
"io"
net2 "net"
"net/http"
"regexp"
"strconv"
"strings"
@@ -20,6 +22,53 @@ type NetworkMetric struct {
RecvSpeed float64
}
type PublicNetworkInfo struct {
IP string `json:"ip,omitempty"`
City string `json:"city,omitempty"`
Region string `json:"region,omitempty"`
Country string `json:"country,omitempty"`
Loc string `json:"loc,omitempty"`
Org string `json:"org,omitempty"`
Postal string `json:"postal,omitempty"`
Timezone string `json:"timezone,omitempty"`
}
func (pn *PublicNetworkInfo) GetPublicNetworkInfo() {
url := "https://ipinfo.io"
req, err := http.NewRequest("GET", url, nil)
if err != nil {
fmt.Println(err)
return
}
req.Header.Set("Authorization", "Bearer 6ecb0db9bd8f19")
client := &http.Client{
Timeout: 5 * time.Second, // set timeout to 10 seconds
}
resp, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
fmt.Println(err)
return
}
err = json.Unmarshal(body, &pn)
if err != nil {
fmt.Println(err)
return
}
//utils.BeautifulPrint(pn)
}
type NetworkInfo struct {
Name string `json:"name"` // e.g., "en0", "lo0", "eth0.100"
MTU int `json:"mtu"` // maximum transmission unit

6
agent-go/a_status/Network_test.go
View File

@@ -42,3 +42,9 @@ func TestGetNetworkInfo(t *testing.T) {
}
}
func TestPublicNetworkInfo_GetPublicNetworkInfo(t *testing.T) {
p := &PublicNetworkInfo{}
p.GetPublicNetworkInfo()
}

8
agent-go/main.go
View File

@@ -13,11 +13,11 @@ func main() {
// 解析命令行参数
var version string
var agentServerInfoConfFile string
//var agentServerInfoConfFile string
var mode string
flag.StringVar(&version, "version", "", "config file version")
flag.StringVar(&mode, "mode", "agent", "agent run mode")
flag.StringVar(&agentServerInfoConfFile, "agentServerInfoConfFile", "", "agent server info conf file")
//flag.StringVar(&agentServerInfoConfFile, "agentServerInfoConfFile", "", "agent server info conf file")
flag.Parse()
if mode == "bastion" {
@@ -30,10 +30,10 @@ func main() {
// 读取对应版本的配置文件
octopusAgentConfigFileName := fmt.Sprintf("octopus-agent-%s.yaml", version)
fmt.Println("config file name is => " + octopusAgentConfigFileName)
fmt.Println("agent server info file is => " + agentServerInfoConfFile)
//fmt.Println("agent server info file is => " + agentServerInfoConfFile)
// 执行初始化之策工作
businessForeverChan := a_init.INIT(octopusAgentConfigFileName, agentServerInfoConfFile)
businessForeverChan := a_init.INIT(octopusAgentConfigFileName)
// 永远等待 runtime的队列消息
<-businessForeverChan

7
agent-operator/CmiiK8sOperator_test.go
View File

@@ -252,7 +252,7 @@ func TestUpdateCmiiDeploymentImageTag(t *testing.T) {
// 计算20:00的时间
now := time.Now()
targetTime := time.Date(now.Year(), now.Month(), now.Day(), 17, 45, 00, 0, now.Location())
targetTime := time.Date(now.Year(), now.Month(), now.Day(), 12, 03, 00, 0, now.Location())
duration := time.Duration(0)
@@ -275,7 +275,10 @@ func TestUpdateCmiiDeploymentImageTag(t *testing.T) {
appNameTagMap := map[string]string{
//"cmii-uav-multilink": "5.5.0",
"cmii-uav-data-post-process": "5.5.0-042501",
"cmii-uav-platform-cms-portal": "5.5.0-042801",
"cmii-uav-platform": "5.5.0-042801",
"cmii-uav-platform-oms": "5.5.0-042801",
"cmii-uav-user": "5.5.0-042801",
}
for appName, newTag := range appNameTagMap {

7
agent-operator/CmiiOperator.go
View File

@@ -193,7 +193,7 @@ func DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileName, loca
// 支持单文件的形式
if !utils.IsFileOrDir(localGzipFolderOrGzipFile) {
// 单个压缩文件
// 单个压缩文件 肯定是离线的形式
if !strings.HasSuffix(localGzipFolderOrGzipFile, ".tar.gz") {
log.ErrorF("local gzip file %s is not a .tar.gz file !", localGzipFolderOrGzipFile)
return nil
@@ -201,8 +201,7 @@ func DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileName, loca
// load
image.LoadFromGzipFilePath(localGzipFolderOrGzipFile)
}
} else {
separator := os.PathSeparator
if !strings.HasSuffix(localGzipFolderOrGzipFile, string(separator)) {
localGzipFolderOrGzipFile += string(separator)
@@ -215,8 +214,10 @@ func DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileName, loca
return nil
}
}
// load
loadAllGzipImageFromLocalFolder(localGzipFolderOrGzipFile)
}
// tag
// push

1
agent-operator/K8sOperator.go
View File

@@ -387,6 +387,7 @@ func (op *CmiiK8sOperator) DeploymentScale(cmiiEnv, appName string, scaleCount i
return true
}
func (op *CmiiK8sOperator) DeploymentUpdateTagByImageFullName(cmiiEnv, imageFullName string) bool {
// todo
split := strings.Split(imageFullName, ":")
// harbor

7
agent-operator/deploy/c_app/DeployCmiiApp.go
View File

@@ -4,6 +4,7 @@ import (
"github.com/go-playground/validator/v10"
"os"
"wdd.io/agent-common/logger"
"wdd.io/agent-common/utils"
"wdd.io/agent-operator/deploy/z_dep"
)
@@ -55,7 +56,7 @@ func init() {
func (backend *CmiiBackendConfig) BackendDeploy(common *z_dep.CommonEnvironmentConfig) bool {
// copy
z_dep.CopySameFields(common, backend)
utils.CopySameFields(common, backend)
validate := validator.New()
err := validate.Struct(backend)
@@ -84,7 +85,7 @@ func (backend *CmiiBackendConfig) BackendDeploy(common *z_dep.CommonEnvironmentC
func (frontend *CmiiFrontendConfig) FrontendDeploy(common *z_dep.CommonEnvironmentConfig) bool {
// copy
z_dep.CopySameFields(common, frontend)
utils.CopySameFields(common, frontend)
validate := validator.New()
err := validate.Struct(frontend)
@@ -106,7 +107,7 @@ func (frontend *CmiiFrontendConfig) FrontendDeploy(common *z_dep.CommonEnvironme
func (frontend *CmiiFrontendConfig) ConfigMapDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool {
// copy
z_dep.CopySameFields(commonEnv, frontend)
utils.CopySameFields(commonEnv, frontend)
// manual validate
if frontend.ShortName == "" || frontend.ClientId == "" {

14
agent-operator/deploy/z_dep/G.go
View File

@@ -4,7 +4,6 @@ import (
"bytes"
"fmt"
"github.com/go-playground/validator/v10"
"reflect"
"runtime"
"text/template"
"wdd.io/agent-common/assert"
@@ -104,16 +103,3 @@ func ParseEnvToApplyFile(environment any, applyTemplate string, applyFilePath st
return true
}
// CopySameFields 利用反射，将a中的所有同名字段的值 复制到b中的对应字段
func CopySameFields(a, b interface{}) {
va := reflect.ValueOf(a).Elem()
vb := reflect.ValueOf(b).Elem()
for i := 0; i < va.NumField(); i++ {
fieldName := va.Type().Field(i).Name
if vb.FieldByName(fieldName).IsValid() {
vb.FieldByName(fieldName).Set(va.Field(i))
}
}
}

1
agent-operator/image/CmiiImageSync.go
View File

@@ -211,6 +211,7 @@ func UploadToHarbor(targetImageName string) (uploadOK bool) {
fmt.Println()
log.InfoF("[UploadToHarbor] - upload %s success!", targetImageName)
fmt.Println()
return true
}
// TagFromListAndPushToCHarbor 需要支持 harbor.cdcyy.cn ip:8033 rancher/rancher:v2.5.7 nginx:latest

15
agent-operator/main.go
View File

@@ -7,7 +7,6 @@ import (
"os"
"strings"
"wdd.io/agent-common/utils"
"wdd.io/agent-operator/real_project/bgtg"
)
var LocalKubeConfigFile = "/root/.kube/config"
@@ -16,7 +15,8 @@ var LocalKubeConfigFile = "/root/.kube/config"
// C:\Users\wddsh\go\bin\gox.exe -osarch="linux/amd64" -output "build/agent-operator_{{.OS}}_{{.Arch}}"
// C:\Users\wddsh\go\bin\gox.exe -osarch="linux/amd64 linux/arm64" -output "build/agent-operator_{{.OS}}_{{.Arch}}"
func RealProjectRunner() {
func BuildDefaultK8sOperator() {
// build from local LocalKubeConfigFile
if !utils.FileExists(LocalKubeConfigFile) {
log.ErrorF("%s not exits! error!", LocalKubeConfigFile)
@@ -29,18 +29,19 @@ func RealProjectRunner() {
return
}
realNamespace := "bjtg"
op := CmiiK8sOperator{}
op.BuildCurrentClientFromConfig(readFile)
DefaultCmiiOperator = op
}
func RealProjectRunner() {
// ops
// update
result := UpdateCmiiImageTagFromNameTagList(realNamespace, bgtg.AllCmiiImageTagList)
//result := UpdateCmiiImageTagFromNameTagList(realNamespace, bgtg.AllCmiiImageTagList)
//result := UpdateCmiiImageTagFromNameTagMap(realNamespace, xmyd.Real540ImageTagMap)
utils.BeautifulPrint(result)
//utils.BeautifulPrint(result)
//for _, imageFullName := range xmyd.AllCmiiImageTagList {
// appName := image.ImageFullNameToAppName(imageFullName)
@@ -92,6 +93,8 @@ func main() {
flag.Parse()
if mode == "image" {
BuildDefaultK8sOperator()
// 堡垒机模式 的 镜像美容
// 初始化堡垒机模式
for {

5
cmii_operator/log/cmii-update-log.txt
View File

@@ -21,3 +21,8 @@
2024-04-25-17-42-00 uavcloud-demo cmii-uav-platform 5.5.0-042501 5.5.0-042503
2024-04-25-17-42-06 uavcloud-demo cmii-uav-platform-splice 5.5.0 5.5.0-042501
2024-04-25-17-45-00 uavcloud-demo cmii-uav-data-post-process 5.5.0 5.5.0-042501
2024-04-26-17-55-00 uavcloud-demo cmii-uav-platform-splice 5.5.0-042501 5.5.0-042601
2024-04-28-12-03-00 uavcloud-demo cmii-uav-platform-cms-portal 5.5.0 5.5.0-042801
2024-04-28-12-03-05 uavcloud-demo cmii-uav-platform 5.5.0-042503 5.5.0-042801
2024-04-28-12-03-10 uavcloud-demo cmii-uav-platform-oms 5.5.0 5.5.0-042801
2024-04-28-12-03-13 uavcloud-demo cmii-uav-user 5.5.0 5.5.0-042801

9
server/src/main/resources/META-INF/additional-spring-configuration-metadata.json Normal file
View File

@@ -0,0 +1,9 @@
{
"properties": [
{
"name": "octopus.notify.pusher_url",
"type": "java.lang.String",
"description": "Description for octopus.notify.pusher_url."
}
]
}

5
server/src/main/resources/application.yml
View File

@@ -93,6 +93,11 @@ mybatis-plus:
# mapper-locations: classpath*:/real-mappers/**/*.xml
octopus:
notify:
# 消息通知的URL地址
pusher_url: http://192.168.35.71:8080
# 发送消息的密钥
access_token: 123456
message:
# agent boot up default common exchange
init_exchange: InitExchange

4
server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java
View File

@@ -44,7 +44,7 @@ public class TestImageSyncScheduler {
ArrayList<String> ImageFullNameList = new ArrayList<>(List.of(
// "harbor.cdcyy.com.cn/cmii/cmii-live-operator:5.2.0",
// "harbor.cdcyy.com.cn/cmii/cmii/srs:v5.0.195"
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform:5.3.0-cqly-042601"
"harbor.cdcyy.com.cn/cmii/cmii-uav-industrial-portfolio:5.4.0-cqly-042802"
));
Boolean downloadAndCompressOnly = false;
@@ -102,7 +102,7 @@ public class TestImageSyncScheduler {
));
ArrayList<String> ImageFullNameList = new ArrayList<>(List.of(
"harbor.cdcyy.com.cn/cmii/cmii-uav-industrial-portfolio:5.4.0-cqly-032802"
"harbor.cdcyy.com.cn/cmii/cmii-uav-industrial-portfolio:5.4.0-cqly-042801"
));
Boolean downloadAndCompressOnly = true;