From cdfa4b5d0406a2ef652b4ef4c18894699b78b200 Mon Sep 17 00:00:00 2001 From: zeaslity Date: Thu, 9 Nov 2023 13:58:58 +0800 Subject: [PATCH] [ Agent ] [ Initialization ] - add harbor func - 1 --- agent-go/executor/BaseFunction.go | 16 +-- agent-go/executor/InitFunction.go | 9 +- .../func/auto/service/BaseFuncScheduler.java | 17 ++- .../io/wdd/func/auto/service/FuncService.java | 3 +- .../func/auto/service/FuncServiceImpl.java | 15 ++- .../execute/service/ExecutionServiceImpl.java | 107 +++++++++--------- .../impl/CoreServerServiceImpl.java | 48 ++++---- .../server/func/TestBaseFuncScheduler.java | 6 +- 8 files changed, 120 insertions(+), 101 deletions(-) diff --git a/agent-go/executor/BaseFunction.go b/agent-go/executor/BaseFunction.go index 1304b10..5e4b401 100644 --- a/agent-go/executor/BaseFunction.go +++ b/agent-go/executor/BaseFunction.go @@ -1,7 +1,6 @@ package executor import ( - "agent-go/g" "agent-go/register" "fmt" "strings" @@ -802,7 +801,7 @@ func (op *AgentOsOperator) installHelm() [][]string { { "wget", "--no-check-certificate", - g.BaseFuncOssUrlPrefix + "helm-v3.12.1-linux-amd64.tar.gz", + op.OssOfflinePrefix + "helm-v3.12.1-linux-amd64.tar.gz", "-O", "/root/wdd/helm-v3.12.1-linux-amd64.tar.gz", }, @@ -873,7 +872,7 @@ func (op *AgentOsOperator) modifyDockerConfig(args []string) [][]string { }, { "wget", - g.BaseFuncOssUrlPrefix + "daemon-config.json", + op.OssOfflinePrefix + "daemon-config.json", "-O", "/etc/docker/daemon.json", }, @@ -913,7 +912,7 @@ func (op *AgentOsOperator) installHarbor() [][]string { //{ // "wget", // "--no-check-certificate", - // g.BaseFuncOssUrlPrefix + "harbor-offline-installer-v2.1.0.tgz", + // op.OssOfflinePrefix + "harbor-offline-installer-v2.1.0.tgz", // "-O", // "/root/wdd/harbor-offline-installer-v2.1.0.tgz", //}, @@ -932,7 +931,7 @@ func (op *AgentOsOperator) installHarbor() [][]string { { "wget", "--no-check-certificate", - g.BaseFuncOssUrlPrefix + "harbor-config-template.yml", + op.OssOfflinePrefix + "harbor-config-template.yml", "-O", "/root/wdd/harbor/harbor.yml", }, @@ -1009,7 +1008,8 @@ func (op *AgentOsOperator) installHarborExec() (bool, []string) { } // download offline file - log.Info("[install harbor] - start to download harbor offline installer !") + harborOfflineFileURL := op.OssOfflinePrefix + "harbor-offline-installer-v2.1.0.tgz" + log.InfoF("[install harbor] - start to download harbor offline installer from => %s !", harborOfflineFileURL) resultOk, resultLog := AllCompleteExecutor([][]string{ { "mkdir", @@ -1029,7 +1029,7 @@ func (op *AgentOsOperator) installHarborExec() (bool, []string) { { "wget", "--no-check-certificate", - g.BaseFuncOssUrlPrefix + "harbor-offline-installer-v2.1.0.tgz", + harborOfflineFileURL, "-qO", "/root/wdd/harbor-offline-installer-v2.1.0.tgz", }}, @@ -1063,7 +1063,7 @@ func (op *AgentOsOperator) installHarborExec() (bool, []string) { AllCommandExecutor([]string{ "wget", "--no-check-certificate", - g.BaseFuncOssUrlPrefix + "harbor-config-template.yml", + op.OssOfflinePrefix + "harbor-config-template.yml", "-qO", "/root/wdd/harbor/harbor.yml", }) diff --git a/agent-go/executor/InitFunction.go b/agent-go/executor/InitFunction.go index 7a5653f..2ae464e 100644 --- a/agent-go/executor/InitFunction.go +++ b/agent-go/executor/InitFunction.go @@ -1,6 +1,9 @@ package executor -import "strings" +import ( + "strconv" + "strings" +) func BuildAgentOsOperator(osInfo string, ossOfflinePrefix string) *AgentOsOperator { @@ -84,8 +87,6 @@ func detectByOsType(os *AgentOsOperator, osInfo string) { func detectByInternet(os *AgentOsOperator) { - log.Info("开始检测Agent的网络连通情况!") - outsideTestUrl := "www.google.com" innerTestUrl := "www.baidu.com" @@ -109,4 +110,6 @@ func detectByInternet(os *AgentOsOperator) { os.IsAgentInnerWall = true } + log.InfoF("[Agent Network Status] - Can Access Internet => %s Inner CN => %s", strconv.FormatBool(os.CanAccessInternet), strconv.FormatBool(os.IsAgentInnerWall)) + } diff --git a/server/src/main/java/io/wdd/func/auto/service/BaseFuncScheduler.java b/server/src/main/java/io/wdd/func/auto/service/BaseFuncScheduler.java index bef7375..db1c09e 100644 --- a/server/src/main/java/io/wdd/func/auto/service/BaseFuncScheduler.java +++ b/server/src/main/java/io/wdd/func/auto/service/BaseFuncScheduler.java @@ -92,10 +92,9 @@ public class BaseFuncScheduler { // BaseFunctionEnum.CHRONY_TO_PUBLIC_NTP, // BaseFunctionEnum.INSTALL_DEFAULT_SSH_KEY, // BaseFunctionEnum.INSTALL_DOCKER, -// BaseFunctionEnum.INSTALL_DOCKER_COMPOSE, +// BaseFunctionEnum.INSTALL_DOCKER_COMPOSE // BaseFunctionEnum.INSTALL_HARBOR, // BaseFunctionEnum.INSTALL_ZSH - BaseFunctionEnum.INSTALL_HARBOR ); @@ -103,7 +102,8 @@ public class BaseFuncScheduler { if (!funcService.callBaseFuncAndJudge( masterTopicName, procedure, - null + null, + false )) { log.error( @@ -115,6 +115,17 @@ public class BaseFuncScheduler { } } + + if (!funcService.callBaseFuncAndJudge( + masterTopicName, + BaseFunctionEnum.INSTALL_HARBOR, + null, + true + )) { + log.error("Master Install Harbor Failed !"); + return false; + } + return true; } diff --git a/server/src/main/java/io/wdd/func/auto/service/FuncService.java b/server/src/main/java/io/wdd/func/auto/service/FuncService.java index 60a5462..280131d 100644 --- a/server/src/main/java/io/wdd/func/auto/service/FuncService.java +++ b/server/src/main/java/io/wdd/func/auto/service/FuncService.java @@ -18,8 +18,9 @@ public interface FuncService { * * @param baseFunctionEnum * @param funcArgs + * @param durationTask * @return */ - boolean callBaseFuncAndJudge(String agentTopicName, BaseFunctionEnum baseFunctionEnum, List funcArgs); + boolean callBaseFuncAndJudge(String agentTopicName, BaseFunctionEnum baseFunctionEnum, List funcArgs, boolean durationTask); } diff --git a/server/src/main/java/io/wdd/func/auto/service/FuncServiceImpl.java b/server/src/main/java/io/wdd/func/auto/service/FuncServiceImpl.java index bb3dfce..a3486de 100644 --- a/server/src/main/java/io/wdd/func/auto/service/FuncServiceImpl.java +++ b/server/src/main/java/io/wdd/func/auto/service/FuncServiceImpl.java @@ -26,7 +26,8 @@ public class FuncServiceImpl implements FuncService { agentTopicName, ExecutionMessageType.BASE, baseFunctionName, - funcArgs + funcArgs, + false ); } @@ -37,12 +38,13 @@ public class FuncServiceImpl implements FuncService { agentTopicName, ExecutionMessageType.APP, appFunctionName, - funcArgs + funcArgs, + false ); } @Override - public boolean callBaseFuncAndJudge(String agentTopicName, BaseFunctionEnum baseFunctionEnum, List funcArgs) { + public boolean callBaseFuncAndJudge(String agentTopicName, BaseFunctionEnum baseFunctionEnum, List funcArgs, boolean durationTask) { log.debug( "调用基本脚本 => {}", @@ -53,7 +55,8 @@ public class FuncServiceImpl implements FuncService { agentTopicName, ExecutionMessageType.BASE, baseFunctionEnum.getFuncName(), - funcArgs + funcArgs, + durationTask ); if (CollectionUtils.isEmpty(syncResultLog)) { @@ -83,7 +86,7 @@ public class FuncServiceImpl implements FuncService { // 归一化调用 - private List syncCallFunction(String agentTopicName, ExecutionMessageType emType, String funcName, List funcArgs) { + private List syncCallFunction(String agentTopicName, ExecutionMessageType emType, String funcName, List funcArgs, boolean durationTask) { // 重新构造内容，增加Function Name if (CollectionUtils.isEmpty(funcArgs)) { @@ -103,7 +106,7 @@ public class FuncServiceImpl implements FuncService { null, null, true, - false + durationTask ); return resultLog; diff --git a/server/src/main/java/io/wdd/rpc/execute/service/ExecutionServiceImpl.java b/server/src/main/java/io/wdd/rpc/execute/service/ExecutionServiceImpl.java index 8107279..2da695a 100644 --- a/server/src/main/java/io/wdd/rpc/execute/service/ExecutionServiceImpl.java +++ b/server/src/main/java/io/wdd/rpc/execute/service/ExecutionServiceImpl.java @@ -76,66 +76,69 @@ public class ExecutionServiceImpl implements ExecutionService { ); // 需要返回结果 - if (!durationTask) { - // 等待结果 - // countDownLatch的个数约定为1, agent会合并多行命令的结果一并返回 - OMessageReplayContent replayContent = WaitFromAgent( - octopusMessage, - 1 - ); - CountDownLatch replayLatch = replayContent.getCountDownLatch(); - boolean waitOK = false; + // 超时等待时间，默认为 COMMAND_MAX_WAIT_TIMEOUT + int commandMaxWaitTimeout = COMMAND_MAX_WAIT_TIMEOUT; + if (durationTask) { + // 2023年11月9日 超长的任务时间设置为5分钟 + commandMaxWaitTimeout = COMMAND_MAX_WAIT_TIMEOUT * 5; + } + // 等待结果 + // countDownLatch的个数约定为1, agent会合并多行命令的结果一并返回 + OMessageReplayContent replayContent = WaitFromAgent( + octopusMessage, + 1 + ); - try { + CountDownLatch replayLatch = replayContent.getCountDownLatch(); + boolean waitOK = false; - // 2023年10月8日 根据执行的命令数量设定超时等待时间 - int commandMaxWaitTimeout = COMMAND_MAX_WAIT_TIMEOUT; - if (!CollectionUtils.isEmpty(commandListComplete)) { - commandMaxWaitTimeout = COMMAND_MAX_WAIT_TIMEOUT * commandListComplete.size(); - } - - waitOK = replayLatch.await( - commandMaxWaitTimeout, - TimeUnit.SECONDS - ); - - } catch (InterruptedException e) { - throw new RuntimeException(e); - } finally { - - // 释放等待队列 - StopWaitingResult(octopusMessage); - - // 转换结果 - commandResultLog = (ArrayList) octopusMessage.getResult(); - - - Object executionContent; - if (executionMessage - .getExecutionType() - .equals("BASE")) { - executionContent = executionMessage.getExecutionType() + "==" + executionMessage.getFuncContent(); - } else if (executionMessage.getSingleLineCommand() != null) { - executionContent = executionMessage.getSingleLineCommand(); - } else { - executionContent = executionMessage.getMultiLineCommand(); - } - - - // debug - log.debug( - "执行命令 {} {} 在规定时间内结束， 结果为 {} 返回内容为 {}", - executionContent, - waitOK ? "已经" : "未", - octopusMessage.getResultCode(), - commandResultLog - ); + try { + // 2023年10月8日 根据执行的命令数量设定超时等待时间 + if (!CollectionUtils.isEmpty(commandListComplete)) { + commandMaxWaitTimeout = COMMAND_MAX_WAIT_TIMEOUT * commandListComplete.size(); } + waitOK = replayLatch.await( + commandMaxWaitTimeout, + TimeUnit.SECONDS + ); + + } catch (InterruptedException e) { + throw new RuntimeException(e); + } finally { + + // 释放等待队列 + StopWaitingResult(octopusMessage); + + // 转换结果 + commandResultLog = (ArrayList) octopusMessage.getResult(); + + + Object executionContent; + if (executionMessage + .getExecutionType() + .equals("BASE")) { + executionContent = executionMessage.getExecutionType() + "==" + executionMessage.getFuncContent(); + } else if (executionMessage.getSingleLineCommand() != null) { + executionContent = executionMessage.getSingleLineCommand(); + } else { + executionContent = executionMessage.getMultiLineCommand(); + } + + // debug + log.debug( + "执行命令 {} {} 在规定时间内结束， 结果为 {} 返回内容为 {}", + executionContent, + waitOK ? "已经" : "未", + octopusMessage.getResultCode(), + commandResultLog + ); + } + return commandResultLog; } diff --git a/server/src/main/java/io/wdd/server/coreService/impl/CoreServerServiceImpl.java b/server/src/main/java/io/wdd/server/coreService/impl/CoreServerServiceImpl.java index 06e71dd..623f661 100644 --- a/server/src/main/java/io/wdd/server/coreService/impl/CoreServerServiceImpl.java +++ b/server/src/main/java/io/wdd/server/coreService/impl/CoreServerServiceImpl.java @@ -1,6 +1,5 @@ package io.wdd.server.coreService.impl; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.extension.conditions.query.LambdaQueryChainWrapper; import com.baomidou.mybatisplus.extension.conditions.update.LambdaUpdateChainWrapper; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @@ -88,14 +87,17 @@ public class CoreServerServiceImpl implements CoreServerService { public Page serverGetByPage(ServerQueryEntity serverQueryEntity) { // 默认数值处理 - if (serverQueryEntity == null || serverQueryEntity.getPageSize() == null || serverQueryEntity.getPageNumber() == null) { + if (serverQueryEntity == null) { serverQueryEntity = ServerQueryEntity .builder() .pageNumber(1) .pageSize(10) .build(); - } else { + } + if (serverQueryEntity.getPageSize() == null || serverQueryEntity.getPageNumber() == null) { + serverQueryEntity.setPageSize(10); + serverQueryEntity.setPageNumber(1); } Page serverInfoPOPage = new Page<>( @@ -104,29 +106,23 @@ public class CoreServerServiceImpl implements CoreServerService { ); // 查询Page，会直接写入到serverInfoPOPage中 - serverInfoService - .getBaseMapper() - .selectPage( - serverInfoPOPage, - new QueryWrapper() - .like( - StringUtils.isNoneEmpty(serverQueryEntity.getServerName()), - "server_name", - serverQueryEntity.getServerName() - ) - .likeRight( - StringUtils.isNoneEmpty(serverQueryEntity.getServerIPv4()), - "server_ip_pb_v4", - serverQueryEntity.getServerIPv4() - ) - .likeRight( - StringUtils.isNoneEmpty(serverQueryEntity.getServerLocation()), - "location", - serverQueryEntity.getServerLocation() - ) - .orderByAsc("server_id") - ); - + new LambdaQueryChainWrapper(serverInfoService.getBaseMapper()) + .likeRight( + StringUtils.isNotEmpty(serverQueryEntity.getServerName()), + ServerInfoPO::getServerName, + serverQueryEntity.getServerName() + ) + .likeRight( + StringUtils.isNotEmpty(serverQueryEntity.getServerIPv4()), + ServerInfoPO::getServerIpPbV4, + serverQueryEntity.getServerIPv4() + ) + .likeRight( + StringUtils.isNotEmpty(serverQueryEntity.getServerLocation()), + ServerInfoPO::getLocation, + serverQueryEntity.getServerLocation() + ) + .page(serverInfoPOPage); // 将查询结果中的User对象转换为UserVO对象 /*Page serverInfoVOPage = new Page<>( diff --git a/server/src/test/java/io/wdd/server/func/TestBaseFuncScheduler.java b/server/src/test/java/io/wdd/server/func/TestBaseFuncScheduler.java index 2563413..1f78f35 100644 --- a/server/src/test/java/io/wdd/server/func/TestBaseFuncScheduler.java +++ b/server/src/test/java/io/wdd/server/func/TestBaseFuncScheduler.java @@ -24,10 +24,12 @@ public class TestBaseFuncScheduler { ProjectDeployContext projectDeployContext = new ProjectDeployContext(); - projectDeployContext.setProjectId(1716372290994155522L); +// projectDeployContext.setProjectId(1716372290994155522L); // vultr + projectDeployContext.setProjectId(1722453318596550657L); // lappro + ServerQueryEntity serverQueryEntity = new ServerQueryEntity(); - serverQueryEntity.setServerName("Paripark"); + serverQueryEntity.setServerName("Chengdu"); ServerInfoPO serverInfoPO = serverService .serverGetByPage(serverQueryEntity) .getRecords()