zeaslity/ProjectOctopus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[ Cmii ] [ Octopus ] - optimize octopus procedure

Browse Source
This commit is contained in:
zeaslity
2024-01-31 17:46:58 +08:00
parent 04e2712d9c
commit a83e7da008
11 changed files with 157 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
agent-go/executor/BaseFunction.go
View File

@@ -729,6 +729,7 @@ func (op *AgentOsOperator) installDockerExec(args []string) (bool, []string) {
"20",
}
}
// todo ubuntu 22.04
// ubuntu 内部
var specificDockerVersion string

27
agent-go/executor/HarborExecutor.go
View File

@@ -69,7 +69,8 @@ func (hOp *HarborOperator) Exec(baseFuncName string, funcArgs ...string) (bool,
func (hOp *HarborOperator) CreateProjectExec(funcArgs []string) (bool, []string) {
if hOp.TargetHarborClient == nil {
ok, createClient := hOp.CheckAndBuildHarborClient(funcArgs[0], true)
ok, createClient := hOp.CheckAndBuildHarborClient(funcArgs[0], "", true)
if !ok {
return false, []string{
"[Harbor Create Project] - Error !",
@@ -116,7 +117,7 @@ func (hOp *HarborOperator) CreateProjectExec(funcArgs []string) (bool, []string)
return true, []string{successLog}
}
func (hOp *HarborOperator) CheckAndBuildHarborClient(targetHarborHost string, isTarget bool) (bool, *apiv2.RESTClient) {
func (hOp *HarborOperator) CheckAndBuildHarborClient(targetHarborHost string, targetHarborPort string, isTarget bool) (bool, *apiv2.RESTClient) {
log.InfoF("[Harbor Client Create] - start to create harbor client %s", targetHarborHost)
@@ -127,12 +128,15 @@ func (hOp *HarborOperator) CheckAndBuildHarborClient(targetHarborHost string, is
)
return false, nil
}
if targetHarborPort == "" {
targetHarborPort = hOp.HarborPort
}
if isTarget {
hOp.TargetHarborHost = "http://" + targetHarborHost + ":" + hOp.HarborPort + "/api/"
hOp.TargetHarborHost = "http://" + targetHarborHost + ":" + targetHarborPort + "/api/"
log.DebugF("[Harbor Client Create] - harbor host is => %s", hOp.TargetHarborHost)
} else {
hOp.SourceHarborHost = "http://" + targetHarborHost + ":" + hOp.HarborPort + "/api/"
hOp.SourceHarborHost = "http://" + targetHarborHost + ":" + targetHarborPort + "/api/"
log.DebugF("[Harbor Client Create] - harbor host is => %s", hOp.SourceHarborHost)
}
@@ -156,7 +160,7 @@ func (hOp *HarborOperator) CheckAndBuildHarborClient(targetHarborHost string, is
func (hOp *HarborOperator) ListProjectExec(funcArgs []string) (bool, []string) {
if hOp.TargetHarborClient == nil {
ok, createClient := hOp.CheckAndBuildHarborClient(funcArgs[0], true)
ok, createClient := hOp.CheckAndBuildHarborClient(funcArgs[0], "", true)
if !ok {
return false, []string{
"[Harbor Create Project ] - Error !",
@@ -190,7 +194,7 @@ func (hOp *HarborOperator) ListProjectExec(funcArgs []string) (bool, []string) {
func (hOp *HarborOperator) SyncProjectExec(funcArgs []string) (bool, []string) {
if hOp.TargetHarborClient == nil {
ok, createClient := hOp.CheckAndBuildHarborClient(funcArgs[0], true)
ok, createClient := hOp.CheckAndBuildHarborClient(funcArgs[0], "", true)
if !ok {
return false, []string{
"[Harbor Sync Project ] - Error !",
@@ -201,7 +205,14 @@ func (hOp *HarborOperator) SyncProjectExec(funcArgs []string) (bool, []string) {
targetClient := hOp.TargetHarborClient
if hOp.SourceHarborClient == nil {
ok, createClient := hOp.CheckAndBuildHarborClient(funcArgs[1], false)
realHost := funcArgs[1]
realPort := ""
if strings.Contains(funcArgs[1], ":") {
split := strings.Split(funcArgs[1], ":")
realHost = split[0]
realPort = split[1]
}
ok, createClient := hOp.CheckAndBuildHarborClient(realHost, realPort, false)
if !ok {
return false, []string{
"[Harbor Sync Project ] - Error !",
@@ -355,7 +366,7 @@ func (hOp *HarborOperator) SyncProjectExec(funcArgs []string) (bool, []string) {
func (hOp *HarborOperator) SyncStatusExec(funcArgs []string) (bool, []string) {
if hOp.TargetHarborClient == nil {
ok, createClient := hOp.CheckAndBuildHarborClient(funcArgs[0], true)
ok, createClient := hOp.CheckAndBuildHarborClient(funcArgs[0], "", true)
if !ok {
return false, []string{
"[ Sync Status ] - Error !",

1
cmii_operator/CmiiImageSync.go
View File

@@ -422,6 +422,7 @@ func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) {
gzipFileName += "="
gzipFileName += first[2]
gzipFileName += "="
} else if len(first) == 2 {
//
gzipFileName += "docker="

8
server/src/main/java/io/wdd/func/auto/beans/BaseFunctionEnum.java
View File

@@ -1,5 +1,9 @@
package io.wdd.func.auto.beans;
/**
* 参数固定顺序为 A1C2IP
*/
public enum BaseFunctionEnum {
SHUTDOWN_FIREWALL(
@@ -35,7 +39,7 @@ public enum BaseFunctionEnum {
INSTALL_DOCKER(
"installDocker",
"安装Docker, args固定为19或20"
"安装Docker, 默认安装20.10.15版本"
),
INSTALL_DOCKER_COMPOSE(
@@ -66,7 +70,7 @@ public enum BaseFunctionEnum {
INSTALL_CHRONY_DOCKER(
"INSTALL_CHRONY_DOCKER",
"安装Chrony服务器, DOCKER版本"
"安装Chrony服务器的DOCKER版本"
),
CHRONY_TO_PUBLIC_NTP_DISCARD(

2
server/src/main/java/io/wdd/func/auto/beans/HarborFunctionEnum.java
View File

@@ -16,7 +16,7 @@ public enum HarborFunctionEnum {
SYNC_PROJECT_BETWEEN_HARBOR(
"SYNC_PROJECT_BETWEEN_HARBOR",
"在源Harbor和目标Harbor之间同步Project， args 为目标Harbor的HostName，不带端口; 源Harbor的HostName，不带端口; 需要同步的仓库project(rancher cmii)名称"
"在源Harbor和目标Harbor之间同步Project， args 为目标Harbor的HostName，不带端口; 源Harbor的HostName，可以自定义端口; 需要同步的仓库project(rancher cmii)名称"
),
SYNC_STATUS_HARBOR(

6
server/src/main/java/io/wdd/func/auto/beans/ProjectDeployContext.java
View File

@@ -35,5 +35,11 @@ public class ProjectDeployContext {
String currentSynchronizingProject;
// app
List<BaseFunctionEnum> masterNodeProcedure;
List<BaseFunctionEnum> agentNodeProcedure;
ArrayList<String> baseFunctionArgs;
ArrayList<String> appFunctionArgs;
}

148
server/src/main/java/io/wdd/func/auto/service/BaseFuncScheduler.java
View File

@@ -45,28 +45,35 @@ public class BaseFuncScheduler {
private void doRunProcedure(ProjectDeployContext projectDeployContext) {
if (!MasterNodeBaseProcedure(projectDeployContext)) {
if (!MasterNodeProcedure(projectDeployContext)) {
log.error("Master 初始化过程错误! 即将终止!");
}
if (!CollectionUtils.isEmpty(projectDeployContext
.getAgentNodeList())) {
List<String> agentTopicNameList = projectDeployContext
.getAgentNodeList()
.stream()
.map(agentNode -> agentNode.getTopicName())
.collect(Collectors.toList());
if (!AgentNodeProcedure(agentTopicNameList)) {
log.error("Agent 初始化过程错误! 即将终止!");
}
if (!AgentNodeProcedure(projectDeployContext)) {
log.error("Agent 初始化过程错误! 即将终止!");
}
log.info("初始化过程完成!");
}
private boolean MasterNodeProcedure(ProjectDeployContext projectDeployContext) {
String masterTopicName = projectDeployContext
.getMasterNode()
.getTopicName();
if (agentNodeRunProcedure(
true,
masterTopicName,
projectDeployContext
)) {
return true;
}
return false;
}
private void beforeRunProcedure(ProjectDeployContext projectDeployContext) {
@@ -76,68 +83,67 @@ public class BaseFuncScheduler {
// 打印施工信息
// base function args
// 执行 基础功能施工的内容
// base func args
ArrayList<String> baseFuncArgList = new ArrayList<>();
baseFuncArgList.add("20");
baseFuncArgList.add(projectDeployContext
.getMasterNode()
.getServerIpInV4());
projectDeployContext.setBaseFunctionArgs(baseFuncArgList);
log.debug("beforeRunProcedure complete!");
}
private boolean MasterNodeBaseProcedure(ProjectDeployContext projectDeployContext) {
// 执行 基础功能施工的内容
String masterTopicName = projectDeployContext
.getMasterNode()
.getTopicName();
// base func args
ArrayList<String> baseFuncArgList = new ArrayList<>();
baseFuncArgList.add(projectDeployContext
.getMasterNode()
.getServerIpInV4());
List<BaseFunctionEnum> masterNodeProcedureList = List.of(
BaseFunctionEnum.DISABLE_SWAP,
BaseFunctionEnum.SHUTDOWN_FIREWALL,
BaseFunctionEnum.DISABLE_SELINUX
);
for (BaseFunctionEnum procedure : masterNodeProcedureList) {
// add op name
baseFuncArgList.add(
0,
procedure.getFuncName()
);
if (!funcService.callBaseFuncAndJudge(
masterTopicName,
procedure,
baseFuncArgList,
false
)) {
log.error(
"MasterNode初始化错误 => {}",
procedure
);
return false;
}
// remove first args
baseFuncArgList.remove(0);
private boolean AgentNodeProcedure(ProjectDeployContext projectDeployContext) {
if (CollectionUtils.isEmpty(projectDeployContext.getAgentNodeList())) {
return true;
}
List<BaseFunctionEnum> masterNodeDurationTaskList = List.of(
BaseFunctionEnum.INSTALL_DOCKER,
BaseFunctionEnum.INSTALL_DOCKER_COMPOSE,
BaseFunctionEnum.MODIFY_DOCKER_CONFIG
// BaseFunctionEnum.INSTALL_HARBOR
List<String> agentTopicNameList = projectDeployContext
.getAgentNodeList()
.stream()
.map(agentNode -> agentNode.getTopicName())
.collect(Collectors.toList());
agentTopicNameList.forEach(
agentTopicName -> {
if (!agentNodeRunProcedure(
false,
agentTopicName,
projectDeployContext
)) {
log.error("{} agent run base func failed !", agentTopicName);
}
}
);
for (BaseFunctionEnum durationBaseFunc : masterNodeDurationTaskList) {
return true;
}
private boolean agentNodeRunProcedure(boolean masterNode, String agentTopicName, ProjectDeployContext projectDeployContext) {
List<BaseFunctionEnum> agentNodeProcedureList;
if (masterNode) {
agentNodeProcedureList = projectDeployContext.getMasterNodeProcedure();
} else {
agentNodeProcedureList = projectDeployContext.getAgentNodeProcedure();
}
if (CollectionUtils.isEmpty(agentNodeProcedureList)) {
return true;
}
ArrayList<String> baseFuncArgList = projectDeployContext.getBaseFunctionArgs();
for (BaseFunctionEnum durationBaseFunc : agentNodeProcedureList) {
// add op name
baseFuncArgList.add(
@@ -146,13 +152,13 @@ public class BaseFuncScheduler {
);
if (!funcService.callBaseFuncAndJudge(
masterTopicName,
agentTopicName,
durationBaseFunc,
baseFuncArgList,
true
)) {
log.error(
"Master Duration Task Failed ! => {}",
"Agent Base Func Failed ! => {}",
durationBaseFunc
);
return false;
@@ -166,9 +172,5 @@ public class BaseFuncScheduler {
}
private boolean AgentNodeProcedure(List<String> agentTopicNameList) {
return true;
}
}

2
server/src/main/java/io/wdd/func/auto/service/HarborFuncScheduler.java
View File

@@ -179,9 +179,9 @@ public class HarborFuncScheduler {
ArrayList<String> syncHarborArgList = new ArrayList<>();
// zero arg
syncHarborArgList.add(HarborFunctionEnum.SYNC_PROJECT_BETWEEN_HARBOR.getOpName());
// first func args
syncHarborArgList.add(masterNode.getServerIpInV4());
// second arg sourceHarborHost
// String sourceHarborHost = getIPv4Address();
syncHarborArgList.add("harbor.wdd.io");

2
server/src/main/java/io/wdd/rpc/execute/service/ExecutionServiceImpl.java
View File

@@ -27,7 +27,7 @@ public class ExecutionServiceImpl implements ExecutionService {
private static final String MANUAL_COMMAND_TYPE = "manual-command";
private static final int COMMAND_MAX_WAIT_TIMEOUT = 120;
private static final int COMMAND_MAX_WAIT_TIMEOUT = 300;
@Resource
OMessageToAgentSender oMessageToAgentSender;

12
server/src/main/resources/application.yml
View File

@@ -1,15 +1,15 @@
server:
port: 9999
# shanghai 20.4.16.200
# local 20.4.16.200
# cqga 20.4.16.200
# shanghai 42.192.52.227
# local 42.192.52.227
# cqga 42.192.52.227
spring:
main:
allow-circular-references: true
allow-bean-definition-overriding: true
rabbitmq:
host: 20.4.16.200
host: 42.192.52.227
port: 20672
username: boge
password: boge8tingH
@@ -24,7 +24,7 @@ spring:
max-interval: 65000
initial-interval: 65000
redis:
host: 20.4.16.200
host: 42.192.52.227
port: 21370
database: 0
password: boge8tingH
@@ -47,7 +47,7 @@ spring:
datasource:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://20.4.16.200:21306/octopus_server?autoReconnect=true&useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2B8
url: jdbc:mysql://42.192.52.227:21306/octopus_server?autoReconnect=true&useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2B8
username: boge
password: boge8tingH
type: com.zaxxer.hikari.HikariDataSource

56
server/src/test/java/io/wdd/server/func/TestBaseFuncScheduler.java
View File

@@ -1,5 +1,6 @@
package io.wdd.server.func;
import io.wdd.func.auto.beans.BaseFunctionEnum;
import io.wdd.func.auto.beans.ProjectDeployContext;
import io.wdd.func.auto.service.AppFuncScheduler;
import io.wdd.func.auto.service.BaseFuncScheduler;
@@ -13,6 +14,8 @@ import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@SpringBootTest
@@ -38,8 +41,9 @@ public class TestBaseFuncScheduler {
ProjectDeployContext projectDeployContext = new ProjectDeployContext();
Long projectServerId = 1751084188582440961L;// cqga
// Long projectServerId = 1751084188582440961L;// cqga
// Long projectServerId = 1722453318596550657L;// lappro
Long projectServerId = 1752602668144975873L;// cqlyj
projectDeployContext.setProjectId(projectServerId);
@@ -57,17 +61,17 @@ public class TestBaseFuncScheduler {
// .getRecords()
// .get(0);
String serverName = "Chongqing-amd64-01"; // cgga
// String serverName = "Chengdu-amd64-99"; // lap pro
String masterNodeServerName = "Chongqing-amd64-01"; // cgga
// String masterNodeServerName = "Chengdu-amd64-99"; // lap pro
ProjectServerVO projectServerVO = coreProjectServerService.projectServerOne(projectServerId);
Optional<ServerInfoPO> serverInfoPOOptional = projectServerVO.getBindingServerList().stream().filter(
serverInfoPO -> StringUtils.contains(serverInfoPO.getServerName(), serverName)
serverInfoPO -> StringUtils.contains(serverInfoPO.getServerName(), masterNodeServerName)
).findFirst();
if (serverInfoPOOptional.isEmpty()) {
System.out.printf("project of %s server of %s is empty", projectServerVO, serverName);
System.out.printf("project of %s server of %s is empty", projectServerVO, masterNodeServerName);
return;
}
@@ -75,22 +79,42 @@ public class TestBaseFuncScheduler {
System.out.println("serverInfoPO = " + serverInfoPO);
projectDeployContext.setMasterNode(serverInfoPO);
ArrayList<ServerInfoPO> agentNodeList = new ArrayList<>();
// baseFuncScheduler.runProcedure(projectDeployContext);
projectServerVO.getBindingServerList().forEach(
po -> {
if (!StringUtils.contains(po.getServerName(), masterNodeServerName)) {
agentNodeList.add(po);
}
}
);
projectDeployContext.setAgentNodeList(agentNodeList);
List<BaseFunctionEnum> masterNodeProcedure = List.of(
// BaseFunctionEnum.INSTALL_DOCKER,
BaseFunctionEnum.INSTALL_DOCKER_COMPOSE,
BaseFunctionEnum.MODIFY_DOCKER_CONFIG,
BaseFunctionEnum.INSTALL_HARBOR
);
List<BaseFunctionEnum> agentNodeProcedure = List.of(
BaseFunctionEnum.DISABLE_SWAP,
BaseFunctionEnum.SHUTDOWN_FIREWALL,
BaseFunctionEnum.INSTALL_DOCKER,
BaseFunctionEnum.INSTALL_DOCKER_COMPOSE
// BaseFunctionEnum.MODIFY_DOCKER_CONFIG
);
projectDeployContext.setAgentNodeProcedure(agentNodeProcedure);
baseFuncScheduler.runProcedure(projectDeployContext);
// projectServerVO.getBindingServerList().forEach(
// serverInfoPO1 -> {
// if (!StringUtils.contains(serverInfoPO1.getServerName(), "01")) {
// projectDeployContext.setMasterNode(serverInfoPO1);
//
// baseFuncScheduler.runProcedure(projectDeployContext);
// }
// }
// );
// harborFuncScheduler.runProcedure(projectDeployContext);
appFuncScheduler.runProcedure(projectDeployContext);
// appFuncScheduler.runProcedure(projectDeployContext);
}
}