zeaslity/ProjectOctopus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[ Cmii ] [ Octopus ] - add Image Function Scheduler - 1

Browse Source
This commit is contained in:
zeaslity
2024-03-27 10:56:50 +08:00
parent cde509d137
commit 08c526c4a4
17 changed files with 373 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
agent-go/executor/CommandExecutor.go
View File

@@ -61,6 +61,16 @@ func Execute(em *ExecutionMessage) (bool, []string) {
// Harbor Execute
ok, resultLog = HarborOperatorCache.Exec(em.FuncContent[0], em.FuncContent[1:]...)
} else if strings.HasPrefix(em.ExecutionType, "IMAGE") {
// image function
if em.FuncContent == nil || len(em.FuncContent) <= 1 {
ok = false
resultLog = []string{
"[Harbor Execute] - functions args is wrong!",
}
}
// Harbor Execute
ok, resultLog = AgentOsOperatorCache.Sync(em.FuncContent[0], em.FuncContent[1:]...)
} else {
// deprecated
// shell command

32
server/src/main/java/io/wdd/func/auto/beans/ImageSyncContext.java Normal file
View File

@@ -0,0 +1,32 @@
package io.wdd.func.auto.beans;
import io.wdd.server.beans.po.ProjectInfoPO;
import io.wdd.server.beans.po.ServerInfoPO;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
import java.util.ArrayList;
@Data
@SuperBuilder(toBuilder = true)
@AllArgsConstructor
@NoArgsConstructor
public class ImageSyncContext {
// single multi file
ArrayList<String> cmiiAppNameTagList;
ArrayList<String> imageFullNameList;
// function args
ArrayList<String> imageSyncFuncArgs;
// target
ProjectInfoPO projectInfoPO;
ServerInfoPO projectMasterNode;
// 我方的 机器 默认值为 192.168.35.71
String innerWorkerAgentName;
}

6
server/src/main/java/io/wdd/func/auto/service/FuncService.java
View File

@@ -4,6 +4,7 @@ package io.wdd.func.auto.service;
import io.wdd.func.auto.beans.AppFunctionEnum;
import io.wdd.func.auto.beans.BaseFunctionEnum;
import io.wdd.func.auto.beans.HarborFunctionEnum;
import io.wdd.func.auto.beans.ImageFunctionEnum;
import java.util.ArrayList;
import java.util.List;
@@ -29,4 +30,9 @@ public interface FuncService {
boolean callAppFuncAndJudge(String agentTopicName, AppFunctionEnum appFunctionEnum, List<String> funcArgs);
boolean callHarborFuncAndJudge(String agentTopicName, HarborFunctionEnum harborFunctionEnum, ArrayList<String> funcArgs);
List<String> callImageFuncService(String agentTopicName, ImageFunctionEnum imageFunctionEnum, List<String> funcArgs);
}

45
server/src/main/java/io/wdd/func/auto/service/FuncServiceImpl.java
View File

@@ -3,6 +3,7 @@ package io.wdd.func.auto.service;
import io.wdd.func.auto.beans.AppFunctionEnum;
import io.wdd.func.auto.beans.BaseFunctionEnum;
import io.wdd.func.auto.beans.HarborFunctionEnum;
import io.wdd.func.auto.beans.ImageFunctionEnum;
import io.wdd.rpc.execute.ExecutionMessageType;
import io.wdd.rpc.execute.service.ExecutionService;
import lombok.extern.slf4j.Slf4j;
@@ -75,22 +76,7 @@ public class FuncServiceImpl implements FuncService {
return appFuncResult;
}
@Override
public boolean callHarborFuncAndJudge(String agentTopicName, HarborFunctionEnum harborFunctionEnum, ArrayList<String> funcArgs) {
List<String> syncResultLog = syncCallFunction(
agentTopicName,
ExecutionMessageType.HARBOR,
funcArgs,
true
);
boolean harborResult = JudgeSyncBaseCommandResult(syncResultLog);
return harborResult;
}
private boolean JudgeSyncBaseCommandResult(List<String> syncResultLog) {
public static boolean JudgeSyncBaseCommandResult(List<String> syncResultLog) {
if (CollectionUtils.isEmpty(syncResultLog)) {
log.error("基本脚本调用失败!");
return false;
@@ -116,6 +102,33 @@ public class FuncServiceImpl implements FuncService {
return false;
}
@Override
public boolean callHarborFuncAndJudge(String agentTopicName, HarborFunctionEnum harborFunctionEnum, ArrayList<String> funcArgs) {
List<String> syncResultLog = syncCallFunction(
agentTopicName,
ExecutionMessageType.HARBOR,
funcArgs,
true
);
return JudgeSyncBaseCommandResult(syncResultLog);
}
@Override
public List<String> callImageFuncService(String agentTopicName, ImageFunctionEnum imageFunctionEnum, List<String> funcArgs) {
// add in
funcArgs.add(0, imageFunctionEnum.getFuncName());
return syncCallFunction(
agentTopicName,
ExecutionMessageType.IMAGE,
funcArgs,
true
);
}
// 归一化调用
private List<String> syncCallFunction(String agentTopicName, ExecutionMessageType emType, List<String> funcArgs, boolean durationTask) {

145
server/src/main/java/io/wdd/func/auto/service/ImageFuncScheduler.java Normal file
View File

@@ -0,0 +1,145 @@
package io.wdd.func.auto.service;
import io.wdd.func.auto.beans.ImageSyncContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Service;
import org.springframework.util.Assert;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import static io.wdd.func.auto.beans.ImageFunctionEnum.*;
import static io.wdd.func.auto.service.FuncServiceImpl.JudgeSyncBaseCommandResult;
@Service
@Slf4j
public class ImageFuncScheduler {
public static final String CmiiHarborHostPrefix = "harbor.cdcyy.com.cn/cmii/";
@Resource
FuncService funcService;
public boolean runProcedure(ImageSyncContext imageSyncContext) {
// before run
// check base requirement
beforeRunProcedure(imageSyncContext);
// during run
doRunProcedure(imageSyncContext);
// after run
afterRunProcedure(imageSyncContext);
return true;
}
private void beforeRunProcedure(ImageSyncContext imageSyncContext) {
// build args
// imageFullName gzipFolderPrefix gzipFileName ossUrlPrefix proxyUrl targetHarborHost namespace targetImageFullName
ArrayList<String> arrayList = new ArrayList<>();
arrayList.add(""); //imageFullName
arrayList.add("/var/lib/docker/octopus_image/"); //gzipFolderPrefix
arrayList.add(""); //gzipFileName
arrayList.add("https://oss.demo.uavcmlc.com:18000/cmlc-installation/tmp/");//ossUrlPrefix
arrayList.add("");//proxyUrl
arrayList.add(imageSyncContext.getProjectMasterNode().getServerIpInV4());//targetHarborHost
arrayList.add(imageSyncContext.getProjectInfoPO().getProjectNamespace());//namespace
arrayList.add("");//targetImageFullName
imageSyncContext.setImageSyncFuncArgs(arrayList);
log.debug("beforeRunProcedure complete!");
}
private void afterRunProcedure(ImageSyncContext imageSyncContext) {
// message-pusher
// 检查是否安装完成, 对安装环境进行判定
log.debug("afterRunProcedure complete!");
}
private void doRunProcedure(ImageSyncContext imageSyncContext) {
List<String> realFullNameList = null;
ArrayList<String> cmiiAppNameTagList = imageSyncContext.getCmiiAppNameTagList();
ArrayList<String> imageFullNameList = imageSyncContext.getImageFullNameList();
if (CollectionUtils.isNotEmpty(cmiiAppNameTagList)) {
realFullNameList = cmiiAppNameTagList.stream().map(CmiiHarborHostPrefix::concat).collect(Collectors.toList());
}
if (CollectionUtils.isNotEmpty(cmiiAppNameTagList)) {
if (CollectionUtils.isEmpty(realFullNameList)) {
realFullNameList = imageFullNameList;
} else {
realFullNameList.addAll(imageFullNameList);
}
}
if (realFullNameList == null) {
log.info("Image Sync image list is null, return !");
return;
}
String innerWorkerAgentName = imageSyncContext.getInnerWorkerAgentName();
Assert.notNull(innerWorkerAgentName, "inner worker agent name cant not be null !");
ArrayList<String> imageSyncFuncArgs = imageSyncContext.getImageSyncFuncArgs();
String outsideAgentTopicName = imageSyncContext.getProjectMasterNode().getTopicName();
//
List<String> resultLog;
for (String fullName : realFullNameList) {
// innerWorkerAgent
imageSyncFuncArgs.set(0, fullName);
resultLog = funcService.callImageFuncService(innerWorkerAgentName, DOWNLOAD_DOCKER_IMAGE, imageSyncFuncArgs);
if (!JudgeSyncBaseCommandResult(resultLog)) {
return;
}
resultLog = funcService.callImageFuncService(innerWorkerAgentName, COMPRESS_IMAGE_TO_GZIP, imageSyncFuncArgs);
if (!JudgeSyncBaseCommandResult(resultLog)) {
return;
}
String gzipFileName = resultLog.get(0);
Assert.notNull(gzipFileName, "COMPRESS_IMAGE_TO_GZIP return gzip file name is null !");
imageSyncFuncArgs.set(2, gzipFileName);
resultLog = funcService.callImageFuncService(innerWorkerAgentName, UPLOAD_GZIP_TO_OSS, imageSyncFuncArgs);
if (!JudgeSyncBaseCommandResult(resultLog)) {
return;
}
// outside agent
resultLog = funcService.callImageFuncService(outsideAgentTopicName, DOWNLOAD_GZIP_IMAGE_FILE, imageSyncFuncArgs);
if (!JudgeSyncBaseCommandResult(resultLog)) {
return;
}
resultLog = funcService.callImageFuncService(outsideAgentTopicName, LOAD_DOCKER_IMAGE_FROM_GZIP, imageSyncFuncArgs);
if (!JudgeSyncBaseCommandResult(resultLog)) {
return;
}
resultLog = funcService.callImageFuncService(outsideAgentTopicName, PUSH_IMAGE_TO_TARGET_HARBOR, imageSyncFuncArgs);
if (!JudgeSyncBaseCommandResult(resultLog)) {
return;
}
resultLog = funcService.callImageFuncService(outsideAgentTopicName, UPDATE_IMAGE_TAG, imageSyncFuncArgs);
if (!JudgeSyncBaseCommandResult(resultLog)) {
return;
}
}
}
}

3
server/src/main/java/io/wdd/rpc/execute/ExecutionMessageType.java
View File

@@ -12,4 +12,7 @@ public enum ExecutionMessageType {
// 执行Harbor仓库操作相关的内容 HARBOR
HARBOR,
// 执行Image Sync相关的操作
IMAGE,
}

13
server/src/main/java/io/wdd/server/beans/po/ServerInfoPO.java
View File

@@ -63,6 +63,7 @@ public class ServerInfoPO implements Serializable {
/**
* server inner ipv6
*/
@TableField(value = "server_ip_in_v6")
private String serverIpInV6;
@@ -163,6 +164,12 @@ public class ServerInfoPO implements Serializable {
@TableField(value = "os_kernel_info")
private String osKernelInfo;
/**
* 服务器能否访问公网
*/
@TableField(value = "can_access_public")
private Integer canAccessPublic;
/**
* machine uuid from /etc/machineid
*/
@@ -187,6 +194,12 @@ public class ServerInfoPO implements Serializable {
@TableField(value = "version")
private Integer version;
/**
* 服务器的角色信息
*/
@TableField(value = "role")
private String role;
@TableField(exist = false)
private static final long serialVersionUID = 1L;
}

33
server/src/main/java/io/wdd/server/beans/po/ServerInfoRoleEnum.java Normal file
View File

@@ -0,0 +1,33 @@
package io.wdd.server.beans.po;
public enum ServerInfoRoleEnum {
MASTER_NODE(
"MASTER_NODE",
""
);
String RoleName;
String Des;
ServerInfoRoleEnum(String roleName, String des) {
RoleName = roleName;
Des = des;
}
public String getRoleName() {
return RoleName;
}
public void setRoleName(String roleName) {
RoleName = roleName;
}
public String getDes() {
return Des;
}
public void setDes(String des) {
Des = des;
}
}

2
server/src/main/java/io/wdd/server/beans/request/ProjectQueryEntity.java
View File

@@ -21,6 +21,6 @@ public class ProjectQueryEntity {
/*条件查询*/
String projectName;
String projectProvince;
String projectNamespace;
}

1
server/src/main/java/io/wdd/server/coreService/impl/CoreProjectServerServiceImpl.java
View File

@@ -72,7 +72,6 @@ public class CoreProjectServerServiceImpl implements CoreProjectServerService {
// 首先查询projectId是否存在 pass
// 查询与之绑定的所有server
// serverId projectId
List<ProjectServerRelationPO> projectServerRelationPOList = new LambdaQueryChainWrapper<ProjectServerRelationPO>(projectServerRelationService.getBaseMapper())

8
server/src/main/java/io/wdd/server/coreService/impl/CoreProjectServiceImpl.java
View File

@@ -31,7 +31,7 @@ public class CoreProjectServiceImpl implements CoreProjectService {
@Override
public Page<ProjectInfoPO> projectQueryByEntity(ProjectQueryEntity projectQueryEntity) {
if (projectQueryEntity == null || projectQueryEntity.getProjectName() == null || projectQueryEntity.getProjectProvince() == null) {
if (projectQueryEntity == null || projectQueryEntity.getProjectName() == null || projectQueryEntity.getProjectNamespace() == null) {
return null;
}
@@ -52,9 +52,9 @@ public class CoreProjectServiceImpl implements CoreProjectService {
projectQueryEntity.getProjectName()
)
.likeRight(
StringUtils.isNotBlank(projectQueryEntity.getProjectProvince()),
ProjectInfoPO::getProjectProvince,
projectQueryEntity.getProjectProvince()
StringUtils.isNotBlank(projectQueryEntity.getProjectNamespace()),
ProjectInfoPO::getProjectNamespace,
projectQueryEntity.getProjectNamespace()
)
.page(poPage);

4
server/src/main/java/io/wdd/server/mapper/ServerInfoMapper.java
View File

@@ -4,9 +4,9 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import io.wdd.server.beans.po.ServerInfoPO;
/**
* @author wdd
* @author wddsh
* @description 针对表【server_info】的数据库操作Mapper
* @createDate 2023-10-08 11:24:01
* @createDate 2024-03-26 18:11:29
* @Entity io.wdd.server.beans.po.ServerInfoPO
*/
public interface ServerInfoMapper extends BaseMapper<ServerInfoPO> {

4
server/src/main/java/io/wdd/server/service/ServerInfoService.java
View File

@@ -4,9 +4,9 @@ import com.baomidou.mybatisplus.extension.service.IService;
import io.wdd.server.beans.po.ServerInfoPO;
/**
* @author wdd
* @author wddsh
* @description 针对表【server_info】的数据库操作Service
* @createDate 2023-10-08 11:24:01
* @createDate 2024-03-26 18:11:29
*/
public interface ServerInfoService extends IService<ServerInfoPO> {

4
server/src/main/java/io/wdd/server/service/impl/ServerInfoServiceImpl.java
View File

@@ -7,9 +7,9 @@ import io.wdd.server.service.ServerInfoService;
import org.springframework.stereotype.Service;
/**
* @author wdd
* @author wddsh
* @description 针对表【server_info】的数据库操作Service实现
* @createDate 2023-10-08 11:24:01
* @createDate 2024-03-26 18:11:29
*/
@Service
public class ServerInfoServiceImpl extends ServiceImpl<ServerInfoMapper, ServerInfoPO>

6
server/src/main/resources/mapper/ServerInfoMapper.xml
View File

@@ -29,10 +29,12 @@
<result property="virtualization" column="virtualization" jdbcType="VARCHAR"/>
<result property="osInfo" column="os_info" jdbcType="VARCHAR"/>
<result property="osKernelInfo" column="os_kernel_info" jdbcType="VARCHAR"/>
<result property="canAccessPublic" column="can_access_public" jdbcType="TINYINT"/>
<result property="machineId" column="machine_id" jdbcType="VARCHAR"/>
<result property="comment" column="comment" jdbcType="VARCHAR"/>
<result property="isDelete" column="is_delete" jdbcType="TINYINT"/>
<result property="version" column="version" jdbcType="INTEGER"/>
<result property="role" column="role" jdbcType="VARCHAR"/>
</resultMap>
<sql id="Base_Column_List">
@@ -45,7 +47,7 @@
cpu_core,memory_total,disk_total,
disk_usage,io_speed,tcp_control,
virtualization,os_info,os_kernel_info,
machine_id,comment,is_delete,
version
can_access_public,machine_id,comment,
is_delete,version,role
</sql>
</mapper>

2
server/src/test/java/io/wdd/server/controller/ProjectInfoTest.java
View File

@@ -95,7 +95,7 @@ public class ProjectInfoTest {
projectQueryEntity.setPageNumber(1);
projectQueryEntity.setPageSize(10);
projectQueryEntity.setProjectName("Test Project");
projectQueryEntity.setProjectProvince("Test Province");
projectQueryEntity.setProjectNamespace("Test Province");
// Act
R<Page<ProjectInfoPO>> result = projectController.projectQueryOne(projectQueryEntity);

84
server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java Normal file
View File

@@ -0,0 +1,84 @@
package io.wdd.server.func;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import io.wdd.func.auto.beans.ImageSyncContext;
import io.wdd.func.auto.service.ImageFuncScheduler;
import io.wdd.server.beans.po.ProjectInfoPO;
import io.wdd.server.beans.po.ServerInfoPO;
import io.wdd.server.beans.request.ProjectQueryEntity;
import io.wdd.server.beans.vo.ProjectServerVO;
import io.wdd.server.coreService.CoreProjectServerService;
import io.wdd.server.coreService.CoreProjectService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.junit.Test;
import org.springframework.boot.test.context.SpringBootTest;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@SpringBootTest
@Slf4j
public class TestImageSyncScheduler {
@Resource
ImageFuncScheduler imageFuncScheduler;
@Resource
CoreProjectService coreProjectService;
@Resource
CoreProjectServerService coreProjectServerService;
@Test
public void runImageSync() {
ArrayList<String> CmiiAppNameList = new ArrayList<>(List.of(
));
ArrayList<String> ImageFullNameList = new ArrayList<>(List.of(
));
String projectNamespace = "xmyd";
// start
ImageSyncContext imageSyncContext = new ImageSyncContext();
// inner
imageSyncContext.setInnerWorkerAgentName("Chengdu-amd64-71-3571gd");
ProjectQueryEntity projectQueryEntity = ProjectQueryEntity.builder().projectNamespace(projectNamespace).build();
Page<ProjectInfoPO> page = coreProjectService.projectQueryByEntity(projectQueryEntity);
if (page.getSize() < 1) {
log.error(" project name space error !");
return;
}
ProjectInfoPO projectInfoPO = page.getRecords().get(0);
imageSyncContext.setProjectInfoPO(projectInfoPO);
log.info("project name => {} namespace => {}", projectInfoPO.getProjectName(), projectInfoPO.getProjectNamespace());
// master
ProjectServerVO projectServerVO = coreProjectServerService.projectServerOne(projectInfoPO.getProjectId());
Optional<ServerInfoPO> master = projectServerVO.getBindingServerList().stream().filter(server -> StringUtils.startsWith(server.getRole(), "master")).findFirst();
if (master.isEmpty()) {
log.error("project master node not set ！");
return;
}
imageSyncContext.setProjectMasterNode(master.get());
log.info("project master node inner ipv4 is => {}", master.get().getServerIpInV4());
imageSyncContext.setImageFullNameList(ImageFullNameList);
imageSyncContext.setCmiiAppNameTagList(CmiiAppNameList);
imageFuncScheduler.runProcedure(imageSyncContext);
}
}