diff --git a/agent-go/executor/CommandExecutor.go b/agent-go/executor/CommandExecutor.go index af28461..8587270 100644 --- a/agent-go/executor/CommandExecutor.go +++ b/agent-go/executor/CommandExecutor.go @@ -61,6 +61,16 @@ func Execute(em *ExecutionMessage) (bool, []string) { // Harbor Execute ok, resultLog = HarborOperatorCache.Exec(em.FuncContent[0], em.FuncContent[1:]...) + } else if strings.HasPrefix(em.ExecutionType, "IMAGE") { + // image function + if em.FuncContent == nil || len(em.FuncContent) <= 1 { + ok = false + resultLog = []string{ + "[Harbor Execute] - functions args is wrong!", + } + } + // Harbor Execute + ok, resultLog = AgentOsOperatorCache.Sync(em.FuncContent[0], em.FuncContent[1:]...) } else { // deprecated // shell command diff --git a/server/src/main/java/io/wdd/func/auto/beans/ImageSyncContext.java b/server/src/main/java/io/wdd/func/auto/beans/ImageSyncContext.java new file mode 100644 index 0000000..da48c4b --- /dev/null +++ b/server/src/main/java/io/wdd/func/auto/beans/ImageSyncContext.java @@ -0,0 +1,32 @@ +package io.wdd.func.auto.beans; + +import io.wdd.server.beans.po.ProjectInfoPO; +import io.wdd.server.beans.po.ServerInfoPO; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.ArrayList; + +@Data +@SuperBuilder(toBuilder = true) +@AllArgsConstructor +@NoArgsConstructor +public class ImageSyncContext { + // single multi file + ArrayList cmiiAppNameTagList; + + ArrayList imageFullNameList; + + // function args + ArrayList imageSyncFuncArgs; + + // target + ProjectInfoPO projectInfoPO; + ServerInfoPO projectMasterNode; + + // 我方的 机器 默认值为 192.168.35.71 + String innerWorkerAgentName; + +} diff --git a/server/src/main/java/io/wdd/func/auto/service/FuncService.java b/server/src/main/java/io/wdd/func/auto/service/FuncService.java index d833230..1dade63 100644 --- a/server/src/main/java/io/wdd/func/auto/service/FuncService.java +++ b/server/src/main/java/io/wdd/func/auto/service/FuncService.java @@ -4,6 +4,7 @@ package io.wdd.func.auto.service; import io.wdd.func.auto.beans.AppFunctionEnum; import io.wdd.func.auto.beans.BaseFunctionEnum; import io.wdd.func.auto.beans.HarborFunctionEnum; +import io.wdd.func.auto.beans.ImageFunctionEnum; import java.util.ArrayList; import java.util.List; @@ -29,4 +30,9 @@ public interface FuncService { boolean callAppFuncAndJudge(String agentTopicName, AppFunctionEnum appFunctionEnum, List funcArgs); boolean callHarborFuncAndJudge(String agentTopicName, HarborFunctionEnum harborFunctionEnum, ArrayList funcArgs); + + + List callImageFuncService(String agentTopicName, ImageFunctionEnum imageFunctionEnum, List funcArgs); + + } diff --git a/server/src/main/java/io/wdd/func/auto/service/FuncServiceImpl.java b/server/src/main/java/io/wdd/func/auto/service/FuncServiceImpl.java index 8662a69..bb9b858 100644 --- a/server/src/main/java/io/wdd/func/auto/service/FuncServiceImpl.java +++ b/server/src/main/java/io/wdd/func/auto/service/FuncServiceImpl.java @@ -3,6 +3,7 @@ package io.wdd.func.auto.service; import io.wdd.func.auto.beans.AppFunctionEnum; import io.wdd.func.auto.beans.BaseFunctionEnum; import io.wdd.func.auto.beans.HarborFunctionEnum; +import io.wdd.func.auto.beans.ImageFunctionEnum; import io.wdd.rpc.execute.ExecutionMessageType; import io.wdd.rpc.execute.service.ExecutionService; import lombok.extern.slf4j.Slf4j; @@ -75,22 +76,7 @@ public class FuncServiceImpl implements FuncService { return appFuncResult; } - @Override - public boolean callHarborFuncAndJudge(String agentTopicName, HarborFunctionEnum harborFunctionEnum, ArrayList funcArgs) { - - List syncResultLog = syncCallFunction( - agentTopicName, - ExecutionMessageType.HARBOR, - funcArgs, - true - ); - - boolean harborResult = JudgeSyncBaseCommandResult(syncResultLog); - - return harborResult; - } - - private boolean JudgeSyncBaseCommandResult(List syncResultLog) { + public static boolean JudgeSyncBaseCommandResult(List syncResultLog) { if (CollectionUtils.isEmpty(syncResultLog)) { log.error("基本脚本调用失败!"); return false; @@ -116,6 +102,33 @@ public class FuncServiceImpl implements FuncService { return false; } + @Override + public boolean callHarborFuncAndJudge(String agentTopicName, HarborFunctionEnum harborFunctionEnum, ArrayList funcArgs) { + + List syncResultLog = syncCallFunction( + agentTopicName, + ExecutionMessageType.HARBOR, + funcArgs, + true + ); + + return JudgeSyncBaseCommandResult(syncResultLog); + } + + @Override + public List callImageFuncService(String agentTopicName, ImageFunctionEnum imageFunctionEnum, List funcArgs) { + + // add in + funcArgs.add(0, imageFunctionEnum.getFuncName()); + + return syncCallFunction( + agentTopicName, + ExecutionMessageType.IMAGE, + funcArgs, + true + ); + } + // 归一化调用 private List syncCallFunction(String agentTopicName, ExecutionMessageType emType, List funcArgs, boolean durationTask) { diff --git a/server/src/main/java/io/wdd/func/auto/service/ImageFuncScheduler.java b/server/src/main/java/io/wdd/func/auto/service/ImageFuncScheduler.java new file mode 100644 index 0000000..298ef91 --- /dev/null +++ b/server/src/main/java/io/wdd/func/auto/service/ImageFuncScheduler.java @@ -0,0 +1,145 @@ +package io.wdd.func.auto.service; + +import io.wdd.func.auto.beans.ImageSyncContext; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.stereotype.Service; +import org.springframework.util.Assert; + +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static io.wdd.func.auto.beans.ImageFunctionEnum.*; +import static io.wdd.func.auto.service.FuncServiceImpl.JudgeSyncBaseCommandResult; + +@Service +@Slf4j +public class ImageFuncScheduler { + + public static final String CmiiHarborHostPrefix = "harbor.cdcyy.com.cn/cmii/"; + @Resource + FuncService funcService; + + public boolean runProcedure(ImageSyncContext imageSyncContext) { + + // before run + // check base requirement + beforeRunProcedure(imageSyncContext); + + + // during run + doRunProcedure(imageSyncContext); + + + // after run + afterRunProcedure(imageSyncContext); + + + return true; + } + + private void beforeRunProcedure(ImageSyncContext imageSyncContext) { + + // build args + + // imageFullName gzipFolderPrefix gzipFileName ossUrlPrefix proxyUrl targetHarborHost namespace targetImageFullName + ArrayList arrayList = new ArrayList<>(); + arrayList.add(""); //imageFullName + arrayList.add("/var/lib/docker/octopus_image/"); //gzipFolderPrefix + arrayList.add(""); //gzipFileName + arrayList.add("https://oss.demo.uavcmlc.com:18000/cmlc-installation/tmp/");//ossUrlPrefix + arrayList.add("");//proxyUrl + arrayList.add(imageSyncContext.getProjectMasterNode().getServerIpInV4());//targetHarborHost + arrayList.add(imageSyncContext.getProjectInfoPO().getProjectNamespace());//namespace + arrayList.add("");//targetImageFullName + + + imageSyncContext.setImageSyncFuncArgs(arrayList); + + log.debug("beforeRunProcedure complete!"); + } + + private void afterRunProcedure(ImageSyncContext imageSyncContext) { + + // message-pusher + + + // 检查是否安装完成, 对安装环境进行判定 + log.debug("afterRunProcedure complete!"); + } + + private void doRunProcedure(ImageSyncContext imageSyncContext) { + + List realFullNameList = null; + ArrayList cmiiAppNameTagList = imageSyncContext.getCmiiAppNameTagList(); + ArrayList imageFullNameList = imageSyncContext.getImageFullNameList(); + + if (CollectionUtils.isNotEmpty(cmiiAppNameTagList)) { + realFullNameList = cmiiAppNameTagList.stream().map(CmiiHarborHostPrefix::concat).collect(Collectors.toList()); + } + + + if (CollectionUtils.isNotEmpty(cmiiAppNameTagList)) { + if (CollectionUtils.isEmpty(realFullNameList)) { + realFullNameList = imageFullNameList; + } else { + realFullNameList.addAll(imageFullNameList); + } + } + + if (realFullNameList == null) { + log.info("Image Sync image list is null, return !"); + return; + } + + String innerWorkerAgentName = imageSyncContext.getInnerWorkerAgentName(); + Assert.notNull(innerWorkerAgentName, "inner worker agent name cant not be null !"); + + ArrayList imageSyncFuncArgs = imageSyncContext.getImageSyncFuncArgs(); + + String outsideAgentTopicName = imageSyncContext.getProjectMasterNode().getTopicName(); + + // + List resultLog; + for (String fullName : realFullNameList) { + // innerWorkerAgent + imageSyncFuncArgs.set(0, fullName); + resultLog = funcService.callImageFuncService(innerWorkerAgentName, DOWNLOAD_DOCKER_IMAGE, imageSyncFuncArgs); + if (!JudgeSyncBaseCommandResult(resultLog)) { + return; + } + resultLog = funcService.callImageFuncService(innerWorkerAgentName, COMPRESS_IMAGE_TO_GZIP, imageSyncFuncArgs); + if (!JudgeSyncBaseCommandResult(resultLog)) { + return; + } + String gzipFileName = resultLog.get(0); + Assert.notNull(gzipFileName, "COMPRESS_IMAGE_TO_GZIP return gzip file name is null !"); + imageSyncFuncArgs.set(2, gzipFileName); + resultLog = funcService.callImageFuncService(innerWorkerAgentName, UPLOAD_GZIP_TO_OSS, imageSyncFuncArgs); + if (!JudgeSyncBaseCommandResult(resultLog)) { + return; + } + + // outside agent + resultLog = funcService.callImageFuncService(outsideAgentTopicName, DOWNLOAD_GZIP_IMAGE_FILE, imageSyncFuncArgs); + if (!JudgeSyncBaseCommandResult(resultLog)) { + return; + } + resultLog = funcService.callImageFuncService(outsideAgentTopicName, LOAD_DOCKER_IMAGE_FROM_GZIP, imageSyncFuncArgs); + if (!JudgeSyncBaseCommandResult(resultLog)) { + return; + } + resultLog = funcService.callImageFuncService(outsideAgentTopicName, PUSH_IMAGE_TO_TARGET_HARBOR, imageSyncFuncArgs); + if (!JudgeSyncBaseCommandResult(resultLog)) { + return; + } + resultLog = funcService.callImageFuncService(outsideAgentTopicName, UPDATE_IMAGE_TAG, imageSyncFuncArgs); + if (!JudgeSyncBaseCommandResult(resultLog)) { + return; + } + } + + } +} diff --git a/server/src/main/java/io/wdd/rpc/execute/ExecutionMessageType.java b/server/src/main/java/io/wdd/rpc/execute/ExecutionMessageType.java index f007d88..ce94450 100644 --- a/server/src/main/java/io/wdd/rpc/execute/ExecutionMessageType.java +++ b/server/src/main/java/io/wdd/rpc/execute/ExecutionMessageType.java @@ -12,4 +12,7 @@ public enum ExecutionMessageType { // 执行Harbor仓库操作相关的内容 HARBOR HARBOR, + // 执行Image Sync相关的操作 + IMAGE, + } diff --git a/server/src/main/java/io/wdd/server/beans/po/ServerInfoPO.java b/server/src/main/java/io/wdd/server/beans/po/ServerInfoPO.java index 3514f09..45cb18d 100644 --- a/server/src/main/java/io/wdd/server/beans/po/ServerInfoPO.java +++ b/server/src/main/java/io/wdd/server/beans/po/ServerInfoPO.java @@ -63,6 +63,7 @@ public class ServerInfoPO implements Serializable { /** * server inner ipv6 + */ @TableField(value = "server_ip_in_v6") private String serverIpInV6; @@ -80,13 +81,13 @@ public class ServerInfoPO implements Serializable { private LocalDateTime expireTime; /** - * + * */ @TableField(value = "update_time") private LocalDateTime updateTime; /** - * + * */ @TableField(value = "create_time") private LocalDateTime createTime; @@ -163,6 +164,12 @@ public class ServerInfoPO implements Serializable { @TableField(value = "os_kernel_info") private String osKernelInfo; + /** + * 服务器能否访问公网 + */ + @TableField(value = "can_access_public") + private Integer canAccessPublic; + /** * machine uuid from /etc/machineid */ @@ -187,6 +194,12 @@ public class ServerInfoPO implements Serializable { @TableField(value = "version") private Integer version; + /** + * 服务器的角色信息 + */ + @TableField(value = "role") + private String role; + @TableField(exist = false) private static final long serialVersionUID = 1L; } \ No newline at end of file diff --git a/server/src/main/java/io/wdd/server/beans/po/ServerInfoRoleEnum.java b/server/src/main/java/io/wdd/server/beans/po/ServerInfoRoleEnum.java new file mode 100644 index 0000000..4041dea --- /dev/null +++ b/server/src/main/java/io/wdd/server/beans/po/ServerInfoRoleEnum.java @@ -0,0 +1,33 @@ +package io.wdd.server.beans.po; + +public enum ServerInfoRoleEnum { + + MASTER_NODE( + "MASTER_NODE", + "" + ); + + String RoleName; + String Des; + + ServerInfoRoleEnum(String roleName, String des) { + RoleName = roleName; + Des = des; + } + + public String getRoleName() { + return RoleName; + } + + public void setRoleName(String roleName) { + RoleName = roleName; + } + + public String getDes() { + return Des; + } + + public void setDes(String des) { + Des = des; + } +} diff --git a/server/src/main/java/io/wdd/server/beans/request/ProjectQueryEntity.java b/server/src/main/java/io/wdd/server/beans/request/ProjectQueryEntity.java index f9c0b50..74dd1a0 100644 --- a/server/src/main/java/io/wdd/server/beans/request/ProjectQueryEntity.java +++ b/server/src/main/java/io/wdd/server/beans/request/ProjectQueryEntity.java @@ -21,6 +21,6 @@ public class ProjectQueryEntity { /*条件查询*/ String projectName; - String projectProvince; + String projectNamespace; } diff --git a/server/src/main/java/io/wdd/server/coreService/impl/CoreProjectServerServiceImpl.java b/server/src/main/java/io/wdd/server/coreService/impl/CoreProjectServerServiceImpl.java index 673a34a..96a74d4 100644 --- a/server/src/main/java/io/wdd/server/coreService/impl/CoreProjectServerServiceImpl.java +++ b/server/src/main/java/io/wdd/server/coreService/impl/CoreProjectServerServiceImpl.java @@ -72,7 +72,6 @@ public class CoreProjectServerServiceImpl implements CoreProjectServerService { // 首先查询projectId是否存在 pass - // 查询与之绑定的所有server // serverId projectId List projectServerRelationPOList = new LambdaQueryChainWrapper(projectServerRelationService.getBaseMapper()) diff --git a/server/src/main/java/io/wdd/server/coreService/impl/CoreProjectServiceImpl.java b/server/src/main/java/io/wdd/server/coreService/impl/CoreProjectServiceImpl.java index 7445d77..de07d3b 100644 --- a/server/src/main/java/io/wdd/server/coreService/impl/CoreProjectServiceImpl.java +++ b/server/src/main/java/io/wdd/server/coreService/impl/CoreProjectServiceImpl.java @@ -31,7 +31,7 @@ public class CoreProjectServiceImpl implements CoreProjectService { @Override public Page projectQueryByEntity(ProjectQueryEntity projectQueryEntity) { - if (projectQueryEntity == null || projectQueryEntity.getProjectName() == null || projectQueryEntity.getProjectProvince() == null) { + if (projectQueryEntity == null || projectQueryEntity.getProjectName() == null || projectQueryEntity.getProjectNamespace() == null) { return null; } @@ -52,9 +52,9 @@ public class CoreProjectServiceImpl implements CoreProjectService { projectQueryEntity.getProjectName() ) .likeRight( - StringUtils.isNotBlank(projectQueryEntity.getProjectProvince()), - ProjectInfoPO::getProjectProvince, - projectQueryEntity.getProjectProvince() + StringUtils.isNotBlank(projectQueryEntity.getProjectNamespace()), + ProjectInfoPO::getProjectNamespace, + projectQueryEntity.getProjectNamespace() ) .page(poPage); diff --git a/server/src/main/java/io/wdd/server/mapper/ServerInfoMapper.java b/server/src/main/java/io/wdd/server/mapper/ServerInfoMapper.java index f06c67f..6fe891e 100644 --- a/server/src/main/java/io/wdd/server/mapper/ServerInfoMapper.java +++ b/server/src/main/java/io/wdd/server/mapper/ServerInfoMapper.java @@ -4,9 +4,9 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper; import io.wdd.server.beans.po.ServerInfoPO; /** - * @author wdd + * @author wddsh * @description 针对表【server_info】的数据库操作Mapper - * @createDate 2023-10-08 11:24:01 + * @createDate 2024-03-26 18:11:29 * @Entity io.wdd.server.beans.po.ServerInfoPO */ public interface ServerInfoMapper extends BaseMapper { diff --git a/server/src/main/java/io/wdd/server/service/ServerInfoService.java b/server/src/main/java/io/wdd/server/service/ServerInfoService.java index 7c5eb79..f51d195 100644 --- a/server/src/main/java/io/wdd/server/service/ServerInfoService.java +++ b/server/src/main/java/io/wdd/server/service/ServerInfoService.java @@ -4,9 +4,9 @@ import com.baomidou.mybatisplus.extension.service.IService; import io.wdd.server.beans.po.ServerInfoPO; /** - * @author wdd + * @author wddsh * @description 针对表【server_info】的数据库操作Service - * @createDate 2023-10-08 11:24:01 + * @createDate 2024-03-26 18:11:29 */ public interface ServerInfoService extends IService { diff --git a/server/src/main/java/io/wdd/server/service/impl/ServerInfoServiceImpl.java b/server/src/main/java/io/wdd/server/service/impl/ServerInfoServiceImpl.java index 59b53a4..45cacef 100644 --- a/server/src/main/java/io/wdd/server/service/impl/ServerInfoServiceImpl.java +++ b/server/src/main/java/io/wdd/server/service/impl/ServerInfoServiceImpl.java @@ -7,9 +7,9 @@ import io.wdd.server.service.ServerInfoService; import org.springframework.stereotype.Service; /** - * @author wdd + * @author wddsh * @description 针对表【server_info】的数据库操作Service实现 - * @createDate 2023-10-08 11:24:01 + * @createDate 2024-03-26 18:11:29 */ @Service public class ServerInfoServiceImpl extends ServiceImpl diff --git a/server/src/main/resources/mapper/ServerInfoMapper.xml b/server/src/main/resources/mapper/ServerInfoMapper.xml index bb00fdd..4589227 100644 --- a/server/src/main/resources/mapper/ServerInfoMapper.xml +++ b/server/src/main/resources/mapper/ServerInfoMapper.xml @@ -29,10 +29,12 @@ + + @@ -45,7 +47,7 @@ cpu_core,memory_total,disk_total, disk_usage,io_speed,tcp_control, virtualization,os_info,os_kernel_info, - machine_id,comment,is_delete, - version + can_access_public,machine_id,comment, + is_delete,version,role diff --git a/server/src/test/java/io/wdd/server/controller/ProjectInfoTest.java b/server/src/test/java/io/wdd/server/controller/ProjectInfoTest.java index 416018d..ce9a06d 100644 --- a/server/src/test/java/io/wdd/server/controller/ProjectInfoTest.java +++ b/server/src/test/java/io/wdd/server/controller/ProjectInfoTest.java @@ -95,7 +95,7 @@ public class ProjectInfoTest { projectQueryEntity.setPageNumber(1); projectQueryEntity.setPageSize(10); projectQueryEntity.setProjectName("Test Project"); - projectQueryEntity.setProjectProvince("Test Province"); + projectQueryEntity.setProjectNamespace("Test Province"); // Act R> result = projectController.projectQueryOne(projectQueryEntity); diff --git a/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java b/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java new file mode 100644 index 0000000..55d3292 --- /dev/null +++ b/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java @@ -0,0 +1,84 @@ +package io.wdd.server.func; + +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import io.wdd.func.auto.beans.ImageSyncContext; +import io.wdd.func.auto.service.ImageFuncScheduler; +import io.wdd.server.beans.po.ProjectInfoPO; +import io.wdd.server.beans.po.ServerInfoPO; +import io.wdd.server.beans.request.ProjectQueryEntity; +import io.wdd.server.beans.vo.ProjectServerVO; +import io.wdd.server.coreService.CoreProjectServerService; +import io.wdd.server.coreService.CoreProjectService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.junit.Test; +import org.springframework.boot.test.context.SpringBootTest; + +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +@SpringBootTest +@Slf4j +public class TestImageSyncScheduler { + + @Resource + ImageFuncScheduler imageFuncScheduler; + + + @Resource + CoreProjectService coreProjectService; + + @Resource + CoreProjectServerService coreProjectServerService; + + @Test + public void runImageSync() { + + ArrayList CmiiAppNameList = new ArrayList<>(List.of( + + )); + + ArrayList ImageFullNameList = new ArrayList<>(List.of( + + )); + + String projectNamespace = "xmyd"; + + + // start + ImageSyncContext imageSyncContext = new ImageSyncContext(); + + // inner + imageSyncContext.setInnerWorkerAgentName("Chengdu-amd64-71-3571gd"); + + ProjectQueryEntity projectQueryEntity = ProjectQueryEntity.builder().projectNamespace(projectNamespace).build(); + + Page page = coreProjectService.projectQueryByEntity(projectQueryEntity); + if (page.getSize() < 1) { + log.error(" project name space error !"); + return; + } + ProjectInfoPO projectInfoPO = page.getRecords().get(0); + imageSyncContext.setProjectInfoPO(projectInfoPO); + + log.info("project name => {} namespace => {}", projectInfoPO.getProjectName(), projectInfoPO.getProjectNamespace()); + // master + ProjectServerVO projectServerVO = coreProjectServerService.projectServerOne(projectInfoPO.getProjectId()); + Optional master = projectServerVO.getBindingServerList().stream().filter(server -> StringUtils.startsWith(server.getRole(), "master")).findFirst(); + if (master.isEmpty()) { + log.error("project master node not set ！"); + return; + } + imageSyncContext.setProjectMasterNode(master.get()); + + log.info("project master node inner ipv4 is => {}", master.get().getServerIpInV4()); + + imageSyncContext.setImageFullNameList(ImageFullNameList); + imageSyncContext.setCmiiAppNameTagList(CmiiAppNameList); + + imageFuncScheduler.runProcedure(imageSyncContext); + } + +}