[ Cmii ] [ Octopus ] - add Image Function Scheduler - 1

agent-go/executor/CommandExecutor.go

@@ -61,6 +61,16 @@ func Execute(em *ExecutionMessage) (bool, []string) {
 		// Harbor Execute
 		ok, resultLog = HarborOperatorCache.Exec(em.FuncContent[0], em.FuncContent[1:]...)
 
+	} else if strings.HasPrefix(em.ExecutionType, "IMAGE") {
+		// image function
+		if em.FuncContent == nil || len(em.FuncContent) <= 1 {
+			ok = false
+			resultLog = []string{
+				"[Harbor Execute] - functions args is wrong!",
+			}
+		}
+		// Harbor Execute
+		ok, resultLog = AgentOsOperatorCache.Sync(em.FuncContent[0], em.FuncContent[1:]...)
 	} else {
 		// deprecated
 		// shell command

server/src/main/java/io/wdd/func/auto/beans/ImageSyncContext.java

@@ -0,0 +1,32 @@
+package io.wdd.func.auto.beans;
+
+import io.wdd.server.beans.po.ProjectInfoPO;
+import io.wdd.server.beans.po.ServerInfoPO;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+import java.util.ArrayList;
+
+@Data
+@SuperBuilder(toBuilder = true)
+@AllArgsConstructor
+@NoArgsConstructor
+public class ImageSyncContext {
+    // single multi file
+    ArrayList<String> cmiiAppNameTagList;
+
+    ArrayList<String> imageFullNameList;
+
+    // function args
+    ArrayList<String> imageSyncFuncArgs;
+
+    // target
+    ProjectInfoPO projectInfoPO;
+    ServerInfoPO projectMasterNode;
+
+    // 我方的 机器 默认值为 192.168.35.71
+    String innerWorkerAgentName;
+
+}

server/src/main/java/io/wdd/func/auto/service/FuncService.java

@@ -4,6 +4,7 @@ package io.wdd.func.auto.service;
 import io.wdd.func.auto.beans.AppFunctionEnum;
 import io.wdd.func.auto.beans.BaseFunctionEnum;
 import io.wdd.func.auto.beans.HarborFunctionEnum;
+import io.wdd.func.auto.beans.ImageFunctionEnum;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -29,4 +30,9 @@ public interface FuncService {
     boolean callAppFuncAndJudge(String agentTopicName, AppFunctionEnum appFunctionEnum, List<String> funcArgs);
 
     boolean callHarborFuncAndJudge(String agentTopicName, HarborFunctionEnum harborFunctionEnum, ArrayList<String> funcArgs);
+
+
+    List<String> callImageFuncService(String agentTopicName, ImageFunctionEnum imageFunctionEnum, List<String> funcArgs);
+
+
 }

server/src/main/java/io/wdd/func/auto/service/FuncServiceImpl.java

@@ -3,6 +3,7 @@ package io.wdd.func.auto.service;
 import io.wdd.func.auto.beans.AppFunctionEnum;
 import io.wdd.func.auto.beans.BaseFunctionEnum;
 import io.wdd.func.auto.beans.HarborFunctionEnum;
+import io.wdd.func.auto.beans.ImageFunctionEnum;
 import io.wdd.rpc.execute.ExecutionMessageType;
 import io.wdd.rpc.execute.service.ExecutionService;
 import lombok.extern.slf4j.Slf4j;
@@ -75,22 +76,7 @@ public class FuncServiceImpl implements FuncService {
         return appFuncResult;
     }
 
-    @Override
+    public static boolean JudgeSyncBaseCommandResult(List<String> syncResultLog) {
-    public boolean callHarborFuncAndJudge(String agentTopicName, HarborFunctionEnum harborFunctionEnum, ArrayList<String> funcArgs) {
-
-        List<String> syncResultLog = syncCallFunction(
-                agentTopicName,
-                ExecutionMessageType.HARBOR,
-                funcArgs,
-                true
-        );
-
-        boolean harborResult = JudgeSyncBaseCommandResult(syncResultLog);
-
-        return harborResult;
-    }
-
-    private boolean JudgeSyncBaseCommandResult(List<String> syncResultLog) {
         if (CollectionUtils.isEmpty(syncResultLog)) {
             log.error("基本脚本调用失败!");
             return false;
@@ -116,6 +102,33 @@ public class FuncServiceImpl implements FuncService {
         return false;
     }
 
+    @Override
+    public boolean callHarborFuncAndJudge(String agentTopicName, HarborFunctionEnum harborFunctionEnum, ArrayList<String> funcArgs) {
+
+        List<String> syncResultLog = syncCallFunction(
+                agentTopicName,
+                ExecutionMessageType.HARBOR,
+                funcArgs,
+                true
+        );
+
+        return JudgeSyncBaseCommandResult(syncResultLog);
+    }
+
+    @Override
+    public List<String> callImageFuncService(String agentTopicName, ImageFunctionEnum imageFunctionEnum, List<String> funcArgs) {
+
+        // add in
+        funcArgs.add(0, imageFunctionEnum.getFuncName());
+
+        return syncCallFunction(
+                agentTopicName,
+                ExecutionMessageType.IMAGE,
+                funcArgs,
+                true
+        );
+    }
+
 
     // 归一化调用
     private List<String> syncCallFunction(String agentTopicName, ExecutionMessageType emType, List<String> funcArgs, boolean durationTask) {

server/src/main/java/io/wdd/func/auto/service/ImageFuncScheduler.java

@@ -0,0 +1,145 @@
+package io.wdd.func.auto.service;
+
+import io.wdd.func.auto.beans.ImageSyncContext;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
+import org.springframework.stereotype.Service;
+import org.springframework.util.Assert;
+
+import javax.annotation.Resource;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static io.wdd.func.auto.beans.ImageFunctionEnum.*;
+import static io.wdd.func.auto.service.FuncServiceImpl.JudgeSyncBaseCommandResult;
+
+@Service
+@Slf4j
+public class ImageFuncScheduler {
+
+    public static final String CmiiHarborHostPrefix = "harbor.cdcyy.com.cn/cmii/";
+    @Resource
+    FuncService funcService;
+
+    public boolean runProcedure(ImageSyncContext imageSyncContext) {
+
+        // before run
+        // check base requirement
+        beforeRunProcedure(imageSyncContext);
+
+
+        // during run
+        doRunProcedure(imageSyncContext);
+
+
+        // after run
+        afterRunProcedure(imageSyncContext);
+
+
+        return true;
+    }
+
+    private void beforeRunProcedure(ImageSyncContext imageSyncContext) {
+
+        // build args
+
+        // imageFullName gzipFolderPrefix gzipFileName ossUrlPrefix proxyUrl targetHarborHost namespace targetImageFullName
+        ArrayList<String> arrayList = new ArrayList<>();
+        arrayList.add(""); //imageFullName
+        arrayList.add("/var/lib/docker/octopus_image/"); //gzipFolderPrefix
+        arrayList.add(""); //gzipFileName
+        arrayList.add("https://oss.demo.uavcmlc.com:18000/cmlc-installation/tmp/");//ossUrlPrefix
+        arrayList.add("");//proxyUrl
+        arrayList.add(imageSyncContext.getProjectMasterNode().getServerIpInV4());//targetHarborHost
+        arrayList.add(imageSyncContext.getProjectInfoPO().getProjectNamespace());//namespace
+        arrayList.add("");//targetImageFullName
+
+
+        imageSyncContext.setImageSyncFuncArgs(arrayList);
+
+        log.debug("beforeRunProcedure complete!");
+    }
+
+    private void afterRunProcedure(ImageSyncContext imageSyncContext) {
+
+        // message-pusher
+
+
+        // 检查是否安装完成, 对安装环境进行判定
+        log.debug("afterRunProcedure complete!");
+    }
+
+    private void doRunProcedure(ImageSyncContext imageSyncContext) {
+
+        List<String> realFullNameList = null;
+        ArrayList<String> cmiiAppNameTagList = imageSyncContext.getCmiiAppNameTagList();
+        ArrayList<String> imageFullNameList = imageSyncContext.getImageFullNameList();
+
+        if (CollectionUtils.isNotEmpty(cmiiAppNameTagList)) {
+            realFullNameList = cmiiAppNameTagList.stream().map(CmiiHarborHostPrefix::concat).collect(Collectors.toList());
+        }
+
+
+        if (CollectionUtils.isNotEmpty(cmiiAppNameTagList)) {
+            if (CollectionUtils.isEmpty(realFullNameList)) {
+                realFullNameList = imageFullNameList;
+            } else {
+                realFullNameList.addAll(imageFullNameList);
+            }
+        }
+
+        if (realFullNameList == null) {
+            log.info("Image Sync image list is null, return !");
+            return;
+        }
+
+        String innerWorkerAgentName = imageSyncContext.getInnerWorkerAgentName();
+        Assert.notNull(innerWorkerAgentName, "inner worker agent name cant not be null !");
+
+        ArrayList<String> imageSyncFuncArgs = imageSyncContext.getImageSyncFuncArgs();
+
+        String outsideAgentTopicName = imageSyncContext.getProjectMasterNode().getTopicName();
+
+        //
+        List<String> resultLog;
+        for (String fullName : realFullNameList) {
+            // innerWorkerAgent
+            imageSyncFuncArgs.set(0, fullName);
+            resultLog = funcService.callImageFuncService(innerWorkerAgentName, DOWNLOAD_DOCKER_IMAGE, imageSyncFuncArgs);
+            if (!JudgeSyncBaseCommandResult(resultLog)) {
+                return;
+            }
+            resultLog = funcService.callImageFuncService(innerWorkerAgentName, COMPRESS_IMAGE_TO_GZIP, imageSyncFuncArgs);
+            if (!JudgeSyncBaseCommandResult(resultLog)) {
+                return;
+            }
+            String gzipFileName = resultLog.get(0);
+            Assert.notNull(gzipFileName, "COMPRESS_IMAGE_TO_GZIP return gzip file name is null !");
+            imageSyncFuncArgs.set(2, gzipFileName);
+            resultLog = funcService.callImageFuncService(innerWorkerAgentName, UPLOAD_GZIP_TO_OSS, imageSyncFuncArgs);
+            if (!JudgeSyncBaseCommandResult(resultLog)) {
+                return;
+            }
+
+            // outside agent
+            resultLog = funcService.callImageFuncService(outsideAgentTopicName, DOWNLOAD_GZIP_IMAGE_FILE, imageSyncFuncArgs);
+            if (!JudgeSyncBaseCommandResult(resultLog)) {
+                return;
+            }
+            resultLog = funcService.callImageFuncService(outsideAgentTopicName, LOAD_DOCKER_IMAGE_FROM_GZIP, imageSyncFuncArgs);
+            if (!JudgeSyncBaseCommandResult(resultLog)) {
+                return;
+            }
+            resultLog = funcService.callImageFuncService(outsideAgentTopicName, PUSH_IMAGE_TO_TARGET_HARBOR, imageSyncFuncArgs);
+            if (!JudgeSyncBaseCommandResult(resultLog)) {
+                return;
+            }
+            resultLog = funcService.callImageFuncService(outsideAgentTopicName, UPDATE_IMAGE_TAG, imageSyncFuncArgs);
+            if (!JudgeSyncBaseCommandResult(resultLog)) {
+                return;
+            }
+        }
+
+    }
+}

server/src/main/java/io/wdd/rpc/execute/ExecutionMessageType.java

@@ -12,4 +12,7 @@ public enum ExecutionMessageType {
     // 执行Harbor仓库操作相关的内容 HARBOR
     HARBOR,
 
+    // 执行Image Sync相关的操作
+    IMAGE,
+
 }

server/src/main/java/io/wdd/server/beans/po/ServerInfoPO.java

@@ -63,6 +63,7 @@ public class ServerInfoPO implements Serializable {
 
     /**
      * server inner ipv6
+
      */
     @TableField(value = "server_ip_in_v6")
     private String serverIpInV6;
@@ -80,13 +81,13 @@ public class ServerInfoPO implements Serializable {
     private LocalDateTime expireTime;
 
     /**
-     *
+     * 
      */
     @TableField(value = "update_time")
     private LocalDateTime updateTime;
 
     /**
-     *
+     * 
      */
     @TableField(value = "create_time")
     private LocalDateTime createTime;
@@ -163,6 +164,12 @@ public class ServerInfoPO implements Serializable {
     @TableField(value = "os_kernel_info")
     private String osKernelInfo;
 
+    /**
+     * 服务器能否访问公网
+     */
+    @TableField(value = "can_access_public")
+    private Integer canAccessPublic;
+
     /**
      * machine uuid from /etc/machineid
      */
@@ -187,6 +194,12 @@ public class ServerInfoPO implements Serializable {
     @TableField(value = "version")
     private Integer version;
 
+    /**
+     * 服务器的角色信息
+     */
+    @TableField(value = "role")
+    private String role;
+
     @TableField(exist = false)
     private static final long serialVersionUID = 1L;
 }

server/src/main/java/io/wdd/server/beans/po/ServerInfoRoleEnum.java

@@ -0,0 +1,33 @@
+package io.wdd.server.beans.po;
+
+public enum ServerInfoRoleEnum {
+
+    MASTER_NODE(
+            "MASTER_NODE",
+            ""
+    );
+
+    String RoleName;
+    String Des;
+
+    ServerInfoRoleEnum(String roleName, String des) {
+        RoleName = roleName;
+        Des = des;
+    }
+
+    public String getRoleName() {
+        return RoleName;
+    }
+
+    public void setRoleName(String roleName) {
+        RoleName = roleName;
+    }
+
+    public String getDes() {
+        return Des;
+    }
+
+    public void setDes(String des) {
+        Des = des;
+    }
+}

server/src/main/java/io/wdd/server/beans/request/ProjectQueryEntity.java

@@ -21,6 +21,6 @@ public class ProjectQueryEntity {
     /*条件查询*/
     String projectName;
 
-    String projectProvince;
+    String projectNamespace;
 
 }

server/src/main/java/io/wdd/server/coreService/impl/CoreProjectServerServiceImpl.java

@@ -72,7 +72,6 @@ public class CoreProjectServerServiceImpl implements CoreProjectServerService {
 
         // 首先查询projectId是否存在 pass
 
-
         // 查询与之绑定的所有server
         // serverId projectId
         List<ProjectServerRelationPO> projectServerRelationPOList = new LambdaQueryChainWrapper<ProjectServerRelationPO>(projectServerRelationService.getBaseMapper())

server/src/main/java/io/wdd/server/coreService/impl/CoreProjectServiceImpl.java

@@ -31,7 +31,7 @@ public class CoreProjectServiceImpl implements CoreProjectService {
     @Override
     public Page<ProjectInfoPO> projectQueryByEntity(ProjectQueryEntity projectQueryEntity) {
 
-        if (projectQueryEntity == null || projectQueryEntity.getProjectName() == null || projectQueryEntity.getProjectProvince() == null) {
+        if (projectQueryEntity == null || projectQueryEntity.getProjectName() == null || projectQueryEntity.getProjectNamespace() == null) {
             return null;
         }
 
@@ -52,9 +52,9 @@ public class CoreProjectServiceImpl implements CoreProjectService {
                         projectQueryEntity.getProjectName()
                 )
                 .likeRight(
-                        StringUtils.isNotBlank(projectQueryEntity.getProjectProvince()),
+                        StringUtils.isNotBlank(projectQueryEntity.getProjectNamespace()),
-                        ProjectInfoPO::getProjectProvince,
+                        ProjectInfoPO::getProjectNamespace,
-                        projectQueryEntity.getProjectProvince()
+                        projectQueryEntity.getProjectNamespace()
                 )
                 .page(poPage);
 

server/src/main/java/io/wdd/server/mapper/ServerInfoMapper.java

@@ -4,9 +4,9 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import io.wdd.server.beans.po.ServerInfoPO;
 
 /**
- * @author wdd
+ * @author wddsh
  * @description 针对表【server_info】的数据库操作Mapper
- * @createDate 2023-10-08 11:24:01
+ * @createDate 2024-03-26 18:11:29
  * @Entity io.wdd.server.beans.po.ServerInfoPO
  */
 public interface ServerInfoMapper extends BaseMapper<ServerInfoPO> {

server/src/main/java/io/wdd/server/service/ServerInfoService.java

@@ -4,9 +4,9 @@ import com.baomidou.mybatisplus.extension.service.IService;
 import io.wdd.server.beans.po.ServerInfoPO;
 
 /**
- * @author wdd
+ * @author wddsh
  * @description 针对表【server_info】的数据库操作Service
- * @createDate 2023-10-08 11:24:01
+ * @createDate 2024-03-26 18:11:29
  */
 public interface ServerInfoService extends IService<ServerInfoPO> {
 

server/src/main/java/io/wdd/server/service/impl/ServerInfoServiceImpl.java

@@ -7,9 +7,9 @@ import io.wdd.server.service.ServerInfoService;
 import org.springframework.stereotype.Service;
 
 /**
- * @author wdd
+ * @author wddsh
  * @description 针对表【server_info】的数据库操作Service实现
- * @createDate 2023-10-08 11:24:01
+ * @createDate 2024-03-26 18:11:29
  */
 @Service
 public class ServerInfoServiceImpl extends ServiceImpl<ServerInfoMapper, ServerInfoPO>

server/src/main/resources/mapper/ServerInfoMapper.xml

@@ -29,10 +29,12 @@
         <result property="virtualization" column="virtualization" jdbcType="VARCHAR"/>
         <result property="osInfo" column="os_info" jdbcType="VARCHAR"/>
         <result property="osKernelInfo" column="os_kernel_info" jdbcType="VARCHAR"/>
+        <result property="canAccessPublic" column="can_access_public" jdbcType="TINYINT"/>
         <result property="machineId" column="machine_id" jdbcType="VARCHAR"/>
         <result property="comment" column="comment" jdbcType="VARCHAR"/>
         <result property="isDelete" column="is_delete" jdbcType="TINYINT"/>
         <result property="version" column="version" jdbcType="INTEGER"/>
+        <result property="role" column="role" jdbcType="VARCHAR"/>
     </resultMap>
 
     <sql id="Base_Column_List">
@@ -45,7 +47,7 @@
         cpu_core,memory_total,disk_total,
         disk_usage,io_speed,tcp_control,
         virtualization,os_info,os_kernel_info,
-        machine_id,comment,is_delete,
+        can_access_public,machine_id,comment,
-        version
+        is_delete,version,role
     </sql>
 </mapper>

server/src/test/java/io/wdd/server/controller/ProjectInfoTest.java

@@ -95,7 +95,7 @@ public class ProjectInfoTest {
         projectQueryEntity.setPageNumber(1);
         projectQueryEntity.setPageSize(10);
         projectQueryEntity.setProjectName("Test Project");
-        projectQueryEntity.setProjectProvince("Test Province");
+        projectQueryEntity.setProjectNamespace("Test Province");
 
         // Act
         R<Page<ProjectInfoPO>> result = projectController.projectQueryOne(projectQueryEntity);

server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java

@@ -0,0 +1,84 @@
+package io.wdd.server.func;
+
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import io.wdd.func.auto.beans.ImageSyncContext;
+import io.wdd.func.auto.service.ImageFuncScheduler;
+import io.wdd.server.beans.po.ProjectInfoPO;
+import io.wdd.server.beans.po.ServerInfoPO;
+import io.wdd.server.beans.request.ProjectQueryEntity;
+import io.wdd.server.beans.vo.ProjectServerVO;
+import io.wdd.server.coreService.CoreProjectServerService;
+import io.wdd.server.coreService.CoreProjectService;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Test;
+import org.springframework.boot.test.context.SpringBootTest;
+
+import javax.annotation.Resource;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+@SpringBootTest
+@Slf4j
+public class TestImageSyncScheduler {
+
+    @Resource
+    ImageFuncScheduler imageFuncScheduler;
+
+
+    @Resource
+    CoreProjectService coreProjectService;
+
+    @Resource
+    CoreProjectServerService coreProjectServerService;
+
+    @Test
+    public void runImageSync() {
+
+        ArrayList<String> CmiiAppNameList = new ArrayList<>(List.of(
+
+        ));
+
+        ArrayList<String> ImageFullNameList = new ArrayList<>(List.of(
+
+        ));
+
+        String projectNamespace = "xmyd";
+
+
+        // start
+        ImageSyncContext imageSyncContext = new ImageSyncContext();
+
+        // inner
+        imageSyncContext.setInnerWorkerAgentName("Chengdu-amd64-71-3571gd");
+
+        ProjectQueryEntity projectQueryEntity = ProjectQueryEntity.builder().projectNamespace(projectNamespace).build();
+
+        Page<ProjectInfoPO> page = coreProjectService.projectQueryByEntity(projectQueryEntity);
+        if (page.getSize() < 1) {
+            log.error(" project name space error !");
+            return;
+        }
+        ProjectInfoPO projectInfoPO = page.getRecords().get(0);
+        imageSyncContext.setProjectInfoPO(projectInfoPO);
+
+        log.info("project name => {}  namespace => {}", projectInfoPO.getProjectName(), projectInfoPO.getProjectNamespace());
+        // master
+        ProjectServerVO projectServerVO = coreProjectServerService.projectServerOne(projectInfoPO.getProjectId());
+        Optional<ServerInfoPO> master = projectServerVO.getBindingServerList().stream().filter(server -> StringUtils.startsWith(server.getRole(), "master")).findFirst();
+        if (master.isEmpty()) {
+            log.error("project master node not set ！");
+            return;
+        }
+        imageSyncContext.setProjectMasterNode(master.get());
+
+        log.info("project master node inner ipv4 is => {}", master.get().getServerIpInV4());
+
+        imageSyncContext.setImageFullNameList(ImageFullNameList);
+        imageSyncContext.setCmiiAppNameTagList(CmiiAppNameList);
+
+        imageFuncScheduler.runProcedure(imageSyncContext);
+    }
+
+}