[ server ] [ execution ]- execution log persistence accomplish

agent/src/main/java/io/wdd/agent/executor/status/AppStatusExecutor.java

@@ -2,7 +2,6 @@ package io.wdd.agent.executor.status;
 
 
 import io.wdd.agent.config.utils.AgentCommonThreadPool;
-import io.wdd.agent.executor.CheckSingleAppStatusCallable;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Service;

server/src/main/java/io/wdd/rpc/execute/config/CommandReaderConfig.java

@@ -28,6 +28,8 @@ public class CommandReaderConfig {
 
     private String streamKey;
 
+    private String recordId;
+
     /**
      * 执行的结果对象，保存在此处
      */

server/src/main/java/io/wdd/rpc/execute/config/CommandReaderConfigBean.java

@@ -8,6 +8,8 @@ import static io.wdd.rpc.execute.result.RedisStreamReaderConfig.REDIS_STREAM_LIS
 @Configuration
 public class CommandReaderConfigBean {
 
+    // todo must support for multi thread
+    // its not thread safe now
     @Bean
     public CommandReaderConfig commandReaderConfig() {
 
@@ -16,6 +18,7 @@ public class CommandReaderConfigBean {
                 .consumerName(REDIS_STREAM_LISTENER_CONSUMER_NAME)
                 .streamKey("ccc")
                 .consumerType(REDIS_STREAM_LISTENER_CONSUMER_NAME)
+                .group("ccc")
                 .ExecutionResult(null)
                 .build();
     }

server/src/main/java/io/wdd/rpc/execute/config/ExecutionLog.java

@@ -1,18 +1,16 @@
 package io.wdd.rpc.execute.config;
 
 import io.swagger.annotations.ApiModel;
+import io.wdd.server.beans.po.ExecutionLogPO;
 import lombok.AllArgsConstructor;
 import lombok.Data;
-import lombok.NoArgsConstructor;
+
-import lombok.experimental.SuperBuilder;
+import java.time.LocalDateTime;
 
 @Data
 @AllArgsConstructor
-@NoArgsConstructor
-@SuperBuilder(toBuilder = true)
 @ApiModel("Execution模快持久化Bean对象")
-public class ExecutionLogBean {
+public class ExecutionLog extends ExecutionLogPO {
 
-    private String name;
 
 }

server/src/main/java/io/wdd/rpc/execute/result/CommandResultReader.java

@@ -66,6 +66,7 @@ public class CommandResultReader implements StreamListener<String, MapRecord<Str
 
         // 赋值给外部的结果，是的执行的结果可以被拿到
         this.commandReaderConfig.setExecutionResult(executionResultFormat);
+        this.commandReaderConfig.setRecordId(String.valueOf(messageId));
 
         log.info("Octopus Agent [ {} ] execution of [ {} ] Time is [ {} ] stream recordId is [{}]",
                  streamKey,

server/src/main/java/io/wdd/rpc/execute/service/CoreExecutionServiceImpl.java

@@ -5,7 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import io.wdd.common.beans.executor.ExecutionMessage;
 import io.wdd.common.beans.rabbitmq.OctopusMessage;
 import io.wdd.common.beans.rabbitmq.OctopusMessageType;
-import io.wdd.rpc.execute.config.ExecutionLogBean;
+import io.wdd.rpc.execute.config.ExecutionLog;
 import io.wdd.rpc.execute.result.BuildStreamReader;
 import io.wdd.rpc.message.sender.ToAgentMessageSender;
 import lombok.extern.slf4j.Slf4j;
@@ -37,91 +37,113 @@ public class CoreExecutionServiceImpl implements CoreExecutionService {
 
     @Override
     public String SendCommandToAgent(String topicName, String command) {
-        return this.SendCommandToAgent(topicName, List.of(command));
+        return this.SendCommandToAgent(topicName,
+                                       List.of(command));
     }
 
     @Override
     public String SendCommandToAgent(String topicName, List<String> commandList) {
-        return this.SendCommandToAgent(topicName,"manual-command", commandList);
+        return this.SendCommandToAgent(topicName,
+                                       "manual-command",
+                                       commandList);
     }
 
     @Override
     public String SendCommandToAgent(String topicName, String type, List<String> commandList) {
 
-        // 构造 Execution Command对应的消息体
+        // 归一化type类型 不行
-        OctopusMessage octopusMessage = this.generateOctopusMessage(topicName, type, commandList);
 
-        // 获取 ResultKey
+        // 构造 Execution Command对应的消息体
-        ExecutionMessage executionMessage = (ExecutionMessage) octopusMessage.getContent();
+        ExecutionMessage executionMessage =
-        String executionMsg;
+                ExecutionMessage.builder()
+                                .type(type)
+                                .commandList(commandList)
+                                .resultKey(ExecutionMessage.GetResultKey(topicName))
+                                .build();
+
+
+        String executionMessageString;
         try {
-            executionMsg = objectMapper.writeValueAsString(executionMessage);
+            executionMessageString = objectMapper.writeValueAsString(executionMessage);
-            octopusMessage.setContent(executionMsg);
 
         } catch (JsonProcessingException e) {
             throw new RuntimeException(e);
         }
+
+        OctopusMessage octopusMessage = OctopusMessage
+                .builder()
+                .type(OctopusMessageType.EXECUTOR)
+                .init_time(LocalDateTime.now())
+                .content(executionMessageString)
+                .uuid(topicName)
+                .build();
+
         String resultKey = executionMessage.getResultKey();
 
         // send the message
         messageSender.send(octopusMessage);
 
         // set up the stream read group
-        String group = redisTemplate.opsForStream().createGroup(resultKey, resultKey);
+        String group = redisTemplate
-        log.info("set consumer group [{}] for the stream key with => [ {} ]", group, resultKey);
+                .opsForStream()
+                .createGroup(resultKey,
+                             resultKey);
+        log.info("set consumer group [{}] for the stream key with => [ {} ]",
+                 group,
+                 resultKey);
 
         // change the redis stream listener container
         // createStreamReader.registerStreamReader(COMMAND_RESULT_REDIS_STREAM_LISTENER_CONTAINER, resultKey);
 
         // construct the persistent Bean
-        ExecutionLogBean executionLogBean = buildPersistentLogBeanFromOctopusMessage(octopusMessage);
+        ExecutionLog executionLog = buildPersistentLogBeanFromOctopusMessage(octopusMessage, executionMessage);
         // send resultKey to ExecutionResultDaemonHandler
-        WAIT_EXECUTION_RESULT_LIST.put(resultKey, executionLogBean);
+        WAIT_EXECUTION_RESULT_LIST.put(resultKey,
+                                       executionLog);
+
+        // help gc
+        executionMessage = null;
+        octopusMessage =null;
 
         return resultKey;
     }
 
-    private ExecutionLogBean buildPersistentLogBeanFromOctopusMessage(OctopusMessage octopusMessage) {
+    private ExecutionLog buildPersistentLogBeanFromOctopusMessage(OctopusMessage octopusMessage, ExecutionMessage executionMessage) {
-
+        ExecutionLog executionLog = new ExecutionLog();
-        return new ExecutionLogBean();
+        executionLog.setAgentTopicName(octopusMessage.getUuid());
+        executionLog.setResultKey((String) octopusMessage.getContent());
+        executionLog.setCommandList(String.valueOf(executionMessage.getCommandList()));
+        executionLog.setType(executionMessage.getType());
+        executionLog.setResultKey(executionMessage.getResultKey());
+        return executionLog;
     }
 
 
     @Override
     public List<String> SendCommandToAgent(List<String> topicNameList, String type, List<String> command) {
-        return topicNameList.stream().map(
+        return topicNameList.stream()
-                topicName -> {
+                            .map(
-                    return this.SendCommandToAgent(topicName, type, command);
+                                    topicName -> {
-                }
+                                        return this.SendCommandToAgent(topicName,
-        ).collect(Collectors.toList());
+                                                                       type,
+                                                                       command);
+                                    }
+                            )
+                            .collect(Collectors.toList());
     }
 
-    private OctopusMessage generateOctopusMessage(String topicName, String type, List<String> commandList){
+    @Deprecated
+    private OctopusMessage generateOctopusMessage(String topicName, String type, List<String> commandList) {
 
-        ExecutionMessage executionMessage = generateExecutionMessage(
+        return null;
-                type,
-                commandList,
-                ExecutionMessage.GetResultKey(topicName)
-        );
-
-        return OctopusMessage.builder()
-                .type(OctopusMessageType.EXECUTOR)
-                .init_time(LocalDateTime.now())
-                .content(executionMessage)
-                .uuid(topicName)
-                .build();
     }
 
+    @Deprecated
     private ExecutionMessage generateExecutionMessage(String type, List<String> commandList, String resultKey) {
 
-        return ExecutionMessage.builder()
+        return null;
-                .type(type)
-                .commandList(commandList)
-                .resultKey(resultKey)
-                .build();
 
     }
 
 
-
 }

server/src/main/java/io/wdd/rpc/execute/service/ExecutionResultDaemonHandler.java

@@ -1,9 +1,11 @@
 package io.wdd.rpc.execute.service;
 
 
+import io.wdd.common.utils.TimeUtils;
 import io.wdd.rpc.execute.config.CommandReaderConfig;
-import io.wdd.rpc.execute.config.ExecutionLogBean;
+import io.wdd.rpc.execute.config.ExecutionLog;
 import io.wdd.rpc.execute.result.BuildStreamReader;
+import io.wdd.server.service.ExecutionLogService;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.springframework.stereotype.Service;
@@ -11,6 +13,7 @@ import org.springframework.stereotype.Service;
 import javax.annotation.PostConstruct;
 import javax.annotation.Resource;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.*;
 
@@ -28,7 +31,7 @@ public class ExecutionResultDaemonHandler {
      * <p>
      * which means there are execution running , waiting for their result to handle
      */
-    public static final ConcurrentHashMap<String, ExecutionLogBean> WAIT_EXECUTION_RESULT_LIST = new ConcurrentHashMap<>(32);
+    public static final ConcurrentHashMap<String, ExecutionLog> WAIT_EXECUTION_RESULT_LIST = new ConcurrentHashMap<>(32);
     private final int MAX_TIMEOUT_WAITING_FOR_EXECUTION_RESULT = 70;
 
     @Resource
@@ -37,6 +40,9 @@ public class ExecutionResultDaemonHandler {
     @Resource
     CommandReaderConfig commandReaderConfig;
 
+    @Resource
+    ExecutionLogService executionLogService;
+
     @PostConstruct
     public void startExecutionDaemonHandler() {
 
@@ -133,9 +139,7 @@ public class ExecutionResultDaemonHandler {
                             throw new RuntimeException(e);
                         }
 
-                        return new ArrayList<>(
+                        return null;
-                                List.of("[ Failed ] - execution has failed !")
-                        );
                     }
             );
 
@@ -144,22 +148,34 @@ public class ExecutionResultDaemonHandler {
                     .anyOf(falloutTimeFuture,
                            executionResultFuture);
 
-            complete.whenComplete(
+            complete
-                    (resultString, e) -> {
+                    .whenComplete(
+                            (result, e) -> {
 
-                        log.info("execution result are => {}",
+                                log.info("execution result are => {}",
-                                 resultString);
+                                         result);
 
-                        // 持久化存储对应的结果
+                                // 持久化存储对应的结果
+                                ExecutionLog executionLog = WAIT_EXECUTION_RESULT_LIST.get(resultKey);
+                                executionLog.setAcTime(TimeUtils.currentTime());
+                                executionLog.setResultContent(String.valueOf(commandReaderConfig.getExecutionResult()));
+                                executionLog.setResultCode(
+                                        CollectionUtils.isEmpty((Collection) result) ? 1 : 0
+                                );
+                                executionLog.setRecordId(commandReaderConfig.getRecordId());
+                                executionLogService.save(executionLog);
 
 
-                        // 清除此次任务的内容
+                                // 清除此次任务的内容
-                        WAIT_EXECUTION_RESULT_LIST.remove(resultKey);
+                                WAIT_EXECUTION_RESULT_LIST.remove(resultKey);
 
-                        log.info("[Execution] - whole process are complete !");
+                                log.info("[Execution] - whole process are complete !");
-                    }
+                            }
-            );
+                    );
 
+            // very important
+            // stuck the main thread , otherwise it will create a dead loop , really bad
+            complete.join();
 
         }
 

server/src/main/java/io/wdd/server/beans/po/ExecutionLogPO.java

@@ -0,0 +1,126 @@
+package io.wdd.server.beans.po;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import java.io.Serializable;
+import java.time.LocalDateTime;
+import java.util.Date;
+import lombok.Data;
+
+/**
+ * 
+ * @TableName execution_log
+ */
+@TableName(value ="execution_log")
+@Data
+public class ExecutionLogPO implements Serializable {
+    /**
+     * 
+     */
+    @TableId
+    private Long id;
+
+    /**
+     * 
+     */
+    private String agentTopicName;
+
+    /**
+     * 
+     */
+    private String resultKey;
+
+    /**
+     * 
+     */
+    private LocalDateTime acTime;
+
+    /**
+     * 
+     */
+    private Integer resultCode;
+
+    /**
+     * 执行类型，命令行或者Function
+     */
+    private String type;
+
+    /**
+     * 
+     */
+    private String commandList;
+
+    /**
+     * 命令执行结果
+     */
+    private String resultContent;
+
+    /**
+     * redis stream key中任务结果对应的RecordId
+     */
+    private String recordId;
+
+    @TableField(exist = false)
+    private static final long serialVersionUID = 1L;
+
+    @Override
+    public boolean equals(Object that) {
+        if (this == that) {
+            return true;
+        }
+        if (that == null) {
+            return false;
+        }
+        if (getClass() != that.getClass()) {
+            return false;
+        }
+        ExecutionLogPO other = (ExecutionLogPO) that;
+        return (this.getId() == null ? other.getId() == null : this.getId().equals(other.getId()))
+            && (this.getAgentTopicName() == null ? other.getAgentTopicName() == null : this.getAgentTopicName().equals(other.getAgentTopicName()))
+            && (this.getResultKey() == null ? other.getResultKey() == null : this.getResultKey().equals(other.getResultKey()))
+            && (this.getAcTime() == null ? other.getAcTime() == null : this.getAcTime().equals(other.getAcTime()))
+            && (this.getResultCode() == null ? other.getResultCode() == null : this.getResultCode().equals(other.getResultCode()))
+            && (this.getType() == null ? other.getType() == null : this.getType().equals(other.getType()))
+            && (this.getCommandList() == null ? other.getCommandList() == null : this.getCommandList().equals(other.getCommandList()))
+            && (this.getResultContent() == null ? other.getResultContent() == null : this.getResultContent().equals(other.getResultContent()))
+            && (this.getRecordId() == null ? other.getRecordId() == null : this.getRecordId().equals(other.getRecordId()));
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + ((getId() == null) ? 0 : getId().hashCode());
+        result = prime * result + ((getAgentTopicName() == null) ? 0 : getAgentTopicName().hashCode());
+        result = prime * result + ((getResultKey() == null) ? 0 : getResultKey().hashCode());
+        result = prime * result + ((getAcTime() == null) ? 0 : getAcTime().hashCode());
+        result = prime * result + ((getResultCode() == null) ? 0 : getResultCode().hashCode());
+        result = prime * result + ((getType() == null) ? 0 : getType().hashCode());
+        result = prime * result + ((getCommandList() == null) ? 0 : getCommandList().hashCode());
+        result = prime * result + ((getResultContent() == null) ? 0 : getResultContent().hashCode());
+        result = prime * result + ((getRecordId() == null) ? 0 : getRecordId().hashCode());
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append(getClass().getSimpleName());
+        sb.append(" [");
+        sb.append("Hash = ").append(hashCode());
+        sb.append(", id=").append(id);
+        sb.append(", agentTopicName=").append(agentTopicName);
+        sb.append(", resultKey=").append(resultKey);
+        sb.append(", acTime=").append(acTime);
+        sb.append(", resultCode=").append(resultCode);
+        sb.append(", type=").append(type);
+        sb.append(", commandList=").append(commandList);
+        sb.append(", resultContent=").append(resultContent);
+        sb.append(", recordId=").append(recordId);
+        sb.append(", serialVersionUID=").append(serialVersionUID);
+        sb.append("]");
+        return sb.toString();
+    }
+}

server/src/main/java/io/wdd/server/mapper/ExecutionLogMapper.java

@@ -0,0 +1,18 @@
+package io.wdd.server.mapper;
+
+import io.wdd.server.beans.po.ExecutionLogPO;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+/**
+* @author wdd
+* @description 针对表【execution_log】的数据库操作Mapper
+* @createDate 2023-01-13 17:58:33
+* @Entity io.wdd.server.beans.po.ExecutionLogPO
+*/
+public interface ExecutionLogMapper extends BaseMapper<ExecutionLogPO> {
+
+}
+
+
+
+

server/src/main/java/io/wdd/server/service/ExecutionLogService.java

@@ -0,0 +1,13 @@
+package io.wdd.server.service;
+
+import io.wdd.server.beans.po.ExecutionLogPO;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+/**
+* @author wdd
+* @description 针对表【execution_log】的数据库操作Service
+* @createDate 2023-01-13 17:58:33
+*/
+public interface ExecutionLogService extends IService<ExecutionLogPO> {
+
+}

server/src/main/java/io/wdd/server/service/impl/ExecutionLogServiceImpl.java

@@ -0,0 +1,22 @@
+package io.wdd.server.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import io.wdd.server.beans.po.ExecutionLogPO;
+import io.wdd.server.service.ExecutionLogService;
+import io.wdd.server.mapper.ExecutionLogMapper;
+import org.springframework.stereotype.Service;
+
+/**
+* @author wdd
+* @description 针对表【execution_log】的数据库操作Service实现
+* @createDate 2023-01-13 17:58:33
+*/
+@Service
+public class ExecutionLogServiceImpl extends ServiceImpl<ExecutionLogMapper, ExecutionLogPO>
+    implements ExecutionLogService{
+
+}
+
+
+
+

server/src/main/resources/mapper/ExecutionLogMapper.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper
+        PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
+        "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="io.wdd.server.mapper.ExecutionLogMapper">
+
+    <resultMap id="BaseResultMap" type="io.wdd.server.beans.po.ExecutionLogPO">
+            <id property="id" column="id" jdbcType="BIGINT"/>
+            <result property="agentTopicName" column="agent_topic_name" jdbcType="VARCHAR"/>
+            <result property="resultKey" column="result_key" jdbcType="VARCHAR"/>
+            <result property="acTime" column="ac_time" jdbcType="TIMESTAMP"/>
+            <result property="resultCode" column="result_code" jdbcType="INTEGER"/>
+            <result property="type" column="type" jdbcType="VARCHAR"/>
+            <result property="commandList" column="command_list" jdbcType="VARCHAR"/>
+            <result property="resultContent" column="result_content" jdbcType="VARCHAR"/>
+            <result property="recordId" column="record_id" jdbcType="VARCHAR"/>
+    </resultMap>
+
+    <sql id="Base_Column_List">
+        id,agent_topic_name,result_key,
+        ac_time,result_code,type,
+        command_list,result_content,record_id
+    </sql>
+</mapper>