[ agent ] [executor] Function File Executor -1

agent/agent-entrypoint.sh

@@ -372,10 +372,27 @@ GenerateSystemInfo() {
   FunctionEnd
 }
 
+PrintEnv(){
+
+  FunctionStart
+
+  env
+
+  FunctionEnd
+}
+
 main() {
 
   GenerateSystemInfo
 
+  PrintEnv
+
 }
 
-main && env && java ${JAVA_OPTS} -jar /wdd/agent.jar
+main
+
+scp -r /wdd /host/wdd
+
+chroot /host
+
+java ${JAVA_OPTS} -jar /wdd/agent.jar

agent/src/main/java/io/wdd/agent/config/message/handler/OMHandlerExecutor.java

@@ -9,7 +9,7 @@ import org.springframework.stereotype.Component;
 
 import javax.annotation.Resource;
 
-import static io.wdd.agent.executor.function.CollectAllFunctionToServer.ALL_FUNCTIONS;
+import static io.wdd.agent.executor.function.CollectAllExecutorFunction.ALL_FUNCTION_MAP;
 
 @Component
 public class OMHandlerExecutor extends AbstractOctopusMessageHandler {
@@ -30,7 +30,7 @@ public class OMHandlerExecutor extends AbstractOctopusMessageHandler {
         ExecutionMessage executionMessage = (ExecutionMessage) octopusMessage.getContent();
         String executionType = executionMessage.getType();
 
-        if (ALL_FUNCTIONS.contains(executionType)) {
+        if (ALL_FUNCTION_MAP.containsKey(executionType)) {
             // execute the exist function
             functionExecutor.execute(executionMessage);
 

agent/src/main/java/io/wdd/agent/executor/config/FunctionReader.java

@@ -7,6 +7,7 @@ import org.springframework.util.StringUtils;
 import java.io.BufferedReader;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -25,18 +26,7 @@ public class FunctionReader {
         try {
 
             BufferedReader bufferedReader = new BufferedReader(new FileReader(functionFilePath));
-            String line = bufferedReader.readLine();
+            result = doReadContent(result, bufferedReader);
-
-            if (line != null) {
-                result = new ArrayList<>(64);
-            }
-
-            while (line != null) {
-                if (!StringUtils.isEmpty(line)) {
-                    result.add(this.SplitLineToCommandList(line));
-                }
-                line = bufferedReader.readLine();
-            }
 
 
         } catch (IOException e) {
@@ -47,6 +37,41 @@ public class FunctionReader {
 
     }
 
+
+    public List<List<String>> ReadStringToCommandList(String functionContent) {
+
+        List<List<String>> result = null;
+
+        try {
+
+            BufferedReader bufferedReader = new BufferedReader(new StringReader(functionContent));
+            result = doReadContent(result, bufferedReader);
+
+
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+        return result;
+
+    }
+
+    private List<List<String>> doReadContent(List<List<String>> result, BufferedReader bufferedReader) throws IOException {
+        String line = bufferedReader.readLine();
+
+        if (line != null) {
+            result = new ArrayList<>(64);
+        }
+
+        while (line != null) {
+            if (!StringUtils.isEmpty(line)) {
+                result.add(this.SplitLineToCommandList(line));
+            }
+            line = bufferedReader.readLine();
+        }
+        return result;
+    }
+
     public List<String> SplitLineToCommandList(String commandLine) {
 
         return Arrays.stream(commandLine.split(" ")).collect(Collectors.toList());

agent/src/main/java/io/wdd/agent/executor/function/CollectAllExecutorFunction.java

@@ -0,0 +1,147 @@
+package io.wdd.agent.executor.function;
+
+
+import com.alibaba.nacos.api.NacosFactory;
+import com.alibaba.nacos.api.config.ConfigService;
+import com.alibaba.nacos.api.exception.NacosException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.wdd.agent.executor.config.FunctionReader;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.DirectoryFileFilter;
+import org.apache.commons.io.filefilter.FileFilterUtils;
+import org.apache.commons.io.filefilter.IOFileFilter;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+import org.yaml.snakeyaml.Yaml;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.Resource;
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Component
+@Lazy
+@Slf4j
+public class CollectAllExecutorFunction {
+
+
+    /**
+     * store the Octopus Agent Functions and Reflection Class Path
+     * key: function name
+     * value: function shell List<String> contend
+     */
+    public static HashMap<String, List<List<String>>> ALL_FUNCTION_MAP = new HashMap<>(128);
+
+    @Value("${spring.cloud.nacos.config.server-addr}")
+    String nacosAddr;
+
+    @Value("${spring.cloud.nacos.config.group}")
+    String group;
+
+    @Value("${spring.cloud.nacos.config.file-extension}")
+    String fileExtension;
+
+    @Value("${octopus.executor.name}")
+    String dataId;
+
+    @Resource
+    FunctionReader functionReader;
+
+    @Resource
+    ObjectMapper objectMapper;
+
+    @PostConstruct
+    private void CollectAllFunctionFromNacos() {
+
+        try {
+
+            // Initialize the configuration service, and the console automatically obtains the following parameters through the sample code.
+            String completeDataId = dataId + "." + fileExtension;
+            Properties properties = new Properties();
+            properties.put("serverAddr", nacosAddr);
+
+            ConfigService configService = NacosFactory.createConfigService(properties);
+
+            // Actively get the configuration.
+            String content = configService.getConfig(completeDataId, group, 5000);
+
+            log.info("functions get from nacos are {}", content);
+
+            parseNacosFunctionYamlToMap(content);
+
+
+        } catch (NacosException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void parseNacosFunctionYamlToMap(String content) {
+
+        Yaml yaml = new Yaml();
+
+        yaml.loadAll(content).iterator().forEachRemaining(
+                realFunction -> {
+
+                    if (!(realFunction instanceof LinkedHashMap)) {
+                        System.out.println("realFunction = " + realFunction);
+                    }
+
+                    Map<String, String> stringMap = (Map<String, String>) realFunction;
+
+                    Optional<String> functionName = stringMap.keySet().stream().findFirst();
+
+                    List<List<String>> commandList = functionReader.ReadStringToCommandList(stringMap.get(functionName.get()));
+
+                    /*log.info("Function {} , content is {}", functionName.get(), commandList);*/
+
+                    ALL_FUNCTION_MAP.put(
+                            functionName.get(),
+                            commandList
+                    );
+
+                }
+        );
+
+
+    }
+
+
+    /**
+     * due to can't get shell from the jar file
+     * this is deprecated
+     */
+//    @PostConstruct
+    private void CollectAllFunctionShellScriptName() {
+
+        // scan current package files name and  store them to FUNCTION_REFLECTION
+
+
+//        Path absolutePath = FileSystems.getDefault().getPath("" ).toAbsolutePath();
+        Path absolutePath = Paths.get("").toAbsolutePath();
+        log.info("current absolute path is {}", absolutePath);
+
+        Path currentDirectory = Path.of(absolutePath + "/src/main/java/io/wdd/agent/executor/function").toAbsolutePath();
+
+
+        IOFileFilter fileFilter = FileFilterUtils.suffixFileFilter(".sh");
+        IOFileFilter directoryFileFilter = DirectoryFileFilter.INSTANCE;
+
+        Collection<File> functionFileList = FileUtils.listFiles(currentDirectory.toFile(), fileFilter, directoryFileFilter);
+
+        log.debug("all function shell script files are : {}", functionFileList);
+
+        Map<String, String> collect = functionFileList.stream().collect(Collectors.toMap(
+                functionFile -> functionFile.getName().split("\\.")[0],
+                functionFile -> functionFile.getAbsolutePath()
+        ));
+
+//        ALL_FUNCTION_MAP.putAll(collect);
+
+    }
+
+}

agent/src/main/java/io/wdd/agent/executor/function/CollectAllFunctionToServer.java

@@ -1,63 +0,0 @@
-package io.wdd.agent.executor.function;
-
-
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.filefilter.DirectoryFileFilter;
-import org.apache.commons.io.filefilter.FileFilterUtils;
-import org.apache.commons.io.filefilter.IOFileFilter;
-import org.springframework.context.annotation.Lazy;
-import org.springframework.stereotype.Component;
-
-import javax.annotation.PostConstruct;
-import java.io.File;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.*;
-import java.util.stream.Collectors;
-
-@Component
-@Lazy
-@Slf4j
-public class CollectAllFunctionToServer {
-
-
-    public static Set<String> ALL_FUNCTIONS = new HashSet<>(128);
-
-    /**
-     * store the Octopus Agent Functions and Reflection Class Path
-     *  key: function name
-     *  value: function shell script relative path
-     *
-     */
-    public static HashMap<String, String> FUNCTION_REFLECTION = new HashMap<>(128);
-
-
-    @PostConstruct
-    private void CollectAllFunctionShellScriptName(){
-
-        // scan current package files name and  store them to FUNCTION_REFLECTION
-
-
-        Path absolutePath = Paths.get("").toAbsolutePath();
-
-        Path currentDirectory = Path.of(absolutePath + "/src/main/java/io/wdd/agent/executor/function").toAbsolutePath();
-
-
-        IOFileFilter fileFilter = FileFilterUtils.suffixFileFilter(".sh");
-        IOFileFilter directoryFileFilter = DirectoryFileFilter.INSTANCE;
-
-        Collection<File> functionFileList = FileUtils.listFiles(currentDirectory.toFile(), fileFilter, directoryFileFilter);
-
-        log.debug("all function shell script files are : {}", functionFileList);
-
-        Map<String, String> collect = functionFileList.stream().collect(Collectors.toMap(
-                functionFile -> functionFile.getName().split("\\.")[0],
-                functionFile -> functionFile.getAbsolutePath()
-        ));
-
-        FUNCTION_REFLECTION.putAll(collect);
-
-    }
-
-}

agent/src/main/java/io/wdd/agent/executor/shell/CommandExecutor.java

@@ -5,7 +5,6 @@ import io.wdd.agent.executor.redis.StreamSender;
 import io.wdd.agent.executor.thread.DaemonLogThread;
 import io.wdd.agent.executor.thread.LogToStreamSender;
 import io.wdd.common.beans.executor.ExecutionMessage;
-import io.wdd.common.handler.MyRuntimeException;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.context.annotation.Configuration;
 
@@ -26,8 +25,7 @@ public class CommandExecutor {
 
 
     /**
-     *
+     * handle command from octopus server
-     *  handle command from octopus server
      *
      * @param executionMessage get from EXECUTOR_HANDLER
      */
@@ -52,7 +50,7 @@ public class CommandExecutor {
     }
 
 
-    public void processExecute(String streamKey, ProcessBuilder processBuilder){
+    public void processExecute(String streamKey, ProcessBuilder processBuilder) {
 
         processBuilder.redirectErrorStream(true);
         processBuilder.inheritIO();

agent/src/main/java/io/wdd/agent/executor/shell/FunctionExecutor.java

@@ -1,6 +1,7 @@
 package io.wdd.agent.executor.shell;
 
 import io.wdd.agent.executor.config.FunctionReader;
+import io.wdd.agent.executor.function.CollectAllExecutorFunction;
 import io.wdd.common.beans.executor.ExecutionMessage;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
@@ -9,7 +10,7 @@ import javax.annotation.Resource;
 import java.util.List;
 import java.util.stream.Collectors;
 
-import static io.wdd.agent.executor.function.CollectAllFunctionToServer.FUNCTION_REFLECTION;
+import static io.wdd.agent.executor.function.CollectAllExecutorFunction.ALL_FUNCTION_MAP;
 
 @Service
 @Slf4j
@@ -21,13 +22,17 @@ public class FunctionExecutor {
     @Resource
     CommandExecutor commandExecutor;
 
+    // todo called by timer
+    @Resource
+    CollectAllExecutorFunction collectAllExecutorFunction;
+
     public void execute(ExecutionMessage executionMessage) {
 
         String resultKey = executionMessage.getResultKey();
 
-        String functionShellScriptFileName = FUNCTION_REFLECTION.get(executionMessage.getType());
+        List<List<String>> commandList = ALL_FUNCTION_MAP.get(executionMessage.getType());
 
-        this.execute(resultKey, functionShellScriptFileName);
+        this.execute(resultKey, commandList);
 
         /*Method execute = null;
 
@@ -42,12 +47,13 @@ public class FunctionExecutor {
     }
 
 
-    private void execute(String streamKey, String functionFileName) {
+    private void execute(String streamKey, List<List<String>> commandList) {
 
-        List<List<String>> commandList = functionReader.ReadFileToCommandList(functionFileName);
+//        List<List<String>> commandList = functionReader.ReadFileToCommandList(functionFileName);
 
         log.info("all commands are {}", commandList);
 
+        // todo modify this
         commandList.stream().map(
                 command -> {
                     commandExecutor.execute(streamKey, command);

agent/src/main/java/io/wdd/agent/executor/web/TestCommandExecutorController.java

@@ -2,6 +2,8 @@ package io.wdd.agent.executor.web;
 
 
 import io.wdd.agent.executor.shell.CommandExecutor;
+import io.wdd.agent.executor.shell.FunctionExecutor;
+import io.wdd.common.beans.executor.ExecutionMessage;
 import io.wdd.common.beans.response.R;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
@@ -11,6 +13,8 @@ import org.springframework.web.bind.annotation.RestController;
 import javax.annotation.Resource;
 import java.util.List;
 
+import static io.wdd.agent.executor.function.CollectAllExecutorFunction.ALL_FUNCTION_MAP;
+
 @RestController
 @RequestMapping("testExecutor")
 public class TestCommandExecutorController {
@@ -18,6 +22,9 @@ public class TestCommandExecutorController {
     @Resource
     CommandExecutor commandExecutor;
 
+    @Resource
+    FunctionExecutor functionExecutor;
+
 
     @PostMapping("comand")
     public R<String> testFor(
@@ -29,4 +36,25 @@ public class TestCommandExecutorController {
         return R.ok(streamKey);
     }
 
+
+    @PostMapping("linuxFile")
+    public R<String> testLinuxFile(
+            @RequestParam(value = "streamKey") String streamKey,
+            @RequestParam(value = "command") String messageType
+    ){
+
+        ExecutionMessage executionMessage = ExecutionMessage.builder()
+                .resultKey(streamKey)
+                .type(messageType)
+                .contend(messageType)
+                .build();
+
+
+        System.out.println("FUNCTION_REFLECTION = " + ALL_FUNCTION_MAP);
+
+        functionExecutor.execute(executionMessage);
+
+        return R.ok(streamKey);
+    }
+
 }

agent/src/test/java/io/wdd/agent/AgentApplicationTests.java

@@ -1,9 +1,7 @@
 package io.wdd.agent;
 
-import io.micrometer.core.instrument.FunctionCounter;
+import io.wdd.agent.executor.function.CollectAllExecutorFunction;
-import io.wdd.agent.executor.function.CollectAllFunctionToServer;
 import io.wdd.agent.executor.shell.FunctionExecutor;
-import io.wdd.common.beans.executor.ExecutionMessage;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.filefilter.DirectoryFileFilter;
@@ -29,16 +27,18 @@ class AgentApplicationTests {
     FunctionExecutor functionExecutor;
 
     @Resource
-    CollectAllFunctionToServer collectAllFunctionToServer;
+    CollectAllExecutorFunction collectAllExecutorFunction;
 
 
     @Test
     void testFileExecute(){
 
-        ExecutionMessage executionMessage = ExecutionMessage.builder().type("TestFunction").resultKey("simpleFor-test").contend("123456").build();
+//        ExecutionMessage executionMessage = ExecutionMessage.builder().type("TestFunction").resultKey("simpleFor-test").contend("123456").build();
+//
+//
+//        functionExecutor.execute(executionMessage);
 
 
-        functionExecutor.execute(executionMessage);
 
     }
 

common/src/main/java/io/wdd/common/beans/executor/ExecutorFunction.java

@@ -0,0 +1,19 @@
+package io.wdd.common.beans.executor;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@SuperBuilder(toBuilder = true)
+public class ExecutorFunction {
+
+
+    String functionName;
+
+    String functionContent;
+
+}

source/src/main/java/io/wdd/source/build-my-own-base-image.dockerfile

@@ -2,4 +2,4 @@ FROM eclipse-temurin:11-jre-focal
 
 MAINTAINER zeaslity@gmail.com
 
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends iputils-ping net-tools dnsutils lsof curl wget mtr-tiny vim && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends iputils-ping net-tools dnsutils lsof curl wget mtr-tiny vim openssh-client && rm -rf /var/lib/apt/lists/*