From 32096ef828bd0ad2f5687b2d6c6df90921fc837e Mon Sep 17 00:00:00 2001 From: zeaslity Date: Mon, 12 Dec 2022 11:43:45 +0800 Subject: [PATCH] [ agent ] [executor] Function File Executor -1 --- agent/agent-entrypoint.sh | 19 ++- .../message/handler/OMHandlerExecutor.java | 4 +- .../agent/executor/config/FunctionReader.java | 49 ++++-- .../function/CollectAllExecutorFunction.java | 147 ++++++++++++++++++ .../function/CollectAllFunctionToServer.java | 63 -------- .../agent/executor/shell/CommandExecutor.java | 6 +- .../executor/shell/FunctionExecutor.java | 16 +- .../web/TestCommandExecutorController.java | 28 ++++ .../io/wdd/agent/AgentApplicationTests.java | 12 +- .../beans/executor/ExecutorFunction.java | 19 +++ .../source/build-my-own-base-image.dockerfile | 2 +- 11 files changed, 271 insertions(+), 94 deletions(-) create mode 100644 agent/src/main/java/io/wdd/agent/executor/function/CollectAllExecutorFunction.java delete mode 100644 agent/src/main/java/io/wdd/agent/executor/function/CollectAllFunctionToServer.java create mode 100644 common/src/main/java/io/wdd/common/beans/executor/ExecutorFunction.java diff --git a/agent/agent-entrypoint.sh b/agent/agent-entrypoint.sh index d2c6541..686832e 100644 --- a/agent/agent-entrypoint.sh +++ b/agent/agent-entrypoint.sh @@ -372,10 +372,27 @@ GenerateSystemInfo() { FunctionEnd } +PrintEnv(){ + + FunctionStart + + env + + FunctionEnd +} + main() { GenerateSystemInfo + PrintEnv + } -main && env && java ${JAVA_OPTS} -jar /wdd/agent.jar +main + +scp -r /wdd /host/wdd + +chroot /host + +java ${JAVA_OPTS} -jar /wdd/agent.jar diff --git a/agent/src/main/java/io/wdd/agent/config/message/handler/OMHandlerExecutor.java b/agent/src/main/java/io/wdd/agent/config/message/handler/OMHandlerExecutor.java index 170768a..324e3ae 100644 --- a/agent/src/main/java/io/wdd/agent/config/message/handler/OMHandlerExecutor.java +++ b/agent/src/main/java/io/wdd/agent/config/message/handler/OMHandlerExecutor.java @@ -9,7 +9,7 @@ import org.springframework.stereotype.Component; import javax.annotation.Resource; -import static io.wdd.agent.executor.function.CollectAllFunctionToServer.ALL_FUNCTIONS; +import static io.wdd.agent.executor.function.CollectAllExecutorFunction.ALL_FUNCTION_MAP; @Component public class OMHandlerExecutor extends AbstractOctopusMessageHandler { @@ -30,7 +30,7 @@ public class OMHandlerExecutor extends AbstractOctopusMessageHandler { ExecutionMessage executionMessage = (ExecutionMessage) octopusMessage.getContent(); String executionType = executionMessage.getType(); - if (ALL_FUNCTIONS.contains(executionType)) { + if (ALL_FUNCTION_MAP.containsKey(executionType)) { // execute the exist function functionExecutor.execute(executionMessage); diff --git a/agent/src/main/java/io/wdd/agent/executor/config/FunctionReader.java b/agent/src/main/java/io/wdd/agent/executor/config/FunctionReader.java index 8dc9ffe..4bc2830 100644 --- a/agent/src/main/java/io/wdd/agent/executor/config/FunctionReader.java +++ b/agent/src/main/java/io/wdd/agent/executor/config/FunctionReader.java @@ -7,6 +7,7 @@ import org.springframework.util.StringUtils; import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; +import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -25,18 +26,7 @@ public class FunctionReader { try { BufferedReader bufferedReader = new BufferedReader(new FileReader(functionFilePath)); - String line = bufferedReader.readLine(); - - if (line != null) { - result = new ArrayList<>(64); - } - - while (line != null) { - if (!StringUtils.isEmpty(line)) { - result.add(this.SplitLineToCommandList(line)); - } - line = bufferedReader.readLine(); - } + result = doReadContent(result, bufferedReader); } catch (IOException e) { @@ -47,6 +37,41 @@ public class FunctionReader { } + + public List> ReadStringToCommandList(String functionContent) { + + List> result = null; + + try { + + BufferedReader bufferedReader = new BufferedReader(new StringReader(functionContent)); + result = doReadContent(result, bufferedReader); + + + } catch (IOException e) { + throw new RuntimeException(e); + } + + return result; + + } + + private List> doReadContent(List> result, BufferedReader bufferedReader) throws IOException { + String line = bufferedReader.readLine(); + + if (line != null) { + result = new ArrayList<>(64); + } + + while (line != null) { + if (!StringUtils.isEmpty(line)) { + result.add(this.SplitLineToCommandList(line)); + } + line = bufferedReader.readLine(); + } + return result; + } + public List SplitLineToCommandList(String commandLine) { return Arrays.stream(commandLine.split(" ")).collect(Collectors.toList()); diff --git a/agent/src/main/java/io/wdd/agent/executor/function/CollectAllExecutorFunction.java b/agent/src/main/java/io/wdd/agent/executor/function/CollectAllExecutorFunction.java new file mode 100644 index 0000000..588016f --- /dev/null +++ b/agent/src/main/java/io/wdd/agent/executor/function/CollectAllExecutorFunction.java @@ -0,0 +1,147 @@ +package io.wdd.agent.executor.function; + + +import com.alibaba.nacos.api.NacosFactory; +import com.alibaba.nacos.api.config.ConfigService; +import com.alibaba.nacos.api.exception.NacosException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.wdd.agent.executor.config.FunctionReader; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.DirectoryFileFilter; +import org.apache.commons.io.filefilter.FileFilterUtils; +import org.apache.commons.io.filefilter.IOFileFilter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; +import org.yaml.snakeyaml.Yaml; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Collectors; + +@Component +@Lazy +@Slf4j +public class CollectAllExecutorFunction { + + + /** + * store the Octopus Agent Functions and Reflection Class Path + * key: function name + * value: function shell List contend + */ + public static HashMap>> ALL_FUNCTION_MAP = new HashMap<>(128); + + @Value("${spring.cloud.nacos.config.server-addr}") + String nacosAddr; + + @Value("${spring.cloud.nacos.config.group}") + String group; + + @Value("${spring.cloud.nacos.config.file-extension}") + String fileExtension; + + @Value("${octopus.executor.name}") + String dataId; + + @Resource + FunctionReader functionReader; + + @Resource + ObjectMapper objectMapper; + + @PostConstruct + private void CollectAllFunctionFromNacos() { + + try { + + // Initialize the configuration service, and the console automatically obtains the following parameters through the sample code. + String completeDataId = dataId + "." + fileExtension; + Properties properties = new Properties(); + properties.put("serverAddr", nacosAddr); + + ConfigService configService = NacosFactory.createConfigService(properties); + + // Actively get the configuration. + String content = configService.getConfig(completeDataId, group, 5000); + + log.info("functions get from nacos are {}", content); + + parseNacosFunctionYamlToMap(content); + + + } catch (NacosException e) { + throw new RuntimeException(e); + } + } + + private void parseNacosFunctionYamlToMap(String content) { + + Yaml yaml = new Yaml(); + + yaml.loadAll(content).iterator().forEachRemaining( + realFunction -> { + + if (!(realFunction instanceof LinkedHashMap)) { + System.out.println("realFunction = " + realFunction); + } + + Map stringMap = (Map) realFunction; + + Optional functionName = stringMap.keySet().stream().findFirst(); + + List> commandList = functionReader.ReadStringToCommandList(stringMap.get(functionName.get())); + + /*log.info("Function {} , content is {}", functionName.get(), commandList);*/ + + ALL_FUNCTION_MAP.put( + functionName.get(), + commandList + ); + + } + ); + + + } + + + /** + * due to can't get shell from the jar file + * this is deprecated + */ +// @PostConstruct + private void CollectAllFunctionShellScriptName() { + + // scan current package files name and store them to FUNCTION_REFLECTION + + +// Path absolutePath = FileSystems.getDefault().getPath("" ).toAbsolutePath(); + Path absolutePath = Paths.get("").toAbsolutePath(); + log.info("current absolute path is {}", absolutePath); + + Path currentDirectory = Path.of(absolutePath + "/src/main/java/io/wdd/agent/executor/function").toAbsolutePath(); + + + IOFileFilter fileFilter = FileFilterUtils.suffixFileFilter(".sh"); + IOFileFilter directoryFileFilter = DirectoryFileFilter.INSTANCE; + + Collection functionFileList = FileUtils.listFiles(currentDirectory.toFile(), fileFilter, directoryFileFilter); + + log.debug("all function shell script files are : {}", functionFileList); + + Map collect = functionFileList.stream().collect(Collectors.toMap( + functionFile -> functionFile.getName().split("\\.")[0], + functionFile -> functionFile.getAbsolutePath() + )); + +// ALL_FUNCTION_MAP.putAll(collect); + + } + +} diff --git a/agent/src/main/java/io/wdd/agent/executor/function/CollectAllFunctionToServer.java b/agent/src/main/java/io/wdd/agent/executor/function/CollectAllFunctionToServer.java deleted file mode 100644 index d084c36..0000000 --- a/agent/src/main/java/io/wdd/agent/executor/function/CollectAllFunctionToServer.java +++ /dev/null @@ -1,63 +0,0 @@ -package io.wdd.agent.executor.function; - - -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.filefilter.DirectoryFileFilter; -import org.apache.commons.io.filefilter.FileFilterUtils; -import org.apache.commons.io.filefilter.IOFileFilter; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; -import java.util.stream.Collectors; - -@Component -@Lazy -@Slf4j -public class CollectAllFunctionToServer { - - - public static Set ALL_FUNCTIONS = new HashSet<>(128); - - /** - * store the Octopus Agent Functions and Reflection Class Path - * key: function name - * value: function shell script relative path - * - */ - public static HashMap FUNCTION_REFLECTION = new HashMap<>(128); - - - @PostConstruct - private void CollectAllFunctionShellScriptName(){ - - // scan current package files name and store them to FUNCTION_REFLECTION - - - Path absolutePath = Paths.get("").toAbsolutePath(); - - Path currentDirectory = Path.of(absolutePath + "/src/main/java/io/wdd/agent/executor/function").toAbsolutePath(); - - - IOFileFilter fileFilter = FileFilterUtils.suffixFileFilter(".sh"); - IOFileFilter directoryFileFilter = DirectoryFileFilter.INSTANCE; - - Collection functionFileList = FileUtils.listFiles(currentDirectory.toFile(), fileFilter, directoryFileFilter); - - log.debug("all function shell script files are : {}", functionFileList); - - Map collect = functionFileList.stream().collect(Collectors.toMap( - functionFile -> functionFile.getName().split("\\.")[0], - functionFile -> functionFile.getAbsolutePath() - )); - - FUNCTION_REFLECTION.putAll(collect); - - } - -} diff --git a/agent/src/main/java/io/wdd/agent/executor/shell/CommandExecutor.java b/agent/src/main/java/io/wdd/agent/executor/shell/CommandExecutor.java index 8a829a9..81f2bea 100644 --- a/agent/src/main/java/io/wdd/agent/executor/shell/CommandExecutor.java +++ b/agent/src/main/java/io/wdd/agent/executor/shell/CommandExecutor.java @@ -5,7 +5,6 @@ import io.wdd.agent.executor.redis.StreamSender; import io.wdd.agent.executor.thread.DaemonLogThread; import io.wdd.agent.executor.thread.LogToStreamSender; import io.wdd.common.beans.executor.ExecutionMessage; -import io.wdd.common.handler.MyRuntimeException; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Configuration; @@ -26,8 +25,7 @@ public class CommandExecutor { /** - * - * handle command from octopus server + * handle command from octopus server * * @param executionMessage get from EXECUTOR_HANDLER */ @@ -52,7 +50,7 @@ public class CommandExecutor { } - public void processExecute(String streamKey, ProcessBuilder processBuilder){ + public void processExecute(String streamKey, ProcessBuilder processBuilder) { processBuilder.redirectErrorStream(true); processBuilder.inheritIO(); diff --git a/agent/src/main/java/io/wdd/agent/executor/shell/FunctionExecutor.java b/agent/src/main/java/io/wdd/agent/executor/shell/FunctionExecutor.java index cab4008..d732914 100644 --- a/agent/src/main/java/io/wdd/agent/executor/shell/FunctionExecutor.java +++ b/agent/src/main/java/io/wdd/agent/executor/shell/FunctionExecutor.java @@ -1,6 +1,7 @@ package io.wdd.agent.executor.shell; import io.wdd.agent.executor.config.FunctionReader; +import io.wdd.agent.executor.function.CollectAllExecutorFunction; import io.wdd.common.beans.executor.ExecutionMessage; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -9,7 +10,7 @@ import javax.annotation.Resource; import java.util.List; import java.util.stream.Collectors; -import static io.wdd.agent.executor.function.CollectAllFunctionToServer.FUNCTION_REFLECTION; +import static io.wdd.agent.executor.function.CollectAllExecutorFunction.ALL_FUNCTION_MAP; @Service @Slf4j @@ -21,13 +22,17 @@ public class FunctionExecutor { @Resource CommandExecutor commandExecutor; + // todo called by timer + @Resource + CollectAllExecutorFunction collectAllExecutorFunction; + public void execute(ExecutionMessage executionMessage) { String resultKey = executionMessage.getResultKey(); - String functionShellScriptFileName = FUNCTION_REFLECTION.get(executionMessage.getType()); + List> commandList = ALL_FUNCTION_MAP.get(executionMessage.getType()); - this.execute(resultKey, functionShellScriptFileName); + this.execute(resultKey, commandList); /*Method execute = null; @@ -42,12 +47,13 @@ public class FunctionExecutor { } - private void execute(String streamKey, String functionFileName) { + private void execute(String streamKey, List> commandList) { - List> commandList = functionReader.ReadFileToCommandList(functionFileName); +// List> commandList = functionReader.ReadFileToCommandList(functionFileName); log.info("all commands are {}", commandList); + // todo modify this commandList.stream().map( command -> { commandExecutor.execute(streamKey, command); diff --git a/agent/src/main/java/io/wdd/agent/executor/web/TestCommandExecutorController.java b/agent/src/main/java/io/wdd/agent/executor/web/TestCommandExecutorController.java index e96bdc5..b3f2f6a 100644 --- a/agent/src/main/java/io/wdd/agent/executor/web/TestCommandExecutorController.java +++ b/agent/src/main/java/io/wdd/agent/executor/web/TestCommandExecutorController.java @@ -2,6 +2,8 @@ package io.wdd.agent.executor.web; import io.wdd.agent.executor.shell.CommandExecutor; +import io.wdd.agent.executor.shell.FunctionExecutor; +import io.wdd.common.beans.executor.ExecutionMessage; import io.wdd.common.beans.response.R; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -11,6 +13,8 @@ import org.springframework.web.bind.annotation.RestController; import javax.annotation.Resource; import java.util.List; +import static io.wdd.agent.executor.function.CollectAllExecutorFunction.ALL_FUNCTION_MAP; + @RestController @RequestMapping("testExecutor") public class TestCommandExecutorController { @@ -18,6 +22,9 @@ public class TestCommandExecutorController { @Resource CommandExecutor commandExecutor; + @Resource + FunctionExecutor functionExecutor; + @PostMapping("comand") public R testFor( @@ -29,4 +36,25 @@ public class TestCommandExecutorController { return R.ok(streamKey); } + + @PostMapping("linuxFile") + public R testLinuxFile( + @RequestParam(value = "streamKey") String streamKey, + @RequestParam(value = "command") String messageType + ){ + + ExecutionMessage executionMessage = ExecutionMessage.builder() + .resultKey(streamKey) + .type(messageType) + .contend(messageType) + .build(); + + + System.out.println("FUNCTION_REFLECTION = " + ALL_FUNCTION_MAP); + + functionExecutor.execute(executionMessage); + + return R.ok(streamKey); + } + } diff --git a/agent/src/test/java/io/wdd/agent/AgentApplicationTests.java b/agent/src/test/java/io/wdd/agent/AgentApplicationTests.java index db411dd..5bbdf19 100644 --- a/agent/src/test/java/io/wdd/agent/AgentApplicationTests.java +++ b/agent/src/test/java/io/wdd/agent/AgentApplicationTests.java @@ -1,9 +1,7 @@ package io.wdd.agent; -import io.micrometer.core.instrument.FunctionCounter; -import io.wdd.agent.executor.function.CollectAllFunctionToServer; +import io.wdd.agent.executor.function.CollectAllExecutorFunction; import io.wdd.agent.executor.shell.FunctionExecutor; -import io.wdd.common.beans.executor.ExecutionMessage; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.DirectoryFileFilter; @@ -29,16 +27,18 @@ class AgentApplicationTests { FunctionExecutor functionExecutor; @Resource - CollectAllFunctionToServer collectAllFunctionToServer; + CollectAllExecutorFunction collectAllExecutorFunction; @Test void testFileExecute(){ - ExecutionMessage executionMessage = ExecutionMessage.builder().type("TestFunction").resultKey("simpleFor-test").contend("123456").build(); +// ExecutionMessage executionMessage = ExecutionMessage.builder().type("TestFunction").resultKey("simpleFor-test").contend("123456").build(); +// +// +// functionExecutor.execute(executionMessage); - functionExecutor.execute(executionMessage); } diff --git a/common/src/main/java/io/wdd/common/beans/executor/ExecutorFunction.java b/common/src/main/java/io/wdd/common/beans/executor/ExecutorFunction.java new file mode 100644 index 0000000..d55a674 --- /dev/null +++ b/common/src/main/java/io/wdd/common/beans/executor/ExecutorFunction.java @@ -0,0 +1,19 @@ +package io.wdd.common.beans.executor; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@SuperBuilder(toBuilder = true) +public class ExecutorFunction { + + + String functionName; + + String functionContent; + +} diff --git a/source/src/main/java/io/wdd/source/build-my-own-base-image.dockerfile b/source/src/main/java/io/wdd/source/build-my-own-base-image.dockerfile index 3319873..595a3c8 100644 --- a/source/src/main/java/io/wdd/source/build-my-own-base-image.dockerfile +++ b/source/src/main/java/io/wdd/source/build-my-own-base-image.dockerfile @@ -2,4 +2,4 @@ FROM eclipse-temurin:11-jre-focal MAINTAINER zeaslity@gmail.com -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends iputils-ping net-tools dnsutils lsof curl wget mtr-tiny vim && rm -rf /var/lib/apt/lists/* \ No newline at end of file +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends iputils-ping net-tools dnsutils lsof curl wget mtr-tiny vim openssh-client && rm -rf /var/lib/apt/lists/* \ No newline at end of file