zeaslity/ProjectOctopus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[ agent ] [executor] Function File Executor -1

Browse Source
This commit is contained in:
zeaslity
2022-12-12 11:43:45 +08:00
parent 889976f9e2
commit 32096ef828
11 changed files with 271 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
agent/agent-entrypoint.sh
View File

@@ -372,10 +372,27 @@ GenerateSystemInfo() {
FunctionEnd
}
PrintEnv(){
FunctionStart
env
FunctionEnd
}
main() {
GenerateSystemInfo
PrintEnv
}
main && env && java ${JAVA_OPTS} -jar /wdd/agent.jar
main
scp -r /wdd /host/wdd
chroot /host
java ${JAVA_OPTS} -jar /wdd/agent.jar

4
agent/src/main/java/io/wdd/agent/config/message/handler/OMHandlerExecutor.java
View File

@@ -9,7 +9,7 @@ import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import static io.wdd.agent.executor.function.CollectAllFunctionToServer.ALL_FUNCTIONS;
import static io.wdd.agent.executor.function.CollectAllExecutorFunction.ALL_FUNCTION_MAP;
@Component
public class OMHandlerExecutor extends AbstractOctopusMessageHandler {
@@ -30,7 +30,7 @@ public class OMHandlerExecutor extends AbstractOctopusMessageHandler {
ExecutionMessage executionMessage = (ExecutionMessage) octopusMessage.getContent();
String executionType = executionMessage.getType();
if (ALL_FUNCTIONS.contains(executionType)) {
if (ALL_FUNCTION_MAP.containsKey(executionType)) {
// execute the exist function
functionExecutor.execute(executionMessage);

49
agent/src/main/java/io/wdd/agent/executor/config/FunctionReader.java
View File

@@ -7,6 +7,7 @@ import org.springframework.util.StringUtils;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -25,18 +26,7 @@ public class FunctionReader {
try {
BufferedReader bufferedReader = new BufferedReader(new FileReader(functionFilePath));
String line = bufferedReader.readLine();
if (line != null) {
result = new ArrayList<>(64);
}
while (line != null) {
if (!StringUtils.isEmpty(line)) {
result.add(this.SplitLineToCommandList(line));
}
line = bufferedReader.readLine();
}
result = doReadContent(result, bufferedReader);
} catch (IOException e) {
@@ -47,6 +37,41 @@ public class FunctionReader {
}
public List<List<String>> ReadStringToCommandList(String functionContent) {
List<List<String>> result = null;
try {
BufferedReader bufferedReader = new BufferedReader(new StringReader(functionContent));
result = doReadContent(result, bufferedReader);
} catch (IOException e) {
throw new RuntimeException(e);
}
return result;
}
private List<List<String>> doReadContent(List<List<String>> result, BufferedReader bufferedReader) throws IOException {
String line = bufferedReader.readLine();
if (line != null) {
result = new ArrayList<>(64);
}
while (line != null) {
if (!StringUtils.isEmpty(line)) {
result.add(this.SplitLineToCommandList(line));
}
line = bufferedReader.readLine();
}
return result;
}
public List<String> SplitLineToCommandList(String commandLine) {
return Arrays.stream(commandLine.split(" ")).collect(Collectors.toList());

147
agent/src/main/java/io/wdd/agent/executor/function/CollectAllExecutorFunction.java Normal file
View File

@@ -0,0 +1,147 @@
package io.wdd.agent.executor.function;
import com.alibaba.nacos.api.NacosFactory;
import com.alibaba.nacos.api.config.ConfigService;
import com.alibaba.nacos.api.exception.NacosException;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.wdd.agent.executor.config.FunctionReader;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.DirectoryFileFilter;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import org.yaml.snakeyaml.Yaml;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;
@Component
@Lazy
@Slf4j
public class CollectAllExecutorFunction {
/**
* store the Octopus Agent Functions and Reflection Class Path
* key: function name
* value: function shell List<String> contend
*/
public static HashMap<String, List<List<String>>> ALL_FUNCTION_MAP = new HashMap<>(128);
@Value("${spring.cloud.nacos.config.server-addr}")
String nacosAddr;
@Value("${spring.cloud.nacos.config.group}")
String group;
@Value("${spring.cloud.nacos.config.file-extension}")
String fileExtension;
@Value("${octopus.executor.name}")
String dataId;
@Resource
FunctionReader functionReader;
@Resource
ObjectMapper objectMapper;
@PostConstruct
private void CollectAllFunctionFromNacos() {
try {
// Initialize the configuration service, and the console automatically obtains the following parameters through the sample code.
String completeDataId = dataId + "." + fileExtension;
Properties properties = new Properties();
properties.put("serverAddr", nacosAddr);
ConfigService configService = NacosFactory.createConfigService(properties);
// Actively get the configuration.
String content = configService.getConfig(completeDataId, group, 5000);
log.info("functions get from nacos are {}", content);
parseNacosFunctionYamlToMap(content);
} catch (NacosException e) {
throw new RuntimeException(e);
}
}
private void parseNacosFunctionYamlToMap(String content) {
Yaml yaml = new Yaml();
yaml.loadAll(content).iterator().forEachRemaining(
realFunction -> {
if (!(realFunction instanceof LinkedHashMap)) {
System.out.println("realFunction = " + realFunction);
}
Map<String, String> stringMap = (Map<String, String>) realFunction;
Optional<String> functionName = stringMap.keySet().stream().findFirst();
List<List<String>> commandList = functionReader.ReadStringToCommandList(stringMap.get(functionName.get()));
/*log.info("Function {} , content is {}", functionName.get(), commandList);*/
ALL_FUNCTION_MAP.put(
functionName.get(),
commandList
);
}
);
}
/**
* due to can't get shell from the jar file
* this is deprecated
*/
// @PostConstruct
private void CollectAllFunctionShellScriptName() {
// scan current package files name and store them to FUNCTION_REFLECTION
// Path absolutePath = FileSystems.getDefault().getPath("" ).toAbsolutePath();
Path absolutePath = Paths.get("").toAbsolutePath();
log.info("current absolute path is {}", absolutePath);
Path currentDirectory = Path.of(absolutePath + "/src/main/java/io/wdd/agent/executor/function").toAbsolutePath();
IOFileFilter fileFilter = FileFilterUtils.suffixFileFilter(".sh");
IOFileFilter directoryFileFilter = DirectoryFileFilter.INSTANCE;
Collection<File> functionFileList = FileUtils.listFiles(currentDirectory.toFile(), fileFilter, directoryFileFilter);
log.debug("all function shell script files are : {}", functionFileList);
Map<String, String> collect = functionFileList.stream().collect(Collectors.toMap(
functionFile -> functionFile.getName().split("\\.")[0],
functionFile -> functionFile.getAbsolutePath()
));
// ALL_FUNCTION_MAP.putAll(collect);
}
}

63
agent/src/main/java/io/wdd/agent/executor/function/CollectAllFunctionToServer.java
View File

@@ -1,63 +0,0 @@
package io.wdd.agent.executor.function;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.DirectoryFileFilter;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;
@Component
@Lazy
@Slf4j
public class CollectAllFunctionToServer {
public static Set<String> ALL_FUNCTIONS = new HashSet<>(128);
/**
* store the Octopus Agent Functions and Reflection Class Path
* key: function name
* value: function shell script relative path
*
*/
public static HashMap<String, String> FUNCTION_REFLECTION = new HashMap<>(128);
@PostConstruct
private void CollectAllFunctionShellScriptName(){
// scan current package files name and store them to FUNCTION_REFLECTION
Path absolutePath = Paths.get("").toAbsolutePath();
Path currentDirectory = Path.of(absolutePath + "/src/main/java/io/wdd/agent/executor/function").toAbsolutePath();
IOFileFilter fileFilter = FileFilterUtils.suffixFileFilter(".sh");
IOFileFilter directoryFileFilter = DirectoryFileFilter.INSTANCE;
Collection<File> functionFileList = FileUtils.listFiles(currentDirectory.toFile(), fileFilter, directoryFileFilter);
log.debug("all function shell script files are : {}", functionFileList);
Map<String, String> collect = functionFileList.stream().collect(Collectors.toMap(
functionFile -> functionFile.getName().split("\\.")[0],
functionFile -> functionFile.getAbsolutePath()
));
FUNCTION_REFLECTION.putAll(collect);
}
}

6
agent/src/main/java/io/wdd/agent/executor/shell/CommandExecutor.java
View File

@@ -5,7 +5,6 @@ import io.wdd.agent.executor.redis.StreamSender;
import io.wdd.agent.executor.thread.DaemonLogThread;
import io.wdd.agent.executor.thread.LogToStreamSender;
import io.wdd.common.beans.executor.ExecutionMessage;
import io.wdd.common.handler.MyRuntimeException;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Configuration;
@@ -26,8 +25,7 @@ public class CommandExecutor {
/**
*
* handle command from octopus server
* handle command from octopus server
*
* @param executionMessage get from EXECUTOR_HANDLER
*/
@@ -52,7 +50,7 @@ public class CommandExecutor {
}
public void processExecute(String streamKey, ProcessBuilder processBuilder){
public void processExecute(String streamKey, ProcessBuilder processBuilder) {
processBuilder.redirectErrorStream(true);
processBuilder.inheritIO();

16
agent/src/main/java/io/wdd/agent/executor/shell/FunctionExecutor.java
View File

@@ -1,6 +1,7 @@
package io.wdd.agent.executor.shell;
import io.wdd.agent.executor.config.FunctionReader;
import io.wdd.agent.executor.function.CollectAllExecutorFunction;
import io.wdd.common.beans.executor.ExecutionMessage;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@@ -9,7 +10,7 @@ import javax.annotation.Resource;
import java.util.List;
import java.util.stream.Collectors;
import static io.wdd.agent.executor.function.CollectAllFunctionToServer.FUNCTION_REFLECTION;
import static io.wdd.agent.executor.function.CollectAllExecutorFunction.ALL_FUNCTION_MAP;
@Service
@Slf4j
@@ -21,13 +22,17 @@ public class FunctionExecutor {
@Resource
CommandExecutor commandExecutor;
// todo called by timer
@Resource
CollectAllExecutorFunction collectAllExecutorFunction;
public void execute(ExecutionMessage executionMessage) {
String resultKey = executionMessage.getResultKey();
String functionShellScriptFileName = FUNCTION_REFLECTION.get(executionMessage.getType());
List<List<String>> commandList = ALL_FUNCTION_MAP.get(executionMessage.getType());
this.execute(resultKey, functionShellScriptFileName);
this.execute(resultKey, commandList);
/*Method execute = null;
@@ -42,12 +47,13 @@ public class FunctionExecutor {
}
private void execute(String streamKey, String functionFileName) {
private void execute(String streamKey, List<List<String>> commandList) {
List<List<String>> commandList = functionReader.ReadFileToCommandList(functionFileName);
// List<List<String>> commandList = functionReader.ReadFileToCommandList(functionFileName);
log.info("all commands are {}", commandList);
// todo modify this
commandList.stream().map(
command -> {
commandExecutor.execute(streamKey, command);

28
agent/src/main/java/io/wdd/agent/executor/web/TestCommandExecutorController.java
View File

@@ -2,6 +2,8 @@ package io.wdd.agent.executor.web;
import io.wdd.agent.executor.shell.CommandExecutor;
import io.wdd.agent.executor.shell.FunctionExecutor;
import io.wdd.common.beans.executor.ExecutionMessage;
import io.wdd.common.beans.response.R;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
@@ -11,6 +13,8 @@ import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource;
import java.util.List;
import static io.wdd.agent.executor.function.CollectAllExecutorFunction.ALL_FUNCTION_MAP;
@RestController
@RequestMapping("testExecutor")
public class TestCommandExecutorController {
@@ -18,6 +22,9 @@ public class TestCommandExecutorController {
@Resource
CommandExecutor commandExecutor;
@Resource
FunctionExecutor functionExecutor;
@PostMapping("comand")
public R<String> testFor(
@@ -29,4 +36,25 @@ public class TestCommandExecutorController {
return R.ok(streamKey);
}
@PostMapping("linuxFile")
public R<String> testLinuxFile(
@RequestParam(value = "streamKey") String streamKey,
@RequestParam(value = "command") String messageType
){
ExecutionMessage executionMessage = ExecutionMessage.builder()
.resultKey(streamKey)
.type(messageType)
.contend(messageType)
.build();
System.out.println("FUNCTION_REFLECTION = " + ALL_FUNCTION_MAP);
functionExecutor.execute(executionMessage);
return R.ok(streamKey);
}
}

12
agent/src/test/java/io/wdd/agent/AgentApplicationTests.java
View File

@@ -1,9 +1,7 @@
package io.wdd.agent;
import io.micrometer.core.instrument.FunctionCounter;
import io.wdd.agent.executor.function.CollectAllFunctionToServer;
import io.wdd.agent.executor.function.CollectAllExecutorFunction;
import io.wdd.agent.executor.shell.FunctionExecutor;
import io.wdd.common.beans.executor.ExecutionMessage;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.DirectoryFileFilter;
@@ -29,16 +27,18 @@ class AgentApplicationTests {
FunctionExecutor functionExecutor;
@Resource
CollectAllFunctionToServer collectAllFunctionToServer;
CollectAllExecutorFunction collectAllExecutorFunction;
@Test
void testFileExecute(){
ExecutionMessage executionMessage = ExecutionMessage.builder().type("TestFunction").resultKey("simpleFor-test").contend("123456").build();
// ExecutionMessage executionMessage = ExecutionMessage.builder().type("TestFunction").resultKey("simpleFor-test").contend("123456").build();
//
//
// functionExecutor.execute(executionMessage);
functionExecutor.execute(executionMessage);
}

19
common/src/main/java/io/wdd/common/beans/executor/ExecutorFunction.java Normal file
View File

@@ -0,0 +1,19 @@
package io.wdd.common.beans.executor;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
@Data
@NoArgsConstructor
@AllArgsConstructor
@SuperBuilder(toBuilder = true)
public class ExecutorFunction {
String functionName;
String functionContent;
}

2
source/src/main/java/io/wdd/source/build-my-own-base-image.dockerfile
View File

@@ -2,4 +2,4 @@ FROM eclipse-temurin:11-jre-focal
MAINTAINER zeaslity@gmail.com
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends iputils-ping net-tools dnsutils lsof curl wget mtr-tiny vim && rm -rf /var/lib/apt/lists/*
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends iputils-ping net-tools dnsutils lsof curl wget mtr-tiny vim openssh-client && rm -rf /var/lib/apt/lists/*