From 601b6a678c8b35a9c291adbd5a7769967e6a5f9f Mon Sep 17 00:00:00 2001 From: zeaslity Date: Mon, 13 Feb 2023 15:48:03 +0800 Subject: [PATCH] =?UTF-8?q?[=20server=20]=20[=20oss=20]=20-=20OSS=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=20=E5=8A=A8=E6=80=81=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../message/handler/OMHandlerExecutor.java | 3 +- .../wdd/agent/executor/FunctionExecutor.java | 12 +- .../ExecutorFunctionNacosCollector.java} | 38 +-- .../bootup/reference/linux-init-LapPro.sh | 6 +- .../io/wdd/agent/AgentApplicationTests.java | 4 +- .../io/wdd/func/controller/OSSController.java | 6 +- .../OSSConfiguration.java} | 27 +- .../oracle/service/OracleOSSCoreService.java | 6 - .../service/OracleOSSCoreServiceImpl.java | 35 --- .../wdd/func/oss/service/OSSCoreService.java | 16 ++ .../func/oss/service/OSSCoreServiceImpl.java | 71 +++++ setting-aliyun.xml | 264 ++++++++++++++++++ spring-shell.log | 0 13 files changed, 393 insertions(+), 95 deletions(-) rename agent/src/main/java/io/wdd/agent/{config/utils/NacosConfigurationCollector.java => executor/config/ExecutorFunctionNacosCollector.java} (82%) rename server/src/main/java/io/wdd/func/oss/{oracle/config/OracleOSSConfiguration.java => config/OSSConfiguration.java} (73%) delete mode 100644 server/src/main/java/io/wdd/func/oss/oracle/service/OracleOSSCoreService.java delete mode 100644 server/src/main/java/io/wdd/func/oss/oracle/service/OracleOSSCoreServiceImpl.java create mode 100644 server/src/main/java/io/wdd/func/oss/service/OSSCoreService.java create mode 100644 server/src/main/java/io/wdd/func/oss/service/OSSCoreServiceImpl.java create mode 100644 setting-aliyun.xml delete mode 100644 spring-shell.log diff --git a/agent/src/main/java/io/wdd/agent/config/message/handler/OMHandlerExecutor.java b/agent/src/main/java/io/wdd/agent/config/message/handler/OMHandlerExecutor.java index 7e1631b..32dbb61 100644 --- a/agent/src/main/java/io/wdd/agent/config/message/handler/OMHandlerExecutor.java +++ b/agent/src/main/java/io/wdd/agent/config/message/handler/OMHandlerExecutor.java @@ -12,9 +12,8 @@ import org.springframework.stereotype.Component; import javax.annotation.Resource; import java.io.IOException; -import java.util.List; -import static io.wdd.agent.config.utils.NacosConfigurationCollector.ALL_FUNCTION_MAP; +import static io.wdd.agent.executor.config.ExecutorFunctionNacosCollector.ALL_FUNCTION_MAP; @Component public class OMHandlerExecutor extends AbstractOctopusMessageHandler { diff --git a/agent/src/main/java/io/wdd/agent/executor/FunctionExecutor.java b/agent/src/main/java/io/wdd/agent/executor/FunctionExecutor.java index 2ce28ff..dd521fb 100644 --- a/agent/src/main/java/io/wdd/agent/executor/FunctionExecutor.java +++ b/agent/src/main/java/io/wdd/agent/executor/FunctionExecutor.java @@ -3,7 +3,7 @@ package io.wdd.agent.executor; import com.alibaba.nacos.api.config.listener.Listener; import com.alibaba.nacos.api.exception.NacosException; import io.wdd.agent.config.utils.AgentCommonThreadPool; -import io.wdd.agent.config.utils.NacosConfigurationCollector; +import io.wdd.agent.executor.config.ExecutorFunctionNacosCollector; import io.wdd.common.beans.executor.ExecutionMessage; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Bean; @@ -14,8 +14,8 @@ import java.util.Iterator; import java.util.List; import java.util.concurrent.Executor; -import static io.wdd.agent.config.utils.NacosConfigurationCollector.ALL_FUNCTION_MAP; -import static io.wdd.agent.config.utils.NacosConfigurationCollector.NacosConfigService; +import static io.wdd.agent.executor.config.ExecutorFunctionNacosCollector.ALL_FUNCTION_MAP; +import static io.wdd.agent.executor.config.ExecutorFunctionNacosCollector.NacosConfigService; @Service @Slf4j @@ -26,7 +26,7 @@ public class FunctionExecutor { // todo called by timer @Resource - NacosConfigurationCollector nacosConfigurationCollector; + ExecutorFunctionNacosCollector executorFunctionNacosCollector; @@ -78,7 +78,7 @@ public class FunctionExecutor { // add listener to listen to the real-time change of the Function Shell Scripts try { - NacosConfigService.addListener(nacosConfigurationCollector.executorFunctionDataId + "." + nacosConfigurationCollector.fileExtension, nacosConfigurationCollector.group, new Listener() { + NacosConfigService.addListener(executorFunctionNacosCollector.executorFunctionDataId + "." + executorFunctionNacosCollector.fileExtension, executorFunctionNacosCollector.group, new Listener() { @Override public Executor getExecutor() { return null; @@ -89,7 +89,7 @@ public class FunctionExecutor { log.info("detected nacos function shell update ! {}", s); - nacosConfigurationCollector.parseNacosFunctionYamlToMap(s); + executorFunctionNacosCollector.parseNacosFunctionYamlToMap(s); } }); diff --git a/agent/src/main/java/io/wdd/agent/config/utils/NacosConfigurationCollector.java b/agent/src/main/java/io/wdd/agent/executor/config/ExecutorFunctionNacosCollector.java similarity index 82% rename from agent/src/main/java/io/wdd/agent/config/utils/NacosConfigurationCollector.java rename to agent/src/main/java/io/wdd/agent/executor/config/ExecutorFunctionNacosCollector.java index 1bdeb7c..d58afe1 100644 --- a/agent/src/main/java/io/wdd/agent/config/utils/NacosConfigurationCollector.java +++ b/agent/src/main/java/io/wdd/agent/executor/config/ExecutorFunctionNacosCollector.java @@ -1,4 +1,4 @@ -package io.wdd.agent.config.utils; +package io.wdd.agent.executor.config; import com.alibaba.nacos.api.NacosFactory; @@ -22,7 +22,7 @@ import static io.wdd.agent.executor.status.AppStatusExecutor.ALL_APP_NEED_TO_MON @Component @Lazy @Slf4j -public class NacosConfigurationCollector { +public class ExecutorFunctionNacosCollector { /** @@ -80,6 +80,7 @@ public class NacosConfigurationCollector { * https://nacos.io/zh-cn/docs/v2/guide/user/sdk.html * * dynamically listen to the nacos + * 动态监听Nacos的变化 * * Actively get the executor functions configuration. * */ @@ -175,37 +176,4 @@ public class NacosConfigurationCollector { } - - /** - * due to can't get shell from the jar file - * this is deprecated - */ -// @PostConstruct -// private void CollectAllFunctionShellScriptName() { -// -// // scan current package files name and store them to FUNCTION_REFLECTION -// -// -// Path absolutePath = Paths.get("").toAbsolutePath(); -// log.info("current absolute path is {}", absolutePath); -// -// Path currentDirectory = Path.of(absolutePath + "/src/main/java/io/wdd/agent/executor/function").toAbsolutePath(); -// -// -// IOFileFilter fileFilter = FileFilterUtils.suffixFileFilter(".sh"); -// IOFileFilter directoryFileFilter = DirectoryFileFilter.INSTANCE; -// -// Collection functionFileList = FileUtils.listFiles(currentDirectory.toFile(), fileFilter, directoryFileFilter); -// -// log.debug("all function shell script files are : {}", functionFileList); -// -// Map collect = functionFileList.stream().collect(Collectors.toMap( -// functionFile -> functionFile.getName().split("\\.")[0], -// functionFile -> functionFile.getAbsolutePath() -// )); -// -// ALL_FUNCTION_MAP.putAll(collect); -// -// } - } diff --git a/agent/src/main/java/io/wdd/agent/initialization/bootup/reference/linux-init-LapPro.sh b/agent/src/main/java/io/wdd/agent/initialization/bootup/reference/linux-init-LapPro.sh index 7f2d8f8..832f511 100644 --- a/agent/src/main/java/io/wdd/agent/initialization/bootup/reference/linux-init-LapPro.sh +++ b/agent/src/main/java/io/wdd/agent/initialization/bootup/reference/linux-init-LapPro.sh @@ -276,7 +276,7 @@ EOF InstallDocker() { Docker_Source="cn" - local dockerVersion=$(echo $DOCKER_VERSION | cut -d"." -f-2) + dockerVersion=$(echo $DOCKER_VERSION | cut -d"." -f-2) if [[ "$1" -ne " " ]]; then Docker_Source="$1" @@ -362,7 +362,9 @@ InstallDocker() { colorEcho ${GREEN} "开始安装docker-ce，版本为${DOCKER_VERSION}" realDockerSTag=$(apt-cache madison docker-ce | grep -w ${dockerVersion} | awk '{print$3}' | grep ${DOCKER_VERSION}) - installDemandSoftwares docker-ce=${realDockerSTag} || return $? + echo " real is $realDockerSTag" + + installDemandSoftwares "docker-ce=${realDockerSTag}" || return $? fi echo "" diff --git a/agent/src/test/java/io/wdd/agent/AgentApplicationTests.java b/agent/src/test/java/io/wdd/agent/AgentApplicationTests.java index 1df7133..a68f329 100644 --- a/agent/src/test/java/io/wdd/agent/AgentApplicationTests.java +++ b/agent/src/test/java/io/wdd/agent/AgentApplicationTests.java @@ -1,6 +1,6 @@ package io.wdd.agent; -import io.wdd.agent.config.utils.NacosConfigurationCollector; +import io.wdd.agent.executor.config.ExecutorFunctionNacosCollector; import io.wdd.agent.executor.FunctionExecutor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; @@ -25,7 +25,7 @@ class AgentApplicationTests { FunctionExecutor functionExecutor; @Resource - NacosConfigurationCollector nacosConfigurationCollector; + ExecutorFunctionNacosCollector executorFunctionNacosCollector; // @Test diff --git a/server/src/main/java/io/wdd/func/controller/OSSController.java b/server/src/main/java/io/wdd/func/controller/OSSController.java index 6d61666..40b703e 100644 --- a/server/src/main/java/io/wdd/func/controller/OSSController.java +++ b/server/src/main/java/io/wdd/func/controller/OSSController.java @@ -2,7 +2,7 @@ package io.wdd.func.controller; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; -import io.wdd.func.oss.oracle.service.OracleOSSCoreService; +import io.wdd.func.oss.service.OSSCoreService; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -15,13 +15,13 @@ import javax.annotation.Resource; public class OSSController { @Resource - OracleOSSCoreService oracleOSSCoreService; + OSSCoreService OSSCoreService; @GetMapping("/bucket/list/all") @ApiOperation("列出所有的桶") public void bucketListAll() { - oracleOSSCoreService.listBucketList(); + OSSCoreService.listBucketList(); System.out.println(" = "); } diff --git a/server/src/main/java/io/wdd/func/oss/oracle/config/OracleOSSConfiguration.java b/server/src/main/java/io/wdd/func/oss/config/OSSConfiguration.java similarity index 73% rename from server/src/main/java/io/wdd/func/oss/oracle/config/OracleOSSConfiguration.java rename to server/src/main/java/io/wdd/func/oss/config/OSSConfiguration.java index d061d72..aababf6 100644 --- a/server/src/main/java/io/wdd/func/oss/oracle/config/OracleOSSConfiguration.java +++ b/server/src/main/java/io/wdd/func/oss/config/OSSConfiguration.java @@ -1,27 +1,42 @@ -package io.wdd.func.oss.oracle.config; +package io.wdd.func.oss.config; +import com.alibaba.nacos.api.config.annotation.NacosConfigListener; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; +import kotlin.PublishedApi; +import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.annotation.Configuration; +import org.yaml.snakeyaml.Yaml; import javax.annotation.PostConstruct; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +/** + * 从Nacos中获取配置，获取所有S3的配置，并且自动更新 + *

+ * https://developer.aliyun.com/article/1111261 + * https://www.cnblogs.com/cndarren/p/16833526.html + */ @Configuration -public class OracleOSSConfiguration { +@ConfigurationProperties(prefix = "oss") +@Data +@RefreshScope +public class OSSConfiguration { /** * 缓存所有的S3客户端 */ public List ALL_S3_CLIENT = new ArrayList<>(); - - /** * 从Nacos中获取到所有的对象存储对象 */ @@ -33,10 +48,14 @@ public class OracleOSSConfiguration { String Seoul2Key; @Value("${oss.oracle.seoul2.secret}") String Seoul2Secret; + private LinkedHashMap> oracle = new LinkedHashMap<>(); + @PostConstruct public void buildAllS3Client() { + System.out.println("oracle = " + oracle); + AWSCredentialsProvider credentials = new AWSStaticCredentialsProvider(new BasicAWSCredentials( Seoul2Key, Seoul2Secret diff --git a/server/src/main/java/io/wdd/func/oss/oracle/service/OracleOSSCoreService.java b/server/src/main/java/io/wdd/func/oss/oracle/service/OracleOSSCoreService.java deleted file mode 100644 index b6cf3c8..0000000 --- a/server/src/main/java/io/wdd/func/oss/oracle/service/OracleOSSCoreService.java +++ /dev/null @@ -1,6 +0,0 @@ -package io.wdd.func.oss.oracle.service; - -public interface OracleOSSCoreService { - - void listBucketList(); -} diff --git a/server/src/main/java/io/wdd/func/oss/oracle/service/OracleOSSCoreServiceImpl.java b/server/src/main/java/io/wdd/func/oss/oracle/service/OracleOSSCoreServiceImpl.java deleted file mode 100644 index f9c38e4..0000000 --- a/server/src/main/java/io/wdd/func/oss/oracle/service/OracleOSSCoreServiceImpl.java +++ /dev/null @@ -1,35 +0,0 @@ -package io.wdd.func.oss.oracle.service; - - -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.Bucket; -import io.wdd.func.oss.oracle.config.OracleOSSConfiguration; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -import javax.annotation.Resource; -import java.util.List; - -/** - * https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm#usingAPI - */ -@Service -@Slf4j -public class OracleOSSCoreServiceImpl implements OracleOSSCoreService { - - - @Resource - OracleOSSConfiguration ossConfiguration; - - - @Override - public void listBucketList() { - - AmazonS3 amazonS3 = ossConfiguration.ALL_S3_CLIENT.get(0); - - List buckets = amazonS3.listBuckets(); - - - System.out.println("buckets = " + buckets); - } -} diff --git a/server/src/main/java/io/wdd/func/oss/service/OSSCoreService.java b/server/src/main/java/io/wdd/func/oss/service/OSSCoreService.java new file mode 100644 index 0000000..32552c9 --- /dev/null +++ b/server/src/main/java/io/wdd/func/oss/service/OSSCoreService.java @@ -0,0 +1,16 @@ +package io.wdd.func.oss.service; + +public interface OSSCoreService { + + /** + * 对桶的操作 + */ + + /** + * 列出所有的桶 + * */ + void listBucketList(); + + + void deleteBucket(); +} diff --git a/server/src/main/java/io/wdd/func/oss/service/OSSCoreServiceImpl.java b/server/src/main/java/io/wdd/func/oss/service/OSSCoreServiceImpl.java new file mode 100644 index 0000000..4cf14c6 --- /dev/null +++ b/server/src/main/java/io/wdd/func/oss/service/OSSCoreServiceImpl.java @@ -0,0 +1,71 @@ +package io.wdd.func.oss.service; + + +import com.alibaba.nacos.api.config.annotation.NacosConfigListener; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.Bucket; +import io.wdd.func.oss.config.OSSConfiguration; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.yaml.snakeyaml.Yaml; + +import javax.annotation.Resource; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm#usingAPI + */ +@Service +@Slf4j +public class OSSCoreServiceImpl implements OSSCoreService { + + + @Resource + OSSConfiguration ossConfiguration; + + + /** + * 使用Linstener 更新OSS的配置信息 + * @param content 获取到的更新的Nacos + */ + @NacosConfigListener(dataId = "octopus-server-k3s.yaml", groupId = "k3s") + public void listener(String content){ + + System.out.println("content = " + content); + + Yaml yaml = new Yaml(); + + Map contextMap = yaml.load(content); + + Map ossObject = (Map) contextMap.get("oss"); + LinkedHashMap> map = (LinkedHashMap>) ossObject.get("oralce"); + + // 需要同步这些 +// synchronized (oracle) { +// +// oracle.clear(); +// oracle.putAll(map); +// +// } + + System.out.println("map = " + map); + } + + @Override + public void listBucketList() { + + AmazonS3 amazonS3 = ossConfiguration.ALL_S3_CLIENT.get(0); + + List buckets = amazonS3.listBuckets(); + + + System.out.println("buckets = " + buckets); + } + + @Override + public void deleteBucket() { + + } +} diff --git a/setting-aliyun.xml b/setting-aliyun.xml new file mode 100644 index 0000000..96d7de6 --- /dev/null +++ b/setting-aliyun.xml @@ -0,0 +1,264 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + aliyunmaven + * + 阿里云公共仓库 + https://maven.aliyun.com/repository/public + + + + + + + + + + + + + \ No newline at end of file diff --git a/spring-shell.log b/spring-shell.log deleted file mode 100644 index e69de29..0000000