[ server ] [ oss ] - OSS配置 动态获取

agent/src/main/java/io/wdd/agent/config/message/handler/OMHandlerExecutor.java

@@ -12,9 +12,8 @@ import org.springframework.stereotype.Component;
 
 import javax.annotation.Resource;
 import java.io.IOException;
-import java.util.List;
 
-import static io.wdd.agent.config.utils.NacosConfigurationCollector.ALL_FUNCTION_MAP;
+import static io.wdd.agent.executor.config.ExecutorFunctionNacosCollector.ALL_FUNCTION_MAP;
 
 @Component
 public class OMHandlerExecutor extends AbstractOctopusMessageHandler {

agent/src/main/java/io/wdd/agent/executor/FunctionExecutor.java

@@ -3,7 +3,7 @@ package io.wdd.agent.executor;
 import com.alibaba.nacos.api.config.listener.Listener;
 import com.alibaba.nacos.api.exception.NacosException;
 import io.wdd.agent.config.utils.AgentCommonThreadPool;
-import io.wdd.agent.config.utils.NacosConfigurationCollector;
+import io.wdd.agent.executor.config.ExecutorFunctionNacosCollector;
 import io.wdd.common.beans.executor.ExecutionMessage;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.context.annotation.Bean;
@@ -14,8 +14,8 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.Executor;
 
-import static io.wdd.agent.config.utils.NacosConfigurationCollector.ALL_FUNCTION_MAP;
+import static io.wdd.agent.executor.config.ExecutorFunctionNacosCollector.ALL_FUNCTION_MAP;
-import static io.wdd.agent.config.utils.NacosConfigurationCollector.NacosConfigService;
+import static io.wdd.agent.executor.config.ExecutorFunctionNacosCollector.NacosConfigService;
 
 @Service
 @Slf4j
@@ -26,7 +26,7 @@ public class FunctionExecutor {
 
     // todo called by timer
     @Resource
-    NacosConfigurationCollector nacosConfigurationCollector;
+    ExecutorFunctionNacosCollector executorFunctionNacosCollector;
 
 
 
@@ -78,7 +78,7 @@ public class FunctionExecutor {
 
         // add listener to listen to the real-time change of the Function Shell Scripts
         try {
-            NacosConfigService.addListener(nacosConfigurationCollector.executorFunctionDataId + "." + nacosConfigurationCollector.fileExtension, nacosConfigurationCollector.group, new Listener() {
+            NacosConfigService.addListener(executorFunctionNacosCollector.executorFunctionDataId + "." + executorFunctionNacosCollector.fileExtension, executorFunctionNacosCollector.group, new Listener() {
                 @Override
                 public Executor getExecutor() {
                     return null;
@@ -89,7 +89,7 @@ public class FunctionExecutor {
 
                     log.info("detected nacos function shell update ! {}", s);
 
-                    nacosConfigurationCollector.parseNacosFunctionYamlToMap(s);
+                    executorFunctionNacosCollector.parseNacosFunctionYamlToMap(s);
 
                 }
             });

agent/src/main/java/io/wdd/agent/executor/config/ExecutorFunctionNacosCollector.java

@@ -1,4 +1,4 @@
-package io.wdd.agent.config.utils;
+package io.wdd.agent.executor.config;
 
 
 import com.alibaba.nacos.api.NacosFactory;
@@ -22,7 +22,7 @@ import static io.wdd.agent.executor.status.AppStatusExecutor.ALL_APP_NEED_TO_MON
 @Component
 @Lazy
 @Slf4j
-public class NacosConfigurationCollector {
+public class ExecutorFunctionNacosCollector {
 
 
     /**
@@ -80,6 +80,7 @@ public class NacosConfigurationCollector {
              *  https://nacos.io/zh-cn/docs/v2/guide/user/sdk.html
              *
              *  dynamically listen to the nacos
+             *  动态监听Nacos的变化
              *
              * Actively get the executor functions configuration.
              * */
@@ -175,37 +176,4 @@ public class NacosConfigurationCollector {
 
     }
 
-
-    /**
-     * due to can't get shell from the jar file
-     * this is deprecated
-     */
-//    @PostConstruct
-//    private void CollectAllFunctionShellScriptName() {
-//
-//        // scan current package files name and  store them to FUNCTION_REFLECTION
-//
-//
-//        Path absolutePath = Paths.get("").toAbsolutePath();
-//        log.info("current absolute path is {}", absolutePath);
-//
-//        Path currentDirectory = Path.of(absolutePath + "/src/main/java/io/wdd/agent/executor/function").toAbsolutePath();
-//
-//
-//        IOFileFilter fileFilter = FileFilterUtils.suffixFileFilter(".sh");
-//        IOFileFilter directoryFileFilter = DirectoryFileFilter.INSTANCE;
-//
-//        Collection<File> functionFileList = FileUtils.listFiles(currentDirectory.toFile(), fileFilter, directoryFileFilter);
-//
-//        log.debug("all function shell script files are : {}", functionFileList);
-//
-//        Map<String, String> collect = functionFileList.stream().collect(Collectors.toMap(
-//                functionFile -> functionFile.getName().split("\\.")[0],
-//                functionFile -> functionFile.getAbsolutePath()
-//        ));
-//
-//        ALL_FUNCTION_MAP.putAll(collect);
-//
-//    }
-
 }

agent/src/main/java/io/wdd/agent/initialization/bootup/reference/linux-init-LapPro.sh

@@ -276,7 +276,7 @@ EOF
 
 InstallDocker() {
   Docker_Source="cn"
-  local dockerVersion=$(echo $DOCKER_VERSION | cut -d"." -f-2)
+  dockerVersion=$(echo $DOCKER_VERSION | cut -d"." -f-2)
 
   if [[ "$1" -ne " " ]]; then
       Docker_Source="$1"
@@ -362,7 +362,9 @@ InstallDocker() {
 
     colorEcho ${GREEN} "开始安装docker-ce，版本为${DOCKER_VERSION}"
     realDockerSTag=$(apt-cache madison docker-ce | grep -w ${dockerVersion} | awk '{print$3}' | grep ${DOCKER_VERSION})
-    installDemandSoftwares docker-ce=${realDockerSTag} || return $?
+    echo " real is $realDockerSTag"
+
+    installDemandSoftwares "docker-ce=${realDockerSTag}" || return $?
   fi
   echo ""
 

agent/src/test/java/io/wdd/agent/AgentApplicationTests.java

@@ -1,6 +1,6 @@
 package io.wdd.agent;
 
-import io.wdd.agent.config.utils.NacosConfigurationCollector;
+import io.wdd.agent.executor.config.ExecutorFunctionNacosCollector;
 import io.wdd.agent.executor.FunctionExecutor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.FileUtils;
@@ -25,7 +25,7 @@ class AgentApplicationTests {
     FunctionExecutor functionExecutor;
 
     @Resource
-    NacosConfigurationCollector nacosConfigurationCollector;
+    ExecutorFunctionNacosCollector executorFunctionNacosCollector;
 
 
 //    @Test

server/src/main/java/io/wdd/func/controller/OSSController.java

@@ -2,7 +2,7 @@ package io.wdd.func.controller;
 
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
-import io.wdd.func.oss.oracle.service.OracleOSSCoreService;
+import io.wdd.func.oss.service.OSSCoreService;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
@@ -15,13 +15,13 @@ import javax.annotation.Resource;
 public class OSSController {
 
     @Resource
-    OracleOSSCoreService oracleOSSCoreService;
+    OSSCoreService OSSCoreService;
 
     @GetMapping("/bucket/list/all")
     @ApiOperation("列出所有的桶")
     public void bucketListAll() {
 
-        oracleOSSCoreService.listBucketList();
+        OSSCoreService.listBucketList();
         System.out.println(" = ");
     }
 

server/src/main/java/io/wdd/func/oss/config/OSSConfiguration.java

@@ -1,27 +1,42 @@
-package io.wdd.func.oss.oracle.config;
+package io.wdd.func.oss.config;
 
+import com.alibaba.nacos.api.config.annotation.NacosConfigListener;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.client.builder.AwsClientBuilder;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3Client;
+import kotlin.PublishedApi;
+import lombok.Data;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.cloud.context.config.annotation.RefreshScope;
 import org.springframework.context.annotation.Configuration;
+import org.yaml.snakeyaml.Yaml;
 
 import javax.annotation.PostConstruct;
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 
+/**
+ * 从Nacos中获取配置，获取所有S3的配置，并且自动更新
+ * <p>
+ * https://developer.aliyun.com/article/1111261
+ * https://www.cnblogs.com/cndarren/p/16833526.html
+ */
 @Configuration
-public class OracleOSSConfiguration {
+@ConfigurationProperties(prefix = "oss")
+@Data
+@RefreshScope
+public class OSSConfiguration {
 
     /**
      * 缓存所有的S3客户端
      */
     public List<AmazonS3> ALL_S3_CLIENT = new ArrayList<>();
-
-
     /**
      * 从Nacos中获取到所有的对象存储对象
      */
@@ -33,10 +48,14 @@ public class OracleOSSConfiguration {
     String Seoul2Key;
     @Value("${oss.oracle.seoul2.secret}")
     String Seoul2Secret;
+    private LinkedHashMap<String, LinkedHashMap<String, String>> oracle = new LinkedHashMap<>();
+
 
     @PostConstruct
     public void buildAllS3Client() {
 
+        System.out.println("oracle = " + oracle);
+
         AWSCredentialsProvider credentials = new AWSStaticCredentialsProvider(new BasicAWSCredentials(
                 Seoul2Key,
                 Seoul2Secret

server/src/main/java/io/wdd/func/oss/oracle/service/OracleOSSCoreService.java

@@ -1,6 +0,0 @@
-package io.wdd.func.oss.oracle.service;
-
-public interface OracleOSSCoreService {
-
-    void listBucketList();
-}

server/src/main/java/io/wdd/func/oss/oracle/service/OracleOSSCoreServiceImpl.java

@@ -1,35 +0,0 @@
-package io.wdd.func.oss.oracle.service;
-
-
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.model.Bucket;
-import io.wdd.func.oss.oracle.config.OracleOSSConfiguration;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-
-import javax.annotation.Resource;
-import java.util.List;
-
-/**
- * https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm#usingAPI
- */
-@Service
-@Slf4j
-public class OracleOSSCoreServiceImpl implements OracleOSSCoreService {
-
-
-    @Resource
-    OracleOSSConfiguration ossConfiguration;
-
-
-    @Override
-    public void listBucketList() {
-
-        AmazonS3 amazonS3 = ossConfiguration.ALL_S3_CLIENT.get(0);
-
-        List<Bucket> buckets = amazonS3.listBuckets();
-
-
-        System.out.println("buckets = " + buckets);
-    }
-}

server/src/main/java/io/wdd/func/oss/service/OSSCoreService.java

@@ -0,0 +1,16 @@
+package io.wdd.func.oss.service;
+
+public interface OSSCoreService {
+
+    /**
+     * 对桶的操作
+     */
+
+    /**
+     * 列出所有的桶
+     * */
+    void listBucketList();
+
+
+    void deleteBucket();
+}

server/src/main/java/io/wdd/func/oss/service/OSSCoreServiceImpl.java

@@ -0,0 +1,71 @@
+package io.wdd.func.oss.service;
+
+
+import com.alibaba.nacos.api.config.annotation.NacosConfigListener;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.Bucket;
+import io.wdd.func.oss.config.OSSConfiguration;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import org.yaml.snakeyaml.Yaml;
+
+import javax.annotation.Resource;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm#usingAPI
+ */
+@Service
+@Slf4j
+public class OSSCoreServiceImpl implements OSSCoreService {
+
+
+    @Resource
+    OSSConfiguration ossConfiguration;
+
+
+    /**
+     *   使用Linstener 更新OSS的配置信息
+     * @param content 获取到的更新的Nacos
+     */
+    @NacosConfigListener(dataId = "octopus-server-k3s.yaml", groupId = "k3s")
+    public void listener(String content){
+
+        System.out.println("content = " + content);
+
+        Yaml yaml = new Yaml();
+
+        Map<String, Object> contextMap = yaml.load(content);
+
+        Map<String, Object> ossObject = (Map<String, Object>) contextMap.get("oss");
+        LinkedHashMap<String, LinkedHashMap<String, String>> map = (LinkedHashMap<String, LinkedHashMap<String, String>>) ossObject.get("oralce");
+
+        // 需要同步这些
+//        synchronized (oracle) {
+//
+//            oracle.clear();
+//            oracle.putAll(map);
+//
+//        }
+
+        System.out.println("map = " + map);
+    }
+
+    @Override
+    public void listBucketList() {
+
+        AmazonS3 amazonS3 = ossConfiguration.ALL_S3_CLIENT.get(0);
+
+        List<Bucket> buckets = amazonS3.listBuckets();
+
+
+        System.out.println("buckets = " + buckets);
+    }
+
+    @Override
+    public void deleteBucket() {
+
+    }
+}

setting-aliyun.xml

@@ -0,0 +1,264 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<!--
+ | This is the configuration file for Maven. It can be specified at two levels:
+ |
+ |  1. User Level. This 自定义的maven-settings.xml file provides configuration for a single user,
+ |                 and is normally provided in ${user.home}/.m2/自定义的maven-settings.xml.
+ |
+ |                 NOTE: This location can be overridden with the CLI option:
+ |
+ |                 -s /path/to/user/自定义的maven-settings.xml
+ |
+ |  2. Global Level. This 自定义的maven-settings.xml file provides configuration for all Maven
+ |                 users on a machine (assuming they're all using the same Maven
+ |                 installation). It's normally provided in
+ |                 ${maven.conf}/自定义的maven-settings.xml.
+ |
+ |                 NOTE: This location can be overridden with the CLI option:
+ |
+ |                 -gs /path/to/global/自定义的maven-settings.xml
+ |
+ | The sections in this sample file are intended to give you a running start at
+ | getting the most out of your Maven installation. Where appropriate, the default
+ | values (values used when the setting is not specified) are provided.
+ |
+ |-->
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
+    <!-- localRepository
+       | The path to the local repository maven will use to store artifacts.
+       |
+       | Default: ${user.home}/.m2/repository -->
+    <!-- <localRepository>/home/jenkins/agent/.m2/repository</localRepository> -->
+
+
+    <!-- interactiveMode
+       | This will determine whether maven prompts you when it needs input. If set to false,
+       | maven will use a sensible default value, perhaps based on some other setting, for
+       | the parameter in question.
+       |
+       | Default: true
+      <interactiveMode>true</interactiveMode>
+      -->
+
+    <!-- offline
+       | Determines whether maven should attempt to connect to the network when executing a build.
+       | This will have an effect on artifact downloads, artifact deployment, and others.
+       |
+       | Default: false
+      <offline>false</offline>
+      -->
+
+    <!-- pluginGroups
+       | This is a list of additional group identifiers that will be searched when resolving plugins by their prefix, i.e.
+       | when invoking a command line like "mvn prefix:goal". Maven will automatically add the group identifiers
+       | "org.apache.maven.plugins" and "org.codehaus.mojo" if these are not already contained in the list.
+       |-->
+    <pluginGroups>
+        <!-- pluginGroup
+             | Specifies a further group identifier to use for plugin lookup.
+            <pluginGroup>com.your.plugins</pluginGroup>
+            -->
+    </pluginGroups>
+
+    <!-- proxies
+       | This is a list of proxies which can be used on this machine to connect to the network.
+       | Unless otherwise specified (by system property or command-line switch), the first proxy
+       | specification in this list marked as active will be used.
+       |-->
+    <proxies>
+        <!-- proxy
+             | Specification for one proxy, to be used in connecting to the network.
+             |
+            <proxy>
+              <id>optional</id>
+              <active>true</active>
+              <protocol>http</protocol>
+              <username>proxyuser</username>
+              <password>proxypass</password>
+              <host>proxy.host.net</host>
+              <port>80</port>
+              <nonProxyHosts>local.net|some.host.com</nonProxyHosts>
+            </proxy>
+            -->
+    </proxies>
+
+    <!-- servers
+       | This is a list of authentication profiles, keyed by the server-id used within the system.
+       | Authentication profiles can be used whenever maven must make a connection to a remote server.
+       |-->
+    <servers>
+        <!-- server
+             | Specifies the authentication information to use when connecting to a particular server, identified by
+             | a unique name within the system (referred to by the 'id' attribute below).
+             |
+             | NOTE: You should either specify username/password OR privateKey/passphrase, since these pairings are
+             |       used together.
+             |
+            <server>
+              <id>deploymentRepo</id>
+              <username>repouser</username>
+              <password>repopwd</password>
+            </server>
+            -->
+
+        <!-- Another sample, using keys to authenticate.
+            <server>
+              <id>siteServer</id>
+              <privateKey>/path/to/private/key</privateKey>
+              <passphrase>optional; leave empty if not used.</passphrase>
+            </server>
+            -->
+    </servers>
+
+    <!-- mirrors
+       | This is a list of mirrors to be used in downloading artifacts from remote repositories.
+       |
+       | It works like this: a POM may declare a repository to use in resolving certain artifacts.
+       | However, this repository may have problems with heavy traffic at times, so people have mirrored
+       | it to several places.
+       |
+       | That repository definition will have a unique id, so we can create a mirror reference for that
+       | repository, to be used as an alternate download site. The mirror site will be the preferred
+       | server for that repository.
+       |-->
+    <mirrors>
+        <!-- mirror
+             | Specifies a repository mirror site to use instead of a given repository. The repository that
+             | this mirror serves has an ID that matches the mirrorOf element of this mirror. IDs are used
+             | for inheritance and direct lookup purposes, and must be unique across the set of mirrors.
+             |
+            <mirror>
+              <id>mirrorId</id>
+              <mirrorOf>repositoryId</mirrorOf>
+              <name>Human Readable Name for this Mirror.</name>
+              <url>http://my.repository.com/repo/path</url>
+            </mirror>
+             -->
+        <mirror>
+            <id>aliyunmaven</id>
+            <mirrorOf>*</mirrorOf>
+            <name>阿里云公共仓库</name>
+            <url>https://maven.aliyun.com/repository/public</url>
+        </mirror>
+
+    </mirrors>
+
+    <!-- profiles
+       | This is a list of profiles which can be activated in a variety of ways, and which can modify
+       | the build process. Profiles provided in the 自定义的maven-settings.xml are intended to provide local machine-
+       | specific paths and repository locations which allow the build to work in the local environment.
+       |
+       | For example, if you have an integration testing plugin - like cactus - that needs to know where
+       | your Tomcat instance is installed, you can provide a variable here such that the variable is
+       | dereferenced during the build process to configure the cactus plugin.
+       |
+       | As noted above, profiles can be activated in a variety of ways. One way - the activeProfiles
+       | section of this document (自定义的maven-settings.xml) - will be discussed later. Another way essentially
+       | relies on the detection of a system property, either matching a particular value for the property,
+       | or merely testing its existence. Profiles can also be activated by JDK version prefix, where a
+       | value of '1.4' might activate a profile when the build is executed on a JDK version of '1.4.2_07'.
+       | Finally, the list of active profiles can be specified directly from the command line.
+       |
+       | NOTE: For profiles defined in the 自定义的maven-settings.xml, you are restricted to specifying only artifact
+       |       repositories, plugin repositories, and free-form properties to be used as configuration
+       |       variables for plugins in the POM.
+       |
+       |-->
+    <profiles>
+        <!-- profile
+             | Specifies a set of introductions to the build process, to be activated using one or more of the
+             | mechanisms described above. For inheritance purposes, and to activate profiles via <activatedProfiles/>
+             | or the command line, profiles have to have an ID that is unique.
+             |
+             | An encouraged best practice for profile identification is to use a consistent naming convention
+             | for profiles, such as 'env-dev', 'env-test', 'env-production', 'user-jdcasey', 'user-brett', etc.
+             | This will make it more intuitive to understand what the set of introduced profiles is attempting
+             | to accomplish, particularly when you only have a list of profile id's for debug.
+             |
+             | This profile example uses the JDK version to trigger activation, and provides a JDK-specific repo.
+            <profile>
+              <id>jdk-1.4</id>
+
+              <activation>
+                <jdk>1.4</jdk>
+              </activation>
+
+              <repositories>
+                <repository>
+                  <id>jdk14</id>
+                  <name>Repository for JDK 1.4 builds</name>
+                  <url>http://www.myhost.com/maven/jdk14</url>
+                  <layout>default</layout>
+                  <snapshotPolicy>always</snapshotPolicy>
+                </repository>
+              </repositories>
+            </profile>
+            -->
+
+        <!--
+             | Here is another profile, activated by the system property 'target-env' with a value of 'dev',
+             | which provides a specific path to the Tomcat instance. To use this, your plugin configuration
+             | might hypothetically look like:
+             |
+             | ...
+             | <plugin>
+             |   <groupId>org.myco.myplugins</groupId>
+             |   <artifactId>myplugin</artifactId>
+             |
+             |   <configuration>
+             |     <tomcatLocation>${tomcatPath}</tomcatLocation>
+             |   </configuration>
+             | </plugin>
+             | ...
+             |
+             | NOTE: If you just wanted to inject this configuration whenever someone set 'target-env' to
+             |       anything, you could just leave off the <value/> inside the activation-property.
+             |
+            <profile>
+              <id>env-dev</id>
+
+              <activation>
+                <property>
+                  <name>target-env</name>
+                  <value>dev</value>
+                </property>
+              </activation>
+
+              <properties>
+                <tomcatPath>/path/to/tomcat/instance</tomcatPath>
+              </properties>
+            </profile>
+            -->
+    </profiles>
+
+    <!-- activeProfiles
+       | List of profiles that are active for all builds.
+       |
+      <activeProfiles>
+        <activeProfile>alwaysActiveProfile</activeProfile>
+        <activeProfile>anotherAlwaysActiveProfile</activeProfile>
+      </activeProfiles>
+      -->
+</settings>