zeaslity/ProjectOctopus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[ server ] [ oss ] -初步完成对象存储部分代码 - 1

Browse Source
This commit is contained in:
zeaslity
2023-02-14 15:28:12 +08:00
parent 601b6a678c
commit 7158183348
11 changed files with 708 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
.run/RunAgentUbuntuStation.run.xml
View File

@@ -1,109 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="RunAgentUbuntuStation" type="docker-deploy" factoryName="dockerfile" server-name="UbuntuStation">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="octopus-agent-ubuntu" />
<option name="buildOnly" value="true" />
<option name="containerName" value="octopus-agent-ubuntu" />
<option name="envVars">
<list>
<DockerEnvVarImpl>
<option name="name" value="cpuBrand" />
<option name="value" value="i7 8700" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="cpuCore" />
<option name="value" value="12 @ 3.2GHz" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="diskTotal" />
<option name="value" value="1 TB" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="diskUsage" />
<option name="value" value="200GB" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="ioSpeed" />
<option name="value" value="1 GB/s" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="location" />
<option name="value" value="Chengdu" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="machineId" />
<option name="value" value="adasdasdas1q122131" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="managePort" />
<option name="value" value="3389" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="memoryTotal" />
<option name="value" value="64 GB" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="memoryUsage" />
<option name="value" value="12 GB" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="osInfo" />
<option name="value" value="Windwos 11" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="osKernelInfo" />
<option name="value" value="Window NT 10.0" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="provider" />
<option name="value" value="China Mobile" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="serverIpInV4" />
<option name="value" value="&quot;&quot;" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="serverIpInV6" />
<option name="value" value="&quot;&quot;" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="serverIpPbV4" />
<option name="value" value="&quot;&quot;" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="serverIpPbV6" />
<option name="value" value="&quot;&quot;" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="serverName" />
<option name="value" value="Chengdu-amd64-99" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="tcpControl" />
<option name="value" value="bbr" />
</DockerEnvVarImpl>
<DockerEnvVarImpl>
<option name="name" value="virtualization" />
<option name="value" value="dedicated" />
</DockerEnvVarImpl>
</list>
</option>
<option name="commandLineOptions" value="--privileged --net=host --ipc=host " />
<option name="showCommandPreview" value="true" />
<option name="sourceFilePath" value="agent/Dockerfile-wsl2" />
<option name="volumeBindings">
<list>
<DockerVolumeBindingImpl>
<option name="containerPath" value="/host" />
<option name="hostPath" value="/" />
</DockerVolumeBindingImpl>
</list>
</option>
</settings>
</deployment>
<method v="2">
<option name="RunConfigurationTask" enabled="true" run_configuration_name="ProjectOctopus [package]" run_configuration_type="MavenRunConfiguration" />
</method>
</configuration>
</component>

179
server/src/main/java/io/wdd/func/controller/OSSController.java
View File

@@ -1,13 +1,21 @@
package io.wdd.func.controller; package io.wdd.func.controller;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import io.swagger.annotations.Api; import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import io.wdd.common.beans.response.R;
import io.wdd.func.oss.config.OssConfig;
import io.wdd.func.oss.service.OSSCoreService; import io.wdd.func.oss.service.OSSCoreService;
import org.springframework.web.bind.annotation.GetMapping; import io.wdd.func.oss.service.OssBackendSelect;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.*;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.util.List;
@RestController @RestController
@RequestMapping("/server/func/oss/") @RequestMapping("/server/func/oss/")
@@ -17,12 +25,167 @@ public class OSSController {
@Resource @Resource
OSSCoreService OSSCoreService; OSSCoreService OSSCoreService;
@GetMapping("/bucket/list/all") @Resource
@ApiOperation("列出所有的桶") OssBackendSelect ossBackendSelect;
public void bucketListAll() {
OSSCoreService.listBucketList(); @GetMapping("/bucket/list/all/")
System.out.println(" = "); @ApiOperation("[桶] - 所有的桶")
public R<List<List<Bucket>>> bucketListAll() {
return R.ok(OSSCoreService.listAllBucket());
} }
@GetMapping("/bucket/list/one/")
@ApiOperation("[桶] - 一个区域所有的桶")
public R<List<Bucket>> bucketListOne(
@RequestParam(value = "BackendPrefixName", required = false)
@ApiParam(value = "BackendPrefixName") String BackendPrefixName
) {
OssConfig ossConfig = ossBackendSelect.one(BackendPrefixName);
return R.ok(OSSCoreService.listBucketList(ossConfig));
}
@GetMapping("/bucket/list/objects")
@ApiOperation("[桶] - 一个区域一个桶中的对象")
public R<ObjectListing> bucketObjectsListAll(
@RequestParam(value = "BackendPrefixName", required = false)
@ApiParam(value = "BackendPrefixName") String BackendPrefixName,
@RequestParam(value = "bucketName", required = false)
@ApiParam(value = "bucketName") String bucketName
) {
OssConfig ossConfig = ossBackendSelect.one(BackendPrefixName);
return R.ok(OSSCoreService.listBucketObjects(
ossConfig,
bucketName
));
}
@PostMapping("/bucket/delete")
@ApiOperation("[桶] - 删除")
public R<Boolean> bucketDelete(
@RequestParam(value = "BackendPrefixName", required = true)
@ApiParam(value = "BackendPrefixName") String BackendPrefixName,
@RequestParam(value = "bucketName", required = true)
@ApiParam(value = "bucketName") String bucketName
) {
OssConfig ossConfig = ossBackendSelect.one(BackendPrefixName);
return R.ok(OSSCoreService.deleteBucket(
ossConfig,
bucketName
));
}
@PostMapping("/bucket/create")
@ApiOperation("[桶] - 创建")
public R<Bucket> bucketCreate(
@RequestParam(value = "BackendPrefixName", required = true)
@ApiParam(value = "BackendPrefixName") String BackendPrefixName,
@RequestParam(value = "bucketName", required = true)
@ApiParam(value = "bucketName") String bucketName
) {
OssConfig ossConfig = ossBackendSelect.one(BackendPrefixName);
return R.ok(OSSCoreService.createBucket(
ossConfig,
bucketName
));
}
@PostMapping("/object/create")
@ApiOperation("[对象] - 创建, 外部上传 [todo 分片上传]")
public R<ObjectMetadata> objectCreate(
@RequestParam(value = "BackendPrefixName", required = true)
@ApiParam(value = "BackendPrefixName") String BackendPrefixName,
@RequestParam(value = "bucketName", required = true)
@ApiParam(value = "bucketName") String bucketName,
@RequestParam(value = "objectName", required = true)
@ApiParam(value = "objectName") String objectName,
@RequestParam("file") @RequestPart MultipartFile file
) {
OssConfig ossConfig = ossBackendSelect.one(BackendPrefixName);
return R.ok(OSSCoreService.createObject(
ossConfig,
bucketName,
objectName,
file
));
}
@PostMapping("/object/one")
@ApiOperation("[对象] - 查询下载地址")
public R<S3Object> objectDetailOne(
@RequestParam(value = "BackendPrefixName", required = true)
@ApiParam(value = "BackendPrefixName") String BackendPrefixName,
@RequestParam(value = "bucketName", required = true)
@ApiParam(value = "bucketName") String bucketName,
@RequestParam(value = "objectName", required = true)
@ApiParam(value = "objectName") String objectName
) {
OssConfig ossConfig = ossBackendSelect.one(BackendPrefixName);
return R.ok(OSSCoreService.getObject(
ossConfig,
bucketName,
objectName
));
}
@PostMapping("/object/head")
@ApiOperation("[对象] - 查询头信息")
public R<ObjectMetadata> objectDetailHead(
@RequestParam(value = "BackendPrefixName", required = true)
@ApiParam(value = "BackendPrefixName") String BackendPrefixName,
@RequestParam(value = "bucketName", required = true)
@ApiParam(value = "bucketName") String bucketName,
@RequestParam(value = "objectName", required = true)
@ApiParam(value = "objectName") String objectName
) {
OssConfig ossConfig = ossBackendSelect.one(BackendPrefixName);
return R.ok(OSSCoreService.headObject(
ossConfig,
bucketName,
objectName
));
}
@PostMapping("/object/delete")
@ApiOperation("[对象] - 删除")
public R<Boolean> objectDelete(
@RequestParam(value = "BackendPrefixName", required = true)
@ApiParam(value = "BackendPrefixName") String BackendPrefixName,
@RequestParam(value = "bucketName", required = true)
@ApiParam(value = "bucketName") String bucketName,
@RequestParam(value = "objectName", required = true)
@ApiParam(value = "objectName") String objectName
) {
OssConfig ossConfig = ossBackendSelect.one(BackendPrefixName);
return R.ok(OSSCoreService.deleteObject(
ossConfig,
bucketName,
objectName
));
}
} }

90
server/src/main/java/io/wdd/func/oss/config/OSSConfiguration.java
View File

@@ -1,25 +1,20 @@
package io.wdd.func.oss.config; package io.wdd.func.oss.config;
import com.alibaba.nacos.api.config.annotation.NacosConfigListener;
import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.AmazonS3Client;
import kotlin.PublishedApi;
import lombok.Data; import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.yaml.snakeyaml.Yaml;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map;
/** /**
* 从Nacos中获取配置，获取所有S3的配置，并且自动更新 * 从Nacos中获取配置，获取所有S3的配置，并且自动更新
@@ -34,45 +29,77 @@ import java.util.Map;
public class OSSConfiguration { public class OSSConfiguration {
/** /**
* 缓存所有的S3客户端 * 缓存所有的S3的配置对象
*/ */
public List<AmazonS3> ALL_S3_CLIENT = new ArrayList<>(); public static final List<OssConfig> ALL_OSS_CONFIG = new ArrayList<>();
/** /**
* 从Nacos中获取到所有的对象存储对象 * 初始数据
*/ */
@Value("${oss.oracle.seoul2.region}")
String SeoulRegion;
@Value("${oss.oracle.seoul2.namespace}")
String Seoul2Namespace;
@Value("${oss.oracle.seoul2.key}")
String Seoul2Key;
@Value("${oss.oracle.seoul2.secret}")
String Seoul2Secret;
private LinkedHashMap<String, LinkedHashMap<String, String>> oracle = new LinkedHashMap<>(); private LinkedHashMap<String, LinkedHashMap<String, String>> oracle = new LinkedHashMap<>();
@PostConstruct @PostConstruct
public void buildAllS3Client() { public void buildAllS3Client() {
System.out.println("oracle = " + oracle); oracle
.entrySet()
.stream()
.forEach(
entry -> {
LinkedHashMap<String, String> entryValue = entry.getValue();
OssConfig ossConfig = OssConfig
.builder()
.name(entry.getKey())
.namespace(entryValue.get("namespace"))
.region(entryValue.get("region"))
.key(entryValue.get("key"))
.secret(entryValue.get("secret"))
.build();
AmazonS3 client = buildS3ClientFromOssConfig(ossConfig);
ossConfig.setClient(client);
// 添加进入全局cache
ALL_OSS_CONFIG.add(ossConfig);
// todo 异步任务,查询 已使用的容量, 写入ossConfig中
//
}
);
// help gc
oracle = null;
}
private AmazonS3 buildS3ClientFromOssConfig(OssConfig ossConfig) {
AWSCredentialsProvider credentials = new AWSStaticCredentialsProvider(new BasicAWSCredentials( AWSCredentialsProvider credentials = new AWSStaticCredentialsProvider(new BasicAWSCredentials(
Seoul2Key, ossConfig.getKey(),
Seoul2Secret ossConfig.getSecret()
)); ));
String oracleOSSEndpoint = buildOracleOSSEndpoint( String oracleOSSEndpoint = buildOracleOSSEndpoint(
Seoul2Namespace, ossConfig.getNamespace(),
SeoulRegion ossConfig.getRegion()
); );
// Create an S3 client pointing at the region // Create an S3 client pointing at the region
AwsClientBuilder.EndpointConfiguration endpointConfiguration = new AwsClientBuilder AwsClientBuilder.EndpointConfiguration endpointConfiguration = new AwsClientBuilder
.EndpointConfiguration( .EndpointConfiguration(
oracleOSSEndpoint, oracleOSSEndpoint,
SeoulRegion ossConfig.getRegion()
); );
AmazonS3 seoul2OSSClient = AmazonS3Client return AmazonS3Client
.builder() .builder()
.standard() .standard()
.withCredentials(credentials) .withCredentials(credentials)
@@ -80,11 +107,6 @@ public class OSSConfiguration {
.disableChunkedEncoding() .disableChunkedEncoding()
.enablePathStyleAccess() .enablePathStyleAccess()
.build(); .build();
// add cache
ALL_S3_CLIENT.add(seoul2OSSClient);
} }
private String buildOracleOSSEndpoint(String namespace, String region) { private String buildOracleOSSEndpoint(String namespace, String region) {
@@ -95,4 +117,16 @@ public class OSSConfiguration {
region region
); );
} }
/**
* 从Nacos中获取到所有的对象存储对象
*//*
@Value("${oss.oracle.seoul2.region}")
String SeoulRegion;
@Value("${oss.oracle.seoul2.namespace}")
String Seoul2Namespace;
@Value("${oss.oracle.seoul2.key}")
String Seoul2Key;
@Value("${oss.oracle.seoul2.secret}")
String Seoul2Secret;*/
} }

46
server/src/main/java/io/wdd/func/oss/config/OssConfig.java Normal file
View File

@@ -0,0 +1,46 @@
package io.wdd.func.oss.config;
import com.amazonaws.services.s3.AmazonS3;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
@Data
@NoArgsConstructor
@AllArgsConstructor
@SuperBuilder(toBuilder = true)
@ApiModel("保存OSS的配置的实体类")
public class OssConfig {
/**
* 方面自己使用的名称 如 seoul1
*/
String name;
/*------------------------------------------------*/
String namespace;
String region;
String key;
String secret;
@ApiModelProperty("存储后端存储的容量, 单位为GB转为B 1024 * 1024 * 1024 * n GB")
Long capacity;
/*------------------------------------------------*/
@ApiModelProperty("存储后端 已使用 容量, 单位为GB转为B 1024 * 1024 * 1024 * n GB")
Long usedCap;
@ApiModelProperty("保存构造好的 s3客户端")
AmazonS3 client;
}

155
server/src/main/java/io/wdd/func/oss/service/OSSCoreService.java
View File

@@ -1,16 +1,165 @@
package io.wdd.func.oss.service; package io.wdd.func.oss.service;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import io.wdd.func.oss.config.OssConfig;
import org.springframework.web.multipart.MultipartFile;
import java.io.File;
import java.util.List;
public interface OSSCoreService { public interface OSSCoreService {
/** /**
* 对桶的操作 * 对桶的操作
*
*
*/ */
/** /**
* 列出所有的桶 * 列出全部的桶对象
*/
List<List<Bucket>> listAllBucket();
/**
* 列出所有的桶 Lists all buckets owned by the authenticated sender of the request.
* <p>
* https://docs.oracle.com/en-us/iaas/api/#/en/s3objectstorage/20160918/Bucket/GetService
*
* @return
*/
List<Bucket> listBucketList(OssConfig ossConfig);
/**
* Lists up to 1000 objects in the specified bucket.
* https://docs.oracle.com/en-us/iaas/api/#/en/s3objectstorage/20160918/Bucket/ListObjects
*
* @param ossConfig
* @param bucketName
* @return
*/
ObjectListing listBucketObjects(OssConfig ossConfig, String bucketName);
/**
* Checks to see if a bucket exists and that the requester has permission to access it.
*
* @param ossConfig
* @param bucketName
*/
void headBucket(OssConfig ossConfig, String bucketName);
/**
* Creates a new bucket.
* <p>
* https://docs.oracle.com/en-us/iaas/api/#/en/s3objectstorage/20160918/Bucket/PutBucket
*
* @param ossConfig
* @param bucketName
* @return
*/
Bucket createBucket(OssConfig ossConfig, String bucketName);
/**
* Deletes a bucket. All objects in the bucket must be deleted before the bucket itself can be deleted.
* https://docs.oracle.com/en-us/iaas/api/#/en/s3objectstorage/20160918/Bucket/DeleteBucket
*
* @param ossConfig
* @param bucketName
* @return
*/
boolean deleteBucket(OssConfig ossConfig, String bucketName);
/**
* Get the location of bucket's region.
* <p>
* https://docs.oracle.com/en-us/iaas/api/#/en/s3objectstorage/20160918/Bucket/GetLocation
*
* @param ossConfig
* @param bucketName
*/
void getBucketLocation(OssConfig ossConfig, String bucketName);
/* *
* 对象操作
* */ * */
void listBucketList();
void deleteBucket(); /**
* Retrieves an object.
* https://docs.oracle.com/en-us/iaas/api/#/en/s3objectstorage/20160918/Object/GetObject
*
* @param ossConfig
* @param bucketName
* @param objectName
* @return
*/
S3Object getObject(OssConfig ossConfig, String bucketName, String objectName);
/**
* https://docs.oracle.com/en-us/iaas/api/#/en/s3objectstorage/20160918/Object/DeleteObject
* Deletes an object.
*
* @param ossConfig
* @param bucketName
* @param objectName
* @return
*/
boolean deleteObject(OssConfig ossConfig, String bucketName, String objectName);
/**
* Gets the user-defined metadata and entity tag for an object.
* https://docs.oracle.com/en-us/iaas/api/#/en/s3objectstorage/20160918/Object/HeadObject
*
* @param ossConfig
* @param objectName
* @return
*/
ObjectMetadata headObject(OssConfig ossConfig, String bucketName, String objectName);
/**
* Creates a new object or overwrites an existing one.
* <p>
* https://docs.oracle.com/en-us/iaas/api/#/en/s3objectstorage/20160918/Object/PutObject
*
* @param ossConfig
* @param bucketName
* @param objectName
* @return
*/
ObjectMetadata createObject(OssConfig ossConfig, String bucketName, String objectName, MultipartFile file);
/**
* Creates a new object or overwrites an existing one.
* <p>
* https://docs.oracle.com/en-us/iaas/api/#/en/s3objectstorage/20160918/Object/PutObject
*
* @param ossConfig
* @param bucketName
* @param objectName
* @return
*/
ObjectMetadata createObject(OssConfig ossConfig, String bucketName, String objectName, File file);
/**
* Restores the object as defined in the request body.
* https://docs.oracle.com/en-us/iaas/api/#/en/s3objectstorage/20160918/Object/RestoreObjects
*
* @param ossConfig
* @param bucketName
* @param objectName
* @return
*/
String restoreObject(OssConfig ossConfig, String bucketName, String objectName);
} }

223
server/src/main/java/io/wdd/func/oss/service/OSSCoreServiceImpl.java
View File

@@ -1,18 +1,20 @@
package io.wdd.func.oss.service; package io.wdd.func.oss.service;
import com.alibaba.nacos.api.config.annotation.NacosConfigListener; import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.Bucket; import com.amazonaws.services.s3.model.*;
import io.wdd.func.oss.config.OSSConfiguration; import io.wdd.func.oss.config.OssConfig;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.yaml.snakeyaml.Yaml; import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource; import java.io.File;
import java.util.LinkedHashMap; import java.io.IOException;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.stream.Collectors;
import static io.wdd.func.oss.config.OSSConfiguration.ALL_OSS_CONFIG;
/** /**
* https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm#usingAPI * https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm#usingAPI
@@ -22,50 +24,185 @@ import java.util.Map;
public class OSSCoreServiceImpl implements OSSCoreService { public class OSSCoreServiceImpl implements OSSCoreService {
@Resource @Override
OSSConfiguration ossConfiguration; public List<List<Bucket>> listAllBucket() {
return ALL_OSS_CONFIG
/** .stream()
* 使用Linstener 更新OSS的配置信息 .map(
* @param content 获取到的更新的Nacos ossConfig -> listBucketList(ossConfig)
*/ )
@NacosConfigListener(dataId = "octopus-server-k3s.yaml", groupId = "k3s") .collect(Collectors.toList());
public void listener(String content){
System.out.println("content = " + content);
Yaml yaml = new Yaml();
Map<String, Object> contextMap = yaml.load(content);
Map<String, Object> ossObject = (Map<String, Object>) contextMap.get("oss");
LinkedHashMap<String, LinkedHashMap<String, String>> map = (LinkedHashMap<String, LinkedHashMap<String, String>>) ossObject.get("oralce");
// 需要同步这些
// synchronized (oracle) {
//
// oracle.clear();
// oracle.putAll(map);
//
// }
System.out.println("map = " + map);
} }
@Override @Override
public void listBucketList() { public List<Bucket> listBucketList(OssConfig ossConfig) {
AmazonS3 client = ossConfig.getClient();
AmazonS3 amazonS3 = ossConfiguration.ALL_S3_CLIENT.get(0); return client.listBuckets();
List<Bucket> buckets = amazonS3.listBuckets();
System.out.println("buckets = " + buckets);
} }
@Override @Override
public void deleteBucket() { public ObjectListing listBucketObjects(OssConfig ossConfig, String bucketName) {
AmazonS3 client = ossConfig.getClient();
return client.listObjects(bucketName);
}
@Override
public void headBucket(OssConfig ossConfig, String bucketName) {
AmazonS3 client = ossConfig.getClient();
HeadBucketResult headBucketResult = client.headBucket(
new HeadBucketRequest(bucketName)
);
}
@Override
public Bucket createBucket(OssConfig ossConfig, String bucketName) {
AmazonS3 client = ossConfig.getClient();
Bucket bucket = client.createBucket(
bucketName
);
return bucket;
}
@Override
public boolean deleteBucket(OssConfig ossConfig, String bucketName) {
AmazonS3 client = ossConfig.getClient();
try {
client.deleteBucket(bucketName);
} catch (Exception e) {
log.error(
"删除桶错误 ！ ossConfig is {}， bucketName is {}",
ossConfig,
bucketName
);
return false;
}
return true;
}
@Override
public void getBucketLocation(OssConfig ossConfig, String bucketName) {
}
@Override
public S3Object getObject(OssConfig ossConfig, String bucketName, String objectName) {
AmazonS3 client = ossConfig.getClient();
return client.getObject(
bucketName,
objectName
);
}
@Override
public boolean deleteObject(OssConfig ossConfig, String bucketName, String objectName) {
AmazonS3 client = ossConfig.getClient();
try {
client.deleteObject(
bucketName,
objectName
);
} catch (SdkClientException e) {
log.error(
"删除对象错误 ！ ossConfig is {}， bucketName is {}, objectName is {}",
ossConfig,
bucketName,
objectName
);
return false;
}
return true;
}
@Override
public ObjectMetadata headObject(OssConfig ossConfig, String bucketName, String objectName) {
AmazonS3 client = ossConfig.getClient();
try {
return client.getObjectMetadata(
bucketName,
objectName
);
} catch (Exception e) {
log.error("错误原因 => {}", e.getMessage());
return null;
}
}
@Override
public ObjectMetadata createObject(OssConfig ossConfig, String bucketName, String objectName, MultipartFile file) {
AmazonS3 client = ossConfig.getClient();
try {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(file.getSize());
PutObjectResult putObjectResult = client.putObject(
bucketName,
objectName,
file.getInputStream(),
metadata
);
return putObjectResult.getMetadata();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
public ObjectMetadata createObject(OssConfig ossConfig, String bucketName, String objectName, File file) {
AmazonS3 client = ossConfig.getClient();
PutObjectResult putObjectResult = client.putObject(
bucketName,
objectName,
file
);
return putObjectResult.getMetadata();
}
@Override
public String restoreObject(OssConfig ossConfig, String bucketName, String objectName) {
AmazonS3 client = ossConfig.getClient();
RestoreObjectResult restoreObjectResult = client.restoreObjectV2(
new RestoreObjectRequest(
bucketName,
objectName
)
);
return restoreObjectResult.getRestoreOutputPath();
} }
} }

79
server/src/main/java/io/wdd/func/oss/service/OssBackendSelect.java Normal file
View File

@@ -0,0 +1,79 @@
package io.wdd.func.oss.service;
import com.amazonaws.services.s3.AmazonS3;
import io.wdd.func.oss.config.OssConfig;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Random;
import java.util.stream.Collectors;
import static io.wdd.func.oss.config.OSSConfiguration.ALL_OSS_CONFIG;
/**
* 对象存储后端选择
* 指定一系列的策略，选出最为合适的OSS存储后端位置
*/
@Service
@Slf4j
public class OssBackendSelect {
public OssConfig one(String BackendPrefixName){
if (StringUtils.isEmpty(BackendPrefixName)) {
return oneRandom();
}
List<OssConfig> ossConfigList = all(BackendPrefixName);
if (ossConfigList.size() == 0) {
log.error("没能找到合适的OSS后端存储，请确保输入正确！");
} else if (ossConfigList.size() == 1) {
return ossConfigList.get(0);
}
// 出现了多个后端存储，需要筛选出来只有一个的情况
// todo 根据评分计算出来最合适的一个！
// 选择当前储存容量最小的一个
// 随机选择一个, 也算是一个算法吧
Random random = new Random();
int pos = random
.ints(0,
ossConfigList.size())
.findFirst()
.getAsInt();
return ossConfigList.get(pos);
}
public OssConfig oneRandom() {
Random random = new Random();
int pos = random
.ints(0,
ALL_OSS_CONFIG.size())
.findFirst()
.getAsInt();
return ALL_OSS_CONFIG.get(pos);
}
public List<OssConfig> all(String BackendPrefixName){
return ALL_OSS_CONFIG
.stream()
.filter(
ossConfig -> ossConfig
.getName()
.startsWith(BackendPrefixName)
)
.collect(Collectors.toList());
}
}

3
server/src/main/java/io/wdd/rpc/init/AcceptAgentInitInfo.java
View File

@@ -236,7 +236,8 @@ public class AcceptAgentInitInfo {
// validate serverName // validate serverName
String[] split = serverName.split("-"); String[] split = serverName.split("-");
if (split.length <= 2 || !ALL_SERVER_CITY_INFO.contains(split[0]) || !ALL_SERVER_ARCH_INFO.contains(split[1])) { if (split.length <= 2 || !ALL_SERVER_CITY_INFO.contains(split[0]) || !ALL_SERVER_ARCH_INFO.contains(split[1])) {
throw new MyRuntimeException(" server name not validated !"); log.error("server info from agent are {}", serverInfoVO);
throw new MyRuntimeException("server name not validated !");
} }
String machineIdPrefixSixBytes = String.valueOf(serverInfoVO.getMachineId().toCharArray(), 0, 6); String machineIdPrefixSixBytes = String.valueOf(serverInfoVO.getMachineId().toCharArray(), 0, 6);

16
server/src/main/java/io/wdd/server/config/Knife4jConfig.java
View File

@@ -48,6 +48,22 @@ public class Knife4jConfig {
} }
@Bean
public Docket createFuncApiRPC() {
return new Docket(DocumentationType.SWAGGER_2)
.useDefaultResponseMessages(false)
.apiInfo(apiInfo())
.groupName("Server功能业务API")
.select()
// controller包路径，配置不对的话，找不到
.apis(
RequestHandlerSelectors.basePackage("io.wdd.func.controller")
)
.paths(PathSelectors.any())
.build();
}
private ApiInfo apiInfo() { private ApiInfo apiInfo() {
return new ApiInfoBuilder() return new ApiInfoBuilder()
.title("Octopus Server") .title("Octopus Server")

26
server/src/main/resources/application-local.yml
View File

@@ -1,26 +0,0 @@
spring:
application:
name: octopus-server
profiles:
active: local
cloud:
nacos:
config:
group: local
config-retry-time: 3000
file-extension: yaml
max-retry: 3
# server-addr: 43.154.83.213:21060
# server-addr: 140.238.52.228:21060
server-addr: https://nacos.107421.xyz:443
timeout: 5000
config-long-poll-timeout: 5000
extension-configs:
- group: local
data-id: common-local.yaml
debug: true
logging:
level:
io.wdd.server:
debug

87
server/src/main/resources/application.yml
View File

@@ -1,87 +0,0 @@
server:
port: 9999
octopus:
message:
# agent boot up default common exchange
init_exchange: InitExchange
# server will send message to agent using this common queue
init_to_server: InitToServer
# agent boot up default common exchange routing key
init_to_server_key: InitToServerKey
# server will receive message from agent using this common queue
init_from_server: InitFromServer
# agent boot up default common exchange routing key
init_from_server_key: InitFromServerKey
# initialization register time out (unit ms) default is 5 min
init_ttl: "300000"
# Octopus Exchange Name == server comunicate with agent
octopus_exchange: OctopusExchange
# Octopus Message To Server == all agent send info to server queue and topic
octopus_to_server: OctopusToServer
spring:
rabbitmq:
host: 127.0.0.1
port: 35672
username: boge
password: boge14@Level5
virtual-host: /wddserver
listener:
simple:
retry:
# ack failed will reentrant the Rabbit Listener
max-attempts: 5
enabled: true
# retry interval unit ms
max-interval: 5000
initial-interval: 5000
redis:
host: 127.0.0.1
port: 36379
database: 0
password: boge14@Level5
datasource:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://127.0.0.1:33306/wdd_server?autoReconnect=true&useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2B8
username: root
password: boge14@Level5
type: com.zaxxer.hikari.HikariDataSource
hikari:
minimum-idle: 3
# 空闲连接存活最大时间，默认600000（10分钟）
idle-timeout: 180000
# 连接池最大连接数，默认是10
maximum-pool-size: 5
# 此属性控制从池返回的连接的默认自动提交行为,默认值：true
auto-commit: true
connection-test-query: SELECT 1
mybatis-plus:
type-aliases-package: io.wdd.server.beans.po
global-config:
db-column-underline: true
db-config:
# modify ethe id strategy
id-type: assign_id
# logic delete field globally
logicDeleteField: isDelete
logic-not-delete-value: 0
logic-delete-value: 1
banner: false
configuration:
# 希望知道所有的sql是怎么执行的, 配置输出日志
log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
# 数据库下划线--实体类也是下划线 需要为false
map-underscore-to-camel-case: true
# 一级缓存的 缓存级别默认为 session，如果要关闭一级缓存可以设置为 statement
local-cache-scope: session
# 是否开启二级缓存
cache-enabled: false
# 默认地址为 classpath*:/mapper/**/*.xml
# mapper-locations: classpath*:/real-mappers/**/*.xml