[agent-operator] - minio refresh
This commit is contained in:
zeaslity
@@ -1,27 +1,23 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/minio/minio-go"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go"
|
||||
"wdd.io/agent-common/utils"
|
||||
)
|
||||
|
||||
type CmiiMinioOperator struct {
|
||||
LocalEndpoint string
|
||||
PublicEndpoint string
|
||||
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
|
||||
LocalClient *minio.Client
|
||||
PublicClient *minio.Client
|
||||
DemoClient *minio.Client
|
||||
|
||||
OctopusBucketName string
|
||||
LocalMinioOperator *MinioOperator
|
||||
PublicMinioOperator *MinioOperator
|
||||
DemoMinioOperator *MinioOperator
|
||||
}
|
||||
|
||||
var DefaultCmiiMinioOperator = newInstance()
|
||||
|
||||
func newInstance() *CmiiMinioOperator {
|
||||
|
||||
op := &CmiiMinioOperator{}
|
||||
|
||||
op.buildCmiiMinioOperator()
|
||||
@@ -32,87 +28,182 @@ func newInstance() *CmiiMinioOperator {
|
||||
const (
|
||||
DefaultLocalEndpoint = "10.250.0.100:9000"
|
||||
DefaultPublicEndpoint = "42.192.52.227:9000"
|
||||
DemoEndpoint = "oss.demo.uavcmlc.com:18000"
|
||||
DefaultDemoEndpoint = "oss.demo.uavcmlc.com:18000"
|
||||
DefaultAccessKeyID = "cmii"
|
||||
DefaultSecretAccessKey = "B#923fC7mk"
|
||||
DefaultOctopusBucketName = "octopus"
|
||||
DefaultOssUrlPrefix = "https://oss.demo.uavcmlc.com/cmlc-installation/tmp/"
|
||||
DefaultOssUrlPrefix = "https://oss.demo.uavcmlc.com:18000/cmlc-installation/tmp/"
|
||||
)
|
||||
|
||||
func (op *CmiiMinioOperator) buildCmiiMinioOperator() {
|
||||
|
||||
var err error
|
||||
if op.LocalClient == nil {
|
||||
// 初始化Minio客户端对象。
|
||||
op.LocalClient, err = minio.New(DefaultLocalEndpoint, DefaultAccessKeyID, DefaultSecretAccessKey, false)
|
||||
if err != nil {
|
||||
log.ErrorF("[buildCmiiMinioOperator] - build for LocalClient error !=> %s", err.Error())
|
||||
if op.LocalMinioOperator == nil {
|
||||
op.LocalMinioOperator = &MinioOperator{
|
||||
MinioEndpoint: DefaultLocalEndpoint,
|
||||
EndpointSecure: false,
|
||||
AccessKeyID: DefaultAccessKeyID,
|
||||
SecretAccessKey: DefaultSecretAccessKey,
|
||||
Client: nil,
|
||||
}
|
||||
|
||||
op.LocalMinioOperator.BuildMinioOperator()
|
||||
}
|
||||
|
||||
if op.PublicClient == nil {
|
||||
// 初始化Minio客户端对象。
|
||||
op.PublicClient, err = minio.New(DefaultPublicEndpoint, DefaultAccessKeyID, DefaultSecretAccessKey, false)
|
||||
if err != nil {
|
||||
log.ErrorF("[buildCmiiMinioOperator] - build for PublicClient error ! => %s", err.Error())
|
||||
if op.PublicMinioOperator == nil {
|
||||
op.PublicMinioOperator = &MinioOperator{
|
||||
MinioEndpoint: DefaultPublicEndpoint,
|
||||
EndpointSecure: false,
|
||||
AccessKeyID: DefaultAccessKeyID,
|
||||
SecretAccessKey: DefaultSecretAccessKey,
|
||||
Client: nil,
|
||||
}
|
||||
|
||||
op.PublicMinioOperator.BuildMinioOperator()
|
||||
}
|
||||
|
||||
if op.DemoClient == nil {
|
||||
// 初始化Minio客户端对象。
|
||||
op.DemoClient, err = minio.New(DemoEndpoint, DefaultAccessKeyID, DefaultSecretAccessKey, true)
|
||||
if err != nil {
|
||||
log.ErrorF("[buildCmiiMinioOperator] - build for DemoClient error ! => %s", err.Error())
|
||||
if op.DemoMinioOperator == nil {
|
||||
op.DemoMinioOperator = &MinioOperator{
|
||||
MinioEndpoint: DefaultDemoEndpoint,
|
||||
EndpointSecure: true,
|
||||
AccessKeyID: DefaultAccessKeyID,
|
||||
SecretAccessKey: DefaultSecretAccessKey,
|
||||
Client: nil,
|
||||
}
|
||||
|
||||
op.DemoMinioOperator.BuildMinioOperator()
|
||||
}
|
||||
|
||||
log.DebugF("[buildCmiiMinioOperator] - build client success !")
|
||||
}
|
||||
|
||||
func (op *CmiiMinioOperator) UploadToLocalOctopus(filePath, fileName string) bool {
|
||||
return op.uploadToOctopus(op.LocalClient, filePath, fileName)
|
||||
return op.LocalMinioOperator.UploadFile(DefaultOctopusBucketName, filePath, fileName)
|
||||
}
|
||||
|
||||
func (op *CmiiMinioOperator) UploadToPublicOctopus(filePath, fileName string) bool {
|
||||
|
||||
return op.uploadToOctopus(op.PublicClient, filePath, fileName)
|
||||
return op.PublicMinioOperator.UploadFile(DefaultOctopusBucketName, filePath, fileName)
|
||||
}
|
||||
|
||||
func (op *CmiiMinioOperator) UploadToDemo(bucketName, filePath, fileName string) bool {
|
||||
|
||||
return op.uploadToOss(op.DemoClient, bucketName, filePath, fileName)
|
||||
return op.DemoMinioOperator.UploadFile(bucketName, filePath, fileName)
|
||||
}
|
||||
|
||||
func (op *CmiiMinioOperator) uploadToOctopus(client *minio.Client, filePath, fileName string) bool {
|
||||
type MinioOperator struct {
|
||||
MinioEndpoint string
|
||||
EndpointSecure bool
|
||||
|
||||
return op.uploadToOss(client, DefaultOctopusBucketName, filePath, fileName)
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
|
||||
Client *minio.Client
|
||||
}
|
||||
|
||||
func (op *CmiiMinioOperator) uploadToOss(client *minio.Client, bucketName, filePath, fileName string) bool {
|
||||
func (op *MinioOperator) BuildMinioOperator() {
|
||||
|
||||
if !strings.HasSuffix(filePath, "/") {
|
||||
filePath += "/"
|
||||
op.MinioEndpoint = strings.TrimPrefix(op.MinioEndpoint, "http://")
|
||||
op.MinioEndpoint = strings.TrimPrefix(op.MinioEndpoint, "https://")
|
||||
if strings.Contains(op.MinioEndpoint, "/") {
|
||||
op.MinioEndpoint = strings.Split(op.MinioEndpoint, "/")[0]
|
||||
}
|
||||
|
||||
realFileName := fileName
|
||||
// 解析bucket上传的真实名称
|
||||
if strings.Contains(bucketName, "/") {
|
||||
splitN := strings.SplitN(bucketName, "/", 2)
|
||||
bucketName = splitN[0]
|
||||
if !strings.HasSuffix(splitN[1], "/") {
|
||||
splitN[1] = splitN[1] + "/"
|
||||
}
|
||||
realFileName = splitN[1] + fileName
|
||||
}
|
||||
|
||||
// 使用PutObject上传文件
|
||||
// realFileName ==> tmp/123/123.txt
|
||||
n, err := client.FPutObject(bucketName, realFileName, filePath+fileName, minio.PutObjectOptions{})
|
||||
client, err := minio.New(op.MinioEndpoint, op.AccessKeyID, op.SecretAccessKey, op.EndpointSecure)
|
||||
if err != nil {
|
||||
log.ErrorF("[uploadToOss] - upload %s error %s", filePath+fileName, err.Error())
|
||||
log.ErrorF("Failed to create client: %v", err)
|
||||
}
|
||||
log.DebugF("endpoint: %s accessKeyId: %s secretAccessKey: %s", op.MinioEndpoint, op.AccessKeyID, op.SecretAccessKey)
|
||||
|
||||
op.Client = client
|
||||
}
|
||||
|
||||
func (op *MinioOperator) ListFileInBucket(bucketName string) {
|
||||
// 使用ListObjectsV2获取bucket中的所有文件名。
|
||||
doneChan := make(chan struct{})
|
||||
|
||||
objectInfo := op.Client.ListObjectsV2(bucketName, "", false, doneChan)
|
||||
|
||||
for object := range objectInfo {
|
||||
utils.BeautifulPrint(object)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (op *MinioOperator) DownloadFileFromOssFullUrl(ossFullUrl string, filePath string) bool {
|
||||
log.InfoF("[DownloadFileFromOssFullUrl] - download from %s", ossFullUrl)
|
||||
|
||||
// https://oss.demo.uavcmlc.com:18000/cmlc-installation/tmp/docker.tar.gz
|
||||
ossFullUrl = strings.TrimPrefix(ossFullUrl, "http://")
|
||||
ossFullUrl = strings.TrimPrefix(ossFullUrl, "https://")
|
||||
// oss.demo.uavcmlc.com:18000/cmlc-installation/tmp/docker.tar.gz
|
||||
|
||||
split := strings.Split(ossFullUrl, "/")
|
||||
if len(split) < 2 {
|
||||
log.ErrorF("DownloadFileFromOssFullUrl - split error len < 2 !")
|
||||
return false
|
||||
}
|
||||
|
||||
bucketName := split[0]
|
||||
fileName := strings.Join(split[1:], "/")
|
||||
//realFileName := split[len(split)-1]
|
||||
|
||||
return op.DownloadFile(bucketName, filePath, fileName)
|
||||
|
||||
}
|
||||
|
||||
// DownloadFile objectName格式为 cmlc-installation/tmp/123/123.txt
|
||||
func (op *MinioOperator) DownloadFile(bucketName, filePath, fileNameWithPrefix string) bool {
|
||||
|
||||
separator := os.PathSeparator
|
||||
if !strings.HasSuffix(filePath, string(separator)) {
|
||||
filePath += string(separator)
|
||||
}
|
||||
|
||||
fileNameWithPrefix = strings.TrimPrefix(fileNameWithPrefix, "/")
|
||||
|
||||
// 获取下载的真实文件名称 123.txt
|
||||
realFileName := fileNameWithPrefix
|
||||
if strings.Contains(fileNameWithPrefix, "/") {
|
||||
realFileName = strings.Split(fileNameWithPrefix, "/")[len(strings.Split(fileNameWithPrefix, "/"))-1]
|
||||
}
|
||||
|
||||
log.InfoF("[DownloadFile] - download from [%s] to [%s]", op.MinioEndpoint+"/"+bucketName+"/"+fileNameWithPrefix, filePath+realFileName)
|
||||
err := op.Client.FGetObject(bucketName, fileNameWithPrefix, filePath+realFileName, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
log.ErrorF("[DownloadFile] - download %s error %s", filePath+realFileName, err.Error())
|
||||
return false
|
||||
}
|
||||
|
||||
log.InfoF("[uploadToOss] - uploaded %s of size %d", filePath+fileName, n)
|
||||
return true
|
||||
}
|
||||
|
||||
// UploadFile bucketNameWithSuffix格式为 cmlc-installation/tmp/123 的格式
|
||||
func (op *MinioOperator) UploadFile(bucketNameWithSuffix, filePath, fileName string) bool {
|
||||
|
||||
separator := os.PathSeparator
|
||||
if !strings.HasSuffix(filePath, string(separator)) {
|
||||
filePath += string(separator)
|
||||
}
|
||||
|
||||
bucketNameWithSuffix = strings.TrimPrefix(bucketNameWithSuffix, "/")
|
||||
oldBucketName := bucketNameWithSuffix
|
||||
realFileName := fileName
|
||||
// 解析bucket上传的真实名称
|
||||
if strings.Contains(bucketNameWithSuffix, "/") {
|
||||
bucketNameWithSuffix = strings.Split(bucketNameWithSuffix, "/")[0]
|
||||
fileName = strings.TrimPrefix(oldBucketName, bucketNameWithSuffix) + "/" + realFileName
|
||||
}
|
||||
|
||||
// 使用PutObject上传文件
|
||||
// fileName ==> tmp/123/123.txt
|
||||
// realFileName ==> 123.txt
|
||||
log.InfoF("[UploadFile] - upload from [%s] to [%s]", filePath+realFileName, op.MinioEndpoint+"/"+bucketNameWithSuffix+"/"+fileName)
|
||||
n, err := op.Client.FPutObject(bucketNameWithSuffix, fileName, filePath+realFileName, minio.PutObjectOptions{})
|
||||
if err != nil {
|
||||
log.ErrorF("[UploadFile] - upload [%s] to [%s] error %s", filePath+realFileName, op.MinioEndpoint+"/"+bucketNameWithSuffix+"/"+fileName, err.Error())
|
||||
return false
|
||||
}
|
||||
|
||||
log.InfoF("[UploadFile] - uploaded %s of size %d", filePath+fileName, n)
|
||||
return true
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user