新增doris的部署内容

agent-common/SplitProject/监管平台-Doris-集群部署/doris-configmap.yaml

@@ -0,0 +1,71 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fe-configmap
+  namespace: doriscluster
+  labels:
+    app.kubernetes.io/component: fe
+data:
+  fe.conf: |
+    CUR_DATE=`date +%Y%m%d-%H%M%S`
+ 
+    # the output dir of stderr and stdout
+    LOG_DIR = ${DORIS_HOME}/log
+ 
+    JAVA_OPTS="-Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$CUR_DATE"
+ 
+    # For jdk 9+, this JAVA_OPTS will be used as default JVM options
+    JAVA_OPTS_FOR_JDK_9="-Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$CUR_DATE:time"
+ 
+    # INFO, WARN, ERROR, FATAL
+    sys_log_level = INFO
+ 
+    # NORMAL, BRIEF, ASYNC
+    sys_log_mode = NORMAL
+ 
+    # Default dirs to put jdbc drivers,default value is ${DORIS_HOME}/jdbc_drivers
+    # jdbc_drivers_dir = ${DORIS_HOME}/jdbc_drivers
+ 
+    http_port = 8030
+    arrow_flight_sql_port = 9090
+    rpc_port = 9020
+    query_port = 9030
+    edit_log_port = 9010
+    
+    enable_fqdn_mode = true
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: be-configmap
+  namespace: doriscluster
+  labels:
+    app.kubernetes.io/component: be
+data:
+  be.conf: |
+    CUR_DATE=`date +%Y%m%d-%H%M%S`
+ 
+    PPROF_TMPDIR="$DORIS_HOME/log/"
+ 
+    JAVA_OPTS="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -DJDBC_MIN_POOL=1 -DJDBC_MAX_POOL=100 -DJDBC_MAX_IDLE_TIME=300000 -DJDBC_MAX_WAIT_TIME=5000"
+ 
+    # For jdk 9+, this JAVA_OPTS will be used as default JVM options
+    JAVA_OPTS_FOR_JDK_9="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -DJDBC_MIN_POOL=1 -DJDBC_MAX_POOL=100 -DJDBC_MAX_IDLE_TIME=300000 -DJDBC_MAX_WAIT_TIME=5000"
+ 
+    # since 1.2, the JAVA_HOME need to be set to run BE process.
+    # JAVA_HOME=/path/to/jdk/
+ 
+    # https://github.com/apache/doris/blob/master/docs/zh-CN/community/developer-guide/debug-tool.md#jemalloc-heap-profile
+    # https://jemalloc.net/jemalloc.3.html
+    JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,lg_tcache_max:20,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false"
+    JEMALLOC_PROF_PRFIX=""
+ 
+    # INFO, WARNING, ERROR, FATAL
+    sys_log_level = INFO
+ 
+    # ports for admin, web, heartbeat service
+    be_port = 9060
+    webserver_port = 8040
+    heartbeat_service_port = 9050
+    arrow_flight_sql_port = 39091
+    brpc_port = 8060

agent-common/SplitProject/监管平台-Doris-集群部署/doris-deplyment.yaml

@@ -0,0 +1,101 @@
+apiVersion: doris.selectdb.com/v1
+kind: DorisCluster
+metadata:
+  labels:
+    app.kubernetes.io/name: doriscluster
+  name: doriscluster-helm
+  namespace: doriscluster
+spec:
+  feSpec:
+    replicas: 1
+    image: harbor.cdcyy.com.cn/cmii/doris.fe-ubuntu:2.1.6
+    limits:
+      cpu: 8
+      memory: 16Gi
+    requests:
+      cpu: 2
+      memory: 6Gi
+    configMapInfo:
+      # use kubectl create configmap fe-configmap --from-file=fe.conf
+      configMapName: fe-configmap
+      resolveKey: fe.conf
+    nodeSelector:
+      uavcloud.env: demo
+    persistentVolumes:
+    - mountPath: /opt/apache-doris/fe/doris-meta
+      name: doriscluster-storage0
+      persistentVolumeClaimSpec:
+        # when use specific storageclass, the storageClassName should reConfig, example as annotation.
+        storageClassName: nfs-prod-distribute
+        accessModes:
+        - ReadWriteOnce
+        resources:
+          # notice: if the storage size less 5G, fe will not start normal.
+          requests:
+            storage: 300Gi
+    - mountPath: /opt/apache-doris/fe/log
+      name: doriscluster-storage1
+      persistentVolumeClaimSpec:
+        # when use specific storageclass, the storageClassName should reConfig, example as annotation.
+        storageClassName: nfs-prod-distribute
+        accessModes:
+        - ReadWriteOnce
+        resources:
+          requests:
+            storage: 100Gi
+    - mountPath: /opt/apache-doris/fe/jdbc_drivers
+      name: doriscluster-storage-fe-jdbc-drivers
+      persistentVolumeClaimSpec:
+        # when use specific storageclass, the storageClassName should reConfig, example as annotation.
+        storageClassName: nfs-prod-distribute
+        accessModes:
+        - ReadWriteOnce
+        resources:
+          requests:
+            storage: 100Gi
+  beSpec:
+    replicas: 3
+    image: harbor.cdcyy.com.cn/cmii/doris.be-ubuntu:2.1.6
+    limits:
+      cpu: 8
+      memory: 24Gi
+    requests:
+      cpu: 2
+      memory: 6Gi
+    configMapInfo:
+      # use kubectl create configmap be-configmap --from-file=be.conf
+      configMapName: be-configmap
+      resolveKey: be.conf
+    nodeSelector:
+      uavcloud.env: demo
+    persistentVolumes:
+    - mountPath: /opt/apache-doris/be/storage
+      name: doriscluster-storage2
+      persistentVolumeClaimSpec:
+        # when use specific storageclass, the storageClassName should reConfig, example as annotation.
+        storageClassName: nfs-prod-distribute
+        accessModes:
+        - ReadWriteOnce
+        resources:
+          requests:
+            storage: 300Gi
+    - mountPath: /opt/apache-doris/be/log
+      name: doriscluster-storage3
+      persistentVolumeClaimSpec:
+        # when use specific storageclass, the storageClassName should reConfig, example as annotation.
+        storageClassName: nfs-prod-distribute
+        accessModes:
+        - ReadWriteOnce
+        resources:
+          requests:
+            storage: 100Gi
+    - mountPath: /opt/apache-doris/be/jdbc_drivers
+      name: doriscluster-storage-be-jdbc-drivers
+      persistentVolumeClaimSpec:
+        # when use specific storageclass, the storageClassName should reConfig, example as annotation.
+        storageClassName: nfs-prod-distribute
+        accessModes:
+        - ReadWriteOnce
+        resources:
+          requests:
+            storage: 100Gi

agent-common/SplitProject/监管平台-Doris-集群部署/doris-operator.yaml

@@ -0,0 +1,347 @@
+# Source: doris-operator/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/name: serviceaccount
+    app.kubernetes.io/instance: controller-doris-operator-sa
+    app.kubernetes.io/component: rbac
+    app.kubernetes.io/created-by: doris-operator
+    app.kubernetes.io/part-of: doris-operator
+    app.kubernetes.io/managed-by: Helm
+  name: doris-operator
+  namespace: doriscluster
+---
+# Source: doris-operator/templates/clusterrole.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: doris-operator
+rules:
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets/status
+    verbs:
+      - get
+  - apiGroups:
+      - autoscaling
+    resources:
+      - horizontalpodautoscalers
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - persistentvolumeclaims
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - serviceaccounts
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - services
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - doris.selectdb.com
+    resources:
+      - dorisclusters
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - doris.selectdb.com
+    resources:
+      - dorisclusters/finalizers
+    verbs:
+      - update
+  - apiGroups:
+      - doris.selectdb.com
+    resources:
+      - dorisclusters/status
+    verbs:
+      - get
+      - patch
+      - update
+  - apiGroups:
+      - rbac.authorization.k8s.io
+    resources:
+      - clusterrolebindings
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - rbac.authorization.k8s.io
+    resources:
+      - rolebindings
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+---
+# Source: doris-operator/templates/clusterrolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/name: clusterrolebinding
+    app.kubernetes.io/instance: doris-operator-rolebinding
+    app.kubernetes.io/component: rbac
+    app.kubernetes.io/created-by: doris-operator
+    app.kubernetes.io/part-of: doris-operator
+    app.kubernetes.io/managed-by: Helm
+  name: doris-operator-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: doris-operator
+subjects:
+  - kind: ServiceAccount
+    name: doris-operator
+    namespace: doriscluster
+---
+# Source: doris-operator/templates/leader-election-role.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/name: role
+    app.kubernetes.io/instance: leader-election-role
+    app.kubernetes.io/component: rbac
+    app.kubernetes.io/created-by: doris-operator
+    app.kubernetes.io/part-of: doris-operator
+    app.kubernetes.io/managed-by: Helm
+  name: leader-election-role
+  namespace: doriscluster
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
+      - delete
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
+      - delete
+  - apiGroups:
+      - ""
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+---
+# Source: doris-operator/templates/leader-election-role-binding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/name: rolebinding
+    app.kubernetes.io/instance: leader-election-rolebinding
+    app.kubernetes.io/component: rbac
+    app.kubernetes.io/created-by: doris-operator
+    app.kubernetes.io/part-of: doris-operator
+    app.kubernetes.io/managed-by: Helm
+  name: leader-election-rolebinding
+  namespace: doriscluster
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: leader-election-role
+subjects:
+  - kind: ServiceAccount
+    name: doris-operator
+    namespace: doriscluster
+---
+# Source: doris-operator/templates/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: doris-operator
+  namespace: doriscluster
+  labels:
+    control-plane: doris-operator
+    app.kubernetes.io/name: deployment
+    app.kubernetes.io/instance: doris-operator
+    app.kubernetes.io/component: doris-operator
+    app.kubernetes.io/created-by: doris-operator
+    app.kubernetes.io/part-of: doris-operator
+spec:
+  selector:
+    matchLabels:
+      control-plane: doris-operator
+  replicas: 1
+  template:
+    metadata:
+      annotations:
+        kubectl.kubernetes.io/default-container: doris-operator
+      labels:
+        control-plane: doris-operator
+    spec:
+      # TODO(user): Uncomment the following code to configure the nodeAffinity expression
+      # according to the platforms which are supported by your solution.
+      # It is considered best practice to support multiple architectures. You can
+      # build your manager image using the makefile target docker-buildx.
+      # affinity:
+      #   nodeAffinity:
+      #     requiredDuringSchedulingIgnoredDuringExecution:
+      #       nodeSelectorTerms:
+      #         - matchExpressions:
+      #           - key: kubernetes.io/arch
+      #             operator: In
+      #             values:
+      #               - amd64
+      #               - arm64
+      #               - ppc64le
+      #               - s390x
+      #           - key: kubernetes.io/os
+      #             operator: In
+      #             values:
+      #               - linux
+      securityContext:
+        runAsNonRoot: true
+        # TODO(user): For common cases that do not require escalating privileges
+        # it is recommended to ensure that all your Pods/Containers are restrictive.
+        # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted
+        # Please uncomment the following code if your project does NOT have to work on old Kubernetes
+        # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ).
+        # seccompProfile:
+        #   type: RuntimeDefault
+      containers:
+        - command:
+            - /dorisoperator
+          args:
+            - --leader-elect
+          image: harbor.cdcyy.com.cn/cmii/doris.k8s-operator:1.3.1
+          name: dorisoperator
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - "ALL"
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 8081
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: 8081
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          # TODO(user): Configure the resources accordingly based on the project requirements.
+          # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+          resources:
+            requests:
+              cpu: 2
+              memory: 4Gi
+            limits:
+              cpu: 2
+              memory: 4Gi
+      serviceAccountName: doris-operator
+      terminationGracePeriodSeconds: 10
+