From dd725ca41377912be5cce2f0b37e53b00b9fa287 Mon Sep 17 00:00:00 2001 From: doubledna Date: Mon, 13 Oct 2025 10:39:18 +0800 Subject: [PATCH] feat:support external s3 storage aksk uses k8s secret storage Signed-off-by: doubledna --- charts/milvus/Chart.yaml | 2 +- charts/milvus/README.md | 11 +++++++++++ charts/milvus/templates/datacoord-deployment.yaml | 12 ++++++++++++ charts/milvus/templates/datanode-deployment.yaml | 12 ++++++++++++ charts/milvus/templates/indexcoord-deployment.yaml | 12 ++++++++++++ charts/milvus/templates/indexnode-deployment.yaml | 12 ++++++++++++ charts/milvus/templates/mixcoord-deployment.yaml | 12 ++++++++++++ charts/milvus/templates/proxy-deployment.yaml | 12 ++++++++++++ charts/milvus/templates/querycoord-deployment.yaml | 12 ++++++++++++ charts/milvus/templates/querynode-deployment.yaml | 12 ++++++++++++ charts/milvus/templates/rootcoord-deployment.yaml | 12 ++++++++++++ charts/milvus/templates/standalone-deployment.yaml | 12 ++++++++++++ .../milvus/templates/streamingnode-deployment.yaml | 12 ++++++++++++ charts/milvus/values.yaml | 7 ++++++- 14 files changed, 150 insertions(+), 2 deletions(-) diff --git a/charts/milvus/Chart.yaml b/charts/milvus/Chart.yaml index aed24605..dcbb577c 100644 --- a/charts/milvus/Chart.yaml +++ b/charts/milvus/Chart.yaml @@ -3,7 +3,7 @@ name: milvus appVersion: "2.6.3" kubeVersion: "^1.10.0-0" description: Milvus is an open-source vector database built to power AI applications and vector similarity search. -version: 5.0.4 +version: 5.0.5 keywords: - milvus - elastic diff --git a/charts/milvus/README.md b/charts/milvus/README.md index e5685aaa..402a70e7 100644 --- a/charts/milvus/README.md +++ b/charts/milvus/README.md @@ -120,6 +120,17 @@ $ helm upgrade --install my-release zilliztech/milvus --set pulsarv3.enabled=fal By default, milvus cluster uses `mixCoordinator` which contains all coordinators. This is the recommended deployment approach for Milvus v2.6.x and later versions. +### Use external S3 as object storage and store aksk in secret +1. Create a secret with name `s3-credentials` or use existing aksk secret +```bash +$ kubectl create secret generic s3-credentials --from-literal=accessKey=YOUR_ACCESS_KEY --from-literal=secretKey=YOUR_SECRET_KEY +``` +2. Helm upgrade with this secret +```bash +# Helm v3.x +$ helm upgrade --install my-release --set cluster.enabled=false --set etcd.replicaCount=1 --set pulsarv3.enabled=false --set minio.enabled=false --set externalS3.enabled=true --set externalS3.existingSecret.enabled=true --set externalS3.existingSecret.name=s3-credentials --set externalS3.existingSecret.accessKey=accessKey --set externalS3.existingSecret.secretKey=secretKey --set externalS3.host=YOUR_S3_HOST --set externalS3.port=YOUR_S3_PORT --set externalS3.bucketName=milvus-bucket zilliztech/milvus +``` + ### Upgrading from Milvus v2.5.x to v2.6.x Upgrading from Milvus 2.5.x to 2.6.x involves significant architectural changes. Please follow these steps carefully: diff --git a/charts/milvus/templates/datacoord-deployment.yaml b/charts/milvus/templates/datacoord-deployment.yaml index dde916ca..fbeb8ec2 100644 --- a/charts/milvus/templates/datacoord-deployment.yaml +++ b/charts/milvus/templates/datacoord-deployment.yaml @@ -95,6 +95,18 @@ spec: {{- if .Values.dataCoordinator.extraEnv }} {{- toYaml .Values.dataCoordinator.extraEnv | nindent 8 }} {{- end }} + {{- if and .Values.externalS3.enabled .Values.externalS3.existingSecret.enabled }} + - name: MINIO_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.accessKey }} + - name: MINIO_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.secretKey }} + {{- end }} {{ if and (.Values.containerSecurityContext) (not .Values.dataCoordinator.containerSecurityContext) }} securityContext: {{- toYaml .Values.containerSecurityContext | nindent 12 }} diff --git a/charts/milvus/templates/datanode-deployment.yaml b/charts/milvus/templates/datanode-deployment.yaml index 1c381849..9df4b836 100644 --- a/charts/milvus/templates/datanode-deployment.yaml +++ b/charts/milvus/templates/datanode-deployment.yaml @@ -95,6 +95,18 @@ spec: {{- if .Values.dataNode.extraEnv }} {{- toYaml .Values.dataNode.extraEnv | nindent 8 }} {{- end }} + {{- if and .Values.externalS3.enabled .Values.externalS3.existingSecret.enabled }} + - name: MINIO_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.accessKey }} + - name: MINIO_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.secretKey }} + {{- end }} ports: - name: datanode containerPort: 21124 diff --git a/charts/milvus/templates/indexcoord-deployment.yaml b/charts/milvus/templates/indexcoord-deployment.yaml index 23692699..7000e43b 100644 --- a/charts/milvus/templates/indexcoord-deployment.yaml +++ b/charts/milvus/templates/indexcoord-deployment.yaml @@ -103,6 +103,18 @@ spec: {{- if .Values.indexCoordinator.extraEnv }} {{- toYaml .Values.indexCoordinator.extraEnv | nindent 8 }} {{- end }} + {{- if and .Values.externalS3.enabled .Values.externalS3.existingSecret.enabled }} + - name: MINIO_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.accessKey }} + - name: MINIO_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.secretKey }} + {{- end }} ports: - name: indexcoord containerPort: 31000 diff --git a/charts/milvus/templates/indexnode-deployment.yaml b/charts/milvus/templates/indexnode-deployment.yaml index 8adc19b2..a1b48d7d 100644 --- a/charts/milvus/templates/indexnode-deployment.yaml +++ b/charts/milvus/templates/indexnode-deployment.yaml @@ -108,6 +108,18 @@ spec: {{- if .Values.indexNode.extraEnv }} {{- toYaml .Values.indexNode.extraEnv | nindent 8 }} {{- end }} + {{- if and .Values.externalS3.enabled .Values.externalS3.existingSecret.enabled }} + - name: MINIO_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecretKeys.name }} + key: {{ .Values.externalS3.existingSecretKeys.accessKey }} + - name: MINIO_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecretKeys.name }} + key: {{ .Values.externalS3.existingSecretKeys.secretKey }} + {{- end }} ports: - name: indexnode containerPort: 21121 diff --git a/charts/milvus/templates/mixcoord-deployment.yaml b/charts/milvus/templates/mixcoord-deployment.yaml index d39a7509..91827fa5 100644 --- a/charts/milvus/templates/mixcoord-deployment.yaml +++ b/charts/milvus/templates/mixcoord-deployment.yaml @@ -99,6 +99,18 @@ spec: {{- if .Values.mixCoordinator.extraEnv }} {{- toYaml .Values.mixCoordinator.extraEnv | nindent 8 }} {{- end }} + {{- if and .Values.externalS3.enabled .Values.externalS3.existingSecret.enabled }} + - name: MINIO_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.accessKey }} + - name: MINIO_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.secretKey }} + {{- end }} ports: - name: metrics containerPort: 9091 diff --git a/charts/milvus/templates/proxy-deployment.yaml b/charts/milvus/templates/proxy-deployment.yaml index 20f8ae9b..d7d3cec5 100644 --- a/charts/milvus/templates/proxy-deployment.yaml +++ b/charts/milvus/templates/proxy-deployment.yaml @@ -95,6 +95,18 @@ spec: {{- if .Values.proxy.extraEnv }} {{- toYaml .Values.proxy.extraEnv | nindent 8 }} {{- end }} + {{- if and .Values.externalS3.enabled .Values.externalS3.existingSecret.enabled }} + - name: MINIO_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.accessKey }} + - name: MINIO_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.secretKey }} + {{- end }} ports: - name: milvus containerPort: 19530 diff --git a/charts/milvus/templates/querycoord-deployment.yaml b/charts/milvus/templates/querycoord-deployment.yaml index 10572d5f..b3ae09a0 100644 --- a/charts/milvus/templates/querycoord-deployment.yaml +++ b/charts/milvus/templates/querycoord-deployment.yaml @@ -103,6 +103,18 @@ spec: {{- if .Values.queryCoordinator.extraEnv }} {{- toYaml .Values.queryCoordinator.extraEnv | nindent 8 }} {{- end }} + {{- if and .Values.externalS3.enabled .Values.externalS3.existingSecret.enabled }} + - name: MINIO_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.accessKey }} + - name: MINIO_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.secretKey }} + {{- end }} ports: - name: querycoord containerPort: 19531 diff --git a/charts/milvus/templates/querynode-deployment.yaml b/charts/milvus/templates/querynode-deployment.yaml index 15715866..5b52b701 100644 --- a/charts/milvus/templates/querynode-deployment.yaml +++ b/charts/milvus/templates/querynode-deployment.yaml @@ -104,6 +104,18 @@ spec: {{- if .Values.queryNode.extraEnv }} {{- toYaml .Values.queryNode.extraEnv | nindent 8 }} {{- end }} + {{- if and .Values.externalS3.enabled .Values.externalS3.existingSecret.enabled }} + - name: MINIO_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.accessKey }} + - name: MINIO_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.secretKey }} + {{- end }} ports: - name: querynode containerPort: 21123 diff --git a/charts/milvus/templates/rootcoord-deployment.yaml b/charts/milvus/templates/rootcoord-deployment.yaml index 4860163f..19f5ddd4 100644 --- a/charts/milvus/templates/rootcoord-deployment.yaml +++ b/charts/milvus/templates/rootcoord-deployment.yaml @@ -103,6 +103,18 @@ spec: {{- if .Values.rootCoordinator.extraEnv }} {{- toYaml .Values.rootCoordinator.extraEnv | nindent 8 }} {{- end }} + {{- if and .Values.externalS3.enabled .Values.externalS3.existingSecret.enabled }} + - name: MINIO_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.accessKey }} + - name: MINIO_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.secretKey }} + {{- end }} ports: - name: rootcoord containerPort: 53100 diff --git a/charts/milvus/templates/standalone-deployment.yaml b/charts/milvus/templates/standalone-deployment.yaml index 4109457a..c58b0e77 100644 --- a/charts/milvus/templates/standalone-deployment.yaml +++ b/charts/milvus/templates/standalone-deployment.yaml @@ -125,6 +125,18 @@ spec: {{- if .Values.standalone.extraEnv }} {{- toYaml .Values.standalone.extraEnv | nindent 8 }} {{- end }} + {{- if and .Values.externalS3.enabled .Values.externalS3.existingSecret.enabled }} + - name: MINIO_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.accessKey }} + - name: MINIO_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.secretKey }} + {{- end }} volumeMounts: - mountPath: /milvus/tools name: tools diff --git a/charts/milvus/templates/streamingnode-deployment.yaml b/charts/milvus/templates/streamingnode-deployment.yaml index d7d241b5..08a0639a 100644 --- a/charts/milvus/templates/streamingnode-deployment.yaml +++ b/charts/milvus/templates/streamingnode-deployment.yaml @@ -92,6 +92,18 @@ spec: {{- if .Values.streamingNode.extraEnv }} {{- toYaml .Values.streamingNode.extraEnv | nindent 8 }} {{- end }} + {{- if and .Values.externalS3.enabled .Values.externalS3.existingSecret.enabled }} + - name: MINIO_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.accessKey }} + - name: MINIO_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.externalS3.existingSecret.name }} + key: {{ .Values.externalS3.existingSecret.secretKey }} + {{- end }} ports: - name: streamingnode containerPort: 22222 diff --git a/charts/milvus/values.yaml b/charts/milvus/values.yaml index 66d15f9d..720e50bf 100644 --- a/charts/milvus/values.yaml +++ b/charts/milvus/values.yaml @@ -1289,7 +1289,12 @@ externalS3: host: "" port: "" accessKey: "" - secretKey: "" + secretKey: "" + existingSecret: + enabled: false + name: "s3-credentials" + accessKey: "accessKey" + secretKey: "secretKey" useSSL: false bucketName: "" rootPath: ""