Milvus集群部署

#本次部署其中的组件kafka、minio均为外置

将milvus chart包解压拷贝至某个目录下

编辑helm的values.yaml文件

## Enable or disable Milvus Cluster mode
cluster:enabled: trueimage:all:repository: milvusdb/milvustag: v2.4.1pullPolicy: IfNotPresent## Optionally specify an array of imagePullSecrets.## Secrets must be manually created in the namespace.## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/### pullSecrets:#   - myRegistryKeySecretNametools:repository: milvusdb/milvus-config-tooltag: v0.1.2pullPolicy: IfNotPresent# Global node selector
# If set, this will apply to all milvus components
# Individual components can be set to a different node selector
nodeSelector: {}# Global tolerations
# If set, this will apply to all milvus components
# Individual components can be set to a different tolerations
tolerations: []# Global affinity
# If set, this will apply to all milvus components
# Individual components can be set to a different affinity
affinity: {}# Global labels and annotations
# If set, this will apply to all milvus components
labels: {}
annotations: {}# Extra configs for milvus.yaml
# If set, this config will merge into milvus.yaml
# Please follow the config structure in the milvus.yaml
# at https://github.com/milvus-io/milvus/blob/master/configs/milvus.yaml
# Note: this config will be the top priority which will override the config
# in the image and helm chart.
extraConfigFiles:user.yaml: |+#    For example enable rest http for milvus proxy#    proxy:#      http:#        enabled: true#      maxUserNum: 100#      maxRoleNum: 10##  Enable tlsMode and set the tls cert and key#  tls:#    serverPemPath: /etc/milvus/certs/tls.crt#    serverKeyPath: /etc/milvus/certs/tls.key#   common:#     security:#       tlsMode: 1## Expose the Milvus service to be accessed from outside the cluster (LoadBalancer service).
## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it.
## ref: http://kubernetes.io/docs/user-guide/services/
##
service:type: ClusterIPport: 19530portName: milvusnodePort: ""annotations: {}labels: {}## List of IP addresses at which the Milvus service is available## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips##externalIPs: []#   - externalIp1# LoadBalancerSourcesRange is a list of allowed CIDR values, which are combined with ServicePort to# set allowed inbound rules on the security group assigned to the master load balancerloadBalancerSourceRanges:- 0.0.0.0/0# Optionally assign a known public LB IP# loadBalancerIP: 1.2.3.4ingress:enabled: falseannotations:# Annotation example: set nginx ingress type# kubernetes.io/ingress.class: nginxnginx.ingress.kubernetes.io/backend-protocol: GRPCnginx.ingress.kubernetes.io/listen-ports-ssl: '[19530]'nginx.ingress.kubernetes.io/proxy-body-size: 4mnginx.ingress.kubernetes.io/ssl-redirect: "true"labels: {}rules:- host: "milvus-example.local"path: "/"pathType: "Prefix"# - host: "milvus-example2.local"#   path: "/otherpath"#   pathType: "Prefix"tls: []#  - secretName: chart-example-tls#    hosts:#      - milvus-example.localserviceAccount:create: falsename:annotations:labels:metrics:enabled: trueserviceMonitor:# Set this to `true` to create ServiceMonitor for Prometheus operatorenabled: falseinterval: "30s"scrapeTimeout: "10s"# Additional labels that can be used so ServiceMonitor will be discovered by PrometheusadditionalLabels: {}livenessProbe:enabled: trueinitialDelaySeconds: 90periodSeconds: 30timeoutSeconds: 5successThreshold: 1failureThreshold: 5readinessProbe:enabled: trueinitialDelaySeconds: 90periodSeconds: 10timeoutSeconds: 5successThreshold: 1failureThreshold: 5log:level: "info"file:maxSize: 300    # MBmaxAge: 10    # daymaxBackups: 20format: "text"    # text/jsonpersistence:mountPath: "/milvus/logs"## If true, create/use a Persistent Volume Claim## If false, use emptyDir##enabled: trueannotations:helm.sh/resource-policy: keeppersistentVolumeClaim:existingClaim: ""## Milvus Logs Persistent Volume Storage Class## If defined, storageClassName: <storageClass>## If set to "-", storageClassName: "", which disables dynamic provisioning## If undefined (the default) or set to null, no storageClassName spec is##   set, choosing the default provisioner.## ReadWriteMany access mode required for milvus cluster.##storageClass: nfs-clientaccessModes: ReadWriteManysize: 20GisubPath: ""## Heaptrack traces all memory allocations and annotates these events with stack traces.
## See more: https://github.com/KDE/heaptrack
## Enable heaptrack in production is not recommended.
heaptrack:image:repository: milvusdb/heaptracktag: v0.1.0pullPolicy: IfNotPresentstandalone:replicas: 1  # Run standalone mode with replication disabledresources: {}# Set local storage size in resources# resources:#   limits:#     ephemeral-storage: 100GinodeSelector: {}affinity: {}tolerations: []extraEnv: []heaptrack:enabled: falsedisk:enabled: truesize:enabled: false  # Enable local storage size limitprofiling:enabled: false  # Enable live profiling## Default message queue for milvus standalone## Supported value: rocksmq, natsmq, pulsar and kafkamessageQueue: rocksmqpersistence:mountPath: "/var/lib/milvus"## If true, alertmanager will create/use a Persistent Volume Claim## If false, use emptyDir##enabled: trueannotations:helm.sh/resource-policy: keeppersistentVolumeClaim:existingClaim: ""## Milvus Persistent Volume Storage Class## If defined, storageClassName: <storageClass>## If set to "-", storageClassName: "", which disables dynamic provisioning## If undefined (the default) or set to null, no storageClassName spec is##   set, choosing the default provisioner.##storageClass:accessModes: ReadWriteOncesize: 50GisubPath: ""proxy:enabled: true# You can set the number of replicas to -1 to remove the replicas field in case you want to use HPAreplicas: 1resources: {}nodeSelector: {}affinity: {}tolerations: []extraEnv: []heaptrack:enabled: falseprofiling:enabled: false  # Enable live profilinghttp:enabled: true  # whether to enable http rest serverdebugMode:enabled: false# Mount a TLS secret into proxy podtls:enabled: false
## when enabling proxy.tls, all items below should be uncommented and the key and crt values should be populated.
#    enabled: true
#    secretName: milvus-tls
## expecting base64 encoded values here: i.e. $(cat tls.crt | base64 -w 0) and $(cat tls.key | base64 -w 0)
#    key: LS0tLS1CRUdJTiBQU--REDUCT
#    crt: LS0tLS1CRUdJTiBDR--REDUCT
#  volumes:
#  - secret:
#      secretName: milvus-tls
#    name: milvus-tls
#  volumeMounts:
#  - mountPath: /etc/milvus/certs/
#    name: milvus-tls# Deployment strategy, default is RollingUpdate# Ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#rolling-update-deploymentstrategy: {}rootCoordinator:enabled: true# You can set the number of replicas greater than 1, only if enable active standbyreplicas: 1  # Run Root Coordinator mode with replication disabledresources: {}nodeSelector: {}affinity: {}tolerations: []extraEnv: []heaptrack:enabled: falseprofiling:enabled: false  # Enable live profilingactiveStandby:enabled: false  # Enable active-standby when you set multiple replicas for root coordinator# Deployment strategy, default is RollingUpdate# Ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#rolling-update-deploymentstrategy: {}service:port: 53100annotations: {}labels: {}clusterIP: ""queryCoordinator:enabled: true# You can set the number of replicas greater than 1, only if enable active standbyreplicas: 1  # Run Query Coordinator mode with replication disabledresources: {}nodeSelector: {}affinity: {}tolerations: []extraEnv: []heaptrack:enabled: falseprofiling:enabled: false  # Enable live profilingactiveStandby:enabled: false  # Enable active-standby when you set multiple replicas for query coordinator# Deployment strategy, default is RollingUpdate# Ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#rolling-update-deploymentstrategy: {}service:port: 19531annotations: {}labels: {}clusterIP: ""queryNode:enabled: true# You can set the number of replicas to -1 to remove the replicas field in case you want to use HPAreplicas: 1resources: {}# Set local storage size in resources# resources:#   limits:#     ephemeral-storage: 100GinodeSelector: {}affinity: {}tolerations: []extraEnv: []heaptrack:enabled: falsedisk:enabled: true  # Enable querynode load disk index, and search on disk indexsize:enabled: false  # Enable local storage size limitprofiling:enabled: false  # Enable live profiling# Deployment strategy, default is RollingUpdate# Ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#rolling-update-deploymentstrategy: {}indexCoordinator:enabled: true# You can set the number of replicas greater than 1, only if enable active standbyreplicas: 1   # Run Index Coordinator mode with replication disabledresources: {}nodeSelector: {}affinity: {}tolerations: []extraEnv: []heaptrack:enabled: falseprofiling:enabled: false  # Enable live profilingactiveStandby:enabled: false  # Enable active-standby when you set multiple replicas for index coordinator# Deployment strategy, default is RollingUpdate# Ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#rolling-update-deploymentstrategy: {}service:port: 31000annotations: {}labels: {}clusterIP: ""indexNode:enabled: true# You can set the number of replicas to -1 to remove the replicas field in case you want to use HPAreplicas: 1resources: {}# Set local storage size in resources# limits:#    ephemeral-storage: 100GinodeSelector: {}affinity: {}tolerations: []extraEnv: []heaptrack:enabled: falseprofiling:enabled: false  # Enable live profilingdisk:enabled: true  # Enable index node build disk vector indexsize:enabled: false  # Enable local storage size limit# Deployment strategy, default is RollingUpdate# Ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#rolling-update-deploymentstrategy: {}dataCoordinator:enabled: true# You can set the number of replicas greater than 1, only if enable active standbyreplicas: 1           # Run Data Coordinator mode with replication disabledresources: {}nodeSelector: {}affinity: {}tolerations: []extraEnv: []heaptrack:enabled: falseprofiling:enabled: false  # Enable live profilingactiveStandby:enabled: false  # Enable active-standby when you set multiple replicas for data coordinator# Deployment strategy, default is RollingUpdate# Ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#rolling-update-deploymentstrategy: {}service:port: 13333annotations: {}labels: {}clusterIP: ""dataNode:enabled: true# You can set the number of replicas to -1 to remove the replicas field in case you want to use HPAreplicas: 1resources: {}nodeSelector: {}affinity: {}tolerations: []extraEnv: []heaptrack:enabled: falseprofiling:enabled: false  # Enable live profiling# Deployment strategy, default is RollingUpdate# Ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#rolling-update-deploymentstrategy: {}## mixCoordinator contains all coord
## If you want to use mixcoord, enable this and disable all of other coords 
mixCoordinator:enabled: false# You can set the number of replicas greater than 1, only if enable active standbyreplicas: 1           # Run Mixture Coordinator mode with replication disabledresources: {}nodeSelector: {}affinity: {}tolerations: []extraEnv: []heaptrack:enabled: falseprofiling:enabled: false  # Enable live profilingactiveStandby:enabled: false  # Enable active-standby when you set multiple replicas for Mixture coordinator# Deployment strategy, default is RollingUpdate# Ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#rolling-update-deploymentstrategy: {}service:annotations: {}labels: {}clusterIP: ""attu:enabled: falsename: attuimage:repository: zilliz/attutag: v2.3.10pullPolicy: IfNotPresentservice:annotations: {}labels: {}type: ClusterIPport: 3000# loadBalancerIP: ""resources: {}podLabels: {}ingress:enabled: falseannotations: {}# Annotation example: set nginx ingress type# kubernetes.io/ingress.class: nginxlabels: {}hosts:- milvus-attu.localtls: []#  - secretName: chart-attu-tls#    hosts:#      - milvus-attu.local## Configuration values for the minio dependency
## ref: https://github.com/zilliztech/milvus-helm/blob/master/charts/minio/README.md
##minio:enabled: falsename: miniomode: distributedimage:tag: "RELEASE.2023-03-20T20-16-18Z"pullPolicy: IfNotPresentaccessKey: minioadminsecretKey: minioadminexistingSecret: ""bucketName: "milvus-bucket"rootPath: fileuseIAM: falseiamEndpoint: ""region: ""useVirtualHost: falsepodDisruptionBudget:enabled: falseresources:requests:memory: 2Giservice:type: ClusterIPport: 9000persistence:enabled: trueexistingClaim: ""storageClass:accessMode: ReadWriteOncesize: 500GilivenessProbe:enabled: trueinitialDelaySeconds: 5periodSeconds: 5timeoutSeconds: 5successThreshold: 1failureThreshold: 5readinessProbe:enabled: trueinitialDelaySeconds: 5periodSeconds: 5timeoutSeconds: 1successThreshold: 1failureThreshold: 5startupProbe:enabled: trueinitialDelaySeconds: 0periodSeconds: 10timeoutSeconds: 5successThreshold: 1failureThreshold: 60## Configuration values for the etcd dependency
## ref: https://artifacthub.io/packages/helm/bitnami/etcd
##etcd:enabled: truename: etcdreplicaCount: 3pdb:create: falseimage:repository: "milvusdb/etcd"tag: "3.5.5-r4"pullPolicy: IfNotPresentservice:type: ClusterIPport: 2379peerPort: 2380auth:rbac:enabled: falsepersistence:enabled: truestorageClass: nfs-clientaccessMode: ReadWriteOncesize: 20Gi## Change default timeout periods to mitigate zoobie probe processlivenessProbe:enabled: truetimeoutSeconds: 10readinessProbe:enabled: trueperiodSeconds: 20timeoutSeconds: 10## Enable auto compaction## compaction by every 1000 revision##autoCompactionMode: revisionautoCompactionRetention: "1000"## Increase default quota to 4G##extraEnvVars:- name: ETCD_QUOTA_BACKEND_BYTESvalue: "4294967296"- name: ETCD_HEARTBEAT_INTERVALvalue: "500"- name: ETCD_ELECTION_TIMEOUTvalue: "2500"## Configuration values for the pulsar dependency
## ref: https://github.com/apache/pulsar-helm-chart
##pulsar:enabled: falsename: pulsarfullnameOverride: ""persistence: truemaxMessageSize: "5242880"  # 5 * 1024 * 1024 Bytes, Maximum size of each message in pulsar.rbac:enabled: falsepsp: falselimit_to_namespace: trueaffinity:anti_affinity: false## enableAntiAffinity: nocomponents:zookeeper: truebookkeeper: true# bookkeeper - autorecoveryautorecovery: truebroker: truefunctions: falseproxy: truetoolset: falsepulsar_manager: falsemonitoring:prometheus: falsegrafana: falsenode_exporter: falsealert_manager: falseimages:broker:repository: apachepulsar/pulsarpullPolicy: IfNotPresenttag: 2.8.2autorecovery:repository: apachepulsar/pulsartag: 2.8.2pullPolicy: IfNotPresentzookeeper:repository: apachepulsar/pulsarpullPolicy: IfNotPresenttag: 2.8.2bookie:repository: apachepulsar/pulsarpullPolicy: IfNotPresenttag: 2.8.2proxy:repository: apachepulsar/pulsarpullPolicy: IfNotPresenttag: 2.8.2pulsar_manager:repository: apachepulsar/pulsar-managerpullPolicy: IfNotPresenttag: v0.1.0zookeeper:resources:requests:memory: 1024Micpu: 0.3configData:PULSAR_MEM: >-Xms1024m-Xmx1024mPULSAR_GC: >-Dcom.sun.management.jmxremote-Djute.maxbuffer=10485760-XX:+ParallelRefProcEnabled-XX:+UnlockExperimentalVMOptions-XX:+DoEscapeAnalysis-XX:+DisableExplicitGC-XX:+PerfDisableSharedMem-Dzookeeper.forceSync=nopdb:usePolicy: falsebookkeeper:replicaCount: 3volumes:journal:name: journalsize: 100Giledgers:name: ledgerssize: 200Giresources:requests:memory: 2048Micpu: 1configData:PULSAR_MEM: >-Xms4096m-Xmx4096m-XX:MaxDirectMemorySize=8192mPULSAR_GC: >-Dio.netty.leakDetectionLevel=disabled-Dio.netty.recycler.linkCapacity=1024-XX:+UseG1GC -XX:MaxGCPauseMillis=10-XX:+ParallelRefProcEnabled-XX:+UnlockExperimentalVMOptions-XX:+DoEscapeAnalysis-XX:ParallelGCThreads=32-XX:ConcGCThreads=32-XX:G1NewSizePercent=50-XX:+DisableExplicitGC-XX:-ResizePLAB-XX:+ExitOnOutOfMemoryError-XX:+PerfDisableSharedMem-XX:+PrintGCDetailsnettyMaxFrameSizeBytes: "104867840"pdb:usePolicy: falsebroker:component: brokerpodMonitor:enabled: falsereplicaCount: 1resources:requests:memory: 4096Micpu: 1.5configData:PULSAR_MEM: >-Xms4096m-Xmx4096m-XX:MaxDirectMemorySize=8192mPULSAR_GC: >-Dio.netty.leakDetectionLevel=disabled-Dio.netty.recycler.linkCapacity=1024-XX:+ParallelRefProcEnabled-XX:+UnlockExperimentalVMOptions-XX:+DoEscapeAnalysis-XX:ParallelGCThreads=32-XX:ConcGCThreads=32-XX:G1NewSizePercent=50-XX:+DisableExplicitGC-XX:-ResizePLAB-XX:+ExitOnOutOfMemoryErrormaxMessageSize: "104857600"defaultRetentionTimeInMinutes: "10080"defaultRetentionSizeInMB: "-1"backlogQuotaDefaultLimitGB: "8"ttlDurationDefaultInSeconds: "259200"subscriptionExpirationTimeMinutes: "3"backlogQuotaDefaultRetentionPolicy: producer_exceptionpdb:usePolicy: falseautorecovery:resources:requests:memory: 512Micpu: 1proxy:replicaCount: 1podMonitor:enabled: falseresources:requests:memory: 2048Micpu: 1service:type: ClusterIPports:pulsar: 6650configData:PULSAR_MEM: >-Xms2048m -Xmx2048mPULSAR_GC: >-XX:MaxDirectMemorySize=2048mhttpNumThreads: "100"pdb:usePolicy: falsepulsar_manager:service:type: ClusterIPpulsar_metadata:component: pulsar-initimage:# the image used for running `pulsar-cluster-initialize` jobrepository: apachepulsar/pulsartag: 2.8.2## Configuration values for the kafka dependency
## ref: https://artifacthub.io/packages/helm/bitnami/kafka
##kafka:enabled: falsename: kafkareplicaCount: 3image:repository: bitnami/kafkatag: 3.1.0-debian-10-r52## Increase graceful termination for kafka graceful shutdownterminationGracePeriodSeconds: "90"pdb:create: false## Enable startup probe to prevent pod restart during recoveringstartupProbe:enabled: true## Kafka Java Heap sizeheapOpts: "-Xmx4096m -Xms4096m"maxMessageBytes: _10485760defaultReplicationFactor: 3offsetsTopicReplicationFactor: 3## Only enable time based log retentionlogRetentionHours: 168logRetentionBytes: _-1extraEnvVars:- name: KAFKA_CFG_MAX_PARTITION_FETCH_BYTESvalue: "5242880"- name: KAFKA_CFG_MAX_REQUEST_SIZEvalue: "5242880"- name: KAFKA_CFG_REPLICA_FETCH_MAX_BYTESvalue: "10485760"- name: KAFKA_CFG_FETCH_MESSAGE_MAX_BYTESvalue: "5242880"- name: KAFKA_CFG_LOG_ROLL_HOURSvalue: "24"persistence:enabled: truestorageClass:accessMode: ReadWriteOncesize: 300Gimetrics:## Prometheus Kafka exporter: exposes complimentary metrics to JMX exporterkafka:enabled: falseimage:repository: bitnami/kafka-exportertag: 1.4.2-debian-10-r182## Prometheus JMX exporter: exposes the majority of Kafkas metricsjmx:enabled: falseimage:repository: bitnami/jmx-exportertag: 0.16.1-debian-10-r245## To enable serviceMonitor, you must enable either kafka exporter or jmx exporter.## And you can enable them bothserviceMonitor:enabled: falseservice:type: ClusterIPports:client: 9092zookeeper:enabled: truereplicaCount: 3###################################
# External S3
# - these configs are only used when `externalS3.enabled` is true
###################################
externalS3:enabled: truehost: "172.30.24.101"port: "32750"accessKey: "3UXCCIGhziX05FhwdmFH"secretKey: "ZwgDPPHN3HsEH8r1TtKvHPF5E2amNFiQXMjzGhOP"useSSL: falsebucketName: "milvus-test"rootPath: "file"useIAM: falseiamEndpoint: ""region: ""useVirtualHost: false###################################
# GCS Gateway
# - these configs are only used when `minio.gcsgateway.enabled` is true
###################################
externalGcs:bucketName: ""###################################
# External etcd
# - these configs are only used when `externalEtcd.enabled` is true
###################################
externalEtcd:enabled: false## the endpoints of the external etcd##endpoints:- localhost:2379###################################
# External pulsar
# - these configs are only used when `externalPulsar.enabled` is true
###################################
externalPulsar:enabled: falsehost: localhostport: 6650maxMessageSize: "5242880"  # 5 * 1024 * 1024 Bytes, Maximum size of each message in pulsar.tenant: publicnamespace: defaultauthPlugin: ""authParams: ""###################################
# External kafka
# - these configs are only used when `externalKafka.enabled` is true
# - note that the following are just examples, you should confirm the 
#   value of brokerList and mechanisms according to the actual external 
#   Kafka configuration. E.g. If you select the AWS MSK, the configuration
#   should look something like this:
#   externalKafka:
#     enabled: true
#     brokerList: "xxxx:9096"
#     securityProtocol: SASL_SSL
#     sasl:
#       mechanisms: SCRAM-SHA-512
#       password: "xxx"
#       username: "xxx"
###################################
externalKafka:enabled: truebrokerList: 172.30.24.90:9092,172.30.24.91:9092,172.30.24.92:9092securityProtocol: PLAINTEXTsasl:mechanisms: PLAINusername: ""password: ""

helm install milvus-test milvus -f milvus/values.yaml -n milvus-test

#若大部分组件及Root coord均未正常启动则从Root coord日志开始排查，一般为ETCD连接异常或Kafka连接异常导致