1# Default values for kuberay-operator.
2# This is a YAML-formatted file.
3# Declare variables to be passed into your templates.
5# -- String to partially override release name.
6nameOverride: kuberay-operator
7# -- String to fully override release name.
8fullnameOverride: kuberay-operator
9# -- String to override component name.
10componentOverride: kuberay-operator
11# -- Number of replicas for the KubeRay operator Deployment.
14 # -- Image repository.
15 repository: cgr.dev/scratch-images/test-tmp/kuberay-operator-fips
17 tag: 1.7.0-r4@sha256:4b6605316c6bcb2a843901692247c001470bdb2eedafd286bfc17023d3448aea
18 # -- Image pull policy.
19 pullPolicy: IfNotPresent
20# -- Secrets with credentials to pull images from a private registry
22# -- Restrict to run on particular nodes.
24# -- Pod priorityClassName
28# -- Extra annotations.
35 # -- Specifies whether a service account should be created.
37 # -- The name of the service account to use.
38 # If not set and create is true, a name is generated using the fullname template.
39 name: kuberay-operator
41 # -- Log encoder to use for stdout (one of `json` or `console`).
43 # -- Log encoder to use for file logging (one of `json` or `console`).
45 # -- Directory for kuberay-operator log file.
47 # -- File name for kuberay-operator log file.
49 # -- EmptyDir volume size limit for kuberay-operator log file.
51# Enable customized Kubernetes scheduler integration. If enabled, Ray workloads will be scheduled
52# by the customized scheduler.
53# * "enabled" is the legacy option and will be deprecated soon.
54# * "name" is the standard option, expecting a scheduler name, supported values are
55# "default", "volcano", "yunikorn", and "scheduler-plugins".
57# Note: "enabled" and "name" should not be set at the same time. If both are set, an error will be thrown.
60# 1. Use volcano (deprecated)
74# name: scheduler-plugins
80 # Deprecated. This option will be removed in the future.
81 # Note, for backwards compatibility. When it sets to true, it enables volcano scheduler integration.
83 # Set the customized scheduler name, supported values are "volcano", "yunikorn", "kai-scheduler" or "scheduler-plugins", do not set
84 # "batchScheduler.enabled=true" at the same time as it will override this option.
86# Configuration for the KubeRay operator.
88 # -- Whether to enable the configuration feature. If enabled, a ConfigMap will be created and mounted to the operator.
89 # When enabled, flag-based configuration values (leaderElectionEnabled, metrics.enabled, kubeClient.qps, etc.)
90 # will be injected into the ConfigMap. The operator will use the ConfigMap and ignore command-line flags.
92 # -- Default environment variables to inject into all Ray containers in all RayCluster CRs.
93 # This allows user to set feature flags across all Ray pods.
95 # defaultContainerEnvs:
96 # - name: RAY_enable_open_telemetry
98 # - name: RAY_metric_cardinality_level
99 # value: "recommended"
100 defaultContainerEnvs: []
101 # -- Default annotations to add to all Ray pod templates.
102 # User-specified annotations take precedence.
104 # defaultPodAnnotations:
105 # prometheus.io/scrape: "true"
106 # prometheus.io/port: "8080"
107 defaultPodAnnotations: {}
108 # -- Default labels to add to all Ray pod templates.
109 # User-specified labels take precedence.
112 # app.kubernetes.io/managed-by: kuberay
114 # -- Sidecar containers to inject into every Ray head pod.
116 # headSidecarContainers:
118 # image: fluent/fluent-bit:1.9
119 headSidecarContainers: []
120 # -- Sidecar containers to inject into every Ray worker pod.
122 # workerSidecarContainers:
124 # image: fluent/fluent-bit:1.9
125 workerSidecarContainers: []
127 - name: RayClusterStatusConditions
129 - name: RayJobDeletionPolicy
131 - name: RayMultiHostIndexing
133 - name: RayServiceIncrementalUpgrade
137 - name: RayClusterMTLS
139 - name: RayClusterNetworkPolicy
141 - name: RayClusterHistoryServer
143# Configurations for KubeRay operator metrics.
145 # -- Whether KubeRay operator should emit control plane metrics.
148 # -- Enable a prometheus ServiceMonitor
150 # -- Prometheus ServiceMonitor interval
152 # -- When true, honorLabels preserves the metric’s labels when they collide with the target’s labels.
154 # -- Additional labels to add to the ServiceMonitor's metadata.
156 # release: prometheus
157 # -- Prometheus ServiceMonitor namespace
158 namespace: "" # "monitoring"
159# -- Path to the operator binary
160operatorCommand: /manager
161# if userKubernetesProxy is set to true, the KubeRay operator will be configured with the --use-kubernetes-proxy flag.
162# Using this option to configure kuberay-operator to comunitcate to Ray head pods by proxying through the Kubernetes API Server.
163# useKubernetesProxy: true
165# -- If leaderElectionEnabled is set to true, the KubeRay operator will use leader election for high availability.
166leaderElectionEnabled: true
167# -- The maximum number of reconcile operations that can be performed simultaneously.
168# This setting controls the concurrency of the controller reconciliation loops.
169# Higher values can improve throughput in clusters with many resources, but may increase resource consumption.
170reconcileConcurrency: 1
171# -- Kube Client configuration for QPS and burst settings.
172# This setting controls the QPS and burst rate of the kube client when sending requests to the Kubernetes API server.
173# If the QPS and burst values are too low, we may easily hit rate limits on the API server and slow down the controller reconciliation loops.
175 # -- The QPS value for the client communicating with the Kubernetes API server.
176 # Must be a float number.
178 # -- The maximum burst for throttling requests from this client to the Kubernetes API server.
179 # Must be a non-negative integer.
181# -- If rbacEnable is set to false, no RBAC resources will be created, including the Role for leader election, the Role for Pods and Services, and so on.
183# -- When crNamespacedRbacEnable is set to true, the KubeRay operator will create a Role for RayCluster preparation (e.g., Pods, Services)
184# and a corresponding RoleBinding for each namespace listed in the "watchNamespace" parameter. Please note that even if crNamespacedRbacEnable
185# is set to false, the Role and RoleBinding for leader election will still be created.
188# (1) This variable is only effective when rbacEnable and singleNamespaceInstall are both set to true.
189# (2) In most cases, it should be set to true, unless you are using a Kubernetes cluster managed by GitOps tools such as ArgoCD.
190crNamespacedRbacEnable: true
191# -- When singleNamespaceInstall is true:
192# - Install namespaced RBAC resources such as Role and RoleBinding instead of cluster-scoped ones like ClusterRole and ClusterRoleBinding so that
193# the chart can be installed by users with permissions restricted to a single namespace.
194# (Please note that this excludes the CRDs, which can only be installed at the cluster scope.)
195# - If "watchNamespace" is not set, the KubeRay operator will, by default, only listen
196# to resource events within its own namespace.
197singleNamespaceInstall: false
198# The KubeRay operator will watch the custom resources in the namespaces listed in the "watchNamespace" parameter.
203# -- Environment variables.
205# If not set or set to true, kuberay auto injects an init container waiting for ray GCS.
206# If false, you will need to inject your own init container to ensure ray GCS is up before the ray workers start.
207# Warning: we highly recommend setting to true and let kuberay handle for you.
208# - name: ENABLE_INIT_CONTAINER_INJECTION
210# If set to true, kuberay creates a normal ClusterIP service for a Ray Head instead of a Headless service. Default to false.
211# - name: ENABLE_RAY_HEAD_CLUSTER_IP_SERVICE
213# If not set or set to "", kuberay will pick up the default k8s cluster domain `cluster.local`
214# Otherwise, kuberay will use your custom domain
215# - name: CLUSTER_DOMAIN
217# If not set or set to false, when running on OpenShift with Ingress creation enabled, kuberay will create OpenShift route
218# Otherwise, regardless of the type of cluster with Ingress creation enabled, kuberay will create Ingress
219# - name: USE_INGRESS_ON_OPENSHIFT
221# Unconditionally requeue after the number of seconds specified in the
222# environment variable RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV. If the
223# environment variable is not set, requeue after the default value (300).
224# - name: RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV
226# If not set or set to "true", KubeRay will clean up the Redis storage namespace when a GCS FT-enabled RayCluster is deleted.
227# - name: ENABLE_GCS_FT_REDIS_CLEANUP
229# For LLM serving, some users might not have sufficient GPU resources to run two RayClusters simultaneously.
230# Therefore, KubeRay offers ENABLE_ZERO_DOWNTIME as a feature flag for zero-downtime upgrades.
231# - name: ENABLE_ZERO_DOWNTIME
233# This environment variable for the KubeRay operator is used to determine whether to enable
234# the injection of readiness and liveness probes into Ray head and worker containers.
235# Enabling this feature contributes to the robustness of Ray clusters.
236# - name: ENABLE_PROBES_INJECTION
238# If set to true, the RayJob CR itself will be deleted if shutdownAfterJobFinishes is set to true. Note that all resources created by the RayJob CR will be deleted, including the K8s Job. Otherwise, only the RayCluster CR will be deleted. Default is false.
239# - name: DELETE_RAYJOB_CR_AFTER_JOB_FINISHES
241# If set to true, we will use deterministic name for head pod. Otherwise, the non-deterministic name is used.
242# - name: ENABLE_DETERMINISTIC_HEAD_POD_NAME
244# This environment variable determines whether to enable a login shell by passing the -l option to the container command /bin/bash.
245# The -l flag was added by default before KubeRay v1.4.0, but it is no longer added by default starting with v1.4.0.
246# - name: ENABLE_LOGIN_SHELL
248# This KubeRay operator environment variable is used to determine if random Pod
249# deletion should be enabled. Note that this only takes effect when autoscaling
250# is enabled for the RayCluster.
251# - name: ENABLE_RANDOM_POD_DELETE
253# If JobDeploymentStatus does not transition to Complete or Failed within
254# this grace period seconds after JobStatus reaches a terminal state,
255# KubeRay will update JobDeploymentStatus directly.
256# - name: RAYJOB_DEPLOYMENT_STATUS_TRANSITION_GRACE_PERIOD_SECONDS
258# If job status checks keep failing for longer than this timeout in seconds,
259# KubeRay will transition the RayJob's JobDeploymentStatus to Failed. Default is 300.
260# - name: RAYJOB_STATUS_CHECK_TIMEOUT_SECONDS
263# -- Resource requests and limits for containers.
267 # Anecdotally, managing 500 Ray pods requires roughly 500MB memory.
268 # Monitor memory usage and adjust as needed.
273# @Ignore -- Pod liveness probe configuration.
275 initialDelaySeconds: 10
278# @Ignore -- Pod readiness probe configuration.
280 initialDelaySeconds: 10
283# -- Set up `securityContext` to improve Pod security.
284podSecurityContext: {}
285# @ignore -- Set up `securityContext` to improve container security.
287 allowPrivilegeEscalation: false
288 readOnlyRootFilesystem: true