2# commonLabels -- set of labels that will be applied to all the resources for the operator
4# commonAnnotations -- set of annotations that will be applied to all the resources for the operator
7 # look details in `kubectl explain deployment.spec.strategy`
11 # crdHook.enabled -- enable automatic CRD installation/update via pre-install/pre-upgrade hooks
12 # when disabled, CRDs must be installed manually using kubectl apply
15 # crdHook.image.repository -- image repository for CRD installation job
16 repository: cgr.dev/chainguard-private/kubectl
17 # crdHook.image.tag -- image tag for CRD installation job
18 tag: latest-dev@sha256:0fd58c31bd76bc39b93d80d5a2d4c37ea0ae9c2becd3e384fb2a197b89d85656
19 # crdHook.image.pullPolicy -- image pull policy for CRD installation job
20 pullPolicy: IfNotPresent
21 # crdHook.imagePullSecrets -- image pull secrets for CRD installation job
22 # possible value format `[{"name":"your-secret-name"}]`,
23 # check `kubectl explain pod.spec.imagePullSecrets` for details
25 # crdHook.resources -- resource limits and requests for CRD installation job
33 # crdHook.nodeSelector -- node selector for CRD installation job
35 # crdHook.tolerations -- tolerations for CRD installation job
37 # crdHook.affinity -- affinity for CRD installation job
39 # crdHook.annotations -- additional annotations for CRD installation job
41 # crdHook.podAnnotations -- additional annotations for CRD installation job pod template
42 # useful to opt out of service mesh injection, e.g. `sidecar.istio.io/inject: "false"`
44 # crdHook.podSecurityContext -- pod-level security context for CRD installation job
45 # required by some admission policies (e.g. Kyverno `restrict-seccomp-strict`)
46 # check `kubectl explain pod.spec.securityContext` for details
47 podSecurityContext: {}
51 # type: RuntimeDefault
52 # crdHook.containerSecurityContext -- container security context for CRD installation job
53 # check `kubectl explain pod.spec.containers.securityContext` for details
54 containerSecurityContext: {}
55 # allowPrivilegeEscalation: false
61 # type: RuntimeDefault
64 # operator.image.registry -- optional image registry prefix (e.g. 1234567890.dkr.ecr.us-east-1.amazonaws.com)
66 # operator.image.repository -- image repository
67 repository: cgr.dev/chainguard-private/clickhouse-operator
68 # operator.image.tag -- image tag (chart's appVersion value will be used if not set)
69 tag: 0.27.3@sha256:0fc563bd8cdd38b1b4d3e539031110f34bf4508f7c0bfcc555bc11e469652b7a
70 # operator.image.pullPolicy -- image pull policy
71 pullPolicy: IfNotPresent
72 containerSecurityContext: {}
73 # operator.resources -- custom resource configuration, check `kubectl explain pod.spec.containers.resources` for details
82 # operator.priorityClassName -- priority class name for the clickhouse-operator deployment, check `kubectl explain pod.spec.priorityClassName` for details
85 # operator.env -- additional environment variables for the clickhouse-operator container in deployment
86 # possible format value `[{"name": "SAMPLE", "value": "text"}]`
88 # operator.livenessProbe -- optional liveness probe for the clickhouse-operator container
89 # check `kubectl explain pod.spec.containers.livenessProbe` for details
94 # initialDelaySeconds: 10
97 # operator.readinessProbe -- optional readiness probe for the clickhouse-operator container
98 # check `kubectl explain pod.spec.containers.readinessProbe` for details
103 # initialDelaySeconds: 5
109 # metrics.image.registry -- optional image registry prefix (e.g. 1234567890.dkr.ecr.us-east-1.amazonaws.com)
111 # metrics.image.repository -- image repository
112 repository: cgr.dev/chainguard-private/clickhouse-operator-metrics-exporter
113 # metrics.image.tag -- image tag (chart's appVersion value will be used if not set)
114 tag: 0.27.3@sha256:457c4e3612ae0804a0be50762111a3c909c8fc5079f4488fd8a5f42a6ba47846
115 # metrics.image.pullPolicy -- image pull policy
116 pullPolicy: IfNotPresent
117 containerSecurityContext: {}
118 # metrics.resources -- custom resource configuration
127 # metrics.env -- additional environment variables for the deployment of metrics-exporter containers
128 # possible format value `[{"name": "SAMPLE", "value": "text"}]`
130 # metrics.livenessProbe -- optional liveness probe for the metrics-exporter container
131 # check `kubectl explain pod.spec.containers.livenessProbe` for details
136 # initialDelaySeconds: 10
139 # metrics.readinessProbe -- optional readiness probe for the metrics-exporter container
140 # check `kubectl explain pod.spec.containers.readinessProbe` for details
145 # initialDelaySeconds: 5
148# imagePullSecrets -- image pull secret for private images in clickhouse-operator pod
149# possible value format `[{"name":"your-secret-name"}]`,
150# check `kubectl explain pod.spec.imagePullSecrets` for details
152# podLabels -- labels to add to the clickhouse-operator pod
154# podAnnotations -- annotations to add to the clickhouse-operator pod, check `kubectl explain pod.spec.annotations` for details
155# @default -- check the `values.yaml` file
157 prometheus.io/port: '8888'
158 prometheus.io/scrape: 'true'
159 clickhouse-operator-metrics/port: '9999'
160 clickhouse-operator-metrics/scrape: 'true'
161# watchNamespaces -- namespaces where the operator watches for ClickHouseInstallation resources.
162# Sets config.yaml watch.namespaces.include (the exclude list is not exposed here). If empty, the
163# operator watches only its own namespace (or all namespaces when running in kube-system).
164# Use [".*"] to watch all namespaces. Entries are regexps and are Helm-templated, so avoid a literal
165# "{{" in a namespace/regexp.
166# Example: watchNamespaces: ["clickhouse", "my-other-namespace"]
168# nameOverride -- override name of the chart
170# fullnameOverride -- full name of the chart.
173 # serviceAccount.create -- specifies whether a service account should be created
175 # serviceAccount.annotations -- annotations to add to the service account
177 # serviceAccount.name -- the name of the service account to use; if not set and create is true, a name is generated using the fullname template
180 # rbac.create -- specifies whether rbac resources should be created
182 # rbac.namespaceScoped -- specifies whether to create roles and rolebindings at the cluster level or namespace level
183 namespaceScoped: false
185 # secret.create -- create a secret with operator credentials
187 # secret.username -- operator credentials username
188 username: clickhouse_operator
189 # secret.password -- operator credentials password
190 password: clickhouse_operator_password
191# nodeSelector -- node for scheduler pod assignment, check `kubectl explain pod.spec.nodeSelector` for details
193# tolerations -- tolerations for scheduler pod assignment, check `kubectl explain pod.spec.tolerations` for details
195# affinity -- affinity for scheduler pod assignment, check `kubectl explain pod.spec.affinity` for details
197# podSecurityContext - operator deployment SecurityContext, check `kubectl explain pod.spec.securityContext` for details
198podSecurityContext: {}
199# topologySpreadConstraints - topologySpreadConstraints affinity for scheduler pod assignment, check `kubectl explain pod.spec.topologySpreadConstraints` for details
200topologySpreadConstraints: []
202 # serviceMonitor.enabled -- ServiceMonitor Custom resource is created for a [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator)
203 # In serviceMonitor will be created two endpoints ch-metrics on port 8888 and op-metrics # 9999, plus a separate keeper-metrics ServiceMonitor when `serviceMonitor.keeperMetrics.enabled` is also set. You can specify interval, scrapeTimeout, relabelings, metricRelabelings for each endpoint below
205 # serviceMonitor.additionalLabels -- additional labels for service monitor
208 # serviceMonitor.interval for ch-metrics endpoint --
210 # serviceMonitor.scrapeTimeout for ch-metrics endpoint -- Prometheus ServiceMonitor scrapeTimeout. If empty, Prometheus uses the global scrape timeout unless it is less than the target's scrape interval value in which the latter is used.
212 # serviceMonitor.relabelings for ch-metrics endpoint -- Prometheus [RelabelConfigs] to apply to samples before scraping
214 # serviceMonitor.metricRelabelings for ch-metrics endpoint -- Prometheus [MetricRelabelConfigs] to apply to samples before ingestio
215 metricRelabelings: []
217 # serviceMonitor.interval for op-metrics endpoint --
219 # serviceMonitor.scrapeTimeout for op-metrics endpoint -- Prometheus ServiceMonitor scrapeTimeout. If empty, Prometheus uses the global scrape timeout unless it is less than the target's scrape interval value in which the latter is used.
221 # serviceMonitor.relabelings for op-metrics endpoint -- Prometheus [RelabelConfigs] to apply to samples before scraping
223 # serviceMonitor.metricRelabelings for op-metrics endpoint -- Prometheus [MetricRelabelConfigs] to apply to samples before ingestio
224 metricRelabelings: []
226 # serviceMonitor.keeperMetrics.enabled -- create a separate ServiceMonitor scraping the native ClickHouse Keeper prometheus endpoint on the Keeper Services the operator manages. Unlike ClickHouse server metrics, Keeper metrics do not go through the operator's metrics-exporter - Keeper serves Prometheus itself. Off by default because it needs the CHK to opt in first: operator-generated Keeper Services publish only the `zk`/`zk-secure`/`raft` ports, so the CHK must both enable the endpoint via `prometheus/*` settings and name the port in `spec.templates.serviceTemplates` (a serviceTemplate REPLACES the default port list, so re-declare `zk` and `raft` there). Without that the ServiceMonitor matches no targets and reports nothing.
228 # serviceMonitor.keeperMetrics.port -- name of the Keeper Service port exposing the native prometheus endpoint. Must match the port name in the CHK serviceTemplate. `prometheus` follows this repo's own Keeper examples (port 7000); charts that name it `metrics` need this overridden
230 # serviceMonitor.keeperMetrics.selector -- matchLabels selecting the Keeper Services to scrape. The default matches every Keeper Service the operator manages. Note the operator creates several Services per CHK (CR-scope plus per-host peer and client), all carrying this label, so if more than one of them names the metrics port the same pod is scraped more than once - narrow with `clickhouse-keeper.altinity.com/Service: chk` (CR-scope) or `: host` (per-host) to pick a single tier
231 # @default -- check the `values.yaml` file
233 clickhouse-keeper.altinity.com/app: chop
234 # serviceMonitor.keeperMetrics.namespaceSelector -- namespaces where prometheus-operator will discover keeper services, `any: true` means all namespaces. Entries under `matchNames` must be literal namespace names, not the regexps `watchNamespaces` accepts. Cluster-wide discovery also requires the Prometheus service account to hold cluster-scoped RBAC on services/endpoints, and Prometheus to select this ServiceMonitor (see `serviceMonitor.additionalLabels`)
237 # serviceMonitor.keeperMetrics.interval -- Prometheus scrape interval for the keeper-metrics endpoint
239 # serviceMonitor.keeperMetrics.scrapeTimeout -- Prometheus ServiceMonitor scrapeTimeout. If empty, Prometheus uses the global scrape timeout unless it is less than the target's scrape interval value in which the latter is used.
241 # serviceMonitor.keeperMetrics.relabelings -- Prometheus [RelabelConfigs] to apply to samples before scraping
243 # serviceMonitor.keeperMetrics.metricRelabelings -- Prometheus [MetricRelabelConfigs] to apply to samples before ingestion
244 metricRelabelings: []
245# configs -- clickhouse operator configs
246# @default -- check the `values.yaml` file for the config content (auto-generated from latest operator release)
250 01-clickhouse-01-listen.xml: |
252 <!-- This file is auto-generated -->
253 <!-- Do not edit this file - all changes would be lost -->
254 <!-- Edit appropriate template in the following folder: -->
255 <!-- deploy/builder/templates-config -->
258 <!-- Listen wildcard address to allow accepting connections from other containers and host network. -->
259 <listen_host>::</listen_host>
260 <listen_host>0.0.0.0</listen_host>
261 <listen_try>1</listen_try>
263 01-clickhouse-02-logger.xml: |
265 <!-- This file is auto-generated -->
266 <!-- Do not edit this file - all changes would be lost -->
267 <!-- Edit appropriate template in the following folder: -->
268 <!-- deploy/builder/templates-config -->
272 <!-- Possible levels: https://github.com/pocoproject/poco/blob/devel/Foundation/include/Poco/Logger.h#L439 -->
274 <log>/var/log/clickhouse-server/clickhouse-server.log</log>
275 <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
278 <!-- Default behavior is autodetection (log to console if not daemon mode and is tty) -->
282 01-clickhouse-03-query_log.xml: |
284 <!-- This file is auto-generated -->
285 <!-- Do not edit this file - all changes would be lost -->
286 <!-- Edit appropriate template in the following folder: -->
287 <!-- deploy/builder/templates-config -->
290 <query_log replace="1">
291 <database>system</database>
292 <table>query_log</table>
293 <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day</engine>
294 <flush_interval_milliseconds>7500</flush_interval_milliseconds>
296 <query_thread_log remove="1"/>
298 01-clickhouse-04-part_log.xml: |
300 <!-- This file is auto-generated -->
301 <!-- Do not edit this file - all changes would be lost -->
302 <!-- Edit appropriate template in the following folder: -->
303 <!-- deploy/builder/templates-config -->
306 <part_log replace="1">
307 <database>system</database>
308 <table>part_log</table>
309 <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day</engine>
310 <flush_interval_milliseconds>7500</flush_interval_milliseconds>
313 01-clickhouse-05-trace_log.xml: |-
315 <!-- This file is auto-generated -->
316 <!-- Do not edit this file - all changes would be lost -->
317 <!-- Edit appropriate template in the following folder: -->
318 <!-- deploy/builder/templates-config -->
321 <trace_log replace="1">
322 <database>system</database>
323 <table>trace_log</table>
324 <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day</engine>
325 <flush_interval_milliseconds>7500</flush_interval_milliseconds>
331 # This file is auto-generated
332 # Do not edit this file - all changes would be lost
333 # Edit appropriate template in the following folder:
334 # deploy/builder/templates-config
337 # Template parameters available:
341 # CH_CREDENTIALS_SECRET_NAMESPACE=
342 # CH_CREDENTIALS_SECRET_NAME=clickhouse-operator
345 ################################################
349 ################################################
351 # Namespaces where clickhouse-operator watches for events.
352 # Concurrently running operators should watch on different namespaces.
353 # `include` and `exclude` accept literal namespace names or regexp patterns.
354 # Empty `include` watches the operator's own namespace (or all namespaces when
355 # the operator runs in `kube-system`); use [".*"] to force watch-all elsewhere.
356 # Empty `exclude` matches none. `exclude` is applied after `include`.
360 # Behavior when ClickHouseOperatorConfiguration changes: none | restart
365 ################################################
367 ## Configuration files section
369 ################################################
371 # Each 'path' can be either absolute or relative.
372 # In case path is absolute - it is used as is
373 # In case path is relative - it is relative to the folder where configuration file you are reading right now is located.
375 # Path to the folder where ClickHouse configuration files common for all instances within a CHI are located.
377 # Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located.
379 # Path to the folder where ClickHouse configuration files with users' settings are located.
380 # Files are common for all instances within a CHI.
382 ################################################
384 ## Configuration users section
386 ################################################
388 # Default settings for user accounts, created by the operator.
389 # IMPORTANT. These are not access credentials or settings for 'default' user account,
390 # it is a template for filling out missing fields for all user accounts to be created by the operator,
391 # with the following EXCEPTIONS:
392 # 1. 'default' user account DOES NOT use provided password, but uses all the rest of the fields.
393 # Password for 'default' user account has to be provided explicitly, if to be used.
394 # 2. CHOP user account DOES NOT use:
395 # - profile setting. It uses predefined profile called 'clickhouse_operator'
396 # - quota setting. It uses empty quota name.
397 # - networks IP setting. Operator specifies 'networks/ip' user setting to match operators' pod IP only.
398 # - password setting. Password for CHOP account is used from 'clickhouse.access.*' section
400 # Default values for ClickHouse user account(s) created by the operator
401 # 1. user/profile - string
402 # 2. user/quota - string
403 # 3. user/networks/ip - multiple strings
404 # 4. user/password - string
405 # These values can be overwritten on per-user basis.
412 ################################################
414 ## Configuration network section
416 ################################################
418 # Default host_regexp to limit network connectivity from outside
419 hostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$"
420 ################################################
422 ## Configuration restart policy section
423 ## Configuration restart policy describes what configuration changes require ClickHouse restart
425 ################################################
426 configurationRestartPolicy:
429 # Special version of "*" - default version - has to satisfy all ClickHouse versions.
430 # Default version will also be used in case ClickHouse version is unknown.
431 # ClickHouse version may be unknown due to host being down - for example, because of incorrect "settings" section.
432 # ClickHouse is not willing to start in case incorrect/unknown settings are provided in config file.
435 # see https://kb.altinity.com/altinity-kb-setup-and-maintenance/altinity-kb-server-config-files/#server-config-configxml-sections-which-dont-require-restart
436 # to be replaced with "select * from system.server_settings where changeable_without_restart = 'No'"
439 - settings/access_control_path: "no"
440 - settings/dictionaries_config: "no"
441 - settings/max_server_memory_*: "no"
442 - settings/max_*_to_drop: "no"
443 - settings/max_concurrent_queries: "no"
444 - settings/models_config: "no"
445 - settings/user_defined_executable_functions_config: "no"
447 - settings/logger/*: "no"
448 - settings/macros/*: "no"
449 - settings/remote_servers/*: "no"
450 - settings/user_directories/*: "no"
451 # these settings should not lead to pod restarts
452 - settings/display_secrets_in_show_and_select: "no"
455 - files/config.d/*.xml: "yes"
456 - files/config.d/*dict*.xml: "no"
457 - files/config.d/*no_restart*: "no"
458 # exceptions in default profile
459 - profiles/default/background_*_pool_size: "yes"
460 - profiles/default/max_*_for_server: "yes"
463 - settings/logger: "yes"
464 #################################################
466 ## Access to ClickHouse instances
468 ################################################
470 # Possible values for 'scheme' are:
471 # 1. http - force http to be used to connect to ClickHouse instances
472 # 2. https - force https to be used to connect to ClickHouse instances
473 # 3. Auto - either http or https is selected based on open ports
475 # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances.
476 # These credentials are used for:
477 # 1. Metrics requests
478 # 2. Schema maintenance
479 # User with these credentials can be specified in additional ClickHouse .xml config files,
480 # located in 'clickhouse.configuration.file.path.user' folder
483 # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS.
485 # Alternate source for rootCA — a Secret in the operator's namespace.
486 # Mutually exclusive with the inline rootCA above (inline wins). Empty
487 # `name` = not used (no-op). When `key` is empty, the operator tries
488 # "ca.crt" then "tls.crt". Resolved once at config load (an operator
489 # restart picks up a rotated Secret).
493 # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances.
494 # Can be used instead of explicitly specified username and password available in sections:
495 # - clickhouse.access.username
496 # - clickhouse.access.password
497 # Secret should have two keys:
501 # Empty `namespace` means that k8s secret would be looked in the same namespace where operator's pod is running.
503 # Empty `name` means no k8s Secret would be looked for
504 name: '{{ include "altinity-clickhouse-operator.fullname" . }}'
505 # Port where to connect to ClickHouse instances to
507 # Timeouts used to limit connection and queries from the operator to ClickHouse instances
508 # Specified in seconds.
510 # Timout to setup connection from the operator to ClickHouse instances. In seconds.
512 # Timout to perform SQL query from the operator to ClickHouse instances. In seconds.
514 ################################################
516 ## Addons specifies additional configuration sections
517 ## Should it be called something like "templates"?
519 ################################################
534 ### users.d is global while description depends on CH version which may vary on per-host basis
535 ### In case of global-ness this may be better to implement via auto-templates
537 ### As a solution, this may be applied on the whole cluster based on any of its hosts
539 ### What to do when host is just created? CH version is not known prior to CH started and user config is required before CH started.
540 ### We do not have any info about the cluster on initial creation
543 "{clickhouseOperatorUser}/access_management": 1
544 "{clickhouseOperatorUser}/named_collection_control": 1
545 "{clickhouseOperatorUser}/show_named_collections": 1
546 "{clickhouseOperatorUser}/show_named_collections_secrets": 1
556 clickhouse_operator/format_display_secrets_in_show_and_select: 1
560 ## this may be added on per-host basis into host's conf.d folder
562 display_secrets_in_show_and_select: 1
564 #################################################
566 ## Metrics collection
568 ################################################
570 # Timeouts used to limit connection and queries from the metrics exporter to ClickHouse instances
571 # Specified in seconds.
573 # Timeout used to limit metrics collection request. In seconds.
574 # Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle.
575 # All collected metrics are returned.
577 # Regexp to match tables in system database to fetch metrics from.
578 # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function.
579 # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics.
580 tablesRegexp: "^(metrics|custom_metrics)$"
581 # List of regexps to match ClickHouse metrics to exclude from collection/export.
582 # Regexps match internal metric names before Prometheus normalization and prefixing.
583 # Default is the per-CPU OS metrics filter shown below; set to [] to disable.
585 - "^metric\\.(OS.*CPU[0-9]+|CPUFrequencyMHz_[0-9]+)$"
588 ################################################
590 ## Configuration files section
592 ################################################
594 # Each 'path' can be either absolute or relative.
595 # In case path is absolute - it is used as is
596 # In case path is relative - it is relative to the folder where configuration file you are reading right now is located.
598 # Path to the folder where Keeper configuration files common for all instances within a CHK are located.
599 common: chk/keeper_config.d
600 # Path to the folder where Keeper configuration files unique for each instance (host) within a CHK are located.
602 # Path to the folder where Keeper configuration files with users' settings are located.
603 # Files are common for all instances within a CHI.
605 ################################################
609 ## Operator-wide security toggles. All fields default to unset / permissive so
610 ## upgrades from earlier versions preserve identical behavior. Set explicit
611 ## values here to tighten the operator's outbound TLS posture; CHIs may further
612 ## override per-cluster via spec.configuration.clusters[].security.
614 ## Three orthogonal axes govern this section:
615 ## 1. security.policy — TLS-hardening master switch (Permissive | Enforced)
616 ## 2. security.fips.enforced — FIPS cryptographic-module gate (bool)
617 ## 3. security.images.policy — Workload image-tag governance (Permissive | FIPSRequired)
618 ## Each axis is independent: enabling one does not enable the others.
620 ## See docs/chi-examples/70-chop-config.yaml for a fully-annotated example
621 ## and docs/security_hardening.md for the design + per-knob semantics.
623 ################################################
627 # Strict | None | "" (preserve legacy InsecureSkipVerify=true)
629 # "1.2" | "1.3" | "" (Go stdlib default)
631 # SNI / cert-name override; default = dial host
633 # Inline PEM CA bundle (or base64-wrapped)
635 # Alternate source — Secret in operator namespace. Mutually exclusive
636 # with the inline rootCA above. Empty `name` = not used (no-op).
637 # When `key` is empty, the operator tries "ca.crt" then "tls.crt".
647 # Strict refuses an insecure kubeconfig at startup
649 # Floors the K8s API client transport TLS version; coerced to 1.3 under FIPS/Enforced
652 # Plain (default) | Secure (loopback + X-CHOP-Token)
654 # Defaults to 127.0.0.1 when mode=Secure
656 # Defaults to /etc/clickhouse-operator-ipc/token
658 # Operator-wide TLS-hardening master switch. ONLY governs TLS posture across
659 # CH / ZK / K8s transports — NOT the FIPS cryptographic-module gate (see
660 # security.fips.enforced below for that, orthogonal axis).
662 # Permissive (default) preserves 0.27.0 behavior — no coercion, no rejection.
663 # Enforced coerces all TLS knobs above to their Strict positions at startup:
664 # - clickhouse.tls.verify=Strict, clickhouse.tls.minVersion=1.3
665 # - zookeeper.tls.verify=Strict, zookeeper.tls.minVersion=1.3
666 # - kubernetes.tls.verify=Strict, kubernetes.tls.minVersion=1.3
668 # - clickhouse.access.scheme: http is coerced to https
669 # Enforced also rejects CHIs that cannot be served in a hardened posture
670 # (e.g. plaintext external ZooKeeper, ZK digest auth).
672 # Independent of the Go FIPS toolchain — works on non-FIPS builds for pure
673 # TLS hardening. Combine with security.fips.enforced=true for full FIPS
674 # cryptographic-module enforcement.
676 # FIPS cryptographic-module enforcement. Orthogonal to security.policy.
677 # Default operator and metrics-exporter images are FIPS-compatible —
678 # built with GOFIPS140=v1.0.0 and run with GODEBUG=fips140=on, so
679 # crypto/fips140.Enabled() returns true at runtime.
680 # When enforced=true, the operator Fatals at startup unless the binary
681 # reports crypto/fips140 Enabled — guards against accidentally running
682 # a non-FIPS rebuild in a hardened deployment.
686 # Workload image-tag governance gate. Today's only non-default value is
687 # FIPSRequired (admission rejects CRs whose CH/Keeper images lack 'fips'
688 # in tag; post-Ready SELECT version() must contain 'fips' or CR aborts).
689 # Orthogonal to security.policy and security.fips.enforced.
690 # See docs/security_hardening_fips.md → "security.images.policy: FIPSRequired"
691 # for the full policy matrix + detection details + recovery procedure.
693 ################################################
695 ## Template(s) management section
697 ################################################
700 # CHI template updates handling policy
701 # Possible policy values:
702 # - ReadOnStart. Accept CHIT updates on the operator's start only.
703 # - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply new CHITs on next regular reconcile of the CHI
704 policy: ApplyOnNextReconcile
705 # Path to the folder where ClickHouseInstallation templates .yaml manifests are located.
706 # Templates are added to the list of all templates and used when CHI is reconciled.
707 # Templates are applied in sorted alpha-numeric order.
708 path: chi/templates.d
710 # CHK template updates handling policy
711 # Possible policy values:
712 # - ReadOnStart. Accept CHIT updates on the operators start only.
713 # - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply new CHITs on next regular reconcile of the CHI
714 policy: ApplyOnNextReconcile
715 # Path to the folder where ClickHouseInstallation templates .yaml manifests are located.
716 # Templates are added to the list of all templates and used when CHI is reconciled.
717 # Templates are applied in sorted alpha-numeric order.
718 path: chk/templates.d
719 ################################################
723 ################################################
725 # Reconcile runtime settings
727 # Max number of concurrent CHI reconciles in progress
728 reconcileCHIsThreadsNumber: 10
729 # Max number of concurrent CHK reconciles in progress
730 reconcileCHKsThreadsNumber: 1
731 # The operator reconciles shards concurrently in each CHI with the following limitations:
732 # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently
733 # can not be greater than 'reconcileShardsThreadsNumber'.
734 # 2. Percentage of shards being reconciled (and thus having hosts down) in each CHI concurrently
735 # can not be greater than 'reconcileShardsMaxConcurrencyPercent'.
736 # 3. The first shard is always reconciled alone. Concurrency starts from the second shard and onward.
737 # Thus limiting number of shards being reconciled (and thus having hosts down) in each CHI by both number and percentage
739 # Max number of concurrent shard reconciles within one cluster in progress
740 reconcileShardsThreadsNumber: 5
741 # Max percentage of concurrent shard reconciles within one cluster in progress
742 reconcileShardsMaxConcurrencyPercent: 50
743 # Reconcile StatefulSet scenario
745 # Create StatefulSet scenario
747 # What to do in case created StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds
749 # 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is,
750 # do not try to fix or delete or update it, just abort reconcile cycle.
751 # Do not proceed to the next StatefulSet(s) and wait for an admin to assist.
752 # 2. delete - delete newly created problematic StatefulSet and follow 'abort' path afterwards.
753 # 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet.
755 # Update StatefulSet scenario
757 # How many seconds to wait for created/updated StatefulSet to be 'Ready'
759 # How many seconds to wait between checks/polls for created/updated StatefulSet status
761 # What to do in case updated StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds
763 # 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is,
764 # do not try to fix or delete or update it, just abort reconcile cycle.
765 # Do not proceed to the next StatefulSet(s) and wait for an admin to assist.
766 # 2. rollback - delete Pod and rollback StatefulSet to previous Generation.
767 # Pod would be recreated by StatefulSet based on rollback-ed StatefulSet configuration.
768 # Follow 'abort' path afterwards.
769 # 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet.
771 # Recreate StatefulSet scenario
773 # What to do in case operator is in need to recreate StatefulSet?
775 # 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is,
776 # do not try to fix or delete or update it, just abort reconcile cycle.
777 # Do not proceed to the next StatefulSet(s) and wait for an admin to assist.
778 # 2. recreate - proceed and recreate StatefulSet.
780 # Triggered when PVC data loss or missing volumes are detected
782 # Triggered when StatefulSet update fails or StatefulSet is not ready
783 onUpdateFailure: recreate
784 # Reconcile Host scenario
786 # The operator during reconcile procedure should wait for a ClickHouse host to achieve the following conditions:
788 # Whether the operator during reconcile procedure should wait for a ClickHouse host:
789 # - to be excluded from a ClickHouse cluster
790 # - to complete all running queries
791 # - to be included into a ClickHouse cluster
792 # respectfully before moving forward with host reconcile
796 # The operator during reconcile procedure should wait for replicas to catch-up
797 # replication delay a.k.a replication lag for the following replicas
799 # All replicas (new and known earlier) are explicitly requested to wait for replication to catch-up
801 # New replicas only are requested to wait for replication to catch-up
803 # Replication catch-up is considered to be completed as soon as replication delay
804 # a.k.a replication lag - calculated as "MAX(absolute_delay) FROM system.replicas"
805 # is within this specified delay (in seconds)
808 # Whether the operator during host launch procedure should wait for startup probe to succeed.
809 # In case probe is unspecified wait is assumed to be completed successfully.
810 # Default option value is to do not wait.
812 # Whether the operator during host launch procedure should wait for readiness probe to succeed.
813 # In case probe is unspecified wait is assumed to be completed successfully.
814 # Default option value is to wait.
816 # The operator during reconcile procedure should drop the following entities:
819 # Whether the operator during reconcile procedure should drop replicas when replica is deleted
821 # Whether the operator during reconcile procedure should drop replicas when replica volume is lost
823 # Whether the operator during reconcile procedure should drop active replicas when replica is deleted or recreated
825 ################################################
827 ## Coordination with external systems during reconcile
829 ################################################
832 # How long the operator waits for a referenced ClickHouseKeeper to become ready
833 # before aborting CHI reconcile. In seconds.
835 # Reaction when a referenced CHK resource changes:
836 # none — do nothing (default, backward-compatible)
837 # reconcile — trigger CHI reconcile
838 # onKeeperResourceUpdate: none
839 ################################################
841 ## Auto-recovery from aborted reconcile
843 ################################################
845 # Recovery scopes keyed by the CHI .status.status they apply to.
846 # Each scope contains on<Event>: <action> mappings that apply while the CHI
847 # is in that status. Multi-scope design anticipates future states beyond Aborted
848 # (e.g. Failed, Broken).
850 # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete)
851 # when one of its host pods transitions to Ready — auto-resumes the reconcile.
853 # Action when a pod belonging to an Aborted CHI transitions to Ready:
854 # retry (default) — re-enqueue the CHI for reconcile
855 # none — do nothing, CHI stays Aborted
857 # Future events (not yet implemented):
858 # onKeeperReady: retry — retry when a referenced CHK becomes ready
859 # onOperatorRestart: retry — sweep Aborted CHIs on operator startup
860 # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one
861 # of its host pods regresses to Ready=False and stays NotReady (sustained) without
862 # crashing — auto-heals stuck hosts.
864 # Action when a Completed CHI's pod flips Ready=True -> Ready=False and
865 # stays NotReady for at least onPodNotReadyThreshold:
866 # none (default) — do nothing
867 # retry — re-enqueue the CHI so the stuck host is force-restarted
868 # OFF by default: force-recreating a Completed CHI's pod is destructive — it can
869 # interrupt a replica's in-progress recovery and means hard downtime for a
870 # single-replica shard. Opt in with `retry` only where that trade-off is acceptable.
872 # Minimum duration a pod must stay Ready=False before recovery fires, once enabled
873 # (Go duration string; default 5m). Raise it for slow-recovering replicas.
874 onPodNotReadyThreshold: 5m
875 # Future scopes (not yet implemented):
880 # Future global policy knobs (not yet implemented) — flat peers of `onStatus`,
881 # apply across all recovery scopes:
883 # Global kill-switch for auto-recovery:
886 # Cap on consecutive auto-recovery attempts before giving up:
889 # Minimum time between auto-recovery attempts for the same CHI:
892 # Exponential backoff for auto-recovery attempts:
897 ################################################
899 ## Annotations management section
901 ################################################
904 # 1. Propagating annotations from the CHI's `metadata.annotations` to child objects' `metadata.annotations`,
905 # 2. Propagating annotations from the CHI Template's `metadata.annotations` to CHI's `metadata.annotations`,
906 # Include annotations from the following list:
907 # Applied only when not empty. Empty list means "include all, no selection"
909 # Exclude annotations from the following list:
911 ################################################
913 ## Labels management section
915 ################################################
918 # 1. Propagating labels from the CHI's `metadata.labels` to child objects' `metadata.labels`,
919 # 2. Propagating labels from the CHI Template's `metadata.labels` to CHI's `metadata.labels`,
920 # Include labels from the following list:
921 # Applied only when not empty. Empty list means "include all, no selection"
923 # Exclude labels from the following list:
924 # Applied only when not empty. Empty list means "nothing to exclude, no selection"
926 # Whether to append *Scope* labels to StatefulSet and Pod.
927 # Full list of available *scope* labels check in 'labeler.go'
928 # LabelShardScopeIndex
929 # LabelReplicaScopeIndex
931 # LabelCHIScopeCycleSize
932 # LabelCHIScopeCycleIndex
933 # LabelCHIScopeCycleOffset
934 # LabelClusterScopeIndex
935 # LabelClusterScopeCycleSize
936 # LabelClusterScopeCycleIndex
937 # LabelClusterScopeCycleOffset
939 ################################################
941 ## Metrics management section
943 ################################################
947 ################################################
949 ## Status management section
951 ################################################
958 ################################################
960 ## StatefulSet management section
962 ################################################
964 revisionHistoryLimit: 0
965 ################################################
967 ## Pod management section
969 ################################################
971 # Grace period for Pod termination.
972 # How many seconds to wait between sending
973 # SIGTERM and SIGKILL during Pod termination process.
974 # Increase this number is case of slow shutdown.
975 terminationGracePeriod: 30
976 ################################################
978 ## Log parameters section
980 ################################################
983 alsologtostderr: "false"
989 001-templates.json.example: |
991 "apiVersion": "clickhouse.altinity.com/v1",
992 "kind": "ClickHouseInstallationTemplate",
994 "name": "01-default-volumeclaimtemplate"
998 "volumeClaimTemplates": [
1000 "name": "chi-default-volume-claim-template",
1015 "name": "chi-default-oneperhost-pod-template",
1016 "distribution": "OnePerHost",
1020 "name": "clickhouse",
1021 "image": "clickhouse/clickhouse-server:23.8",
1025 "containerPort": 8123
1029 "containerPort": 9000
1032 "name": "interserver",
1033 "containerPort": 9009
1044 default-pod-template.yaml.example: |
1045 apiVersion: "clickhouse.altinity.com/v1"
1046 kind: "ClickHouseInstallationTemplate"
1048 name: "default-oneperhost-pod-template"
1052 - name: default-oneperhost-pod-template
1053 distribution: "OnePerHost"
1054 default-storage-template.yaml.example: |
1055 apiVersion: "clickhouse.altinity.com/v1"
1056 kind: "ClickHouseInstallationTemplate"
1058 name: "default-storage-template-2Gi"
1061 volumeClaimTemplates:
1062 - name: default-storage-template-2Gi
1070 Templates in this folder are packaged with an operator and available via 'useTemplate'
1072 01-clickhouse-operator-profile.xml: |
1074 <!-- This file is auto-generated -->
1075 <!-- Do not edit this file - all changes would be lost -->
1076 <!-- Edit appropriate template in the following folder: -->
1077 <!-- deploy/builder/templates-config -->
1081 # Template parameters available:
1085 <!-- clickhouse-operator user is generated by the operator based on config.yaml in runtime -->
1087 <clickhouse_operator>
1088 <log_queries>0</log_queries>
1089 <skip_unavailable_shards>1</skip_unavailable_shards>
1090 <http_connection_timeout>10</http_connection_timeout>
1091 <max_concurrent_queries_for_all_users>0</max_concurrent_queries_for_all_users>
1092 <os_thread_priority>0</os_thread_priority>
1093 </clickhouse_operator>
1096 02-clickhouse-default-profile.xml: |-
1098 <!-- This file is auto-generated -->
1099 <!-- Do not edit this file - all changes would be lost -->
1100 <!-- Edit appropriate template in the following folder: -->
1101 <!-- deploy/builder/templates-config -->
1106 <os_thread_priority>2</os_thread_priority>
1107 <log_queries>1</log_queries>
1108 <connect_timeout_with_failover_ms>1000</connect_timeout_with_failover_ms>
1109 <distributed_aggregation_memory_efficient>1</distributed_aggregation_memory_efficient>
1110 <parallel_view_processing>1</parallel_view_processing>
1111 <do_not_merge_across_partitions_select_final>1</do_not_merge_across_partitions_select_final>
1112 <load_balancing>nearest_hostname</load_balancing>
1113 <prefer_localhost_replica>0</prefer_localhost_replica>
1114 <!-- materialize_ttl_recalculate_only>1</materialize_ttl_recalculate_only> 21.10 and above -->
1118 keeperConfdFiles: null
1120 01-keeper-01-default-config.xml: |
1122 <!-- This file is auto-generated -->
1123 <!-- Do not edit this file - all changes would be lost -->
1124 <!-- Edit appropriate template in the following folder: -->
1125 <!-- deploy/builder/templates-config -->
1128 <asynchronous_metrics_keeper_metrics_only>1</asynchronous_metrics_keeper_metrics_only>
1130 <coordination_settings>
1131 <async_replication>1</async_replication>
1132 <min_session_timeout_ms>10000</min_session_timeout_ms>
1133 <operation_timeout_ms>10000</operation_timeout_ms>
1134 <raft_logs_level>information</raft_logs_level>
1135 <session_timeout_ms>100000</session_timeout_ms>
1136 <use_xid_64>1</use_xid_64>
1137 </coordination_settings>
1138 <hostname_checks_enabled>true</hostname_checks_enabled>
1139 <log_storage_path>/var/lib/clickhouse-keeper/coordination/logs</log_storage_path>
1140 <snapshot_storage_path>/var/lib/clickhouse-keeper/coordination/snapshots</snapshot_storage_path>
1141 <storage_path>/var/lib/clickhouse-keeper</storage_path>
1142 <tcp_port>2181</tcp_port>
1144 Four-letter-word command allowlist.
1146 Set explicitly to the upstream-default list so the operator-rendered
1147 liveness probe (which sends `ruok` over TCP and expects `imok`) keeps
1148 working even if a user adds their own keeper_server settings.
1150 Without this, a user override that restricts the allowlist
1151 (e.g. `four_letter_word_white_list: "mntr,stat"` for security)
1152 would silently disable `ruok` → liveness probe always fails → CrashLoopBackOff.
1154 The list mirrors ClickHouse Keeper's compiled-in default; users who want a
1155 stricter list can override this value, but they must keep `ruok` if they
1156 also use the default operator probes.
1158 <four_letter_word_white_list>conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro</four_letter_word_white_list>
1160 <listen_host>::</listen_host>
1161 <listen_host>0.0.0.0</listen_host>
1162 <listen_try>1</listen_try>
1164 <console>1</console>
1165 <level>information</level>
1167 <max_connections>4096</max_connections>
1169 01-keeper-02-readiness.xml: |
1171 <!-- This file is auto-generated -->
1172 <!-- Do not edit this file - all changes would be lost -->
1173 <!-- Edit appropriate template in the following folder: -->
1174 <!-- deploy/builder/templates-config -->
1181 <endpoint>/ready</endpoint>
1186 01-keeper-03-enable-reconfig.xml: |-
1188 <!-- This file is auto-generated -->
1189 <!-- Do not edit this file - all changes would be lost -->
1190 <!-- Edit appropriate template in the following folder: -->
1191 <!-- deploy/builder/templates-config -->
1195 <enable_reconfiguration>false</enable_reconfiguration>
1198 keeperTemplatesdFiles:
1200 Templates in this folder are packaged with an operator and available via 'useTemplate'
1201 keeperUsersdFiles: null
1202# additionalResources -- list of additional resources to create (processed via `tpl` function),
1203# useful for create ClickHouse clusters together with clickhouse-operator.
1204# check `kubectl explain chi` for details
1205additionalResources: []
1210# name: {{ include "altinity-clickhouse-operator.fullname" . }}-cm
1211# namespace: {{ include "altinity-clickhouse-operator.namespace" . }}
1216# name: {{ include "altinity-clickhouse-operator.fullname" . }}-s
1217# namespace: {{ include "altinity-clickhouse-operator.namespace" . }}
1221# apiVersion: clickhouse.altinity.com/v1
1222# kind: ClickHouseInstallation
1224# name: {{ include "altinity-clickhouse-operator.fullname" . }}-chi
1225# namespace: {{ include "altinity-clickhouse-operator.namespace" . }}
1234 # dashboards.enabled -- provision grafana dashboards as configMaps (can be synced by grafana dashboards sidecar https://github.com/grafana/helm-charts/blob/grafana-8.3.4/charts/grafana/values.yaml#L778 )
1236 # dashboards.additionalLabels -- labels to add to a secret with dashboards
1238 # dashboards.additionalLabels.grafana_dashboard - will watch when official grafana helm chart sidecar.dashboards.enabled=true
1239 grafana_dashboard: ""
1240 # dashboards.annotations -- annotations to add to a secret with dashboards
1242 # dashboards.annotations.grafana_folder -- folder where will place dashboards, requires define values in official grafana helm chart sidecar.dashboards.folderAnnotation: grafana_folder
1243 grafana_folder: clickhouse-operator