dcgm-exporter

# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

image:

repository: cgr.dev/chainguard-private/dcgm-exporter

pullPolicy: IfNotPresent

# Image tag defaults to AppVersion, but you can use the tag key

# for the image tag, e.g:

tag: 4.8.4-r4@sha256:6287b9f11a08ca3ca96967942147966b9a1f37726b268c798f7388065e8596da

# Optional digest pin. When set, the chart renders repository@digest and

# ignores tag for the image reference.

digest: ""

# Add dcgm-exporter command line arguments here. When empty, the exporter uses

# its built-in default collectors path.

arguments: []

# NOTE: in general, add any command line arguments to arguments above and they

# will be passed through.

# Use "-r", "<HOST>:<PORT>" or a DCGM connection string to connect to an already running DCGM host engine.

# Connection string formats: "tcp://<HOST>:<PORT>", "unix:///<SOCKET_PATH>", or "vsock://<CID>:<PORT>".

# Example arguments: ["-r", "host123:5555"]

# Example VSOCK arguments: ["-r", "vsock://3:5555"]

# Use "-n" to remove the hostname tag from the output.

# Example arguments: ["-n"]

# Use "-d" to specify the devices to monitor. -d must be followed by a string

# in the following format: f, g[:numeric_range], i[:numeric_range], or

# g[:numeric_range]+i[:numeric_range]. i+g ordering is also accepted.

# The flex option f is the default and cannot be combined with g or i.

# Where a numeric range is something like 0-4 or 0,2,4, etc.

# Example arguments: ["-d", "f"] to monitor all GPUs or GPU instances, or

# ["-d", "g:0-3"] to monitor GPUs 0-3, or ["-d", "i"] to monitor GPU instances.

# Use "-m" to specify the namespace and name of a configmap containing

# the watched exporter fields.

# Example arguments: ["-m", "default:exporter-metrics-config-map"]

# If adding "--enable-pprof", also enable tlsServerConfig and/or basicAuth.

# Pprof requires a web config file so /debug/pprof/ is protected by

# exporter-toolkit authentication or TLS.

# Optional dcgm-exporter YAML configuration. YAML is read at exporter startup.

# Changes to the YAML file require restarting the exporter pod.

config:

enabled: false

# When true, the chart creates a ConfigMap from config.data. When false,

# config.name must reference an existing ConfigMap.

create: true

# Defaults to "<release-name>-dcgm-exporter-config" when empty.

name: ""

key: config.yaml

mountPath: /etc/dcgm-exporter/config.yaml

data: |

version: 2

# Detached-GPU support requires DCGM 4.5+ and an NVIDIA 590+ driver.

sources:

dcgm:

detectBindUnbind:

enabled: false

pollInterval: 1s

metrics:

file: /etc/dcgm-exporter/default-counters.csv

enableExporterMetrics: false

collections:

- name: scrape

every: 30s

metrics:

include: ["*"]

# Image pull secrets for container images

imagePullSecrets: []

# Overrides the chart's name

nameOverride: ""

# Overrides the chart's computed fullname

fullnameOverride: ""

# Overrides the deployment namespace

namespaceOverride: ""

# hostPID allows the DCGM-Exporter container to see processes on the host node

# Default: false

hostPID: false

# Defines the runtime class that will be used by the pod

runtimeClassName: ""

# Defines the priority class that will be used by the pod

priorityClassName: "system-node-critical"

# Enable or disable host network

hostNetwork: false

# Defines serviceAccount names for components.

serviceAccount:

# Specifies whether a service account should be created

create: true

# Annotations to add to the service account

annotations: {}

# The name of the service account to use.

# If not set and create is true, a name is generated using the fullname template

name:

# Whether to mount the SA token on the DaemonSet pod and the ServiceAccount.

# If not set, derived from kubernetes.enablePodLabels/enablePodUID with rbac.create, or kubernetesDRA.enabled.

# Setting this to false disables in-cluster Kubernetes API authentication for those features.

automountServiceAccountToken:

rollingUpdate:

# Specifies maximum number of DaemonSet pods that can be unavailable during the update

maxUnavailable: 1

# Specifies maximum number of nodes with an existing available DaemonSet pod that can have an updated DaemonSet pod during during an update

maxSurge: 0

# Labels to be added to dcgm-exporter pods

podLabels: {}

# Annotations to be added to the dcgm-exporter DaemonSet

daemonSetAnnotations: {}

# Annotations to be added to dcgm-exporter pods

podAnnotations: {}

# Using this annotation which is required for prometheus scraping

# prometheus.io/scrape: "true"

# prometheus.io/port: "9400"

# The SecurityContext for the dcgm-exporter pods

podSecurityContext: {}

# fsGroup: 2000

# The SecurityContext for the dcgm-exporter containers

securityContext:

runAsNonRoot: false

runAsUser: 0

capabilities:

add: ["SYS_ADMIN"] # Required for profiling metrics (DCGM_FI_PROF_*)

drop: ["ALL"]

allowPrivilegeEscalation: false

# readOnlyRootFilesystem: true # Enable if tmpfs volumes are configured

# Note: For non-root without profiling metrics, use:

# runAsNonRoot: true, runAsUser: 1000, and remove SYS_ADMIN from capabilities.add

# Defines the dcgm-exporter service

service:

# When enabled, the helm chart will create service

enable: true

type: ClusterIP

# Accepts either "Cluster" or "Local", choose Local if you want to route internal traffic within the node only

internalTrafficPolicy: Cluster

clusterIP: ""

port: 9400

address: ":9400"

# HTTP server read timeout for inbound scrape requests

webReadTimeout: 10s

# HTTP server write timeout for generating and returning scrape responses

webWriteTimeout: 30s

# Annotations to add to the service

annotations: {}

# Allows to control pod resources

resources:

limits:

cpu: 200m

memory: 512Mi

requests:

cpu: 100m

memory: 128Mi

serviceMonitor:

apiVersion: "monitoring.coreos.com/v1"

enabled: true

interval: 30s

scrapeTimeout: 25s

honorLabels: false

additionalLabels: {}

#monitoring: prometheus

relabelings: []

# - sourceLabels: [__meta_kubernetes_pod_node_name]

# separator: ;

# regex: ^(.*)$

# targetLabel: nodename

# replacement: $1

# action: replace

metricRelabelings: []

# - action: replace

# regex: original_metrics_name

# replacement: new_metrics_name

# sourceLabels:

# - name

# targetLabel: __name__

nodeSelector: {}

#node: gpu

tolerations:

- key: node-role.kubernetes.io/control-plane

operator: Exists

effect: NoSchedule

#- operator: Exists

affinity: {}

#nodeAffinity:

# requiredDuringSchedulingIgnoredDuringExecution:

# nodeSelectorTerms:

# - matchExpressions:

# - key: nvidia-gpu

# operator: Exists

extraHostVolumes: []

#- name: host-binaries

# hostPath: /opt/bin

extraConfigMapVolumes:

- name: exporter-metrics-volume

configMap:

name: exporter-metrics-config-map

items:

- key: metrics

path: default-counters.csv

extraVolumeMounts:

- name: exporter-metrics-volume

mountPath: /etc/dcgm-exporter/default-counters.csv

subPath: default-counters.csv

extraEnv: []

# Path to the kubelet socket for /pod-resources

kubeletPath: "/var/lib/kubelet/pod-resources"

# HTTPS configuration

tlsServerConfig:

# Enable or disable HTTPS configuration

enabled: false

# Use autogenerated self-signed TLS certificates. Not recommended for production environments.

autoGenerated: true

# Existing secret containing your own server key and certificate

existingSecret: ""

# Certificate file name

certFilename: "tls.crt"

# Key file name

keyFilename: "tls.key"

# CA certificate file name

caFilename: "ca.crt"

# Server policy for client authentication. Maps to ClientAuth Policies.

# For more detail on clientAuth options:

# https://golang.org/pkg/crypto/tls/#ClientAuthType

#

# NOTE: If you want to enable client authentication, you need to use

# RequireAndVerifyClientCert. Other values are insecure.

clientAuthType: ""

# TLS Key for HTTPS - ignored if existingSecret is provided

key: ""

# TLS Certificate for HTTPS - ignored if existingSecret is provided

cert: ""

# CA Certificate for HTTPS - ignored if existingSecret is provided

ca: ""

basicAuth:

#Object containing <user>:<passwords> key-value pairs for each user that will have access via basic authentication

users: {}

# Kubernetes integration settings

kubernetes:

# Enable Kubernetes pod labels in metrics

# When enabled, metrics will include labels from the pods that are using the GPUs

# This requires cluster-level read permissions to pods

enablePodLabels: false

# Enable Kubernetes pod UID in metrics

# When enabled, metrics will include the pod UID as an attribute for the pods that are using the GPUs

# This requires cluster-level read permissions to pods

enablePodUID: false

# Pod label filtering configuration

# Filter which pod labels are included in metrics using regex patterns

# Empty list means all labels are included (default behavior)

# Examples:

# podLabelAllowlistRegex:

# - "^app$" # Exact match for "app" label

# - "^app\\.kubernetes\\.io/.*" # All labels starting with app.kubernetes.io/

# - "^(tier|environment|version)$" # Match tier, environment, or version labels

podLabelAllowlistRegex: []

# RBAC settings for Kubernetes integration

rbac:

# Automatically creates ClusterRole and ClusterRoleBinding for pod access when enablePodLabels or enablePodUID is true

# Set to false if you want to manage RBAC resources manually

create: true

# Runtime object debug dump configuration for troubleshooting

debugDump:

# Enable file-based debugging dumps for troubleshooting

enabled: false

# Directory to store debug dump files

directory: "/tmp/dcgm-exporter-debug"

# Retention period for debug dump files in hours (0 = no cleanup)

retention: 24

# Use gzip compression for debug dump files

compression: true

# Enable DRA to capture metrics associated with GPUs managed by NVIDIA k8s-dra-driver (https://github.com/NVIDIA/k8s-dra-driver-gpu)

# default: false

kubernetesDRA:

enabled: false

# Customized list of metrics to emit. Expected to be in the same format (CSV) as the default list.

# Must be the complete list and is not additive. If unset, the default list will take effect.

# customMetrics: |

# Format

# If line starts with a '#' it is considered a comment

# DCGM FIELD, Prometheus metric type, help message

# Clocks

# DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz).

# DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz).

# Temperature

# DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C).

# DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C).

# Power

# DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W).

# DCGM_FI_DEV_POWER_MGMT_LIMIT, gauge, Configured power management limit (in W).

# DCGM_FI_DEV_ENFORCED_POWER_LIMIT, gauge, Effective power limit enforced by the driver after all limiters (in W).

# DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since the driver was last reloaded (in mJ).

# PCIE

# PCIe throughput profiling metrics are documented in the DCP metrics section below.

# DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries.

# Utilization (the sample period varies depending on the product)

# DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %).

# DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %).

# DCGM_FI_DEV_ENC_UTIL, gauge, Encoder utilization (in %).

# DCGM_FI_DEV_DEC_UTIL , gauge, Decoder utilization (in %).

# Errors and violations

# DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered.

# DCGM_EXP_CLOCK_EVENTS_TOTAL, counter, cumulative clock events observed since exporter start (edge-counted)

# DCGM_EXP_XID_ERRORS_TOTAL, counter, cumulative XID errors observed since exporter start

# DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in ns).

# DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in ns).

# DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in ns).

# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in ns).

# DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in ns).

# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in ns).

# Fabric health

# DCGM_FI_DEV_FABRIC_HEALTH_SUMMARY, gauge, Overall health of the GPU fabric.

# DCGM_FI_DEV_FABRIC_HEALTH_MASK, gauge, GPU fabric health status mask.

# Memory usage

# DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB).

# DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB).

# ECC

# DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors.

# DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors.

# DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors.

# DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors.

# Retired pages

# DCGM_FI_DEV_RETIRED_SBE, counter, Total number of retired pages due to single-bit errors.

# DCGM_FI_DEV_RETIRED_DBE, counter, Total number of retired pages due to double-bit errors.

# DCGM_FI_DEV_RETIRED_PENDING, gauge, Whether pages are pending retirement (1 means pending and 0 means not pending).

# NVLink

# NVLink error counters for pre-Hopper GPUs

# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_TOTAL, counter, Total number of NVLink flow-control CRC errors across all lanes.

# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_TOTAL, counter, Total number of NVLink data CRC errors across all lanes.

# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_TOTAL, counter, Total number of NVLink replay errors across all lanes.

# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_TOTAL, counter, Total number of NVLink recovery errors across all lanes.

# NVLink data-link error counters for Hopper GPUs

# DCGM_FI_DEV_NVLINK_CRC_ERROR_TOTAL, counter, Total number of NVLink data-link CRC errors.

# DCGM_FI_DEV_NVLINK_RECOVERY_TOTAL, counter, Total number of NVLink data-link recovery errors.

# DCGM_FI_DEV_NVLINK_REPLAY_TOTAL, counter, Total number of NVLink data-link replay errors.

# NVLink recovery counters for Blackwell and newer GPUs

# DCGM_FI_DEV_NVLINK_RECOVERY_SUCCESSFUL_TOTAL, counter, Total number of successful NVLink recovery events across all links.

# DCGM_FI_DEV_NVLINK_RECOVERY_FAILED_TOTAL, counter, Total number of failed NVLink recovery events across all links.

# DCGM_FI_DEV_NVLINK_RECOVERY_EVENT_TOTAL, counter, Total number of NVLink recovery events across all links.

# DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, gauge, Aggregate NVLink throughput across all lanes (in MB/s).

# DCGM_FI_DEV_NVLINK_BANDWIDTH_L0, gauge, NVLink lane 0 throughput (in MB/s).

# VGPU License status

# DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status

# Remapped rows

# DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors

# DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors

# DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed

# DCP metrics

# DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active.

# DCGM_FI_PROF_SM_ACTIVE, gauge, The ratio of cycles an SM has at least 1 warp assigned.

# DCGM_FI_PROF_SM_OCCUPANCY, gauge, The ratio of number of warps resident on an SM.

# DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active.

# DCGM_FI_PROF_DRAM_ACTIVE, gauge, Ratio of cycles the device memory interface is active sending or receiving data.

# DCGM_FI_PROF_PIPE_FP64_ACTIVE, gauge, Ratio of cycles the fp64 pipes are active.

# DCGM_FI_PROF_PIPE_FP32_ACTIVE, gauge, Ratio of cycles the fp32 pipes are active.

# DCGM_FI_PROF_PIPE_FP16_ACTIVE, gauge, Ratio of cycles the fp16 pipes are active.

# DCGM_FI_PROF_PCIE_TX_BYTES, gauge, The rate of data transmitted over the PCIe bus - including both protocol headers and data payloads - in bytes per second.

# DCGM_FI_PROF_PCIE_RX_BYTES, gauge, The rate of data received over the PCIe bus - including both protocol headers and data payloads - in bytes per second.

# DCGM_FI_PROF_SM_CYCLES_ELAPSED_TOTAL, counter, Total elapsed SM cycles.

# DCGM_FI_PROF_SM_CYCLES_ACTIVE_TOTAL, counter, Total SM cycles with active warps.

# DCGM_FI_PROF_MMA_CYCLES_ACTIVE_TOTAL, counter, Total MMA tensor cycles active.

# DCGM_FI_PROF_DMMA_CYCLES_ACTIVE_TOTAL, counter, Total DMMA tensor cycles active.

# DCGM_FI_PROF_HMMA_CYCLES_ACTIVE_TOTAL, counter, Total HMMA tensor cycles active.

# DCGM_FI_PROF_IMMA_CYCLES_ACTIVE_TOTAL, counter, Total IMMA tensor cycles active.

# DCGM_FI_PROF_DFMA_CYCLES_ACTIVE_TOTAL, counter, Total DFMA tensor cycles active.

# DCGM_FI_PROF_PCIE_TX_BYTES_TOTAL, counter, Total PCIe transmitted bytes.

# DCGM_FI_PROF_PCIE_RX_BYTES_TOTAL, counter, Total PCIe received bytes.

# DCGM_FI_PROF_INT_CYCLES_ACTIVE_TOTAL, counter, Total integer pipe cycles active.

# DCGM_FI_PROF_FP64_CYCLES_ACTIVE_TOTAL, counter, Total FP64 pipe cycles active.

# DCGM_FI_PROF_FP32_CYCLES_ACTIVE_TOTAL, counter, Total FP32 pipe cycles active.

# DCGM_FI_PROF_FP16_CYCLES_ACTIVE_TOTAL, counter, Total FP16 pipe cycles active.

livenessProbe:

initialDelaySeconds: 45

periodSeconds: 5

# timeoutSeconds and failureThreshold use the Kubernetes defaults when omitted.

# timeoutSeconds: 1

# failureThreshold: 3

readinessProbe:

initialDelaySeconds: 45

# periodSeconds, timeoutSeconds, and failureThreshold use Kubernetes defaults when omitted.

# periodSeconds: 10

# timeoutSeconds: 1

# failureThreshold: 3