From 2766919e5206f3208829f32ee0722de76575f35d Mon Sep 17 00:00:00 2001 From: Steve McMaster Date: Sun, 21 Jul 2024 23:40:05 -0400 Subject: [PATCH] Add nvidia-smi sidecar when requesting gpus --- base.tf.j2 | 22 ++++++++++++++++++++++ splunk.tf.j2 | 1 - 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/base.tf.j2 b/base.tf.j2 index 4772dc9..ea535f4 100644 --- a/base.tf.j2 +++ b/base.tf.j2 @@ -484,6 +484,28 @@ resource "kubernetes_deployment" "main" { } {% endblock %} + {% block kubernetes_pod_nvidia_sidecar %} + dynamic "container" { + for_each = data.coder_parameter.gpu.value == 0 ? [] : [1] + + content { + name = "nvidia-smi" + image = "nvidia/cuda:12.5.1-cudnn-devel-ubuntu22.04" + command = ["bash", "-c", "watch -n1 nvidia-smi"] + image_pull_policy = "{{ image_pull_policy }}" + + resources { + limits = { + "cpu" = "250m" + "memory" = "512Mi" + "nvidia.com/gpu" = 0 + } + } + } + } + + {% endblock %} + {% block kubernetes_pod_main_volume %} volume { name = "home" diff --git a/splunk.tf.j2 b/splunk.tf.j2 index 6f45da4..664112c 100644 --- a/splunk.tf.j2 +++ b/splunk.tf.j2 @@ -69,7 +69,6 @@ resource "coder_script" "splunk_ansible" { script = <