Skip to content

Commit 19a7768

Browse files
sriram-30sriram-30
committed
GPU Operator Integration with Remediation Workflows using Argo Workflows
1 parent 55f57e4 commit 19a7768

265 files changed

Lines changed: 93982 additions & 864 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Makefile‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,10 @@ ifdef SKIP_INSTALL_DEFAULT_CR
7070
SKIP_INSTALL_DEFAULT_CR_CMD=--set crds.defaultCR.install=false
7171
endif
7272

73+
ifdef SKIP_REMEDIATION_CONTROLLER
74+
SKIP_REMEDIATION_CONTROLLER_CMD=--set remediation.enabled=false
75+
endif
76+
7377
#################################
7478
# OpenShift OLM Bundle varaiables
7579
# BUNDLE_IMG defines the image:tag used for the bundle.
@@ -324,6 +328,9 @@ helm-k8s: helmify manifests kustomize clean-helm-k8s gen-kmm-charts-k8s ## Build
324328
# Patching k8s helm chart kmm subchart
325329
cp $(shell pwd)/hack/k8s-patch/k8s-kmm-patch/metadata-patch/*.yaml $(shell pwd)/helm-charts-k8s/charts/kmm/
326330
cp $(shell pwd)/hack/k8s-patch/k8s-kmm-patch/template-patch/*.yaml $(shell pwd)/helm-charts-k8s/charts/kmm/templates/
331+
mkdir -p $(shell pwd)/helm-charts-k8s/charts/remediation/templates
332+
cp $(shell pwd)/hack/k8s-patch/k8s-remediation-patch/metadata-patch/*.yaml $(shell pwd)/helm-charts-k8s/charts/remediation/
333+
cp $(shell pwd)/hack/k8s-patch/k8s-remediation-patch/template-patch/*.yaml $(shell pwd)/helm-charts-k8s/charts/remediation/templates/
327334
cd $(shell pwd)/helm-charts-k8s; helm dependency update; helm lint; cd ..;
328335
mkdir $(shell pwd)/helm-charts-k8s/crds
329336
echo "moving crd yaml files to crds folder"
@@ -574,7 +581,7 @@ helm-uninstall-openshift:
574581
helm uninstall amd-gpu-operator -n kube-amd-gpu
575582

576583
helm-install-k8s:
577-
helm install -f helm-charts-k8s/values.yaml amd-gpu-operator ${GPU_OPERATOR_CHART} -n kube-amd-gpu --create-namespace ${SKIP_NFD_CMD} ${SKIP_KMM_CMD} ${HELM_OC_CMD} ${SIM_ENABLE_CMD} ${SKIP_INSTALL_DEFAULT_CR_CMD}
584+
helm install -f helm-charts-k8s/values.yaml amd-gpu-operator ${GPU_OPERATOR_CHART} -n kube-amd-gpu --create-namespace ${SKIP_NFD_CMD} ${SKIP_KMM_CMD} ${SKIP_REMEDIATION_CONTROLLER_CMD} ${HELM_OC_CMD} ${SIM_ENABLE_CMD} ${SKIP_INSTALL_DEFAULT_CR_CMD}
578585

579586
helm-uninstall-k8s:
580587
echo "Deleting all device configs before uninstalling operator..."

‎api/v1alpha1/deviceconfig_types.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,28 @@ type DeviceConfigSpec struct {
7575
//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Selector",xDescriptors={"urn:alm:descriptor:com.amd.deviceconfigs:selector"}
7676
// +optional
7777
Selector map[string]string `json:"selector,omitempty"`
78+
79+
// remediation workflow
80+
//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="RemediationWorkflow",xDescriptors={"urn:alm:descriptor:com.amd.deviceconfigs:remediationWorkflow"}
81+
// +optional
82+
RemediationWorkflow RemediationWorkflowSpec `json:"remediationWorkflow,omitempty"`
83+
}
84+
85+
// RemediationWorkflowSpec defines workflows to run based on node conditions
86+
type RemediationWorkflowSpec struct {
87+
// enable remediation workflows. disabled by default
88+
// enable if operator should automatically handle remediation of node incase of gpu issues
89+
//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Enable",xDescriptors={"urn:alm:descriptor:com.amd.deviceconfigs:enable"}
90+
Enable *bool `json:"enable,omitempty"`
91+
92+
// Name of the ConfigMap that holds condition-to-workflow mappings.
93+
//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="ConditionalWorkflows",xDescriptors={"urn:alm:descriptor:com.amd.deviceconfigs:conditionalWorkflows"}
94+
ConditionalWorkflows *v1.LocalObjectReference `json:"conditionalWorkflows,omitempty"`
95+
96+
// Time to live for argo workflow object and its pods for a failed workflow in hours. By default, it is set to 24 hours
97+
//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="TtlForFailedWorkflows",xDescriptors={"urn:alm:descriptor:com.amd.deviceconfigs:ttlForFailedWorkflows"}
98+
// +kubebuilder:default:=24
99+
TtlForFailedWorkflows int `json:"ttlForFailedWorkflows,omitempty"`
78100
}
79101

80102
type RegistryTLS struct {

‎api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 26 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎bundle/manifests/amd-gpu-operator.clusterserviceversion.yaml‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ metadata:
3232
capabilities: Seamless Upgrades
3333
categories: AI/Machine Learning,Monitoring
3434
containerImage: docker.io/rocm/gpu-operator:v1.2.0
35-
createdAt: "2025-06-12T00:51:00Z"
35+
createdAt: "2025-07-17T08:55:25Z"
3636
description: |-
3737
Operator responsible for deploying AMD GPU kernel drivers, device plugin, device test runner and device metrics exporter
3838
For more information, visit [documentation](https://instinct.docs.amd.com/projects/gpu-operator/en/latest/)
@@ -598,6 +598,28 @@ spec:
598598
path: metricsExporter.upgradePolicy.upgradeStrategy
599599
x-descriptors:
600600
- urn:alm:descriptor:com.amd.deviceconfigs:upgradeStrategy
601+
- description: remediation workflow
602+
displayName: RemediationWorkflow
603+
path: remediationWorkflow
604+
x-descriptors:
605+
- urn:alm:descriptor:com.amd.deviceconfigs:remediationWorkflow
606+
- description: Name of the ConfigMap that holds condition-to-workflow mappings.
607+
displayName: ConditionalWorkflows
608+
path: remediationWorkflow.conditionalWorkflows
609+
x-descriptors:
610+
- urn:alm:descriptor:com.amd.deviceconfigs:conditionalWorkflows
611+
- description: enable remediation workflows. disabled by default enable if operator
612+
should automatically handle remediation of node incase of gpu issues
613+
displayName: Enable
614+
path: remediationWorkflow.enable
615+
x-descriptors:
616+
- urn:alm:descriptor:com.amd.deviceconfigs:enable
617+
- description: Time to live for argo workflow object and its pods for a failed
618+
workflow in hours. By default, it is set to 24 hours
619+
displayName: TtlForFailedWorkflows
620+
path: remediationWorkflow.ttlForFailedWorkflows
621+
x-descriptors:
622+
- urn:alm:descriptor:com.amd.deviceconfigs:ttlForFailedWorkflows
601623
- description: Selector describes on which nodes the GPU Operator should enable
602624
the GPU device.
603625
displayName: Selector

‎bundle/manifests/amd.com_deviceconfigs.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1180,6 +1180,36 @@ spec:
11801180
type: string
11811181
type: object
11821182
type: object
1183+
remediationWorkflow:
1184+
description: remediation workflow
1185+
properties:
1186+
conditionalWorkflows:
1187+
description: Name of the ConfigMap that holds condition-to-workflow
1188+
mappings.
1189+
properties:
1190+
name:
1191+
default: ""
1192+
description: |-
1193+
Name of the referent.
1194+
This field is effectively required, but due to backwards compatibility is
1195+
allowed to be empty. Instances of this type with an empty value here are
1196+
almost certainly wrong.
1197+
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
1198+
type: string
1199+
type: object
1200+
x-kubernetes-map-type: atomic
1201+
enable:
1202+
description: |-
1203+
enable remediation workflows. disabled by default
1204+
enable if operator should automatically handle remediation of node incase of gpu issues
1205+
type: boolean
1206+
ttlForFailedWorkflows:
1207+
default: 24
1208+
description: Time to live for argo workflow object and its pods
1209+
for a failed workflow in hours. By default, it is set to 24
1210+
hours
1211+
type: integer
1212+
type: object
11831213
selector:
11841214
additionalProperties:
11851215
type: string

‎cmd/main.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ package main
3535
import (
3636
"flag"
3737

38+
workflowv1alpha1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
3839
kmmv1beta1 "github.com/rh-ecosystem-edge/kernel-module-management/api/v1beta1"
3940
"k8s.io/apimachinery/pkg/runtime"
4041
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -75,6 +76,7 @@ func init() {
7576
utilruntime.Must(kmmv1beta1.AddToScheme(scheme))
7677
utilruntime.Must(apiextensionsv1.AddToScheme(scheme))
7778
utilruntime.Must(monitoringv1.AddToScheme(scheme))
79+
utilruntime.Must(workflowv1alpha1.AddToScheme(scheme))
7880
//+kubebuilder:scaffold:scheme
7981
}
8082

‎config/crd/bases/amd.com_deviceconfigs.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1176,6 +1176,36 @@ spec:
11761176
type: string
11771177
type: object
11781178
type: object
1179+
remediationWorkflow:
1180+
description: remediation workflow
1181+
properties:
1182+
conditionalWorkflows:
1183+
description: Name of the ConfigMap that holds condition-to-workflow
1184+
mappings.
1185+
properties:
1186+
name:
1187+
default: ""
1188+
description: |-
1189+
Name of the referent.
1190+
This field is effectively required, but due to backwards compatibility is
1191+
allowed to be empty. Instances of this type with an empty value here are
1192+
almost certainly wrong.
1193+
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
1194+
type: string
1195+
type: object
1196+
x-kubernetes-map-type: atomic
1197+
enable:
1198+
description: |-
1199+
enable remediation workflows. disabled by default
1200+
enable if operator should automatically handle remediation of node incase of gpu issues
1201+
type: boolean
1202+
ttlForFailedWorkflows:
1203+
default: 24
1204+
description: Time to live for argo workflow object and its pods
1205+
for a failed workflow in hours. By default, it is set to 24
1206+
hours
1207+
type: integer
1208+
type: object
11791209
selector:
11801210
additionalProperties:
11811211
type: string

‎config/manifests/bases/amd-gpu-operator.clusterserviceversion.yaml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -569,6 +569,28 @@ spec:
569569
path: metricsExporter.upgradePolicy.upgradeStrategy
570570
x-descriptors:
571571
- urn:alm:descriptor:com.amd.deviceconfigs:upgradeStrategy
572+
- description: remediation workflow
573+
displayName: RemediationWorkflow
574+
path: remediationWorkflow
575+
x-descriptors:
576+
- urn:alm:descriptor:com.amd.deviceconfigs:remediationWorkflow
577+
- description: Name of the ConfigMap that holds condition-to-workflow mappings.
578+
displayName: ConditionalWorkflows
579+
path: remediationWorkflow.conditionalWorkflows
580+
x-descriptors:
581+
- urn:alm:descriptor:com.amd.deviceconfigs:conditionalWorkflows
582+
- description: enable remediation workflows. disabled by default enable if operator
583+
should automatically handle remediation of node incase of gpu issues
584+
displayName: Enable
585+
path: remediationWorkflow.enable
586+
x-descriptors:
587+
- urn:alm:descriptor:com.amd.deviceconfigs:enable
588+
- description: Time to live for argo workflow object and its pods for a failed
589+
workflow in hours. By default, it is set to 24 hours
590+
displayName: TtlForFailedWorkflows
591+
path: remediationWorkflow.ttlForFailedWorkflows
592+
x-descriptors:
593+
- urn:alm:descriptor:com.amd.deviceconfigs:ttlForFailedWorkflows
572594
- description: Selector describes on which nodes the GPU Operator should enable
573595
the GPU device.
574596
displayName: Selector

‎go.mod‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
module github.com/ROCm/gpu-operator
22

3-
go 1.23.0
3+
go 1.23.1
44

55
toolchain go1.23.4
66

77
require (
8+
github.com/argoproj/argo-workflows/v3 v3.6.5
89
github.com/go-logr/logr v1.4.2
910
github.com/onsi/ginkgo/v2 v2.22.0
1011
github.com/onsi/gomega v1.36.1
@@ -13,7 +14,7 @@ require (
1314
github.com/prometheus/common v0.55.0
1415
github.com/rh-ecosystem-edge/kernel-module-management v0.0.0-20250217131402-3522d8ca4d5f
1516
github.com/sirupsen/logrus v1.9.3
16-
github.com/stretchr/testify v1.9.0
17+
github.com/stretchr/testify v1.10.0
1718
go.uber.org/mock v0.4.0
1819
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
1920
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
@@ -23,7 +24,7 @@ require (
2324
k8s.io/apimachinery v0.32.2
2425
k8s.io/client-go v0.32.2
2526
k8s.io/klog/v2 v2.130.1
26-
k8s.io/kubectl v0.29.3
27+
k8s.io/kubectl v0.30.3
2728
k8s.io/utils v0.0.0-20241210054802-24370beab758
2829
open-cluster-management.io/api v0.13.0
2930
sigs.k8s.io/controller-runtime v0.20.3
@@ -39,7 +40,7 @@ require (
3940
github.com/chai2010/gettext-go v1.0.2 // indirect
4041
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
4142
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
42-
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
43+
github.com/evanphx/json-patch v5.8.0+incompatible // indirect
4344
github.com/evanphx/json-patch/v5 v5.9.11 // indirect
4445
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
4546
github.com/fsnotify/fsnotify v1.7.0 // indirect
@@ -58,9 +59,10 @@ require (
5859
github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect
5960
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
6061
github.com/google/uuid v1.6.0 // indirect
61-
github.com/gorilla/websocket v1.5.0 // indirect
62+
github.com/gorilla/websocket v1.5.1 // indirect
6263
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
63-
github.com/imdario/mergo v0.3.13 // indirect
64+
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
65+
github.com/imdario/mergo v0.3.15 // indirect
6466
github.com/inconshreveable/mousetrap v1.1.0 // indirect
6567
github.com/josharian/intern v1.0.0 // indirect
6668
github.com/json-iterator/go v1.1.12 // indirect
@@ -90,19 +92,23 @@ require (
9092
github.com/xlab/treeprint v1.2.0 // indirect
9193
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
9294
golang.org/x/net v0.34.0 // indirect
93-
golang.org/x/oauth2 v0.25.0 // indirect
94-
golang.org/x/sync v0.10.0 // indirect
95-
golang.org/x/sys v0.29.0 // indirect
96-
golang.org/x/term v0.28.0 // indirect
97-
golang.org/x/text v0.21.0 // indirect
95+
golang.org/x/oauth2 v0.28.0 // indirect
96+
golang.org/x/sync v0.12.0 // indirect
97+
golang.org/x/sys v0.31.0 // indirect
98+
golang.org/x/term v0.30.0 // indirect
99+
golang.org/x/text v0.23.0 // indirect
98100
golang.org/x/time v0.9.0 // indirect
99101
golang.org/x/tools v0.26.0 // indirect
100102
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
103+
google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect
104+
google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect
105+
google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect
106+
google.golang.org/grpc v1.65.0 // indirect
101107
google.golang.org/protobuf v1.36.3 // indirect
102108
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
103109
gopkg.in/inf.v0 v0.9.1 // indirect
104110
gopkg.in/yaml.v2 v2.4.0 // indirect
105-
k8s.io/cli-runtime v0.29.3 // indirect
111+
k8s.io/cli-runtime v0.30.3 // indirect
106112
k8s.io/component-base v0.32.2 // indirect
107113
k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 // indirect
108114
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect

0 commit comments

Comments
 (0)