From 67b5780d310dea8897db46512b1427b0f1a85f93 Mon Sep 17 00:00:00 2001 From: Andrew Bays Date: Tue, 23 Jun 2026 09:48:23 -0400 Subject: [PATCH] [cifmw_backup_restore] Add baremetal backup/restore support Add support for backing up and restoring baremetal-provisioned compute nodes (BaremetalHosts and OpenStackBaremetalSets). This uses the annotation-based approach from OSPRH-29980 and OSPRH-29529, eliminating the previous workaround of scaling down the openstack-baremetal operator and manually deleting its webhooks. Backup: conditionally creates a separate BMH backup when BaremetalHosts are in a different namespace than OpenStack resources. Restore: adds Steps 9a (BMH restore with status + unpause) and 9b (OSBMS restore with status + annotation removal) between the full deployment resume and the DataPlane restore. The resource modifier template conditionally injects BMH pause, OSBMS webhook-skip, and OSBMS reconcile-pause annotations during restore. All baremetal steps are gated behind cifmw_backup_restore_baremetal (default: false) so existing workflows are unaffected. Jira: OSPRH-31756 Co-Authored-By: Claude Opus 4.6 Signed-off-by: Andrew Bays --- roles/cifmw_backup_restore/defaults/main.yml | 4 + roles/cifmw_backup_restore/tasks/backup.yml | 49 ++++++ roles/cifmw_backup_restore/tasks/restore.yml | 140 ++++++++++++++++++ .../00-resource-modifiers-configmap.yaml.j2 | 17 +++ .../templates/backup-bmh.yaml.j2 | 21 +++ .../templates/restore-bmh-secrets.yaml.j2 | 17 +++ .../templates/restore-bmh.yaml.j2 | 20 +++ .../templates/restore-bmset.yaml.j2 | 20 +++ 8 files changed, 288 insertions(+) create mode 100644 roles/cifmw_backup_restore/templates/backup-bmh.yaml.j2 create mode 100644 roles/cifmw_backup_restore/templates/restore-bmh-secrets.yaml.j2 create mode 100644 roles/cifmw_backup_restore/templates/restore-bmh.yaml.j2 create mode 100644 roles/cifmw_backup_restore/templates/restore-bmset.yaml.j2 diff --git a/roles/cifmw_backup_restore/defaults/main.yml b/roles/cifmw_backup_restore/defaults/main.yml index c776bcc63..301689d23 100644 --- a/roles/cifmw_backup_restore/defaults/main.yml +++ b/roles/cifmw_backup_restore/defaults/main.yml @@ -65,6 +65,10 @@ cifmw_backup_restore_swift_xattr_timeout: 600s cifmw_backup_restore_ovn_db: true cifmw_backup_restore_ovn_db_ready_timeout: 5m +# Baremetal-provisioned compute nodes (BMH + OpenStackBaremetalSet) +cifmw_backup_restore_baremetal: false +cifmw_backup_restore_bmh_namespace: "{{ cifmw_backup_restore_namespace }}" + # Restore # cifmw_backup_restore_backup_timestamp: REQUIRED for restore (e.g., 20260311-081234) cifmw_backup_restore_restore_timeout: 900 diff --git a/roles/cifmw_backup_restore/tasks/backup.yml b/roles/cifmw_backup_restore/tasks/backup.yml index 333a2d66c..c7300b8ca 100644 --- a/roles/cifmw_backup_restore/tasks/backup.yml +++ b/roles/cifmw_backup_restore/tasks/backup.yml @@ -295,6 +295,54 @@ msg: "Resources backup ended with phase: {{ _resources_backup_phase.stdout }}" when: _resources_backup_phase.stdout != "Completed" +# ======================================== +# Step 5: OADP Baremetal Backup (optional — separate namespace only) +# ======================================== +- name: Check if BMH namespace differs from OpenStack namespace + ansible.builtin.set_fact: + _bmh_separate_namespace: "{{ cifmw_backup_restore_bmh_namespace != cifmw_backup_restore_namespace }}" + when: cifmw_backup_restore_baremetal | bool + +- name: Render BMH backup CR + ansible.builtin.template: + src: backup-bmh.yaml.j2 + dest: "{{ _cifmw_backup_restore_rendered_dir.path }}/backup-bmh.yaml" + mode: "0644" + when: + - cifmw_backup_restore_baremetal | bool + - _bmh_separate_namespace | default(false) | bool + +- name: Create OADP BMH backup + kubernetes.core.k8s: + src: "{{ _cifmw_backup_restore_rendered_dir.path }}/backup-bmh.yaml" + state: present + when: + - cifmw_backup_restore_baremetal | bool + - _bmh_separate_namespace | default(false) | bool + +- name: Wait for BMH backup to complete + ansible.builtin.command: + cmd: >- + oc get backup openstack-backup-bmh-{{ cifmw_backup_restore_backup_name_suffix }} + -n {{ cifmw_backup_restore_oadp_namespace }} + -o jsonpath='{.status.phase}' + register: _bmh_backup_phase + changed_when: false + until: _bmh_backup_phase.stdout in ["Completed", "Failed", "PartiallyFailed"] + retries: "{{ (cifmw_backup_restore_oadp_backup_timeout | regex_replace('[^0-9]', '') | int * 60 / 10) | int }}" + delay: 10 + when: + - cifmw_backup_restore_baremetal | bool + - _bmh_separate_namespace | default(false) | bool + +- name: Fail if BMH backup did not complete + ansible.builtin.fail: + msg: "BMH backup ended with phase: {{ _bmh_backup_phase.stdout }}" + when: + - cifmw_backup_restore_baremetal | bool + - _bmh_separate_namespace | default(false) | bool + - _bmh_backup_phase.stdout != "Completed" + # ======================================== # Summary # ======================================== @@ -308,6 +356,7 @@ - "Backup name suffix: {{ cifmw_backup_restore_backup_name_suffix }}" - "PVC backup: openstack-backup-pvcs-{{ cifmw_backup_restore_backup_name_suffix }}" - "Resources backup: openstack-backup-resources-{{ cifmw_backup_restore_backup_name_suffix }}" + - "{% if (cifmw_backup_restore_baremetal | bool) and (_bmh_separate_namespace | default(false) | bool) %}BMH backup: openstack-backup-bmh-{{ cifmw_backup_restore_backup_name_suffix }}{% else %}BMH backup: n/a (baremetal disabled or BMHs in openstack namespace){% endif %}" - "" - "Operator version recorded on Backup CRs:" - " CSV: {{ _backup_csv_version }}" diff --git a/roles/cifmw_backup_restore/tasks/restore.yml b/roles/cifmw_backup_restore/tasks/restore.yml index 2eed66b72..06fc9ad86 100644 --- a/roles/cifmw_backup_restore/tasks/restore.yml +++ b/roles/cifmw_backup_restore/tasks/restore.yml @@ -76,6 +76,25 @@ msg: "Backups must be Completed. PVC: {{ _pvc_backup_phase.stdout }}, Resources: {{ _resources_backup_phase.stdout }}" when: _pvc_backup_phase.stdout != "Completed" or _resources_backup_phase.stdout != "Completed" +- name: Verify BMH backup exists (separate namespace) + ansible.builtin.command: + cmd: >- + oc get backup openstack-backup-bmh-{{ cifmw_backup_restore_backup_timestamp }} + -n {{ cifmw_backup_restore_oadp_namespace }} -o jsonpath='{.status.phase}' + register: _bmh_backup_phase + changed_when: false + when: + - cifmw_backup_restore_baremetal | bool + - cifmw_backup_restore_bmh_namespace != cifmw_backup_restore_namespace + +- name: Fail if BMH backup is not completed + ansible.builtin.fail: + msg: "BMH backup must be Completed. Phase: {{ _bmh_backup_phase.stdout }}" + when: + - cifmw_backup_restore_baremetal | bool + - cifmw_backup_restore_bmh_namespace != cifmw_backup_restore_namespace + - _bmh_backup_phase.stdout != "Completed" + # ======================================== # Operator version validation # ======================================== @@ -394,6 +413,127 @@ --timeout={{ cifmw_backup_restore_ctlplane_ready_timeout }} changed_when: false +# ======================================== +# Step 9a: Restore BaremetalHosts (optional — baremetal-provisioned only) +# ======================================== +- name: Derive BMH backup name + ansible.builtin.set_fact: + _bmh_backup_name: "openstack-backup-bmh-{{ cifmw_backup_restore_backup_timestamp }}" + _bmh_separate_namespace: "{{ cifmw_backup_restore_bmh_namespace != cifmw_backup_restore_namespace }}" + when: cifmw_backup_restore_baremetal | bool + +- name: Render BMH secrets restore (separate namespace) + ansible.builtin.template: + src: restore-bmh-secrets.yaml.j2 + dest: "{{ _cifmw_backup_restore_rendered_dir.path }}/restore-bmh-secrets.yaml" + mode: "0644" + vars: + bmh_backup_name: "{{ _bmh_backup_name }}" + restore_suffix: "{{ _restore_suffix }}" + when: + - cifmw_backup_restore_baremetal | bool + - _bmh_separate_namespace | bool + +- name: Create BMH secrets restore (separate namespace) + kubernetes.core.k8s: + src: "{{ _cifmw_backup_restore_rendered_dir.path }}/restore-bmh-secrets.yaml" + state: present + when: + - cifmw_backup_restore_baremetal | bool + - _bmh_separate_namespace | bool + +- name: Wait for BMH secrets restore + ansible.builtin.include_tasks: wait_for_restore.yml + vars: + _restore_name: "openstack-restore-bmh-secrets-{{ _restore_suffix }}" + _step_name: "Step 9a (BMH secrets restore)" + when: + - cifmw_backup_restore_baremetal | bool + - _bmh_separate_namespace | bool + +- name: Render BMH restore + ansible.builtin.template: + src: restore-bmh.yaml.j2 + dest: "{{ _cifmw_backup_restore_rendered_dir.path }}/restore-bmh.yaml" + mode: "0644" + vars: + bmh_backup_name: >- + {{ _bmh_backup_name if (_bmh_separate_namespace | bool) else _resources_backup_name }} + restore_suffix: "{{ _restore_suffix }}" + when: cifmw_backup_restore_baremetal | bool + +- name: Create BMH restore + kubernetes.core.k8s: + src: "{{ _cifmw_backup_restore_rendered_dir.path }}/restore-bmh.yaml" + state: present + when: cifmw_backup_restore_baremetal | bool + +- name: Wait for BMH restore + ansible.builtin.include_tasks: wait_for_restore.yml + vars: + _restore_name: "openstack-restore-bmh-{{ _restore_suffix }}" + _step_name: "Step 9a (BMH restore)" + when: cifmw_backup_restore_baremetal | bool + +- name: Remove BMH pause annotations + ansible.builtin.command: + cmd: >- + oc annotate bmh --all + -n {{ cifmw_backup_restore_bmh_namespace }} + baremetalhost.metal3.io/paused- + changed_when: true + when: cifmw_backup_restore_baremetal | bool + +- name: Verify BMH state + ansible.builtin.command: + cmd: >- + oc get bmh -n {{ cifmw_backup_restore_bmh_namespace }} + -o custom-columns=NAME:.metadata.name,STATE:.status.provisioning.state,ONLINE:.spec.online --no-headers + register: _bmh_state + changed_when: false + when: cifmw_backup_restore_baremetal | bool + +- name: Display BMH state + ansible.builtin.debug: + msg: "{{ _bmh_state.stdout_lines }}" + when: cifmw_backup_restore_baremetal | bool + +# ======================================== +# Step 9b: Restore OpenStackBaremetalSets (optional — baremetal-provisioned only) +# ======================================== +- name: Render OSBMS restore + ansible.builtin.template: + src: restore-bmset.yaml.j2 + dest: "{{ _cifmw_backup_restore_rendered_dir.path }}/restore-bmset.yaml" + mode: "0644" + vars: + resources_backup_name: "{{ _resources_backup_name }}" + restore_suffix: "{{ _restore_suffix }}" + when: cifmw_backup_restore_baremetal | bool + +- name: Create OSBMS restore + kubernetes.core.k8s: + src: "{{ _cifmw_backup_restore_rendered_dir.path }}/restore-bmset.yaml" + state: present + when: cifmw_backup_restore_baremetal | bool + +- name: Wait for OSBMS restore + ansible.builtin.include_tasks: wait_for_restore.yml + vars: + _restore_name: "openstack-restore-55-bmset-{{ _restore_suffix }}" + _step_name: "Step 9b (OpenStackBaremetalSet restore)" + when: cifmw_backup_restore_baremetal | bool + +- name: Remove OSBMS webhook-skip and pause annotations + ansible.builtin.command: + cmd: >- + oc annotate openstackbaremetalset --all + -n {{ cifmw_backup_restore_namespace }} + openstack.org/skip-webhook-validation- + openstack.org/paused- + changed_when: true + when: cifmw_backup_restore_baremetal | bool + # ======================================== # Step 10: Restore DataPlane (Order 60) # ======================================== diff --git a/roles/cifmw_backup_restore/templates/00-resource-modifiers-configmap.yaml.j2 b/roles/cifmw_backup_restore/templates/00-resource-modifiers-configmap.yaml.j2 index 7c214f134..95eab754d 100644 --- a/roles/cifmw_backup_restore/templates/00-resource-modifiers-configmap.yaml.j2 +++ b/roles/cifmw_backup_restore/templates/00-resource-modifiers-configmap.yaml.j2 @@ -31,3 +31,20 @@ data: - patchData: | spec: disabled: "True" +{% if cifmw_backup_restore_baremetal | bool %} + - conditions: + groupResource: baremetalhosts.metal3.io + mergePatches: + - patchData: | + metadata: + annotations: + baremetalhost.metal3.io/paused: "" + - conditions: + groupResource: openstackbaremetalsets.baremetal.openstack.org + mergePatches: + - patchData: | + metadata: + annotations: + openstack.org/skip-webhook-validation: "" + openstack.org/paused: "" +{% endif %} diff --git a/roles/cifmw_backup_restore/templates/backup-bmh.yaml.j2 b/roles/cifmw_backup_restore/templates/backup-bmh.yaml.j2 new file mode 100644 index 000000000..f59231afb --- /dev/null +++ b/roles/cifmw_backup_restore/templates/backup-bmh.yaml.j2 @@ -0,0 +1,21 @@ +--- +# Baremetal Hosts Backup (separate namespace) +# Backs up BaremetalHosts, Secrets, and ConfigMaps from the BMH namespace. +apiVersion: velero.io/v1 +kind: Backup +metadata: + name: openstack-backup-bmh-{{ cifmw_backup_restore_backup_name_suffix }} + namespace: {{ cifmw_backup_restore_oadp_namespace }} + annotations: + openstack.org/csv-version: "{{ _backup_csv_version }}" + openstack.org/catalog-source-image: "{{ _backup_catalog_image }}" + openstack.org/operator-image: "{{ _backup_operator_image }}" +spec: + includedNamespaces: + - {{ cifmw_backup_restore_bmh_namespace }} + includedResources: + - baremetalhosts.metal3.io + - secrets + - configmaps + storageLocation: {{ cifmw_backup_restore_storage_location }} + ttl: {{ cifmw_backup_restore_backup_ttl }} diff --git a/roles/cifmw_backup_restore/templates/restore-bmh-secrets.yaml.j2 b/roles/cifmw_backup_restore/templates/restore-bmh-secrets.yaml.j2 new file mode 100644 index 000000000..1fa5da0d4 --- /dev/null +++ b/roles/cifmw_backup_restore/templates/restore-bmh-secrets.yaml.j2 @@ -0,0 +1,17 @@ +--- +# Restore BMH Secrets and ConfigMaps (separate namespace only) +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: openstack-restore-bmh-secrets-{{ restore_suffix }} + namespace: {{ cifmw_backup_restore_oadp_namespace }} +spec: + backupName: {{ bmh_backup_name }} + includedNamespaces: + - {{ cifmw_backup_restore_bmh_namespace }} + includedResources: + - secrets + - configmaps + resourceModifier: + kind: ConfigMap + name: openstack-restore-resource-modifiers diff --git a/roles/cifmw_backup_restore/templates/restore-bmh.yaml.j2 b/roles/cifmw_backup_restore/templates/restore-bmh.yaml.j2 new file mode 100644 index 000000000..34b45293a --- /dev/null +++ b/roles/cifmw_backup_restore/templates/restore-bmh.yaml.j2 @@ -0,0 +1,20 @@ +--- +# Restore BaremetalHosts with status +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: openstack-restore-bmh-{{ restore_suffix }} + namespace: {{ cifmw_backup_restore_oadp_namespace }} +spec: + backupName: {{ bmh_backup_name }} + includedNamespaces: + - {{ cifmw_backup_restore_bmh_namespace }} + includedResources: + - baremetalhosts.metal3.io + restorePVs: false + restoreStatus: + includedResources: + - baremetalhosts.metal3.io + resourceModifier: + kind: ConfigMap + name: openstack-restore-resource-modifiers diff --git a/roles/cifmw_backup_restore/templates/restore-bmset.yaml.j2 b/roles/cifmw_backup_restore/templates/restore-bmset.yaml.j2 new file mode 100644 index 000000000..9b440aa35 --- /dev/null +++ b/roles/cifmw_backup_restore/templates/restore-bmset.yaml.j2 @@ -0,0 +1,20 @@ +--- +# Restore OpenStackBaremetalSets with status +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: openstack-restore-55-bmset-{{ restore_suffix }} + namespace: {{ cifmw_backup_restore_oadp_namespace }} +spec: + backupName: {{ resources_backup_name }} + includedNamespaces: + - {{ cifmw_backup_restore_namespace }} + includedResources: + - openstackbaremetalsets.baremetal.openstack.org + - openstackprovisionservers.baremetal.openstack.org + restoreStatus: + includedResources: + - openstackbaremetalsets.baremetal.openstack.org + resourceModifier: + kind: ConfigMap + name: openstack-restore-resource-modifiers