DBAAS-1123: delete adopted objects that are obsolete

Makefile

@@ -10,7 +10,7 @@ OPERATOR_SDK_VERSION ?= v1.25.4
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
 VERSION ?= 0.3.0
 
-RDS_VERSION ?= 0.1.3
+RDS_VERSION ?= 1.0.0
 
 # QUAY_ORG indicates the organization that docker images will be build for & pushed to
 # CHANGE THIS TO YOUR OWN QUAY USERNAME FOR DEV/TESTING/PUSHING
@@ -277,7 +277,7 @@ RDS_TEMP = ./controllers/yaml/temp
 RDS_CONFIG = ./controllers/yaml/rds
 rds-crds-clean: ## Cleanup the RDS CRD files.
 	find $(RDS_CONFIG) -type f -delete
-	rm $(RDS_TEMP)/v$(RDS_VERSION).zip
+	rm $(RDS_TEMP)/*
 
 .PHONY: sdk
 SDK = ./bin/operator-sdk.$(OPERATOR_SDK_VERSION)

bundle/manifests/rds-dbaas-operator.clusterserviceversion.yaml

@@ -234,6 +234,7 @@ spec:
           resources:
           - dbclusters
           verbs:
+          - delete
           - get
           - list
           - update

bundle/metadata/dependencies.yaml

@@ -2,4 +2,4 @@ dependencies:
   - type: olm.package
     value:
       packageName: ack-rds-controller
-      version: ">=0.0.27 <=0.1.3"
+      version: "1.0.0"

config/rbac/role.yaml

@@ -142,6 +142,7 @@ rules:
   resources:
   - dbclusters
   verbs:
+  - delete
   - get
   - list
   - update

controllers/rds/test/dbcluster_mock.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"sync/atomic"
 
 	"k8s.io/utils/pointer"
 
@@ -29,6 +30,16 @@ import (
 	controllersrds "github.com/RHEcosystemAppEng/rds-dbaas-operator/controllers/rds"
 )
 
+var inventoryTestDBClusterCounter int32 = 6
+
+func GetInventoryTestDBClusterCounter() int32 {
+	return atomic.LoadInt32(&inventoryTestDBClusterCounter)
+}
+
+func SetInventoryTestDBClusterCounter(counter int32) {
+	atomic.StoreInt32(&inventoryTestDBClusterCounter, counter)
+}
+
 var inventoryTestDBClusters = []*rds.DescribeDBClustersOutput{
 	{
 		DBClusters: []types.DBCluster{
@@ -222,6 +233,17 @@ var inventoryTestDBClusters = []*rds.DescribeDBClustersOutput{
 			},
 		},
 	},
+	{
+		// To be deleted
+		DBClusters: []types.DBCluster{
+			{
+				DBClusterIdentifier: pointer.String("mock-db-cluster-delete-1"),
+				Status:              pointer.String("available"),
+				Engine:              pointer.String("postgres"),
+				DBClusterArn:        pointer.String("mock-db-cluster-delete-1"),
+			},
+		},
+	},
 }
 
 var connectionTestDBClusters = []*rds.DescribeDBClustersOutput{
@@ -250,7 +272,7 @@ type mockDescribeDBClustersPaginator struct {
 func NewDescribeDBClustersPaginator(accessKey, secretKey, region string) controllersrds.DescribeDBClustersPaginatorAPI {
 	counter := 0
 	if strings.HasSuffix(accessKey, ClusterInventoryControllerTestAccessKeySuffix) {
-		counter = 5
+		counter = int(GetInventoryTestDBClusterCounter())
 	} else if strings.HasSuffix(accessKey, ClusterConnectionControllerTestAccessKeySuffix) {
 		counter = 1
 	}

controllers/rds/test/dbinstance_mock.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"sync/atomic"
 
 	"k8s.io/utils/pointer"
 
@@ -28,6 +29,16 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/rds/types"
 )
 
+var inventoryTestDBInstanceCounter int32 = 6
+
+func GetInventoryTestDBInstanceCounter() int32 {
+	return atomic.LoadInt32(&inventoryTestDBInstanceCounter)
+}
+
+func SetInventoryTestDBInstanceCounter(counter int32) {
+	atomic.StoreInt32(&inventoryTestDBInstanceCounter, counter)
+}
+
 var inventoryTestDBInstances = []*rds.DescribeDBInstancesOutput{
 	{
 		DBInstances: []types.DBInstance{
@@ -199,6 +210,17 @@ var inventoryTestDBInstances = []*rds.DescribeDBInstancesOutput{
 			},
 		},
 	},
+	{
+		// To be deleted
+		DBInstances: []types.DBInstance{
+			{
+				DBInstanceIdentifier: pointer.String("mock-db-instance-delete-1"),
+				DBInstanceStatus:     pointer.String("available"),
+				Engine:               pointer.String("postgres"),
+				DBInstanceArn:        pointer.String("mock-db-instance-delete-1"),
+			},
+		},
+	},
 }
 
 var connectionTestDBInstances = []*rds.DescribeDBInstancesOutput{
@@ -240,7 +262,7 @@ type mockDescribeDBInstancesPaginator struct {
 func NewDescribeDBInstancesPaginator(accessKey, secretKey, region string) controllersrds.DescribeDBInstancesPaginatorAPI {
 	counter := 0
 	if strings.HasSuffix(accessKey, InventoryControllerTestAccessKeySuffix) {
-		counter = 5
+		counter = int(GetInventoryTestDBInstanceCounter())
 	} else if strings.HasSuffix(accessKey, ConnectionControllerTestAccessKeySuffix) {
 		counter = 1
 	}

controllers/rdsinventory_controller.go

@@ -83,6 +83,8 @@ const (
 	inventoryStatusMessageUpdateError              = "Failed to update Inventory"
 	inventoryStatusMessageGetInstancesError        = "Failed to get Instances"
 	inventoryStatusMessageGetClustersError         = "Failed to get clusters"
+	inventoryStatusMessageDeleteInstancesError     = "Failed to delete Instances"
+	inventoryStatusMessageDeleteClustersError      = "Failed to delete clusters"
 	inventoryStatusMessageAdoptInstanceError       = "Failed to adopt DB Instance"
 	inventoryStatusMessageAdoptClusterError        = "Failed to adopt DB Cluster"
 	inventoryStatusMessageUpdateInstanceError      = "Failed to update DB Instance"
@@ -124,9 +126,9 @@ type RDSInventoryReconciler struct {
 //+kubebuilder:rbac:groups=dbaas.redhat.com,resources=rdsinventories,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=dbaas.redhat.com,resources=rdsinventories/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=dbaas.redhat.com,resources=rdsinventories/finalizers,verbs=update
-//+kubebuilder:rbac:groups=rds.services.k8s.aws,resources=dbinstances,verbs=get;list;watch;update
+//+kubebuilder:rbac:groups=rds.services.k8s.aws,resources=dbinstances,verbs=get;list;watch;update;delete
 //+kubebuilder:rbac:groups=rds.services.k8s.aws,resources=dbinstances/finalizers,verbs=update
-//+kubebuilder:rbac:groups=rds.services.k8s.aws,resources=dbclusters,verbs=get;list;watch;update
+//+kubebuilder:rbac:groups=rds.services.k8s.aws,resources=dbclusters,verbs=get;list;watch;update;delete
 //+kubebuilder:rbac:groups=rds.services.k8s.aws,resources=dbclusters/finalizers,verbs=update
 //+kubebuilder:rbac:groups=services.k8s.aws,resources=adoptedresources,verbs=get;list;watch;create;delete;update
 //+kubebuilder:rbac:groups="",resources=secrets;configmaps,verbs=get;list;watch;create;delete;update
@@ -503,6 +505,21 @@ func (r *RDSInventoryReconciler) Reconcile(ctx context.Context, req ctrl.Request
 			returnError(e, inventoryStatusReasonBackendError, inventoryStatusMessageGetInstancesError)
 			return true, false
 		}
+
+		adoptedResourceList := &ackv1alpha1.AdoptedResourceList{}
+		if e := r.List(ctx, adoptedResourceList, client.InNamespace(inventory.Namespace),
+			client.MatchingLabels(map[string]string{adoptedDBResourceLabelKey: adoptedDBResourceLabelValue})); e != nil {
+			logger.Error(e, "Failed to read adopted Resources of the Inventory that are created by the operator")
+			returnError(e, inventoryStatusReasonBackendError, inventoryStatusMessageGetInstancesError)
+			return true, false
+		}
+		adoptedDBInstanceMap := make(map[string]ackv1alpha1.AdoptedResource, len(adoptedResourceList.Items))
+		for _, adoptedDBInstance := range adoptedResourceList.Items {
+			if adoptedDBInstance.Spec.AWS != nil && adoptedDBInstance.Spec.AWS.ARN != nil {
+				adoptedDBInstanceMap[string(*adoptedDBInstance.Spec.AWS.ARN)] = adoptedDBInstance
+			}
+		}
+
 		modifyDBInstance := r.GetModifyDBInstanceAPI(accessKey, secretKey, region)
 		waitForAdoptedResource := false
 		for i := range adoptedDBInstanceList.Items {
@@ -527,6 +544,23 @@ func (r *RDSInventoryReconciler) Reconcile(ctx context.Context, req ctrl.Request
 			}
 			awsDBInstance, awsOk := awsDBInstanceMap[string(*adoptedDBInstance.Status.ACKResourceMetadata.ARN)]
 			if !awsOk {
+				if e := r.Delete(ctx, &adoptedDBInstance); e != nil {
+					if !errors.IsNotFound(e) {
+						logger.Error(e, "Failed to delete obsolete adopted DB Instance", "DB Instance", adoptedDBInstance)
+						returnError(e, inventoryStatusReasonBackendError, inventoryStatusMessageDeleteInstancesError)
+						return true, false
+					}
+				}
+				if instance, ok := adoptedDBInstanceMap[string(*adoptedDBInstance.Status.ACKResourceMetadata.ARN)]; ok {
+					if e := r.Delete(ctx, &instance); e != nil {
+						if !errors.IsNotFound(e) {
+							logger.Error(e, "Failed to delete adopting resource", "DB Instance", adoptedDBInstance)
+							returnError(e, inventoryStatusReasonBackendError, inventoryStatusMessageDeleteInstancesError)
+							return true, false
+						}
+					}
+				}
+				logger.Info("Deleted adopted DB Instance that is obsolete", "DB Cluster", adoptedDBInstance)
 				continue
 			}
 
@@ -730,6 +764,21 @@ func (r *RDSInventoryReconciler) Reconcile(ctx context.Context, req ctrl.Request
 			returnError(e, inventoryStatusReasonBackendError, inventoryStatusMessageGetClustersError)
 			return true, false
 		}
+
+		adoptedResourceList := &ackv1alpha1.AdoptedResourceList{}
+		if e := r.List(ctx, adoptedResourceList, client.InNamespace(inventory.Namespace),
+			client.MatchingLabels(map[string]string{adoptedDBResourceLabelKey: adoptedDBResourceLabelValue})); e != nil {
+			logger.Error(e, "Failed to read adopted Resources of the Inventory that are created by the operator")
+			returnError(e, inventoryStatusReasonBackendError, inventoryStatusMessageGetClustersError)
+			return true, false
+		}
+		adoptedDBClusterMap := make(map[string]ackv1alpha1.AdoptedResource, len(adoptedResourceList.Items))
+		for _, adoptedDBCluster := range adoptedResourceList.Items {
+			if adoptedDBCluster.Spec.AWS != nil && adoptedDBCluster.Spec.AWS.ARN != nil {
+				adoptedDBClusterMap[string(*adoptedDBCluster.Spec.AWS.ARN)] = adoptedDBCluster
+			}
+		}
+
 		modifyDBCluster := r.GetModifyDBClusterAPI(accessKey, secretKey, region)
 		waitForAdoptedResource := false
 		for i := range adoptedDBClusterList.Items {
@@ -752,6 +801,23 @@ func (r *RDSInventoryReconciler) Reconcile(ctx context.Context, req ctrl.Request
 			}
 			awsDBCluster, awsOk := awsDBClusterMap[string(*adoptedDBCluster.Status.ACKResourceMetadata.ARN)]
 			if !awsOk {
+				if e := r.Delete(ctx, &adoptedDBCluster); e != nil {
+					if !errors.IsNotFound(e) {
+						logger.Error(e, "Failed to delete obsolete adopted DB Cluster", "DB Cluster", adoptedDBCluster)
+						returnError(e, inventoryStatusReasonBackendError, inventoryStatusMessageDeleteClustersError)
+						return true, false
+					}
+				}
+				if cluster, ok := adoptedDBClusterMap[string(*adoptedDBCluster.Status.ACKResourceMetadata.ARN)]; ok {
+					if e := r.Delete(ctx, &cluster); e != nil {
+						if !errors.IsNotFound(e) {
+							logger.Error(e, "Failed to delete adopting resource", "DB Cluster", adoptedDBCluster)
+							returnError(e, inventoryStatusReasonBackendError, inventoryStatusMessageDeleteClustersError)
+							return true, false
+						}
+					}
+				}
+				logger.Info("Deleted adopted DB Cluster that is obsolete", "DB Cluster", adoptedDBCluster)
 				continue
 			}
 
@@ -1199,6 +1265,9 @@ func createAdoptedResource(resourceIdentifier *string, resourceArn *string, engi
 				"owner.kind":      inventory.Kind,
 				"owner.namespace": inventory.Namespace,
 			},
+			Labels: map[string]string{
+				adoptedDBResourceLabelKey: adoptedDBResourceLabelValue,
+			},
 		},
 		Spec: ackv1alpha1.AdoptedResourceSpec{
 			Kubernetes: &ackv1alpha1.ResourceWithMetadata{