From 4ea7d34b1af44f99a14994d9189f29fb18ff80fd Mon Sep 17 00:00:00 2001 From: r2dedios Date: Thu, 28 May 2026 16:23:01 +0200 Subject: [PATCH 01/22] feat(scanner): redesign scanner as long-running gRPC service with target system and action runs --- Makefile | 3 + cmd/agent/executor_agent_service.go | 67 ++- cmd/api/router.go | 12 + cmd/api/server.go | 3 + cmd/scanner/proto/scanner.proto | 33 ++ cmd/scanner/scanner.go | 441 ++++++++++-------- db/sql/init.sql | 108 ++++- db/test_files/integration_tests_data.sql | 22 +- db/test_files/load_example_data.sql | 70 +-- deployments/compose/compose-devel.yaml | 11 +- .../containerfiles/Containerfile-scanner | 32 +- .../cluster-iq/templates/agent/configmap.yaml | 1 + .../templates/scanner/configmap.yaml | 3 +- .../cluster-iq/templates/scanner/cronjob.yaml | 45 -- .../templates/scanner/deployment.yaml | 64 +++ .../cluster-iq/templates/scanner/service.yaml | 18 + deployments/helm/cluster-iq/values.yaml | 25 + generated/scanner/scanner.pb.go | 377 +++++++++++++++ generated/scanner/scanner_grpc.pb.go | 149 ++++++ internal/actions/action_operation.go | 3 + internal/actions/action_target.go | 9 + internal/api/handlers/action_run_handler.go | 217 +++++++++ internal/clients/agent.go | 2 + internal/clients/scanner.go | 64 +++ internal/cloud_executors/aws.go | 3 + internal/config/agent_config.go | 2 + internal/config/scanner_config.go | 1 + internal/models/convert/generated.go | 23 + internal/models/convert/mapper.go | 4 + internal/models/db/action.go | 25 +- internal/models/db/action_run.go | 13 + internal/models/dto/action_dto.go | 29 +- internal/models/dto/action_run_dto.go | 20 + internal/models/dto/wrappers.go | 6 + internal/repositories/action_repository.go | 167 ++++--- .../repositories/action_run_repository.go | 105 +++++ internal/services/action_run_service.go | 55 +++ .../api_action_runs_integration_test.go | 256 ++++++++++ 38 files changed, 2101 insertions(+), 387 deletions(-) create mode 100644 cmd/scanner/proto/scanner.proto delete mode 100644 deployments/helm/cluster-iq/templates/scanner/cronjob.yaml create mode 100644 deployments/helm/cluster-iq/templates/scanner/deployment.yaml create mode 100644 deployments/helm/cluster-iq/templates/scanner/service.yaml create mode 100644 generated/scanner/scanner.pb.go create mode 100644 generated/scanner/scanner_grpc.pb.go create mode 100644 internal/api/handlers/action_run_handler.go create mode 100644 internal/clients/scanner.go create mode 100644 internal/models/db/action_run.go create mode 100644 internal/models/dto/action_run_dto.go create mode 100644 internal/repositories/action_run_repository.go create mode 100644 internal/services/action_run_service.go create mode 100644 test/integration/api_action_runs_integration_test.go diff --git a/Makefile b/Makefile index 59dde545..fb95b70d 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,7 @@ AGENT_IMG_NAME ?= $(PROJECT_NAME)-agent AGENT_IMAGE ?= $(REGISTRY)/$(REGISTRY_REPO)/$(AGENT_IMG_NAME) AGENT_CONTAINERFILE ?= ./$(DEPLOYMENTS_DIR)/containerfiles/Containerfile-agent AGENT_PROTO_PATH ?= ./cmd/agent/proto/agent.proto +SCANNER_PROTO_PATH ?= ./cmd/scanner/proto/scanner.proto PGSQL_IMG_NAME ?= $(PROJECT_NAME)-pgsql PGSQL_IMAGE ?= $(REGISTRY)/$(REGISTRY_REPO)/$(PGSQL_IMG_NAME) PGSQL_CONTAINERFILE ?= ./$(DEPLOYMENTS_DIR)/containerfiles/Containerfile-pgsql @@ -81,6 +82,8 @@ local-build-api: generate-converters swagger-doc ## Build the API binary local-build-scanner: ## Build the scanner binary @echo "### [Building Scanner] ###" + @[ ! -d $(GENERATED_DIR) ] && { mkdir $(GENERATED_DIR); } || { exit 0; } + @$(PROTOC) --go_out=$(GENERATED_DIR) --go-grpc_out=$(GENERATED_DIR) $(SCANNER_PROTO_PATH) @$(GO) build -o $(BIN_DIR)/scanners/scanner $(LDFLAGS) ./cmd/scanner local-build-agent: ## Build the agent binary diff --git a/cmd/agent/executor_agent_service.go b/cmd/agent/executor_agent_service.go index 5ba8b830..671bdb11 100644 --- a/cmd/agent/executor_agent_service.go +++ b/cmd/agent/executor_agent_service.go @@ -9,6 +9,7 @@ import ( "sync" "github.com/RHEcosystemAppEng/cluster-iq/internal/actions" + "github.com/RHEcosystemAppEng/cluster-iq/internal/clients" cexec "github.com/RHEcosystemAppEng/cluster-iq/internal/cloud_executors" "github.com/RHEcosystemAppEng/cluster-iq/internal/config" "github.com/RHEcosystemAppEng/cluster-iq/internal/credentials" @@ -28,6 +29,8 @@ type ExecutorAgentService struct { client http.Client // HTTP Client for retrieving the schedule from API eventService *eventservice.EventService // Service for handling audit logs actionRepo repositories.ActionRepository + scannerClient *clients.ScannerGRPCClient // gRPC client for the Scanner service + actionRunRepo repositories.ActionRunRepository } // NewExecutorAgentService creates and initializes a new AgentCron instance for managing the scheduled actions @@ -57,6 +60,13 @@ func NewExecutorAgentService(cfg *config.ExecutorAgentServiceConfig, actionsChan eventService := eventservice.NewEventService(db, logger) actionRepo := repositories.NewActionRepository(db) + actionRunRepo := repositories.NewActionRunRepository(db) + + scannerClient, err := clients.NewScannerGRPCClient(cfg.ScannerURL, logger) + if err != nil { + logger.Error("Failed to create Scanner gRPC client", zap.Error(err)) + return nil + } eas := ExecutorAgentService{ cfg: cfg, @@ -66,9 +76,11 @@ func NewExecutorAgentService(cfg *config.ExecutorAgentServiceConfig, actionsChan logger: logger, wg: wg, }, - client: client, - eventService: eventService, - actionRepo: actionRepo, + client: client, + eventService: eventService, + actionRepo: actionRepo, + scannerClient: scannerClient, + actionRunRepo: actionRunRepo, } // Reading credentials file and creating executors per account @@ -198,6 +210,11 @@ func (e *ExecutorAgentService) processAction(action actions.Action) { zap.Any("requester", action.GetRequester()), ) + if action.GetActionOperation() == actions.Scan { + e.processScanAction(action) + return + } + // Initialize event tracker tracker := e.eventService.StartTracking(&eventservice.EventOptions{ Action: action.GetActionOperation(), @@ -235,6 +252,50 @@ func (e *ExecutorAgentService) processAction(action actions.Action) { e.resetCronActionStatus(action) } +// processScanAction dispatches a Scan action to the Scanner gRPC service. +func (e *ExecutorAgentService) processScanAction(action actions.Action) { + if !e.setActionStatus(action, actions.StatusRunning) { + return + } + + target := action.GetTarget() + + runID, err := e.actionRunRepo.Create(context.Background(), action.GetID()) + if err != nil { + e.logger.Error("Failed to create action run for scan", + zap.String("action_id", action.GetID()), zap.Error(err)) + e.setActionStatus(action, actions.StatusFailed) + return + } + + resp, err := e.scannerClient.Scan( + context.Background(), + runID, + target.TargetAccountIDs, + target.SelectAll, + ) + if err != nil { + e.logger.Error("Scanner gRPC call failed", + zap.String("action_id", action.GetID()), zap.Error(err)) + e.setActionStatus(action, actions.StatusFailed) + return + } + + if resp.Error != 0 { + e.logger.Error("Scanner returned error", + zap.String("action_id", action.GetID()), + zap.String("message", resp.Message)) + e.setActionStatus(action, actions.StatusFailed) + return + } + + e.logger.Info("Scan completed successfully", + zap.String("action_id", action.GetID()), + zap.Int32("accounts_scanned", resp.AccountsScanned)) + e.setActionStatus(action, actions.StatusSuccess) + e.resetCronActionStatus(action) +} + // setActionStatus safely updates action status with type assertion. // Returns false if update failed (caller should abort). func (e *ExecutorAgentService) setActionStatus(action actions.Action, status actions.ActionStatus) bool { diff --git a/cmd/api/router.go b/cmd/api/router.go index 7900e2c7..daa810ba 100644 --- a/cmd/api/router.go +++ b/cmd/api/router.go @@ -14,6 +14,7 @@ type APIHandlers struct { ExpenseHandler *handlers.ExpenseHandler EventHandler *handlers.EventHandler ActionHandler *handlers.ActionHandler + ActionRunHandler *handlers.ActionRunHandler OverviewHandler *handlers.OverviewHandler HealthCheckHandler *handlers.HealthCheckHandler } @@ -30,6 +31,7 @@ func Setup(engine *gin.Engine, handlers APIHandlers) { setupExpenseRoutes(baseGroup, handlers.ExpenseHandler) setupEventRoutes(baseGroup, handlers.EventHandler) setupActionRoutes(baseGroup, handlers.ActionHandler) + setupActionRunRoutes(baseGroup, handlers.ActionRunHandler) setupOverviewRoutes(baseGroup, handlers.OverviewHandler) } } @@ -110,6 +112,16 @@ func setupActionRoutes(group *gin.RouterGroup, handler *handlers.ActionHandler) } } +func setupActionRunRoutes(group *gin.RouterGroup, handler *handlers.ActionRunHandler) { + actionRuns := group.Group("/action-runs") + { + actionRuns.GET("", handler.List) + actionRuns.GET("/:id", handler.Get) + actionRuns.POST("", handler.Create) + actionRuns.PATCH("/:id", handler.Update) + } +} + func setupOverviewRoutes(group *gin.RouterGroup, handler *handlers.OverviewHandler) { overview := group.Group("/overview") { diff --git a/cmd/api/server.go b/cmd/api/server.go index e5bd8b3d..bb33aec7 100644 --- a/cmd/api/server.go +++ b/cmd/api/server.go @@ -200,6 +200,7 @@ func main() { expenseRepo := repositories.NewExpenseRepository(dbClient) eventRepo := repositories.NewEventRepository(dbClient) actionRepo := repositories.NewActionRepository(dbClient) + actionRunRepo := repositories.NewActionRunRepository(dbClient) // Initializing services inventoryService := services.NewInventoryService(inventoryRepo) @@ -212,6 +213,7 @@ func main() { expenseService := services.NewExpenseService(expenseRepo) eventService := services.NewEventService(eventRepo) actionService := services.NewActionService(actionRepo) + actionRunService := services.NewActionRunService(actionRunRepo) overviewService := services.NewOverviewService(clusterRepo, instanceRepo, accountRepo) // Initializing handlers @@ -223,6 +225,7 @@ func main() { ExpenseHandler: handlers.NewExpenseHandler(expenseService, logger), EventHandler: handlers.NewEventHandler(eventService, logger), ActionHandler: handlers.NewActionHandler(actionService, logger), + ActionRunHandler: handlers.NewActionRunHandler(actionRunService, logger), OverviewHandler: handlers.NewOverviewHandler(overviewService, logger), HealthCheckHandler: handlers.NewHealthCheckHandler(dbClient, logger), } diff --git a/cmd/scanner/proto/scanner.proto b/cmd/scanner/proto/scanner.proto new file mode 100644 index 00000000..6940f2a7 --- /dev/null +++ b/cmd/scanner/proto/scanner.proto @@ -0,0 +1,33 @@ +syntax = "proto3"; + +package scanner; + +option go_package = "./scanner"; + +// gRPC service for the Scanner +service ScannerService { + rpc Scan (ScanRequest) returns (ScanResponse); + rpc Health (HealthRequest) returns (HealthResponse); +} + +// Message for requesting a scan +message ScanRequest { + int64 run_id = 1; + repeated string account_ids = 2; + bool select_all = 3; +} + +// Message for answering to ScanRequests +message ScanResponse { + int32 error = 1; + string message = 2; + int32 accounts_scanned = 3; +} + +// Message for health check requests +message HealthRequest {} + +// Message for health check responses +message HealthResponse { + bool ready = 1; +} diff --git a/cmd/scanner/scanner.go b/cmd/scanner/scanner.go index e7c5ba16..9285c7d9 100644 --- a/cmd/scanner/scanner.go +++ b/cmd/scanner/scanner.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "io" + "net" "net/http" "os" "os/signal" @@ -15,6 +16,7 @@ import ( "syscall" "time" + pb "github.com/RHEcosystemAppEng/cluster-iq/generated/scanner" responsetypes "github.com/RHEcosystemAppEng/cluster-iq/internal/api/response_types" "github.com/RHEcosystemAppEng/cluster-iq/internal/config" "github.com/RHEcosystemAppEng/cluster-iq/internal/credentials" @@ -23,6 +25,8 @@ import ( "github.com/RHEcosystemAppEng/cluster-iq/internal/models/dto" "github.com/RHEcosystemAppEng/cluster-iq/internal/stocker" "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/reflection" ) const ( @@ -31,6 +35,7 @@ const ( apiClusterEndpoint = "/clusters" apiInstanceEndpoint = "/instances" apiExpenseEndpoint = "/expenses" + apiActionRunEndpoint = "/action-runs" // apiRequestTimeout defines the timeout for HTTP POST requests to the API apiRequestTimeout = 60 * time.Second @@ -57,16 +62,17 @@ var ( // Scanner models the cloud agnostic Scanner for looking up OCP deployments type Scanner struct { - inventory inventory.Inventory - stockers []stocker.Stocker - billingStockers []stocker.Stocker - cfg *config.ScannerConfig - logger *zap.Logger + pb.UnimplementedScannerServiceServer + allAccounts map[string]credentials.AccountConfig + cfg *config.ScannerConfig + logger *zap.Logger + grpcServer *grpc.Server + mu sync.Mutex + scanning bool } // NewScanner creates and returns a new Scanner instance func NewScanner(cfg *config.ScannerConfig, logger *zap.Logger) *Scanner { - // Calculate Credentials file MD5 checksum for checking on runtime hash := md5.Sum([]byte(cfg.CredentialsFile)) credsFileHash = hash[:] @@ -77,81 +83,77 @@ func NewScanner(cfg *config.ScannerConfig, logger *zap.Logger) *Scanner { APIURL = cfg.APIURL return &Scanner{ - inventory: *inventory.NewInventory(), - stockers: make([]stocker.Stocker, 0), - cfg: cfg, - logger: logger, + allAccounts: make(map[string]credentials.AccountConfig), + cfg: cfg, + logger: logger, } } func init() { - // Initialize logging configuration. logger = ciqLogger.NewLogger() } -// readCloudProviderAccounts reads and loads cloud provider accounts from a credentials file. -func (s *Scanner) readCloudProviderAccounts() error { - // Load cloud accounts credentials file. +// loadAccounts reads the credentials file and caches all account configs. +func (s *Scanner) loadAccounts() error { accountConfigs, err := credentials.ReadCloudAccounts(s.cfg.CredentialsFile) if err != nil { - return err + return fmt.Errorf("failed to read cloud accounts: %w", err) + } + + for _, ac := range accountConfigs { + s.allAccounts[ac.ID] = ac } - // Read INI file content. - for _, accountConfig := range accountConfigs { - newAccount, err := inventory.NewAccount( - accountConfig.ID, - accountConfig.Name, - accountConfig.Provider, - accountConfig.User, - accountConfig.Key, - ) + s.logger.Info("Loaded cloud accounts from credentials file", + zap.Int("count", len(s.allAccounts))) + + return nil +} + +// buildInventory creates an Inventory from the given account configs. +func (s *Scanner) buildInventory(configs []credentials.AccountConfig) (*inventory.Inventory, error) { + inv := inventory.NewInventory() + for _, ac := range configs { + newAccount, err := inventory.NewAccount(ac.ID, ac.Name, ac.Provider, ac.User, ac.Key) if err != nil { - return err + return nil, fmt.Errorf("failed to create account %s: %w", ac.ID, err) } - - // Getting billing enabled flag from config - if accountConfig.BillingEnabled { + if ac.BillingEnabled { newAccount.EnableBilling() } - - // Adding account to Inventory for scanning - if err := s.inventory.AddAccount(newAccount); err != nil { - return err + if err := inv.AddAccount(newAccount); err != nil { + return nil, fmt.Errorf("failed to add account %s: %w", ac.ID, err) } } - - return nil + return inv, nil } -// nolint:cyclop // createStockers creates and configures stocker instances for each provided account to be inventoried. -func (s *Scanner) createStockers() error { - for _, account := range s.inventory.Accounts { +// nolint:cyclop +func (s *Scanner) createStockers(inv *inventory.Inventory) ([]stocker.Stocker, []stocker.Stocker) { + var stockers []stocker.Stocker + var billingStockers []stocker.Stocker + + for _, account := range inv.Accounts { switch account.Provider { case inventory.AWSProvider: s.logger.Info("Processing AWS account", zap.String("account_id", account.AccountID), zap.String("account_name", account.AccountName)) - - // AWS API Stoker awsStocker, err := stocker.NewAWSStocker(account, s.cfg.SkipNoOpenShiftInstances, s.logger) if err != nil { s.logger.Error("Failed to create AWS stocker; skipping this account", - zap.String("account", account.AccountName), - zap.Error(err)) + zap.String("account", account.AccountName), zap.Error(err)) continue } - s.stockers = append(s.stockers, awsStocker) + stockers = append(stockers, awsStocker) - // AWS Billing API Stoker if account.IsBillingEnabled() { s.logger.Warn("Enabled AWS Billing Stocker", zap.String("account_id", account.AccountID), zap.String("account_name", account.AccountName)) - instancesToScan, err := s.getInstancesForBillingUpdate(account.AccountID) + instancesToScan, err := getInstancesForBillingUpdate(s.cfg.APIURL, account.AccountID, s.logger) if err != nil { - s.logger.Error("Failed to retrieve the list of instances required for billing information from AWS Cost Explorer.", - zap.String("account_name", account.AccountName), - zap.Error(err)) + s.logger.Error("Failed to retrieve instances for billing", + zap.String("account_name", account.AccountName), zap.Error(err)) } else { if bs := stocker.NewAWSBillingStocker(account, s.logger, instancesToScan); bs != nil { - s.billingStockers = append(s.billingStockers, bs) + billingStockers = append(billingStockers, bs) } } } @@ -159,18 +161,12 @@ func (s *Scanner) createStockers() error { s.logger.Warn("Failed to scan GCP account", zap.String("account_id", account.AccountID), zap.String("account_name", account.AccountName), - zap.String("reason", "not implemented"), - ) - // TODO: Uncomment line below when GCP Stocker is implemented - // gcpStocker = stocker.NewGCPStocker(account, s.cfg.SkipNoOpenShiftInstances, s.logger)) + zap.String("reason", "not implemented")) case inventory.AzureProvider: s.logger.Warn("Failed to scan Azure account", zap.String("account_id", account.AccountID), zap.String("account_name", account.AccountName), - zap.String("reason", "not implemented"), - ) - // TODO: Uncomment line below when Azure Stocker is implemented - // azureStocker = stocker.NewAzureStocker(account, s.cfg.SkipNoOpenShiftInstances, s.logger)) + zap.String("reason", "not implemented")) case inventory.UnknownProvider: s.logger.Warn("Unknown cloud provider, skipping account", zap.String("account_id", account.AccountID), @@ -185,120 +181,244 @@ func (s *Scanner) createStockers() error { } s.logger.Info("Account registration complete", - zap.Int("registeredAccounts", len(s.inventory.Accounts)), - zap.Int("registeredStockers", len(s.stockers)), - zap.Int("skippedAccounts", len(s.inventory.Accounts)-len(s.stockers))) + zap.Int("registeredAccounts", len(inv.Accounts)), + zap.Int("registeredStockers", len(stockers)), + zap.Int("skippedAccounts", len(inv.Accounts)-len(stockers))) - // If there are no stockers, nothing to do - if len(s.stockers) == 0 { - return fmt.Errorf("no valid accounts found for scanning") - } - - // Checking the logLevel before entering on the For loop for optimization - if s.logger.Core().Enabled(zap.DebugLevel) { - s.logger.Debug("Total Stockers created", zap.Int("count", len(s.stockers))) - for i, stocker := range s.stockers { - s.logger.Debug("Stocker", zap.Int("id", i), zap.String("account_id", stocker.GetAccount().AccountID), zap.String("account_name", stocker.GetAccount().AccountName)) - } - } - - return nil + return stockers, billingStockers } -// startStockers runs every stocker instance -func (s *Scanner) startStockers() error { +func runStockers(stockers []stocker.Stocker, billingStockers []stocker.Stocker, l *zap.Logger) error { var wg sync.WaitGroup - errChan := make(chan error, len(s.stockers)+len(s.billingStockers)) + errChan := make(chan error, len(stockers)+len(billingStockers)) - // First iteration for infrastructure stockers - s.logger.Warn("Running Infrastructure Stockers!", zap.Int("stockers_count", len(s.stockers))) - for _, stockerInstance := range s.stockers { + l.Warn("Running Infrastructure Stockers!", zap.Int("stockers_count", len(stockers))) + for _, st := range stockers { wg.Add(1) go func() { defer wg.Done() - if err := stockerInstance.MakeStock(); err != nil { + if err := st.MakeStock(); err != nil { errChan <- err } }() } - - // Waiting for every Stock wg.Wait() - // Second iteration for billing stockers - s.logger.Warn("Running Billing Stockers!", zap.Int("stockers_count", len(s.billingStockers))) - for _, stockerInstance := range s.billingStockers { + l.Warn("Running Billing Stockers!", zap.Int("stockers_count", len(billingStockers))) + for _, st := range billingStockers { wg.Add(1) go func() { defer wg.Done() - if err := stockerInstance.MakeStock(); err != nil { + if err := st.MakeStock(); err != nil { errChan <- err } }() } - // Waiting for every Stock go func() { wg.Wait() close(errChan) }() - // Collecting stockers errors var errorList []error for err := range errChan { errorList = append(errorList, err) } - // Processing errors when every stocker has finished if len(errorList) > 0 { for _, err := range errorList { - s.logger.Error("Stocker Error", zap.Error(err)) + l.Error("Stocker Error", zap.Error(err)) } return fmt.Errorf("error when running Scanner stockers. Failed Stockers: (%d)", len(errorList)) } - s.logger.Info("Stockers executed correctly") + l.Info("Stockers executed correctly") return nil } -// postNewAccount posts into the API an account, its clusters, instances and expenses -func (s *Scanner) postNewAccount(account inventory.Account) error { - s.logger.Debug("Posting new Account", zap.String("account_id", account.AccountID), zap.String("account_name", account.AccountName)) +// selectAccounts returns account configs matching the given IDs, or all accounts if selectAll is true or no IDs are provided. +func (s *Scanner) selectAccounts(accountIDs []string, selectAll bool) []credentials.AccountConfig { + var configs []credentials.AccountConfig + if selectAll || len(accountIDs) == 0 { + for _, ac := range s.allAccounts { + configs = append(configs, ac) + } + return configs + } + + for _, id := range accountIDs { + ac, ok := s.allAccounts[id] + if !ok { + s.logger.Warn("Account not found in credentials, skipping", zap.String("account_id", id)) + continue + } + configs = append(configs, ac) + } + return configs +} + +// ExecuteScan runs the full scan pipeline for the given accounts. +func (s *Scanner) ExecuteScan(accountIDs []string, selectAll bool) (int, error) { + s.mu.Lock() + if s.scanning { + s.mu.Unlock() + return 0, fmt.Errorf("a scan is already in progress") + } + s.scanning = true + s.mu.Unlock() + + defer func() { + s.mu.Lock() + s.scanning = false + s.mu.Unlock() + }() + + configs := s.selectAccounts(accountIDs, selectAll) + if len(configs) == 0 { + return 0, fmt.Errorf("no valid accounts found for scanning") + } + + inv, err := s.buildInventory(configs) + if err != nil { + return 0, fmt.Errorf("failed to build inventory: %w", err) + } + + stockers, billingStockers := s.createStockers(inv) + if len(stockers) == 0 { + return 0, fmt.Errorf("no valid stockers created") + } + + if err := runStockers(stockers, billingStockers, s.logger); err != nil { + return 0, fmt.Errorf("failed to run stockers: %w", err) + } + + inv.PrintInventory() + if err := postScannerInventory(inv, s.logger); err != nil { + return 0, fmt.Errorf("failed to post inventory: %w", err) + } + + return len(configs), nil +} + +// Scan handles a gRPC scan request. +func (s *Scanner) Scan(_ context.Context, req *pb.ScanRequest) (*pb.ScanResponse, error) { + s.logger.Info("Received scan request", + zap.Int64("run_id", req.RunId), + zap.Strings("account_ids", req.AccountIds), + zap.Bool("select_all", req.SelectAll)) + + scanned, err := s.ExecuteScan(req.AccountIds, req.SelectAll) + if err != nil { + s.logger.Error("Scan failed", zap.Error(err)) + if req.RunId > 0 { + s.updateRunStatus(req.RunId, "Failed", err.Error()) + } + return &pb.ScanResponse{ + Error: 1, + Message: err.Error(), + AccountsScanned: 0, + }, nil + } + + s.logger.Info("Scan completed successfully", zap.Int("accounts_scanned", scanned)) + if req.RunId > 0 { + s.updateRunStatus(req.RunId, "Success", "") + } + + return &pb.ScanResponse{ + Error: 0, + Message: "Scan completed successfully", + AccountsScanned: int32(scanned), + }, nil +} + +// Health reports the scanner's readiness. +func (s *Scanner) Health(_ context.Context, _ *pb.HealthRequest) (*pb.HealthResponse, error) { + return &pb.HealthResponse{Ready: true}, nil +} + +func (s *Scanner) updateRunStatus(runID int64, status string, errorMsg string) { + payload := map[string]string{ + "status": status, + "errorMsg": errorMsg, + } + b, err := json.Marshal(payload) + if err != nil { + s.logger.Error("Failed to marshal run status", zap.Error(err)) + return + } + + url := fmt.Sprintf("%s%s/%d", APIURL, apiActionRunEndpoint, runID) + ctx, cancel := context.WithTimeout(context.Background(), apiRequestTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodPatch, url, bytes.NewBuffer(b)) + if err != nil { + s.logger.Error("Failed to create run status request", zap.Error(err)) + return + } + req.Header.Set("Content-Type", "application/json") + + resp, err := client.Do(req) + if resp != nil { + defer resp.Body.Close() + } + if err != nil { + s.logger.Error("Failed to update run status", zap.Int64("run_id", runID), zap.Error(err)) + return + } + + s.logger.Info("Updated action run status", zap.Int64("run_id", runID), zap.String("status", status)) +} + +// startGRPCServer initializes and starts the gRPC server. +func (s *Scanner) startGRPCServer() error { + lc := net.ListenConfig{} + lis, err := lc.Listen(context.Background(), "tcp", s.cfg.ListenURL) + if err != nil { + return fmt.Errorf("failed to listen on %s: %w", s.cfg.ListenURL, err) + } + + s.grpcServer = grpc.NewServer() + pb.RegisterScannerServiceServer(s.grpcServer, s) + reflection.Register(s.grpcServer) + + s.logger.Info("Scanner gRPC server listening", zap.String("address", s.cfg.ListenURL)) + + return s.grpcServer.Serve(lis) +} + +func postNewAccount(account inventory.Account, l *zap.Logger) error { + l.Debug("Posting new Account", zap.String("account_id", account.AccountID), zap.String("account_name", account.AccountName)) - // Converting to Array because API handler assumes a list of accounts var accounts []dto.AccountDTORequest accounts = append(accounts, *dto.ToAccountDTORequest(account)) b, err := json.Marshal(accounts) if err != nil { - s.logger.Error("Failed to marshal account", zap.String("account_id", account.AccountID), zap.String("account_name", account.AccountName), zap.Error(err)) + l.Error("Failed to marshal account", zap.String("account_id", account.AccountID), zap.Error(err)) return err } - // Posting Account data if err := postData(apiAccountEndpoint, b); err != nil { return err } - // Flattering account for posting its elements clusters, instances, expenses := flatternAccount(account) - // Posting Clusters if len(clusters) > 0 { if err := postClusters(clusters); err != nil { return err } } - // Posting Instances if len(instances) > 0 { if err := postInstances(instances); err != nil { return err } } - // Posting Expenses if len(expenses) > 0 { - s.logger.Info("Posting expenses", zap.Int("expenses_count", len(expenses))) + l.Info("Posting expenses", zap.Int("expenses_count", len(expenses))) if err := postExpenses(expenses); err != nil { return err } @@ -306,7 +426,6 @@ func (s *Scanner) postNewAccount(account inventory.Account) error { return nil } -// flatternAccount extracts every Cluster, Instance and Expense from an Account for posting func flatternAccount(account inventory.Account) ([]inventory.Cluster, []inventory.Instance, []inventory.Expense) { var clusters []inventory.Cluster var instances []inventory.Instance @@ -315,95 +434,79 @@ func flatternAccount(account inventory.Account) ([]inventory.Cluster, []inventor for _, instance := range cluster.Instances { expenses = append(expenses, instance.Expenses...) instances = append(instances, instance) - } clusters = append(clusters, *cluster) } - return clusters, instances, expenses } -// postClusters posts into the API, the new instances obtained after scanning func postClusters(clusters []inventory.Cluster) error { b, err := json.Marshal(dto.ToClusterDTORequestList(clusters)) if err != nil { return err } - return postData(apiClusterEndpoint, b) } -// postInstances posts into the API, the instances obtained after scanning func postInstances(instances []inventory.Instance) error { b, err := json.Marshal(dto.ToInstanceDTORequestList(instances)) if err != nil { return err } - return postData(apiInstanceEndpoint, b) } -// postExpenses posts into the API, the expenses obtained after scanning func postExpenses(expenses []inventory.Expense) error { b, err := json.Marshal(dto.ToExpenseDTORequestList(expenses)) if err != nil { return err } - return postData(apiExpenseEndpoint, b) } -// postScannerInventory posts to ClusterIQ API the information obtained of the scanning process -// This function parallelizes the post operations creating a thread by account(or stocker) -func (s *Scanner) postScannerInventory() error { +func postScannerInventory(inv *inventory.Inventory, l *zap.Logger) error { var wg sync.WaitGroup - errChan := make(chan error, len(s.inventory.Accounts)) + errChan := make(chan error, len(inv.Accounts)) - for _, account := range s.inventory.Accounts { + for _, account := range inv.Accounts { wg.Add(1) go func() { defer wg.Done() - if err := s.postNewAccount(*account); err != nil { + if err := postNewAccount(*account, l); err != nil { errChan <- err } }() - } - // Waiting for every Stock + go func() { wg.Wait() close(errChan) }() - // Collecting account posting errors var errorList []error for err := range errChan { errorList = append(errorList, err) } - // Processing errors when every post account operation has finished if len(errorList) > 0 { for _, err := range errorList { - s.logger.Error("Post Account Error", zap.Error(err)) + l.Error("Post Account Error", zap.Error(err)) } return fmt.Errorf("error when posting Scanner inventory") } - s.logger.Info("Inventory posted correctly") + l.Info("Inventory posted correctly") - // HTTP post to /inventory to refresh views if err := postData(apiInventoryEndpoint, []byte{}); err != nil { return err } - s.logger.Info("Inventory refreshed correctly") + l.Info("Inventory refreshed correctly") return nil } func postData(path string, b []byte) error { url := fmt.Sprintf("%s%s", APIURL, path) - - // Create context with timeout for API requests ctx, cancel := context.WithTimeout(context.Background(), apiRequestTimeout) defer cancel() @@ -419,45 +522,41 @@ func postData(path string, b []byte) error { if err != nil { return err } - return nil } -// getInstances fetches instances from the backend API -func (s *Scanner) getInstancesForBillingUpdate(accountID string) ([]inventory.Instance, error) { - s.logger.Debug("Fetching instances for update billing from backend") - - requestURL := s.cfg.APIURL + apiAccountEndpoint + "/" + accountID + "/expense_update" +func getInstancesForBillingUpdate(apiURL string, accountID string, l *zap.Logger) ([]inventory.Instance, error) { + l.Debug("Fetching instances for update billing from backend") + requestURL := apiURL + apiAccountEndpoint + "/" + accountID + "/expense_update" req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, requestURL, nil) if err != nil { - s.logger.Error("Failed preparing last expenses list request", zap.Error(err)) + l.Error("Failed preparing last expenses list request", zap.Error(err)) return nil, err } resp, err := http.DefaultClient.Do(req) if err != nil { - s.logger.Error("Failed to get last expenses from API", zap.Error(err)) + l.Error("Failed to get last expenses from API", zap.Error(err)) return nil, err } - defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - s.logger.Error("Failed to get last expenses from API", zap.Int("status_code", resp.StatusCode)) + l.Error("Failed to get last expenses from API", zap.Int("status_code", resp.StatusCode)) return nil, fmt.Errorf("failed to get last expenses, status code: %d", resp.StatusCode) } body, err := io.ReadAll(resp.Body) if err != nil { - s.logger.Error("Failed to read response body", zap.Error(err)) + l.Error("Failed to read response body", zap.Error(err)) return nil, err } var response responsetypes.ListResponse[dto.InstanceDTOResponse] err = json.Unmarshal(body, &response) if err != nil { - s.logger.Error("Failed to unmarshal instances JSON", zap.Error(err)) + l.Error("Failed to unmarshal instances JSON", zap.Error(err)) return nil, err } @@ -465,7 +564,7 @@ func (s *Scanner) getInstancesForBillingUpdate(accountID string) ([]inventory.In return nil, fmt.Errorf("no instances for billing update") } - s.logger.Debug("Successfully fetched instances from backend", zap.Int("instances_num", response.Count)) + l.Debug("Successfully fetched instances from backend", zap.Int("instances_num", response.Count)) var instances []inventory.Instance for _, instance := range response.Items { @@ -474,23 +573,9 @@ func (s *Scanner) getInstancesForBillingUpdate(accountID string) ([]inventory.In return instances, nil } -// signalHandler for managing incoming OS signals -func signalHandler(sig os.Signal) { - if sig == syscall.SIGTERM { - logger.Fatal("SIGTERM signal received. Stopping ClusterIQ Scanner") - os.Exit(0) - } - - logger.Warn("Ignoring signal: ", zap.String("signal_id", sig.String())) -} - -// Main method func main() { - // Ignore Logger sync error defer func() { _ = logger.Sync() }() - var err error - cfg, err := config.LoadScannerConfig() if err != nil { logger.Fatal("Failed to load config", zap.Error(err)) @@ -503,44 +588,28 @@ func main() { zap.String("commit", commit), zap.String("credentials_file_path", cfg.CredentialsFile), zap.ByteString("credentials_file_hash", credsFileHash), + zap.String("listen_url", cfg.ListenURL), ) - // Listen Signals block for receive OS signals. This is used by K8s/OCP for - // interacting with this software when it's deployed on a Pod - go func() { - quitChan := make(chan os.Signal, 1) - signal.Notify(quitChan, syscall.SIGTERM) - s := <-quitChan - signalHandler(s) - logger.Info("Scanner stopped") - }() - - // Get Cloud Accounts from credentials file - err = scan.readCloudProviderAccounts() - if err != nil { - logger.Error("Failed to get cloud provider accounts", zap.Error(err)) - return + if err := scan.loadAccounts(); err != nil { + logger.Fatal("Failed to load cloud accounts", zap.Error(err)) } - // Run Stockers - err = scan.createStockers() - if err != nil { - logger.Error("Failed to create stockers", zap.Error(err)) - return - } + // Signal handling for graceful shutdown + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) - err = scan.startStockers() - if err != nil { - logger.Error("Failed to start up stocker instances", zap.Error(err)) - return - } + go func() { + s := <-quit + logger.Warn("Received signal, shutting down...", zap.String("signal", s.String())) + if scan.grpcServer != nil { + scan.grpcServer.GracefulStop() + } + }() - // Writing into DB - scan.inventory.PrintInventory() - if err := scan.postScannerInventory(); err != nil { - logger.Error("Can't post scanned results", zap.Error(err)) - return + if err := scan.startGRPCServer(); err != nil { + logger.Fatal("gRPC server failed", zap.Error(err)) } - logger.Info("Scanner finished successfully") + logger.Info("Scanner stopped") } diff --git a/db/sql/init.sql b/db/sql/init.sql index 19d7ecec..97bf0083 100644 --- a/db/sql/init.sql +++ b/db/sql/init.sql @@ -26,7 +26,8 @@ CREATE TYPE RESOURCE_TYPE AS ENUM ( -- Supported values of Action Operations CREATE TYPE ACTION_OPERATION AS ENUM ( 'PowerOn', - 'PowerOff' + 'PowerOff', + 'Scan' ); -- Supported values of action types @@ -237,6 +238,37 @@ CREATE TRIGGER trg_delete_instance_events EXECUTE FUNCTION delete_instance_events(); +-- ############################################################################# +-- ## Targets definition ## +-- ############################################################################# +\! echo '## Creating Targets tables' + +CREATE TABLE IF NOT EXISTS targets ( + id BIGINT GENERATED ALWAYS AS IDENTITY NOT NULL, + target_type RESOURCE_TYPE NOT NULL, + select_all BOOLEAN DEFAULT false, + PRIMARY KEY (id) +); + +CREATE TABLE IF NOT EXISTS target_accounts ( + target_id BIGINT REFERENCES targets(id) ON DELETE CASCADE NOT NULL, + account_id INTEGER REFERENCES accounts(id) ON DELETE CASCADE NOT NULL, + PRIMARY KEY (target_id, account_id) +); + +CREATE TABLE IF NOT EXISTS target_clusters ( + target_id BIGINT REFERENCES targets(id) ON DELETE CASCADE NOT NULL, + cluster_id BIGINT REFERENCES clusters(id) ON DELETE CASCADE NOT NULL, + PRIMARY KEY (target_id, cluster_id) +); + +CREATE TABLE IF NOT EXISTS target_instances ( + target_id BIGINT REFERENCES targets(id) ON DELETE CASCADE NOT NULL, + instance_id BIGINT REFERENCES instances(id) ON DELETE CASCADE NOT NULL, + PRIMARY KEY (target_id, instance_id) +); + + -- ############################################################################# -- ## Actions and Scheduling definition ## -- ############################################################################# @@ -248,10 +280,10 @@ CREATE TABLE IF NOT EXISTS schedule ( time TIMESTAMP WITH TIME ZONE, cron_exp TEXT, operation ACTION_OPERATION NOT NULL, - target INTEGER REFERENCES clusters(id) ON DELETE CASCADE NOT NULL, + target BIGINT REFERENCES targets(id) ON DELETE CASCADE NOT NULL, status ACTION_STATUS DEFAULT 'Unknown' NOT NULL, enabled BOOLEAN DEFAULT false, - PRIMARY KEY (id), + PRIMARY KEY (id), CONSTRAINT chk_schedule_time_or_cron CHECK ((time IS NOT NULL) <> (cron_exp IS NOT NULL)) ); @@ -259,6 +291,25 @@ CREATE INDEX IF NOT EXISTS ix_schedule_target_enabled ON schedule (target, enabl CREATE INDEX IF NOT EXISTS ix_schedule_status ON schedule (status); +-- ############################################################################# +-- ## Action Runs (execution history) ## +-- ############################################################################# +\! echo '## Creating Action Runs table' + +CREATE TABLE IF NOT EXISTS action_runs ( + id BIGINT GENERATED ALWAYS AS IDENTITY NOT NULL, + schedule_id BIGINT REFERENCES schedule(id) ON DELETE CASCADE NOT NULL, + started_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + finished_at TIMESTAMP WITH TIME ZONE, + status ACTION_STATUS NOT NULL DEFAULT 'Running', + error_msg TEXT, + PRIMARY KEY (id) +); + +CREATE INDEX IF NOT EXISTS ix_action_runs_schedule ON action_runs (schedule_id); +CREATE INDEX IF NOT EXISTS ix_action_runs_status ON action_runs (status); + + -- ## Advanced Inventory definition (Views & Functions) ## -- ################################################################################################# @@ -516,29 +567,40 @@ WHERE -- ## Schedule -- ############################################################################# --- Schedule with cluster and instances list view +-- Schedule with target details view CREATE OR REPLACE VIEW schedule_full_view AS SELECT - s.id, - s.type, - s.time, - s.cron_exp, - s.operation, - s.status, - s.enabled, - c.cluster_id, - c.region, - a.account_id, - COALESCE( - array_agg(DISTINCT i.instance_id ORDER BY i.instance_id), - '{}' - ) AS instances -FROM - schedule s -JOIN clusters c ON c.id = s.target + s.id, + s.type, + s.time, + s.cron_exp, + s.operation, + s.status, + s.enabled, + t.target_type, + t.select_all, + c.cluster_id, + c.region, + COALESCE(a_power.account_id, '') AS account_id, + COALESCE( + array_agg(DISTINCT i.instance_id ORDER BY i.instance_id) + FILTER (WHERE i.instance_id IS NOT NULL), + '{}' + ) AS instances, + COALESCE( + (SELECT array_agg(DISTINCT accs.account_id ORDER BY accs.account_id) + FROM target_accounts ta_sub + JOIN accounts accs ON accs.id = ta_sub.account_id + WHERE ta_sub.target_id = t.id), + '{}' + ) AS target_account_ids +FROM schedule s +JOIN targets t ON t.id = s.target +LEFT JOIN target_clusters tc ON tc.target_id = t.id +LEFT JOIN clusters c ON c.id = tc.cluster_id LEFT JOIN instances i ON i.cluster_id = c.id -JOIN accounts a ON c.account_id = a.id -GROUP BY a.account_id, s.id, c.id +LEFT JOIN accounts a_power ON a_power.id = c.account_id +GROUP BY s.id, t.id, c.id, a_power.account_id ORDER BY s.id; diff --git a/db/test_files/integration_tests_data.sql b/db/test_files/integration_tests_data.sql index aaddeaf3..385fba8f 100644 --- a/db/test_files/integration_tests_data.sql +++ b/db/test_files/integration_tests_data.sql @@ -1,7 +1,7 @@ BEGIN; -- Cleaning -TRUNCATE expenses, tags, instances, clusters, accounts RESTART IDENTITY CASCADE; +TRUNCATE action_runs, schedule, targets, expenses, tags, instances, clusters, accounts RESTART IDENTITY CASCADE; -- ## Accounts ## INSERT INTO accounts (account_id, account_name, provider, last_scan_ts, created_at) VALUES @@ -75,12 +75,30 @@ INSERT INTO events (event_timestamp, triggered_by, action, resource_id, resource ('2025-08-02 12:00:00+00', 'cluster-iq-tester', 'test', '10', 'Instance', 'Pending', 'integration test event', 'critical'); +-- ## Targets (for schedule entries) ## +INSERT INTO targets (target_type, select_all) VALUES + ('Cluster', false), + ('Cluster', false), + ('Cluster', false); + +INSERT INTO target_clusters (target_id, cluster_id) VALUES + (1, 1), + (2, 2), + (3, 4); + INSERT INTO schedule (type, time, operation, target, status, enabled) VALUES ('scheduled_action', '1970-01-01 00:00:00+000', 'PowerOff', 1, 'Pending', 't'), - ('scheduled_action', '1970-01-01 00:00:00+000', 'PowerOn', 4, 'Pending', 'f'); + ('scheduled_action', '1970-01-01 00:00:00+000', 'PowerOn', 3, 'Pending', 'f'); INSERT INTO schedule (type, cron_exp, operation, target, status, enabled) VALUES ('scheduled_action', '30 */12 * 6 *', 'PowerOff', 2, 'Pending', 'f'); +-- ## Action Runs ## +INSERT INTO action_runs (schedule_id, status) VALUES + (1, 'Running'); + +INSERT INTO action_runs (schedule_id, status, finished_at, error_msg) VALUES + (2, 'Success', '2025-08-02 13:00:00+00', NULL); + COMMIT; diff --git a/db/test_files/load_example_data.sql b/db/test_files/load_example_data.sql index 7367fd47..104ba80a 100644 --- a/db/test_files/load_example_data.sql +++ b/db/test_files/load_example_data.sql @@ -3,7 +3,7 @@ BEGIN; -- Limpia datos previos (si los hubiera) -TRUNCATE expenses, tags, instances, clusters, accounts RESTART IDENTITY CASCADE; +TRUNCATE action_runs, schedule, targets, expenses, tags, instances, clusters, accounts RESTART IDENTITY CASCADE; -- Inserta 3 cuentas (una por proveedor) y guarda sus IDs WITH ins AS ( INSERT INTO accounts (account_id, account_name, provider, last_scan_ts) @@ -204,35 +204,45 @@ BEGIN END $$; --- Generating scheduled actions -INSERT INTO schedule (type, time, cron_exp, operation, target, status, enabled) -SELECT - 'scheduled_action'::ACTION_TYPE, - now() + (g * interval '1 day') AS time, - NULL, - (ARRAY['PowerOn','PowerOff'])[1 + (random()*1)::int]::ACTION_OPERATION, - c.id AS target, - 'Pending'::ACTION_STATUS, - (random() > 0.5) AS enabled -FROM generate_series(1,3) g -JOIN LATERAL ( - SELECT id FROM clusters ORDER BY random() LIMIT 1 -) c ON true; - --- Generating cron-based action -INSERT INTO schedule (type, time, cron_exp, operation, target, status, enabled) -SELECT - 'cron_action'::ACTION_TYPE, - NULL, - (ARRAY['0 6 * * *', '0 0 * * 0', '*/30 * * * *'])[g] AS cron_exp, - (ARRAY['PowerOn','PowerOff'])[1 + (random()*1)::int]::ACTION_OPERATION, - c.id AS target, - 'Pending'::ACTION_STATUS, - (random() > 0.5) AS enabled -FROM generate_series(1,3) g -JOIN LATERAL ( - SELECT id FROM clusters ORDER BY random() LIMIT 1 -) c ON true; +-- Generating scheduled actions (with targets) +DO $$ +DECLARE + v_target_id BIGINT; + v_cluster_id BIGINT; + v_operation ACTION_OPERATION; +BEGIN + FOR g IN 1..3 LOOP + SELECT id INTO v_cluster_id FROM clusters ORDER BY random() LIMIT 1; + v_operation := (ARRAY['PowerOn','PowerOff'])[1 + (random()*1)::int]::ACTION_OPERATION; + + INSERT INTO targets (target_type, select_all) VALUES ('Cluster', false) RETURNING id INTO v_target_id; + INSERT INTO target_clusters (target_id, cluster_id) VALUES (v_target_id, v_cluster_id); + INSERT INTO schedule (type, time, cron_exp, operation, target, status, enabled) + VALUES ('scheduled_action', now() + (g * interval '1 day'), NULL, v_operation, v_target_id, 'Pending', (random() > 0.5)); + END LOOP; +END +$$; + +-- Generating cron-based actions (with targets) +DO $$ +DECLARE + v_target_id BIGINT; + v_cluster_id BIGINT; + v_operation ACTION_OPERATION; + v_cron TEXT; +BEGIN + FOR g IN 1..3 LOOP + SELECT id INTO v_cluster_id FROM clusters ORDER BY random() LIMIT 1; + v_operation := (ARRAY['PowerOn','PowerOff'])[1 + (random()*1)::int]::ACTION_OPERATION; + v_cron := (ARRAY['0 6 * * *', '0 0 * * 0', '*/30 * * * *'])[g]; + + INSERT INTO targets (target_type, select_all) VALUES ('Cluster', false) RETURNING id INTO v_target_id; + INSERT INTO target_clusters (target_id, cluster_id) VALUES (v_target_id, v_cluster_id); + INSERT INTO schedule (type, time, cron_exp, operation, target, status, enabled) + VALUES ('cron_action', NULL, v_cron, v_operation, v_target_id, 'Pending', (random() > 0.5)); + END LOOP; +END +$$; -- Generating events INSERT INTO events ( diff --git a/deployments/compose/compose-devel.yaml b/deployments/compose/compose-devel.yaml index 6f190c92..96b2eacc 100644 --- a/deployments/compose/compose-devel.yaml +++ b/deployments/compose/compose-devel.yaml @@ -30,16 +30,24 @@ services: scanner: image: quay.io/ecosystem-appeng/cluster-iq-scanner:latest container_name: scanner - restart: "no" + restart: "always" depends_on: api: condition: service_healthy restart: true + ports: + - 50052:50052 environment: CIQ_API_URL: "http://api:8080/api/v1" + CIQ_SCANNER_LISTEN_URL: "0.0.0.0:50052" CIQ_CREDS_FILE: "/credentials" CIQ_SKIP_NO_OPENSHIFT_INSTANCES: "true" CIQ_LOG_LEVEL: "DEBUG" + healthcheck: + test: ["CMD-SHELL", "timeout 2 bash -lc 'echo > /dev/tcp/127.0.0.1/50052'"] + interval: 10s + timeout: 5s + retries: 5 volumes: - ../../secrets/credentials:/credentials:ro,Z networks: @@ -57,6 +65,7 @@ services: CIQ_API_URL: "http://api:8080/api/v1" CIQ_DB_URL: "postgresql://user:password@pgsql:5432/clusteriq?sslmode=disable" CIQ_AGENT_INSTANT_SERVICE_LISTEN_URL: "0.0.0.0:50051" + CIQ_SCANNER_URL: "scanner:50052" CIQ_CREDS_FILE: "/credentials" CIQ_LOG_LEVEL: "DEBUG" CIQ_AGENT_POLLING_SECONDS_INTERVAL: 10 # Seconds diff --git a/deployments/containerfiles/Containerfile-scanner b/deployments/containerfiles/Containerfile-scanner index 2511b17c..52bf3f72 100644 --- a/deployments/containerfiles/Containerfile-scanner +++ b/deployments/containerfiles/Containerfile-scanner @@ -7,18 +7,43 @@ FROM golang:1.25.7 AS builder ARG VERSION ARG COMMIT +# Versions for Protobuf and gRPC +ENV PROTOC_VERSION=29.3 +ENV PROTOC_GEN_GO_VERSION=v1.36.2 +ENV PROTOC_GEN_GO_GRPC_VERSION=v1.5.1 + +# Installing ProtoBuf 29.3 +RUN apt-get update && apt-get install -y --no-install-recommends \ + unzip \ + curl && \ + curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip && \ + unzip protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local && \ + rm -f protoc-${PROTOC_VERSION}-linux-x86_64.zip && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Installing protoc-gen-go and protoc-gen-go-grpc +RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@${PROTOC_GEN_GO_VERSION} && \ + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@${PROTOC_GEN_GO_GRPC_VERSION} + +# Adding /go/bin to path so the 'protoc-gen-go' can be found +ENV PATH="${PATH}:/go/bin" + # Code copy WORKDIR /app COPY . . -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o cluster-iq-scanner -ldflags "-X main.version=${VERSION} -X main.commit=${COMMIT}" ./cmd/scanner/scanner.go +# gRPC code generation +RUN mkdir -p ./generated && \ + protoc --go_out=./generated --go-grpc_out=./generated ./cmd/scanner/proto/scanner.proto +# Scanner building +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o cluster-iq-scanner -ldflags "-X main.version=${VERSION} -X main.commit=${COMMIT}" ./cmd/scanner ## Run #################### FROM registry.access.redhat.com/ubi8/ubi-micro:8.10-15 # Labels -LABEL version="v0.5" +LABEL version="v0.6" LABEL description="ClusterIQ cloud provider Scanner" # Binary @@ -29,6 +54,9 @@ COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifi # Default config ENV CIQ_API_URL: "http://api:8080/api/v1" ENV CIQ_CREDS_FILE: "/credentials" +ENV CIQ_SCANNER_LISTEN_URL: "0.0.0.0:50052" ENV CIQ_LOG_LEVEL: "DEBUG" +EXPOSE 50052 + ENTRYPOINT ["/cluster-iq-scanner"] diff --git a/deployments/helm/cluster-iq/templates/agent/configmap.yaml b/deployments/helm/cluster-iq/templates/agent/configmap.yaml index 3fd9d454..42e6f87e 100644 --- a/deployments/helm/cluster-iq/templates/agent/configmap.yaml +++ b/deployments/helm/cluster-iq/templates/agent/configmap.yaml @@ -8,6 +8,7 @@ metadata: data: CIQ_AGENT_INSTANT_SERVICE_LISTEN_URL: 0.0.0.0:{{ .Values.agent.service.port }} CIQ_API_URL: api:{{ .Values.api.service.port }} + CIQ_SCANNER_URL: 'scanner.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.scanner.service.port }}' CIQ_CREDS_FILE: /credentials/credentials CIQ_LOG_LEVEL: {{ .Values.agent.logLevel }} CIQ_AGENT_POLLING_SECONDS_INTERVAL: "{{ .Values.agent.pollingInterval }}" diff --git a/deployments/helm/cluster-iq/templates/scanner/configmap.yaml b/deployments/helm/cluster-iq/templates/scanner/configmap.yaml index b666a74a..7c1a97e1 100644 --- a/deployments/helm/cluster-iq/templates/scanner/configmap.yaml +++ b/deployments/helm/cluster-iq/templates/scanner/configmap.yaml @@ -4,9 +4,10 @@ metadata: name: scanner labels: {{- include "cluster-iq.labels" . | nindent 4 }} - {{- include "cluster-iq.componentLabels" "api" | nindent 4 }} + {{- include "cluster-iq.componentLabels" "scanner" | nindent 4 }} data: CIQ_API_URL: 'http://api.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.api.service.port }}/api/v1' + CIQ_SCANNER_LISTEN_URL: '0.0.0.0:{{ .Values.scanner.service.port }}' CIQ_CREDS_FILE: /credentials/credentials CIQ_LOG_LEVEL: {{ .Values.scanner.logLevel }} CIQ_SKIP_NO_OPENSHIFT_INSTANCES: "{{ .Values.scanner.skipNoOpenshiftInstances }}" diff --git a/deployments/helm/cluster-iq/templates/scanner/cronjob.yaml b/deployments/helm/cluster-iq/templates/scanner/cronjob.yaml deleted file mode 100644 index e1d6a635..00000000 --- a/deployments/helm/cluster-iq/templates/scanner/cronjob.yaml +++ /dev/null @@ -1,45 +0,0 @@ -kind: CronJob -apiVersion: batch/v1 -metadata: - name: scanner - labels: - {{- include "cluster-iq.labels" . | nindent 4 }} - {{- include "cluster-iq.componentLabels" "scanner" | nindent 4 }} -spec: - schedule: 0 0 * * * - concurrencyPolicy: Allow - suspend: false - successfulJobsHistoryLimit: 1 - failedJobsHistoryLimit: 5 - jobTemplate: - spec: - template: - spec: - serviceAccountName: {{ include "cluster-iq.scannerServiceAccountName" . }} - volumes: - {{- if .Values.vault.enabled }} - - name: credentials - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "cluster-iq-credentials-scanner" - {{- else }} - - name: credentials - secret: - secretName: credentials - {{- end }} - containers: - - name: scanner - image: "{{ .Values.scanner.image.repository }}:{{ .Values.scanner.image.tag | default .Chart.AppVersion }}" - envFrom: - - configMapRef: - name: scanner - volumeMounts: - - name: credentials - readOnly: true - mountPath: /credentials - imagePullPolicy: IfNotPresent - resources: - {{- toYaml .Values.scanner.resources | nindent 16 }} - restartPolicy: OnFailure diff --git a/deployments/helm/cluster-iq/templates/scanner/deployment.yaml b/deployments/helm/cluster-iq/templates/scanner/deployment.yaml new file mode 100644 index 00000000..0cf7b205 --- /dev/null +++ b/deployments/helm/cluster-iq/templates/scanner/deployment.yaml @@ -0,0 +1,64 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: scanner + labels: + {{- include "cluster-iq.labels" . | nindent 4 }} + {{- include "cluster-iq.componentLabels" "scanner" | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "cluster-iq.selectorLabels" . | nindent 6 }} + {{- include "cluster-iq.componentLabels" "scanner" | nindent 6 }} + template: + metadata: + labels: + {{- include "cluster-iq.labels" . | nindent 8 }} + {{- include "cluster-iq.componentLabels" "scanner" | nindent 8 }} + spec: + serviceAccountName: {{ include "cluster-iq.scannerServiceAccountName" . }} + containers: + - name: scanner + image: "{{ .Values.scanner.image.repository }}:{{ .Values.scanner.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.scanner.image.pullPolicy }} + envFrom: + - configMapRef: + name: scanner + ports: + - name: {{ .Values.scanner.service.name }} + containerPort: {{ .Values.scanner.service.port }} + protocol: TCP + startupProbe: + tcpSocket: + port: {{ .Values.scanner.service.port }} + {{- toYaml .Values.scanner.startupProbe | nindent 12 }} + readinessProbe: + tcpSocket: + port: {{ .Values.scanner.service.port }} + {{- toYaml .Values.scanner.readinessProbe | nindent 12 }} + livenessProbe: + tcpSocket: + port: {{ .Values.scanner.service.port }} + {{- toYaml .Values.scanner.livenessProbe | nindent 12 }} + resources: + {{- toYaml .Values.scanner.resources | nindent 12 }} + volumeMounts: + - name: credentials + readOnly: true + mountPath: /credentials + volumes: + {{- if .Values.vault.enabled }} + - name: credentials + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "cluster-iq-credentials-scanner" + {{- else }} + - name: credentials + secret: + secretName: credentials + optional: false + {{- end }} diff --git a/deployments/helm/cluster-iq/templates/scanner/service.yaml b/deployments/helm/cluster-iq/templates/scanner/service.yaml new file mode 100644 index 00000000..406841d0 --- /dev/null +++ b/deployments/helm/cluster-iq/templates/scanner/service.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: scanner + labels: + {{- include "cluster-iq.labels" . | nindent 4 }} + {{- include "cluster-iq.componentLabels" "scanner" | nindent 4 }} +spec: + type: {{ .Values.scanner.service.type }} + ports: + - port: {{ .Values.scanner.service.port }} + targetPort: {{ .Values.scanner.service.name }} + protocol: TCP + name: {{ .Values.scanner.service.name }} + selector: + {{- include "cluster-iq.selectorLabels" . | nindent 4 }} + {{- include "cluster-iq.componentLabels" "scanner" | nindent 4 }} diff --git a/deployments/helm/cluster-iq/values.yaml b/deployments/helm/cluster-iq/values.yaml index f25825e9..c8e42b39 100644 --- a/deployments/helm/cluster-iq/values.yaml +++ b/deployments/helm/cluster-iq/values.yaml @@ -255,6 +255,11 @@ scanner: # Overrides the image tag whose default is the chart appVersion. tag: "latest" + service: + type: ClusterIP + port: 50052 + name: scanner-grpc + resources: requests: memory: "128Mi" @@ -262,6 +267,26 @@ scanner: skipNoOpenshiftInstances: true + startupProbe: + initialDelaySeconds: 0 + periodSeconds: 1 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 5 + livenessProbe: + initialDelaySeconds: 5 + periodSeconds: 15 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + terminationGracePeriodSeconds: 30 + readinessProbe: + initialDelaySeconds: 5 + periodSeconds: 15 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + #This section builds out the service account serviceAccount: # Specifies whether a service account should be created diff --git a/generated/scanner/scanner.pb.go b/generated/scanner/scanner.pb.go new file mode 100644 index 00000000..44d135d3 --- /dev/null +++ b/generated/scanner/scanner.pb.go @@ -0,0 +1,377 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v3.19.6 +// source: cmd/scanner/proto/scanner.proto + +package scanner + +import ( + reflect "reflect" + sync "sync" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Message for requesting a scan +type ScanRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + RunId int64 `protobuf:"varint,1,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + AccountIds []string `protobuf:"bytes,2,rep,name=account_ids,json=accountIds,proto3" json:"account_ids,omitempty"` + SelectAll bool `protobuf:"varint,3,opt,name=select_all,json=selectAll,proto3" json:"select_all,omitempty"` +} + +func (x *ScanRequest) Reset() { + *x = ScanRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cmd_scanner_proto_scanner_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ScanRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ScanRequest) ProtoMessage() {} + +func (x *ScanRequest) ProtoReflect() protoreflect.Message { + mi := &file_cmd_scanner_proto_scanner_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ScanRequest.ProtoReflect.Descriptor instead. +func (*ScanRequest) Descriptor() ([]byte, []int) { + return file_cmd_scanner_proto_scanner_proto_rawDescGZIP(), []int{0} +} + +func (x *ScanRequest) GetRunId() int64 { + if x != nil { + return x.RunId + } + return 0 +} + +func (x *ScanRequest) GetAccountIds() []string { + if x != nil { + return x.AccountIds + } + return nil +} + +func (x *ScanRequest) GetSelectAll() bool { + if x != nil { + return x.SelectAll + } + return false +} + +// Message for answering to ScanRequests +type ScanResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Error int32 `protobuf:"varint,1,opt,name=error,proto3" json:"error,omitempty"` + Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` + AccountsScanned int32 `protobuf:"varint,3,opt,name=accounts_scanned,json=accountsScanned,proto3" json:"accounts_scanned,omitempty"` +} + +func (x *ScanResponse) Reset() { + *x = ScanResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cmd_scanner_proto_scanner_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ScanResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ScanResponse) ProtoMessage() {} + +func (x *ScanResponse) ProtoReflect() protoreflect.Message { + mi := &file_cmd_scanner_proto_scanner_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ScanResponse.ProtoReflect.Descriptor instead. +func (*ScanResponse) Descriptor() ([]byte, []int) { + return file_cmd_scanner_proto_scanner_proto_rawDescGZIP(), []int{1} +} + +func (x *ScanResponse) GetError() int32 { + if x != nil { + return x.Error + } + return 0 +} + +func (x *ScanResponse) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +func (x *ScanResponse) GetAccountsScanned() int32 { + if x != nil { + return x.AccountsScanned + } + return 0 +} + +// Message for health check requests +type HealthRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *HealthRequest) Reset() { + *x = HealthRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cmd_scanner_proto_scanner_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HealthRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HealthRequest) ProtoMessage() {} + +func (x *HealthRequest) ProtoReflect() protoreflect.Message { + mi := &file_cmd_scanner_proto_scanner_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HealthRequest.ProtoReflect.Descriptor instead. +func (*HealthRequest) Descriptor() ([]byte, []int) { + return file_cmd_scanner_proto_scanner_proto_rawDescGZIP(), []int{2} +} + +// Message for health check responses +type HealthResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Ready bool `protobuf:"varint,1,opt,name=ready,proto3" json:"ready,omitempty"` +} + +func (x *HealthResponse) Reset() { + *x = HealthResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cmd_scanner_proto_scanner_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HealthResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HealthResponse) ProtoMessage() {} + +func (x *HealthResponse) ProtoReflect() protoreflect.Message { + mi := &file_cmd_scanner_proto_scanner_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HealthResponse.ProtoReflect.Descriptor instead. +func (*HealthResponse) Descriptor() ([]byte, []int) { + return file_cmd_scanner_proto_scanner_proto_rawDescGZIP(), []int{3} +} + +func (x *HealthResponse) GetReady() bool { + if x != nil { + return x.Ready + } + return false +} + +var File_cmd_scanner_proto_scanner_proto protoreflect.FileDescriptor + +var file_cmd_scanner_proto_scanner_proto_rawDesc = []byte{ + 0x0a, 0x1f, 0x63, 0x6d, 0x64, 0x2f, 0x73, 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x72, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x12, 0x07, 0x73, 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x72, 0x22, 0x64, 0x0a, 0x0b, 0x53, 0x63, + 0x61, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x15, 0x0a, 0x06, 0x72, 0x75, 0x6e, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x72, 0x75, 0x6e, 0x49, 0x64, + 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x73, 0x18, + 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, + 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x5f, 0x61, 0x6c, 0x6c, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x41, 0x6c, 0x6c, + 0x22, 0x69, 0x0a, 0x0c, 0x53, 0x63, 0x61, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x12, 0x29, 0x0a, 0x10, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x5f, 0x73, 0x63, 0x61, + 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x61, 0x63, 0x63, 0x6f, + 0x75, 0x6e, 0x74, 0x73, 0x53, 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x64, 0x22, 0x0f, 0x0a, 0x0d, 0x48, + 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x26, 0x0a, 0x0e, + 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, + 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, + 0x65, 0x61, 0x64, 0x79, 0x32, 0x80, 0x01, 0x0a, 0x0e, 0x53, 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x72, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x33, 0x0a, 0x04, 0x53, 0x63, 0x61, 0x6e, 0x12, + 0x14, 0x2e, 0x73, 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x72, 0x2e, 0x53, 0x63, 0x61, 0x6e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x73, 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x72, 0x2e, + 0x53, 0x63, 0x61, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, + 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x16, 0x2e, 0x73, 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x72, + 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, + 0x2e, 0x73, 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x72, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x0b, 0x5a, 0x09, 0x2e, 0x2f, 0x73, 0x63, 0x61, + 0x6e, 0x6e, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_cmd_scanner_proto_scanner_proto_rawDescOnce sync.Once + file_cmd_scanner_proto_scanner_proto_rawDescData = file_cmd_scanner_proto_scanner_proto_rawDesc +) + +func file_cmd_scanner_proto_scanner_proto_rawDescGZIP() []byte { + file_cmd_scanner_proto_scanner_proto_rawDescOnce.Do(func() { + file_cmd_scanner_proto_scanner_proto_rawDescData = protoimpl.X.CompressGZIP(file_cmd_scanner_proto_scanner_proto_rawDescData) + }) + return file_cmd_scanner_proto_scanner_proto_rawDescData +} + +var file_cmd_scanner_proto_scanner_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_cmd_scanner_proto_scanner_proto_goTypes = []interface{}{ + (*ScanRequest)(nil), // 0: scanner.ScanRequest + (*ScanResponse)(nil), // 1: scanner.ScanResponse + (*HealthRequest)(nil), // 2: scanner.HealthRequest + (*HealthResponse)(nil), // 3: scanner.HealthResponse +} +var file_cmd_scanner_proto_scanner_proto_depIdxs = []int32{ + 0, // 0: scanner.ScannerService.Scan:input_type -> scanner.ScanRequest + 2, // 1: scanner.ScannerService.Health:input_type -> scanner.HealthRequest + 1, // 2: scanner.ScannerService.Scan:output_type -> scanner.ScanResponse + 3, // 3: scanner.ScannerService.Health:output_type -> scanner.HealthResponse + 2, // [2:4] is the sub-list for method output_type + 0, // [0:2] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_cmd_scanner_proto_scanner_proto_init() } +func file_cmd_scanner_proto_scanner_proto_init() { + if File_cmd_scanner_proto_scanner_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_cmd_scanner_proto_scanner_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ScanRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cmd_scanner_proto_scanner_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ScanResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cmd_scanner_proto_scanner_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HealthRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cmd_scanner_proto_scanner_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HealthResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_cmd_scanner_proto_scanner_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_cmd_scanner_proto_scanner_proto_goTypes, + DependencyIndexes: file_cmd_scanner_proto_scanner_proto_depIdxs, + MessageInfos: file_cmd_scanner_proto_scanner_proto_msgTypes, + }.Build() + File_cmd_scanner_proto_scanner_proto = out.File + file_cmd_scanner_proto_scanner_proto_rawDesc = nil + file_cmd_scanner_proto_scanner_proto_goTypes = nil + file_cmd_scanner_proto_scanner_proto_depIdxs = nil +} diff --git a/generated/scanner/scanner_grpc.pb.go b/generated/scanner/scanner_grpc.pb.go new file mode 100644 index 00000000..26e68926 --- /dev/null +++ b/generated/scanner/scanner_grpc.pb.go @@ -0,0 +1,149 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v3.19.6 +// source: cmd/scanner/proto/scanner.proto + +package scanner + +import ( + context "context" + + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + ScannerService_Scan_FullMethodName = "/scanner.ScannerService/Scan" + ScannerService_Health_FullMethodName = "/scanner.ScannerService/Health" +) + +// ScannerServiceClient is the client API for ScannerService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ScannerServiceClient interface { + Scan(ctx context.Context, in *ScanRequest, opts ...grpc.CallOption) (*ScanResponse, error) + Health(ctx context.Context, in *HealthRequest, opts ...grpc.CallOption) (*HealthResponse, error) +} + +type scannerServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewScannerServiceClient(cc grpc.ClientConnInterface) ScannerServiceClient { + return &scannerServiceClient{cc} +} + +func (c *scannerServiceClient) Scan(ctx context.Context, in *ScanRequest, opts ...grpc.CallOption) (*ScanResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ScanResponse) + err := c.cc.Invoke(ctx, ScannerService_Scan_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *scannerServiceClient) Health(ctx context.Context, in *HealthRequest, opts ...grpc.CallOption) (*HealthResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(HealthResponse) + err := c.cc.Invoke(ctx, ScannerService_Health_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ScannerServiceServer is the server API for ScannerService service. +// All implementations must embed UnimplementedScannerServiceServer +// for forward compatibility +type ScannerServiceServer interface { + Scan(context.Context, *ScanRequest) (*ScanResponse, error) + Health(context.Context, *HealthRequest) (*HealthResponse, error) + mustEmbedUnimplementedScannerServiceServer() +} + +// UnimplementedScannerServiceServer must be embedded to have forward compatible implementations. +type UnimplementedScannerServiceServer struct { +} + +func (UnimplementedScannerServiceServer) Scan(context.Context, *ScanRequest) (*ScanResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Scan not implemented") +} +func (UnimplementedScannerServiceServer) Health(context.Context, *HealthRequest) (*HealthResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Health not implemented") +} +func (UnimplementedScannerServiceServer) mustEmbedUnimplementedScannerServiceServer() {} + +// UnsafeScannerServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ScannerServiceServer will +// result in compilation errors. +type UnsafeScannerServiceServer interface { + mustEmbedUnimplementedScannerServiceServer() +} + +func RegisterScannerServiceServer(s grpc.ServiceRegistrar, srv ScannerServiceServer) { + s.RegisterService(&ScannerService_ServiceDesc, srv) +} + +func _ScannerService_Scan_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ScanRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ScannerServiceServer).Scan(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ScannerService_Scan_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ScannerServiceServer).Scan(ctx, req.(*ScanRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ScannerService_Health_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(HealthRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ScannerServiceServer).Health(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ScannerService_Health_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ScannerServiceServer).Health(ctx, req.(*HealthRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// ScannerService_ServiceDesc is the grpc.ServiceDesc for ScannerService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ScannerService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "scanner.ScannerService", + HandlerType: (*ScannerServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Scan", + Handler: _ScannerService_Scan_Handler, + }, + { + MethodName: "Health", + Handler: _ScannerService_Health_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "cmd/scanner/proto/scanner.proto", +} diff --git a/internal/actions/action_operation.go b/internal/actions/action_operation.go index 83b6a8f3..d14afb80 100644 --- a/internal/actions/action_operation.go +++ b/internal/actions/action_operation.go @@ -10,6 +10,9 @@ const ( // PowerOffCluster represents an action to power off a cluster. PowerOff ActionOperation = "PowerOff" + + // Scan represents an action to scan cloud accounts for resource discovery. + Scan ActionOperation = "Scan" ) func NewPowerOnClusterAction(target ActionTarget, requester string, description *string) *InstantAction { diff --git a/internal/actions/action_target.go b/internal/actions/action_target.go index 74034491..2dbb699a 100644 --- a/internal/actions/action_target.go +++ b/internal/actions/action_target.go @@ -14,6 +14,15 @@ type ActionTarget struct { // Instances is a list of instance IDs associated with the target cluster. Instances []string `db:"instances"` + + // TargetType indicates the resource type being targeted (Account, Cluster, Instance). + TargetType string `db:"target_type"` + + // SelectAll when true, targets all resources of the given TargetType. + SelectAll bool `db:"select_all"` + + // TargetAccountIDs lists account IDs for scan-type actions. + TargetAccountIDs []string `db:"target_account_ids"` } // NewActionTarget creates and returns a new instance of ActionTarget. diff --git a/internal/api/handlers/action_run_handler.go b/internal/api/handlers/action_run_handler.go new file mode 100644 index 00000000..6340f6b1 --- /dev/null +++ b/internal/api/handlers/action_run_handler.go @@ -0,0 +1,217 @@ +package handlers + +import ( + "errors" + "net/http" + "strconv" + + responsetypes "github.com/RHEcosystemAppEng/cluster-iq/internal/api/response_types" + "github.com/RHEcosystemAppEng/cluster-iq/internal/models" + "github.com/RHEcosystemAppEng/cluster-iq/internal/models/convert" + "github.com/RHEcosystemAppEng/cluster-iq/internal/models/dto" + "github.com/RHEcosystemAppEng/cluster-iq/internal/repositories" + "github.com/RHEcosystemAppEng/cluster-iq/internal/services" + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +// ActionRunHandler wires HTTP endpoints to the ActionRunService. +type ActionRunHandler struct { + service services.ActionRunService + logger *zap.Logger +} + +// NewActionRunHandler returns an ActionRunHandler with its dependencies. +func NewActionRunHandler(service services.ActionRunService, logger *zap.Logger) *ActionRunHandler { + return &ActionRunHandler{ + service: service, + logger: logger, + } +} + +type actionRunFilterParams struct { + ScheduleID string `form:"schedule_id"` + Status string `form:"status"` +} + +func (f *actionRunFilterParams) toRepoFilters() map[string]interface{} { + filters := make(map[string]interface{}) + if f.ScheduleID != "" { + filters["schedule_id"] = f.ScheduleID + } + if f.Status != "" { + filters["status"] = f.Status + } + return filters +} + +type listActionRunsRequest struct { + dto.PaginationRequest + Filters actionRunFilterParams `form:"inline"` +} + +// List returns a paginated list of action runs. +// +// @Summary List action runs +// @Description Paginated retrieval of action execution history. +// @Tags ActionRuns +// @Accept json +// @Produce json +// @Param schedule_id query string false "Filter by schedule ID" +// @Param status query string false "Filter by status" +// @Param page query int false "Page number" default(1) +// @Param page_size query int false "Items per page" default(10) +// @Success 200 {object} dto.ActionRunListResponse +// @Failure 400 {object} responsetypes.GenericErrorResponse +// @Failure 500 {object} responsetypes.GenericErrorResponse +// @Router /action-runs [get] +func (h *ActionRunHandler) List(c *gin.Context) { + var req listActionRunsRequest + + if err := c.ShouldBindQuery(&req); err != nil { + c.JSON(http.StatusBadRequest, responsetypes.GenericErrorResponse{ + Message: "Invalid query parameters: " + err.Error(), + }) + return + } + + opts := models.ListOptions{ + PageSize: req.PageSize, + Offset: (req.Page - 1) * req.PageSize, + Filters: req.Filters.toRepoFilters(), + } + + runs, total, err := h.service.List(c.Request.Context(), opts) + if err != nil { + h.logger.Error("error listing action runs", zap.Error(err)) + c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ + Message: "Failed to list action runs", + }) + return + } + + response := responsetypes.NewListResponse((&convert.ConverterImpl{}).ToActionRunDTOs(runs), total) + + c.Header("X-Total-Count", strconv.Itoa(total)) + c.JSON(http.StatusOK, response) +} + +// Get returns an action run by ID. +// +// @Summary Get action run by ID +// @Description Return an action run record. +// @Tags ActionRuns +// @Accept json +// @Produce json +// @Param id path string true "Action run ID" +// @Success 200 {object} dto.ActionRunDTOResponse +// @Failure 404 {object} responsetypes.GenericErrorResponse +// @Failure 500 {object} responsetypes.GenericErrorResponse +// @Router /action-runs/{id} [get] +func (h *ActionRunHandler) Get(c *gin.Context) { + runID := c.Param("id") + + run, err := h.service.Get(c.Request.Context(), runID) + if err != nil { + h.logger.Error("error getting action run", zap.String("run_id", runID), zap.Error(err)) + if errors.Is(err, repositories.ErrNotFound) { + c.JSON(http.StatusNotFound, responsetypes.GenericErrorResponse{ + Message: "Action run not found", + }) + return + } + c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ + Message: "Failed to retrieve action run", + }) + return + } + + c.JSON(http.StatusOK, (&convert.ConverterImpl{}).ToActionRunDTO(run)) +} + +// Create creates a new action run. +// +// @Summary Create action run +// @Description Create a new execution record for a scheduled action. +// @Tags ActionRuns +// @Accept json +// @Produce json +// @Param run body dto.ActionRunDTORequest true "Action run to create" +// @Success 201 {object} responsetypes.PostResponse +// @Failure 400 {object} responsetypes.GenericErrorResponse +// @Failure 500 {object} responsetypes.GenericErrorResponse +// @Router /action-runs [post] +func (h *ActionRunHandler) Create(c *gin.Context) { + var req dto.ActionRunDTORequest + + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, responsetypes.GenericErrorResponse{ + Message: "Invalid request body: " + err.Error(), + }) + return + } + + id, err := h.service.Create(c.Request.Context(), req.ScheduleID) + if err != nil { + h.logger.Error("error creating action run", zap.Error(err)) + c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ + Message: "Failed to create action run: " + err.Error(), + }) + return + } + + c.JSON(http.StatusCreated, responsetypes.PostResponse{ + Count: 1, + Status: strconv.FormatInt(id, 10), + }) +} + +// Update updates an action run's status. +// +// @Summary Update action run +// @Description Update the status of an action run (called by scanner on completion). +// @Tags ActionRuns +// @Accept json +// @Produce json +// @Param id path string true "Action run ID" +// @Param run body dto.ActionRunDTORequest true "Updated status" +// @Success 200 {object} nil +// @Failure 400 {object} responsetypes.GenericErrorResponse +// @Failure 404 {object} responsetypes.GenericErrorResponse +// @Failure 500 {object} responsetypes.GenericErrorResponse +// @Router /action-runs/{id} [patch] +func (h *ActionRunHandler) Update(c *gin.Context) { + runID := c.Param("id") + + var req dto.ActionRunDTORequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, responsetypes.GenericErrorResponse{ + Message: "Invalid request body: " + err.Error(), + }) + return + } + + if _, err := h.service.Get(c.Request.Context(), runID); err != nil { + h.logger.Error("error updating action run", zap.String("run_id", runID), zap.Error(err)) + if errors.Is(err, repositories.ErrNotFound) { + c.JSON(http.StatusNotFound, responsetypes.GenericErrorResponse{ + Message: "Action run not found", + }) + return + } + c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ + Message: "Failed to update action run", + }) + return + } + + if err := h.service.Update(c.Request.Context(), runID, req.Status, req.ErrorMsg); err != nil { + h.logger.Error("error updating action run", zap.String("run_id", runID), zap.Error(err)) + c.JSON(http.StatusInternalServerError, responsetypes.GenericErrorResponse{ + Message: "Failed to update action run: " + err.Error(), + }) + return + } + + c.Status(http.StatusOK) +} diff --git a/internal/clients/agent.go b/internal/clients/agent.go index f0d78593..8233fff8 100644 --- a/internal/clients/agent.go +++ b/internal/clients/agent.go @@ -64,6 +64,8 @@ func (a APIGRPCClient) ProcessInstantAction(ctx context.Context, action *actions return a.PowerOffCluster(ctx, action) case actions.PowerOn: return a.PowerOnCluster(ctx, action) + case actions.Scan: + return fmt.Errorf("scan operations are handled by the scanner service, not the agent") default: return fmt.Errorf("received InstantAction with unknown Operation") } diff --git a/internal/clients/scanner.go b/internal/clients/scanner.go new file mode 100644 index 00000000..56f8e244 --- /dev/null +++ b/internal/clients/scanner.go @@ -0,0 +1,64 @@ +package clients + +import ( + "context" + + pb "github.com/RHEcosystemAppEng/cluster-iq/generated/scanner" + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +// ScannerGRPCClient manages the gRPC client connection to the Scanner service. +type ScannerGRPCClient struct { + Client pb.ScannerServiceClient + conn *grpc.ClientConn + logger *zap.Logger +} + +// NewScannerGRPCClient initializes and returns a new ScannerGRPCClient. +func NewScannerGRPCClient(scannerURL string, logger *zap.Logger) (*ScannerGRPCClient, error) { + conn, err := grpc.NewClient(scannerURL, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, err + } + + return &ScannerGRPCClient{ + Client: pb.NewScannerServiceClient(conn), + conn: conn, + logger: logger, + }, nil +} + +// Close closes the underlying gRPC connection. +func (s *ScannerGRPCClient) Close() error { + return s.conn.Close() +} + +// Scan sends a scan request to the Scanner service. +func (s *ScannerGRPCClient) Scan(ctx context.Context, runID int64, accountIDs []string, selectAll bool) (*pb.ScanResponse, error) { + req := &pb.ScanRequest{ + RunId: runID, + AccountIds: accountIDs, + SelectAll: selectAll, + } + + s.logger.Info("Sending scan request to Scanner", + zap.Int64("run_id", runID), + zap.Strings("account_ids", accountIDs), + zap.Bool("select_all", selectAll), + ) + + resp, err := s.Client.Scan(ctx, req) + if err != nil { + return nil, err + } + + s.logger.Info("Scan response received", + zap.Int32("error_code", resp.Error), + zap.String("message", resp.Message), + zap.Int32("accounts_scanned", resp.AccountsScanned), + ) + + return resp, nil +} diff --git a/internal/cloud_executors/aws.go b/internal/cloud_executors/aws.go index f8dcf8a9..70270e40 100644 --- a/internal/cloud_executors/aws.go +++ b/internal/cloud_executors/aws.go @@ -58,6 +58,9 @@ func (e *AWSExecutor) ProcessAction(action actions.Action) error { case actions.PowerOff: return e.PowerOffCluster(target.GetInstances()) + case actions.Scan: + return fmt.Errorf("scan operations are handled by the scanner service, not the cloud executor") + default: // No registered ActionOperation return fmt.Errorf("cannot identify ActionOperation while processing an Action") } diff --git a/internal/config/agent_config.go b/internal/config/agent_config.go index 1c0bcb63..550fc27c 100644 --- a/internal/config/agent_config.go +++ b/internal/config/agent_config.go @@ -10,6 +10,8 @@ type ExecutorAgentServiceConfig struct { // APIURL refers to the ClusterIQ API Endpoint APIURL string `env:"CIQ_API_URL,required"` DBURL string `env:"CIQ_DB_URL,required"` + // ScannerURL refers to the Scanner gRPC endpoint + ScannerURL string `env:"CIQ_SCANNER_URL" envDefault:"scanner:50052"` // Credentials for accessing the cloud providers accounts Credentials CloudCredentialsConfig } diff --git a/internal/config/scanner_config.go b/internal/config/scanner_config.go index 97cc8143..ddc99aa5 100644 --- a/internal/config/scanner_config.go +++ b/internal/config/scanner_config.go @@ -6,6 +6,7 @@ import env "github.com/caarlos0/env/v11" type ScannerConfig struct { CloudCredentialsConfig APIURL string `env:"CIQ_API_URL,required"` + ListenURL string `env:"CIQ_SCANNER_LISTEN_URL" envDefault:"0.0.0.0:50052"` SkipNoOpenShiftInstances bool `env:"CIQ_SKIP_NO_OPENSHIFT_INSTANCES" envDefault:"true"` } diff --git a/internal/models/convert/generated.go b/internal/models/convert/generated.go index 1384b4bf..bfa70d35 100644 --- a/internal/models/convert/generated.go +++ b/internal/models/convert/generated.go @@ -45,10 +45,13 @@ func (c *ConverterImpl) ToActionDTO(source db.ActionDBResponse) dto.ActionDTORes dtoActionDTOResponse.Operation = source.Operation dtoActionDTOResponse.Status = source.Status dtoActionDTOResponse.Enabled = source.Enabled + dtoActionDTOResponse.TargetType = source.TargetType + dtoActionDTOResponse.SelectAll = source.SelectAll dtoActionDTOResponse.ClusterID = source.ClusterID dtoActionDTOResponse.Region = source.Region dtoActionDTOResponse.AccountID = source.AccountID dtoActionDTOResponse.Instances = StringArray(source.Instances) + dtoActionDTOResponse.TargetAccountIDs = StringArray(source.TargetAccountIDs) return dtoActionDTOResponse } func (c *ConverterImpl) ToActionDTOs(source []db.ActionDBResponse) []dto.ActionDTOResponse { @@ -61,6 +64,26 @@ func (c *ConverterImpl) ToActionDTOs(source []db.ActionDBResponse) []dto.ActionD } return dtoActionDTOResponseList } +func (c *ConverterImpl) ToActionRunDTO(source db.ActionRunDBResponse) dto.ActionRunDTOResponse { + var dtoActionRunDTOResponse dto.ActionRunDTOResponse + dtoActionRunDTOResponse.ID = source.ID + dtoActionRunDTOResponse.ScheduleID = source.ScheduleID + dtoActionRunDTOResponse.StartedAt = NullTime(source.StartedAt) + dtoActionRunDTOResponse.FinishedAt = NullTime(source.FinishedAt) + dtoActionRunDTOResponse.Status = source.Status + dtoActionRunDTOResponse.ErrorMsg = NullString(source.ErrorMsg) + return dtoActionRunDTOResponse +} +func (c *ConverterImpl) ToActionRunDTOs(source []db.ActionRunDBResponse) []dto.ActionRunDTOResponse { + var dtoActionRunDTOResponseList []dto.ActionRunDTOResponse + if source != nil { + dtoActionRunDTOResponseList = make([]dto.ActionRunDTOResponse, len(source)) + for i := 0; i < len(source); i++ { + dtoActionRunDTOResponseList[i] = c.ToActionRunDTO(source[i]) + } + } + return dtoActionRunDTOResponseList +} func (c *ConverterImpl) ToClusterDTO(source db.ClusterDBResponse) dto.ClusterDTOResponse { var dtoClusterDTOResponse dto.ClusterDTOResponse dtoClusterDTOResponse.ClusterID = source.ClusterID diff --git a/internal/models/convert/mapper.go b/internal/models/convert/mapper.go index 975953a6..e82611a9 100644 --- a/internal/models/convert/mapper.go +++ b/internal/models/convert/mapper.go @@ -49,6 +49,10 @@ type Converter interface { ToActionDTO(src db.ActionDBResponse) dto.ActionDTOResponse ToActionDTOs(src []db.ActionDBResponse) []dto.ActionDTOResponse + // ActionRun + ToActionRunDTO(src db.ActionRunDBResponse) dto.ActionRunDTOResponse + ToActionRunDTOs(src []db.ActionRunDBResponse) []dto.ActionRunDTOResponse + // Instance ToInstanceDTO(src db.InstanceDBResponse) dto.InstanceDTOResponse ToInstanceDTOs(src []db.InstanceDBResponse) []dto.InstanceDTOResponse diff --git a/internal/models/db/action.go b/internal/models/db/action.go index a2a31000..4f597cd4 100644 --- a/internal/models/db/action.go +++ b/internal/models/db/action.go @@ -9,15 +9,18 @@ import ( // ActionDBResponse represents the database schema for action details, // linking each field to a corresponding column in the database. type ActionDBResponse struct { - ID string `db:"id"` - Type string `db:"type"` - Time sql.NullTime `db:"time"` - CronExp sql.NullString `db:"cron_exp"` - Operation string `db:"operation"` - Status string `db:"status"` - Enabled bool `db:"enabled"` - ClusterID string `db:"cluster_id"` - Region string `db:"region"` - AccountID string `db:"account_id"` - Instances pq.StringArray `db:"instances"` + ID string `db:"id"` + Type string `db:"type"` + Time sql.NullTime `db:"time"` + CronExp sql.NullString `db:"cron_exp"` + Operation string `db:"operation"` + Status string `db:"status"` + Enabled bool `db:"enabled"` + TargetType string `db:"target_type"` + SelectAll bool `db:"select_all"` + ClusterID string `db:"cluster_id"` + Region string `db:"region"` + AccountID string `db:"account_id"` + Instances pq.StringArray `db:"instances"` + TargetAccountIDs pq.StringArray `db:"target_account_ids"` } diff --git a/internal/models/db/action_run.go b/internal/models/db/action_run.go new file mode 100644 index 00000000..d66a5f6f --- /dev/null +++ b/internal/models/db/action_run.go @@ -0,0 +1,13 @@ +package db + +import "database/sql" + +// ActionRunDBResponse represents the database schema for action execution history. +type ActionRunDBResponse struct { + ID string `db:"id"` + ScheduleID string `db:"schedule_id"` + StartedAt sql.NullTime `db:"started_at"` + FinishedAt sql.NullTime `db:"finished_at"` + Status string `db:"status"` + ErrorMsg sql.NullString `db:"error_msg"` +} diff --git a/internal/models/dto/action_dto.go b/internal/models/dto/action_dto.go index 37f4b770..4e71c46d 100644 --- a/internal/models/dto/action_dto.go +++ b/internal/models/dto/action_dto.go @@ -89,19 +89,22 @@ func ToModelActionList(dtos []ActionDTORequest) *[]actions.Action { // ActionDTOResponse represents the data transfer object for an action response, // containing action details including schedule, cron expression, and target resources. type ActionDTOResponse struct { - ID string `json:"id"` - Type string `json:"type"` - Time time.Time `json:"time"` - CronExp string `json:"cronExpression"` - Operation string `json:"operation"` - Status string `json:"status"` - Enabled bool `json:"enabled"` - ClusterID string `json:"clusterId"` - Region string `json:"region"` - AccountID string `json:"accountId"` - Instances []string `json:"instances"` - Requester string `json:"requester"` - Description *string `json:"description"` + ID string `json:"id"` + Type string `json:"type"` + Time time.Time `json:"time"` + CronExp string `json:"cronExpression"` + Operation string `json:"operation"` + Status string `json:"status"` + Enabled bool `json:"enabled"` + TargetType string `json:"targetType"` + SelectAll bool `json:"selectAll"` + ClusterID string `json:"clusterId"` + Region string `json:"region"` + AccountID string `json:"accountId"` + Instances []string `json:"instances"` + TargetAccountIDs []string `json:"targetAccountIds"` + Requester string `json:"requester"` + Description *string `json:"description"` } // @name ActionResponse // ToModelAction converts ActionDTOResponse to actions.Action diff --git a/internal/models/dto/action_run_dto.go b/internal/models/dto/action_run_dto.go new file mode 100644 index 00000000..02c9d801 --- /dev/null +++ b/internal/models/dto/action_run_dto.go @@ -0,0 +1,20 @@ +package dto + +import "time" + +// ActionRunDTORequest represents a request to create or update an action run. +type ActionRunDTORequest struct { + ScheduleID string `json:"scheduleId" binding:"required"` + Status string `json:"status"` + ErrorMsg string `json:"errorMsg"` +} // @name ActionRunRequest + +// ActionRunDTOResponse represents the response for an action execution record. +type ActionRunDTOResponse struct { + ID string `json:"id"` + ScheduleID string `json:"scheduleId"` + StartedAt time.Time `json:"startedAt"` + FinishedAt time.Time `json:"finishedAt"` + Status string `json:"status"` + ErrorMsg string `json:"errorMsg"` +} // @name ActionRunResponse diff --git a/internal/models/dto/wrappers.go b/internal/models/dto/wrappers.go index 5ae29bd7..14daf5a1 100644 --- a/internal/models/dto/wrappers.go +++ b/internal/models/dto/wrappers.go @@ -45,3 +45,9 @@ type SystemEventListResponse struct { type ClusterEventListResponse struct { responsetypes.ListResponse[ClusterEventDTOResponse] } // @name ClusterEventListResponse + +// ActionRunListResponse wraps a paginated list of ActionRunDTOResponse items +// for OpenAPI schema generation. +type ActionRunListResponse struct { + responsetypes.ListResponse[ActionRunDTOResponse] +} // @name ActionRunListResponse diff --git a/internal/repositories/action_repository.go b/internal/repositories/action_repository.go index d79771aa..fe5f3cc0 100644 --- a/internal/repositories/action_repository.go +++ b/internal/repositories/action_repository.go @@ -33,60 +33,31 @@ const ( enabled = false WHERE id = $1 ` - // InsertAction inserts a new action returning the ID - InsertActionsQuery = ` - INSERT INTO schedule ( - type, - time, - operation, - target, - status, - enabled - ) VALUES ( - :type, - (SELECT now()), - :operation, - (SELECT id FROM clusters WHERE cluster_id = :target.cluster_id), - :status, - :enabled - ) RETURNING id + + InsertTargetQuery = `INSERT INTO targets (target_type, select_all) VALUES ($1, $2) RETURNING id` + + LinkTargetClusterQuery = `INSERT INTO target_clusters (target_id, cluster_id) SELECT $1, id FROM clusters WHERE cluster_id = $2` + + LinkTargetAccountQuery = `INSERT INTO target_accounts (target_id, account_id) SELECT $1, id FROM accounts WHERE account_id = $2` + + InsertScheduledActionWithTargetQuery = ` + INSERT INTO schedule (type, time, operation, target, status, enabled) + VALUES ('scheduled_action', $1, $2, $3, $4, $5) + RETURNING id ` - // InsertScheduledActionQuery inserts new scheduled actions on the DB - InsertScheduledActionsQuery = ` - INSERT INTO schedule ( - type, - time, - operation, - target, - status, - enabled - ) VALUES ( - 'scheduled_action', - :time, - :operation, - (SELECT id FROM clusters WHERE cluster_id=:target.cluster_id), - :status, - :enabled - ) + + InsertCronActionWithTargetQuery = ` + INSERT INTO schedule (type, cron_exp, operation, target, status, enabled) + VALUES ('cron_action', $1, $2, $3, $4, $5) + RETURNING id ` - // InsertCronActionQuery inserts new Cron actions on the DB - InsertCronActionsQuery = ` - INSERT INTO schedule ( - type, - cron_exp, - operation, - target, - status, - enabled - ) VALUES ( - 'cron_action', - :cron_exp, - :operation, - (SELECT id FROM clusters WHERE cluster_id=:target.cluster_id), - :status, - :enabled - ) + + InsertInstantActionWithTargetQuery = ` + INSERT INTO schedule (type, time, operation, target, status, enabled) + VALUES ('instant_action', NOW(), $1, $2, $3, $4) + RETURNING id ` + // UpdateActionQuery updates a single action on the DB UpdateActionQuery = ` UPDATE schedule @@ -192,11 +163,7 @@ func (r *actionRepositoryImpl) GetByID(ctx context.Context, actionID string) (db // // Returns: // - An error if the insert fails -// -// TODO: Temporal fix returning TX from DBClient to manage both insertions in the same sql transaction func (r *actionRepositoryImpl) Create(ctx context.Context, newActions []actions.Action) (err error) { - schedActions, cronActions := actions.SplitActionsByType(newActions) - tx, err := r.db.NewTx(ctx) if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) @@ -207,33 +174,93 @@ func (r *actionRepositoryImpl) Create(ctx context.Context, newActions []actions. } }() - // Writing Scheduled Actions - if len(schedActions) > 0 { - if _, err := tx.NamedExecContext(ctx, InsertScheduledActionsQuery, schedActions); err != nil { - return fmt.Errorf("failed to insert scheduled actions: %w", err) + for _, action := range newActions { + targetID, targetErr := createTargetForAction(ctx, tx, action) + if targetErr != nil { + return fmt.Errorf("failed to create target: %w", targetErr) } - } - // Writing Cron Actions - if len(cronActions) > 0 { - if _, err := tx.NamedExecContext(ctx, InsertCronActionsQuery, cronActions); err != nil { - return fmt.Errorf("failed to insert cron actions: %w", err) + switch a := action.(type) { + case *actions.ScheduledAction: + _, err = tx.ExecContext(ctx, InsertScheduledActionWithTargetQuery, + a.When, a.Operation, targetID, a.Status, a.Enabled) + case *actions.CronAction: + _, err = tx.ExecContext(ctx, InsertCronActionWithTargetQuery, + a.Expression, a.Operation, targetID, a.Status, a.Enabled) + default: + return fmt.Errorf("unsupported action type for batch create: %T", action) + } + if err != nil { + return fmt.Errorf("failed to insert schedule: %w", err) } } - // Commit the transaction return tx.Commit() } -// AddEvent inserts a new audit event into the database and returns the event ID. func (r *actionRepositoryImpl) CreateAction(ctx context.Context, action actions.Action) (int64, error) { - var returnedValue int64 - returnedValue, err := r.db.InsertWithReturnWithContext(ctx, InsertActionsQuery, action) + tx, err := r.db.NewTx(ctx) if err != nil { - return -1, err + return -1, fmt.Errorf("failed to begin transaction: %w", err) + } + defer func() { + if err != nil { + _ = tx.Rollback() + } + }() + + targetID, err := createTargetForAction(ctx, tx, action) + if err != nil { + return -1, fmt.Errorf("failed to create target: %w", err) + } + + var scheduleID int64 + err = tx.QueryRowContext(ctx, InsertInstantActionWithTargetQuery, + action.GetActionOperation(), targetID, action.(*actions.InstantAction).Status, action.(*actions.InstantAction).Enabled, + ).Scan(&scheduleID) + if err != nil { + return -1, fmt.Errorf("failed to insert action: %w", err) + } + + if err = tx.Commit(); err != nil { + return -1, fmt.Errorf("failed to commit action: %w", err) + } + + return scheduleID, nil +} + +func createTargetForAction(ctx context.Context, tx interface { + QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row + ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) +}, action actions.Action) (int64, error) { + target := action.GetTarget() + + targetType := target.TargetType + if targetType == "" { + targetType = "Cluster" + } + + var targetID int64 + if err := tx.QueryRowContext(ctx, InsertTargetQuery, targetType, target.SelectAll).Scan(&targetID); err != nil { + return 0, fmt.Errorf("failed to insert target: %w", err) + } + + switch targetType { + case "Cluster": + if target.ClusterID != "" { + if _, err := tx.ExecContext(ctx, LinkTargetClusterQuery, targetID, target.ClusterID); err != nil { + return 0, fmt.Errorf("failed to link target cluster: %w", err) + } + } + case "Account": + for _, accountID := range target.TargetAccountIDs { + if _, err := tx.ExecContext(ctx, LinkTargetAccountQuery, targetID, accountID); err != nil { + return 0, fmt.Errorf("failed to link target account: %w", err) + } + } } - return returnedValue, nil + return targetID, nil } // Delete removes an actions.ScheduledAction action from the DB based on its ID diff --git a/internal/repositories/action_run_repository.go b/internal/repositories/action_run_repository.go new file mode 100644 index 00000000..1bed2316 --- /dev/null +++ b/internal/repositories/action_run_repository.go @@ -0,0 +1,105 @@ +package repositories + +import ( + "context" + "database/sql" + "errors" + "fmt" + + dbclient "github.com/RHEcosystemAppEng/cluster-iq/internal/db_client" + "github.com/RHEcosystemAppEng/cluster-iq/internal/models" + "github.com/RHEcosystemAppEng/cluster-iq/internal/models/db" +) + +const ( + ActionRunsTable = "action_runs" + + InsertActionRunQuery = ` + INSERT INTO action_runs (schedule_id) + VALUES ($1) + RETURNING id + ` + + UpdateActionRunQuery = ` + UPDATE action_runs + SET + status = $1, + finished_at = NOW(), + error_msg = $2 + WHERE id = $3 + ` +) + +var _ ActionRunRepository = (*actionRunRepositoryImpl)(nil) + +// ActionRunRepository defines the interface for action run data access operations. +type ActionRunRepository interface { + List(ctx context.Context, opts models.ListOptions) ([]db.ActionRunDBResponse, int, error) + GetByID(ctx context.Context, runID string) (db.ActionRunDBResponse, error) + Create(ctx context.Context, scheduleID string) (int64, error) + Update(ctx context.Context, runID string, status string, errorMsg string) error +} + +type actionRunRepositoryImpl struct { + db *dbclient.DBClient +} + +func NewActionRunRepository(db *dbclient.DBClient) ActionRunRepository { + return &actionRunRepositoryImpl{db: db} +} + +func (r *actionRunRepositoryImpl) List(ctx context.Context, opts models.ListOptions) ([]db.ActionRunDBResponse, int, error) { + runs := []db.ActionRunDBResponse{} + + if err := r.db.SelectWithContext(ctx, &runs, ActionRunsTable, opts, "id", "*"); err != nil { + return runs, 0, fmt.Errorf("failed to list action runs: %w", err) + } + + return runs, len(runs), nil +} + +func (r *actionRunRepositoryImpl) GetByID(ctx context.Context, runID string) (db.ActionRunDBResponse, error) { + var run db.ActionRunDBResponse + + opts := models.ListOptions{ + Filters: map[string]interface{}{ + "id": runID, + }, + } + + if err := r.db.GetWithContext(ctx, &run, ActionRunsTable, opts, "id", "*"); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return run, ErrNotFound + } + return run, err + } + + return run, nil +} + +func (r *actionRunRepositoryImpl) Create(ctx context.Context, scheduleID string) (int64, error) { + var id int64 + err := r.db.QueryRowContext(ctx, &id, InsertActionRunQuery, scheduleID) + if err != nil { + return -1, fmt.Errorf("failed to create action run: %w", err) + } + return id, nil +} + +func (r *actionRunRepositoryImpl) Update(ctx context.Context, runID string, status string, errorMsg string) error { + tx, err := r.db.NewTx(ctx) + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + defer func() { + if err != nil { + _ = tx.Rollback() + } + }() + + if _, err = tx.ExecContext(ctx, UpdateActionRunQuery, status, errorMsg, runID); err != nil { + return fmt.Errorf("failed to update action run: %w", err) + } + + return tx.Commit() +} diff --git a/internal/services/action_run_service.go b/internal/services/action_run_service.go new file mode 100644 index 00000000..ed4b909e --- /dev/null +++ b/internal/services/action_run_service.go @@ -0,0 +1,55 @@ +package services + +import ( + "context" + "fmt" + + "github.com/RHEcosystemAppEng/cluster-iq/internal/models" + "github.com/RHEcosystemAppEng/cluster-iq/internal/models/db" + "github.com/RHEcosystemAppEng/cluster-iq/internal/repositories" +) + +// ActionRunService defines the interface for action run business logic. +type ActionRunService interface { + List(ctx context.Context, options models.ListOptions) ([]db.ActionRunDBResponse, int, error) + Get(ctx context.Context, runID string) (db.ActionRunDBResponse, error) + Create(ctx context.Context, scheduleID string) (int64, error) + Update(ctx context.Context, runID string, status string, errorMsg string) error +} + +var _ ActionRunService = (*actionRunServiceImpl)(nil) + +type actionRunServiceImpl struct { + repo repositories.ActionRunRepository +} + +func NewActionRunService(repo repositories.ActionRunRepository) ActionRunService { + return &actionRunServiceImpl{repo: repo} +} + +func (s *actionRunServiceImpl) List(ctx context.Context, options models.ListOptions) ([]db.ActionRunDBResponse, int, error) { + return s.repo.List(ctx, options) +} + +func (s *actionRunServiceImpl) Get(ctx context.Context, runID string) (db.ActionRunDBResponse, error) { + run, err := s.repo.GetByID(ctx, runID) + if err != nil { + return run, fmt.Errorf("get action run %s: %w", runID, err) + } + return run, nil +} + +func (s *actionRunServiceImpl) Create(ctx context.Context, scheduleID string) (int64, error) { + id, err := s.repo.Create(ctx, scheduleID) + if err != nil { + return -1, fmt.Errorf("create action run: %w", err) + } + return id, nil +} + +func (s *actionRunServiceImpl) Update(ctx context.Context, runID string, status string, errorMsg string) error { + if err := s.repo.Update(ctx, runID, status, errorMsg); err != nil { + return fmt.Errorf("update action run %s: %w", runID, err) + } + return nil +} diff --git a/test/integration/api_action_runs_integration_test.go b/test/integration/api_action_runs_integration_test.go new file mode 100644 index 00000000..ebb226fd --- /dev/null +++ b/test/integration/api_action_runs_integration_test.go @@ -0,0 +1,256 @@ +package integration + +import ( + "bytes" + "encoding/json" + "net/http" + "testing" + + responsetypes "github.com/RHEcosystemAppEng/cluster-iq/internal/api/response_types" + "github.com/RHEcosystemAppEng/cluster-iq/internal/models/dto" +) + +const ( + APIActionRunsURL = APIBaseURL + "/action-runs" +) + +func TestAPIActionRuns(t *testing.T) { + waitForAPIReady(t) + + if err := refreshInventory(); err != nil { + t.Fatal("Error refreshing inventory") + } + + t.Run("List Action Runs", func(t *testing.T) { testListActionRuns(t) }) + t.Run("List Action Runs with Pagination", func(t *testing.T) { testListActionRunsWithPagination(t) }) + t.Run("List Action Runs filtered by Status", func(t *testing.T) { testListActionRunsFilteredByStatus(t) }) + t.Run("Get Action Run By ID Success", func(t *testing.T) { testGetActionRunByID_Exists(t) }) + t.Run("Get Action Run By ID Not Found", func(t *testing.T) { testGetActionRunByID_NoExists(t) }) + t.Run("Post Action Run", func(t *testing.T) { testPostActionRun(t) }) + t.Run("Update Action Run", func(t *testing.T) { testUpdateActionRun(t) }) + t.Run("Update Action Run Not Found", func(t *testing.T) { testUpdateActionRun_NoExists(t) }) +} + +func testListActionRuns(t *testing.T) { + expectedCount := 2 + expectedHTTPCode := http.StatusOK + + resp, err := http.Get(APIActionRunsURL) + if err != nil { + t.Fatalf("Failed to make request: %v", err) + } + defer resp.Body.Close() + + checkHTTPResponseCode(t, resp, expectedHTTPCode) + + var response responsetypes.ListResponse[dto.ActionRunDTOResponse] + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + t.Fatalf("Failed to decode response body: %v", err) + } + + if response.Count != expectedCount { + t.Fatalf("Expected Count: '%d', got: '%d'", expectedCount, response.Count) + } + + if len := len(response.Items); len != expectedCount { + t.Fatalf("Expected Items: '%d', got: '%d'", expectedCount, len) + } +} + +func testListActionRunsWithPagination(t *testing.T) { + expectedCount := 1 + expectedHTTPCode := http.StatusOK + + resp, err := http.Get(APIActionRunsURL + "?page=1&page_size=1") + if err != nil { + t.Fatalf("Failed to make request: %v", err) + } + defer resp.Body.Close() + + checkHTTPResponseCode(t, resp, expectedHTTPCode) + + var response responsetypes.ListResponse[dto.ActionRunDTOResponse] + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + t.Fatalf("Failed to decode response body: %v", err) + } + + if response.Count != expectedCount { + t.Fatalf("Expected Count: '%d', got: '%d'", expectedCount, response.Count) + } + + if len := len(response.Items); len != expectedCount { + t.Fatalf("Expected Items: '%d', got: '%d'", expectedCount, len) + } +} + +func testListActionRunsFilteredByStatus(t *testing.T) { + expectedCount := 1 + expectedHTTPCode := http.StatusOK + + resp, err := http.Get(APIActionRunsURL + "?status=Running") + if err != nil { + t.Fatalf("Failed to make request: %v", err) + } + defer resp.Body.Close() + + checkHTTPResponseCode(t, resp, expectedHTTPCode) + + var response responsetypes.ListResponse[dto.ActionRunDTOResponse] + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + t.Fatalf("Failed to decode response body: %v", err) + } + + if response.Count != expectedCount { + t.Fatalf("Expected Count: '%d', got: '%d'", expectedCount, response.Count) + } + + if len := len(response.Items); len != expectedCount { + t.Fatalf("Expected Items: '%d', got: '%d'", expectedCount, len) + } + + if response.Items[0].Status != "Running" { + t.Fatalf("Expected Status: 'Running', got: '%s'", response.Items[0].Status) + } +} + +func testGetActionRunByID_Exists(t *testing.T) { + expectedRunID := "1" + expectedHTTPCode := http.StatusOK + + resp, err := http.Get(APIActionRunsURL + "/" + expectedRunID) + if err != nil { + t.Fatalf("Failed to make request: %v", err) + } + defer resp.Body.Close() + + checkHTTPResponseCode(t, resp, expectedHTTPCode) + + var response dto.ActionRunDTOResponse + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + t.Fatalf("Failed to decode response body: %v", err) + } + + if response.ID != expectedRunID { + t.Fatalf("Expected ID: '%s', got: '%s'", expectedRunID, response.ID) + } +} + +func testGetActionRunByID_NoExists(t *testing.T) { + expectedMsg := "Action run not found" + expectedHTTPCode := http.StatusNotFound + + resp, err := http.Get(APIActionRunsURL + "/" + "9999") + if err != nil { + t.Fatalf("Failed to make request: %v", err) + } + defer resp.Body.Close() + + checkHTTPResponseCode(t, resp, expectedHTTPCode) + + var response responsetypes.GenericErrorResponse + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + t.Fatalf("Failed to decode response body: %v", err) + } + + if response.Message != expectedMsg { + t.Fatalf("Expected Message: '%s', got: '%s'", expectedMsg, response.Message) + } +} + +func testPostActionRun(t *testing.T) { + expectedHTTPCode := http.StatusCreated + expectedCount := 1 + + payload := dto.ActionRunDTORequest{ + ScheduleID: "1", + } + b, err := json.Marshal(payload) + if err != nil { + t.Fatalf("Failed to marshal data in post request: %v", err) + } + + resp, err := http.Post(APIActionRunsURL, "application/json", bytes.NewBuffer(b)) + if err != nil { + t.Fatalf("Failed to make POST request: %v", err) + } + defer resp.Body.Close() + + checkHTTPResponseCode(t, resp, expectedHTTPCode) + + var response responsetypes.PostResponse + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + t.Fatalf("Failed to decode response body: %v", err) + } + + if response.Count != expectedCount { + t.Fatalf("Expected Count: '%d', got '%d'", expectedCount, response.Count) + } +} + +func testUpdateActionRun(t *testing.T) { + expectedHTTPCode := http.StatusOK + + payload := dto.ActionRunDTORequest{ + ScheduleID: "1", + Status: "Failed", + ErrorMsg: "test error", + } + b, err := json.Marshal(payload) + if err != nil { + t.Fatalf("Failed to marshal data in patch request: %v", err) + } + + req, err := http.NewRequest(http.MethodPatch, APIActionRunsURL+"/1", bytes.NewBuffer(b)) + if err != nil { + t.Fatalf("Failed to create PATCH request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + + client := http.Client{} + resp, err := client.Do(req) + if err != nil { + t.Fatalf("Failed to make PATCH request: %v", err) + } + defer resp.Body.Close() + + checkHTTPResponseCode(t, resp, expectedHTTPCode) +} + +func testUpdateActionRun_NoExists(t *testing.T) { + expectedMsg := "Action run not found" + expectedHTTPCode := http.StatusNotFound + + payload := dto.ActionRunDTORequest{ + ScheduleID: "1", + Status: "Failed", + ErrorMsg: "test error", + } + b, err := json.Marshal(payload) + if err != nil { + t.Fatalf("Failed to marshal data in patch request: %v", err) + } + + req, err := http.NewRequest(http.MethodPatch, APIActionRunsURL+"/9999", bytes.NewBuffer(b)) + if err != nil { + t.Fatalf("Failed to create PATCH request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + + client := http.Client{} + resp, err := client.Do(req) + if err != nil { + t.Fatalf("Failed to make PATCH request: %v", err) + } + defer resp.Body.Close() + + checkHTTPResponseCode(t, resp, expectedHTTPCode) + + var response responsetypes.GenericErrorResponse + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + t.Fatalf("Failed to decode response body: %v", err) + } + + if response.Message != expectedMsg { + t.Fatalf("Expected Message: '%s', got: '%s'", expectedMsg, response.Message) + } +} From ec521528ca078e83808530dc6f3c5f210f40296f Mon Sep 17 00:00:00 2001 From: r2dedios Date: Fri, 29 May 2026 19:34:13 +0200 Subject: [PATCH 02/22] fix(compose): ensure API waits for DB initialization --- deployments/compose/compose-devel.yaml | 6 +++--- deployments/compose/compose-gh.yaml | 11 +++++++++++ deployments/compose/compose-integration-tests.yaml | 11 +++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/deployments/compose/compose-devel.yaml b/deployments/compose/compose-devel.yaml index 96b2eacc..d3f86112 100644 --- a/deployments/compose/compose-devel.yaml +++ b/deployments/compose/compose-devel.yaml @@ -10,8 +10,8 @@ services: container_name: api restart: "always" depends_on: - pgsql: - condition: service_started + init-pgsql: + condition: service_completed_successfully ports: - 8081:8080 environment: @@ -150,7 +150,7 @@ services: echo "Done!" ' environment: - CIQ_DB_PRELOAD_DATA: "true" + CIQ_DB_PRELOAD_DATA: "false" volumes: - ./../../db/sql/init.sql:/init.sql:ro,Z - ./../../db/sql/cron.sql:/cron.sql:ro,Z diff --git a/deployments/compose/compose-gh.yaml b/deployments/compose/compose-gh.yaml index 5e1139cd..040a15b1 100644 --- a/deployments/compose/compose-gh.yaml +++ b/deployments/compose/compose-gh.yaml @@ -7,6 +7,9 @@ services: image: quay.io/ecosystem-appeng/cluster-iq-api:latest container_name: api restart: always + depends_on: + init-pgsql: + condition: service_completed_successfully ports: - 8081:8080 environment: @@ -28,6 +31,11 @@ services: POSTGRESQL_PASSWORD: "password" POSTGRESQL_DATABASE: "clusteriq" POSTGRESQL_ADMIN_PASSWORD: "admin" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 networks: - cluster_iq @@ -35,6 +43,9 @@ services: image: quay.io/fedora/postgresql-16:16 container_name: init-pgsql restart: "no" + depends_on: + pgsql: + condition: service_healthy command: | sh -c 'while true; do psql postgresql://user:password@pgsql:5432/clusteriq -c "SELECT true" && break || sleep 2; diff --git a/deployments/compose/compose-integration-tests.yaml b/deployments/compose/compose-integration-tests.yaml index 558c2b76..93abe1ac 100644 --- a/deployments/compose/compose-integration-tests.yaml +++ b/deployments/compose/compose-integration-tests.yaml @@ -7,6 +7,9 @@ services: image: quay.io/ecosystem-appeng/cluster-iq-api:latest container_name: api-test restart: always + depends_on: + init-pgsql-test: + condition: service_completed_successfully ports: - 8081:8080 environment: @@ -30,6 +33,11 @@ services: POSTGRESQL_ADMIN_PASSWORD: "admin" POSTGRESQL_LIBRARIES: "pg_cron" POSTGRESQL_EXTENSIONS: "pg_cron" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 networks: - cluster_iq @@ -37,6 +45,9 @@ services: image: quay.io/ecosystem-appeng/cluster-iq-pgsql:latest container_name: init-pgsql-test restart: "no" + depends_on: + pgsql-test: + condition: service_healthy command: | sh -c ' echo "Waiting for PGSQL to start up" From 01218a71e5978831fdb6630f02ca09026e80a9d3 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Fri, 29 May 2026 19:34:21 +0200 Subject: [PATCH 03/22] fix(scanner): add API readiness check, account seeding, and billing optimization --- cmd/scanner/scanner.go | 147 +++++++++++++----------- internal/stocker/aws_billing_stocker.go | 27 ++--- 2 files changed, 95 insertions(+), 79 deletions(-) diff --git a/cmd/scanner/scanner.go b/cmd/scanner/scanner.go index 9285c7d9..d78335a1 100644 --- a/cmd/scanner/scanner.go +++ b/cmd/scanner/scanner.go @@ -6,6 +6,7 @@ import ( "crypto/md5" "crypto/tls" "encoding/json" + "errors" "fmt" "io" "net" @@ -35,8 +36,6 @@ const ( apiClusterEndpoint = "/clusters" apiInstanceEndpoint = "/instances" apiExpenseEndpoint = "/expenses" - apiActionRunEndpoint = "/action-runs" - // apiRequestTimeout defines the timeout for HTTP POST requests to the API apiRequestTimeout = 60 * time.Second ) @@ -110,6 +109,58 @@ func (s *Scanner) loadAccounts() error { return nil } +// waitForAPI blocks until the API server responds to /healthcheck. +func (s *Scanner) waitForAPI() { + url := fmt.Sprintf("%s/healthcheck", APIURL) + for { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err == nil { + resp, err := client.Do(req) + if resp != nil { + resp.Body.Close() + } + if err == nil && resp.StatusCode == http.StatusOK { + cancel() + s.logger.Info("API server is ready") + return + } + } + cancel() + s.logger.Info("Waiting for API server...", zap.String("url", url)) + time.Sleep(3 * time.Second) + } +} + +// seedAccounts posts account records from the credentials file to the API +// so they are available in the console before the first scan runs. +func (s *Scanner) seedAccounts() error { + var accounts []dto.AccountDTORequest + for _, ac := range s.allAccounts { + accounts = append(accounts, dto.AccountDTORequest{ + AccountID: ac.ID, + AccountName: ac.Name, + Provider: ac.Provider, + }) + } + + b, err := json.Marshal(accounts) + if err != nil { + return fmt.Errorf("failed to marshal seed accounts: %w", err) + } + + if err := postData(apiAccountEndpoint, b); err != nil { + return fmt.Errorf("failed to seed accounts: %w", err) + } + + if err := refreshInventory(s.logger); err != nil { + return fmt.Errorf("failed to refresh materialized views after seed: %w", err) + } + + s.logger.Info("Seeded accounts into database", zap.Int("count", len(accounts))) + return nil +} + // buildInventory creates an Inventory from the given account configs. func (s *Scanner) buildInventory(configs []credentials.AccountConfig) (*inventory.Inventory, error) { inv := inventory.NewInventory() @@ -129,9 +180,10 @@ func (s *Scanner) buildInventory(configs []credentials.AccountConfig) (*inventor } // nolint:cyclop -func (s *Scanner) createStockers(inv *inventory.Inventory) ([]stocker.Stocker, []stocker.Stocker) { +func (s *Scanner) createStockers(inv *inventory.Inventory) ([]stocker.Stocker, []stocker.Stocker, []error) { var stockers []stocker.Stocker var billingStockers []stocker.Stocker + var errs []error for _, account := range inv.Accounts { switch account.Provider { @@ -141,12 +193,13 @@ func (s *Scanner) createStockers(inv *inventory.Inventory) ([]stocker.Stocker, [ if err != nil { s.logger.Error("Failed to create AWS stocker; skipping this account", zap.String("account", account.AccountName), zap.Error(err)) + errs = append(errs, fmt.Errorf("account %s: %w", account.AccountName, err)) continue } stockers = append(stockers, awsStocker) if account.IsBillingEnabled() { - s.logger.Warn("Enabled AWS Billing Stocker", zap.String("account_id", account.AccountID), zap.String("account_name", account.AccountName)) + s.logger.Info("Enabled AWS Billing Stocker", zap.String("account_id", account.AccountID), zap.String("account_name", account.AccountName)) instancesToScan, err := getInstancesForBillingUpdate(s.cfg.APIURL, account.AccountID, s.logger) if err != nil { s.logger.Error("Failed to retrieve instances for billing", @@ -185,7 +238,7 @@ func (s *Scanner) createStockers(inv *inventory.Inventory) ([]stocker.Stocker, [ zap.Int("registeredStockers", len(stockers)), zap.Int("skippedAccounts", len(inv.Accounts)-len(stockers))) - return stockers, billingStockers + return stockers, billingStockers, errs } func runStockers(stockers []stocker.Stocker, billingStockers []stocker.Stocker, l *zap.Logger) error { @@ -283,9 +336,9 @@ func (s *Scanner) ExecuteScan(accountIDs []string, selectAll bool) (int, error) return 0, fmt.Errorf("failed to build inventory: %w", err) } - stockers, billingStockers := s.createStockers(inv) + stockers, billingStockers, stockerErrors := s.createStockers(inv) if len(stockers) == 0 { - return 0, fmt.Errorf("no valid stockers created") + return 0, fmt.Errorf("no valid stockers created: %w", errors.Join(stockerErrors...)) } if err := runStockers(stockers, billingStockers, s.logger); err != nil { @@ -310,9 +363,6 @@ func (s *Scanner) Scan(_ context.Context, req *pb.ScanRequest) (*pb.ScanResponse scanned, err := s.ExecuteScan(req.AccountIds, req.SelectAll) if err != nil { s.logger.Error("Scan failed", zap.Error(err)) - if req.RunId > 0 { - s.updateRunStatus(req.RunId, "Failed", err.Error()) - } return &pb.ScanResponse{ Error: 1, Message: err.Error(), @@ -321,10 +371,6 @@ func (s *Scanner) Scan(_ context.Context, req *pb.ScanRequest) (*pb.ScanResponse } s.logger.Info("Scan completed successfully", zap.Int("accounts_scanned", scanned)) - if req.RunId > 0 { - s.updateRunStatus(req.RunId, "Success", "") - } - return &pb.ScanResponse{ Error: 0, Message: "Scan completed successfully", @@ -337,40 +383,6 @@ func (s *Scanner) Health(_ context.Context, _ *pb.HealthRequest) (*pb.HealthResp return &pb.HealthResponse{Ready: true}, nil } -func (s *Scanner) updateRunStatus(runID int64, status string, errorMsg string) { - payload := map[string]string{ - "status": status, - "errorMsg": errorMsg, - } - b, err := json.Marshal(payload) - if err != nil { - s.logger.Error("Failed to marshal run status", zap.Error(err)) - return - } - - url := fmt.Sprintf("%s%s/%d", APIURL, apiActionRunEndpoint, runID) - ctx, cancel := context.WithTimeout(context.Background(), apiRequestTimeout) - defer cancel() - - req, err := http.NewRequestWithContext(ctx, http.MethodPatch, url, bytes.NewBuffer(b)) - if err != nil { - s.logger.Error("Failed to create run status request", zap.Error(err)) - return - } - req.Header.Set("Content-Type", "application/json") - - resp, err := client.Do(req) - if resp != nil { - defer resp.Body.Close() - } - if err != nil { - s.logger.Error("Failed to update run status", zap.Int64("run_id", runID), zap.Error(err)) - return - } - - s.logger.Info("Updated action run status", zap.Int64("run_id", runID), zap.String("status", status)) -} - // startGRPCServer initializes and starts the gRPC server. func (s *Scanner) startGRPCServer() error { lc := net.ListenConfig{} @@ -497,11 +509,18 @@ func postScannerInventory(inv *inventory.Inventory, l *zap.Logger) error { l.Info("Inventory posted correctly") - if err := postData(apiInventoryEndpoint, []byte{}); err != nil { + if err := refreshInventory(l); err != nil { return err } - l.Info("Inventory refreshed correctly") + return nil +} + +func refreshInventory(l *zap.Logger) error { + if err := postData(apiInventoryEndpoint, []byte{}); err != nil { + return err + } + l.Info("Inventory refreshed") return nil } @@ -522,10 +541,14 @@ func postData(path string, b []byte) error { if err != nil { return err } + + if response.StatusCode < 200 || response.StatusCode >= 300 { + return fmt.Errorf("API returned HTTP %d for %s", response.StatusCode, path) + } return nil } -func getInstancesForBillingUpdate(apiURL string, accountID string, l *zap.Logger) ([]inventory.Instance, error) { +func getInstancesForBillingUpdate(apiURL string, accountID string, l *zap.Logger) ([]string, error) { l.Debug("Fetching instances for update billing from backend") requestURL := apiURL + apiAccountEndpoint + "/" + accountID + "/expense_update" @@ -553,24 +576,15 @@ func getInstancesForBillingUpdate(apiURL string, accountID string, l *zap.Logger return nil, err } - var response responsetypes.ListResponse[dto.InstanceDTOResponse] + var response responsetypes.ListResponse[string] err = json.Unmarshal(body, &response) if err != nil { - l.Error("Failed to unmarshal instances JSON", zap.Error(err)) + l.Error("Failed to unmarshal instance IDs JSON", zap.Error(err)) return nil, err } - if response.Count == 0 { - return nil, fmt.Errorf("no instances for billing update") - } - - l.Debug("Successfully fetched instances from backend", zap.Int("instances_num", response.Count)) - - var instances []inventory.Instance - for _, instance := range response.Items { - instances = append(instances, *instance.ToInventoryInstance()) - } - return instances, nil + l.Debug("Successfully fetched instance IDs from backend", zap.Int("instances_num", response.Count)) + return response.Items, nil } func main() { @@ -595,6 +609,11 @@ func main() { logger.Fatal("Failed to load cloud accounts", zap.Error(err)) } + scan.waitForAPI() + if err := scan.seedAccounts(); err != nil { + logger.Warn("Failed to seed accounts", zap.Error(err)) + } + // Signal handling for graceful shutdown quit := make(chan os.Signal, 1) signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) diff --git a/internal/stocker/aws_billing_stocker.go b/internal/stocker/aws_billing_stocker.go index f8ae4a4f..23544ff7 100644 --- a/internal/stocker/aws_billing_stocker.go +++ b/internal/stocker/aws_billing_stocker.go @@ -19,19 +19,17 @@ type AWSBillingStocker struct { logger *zap.Logger // AWS connection interface conn *cp.AWSConnection - // List of instances to obtain its expenses - Instances []inventory.Instance + // List of instance IDs to obtain their expenses + InstanceIDs []string } // NewAWSBillingStocker create and returns a pointer to a new AWSBillingStocker instance -func NewAWSBillingStocker(account *inventory.Account, logger *zap.Logger, instances []inventory.Instance) *AWSBillingStocker { - // Check if there are instances to get billing information - if len(instances) == 0 { - logger.Error("No instances to get billing information") +func NewAWSBillingStocker(account *inventory.Account, logger *zap.Logger, instanceIDs []string) *AWSBillingStocker { + if len(instanceIDs) == 0 { + logger.Info("No instances pending billing update, skipping billing stocker") return nil } - // Leaving the region empty forces to the AWSConnection to use the default region until a new one is configured conn, err := cp.NewAWSConnection(account.User(), account.Password(), "", cp.WithCostExplorer()) if err != nil { logger.Error("Error creating a new AWSBillingStocker", zap.String("account", account.AccountName), zap.Error(err)) @@ -39,10 +37,10 @@ func NewAWSBillingStocker(account *inventory.Account, logger *zap.Logger, instan } return &AWSBillingStocker{ - Account: account, - logger: logger, - Instances: instances, - conn: conn, + Account: account, + logger: logger, + InstanceIDs: instanceIDs, + conn: conn, } } @@ -59,9 +57,9 @@ func (s *AWSBillingStocker) MakeStock() error { cluster := s.Account.Clusters[i] for j := range cluster.Instances { instance := &cluster.Instances[j] - for _, targetInstance := range s.Instances { - if targetInstance.InstanceID == instance.InstanceID { - s.logger.Info("Getting expenses for instance", zap.String("instance_id", targetInstance.InstanceID)) + for _, targetID := range s.InstanceIDs { + if targetID == instance.InstanceID { + s.logger.Info("Getting expenses for instance", zap.String("instance_id", targetID)) err := s.getInstanceExpenses(instance) if err != nil { s.logger.Error("Error querying billing info for an instance", @@ -69,7 +67,6 @@ func (s *AWSBillingStocker) MakeStock() error { zap.String("instance_id", instance.InstanceID), zap.String("error", err.Error()), ) - // Continue to the next region even if an error occurs continue } break From 4deaec0f68b37b5fd4314d629f5e5e29213a2152 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Fri, 29 May 2026 19:34:31 +0200 Subject: [PATCH 04/22] fix(api): use lightweight struct for expense update instances --- internal/api/handlers/account_handler.go | 10 ++++++---- internal/models/db/instance.go | 6 ++++++ internal/repositories/account_repository.go | 10 +++------- internal/repositories/errors.go | 3 --- internal/services/account_service.go | 4 ++-- 5 files changed, 17 insertions(+), 16 deletions(-) diff --git a/internal/api/handlers/account_handler.go b/internal/api/handlers/account_handler.go index bdccb103..63ed6c71 100644 --- a/internal/api/handlers/account_handler.go +++ b/internal/api/handlers/account_handler.go @@ -176,12 +176,10 @@ func (h *AccountHandler) GetAccountClustersByID(c *gin.Context) { // @Accept json // @Produce json // @Param id path string true "Account ID" -// @Success 200 {object} dto.InstanceListResponse +// @Success 200 {object} responsetypes.ListResponse[string] // @Failure 404 {object} responsetypes.GenericErrorResponse // @Failure 500 {object} responsetypes.GenericErrorResponse // @Router /accounts/{id}/expense_update [get] -// -// NOTE: Align the documented route with the actual router configuration. func (h *AccountHandler) GetExpensesUpdateInstances(c *gin.Context) { accountID := c.Param("id") @@ -201,8 +199,12 @@ func (h *AccountHandler) GetExpensesUpdateInstances(c *gin.Context) { return } - response := responsetypes.NewListResponse((&convert.ConverterImpl{}).ToInstanceDTOs(instances), len(instances)) + instanceIDs := make([]string, len(instances)) + for i, inst := range instances { + instanceIDs[i] = inst.InstanceID + } + response := responsetypes.NewListResponse(instanceIDs, len(instanceIDs)) c.JSON(http.StatusOK, response) } diff --git a/internal/models/db/instance.go b/internal/models/db/instance.go index cdb7f80f..291f4ec6 100644 --- a/internal/models/db/instance.go +++ b/internal/models/db/instance.go @@ -26,3 +26,9 @@ type InstanceDBResponse struct { CurrentMonthSoFarCost float64 `db:"current_month_so_far_cost"` Tags TagDBResponses `db:"tags_json"` } + +// InstancePendingExpenseDB maps the instances_pending_expense_update view. +type InstancePendingExpenseDB struct { + AccountID string `db:"account_id"` + InstanceID string `db:"instance_id"` +} diff --git a/internal/repositories/account_repository.go b/internal/repositories/account_repository.go index 8d86995d..73384458 100644 --- a/internal/repositories/account_repository.go +++ b/internal/repositories/account_repository.go @@ -50,7 +50,7 @@ type AccountRepository interface { CountAccounts(ctx context.Context, opts models.ListOptions) (int, error) GetAccountByID(ctx context.Context, accountID string) (db.AccountDBResponse, error) GetAccountClustersByID(ctx context.Context, accountID string) ([]db.ClusterDBResponse, error) - GetExpenseUpdateInstances(ctx context.Context, accountID string) ([]db.InstanceDBResponse, error) + GetExpenseUpdateInstances(ctx context.Context, accountID string) ([]db.InstancePendingExpenseDB, error) GetScannerTimestamp(ctx context.Context) (time.Time, error) CreateAccount(ctx context.Context, accounts []inventory.Account) error UpdateAccount(ctx context.Context, accountID string, patch dto.AccountPatchRequest) error @@ -153,10 +153,6 @@ func (r *accountRepositoryImpl) GetAccountClustersByID(ctx context.Context, acco return clusters, err } - if len(clusters) == 0 { - return clusters, ErrNoClustersInAccount - } - return clusters, nil } @@ -167,8 +163,8 @@ func (r *accountRepositoryImpl) GetAccountClustersByID(ctx context.Context, acco // Returns: // - A slice of inventory.Instance objects. // - An error if the query fails. -func (r *accountRepositoryImpl) GetExpenseUpdateInstances(ctx context.Context, accountID string) ([]db.InstanceDBResponse, error) { - instances := []db.InstanceDBResponse{} +func (r *accountRepositoryImpl) GetExpenseUpdateInstances(ctx context.Context, accountID string) ([]db.InstancePendingExpenseDB, error) { + instances := []db.InstancePendingExpenseDB{} opts := models.ListOptions{ PageSize: 0, diff --git a/internal/repositories/errors.go b/internal/repositories/errors.go index 5e014292..8eb3555c 100644 --- a/internal/repositories/errors.go +++ b/internal/repositories/errors.go @@ -4,6 +4,3 @@ import "errors" // ErrNotFound is returned when a resource is not found in the database. var ErrNotFound = errors.New("requested resource not found") - -// ErrNoClustersInAccount is returned when an account exists but has no associated clusters. -var ErrNoClustersInAccount = errors.New("no clusters found for this account") diff --git a/internal/services/account_service.go b/internal/services/account_service.go index 4066da40..cb38f38d 100644 --- a/internal/services/account_service.go +++ b/internal/services/account_service.go @@ -16,7 +16,7 @@ type AccountService interface { List(ctx context.Context, options models.ListOptions) ([]db.AccountDBResponse, int, error) GetByID(ctx context.Context, accountID string) (db.AccountDBResponse, error) GetAccountClustersByID(ctx context.Context, accountID string) ([]db.ClusterDBResponse, error) - GetExpenseUpdateInstances(ctx context.Context, accountID string) ([]db.InstanceDBResponse, error) + GetExpenseUpdateInstances(ctx context.Context, accountID string) ([]db.InstancePendingExpenseDB, error) Create(ctx context.Context, accounts []inventory.Account) error Update(ctx context.Context, accountID string, patch dto.AccountPatchRequest) error Delete(ctx context.Context, accountID string) error @@ -57,7 +57,7 @@ func (s *accountServiceImpl) GetAccountClustersByID(ctx context.Context, account } // GetExpenseUpdateInstances retrieves instances with outdated billing information. -func (s *accountServiceImpl) GetExpenseUpdateInstances(ctx context.Context, accountID string) ([]db.InstanceDBResponse, error) { +func (s *accountServiceImpl) GetExpenseUpdateInstances(ctx context.Context, accountID string) ([]db.InstancePendingExpenseDB, error) { instances, err := s.repo.GetExpenseUpdateInstances(ctx, accountID) if err != nil { return instances, fmt.Errorf("get expense update instances for account %s: %w", accountID, err) From e980741229aaba6007314880194b105fd78df2c4 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Fri, 29 May 2026 19:34:44 +0200 Subject: [PATCH 05/22] fix(db): handle NULL columns in system events and schedule models --- internal/models/convert/generated.go | 10 +++++---- internal/models/db/action.go | 14 +++++++------ internal/models/db/action_test.go | 24 ++++++++++----------- internal/models/db/events.go | 5 +++-- internal/models/db/events_test.go | 13 ++++++------ internal/models/dto/action_dto.go | 31 ++++++++++++++++++++-------- 6 files changed, 58 insertions(+), 39 deletions(-) diff --git a/internal/models/convert/generated.go b/internal/models/convert/generated.go index bfa70d35..dbc5d8e5 100644 --- a/internal/models/convert/generated.go +++ b/internal/models/convert/generated.go @@ -47,11 +47,13 @@ func (c *ConverterImpl) ToActionDTO(source db.ActionDBResponse) dto.ActionDTORes dtoActionDTOResponse.Enabled = source.Enabled dtoActionDTOResponse.TargetType = source.TargetType dtoActionDTOResponse.SelectAll = source.SelectAll - dtoActionDTOResponse.ClusterID = source.ClusterID - dtoActionDTOResponse.Region = source.Region + dtoActionDTOResponse.ClusterID = NullString(source.ClusterID) + dtoActionDTOResponse.ClusterName = NullString(source.ClusterName) + dtoActionDTOResponse.Region = NullString(source.Region) dtoActionDTOResponse.AccountID = source.AccountID dtoActionDTOResponse.Instances = StringArray(source.Instances) dtoActionDTOResponse.TargetAccountIDs = StringArray(source.TargetAccountIDs) + dtoActionDTOResponse.TargetAccountNames = StringArray(source.TargetAccountNames) return dtoActionDTOResponse } func (c *ConverterImpl) ToActionDTOs(source []db.ActionDBResponse) []dto.ActionDTOResponse { @@ -195,8 +197,8 @@ func (c *ConverterImpl) ToInstanceDTOs(source []db.InstanceDBResponse) []dto.Ins func (c *ConverterImpl) ToSystemEventDTO(source db.SystemEventDBResponse) dto.SystemEventDTOResponse { var dtoSystemEventDTOResponse dto.SystemEventDTOResponse dtoSystemEventDTOResponse.ClusterEventDTOResponse = c.ToClusterEventDTO(source.ClusterEventDBResponse) - dtoSystemEventDTOResponse.AccountID = source.AccountID - dtoSystemEventDTOResponse.Provider = source.Provider + dtoSystemEventDTOResponse.AccountID = NullString(source.AccountID) + dtoSystemEventDTOResponse.Provider = NullString(source.Provider) return dtoSystemEventDTOResponse } func (c *ConverterImpl) ToSystemEventDTOs(source []db.SystemEventDBResponse) []dto.SystemEventDTOResponse { diff --git a/internal/models/db/action.go b/internal/models/db/action.go index 4f597cd4..eae12c64 100644 --- a/internal/models/db/action.go +++ b/internal/models/db/action.go @@ -17,10 +17,12 @@ type ActionDBResponse struct { Status string `db:"status"` Enabled bool `db:"enabled"` TargetType string `db:"target_type"` - SelectAll bool `db:"select_all"` - ClusterID string `db:"cluster_id"` - Region string `db:"region"` - AccountID string `db:"account_id"` - Instances pq.StringArray `db:"instances"` - TargetAccountIDs pq.StringArray `db:"target_account_ids"` + SelectAll bool `db:"select_all"` + ClusterID sql.NullString `db:"cluster_id"` + ClusterName sql.NullString `db:"cluster_name"` + Region sql.NullString `db:"region"` + AccountID string `db:"account_id"` + Instances pq.StringArray `db:"instances"` + TargetAccountIDs pq.StringArray `db:"target_account_ids"` + TargetAccountNames pq.StringArray `db:"target_account_names"` } diff --git a/internal/models/db/action_test.go b/internal/models/db/action_test.go index 723a9cb7..2ef561b0 100644 --- a/internal/models/db/action_test.go +++ b/internal/models/db/action_test.go @@ -29,8 +29,8 @@ func testActionDBResponse_ToActionDTOResponse_WithValidFields(t *testing.T) { Operation: "START", Status: "Pending", Enabled: true, - ClusterID: "cluster-1", - Region: "eu-west-1", + ClusterID: sql.NullString{String: "cluster-1", Valid: true}, + Region: sql.NullString{String: "eu-west-1", Valid: true}, AccountID: "acc-1", Instances: pq.StringArray{"i-1", "i-2"}, } @@ -44,8 +44,8 @@ func testActionDBResponse_ToActionDTOResponse_WithValidFields(t *testing.T) { assert.Equal(t, model.Operation, dto.Operation) assert.Equal(t, model.Status, dto.Status) assert.Equal(t, model.Enabled, dto.Enabled) - assert.Equal(t, model.ClusterID, dto.ClusterID) - assert.Equal(t, model.Region, dto.Region) + assert.Equal(t, model.ClusterID.String, dto.ClusterID) + assert.Equal(t, model.Region.String, dto.Region) assert.Equal(t, model.AccountID, dto.AccountID) assert.Equal(t, []string{"i-1", "i-2"}, dto.Instances) } @@ -61,8 +61,8 @@ func testActionDBResponse_ToActionDTOResponse_WithInvalidFields(t *testing.T) { Operation: "STOP", Status: "Failed", Enabled: false, - ClusterID: "cluster-2", - Region: "us-east-1", + ClusterID: sql.NullString{String: "cluster-2", Valid: true}, + Region: sql.NullString{String: "us-east-1", Valid: true}, AccountID: "acc-2", Instances: pq.StringArray{"i-9"}, } @@ -79,8 +79,8 @@ func testActionDBResponse_ToActionDTOResponse_WithInvalidFields(t *testing.T) { assert.Equal(t, model.Operation, dto.Operation) assert.Equal(t, model.Status, dto.Status) assert.Equal(t, model.Enabled, dto.Enabled) - assert.Equal(t, model.ClusterID, dto.ClusterID) - assert.Equal(t, model.Region, dto.Region) + assert.Equal(t, model.ClusterID.String, dto.ClusterID) + assert.Equal(t, model.Region.String, dto.Region) assert.Equal(t, model.AccountID, dto.AccountID) assert.Equal(t, []string{"i-9"}, dto.Instances) } @@ -103,8 +103,8 @@ func testToActionDTOResponseList_Correct(t *testing.T) { Operation: "START", Status: "Pending", Enabled: true, - ClusterID: "cluster-1", - Region: "eu-west-1", + ClusterID: sql.NullString{String: "cluster-1", Valid: true}, + Region: sql.NullString{String: "eu-west-1", Valid: true}, AccountID: "acc-1", Instances: pq.StringArray{"i-1"}, }, @@ -116,8 +116,8 @@ func testToActionDTOResponseList_Correct(t *testing.T) { Operation: "STOP", Status: "Running", Enabled: false, - ClusterID: "cluster-2", - Region: "us-east-1", + ClusterID: sql.NullString{String: "cluster-2", Valid: true}, + Region: sql.NullString{String: "us-east-1", Valid: true}, AccountID: "acc-2", Instances: pq.StringArray{"i-2"}, }, diff --git a/internal/models/db/events.go b/internal/models/db/events.go index f787924c..62011f69 100644 --- a/internal/models/db/events.go +++ b/internal/models/db/events.go @@ -1,6 +1,7 @@ package db import ( + "database/sql" "time" "github.com/RHEcosystemAppEng/cluster-iq/internal/actions" @@ -24,6 +25,6 @@ type ClusterEventDBResponse struct { // extending ClusterEventDBResponse with account and provider information. type SystemEventDBResponse struct { ClusterEventDBResponse - AccountID string `db:"account_id"` - Provider string `db:"provider"` + AccountID sql.NullString `db:"account_id"` + Provider sql.NullString `db:"provider"` } diff --git a/internal/models/db/events_test.go b/internal/models/db/events_test.go index fdd73855..70a007d2 100644 --- a/internal/models/db/events_test.go +++ b/internal/models/db/events_test.go @@ -1,6 +1,7 @@ package db_test import ( + "database/sql" "testing" "time" @@ -125,8 +126,8 @@ func testSystemEventDBResponse_ToSystemEventDTOResponse_Correct(t *testing.T) { Description: &desc, Severity: "Warning", }, - AccountID: "acc-1", - Provider: "AWS", + AccountID: sql.NullString{String: "acc-1", Valid: true}, + Provider: sql.NullString{String: "AWS", Valid: true}, } dto := conv.ToSystemEventDTO(model) @@ -169,8 +170,8 @@ func testToSystemEventDTOResponseList_Correct(t *testing.T) { Result: "Success", Severity: "Info", }, - AccountID: "acc-1", - Provider: "AWS", + AccountID: sql.NullString{String: "acc-1", Valid: true}, + Provider: sql.NullString{String: "AWS", Valid: true}, }, { ClusterEventDBResponse: db.ClusterEventDBResponse{ @@ -183,8 +184,8 @@ func testToSystemEventDTOResponseList_Correct(t *testing.T) { Result: "Failed", Severity: "Error", }, - AccountID: "acc-2", - Provider: "GCP", + AccountID: sql.NullString{String: "acc-2", Valid: true}, + Provider: sql.NullString{String: "GCP", Valid: true}, }, } diff --git a/internal/models/dto/action_dto.go b/internal/models/dto/action_dto.go index 4e71c46d..de23d3b9 100644 --- a/internal/models/dto/action_dto.go +++ b/internal/models/dto/action_dto.go @@ -32,6 +32,17 @@ func (a ActionDTORequest) ToModelAction() actions.Action { Instances: a.Instances, } + if actions.ActionOperation(a.Operation) == actions.Scan { + target.TargetType = "Account" + if a.AccountID != "" { + target.TargetAccountIDs = []string{a.AccountID} + } else { + target.SelectAll = true + } + } else { + target.TargetType = "Cluster" + } + switch actions.ActionType(a.Type) { case actions.ScheduledActionType: action := actions.NewScheduledAction( @@ -96,15 +107,17 @@ type ActionDTOResponse struct { Operation string `json:"operation"` Status string `json:"status"` Enabled bool `json:"enabled"` - TargetType string `json:"targetType"` - SelectAll bool `json:"selectAll"` - ClusterID string `json:"clusterId"` - Region string `json:"region"` - AccountID string `json:"accountId"` - Instances []string `json:"instances"` - TargetAccountIDs []string `json:"targetAccountIds"` - Requester string `json:"requester"` - Description *string `json:"description"` + TargetType string `json:"targetType"` + SelectAll bool `json:"selectAll"` + ClusterID string `json:"clusterId"` + ClusterName string `json:"clusterName"` + Region string `json:"region"` + AccountID string `json:"accountId"` + Instances []string `json:"instances"` + TargetAccountIDs []string `json:"targetAccountIds"` + TargetAccountNames []string `json:"targetAccountNames"` + Requester string `json:"requester"` + Description *string `json:"description"` } // @name ActionResponse // ToModelAction converts ActionDTOResponse to actions.Action From 212603783d6b7c928ab18d316b741fc93b65e1a4 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Fri, 29 May 2026 19:35:07 +0200 Subject: [PATCH 06/22] feat(events): add Account resource type to audit event system --- cmd/agent/executor_agent_service.go | 34 ++++++++++++++++++----- cmd/agent/schedule_agent_service.go | 15 ++++++++++ db/sql/init.sql | 8 ++++-- internal/inventory/types.go | 1 + internal/repositories/event_repository.go | 2 ++ 5 files changed, 51 insertions(+), 9 deletions(-) diff --git a/cmd/agent/executor_agent_service.go b/cmd/agent/executor_agent_service.go index 671bdb11..29bd537f 100644 --- a/cmd/agent/executor_agent_service.go +++ b/cmd/agent/executor_agent_service.go @@ -6,6 +6,7 @@ import ( "crypto/tls" "fmt" "net/http" + "strconv" "sync" "github.com/RHEcosystemAppEng/cluster-iq/internal/actions" @@ -210,22 +211,32 @@ func (e *ExecutorAgentService) processAction(action actions.Action) { zap.Any("requester", action.GetRequester()), ) + target := action.GetTarget() + + resourceType := inventory.ClusterResourceType + resourceID := target.ClusterID if action.GetActionOperation() == actions.Scan { - e.processScanAction(action) - return + resourceType = inventory.AccountResourceType + if len(target.TargetAccountIDs) > 0 { + resourceID = target.TargetAccountIDs[0] + } } - // Initialize event tracker tracker := e.eventService.StartTracking(&eventservice.EventOptions{ Action: action.GetActionOperation(), Description: action.GetDescription(), - ResourceID: action.GetTarget().ClusterID, - ResourceType: inventory.ClusterResourceType, + ResourceID: resourceID, + ResourceType: resourceType, Result: eventservice.ResultPending, Severity: eventservice.SeverityInfo, TriggeredBy: action.GetRequester(), }) + if action.GetActionOperation() == actions.Scan { + e.processScanAction(action, tracker) + return + } + // Mark as running if !e.setActionStatus(action, actions.StatusRunning) { tracker.Failed() @@ -233,7 +244,7 @@ func (e *ExecutorAgentService) processAction(action actions.Action) { } // Get executor - executor := e.GetExecutor(action.GetTarget().AccountID) + executor := e.GetExecutor(target.AccountID) if executor == nil { e.handleMissingExecutor(action, tracker) return @@ -253,8 +264,9 @@ func (e *ExecutorAgentService) processAction(action actions.Action) { } // processScanAction dispatches a Scan action to the Scanner gRPC service. -func (e *ExecutorAgentService) processScanAction(action actions.Action) { +func (e *ExecutorAgentService) processScanAction(action actions.Action, tracker *eventservice.EventTracker) { if !e.setActionStatus(action, actions.StatusRunning) { + tracker.Failed() return } @@ -265,8 +277,10 @@ func (e *ExecutorAgentService) processScanAction(action actions.Action) { e.logger.Error("Failed to create action run for scan", zap.String("action_id", action.GetID()), zap.Error(err)) e.setActionStatus(action, actions.StatusFailed) + tracker.Failed() return } + runIDStr := strconv.FormatInt(runID, 10) resp, err := e.scannerClient.Scan( context.Background(), @@ -278,6 +292,8 @@ func (e *ExecutorAgentService) processScanAction(action actions.Action) { e.logger.Error("Scanner gRPC call failed", zap.String("action_id", action.GetID()), zap.Error(err)) e.setActionStatus(action, actions.StatusFailed) + _ = e.actionRunRepo.Update(context.Background(), runIDStr, "Failed", err.Error()) + tracker.Failed() return } @@ -286,6 +302,8 @@ func (e *ExecutorAgentService) processScanAction(action actions.Action) { zap.String("action_id", action.GetID()), zap.String("message", resp.Message)) e.setActionStatus(action, actions.StatusFailed) + _ = e.actionRunRepo.Update(context.Background(), runIDStr, "Failed", resp.Message) + tracker.Failed() return } @@ -293,6 +311,8 @@ func (e *ExecutorAgentService) processScanAction(action actions.Action) { zap.String("action_id", action.GetID()), zap.Int32("accounts_scanned", resp.AccountsScanned)) e.setActionStatus(action, actions.StatusSuccess) + _ = e.actionRunRepo.Update(context.Background(), runIDStr, "Success", "") + tracker.Success() e.resetCronActionStatus(action) } diff --git a/cmd/agent/schedule_agent_service.go b/cmd/agent/schedule_agent_service.go index f2b23382..c9c613d4 100644 --- a/cmd/agent/schedule_agent_service.go +++ b/cmd/agent/schedule_agent_service.go @@ -284,6 +284,21 @@ func (a *ScheduleAgentService) ScheduleNewActions(newSchedule []actions.Action) // managing actions based on type switch t := action.(type) { + case *actions.InstantAction: + if _, exists := a.schedule[action.GetID()]; exists { + continue + } + a.logger.Info("Dispatching InstantAction for immediate execution", zap.String("action_id", t.GetID())) + a.schedule[t.GetID()] = scheduleItem{ + cancel: func() {}, + action: t, + } + go func() { + a.actionsChannel <- t + a.mutex.Lock() + delete(a.schedule, t.GetID()) + a.mutex.Unlock() + }() case *actions.ScheduledAction: scheduledFunc(t) case *actions.CronAction: diff --git a/db/sql/init.sql b/db/sql/init.sql index 97bf0083..e10cff8d 100644 --- a/db/sql/init.sql +++ b/db/sql/init.sql @@ -642,7 +642,7 @@ SELECT ev.event_timestamp, ev.triggered_by, ev.action, - COALESCE(c.cluster_id, i.instance_id) AS resource_id, + COALESCE(c.cluster_id, i.instance_id, a.account_id) AS resource_id, ev.resource_type, ev.result, ev.description, @@ -650,6 +650,7 @@ SELECT FROM events ev LEFT JOIN clusters c ON ev.resource_type = 'Cluster'::RESOURCE_TYPE AND c.id = ev.resource_id LEFT JOIN instances i ON ev.resource_type = 'Instance'::RESOURCE_TYPE AND i.id = ev.resource_id +LEFT JOIN accounts a ON ev.resource_type = 'Account'::RESOURCE_TYPE AND a.id = ev.resource_id ORDER BY event_timestamp DESC; -- View for System Events @@ -659,7 +660,7 @@ SELECT ev.event_timestamp, ev.triggered_by, ev.action, - COALESCE(c.cluster_id, i.instance_id) AS resource_id, + COALESCE(c.cluster_id, i.instance_id, a.account_id) AS resource_id, ev.resource_type, ev.result, ev.description, @@ -669,12 +670,15 @@ SELECT FROM events ev LEFT JOIN clusters c ON ev.resource_type = 'Cluster'::RESOURCE_TYPE AND c.id = ev.resource_id LEFT JOIN instances i ON ev.resource_type = 'Instance'::RESOURCE_TYPE AND i.id = ev.resource_id +LEFT JOIN accounts a ON ev.resource_type = 'Account'::RESOURCE_TYPE AND a.id = ev.resource_id LEFT JOIN accounts acc ON acc.id = ( CASE WHEN ev.resource_type = 'Cluster'::RESOURCE_TYPE THEN (SELECT c.account_id FROM clusters c WHERE c.id = ev.resource_id) WHEN ev.resource_type = 'Instance'::RESOURCE_TYPE THEN (SELECT c.account_id FROM clusters c WHERE c.id = (SELECT i.cluster_id FROM instances i WHERE i.id = ev.resource_id)) + WHEN ev.resource_type = 'Account'::RESOURCE_TYPE + THEN ev.resource_id END ) ORDER BY ev.event_timestamp DESC; diff --git a/internal/inventory/types.go b/internal/inventory/types.go index 2945ecbc..952502ee 100644 --- a/internal/inventory/types.go +++ b/internal/inventory/types.go @@ -9,6 +9,7 @@ const ( ClusterPowerOffAction = "PowerOff" // Resource types + AccountResourceType = "Account" ClusterResourceType = "Cluster" InstanceResourceType = "Instance" ) diff --git a/internal/repositories/event_repository.go b/internal/repositories/event_repository.go index 96373fb1..47d2b8a6 100644 --- a/internal/repositories/event_repository.go +++ b/internal/repositories/event_repository.go @@ -38,6 +38,8 @@ const ( THEN (SELECT id FROM clusters c WHERE c.cluster_id = :resource_id) WHEN :resource_type = 'Instance' THEN (SELECT id FROM instances i WHERE i.instance_id = :resource_id) + WHEN :resource_type = 'Account' + THEN (SELECT id FROM accounts a WHERE a.account_id = :resource_id) END ), :resource_type, From 2c62af52cac4343b5154113dff444e8b5d6c3c07 Mon Sep 17 00:00:00 2001 From: r2dedios Date: Fri, 29 May 2026 19:35:18 +0200 Subject: [PATCH 07/22] feat(db): add cluster and account names to schedule view --- db/sql/init.sql | 10 +++++++++- internal/repositories/action_repository.go | 3 +++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/db/sql/init.sql b/db/sql/init.sql index e10cff8d..48bb4ec5 100644 --- a/db/sql/init.sql +++ b/db/sql/init.sql @@ -580,6 +580,7 @@ SELECT t.target_type, t.select_all, c.cluster_id, + c.cluster_name, c.region, COALESCE(a_power.account_id, '') AS account_id, COALESCE( @@ -593,7 +594,14 @@ SELECT JOIN accounts accs ON accs.id = ta_sub.account_id WHERE ta_sub.target_id = t.id), '{}' - ) AS target_account_ids + ) AS target_account_ids, + COALESCE( + (SELECT array_agg(DISTINCT accs.account_name ORDER BY accs.account_name) + FROM target_accounts ta_sub + JOIN accounts accs ON accs.id = ta_sub.account_id + WHERE ta_sub.target_id = t.id), + '{}' + ) AS target_account_names FROM schedule s JOIN targets t ON t.id = s.target LEFT JOIN target_clusters tc ON tc.target_id = t.id diff --git a/internal/repositories/action_repository.go b/internal/repositories/action_repository.go index fe5f3cc0..59955189 100644 --- a/internal/repositories/action_repository.go +++ b/internal/repositories/action_repository.go @@ -187,6 +187,9 @@ func (r *actionRepositoryImpl) Create(ctx context.Context, newActions []actions. case *actions.CronAction: _, err = tx.ExecContext(ctx, InsertCronActionWithTargetQuery, a.Expression, a.Operation, targetID, a.Status, a.Enabled) + case *actions.InstantAction: + _, err = tx.ExecContext(ctx, InsertInstantActionWithTargetQuery, + a.Operation, targetID, a.Status, a.Enabled) default: return fmt.Errorf("unsupported action type for batch create: %T", action) } From af023e21f03abd2961ece7c490f3e543a537925a Mon Sep 17 00:00:00 2001 From: r2dedios Date: Fri, 29 May 2026 19:36:13 +0200 Subject: [PATCH 08/22] feat(console): replace ModalPowerManagement with ModalCreateAction --- console/package-lock.json | 4 +- console/src/api/data-contracts.ts | 6 + console/src/api/index.ts | 12 - .../src/app/Actions/Scheduler/Schedule.tsx | 8 +- .../Scheduler/components/AccountSelector.tsx | 71 ++++-- ...erManagement.tsx => ModalCreateAction.tsx} | 233 +++++++++--------- .../components/ClusterDetailsDropdown.tsx | 30 +-- console/src/app/types/types.tsx | 1 + 8 files changed, 192 insertions(+), 173 deletions(-) rename console/src/app/Actions/Scheduler/components/{ModalPowerManagement.tsx => ModalCreateAction.tsx} (61%) diff --git a/console/package-lock.json b/console/package-lock.json index 932d0017..c9210e26 100644 --- a/console/package-lock.json +++ b/console/package-lock.json @@ -1,12 +1,12 @@ { "name": "cluster-iq-console", - "version": "0.5.0", + "version": "0.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cluster-iq-console", - "version": "0.5.0", + "version": "0.6.0", "dependencies": { "@patternfly/react-core": "^6.4.0", "@patternfly/react-icons": "^6.4.0", diff --git a/console/src/api/data-contracts.ts b/console/src/api/data-contracts.ts index a1159237..ef106bff 100644 --- a/console/src/api/data-contracts.ts +++ b/console/src/api/data-contracts.ts @@ -45,11 +45,13 @@ export interface ActionRequestApi { accountId?: string; clusterId?: string; cronExpression?: string; + description?: string; enabled?: boolean; id?: string; instances?: string[]; operation?: string; region?: string; + requester?: string; status?: string; time?: string; type?: string; @@ -58,13 +60,17 @@ export interface ActionRequestApi { export interface ActionResponseApi { accountId?: string; clusterId?: string; + clusterName?: string; cronExpression?: string; enabled?: boolean; id?: string; instances?: string[]; operation?: string; region?: string; + selectAll?: boolean; status?: string; + targetAccountIds?: string[]; + targetAccountNames?: string[]; time?: string; type?: string; } diff --git a/console/src/api/index.ts b/console/src/api/index.ts index fd8a1536..227b183c 100644 --- a/console/src/api/index.ts +++ b/console/src/api/index.ts @@ -48,18 +48,6 @@ export const api = { schedule: new Schedule(http), }; -export const startCluster = (clusterID: string, userEmail?: string, description?: string) => - http.instance.post(`/clusters/${clusterID}/power_on`, { - requester: userEmail || 'unknown', - description: description, - }); - -export const stopCluster = (clusterID: string, userEmail?: string, description?: string) => - http.instance.post(`/clusters/${clusterID}/power_off`, { - requester: userEmail || 'unknown', - description: description, - }); - export type { ClusterResponseApi, InstanceResponseApi, diff --git a/console/src/app/Actions/Scheduler/Schedule.tsx b/console/src/app/Actions/Scheduler/Schedule.tsx index cb88a80c..31a72333 100644 --- a/console/src/app/Actions/Scheduler/Schedule.tsx +++ b/console/src/app/Actions/Scheduler/Schedule.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { ModalPowerManagement } from './components/ModalPowerManagement'; +import { ModalCreateAction } from './components/ModalCreateAction'; import { Flex, FlexItem, Button, PageSection, Panel, Content } from '@patternfly/react-core'; import ScheduleActionsTable from './components/ActionsTable'; import ScheduleActionsTableToolbar from './components/ActionsToolBar'; @@ -70,11 +70,7 @@ const Scheduler: React.FunctionComponent = () => { reloadFlag={reloadFlag} /> - setReloadFlag(k => k + 1)} - /> + setReloadFlag(k => k + 1)} /> ); diff --git a/console/src/app/Actions/Scheduler/components/AccountSelector.tsx b/console/src/app/Actions/Scheduler/components/AccountSelector.tsx index 1a785a68..b947f4f7 100644 --- a/console/src/app/Actions/Scheduler/components/AccountSelector.tsx +++ b/console/src/app/Actions/Scheduler/components/AccountSelector.tsx @@ -2,6 +2,7 @@ import React from 'react'; import { Button, FormGroup, + Popover, Select, SelectOption, MenuToggle, @@ -10,14 +11,18 @@ import { TextInputGroupUtilities, Tooltip, } from '@patternfly/react-core'; +import { HelpIcon } from '@patternfly/react-icons'; import { AccountResponseApi } from '@api'; import TimesIcon from '@patternfly/react-icons/dist/esm/icons/times-icon'; +export const ALL_ACCOUNTS_ID = '__all__'; + interface AccountTypeaheadSelectProps { accounts: AccountResponseApi[]; selectedAccount: AccountResponseApi | null; onSelectAccount: (account: AccountResponseApi | null) => void; onClearAccount: () => void; + showAllOption?: boolean; } export const AccountTypeaheadSelect: React.FunctionComponent = ({ @@ -25,34 +30,66 @@ export const AccountTypeaheadSelect: React.FunctionComponent { const [isOpen, setIsOpen] = React.useState(false); const [inputValue, setInputValue] = React.useState(''); const safeAccounts = React.useMemo(() => (Array.isArray(accounts) ? accounts : []), [accounts]); + const allAccountsEntry: AccountResponseApi = React.useMemo( + () => ({ accountId: ALL_ACCOUNTS_ID, accountName: 'All Accounts' }), + [] + ); + const filteredAccounts = React.useMemo(() => { const q = inputValue.trim().toLowerCase(); - if (!q) return safeAccounts; + const filtered = q + ? safeAccounts.filter(a => { + const haystack = `${a.accountName ?? ''} ${a.accountId ?? ''}`.toLowerCase(); + return haystack.includes(q); + }) + : safeAccounts; - return safeAccounts.filter(a => { - const haystack = `${a.accountName ?? ''} ${a.accountId ?? ''}`.toLowerCase(); - return haystack.includes(q); - }); - }, [safeAccounts, inputValue]); + if (showAllOption) { + const allMatches = !q || 'all accounts'.includes(q); + return allMatches ? [allAccountsEntry, ...filtered] : filtered; + } + return filtered; + }, [safeAccounts, inputValue, showAllOption, allAccountsEntry]); const onSelect = (_event?: React.MouseEvent, value?: string | number) => { const id = String(value ?? ''); - const acc = safeAccounts.find(a => a.accountId === id) ?? null; + const acc = + (showAllOption && id === ALL_ACCOUNTS_ID ? allAccountsEntry : null) ?? + safeAccounts.find(a => a.accountId === id) ?? + null; - // Keep input in sync with selection for a predictable UX - setInputValue(acc ? `${acc.accountName} (${acc.accountId})` : ''); + setInputValue( + acc ? (acc.accountId === ALL_ACCOUNTS_ID ? (acc.accountName ?? '') : `${acc.accountName} (${acc.accountId})`) : '' + ); onSelectAccount(acc); setIsOpen(false); }; return ( - + +