feat: Add switch leak detection and powering off switches on detection of l…

flow/internal/nicoapi/grpc.go

@@ -136,6 +136,27 @@ func (c *grpcClient) GetLeakingMachineIds(ctx context.Context) ([]string, error)
 	return ids, nil
 }
 
+func (c *grpcClient) GetLeakingSwitchIds(ctx context.Context) ([]string, error) {
+	ctx, cancel := context.WithTimeout(ctx, c.grpcTimeout)
+	defer cancel()
+
+	alert := "hardware-health.tray-leak-detection"
+	searchConfig := pb.SwitchSearchFilter{
+		OnlyWithHealthAlert: &alert,
+	}
+
+	switchIDs, err := c.gclient.FindSwitchIds(ctx, &searchConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	ids := make([]string, 0, len(switchIDs.GetIds()))
+	for _, switchID := range switchIDs.GetIds() {
+		ids = append(ids, switchID.GetId())
+	}
+	return ids, nil
+}
+
 // Version returns the version string of nico-core-api, mainly as a "ping"
 func (c *grpcClient) Version(ctx context.Context) (string, error) {
 	ctx, cancel := context.WithTimeout(ctx, c.grpcTimeout)

flow/internal/nicoapi/mock.go

@@ -31,6 +31,7 @@ type mockClient struct {
 	machineInterfaces           map[string]MachineInterface
 	expectedSwitches            map[string]ExpectedSwitchInfo // keyed by BMC MAC
 	leakingMachineIds           []string
+	leakingSwitchIds            []string
 	firmwareUpdateTimeWindowErr error // If set, SetFirmwareUpdateTimeWindow will return this error
 	adminPowerControlErr        error // If set, AdminPowerControl will return this error
 	desiredFirmwareVersions     []*pb.DesiredFirmwareVersionEntry
@@ -62,6 +63,10 @@ func (c *mockClient) GetLeakingMachineIds(ctx context.Context) ([]string, error)
 	return c.leakingMachineIds, nil
 }
 
+func (c *mockClient) GetLeakingSwitchIds(ctx context.Context) ([]string, error) {
+	return c.leakingSwitchIds, nil
+}
+
 func (c *mockClient) SetLeakingMachineIds(ids []string) {
 	c.leakingMachineIds = ids
 }

flow/internal/nicoapi/mod.go

@@ -32,6 +32,7 @@ type Client interface {
 	Version(ctx context.Context) (string, error)
 	GetMachines(ctx context.Context) ([]MachineDetail, error)
 	GetLeakingMachineIds(ctx context.Context) ([]string, error)
+	GetLeakingSwitchIds(ctx context.Context) ([]string, error)
 	GetPowerStates(ctx context.Context, machineIds []string) (ret []MachinePowerState, err error)
 	SetFirmwareUpdateTimeWindow(ctx context.Context, machineIds []string, startTime, endTime time.Time) error
 	// FindInterfaces returns all machine interfaces known by nico-core-api, keyed by MAC address

flow/internal/nicoapi/nicoproto/nico.proto

@@ -3976,6 +3976,7 @@ message ManagedHostNetworkConfigResponse {
 
   // Route imports and tagging details for exports
   // used by FNN configs.
+  // Deprecated: use FlatInterfaceConfig.routing_profile for per-VPC routing.
   // NOTE: This will replace internet_l3_vni and common_internal_route_target but could allow
   //       common_internal_route_target to be renamed/repurposed as a site-level RT.
   //       to become a site-level common route target.
@@ -4200,6 +4201,10 @@ message FlatInterfaceConfig {
   // IPv6 configuration for dual-stack FNN interfaces.
   optional FlatInterfaceIpv6Config ipv6_interface_config = 19;
 
+  // Route imports and tagging details for exports used by FNN configs.
+  // This is scoped to the VPC that owns this interface.
+  optional RoutingProfile routing_profile = 20;
+
   // The details of the network security group associated with
   // either the instance or its parent VPC.
   // Currently, source would either be INSTANCE or VPC.

flow/internal/scheduler/jobs/leakdetection/leakdetection.go

@@ -48,17 +48,37 @@ func runLeakDetectionOne(
 	for _, machineID := range leakingMachineIds {
 		log.Info().Msgf("Leaking machine ID: %s, submitting force power-off task", machineID)
 
-		if err := submitPowerOffTask(ctx, taskMgr, machineID); err != nil {
+		err := submitPowerOffTask(ctx, taskMgr, machineID, devicetypes.ComponentTypeCompute)
+		if err != nil {
 			log.Error().Err(err).Str("machine_id", machineID).
 				Msg("Failed to submit power-off task for leaking machine")
 		}
 	}
+
+	leakingSwitchIds, err := nicoClient.GetLeakingSwitchIds(ctx)
+	if err != nil {
+		log.Error().Err(err).Msg("Unable to retrieve leaking switch IDs from NICo")
+		return
+	}
+
+	log.Info().Msgf("Found %d leaking switch IDs", len(leakingSwitchIds))
+
+	for _, switchID := range leakingSwitchIds {
+		log.Info().Msgf("Leaking switch ID: %s, submitting force power-off task", switchID)
+
+		err := submitPowerOffTask(ctx, taskMgr, switchID, devicetypes.ComponentTypeNVSwitch)
+		if err != nil {
+			log.Error().Err(err).Str("switch_id", switchID).
+				Msg("Failed to submit power-off task for leaking switch")
+		}
+	}
 }
 
 func submitPowerOffTask(
 	ctx context.Context,
 	taskMgr taskmanager.Manager,
 	machineID string,
+	componentType devicetypes.ComponentType,
 ) error {
 	info := &operations.PowerControlTaskInfo{
 		Operation: operations.PowerOperationForcePowerOff,
@@ -80,7 +100,7 @@ func submitPowerOffTask(
 			Components: []operation.ComponentTarget{
 				{
 					External: &operation.ExternalRef{
-						Type: devicetypes.ComponentTypeCompute,
+						Type: componentType,
 						ID:   machineID,
 					},
 				},

flow/internal/scheduler/jobs/leakdetection/leakdetection_test.go

@@ -65,7 +65,7 @@ func TestSubmitPowerOffTask_Success(t *testing.T) {
 	mgr := &mockManager{}
 	machineID := "machine-abc-123"
 
-	err := submitPowerOffTask(ctx, mgr, machineID)
+	err := submitPowerOffTask(ctx, mgr, machineID, devicetypes.ComponentTypeCompute)
 	require.NoError(t, err)
 	require.Len(t, mgr.requests, 1)
 
@@ -92,7 +92,7 @@ func TestSubmitPowerOffTask_NoTasksCreated(t *testing.T) {
 	ctx := context.Background()
 	mgr := &mockManager{returnNoTaskID: true}
 
-	err := submitPowerOffTask(ctx, mgr, "machine-xyz")
+	err := submitPowerOffTask(ctx, mgr, "machine-xyz", devicetypes.ComponentTypeCompute)
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "failed to create any power-off tasks")
 }
@@ -101,7 +101,7 @@ func TestSubmitPowerOffTask_SubmitError(t *testing.T) {
 	ctx := context.Background()
 	mgr := &mockManager{submitErr: errors.New("submit failed")}
 
-	err := submitPowerOffTask(ctx, mgr, "machine-xyz")
+	err := submitPowerOffTask(ctx, mgr, "machine-xyz", devicetypes.ComponentTypeCompute)
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "submit failed")
 }