feat: implement circuit breaker for external service calls

src/common/common.module.ts

@@ -2,13 +2,16 @@ import { Module } from '@nestjs/common';
 import { ConfigModule } from '@nestjs/config';
 import { TransactionHelperService } from './database/transaction-helper.service';
 import { LogShipperService } from './services/log-shipper.service';
+import { EnhancedCircuitBreakerService } from './services/circuit-breaker.service';
+import { CircuitBreakerController } from './controllers/circuit-breaker.controller';
 
 /**
  * Registers the common module.
  */
 @Module({
   imports: [ConfigModule],
-  providers: [TransactionHelperService, LogShipperService],
-  exports: [TransactionHelperService, LogShipperService],
+  controllers: [CircuitBreakerController],
+  providers: [TransactionHelperService, LogShipperService, EnhancedCircuitBreakerService],
+  exports: [TransactionHelperService, LogShipperService, EnhancedCircuitBreakerService],
 })
 export class CommonModule {}

src/common/services/circuit-breaker.service.spec.ts

@@ -0,0 +1,167 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { ConfigService } from '@nestjs/config';
+import { EnhancedCircuitBreakerService } from './circuit-breaker.service';
+
+describe('EnhancedCircuitBreakerService', () => {
+  let service: EnhancedCircuitBreakerService;
+
+  const mockConfigService = {
+    get: jest.fn((key: string, defaultValue?: number) => defaultValue),
+  };
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [
+        EnhancedCircuitBreakerService,
+        { provide: ConfigService, useValue: mockConfigService },
+      ],
+    }).compile();
+
+    service = module.get<EnhancedCircuitBreakerService>(EnhancedCircuitBreakerService);
+    service.onModuleInit();
+  });
+
+  afterEach(async () => {
+    await service.shutdown();
+  });
+
+  describe('execute', () => {
+    it('should execute a successful operation', async () => {
+      const result = await service.execute('test-key', async () => 'success');
+      expect(result).toBe('success');
+    });
+
+    it('should return fallback result when operation fails', async () => {
+      const result = await service.execute(
+        'test-fallback',
+        async () => {
+          throw new Error('service down');
+        },
+        {
+          fallback: () => 'fallback-value',
+          errorThresholdPercentage: 1,
+          resetTimeout: 100,
+        },
+      );
+      expect(result).toBe('fallback-value');
+    });
+
+    it('should throw when operation fails and no fallback provided', async () => {
+      await expect(
+        service.execute('test-no-fallback', async () => {
+          throw new Error('service down');
+        }),
+      ).rejects.toThrow();
+    });
+
+    it('should reuse the same circuit breaker for the same key', async () => {
+      await service.execute('reuse-key', async () => 'first');
+      await service.execute('reuse-key', async () => 'second');
+      const stats = service.getStats('reuse-key');
+      expect(stats).not.toBeNull();
+      expect(stats!.stats.successes).toBe(2);
+    });
+  });
+
+  describe('getStats', () => {
+    it('should return null for unknown key', () => {
+      expect(service.getStats('unknown')).toBeNull();
+    });
+
+    it('should return stats after execution', async () => {
+      await service.execute('stats-key', async () => 'ok');
+      const stats = service.getStats('stats-key');
+      expect(stats).not.toBeNull();
+      expect(stats!.name).toBe('stats-key');
+      expect(stats!.closed).toBe(true);
+    });
+  });
+
+  describe('getAllStats', () => {
+    it('should return empty object when no breakers exist', () => {
+      expect(service.getAllStats()).toEqual({});
+    });
+
+    it('should return stats for all registered breakers', async () => {
+      await service.execute('breaker-a', async () => 'a');
+      await service.execute('breaker-b', async () => 'b');
+      const all = service.getAllStats();
+      expect(Object.keys(all)).toContain('breaker-a');
+      expect(Object.keys(all)).toContain('breaker-b');
+    });
+  });
+
+  describe('half-open state', () => {
+    it('should transition through open and half-open states', async () => {
+      // Force the circuit open by failing repeatedly
+      const failingOp = async () => {
+        throw new Error('fail');
+      };
+      const opts = {
+        errorThresholdPercentage: 1,
+        resetTimeout: 100,
+        rollingCountTimeout: 500,
+        rollingCountBuckets: 2,
+        fallback: () => null,
+      };
+
+      // Trigger failures to open the circuit
+      for (let i = 0; i < 5; i++) {
+        await service.execute('half-open-test', failingOp, opts);
+      }
+
+      const stats = service.getStats('half-open-test');
+      expect(stats).not.toBeNull();
+      // Circuit should be open (not closed) after repeated failures
+      expect(stats!.closed).toBe(false);
+    });
+  });
+
+  describe('enable / disable', () => {
+    it('should enable and disable a circuit breaker', async () => {
+      await service.execute('toggle-key', async () => 'ok');
+      service.disable('toggle-key');
+      service.enable('toggle-key');
+      // No errors thrown — just verifying the methods work
+      const stats = service.getStats('toggle-key');
+      expect(stats).not.toBeNull();
+    });
+
+    it('should not throw when enabling/disabling unknown key', () => {
+      expect(() => service.enable('nonexistent')).not.toThrow();
+      expect(() => service.disable('nonexistent')).not.toThrow();
+    });
+  });
+
+  describe('close (reset)', () => {
+    it('should remove the circuit breaker on close', async () => {
+      await service.execute('reset-key', async () => 'ok');
+      service.close('reset-key');
+      expect(service.getStats('reset-key')).toBeNull();
+    });
+  });
+
+  describe('getHealthStatus', () => {
+    it('should return zero totals when no breakers exist', () => {
+      const health = service.getHealthStatus();
+      expect(health.total).toBe(0);
+      expect(health.healthy).toBe(0);
+      expect(health.unhealthy).toBe(0);
+    });
+
+    it('should report healthy breaker after successful calls', async () => {
+      await service.execute('healthy-key', async () => 'ok');
+      const health = service.getHealthStatus();
+      expect(health.total).toBe(1);
+      expect(health.healthy).toBe(1);
+    });
+  });
+
+  describe('shutdown', () => {
+    it('should shut down all circuit breakers', async () => {
+      await service.execute('shutdown-key', async () => 'ok');
+      await service.shutdown();
+      expect(service.getAllStats()).toEqual({});
+    });
+  });
+});

src/incident-management/incident-management.module.ts

@@ -14,6 +14,7 @@ import {
   RunbookExecutionService,
   NotificationAndEscalationService,
 } from './services';
+import { CommonModule } from '../common/common.module';
 
 @Module({
   imports: [
@@ -23,6 +24,7 @@ import {
       RunbookExecution,
     ]),
     ConfigModule,
+    CommonModule,
   ],
   controllers: [IncidentManagementController],
   providers: [

src/incident-management/services/notification-and-escalation.service.ts

@@ -4,6 +4,7 @@ import * as nodemailer from 'nodemailer';
 import axios from 'axios';
 import { Incident, IncidentSeverity } from '../entities/incident.entity';
 import { RemediationAction } from '../entities/remediation-action.entity';
+import { EnhancedCircuitBreakerService } from '../../common/services/circuit-breaker.service';
 
 export enum NotificationChannel {
   EMAIL = 'email',
@@ -31,7 +32,10 @@ export class NotificationAndEscalationService {
   private emailTransporter: nodemailer.Transporter;
   private escalationPolicies: Map<string, EscalationPolicy> = new Map();
 
-  constructor(private configService: ConfigService) {
+  constructor(
+    private configService: ConfigService,
+    private circuitBreakerService: EnhancedCircuitBreakerService,
+  ) {
     this.initializeEmailTransport();
     this.initializeEscalationPolicies();
   }
@@ -377,34 +381,47 @@ export class NotificationAndEscalationService {
 
     const text = this.buildSlackMessage(incident, eventType, remediationAction);
 
-    await axios.post(slackWebhook, {
-      channel,
-      attachments: [
-        {
-          color,
-          title: incident.title,
-          text,
-          fields: [
+    await this.circuitBreakerService.execute(
+      'slack-notification',
+      () =>
+        axios.post(slackWebhook, {
+          channel,
+          attachments: [
             {
-              title: 'Severity',
-              value: incident.severity,
-              short: true,
-            },
-            {
-              title: 'Status',
-              value: incident.status,
-              short: true,
-            },
-            {
-              title: 'Incident ID',
-              value: incident.id,
-              short: false,
+              color,
+              title: incident.title,
+              text,
+              fields: [
+                {
+                  title: 'Severity',
+                  value: incident.severity,
+                  short: true,
+                },
+                {
+                  title: 'Status',
+                  value: incident.status,
+                  short: true,
+                },
+                {
+                  title: 'Incident ID',
+                  value: incident.id,
+                  short: false,
+                },
+              ],
+              ts: Math.floor(Date.now() / 1000),
             },
           ],
-          ts: Math.floor(Date.now() / 1000),
+        }),
+      {
+        timeout: 5000,
+        errorThresholdPercentage: 50,
+        resetTimeout: 30000,
+        fallback: (error: Error) => {
+          this.logger.warn(`Slack notification fallback triggered: ${error.message}`);
+          return null;
         },
-      ],
-    });
+      },
+    );
 
     this.logger.log(`Slack notification sent to ${channel}`);
   }
@@ -429,20 +446,33 @@ export class NotificationAndEscalationService {
           ? 'resolve'
           : 'acknowledge';
 
-    await axios.post('https://events.pagerduty.com/v2/enqueue', {
-      routing_key: pagerDutyKey,
-      event_action: eventAction,
-      dedup_key: incident.id,
-      payload: {
-        summary: incident.title,
-        severity: incident.severity.toLowerCase(),
-        source: 'TeachLink Incident Management',
-        custom_details: {
-          description: incident.description,
-          incidentId: incident.id,
+    await this.circuitBreakerService.execute(
+      'pagerduty-notification',
+      () =>
+        axios.post('https://events.pagerduty.com/v2/enqueue', {
+          routing_key: pagerDutyKey,
+          event_action: eventAction,
+          dedup_key: incident.id,
+          payload: {
+            summary: incident.title,
+            severity: incident.severity.toLowerCase(),
+            source: 'TeachLink Incident Management',
+            custom_details: {
+              description: incident.description,
+              incidentId: incident.id,
+            },
+          },
+        }),
+      {
+        timeout: 5000,
+        errorThresholdPercentage: 50,
+        resetTimeout: 30000,
+        fallback: (error: Error) => {
+          this.logger.warn(`PagerDuty notification fallback triggered: ${error.message}`);
+          return null;
         },
       },
-    });
+    );
 
     this.logger.log(`PagerDuty notification sent for incident ${incident.id}`);
   }
@@ -455,17 +485,30 @@ export class NotificationAndEscalationService {
     incident: Incident,
     eventType: string,
   ): Promise<void> {
-    await axios.post(webhookUrl, {
-      eventType,
-      incident: {
-        id: incident.id,
-        title: incident.title,
-        description: incident.description,
-        severity: incident.severity,
-        status: incident.status,
-        detectedAt: incident.detectedAt,
+    await this.circuitBreakerService.execute(
+      `webhook-notification-${webhookUrl}`,
+      () =>
+        axios.post(webhookUrl, {
+          eventType,
+          incident: {
+            id: incident.id,
+            title: incident.title,
+            description: incident.description,
+            severity: incident.severity,
+            status: incident.status,
+            detectedAt: incident.detectedAt,
+          },
+        }),
+      {
+        timeout: 5000,
+        errorThresholdPercentage: 50,
+        resetTimeout: 30000,
+        fallback: (error: Error) => {
+          this.logger.warn(`Webhook notification fallback triggered for ${webhookUrl}: ${error.message}`);
+          return null;
+        },
       },
-    });
+    );
 
     this.logger.log(`Webhook notification sent to ${webhookUrl}`);
   }