diff --git a/src/common/common.module.ts b/src/common/common.module.ts index a61e7539..cf9fbc48 100644 --- a/src/common/common.module.ts +++ b/src/common/common.module.ts @@ -2,13 +2,16 @@ import { Module } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; import { TransactionHelperService } from './database/transaction-helper.service'; import { LogShipperService } from './services/log-shipper.service'; +import { EnhancedCircuitBreakerService } from './services/circuit-breaker.service'; +import { CircuitBreakerController } from './controllers/circuit-breaker.controller'; /** * Registers the common module. */ @Module({ imports: [ConfigModule], - providers: [TransactionHelperService, LogShipperService], - exports: [TransactionHelperService, LogShipperService], + controllers: [CircuitBreakerController], + providers: [TransactionHelperService, LogShipperService, EnhancedCircuitBreakerService], + exports: [TransactionHelperService, LogShipperService, EnhancedCircuitBreakerService], }) export class CommonModule {} diff --git a/src/common/services/circuit-breaker.service.spec.ts b/src/common/services/circuit-breaker.service.spec.ts new file mode 100644 index 00000000..5821907e --- /dev/null +++ b/src/common/services/circuit-breaker.service.spec.ts @@ -0,0 +1,167 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ConfigService } from '@nestjs/config'; +import { EnhancedCircuitBreakerService } from './circuit-breaker.service'; + +describe('EnhancedCircuitBreakerService', () => { + let service: EnhancedCircuitBreakerService; + + const mockConfigService = { + get: jest.fn((key: string, defaultValue?: number) => defaultValue), + }; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + EnhancedCircuitBreakerService, + { provide: ConfigService, useValue: mockConfigService }, + ], + }).compile(); + + service = module.get(EnhancedCircuitBreakerService); + service.onModuleInit(); + }); + + afterEach(async () => { + await service.shutdown(); + }); + + describe('execute', () => { + it('should execute a successful operation', async () => { + const result = await service.execute('test-key', async () => 'success'); + expect(result).toBe('success'); + }); + + it('should return fallback result when operation fails', async () => { + const result = await service.execute( + 'test-fallback', + async () => { + throw new Error('service down'); + }, + { + fallback: () => 'fallback-value', + errorThresholdPercentage: 1, + resetTimeout: 100, + }, + ); + expect(result).toBe('fallback-value'); + }); + + it('should throw when operation fails and no fallback provided', async () => { + await expect( + service.execute('test-no-fallback', async () => { + throw new Error('service down'); + }), + ).rejects.toThrow(); + }); + + it('should reuse the same circuit breaker for the same key', async () => { + await service.execute('reuse-key', async () => 'first'); + await service.execute('reuse-key', async () => 'second'); + const stats = service.getStats('reuse-key'); + expect(stats).not.toBeNull(); + expect(stats!.stats.successes).toBe(2); + }); + }); + + describe('getStats', () => { + it('should return null for unknown key', () => { + expect(service.getStats('unknown')).toBeNull(); + }); + + it('should return stats after execution', async () => { + await service.execute('stats-key', async () => 'ok'); + const stats = service.getStats('stats-key'); + expect(stats).not.toBeNull(); + expect(stats!.name).toBe('stats-key'); + expect(stats!.closed).toBe(true); + }); + }); + + describe('getAllStats', () => { + it('should return empty object when no breakers exist', () => { + expect(service.getAllStats()).toEqual({}); + }); + + it('should return stats for all registered breakers', async () => { + await service.execute('breaker-a', async () => 'a'); + await service.execute('breaker-b', async () => 'b'); + const all = service.getAllStats(); + expect(Object.keys(all)).toContain('breaker-a'); + expect(Object.keys(all)).toContain('breaker-b'); + }); + }); + + describe('half-open state', () => { + it('should transition through open and half-open states', async () => { + // Force the circuit open by failing repeatedly + const failingOp = async () => { + throw new Error('fail'); + }; + const opts = { + errorThresholdPercentage: 1, + resetTimeout: 100, + rollingCountTimeout: 500, + rollingCountBuckets: 2, + fallback: () => null, + }; + + // Trigger failures to open the circuit + for (let i = 0; i < 5; i++) { + await service.execute('half-open-test', failingOp, opts); + } + + const stats = service.getStats('half-open-test'); + expect(stats).not.toBeNull(); + // Circuit should be open (not closed) after repeated failures + expect(stats!.closed).toBe(false); + }); + }); + + describe('enable / disable', () => { + it('should enable and disable a circuit breaker', async () => { + await service.execute('toggle-key', async () => 'ok'); + service.disable('toggle-key'); + service.enable('toggle-key'); + // No errors thrown — just verifying the methods work + const stats = service.getStats('toggle-key'); + expect(stats).not.toBeNull(); + }); + + it('should not throw when enabling/disabling unknown key', () => { + expect(() => service.enable('nonexistent')).not.toThrow(); + expect(() => service.disable('nonexistent')).not.toThrow(); + }); + }); + + describe('close (reset)', () => { + it('should remove the circuit breaker on close', async () => { + await service.execute('reset-key', async () => 'ok'); + service.close('reset-key'); + expect(service.getStats('reset-key')).toBeNull(); + }); + }); + + describe('getHealthStatus', () => { + it('should return zero totals when no breakers exist', () => { + const health = service.getHealthStatus(); + expect(health.total).toBe(0); + expect(health.healthy).toBe(0); + expect(health.unhealthy).toBe(0); + }); + + it('should report healthy breaker after successful calls', async () => { + await service.execute('healthy-key', async () => 'ok'); + const health = service.getHealthStatus(); + expect(health.total).toBe(1); + expect(health.healthy).toBe(1); + }); + }); + + describe('shutdown', () => { + it('should shut down all circuit breakers', async () => { + await service.execute('shutdown-key', async () => 'ok'); + await service.shutdown(); + expect(service.getAllStats()).toEqual({}); + }); + }); +}); diff --git a/src/incident-management/incident-management.module.ts b/src/incident-management/incident-management.module.ts index f42c3057..d34b138e 100644 --- a/src/incident-management/incident-management.module.ts +++ b/src/incident-management/incident-management.module.ts @@ -14,6 +14,7 @@ import { RunbookExecutionService, NotificationAndEscalationService, } from './services'; +import { CommonModule } from '../common/common.module'; @Module({ imports: [ @@ -23,6 +24,7 @@ import { RunbookExecution, ]), ConfigModule, + CommonModule, ], controllers: [IncidentManagementController], providers: [ diff --git a/src/incident-management/services/notification-and-escalation.service.ts b/src/incident-management/services/notification-and-escalation.service.ts index 01a1291a..44a0a76f 100644 --- a/src/incident-management/services/notification-and-escalation.service.ts +++ b/src/incident-management/services/notification-and-escalation.service.ts @@ -4,6 +4,7 @@ import * as nodemailer from 'nodemailer'; import axios from 'axios'; import { Incident, IncidentSeverity } from '../entities/incident.entity'; import { RemediationAction } from '../entities/remediation-action.entity'; +import { EnhancedCircuitBreakerService } from '../../common/services/circuit-breaker.service'; export enum NotificationChannel { EMAIL = 'email', @@ -31,7 +32,10 @@ export class NotificationAndEscalationService { private emailTransporter: nodemailer.Transporter; private escalationPolicies: Map = new Map(); - constructor(private configService: ConfigService) { + constructor( + private configService: ConfigService, + private circuitBreakerService: EnhancedCircuitBreakerService, + ) { this.initializeEmailTransport(); this.initializeEscalationPolicies(); } @@ -377,34 +381,47 @@ export class NotificationAndEscalationService { const text = this.buildSlackMessage(incident, eventType, remediationAction); - await axios.post(slackWebhook, { - channel, - attachments: [ - { - color, - title: incident.title, - text, - fields: [ + await this.circuitBreakerService.execute( + 'slack-notification', + () => + axios.post(slackWebhook, { + channel, + attachments: [ { - title: 'Severity', - value: incident.severity, - short: true, - }, - { - title: 'Status', - value: incident.status, - short: true, - }, - { - title: 'Incident ID', - value: incident.id, - short: false, + color, + title: incident.title, + text, + fields: [ + { + title: 'Severity', + value: incident.severity, + short: true, + }, + { + title: 'Status', + value: incident.status, + short: true, + }, + { + title: 'Incident ID', + value: incident.id, + short: false, + }, + ], + ts: Math.floor(Date.now() / 1000), }, ], - ts: Math.floor(Date.now() / 1000), + }), + { + timeout: 5000, + errorThresholdPercentage: 50, + resetTimeout: 30000, + fallback: (error: Error) => { + this.logger.warn(`Slack notification fallback triggered: ${error.message}`); + return null; }, - ], - }); + }, + ); this.logger.log(`Slack notification sent to ${channel}`); } @@ -429,20 +446,33 @@ export class NotificationAndEscalationService { ? 'resolve' : 'acknowledge'; - await axios.post('https://events.pagerduty.com/v2/enqueue', { - routing_key: pagerDutyKey, - event_action: eventAction, - dedup_key: incident.id, - payload: { - summary: incident.title, - severity: incident.severity.toLowerCase(), - source: 'TeachLink Incident Management', - custom_details: { - description: incident.description, - incidentId: incident.id, + await this.circuitBreakerService.execute( + 'pagerduty-notification', + () => + axios.post('https://events.pagerduty.com/v2/enqueue', { + routing_key: pagerDutyKey, + event_action: eventAction, + dedup_key: incident.id, + payload: { + summary: incident.title, + severity: incident.severity.toLowerCase(), + source: 'TeachLink Incident Management', + custom_details: { + description: incident.description, + incidentId: incident.id, + }, + }, + }), + { + timeout: 5000, + errorThresholdPercentage: 50, + resetTimeout: 30000, + fallback: (error: Error) => { + this.logger.warn(`PagerDuty notification fallback triggered: ${error.message}`); + return null; }, }, - }); + ); this.logger.log(`PagerDuty notification sent for incident ${incident.id}`); } @@ -455,17 +485,30 @@ export class NotificationAndEscalationService { incident: Incident, eventType: string, ): Promise { - await axios.post(webhookUrl, { - eventType, - incident: { - id: incident.id, - title: incident.title, - description: incident.description, - severity: incident.severity, - status: incident.status, - detectedAt: incident.detectedAt, + await this.circuitBreakerService.execute( + `webhook-notification-${webhookUrl}`, + () => + axios.post(webhookUrl, { + eventType, + incident: { + id: incident.id, + title: incident.title, + description: incident.description, + severity: incident.severity, + status: incident.status, + detectedAt: incident.detectedAt, + }, + }), + { + timeout: 5000, + errorThresholdPercentage: 50, + resetTimeout: 30000, + fallback: (error: Error) => { + this.logger.warn(`Webhook notification fallback triggered for ${webhookUrl}: ${error.message}`); + return null; + }, }, - }); + ); this.logger.log(`Webhook notification sent to ${webhookUrl}`); } diff --git a/src/monitoring/alerting/alerting.service.ts b/src/monitoring/alerting/alerting.service.ts index 055bcc4b..988b43c0 100644 --- a/src/monitoring/alerting/alerting.service.ts +++ b/src/monitoring/alerting/alerting.service.ts @@ -2,6 +2,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import * as nodemailer from 'nodemailer'; import axios from 'axios'; +import { EnhancedCircuitBreakerService } from '../../common/services/circuit-breaker.service'; export type AlertSeverity = 'INFO' | 'WARNING' | 'CRITICAL'; @@ -121,7 +122,10 @@ export class AlertingService { private readonly emailFrom: string; private mailerTransport: nodemailer.Transporter | null = null; - constructor(private readonly configService: ConfigService) { + constructor( + private readonly configService: ConfigService, + private readonly circuitBreakerService: EnhancedCircuitBreakerService, + ) { this.emailFrom = this.configService.get('EMAIL_FROM', 'noreply@teachlink.io'); const recipientRaw = this.configService.get('ALERT_EMAIL_RECIPIENTS', ''); this.alertEmailRecipients = recipientRaw @@ -369,6 +373,18 @@ export class AlertingService { ], }; - await axios.post(this.slackWebhookUrl, body); + await this.circuitBreakerService.execute( + 'alerting-slack', + () => axios.post(this.slackWebhookUrl, body), + { + timeout: 5000, + errorThresholdPercentage: 50, + resetTimeout: 30000, + fallback: (error: Error) => { + this.logger.warn(`Alerting Slack fallback triggered: ${error.message}`); + return null; + }, + }, + ); } } diff --git a/src/monitoring/monitoring.module.ts b/src/monitoring/monitoring.module.ts index 084c6e3a..b1387020 100644 --- a/src/monitoring/monitoring.module.ts +++ b/src/monitoring/monitoring.module.ts @@ -3,9 +3,10 @@ import { ConfigModule } from '@nestjs/config'; import { AlertingService } from './alerting/alerting.service'; import { MetricsCollectionService } from './metrics/metrics-collection.service'; import { CustomMetricsService } from './custom-metrics.service'; +import { CommonModule } from '../common/common.module'; @Module({ - imports: [ConfigModule], + imports: [ConfigModule, CommonModule], providers: [AlertingService, MetricsCollectionService, CustomMetricsService], exports: [AlertingService, MetricsCollectionService, CustomMetricsService], })