From e50f4ae3727ed48569d65253363da93c3303b7a5 Mon Sep 17 00:00:00 2001 From: Esteban Galvis Date: Fri, 20 Mar 2026 16:37:00 -0500 Subject: [PATCH 1/2] feat(backup): enhance backup progress tracking and state management --- src/apps/backups/BackupService.ts | 17 +- .../features/backup/backup-progress-state.ts | 91 ++++++ .../backup/backup-progress-tracker.test.ts | 302 ++++++++++++++---- .../backup/backup-progress-tracker.ts | 46 ++- .../backup/launch-backup-processes.ts | 90 ++++-- .../precalculate-backup-item-count.test.ts | 107 +++++++ .../backup/precalculate-backup-item-count.ts | 51 +++ .../update/FileBatchUpdater.test.ts | 5 +- .../application/update/FileBatchUpdater.ts | 2 + .../upload/FileBatchUploader.test.ts | 170 ++++++++++ .../application/upload/FileBatchUploader.ts | 6 + 11 files changed, 785 insertions(+), 102 deletions(-) create mode 100644 src/backend/features/backup/backup-progress-state.ts create mode 100644 src/backend/features/backup/precalculate-backup-item-count.test.ts create mode 100644 src/backend/features/backup/precalculate-backup-item-count.ts create mode 100644 src/context/local/localFile/application/upload/FileBatchUploader.test.ts diff --git a/src/apps/backups/BackupService.ts b/src/apps/backups/BackupService.ts index a89520767..e4b28666d 100644 --- a/src/apps/backups/BackupService.ts +++ b/src/apps/backups/BackupService.ts @@ -72,8 +72,8 @@ export class BackupService { await this.isThereEnoughSpace(filesDiff); logger.debug({ tag: 'BACKUPS', msg: 'Space check completed' }); - const itemsAlreadyBacked = filesDiff.unmodified.length + foldersDiff.unmodified.length; - tracker.addToTotal(filesDiff.total + foldersDiff.total); + const emptyAddedFiles = filesDiff.added.filter((f) => f.size === 0).length; + const itemsAlreadyBacked = filesDiff.unmodified.length + foldersDiff.unmodified.length + emptyAddedFiles; tracker.incrementProcessed(itemsAlreadyBacked); logger.debug({ tag: 'BACKUPS', msg: 'Starting folder backup' }); @@ -212,8 +212,9 @@ export class BackupService { return; } // eslint-disable-next-line no-await-in-loop - await this.fileBatchUploader.run(localRootPath, tree, batch, signal); - tracker.incrementProcessed(batch.length); + await this.fileBatchUploader.run(localRootPath, tree, batch, signal, () => { + tracker.incrementProcessed(1); + }); } } @@ -227,13 +228,13 @@ export class BackupService { const batches = ModifiedFilesBatchCreator.run(modified); for (const batch of batches) { - logger.debug({ tag: 'BACKUPS', msg: 'Signal aborted', aborted: signal.aborted }); if (signal.aborted) { return; } // eslint-disable-next-line no-await-in-loop - await this.fileBatchUpdater.run(localTree.root, remoteTree, Array.from(batch.keys()), signal); - tracker.incrementProcessed(batch.size); + await this.fileBatchUpdater.run(localTree.root, remoteTree, Array.from(batch.keys()), signal, () => { + tracker.incrementProcessed(1); + }); } } @@ -249,7 +250,7 @@ export class BackupService { // eslint-disable-next-line no-await-in-loop await addFileToTrash(file.uuid); + tracker.incrementProcessed(1); } - tracker.incrementProcessed(deleted.length); } } diff --git a/src/backend/features/backup/backup-progress-state.ts b/src/backend/features/backup/backup-progress-state.ts new file mode 100644 index 000000000..814a974b4 --- /dev/null +++ b/src/backend/features/backup/backup-progress-state.ts @@ -0,0 +1,91 @@ +export type BackupProgressState = { + readonly processedItems: number; + readonly backupWeights: ReadonlyMap; + readonly backupTotals: ReadonlyMap; + readonly currentBackupId: string; + readonly completedBackups: ReadonlySet; +}; + +export const createInitialState = (): BackupProgressState => ({ + processedItems: 0, + backupWeights: new Map(), + backupTotals: new Map(), + currentBackupId: '', + completedBackups: new Set(), +}); + +export const initializeWeights = ( + state: BackupProgressState, + backupIds: string[], + fileCounts: ReadonlyMap, +): BackupProgressState => { + const totalFiles = Array.from(fileCounts.values()).reduce((a, b) => a + b, 0); + + if (totalFiles === 0) { + return state; + } + + const weights = new Map(); + const totals = new Map(); + + backupIds.forEach((id) => { + const count = fileCounts.get(id) || 1; + weights.set(id, count / totalFiles); + totals.set(id, count); + }); + + return { + ...state, + backupWeights: weights, + backupTotals: totals, + }; +}; + +export const setCurrentBackupId = (state: BackupProgressState, backupId: string): BackupProgressState => ({ + ...state, + currentBackupId: backupId, + processedItems: 0, +}); + +export const incrementProcessed = (state: BackupProgressState, count: number = 1): BackupProgressState => ({ + ...state, + processedItems: state.processedItems + count, +}); + +export const markBackupAsCompleted = (state: BackupProgressState, backupId: string): BackupProgressState => ({ + ...state, + completedBackups: new Set([...state.completedBackups, backupId]), +}); + +export const getPercentage = (state: BackupProgressState): number => { + let weightedProgress = 0; + + for (const backupId of state.completedBackups) { + const weight = state.backupWeights.get(backupId) || 0; + weightedProgress += weight * 100; + } + + if (state.backupWeights.has(state.currentBackupId) && !state.completedBackups.has(state.currentBackupId)) { + const currentWeight = state.backupWeights.get(state.currentBackupId)!; + const currentTotal = state.backupTotals.get(state.currentBackupId) || 1; + + if (currentTotal > 0) { + const backupProgress = (state.processedItems / currentTotal) * 100; + weightedProgress += currentWeight * backupProgress; + } + } + + return Math.min(100, Math.round(weightedProgress)); +}; + +export const resetState = (): BackupProgressState => createInitialState(); + +export const initializeAndSetBackup = ( + state: BackupProgressState, + backupIds: string[], + fileCounts: ReadonlyMap, + firstBackupId: string, +): BackupProgressState => { + const weightedState = initializeWeights(state, backupIds, fileCounts); + return setCurrentBackupId(weightedState, firstBackupId); +}; diff --git a/src/backend/features/backup/backup-progress-tracker.test.ts b/src/backend/features/backup/backup-progress-tracker.test.ts index ab1d33b74..e1eeab3e9 100644 --- a/src/backend/features/backup/backup-progress-tracker.test.ts +++ b/src/backend/features/backup/backup-progress-tracker.test.ts @@ -1,12 +1,21 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { BackupProgressTracker } from './backup-progress-tracker'; +import { + createInitialState, + initializeWeights, + setCurrentBackupId, + incrementProcessed, + markBackupAsCompleted, + getPercentage, + resetState, +} from './backup-progress-state'; import { broadcastToWindows } from '../../../apps/main/windows'; vi.mock('../../../apps/main/windows', () => ({ broadcastToWindows: vi.fn(), })); -describe('BackupProgressTracker', () => { +describe('BackupProgressTracker - Functional approach', () => { let tracker: BackupProgressTracker; beforeEach(() => { @@ -14,119 +23,290 @@ describe('BackupProgressTracker', () => { vi.clearAllMocks(); }); - describe('addToTotal', () => { - it('should add to total items given a number', () => { - tracker.addToTotal(5); + describe('initializeWeights', () => { + it('should initialize weights for multiple backups', () => { + const backupIds = ['backup-a', 'backup-b', 'backup-c']; + const fileCounts = new Map([ + ['backup-a', 100], + ['backup-b', 200], + ['backup-c', 200], + ]); + tracker.initializeWeights(backupIds, fileCounts); + + // After initialization, percentages should still be 0 expect(tracker.getPercentage()).toBe(0); }); - it('should accumulate total items when called multiple times', () => { - tracker.addToTotal(5); - tracker.addToTotal(10); - tracker.incrementProcessed(15); + it('should handle single backup', () => { + const backupIds = ['backup-a']; + const fileCounts = new Map([['backup-a', 100]]); - expect(tracker.getPercentage()).toBe(100); + tracker.initializeWeights(backupIds, fileCounts); + + expect(tracker.getPercentage()).toBe(0); }); - it('should handle adding zero to total items', () => { - tracker.addToTotal(0); + it('should handle empty file counts gracefully', () => { + const backupIds = ['backup-a']; + const fileCounts = new Map([['backup-a', 0]]); + + tracker.initializeWeights(backupIds, fileCounts); expect(tracker.getPercentage()).toBe(0); }); + }); - it('should handle adding negative numbers to total items', () => { - tracker.addToTotal(10); - tracker.addToTotal(-5); - tracker.incrementProcessed(5); + describe('setCurrentBackupId', () => { + it('should set current backup id', () => { + const backupIds = ['backup-a', 'backup-b']; + const fileCounts = new Map([ + ['backup-a', 100], + ['backup-b', 50], + ]); - expect(tracker.getPercentage()).toBe(100); + tracker.initializeWeights(backupIds, fileCounts); + tracker.setCurrentBackupId('backup-a'); + + // Should still be 0 initially + expect(tracker.getPercentage()).toBe(0); }); - }); - describe('reset', () => { - it('should reset processed and total items to zero', () => { - tracker.addToTotal(100); + it('should reset processed items when setting new backup', () => { + const backupIds = ['backup-a', 'backup-b']; + const fileCounts = new Map([ + ['backup-a', 100], + ['backup-b', 50], + ]); + + tracker.initializeWeights(backupIds, fileCounts); + tracker.setCurrentBackupId('backup-a'); tracker.incrementProcessed(50); - tracker.reset(); + expect(tracker.getPercentage()).toBeGreaterThan(0); - expect(tracker.getPercentage()).toBe(0); + tracker.setCurrentBackupId('backup-b'); + // Processed should reset to 0 for new backup + expect(tracker.getPercentage()).toBeLessThan(50); // Should be backup-a (67%) * 50% = ~33% }); }); - describe('incrementProcessed', () => { - it('should increment processed items given a number', () => { - tracker.addToTotal(100); + describe('incrementProcessed with weighted calculation', () => { + it('should calculate weighted progress for single backup', () => { + const backupIds = ['backup-a']; + const fileCounts = new Map([['backup-a', 100]]); + + tracker.initializeWeights(backupIds, fileCounts); + tracker.setCurrentBackupId('backup-a'); tracker.incrementProcessed(25); + // 25/100 * 100% = 25% expect(tracker.getPercentage()).toBe(25); }); - it('should accumulate processed items when called multiple times', () => { - tracker.addToTotal(100); - tracker.incrementProcessed(10); - tracker.incrementProcessed(20); - tracker.incrementProcessed(30); + it('should calculate weighted progress for multiple backups', () => { + const backupIds = ['backup-a', 'backup-b']; + const fileCounts = new Map([ + ['backup-a', 100], // 66.7% weight + ['backup-b', 50], // 33.3% weight + ]); - expect(tracker.getPercentage()).toBe(60); - }); + tracker.initializeWeights(backupIds, fileCounts); - it('should handle incrementing by zero', () => { - tracker.addToTotal(100); + // Process backup-a halfway + tracker.setCurrentBackupId('backup-a'); tracker.incrementProcessed(50); - tracker.incrementProcessed(0); - expect(tracker.getPercentage()).toBe(50); + // Should be around 33% (50/100 * 0.667 = 0.333) + expect(tracker.getPercentage()).toBe(33); }); - it('should handle incrementing by negative numbers', () => { - tracker.addToTotal(100); - tracker.incrementProcessed(50); - tracker.incrementProcessed(-10); + it('should accumulate progress correctly across backups', () => { + const backupIds = ['backup-a', 'backup-b']; + const fileCounts = new Map([ + ['backup-a', 100], // 66.7% weight + ['backup-b', 50], // 33.3% weight + ]); + + tracker.initializeWeights(backupIds, fileCounts); - expect(tracker.getPercentage()).toBe(40); + // Complete backup-a + tracker.setCurrentBackupId('backup-a'); + tracker.incrementProcessed(100); + expect(tracker.getPercentage()).toBe(67); + + // Mark backup-a as completed + tracker.markBackupAsCompleted('backup-a'); + + // Start backup-b + tracker.setCurrentBackupId('backup-b'); + tracker.incrementProcessed(25); + + // Should be 67% (completed a) + 25/50 * 33% = 67% + 16% = 83% + expect(tracker.getPercentage()).toBe(83); }); - it('should emit progress after incrementing processed items', () => { - tracker.addToTotal(100); + it('should emit progress after incrementing', () => { + const backupIds = ['backup-a']; + const fileCounts = new Map([['backup-a', 100]]); + + tracker.initializeWeights(backupIds, fileCounts); + tracker.setCurrentBackupId('backup-a'); tracker.incrementProcessed(50); expect(broadcastToWindows).toHaveBeenCalledWith('backup-progress', 50); }); + + it('should handle increments accumulating', () => { + const backupIds = ['backup-a']; + const fileCounts = new Map([['backup-a', 100]]); + + tracker.initializeWeights(backupIds, fileCounts); + tracker.setCurrentBackupId('backup-a'); + + tracker.incrementProcessed(25); + expect(tracker.getPercentage()).toBe(25); + + tracker.incrementProcessed(25); + expect(tracker.getPercentage()).toBe(50); + + tracker.incrementProcessed(50); + expect(tracker.getPercentage()).toBe(100); + }); }); - describe('getPercentage', () => { - it('should return 0% when total items is zero', () => { - expect(tracker.getPercentage()).toBe(0); + describe('markBackupAsCompleted', () => { + it('should mark backup as completed', () => { + const backupIds = ['backup-a', 'backup-b']; + const fileCounts = new Map([ + ['backup-a', 10], + ['backup-b', 5], + ]); + + tracker.initializeWeights(backupIds, fileCounts); + tracker.setCurrentBackupId('backup-a'); + tracker.incrementProcessed(10); + + // At 66% (10/15 items) + expect(tracker.getPercentage()).toBe(67); + + tracker.markBackupAsCompleted('backup-a'); + + // Should stay at 67% + expect(tracker.getPercentage()).toBe(67); }); + }); - it('should return correct percentage based on processed and total items', () => { - tracker.addToTotal(200); + describe('reset', () => { + it('should reset to initial state', () => { + const backupIds = ['backup-a']; + const fileCounts = new Map([['backup-a', 100]]); + + tracker.initializeWeights(backupIds, fileCounts); + tracker.setCurrentBackupId('backup-a'); tracker.incrementProcessed(50); - expect(tracker.getPercentage()).toBe(25); + expect(tracker.getPercentage()).toBe(50); + + tracker.reset(); + + expect(tracker.getPercentage()).toBe(0); }); + }); +}); - it('should return 100% when processed items equal total items', () => { - tracker.addToTotal(50); - tracker.incrementProcessed(50); +describe('Pure functional backup progress state', () => { + describe('createInitialState', () => { + it('should create empty state', () => { + const state = createInitialState(); - expect(tracker.getPercentage()).toBe(100); + expect(state.processedItems).toBe(0); + expect(state.backupWeights.size).toBe(0); + expect(state.backupTotals.size).toBe(0); + expect(state.currentBackupId).toBe(''); + expect(state.completedBackups.size).toBe(0); }); + }); - it('should not exceed 100% even if processed items exceed total items', () => { - tracker.addToTotal(50); - tracker.incrementProcessed(100); + describe('initializeWeights', () => { + it('should calculate correct weights', () => { + const state = createInitialState(); + const backupIds = ['a', 'b']; + const fileCounts = new Map([ + ['a', 100], + ['b', 100], + ]); - expect(tracker.getPercentage()).toBe(100); + const newState = initializeWeights(state, backupIds, fileCounts); + + expect(newState.backupWeights.get('a')).toBe(0.5); + expect(newState.backupWeights.get('b')).toBe(0.5); }); - it('should round percentage to nearest integer', () => { - tracker.addToTotal(3); - tracker.incrementProcessed(1); + it('should handle unequal weights', () => { + const state = createInitialState(); + const backupIds = ['a', 'b']; + const fileCounts = new Map([ + ['a', 100], + ['b', 50], + ]); - expect(tracker.getPercentage()).toBe(33); + const newState = initializeWeights(state, backupIds, fileCounts); + + expect(newState.backupWeights.get('a')).toBeCloseTo(0.667, 2); + expect(newState.backupWeights.get('b')).toBeCloseTo(0.333, 2); + }); + }); + + describe('getPercentage', () => { + it('should calculate percentage with weights', () => { + let state = createInitialState(); + + const backupIds = ['a', 'b']; + const fileCounts = new Map([ + ['a', 100], + ['b', 50], + ]); + + state = initializeWeights(state, backupIds, fileCounts); + state = setCurrentBackupId(state, 'a'); + state = incrementProcessed(state, 50); + + // 50/100 * 0.667 = 33.35% ≈ 33% + expect(getPercentage(state)).toBe(33); + }); + + it('should accumulate completed backups', () => { + let state = createInitialState(); + + const backupIds = ['a', 'b']; + const fileCounts = new Map([ + ['a', 100], + ['b', 50], + ]); + + state = initializeWeights(state, backupIds, fileCounts); + state = setCurrentBackupId(state, 'a'); + state = incrementProcessed(state, 100); + state = markBackupAsCompleted(state, 'a'); + + expect(getPercentage(state)).toBeCloseTo(67, 0); + + state = setCurrentBackupId(state, 'b'); + state = incrementProcessed(state, 50); + + // 67% (completed a) + 100/50 * 33% = 100% + expect(getPercentage(state)).toBe(100); + }); + }); + + describe('resetState', () => { + it('should return initial state', () => { + const state = resetState(); + + expect(state.processedItems).toBe(0); + expect(state.backupWeights.size).toBe(0); + expect(state.completedBackups.size).toBe(0); }); }); }); diff --git a/src/backend/features/backup/backup-progress-tracker.ts b/src/backend/features/backup/backup-progress-tracker.ts index e921eac2e..672bce85d 100644 --- a/src/backend/features/backup/backup-progress-tracker.ts +++ b/src/backend/features/backup/backup-progress-tracker.ts @@ -1,27 +1,51 @@ import { logger } from '@internxt/drive-desktop-core/build/backend'; import { broadcastToWindows } from '../../../apps/main/windows'; +import { + BackupProgressState, + createInitialState, + initializeWeights, + setCurrentBackupId as setCurrentBackupIdFn, + markBackupAsCompleted as markBackupAsCompletedFn, + incrementProcessed as incrementProcessedFn, + getPercentage as getPercentageFn, + resetState, +} from './backup-progress-state'; export class BackupProgressTracker { - private totalItems = 0; - private processedItems = 0; + private state: BackupProgressState; - addToTotal(totalBackups: number): void { - this.totalItems += totalBackups; + constructor() { + this.state = createInitialState(); } - reset() { - this.processedItems = 0; - this.totalItems = 0; + initializeWeights(backupIds: string[], fileCounts: ReadonlyMap): void { + this.state = initializeWeights(this.state, backupIds, fileCounts); + logger.debug({ + tag: 'BACKUPS', + msg: 'Backup progress weights initialized', + weights: Object.fromEntries(this.state.backupWeights), + }); } - incrementProcessed(count: number): void { - this.processedItems += count; + setCurrentBackupId(backupId: string): void { + this.state = setCurrentBackupIdFn(this.state, backupId); + } + + markBackupAsCompleted(backupId: string): void { + this.state = markBackupAsCompletedFn(this.state, backupId); + } + + incrementProcessed(count: number = 1): void { + this.state = incrementProcessedFn(this.state, count); this.emitProgress(); } getPercentage(): number { - if (this.totalItems === 0) return 0; - return Math.min(100, Math.round((this.processedItems / this.totalItems) * 100)); + return getPercentageFn(this.state); + } + + reset(): void { + this.state = resetState(); } private emitProgress(): void { diff --git a/src/backend/features/backup/launch-backup-processes.ts b/src/backend/features/backup/launch-backup-processes.ts index 6f0fb2109..450ff77b9 100644 --- a/src/backend/features/backup/launch-backup-processes.ts +++ b/src/backend/features/backup/launch-backup-processes.ts @@ -7,36 +7,88 @@ import { backupsConfig } from '.'; import { BackupService } from '../../../apps/backups/BackupService'; import { BackupsDependencyContainerFactory } from '../../../apps/backups/dependency-injection/BackupsDependencyContainerFactory'; import { DriveDesktopError } from '../../../context/shared/domain/errors/DriveDesktopError'; +import { precalculateBackupItemCount } from './precalculate-backup-item-count'; export async function launchBackupProcesses( tracker: BackupProgressTracker, errors: BackupErrorsTracker, signal: AbortSignal, -): Promise { +) { const suspensionBlockId = powerSaveBlocker.start('prevent-display-sleep'); - const backups = await backupsConfig.obtainBackupsInfo(); - const container = await BackupsDependencyContainerFactory.build(); - const backupService = container.get(BackupService); + try { + const backups = await backupsConfig.obtainBackupsInfo(); + const container = await BackupsDependencyContainerFactory.build(); + const backupService = container.get(BackupService); - for (const backupInfo of backups) { - logger.debug({ tag: 'BACKUPS', msg: 'Backup info obtained:', backupInfo }); - if (signal.aborted) { - logger.debug({ tag: 'BACKUPS', msg: 'Backup aborted' }); - break; + logger.debug({ + tag: 'BACKUPS', + msg: 'Starting backup item count precalculation', + count: backups.length, + }); + + const itemCounts = new Map(); + + for (const backup of backups) { + if (signal.aborted) { + logger.debug({ tag: 'BACKUPS', msg: 'Precalculation aborted' }); + break; + } + + const count = await precalculateBackupItemCount(backup, container); + itemCounts.set(backup.folderUuid, count); } - // eslint-disable-next-line no-await-in-loop - const result = await backupService.runWithRetry(backupInfo, signal, tracker); - if (result.isLeft()) { - const error = result.getLeft(); - logger.debug({ tag: 'BACKUPS', msg: 'failed', error: error.cause }); - // TODO: Make retryError extend DriveDesktopError to avoid this check - if (error instanceof DriveDesktopError && 'cause' in error && error.cause && isSyncError(error.cause)) { - errors.add(backupInfo.folderId, { name: backupInfo.name, error: error.cause }); + const backupIds = backups.map((b) => b.folderUuid); + tracker.initializeWeights(backupIds, itemCounts); + + logger.debug({ + tag: 'BACKUPS', + msg: 'Starting backup execution with weighted progress', + }); + + for (const backupInfo of backups) { + logger.debug({ + tag: 'BACKUPS', + msg: 'Backup info obtained', + pathname: backupInfo.pathname, + }); + + if (signal.aborted) { + logger.debug({ tag: 'BACKUPS', msg: 'Backup execution aborted' }); + break; } + + tracker.setCurrentBackupId(backupInfo.folderUuid); + + const result = await backupService.runWithRetry(backupInfo, signal, tracker); + + tracker.markBackupAsCompleted(backupInfo.folderUuid); + + if (result.isLeft()) { + const error = result.getLeft(); + logger.debug({ + tag: 'BACKUPS', + msg: 'Backup failed', + pathname: backupInfo.pathname, + error: error.cause, + }); + + if (error instanceof DriveDesktopError && 'cause' in error && error.cause && isSyncError(error.cause)) { + errors.add(backupInfo.folderId, { + name: backupInfo.name, + error: error.cause, + }); + } + } + + logger.debug({ + tag: 'BACKUPS', + msg: 'Backup execution completed', + pathname: backupInfo.pathname, + }); } - logger.debug({ tag: 'BACKUPS', msg: `Backup of folder ${backupInfo.pathname} completed successfully` }); + } finally { + powerSaveBlocker.stop(suspensionBlockId); } - powerSaveBlocker.stop(suspensionBlockId); } diff --git a/src/backend/features/backup/precalculate-backup-item-count.test.ts b/src/backend/features/backup/precalculate-backup-item-count.test.ts new file mode 100644 index 000000000..8048f10af --- /dev/null +++ b/src/backend/features/backup/precalculate-backup-item-count.test.ts @@ -0,0 +1,107 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { left, right } from '../../../context/shared/domain/Either'; +import LocalTreeBuilder from '../../../context/local/localTree/application/LocalTreeBuilder'; +import { RemoteTreeBuilder } from '../../../context/virtual-drive/remoteTree/application/RemoteTreeBuilder'; +import { DiffFilesCalculatorService } from '../../../apps/backups/diff/DiffFilesCalculatorService'; +import { FoldersDiffCalculator } from '../../../apps/backups/diff/FoldersDiffCalculator'; +import { Container } from 'diod'; +import { precalculateBackupItemCount } from './precalculate-backup-item-count'; + +describe('precalculateBackupItemCount', () => { + const backupInfo = { + folderUuid: 'folder-uuid', + folderId: 42, + tmpPath: '/tmp/backup', + backupsBucket: 'bucket', + pathname: '/home/user/Documents', + name: 'Documents', + }; + + const localTree = { root: { path: '/home/user/Documents' } }; + const remoteTree = { root: { path: '/remote/Documents' } }; + + let localTreeBuilder: { run: ReturnType }; + let remoteTreeBuilder: { run: ReturnType }; + let container: Container; + + beforeEach(() => { + vi.clearAllMocks(); + + localTreeBuilder = { + run: vi.fn(), + }; + + remoteTreeBuilder = { + run: vi.fn(), + }; + + const get = vi.fn((token: unknown) => { + if (token === LocalTreeBuilder) return localTreeBuilder; + if (token === RemoteTreeBuilder) return remoteTreeBuilder; + return undefined; + }); + + container = { get } as unknown as Container; + }); + + it('returns total item count when precalculation succeeds', async () => { + localTreeBuilder.run.mockResolvedValue(right(localTree)); + remoteTreeBuilder.run.mockResolvedValue(remoteTree); + + vi.spyOn(DiffFilesCalculatorService, 'calculate').mockReturnValue({ total: 7 } as never); + vi.spyOn(FoldersDiffCalculator, 'calculate').mockReturnValue({ total: 3 } as never); + + const count = await precalculateBackupItemCount(backupInfo, container); + + expect(count).toBe(10); + expect(localTreeBuilder.run).toHaveBeenCalledWith(backupInfo.pathname); + expect(remoteTreeBuilder.run).toHaveBeenCalledWith(backupInfo.folderId, backupInfo.folderUuid); + expect(logger.debug).toHaveBeenCalledWith( + expect.objectContaining({ + tag: 'BACKUPS', + msg: 'Backup item count precalculated', + pathname: backupInfo.pathname, + count: 10, + }), + ); + }); + + it('returns 0 when local tree build returns left', async () => { + localTreeBuilder.run.mockResolvedValue(left(new Error('local tree error'))); + + const filesSpy = vi.spyOn(DiffFilesCalculatorService, 'calculate'); + const foldersSpy = vi.spyOn(FoldersDiffCalculator, 'calculate'); + + const count = await precalculateBackupItemCount(backupInfo, container); + + expect(count).toBe(0); + expect(remoteTreeBuilder.run).not.toHaveBeenCalled(); + expect(filesSpy).not.toHaveBeenCalled(); + expect(foldersSpy).not.toHaveBeenCalled(); + expect(logger.error).toHaveBeenCalledWith( + expect.objectContaining({ + tag: 'BACKUPS', + msg: 'Error building local tree during precalculation', + pathname: backupInfo.pathname, + }), + ); + }); + + it('returns 0 when an exception is thrown', async () => { + const runError = new Error('unexpected failure'); + localTreeBuilder.run.mockRejectedValue(runError); + + const count = await precalculateBackupItemCount(backupInfo, container); + + expect(count).toBe(0); + expect(logger.error).toHaveBeenCalledWith( + expect.objectContaining({ + tag: 'BACKUPS', + msg: 'Error during backup item precalculation', + pathname: backupInfo.pathname, + error: runError, + }), + ); + }); +}); diff --git a/src/backend/features/backup/precalculate-backup-item-count.ts b/src/backend/features/backup/precalculate-backup-item-count.ts new file mode 100644 index 000000000..5ac54cd89 --- /dev/null +++ b/src/backend/features/backup/precalculate-backup-item-count.ts @@ -0,0 +1,51 @@ +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { BackupInfo } from '../../../apps/backups/BackupInfo'; +import { DiffFilesCalculatorService } from '../../../apps/backups/diff/DiffFilesCalculatorService'; +import { FoldersDiffCalculator } from '../../../apps/backups/diff/FoldersDiffCalculator'; +import { AbsolutePath } from '../../../context/local/localFile/infrastructure/AbsolutePath'; +import LocalTreeBuilder from '../../../context/local/localTree/application/LocalTreeBuilder'; +import { RemoteTreeBuilder } from '../../../context/virtual-drive/remoteTree/application/RemoteTreeBuilder'; +import { Container } from 'diod'; + +export const precalculateBackupItemCount = async (backupInfo: BackupInfo, container: Container) => { + try { + const localTreeBuilder = container.get(LocalTreeBuilder); + const remoteTreeBuilder = container.get(RemoteTreeBuilder); + + const localTreeEither = await localTreeBuilder.run(backupInfo.pathname as AbsolutePath); + + if (localTreeEither.isLeft()) { + logger.error({ + tag: 'BACKUPS', + msg: 'Error building local tree during precalculation', + pathname: backupInfo.pathname, + }); + return 0; + } + + const local = localTreeEither.getRight(); + const remote = await remoteTreeBuilder.run(backupInfo.folderId, backupInfo.folderUuid); + + const filesDiff = DiffFilesCalculatorService.calculate(local, remote); + const foldersDiff = FoldersDiffCalculator.calculate(local, remote); + + const totalItems = filesDiff.total + foldersDiff.total; + + logger.debug({ + tag: 'BACKUPS', + msg: 'Backup item count precalculated', + pathname: backupInfo.pathname, + count: totalItems, + }); + + return totalItems; + } catch (error) { + logger.error({ + tag: 'BACKUPS', + msg: 'Error during backup item precalculation', + pathname: backupInfo.pathname, + error, + }); + return 0; + } +}; diff --git a/src/context/local/localFile/application/update/FileBatchUpdater.test.ts b/src/context/local/localFile/application/update/FileBatchUpdater.test.ts index 896ddf0ee..52b41f16a 100644 --- a/src/context/local/localFile/application/update/FileBatchUpdater.test.ts +++ b/src/context/local/localFile/application/update/FileBatchUpdater.test.ts @@ -2,7 +2,6 @@ import { FileBatchUpdater } from './FileBatchUpdater'; import { AbsolutePath } from '../../infrastructure/AbsolutePath'; import { right } from '../../../../shared/domain/Either'; import { SimpleFileOverrider } from '../../../../virtual-drive/files/application/override/SimpleFileOverrider'; -import { RemoteFileSystem } from '../../../../virtual-drive/files/domain/file-systems/RemoteFileSystem'; import { FileMother } from '../../../../virtual-drive/files/domain/__test-helpers__/FileMother'; import { RemoteTreeMother } from '../../../../virtual-drive/remoteTree/domain/__test-helpers__/RemoteTreeMother'; import { LocalFolderMother } from '../../../localFolder/domain/__test-helpers__/LocalFolderMother'; @@ -24,7 +23,7 @@ describe('File Batch Updater', () => { beforeAll(() => { uploader = new LocalFileUploaderMock(); - simpleFileOverrider = new SimpleFileOverrider({} as RemoteFileSystem); + simpleFileOverrider = new SimpleFileOverrider(); simpleFileOverriderSpy = vi.spyOn(simpleFileOverrider, 'run'); @@ -54,7 +53,7 @@ describe('File Batch Updater', () => { vi.spyOn(uploader, 'upload').mockReturnValue(Promise.resolve(right(mockContentsId))); simpleFileOverriderSpy.mockReturnValue(right(Promise.resolve())); - await SUT.run(localRoot, tree, localFiles, abortController.signal); + await SUT.run(localRoot, tree, localFiles, abortController.signal, vi.fn()); expect(simpleFileOverriderSpy).toBeCalledTimes(numberOfFilesToUpdate); }); diff --git a/src/context/local/localFile/application/update/FileBatchUpdater.ts b/src/context/local/localFile/application/update/FileBatchUpdater.ts index 89b7f8f9a..7a8ff89ac 100644 --- a/src/context/local/localFile/application/update/FileBatchUpdater.ts +++ b/src/context/local/localFile/application/update/FileBatchUpdater.ts @@ -18,6 +18,7 @@ export class FileBatchUpdater { remoteTree: RemoteTree, batch: Array, signal: AbortSignal, + onFileProcessed: () => void, ): Promise { for (const localFile of batch) { if (signal.aborted) { @@ -43,6 +44,7 @@ export class FileBatchUpdater { // eslint-disable-next-line no-await-in-loop await this.simpleFileOverrider.run(file, contentsId, localFile.size); + onFileProcessed(); } } } diff --git a/src/context/local/localFile/application/upload/FileBatchUploader.test.ts b/src/context/local/localFile/application/upload/FileBatchUploader.test.ts new file mode 100644 index 000000000..6ddd373a0 --- /dev/null +++ b/src/context/local/localFile/application/upload/FileBatchUploader.test.ts @@ -0,0 +1,170 @@ +import { describe, it, expect, vi, beforeAll, beforeEach } from 'vitest'; +import { FileBatchUploader } from './FileBatchUploader'; +import { LocalFileUploaderMock } from '../__mocks__/LocalFileUploaderMock'; +import { SimpleFileCreator } from '../../../../virtual-drive/files/application/create/SimpleFileCreator'; +import { RemoteFileSystem } from '../../../../virtual-drive/files/domain/file-systems/RemoteFileSystem'; +import { LocalFileMother } from '../../domain/__test-helpers__/LocalFileMother'; +import { RemoteTreeMother } from '../../../../virtual-drive/remoteTree/domain/__test-helpers__/RemoteTreeMother'; +import { FileMother } from '../../../../virtual-drive/files/domain/__test-helpers__/FileMother'; +import { AbsolutePath } from '../../infrastructure/AbsolutePath'; +import { left, right } from '../../../../shared/domain/Either'; +import { DriveDesktopError } from '../../../../shared/domain/errors/DriveDesktopError'; +import path from 'path'; + +vi.mock('../../../../../backend/features/backup', () => ({ + backupErrorsTracker: { add: vi.fn() }, +})); + +vi.mock('../../../../../infra/drive-server/services/files/services/delete-file-content-from-bucket', () => ({ + deleteFileFromStorageByFileId: vi.fn().mockResolvedValue(undefined), +})); + +const ROOT = '/local/backup' as AbsolutePath; +const BUCKET = 'test-bucket'; +const CONTENTS_ID = 'mock-contents-id'; + +describe('FileBatchUploader', () => { + let SUT: FileBatchUploader; + let uploader: LocalFileUploaderMock; + let creator: SimpleFileCreator; + let abortController: AbortController; + let onFileProcessed: ReturnType; + + beforeAll(() => { + uploader = new LocalFileUploaderMock(); + creator = new SimpleFileCreator({} as RemoteFileSystem); + SUT = new FileBatchUploader(uploader, creator, BUCKET); + }); + + beforeEach(() => { + vi.resetAllMocks(); + abortController = new AbortController(); + onFileProcessed = vi.fn(); + }); + + describe('successful upload', () => { + it('calls onFileProcessed once per file when all succeed', async () => { + const files = LocalFileMother.array(3, (i) => ({ + path: path.join(ROOT, `file-${i}.txt`) as AbsolutePath, + })); + + const tree = RemoteTreeMother.onlyRoot(); + + vi.spyOn(uploader, 'upload').mockResolvedValue(right(CONTENTS_ID)); + vi.spyOn(creator, 'run').mockImplementation(() => Promise.resolve(right(FileMother.any()))); + + await SUT.run(ROOT, tree, files, abortController.signal, onFileProcessed); + + expect(onFileProcessed).toHaveBeenCalledTimes(files.length); + }); + + it('adds the created file to the remote tree', async () => { + const file = LocalFileMother.fromPartial({ path: path.join(ROOT, 'new.txt') as AbsolutePath }); + const remoteFile = FileMother.fromPartial({ path: '/new.txt' }); + + const tree = RemoteTreeMother.onlyRoot(); + const addFileSpy = vi.spyOn(tree, 'addFile'); + + vi.spyOn(uploader, 'upload').mockResolvedValue(right(CONTENTS_ID)); + vi.spyOn(creator, 'run').mockResolvedValue(right(remoteFile)); + + await SUT.run(ROOT, tree, [file], abortController.signal, onFileProcessed); + + expect(addFileSpy).toHaveBeenCalledWith(tree.root, remoteFile); + }); + }); + + describe('upload errors', () => { + it('calls onFileProcessed and continues when upload throws', async () => { + const files = LocalFileMother.array(2, (i) => ({ + path: path.join(ROOT, `file-${i}.txt`) as AbsolutePath, + })); + + const tree = RemoteTreeMother.onlyRoot(); + + vi.spyOn(uploader, 'upload') + .mockRejectedValueOnce(new Error('network error')) + .mockResolvedValueOnce(right(CONTENTS_ID)); + vi.spyOn(creator, 'run').mockImplementation(() => Promise.resolve(right(FileMother.any()))); + + await SUT.run(ROOT, tree, files, abortController.signal, onFileProcessed); + + expect(onFileProcessed).toHaveBeenCalledTimes(2); + }); + + it('calls onFileProcessed and continues on non-fatal upload failure', async () => { + const file = LocalFileMother.fromPartial({ path: path.join(ROOT, 'fail.txt') as AbsolutePath }); + const tree = RemoteTreeMother.onlyRoot(); + + vi.spyOn(uploader, 'upload').mockResolvedValue(left(new DriveDesktopError('BAD_RESPONSE', 'Upload failed'))); + + await SUT.run(ROOT, tree, [file], abortController.signal, onFileProcessed); + + expect(onFileProcessed).toHaveBeenCalledTimes(1); + }); + + it('throws and does NOT call onFileProcessed on fatal upload failure', async () => { + const file = LocalFileMother.fromPartial({ path: path.join(ROOT, 'fatal.txt') as AbsolutePath }); + const tree = RemoteTreeMother.onlyRoot(); + + vi.spyOn(uploader, 'upload').mockResolvedValue(left(new DriveDesktopError('NO_INTERNET', 'No connection'))); + + await expect(SUT.run(ROOT, tree, [file], abortController.signal, onFileProcessed)).rejects.toThrow(); + expect(onFileProcessed).not.toHaveBeenCalled(); + }); + }); + + describe('file creation errors', () => { + it('calls onFileProcessed and continues on FILE_ALREADY_EXISTS', async () => { + const file = LocalFileMother.fromPartial({ path: path.join(ROOT, 'exists.txt') as AbsolutePath }); + const tree = RemoteTreeMother.onlyRoot(); + + vi.spyOn(uploader, 'upload').mockResolvedValue(right(CONTENTS_ID)); + vi.spyOn(creator, 'run').mockResolvedValue(left(new DriveDesktopError('FILE_ALREADY_EXISTS', 'Already exists'))); + + await SUT.run(ROOT, tree, [file], abortController.signal, onFileProcessed); + + expect(onFileProcessed).toHaveBeenCalledTimes(1); + }); + + it('calls onFileProcessed and continues on BAD_RESPONSE from creator', async () => { + const file = LocalFileMother.fromPartial({ path: path.join(ROOT, 'bad.txt') as AbsolutePath }); + const tree = RemoteTreeMother.onlyRoot(); + + vi.spyOn(uploader, 'upload').mockResolvedValue(right(CONTENTS_ID)); + vi.spyOn(creator, 'run').mockResolvedValue(left(new DriveDesktopError('BAD_RESPONSE', 'Server error'))); + + await SUT.run(ROOT, tree, [file], abortController.signal, onFileProcessed); + + expect(onFileProcessed).toHaveBeenCalledTimes(1); + }); + + it('throws and does NOT call onFileProcessed on unexpected creator error', async () => { + const file = LocalFileMother.fromPartial({ path: path.join(ROOT, 'unk.txt') as AbsolutePath }); + const tree = RemoteTreeMother.onlyRoot(); + + vi.spyOn(uploader, 'upload').mockResolvedValue(right(CONTENTS_ID)); + vi.spyOn(creator, 'run').mockResolvedValue(left(new DriveDesktopError('UNKNOWN', 'Unexpected'))); + + await expect(SUT.run(ROOT, tree, [file], abortController.signal, onFileProcessed)).rejects.toThrow(); + expect(onFileProcessed).not.toHaveBeenCalled(); + }); + }); + + describe('abort signal', () => { + it('stops processing when signal is already aborted', async () => { + const files = LocalFileMother.array(3, (i) => ({ + path: path.join(ROOT, `file-${i}.txt`) as AbsolutePath, + })); + const tree = RemoteTreeMother.onlyRoot(); + const uploadSpy = vi.spyOn(uploader, 'upload'); + + abortController.abort(); + + await SUT.run(ROOT, tree, files, abortController.signal, onFileProcessed); + + expect(uploadSpy).not.toHaveBeenCalled(); + expect(onFileProcessed).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/src/context/local/localFile/application/upload/FileBatchUploader.ts b/src/context/local/localFile/application/upload/FileBatchUploader.ts index 9d02325ab..e40678307 100644 --- a/src/context/local/localFile/application/upload/FileBatchUploader.ts +++ b/src/context/local/localFile/application/upload/FileBatchUploader.ts @@ -22,6 +22,7 @@ export class FileBatchUploader { remoteTree: RemoteTree, batch: Array, signal: AbortSignal, + onFileProcessed: () => void, ): Promise { for (const localFile of batch) { if (signal.aborted) { @@ -37,6 +38,7 @@ export class FileBatchUploader { uploadEither = await this.localHandler.upload(localFile.path, localFile.size, signal); } catch (error) { logger.error({ msg: '[UPLOAD ERROR]', error }); + onFileProcessed(); continue; } @@ -48,6 +50,7 @@ export class FileBatchUploader { throw error; } backupErrorsTracker.add(parent.id, { name: localFile.nameWithExtension(), error: error.cause }); + onFileProcessed(); continue; } @@ -69,11 +72,13 @@ export class FileBatchUploader { logger.debug({ msg: `[FILE ALREADY EXISTS] Skipping file ${localFile.path} - already exists remotely`, }); + onFileProcessed(); continue; } if (error.cause === 'BAD_RESPONSE') { backupErrorsTracker.add(parent.id, { name: localFile.nameWithExtension(), error: error.cause }); + onFileProcessed(); continue; } @@ -83,6 +88,7 @@ export class FileBatchUploader { const file = either.getRight(); remoteTree.addFile(parent, file); + onFileProcessed(); } } } From 0a581de59fd40efd3afa549d08d46d7244ff0a63 Mon Sep 17 00:00:00 2001 From: Esteban Galvis Date: Fri, 20 Mar 2026 16:45:43 -0500 Subject: [PATCH 2/2] fix(backup): standardize comment formatting in weighted progress tests --- .../features/backup/backup-progress-tracker.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/features/backup/backup-progress-tracker.test.ts b/src/backend/features/backup/backup-progress-tracker.test.ts index e1eeab3e9..a769b17b3 100644 --- a/src/backend/features/backup/backup-progress-tracker.test.ts +++ b/src/backend/features/backup/backup-progress-tracker.test.ts @@ -107,8 +107,8 @@ describe('BackupProgressTracker - Functional approach', () => { it('should calculate weighted progress for multiple backups', () => { const backupIds = ['backup-a', 'backup-b']; const fileCounts = new Map([ - ['backup-a', 100], // 66.7% weight - ['backup-b', 50], // 33.3% weight + ['backup-a', 100], // 66.7% weight + ['backup-b', 50], // 33.3% weight ]); tracker.initializeWeights(backupIds, fileCounts); @@ -124,8 +124,8 @@ describe('BackupProgressTracker - Functional approach', () => { it('should accumulate progress correctly across backups', () => { const backupIds = ['backup-a', 'backup-b']; const fileCounts = new Map([ - ['backup-a', 100], // 66.7% weight - ['backup-b', 50], // 33.3% weight + ['backup-a', 100], // 66.7% weight + ['backup-b', 50], // 33.3% weight ]); tracker.initializeWeights(backupIds, fileCounts);