diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index b2aaa11..72764b4 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -15,8 +15,8 @@ - - + + diff --git a/src/Sa.HybridFileStorage.FileSystem/FileRetryHelper.cs b/src/Sa.HybridFileStorage.FileSystem/FileRetryHelper.cs new file mode 100644 index 0000000..043d3f3 --- /dev/null +++ b/src/Sa.HybridFileStorage.FileSystem/FileRetryHelper.cs @@ -0,0 +1,31 @@ +namespace Sa.HybridFileStorage.FileSystem; + +internal static class FileRetryHelper +{ + public static async Task RetryAsync( + Action action, + int maxRetries = 3, + int baseDelayMs = 100, + CancellationToken cancellationToken = default) + { + Exception? lastException = null; + + for (int attempt = 0; attempt < maxRetries; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + action(); + return; + } + catch (IOException ex) when (attempt < maxRetries - 1) + { + lastException = ex; + await Task.Delay(baseDelayMs * (int)Math.Pow(2, attempt), cancellationToken); + } + } + + throw lastException ?? new InvalidOperationException("Retry failed"); + } +} diff --git a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs index 276d33f..594fe72 100644 --- a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs +++ b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs @@ -1,68 +1,150 @@ using Sa.HybridFileStorage.Domain; +using System.Security; namespace Sa.HybridFileStorage.FileSystem; -internal sealed class FileSystemStorage(FileSystemStorageOptions options, TimeProvider? timeProvider = null) : IFileStorage +internal sealed class FileSystemStorage( + FileSystemStorageOptions options, + TimeProvider? timeProvider = null) : IFileStorage { - private readonly string _basePath = Path.TrimEndingDirectorySeparator(options.BasePath); + + private readonly string _basePath = Path.TrimEndingDirectorySeparator( + Path.GetFullPath(options.BasePath ?? throw new ArgumentNullException(nameof(options.BasePath)))); + + private readonly TimeProvider _timeProvider = timeProvider ?? TimeProvider.System; + + public string? ScopeName => options.ScopeName; public string StorageType { get; } = options.StorageType ?? "file"; - public bool IsReadOnly { get; } = options.IsReadOnly ?? false; + public bool IsReadOnly { get; } = options.IsReadOnly; private void EnsureWritable() { if (IsReadOnly) { - throw new InvalidOperationException("Cannot perform this operation. The storage is read-only."); + throw new HybridFileStorageWritableException(); } } - public async Task UploadAsync(UploadFileInput metadata, Stream fileStream, CancellationToken cancellationToken) + public async Task UploadAsync( + UploadFileInput metadata, + Stream fileStream, + CancellationToken cancellationToken) { + ArgumentNullException.ThrowIfNull(metadata); + ArgumentException.ThrowIfNullOrWhiteSpace(metadata.FileName); + ArgumentNullException.ThrowIfNull(fileStream); EnsureWritable(); - string filePath = $"{_basePath}/{metadata.TenantId}/{metadata.FileName.Replace('\\', '/')}"; + string relativePath = PathSanitizer.SanitizeRelativePath(metadata.FileName); - string? dir = Path.GetDirectoryName(filePath); - EnsureDirectory(dir); + string filePath = Path.Combine(_basePath, metadata.TenantId.ToString(), relativePath); + + EnsurePathWithinBase(filePath); + + string? directory = Path.GetDirectoryName(filePath); + EnsureDirectory(directory); + + await using var fileStreamOutput = new FileStream(filePath, new FileStreamOptions + { + Mode = FileMode.Create, + Access = FileAccess.Write, + Share = FileShare.None, + BufferSize = 4096, + Options = FileOptions.Asynchronous | FileOptions.SequentialScan, + PreallocationSize = 10 * 1024 * 1024 + }); - using var fileStreamOutput = new FileStream(filePath, FileMode.Create, FileAccess.Write); await fileStream.CopyToAsync(fileStreamOutput, cancellationToken); - var fileId = FilePathToId(filePath); - var fileAbsolute = Path.GetFullPath(filePath); + var absolutePath = Path.GetFullPath(filePath); - return new StorageResult(fileId, fileAbsolute, StorageType, timeProvider?.GetUtcNow() ?? TimeProvider.System.GetUtcNow()); + return new StorageResult( + FilePathToId(filePath), + absolutePath, + StorageType, + _timeProvider.GetUtcNow()); } - public async Task DownloadAsync(string fileId, Func loadStream, CancellationToken cancellationToken) + + public async Task DownloadAsync( + string fileId, + Func loadStream, + CancellationToken cancellationToken) { + ArgumentNullException.ThrowIfNull(fileId); + ArgumentNullException.ThrowIfNull(loadStream); + + var filePath = FileIdToPath(fileId); - if (File.Exists(filePath)) + + EnsurePathWithinBase(filePath); + + + try { - using var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read); + await using var fs = new FileStream(filePath, new FileStreamOptions + { + Mode = FileMode.Open, + Access = FileAccess.Read, + Share = FileShare.Read, + BufferSize = 81_920, + Options = FileOptions.Asynchronous | FileOptions.SequentialScan, + }); + await loadStream(fs, cancellationToken); + return true; + + } + catch (FileNotFoundException) + { + return false; + } + catch (DirectoryNotFoundException) + { + return false; } - return false; } - public Task DeleteAsync(string fileId, CancellationToken cancellationToken) + public async Task DeleteAsync(string fileId, CancellationToken cancellationToken) { + ArgumentNullException.ThrowIfNull(fileId); EnsureWritable(); var filePath = FileIdToPath(fileId); - if (File.Exists(filePath)) + EnsurePathWithinBase(filePath); + + if (!File.Exists(filePath)) { - File.Delete(filePath); - return Task.FromResult(true); + return false; + } + + try + { + await FileRetryHelper.RetryAsync( + () => File.Delete(filePath), + cancellationToken: cancellationToken); + + return true; + } + catch (IOException) + { + return false; } - return Task.FromResult(false); } public bool CanProcess(string fileId) - => fileId.StartsWith(FilePathToId(_basePath)); + { + if (string.IsNullOrWhiteSpace(fileId)) + { + return false; + } + + var expectedPrefix = $"{StorageType}://"; + return fileId.StartsWith(expectedPrefix, StringComparison.Ordinal); + } private static void EnsureDirectory(string? dir) { @@ -93,4 +175,28 @@ private static string FileIdToPath(string fileId) return filePath.ToString(); } + + private void EnsurePathWithinBase(string path) + { + var fullPath = Path.GetFullPath(path); + + if (!fullPath.StartsWith(_basePath, StringComparison.Ordinal)) + { + throw new SecurityException($""" + Access denied. Path '{fullPath}' is outside the allowed base directory '{_basePath}'. + """); + } + + // Edge-case защита: basePath="C:/Data", path="C:/DataOther" + if (fullPath.Length > _basePath.Length) + { + var nextChar = fullPath[_basePath.Length]; + if (nextChar != Path.DirectorySeparatorChar && nextChar != Path.AltDirectorySeparatorChar) + { + throw new SecurityException($""" + Access denied. Path '{fullPath}' is outside the allowed base directory. + """); + } + } + } } diff --git a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageOptions.cs b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageOptions.cs index 87fc232..8a231e6 100644 --- a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageOptions.cs +++ b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageOptions.cs @@ -1,8 +1,9 @@ namespace Sa.HybridFileStorage.FileSystem; -public sealed class FileSystemStorageOptions +public sealed record FileSystemStorageOptions { - public string StorageType { get; set; } = "file"; - public required string BasePath { get; set; } - public bool? IsReadOnly { get; set; } + public string StorageType { get; init; } = "file"; + public required string BasePath { get; init; } + public bool IsReadOnly { get; init; } = false; + public string? ScopeName { get; init; } = null; } diff --git a/src/Sa.HybridFileStorage.FileSystem/PathSanitizer.cs b/src/Sa.HybridFileStorage.FileSystem/PathSanitizer.cs new file mode 100644 index 0000000..11731b8 --- /dev/null +++ b/src/Sa.HybridFileStorage.FileSystem/PathSanitizer.cs @@ -0,0 +1,141 @@ +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Security; +using System.Text; + +namespace Sa.HybridFileStorage.FileSystem; + +internal static class PathSanitizer +{ + private static readonly SearchValues s_separators = SearchValues.Create(['/', '\\']); + private static readonly SearchValues s_invalidChars = SearchValues.Create(Path.GetInvalidFileNameChars()); + + public static string SanitizeRelativePath(string relativePath) + { + ArgumentNullException.ThrowIfNull(relativePath); + + if (relativePath.Length == 0) + { + throw new ArgumentException("File name cannot be empty.", nameof(relativePath)); + } + + if (IsSimplePath(relativePath)) + { + return SanitizeSimplePath(relativePath); + } + + return SanitizeComplexPath(relativePath); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsSimplePath(ReadOnlySpan path) + { + foreach (var c in path) + { + if (c is '/' or '\\' or '.' or '<' or '>' or ':' or '"' or '|' or '?' or '*') + { + return false; + } + } + return true; + } + + private static string SanitizeSimplePath(string path) + => path.Replace('/', Path.DirectorySeparatorChar).Replace('\\', Path.DirectorySeparatorChar); + + private static string SanitizeComplexPath(string relativePath) + { + ReadOnlySpan span = relativePath.AsSpan(); + + var estimatedCapacity = span.Length + 16; + var builder = new StringBuilder(estimatedCapacity); + + var currentPartStart = 0; + var hasContent = false; + + for (var i = 0; i <= span.Length; i++) + { + if (i == span.Length || IsSeparator(span[i])) + { + if (i > currentPartStart) + { + var part = span[currentPartStart..i]; + var sanitized = SanitizePathPart(part); + + if (sanitized.Length > 0) + { + if (hasContent) + { + builder.Append(Path.DirectorySeparatorChar); + } + builder.Append(sanitized); + hasContent = true; + } + } + + currentPartStart = i + 1; + } + } + + if (!hasContent) + { + throw new ArgumentException(""" +Resulting path is empty after sanitization. +Ensure the filename contains valid characters. +""", nameof(relativePath)); + } + + return builder.ToString(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsSeparator(char c) => s_separators.Contains(c); + + + private static string SanitizePathPart(ReadOnlySpan part) + { + + var trimmed = part.Trim(); + + if (trimmed.Length == 0) + { + return string.Empty; + } + + if (trimmed.Equals("..", StringComparison.Ordinal) || + trimmed.Equals(".", StringComparison.Ordinal)) + { + throw new SecurityException($""" +Path segment '{trimmed}' is not allowed. +Relative paths (.. / .) are prohibited for security reasons. +"""); + } + + if (!ContainsInvalidChars(trimmed)) + { + return trimmed.ToString(); + } + + return ReplaceInvalidChars(trimmed); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool ContainsInvalidChars(ReadOnlySpan span) + => span.IndexOfAny(s_invalidChars) >= 0; + + + private static string ReplaceInvalidChars(ReadOnlySpan span) + { + Span buffer = span.Length <= 256 + ? stackalloc char[span.Length] + : new char[span.Length]; + + for (var i = 0; i < span.Length; i++) + { + var c = span[i]; + buffer[i] = s_invalidChars.Contains(c) ? '_' : c; + } + + return new string(buffer); + } +} diff --git a/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj b/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj index a4aba65..d77f3da 100644 --- a/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj +++ b/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj @@ -3,7 +3,7 @@ - 0.8.0 + 0.8.1 File storage management diff --git a/src/Sa.HybridFileStorage.FileSystem/Setup.cs b/src/Sa.HybridFileStorage.FileSystem/Setup.cs index bed5fd9..397188e 100644 --- a/src/Sa.HybridFileStorage.FileSystem/Setup.cs +++ b/src/Sa.HybridFileStorage.FileSystem/Setup.cs @@ -1,5 +1,4 @@ using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; using Sa.HybridFileStorage.Domain; namespace Sa.HybridFileStorage.FileSystem; @@ -7,12 +6,11 @@ namespace Sa.HybridFileStorage.FileSystem; public static class Setup { - public static IServiceCollection AddSaFileSystemFileStorage(this IServiceCollection services, FileSystemStorageOptions options) + public static IServiceCollection AddSaFileSystemFileStorage( + this IServiceCollection services, + FileSystemStorageOptions options) { - services.TryAddSingleton(TimeProvider.System); - services.TryAddKeyedSingleton(options); - services.TryAddKeyedSingleton(options, (sp, o) => new FileSystemStorage(options, sp.GetService())); - services.AddSingleton(sp => sp.GetRequiredKeyedService(options)); + services.AddSingleton(sp => new FileSystemStorage(options, sp.GetService())); return services; } diff --git a/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs b/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs index d486820..f8a61dd 100644 --- a/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs +++ b/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs @@ -39,7 +39,8 @@ public static (int tenantId, long timestamp) ParseFromFileId(string fileId, stri ReadOnlySpan dateSpan = subParts.Slice(firstSlashIndex + 1, DateFormat.Length); - if (!DateTimeOffset.TryParseExact(dateSpan, DateFormat, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset date)) + if (!DateTimeOffset.TryParseExact( + dateSpan, DateFormat, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset date)) { throw new FormatException("Invalid timestamp format in file ID."); } @@ -49,8 +50,13 @@ public static (int tenantId, long timestamp) ParseFromFileId(string fileId, stri } - public static string FormatToFileId(string storageType, string tableName, int tenantId, DateTimeOffset date, string fileName) - => $"{storageType}://{tableName}/{tenantId}/{date.ToString(DateFormat, CultureInfo.InvariantCulture)}/{NormalizeFileName(fileName)}"; + public static string FormatToFileId( + string storageType, + string tableName, + int tenantId, + DateTimeOffset date, + string fileName) + => $"{storageType}://{tableName}/{tenantId}/{date.ToString(DateFormat, CultureInfo.InvariantCulture)}/{NormalizeFileName(fileName)}"; public static string NormalizeFileName(string fileName) => fileName.TrimStart('\\', '/').Replace('\\', '/'); diff --git a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs index bdf74dc..2e99ee9 100644 --- a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs +++ b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs @@ -11,32 +11,44 @@ internal sealed class PostgresFileStorage( IPartitionManager partManager, RecyclableMemoryStreamManager streamManager, StorageOptions options, + string? scopeName, TimeProvider? timeProvider = null) : IFileStorage { - private readonly string _qualifiedTableName = $"{options.SchemaName}.\"{options.TableName}\""; + private readonly string _partName = Sanitize(scopeName ?? "root"); - public string StorageType { get; } = options.StorageType; + private readonly string _qualifiedTableName = $"{options.SchemaName}.\"{Sanitize(options.TableName)}\""; - public bool IsReadOnly { get; } = options.IsReadOnly ?? false; + private readonly TimeProvider _timeProvider = timeProvider ?? TimeProvider.System; + + public string StorageType => options.StorageType; + + public bool IsReadOnly => options.IsReadOnly; + + public string? ScopeName => scopeName; private void EnsureWritable() { if (IsReadOnly) { - throw new InvalidOperationException("Cannot perform this operation. The storage is read-only."); + throw new HybridFileStorageWritableException(); } } - public async Task UploadAsync(UploadFileInput metadata, Stream fileStream, CancellationToken cancellationToken) + public async Task UploadAsync( + UploadFileInput metadata, + Stream fileStream, + CancellationToken cancellationToken) { EnsureWritable(); - var now = timeProvider?.GetUtcNow() ?? TimeProvider.System.GetUtcNow(); + var now = _timeProvider.GetUtcNow(); + + await partManager.EnsureParts(_qualifiedTableName, now, [metadata.TenantId, _partName], cancellationToken); - await partManager.EnsureParts(_qualifiedTableName, now, [metadata.TenantId], cancellationToken); + string fileId = FileIdParser.FormatToFileId( + StorageType, options.TableName, metadata.TenantId, now, metadata.FileName); - string fileId = FileIdParser.FormatToFileId(StorageType, options.TableName, metadata.TenantId, now, metadata.FileName); string fileExtension = FileIdParser.GetFileExtension(metadata.FileName); long createdAt = now.ToUnixTimeSeconds(); @@ -56,8 +68,8 @@ public async Task UploadAsync(UploadFileInput metadata, Stream fi } await dataSource.ExecuteNonQuery($""" -INSERT INTO {_qualifiedTableName} (id, name, file_ext, data, size, tenant_id, created_at) -VALUES (@id, @name, @file_ext, @data, @size, @tenant_id, @created_at) +INSERT INTO {_qualifiedTableName} (id, name, file_ext, data, size, tenant_id, scope_name, created_at) +VALUES (@id, @name, @file_ext, @data, @size, @tenant_id, @scope_name, @created_at) ON CONFLICT DO NOTHING """ , @@ -68,6 +80,7 @@ ON CONFLICT DO NOTHING , new NpgsqlParameter("data", fileStream) , new NpgsqlParameter("size", (int)memoryStream.Length) , new NpgsqlParameter("tenant_id", metadata.TenantId) + , new NpgsqlParameter("scope_name", _partName) , new NpgsqlParameter("created_at", createdAt) ] , cancellationToken); @@ -90,13 +103,17 @@ public async Task DeleteAsync(string fileId, CancellationToken cancellatio EnsureWritable(); (int tenantId, long timestamp) = FileIdParser.ParseFromFileId(fileId, options.TableName); - int rowsAffected = await dataSource.ExecuteNonQuery( - $""" - DELETE FROM {_qualifiedTableName} WHERE tenant_id = @tenant_id AND created_at >= @timestamp AND id = @id + int rowsAffected = await dataSource.ExecuteNonQuery($""" + DELETE FROM {_qualifiedTableName} + WHERE + tenant_id = @tenant_id AND scope_name = @scope_name + AND created_at >= @timestamp + AND id = @id """ , [ - new NpgsqlParameter("tenant_id", tenantId), + new NpgsqlParameter("tenant_id", tenantId), + new NpgsqlParameter("scope_name", _partName), new NpgsqlParameter("timestamp", timestamp), new NpgsqlParameter("id", fileId) ] @@ -104,13 +121,20 @@ public async Task DeleteAsync(string fileId, CancellationToken cancellatio return rowsAffected > 0; } - public async Task DownloadAsync(string fileId, Func loadStream, CancellationToken cancellationToken) + public async Task DownloadAsync( + string fileId, + Func loadStream, + CancellationToken cancellationToken) { (int tenantId, long timestamp) = FileIdParser.ParseFromFileId(fileId, options.TableName); int rowsAffected = await dataSource.ExecuteReader( $""" - SELECT data FROM {_qualifiedTableName} WHERE tenant_id = @tenant_id AND created_at >= @timestamp AND id = @id + SELECT data FROM {_qualifiedTableName} + WHERE + tenant_id = @tenant_id AND scope_name = @scope_name + AND created_at >= @timestamp + AND id = @id """ , async (reader, i) => { @@ -119,7 +143,8 @@ public async Task DownloadAsync(string fileId, Func("tenant_id", tenantId), + new NpgsqlParameter("tenant_id", tenantId), + new NpgsqlParameter("scope_name", _partName), new NpgsqlParameter("timestamp", timestamp), new NpgsqlParameter("id", fileId) ] @@ -128,4 +153,18 @@ public async Task DownloadAsync(string fileId, Func 0; } + + + private static string Sanitize(string tableName) + { + var result = new char[tableName.Length]; + + for (int i = 0; i < tableName.Length; i++) + { + char c = tableName[i]; + result[i] = char.IsLetter(c) || (i > 0 && char.IsDigit(c)) || c == '_' ? c : '_'; + } + + return new string(result); + } } diff --git a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageConfiguration.cs b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageConfiguration.cs index 811f5b2..d714666 100644 --- a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageConfiguration.cs +++ b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageConfiguration.cs @@ -15,7 +15,6 @@ internal sealed class PostgresFileStorageConfiguration : IPostgresFileStorageCon public PostgresFileStorageConfiguration(IServiceCollection services) { - services.TryAddSingleton(TimeProvider.System); services.TryAddSingleton(); _partConfiguration = services.AddSaPartitional((sp, builder) => @@ -28,9 +27,10 @@ public PostgresFileStorageConfiguration(IServiceCollection services) "size INT NOT NULL", "file_ext TEXT NOT NULL", "tenant_id INT NOT NULL", + "scope_name TEXT NOT NULL", "data BYTEA NOT NULL" ) - .PartByList("tenant_id") + .PartByList("tenant_id", "scope_name") .PartByRange(_options.PartOptions.PgPartBy, "created_at"); }); }) @@ -62,7 +62,7 @@ public PostgresFileStorageConfiguration(IServiceCollection services) TableName = _options.StorageOptions.TableName.Trim('"') }; - var storage = new PostgresFileStorage(dataSource, pm, sm, options, time); + var storage = new PostgresFileStorage(dataSource, pm, sm, options, _options.ScopeName, time); return storage; }); } diff --git a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs index d89f938..185602b 100644 --- a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs +++ b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs @@ -2,28 +2,29 @@ namespace Sa.HybridFileStorage.Postgres; -public record StorageOptions +public sealed record StorageOptions { public string SchemaName { get; set; } = "public"; public string TableName { get; set; } = "files"; public string StorageType { get; set; } = "pg"; - public bool? IsReadOnly { get; set; } + public bool IsReadOnly { get; set; } } -public class CleanupOptions +public sealed class CleanupOptions { public int ExpireDays { get; set; } = 365 * 3; } -public class PartOptions +public sealed class PartOptions { public int MigrationScheduleForwardDays { get; set; } = 2; public PgPartBy PgPartBy { get; set; } = PgPartBy.Day; } -public class PostgresFileStorageOptions +public sealed class PostgresFileStorageOptions { public StorageOptions StorageOptions { get; } = new(); public PartOptions PartOptions { get; } = new(); public CleanupOptions CleanupOptions { get; } = new(); + public string? ScopeName { get; init; } = null; } diff --git a/src/Sa.HybridFileStorage.Postgres/Sa.HybridFileStorage.Postgres.csproj b/src/Sa.HybridFileStorage.Postgres/Sa.HybridFileStorage.Postgres.csproj index 0ecd96a..08931fa 100644 --- a/src/Sa.HybridFileStorage.Postgres/Sa.HybridFileStorage.Postgres.csproj +++ b/src/Sa.HybridFileStorage.Postgres/Sa.HybridFileStorage.Postgres.csproj @@ -3,7 +3,7 @@ - 0.8.0 + 0.8.1 File storage management in Pg diff --git a/src/Sa.HybridFileStorage.Postgres/Setup.cs b/src/Sa.HybridFileStorage.Postgres/Setup.cs index 380e55f..e441bbd 100644 --- a/src/Sa.HybridFileStorage.Postgres/Setup.cs +++ b/src/Sa.HybridFileStorage.Postgres/Setup.cs @@ -5,9 +5,11 @@ namespace Sa.HybridFileStorage.Postgres; public static class Setup { - public static IServiceCollection AddSaPostgreSqlFileStorage(this IServiceCollection services, Action? configure = null) + public static IServiceCollection AddSaPostgreSqlFileStorage( + this IServiceCollection services, + Action? configure = null) { - var configurator = new PostgresFileStorageConfiguration(services); + PostgresFileStorageConfiguration configurator = new(services); configure?.Invoke(configurator); return services; } diff --git a/src/Sa.HybridFileStorage.S3/S3FileStorage.cs b/src/Sa.HybridFileStorage.S3/S3FileStorage.cs index a009475..245667f 100644 --- a/src/Sa.HybridFileStorage.S3/S3FileStorage.cs +++ b/src/Sa.HybridFileStorage.S3/S3FileStorage.cs @@ -5,19 +5,26 @@ namespace Sa.HybridFileStorage.S3; -internal sealed class S3FileStorage(IS3BucketClient client, S3FileStorageOptions options, TimeProvider? timeProvider = null) : IFileStorage +internal sealed class S3FileStorage( + IS3BucketClient client, + S3FileStorageOptions options, + TimeProvider? timeProvider = null) : IFileStorage { private const string DateFormat = "yyyy/MM/dd/HH"; - public string StorageType { get; } = options.StorageType; + private readonly TimeProvider _timeProvider = timeProvider ?? TimeProvider.System; - public bool IsReadOnly { get; } = options.IsReadOnly ?? false; + public string StorageType => options.StorageType; + + public bool IsReadOnly => options.IsReadOnly; + + public string? ScopeName => options.ScopeName; private void EnsureWritable() { if (IsReadOnly) { - throw new InvalidOperationException("Cannot perform this operation. The storage is read-only."); + throw new HybridFileStorageWritableException(); } } @@ -49,7 +56,7 @@ public async Task UploadAsync(UploadFileInput metadata, Stream fi EnsureWritable(); await EnsureBucket(cancellationToken); - var now = timeProvider?.GetUtcNow() ?? TimeProvider.System.GetUtcNow(); + var now = _timeProvider.GetUtcNow(); var eventTime = now.ToString(DateFormat, CultureInfo.InvariantCulture); var filePath = $"{metadata.TenantId}/{eventTime}/{metadata.FileName}"; @@ -87,7 +94,7 @@ private static string FileIdToPath(string fileId) throw new FormatException("Invalid file ID format."); } - ReadOnlySpan filePath = span[(separatorIndex + 3)..]; // +3 for skip "://" + ReadOnlySpan filePath = span[(separatorIndex + "://".Length)..]; // +3 for skip "://" return filePath.ToString(); } diff --git a/src/Sa.HybridFileStorage.S3/S3FileStorageOptions.cs b/src/Sa.HybridFileStorage.S3/S3FileStorageOptions.cs index f00c7a6..3a66425 100644 --- a/src/Sa.HybridFileStorage.S3/S3FileStorageOptions.cs +++ b/src/Sa.HybridFileStorage.S3/S3FileStorageOptions.cs @@ -1,17 +1,18 @@ namespace Sa.HybridFileStorage.S3; -public class S3FileStorageOptions +public sealed class S3FileStorageOptions { - public string StorageType { get; set; } = "s3"; + public string StorageType { get; init; } = "s3"; + public string? ScopeName { get; init; } = null; /// /// http://localhost:9000 /// - public required string Endpoint { get; set; } - public required string AccessKey { get; set; } - public required string SecretKey { get; set; } - public required string Bucket { get; set; } - public string? Region { get; set; } + public required string Endpoint { get; init; } + public required string AccessKey { get; init; } + public required string SecretKey { get; init; } + public required string Bucket { get; init; } + public string Region { get; init; } = "eu-central-1"; - public bool? IsReadOnly { get; set; } + public bool IsReadOnly { get; init; } = false; } diff --git a/src/Sa.HybridFileStorage.S3/Sa.HybridFileStorage.S3.csproj b/src/Sa.HybridFileStorage.S3/Sa.HybridFileStorage.S3.csproj index 7184cea..51e9b78 100644 --- a/src/Sa.HybridFileStorage.S3/Sa.HybridFileStorage.S3.csproj +++ b/src/Sa.HybridFileStorage.S3/Sa.HybridFileStorage.S3.csproj @@ -3,7 +3,7 @@ - 0.8.0 + 0.8.1 File storage management in S3 diff --git a/src/Sa.HybridFileStorage.S3/Setup.cs b/src/Sa.HybridFileStorage.S3/Setup.cs index 6dd740c..fbce651 100644 --- a/src/Sa.HybridFileStorage.S3/Setup.cs +++ b/src/Sa.HybridFileStorage.S3/Setup.cs @@ -1,5 +1,4 @@ using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; using Sa.Data.S3; using Sa.HybridFileStorage.Domain; @@ -18,14 +17,10 @@ public static IServiceCollection AddSaS3FileStorage(this IServiceCollection serv Region = options.Region ?? "eu-central-1", }; - - services.TryAddSingleton(TimeProvider.System); - services.AddSaS3BucketClient(settings); - - services.TryAddSingleton(options); - services.AddSingleton(); + services.AddSingleton(sp + => new S3FileStorage(sp.GetRequiredService(), options, sp.GetService())); return services; } diff --git a/src/Sa.HybridFileStorage/Domain/IFileStorage.cs b/src/Sa.HybridFileStorage/Domain/IFileStorage.cs index 736bd11..7ebbfa5 100644 --- a/src/Sa.HybridFileStorage/Domain/IFileStorage.cs +++ b/src/Sa.HybridFileStorage/Domain/IFileStorage.cs @@ -5,6 +5,11 @@ /// public interface IFileStorage { + /// + /// scope domain + /// + string? ScopeName { get; } + /// /// Gets the type of the storage. /// @@ -37,7 +42,10 @@ public interface IFileStorage /// File ID. /// Cancellation token. /// The file stream. - Task DownloadAsync(string fileId, Func loadStream, CancellationToken cancellationToken); + Task DownloadAsync( + string fileId, + Func loadStream, + CancellationToken cancellationToken); /// /// Deletes a file from the storage by its ID. diff --git a/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs b/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs index be5b960..8b5f536 100644 --- a/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs +++ b/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs @@ -2,6 +2,6 @@ public sealed record UploadFileInput { - public int TenantId { get; set; } - public string FileName { get; set; } = string.Empty; + public int TenantId { get; init; } + public string FileName { get; init; } = string.Empty; } diff --git a/src/Sa.HybridFileStorage/Exceptions.cs b/src/Sa.HybridFileStorage/Exceptions.cs new file mode 100644 index 0000000..e3dee87 --- /dev/null +++ b/src/Sa.HybridFileStorage/Exceptions.cs @@ -0,0 +1,11 @@ +namespace Sa.HybridFileStorage; + +public class HybridFileStorageNoAvailableException() : Exception("No storage available."); + + +public class HybridFileStorageAggregateException(IEnumerable innerExceptions) + : AggregateException("Operation failed for some available storages.", innerExceptions); + + +public class HybridFileStorageWritableException() + : Exception("Cannot perform operation. All storage options are read-only."); diff --git a/src/Sa.HybridFileStorage/FileStorageExtensions.cs b/src/Sa.HybridFileStorage/FileStorageExtensions.cs new file mode 100644 index 0000000..2dcc28c --- /dev/null +++ b/src/Sa.HybridFileStorage/FileStorageExtensions.cs @@ -0,0 +1,12 @@ +using Sa.HybridFileStorage.Domain; + +namespace Sa.HybridFileStorage; + +internal static class FileStorageExtensions +{ + public static IEnumerable GetScopeStorages( + this IReadOnlyCollection storages, + string fileId, + string? scopeName) + => storages.Where(c => c.ScopeName == scopeName && c.CanProcess(fileId)); +} diff --git a/src/Sa.HybridFileStorage/HybridFileStorage.cs b/src/Sa.HybridFileStorage/HybridFileStorage.cs index 675717f..c596e7d 100644 --- a/src/Sa.HybridFileStorage/HybridFileStorage.cs +++ b/src/Sa.HybridFileStorage/HybridFileStorage.cs @@ -4,25 +4,35 @@ namespace Sa.HybridFileStorage; -internal sealed class HybridFileStorage(IHybridFileStorageContainer container, InterceptorContainer interceptors) : IHybridFileStorage +internal sealed class HybridFileStorage( + IHybridFileStorageContainer container, + InterceptorContainer interceptors) : IHybridFileStorage { - public string StorageType => container.StorageType; - public bool IsReadOnly => container.IsReadOnly; + public IReadOnlyCollection Storages => container.Storages; - public bool CanProcess(string fileId) => container.CanProcess(fileId); - - private void EnsureWritable() + private void EnsureWritable(string? scopeName) { - if (IsReadOnly) + if (!container.Storages.Any(c => c.ScopeName == scopeName)) { - throw new InvalidOperationException("Cannot perform operation. All storage options are read-only."); + throw new HybridFileStorageNoAvailableException(); + } + + + if (Storages.All(f => f.ScopeName == scopeName && f.IsReadOnly)) + { + throw new HybridFileStorageWritableException(); } } - public async Task UploadAsync(UploadFileInput input, Stream fileStream, CancellationToken cancellationToken) + public async Task UploadAsync( + UploadFileInput input, + string? scopeName, + Stream fileStream, + CancellationToken cancellationToken = default) { - EnsureWritable(); + EnsureWritable(scopeName); + return await ExecuteStorageOperationAsync( container.Storages.Where(c => !c.IsReadOnly), async (storage, ct) => await interceptors.ExecuteBeforeUploadAsync(storage, input, fileStream, ct), @@ -33,10 +43,14 @@ public async Task UploadAsync(UploadFileInput input, Stream fileS ); } - public async Task DownloadAsync(string fileId, Func loadStream, CancellationToken cancellationToken) + public async Task DownloadAsync( + string fileId, + string? scopeName, + Func loadStream, + CancellationToken cancellationToken = default) { return await ExecuteStorageOperationAsync( - container.GetStorages(fileId), + container.Storages.GetScopeStorages(fileId, scopeName), async (storage, ct) => await interceptors.ExecuteBeforeDownloadAsync(storage, fileId, loadStream, ct), async (storage, ct) => await storage.DownloadAsync(fileId, loadStream, ct), async (storage, result, ct) => await interceptors.ExecuteAfterDownloadAsync(storage, fileId, result, ct), @@ -45,11 +59,12 @@ public async Task DownloadAsync(string fileId, Func DeleteAsync(string fileId, CancellationToken cancellationToken) + public async Task DeleteAsync(string fileId, string? scopeName, CancellationToken cancellationToken = default) { - EnsureWritable(); + EnsureWritable(scopeName); + return await ExecuteStorageOperationAsync( - container.GetStorages(fileId), + container.Storages.GetScopeStorages(fileId, scopeName), async (storage, ct) => await interceptors.ExecuteBeforeDeleteAsync(storage, fileId, ct), async (storage, ct) => await storage.DeleteAsync(fileId, ct), async (storage, result, ct) => await interceptors.ExecuteAfterDeleteAsync(storage, fileId, result, ct), @@ -60,12 +75,12 @@ public async Task DeleteAsync(string fileId, CancellationToken cancellatio private static async Task ExecuteStorageOperationAsync( - IEnumerable storages, - Func> beforeOperation, - Func> operation, - Func afterOperation, - Func onError, - CancellationToken cancellationToken) + IEnumerable storages, + Func> beforeOperation, + Func> operation, + Func afterOperation, + Func onError, + CancellationToken cancellationToken) { var exceptions = new List(); @@ -87,9 +102,9 @@ private static async Task ExecuteStorageOperationAsync( if (exceptions.Count > 0) { - throw new AggregateException("Operation failed for all available storages.", exceptions); + throw new HybridFileStorageAggregateException(exceptions); } - throw new InvalidOperationException("No storage available."); + throw new HybridFileStorageNoAvailableException(); } } diff --git a/src/Sa.HybridFileStorage/HybridFileStorageContainer.cs b/src/Sa.HybridFileStorage/HybridFileStorageContainer.cs index a37597b..bd1f9a4 100644 --- a/src/Sa.HybridFileStorage/HybridFileStorageContainer.cs +++ b/src/Sa.HybridFileStorage/HybridFileStorageContainer.cs @@ -8,7 +8,10 @@ internal sealed class HybridFileStorageContainer(IEnumerable stora public IHybridFileStorageContainerConfiguration AddStorage(IFileStorage storage) { - _storages.Add(storage); + if (!_storages.Contains(storage)) + { + _storages.Add(storage); + } return this; } diff --git a/src/Sa.HybridFileStorage/HybridStorageBuilder.cs b/src/Sa.HybridFileStorage/HybridStorageBuilder.cs index 96b4722..e7b80fc 100644 --- a/src/Sa.HybridFileStorage/HybridStorageBuilder.cs +++ b/src/Sa.HybridFileStorage/HybridStorageBuilder.cs @@ -9,14 +9,17 @@ internal sealed class HybridStorageBuilder(IServiceCollection services) : IHybri { private Action? _configureStorage; private Action? _configureInterceptors; + private bool _logged = false; - public IHybridFileStorageConfiguration ConfigureStorage(Action configure) + public IHybridFileStorageConfiguration ConfigureStorage( + Action configure) { _configureStorage = configure; return this; } - public IHybridFileStorageConfiguration ConfigureInterceptors(Action configure) + public IHybridFileStorageConfiguration ConfigureInterceptors( + Action configure) { _configureInterceptors = configure; return this; @@ -24,29 +27,28 @@ public IHybridFileStorageConfiguration ConfigureInterceptors(Action(); + _logged = true; return this; } public void Build() { - services.TryAddSingleton(TimeProvider.System); + if (_logged) + { + services.TryAddSingleton(); + } services.TryAddSingleton(sp => { - - var interceptorContainer = new InterceptorContainer(); + InterceptorContainer interceptorContainer = new(); interceptorContainer.AddLoggingInterceptor(sp.GetService()); _configureInterceptors?.Invoke(sp, interceptorContainer); - - IHybridFileStorageContainer storageContainer = sp.GetService() - ?? new HybridFileStorageContainer(sp.GetServices()); + HybridFileStorageContainer storageContainer = new(sp.GetServices()); _configureStorage?.Invoke(sp, storageContainer); - return new HybridFileStorage(storageContainer, interceptorContainer); }); } diff --git a/src/Sa.HybridFileStorage/IHybridFileStorage.cs b/src/Sa.HybridFileStorage/IHybridFileStorage.cs index 93ee46f..aae7bd5 100644 --- a/src/Sa.HybridFileStorage/IHybridFileStorage.cs +++ b/src/Sa.HybridFileStorage/IHybridFileStorage.cs @@ -10,21 +10,9 @@ namespace Sa.HybridFileStorage; public interface IHybridFileStorage { /// - /// Gets a value indicating whether the storage is read-only. + /// storages /// - bool IsReadOnly { get; } - - /// - /// Gets the type of storage (e.g., "pg", "s3", "file"). - /// - string StorageType { get; } - - /// - /// Determines whether the storage can process the specified file ID. - /// - /// The unique identifier for the file. - /// True if the storage can process the file ID; otherwise, false. - bool CanProcess(string fileId); + IReadOnlyCollection Storages { get; } /// /// Deletes the file associated with the specified file ID asynchronously. @@ -32,7 +20,7 @@ public interface IHybridFileStorage /// The unique identifier for the file to be deleted. /// A cancellation token to cancel the operation if needed. /// True if the file was successfully deleted; otherwise, false. - Task DeleteAsync(string fileId, CancellationToken cancellationToken); + Task DeleteAsync(string fileId, string? scopeName, CancellationToken cancellationToken = default); /// /// Downloads the file associated with the specified file ID asynchronously. @@ -41,7 +29,11 @@ public interface IHybridFileStorage /// A function that processes the downloaded file stream. /// A cancellation token to cancel the operation if needed. /// True if the file was successfully downloaded; otherwise, false. - Task DownloadAsync(string fileId, Func loadStream, CancellationToken cancellationToken); + Task DownloadAsync( + string fileId, + string? scopeName, + Func loadStream, + CancellationToken cancellationToken = default); /// /// Uploads a file asynchronously using the provided input and file stream. @@ -50,5 +42,9 @@ public interface IHybridFileStorage /// A stream containing the file data to be uploaded. /// A cancellation token to cancel the operation if needed. /// A containing the result of the upload operation. - Task UploadAsync(UploadFileInput input, Stream fileStream, CancellationToken cancellationToken); + Task UploadAsync( + UploadFileInput input, + string? scopeName, + Stream fileStream, + CancellationToken cancellationToken = default); } diff --git a/src/Sa.HybridFileStorage/IHybridFileStorageConfiguration.cs b/src/Sa.HybridFileStorage/IHybridFileStorageConfiguration.cs index 59d1280..ed5ccec 100644 --- a/src/Sa.HybridFileStorage/IHybridFileStorageConfiguration.cs +++ b/src/Sa.HybridFileStorage/IHybridFileStorageConfiguration.cs @@ -4,7 +4,11 @@ namespace Sa.HybridFileStorage; public interface IHybridFileStorageConfiguration { - IHybridFileStorageConfiguration ConfigureInterceptors(Action configure); - IHybridFileStorageConfiguration ConfigureStorage(Action configure); + IHybridFileStorageConfiguration ConfigureInterceptors( + Action configure); + + IHybridFileStorageConfiguration ConfigureStorage( + Action configure); + IHybridFileStorageConfiguration AddLogging(); } diff --git a/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs b/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs index ccfef3c..0f3108f 100644 --- a/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs +++ b/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs @@ -5,9 +5,4 @@ namespace Sa.HybridFileStorage; public interface IHybridFileStorageContainer : IHybridFileStorageContainerConfiguration { IReadOnlyCollection Storages { get; } - string StorageType => string.Join(',', Storages.Select(c => c.StorageType)); - bool IsReadOnly => Storages.All(f => f.IsReadOnly); - bool CanProcess(string fileId) => Storages.Any(c => c.CanProcess(fileId)); - - IEnumerable GetStorages(string fileId) => Storages.Where(c => c.CanProcess(fileId)); } diff --git a/src/Sa.HybridFileStorage/InMemoryFileStorage.cs b/src/Sa.HybridFileStorage/InMemoryFileStorage.cs index 2f81776..74ce2f9 100644 --- a/src/Sa.HybridFileStorage/InMemoryFileStorage.cs +++ b/src/Sa.HybridFileStorage/InMemoryFileStorage.cs @@ -3,25 +3,38 @@ namespace Sa.HybridFileStorage; -public sealed class InMemoryFileStorage(TimeProvider? currentTimeProvider = null, bool isReadOnly = false) : IFileStorage + +public sealed class InMemoryFileStorage( + InMemoryFileStorageOptions? options = null, + TimeProvider? timeProvider = null) : IFileStorage { + private readonly InMemoryFileStorageOptions _options = options ?? new (); + + private readonly TimeProvider _timeProvider = timeProvider ?? TimeProvider.System; + public const string DefaultStorageType = "mem"; private readonly ConcurrentDictionary _storage = []; + + public string? ScopeName => _options.ScopeName; + public string StorageType => DefaultStorageType; - public bool IsReadOnly => isReadOnly; + public bool IsReadOnly => _options.IsReadOnly; private void EnsureWritable() { if (IsReadOnly) { - throw new InvalidOperationException("Cannot perform this operation. The storage is read-only."); + throw new HybridFileStorageWritableException(); } } - public async Task UploadAsync(UploadFileInput metadata, Stream fileStream, CancellationToken cancellationToken) + public async Task UploadAsync( + UploadFileInput metadata, + Stream fileStream, + CancellationToken cancellationToken) { EnsureWritable(); @@ -34,12 +47,13 @@ public async Task UploadAsync(UploadFileInput metadata, Stream fi _storage[fileId] = fileData; - var now = currentTimeProvider?.GetUtcNow() ?? TimeProvider.System.GetUtcNow(); - - return new StorageResult(fileId, fileId, StorageType, now); + return new StorageResult(fileId, fileId, StorageType, _timeProvider.GetUtcNow()); } - public async Task DownloadAsync(string fileId, Func loadStream, CancellationToken cancellationToken) + public async Task DownloadAsync( + string fileId, + Func loadStream, + CancellationToken cancellationToken) { if (_storage.TryGetValue(fileId, out var fileData)) { diff --git a/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs b/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs new file mode 100644 index 0000000..9712640 --- /dev/null +++ b/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs @@ -0,0 +1,3 @@ +namespace Sa.HybridFileStorage; + +public sealed record InMemoryFileStorageOptions(string? ScopeName = null, bool IsReadOnly = false); diff --git a/src/Sa.HybridFileStorage/Interceptors/IDeleteInterceptor.cs b/src/Sa.HybridFileStorage/Interceptors/IDeleteInterceptor.cs index bd60a07..b974e1a 100644 --- a/src/Sa.HybridFileStorage/Interceptors/IDeleteInterceptor.cs +++ b/src/Sa.HybridFileStorage/Interceptors/IDeleteInterceptor.cs @@ -5,6 +5,15 @@ namespace Sa.HybridFileStorage.Interceptors; public interface IDeleteInterceptor { ValueTask CanDeleteAsync(IFileStorage storage, string fileId, CancellationToken cancellationToken); - ValueTask AfterDeleteAsync(IFileStorage storage, string fileId, bool success, CancellationToken cancellationToken); - ValueTask OnDeleteErrorAsync(IFileStorage storage, string fileId, Exception exception, CancellationToken cancellationToken); + + ValueTask AfterDeleteAsync( + IFileStorage storage, + string fileId, + bool success, + CancellationToken cancellationToken); + + ValueTask OnDeleteErrorAsync(IFileStorage storage, + string fileId, + Exception exception, + CancellationToken cancellationToken); } diff --git a/src/Sa.HybridFileStorage/Interceptors/IDownloadInterceptor.cs b/src/Sa.HybridFileStorage/Interceptors/IDownloadInterceptor.cs index 69b0c10..3e06422 100644 --- a/src/Sa.HybridFileStorage/Interceptors/IDownloadInterceptor.cs +++ b/src/Sa.HybridFileStorage/Interceptors/IDownloadInterceptor.cs @@ -4,7 +4,21 @@ namespace Sa.HybridFileStorage.Interceptors; public interface IDownloadInterceptor { - ValueTask CanDownloadAsync(IFileStorage storage, string fileId, Func loadStream, CancellationToken cancellationToken); - ValueTask AfterDownloadAsync(IFileStorage storage, string fileId, bool success, CancellationToken cancellationToken); - ValueTask OnDownloadErrorAsync(IFileStorage storage, string fileId, Exception exception, CancellationToken cancellationToken); + ValueTask CanDownloadAsync( + IFileStorage storage, + string fileId, + Func loadStream, + CancellationToken cancellationToken); + + ValueTask AfterDownloadAsync( + IFileStorage storage, + string fileId, + bool success, + CancellationToken cancellationToken); + + ValueTask OnDownloadErrorAsync( + IFileStorage storage, + string fileId, + Exception exception, + CancellationToken cancellationToken); } diff --git a/src/Sa.HybridFileStorage/Interceptors/IUploadInterceptor.cs b/src/Sa.HybridFileStorage/Interceptors/IUploadInterceptor.cs index 27f23b3..6876566 100644 --- a/src/Sa.HybridFileStorage/Interceptors/IUploadInterceptor.cs +++ b/src/Sa.HybridFileStorage/Interceptors/IUploadInterceptor.cs @@ -4,7 +4,11 @@ namespace Sa.HybridFileStorage.Interceptors; public interface IUploadInterceptor { - ValueTask CanUploadAsync(IFileStorage storage, UploadFileInput input, Stream fileStream, CancellationToken cancellationToken); + ValueTask CanUploadAsync( + IFileStorage storage, + UploadFileInput input, + Stream fileStream, + CancellationToken cancellationToken); ValueTask AfterUploadAsync(IFileStorage storage, StorageResult result, CancellationToken cancellationToken); ValueTask OnUploadErrorAsync(IFileStorage storage, Exception exception, CancellationToken cancellationToken); } diff --git a/src/Sa.HybridFileStorage/Interceptors/InterceptorContainer.cs b/src/Sa.HybridFileStorage/Interceptors/InterceptorContainer.cs index 6be0ab1..6737d4e 100644 --- a/src/Sa.HybridFileStorage/Interceptors/InterceptorContainer.cs +++ b/src/Sa.HybridFileStorage/Interceptors/InterceptorContainer.cs @@ -27,7 +27,11 @@ public IInterceptorContainer AddDeleteInterceptor(IDeleteInterceptor interceptor return this; } - public async Task ExecuteBeforeUploadAsync(IFileStorage storage, UploadFileInput input, Stream fileStream, CancellationToken cancellationToken) + public async Task ExecuteBeforeUploadAsync( + IFileStorage storage, + UploadFileInput input, + Stream fileStream, + CancellationToken cancellationToken) { foreach (var interceptor in _uploadInterceptors) { @@ -39,7 +43,10 @@ public async Task ExecuteBeforeUploadAsync(IFileStorage storage, UploadFil return true; } - public async Task ExecuteAfterUploadAsync(IFileStorage storage, StorageResult result, CancellationToken cancellationToken) + public async Task ExecuteAfterUploadAsync( + IFileStorage storage, + StorageResult result, + CancellationToken cancellationToken) { foreach (var interceptor in _uploadInterceptors) { @@ -47,7 +54,10 @@ public async Task ExecuteAfterUploadAsync(IFileStorage storage, StorageResult re } } - public async Task ExecuteOnUploadErrorAsync(IFileStorage storage, Exception exception, CancellationToken cancellationToken) + public async Task ExecuteOnUploadErrorAsync( + IFileStorage storage, + Exception exception, + CancellationToken cancellationToken) { foreach (var interceptor in _uploadInterceptors) { @@ -55,7 +65,11 @@ public async Task ExecuteOnUploadErrorAsync(IFileStorage storage, Exception exce } } - public async Task ExecuteBeforeDownloadAsync(IFileStorage storage, string fileId, Func loadStream, CancellationToken cancellationToken) + public async Task ExecuteBeforeDownloadAsync( + IFileStorage storage, + string fileId, + Func loadStream, + CancellationToken cancellationToken) { foreach (var interceptor in _downloadInterceptors) { @@ -67,7 +81,11 @@ public async Task ExecuteBeforeDownloadAsync(IFileStorage storage, string return true; } - public async Task ExecuteAfterDownloadAsync(IFileStorage storage, string fileId, bool success, CancellationToken cancellationToken) + public async Task ExecuteAfterDownloadAsync( + IFileStorage storage, + string fileId, + bool success, + CancellationToken cancellationToken) { foreach (var interceptor in _downloadInterceptors) { @@ -75,7 +93,11 @@ public async Task ExecuteAfterDownloadAsync(IFileStorage storage, string fileId, } } - public async Task ExecuteOnDownloadErrorAsync(IFileStorage storage, string fileId, Exception exception, CancellationToken cancellationToken) + public async Task ExecuteOnDownloadErrorAsync( + IFileStorage storage, + string fileId, + Exception exception, + CancellationToken cancellationToken) { foreach (var interceptor in _downloadInterceptors) { @@ -83,7 +105,10 @@ public async Task ExecuteOnDownloadErrorAsync(IFileStorage storage, string fileI } } - public async Task ExecuteBeforeDeleteAsync(IFileStorage storage, string fileId, CancellationToken cancellationToken) + public async Task ExecuteBeforeDeleteAsync( + IFileStorage storage, + string fileId, + CancellationToken cancellationToken) { foreach (var interceptor in _deleteInterceptors) { @@ -95,7 +120,11 @@ public async Task ExecuteBeforeDeleteAsync(IFileStorage storage, string fi return true; } - public async Task ExecuteAfterDeleteAsync(IFileStorage storage, string fileId, bool success, CancellationToken cancellationToken) + public async Task ExecuteAfterDeleteAsync( + IFileStorage storage, + string fileId, + bool success, + CancellationToken cancellationToken) { foreach (var interceptor in _deleteInterceptors) { @@ -103,7 +132,11 @@ public async Task ExecuteAfterDeleteAsync(IFileStorage storage, string fileId, b } } - public async Task ExecuteOnDeleteErrorAsync(IFileStorage storage, string fileId, Exception exception, CancellationToken cancellationToken) + public async Task ExecuteOnDeleteErrorAsync( + IFileStorage storage, + string fileId, + Exception exception, + CancellationToken cancellationToken) { foreach (var interceptor in _deleteInterceptors) { diff --git a/src/Sa.HybridFileStorage/Interceptors/LoggingInterceptor.cs b/src/Sa.HybridFileStorage/Interceptors/LoggingInterceptor.cs index 63ed6f5..0e02fa8 100644 --- a/src/Sa.HybridFileStorage/Interceptors/LoggingInterceptor.cs +++ b/src/Sa.HybridFileStorage/Interceptors/LoggingInterceptor.cs @@ -15,7 +15,11 @@ public ValueTask CanDeleteAsync(IFileStorage storage, string fileId, Cance return ValueTask.FromResult(true); } - public ValueTask AfterDeleteAsync(IFileStorage storage, string fileId, bool success, CancellationToken cancellationToken) + public ValueTask AfterDeleteAsync( + IFileStorage storage, + string fileId, + bool success, + CancellationToken cancellationToken) { if (success) { @@ -28,19 +32,31 @@ public ValueTask AfterDeleteAsync(IFileStorage storage, string fileId, bool succ return ValueTask.CompletedTask; } - public ValueTask OnDeleteErrorAsync(IFileStorage storage, string fileId, Exception exception, CancellationToken cancellationToken) + public ValueTask OnDeleteErrorAsync( + IFileStorage storage, + string fileId, + Exception exception, + CancellationToken cancellationToken) { LogDeleteError(_logger, exception, fileId, storage.StorageType); return ValueTask.CompletedTask; } - public ValueTask CanDownloadAsync(IFileStorage storage, string fileId, Func loadStream, CancellationToken cancellationToken) + public ValueTask CanDownloadAsync( + IFileStorage storage, + string fileId, + Func loadStream, + CancellationToken cancellationToken) { LogCanDownload(_logger, fileId, storage.StorageType); return ValueTask.FromResult(true); } - public ValueTask AfterDownloadAsync(IFileStorage storage, string fileId, bool success, CancellationToken cancellationToken) + public ValueTask AfterDownloadAsync( + IFileStorage storage, + string fileId, + bool success, + CancellationToken cancellationToken) { if (success) { @@ -53,25 +69,39 @@ public ValueTask AfterDownloadAsync(IFileStorage storage, string fileId, bool su return ValueTask.CompletedTask; } - public ValueTask OnDownloadErrorAsync(IFileStorage storage, string fileId, Exception exception, CancellationToken cancellationToken) + public ValueTask OnDownloadErrorAsync( + IFileStorage storage, + string fileId, + Exception exception, + CancellationToken cancellationToken) { LogDownloadError(_logger, exception, fileId, storage.StorageType); return ValueTask.CompletedTask; } - public ValueTask CanUploadAsync(IFileStorage storage, UploadFileInput input, Stream fileStream, CancellationToken cancellationToken) + public ValueTask CanUploadAsync( + IFileStorage storage, + UploadFileInput input, + Stream fileStream, + CancellationToken cancellationToken) { LogCanUpload(_logger, input, storage.StorageType); return ValueTask.FromResult(true); } - public ValueTask AfterUploadAsync(IFileStorage storage, StorageResult result, CancellationToken cancellationToken) + public ValueTask AfterUploadAsync( + IFileStorage storage, + StorageResult result, + CancellationToken cancellationToken) { LogUploadSuccess(_logger, storage.StorageType, result); return ValueTask.CompletedTask; } - public ValueTask OnUploadErrorAsync(IFileStorage storage, Exception exception, CancellationToken cancellationToken) + public ValueTask OnUploadErrorAsync( + IFileStorage storage, + Exception exception, + CancellationToken cancellationToken) { LogUploadError(_logger, exception, storage.StorageType); return ValueTask.CompletedTask; diff --git a/src/Sa.HybridFileStorage/Interceptors/Setup.cs b/src/Sa.HybridFileStorage/Interceptors/Setup.cs index c31e263..6ba8ea7 100644 --- a/src/Sa.HybridFileStorage/Interceptors/Setup.cs +++ b/src/Sa.HybridFileStorage/Interceptors/Setup.cs @@ -2,7 +2,9 @@ internal static class Setup { - internal static IInterceptorContainer AddLoggingInterceptor(this IInterceptorContainer container, LoggingInterceptor? loggingInterceptor = null) + internal static IInterceptorContainer AddLoggingInterceptor( + this IInterceptorContainer container, + LoggingInterceptor? loggingInterceptor = null) { if (loggingInterceptor != null) { diff --git a/src/Sa.HybridFileStorage/Readme.md b/src/Sa.HybridFileStorage/Readme.md index 660fa4f..a463dcb 100644 --- a/src/Sa.HybridFileStorage/Readme.md +++ b/src/Sa.HybridFileStorage/Readme.md @@ -5,15 +5,36 @@ The IHybridFileStorage interface enhances the resilience and availability of fil This interface defines a contract for hybrid file storage systems capable of handling file operations such as uploading, downloading, and deleting files. The integration of multiple storage providers (such as file system, S3, and PostgreSQL) ensures reliable file storage, as the system can automatically switch between different providers in the event that one becomes unavailable. ```csharp + public interface IHybridFileStorage { - bool IsReadOnly { get; } - string StorageType { get; } - - bool CanProcess(string fileId); - Task DeleteAsync(string fileId, CancellationToken cancellationToken); - Task DownloadAsync(string fileId, Func loadStream, CancellationToken cancellationToken); - Task UploadAsync(UploadFileInput input, Stream fileStream, CancellationToken cancellationToken); + /// + /// storages + /// + IReadOnlyCollection Storages { get; } + + /// + /// Deletes the file associated with the specified file ID asynchronously. + /// + Task DeleteAsync(string fileId, string? scopeName, CancellationToken cancellationToken = default); + + /// + /// Downloads the file associated with the specified file ID asynchronously. + /// + Task DownloadAsync( + string fileId, + string? scopeName, + Func loadStream, + CancellationToken cancellationToken = default); + + /// + /// Uploads a file asynchronously using the provided input and file stream. + /// + Task UploadAsync( + UploadFileInput input, + string? scopeName, + Stream fileStream, + CancellationToken cancellationToken = default); } ``` @@ -21,34 +42,20 @@ public interface IHybridFileStorage ```csharp -IHostBuilder builder = Host.CreateDefaultBuilder(); - +// di builder.Services.AddSaHybridStorage((_, b) => b.AddStorage(new InMemoryFileStorage())); -builder.Services.TryAddSingleton(); - -builder.UseConsoleLifetime(); -var host = builder.Build(); -await host.Services.GetRequiredService().Run(); +// some test +using var stream = "Hello, HybridFileStorage!".ToStream(); -namespace HybridFileStorage.Console -{ - public class Proccessor(IHybridFileStorage storage) - { - public async Task Run(CancellationToken cancellationToken = default) - { - var expected = "Hello, HybridFileStorage!"; - using var stream = expected.ToStream(); - - var result = await storage.UploadAsync(new UploadFileInput { FileName = "file.txt" }, stream, cancellationToken); +await storage.UploadAsync( + new UploadFileInput { FileName = "file.txt" }, + stream, + cancellationToken); - string? actual = null; +await storage.DownloadAsync( + result.FileId, + async (fs, t) => actual = await fs.ToStrAsync(t), + cancellationToken); - var isDownload = await storage.DownloadAsync(result.FileId, async (fs, t) => actual = await fs.ToStrAsync(t), cancellationToken); - - Debug.Assert(isDownload); - Debug.Assert(expected == actual); - } - } -} ``` diff --git a/src/Sa.HybridFileStorage/Sa.HybridFileStorage.csproj b/src/Sa.HybridFileStorage/Sa.HybridFileStorage.csproj index 15630b3..d65fc9c 100644 --- a/src/Sa.HybridFileStorage/Sa.HybridFileStorage.csproj +++ b/src/Sa.HybridFileStorage/Sa.HybridFileStorage.csproj @@ -3,7 +3,7 @@ - 0.8.0 + 0.8.1 File storage management diff --git a/src/Sa.HybridFileStorage/Setup.cs b/src/Sa.HybridFileStorage/Setup.cs index f7708d2..c235d56 100644 --- a/src/Sa.HybridFileStorage/Setup.cs +++ b/src/Sa.HybridFileStorage/Setup.cs @@ -5,7 +5,9 @@ namespace Sa.HybridFileStorage; public static class Setup { - public static IServiceCollection AddSaHybridFileStorage(this IServiceCollection services, Action? configure = null) + public static IServiceCollection AddSaHybridFileStorage( + this IServiceCollection services, + Action? configure = null) { HybridStorageBuilder builder = new(services); configure?.Invoke(builder); @@ -14,9 +16,14 @@ public static IServiceCollection AddSaHybridFileStorage(this IServiceCollection } - public static IServiceCollection AddSaInMemoryFileStorage(this IServiceCollection services) + public static IServiceCollection AddSaInMemoryFileStorage( + this IServiceCollection services, + InMemoryFileStorageOptions? options = null) { - services.AddSingleton(); + options ??= new InMemoryFileStorageOptions(); + + services.AddSingleton( + sp => new InMemoryFileStorage(options, sp.GetService())); return services; } } diff --git a/src/Samples/HybridFileStorage.Console/Program.cs b/src/Samples/HybridFileStorage.Console/Program.cs index fae9118..b00287b 100644 --- a/src/Samples/HybridFileStorage.Console/Program.cs +++ b/src/Samples/HybridFileStorage.Console/Program.cs @@ -45,11 +45,13 @@ public async Task Run(CancellationToken cancellationToken = default) var expected = "Hello, HybridFileStorage!"; using var stream = expected.ToStream(); - var result = await storage.UploadAsync(new UploadFileInput { FileName = "file.txt" }, stream, cancellationToken); + var result = await storage.UploadAsync( + new UploadFileInput { FileName = "file.txt" }, null, stream, cancellationToken); string? actual = default; - var isDowload = await storage.DownloadAsync(result.FileId, async (fs, t) => actual = await fs.ToStrAsync(t), cancellationToken); + var isDowload = await storage.DownloadAsync( + result.FileId, null, async (fs, t) => actual = await fs.ToStrAsync(t), cancellationToken); Debug.Assert(isDowload); diff --git a/src/Tests/Sa.HybridFileStorage.FileSystemTests/FileSystemStorageTests.cs b/src/Tests/Sa.HybridFileStorage.FileSystemTests/FileSystemStorageTests.cs index 38ad8f0..597862b 100644 --- a/src/Tests/Sa.HybridFileStorage.FileSystemTests/FileSystemStorageTests.cs +++ b/src/Tests/Sa.HybridFileStorage.FileSystemTests/FileSystemStorageTests.cs @@ -72,4 +72,27 @@ private async Task EnsureFileSame(string fileName, MemoryStream expectedBy return isDownloaded; } + + + [Fact] + public async Task CrudEx() + { + var metadata = new UploadFileInput { FileName = "/api/files/download/file/var/www/uploads/image.bin", TenantId = 1 }; + using MemoryStream fileContent = FixtureHelper.GetByteStream(); + + var result = await Storage.UploadAsync(metadata, fileContent, fixture.CancellationToken); + + Assert.NotNull(result); + Assert.NotEmpty(result.FileId); + + bool canProcessed = Storage.CanProcess(result.FileId); + Assert.True(canProcessed); + + var isSame = await EnsureFileSame(result.FileId, fileContent); + Assert.True(isSame); + + var isDeleted = await Storage.DeleteAsync(result.FileId, fixture.CancellationToken); + + Assert.True(isDeleted); + } } diff --git a/src/Tests/Sa.HybridFileStorageTests/HybridFileStorageTests.cs b/src/Tests/Sa.HybridFileStorageTests/HybridFileStorageTests.cs index a9c72f6..812bc86 100644 --- a/src/Tests/Sa.HybridFileStorageTests/HybridFileStorageTests.cs +++ b/src/Tests/Sa.HybridFileStorageTests/HybridFileStorageTests.cs @@ -43,14 +43,18 @@ public override ValueTask DisposeAsync() class MemUploadSomeInterceptor : IUploadInterceptor { - public ValueTask AfterUploadAsync(IFileStorage storage, StorageResult result, CancellationToken cancellationToken) => ValueTask.CompletedTask; + public ValueTask AfterUploadAsync + (IFileStorage storage, StorageResult result, CancellationToken cancellationToken) => ValueTask.CompletedTask; - public ValueTask CanUploadAsync(IFileStorage storage, UploadFileInput input, Stream fileStream, CancellationToken cancellationToken) + public ValueTask CanUploadAsync + (IFileStorage storage, UploadFileInput input, Stream fileStream, CancellationToken cancellationToken) { - return ValueTask.FromResult(input.FileName != "some.bin" || storage.StorageType == InMemoryFileStorage.DefaultStorageType); + return ValueTask.FromResult(input.FileName != "some.bin" + || storage.StorageType == InMemoryFileStorage.DefaultStorageType); } - public ValueTask OnUploadErrorAsync(IFileStorage storage, Exception exception, CancellationToken cancellationToken) => ValueTask.CompletedTask; + public ValueTask OnUploadErrorAsync + (IFileStorage storage, Exception exception, CancellationToken cancellationToken) => ValueTask.CompletedTask; } private IHybridFileStorage Storage => fixture.Sub; @@ -62,18 +66,18 @@ public async Task Crud() var input = new UploadFileInput { FileName = "test.bin", TenantId = 2 }; using MemoryStream fileContent = FixtureHelper.GetByteStream(); - var result = await Storage.UploadAsync(input, fileContent, fixture.CancellationToken); + var result = await Storage.UploadAsync(input, null, fileContent, fixture.CancellationToken); Assert.NotNull(result); Assert.NotEmpty(result.FileId); - bool canProcessed = Storage.CanProcess(result.FileId); + bool canProcessed = Storage.Storages.Any(c => c.CanProcess(result.FileId)); Assert.True(canProcessed); var isSame = await EnsureFileSame(result.FileId, fileContent); Assert.True(isSame); - var isDeleted = await Storage.DeleteAsync(result.FileId, fixture.CancellationToken); + var isDeleted = await Storage.DeleteAsync(result.FileId, null, fixture.CancellationToken); Assert.True(isDeleted); @@ -88,7 +92,7 @@ public async Task UploadInMemStorageBySomeInterceptor() var input = new UploadFileInput { FileName = "some.bin", TenantId = 1 }; using MemoryStream fileContent = FixtureHelper.GetByteStream(); - var result = await Storage.UploadAsync(input, fileContent, fixture.CancellationToken); + var result = await Storage.UploadAsync(input, null, fileContent, fixture.CancellationToken); Assert.NotNull(result); Assert.Equal(InMemoryFileStorage.DefaultStorageType, result.StorageType); @@ -96,7 +100,7 @@ public async Task UploadInMemStorageBySomeInterceptor() var isSame = await EnsureFileSame(result.FileId, fileContent); Assert.True(isSame); - var isDeleted = await Storage.DeleteAsync(result.FileId, fixture.CancellationToken); + var isDeleted = await Storage.DeleteAsync(result.FileId, null, fixture.CancellationToken); Assert.True(isDeleted); } @@ -109,8 +113,9 @@ public async Task WhenStorageIsEmptyThrowsInvalidOperationException() using var sp = services.BuildServiceProvider(); using MemoryStream fileContent = FixtureHelper.GetByteStream(); - await Assert.ThrowsAsync(() => - sp.GetRequiredService().UploadAsync(new UploadFileInput { FileName = "", TenantId = 1 }, fileContent, fixture.CancellationToken)); + await Assert.ThrowsAsync(() => + sp.GetRequiredService().UploadAsync( + new UploadFileInput { FileName = "", TenantId = 1 }, null, fileContent, fixture.CancellationToken)); } @@ -120,13 +125,18 @@ public async Task WhenStorageIsReadOnlyThrowsInvalidOperationException() ServiceCollection services = new(); services.AddSaHybridFileStorage(b => b.ConfigureStorage((_, c) - => c.AddStorage(new InMemoryFileStorage(null, true)))); + => c.AddStorage(new InMemoryFileStorage(new InMemoryFileStorageOptions(IsReadOnly: true))))); using var sp = services.BuildServiceProvider(); using MemoryStream fileContent = FixtureHelper.GetByteStream(); - await Assert.ThrowsAsync(() => - sp.GetRequiredService().UploadAsync(new UploadFileInput { FileName = "", TenantId = 1 }, fileContent, fixture.CancellationToken)); + + await Assert.ThrowsAsync(() => + sp.GetRequiredService().UploadAsync( + new UploadFileInput { FileName = "", TenantId = 1 }, + null, + fileContent, + fixture.CancellationToken)); } @@ -137,6 +147,7 @@ private async Task EnsureFileSame(string fileName, MemoryStream expectedBy using MemoryStream memoryStream = new(); var isDownloaded = await Storage.DownloadAsync(fileName + , scopeName: null , (stream, ct) => stream.CopyToAsync(memoryStream, ct) , fixture.CancellationToken);