diff --git a/src/CsvToOfx.Core/Models/HeaderMap.cs b/src/CsvToOfx.Core/Models/HeaderMap.cs index 5c85d96..ecca6b3 100644 --- a/src/CsvToOfx.Core/Models/HeaderMap.cs +++ b/src/CsvToOfx.Core/Models/HeaderMap.cs @@ -1,8 +1,28 @@ using System.Collections.Generic; +using System.Linq; namespace CsvToOfx.Core.Models; public sealed record HeaderMap( string Name, - IReadOnlyDictionary Columns -); + IReadOnlyDictionary Columns, + IReadOnlyCollection? RequiredFields = null) +{ + public IEnumerable EffectiveRequiredFields => + RequiredFields is { Count: > 0 } ? RequiredFields : Columns.Values.Distinct(); + + public bool TryGetColumnName(CanonicalField field, out string columnName) + { + foreach (var entry in Columns) + { + if (entry.Value != field) + continue; + + columnName = entry.Key; + return true; + } + + columnName = string.Empty; + return false; + } +} diff --git a/src/CsvToOfx.Core/Parsing/HeaderMaps/FidelityIraHeaderMap.cs b/src/CsvToOfx.Core/Parsing/HeaderMaps/FidelityIraHeaderMap.cs index 5585f29..ea843b7 100644 --- a/src/CsvToOfx.Core/Parsing/HeaderMaps/FidelityIraHeaderMap.cs +++ b/src/CsvToOfx.Core/Parsing/HeaderMaps/FidelityIraHeaderMap.cs @@ -6,6 +6,18 @@ namespace CsvToOfx.Core.Parsing.HeaderMaps; public static class FidelityIraHeaderMap { + private static readonly CanonicalField[] RequiredFields = + { + CanonicalField.TradeDate, + CanonicalField.Action, + CanonicalField.Symbol, + CanonicalField.Description, + CanonicalField.Price, + CanonicalField.Quantity, + CanonicalField.Fees, + CanonicalField.Amount + }; + public static HeaderMap Instance { get; } = new HeaderMap( "Fidelity-IRA", new Dictionary(StringComparer.OrdinalIgnoreCase) @@ -23,6 +35,6 @@ public static class FidelityIraHeaderMap ["Amount ($)"] = CanonicalField.Amount, ["Cash Balance ($)"] = CanonicalField.CashBalance, ["Settlement Date"] = CanonicalField.SettlementDate - }); + }, + RequiredFields); } - diff --git a/src/CsvToOfx.Core/Parsing/HeaderMaps/FidelityTradingHeaderMap.cs b/src/CsvToOfx.Core/Parsing/HeaderMaps/FidelityTradingHeaderMap.cs index 7a30ab5..4b1f45b 100644 --- a/src/CsvToOfx.Core/Parsing/HeaderMaps/FidelityTradingHeaderMap.cs +++ b/src/CsvToOfx.Core/Parsing/HeaderMaps/FidelityTradingHeaderMap.cs @@ -6,6 +6,18 @@ namespace CsvToOfx.Core.Parsing.HeaderMaps; public static class FidelityTradingHeaderMap { + private static readonly CanonicalField[] RequiredFields = + { + CanonicalField.TradeDate, + CanonicalField.Action, + CanonicalField.Symbol, + CanonicalField.Description, + CanonicalField.Price, + CanonicalField.Quantity, + CanonicalField.Fees, + CanonicalField.Amount + }; + public static HeaderMap Instance { get; } = new HeaderMap( "Fidelity-Trading", new Dictionary(StringComparer.OrdinalIgnoreCase) @@ -27,6 +39,6 @@ public static class FidelityTradingHeaderMap ["Amount"] = CanonicalField.Amount, ["Cash Balance"] = CanonicalField.CashBalance, ["Settlement Date"] = CanonicalField.SettlementDate - }); + }, + RequiredFields); } - diff --git a/src/CsvToOfx.Parsers/Providers/Fidelity/FidelityParser.cs b/src/CsvToOfx.Parsers/Providers/Fidelity/FidelityParser.cs index 22cf1f7..6aad1bd 100644 --- a/src/CsvToOfx.Parsers/Providers/Fidelity/FidelityParser.cs +++ b/src/CsvToOfx.Parsers/Providers/Fidelity/FidelityParser.cs @@ -1,4 +1,5 @@ using CsvToOfx.Core.Models; +using CsvToOfx.Core.Parsing.HeaderMaps; using CsvToOfx.Parsers.Abstractions; using CsvToOfx.Parsers.Shared; @@ -10,8 +11,10 @@ public sealed class FidelityParser : IStatementParser public string ProviderCode => "fidelity"; public ParserCapabilities Capabilities => ParserCapabilities.Csv | ParserCapabilities.Brokerage; - private static readonly string[] RequiredFields = { - "Run Date","Action","Symbol","Description","Type","Price","Quantity","Commission","Fees","Amount" + private static readonly HeaderMap[] HeaderMaps = + { + FidelityTradingHeaderMap.Instance, + FidelityIraHeaderMap.Instance }; public ParseResult Parse(RawStatement input, ParserContext ctx) @@ -20,19 +23,24 @@ public ParseResult Parse(RawStatement input, ParserContext ctx) var account = new AccountRef("Fidelity", ctx.AccountId, "Brokerage"); var transactions = new List(); var securities = new Dictionary(StringComparer.OrdinalIgnoreCase); + var readResult = reader.ReadRows(input.Content, HeaderMaps); + if (readResult is null) + return new ParseResult(account, transactions, securities.Values.ToList()); + + var columnsByField = BuildFieldLookup(readResult.HeaderMap); - foreach (var row in reader.ReadRows(input.Content, RequiredFields)) + foreach (var row in readResult.Rows) { // skip empty rows var nonEmpty = row.Values.Count(v => !string.IsNullOrWhiteSpace(v)); if (nonEmpty <= 1) continue; // date filter - var dt = ctx.DateParser.ParseOrNull(Get(row, "Run Date")); + var dt = ctx.DateParser.ParseOrNull(Get(row, columnsByField, CanonicalField.TradeDate)); if (ctx.StartDateFilter.HasValue && dt.HasValue && dt.Value < ctx.StartDateFilter.Value) continue; - var action = FidelityActionMap.Normalize(Get(row, "Action")); - var symbol = (Get(row, "Symbol") ?? "").Trim(); + var action = FidelityActionMap.Normalize(Get(row, columnsByField, CanonicalField.Action)); + var symbol = (Get(row, columnsByField, CanonicalField.Symbol) ?? "").Trim(); var security = ctx.SecurityResolver.ResolveFromRow(row); if (security is not null) { @@ -55,12 +63,12 @@ public ParseResult Parse(RawStatement input, ParserContext ctx) } } - var units = TryParseDecimal(Get(row, "Quantity")); - var unitPrice = TryParseDecimal(Get(row, "Price")); - var amount = ctx.AmountParser.ParseAbsOrNull(Get(row, "Amount")) ?? 0m; - var memo = BuildMemo(action, row); - var fees = TryParseDecimal(Get(row, "Fees")); - var currency = (Get(row, "Currency") ?? ctx.CurrencyDefault).Trim(); + var units = TryParseDecimal(Get(row, columnsByField, CanonicalField.Quantity)); + var unitPrice = TryParseDecimal(Get(row, columnsByField, CanonicalField.Price)); + var amount = ctx.AmountParser.ParseAbsOrNull(Get(row, columnsByField, CanonicalField.Amount)) ?? 0m; + var memo = BuildMemo(action, row, columnsByField); + var fees = TryParseDecimal(Get(row, columnsByField, CanonicalField.Fees)); + var currency = (Get(row, columnsByField, CanonicalField.Currency) ?? ctx.CurrencyDefault).Trim(); var fitid = ctx.FitIdGenerator.FromSortedRow(row); transactions.Add(new NormalizedTransaction( @@ -79,22 +87,45 @@ public ParseResult Parse(RawStatement input, ParserContext ctx) return new ParseResult(account, transactions, securities.Values.ToList()); } - private static string? Get(IDictionary row, string key) - => row.TryGetValue(key, out var value) ? value : null; + private static Dictionary BuildFieldLookup(HeaderMap headerMap) + { + var lookup = new Dictionary(); + foreach (var field in headerMap.Columns.Values.Distinct()) + { + if (headerMap.TryGetColumnName(field, out var columnName)) + lookup[field] = columnName; + } + + return lookup; + } + + private static string? Get( + IDictionary row, + IReadOnlyDictionary columnsByField, + CanonicalField field) + { + if (!columnsByField.TryGetValue(field, out var columnName)) + return null; + + return row.TryGetValue(columnName, out var value) ? value : null; + } private static decimal? TryParseDecimal(string? s) => decimal.TryParse((s ?? "").Replace(",", ""), out var v) ? v : null; - private static string? BuildMemo(CanonicalAction action, IDictionary row) + private static string? BuildMemo( + CanonicalAction action, + IDictionary row, + IReadOnlyDictionary columnsByField) { - var symbol = (Get(row, "Symbol") ?? "").Trim(); + var symbol = (Get(row, columnsByField, CanonicalField.Symbol) ?? "").Trim(); return action switch { CanonicalAction.Income => string.IsNullOrWhiteSpace(symbol) ? "Dividend Received" : $"Dividend Received - {symbol}", - CanonicalAction.CashTransfer => (Get(row, "Action") ?? "").Trim(), + CanonicalAction.CashTransfer => (Get(row, columnsByField, CanonicalField.Action) ?? "").Trim(), CanonicalAction.BuyStock => string.IsNullOrWhiteSpace(symbol) ? "You Bought" : $"You Bought - {symbol}", CanonicalAction.SellStock => string.IsNullOrWhiteSpace(symbol) ? "You Sold" : $"You Sold - {symbol}", - _ => Get(row, "Description") + _ => Get(row, columnsByField, CanonicalField.Description) }; } } diff --git a/src/CsvToOfx.Parsers/Shared/CsvReadResult.cs b/src/CsvToOfx.Parsers/Shared/CsvReadResult.cs new file mode 100644 index 0000000..995a542 --- /dev/null +++ b/src/CsvToOfx.Parsers/Shared/CsvReadResult.cs @@ -0,0 +1,8 @@ +using CsvToOfx.Core.Models; + +namespace CsvToOfx.Parsers.Shared; + +public sealed record CsvReadResult( + HeaderMap HeaderMap, + IReadOnlyList> Rows +); diff --git a/src/CsvToOfx.Parsers/Shared/CsvRowReader.cs b/src/CsvToOfx.Parsers/Shared/CsvRowReader.cs index ce1569a..9b852c0 100644 --- a/src/CsvToOfx.Parsers/Shared/CsvRowReader.cs +++ b/src/CsvToOfx.Parsers/Shared/CsvRowReader.cs @@ -1,5 +1,6 @@ using CsvHelper; using CsvHelper.Configuration; +using CsvToOfx.Core.Models; using System.Globalization; namespace CsvToOfx.Parsers.Shared; @@ -62,6 +63,56 @@ public sealed class CsvRowReader } } + public CsvReadResult? ReadRows(Stream csv, IEnumerable headerMaps) + { + using var reader = new StreamReader(csv); + using var csvr = new CsvReader(reader, _conf); + + var candidates = headerMaps.ToList(); + HeaderMap? matchedHeaderMap = null; + string[]? headerRecord = null; + var rows = new List>(); + var seenData = false; + + while (csvr.Read()) + { + var record = csvr.Parser.Record; + if (record is null || record.Length == 0) + continue; + + var nonEmpty = record.Count(v => !string.IsNullOrWhiteSpace(v)); + if (nonEmpty == 0) + { + if (seenData) + break; + + continue; + } + + if (matchedHeaderMap is null) + { + matchedHeaderMap = MatchHeader(record, candidates); + if (matchedHeaderMap is null) + continue; + + headerRecord = record.Select(v => v?.Trim() ?? string.Empty).ToArray(); + continue; + } + + if (headerRecord is null || record.Length < headerRecord.Length) + continue; + + var converted = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (var i = 0; i < headerRecord.Length; i++) + converted[headerRecord[i]] = i < record.Length ? record[i] : null; + + seenData = true; + rows.Add(converted); + } + + return matchedHeaderMap is null ? null : new CsvReadResult(matchedHeaderMap, rows); + } + private static bool MatchesHeader(string[] record, HashSet? requiredHeaders) { var normalized = record @@ -77,4 +128,38 @@ private static bool MatchesHeader(string[] record, HashSet? requiredHead return requiredHeaders.IsSubsetOf(normalized); } + + private static HeaderMap? MatchHeader(string[] record, IReadOnlyCollection headerMaps) + { + var normalized = record + .Where(v => !string.IsNullOrWhiteSpace(v)) + .Select(v => v.Trim()) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + if (normalized.Count == 0) + return null; + + HeaderMap? bestMatch = null; + var bestScore = -1; + + foreach (var headerMap in headerMaps) + { + var matchedFields = headerMap.Columns + .Where(entry => normalized.Contains(entry.Key)) + .Select(entry => entry.Value) + .ToHashSet(); + + if (!headerMap.EffectiveRequiredFields.All(matchedFields.Contains)) + continue; + + var score = matchedFields.Count; + if (score <= bestScore) + continue; + + bestMatch = headerMap; + bestScore = score; + } + + return bestMatch; + } } diff --git a/tests/CsvToOfx.Parsers.Tests/UnitTest1.cs b/tests/CsvToOfx.Parsers.Tests/UnitTest1.cs index 690c260..1c65b32 100644 --- a/tests/CsvToOfx.Parsers.Tests/UnitTest1.cs +++ b/tests/CsvToOfx.Parsers.Tests/UnitTest1.cs @@ -10,14 +10,14 @@ namespace CsvToOfx.Parsers.Tests; public class FidelityParserTests { [Fact] - public void Parse_SkipsLeadingBlankRows_AndStopsBeforeFooterDisclaimer() + public void Parse_SkipsLeadingBlankRows_AndStopsBeforeFooterDisclaimer_ForTradingHeader() { const string csv = """ -Run Date,Action,Symbol,Description,Type,Price,Quantity,Commission,Fees,Amount -3/1/26,You bought,ABC,Alpha Inc,Common Stock,10.50,2,0,0,-21.00 -3/2/26,Dividend Received,XYZ,XYZ Dividend,Cash,0,0,0,0,5.25 +Run Date,Action,Symbol,Description,Type,Exchange Quantity,Exchange Currency,Currency,Price,Quantity,Exchange Rate,Commission,Fees,Accrued Interest,Amount,Cash Balance,Settlement Date +3/1/26,You bought,ABC,Alpha Inc,Common Stock,,,USD,10.50,2,,0,0,0,-21.00,100.00,3/3/26 +3/2/26,Dividend Received,XYZ,XYZ Dividend,Cash,,,USD,0,0,,0,0,0,5.25,105.25,3/2/26 The data in this file is for informational purposes only. @@ -46,5 +46,73 @@ public void Parse_SkipsLeadingBlankRows_AndStopsBeforeFooterDisclaimer() result.Transactions[0].Amount.Should().Be(21.00m); result.Transactions[1].Security!.Id.Should().Be("XYZ"); result.Transactions[1].Amount.Should().Be(5.25m); + result.Transactions[0].Currency.Should().Be("USD"); + } + + [Fact] + public void Parse_HandlesIraHeaderVariant_AndDefaultsCurrency() + { + const string csv = """ + +Run Date,Action,Symbol,Description,Type,Price ($),Quantity,Commission ($),Fees ($),Accrued Interest ($),Amount ($),Cash Balance ($),Settlement Date +3/10/26,You bought,VOO,Vanguard 500 Index Fund,Mutual Fund,510.25,1.5,0,0,0,-765.38,1200.00,3/11/26 +3/12/26,Dividend Received,VOO,Vanguard 500 Index Fund,Cash,0,0,0,0,0,7.42,1207.42,3/12/26 +"""; + + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(csv)); + + var parser = new FidelityParser(); + var ctx = new ParserContext + { + AccountId = "acct-ira", + Institution = "fidelity", + CurrencyDefault = "USD", + DateParser = new DateParser(), + AmountParser = new AmountParser(), + FitIdGenerator = new FitIdGenerator(), + SubacctResolver = new SubacctResolver(), + SecurityResolver = new SecurityResolver(preferCusip: false) + }; + + var result = parser.Parse(new RawStatement("fidelity", stream, ".csv"), ctx); + + result.Transactions.Should().HaveCount(2); + result.Transactions[0].Security!.Id.Should().Be("VOO"); + result.Transactions[0].UnitPrice.Should().Be(510.25m); + result.Transactions[0].Amount.Should().Be(765.38m); + result.Transactions[0].Currency.Should().Be("USD"); + result.Transactions[1].Amount.Should().Be(7.42m); + result.Transactions[1].Memo.Should().Be("Dividend Received - VOO"); + } + + [Fact] + public void Parse_StillHandlesSimplifiedTradingHeaders() + { + const string csv = """ +Run Date,Action,Symbol,Description,Type,Price,Quantity,Commission,Fees,Amount +3/15/26,You sold,MSFT,Microsoft Corp,Common Stock,420.10,1,0,0,420.10 +"""; + + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(csv)); + + var parser = new FidelityParser(); + var ctx = new ParserContext + { + AccountId = "acct-simple", + Institution = "fidelity", + CurrencyDefault = "USD", + DateParser = new DateParser(), + AmountParser = new AmountParser(), + FitIdGenerator = new FitIdGenerator(), + SubacctResolver = new SubacctResolver(), + SecurityResolver = new SecurityResolver(preferCusip: false) + }; + + var result = parser.Parse(new RawStatement("fidelity", stream, ".csv"), ctx); + + result.Transactions.Should().HaveCount(1); + result.Transactions[0].Security!.Id.Should().Be("MSFT"); + result.Transactions[0].Action.Should().Be(CanonicalAction.SellStock); + result.Transactions[0].Currency.Should().Be("USD"); } }