From 2af677a4b6c5df66fd75e6b39a6038a5a9e3aeb9 Mon Sep 17 00:00:00 2001 From: DoLearnWhileAlive Date: Tue, 26 May 2026 15:28:24 +0200 Subject: [PATCH 1/6] =?UTF-8?q?Refactored=20`Merge-DatumArray`=20for=20per?= =?UTF-8?q?formance=20optimization=20-=20Replaced=20O(n=C2=B2)=20nested=20?= =?UTF-8?q?loop=20comparisons=20using=20`Compare-Hashtable`=20with=20=20?= =?UTF-8?q?=20O(n+m)=20hash-based=20indexing=20-=20Pre-computed=20knockout?= =?UTF-8?q?=20reference=20items=20to=20avoid=20repeated=20checks=20during?= =?UTF-8?q?=20=20=20merge=20-=20Changed=20output=20type=20from=20`[System.?= =?UTF-8?q?Collections.ArrayList]`=20to=20=20=20`[System.Collections.Gener?= =?UTF-8?q?ic.List[object]]`=20-=20Added=20private=20function=20`Get-Datum?= =?UTF-8?q?TupleKeyValueString`=20for=20consistent=20=20=20composite=20key?= =?UTF-8?q?=20generation=20-=20Added=20private=20function=20`Test-DatumKno?= =?UTF-8?q?ckout`=20for=20efficient=20knockout=20=20=20matching=20-=20Remo?= =?UTF-8?q?ved=20private=20function=20`Compare-Hashtable.ps1`=20(no=20long?= =?UTF-8?q?er=20needed)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 10 + .../Private/Get-DatumTupleKeyValueString.ps1 | 20 + source/Private/Merge-DatumArray.ps1 | 382 +++++++++--------- source/Private/Test-DatumKnockout.ps1 | 21 + 4 files changed, 243 insertions(+), 190 deletions(-) create mode 100644 source/Private/Get-DatumTupleKeyValueString.ps1 create mode 100644 source/Private/Test-DatumKnockout.ps1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 65cee6c..e1fe62b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 evaluate to `$null` or empty strings are now silently skipped instead of causing lookup errors. - Added knockout support for hashtable array items. +- Added private function `Get-DatumTupleKeyValueString` for consistent composite key generation +- Added private function `Test-DatumKnockout` for efficient knockout matching ### Changed @@ -52,6 +54,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 using an InvokeCommand expression that returns a path for some nodes and nothing for others. - Updated build pipeline to also test against Linux and MacOS +- Refactored `Merge-DatumArray` for performance optimization: + - Replaced O(n²) nested loop comparisons using `Compare-Hashtable` with O(n+m) hash-based indexing + - Pre-computed knockout reference items to avoid repeated checks during merge + - Changed output type from `[System.Collections.ArrayList]` to `[System.Collections.Generic.List[object]]` ### Fixed @@ -76,6 +82,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `lookup_options`. - Fixed issues running integration tests with PowerShell on Linux. +### Removed + +- Removed private function `Compare-Hashtable.ps1` (no longer needed) + ## [0.41.0] - 2026-02-03 ### Added diff --git a/source/Private/Get-DatumTupleKeyValueString.ps1 b/source/Private/Get-DatumTupleKeyValueString.ps1 new file mode 100644 index 0000000..ef25c92 --- /dev/null +++ b/source/Private/Get-DatumTupleKeyValueString.ps1 @@ -0,0 +1,20 @@ +function Get-DatumTupleKeyValueString +{ + [OutputType([string])] + [CmdletBinding()] + param( + [hashtable]$Item, + [string[]]$Keys + ) + if (-not $Keys) + { + $Keys = $Item.Keys + } + + $sep = '#' + $values = foreach ($k in $Keys) + { + $Item[$k] + } + return ($values -join $sep) +} diff --git a/source/Private/Merge-DatumArray.ps1 b/source/Private/Merge-DatumArray.ps1 index e5cfc0c..361cdc6 100644 --- a/source/Private/Merge-DatumArray.ps1 +++ b/source/Private/Merge-DatumArray.ps1 @@ -1,6 +1,6 @@ function Merge-DatumArray { - [OutputType([System.Collections.ArrayList])] + [OutputType([System.Collections.Generic.List[object]])] [CmdletBinding()] param ( [Parameter(Mandatory = $true)] @@ -29,253 +29,255 @@ function Merge-DatumArray Write-Debug -Message "`tMerge-DatumArray -StartingPath <$StartingPath>" $hashArrayStrategy = $Strategy.merge_hash_array Write-Debug -Message "`t`tHash Array Strategy: $hashArrayStrategy" - $mergeBasetypeArraysStrategy = $Strategy.merge_basetype_array - $mergedArray = [System.Collections.ArrayList]::new() - $sortParams = @{} - if ($propertyNames = [string[]]$Strategy.merge_options.tuple_keys) + $tupleKeys = if ($Strategy.merge_options.tuple_keys) { - $sortParams.Add('Property', $propertyNames) + [string[]]$Strategy.merge_options.tuple_keys + } + else + { + @() } - if ($ReferenceArray -as [hashtable[]]) + $mergedArray = [System.Collections.Generic.List[object]]::new() + + # Early exit if not an array of hashtables + if (-not $ReferenceArray -as [hashtable[]]) { - Write-Debug -Message "`t`tMERGING Array of Hashtables" - if (-not $hashArrayStrategy -or $hashArrayStrategy -match 'MostSpecific') + return , $mergedArray + } + + # MostSpecific strategy: return reference array as-is (with optional sorting) + if (-not $hashArrayStrategy -or $hashArrayStrategy -match 'MostSpecific') + { + Write-Debug -Message "`t`tMerge_hash_arrays Disabled. value: $hashArrayStrategy" + if ($Strategy.sort_merged_arrays -and $tupleKeys.Count -gt 0) { - Write-Debug -Message "`t`tMerge_hash_arrays Disabled. value: $hashArrayStrategy" - $mergedArray = $ReferenceArray - if ($Strategy.sort_merged_arrays) - { - $mergedArray = $mergedArray | Sort-Object @sortParams - } - return $mergedArray + $ReferenceArray = $ReferenceArray | Sort-Object -Property $tupleKeys } + return , $ReferenceArray + } + + Write-Debug -Message "`t`tMERGING Array of Hashtables" - $knockedOutTupleKeyValues = [System.Collections.ArrayList]@() + # Precompute knockout regex for identifying knockout-prefixed values + $knockoutPrefixMatcher = if ($Strategy.merge_options.knockout_prefix) + { + '^' + [regex]::Escape($Strategy.merge_options.knockout_prefix) + } + else + { + $null + } + + $result = $null + + # Precompute list of knockout reference items + # Stores only properties with knockout values to efficiently check during merge + $knockoutReferenceItems = @( foreach ($referenceItem in $ReferenceArray) { - $currentRefItem = [ordered]@{} + $referenceItem - - # make sure property values are converted before merge - $result = $null - foreach ($prop in $propertyNames.Where{ $currentRefItem.Contains($_) }) + $knockoutItem = @{} + foreach ($prop in $tupleKeys) { - if (Invoke-DatumHandler -InputObject $currentRefItem[$prop] -DatumHandlers $Datum.__Definition.DatumHandlers -Result ([ref]$result)) + if ($referenceItem.Contains($prop)) { - $currentRefItem[$prop] = ConvertTo-Datum -InputObject $result -DatumHandlers $Datum.__Definition.DatumHandlers + # Make sure property values are converted before comparing + if (Invoke-DatumHandler -InputObject $referenceItem[$prop] -DatumHandlers $Datum.__Definition.DatumHandlers -Result ([ref]$result)) + { + $referenceItem[$prop] = ConvertTo-Datum -InputObject $result -DatumHandlers $Datum.__Definition.DatumHandlers + } + + if ($knockoutPrefixMatcher) + { + if ($referenceItem[$prop] -match $knockoutPrefixMatcher) + { + $knockoutItem[$prop] = $referenceItem[$prop] + } + } + } + if ($knockoutItem.Count -gt 0) + { + $knockoutItem } } + } + ) - if ($knockoutPrefixMatcher = $Strategy.merge_options.knockout_prefix) + switch -Regex ($hashArrayStrategy) + { + # Sum/Add strategy: combine all items from both arrays + '^Sum|^Add' + { + foreach ($referenceItem in $ReferenceArray) + { + $mergedArray.Add($referenceItem) + } + foreach ($differenceItem in $DifferenceArray) { - $knockoutPrefixMatcher = [regex]::Escape($Strategy.merge_options.knockout_prefix).Insert(0, '^') + $mergedArray.Add($differenceItem) + } + } - if ($tupleKeyNames = [string[]]$strategy.merge_options.tuple_keys) + # Deep/Merge strategy: merge items with matching tuple keys + '^Deep|^Merge' + { + Write-Debug -Message "`t`t`tStrategy for Array Items: Merge Hash By tuple`r`n" + + # Build differenceItem index hashtable for O(1) lookups + # Key = composite tuple key string, Value = the difference item + $diffIndex = @{} + foreach ($differenceItem in $DifferenceArray) + { + # Make sure property values are converted before comparing + foreach ($prop in $tupleKeys) { - if ($currentRefItemKeysWithKnockOutValues = $currentRefItem.Keys.Where{ $_ -in $tupleKeyNames -and $currentRefItem[$_] -match $knockoutPrefixMatcher }) + if ($differenceItem.Contains($prop)) { - $ht = @{} - foreach ($key in $currentRefItemKeysWithKnockOutValues) + if (Invoke-DatumHandler -InputObject $differenceItem[$prop] -DatumHandlers $Datum.__Definition.DatumHandlers -Result ([ref]$result)) { - $ht.$key = $currentRefItem.$key -replace $knockoutPrefixMatcher + $differenceItem[$prop] = ConvertTo-Datum -InputObject $result -DatumHandlers $Datum.__Definition.DatumHandlers } - - $null = $knockedOutTupleKeyValues.Add($ht) } } + $key = Get-DatumTupleKeyValueString $differenceItem $tupleKeys + $diffIndex[$key] = $differenceItem } - } - switch -Regex ($hashArrayStrategy) - { - '^Sum|^Add' + # Track which difference keys have been used for merging + $usedDiffKeys = @{} + + # Process reference items: merge with matching difference items or keep as-is + foreach ($referenceItem in $ReferenceArray) { - foreach ($referenceItem in $ReferenceArray) + # Determine which keys to use for comparison + $compareKeys = if ($tupleKeys.Count -gt 0) + { + $tupleKeys + } + else { - $null = $mergedArray.Add(([ordered]@{} + $referenceItem)) + Write-Debug -Message "`t`t`t ..No PropertyName defined: Use ReferenceItem Keys" + $referenceItem.Keys + } + $key = Get-DatumTupleKeyValueString $referenceItem $compareKeys + + if ($diffIndex.ContainsKey($key)) + { + # Match found - merge the items + $usedDiffKeys[$key] = $true + $paramsMergeHt = @{ + ParentPath = $StartingPath + Strategy = $Strategy + ReferenceHashtable = $referenceItem + DifferenceHashtable = $diffIndex[$key] + ChildStrategies = $ChildStrategies + } + $mergedArray.Add((Merge-Hashtable @paramsMergeHt)) } - foreach ($differenceItem in $DifferenceArray) + else { - $null = $mergedArray.Add(([ordered]@{} + $differenceItem)) + # No match - keep reference item as-is + $mergedArray.Add($referenceItem) } } - # MergeHashesByProperties - '^Deep|^Merge' + # Process remaining difference items that weren't merged + foreach ($differenceItem in $DifferenceArray) { - Write-Debug -Message "`t`t`tStrategy for Array Items: Merge Hash By tuple`r`n" - # look at each $RefItems in $RefArray - # if no PropertyNames defined, use all Properties of $RefItem - # else use defined propertyNames - # Search for DiffItem that has the same Property/Value pairs - # if found, Merge-Datum (or MergeHashtable?) - # if not found, add $DiffItem to $RefArray - - # look at each $RefItems in $RefArray - $usedOrKnockedOutDiffItems = [System.Collections.ArrayList]@() - foreach ($referenceItem in $ReferenceArray) + # Check if this item should be knocked out + $shouldKnockout = $false + foreach ($knockoutReferenceItem in $knockoutReferenceItems) { - $referenceItem = [ordered]@{} + $referenceItem - Write-Debug -Message "`t`t`t .. Working on Merged Element $($mergedArray.Count)`r`n" - # if no PropertyNames defined, use all Properties of $RefItem - if (-not $propertyNames) + if (Test-DatumKnockout -DiffItem $differenceItem -RefKnockoutItem $knockoutReferenceItem -KnockoutPrefixMatcher $knockoutPrefixMatcher) { - Write-Debug -Message "`t`t`t ..No PropertyName defined: Use ReferenceItem Keys" - $propertyNames = $referenceItem.Keys + $shouldKnockout = $true + break } - $mergedItem = @{} + $referenceItem - $diffItemsToMerge = $DifferenceArray.Where{ - $differenceItem = [ordered]@{} + $_ - # make sure property values are converted before merge - $result = $null - foreach ($prop in $propertyNames) - { - if ($differenceItem.Contains($prop)) - { - if (Invoke-DatumHandler -InputObject $differenceItem.$prop -DatumHandlers $Datum.__Definition.DatumHandlers -Result ([ref]$result)) - { - $differenceItem.$prop = ConvertTo-Datum -InputObject $result -DatumHandlers $Datum.__Definition.DatumHandlers - } - } - } - $itemKnockedOut = $false - foreach ($knockedOutTupleKeyValue in $knockedOutTupleKeyValues) - { - $filterStrings = foreach ($knockedOutTupleKeyValueKey in $knockedOutTupleKeyValue.Keys) - { - "`$knockedOutTupleKeyValue.'$knockedOutTupleKeyValueKey' -eq `$differenceItem.'$knockedOutTupleKeyValueKey'" - } - $filterScript = [scriptblock]::Create($filterStrings -join ' -and ') - if ( &$filterScript ) - { - $null = $usedOrKnockedOutDiffItems.Add($_) - $itemKnockedOut = $true - break - } - } - if ($itemKnockedOut -eq $false) - { - # Search for DiffItem that has the same Property/Value pairs than RefItem - $compareHashParams = @{ - ReferenceHashtable = [ordered]@{} + $referenceItem - DifferenceHashtable = $differenceItem - Property = $propertyNames - } - (-not (Compare-Hashtable @compareHashParams)) - } - } - Write-Debug -Message "`t`t`t ..Items to merge: $($diffItemsToMerge.Count)" - $diffItemsToMerge | ForEach-Object { - $mergeItemsParams = @{ - ParentPath = $StartingPath - Strategy = $Strategy - ReferenceHashtable = $mergedItem - DifferenceHashtable = $_ - ChildStrategies = $ChildStrategies - } - $mergedItem = Merge-Hashtable @mergeItemsParams - } - # If a diff Item has been used, save it to find the unused ones - $null = $usedOrKnockedOutDiffItems.AddRange($diffItemsToMerge) - $null = $mergedArray.Add($mergedItem) } - $unMergedItems = $DifferenceArray | ForEach-Object { - if (-not $usedOrKnockedOutDiffItems.Contains($_)) - { - ([ordered]@{} + $_) - } + if ($shouldKnockout) + { + continue } - if ($null -ne $unMergedItems) + + $key = Get-DatumTupleKeyValueString $differenceItem $tupleKeys + # Only add if not already merged with a reference item + if (-not $usedDiffKeys.ContainsKey($key)) { - if ($unMergedItems -is [System.Array]) - { - $null = $mergedArray.AddRange($unMergedItems) - } - else - { - $null = $mergedArray.Add($unMergedItems) - } + $mergedArray.Add($differenceItem) } } + } - # UniqueByProperties - '^Unique' - { - Write-Debug -Message "`t`t`tSelecting Unique Hashes accross both arrays based on Property tuples" - # look at each $DiffItems in $DiffArray - # if no PropertyNames defined, use all Properties of $DiffItem - # else use defined PropertyNames - # Search for a RefItem that has the same Property/Value pairs - # if Nothing is found - # add current DiffItem to RefArray + # Unique strategy: keep only unique items across both arrays based on tuple keys + '^Unique' + { + Write-Debug -Message "`t`t`tSelecting Unique Hashes accross both arrays based on Property tuples" + + # Track seen keys to ensure uniqueness + $seenKeys = @{} - if (-not $propertyNames) + # Process reference items first + foreach ($referenceItem in $ReferenceArray) + { + # Determine which keys to use for comparison + $compareKeys = if ($tupleKeys.Count -gt 0) + { + $tupleKeys + } + else { Write-Debug -Message "`t`t`t ..No PropertyName defined: Use ReferenceItem Keys" - $propertyNames = $referenceItem.Keys + $referenceItem.Keys } + $key = Get-DatumTupleKeyValueString $referenceItem $compareKeys + if (-not $seenKeys.ContainsKey($key)) + { + $seenKeys[$key] = $true + $mergedArray.Add($referenceItem) + } + } - $mergedArray = [System.Collections.ArrayList]::new() - $ReferenceArray | ForEach-Object { - $currentRefItem = [ordered]@{} + $_ - # make sure property values are converted before merge - $result = $null - foreach ($prop in $propertyNames) + # Process difference items, skipping knockouts and duplicates + foreach ($differenceItem in $DifferenceArray) + { + # Make sure property values are converted before comparing + foreach ($prop in $tupleKeys) + { + if ($differenceItem.Contains($prop)) { - if ($currentRefItem.Contains($prop)) + if (Invoke-DatumHandler -InputObject $differenceItem[$prop] -DatumHandlers $Datum.__Definition.DatumHandlers -Result ([ref]$result)) { - if (Invoke-DatumHandler -InputObject $currentRefItem.$prop -DatumHandlers $Datum.__Definition.DatumHandlers -Result ([ref]$result)) - { - $currentRefItem.$prop = ConvertTo-Datum -InputObject $result -DatumHandlers $Datum.__Definition.DatumHandlers - } + $differenceItem[$prop] = ConvertTo-Datum -InputObject $result -DatumHandlers $Datum.__Definition.DatumHandlers } } - if (-not ($mergedArray.Where{ -not (Compare-Hashtable -Property $propertyNames -ReferenceHashtable $currentRefItem -DifferenceHashtable $_ ) })) - { - $null = $mergedArray.Add(([ordered]@{} + $_)) - } } - $DifferenceArray | ForEach-Object { - $currentDiffItem = [ordered]@{} + $_ - # make sure property values are converted before merge - $result = $null - foreach ($prop in $propertyNames) - { - if ($currentDiffItem.Contains($prop)) - { - if (Invoke-DatumHandler -InputObject $currentDiffItem.$prop -DatumHandlers $Datum.__Definition.DatumHandlers -Result ([ref]$result)) - { - $currentDiffItem.$prop = ConvertTo-Datum -InputObject $result -DatumHandlers $Datum.__Definition.DatumHandlers - } - } - } - $itemKnockedOut = $false - foreach ($knockedOutTupleKeyValue in $knockedOutTupleKeyValues) - { - $filterStrings = foreach ($knockedOutTupleKeyValueKey in $knockedOutTupleKeyValue.Keys) - { - "`$knockedOutTupleKeyValue.'$knockedOutTupleKeyValueKey' -eq `$currentDiffItem.'$knockedOutTupleKeyValueKey'" - } - $filterScript = [scriptblock]::Create($filterStrings -join ' -and ') - if ( &$filterScript ) - { - $itemKnockedOut = $true - break - } - } - if ($itemKnockedOut -eq $false) + # Check if this item should be knocked out + $shouldKnockout = $false + foreach ($knockoutReferenceItem in $knockoutReferenceItems) + { + if (Test-DatumKnockout -DiffItem $differenceItem -RefKnockoutItem $knockoutReferenceItem -KnockoutPrefixMatcher $knockoutPrefixMatcher) { - if (-not ($mergedArray.Where{ -not (Compare-Hashtable -Property $propertyNames -ReferenceHashtable $currentDiffItem -DifferenceHashtable $_ ) })) - { - $null = $mergedArray.Add(([ordered]@{} + $_)) - } + $shouldKnockout = $true + break } } + if ($shouldKnockout) + { + continue + } + + $key = Get-DatumTupleKeyValueString $differenceItem $tupleKeys + # Only add if not already seen (ensures uniqueness) + if (-not $seenKeys.ContainsKey($key)) + { + $seenKeys[$key] = $true + $mergedArray.Add($differenceItem) + } } } } - return (, $mergedArray) + return , $mergedArray } diff --git a/source/Private/Test-DatumKnockout.ps1 b/source/Private/Test-DatumKnockout.ps1 new file mode 100644 index 0000000..c7c3f0e --- /dev/null +++ b/source/Private/Test-DatumKnockout.ps1 @@ -0,0 +1,21 @@ +function Test-DatumKnockout +{ + [OutputType([bool])] + [CmdletBinding()] + param( + [hashtable]$DiffItem, + [hashtable]$RefKnockoutItem, + [regex]$KnockoutPrefixMatcher + ) + + foreach ($k in $RefKnockoutItem.Keys) + { + $refVal = $RefKnockoutItem[$k] -replace $KnockoutPrefixMatcher + $diffVal = $DiffItem[$k] + if ($diffVal -ne $refVal) + { + return $false + } + } + return $true +} From b00f776a26e677cdef13e6af7ae2a1e790e2a838 Mon Sep 17 00:00:00 2001 From: DoLearnWhileAlive Date: Thu, 28 May 2026 14:09:58 +0200 Subject: [PATCH 2/6] Updated RsopWithInvokCommandHandler integration tests to ensure also node specific Datum handler values from lower layers are merged correctly. --- CHANGELOG.md | 4 +- .../RsopWithInvokCommandHandler.tests.ps1 | 46 ++++++++++++------- .../AllNodes/DSCFile01.yml | 6 +-- .../AllNodes/DSCWeb01.yml | 2 +- .../AllNodes/DSCWeb02.yml | 2 +- .../Baselines/ServerBaseline.yml | 7 +-- .../Roles/FileServer.yml | 2 +- .../Roles/WebServer.yml | 2 +- 8 files changed, 43 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1fe62b..53d090b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,7 +58,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Replaced O(n²) nested loop comparisons using `Compare-Hashtable` with O(n+m) hash-based indexing - Pre-computed knockout reference items to avoid repeated checks during merge - Changed output type from `[System.Collections.ArrayList]` to `[System.Collections.Generic.List[object]]` - + - Updated RsopWithInvokCommandHandler integration tests to ensure also + node specific Datum handler values from lower layers are merged + correctly. ### Fixed - Fix `ConvertTo-Json` truncation warnings for deep data structures diff --git a/tests/Integration/RsopWithInvokCommandHandler.tests.ps1 b/tests/Integration/RsopWithInvokCommandHandler.tests.ps1 index 8941738..aeedd1a 100644 --- a/tests/Integration/RsopWithInvokCommandHandler.tests.ps1 +++ b/tests/Integration/RsopWithInvokCommandHandler.tests.ps1 @@ -55,18 +55,18 @@ Describe "RSOP tests based on 'MergeTestDataWithInvokCommandHandler' test data" } @{ Node = 'DSCFile01' - PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 1"}.Destination' - Value = '192.168.11.0/24', '192.168.22.0/24', '192.168.33.0/24', '192.168.10.0/24', '192.168.20.0/24', '192.168.30.0/24', '192.168.40.0/24', '192.168.50.0/24', '192.168.60.0/24' + PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 1 on DSCFile01"}.Destination' + Value = '192.168.11.0/24', '192.168.22.0/24', '192.168.33.0/24', '192.168.10.0/24', '192.168.20.0/24', '192.168.30.0/24', '192.168.40.0/24', '192.168.50.0/24', '192.168.60.0/24', '192.168.90.0/24' } @{ Node = 'DSCWeb01' - PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 1"}.Destination' - Value = '192.168.12.0/24', '192.168.23.0/24', '192.168.34.0/24', '192.168.40.0/24', '192.168.50.0/24', '192.168.60.0/24' + PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 1 on DSCWeb01"}.Destination' + Value = '192.168.12.0/24', '192.168.23.0/24', '192.168.34.0/24', '192.168.40.0/24', '192.168.50.0/24', '192.168.60.0/24', '192.168.80.0/24' } @{ Node = 'DSCWeb02' - PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 1"}.Destination' - Value = '192.168.12.0/24', '192.168.23.0/24', '192.168.34.0/24', '192.168.10.0/24', '192.168.20.0/24', '192.168.30.0/24', '192.168.40.0/24', '192.168.50.0/24', '192.168.60.0/24' + PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 1 on DSCWeb02"}.Destination' + Value = '192.168.12.0/24', '192.168.23.0/24', '192.168.34.0/24', '192.168.10.0/24', '192.168.20.0/24', '192.168.30.0/24', '192.168.40.0/24', '192.168.50.0/24', '192.168.60.0/24', '192.168.80.0/24' } @{ Node = 'DSCFile01' @@ -107,39 +107,39 @@ Describe "RSOP tests based on 'MergeTestDataWithInvokCommandHandler' test data" #DSCFile01 @{ Node = 'DSCFile01' - PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 1"}.IpAddress' + PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 1 on DSCFile01"}.IpAddress' Value = '192.168.10.100' } @{ Node = 'DSCFile01' - PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 1"}.Gateway' + PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 1 on DSCFile01"}.Gateway' Value = '192.168.10.50' } @{ Node = 'DSCFile01' - PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 2"}.IpAddress' + PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 2 on DSCFile01"}.IpAddress' Value = '192.168.20.100' } @{ Node = 'DSCFile01' - PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 2"}.Gateway' + PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 2 on DSCFile01"}.Gateway' Value = '192.168.20.50' } @{ Node = 'DSCFile01' - PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 3"}.IpAddress' + PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 3 on DSCFile01"}.IpAddress' Value = '192.168.30.100' } @{ Node = 'DSCFile01' - PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 3"}.Gateway' + PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 3 on DSCFile01"}.Gateway' Value = '192.168.30.1' } @{ Node = 'DSCFile01' - PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 3"}.DnsServer' + PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 3 on DSCFile01"}.DnsServer' Value = '192.168.30.10', '192.168.30.20' } @{ @@ -151,24 +151,36 @@ Describe "RSOP tests based on 'MergeTestDataWithInvokCommandHandler' test data" #DSCWeb01 @{ Node = 'DSCWeb01' - PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 1"}.IpAddress' + PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 1 on DSCWeb01"}.IpAddress' Value = '192.168.10.101' } @{ Node = 'DSCWeb01' - PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 1"}.Gateway' + PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 1 on DSCWeb01"}.Gateway' Value = $null } + @{ + Node = 'DSCWeb01' + PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Count' + Value = '3' + } + + #DSCWeb02 @{ Node = 'DSCWeb02' - PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 1"}.IpAddress' + PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 1 on DSCWeb02"}.IpAddress' Value = '192.168.10.102' } @{ Node = 'DSCWeb02' - PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 1"}.Gateway' + PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Where{$_.InterfaceAlias -eq "Ethernet 1 on DSCWeb02"}.Gateway' Value = '192.168.10.50' } + @{ + Node = 'DSCWeb02' + PropertyPath = 'NetworkIpConfigurationMerged.Interfaces.Count' + Value = '3' + } ) It "The value of Datum RSOP property '' for node '' should be ''." -ForEach $script:testCases { diff --git a/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/AllNodes/DSCFile01.yml b/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/AllNodes/DSCFile01.yml index 1ad19bf..d3e0b22 100644 --- a/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/AllNodes/DSCFile01.yml +++ b/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/AllNodes/DSCFile01.yml @@ -8,11 +8,11 @@ SomeData: '[Command=Get-Random]' NetworkIpConfigurationMerged: Interfaces: - - InterfaceAlias: Ethernet 1 + - InterfaceAlias: Ethernet 1 on DSCFile01 IpAddress: 192.168.10.100 - - InterfaceAlias: Ethernet 2 + - InterfaceAlias: Ethernet 2 on DSCFile01 IpAddress: 192.168.20.100 - - InterfaceAlias: Ethernet 3 + - InterfaceAlias: Ethernet 3 on DSCFile01 IpAddress: 192.168.30.100 Gateway: 192.168.30.1 DnsServer: diff --git a/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/AllNodes/DSCWeb01.yml b/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/AllNodes/DSCWeb01.yml index dc6a7e1..08783f6 100644 --- a/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/AllNodes/DSCWeb01.yml +++ b/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/AllNodes/DSCWeb01.yml @@ -9,7 +9,7 @@ NetworkIpConfigurationMerged: ConfigureIPv6: 2 --DisableNetBios: null Interfaces: - - InterfaceAlias: Ethernet 1 + - InterfaceAlias: Ethernet 1 on DSCWeb01 IpAddress: 192.168.10.101 --Gateway: null Destination: diff --git a/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/AllNodes/DSCWeb02.yml b/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/AllNodes/DSCWeb02.yml index 415a375..ea5eea9 100644 --- a/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/AllNodes/DSCWeb02.yml +++ b/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/AllNodes/DSCWeb02.yml @@ -14,7 +14,7 @@ Configurations: NetworkIpConfigurationMerged: Interfaces: - - InterfaceAlias: Ethernet 1 + - InterfaceAlias: 'Ethernet 1 on DSCWeb02' IpAddress: 192.168.10.102 NetworkIpConfigurationNonMerged: diff --git a/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/Baselines/ServerBaseline.yml b/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/Baselines/ServerBaseline.yml index 93b5cb9..b32baee 100644 --- a/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/Baselines/ServerBaseline.yml +++ b/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/Baselines/ServerBaseline.yml @@ -10,7 +10,7 @@ NetworkIpConfigurationMerged: ConfigureIPv6: 0 DisableNetBios: true Interfaces: - - InterfaceAlias: Ethernet 1 + - InterfaceAlias: '[x={ "Ethernet 1 on $($Node.Name)" }=]' Prefix: 24 Gateway: 192.168.10.50 DnsServer: 192.168.10.10 @@ -22,14 +22,15 @@ NetworkIpConfigurationMerged: - 192.168.40.0/24 - 192.168.50.0/24 - 192.168.60.0/24 - - InterfaceAlias: Ethernet 2 + - '[x={ "192.168.{0}0.0/24" -f $Node.Name.Length }=]' + - InterfaceAlias: '[x={ "Ethernet 2 on $($Node.Name)" }=]' Prefix: 24 Gateway: 192.168.20.50 DnsServer: - 192.168.20.10 - 192.168.20.11 DisableNetbios: true - - InterfaceAlias: '[x={ "Ethernet 3" }=]' + - InterfaceAlias: '[x={ "Ethernet 3 on $($Node.Name)" }=]' Prefix: 24 Gateway: 192.168.30.50 DnsServer: diff --git a/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/Roles/FileServer.yml b/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/Roles/FileServer.yml index b529a9d..e0b78ff 100644 --- a/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/Roles/FileServer.yml +++ b/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/Roles/FileServer.yml @@ -14,7 +14,7 @@ FilesAndFolders: NetworkIpConfigurationMerged: ConfigureIPv6: -1 Interfaces: - - InterfaceAlias: Ethernet 1 + - InterfaceAlias: '[x={ "Ethernet 1 on $($Node.Name)" }=]' Prefix: 24 Gateway: 192.168.10.50 DnsServer: 192.168.10.10 diff --git a/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/Roles/WebServer.yml b/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/Roles/WebServer.yml index 42e8e42..90907be 100644 --- a/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/Roles/WebServer.yml +++ b/tests/Integration/assets/MergeTestDataWithInvokCommandHandler/Roles/WebServer.yml @@ -9,7 +9,7 @@ WindowsFeatures: NetworkIpConfigurationMerged: DisableNetBios: false Interfaces: - - InterfaceAlias: Ethernet 1 + - InterfaceAlias: '[x={ "Ethernet 1 on $($Node.Name)" }=]' Prefix: 24 Gateway: 192.168.10.50 DnsServer: 192.168.10.10 From 072c0201a8deccf4eb20d980b7247be3ab9318d0 Mon Sep 17 00:00:00 2001 From: DoLearnWhileAlive Date: Thu, 28 May 2026 14:11:13 +0200 Subject: [PATCH 3/6] Remove 'Known Skipped Tests' section from systemPatterns in memory-bank since bug is fixed and tests are enabled --- memory-bank/systemPatterns.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/memory-bank/systemPatterns.md b/memory-bank/systemPatterns.md index 6da58e2..1eea61d 100644 --- a/memory-bank/systemPatterns.md +++ b/memory-bank/systemPatterns.md @@ -133,12 +133,6 @@ tests/ - **DscWorkshopConfigData**: Full hierarchy with ProtectedData credentials + domain join - **Demo3**: Node1, Node2, Node3 — simple role merge + `$false` value test -### Known Skipped Tests (3 tests) -All in `RsopWithInvokCommandHandler.tests.ps1`, tagged with "There is a bug in the merge logic": -- Ethernet 3 Gateway for DSCFile01 -- Ethernet 3 DnsServer for DSCFile01 -- Interface Count for DSCFile01 - ## Build System - **Sampler-based**: Uses the Sampler module for build/test/publish pipeline - **ModuleBuilder**: Merges source files into single module output From 0378cf1065a04b8c9e7b1cce4785f5b3b5365ae2 Mon Sep 17 00:00:00 2001 From: DoLearnWhileAlive Date: Thu, 28 May 2026 14:13:18 +0200 Subject: [PATCH 4/6] Fix regression by shallow copy reference and difference items again otherwise follow-up nodes merge the same tuple key values as the first node. --- source/Private/Merge-DatumArray.ps1 | 73 +++++++++++++++++------------ 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/source/Private/Merge-DatumArray.ps1 b/source/Private/Merge-DatumArray.ps1 index 361cdc6..92cdf25 100644 --- a/source/Private/Merge-DatumArray.ps1 +++ b/source/Private/Merge-DatumArray.ps1 @@ -60,6 +60,50 @@ function Merge-DatumArray Write-Debug -Message "`t`tMERGING Array of Hashtables" + $ReferenceArray = @( + foreach ($referenceItem in $ReferenceArray) + { + # Shallow copy reference items to avoid converted tuple key values + # land in original DatumTree. Otherwise follow-up nodes merge the + # same tuple key values as the first node. + $clonedReferenceItem = [ordered]@{} + $referenceItem + foreach ($prop in $tupleKeys) + { + if ($clonedReferenceItem.Contains($prop)) + { + # Make sure property values are converted before comparing + if (Invoke-DatumHandler -InputObject $clonedReferenceItem[$prop] -DatumHandlers $Datum.__Definition.DatumHandlers -Result ([ref]$result)) + { + $clonedReferenceItem[$prop] = ConvertTo-Datum -InputObject $result -DatumHandlers $Datum.__Definition.DatumHandlers + } + } + } + $clonedReferenceItem + } + ) + + $DifferenceArray = @( + foreach ($differenceItem in $DifferenceArray) + { + # Shallow copy difference items to avoid converted tuple key values + # land in original DatumTree. Otherwise follow-up nodes merge the + # same tuple key values as the first node. + $clonedDifferenceItem = [ordered]@{} + $differenceItem + foreach ($prop in $tupleKeys) + { + if ($clonedDifferenceItem.Contains($prop)) + { + # Make sure property values are converted before comparing + if (Invoke-DatumHandler -InputObject $clonedDifferenceItem[$prop] -DatumHandlers $Datum.__Definition.DatumHandlers -Result ([ref]$result)) + { + $clonedDifferenceItem[$prop] = ConvertTo-Datum -InputObject $result -DatumHandlers $Datum.__Definition.DatumHandlers + } + } + } + $clonedDifferenceItem + } + ) + # Precompute knockout regex for identifying knockout-prefixed values $knockoutPrefixMatcher = if ($Strategy.merge_options.knockout_prefix) { @@ -82,12 +126,6 @@ function Merge-DatumArray { if ($referenceItem.Contains($prop)) { - # Make sure property values are converted before comparing - if (Invoke-DatumHandler -InputObject $referenceItem[$prop] -DatumHandlers $Datum.__Definition.DatumHandlers -Result ([ref]$result)) - { - $referenceItem[$prop] = ConvertTo-Datum -InputObject $result -DatumHandlers $Datum.__Definition.DatumHandlers - } - if ($knockoutPrefixMatcher) { if ($referenceItem[$prop] -match $knockoutPrefixMatcher) @@ -129,17 +167,6 @@ function Merge-DatumArray $diffIndex = @{} foreach ($differenceItem in $DifferenceArray) { - # Make sure property values are converted before comparing - foreach ($prop in $tupleKeys) - { - if ($differenceItem.Contains($prop)) - { - if (Invoke-DatumHandler -InputObject $differenceItem[$prop] -DatumHandlers $Datum.__Definition.DatumHandlers -Result ([ref]$result)) - { - $differenceItem[$prop] = ConvertTo-Datum -InputObject $result -DatumHandlers $Datum.__Definition.DatumHandlers - } - } - } $key = Get-DatumTupleKeyValueString $differenceItem $tupleKeys $diffIndex[$key] = $differenceItem } @@ -241,18 +268,6 @@ function Merge-DatumArray # Process difference items, skipping knockouts and duplicates foreach ($differenceItem in $DifferenceArray) { - # Make sure property values are converted before comparing - foreach ($prop in $tupleKeys) - { - if ($differenceItem.Contains($prop)) - { - if (Invoke-DatumHandler -InputObject $differenceItem[$prop] -DatumHandlers $Datum.__Definition.DatumHandlers -Result ([ref]$result)) - { - $differenceItem[$prop] = ConvertTo-Datum -InputObject $result -DatumHandlers $Datum.__Definition.DatumHandlers - } - } - } - # Check if this item should be knocked out $shouldKnockout = $false foreach ($knockoutReferenceItem in $knockoutReferenceItems) From 3d9cfc52389a33f547b691bc7d5f5330d16891f1 Mon Sep 17 00:00:00 2001 From: DoLearnWhileAlive Date: Thu, 28 May 2026 14:14:30 +0200 Subject: [PATCH 5/6] Delete Compare-Hashtable.ps1 --- source/Private/Compare-Hashtable.ps1 | 80 ---------------------------- 1 file changed, 80 deletions(-) delete mode 100644 source/Private/Compare-Hashtable.ps1 diff --git a/source/Private/Compare-Hashtable.ps1 b/source/Private/Compare-Hashtable.ps1 deleted file mode 100644 index 65613de..0000000 --- a/source/Private/Compare-Hashtable.ps1 +++ /dev/null @@ -1,80 +0,0 @@ -function Compare-Hashtable -{ - [CmdletBinding()] - param ( - [Parameter(Mandatory = $true)] - [hashtable] - $ReferenceHashtable, - - [Parameter(Mandatory = $true)] - [hashtable] - $DifferenceHashtable, - - [Parameter()] - [string[]] - $Property = ($ReferenceHashtable.Keys + $DifferenceHashtable.Keys | Select-Object -Unique) - ) - - Write-Debug -Message "Compare-Hashtable -Ref @{$($ReferenceHashtable.keys -join ';')} -Diff @{$($DifferenceHashtable.keys -join ';')} -Property [$($Property -join ', ')]" - #Write-Debug -Message "REF:`r`n$($ReferenceHashtable | ConvertTo-Json)" - #Write-Debug -Message "DIFF:`r`n$($DifferenceHashtable | ConvertTo-Json)" - - foreach ($propertyName in $Property) - { - Write-Debug -Message " Testing <$propertyName>'s value" - if (($inRef = $ReferenceHashtable.Contains($propertyName)) -and - ($inDiff = $DifferenceHashtable.Contains($propertyName))) - { - if ($ReferenceHashtable[$propertyName] -as [hashtable[]] -or $DifferenceHashtable[$propertyName] -as [hashtable[]]) - { - if ((Compare-Hashtable -ReferenceHashtable $ReferenceHashtable[$propertyName] -DifferenceHashtable $DifferenceHashtable[$propertyName])) - { - Write-Debug -Message " Skipping $propertyName...." - # If compare returns something, they're not the same - continue - } - } - else - { - Write-Debug -Message "Comparing: $($ReferenceHashtable[$propertyName]) With $($DifferenceHashtable[$propertyName])" - if ($ReferenceHashtable[$propertyName] -ne $DifferenceHashtable[$propertyName]) - { - [PSCustomObject]@{ - SideIndicator = '<=' - PropertyName = $propertyName - Value = $ReferenceHashtable[$propertyName] - } - - [PSCustomObject]@{ - SideIndicator = '=>' - PropertyName = $propertyName - Value = $DifferenceHashtable[$propertyName] - } - } - } - } - else - { - Write-Debug -Message " Property $propertyName Not in one Side: Ref: [$($ReferenceHashtable.Keys -join ',')] | [$($DifferenceHashtable.Keys -join ',')]" - if ($inRef) - { - Write-Debug -Message "$propertyName found in Reference hashtable" - [PSCustomObject]@{ - SideIndicator = '<=' - PropertyName = $propertyName - Value = $ReferenceHashtable[$propertyName] - } - } - else - { - Write-Debug -Message "$propertyName found in Difference hashtable" - [PSCustomObject]@{ - SideIndicator = '=>' - PropertyName = $propertyName - Value = $DifferenceHashtable[$propertyName] - } - } - } - } - -} From 93991c8bcfc40a44fa05a2855b996b73582b94ed Mon Sep 17 00:00:00 2001 From: DoLearnWhileAlive Date: Sun, 31 May 2026 00:10:19 +0200 Subject: [PATCH 6/6] Fixed build issue with latest ModuleBuilder version by pinning to v3.1.8 --- CHANGELOG.md | 2 ++ RequiredModules.psd1 | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53d090b..1793757 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated RsopWithInvokCommandHandler integration tests to ensure also node specific Datum handler values from lower layers are merged correctly. + ### Fixed - Fix `ConvertTo-Json` truncation warnings for deep data structures @@ -83,6 +84,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 property separator under `ResolutionPrecedence` and `lookup_options`. - Fixed issues running integration tests with PowerShell on Linux. +- Fixed build issue with latest ModuleBuilder version by pinning to v3.1.8 ### Removed diff --git a/RequiredModules.psd1 b/RequiredModules.psd1 index 0ebd8e2..5eadba2 100644 --- a/RequiredModules.psd1 +++ b/RequiredModules.psd1 @@ -14,7 +14,7 @@ 'DscResource.AnalyzerRules' = 'latest' #'DscResource.Common' = 'latest' Plaster = 'latest' - ModuleBuilder = 'latest' + ModuleBuilder = '3.1.8' ChangelogManagement = 'latest' Sampler = 'latest' 'Sampler.GitHubTasks' = 'latest'