From ed572e47f9d20a4ffa40400cd0467f664f717f3e Mon Sep 17 00:00:00 2001 From: Maximum Code Date: Sat, 9 May 2026 22:02:08 -0700 Subject: [PATCH 1/8] Add Access-style C# code modules Introduce CSharpDB.CodeModules with database-owned module storage, workspace export/import, Roslyn build diagnostics, local trust, and Admin Forms runtime contracts. Wire Admin Forms events and selected-control events to optional CodeModuleHandler dispatch, add handler stub generation, Code Modules admin tab, and safe form command API support. Refresh UDF/Admin Forms documentation and add the CodeModules README plus a real-world website blog walkthrough. Also fix the extra nested code-block lip in generated www pages. --- CSharpDB.slnx | 2 + docs/admin-forms-access-parity/README.md | 12 +- .../access-style-functions-and-macros.md | 277 ++++---- docs/roadmap.md | 8 +- docs/trusted-csharp-functions/README.md | 46 +- docs/user-defined-functions/README.md | 66 ++ .../CSharpDB.Admin.Forms.csproj | 1 + .../Designer/ControlEventBindingsEditor.razor | 82 +++ .../Designer/FormEventBindingsEditor.razor | 82 +++ .../Components/Designer/FormRenderer.razor | 52 +- .../Designer/PropertyInspector.razor | 1 + .../Models/ControlEventBinding.cs | 4 +- .../Models/FormEventBinding.cs | 4 +- src/CSharpDB.Admin.Forms/README.md | 38 +- .../Services/AdminFormCodeModuleCommandApi.cs | 190 ++++++ .../AdminFormsServiceCollectionExtensions.cs | 9 + .../Services/DefaultFormEventDispatcher.cs | 63 +- .../Services/FormCodeModuleDesignerService.cs | 148 +++++ src/CSharpDB.Admin/CSharpDB.Admin.csproj | 1 + .../Components/Layout/CommandPalette.razor | 1 + .../Components/Layout/MainLayout.razor | 4 + .../Components/Layout/TitleBar.razor | 3 + .../Components/Tabs/CodeModulesTab.razor | 253 ++++++++ src/CSharpDB.Admin/Models/TabDescriptor.cs | 1 + src/CSharpDB.Admin/Program.cs | 3 + .../Services/TabManagerService.cs | 15 + .../CSharpDB.CodeModules.csproj | 21 + .../CSharpDbCodeModuleClient.cs | 614 ++++++++++++++++++ src/CSharpDB.CodeModules/CodeModuleModels.cs | 118 ++++ .../CodeModulesServiceCollectionExtensions.cs | 24 + .../Infrastructure/CodeModuleHashing.cs | 38 ++ .../Infrastructure/CodeModuleSql.cs | 73 +++ src/CSharpDB.CodeModules/README.md | 165 +++++ .../Runtime/CodeModuleFormDispatch.cs | 251 +++++++ .../Runtime/FormRuntimeContracts.cs | 218 +++++++ .../Trust/CodeModuleTrustStore.cs | 175 +++++ src/CSharpDB/CSharpDB.csproj | 1 + .../CSharpDB.Admin.Forms.Tests.csproj | 1 + .../Services/FormCodeModuleEventTests.cs | 247 +++++++ .../CSharpDB.CodeModules.Tests.csproj | 23 + .../CSharpDbCodeModuleClientTests.cs | 145 +++++ .../TestDatabaseScope.cs | 52 ++ .../access-style-csharp-code-modules.html | 477 ++++++++++++++ www/blog/index.html | 10 + www/css/style.css | 10 + www/sitemap.xml | 8 +- 46 files changed, 3850 insertions(+), 187 deletions(-) create mode 100644 docs/user-defined-functions/README.md create mode 100644 src/CSharpDB.Admin.Forms/Services/AdminFormCodeModuleCommandApi.cs create mode 100644 src/CSharpDB.Admin.Forms/Services/FormCodeModuleDesignerService.cs create mode 100644 src/CSharpDB.Admin/Components/Tabs/CodeModulesTab.razor create mode 100644 src/CSharpDB.CodeModules/CSharpDB.CodeModules.csproj create mode 100644 src/CSharpDB.CodeModules/CSharpDbCodeModuleClient.cs create mode 100644 src/CSharpDB.CodeModules/CodeModuleModels.cs create mode 100644 src/CSharpDB.CodeModules/CodeModulesServiceCollectionExtensions.cs create mode 100644 src/CSharpDB.CodeModules/Infrastructure/CodeModuleHashing.cs create mode 100644 src/CSharpDB.CodeModules/Infrastructure/CodeModuleSql.cs create mode 100644 src/CSharpDB.CodeModules/README.md create mode 100644 src/CSharpDB.CodeModules/Runtime/CodeModuleFormDispatch.cs create mode 100644 src/CSharpDB.CodeModules/Runtime/FormRuntimeContracts.cs create mode 100644 src/CSharpDB.CodeModules/Trust/CodeModuleTrustStore.cs create mode 100644 tests/CSharpDB.Admin.Forms.Tests/Services/FormCodeModuleEventTests.cs create mode 100644 tests/CSharpDB.CodeModules.Tests/CSharpDB.CodeModules.Tests.csproj create mode 100644 tests/CSharpDB.CodeModules.Tests/CSharpDbCodeModuleClientTests.cs create mode 100644 tests/CSharpDB.CodeModules.Tests/TestDatabaseScope.cs create mode 100644 www/blog/access-style-csharp-code-modules.html diff --git a/CSharpDB.slnx b/CSharpDB.slnx index 691caea1..f7a6e573 100644 --- a/CSharpDB.slnx +++ b/CSharpDB.slnx @@ -29,6 +29,7 @@ + @@ -56,6 +57,7 @@ + diff --git a/docs/admin-forms-access-parity/README.md b/docs/admin-forms-access-parity/README.md index f04845df..9cea97d1 100644 --- a/docs/admin-forms-access-parity/README.md +++ b/docs/admin-forms-access-parity/README.md @@ -29,7 +29,11 @@ The current forms surface already includes: - declarative form action sequences for run-command, reusable sequence, set-field, show-message, stop, built-in navigation, save, delete, refresh, and go-to-record steps +- open/close form, apply/clear filter, run SQL/procedure, and control-property + action steps with rendered-host policy gates where needed - conditional action steps and reusable named form action sequences +- conditional UI rules for visible, enabled, read-only, text, placeholder, and + value effects - visual designer editing for form and selected-control action sequences - generated automation metadata for export/import host callback requirements @@ -106,10 +110,10 @@ Expected fix: | Feature | Status | Notes | | --- | --- | --- | -| Command button control | Partial | Trusted command buttons can invoke host-registered C# commands, action-only click sequences, reusable action sequences, and built-in rendered-form navigation/save/delete actions; richer button styling and command presets remain future work. | -| Action model | Partial | Declarative action sequences support run-command, reusable sequence, set-field, show-message, stop, built-in navigation/save/delete/refresh/go-to, simple per-step conditions, visual designer editing, and generated automation metadata for form and selected-control events; open form, apply filter, clear filter, run SQL/procedure, loops, and conditional UI rules remain future work. | -| Event hooks | Partial | Form lifecycle events, command-button clicks, and selected control events can call trusted commands; additional Access-style events remain future work. | -| Conditional UI rules | Planned | Add visible/enabled/read-only expressions for controls. | +| Command button control | Partial | Trusted command buttons can invoke host-registered C# commands, action-only click sequences, reusable action sequences, and built-in rendered-form navigation/save/delete/filter/control actions; richer button styling and command presets remain future work. | +| Action model | Partial | Declarative action sequences support run-command, reusable sequence, set-field, show-message, stop, rendered record navigation/save/delete/refresh/go-to, open/close form, apply/clear filter, run SQL/procedure, control property changes, simple per-step conditions, visual designer editing, diagnostics, and generated automation metadata; database-owned C# form modules can now run trusted local handlers with explicit trust. Loops, on-error handling, temp/session variables, broader report/query/import/export actions, report modules, and sandboxed code remain future work. | +| Event hooks | Partial | Form lifecycle events, command-button clicks, and selected control events can call trusted commands, C# code-module handlers, or action sequences; additional Access-style events such as double-click, key, mouse, timer, dirty, and current events remain future work. | +| Conditional UI rules | Partial | Form-level rules can set rendered control properties such as visible, enabled, read-only, text, placeholder, and value; reusable rule presets and broader expression/event surfaces remain future work. | ### Phase 5: Broader Control and Property Coverage diff --git a/docs/admin-forms-access-parity/access-style-functions-and-macros.md b/docs/admin-forms-access-parity/access-style-functions-and-macros.md index 32f29b80..637ec46c 100644 --- a/docs/admin-forms-access-parity/access-style-functions-and-macros.md +++ b/docs/admin-forms-access-parity/access-style-functions-and-macros.md @@ -12,13 +12,15 @@ feel productive without requiring host code for every common task. - Keep user-facing form actions as declarative Admin Form actions where possible. - Route dangerous or host-specific actions through trusted callbacks, policy, and diagnostics. -- Treat database-owned C# code modules as the later `RunCode` target. +- Treat database-owned C# code modules as trusted event handlers first; a + declarative `RunCode` macro action can build on that runtime later. - Preserve the existing saved form wire shape by storing action and expression settings in metadata/property bags. Implementation note: Admin Forms formulas now include the expression functions -listed below as built-ins. Macro/action commands remain roadmap items unless -already covered by the existing form action model. +listed below as built-ins. The rendered Forms runtime also supports the current +declarative action model shown below; entries marked future remain roadmap +items. ## Included Expression Functions @@ -27,90 +29,89 @@ already covered by the existing form action model. These should be first because they appear constantly in form defaults, calculated controls, validation messages, and visibility/enabled rules. -| Function | Purpose | Priority | +| Function | Purpose | Status | | --- | --- | --- | -| `Nz(value, fallback)` | Replace null/empty values with a fallback. | V1 | -| `IsNull(value)` | Test for null. | V1 | -| `IsEmpty(value)` | Test for empty/unset values where applicable. | V1 | -| `IIf(condition, trueValue, falseValue)` | Inline conditional. | V1 | -| `Switch(condition1, value1, ...)` | Multi-branch conditional. | V2 | -| `Choose(index, value1, value2, ...)` | Pick from positional values. | V2 | +| `Nz(value, fallback)` | Replace null/empty values with a fallback. | Shipped | +| `IsNull(value)` | Test for null. | Shipped | +| `IsEmpty(value)` | Test for empty/unset values where applicable. | Shipped | +| `IIf(condition, trueValue, falseValue)` | Inline conditional. | Shipped | +| `Switch(condition1, value1, ...)` | Multi-branch conditional. | Shipped | +| `Choose(index, value1, value2, ...)` | Pick from positional values. | Shipped | ### Text -| Function | Purpose | Priority | +| Function | Purpose | Status | | --- | --- | --- | -| `Len(value)` | Text length. | V1 | -| `Left(value, count)` | Left substring. | V1 | -| `Right(value, count)` | Right substring. | V1 | -| `Mid(value, start, count)` | Middle substring. | V1 | -| `Trim(value)` | Trim both ends. | V1 | -| `LTrim(value)` | Trim left. | V2 | -| `RTrim(value)` | Trim right. | V2 | -| `UCase(value)` | Uppercase text. | V1 | -| `LCase(value)` | Lowercase text. | V1 | -| `InStr(value, search)` | Find substring position. | V1 | -| `Replace(value, search, replacement)` | Replace text. | V1 | -| `StrComp(left, right, comparison)` | Compare strings. | V2 | -| `Val(value)` | Parse leading numeric text. | V2 | +| `Len(value)` | Text length. | Shipped | +| `Left(value, count)` | Left substring. | Shipped | +| `Right(value, count)` | Right substring. | Shipped | +| `Mid(value, start, count)` | Middle substring. | Shipped | +| `Trim(value)` | Trim both ends. | Shipped | +| `LTrim(value)` | Trim left. | Shipped | +| `RTrim(value)` | Trim right. | Shipped | +| `UCase(value)` | Uppercase text. | Shipped | +| `LCase(value)` | Lowercase text. | Shipped | +| `InStr(value, search)` | Find substring position. | Shipped | +| `Replace(value, search, replacement)` | Replace text. | Shipped | +| `StrComp(left, right, comparison)` | Compare strings. | Shipped | +| `Val(value)` | Parse leading numeric text. | Shipped | ### Date and Time -| Function | Purpose | Priority | +| Function | Purpose | Status | | --- | --- | --- | -| `Date()` | Current date. | V1 | -| `Time()` | Current time. | V1 | -| `Now()` | Current date/time. | V1 | -| `Year(value)` | Extract year. | V1 | -| `Month(value)` | Extract month number. | V1 | -| `Day(value)` | Extract day of month. | V1 | -| `Hour(value)` | Extract hour. | V2 | -| `Minute(value)` | Extract minute. | V2 | -| `Second(value)` | Extract second. | V2 | -| `DateAdd(interval, amount, value)` | Add date/time interval. | V1 | -| `DateDiff(interval, start, end)` | Difference between dates. | V1 | -| `DatePart(interval, value)` | Extract date/time part. | V2 | -| `DateSerial(year, month, day)` | Construct date. | V2 | -| `TimeSerial(hour, minute, second)` | Construct time. | V2 | -| `Weekday(value)` | Day of week number. | V2 | -| `MonthName(month)` | Month display name. | V2 | +| `Date()` | Current date. | Shipped | +| `Time()` | Current time. | Shipped | +| `Now()` | Current date/time. | Shipped | +| `Year(value)` | Extract year. | Shipped | +| `Month(value)` | Extract month number. | Shipped | +| `Day(value)` | Extract day of month. | Shipped | +| `Hour(value)` | Extract hour. | Shipped | +| `Minute(value)` | Extract minute. | Shipped | +| `Second(value)` | Extract second. | Shipped | +| `DateAdd(interval, amount, value)` | Add date/time interval. | Shipped | +| `DateDiff(interval, start, end)` | Difference between dates. | Shipped | +| `DatePart(interval, value)` | Extract date/time part. | Shipped | +| `DateSerial(year, month, day)` | Construct date. | Shipped | +| `TimeSerial(hour, minute, second)` | Construct time. | Shipped | +| `Weekday(value)` | Day of week number. | Shipped | +| `MonthName(month)` | Month display name. | Shipped | ### Number and Conversion -| Function | Purpose | Priority | +| Function | Purpose | Status | | --- | --- | --- | -| `Abs(value)` | Absolute value. | V1 | -| `Round(value, digits)` | Round number. | V1 | -| `Int(value)` | Floor-like integer conversion. | V1 | -| `Fix(value)` | Truncate toward zero. | V2 | -| `Sgn(value)` | Sign of number. | V2 | -| `CStr(value)` | Convert to string. | V1 | -| `CInt(value)` | Convert to integer. | V1 | -| `CLng(value)` | Convert to long integer. | V2 | -| `CDbl(value)` | Convert to double. | V1 | -| `CBool(value)` | Convert to boolean. | V1 | -| `CDate(value)` | Convert to date/time. | V1 | -| `Format(value, format)` | Format date/number/text. | V2 | +| `Abs(value)` | Absolute value. | Shipped | +| `Round(value, digits)` | Round number. | Shipped | +| `Int(value)` | Floor-like integer conversion. | Shipped | +| `Fix(value)` | Truncate toward zero. | Shipped | +| `Sgn(value)` | Sign of number. | Shipped | +| `CStr(value)` | Convert to string. | Shipped | +| `CInt(value)` | Convert to integer. | Shipped | +| `CLng(value)` | Convert to long integer. | Shipped | +| `CDbl(value)` | Convert to double. | Shipped | +| `CBool(value)` | Convert to boolean. | Shipped | +| `CDate(value)` | Convert to date/time. | Shipped | +| `Format(value, format)` | Format date/number/text. | Shipped | ### Domain Aggregates -Domain aggregate functions are important for Access familiarity, but they need a -careful implementation because they read other rows/tables during form -evaluation. +Domain aggregate functions are important for Access familiarity. They are +available in Admin Forms formulas through the rendered Forms runtime, which +loads referenced domains with a row limit and evaluates criteria through the +Forms filter parser. -| Function | Purpose | Priority | +| Function | Purpose | Status | | --- | --- | --- | -| `DLookup(expr, domain, criteria)` | Read one value from a table/query. | V2 | -| `DCount(expr, domain, criteria)` | Count matching rows. | V2 | -| `DSum(expr, domain, criteria)` | Sum matching rows. | V2 | -| `DAvg(expr, domain, criteria)` | Average matching rows. | V2 | -| `DMin(expr, domain, criteria)` | Minimum matching value. | V2 | -| `DMax(expr, domain, criteria)` | Maximum matching value. | V2 | - -V2 should enforce query/table access through the same callback and diagnostics -boundary used for trusted extensions where relevant. These functions should also -have row limits and clear error handling so formula evaluation cannot become an -unbounded database workload. +| `DLookup(expr, domain, criteria)` | Read one value from a table/query. | Shipped in Admin Forms | +| `DCount(expr, domain, criteria)` | Count matching rows. | Shipped in Admin Forms | +| `DSum(expr, domain, criteria)` | Sum matching rows. | Shipped in Admin Forms | +| `DAvg(expr, domain, criteria)` | Average matching rows. | Shipped in Admin Forms | +| `DMin(expr, domain, criteria)` | Minimum matching value. | Shipped in Admin Forms | +| `DMax(expr, domain, criteria)` | Maximum matching value. | Shipped in Admin Forms | + +Future work can broaden these beyond rendered Admin Forms and add more +diagnostics for expensive domain reads. ## Included Macro and Action Commands @@ -119,115 +120,113 @@ unbounded database workload. These map cleanly to existing form runtime behavior and should remain declarative actions rather than host callbacks. -| Action | Purpose | Priority | +| Action | Purpose | Status | | --- | --- | --- | -| `NewRecord` | Start a new record. | V1 | -| `SaveRecord` | Save the current record. | V1 | -| `DeleteRecord` | Delete the current record. | V1 | -| `UndoRecord` | Revert unsaved edits. | V1 | -| `RefreshRecord` | Reload the current record. | V1 | -| `Requery` | Reload the form record source. | V1 | -| `GoToRecord` | Navigate to a specific record. | V1 | -| `FindRecord` | Search/navigate by criteria. | V2 | -| `NextRecord` | Navigate forward. | V1 | -| `PreviousRecord` | Navigate backward. | V1 | +| `NewRecord` | Start a new record. | Shipped | +| `SaveRecord` | Save the current record. | Shipped | +| `DeleteRecord` | Delete the current record. | Shipped | +| `UndoRecord` | Revert unsaved edits. | Future action | +| `RefreshRecords` | Reload the current record/page. | Shipped | +| `Requery` | Reload the form record source. | Covered by `RefreshRecords`; broader record-source requery is future | +| `GoToRecord` | Navigate to a specific record. | Shipped | +| `FindRecord` | Search/navigate by criteria. | Future action | +| `NextRecord` | Navigate forward. | Shipped | +| `PreviousRecord` | Navigate backward. | Shipped | ### Form, Window, and Report Actions -| Action | Purpose | Priority | +| Action | Purpose | Status | | --- | --- | --- | -| `OpenForm` | Open another saved form. | V1 | -| `CloseForm` | Close current or named form. | V1 | -| `OpenReport` | Open a saved report. | V2 | -| `CloseReport` | Close a report surface. | V2 | -| `PreviewReport` | Open report preview. | V2 | -| `PrintReport` | Print or export through report pipeline. | V2 | +| `OpenForm` | Open another saved form. | Shipped | +| `CloseForm` | Close current or named form. | Shipped | +| `OpenReport` | Open a saved report. | Future action | +| `CloseReport` | Close a report surface. | Future action | +| `PreviewReport` | Open report preview. | Future action | +| `PrintReport` | Print or export through report pipeline. | Future action | ### Filter and Sort Actions -| Action | Purpose | Priority | +| Action | Purpose | Status | | --- | --- | --- | -| `ApplyFilter` | Apply a form filter. | V1 | -| `ClearFilter` | Clear current filter. | V1 | -| `SetOrderBy` | Apply sort order. | V1 | -| `ClearOrderBy` | Clear sort order. | V1 | -| `SearchRecords` | Search over configured searchable fields. | V2 | +| `ApplyFilter` | Apply a form or data-grid filter. | Shipped | +| `ClearFilter` | Clear current form or data-grid filter. | Shipped | +| `SetOrderBy` | Apply sort order. | Future action | +| `ClearOrderBy` | Clear sort order. | Future action | +| `SearchRecords` | Search over configured searchable fields. | Future action | ### UI and Control Actions These are needed for Access-style command buttons and conditional workflows. -| Action | Purpose | Priority | +| Action | Purpose | Status | | --- | --- | --- | -| `SetValue` | Set a field/control value. | V1 | -| `SetProperty` | Set visible/enabled/read-only/text/style properties. | V1 | -| `SetFocus` | Move focus to a control. | V1 | -| `EnableControl` | Set enabled state. | V1 | -| `DisableControl` | Clear enabled state. | V1 | -| `ShowControl` | Set visible state. | V1 | -| `HideControl` | Clear visible state. | V1 | -| `LockControl` | Set read-only state. | V1 | -| `UnlockControl` | Clear read-only state. | V1 | -| `MsgBox` | Show a message dialog. | V1 | -| `InputBox` | Prompt for a value. | V2 | +| `SetFieldValue` | Set a current record field value. | Shipped | +| `SetControlProperty` | Set visible/enabled/read-only/text/placeholder/value properties. | Shipped | +| `SetFocus` | Move focus to a control. | Future action | +| `SetControlEnabled` | Set enabled state. | Shipped | +| `SetControlVisibility` | Set visible state. | Shipped | +| `SetControlReadOnly` | Set read-only state. | Shipped | +| `ShowMessage` | Show a message through the current Forms surface. | Shipped | +| `InputBox` | Prompt for a value. | Future action | ### Flow Actions -| Action | Purpose | Priority | +| Action | Purpose | Status | | --- | --- | --- | -| `If` / `Else` | Conditional action branching. | V1 | -| `StopMacro` | Stop the current action sequence. | V1 | -| `RunMacro` | Run a named action sequence. | V1 | -| `RunActionSequence` | Existing explicit reusable sequence action. | V1 | -| `OnError` | Configure failure handling. | V2 | +| Per-step `Condition` | Conditionally run or skip a step. | Shipped | +| `Stop` | Stop the current action sequence successfully. | Shipped | +| `RunActionSequence` | Run a named reusable sequence. | Shipped | +| `If` / `Else` blocks | Conditional action branching. | Future action | +| `OnError` | Configure failure handling. | Future action | ### Data and Query Actions These must be gated carefully because they can mutate data outside the current form record. -| Action | Purpose | Priority | +| Action | Purpose | Status | | --- | --- | --- | -| `OpenQuery` | Open a saved query result. | V2 | -| `RunQuery` | Execute a saved query. | V2 trusted | -| `RunProcedure` | Execute a saved procedure. | V2 trusted | -| `RunSQL` | Execute SQL text. | Later trusted | -| `ExportData` | Export records. | Later trusted | -| `ImportData` | Import records. | Later trusted | +| `OpenQuery` | Open a saved query result. | Future action | +| `RunQuery` | Execute a saved query. | Future trusted action | +| `RunProcedure` | Execute a saved procedure. | Shipped with host opt-in | +| `RunSql` | Execute SQL text. | Shipped with host opt-in | +| `ExportData` | Export records. | Future trusted action | +| `ImportData` | Import records. | Future trusted action | ### Code Bridge Actions -| Action | Purpose | Priority | +| Action | Purpose | Status | | --- | --- | --- | -| `RunCommand` | Invoke host-registered trusted command callback. | Existing | -| `RunCode` | Invoke database-owned C# code module function. | Later | +| `RunCommand` | Invoke host-registered trusted command callback. | Shipped | +| `RunCode` | Invoke database-owned C# code module function from a declarative action sequence. | Future action | -`RunCode` should wait for the database code modules work. It should compile and -execute database-owned C# through a trusted build/runtime model, not arbitrary -source text embedded directly inside form JSON. +Database-owned C# form modules now compile and execute through a trusted +build/runtime model for form and control event handlers. `RunCode` remains a +future macro action that should reuse that model rather than embedding arbitrary +source text directly inside form JSON. ### Temp and Session Variables -| Action | Purpose | Priority | +| Action | Purpose | Status | | --- | --- | --- | -| `SetTempVar` | Store a session-scoped value. | V2 | -| `RemoveTempVar` | Remove one session value. | V2 | -| `RemoveAllTempVars` | Clear session values. | V2 | +| `SetTempVar` | Store a session-scoped value. | Future action | +| `RemoveTempVar` | Remove one session value. | Future action | +| `RemoveAllTempVars` | Clear session values. | Future action | Temp variables should be scoped to the current user/session and be available to form expressions, filters, and action sequences. -## Recommended Implementation Order - -1. Add core formula functions: `Nz`, `IIf`, `Date`, `Now`, common text helpers, - numeric helpers, and conversions. -2. Add declarative form actions: `MsgBox`, `SetValue`, `SetProperty`, - `SetFocus`, `ApplyFilter`, `ClearFilter`, `Requery`, `OpenForm`, and - `CloseForm`. -3. Add domain aggregates with clear access checks, row limits, and diagnostics. -4. Add temp/session variables and make them visible to expressions and actions. -5. Add `RunCode` after database code modules exist. -6. Add import/export/file/app-launch style actions only as trusted operations. +## Recommended Remaining Implementation Order + +1. Add sort/search/find actions and any small aliases needed for Access naming + compatibility. +2. Add temp/session variables and make them visible to expressions, filters, + and action sequences. +3. Add richer macro flow such as `If` / `Else` blocks and `OnError`. +4. Add report/query/import/export actions behind explicit trusted boundaries. +5. Add `RunCode` after the code-module event-handler MVP is extended to macro + action invocation. +6. Add file/app-launch style actions only as trusted operations. ## Notes on Access Compatibility diff --git a/docs/roadmap.md b/docs/roadmap.md index d47e660f..3ed1427f 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,6 +1,6 @@ # CSharpDB Roadmap -This document outlines the planned direction for CSharpDB, organized by timeframe and priority. Items are roughly ordered by expected impact within each tier, and statuses are intended to reflect the current `v3.4.0` state of the repo. +This document outlines the planned direction for CSharpDB, organized by timeframe and priority. Items are roughly ordered by expected impact within each tier, and statuses are intended to reflect the current `main` state of the repo. --- @@ -42,7 +42,7 @@ SQL feature parity, provider/tooling compatibility, and ecosystem expansion. | Feature | Description | Status | |---------|-------------|--------| -| **User-defined functions and commands** | Trusted in-process C# scalar functions are implemented for SQL, triggers/procedures, direct clients, Admin Forms/Reports, and pipelines; trusted commands now back Admin Forms lifecycle events, command-button clicks, selected control events, Admin Reports render lifecycle events, and pipeline run hooks; Admin Forms now have declarative action sequences for run-command, set-field, show-message, and stop steps. Broader built-in scalar functions, native plugin extensions, aggregate/table-valued UDFs, richer macro flow, additional Access-style control events, and sandboxed UDFs remain future work | Partial | +| **User-defined functions and commands** | Trusted in-process C# scalar functions are implemented for SQL, triggers/procedures, direct clients, Admin Forms/Reports, and pipelines; common SQL/Admin scalar built-ins cover text, date/time, numeric, conversion, null/conditional, and Admin Forms domain helper scenarios; trusted commands back Admin Forms lifecycle events, command-button clicks, selected-control events, Admin Reports render lifecycle events, and pipeline run hooks; Admin Forms action sequences cover command, field/message/stop, reusable sequence, rendered record navigation/save/delete/refresh/go-to, open/close form, filter, SQL/procedure, control-property, condition, and rule workflows. Database-owned C# code modules now cover the local Admin Forms MVP with metadata storage, VS Code file-sync export/import, Roslyn diagnostics, local trust, form/control event binding, and trusted in-process handler dispatch. Aggregate/table-valued UDFs, native plugin extensions, report modules, remote delegate serialization, additional Access-style events, macro loops/on-error/temp vars, broader report/query/import/export actions, and sandboxed UDFs remain future work | Partial | | **Subqueries** | Scalar subqueries, `IN (SELECT ...)`, `EXISTS (SELECT ...)`, including correlated evaluation in `WHERE`, non-aggregate projection, and `UPDATE`/`DELETE` expressions | Done | | **`UNION` / `INTERSECT` / `EXCEPT`** | Set operations across SELECT results, including use in top-level queries, views, and CTE query bodies | Done | | **Window functions** | `ROW_NUMBER()`, `RANK()`, `DENSE_RANK()`, `LEAD()`, `LAG()` | Planned | @@ -102,7 +102,7 @@ These are known simplifications in the current implementation: | Area | Limitation | |------|-----------| -| **Functions and automation** | Trusted in-process C# scalar functions are supported when registered by the host; Admin Forms, Admin Reports, and pipelines can invoke trusted host commands from supported event/hook surfaces; Admin Forms support declarative action sequences for run-command, set-field, show-message, and stop steps. Broader built-in scalar functions, aggregate/table-valued UDFs, stored C# modules, remote delegate serialization, additional Access-style control events, richer macro flow, and sandboxed UDFs are not implemented | +| **Functions and automation** | Trusted in-process C# scalar functions, common built-in scalar helpers, Admin Forms domain helpers, trusted host commands, pipeline/report/form command hooks, declarative Admin Forms action sequences, and local trusted Admin Forms C# code modules are implemented for the current supported surfaces. Remaining gaps are aggregate/table-valued UDFs, native plugin extensions, report modules, remote delegate serialization, additional Access-style events, macro loops/on-error/temp vars, broader report/query/import/export actions, and sandboxed UDFs | | **Query** | Scalar/`IN`/`EXISTS` subqueries are supported, including correlated cases in `WHERE`, non-aggregate projection, and `UPDATE`/`DELETE` expressions; correlated subqueries are not yet supported in `JOIN ON`, `GROUP BY`, `HAVING`, `ORDER BY`, or aggregate projections | | **Query** | `UNION`, `INTERSECT`, and `EXCEPT` are supported; `UNION ALL` is not implemented yet | | **Query** | No window functions | @@ -112,7 +112,7 @@ These are known simplifications in the current implementation: | **Collections** | `FindByIndexAsync` supports declared field-equality lookups; `FindByPathAsync` and `FindByPathRangeAsync` support path-based queries on indexed paths; `FindAsync` remains a full scan for unindexed predicates. Generated collections require registered descriptors for existing collection indexes; unsupported generated model shapes warn and use the source-generated JSON fallback instead of binary direct payloads | | **Networking** | `CSharpDB.Daemon` now hosts both REST and gRPC from one process; named pipes remain reserved but are not implemented end to end today | | **Security** | Remote REST and daemon gRPC support opt-in API-key authentication, defaulting to `None` for backward compatibility. JWT, RBAC, mTLS helpers, TLS-specific configuration, and at-rest encryption are not implemented | -| **Admin Forms** | The Forms designer/runtime supports the core generated-form and data-entry path plus initial trusted command-backed automation, including lifecycle events, command buttons, selected control events, and declarative action sequences, but still needs Access-parity work for responsive runtime rendering, complete inferred validation, richer form modes, broader built-in actions, additional events, advanced filtering/sorting, and broader controls | +| **Admin Forms** | The Forms designer/runtime supports the core generated-form and data-entry path plus trusted command-backed automation, including lifecycle events, command buttons, selected-control events, conditional UI rules, domain formula helpers, and declarative action sequences for current record, form navigation, filtering, SQL/procedure, and control-property workflows. It still needs Access-parity work for responsive runtime rendering, complete inferred validation, richer form modes, additional events, advanced filtering/sorting, report/query/import/export actions, macro loops/on-error/temp vars, and broader controls | | **Admin Reports** | The Reports designer/runtime supports the core banded preview path plus trusted command-backed preview lifecycle events, but still needs Access-parity work for bounded saved-query previews, full report output/export, parameters, richer grouping and totals semantics, conditional formatting, subreports, and broader controls | | **Text / Multilingual** | Text is stored as UTF-8 and supports all Unicode languages; default semantics remain ordinal, but opt-in `BINARY`, `NOCASE`, `NOCASE_AI`, and `ICU:` collation are implemented for SQL and collection indexes. Dedicated ordered SQL text index optimization remains planned | | **Concurrency** | The physical WAL commit path is still serialized at the storage boundary. Initial multi-writer support is shipped, but observed gains still depend on conflict shape and whether shared auto-commit `INSERT` is left on the default serialized path | diff --git a/docs/trusted-csharp-functions/README.md b/docs/trusted-csharp-functions/README.md index d7df8bb6..b7e22d07 100644 --- a/docs/trusted-csharp-functions/README.md +++ b/docs/trusted-csharp-functions/README.md @@ -2,7 +2,7 @@ CSharpDB can call host-registered C# scalar functions from SQL and the embedded expression surfaces that sit on top of the engine. This is the CSharpDB equivalent of an Access-style application function integration: the application owns the C# code, registers it while opening or hosting the database, and users call the function by name in database expressions. -This feature is intentionally trusted and in-process. It does not store C# source code in the database, sandbox user code, load plugin assemblies from database files, or serialize delegates over HTTP or gRPC. +This callback feature is intentionally trusted and in-process. Host-registered callbacks do not store C# source code in the database, sandbox user code, load plugin assemblies from database files, or serialize delegates over HTTP or gRPC. For the newer database-owned Admin Forms C# module workflow, see [User-Defined Functions And Commands](../user-defined-functions/README.md). For an end-to-end app-builder walkthrough that combines Admin Forms, collections, macro actions, reports, trusted callbacks, and callback readiness, see the @@ -631,7 +631,7 @@ var shipButton = existingButton with }; ``` -The action set is intentionally small and form-focused: +The action set is intentionally form-focused: | Action | Behavior | | --- | --- | @@ -647,6 +647,16 @@ The action set is intentionally small and form-focused: | `PreviousRecord` | Moves the rendered form to the previous record. | | `NextRecord` | Moves the rendered form to the next record. | | `GoToRecord` | Navigates to a primary-key value from `Value`, `Arguments["value"]`, `Arguments["recordId"]`, `Arguments["primaryKey"]`, or the field named by `Target`. | +| `OpenForm` | Opens another saved form through the rendered form host. | +| `CloseForm` | Closes the current or named rendered form surface. | +| `ApplyFilter` | Applies a form or data-grid filter. | +| `ClearFilter` | Clears a form or data-grid filter. | +| `RunSql` | Executes SQL through the rendered host when SQL actions are enabled by policy. | +| `RunProcedure` | Executes a saved procedure through the rendered host when procedure actions are enabled by policy. | +| `SetControlProperty` | Overrides a rendered control property such as `visible`, `enabled`, `readOnly`, `text`, `placeholder`, or bound `value`. | +| `SetControlVisibility` | Short form for setting a rendered control's `visible` property. | +| `SetControlEnabled` | Short form for setting a rendered control's `enabled` property. | +| `SetControlReadOnly` | Short form for setting a rendered control's `readOnly` property. | Reusable action sequences are stored once on the form and invoked by name from form events, control events, or command buttons: @@ -709,9 +719,10 @@ var form = existingForm with The Admin Forms property inspector exposes action sequences with a visual editor on form-level and selected-control event bindings. Designers can add a -sequence, name it, add command, reusable-sequence, field, message, stop, and -built-in record steps, reorder or remove steps, choose registered commands or -reusable sequences when available, and set per-step conditions and +sequence, name it, add command, reusable-sequence, field, message, stop, +rendered record, form, filter, SQL/procedure, and control-property steps, +reorder or remove steps, choose registered commands or reusable sequences when +available, and set per-step conditions and `StopOnFailure`. The form-level property inspector also includes a reusable action-sequence library editor. JSON editing remains only for optional binding, `RunCommand`, or `RunActionSequence` argument payloads. @@ -753,16 +764,18 @@ backfilled when it is loaded. `BeforeInsert` and `BeforeUpdate`, and it can update the current rendered record from control events or command-button clicks. -Built-in record actions require a rendered Admin Forms data-entry runtime. -They are intended for command buttons and selected-control events. Headless -form lifecycle dispatch can still run `SetFieldValue`, `ShowMessage`, `Stop`, -and `RunCommand`, but it reports a failure if a sequence asks for rendered-form -navigation or save/delete actions. +Built-in record, form navigation, filter, SQL/procedure, and control-property +actions require a rendered Admin Forms data-entry runtime. They are intended for +command buttons and selected-control events. Headless form lifecycle dispatch +can still run `SetFieldValue`, `ShowMessage`, `Stop`, and `RunCommand`, but it +reports a failure if a sequence asks for actions that need the rendered form +instance. -Action sequences do not include loops, stored C# source, database-owned -plugins, or remote delegate serialization. Rendered Admin form runtimes support -direct SQL and procedure actions only when the host explicitly enables those -capabilities. +Action sequences do not include loops, a `RunCode` macro action, +database-owned plugins, or remote delegate serialization. Database-owned C# +form modules are handled as trusted event handlers through the separate code +module runtime. Rendered Admin form runtimes support direct SQL and procedure +actions only when the host explicitly enables those capabilities. --- @@ -1026,11 +1039,12 @@ V1 does not support: - Aggregate UDFs. - Table-valued UDFs. -- Stored C# source code or database-owned compiled modules. +- Database-owned C# modules beyond the local Admin Forms event-handler MVP, + such as report modules, procedure modules, or remote/daemon execution. - Sandboxed execution. - Async scalar delegates. - Passing a database handle into the function context. - Sending delegates over HTTP, gRPC, or pipeline package files. - Optimizer pushdown, expression indexes, generated columns, or constant folding based on custom function metadata. - Additional Access-style control events such as double-click, key, mouse, timer, and dirty/current events. -- Richer macro/action scripts with loops, reusable UI rule presets, additional event surfaces, or database-owned executable code. +- Richer macro/action scripts with loops, reusable UI rule presets, additional event surfaces, or `RunCode` action invocation. diff --git a/docs/user-defined-functions/README.md b/docs/user-defined-functions/README.md new file mode 100644 index 00000000..7969d657 --- /dev/null +++ b/docs/user-defined-functions/README.md @@ -0,0 +1,66 @@ +# User-Defined Functions And Commands Plan + +This page tracks the shipped and remaining work for CSharpDB function, +command, and automation extensibility. The implementation separates portable +database metadata from local execution trust: databases can reference functions, +commands, validation rules, declarative action sequences, and C# code modules, +while executable code runs only in trusted host contexts. + +## Shipped Capabilities + +- Trusted in-process C# scalar functions through `DbFunctionRegistry`, available + to SQL, triggers/procedures, direct clients, Admin Forms/Reports, and + pipelines. +- Common built-in scalar functions for SQL and Admin formulas, including text, + date/time, numeric, conversion, null, and conditional helpers. +- Admin Forms domain helper functions such as `DLookup`, `DCount`, `DSum`, + `DAvg`, `DMin`, and `DMax`, resolved by the rendered Forms runtime with + bounded row loading. +- Trusted command callbacks through `DbCommandRegistry`, used by Admin Forms, + Admin Reports, and pipeline lifecycle hooks. +- Admin Forms action sequences for host commands, reusable sequences, field + updates, messages, stop, rendered record navigation/save/delete/refresh/go-to, + open/close form, filter, SQL/procedure, control-property, conditional, and + rule workflows. +- Database-owned C# code modules for the local Admin Forms MVP: module source is + stored in `__code_modules`, build diagnostics are stored in + `__code_module_builds`, VS Code-friendly file sync exports/imports + `.csharpdb-code/csharpdb.codeproj.json` plus `.cs` files, and form/control + events can bind to trusted in-process handlers. +- Automation metadata and validation surfaces that let exported forms, reports, + and pipeline packages describe required host callbacks. + +## Current Boundaries + +- Scalar delegates are host-owned and in-process; delegates are not serialized + over HTTP, gRPC, or package files. +- Trusted callbacks are policy-mediated but not sandboxed. They run with the + permissions of the host process that registered them. +- Database-owned C# source execution requires host opt-in, successful Roslyn + build, and explicit local trust for the current module-set hash. Trust is not + stored in the database and source changes invalidate it. +- The current code-module runtime is limited to local Admin Forms form/control + handlers; reports, procedures, daemon/remote execution, in-browser editing, + and debugging integration remain outside this slice. +- Custom scalar functions are intentionally not used for optimizer pushdown, + expression indexes, generated columns, or constant folding. + +## Future Work + +- Aggregate and table-valued UDFs. +- Native/plugin extension loading with an explicit trust and packaging model. +- Report modules and broader database-owned code module surfaces beyond the + local Admin Forms MVP. +- Sandboxed UDF execution, including the WebAssembly/Wasmtime research track. +- Remote delegate or extension registration for daemon-hosted deployments. +- Additional Access-style form/control events, macro loops, on-error handling, + temp/session variables, and broader report/query/import/export actions. + +## Related Docs + +- [Trusted C# Callbacks](../trusted-csharp-functions/README.md) +- [Trusted Validation Rules](../trusted-csharp-functions/validation-rules.md) +- [Access-Style Macro Actions](../trusted-csharp-functions/access-style-macro-actions.md) +- [Access-Style Functions and Macros](../admin-forms-access-parity/access-style-functions-and-macros.md) +- [Database Code Modules With VS Code Sync Plan](../trusted-csharp-functions/database-code-modules-vscode-plan.md) +- [WebAssembly sandboxed UDFs](../roadmap.md#long-term) diff --git a/src/CSharpDB.Admin.Forms/CSharpDB.Admin.Forms.csproj b/src/CSharpDB.Admin.Forms/CSharpDB.Admin.Forms.csproj index 6b923cd2..c9e88ff8 100644 --- a/src/CSharpDB.Admin.Forms/CSharpDB.Admin.Forms.csproj +++ b/src/CSharpDB.Admin.Forms/CSharpDB.Admin.Forms.csproj @@ -9,6 +9,7 @@ + diff --git a/src/CSharpDB.Admin.Forms/Components/Designer/ControlEventBindingsEditor.razor b/src/CSharpDB.Admin.Forms/Components/Designer/ControlEventBindingsEditor.razor index 2f74b955..0638bf42 100644 --- a/src/CSharpDB.Admin.Forms/Components/Designer/ControlEventBindingsEditor.razor +++ b/src/CSharpDB.Admin.Forms/Components/Designer/ControlEventBindingsEditor.razor @@ -1,8 +1,10 @@ @using System.Text.Json @using CSharpDB.Admin.Forms.Models @using CSharpDB.Admin.Forms.Serialization +@using CSharpDB.CodeModules @using CSharpDB.Primitives @inject DbCommandRegistry CommandRegistry +@inject IFormCodeModuleDesignerService CodeModuleDesigner
@if (EventBindings.Count == 0) @@ -69,6 +71,28 @@ @onchange="@(e => UpdateArguments(idx, binding, e.Value?.ToString() ?? string.Empty))">
+
+ +
+ + +
+ + +
+
@_argumentError
} + @if (!string.IsNullOrWhiteSpace(_codeHandlerError)) + { +
@_codeHandlerError
+ } @@ -89,12 +117,15 @@ @code { [Parameter, EditorRequired] public IReadOnlyList EventBindings { get; set; } = []; [Parameter] public IReadOnlyList ActionSequences { get; set; } = []; + [Parameter] public ControlDefinition? Control { get; set; } [Parameter] public EventCallback> EventBindingsChanged { get; set; } private readonly Dictionary _argumentText = []; private string? _argumentError; + private string? _codeHandlerError; private static readonly ControlEventKind[] EventKinds = Enum.GetValues(); + [CascadingParameter] public DesignerState? State { get; set; } private IReadOnlyList RegisteredCommands => CommandRegistry.Commands.ToList(); @@ -168,6 +199,51 @@ private Task UpdateActionSequence(int index, ControlEventBinding binding, DbActionSequence? actionSequence) => ReplaceBinding(index, binding with { ActionSequence = actionSequence }); + private Task UpdateCodeHandler( + int index, + ControlEventBinding binding, + string? moduleId = null, + string? typeName = null, + string? methodName = null) + { + string resolvedModuleId = moduleId ?? binding.CodeHandler?.ModuleId ?? string.Empty; + string resolvedTypeName = typeName ?? binding.CodeHandler?.TypeName ?? string.Empty; + string resolvedMethodName = methodName ?? binding.CodeHandler?.MethodName ?? string.Empty; + CodeModuleHandler? handler = string.IsNullOrWhiteSpace(resolvedModuleId) && + string.IsNullOrWhiteSpace(resolvedTypeName) && + string.IsNullOrWhiteSpace(resolvedMethodName) + ? null + : new CodeModuleHandler(resolvedModuleId.Trim(), resolvedTypeName.Trim(), resolvedMethodName.Trim()); + return ReplaceBinding(index, binding with { CodeHandler = handler }); + } + + private async Task CreateCodeHandlerAsync(int index, ControlEventBinding binding) + { + _codeHandlerError = null; + if (State is null || Control is null) + { + _codeHandlerError = "Form designer state is unavailable."; + return; + } + + try + { + CodeModuleHandler handler = await CodeModuleDesigner.CreateHandlerAsync(new FormCodeModuleHandlerRequest( + State.FormId, + State.FormName, + State.TableName, + binding.Event.ToString(), + IsCancelable: false, + Control.ControlId, + Control.ControlType)); + await ReplaceBinding(index, binding with { CodeHandler = handler, CommandName = string.Empty }); + } + catch (Exception ex) + { + _codeHandlerError = ex.Message; + } + } + private async Task ReplaceBinding(int index, ControlEventBinding binding) { var updated = EventBindings.ToList(); @@ -199,6 +275,12 @@ => !string.IsNullOrWhiteSpace(commandName) && RegisteredCommands.All(command => !string.Equals(command.Name, commandName, StringComparison.OrdinalIgnoreCase)); + private static string GetCodeModuleId(ControlEventBinding binding) => binding.CodeHandler?.ModuleId ?? string.Empty; + + private static string GetCodeTypeName(ControlEventBinding binding) => binding.CodeHandler?.TypeName ?? string.Empty; + + private static string GetCodeMethodName(ControlEventBinding binding) => binding.CodeHandler?.MethodName ?? string.Empty; + private static string FormatArguments(IReadOnlyDictionary? arguments) => arguments is null || arguments.Count == 0 ? string.Empty diff --git a/src/CSharpDB.Admin.Forms/Components/Designer/FormEventBindingsEditor.razor b/src/CSharpDB.Admin.Forms/Components/Designer/FormEventBindingsEditor.razor index c1f14137..fc6fa01d 100644 --- a/src/CSharpDB.Admin.Forms/Components/Designer/FormEventBindingsEditor.razor +++ b/src/CSharpDB.Admin.Forms/Components/Designer/FormEventBindingsEditor.razor @@ -1,7 +1,9 @@ @using System.Text.Json @using CSharpDB.Admin.Forms.Serialization +@using CSharpDB.CodeModules @using CSharpDB.Primitives @inject DbCommandRegistry CommandRegistry +@inject IFormCodeModuleDesignerService CodeModuleDesigner
@if (EventBindings.Count == 0) @@ -68,6 +70,28 @@ @onchange="@(e => UpdateArguments(idx, binding, e.Value?.ToString() ?? string.Empty))">
+
+ +
+ + +
+ + +
+
@_argumentError
} + @if (!string.IsNullOrWhiteSpace(_codeHandlerError)) + { +
@_codeHandlerError
+ } @@ -92,8 +120,10 @@ private readonly Dictionary _argumentText = []; private string? _argumentError; + private string? _codeHandlerError; private static readonly FormEventKind[] EventKinds = Enum.GetValues(); + [CascadingParameter] public DesignerState? State { get; set; } private IReadOnlyList RegisteredCommands => CommandRegistry.Commands.ToList(); @@ -167,6 +197,49 @@ private Task UpdateActionSequence(int index, FormEventBinding binding, DbActionSequence? actionSequence) => ReplaceBinding(index, binding with { ActionSequence = actionSequence }); + private Task UpdateCodeHandler( + int index, + FormEventBinding binding, + string? moduleId = null, + string? typeName = null, + string? methodName = null) + { + string resolvedModuleId = moduleId ?? binding.CodeHandler?.ModuleId ?? string.Empty; + string resolvedTypeName = typeName ?? binding.CodeHandler?.TypeName ?? string.Empty; + string resolvedMethodName = methodName ?? binding.CodeHandler?.MethodName ?? string.Empty; + CodeModuleHandler? handler = string.IsNullOrWhiteSpace(resolvedModuleId) && + string.IsNullOrWhiteSpace(resolvedTypeName) && + string.IsNullOrWhiteSpace(resolvedMethodName) + ? null + : new CodeModuleHandler(resolvedModuleId.Trim(), resolvedTypeName.Trim(), resolvedMethodName.Trim()); + return ReplaceBinding(index, binding with { CodeHandler = handler }); + } + + private async Task CreateCodeHandlerAsync(int index, FormEventBinding binding) + { + _codeHandlerError = null; + if (State is null) + { + _codeHandlerError = "Form designer state is unavailable."; + return; + } + + try + { + CodeModuleHandler handler = await CodeModuleDesigner.CreateHandlerAsync(new FormCodeModuleHandlerRequest( + State.FormId, + State.FormName, + State.TableName, + binding.Event.ToString(), + IsCancelable(binding.Event))); + await ReplaceBinding(index, binding with { CodeHandler = handler, CommandName = string.Empty }); + } + catch (Exception ex) + { + _codeHandlerError = ex.Message; + } + } + private async Task ReplaceBinding(int index, FormEventBinding binding) { var updated = EventBindings.ToList(); @@ -198,6 +271,15 @@ => !string.IsNullOrWhiteSpace(commandName) && RegisteredCommands.All(command => !string.Equals(command.Name, commandName, StringComparison.OrdinalIgnoreCase)); + private static string GetCodeModuleId(FormEventBinding binding) => binding.CodeHandler?.ModuleId ?? string.Empty; + + private static string GetCodeTypeName(FormEventBinding binding) => binding.CodeHandler?.TypeName ?? string.Empty; + + private static string GetCodeMethodName(FormEventBinding binding) => binding.CodeHandler?.MethodName ?? string.Empty; + + private static bool IsCancelable(FormEventKind eventKind) + => eventKind is FormEventKind.BeforeInsert or FormEventKind.BeforeUpdate or FormEventKind.BeforeDelete; + private static string FormatArguments(IReadOnlyDictionary? arguments) => arguments is null || arguments.Count == 0 ? string.Empty diff --git a/src/CSharpDB.Admin.Forms/Components/Designer/FormRenderer.razor b/src/CSharpDB.Admin.Forms/Components/Designer/FormRenderer.razor index 64c761da..5c99f372 100644 --- a/src/CSharpDB.Admin.Forms/Components/Designer/FormRenderer.razor +++ b/src/CSharpDB.Admin.Forms/Components/Designer/FormRenderer.razor @@ -4,9 +4,11 @@ @using CSharpDB.Admin.Forms.Models @using CSharpDB.Admin.Forms.Pages @using CSharpDB.Admin.Forms.Services +@using CSharpDB.CodeModules @using CSharpDB.Primitives @inject DbCommandRegistry Commands @inject DbExtensionPolicy? CallbackPolicy +@inject ICodeModuleFormEventDispatcher CodeModules
@foreach (var control in GetControlsToRender()) @@ -1380,16 +1382,60 @@ return; } - if (binding.ActionSequence is null) + if (binding.CodeHandler is null && binding.ActionSequence is null) continue; } } - else if (binding.ActionSequence is null) + else if (binding.CodeHandler is null && binding.ActionSequence is null) { - await ReportCommandErrorAsync($"Control event '{eventKind}' has no command or action sequence."); + await ReportCommandErrorAsync($"Control event '{eventKind}' has no command, code handler, or action sequence."); return; } + if (binding.CodeHandler is not null) + { + var commandApi = new AdminFormCodeModuleCommandApi( + Form, + Commands, + EffectiveCallbackPolicy, + Record, + binding.Arguments, + runtimeArguments, + metadata, + Form.ActionSequences, + ActionRuntime ?? NullFormActionRuntime.Instance, + SetActionFieldValueAsync, + ReportCommandErrorAsync, + OnBuiltInAction); + CodeModuleFormDispatchResult codeResult = await CodeModules.DispatchAsync( + binding.CodeHandler, + new CodeModuleFormEventDispatchContext( + Form.FormId, + Form.Name, + Form.TableName, + eventKind.ToString(), + Record, + binding.Arguments, + runtimeArguments, + metadata, + commandApi, + IsCancelable: false, + control.ControlId, + control.ControlType)); + + if (!codeResult.Succeeded) + { + if (binding.StopOnFailure) + { + await ReportCommandErrorAsync(codeResult.Message ?? $"Control event '{eventKind}' code handler failed."); + return; + } + + if (binding.ActionSequence is null) + continue; + } + } + if (binding.ActionSequence is not null) { FormEventDispatchResult actionResult = await FormActionSequenceExecutor.ExecuteAsync( diff --git a/src/CSharpDB.Admin.Forms/Components/Designer/PropertyInspector.razor b/src/CSharpDB.Admin.Forms/Components/Designer/PropertyInspector.razor index ee78c33d..44228c13 100644 --- a/src/CSharpDB.Admin.Forms/Components/Designer/PropertyInspector.razor +++ b/src/CSharpDB.Admin.Forms/Components/Designer/PropertyInspector.razor @@ -92,6 +92,7 @@
Events
diff --git a/src/CSharpDB.Admin.Forms/Models/ControlEventBinding.cs b/src/CSharpDB.Admin.Forms/Models/ControlEventBinding.cs index c4f21be1..1fe64bd5 100644 --- a/src/CSharpDB.Admin.Forms/Models/ControlEventBinding.cs +++ b/src/CSharpDB.Admin.Forms/Models/ControlEventBinding.cs @@ -1,3 +1,4 @@ +using CSharpDB.CodeModules; using CSharpDB.Primitives; namespace CSharpDB.Admin.Forms.Models; @@ -15,4 +16,5 @@ public sealed record ControlEventBinding( string CommandName, IReadOnlyDictionary? Arguments = null, bool StopOnFailure = true, - DbActionSequence? ActionSequence = null); + DbActionSequence? ActionSequence = null, + CodeModuleHandler? CodeHandler = null); diff --git a/src/CSharpDB.Admin.Forms/Models/FormEventBinding.cs b/src/CSharpDB.Admin.Forms/Models/FormEventBinding.cs index 754b031c..bc3d63a8 100644 --- a/src/CSharpDB.Admin.Forms/Models/FormEventBinding.cs +++ b/src/CSharpDB.Admin.Forms/Models/FormEventBinding.cs @@ -1,3 +1,4 @@ +using CSharpDB.CodeModules; using CSharpDB.Primitives; namespace CSharpDB.Admin.Forms.Models; @@ -19,4 +20,5 @@ public sealed record FormEventBinding( string CommandName, IReadOnlyDictionary? Arguments = null, bool StopOnFailure = true, - DbActionSequence? ActionSequence = null); + DbActionSequence? ActionSequence = null, + CodeModuleHandler? CodeHandler = null); diff --git a/src/CSharpDB.Admin.Forms/README.md b/src/CSharpDB.Admin.Forms/README.md index 79126d1c..939cb819 100644 --- a/src/CSharpDB.Admin.Forms/README.md +++ b/src/CSharpDB.Admin.Forms/README.md @@ -18,6 +18,9 @@ This project is consumed by `CSharpDB.Admin`. It is not a standalone web host. - trusted command-backed form events and command buttons - trusted command-backed selected-control events - declarative action sequences for form and selected-control events +- Access-style rendered-form action steps for record navigation, save/delete, + open/close form, filtering, SQL/procedure, and control property workflows +- conditional action steps and conditional UI rules - generated automation metadata for import/export host callback requirements ## Main Components @@ -112,21 +115,30 @@ properties, optional validation overrides, optional renderer hints, and optional `ControlEventBinding` entries for selected control events such as `OnClick`, `OnChange`, `OnGotFocus`, and `OnLostFocus`. -Form and control event bindings can reference a trusted command name and can -optionally include a `DbActionSequence`. Forms can also store reusable named -action sequences in `ActionSequences`, and event/button sequences can invoke -them with `RunActionSequence`. Action sequences store declarative steps such as +Form and control event bindings can reference a trusted command name, a +`CodeModuleHandler`, and optionally include a `DbActionSequence`. Dispatch order +is trusted command, C# code-module handler, then declarative action sequence. +Forms can also store reusable named action sequences in `ActionSequences`, and +event/button sequences can invoke them with `RunActionSequence`. Action +sequences store declarative steps such as `RunCommand`, `RunActionSequence`, `SetFieldValue`, `ShowMessage`, `Stop`, `NewRecord`, `SaveRecord`, `DeleteRecord`, `RefreshRecords`, `PreviousRecord`, -`NextRecord`, and `GoToRecord`; they do not store C# source or serialized -delegates. The property inspector exposes a visual action-sequence editor on -form-level and selected-control event bindings plus a reusable action library -when editing form properties. JSON editing is limited to optional command or -nested-sequence argument payloads. - -The built-in record actions run only in the rendered Forms data-entry runtime. -Headless form event dispatch can still run command, field, message, and stop -steps, but navigation and save/delete actions require a rendered form instance. +`NextRecord`, `GoToRecord`, `OpenForm`, `CloseForm`, `ApplyFilter`, +`ClearFilter`, `RunSql`, `RunProcedure`, `SetControlProperty`, +`SetControlVisibility`, `SetControlEnabled`, and `SetControlReadOnly`; they do +not store serialized delegates. Database-owned C# source is stored separately +through `CSharpDB.CodeModules` and requires host opt-in, successful build, and +explicit local trust before execution. The property inspector exposes a +visual action-sequence editor on form-level and selected-control event bindings +plus a reusable action library when editing form properties. JSON editing is +limited to optional command or nested-sequence argument payloads. + +The built-in record, form navigation, filter, SQL/procedure, and control +property actions run only in the rendered Forms data-entry runtime. Headless +form event dispatch can still run command, field, message, stop, and other +runtime-independent steps, but actions that need a loaded rendered form instance +report a failure outside that surface. SQL and procedure actions also require +the rendered host to enable those capabilities explicitly. Every action step can also store a simple condition such as `Status = 'Ready'`, `Amount > 0`, or `IsActive`. False conditions skip that step; malformed diff --git a/src/CSharpDB.Admin.Forms/Services/AdminFormCodeModuleCommandApi.cs b/src/CSharpDB.Admin.Forms/Services/AdminFormCodeModuleCommandApi.cs new file mode 100644 index 00000000..28eb98fc --- /dev/null +++ b/src/CSharpDB.Admin.Forms/Services/AdminFormCodeModuleCommandApi.cs @@ -0,0 +1,190 @@ +using CSharpDB.Admin.Forms.Contracts; +using CSharpDB.Admin.Forms.Models; +using CSharpDB.CodeModules.Runtime; +using CSharpDB.Primitives; + +namespace CSharpDB.Admin.Forms.Services; + +internal sealed class AdminFormCodeModuleCommandApi( + FormDefinition form, + DbCommandRegistry commands, + DbExtensionPolicy callbackPolicy, + IReadOnlyDictionary? record, + IReadOnlyDictionary? bindingArguments, + IReadOnlyDictionary? runtimeArguments, + IReadOnlyDictionary metadata, + IReadOnlyList? reusableSequences, + IFormActionRuntime actionRuntime, + Func? setFieldValue = null, + Func? showMessage = null, + Func>? executeBuiltInFormAction = null) : IFormCommandApi +{ + public async ValueTask SetFieldAsync(string fieldName, object? value, CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(fieldName); + ct.ThrowIfCancellationRequested(); + + if (setFieldValue is not null) + { + await setFieldValue(fieldName, value); + return; + } + + if (record is not IDictionary mutable) + throw new InvalidOperationException($"The current form record is read-only and field '{fieldName}' cannot be changed."); + + string key = mutable.Keys.FirstOrDefault(candidate => string.Equals(candidate, fieldName, StringComparison.OrdinalIgnoreCase)) + ?? throw new InvalidOperationException($"Unknown form field '{fieldName}'."); + mutable[key] = value; + } + + public async ValueTask ShowMessageAsync(string message, CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(message); + ct.ThrowIfCancellationRequested(); + if (showMessage is not null) + await showMessage(message); + + return FormCommandApiResult.Success(message); + } + + public async ValueTask RunActionSequenceAsync( + string sequenceName, + IReadOnlyDictionary? arguments = null, + CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sequenceName); + FormEventDispatchResult result = await FormActionSequenceExecutor.ExecuteAsync( + new DbActionSequence( + [new DbActionStep(DbActionKind.RunActionSequence, SequenceName: sequenceName, Arguments: arguments)], + Name: "__CodeModule"), + commands, + record, + bindingArguments, + runtimeArguments, + metadata, + reusableSequences, + setFieldValue, + showMessage, + executeBuiltInFormAction, + actionRuntime, + callbackPolicy, + ct); + + return ToApiResult(result); + } + + public async ValueTask RunHostCommandAsync( + string commandName, + IReadOnlyDictionary? arguments = null, + CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(commandName); + if (!commands.TryGetCommand(commandName, out DbCommandDefinition definition)) + return DbCommandResult.Failure($"Unknown form command '{commandName}'."); + + Dictionary commandArguments = DbCommandArguments.FromObjectDictionaries(record, runtimeArguments, bindingArguments, arguments); + return await definition.InvokeAsync( + commandArguments, + metadata, + callbackPolicy, + DbExtensionHostMode.Embedded, + ct); + } + + public ValueTask SaveRecordAsync(CancellationToken ct = default) + => ExecuteRecordActionAsync(DbActionKind.SaveRecord, ct); + + public ValueTask NewRecordAsync(CancellationToken ct = default) + => ExecuteRecordActionAsync(DbActionKind.NewRecord, ct); + + public ValueTask RefreshRecordsAsync(CancellationToken ct = default) + => ExecuteRecordActionAsync(DbActionKind.RefreshRecords, ct); + + public async ValueTask OpenFormAsync( + string formName, + IReadOnlyDictionary? arguments = null, + CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(formName); + FormEventDispatchResult result = await actionRuntime.OpenFormAsync( + BuildRuntimeContext(null, arguments), + formName, + arguments ?? EmptyObjectDictionary.Instance, + ct); + return ToApiResult(result); + } + + public async ValueTask CloseFormAsync(string? formName = null, CancellationToken ct = default) + { + FormEventDispatchResult result = await actionRuntime.CloseFormAsync( + BuildRuntimeContext(null, null), + formName, + ct); + return ToApiResult(result); + } + + public async ValueTask ApplyFilterAsync( + string filter, + string target = "form", + IReadOnlyDictionary? arguments = null, + CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(filter); + string resolvedTarget = string.IsNullOrWhiteSpace(target) ? "form" : target; + FormEventDispatchResult result = await actionRuntime.ApplyFilterAsync( + BuildRuntimeContext(null, arguments), + resolvedTarget, + filter, + arguments ?? EmptyObjectDictionary.Instance, + ct); + return ToApiResult(result); + } + + public async ValueTask ClearFilterAsync(string target = "form", CancellationToken ct = default) + { + string resolvedTarget = string.IsNullOrWhiteSpace(target) ? "form" : target; + FormEventDispatchResult result = await actionRuntime.ClearFilterAsync( + BuildRuntimeContext(null, null), + resolvedTarget, + ct); + return ToApiResult(result); + } + + private async ValueTask ExecuteRecordActionAsync(DbActionKind kind, CancellationToken ct) + { + var step = new DbActionStep(kind); + FormEventDispatchResult result = executeBuiltInFormAction is not null && actionRuntime is NullFormActionRuntime + ? await executeBuiltInFormAction(step, ct) + : await actionRuntime.ExecuteRecordActionAsync(BuildRuntimeContext(step, null), step, ct); + + return ToApiResult(result); + } + + private FormActionRuntimeContext BuildRuntimeContext( + DbActionStep? step, + IReadOnlyDictionary? stepArguments) + => new( + form.FormId, + form.Name, + form.TableName, + metadata.TryGetValue("event", out string? eventName) ? eventName : null, + "__CodeModule", + 0, + record, + bindingArguments, + runtimeArguments, + stepArguments, + metadata); + + private static FormCommandApiResult ToApiResult(FormEventDispatchResult result) + => result.Succeeded + ? FormCommandApiResult.Success(result.Message) + : FormCommandApiResult.Failure(result.Message ?? "The form command failed."); + + private static class EmptyObjectDictionary + { + public static readonly IReadOnlyDictionary Instance = + new Dictionary(StringComparer.OrdinalIgnoreCase); + } +} diff --git a/src/CSharpDB.Admin.Forms/Services/AdminFormsServiceCollectionExtensions.cs b/src/CSharpDB.Admin.Forms/Services/AdminFormsServiceCollectionExtensions.cs index 4bea54cd..284a21af 100644 --- a/src/CSharpDB.Admin.Forms/Services/AdminFormsServiceCollectionExtensions.cs +++ b/src/CSharpDB.Admin.Forms/Services/AdminFormsServiceCollectionExtensions.cs @@ -1,4 +1,5 @@ using CSharpDB.Admin.Forms.Contracts; +using CSharpDB.CodeModules; using CSharpDB.Primitives; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; @@ -13,6 +14,8 @@ public static IServiceCollection AddCSharpDbAdminForms(this IServiceCollection s services.TryAddSingleton(DbValidationRuleRegistry.Empty); services.TryAddSingleton(DbExtensionPolicies.DefaultHostCallbackPolicy); services.TryAddSingleton(NullFormActionRuntime.Instance); + services.TryAddSingleton(NullCodeModuleFormEventDispatcher.Instance); + services.TryAddSingleton(NullFormCodeModuleDesignerService.Instance); services.TryAddFormControlRegistry(); services.AddScoped(); services.AddScoped(); @@ -33,6 +36,12 @@ public static IServiceCollection AddCSharpDbAdminForms( return services.AddCSharpDbAdminForms(); } + public static IServiceCollection AddCSharpDbAdminFormCodeModules(this IServiceCollection services) + { + services.AddScoped(); + return services; + } + public static IServiceCollection AddCSharpDbAdminFormValidationRules( this IServiceCollection services, Action configureRules) diff --git a/src/CSharpDB.Admin.Forms/Services/DefaultFormEventDispatcher.cs b/src/CSharpDB.Admin.Forms/Services/DefaultFormEventDispatcher.cs index 0db65f52..9e1f4740 100644 --- a/src/CSharpDB.Admin.Forms/Services/DefaultFormEventDispatcher.cs +++ b/src/CSharpDB.Admin.Forms/Services/DefaultFormEventDispatcher.cs @@ -1,5 +1,6 @@ using CSharpDB.Admin.Forms.Contracts; using CSharpDB.Admin.Forms.Models; +using CSharpDB.CodeModules; using CSharpDB.Primitives; namespace CSharpDB.Admin.Forms.Services; @@ -9,14 +10,15 @@ public sealed class DefaultFormEventDispatcher : IFormEventDispatcher private readonly DbCommandRegistry _commands; private readonly DbExtensionPolicy _callbackPolicy; private readonly IFormActionRuntime _actionRuntime; + private readonly ICodeModuleFormEventDispatcher _codeModules; public DefaultFormEventDispatcher(DbCommandRegistry commands) - : this(commands, DbExtensionPolicies.DefaultHostCallbackPolicy, NullFormActionRuntime.Instance) + : this(commands, DbExtensionPolicies.DefaultHostCallbackPolicy, NullFormActionRuntime.Instance, NullCodeModuleFormEventDispatcher.Instance) { } public DefaultFormEventDispatcher(DbCommandRegistry commands, IFormActionRuntime actionRuntime) - : this(commands, DbExtensionPolicies.DefaultHostCallbackPolicy, actionRuntime) + : this(commands, DbExtensionPolicies.DefaultHostCallbackPolicy, actionRuntime, NullCodeModuleFormEventDispatcher.Instance) { } @@ -24,14 +26,25 @@ public DefaultFormEventDispatcher( DbCommandRegistry commands, DbExtensionPolicy callbackPolicy, IFormActionRuntime actionRuntime) + : this(commands, callbackPolicy, actionRuntime, NullCodeModuleFormEventDispatcher.Instance) + { + } + + public DefaultFormEventDispatcher( + DbCommandRegistry commands, + DbExtensionPolicy callbackPolicy, + IFormActionRuntime actionRuntime, + ICodeModuleFormEventDispatcher codeModules) { ArgumentNullException.ThrowIfNull(commands); ArgumentNullException.ThrowIfNull(callbackPolicy); ArgumentNullException.ThrowIfNull(actionRuntime); + ArgumentNullException.ThrowIfNull(codeModules); _commands = commands; _callbackPolicy = callbackPolicy; _actionRuntime = actionRuntime; + _codeModules = codeModules; } public async Task DispatchAsync( @@ -100,13 +113,50 @@ public async Task DispatchAsync( if (binding.StopOnFailure) return FormEventDispatchResult.Failure(commandFailureMessage!); - if (binding.ActionSequence is null) + if (binding.CodeHandler is null && binding.ActionSequence is null) continue; } } - else if (binding.ActionSequence is null) + else if (binding.CodeHandler is null && binding.ActionSequence is null) { - return FormEventDispatchResult.Failure($"Form event '{eventKind}' has no command or action sequence."); + return FormEventDispatchResult.Failure($"Form event '{eventKind}' has no command, code handler, or action sequence."); + } + + if (binding.CodeHandler is not null) + { + var commandApi = new AdminFormCodeModuleCommandApi( + form, + _commands, + _callbackPolicy, + record, + binding.Arguments, + runtimeArguments: null, + metadata, + form.ActionSequences, + actionRuntime); + CodeModuleFormDispatchResult codeResult = await _codeModules.DispatchAsync( + binding.CodeHandler, + new CodeModuleFormEventDispatchContext( + form.FormId, + form.Name, + form.TableName, + eventKind.ToString(), + record, + binding.Arguments, + RuntimeArguments: null, + metadata, + commandApi, + IsCancelableEvent(eventKind)), + ct); + + if (!codeResult.Succeeded) + { + if (binding.StopOnFailure) + return FormEventDispatchResult.Failure(codeResult.Message ?? $"Form event '{eventKind}' code handler failed."); + + if (binding.ActionSequence is null) + continue; + } } if (binding.ActionSequence is not null) @@ -130,4 +180,7 @@ public async Task DispatchAsync( return FormEventDispatchResult.Success(); } + + private static bool IsCancelableEvent(FormEventKind eventKind) + => eventKind is FormEventKind.BeforeInsert or FormEventKind.BeforeUpdate or FormEventKind.BeforeDelete; } diff --git a/src/CSharpDB.Admin.Forms/Services/FormCodeModuleDesignerService.cs b/src/CSharpDB.Admin.Forms/Services/FormCodeModuleDesignerService.cs new file mode 100644 index 00000000..5f885120 --- /dev/null +++ b/src/CSharpDB.Admin.Forms/Services/FormCodeModuleDesignerService.cs @@ -0,0 +1,148 @@ +using System.Text; +using CSharpDB.Admin.Forms.Models; +using CSharpDB.CodeModules; +using CSharpDB.CodeModules.Runtime; + +namespace CSharpDB.Admin.Forms.Services; + +public interface IFormCodeModuleDesignerService +{ + Task CreateHandlerAsync( + FormCodeModuleHandlerRequest request, + CancellationToken ct = default); +} + +public sealed record FormCodeModuleHandlerRequest( + string FormId, + string FormName, + string TableName, + string EventName, + bool IsCancelable, + string? ControlId = null, + string? ControlType = null); + +public sealed class NullFormCodeModuleDesignerService : IFormCodeModuleDesignerService +{ + public static NullFormCodeModuleDesignerService Instance { get; } = new(); + + private NullFormCodeModuleDesignerService() + { + } + + public Task CreateHandlerAsync( + FormCodeModuleHandlerRequest request, + CancellationToken ct = default) + => Task.FromException( + new InvalidOperationException("C# code module designer support is not configured for this host.")); +} + +public sealed class CSharpDbFormCodeModuleDesignerService(CSharpDbCodeModuleClient codeModules) : IFormCodeModuleDesignerService +{ + private const string NamespaceName = "CSharpDB.UserCode.Forms"; + + public async Task CreateHandlerAsync( + FormCodeModuleHandlerRequest request, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(request); + if (string.IsNullOrWhiteSpace(request.FormId)) + throw new InvalidOperationException("Save the form before creating C# handlers."); + + string moduleId = $"form:{request.FormId}"; + string typeName = $"{NamespaceName}.{ToIdentifier(request.FormName, "Form")}Module"; + string methodName = string.IsNullOrWhiteSpace(request.ControlId) + ? $"On{ToIdentifier(request.EventName, "Event")}" + : $"{ToIdentifier(request.ControlId, "Control")}_{ToIdentifier(request.EventName, "Event")}"; + string contextType = string.IsNullOrWhiteSpace(request.ControlId) + ? request.IsCancelable ? nameof(FormBeforeEventContext) : nameof(FormEventContext) + : nameof(FormControlEventContext); + + CodeModuleDefinition? existing = await codeModules.GetAsync(moduleId, ct); + string source = existing is null + ? CreateFormModuleSource(typeName, methodName, contextType) + : EnsureMethodStub(existing.Source, methodName, contextType); + + await codeModules.UpsertAsync(new CodeModuleDefinition( + moduleId, + string.IsNullOrWhiteSpace(request.FormName) ? moduleId : request.FormName, + CodeModuleKind.Form, + source, + OwnerKind: "Form", + OwnerId: request.FormId, + TypeName: typeName, + Metadata: new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["tableName"] = request.TableName, + }), ct); + + return new CodeModuleHandler(moduleId, typeName, methodName); + } + + private static string CreateFormModuleSource(string typeName, string methodName, string contextType) + { + string shortTypeName = typeName.Split('.').Last(); + return $$""" + using CSharpDB.CodeModules.Runtime; + + namespace {{NamespaceName}}; + + public sealed class {{shortTypeName}} : FormCodeModule + { + public void {{methodName}}({{contextType}} context) + { + } + } + """; + } + + private static string EnsureMethodStub(string source, string methodName, string contextType) + { + if (source.Contains($" {methodName}(", StringComparison.Ordinal) || + source.Contains($"\t{methodName}(", StringComparison.Ordinal)) + { + return source; + } + + int insertAt = source.LastIndexOf('}'); + if (insertAt < 0) + return source + Environment.NewLine + CreateDetachedMethod(methodName, contextType); + + var builder = new StringBuilder(source.Length + 160); + builder.Append(source.AsSpan(0, insertAt)); + if (!source[..insertAt].EndsWith(Environment.NewLine, StringComparison.Ordinal)) + builder.AppendLine(); + + builder.AppendLine(); + builder.AppendLine($" public void {methodName}({contextType} context)"); + builder.AppendLine(" {"); + builder.AppendLine(" }"); + builder.Append(source.AsSpan(insertAt)); + return builder.ToString(); + } + + private static string CreateDetachedMethod(string methodName, string contextType) + => $$""" + + public void {{methodName}}({{contextType}} context) + { + } + """; + + private static string ToIdentifier(string? value, string fallback) + { + string text = string.IsNullOrWhiteSpace(value) ? fallback : value; + var builder = new StringBuilder(text.Length + 1); + foreach (char ch in text) + { + if (char.IsLetterOrDigit(ch) || ch == '_') + builder.Append(ch); + } + + if (builder.Length == 0) + builder.Append(fallback); + if (!char.IsLetter(builder[0]) && builder[0] != '_') + builder.Insert(0, '_'); + + return builder.ToString(); + } +} diff --git a/src/CSharpDB.Admin/CSharpDB.Admin.csproj b/src/CSharpDB.Admin/CSharpDB.Admin.csproj index 2c073bb4..bd5e62d0 100644 --- a/src/CSharpDB.Admin/CSharpDB.Admin.csproj +++ b/src/CSharpDB.Admin/CSharpDB.Admin.csproj @@ -10,6 +10,7 @@ + diff --git a/src/CSharpDB.Admin/Components/Layout/CommandPalette.razor b/src/CSharpDB.Admin/Components/Layout/CommandPalette.razor index 40fb3611..3423336d 100644 --- a/src/CSharpDB.Admin/Components/Layout/CommandPalette.razor +++ b/src/CSharpDB.Admin/Components/Layout/CommandPalette.razor @@ -222,6 +222,7 @@ _items.Add(new PaletteItem("Command", "New Report", "Open report designer", "bi-file-earmark-richtext", "icon-report", () => { TabManager.OpenReportDesignerTab(); return Task.CompletedTask; })); _items.Add(new PaletteItem("Command", "New Procedure", "Open procedure editor", "bi-gear-wide-connected", "icon-trigger", () => { TabManager.OpenNewProcedureTab(); return Task.CompletedTask; })); _items.Add(new PaletteItem("Command", "New Pipeline", "Open pipeline builder", "bi-diagram-3", "icon-view", () => { TabManager.OpenPipelineTab(); return Task.CompletedTask; })); + _items.Add(new PaletteItem("Command", "Code Modules", "Open C# code modules", "bi-filetype-cs", "icon-system", () => { TabManager.OpenCodeModulesTab(); return Task.CompletedTask; })); _items.Add(new PaletteItem("Command", "Storage Inspector", "Open storage diagnostics", "bi-hdd-stack", "icon-system", () => { TabManager.OpenStorageTab(); return Task.CompletedTask; })); } diff --git a/src/CSharpDB.Admin/Components/Layout/MainLayout.razor b/src/CSharpDB.Admin/Components/Layout/MainLayout.razor index 9b77d2d7..3c02b127 100644 --- a/src/CSharpDB.Admin/Components/Layout/MainLayout.razor +++ b/src/CSharpDB.Admin/Components/Layout/MainLayout.razor @@ -51,6 +51,10 @@ break; + case TabKind.CodeModules: + + break; case TabKind.Procedure: diff --git a/src/CSharpDB.Admin/Components/Layout/TitleBar.razor b/src/CSharpDB.Admin/Components/Layout/TitleBar.razor index f2f00423..ab022eba 100644 --- a/src/CSharpDB.Admin/Components/Layout/TitleBar.razor +++ b/src/CSharpDB.Admin/Components/Layout/TitleBar.razor @@ -42,6 +42,9 @@ + diff --git a/src/CSharpDB.Admin/Components/Tabs/CodeModulesTab.razor b/src/CSharpDB.Admin/Components/Tabs/CodeModulesTab.razor new file mode 100644 index 00000000..16f7be65 --- /dev/null +++ b/src/CSharpDB.Admin/Components/Tabs/CodeModulesTab.razor @@ -0,0 +1,253 @@ +@using CSharpDB.CodeModules +@inject CSharpDbCodeModuleClient CodeModules +@inject ToastService Toast + +
+
+
+ + / + code modules +
+ +
+ + + + + +
+ +
+ +
+ +
+ @GetBuildText() + @GetTrustText() +
+
+ +
+
+ @if (_modules.Count == 0) + { +
No C# code modules are stored in this database.
+ } + else + { + + + + + + + + + + + @foreach (CodeModuleSummary module in _modules) + { + + + + + + + } + +
NameKindOwnerHash
@module.Name@module.Kind@FormatOwner(module)@ShortHash(module.SourceHash)
+ } +
+ +
+ @if (_selected is null) + { +
Select a module to preview source.
+ } + else + { +
+
+
@_selected.Kind
+

@_selected.Name

+
+ @ShortHash(_selected.SourceHash ?? string.Empty) +
+ + + + + + + + +
Module Id@_selected.ModuleId
Type@(_selected.TypeName ?? "-")
Owner@((_selected.OwnerKind ?? "-") + "/" + (_selected.OwnerId ?? "-"))
Updated@_selected.UpdatedUtc?.LocalDateTime.ToString("g")
+ +
+

Source Preview

+ 
@_selected.Source
+
+ } + +
+

Diagnostics

+ @if (_lastBuild is null) + { +
Build has not run in this tab.
+ } + else if (_lastBuild.Diagnostics.Count == 0) + { +
No diagnostics.
+ } + else + { + + + + + + + + + + + @foreach (CodeModuleDiagnostic diagnostic in _lastBuild.Diagnostics) + { + + + + + + + } + +
SeverityModuleLineMessage
@diagnostic.Severity@(diagnostic.ModuleId ?? diagnostic.Path ?? "-")@diagnostic.Line:@diagnostic.Column@diagnostic.Code @diagnostic.Message
+ } +
+ + @if (_lastImport is not null) + { +
+

Last Import

+ + + @foreach (CodeModuleImportChange change in _lastImport.Changes) + { + + } + +
@change.Kind@change.ModuleId - @(change.Message ?? change.Path)
+
+ } +
+
+
+ +@code { + [Parameter, EditorRequired] public TabDescriptor Tab { get; set; } = default!; + + private List _modules = []; + private CodeModuleDefinition? _selected; + private string? _selectedModuleId; + private CodeModuleBuildResult? _lastBuild; + private CodeModuleTrustState? _trustState; + private CodeModuleImportResult? _lastImport; + private string _workspacePath = string.Empty; + + private bool CanTrust => _lastBuild?.Succeeded == true && _trustState?.IsTrusted != true; + + protected override async Task OnInitializedAsync() + { + _workspacePath = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments); + await RefreshAsync(); + } + + private async Task RefreshAsync() + { + _modules = (await CodeModules.ListAsync()).ToList(); + _trustState = await CodeModules.GetTrustStateAsync(); + if (_selectedModuleId is not null) + _selected = await CodeModules.GetAsync(_selectedModuleId); + if (_selected is null && _modules.Count > 0) + await SelectModuleAsync(_modules[0].ModuleId); + } + + private async Task SelectModuleAsync(string moduleId) + { + _selectedModuleId = moduleId; + _selected = await CodeModules.GetAsync(moduleId); + } + + private async Task BuildAsync() + { + _lastBuild = await CodeModules.BuildAsync(); + _trustState = await CodeModules.GetTrustStateAsync(_lastBuild.ModuleSetHash); + Toast.Show(_lastBuild.Succeeded ? "Code modules built successfully." : "Code module build failed."); + } + + private async Task TrustAsync() + { + await CodeModules.TrustAsync(); + _lastBuild = await CodeModules.BuildAsync(); + _trustState = await CodeModules.GetTrustStateAsync(_lastBuild.ModuleSetHash); + Toast.Show("Current C# code module build is trusted locally."); + } + + private async Task ExportAsync() + { + CodeModuleExportResult result = await CodeModules.ExportAsync(GetWorkspacePath()); + Toast.Show($"Exported {result.ModuleCount} code module(s)."); + } + + private async Task ImportAsync() + { + _lastImport = await CodeModules.ImportAsync(GetWorkspacePath()); + await RefreshAsync(); + Toast.Show(_lastImport.HasConflicts ? "Code module import completed with conflicts." : "Code module import completed."); + } + + private string GetWorkspacePath() + => string.IsNullOrWhiteSpace(_workspacePath) + ? Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments) + : _workspacePath; + + private string GetBuildText() + => _lastBuild is null ? "Not built" : _lastBuild.Succeeded ? "Build passed" : "Build failed"; + + private string GetBuildBadgeClass() + => _lastBuild is null + ? "callbacks-policy-badge" + : _lastBuild.Succeeded ? "callbacks-policy-badge allowed" : "callbacks-policy-badge denied"; + + private string GetTrustText() + => _trustState?.IsTrusted == true ? "Trusted" : "Untrusted"; + + private string GetTrustBadgeClass() + => _trustState?.IsTrusted == true ? "callbacks-policy-badge allowed" : "callbacks-policy-badge denied"; + + private static string ShortHash(string hash) + => string.IsNullOrWhiteSpace(hash) ? "-" : hash[..Math.Min(12, hash.Length)]; + + private static string FormatOwner(CodeModuleSummary module) + => string.IsNullOrWhiteSpace(module.OwnerKind) && string.IsNullOrWhiteSpace(module.OwnerId) + ? "-" + : $"{module.OwnerKind ?? "-"}:{module.OwnerId ?? "-"}"; +} diff --git a/src/CSharpDB.Admin/Models/TabDescriptor.cs b/src/CSharpDB.Admin/Models/TabDescriptor.cs index c2848441..ae618999 100644 --- a/src/CSharpDB.Admin/Models/TabDescriptor.cs +++ b/src/CSharpDB.Admin/Models/TabDescriptor.cs @@ -8,6 +8,7 @@ public enum TabKind ViewData, CollectionData, HostCallbacks, + CodeModules, Procedure, Pipeline, Storage, diff --git a/src/CSharpDB.Admin/Program.cs b/src/CSharpDB.Admin/Program.cs index 4896c433..2a460f2e 100644 --- a/src/CSharpDB.Admin/Program.cs +++ b/src/CSharpDB.Admin/Program.cs @@ -5,6 +5,7 @@ using CSharpDB.Admin.Reports.Services; using CSharpDB.Admin.Services; using CSharpDB.Client; +using CSharpDB.CodeModules; using CSharpDB.Primitives; var builder = WebApplication.CreateBuilder(args); @@ -44,7 +45,9 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddSingleton(); +builder.Services.AddCSharpDbCodeModules(options => options.EnableInProcessExecution = true); builder.Services.AddCSharpDbAdminForms(); +builder.Services.AddCSharpDbAdminFormCodeModules(); if (builder.Configuration.GetValue("AdminForms:EnableSampleControls")) builder.Services.AddSampleFormControls(); builder.Services.AddCSharpDbAdminReports(); diff --git a/src/CSharpDB.Admin/Services/TabManagerService.cs b/src/CSharpDB.Admin/Services/TabManagerService.cs index 179fac0f..5326cb2b 100644 --- a/src/CSharpDB.Admin/Services/TabManagerService.cs +++ b/src/CSharpDB.Admin/Services/TabManagerService.cs @@ -117,6 +117,21 @@ public TabDescriptor OpenCallbacksTab(string? selectedCallbackName = null, strin return _tabs.First(t => t.Id == tab.Id); } + public TabDescriptor OpenCodeModulesTab() + { + const string tabId = "code-modules"; + TabDescriptor? existing = _tabs.FirstOrDefault(t => t.Id == tabId); + if (existing is not null) + { + ActivateTab(existing.Id); + return existing; + } + + var tab = new TabDescriptor(tabId, "Code Modules", "bi-filetype-cs", TabKind.CodeModules); + OpenTab(tab); + return _tabs.First(t => t.Id == tab.Id); + } + public TabDescriptor OpenQueryTab(string? initialSql = null) { int num = Interlocked.Increment(ref _queryCounter); diff --git a/src/CSharpDB.CodeModules/CSharpDB.CodeModules.csproj b/src/CSharpDB.CodeModules/CSharpDB.CodeModules.csproj new file mode 100644 index 00000000..c4d437c2 --- /dev/null +++ b/src/CSharpDB.CodeModules/CSharpDB.CodeModules.csproj @@ -0,0 +1,21 @@ + + + + net10.0 + enable + enable + CSharpDB.CodeModules + Database-owned C# code modules, local trust, workspace sync, and Admin Forms runtime contracts for CSharpDB. + + + + + + + + + + + + + diff --git a/src/CSharpDB.CodeModules/CSharpDbCodeModuleClient.cs b/src/CSharpDB.CodeModules/CSharpDbCodeModuleClient.cs new file mode 100644 index 00000000..f413f58e --- /dev/null +++ b/src/CSharpDB.CodeModules/CSharpDbCodeModuleClient.cs @@ -0,0 +1,614 @@ +using System.Collections.Immutable; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; +using CSharpDB.Client; +using CSharpDB.CodeModules.Infrastructure; +using CSharpDB.CodeModules.Runtime; +using CSharpDB.CodeModules.Trust; +using CSharpDB.Primitives; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; + +namespace CSharpDB.CodeModules; + +public sealed class CSharpDbCodeModuleClient( + ICSharpDbClient dbClient, + ICodeModuleTrustStore trustStore) +{ + public CSharpDbCodeModuleClient(ICSharpDbClient dbClient) + : this(dbClient, new FileCodeModuleTrustStore()) + { + } + + public const string CodeModulesTableName = "__code_modules"; + public const string CodeModuleBuildsTableName = "__code_module_builds"; + public const string WorkspaceDirectoryName = ".csharpdb-code"; + public const string ManifestFileName = "csharpdb.codeproj.json"; + + private static readonly JsonSerializerOptions s_jsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false, + Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) }, + }; + + private static readonly JsonSerializerOptions s_manifestJsonOptions = new(s_jsonOptions) + { + WriteIndented = true, + }; + + private readonly SemaphoreSlim _initLock = new(1, 1); + private bool _initialized; + + public async Task> ListAsync(CancellationToken ct = default) + { + await EnsureInitializedAsync(ct); + string sql = $""" + SELECT module_id, name, module_kind, owner_kind, owner_id, type_name, source_hash, created_utc, updated_utc + FROM {CodeModulesTableName} + ORDER BY module_kind, name, module_id; + """; + + return CodeModuleSql.ReadRows(await dbClient.ExecuteSqlAsync(sql, ct)) + .Select(ReadSummary) + .ToArray(); + } + + public async Task GetAsync(string moduleId, CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(moduleId); + await EnsureInitializedAsync(ct); + string sql = $""" + SELECT module_id, name, module_kind, language, owner_kind, owner_id, type_name, source, source_hash, metadata_json, created_utc, updated_utc + FROM {CodeModulesTableName} + WHERE module_id = {CodeModuleSql.FormatLiteral(moduleId)} + LIMIT 1; + """; + + Dictionary? row = CodeModuleSql.ReadRows(await dbClient.ExecuteSqlAsync(sql, ct)).FirstOrDefault(); + return row is null ? null : ReadDefinition(row); + } + + public async Task UpsertAsync(CodeModuleDefinition module, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(module); + await EnsureInitializedAsync(ct); + + CodeModuleDefinition normalized = NormalizeForStorage(module); + string now = DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture); + CodeModuleDefinition? existing = await GetAsync(normalized.ModuleId, ct); + string metadataJson = JsonSerializer.Serialize(normalized.SafeMetadata, s_jsonOptions); + + string sql = existing is null + ? $""" + INSERT INTO {CodeModulesTableName} ( + module_id, + name, + module_kind, + language, + owner_kind, + owner_id, + type_name, + source, + source_hash, + metadata_json, + created_utc, + updated_utc + ) + VALUES ( + {CodeModuleSql.FormatLiteral(normalized.ModuleId)}, + {CodeModuleSql.FormatLiteral(normalized.Name)}, + {CodeModuleSql.FormatLiteral(normalized.Kind.ToString())}, + {CodeModuleSql.FormatLiteral(normalized.Language)}, + {CodeModuleSql.FormatLiteral(normalized.OwnerKind)}, + {CodeModuleSql.FormatLiteral(normalized.OwnerId)}, + {CodeModuleSql.FormatLiteral(normalized.TypeName)}, + {CodeModuleSql.FormatLiteral(normalized.Source)}, + {CodeModuleSql.FormatLiteral(normalized.SourceHash)}, + {CodeModuleSql.FormatLiteral(metadataJson)}, + {CodeModuleSql.FormatLiteral(now)}, + {CodeModuleSql.FormatLiteral(now)} + ); + """ + : $""" + UPDATE {CodeModulesTableName} + SET name = {CodeModuleSql.FormatLiteral(normalized.Name)}, + module_kind = {CodeModuleSql.FormatLiteral(normalized.Kind.ToString())}, + language = {CodeModuleSql.FormatLiteral(normalized.Language)}, + owner_kind = {CodeModuleSql.FormatLiteral(normalized.OwnerKind)}, + owner_id = {CodeModuleSql.FormatLiteral(normalized.OwnerId)}, + type_name = {CodeModuleSql.FormatLiteral(normalized.TypeName)}, + source = {CodeModuleSql.FormatLiteral(normalized.Source)}, + source_hash = {CodeModuleSql.FormatLiteral(normalized.SourceHash)}, + metadata_json = {CodeModuleSql.FormatLiteral(metadataJson)}, + updated_utc = {CodeModuleSql.FormatLiteral(now)} + WHERE module_id = {CodeModuleSql.FormatLiteral(normalized.ModuleId)}; + """; + + CodeModuleSql.ThrowIfError(await dbClient.ExecuteSqlAsync(sql, ct)); + return await GetAsync(normalized.ModuleId, ct) + ?? throw new InvalidOperationException($"Code module '{normalized.ModuleId}' could not be loaded after save."); + } + + public async Task DeleteAsync(string moduleId, CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(moduleId); + await EnsureInitializedAsync(ct); + string sql = $""" + DELETE FROM {CodeModulesTableName} + WHERE module_id = {CodeModuleSql.FormatLiteral(moduleId)}; + """; + + var result = await dbClient.ExecuteSqlAsync(sql, ct); + CodeModuleSql.ThrowIfError(result); + return result.RowsAffected > 0; + } + + public async Task ExportAsync(string workspaceDirectory, CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(workspaceDirectory); + await EnsureInitializedAsync(ct); + string root = PrepareWorkspaceRoot(workspaceDirectory); + IReadOnlyList modules = await ListDefinitionsAsync(ct); + string moduleSetHash = CodeModuleHashing.ComputeModuleSetHash(modules); + + var entries = new List(modules.Count); + var usedPaths = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (CodeModuleDefinition module in modules) + { + string relativePath = BuildWorkspacePath(module, usedPaths); + string absolutePath = Path.Combine(root, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(absolutePath)!); + await File.WriteAllTextAsync(absolutePath, module.Source, ct); + entries.Add(new CodeModuleWorkspaceManifestEntry( + module.ModuleId, + module.Name, + module.Kind, + module.OwnerKind, + module.OwnerId, + module.TypeName, + relativePath.Replace('\\', '/'), + module.SourceHash ?? CodeModuleHashing.ComputeSourceHash(module.Source), + module.SafeMetadata)); + } + + var manifest = new CodeModuleWorkspaceManifest(1, moduleSetHash, entries); + string manifestPath = Path.Combine(root, ManifestFileName); + await File.WriteAllTextAsync(manifestPath, JsonSerializer.Serialize(manifest, s_manifestJsonOptions), ct); + return new CodeModuleExportResult(root, manifestPath, modules.Count, moduleSetHash); + } + + public async Task ImportAsync(string workspaceDirectory, CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(workspaceDirectory); + await EnsureInitializedAsync(ct); + string root = PrepareWorkspaceRoot(workspaceDirectory); + string manifestPath = Path.Combine(root, ManifestFileName); + if (!File.Exists(manifestPath)) + throw new FileNotFoundException("The CSharpDB code module manifest was not found.", manifestPath); + + CodeModuleWorkspaceManifest manifest = JsonSerializer.Deserialize( + await File.ReadAllTextAsync(manifestPath, ct), + s_manifestJsonOptions) + ?? throw new InvalidOperationException("The CSharpDB code module manifest could not be read."); + + var changes = new List(); + foreach (CodeModuleWorkspaceManifestEntry entry in manifest.Modules) + { + string relativePath = NormalizeRelativePath(entry.Path); + string absolutePath = Path.Combine(root, relativePath); + if (!File.Exists(absolutePath)) + { + changes.Add(new CodeModuleImportChange(entry.ModuleId, entry.Path, CodeModuleImportChangeKind.Skipped, "Source file is missing.")); + continue; + } + + string source = await File.ReadAllTextAsync(absolutePath, ct); + string fileHash = CodeModuleHashing.ComputeSourceHash(source); + CodeModuleDefinition? existing = await GetAsync(entry.ModuleId, ct); + if (existing is not null && + !string.Equals(existing.SourceHash, entry.SourceHash, StringComparison.OrdinalIgnoreCase) && + !string.Equals(fileHash, entry.SourceHash, StringComparison.OrdinalIgnoreCase)) + { + changes.Add(new CodeModuleImportChange( + entry.ModuleId, + entry.Path, + CodeModuleImportChangeKind.Conflict, + "Both the database module and exported file changed since export.")); + continue; + } + + if (existing is not null && string.Equals(existing.SourceHash, fileHash, StringComparison.OrdinalIgnoreCase)) + { + changes.Add(new CodeModuleImportChange(entry.ModuleId, entry.Path, CodeModuleImportChangeKind.Unchanged)); + continue; + } + + if (existing is not null && !string.Equals(existing.SourceHash, entry.SourceHash, StringComparison.OrdinalIgnoreCase)) + { + changes.Add(new CodeModuleImportChange(entry.ModuleId, entry.Path, CodeModuleImportChangeKind.Skipped, "Database module changed and exported file is unchanged.")); + continue; + } + + var updated = new CodeModuleDefinition( + entry.ModuleId, + entry.Name, + entry.Kind, + source, + entry.OwnerKind, + entry.OwnerId, + entry.TypeName, + entry.Metadata, + SourceHash: fileHash); + await UpsertAsync(updated, ct); + changes.Add(new CodeModuleImportChange( + entry.ModuleId, + entry.Path, + existing is null ? CodeModuleImportChangeKind.Added : CodeModuleImportChangeKind.Updated)); + } + + return new CodeModuleImportResult(root, changes); + } + + public async Task BuildAsync(CancellationToken ct = default) + { + await EnsureInitializedAsync(ct); + IReadOnlyList modules = await ListDefinitionsAsync(ct); + string moduleSetHash = CodeModuleHashing.ComputeModuleSetHash(modules); + DateTimeOffset builtUtc = DateTimeOffset.UtcNow; + + Dictionary treeModuleMap = new(); + List syntaxTrees = []; + foreach (CodeModuleDefinition module in modules) + { + string path = BuildDiagnosticPath(module); + SyntaxTree tree = CSharpSyntaxTree.ParseText( + CodeModuleHashing.NormalizeSource(module.Source), + new CSharpParseOptions(LanguageVersion.Preview), + path, + cancellationToken: ct); + treeModuleMap[tree] = module; + syntaxTrees.Add(tree); + } + + CSharpCompilation compilation = CSharpCompilation.Create( + $"CSharpDB.CodeModules.Database.{moduleSetHash}", + syntaxTrees, + BuildReferences(), + new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary)); + + using var assemblyStream = new MemoryStream(); + Microsoft.CodeAnalysis.Emit.EmitResult emit = compilation.Emit(assemblyStream, cancellationToken: ct); + IReadOnlyList diagnostics = emit.Diagnostics + .Select(diagnostic => ConvertDiagnostic(diagnostic, treeModuleMap)) + .ToArray(); + var result = new CodeModuleBuildResult( + moduleSetHash, + emit.Success ? CodeModuleBuildStatus.Succeeded : CodeModuleBuildStatus.Failed, + diagnostics, + builtUtc, + emit.Success ? assemblyStream.ToArray() : null); + await StoreBuildResultAsync(result with { AssemblyBytes = null }, ct); + return result; + } + + public async Task TrustAsync(CancellationToken ct = default) + { + CodeModuleBuildResult build = await BuildAsync(ct); + if (!build.Succeeded) + throw new InvalidOperationException("C# code modules must build successfully before they can be trusted."); + + string databasePath = await GetDatabasePathAsync(ct); + await trustStore.TrustAsync(databasePath, build.ModuleSetHash, ct); + } + + public async Task GetTrustStateAsync(string? moduleSetHash = null, CancellationToken ct = default) + { + string resolvedHash = string.IsNullOrWhiteSpace(moduleSetHash) + ? CodeModuleHashing.ComputeModuleSetHash(await ListDefinitionsAsync(ct)) + : moduleSetHash; + string databasePath = await GetDatabasePathAsync(ct); + return await trustStore.GetTrustStateAsync(databasePath, resolvedHash, ct); + } + + public async Task GetCurrentModuleSetHashAsync(CancellationToken ct = default) + => CodeModuleHashing.ComputeModuleSetHash(await ListDefinitionsAsync(ct)); + + private async Task> ListDefinitionsAsync(CancellationToken ct) + { + await EnsureInitializedAsync(ct); + string sql = $""" + SELECT module_id, name, module_kind, language, owner_kind, owner_id, type_name, source, source_hash, metadata_json, created_utc, updated_utc + FROM {CodeModulesTableName} + ORDER BY module_id; + """; + + return CodeModuleSql.ReadRows(await dbClient.ExecuteSqlAsync(sql, ct)) + .Select(ReadDefinition) + .ToArray(); + } + + private async Task EnsureInitializedAsync(CancellationToken ct) + { + if (_initialized) + return; + + await _initLock.WaitAsync(ct); + try + { + if (_initialized) + return; + + string sql = $""" + CREATE TABLE IF NOT EXISTS {CodeModulesTableName} ( + module_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + module_kind TEXT NOT NULL, + language TEXT NOT NULL, + owner_kind TEXT, + owner_id TEXT, + type_name TEXT, + source TEXT NOT NULL, + source_hash TEXT NOT NULL, + metadata_json TEXT, + created_utc TEXT NOT NULL, + updated_utc TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx___code_modules_owner + ON {CodeModulesTableName} (owner_kind, owner_id); + CREATE TABLE IF NOT EXISTS {CodeModuleBuildsTableName} ( + build_id TEXT PRIMARY KEY, + module_set_hash TEXT NOT NULL, + status TEXT NOT NULL, + diagnostics_json TEXT NOT NULL, + created_utc TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx___code_module_builds_module_set_hash + ON {CodeModuleBuildsTableName} (module_set_hash); + """; + + CodeModuleSql.ThrowIfError(await dbClient.ExecuteSqlAsync(sql, ct)); + _initialized = true; + } + finally + { + _initLock.Release(); + } + } + + private async Task StoreBuildResultAsync(CodeModuleBuildResult result, CancellationToken ct) + { + string diagnosticsJson = JsonSerializer.Serialize(result.Diagnostics, s_jsonOptions); + string sql = $""" + INSERT INTO {CodeModuleBuildsTableName} ( + build_id, + module_set_hash, + status, + diagnostics_json, + created_utc + ) + VALUES ( + {CodeModuleSql.FormatLiteral(Guid.NewGuid().ToString("N"))}, + {CodeModuleSql.FormatLiteral(result.ModuleSetHash)}, + {CodeModuleSql.FormatLiteral(result.Status.ToString())}, + {CodeModuleSql.FormatLiteral(diagnosticsJson)}, + {CodeModuleSql.FormatLiteral(result.BuiltUtc.ToString("O", CultureInfo.InvariantCulture))} + ); + """; + + CodeModuleSql.ThrowIfError(await dbClient.ExecuteSqlAsync(sql, ct)); + } + + private async Task GetDatabasePathAsync(CancellationToken ct) + { + string dataSource = dbClient.DataSource; + if (!string.IsNullOrWhiteSpace(dataSource)) + return dataSource; + + return (await dbClient.GetInfoAsync(ct)).DataSource; + } + + private static CodeModuleDefinition NormalizeForStorage(CodeModuleDefinition module) + { + if (string.IsNullOrWhiteSpace(module.ModuleId)) + throw new ArgumentException("Code modules require a module id.", nameof(module)); + if (string.IsNullOrWhiteSpace(module.Name)) + throw new ArgumentException("Code modules require a name.", nameof(module)); + if (string.IsNullOrWhiteSpace(module.Language)) + throw new ArgumentException("Code modules require a language.", nameof(module)); + + string source = CodeModuleHashing.NormalizeSource(module.Source); + string sourceHash = CodeModuleHashing.ComputeSourceHash(source); + return module with + { + ModuleId = module.ModuleId.Trim(), + Name = module.Name.Trim(), + Language = module.Language.Trim(), + OwnerKind = string.IsNullOrWhiteSpace(module.OwnerKind) ? null : module.OwnerKind.Trim(), + OwnerId = string.IsNullOrWhiteSpace(module.OwnerId) ? null : module.OwnerId.Trim(), + TypeName = string.IsNullOrWhiteSpace(module.TypeName) ? null : module.TypeName.Trim(), + Source = source, + SourceHash = sourceHash, + }; + } + + private static CodeModuleSummary ReadSummary(Dictionary row) + => new( + ReadRequired(row, "module_id"), + ReadRequired(row, "name"), + ReadKind(row), + ReadOptional(row, "owner_kind"), + ReadOptional(row, "owner_id"), + ReadOptional(row, "type_name"), + ReadRequired(row, "source_hash"), + ReadUtc(row, "created_utc"), + ReadUtc(row, "updated_utc")); + + private static CodeModuleDefinition ReadDefinition(Dictionary row) + { + IReadOnlyDictionary? metadata = null; + string? metadataJson = ReadOptional(row, "metadata_json"); + if (!string.IsNullOrWhiteSpace(metadataJson)) + { + metadata = JsonSerializer.Deserialize>(metadataJson, s_jsonOptions) + ?? new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + return new CodeModuleDefinition( + ReadRequired(row, "module_id"), + ReadRequired(row, "name"), + ReadKind(row), + ReadRequired(row, "source"), + ReadOptional(row, "owner_kind"), + ReadOptional(row, "owner_id"), + ReadOptional(row, "type_name"), + metadata, + ReadOptional(row, "language") ?? "csharp", + ReadRequired(row, "source_hash"), + ReadUtc(row, "created_utc"), + ReadUtc(row, "updated_utc")); + } + + private static CodeModuleKind ReadKind(Dictionary row) + => Enum.TryParse(ReadRequired(row, "module_kind"), ignoreCase: true, out CodeModuleKind kind) + ? kind + : throw new InvalidOperationException($"Unknown code module kind '{ReadRequired(row, "module_kind")}'."); + + private static string ReadRequired(Dictionary row, string column) + => row.TryGetValue(column, out object? value) && value is not null + ? Convert.ToString(value, CultureInfo.InvariantCulture) ?? string.Empty + : throw new InvalidOperationException($"Code module row is missing '{column}'."); + + private static string? ReadOptional(Dictionary row, string column) + => row.TryGetValue(column, out object? value) && value is not null + ? Convert.ToString(value, CultureInfo.InvariantCulture) + : null; + + private static DateTimeOffset ReadUtc(Dictionary row, string column) + => DateTimeOffset.TryParse(ReadRequired(row, column), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset value) + ? value + : DateTimeOffset.MinValue; + + private static CodeModuleDiagnostic ConvertDiagnostic( + Diagnostic diagnostic, + IReadOnlyDictionary treeModuleMap) + { + FileLinePositionSpan span = diagnostic.Location.GetLineSpan(); + string? path = span.Path; + CodeModuleDefinition? module = null; + if (diagnostic.Location.SourceTree is not null) + treeModuleMap.TryGetValue(diagnostic.Location.SourceTree, out module); + + return new CodeModuleDiagnostic( + module?.ModuleId, + string.IsNullOrWhiteSpace(path) ? null : path, + span.StartLinePosition.Line + 1, + span.StartLinePosition.Character + 1, + diagnostic.Severity switch + { + DiagnosticSeverity.Hidden => CodeModuleDiagnosticSeverity.Hidden, + DiagnosticSeverity.Info => CodeModuleDiagnosticSeverity.Info, + DiagnosticSeverity.Warning => CodeModuleDiagnosticSeverity.Warning, + DiagnosticSeverity.Error => CodeModuleDiagnosticSeverity.Error, + _ => CodeModuleDiagnosticSeverity.Info, + }, + diagnostic.Id, + diagnostic.GetMessage(CultureInfo.InvariantCulture)); + } + + private static IReadOnlyList BuildReferences() + { + var references = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (AppContext.GetData("TRUSTED_PLATFORM_ASSEMBLIES") is string tpa) + { + foreach (string path in tpa.Split(Path.PathSeparator)) + AddReference(path); + } + + AddReference(typeof(FormCodeModule).Assembly.Location); + AddReference(typeof(DbCommandResult).Assembly.Location); + AddReference(typeof(object).Assembly.Location); + + return references.Values.ToImmutableArray(); + + void AddReference(string? path) + { + if (string.IsNullOrWhiteSpace(path) || !File.Exists(path) || references.ContainsKey(path)) + return; + + references[path] = MetadataReference.CreateFromFile(path); + } + } + + private static string PrepareWorkspaceRoot(string workspaceDirectory) + { + string root = Path.GetFullPath(workspaceDirectory); + if (!string.Equals(Path.GetFileName(root), WorkspaceDirectoryName, StringComparison.OrdinalIgnoreCase)) + root = Path.Combine(root, WorkspaceDirectoryName); + + Directory.CreateDirectory(root); + return root; + } + + private static string BuildWorkspacePath(CodeModuleDefinition module, HashSet usedPaths) + { + string folder = module.Kind switch + { + CodeModuleKind.Form => "forms", + CodeModuleKind.Class => "classes", + _ => "modules", + }; + string baseName = ToSafeFileName(module.Name); + string relativePath = Path.Combine(folder, $"{baseName}.cs"); + int suffix = 2; + while (!usedPaths.Add(relativePath)) + { + relativePath = Path.Combine(folder, $"{baseName}-{suffix}.cs"); + suffix++; + } + + return relativePath; + } + + private static string BuildDiagnosticPath(CodeModuleDefinition module) + => module.Kind switch + { + CodeModuleKind.Form => $"forms/{ToSafeFileName(module.Name)}.cs", + CodeModuleKind.Class => $"classes/{ToSafeFileName(module.Name)}.cs", + _ => $"modules/{ToSafeFileName(module.Name)}.cs", + }; + + private static string NormalizeRelativePath(string path) + { + string normalized = path.Replace('/', Path.DirectorySeparatorChar).Replace('\\', Path.DirectorySeparatorChar); + if (Path.IsPathRooted(normalized)) + throw new InvalidOperationException("Code module workspace paths must be relative."); + + return normalized; + } + + private static string ToSafeFileName(string value) + { + string safe = new(value.Select(ch => char.IsLetterOrDigit(ch) || ch is '_' or '-' ? ch : '_').ToArray()); + return string.IsNullOrWhiteSpace(safe) ? "module" : safe; + } + + private sealed record CodeModuleWorkspaceManifest( + int Version, + string ModuleSetHash, + IReadOnlyList Modules); + + private sealed record CodeModuleWorkspaceManifestEntry( + string ModuleId, + string Name, + CodeModuleKind Kind, + string? OwnerKind, + string? OwnerId, + string? TypeName, + string Path, + string SourceHash, + IReadOnlyDictionary? Metadata); +} diff --git a/src/CSharpDB.CodeModules/CodeModuleModels.cs b/src/CSharpDB.CodeModules/CodeModuleModels.cs new file mode 100644 index 00000000..74d8bacc --- /dev/null +++ b/src/CSharpDB.CodeModules/CodeModuleModels.cs @@ -0,0 +1,118 @@ +using System.Collections.ObjectModel; + +namespace CSharpDB.CodeModules; + +public enum CodeModuleKind +{ + Form, + Standard, + Class, +} + +public enum CodeModuleBuildStatus +{ + Succeeded, + Failed, +} + +public enum CodeModuleDiagnosticSeverity +{ + Hidden, + Info, + Warning, + Error, +} + +public sealed record CodeModuleHandler( + string ModuleId, + string TypeName, + string MethodName); + +public sealed record CodeModuleDefinition( + string ModuleId, + string Name, + CodeModuleKind Kind, + string Source, + string? OwnerKind = null, + string? OwnerId = null, + string? TypeName = null, + IReadOnlyDictionary? Metadata = null, + string Language = "csharp", + string? SourceHash = null, + DateTimeOffset? CreatedUtc = null, + DateTimeOffset? UpdatedUtc = null) +{ + public IReadOnlyDictionary SafeMetadata => + Metadata is null + ? new ReadOnlyDictionary(new Dictionary(StringComparer.OrdinalIgnoreCase)) + : new ReadOnlyDictionary(new Dictionary(Metadata, StringComparer.OrdinalIgnoreCase)); +} + +public sealed record CodeModuleSummary( + string ModuleId, + string Name, + CodeModuleKind Kind, + string? OwnerKind, + string? OwnerId, + string? TypeName, + string SourceHash, + DateTimeOffset CreatedUtc, + DateTimeOffset UpdatedUtc); + +public sealed record CodeModuleDiagnostic( + string? ModuleId, + string? Path, + int Line, + int Column, + CodeModuleDiagnosticSeverity Severity, + string Code, + string Message); + +public sealed record CodeModuleBuildResult( + string ModuleSetHash, + CodeModuleBuildStatus Status, + IReadOnlyList Diagnostics, + DateTimeOffset BuiltUtc, + byte[]? AssemblyBytes = null) +{ + public bool Succeeded => Status == CodeModuleBuildStatus.Succeeded; +} + +public sealed record CodeModuleTrustState( + string DatabasePath, + string ModuleSetHash, + bool IsTrusted, + DateTimeOffset? TrustedUtc = null); + +public sealed record CodeModuleExportResult( + string WorkspaceDirectory, + string ManifestPath, + int ModuleCount, + string ModuleSetHash); + +public enum CodeModuleImportChangeKind +{ + Added, + Updated, + Unchanged, + Skipped, + Conflict, +} + +public sealed record CodeModuleImportChange( + string ModuleId, + string Path, + CodeModuleImportChangeKind Kind, + string? Message = null); + +public sealed record CodeModuleImportResult( + string WorkspaceDirectory, + IReadOnlyList Changes) +{ + public bool HasConflicts => Changes.Any(change => change.Kind == CodeModuleImportChangeKind.Conflict); +} + +public sealed record CodeModuleRuntimeOptions +{ + public bool EnableInProcessExecution { get; set; } +} diff --git a/src/CSharpDB.CodeModules/CodeModulesServiceCollectionExtensions.cs b/src/CSharpDB.CodeModules/CodeModulesServiceCollectionExtensions.cs new file mode 100644 index 00000000..6e1519f9 --- /dev/null +++ b/src/CSharpDB.CodeModules/CodeModulesServiceCollectionExtensions.cs @@ -0,0 +1,24 @@ +using CSharpDB.CodeModules.Trust; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace CSharpDB.CodeModules; + +public static class CodeModulesServiceCollectionExtensions +{ + public static IServiceCollection AddCSharpDbCodeModules( + this IServiceCollection services, + Action? configureRuntime = null) + { + ArgumentNullException.ThrowIfNull(services); + + var options = new CodeModuleRuntimeOptions(); + configureRuntime?.Invoke(options); + + services.TryAddSingleton(options); + services.TryAddSingleton(); + services.TryAddScoped(); + services.TryAddScoped(); + return services; + } +} diff --git a/src/CSharpDB.CodeModules/Infrastructure/CodeModuleHashing.cs b/src/CSharpDB.CodeModules/Infrastructure/CodeModuleHashing.cs new file mode 100644 index 00000000..5d7bcee3 --- /dev/null +++ b/src/CSharpDB.CodeModules/Infrastructure/CodeModuleHashing.cs @@ -0,0 +1,38 @@ +using System.Security.Cryptography; +using System.Text; + +namespace CSharpDB.CodeModules.Infrastructure; + +internal static class CodeModuleHashing +{ + public static string ComputeSourceHash(string source) + => ComputeHash(NormalizeSource(source)); + + public static string ComputeModuleSetHash(IEnumerable modules) + { + var builder = new StringBuilder(); + foreach (CodeModuleDefinition module in modules.OrderBy(module => module.ModuleId, StringComparer.Ordinal)) + { + string sourceHash = string.IsNullOrWhiteSpace(module.SourceHash) + ? ComputeSourceHash(module.Source) + : module.SourceHash; + builder + .Append(module.ModuleId).Append('\n') + .Append(module.Kind).Append('\n') + .Append(sourceHash).Append('\n'); + } + + return ComputeHash(builder.ToString()); + } + + public static string NormalizeSource(string source) + => (source ?? string.Empty) + .Replace("\r\n", "\n", StringComparison.Ordinal) + .Replace('\r', '\n'); + + private static string ComputeHash(string value) + { + byte[] bytes = SHA256.HashData(Encoding.UTF8.GetBytes(value)); + return Convert.ToHexString(bytes).ToLowerInvariant(); + } +} diff --git a/src/CSharpDB.CodeModules/Infrastructure/CodeModuleSql.cs b/src/CSharpDB.CodeModules/Infrastructure/CodeModuleSql.cs new file mode 100644 index 00000000..c33bfaf7 --- /dev/null +++ b/src/CSharpDB.CodeModules/Infrastructure/CodeModuleSql.cs @@ -0,0 +1,73 @@ +using System.Globalization; +using System.Text.Json; +using CSharpDB.Client.Models; + +namespace CSharpDB.CodeModules.Infrastructure; + +internal static class CodeModuleSql +{ + public static string FormatLiteral(object? value) + { + object? normalized = NormalizeValue(value); + return normalized switch + { + null => "NULL", + long integer => integer.ToString(CultureInfo.InvariantCulture), + double real => real.ToString(CultureInfo.InvariantCulture), + string text => $"'{text.Replace("'", "''", StringComparison.Ordinal)}'", + byte[] blob => $"X'{Convert.ToHexString(blob)}'", + _ => $"'{Convert.ToString(normalized, CultureInfo.InvariantCulture)?.Replace("'", "''", StringComparison.Ordinal) ?? string.Empty}'", + }; + } + + public static IReadOnlyList> ReadRows(SqlExecutionResult result) + { + ThrowIfError(result); + if (result.ColumnNames is null || result.Rows is null) + return []; + + var rows = new List>(result.Rows.Count); + foreach (object?[] row in result.Rows) + { + var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (int i = 0; i < result.ColumnNames.Length && i < row.Length; i++) + dict[result.ColumnNames[i]] = row[i]; + + rows.Add(dict); + } + + return rows; + } + + public static void ThrowIfError(SqlExecutionResult result) + { + if (!string.IsNullOrWhiteSpace(result.Error)) + throw new InvalidOperationException(result.Error); + } + + public static object? NormalizeValue(object? value) => value switch + { + null => null, + JsonElement json => NormalizeJsonElement(json), + bool boolean => boolean ? 1L : 0L, + byte or sbyte or short or ushort or int or uint or long => Convert.ToInt64(value, CultureInfo.InvariantCulture), + float or double or decimal => Convert.ToDouble(value, CultureInfo.InvariantCulture), + DateTime dateTime => dateTime.ToString("O", CultureInfo.InvariantCulture), + DateTimeOffset dateTimeOffset => dateTimeOffset.ToString("O", CultureInfo.InvariantCulture), + Guid guid => guid.ToString("D"), + string text => text, + byte[] blob => blob, + _ => Convert.ToString(value, CultureInfo.InvariantCulture), + }; + + private static object? NormalizeJsonElement(JsonElement value) => value.ValueKind switch + { + JsonValueKind.Null => null, + JsonValueKind.String => value.GetString(), + JsonValueKind.False => 0L, + JsonValueKind.True => 1L, + JsonValueKind.Number when value.TryGetInt64(out long integer) => integer, + JsonValueKind.Number => value.GetDouble(), + _ => value.GetRawText(), + }; +} diff --git a/src/CSharpDB.CodeModules/README.md b/src/CSharpDB.CodeModules/README.md new file mode 100644 index 00000000..40629770 --- /dev/null +++ b/src/CSharpDB.CodeModules/README.md @@ -0,0 +1,165 @@ +# CSharpDB.CodeModules + +Database-owned C# code modules, workspace sync, local trust, and Admin Forms +runtime contracts for CSharpDB. + +This project is consumed by `CSharpDB.Admin.Forms` and `CSharpDB.Admin`. It is +not a standalone web host or an in-browser code editor. + +## What This Project Provides + +- database-backed module storage in `__code_modules` +- build history and diagnostics storage in `__code_module_builds` +- module CRUD through `CSharpDbCodeModuleClient` +- stable source hashes and module-set hashes +- export/import workspace sync for normal editor workflows +- Roslyn-based in-memory builds with structured diagnostics +- local trust grants keyed by normalized database path and module-set hash +- Admin Forms runtime contracts for Access-like form code modules +- trusted in-process form event dispatch with explicit host opt-in + +## Module Kinds + +| Kind | Purpose | +| --- | --- | +| `Form` | Event-handler module associated with one Admin Form. | +| `Standard` | Shared helper methods used by form modules. | +| `Class` | Shared helper classes used by form modules. | + +Form modules normally derive from `FormCodeModule`. Standard and class modules +compile into the same database-owned assembly and can be referenced by form +handlers. + +## Service Registration + +Register the code-module services in a host that already provides +`ICSharpDbClient`: + +```csharp +using CSharpDB.CodeModules; + +builder.Services.AddCSharpDbCodeModules(); +``` + +Hosts that want to execute trusted code modules in-process must opt in +explicitly: + +```csharp +builder.Services.AddCSharpDbCodeModules(options => +{ + options.EnableInProcessExecution = true; +}); +``` + +The extension registers: + +- `CSharpDbCodeModuleClient` +- `ICodeModuleTrustStore` +- `ICodeModuleFormEventDispatcher` + +The default trust store writes local trust state to +`%LOCALAPPDATA%\CSharpDB\code-module-trust.json`. Trust is intentionally not +stored in the database, so moving or editing a database does not silently grant +execution permission on another machine or for another module-set hash. + +## Core Contracts + +| Contract | Purpose | +| --- | --- | +| `CSharpDbCodeModuleClient` | List, get, upsert, delete, export, import, build, trust, and inspect module trust state. | +| `CodeModuleDefinition` | Persisted module source, kind, owner, type name, metadata, source hash, and timestamps. | +| `CodeModuleBuildResult` | Build status, module-set hash, diagnostics, build timestamp, and optional in-memory assembly bytes. | +| `CodeModuleHandler` | Event binding reference to a module id, type name, and method name. | +| `FormCodeModule` | Base type for Admin Forms code modules. Exposes `Me`, `DoCmd`, and `CurrentEvent`. | +| `FormEventContext` | Runtime form event data, arguments, metadata, message, and cancellation state. | +| `FormBeforeEventContext` | Cancelable before-event context for insert, update, and delete workflows. | +| `FormControlEventContext` | Control event context with selected control id and control type. | +| `IFormCommandApi` | Safe form actions such as set field, show message, run action sequence, run host command, save, new, refresh, open/close form, and apply/clear filter. | + +## Execution Model + +Code module execution is gated by three checks: + +1. The host must enable in-process execution. +2. The current module set must build successfully. +3. The current module-set hash must be trusted locally for the current database + path. + +Untrusted modules, missing handlers, compile failures, and thrown handler +exceptions fail the event with diagnostics. Before-event handlers can cancel an +operation by calling `Cancel` on the event context. Form handlers can mutate the +current record through `Me`; writes must target fields that exist on the current +form record. + +```csharp +using System; +using System.Threading.Tasks; +using CSharpDB.CodeModules.Runtime; + +namespace CSharpDB.UserCode.Forms; + +public sealed class CustomersModule : FormCodeModule +{ + public void BeforeUpdate(FormBeforeEventContext context) + { + if (string.Equals((string?)Me.Status, "Closed", StringComparison.OrdinalIgnoreCase)) + context.Cancel("Closed customers cannot be edited."); + } + + public async Task OnClick(FormControlEventContext context) + { + await DoCmd.ShowMessageAsync($"Clicked {context.ControlId}"); + } +} +``` + +Handlers may return `void`, `Task`, or `ValueTask`. They may accept no +parameters or one parameter assignable from the runtime event context. + +## Workspace Sync + +`ExportAsync` writes a normal file workspace: + +```text +.csharpdb-code/ + csharpdb.codeproj.json + forms/ + modules/ + classes/ +``` + +Users can edit the exported `.cs` files in their preferred editor. `ImportAsync` +uses the manifest source hash and current database source hash to detect +conflicts. If both the database module and exported file changed since export, +the import reports a conflict and does not overwrite either side. + +## Runtime Boundaries + +This package does not provide sandboxing. Safety comes from explicit host opt-in, +restricted runtime contracts, successful build validation, and local trust for a +specific module-set hash. + +The current scope does not include an in-browser IDE, VS Code sidecar commands, +file watching, debugger integration, report modules, aggregate/table-valued UDFs, +native plugin extensions, remote delegate serialization, or database-owned code +execution through daemon transports. + +## Build + +```powershell +dotnet build src/CSharpDB.CodeModules/CSharpDB.CodeModules.csproj +dotnet test tests/CSharpDB.CodeModules.Tests/CSharpDB.CodeModules.Tests.csproj +``` + +Related Admin Forms coverage: + +```powershell +dotnet test tests/CSharpDB.Admin.Forms.Tests/CSharpDB.Admin.Forms.Tests.csproj --filter "CodeModule|FormEvent|ControlEvent|JsonRoundtrip" +``` + +## Dependencies + +- `CSharpDB.Client` +- `CSharpDB.Primitives` +- `Microsoft.CodeAnalysis.CSharp` +- `Microsoft.Extensions.DependencyInjection.Abstractions` diff --git a/src/CSharpDB.CodeModules/Runtime/CodeModuleFormDispatch.cs b/src/CSharpDB.CodeModules/Runtime/CodeModuleFormDispatch.cs new file mode 100644 index 00000000..4ce81e4b --- /dev/null +++ b/src/CSharpDB.CodeModules/Runtime/CodeModuleFormDispatch.cs @@ -0,0 +1,251 @@ +using System.Reflection; +using System.Runtime.Loader; +using CSharpDB.CodeModules.Runtime; + +namespace CSharpDB.CodeModules; + +public interface ICodeModuleFormEventDispatcher +{ + Task DispatchAsync( + CodeModuleHandler handler, + CodeModuleFormEventDispatchContext context, + CancellationToken ct = default); +} + +public sealed record CodeModuleFormEventDispatchContext( + string? FormId, + string? FormName, + string? TableName, + string EventName, + IReadOnlyDictionary? Record, + IReadOnlyDictionary? BindingArguments, + IReadOnlyDictionary? RuntimeArguments, + IReadOnlyDictionary? Metadata, + IFormCommandApi CommandApi, + bool IsCancelable, + string? ControlId = null, + string? ControlType = null); + +public sealed record CodeModuleFormDispatchResult(bool Succeeded, string? Message = null) +{ + public static CodeModuleFormDispatchResult Success(string? message = null) => new(true, message); + + public static CodeModuleFormDispatchResult Failure(string message) + { + ArgumentException.ThrowIfNullOrWhiteSpace(message); + return new(false, message); + } +} + +public sealed class NullCodeModuleFormEventDispatcher : ICodeModuleFormEventDispatcher +{ + public static NullCodeModuleFormEventDispatcher Instance { get; } = new(); + + private NullCodeModuleFormEventDispatcher() + { + } + + public Task DispatchAsync( + CodeModuleHandler handler, + CodeModuleFormEventDispatchContext context, + CancellationToken ct = default) + => Task.FromResult(CodeModuleFormDispatchResult.Failure( + "C# code module execution is not configured for this host.")); +} + +public sealed class CodeModuleFormEventDispatcher( + CSharpDbCodeModuleClient client, + CodeModuleRuntimeOptions options) : ICodeModuleFormEventDispatcher +{ + public async Task DispatchAsync( + CodeModuleHandler handler, + CodeModuleFormEventDispatchContext context, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(handler); + ArgumentNullException.ThrowIfNull(context); + + if (!options.EnableInProcessExecution) + return CodeModuleFormDispatchResult.Failure("C# code module execution is disabled by the host."); + + CodeModuleBuildResult build = await client.BuildAsync(ct); + if (!build.Succeeded || build.AssemblyBytes is null) + { + string message = build.Diagnostics.FirstOrDefault(diagnostic => diagnostic.Severity == CodeModuleDiagnosticSeverity.Error)?.Message + ?? "C# code modules failed to build."; + return CodeModuleFormDispatchResult.Failure(message); + } + + CodeModuleTrustState trust = await client.GetTrustStateAsync(build.ModuleSetHash, ct); + if (!trust.IsTrusted) + { + return CodeModuleFormDispatchResult.Failure( + $"C# code modules are not trusted locally for module set '{build.ModuleSetHash}'. Build and trust the current code modules before running this handler."); + } + + AssemblyLoadContext loadContext = new($"csharpdb-code-modules-{Guid.NewGuid():N}", isCollectible: true); + try + { + using var stream = new MemoryStream(build.AssemblyBytes); + Assembly assembly = loadContext.LoadFromStream(stream); + return await InvokeHandlerAsync(assembly, handler, context, ct); + } + finally + { + loadContext.Unload(); + } + } + + private static async Task InvokeHandlerAsync( + Assembly assembly, + CodeModuleHandler handler, + CodeModuleFormEventDispatchContext context, + CancellationToken ct) + { + Type? moduleType = assembly.GetType(handler.TypeName, throwOnError: false, ignoreCase: false) + ?? assembly.GetTypes().FirstOrDefault(type => string.Equals(type.Name, handler.TypeName, StringComparison.Ordinal)); + if (moduleType is null) + return CodeModuleFormDispatchResult.Failure($"C# code module type '{handler.TypeName}' was not found."); + + if (!typeof(FormCodeModule).IsAssignableFrom(moduleType)) + return CodeModuleFormDispatchResult.Failure($"C# code module type '{handler.TypeName}' must derive from FormCodeModule."); + + MethodInfo? method = moduleType.GetMethod( + handler.MethodName, + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + if (method is null) + return CodeModuleFormDispatchResult.Failure($"C# code module handler '{handler.MethodName}' was not found on '{handler.TypeName}'."); + + object? instance = Activator.CreateInstance(moduleType); + if (instance is not FormCodeModule formModule) + return CodeModuleFormDispatchResult.Failure($"C# code module type '{handler.TypeName}' could not be created."); + + FormEventContext eventContext = CreateEventContext(context); + formModule.Initialize(new FormCodeModuleRuntimeContext( + new FormModuleRecord(ToMutableRecord(context.Record)), + context.CommandApi, + eventContext)); + + try + { + object?[]? args = BuildMethodArguments(method, eventContext); + object? result = method.Invoke(formModule, args); + await AwaitResultAsync(result, ct); + } + catch (TargetInvocationException ex) when (ex.InnerException is not null) + { + return CodeModuleFormDispatchResult.Failure( + $"C# code module handler '{handler.MethodName}' failed: {ex.InnerException.Message}"); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + return CodeModuleFormDispatchResult.Failure( + $"C# code module handler '{handler.MethodName}' failed: {ex.Message}"); + } + + if (eventContext.Canceled) + return CodeModuleFormDispatchResult.Failure(eventContext.Message ?? "The C# code module canceled the event."); + + return CodeModuleFormDispatchResult.Success(eventContext.Message); + } + + private static IDictionary ToMutableRecord(IReadOnlyDictionary? record) + { + if (record is IDictionary mutable) + return mutable; + + return record is null + ? new Dictionary(StringComparer.OrdinalIgnoreCase) + : new Dictionary(record, StringComparer.OrdinalIgnoreCase); + } + + private static FormEventContext CreateEventContext(CodeModuleFormEventDispatchContext context) + { + IReadOnlyDictionary arguments = MergeArguments(context.BindingArguments, context.RuntimeArguments); + if (!string.IsNullOrWhiteSpace(context.ControlId)) + { + return new FormControlEventContext( + context.FormId, + context.FormName, + context.TableName, + context.EventName, + context.ControlId, + context.ControlType, + arguments, + context.Metadata); + } + + return context.IsCancelable + ? new FormBeforeEventContext( + context.FormId, + context.FormName, + context.TableName, + context.EventName, + arguments, + context.Metadata) + : new FormEventContext( + context.FormId, + context.FormName, + context.TableName, + context.EventName, + arguments, + context.Metadata); + } + + private static IReadOnlyDictionary MergeArguments( + IReadOnlyDictionary? bindingArguments, + IReadOnlyDictionary? runtimeArguments) + { + if ((bindingArguments is null || bindingArguments.Count == 0) && + (runtimeArguments is null || runtimeArguments.Count == 0)) + { + return new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (runtimeArguments is not null) + { + foreach ((string key, object? value) in runtimeArguments) + result[key] = value; + } + + if (bindingArguments is not null) + { + foreach ((string key, object? value) in bindingArguments) + result[key] = value; + } + + return result; + } + + private static object?[]? BuildMethodArguments(MethodInfo method, FormEventContext context) + { + ParameterInfo[] parameters = method.GetParameters(); + if (parameters.Length == 0) + return null; + + if (parameters.Length == 1 && parameters[0].ParameterType.IsInstanceOfType(context)) + return [context]; + + throw new InvalidOperationException( + $"C# code module handler '{method.Name}' must accept no parameters or one parameter assignable from '{context.GetType().Name}'."); + } + + private static async Task AwaitResultAsync(object? result, CancellationToken ct) + { + switch (result) + { + case null: + return; + case Task task: + await task.WaitAsync(ct); + return; + case ValueTask valueTask: + await valueTask.AsTask().WaitAsync(ct); + return; + default: + throw new InvalidOperationException( + $"C# code module handlers must return void, Task, or ValueTask. Actual return type was '{result.GetType().Name}'."); + } + } +} diff --git a/src/CSharpDB.CodeModules/Runtime/FormRuntimeContracts.cs b/src/CSharpDB.CodeModules/Runtime/FormRuntimeContracts.cs new file mode 100644 index 00000000..c304c15c --- /dev/null +++ b/src/CSharpDB.CodeModules/Runtime/FormRuntimeContracts.cs @@ -0,0 +1,218 @@ +using System.Collections.ObjectModel; +using System.Dynamic; +using CSharpDB.Primitives; + +namespace CSharpDB.CodeModules.Runtime; + +public abstract class FormCodeModule +{ + private FormCodeModuleRuntimeContext? _runtime; + + public dynamic Me => _runtime?.Record + ?? throw new InvalidOperationException("The form code module has not been initialized."); + + public IFormCommandApi DoCmd => _runtime?.Commands + ?? throw new InvalidOperationException("The form code module has not been initialized."); + + public FormEventContext CurrentEvent => _runtime?.EventContext + ?? throw new InvalidOperationException("The form code module has not been initialized."); + + public void Initialize(FormCodeModuleRuntimeContext runtime) + { + ArgumentNullException.ThrowIfNull(runtime); + _runtime = runtime; + } +} + +public sealed record FormCodeModuleRuntimeContext( + FormModuleRecord Record, + IFormCommandApi Commands, + FormEventContext EventContext); + +public class FormEventContext +{ + public FormEventContext( + string? formId, + string? formName, + string? tableName, + string eventName, + IReadOnlyDictionary? arguments, + IReadOnlyDictionary? metadata) + { + FormId = formId; + FormName = formName; + TableName = tableName; + EventName = eventName; + Arguments = arguments ?? EmptyObjectDictionary.Instance; + Metadata = metadata ?? EmptyStringDictionary.Instance; + } + + public string? FormId { get; } + public string? FormName { get; } + public string? TableName { get; } + public string EventName { get; } + public IReadOnlyDictionary Arguments { get; } + public IReadOnlyDictionary Metadata { get; } + public bool Canceled { get; private set; } + public string? Message { get; private set; } + + public void Cancel(string? message = null) + { + Canceled = true; + Message = message; + } + + public void SetMessage(string message) + { + ArgumentException.ThrowIfNullOrWhiteSpace(message); + Message = message; + } +} + +public sealed class FormBeforeEventContext : FormEventContext +{ + public FormBeforeEventContext( + string? formId, + string? formName, + string? tableName, + string eventName, + IReadOnlyDictionary? arguments, + IReadOnlyDictionary? metadata) + : base(formId, formName, tableName, eventName, arguments, metadata) + { + } +} + +public sealed class FormControlEventContext : FormEventContext +{ + public FormControlEventContext( + string? formId, + string? formName, + string? tableName, + string eventName, + string controlId, + string? controlType, + IReadOnlyDictionary? arguments, + IReadOnlyDictionary? metadata) + : base(formId, formName, tableName, eventName, arguments, metadata) + { + ControlId = controlId; + ControlType = controlType; + } + + public string ControlId { get; } + public string? ControlType { get; } +} + +public interface IFormCommandApi +{ + ValueTask SetFieldAsync(string fieldName, object? value, CancellationToken ct = default); + + ValueTask ShowMessageAsync(string message, CancellationToken ct = default); + + ValueTask RunActionSequenceAsync( + string sequenceName, + IReadOnlyDictionary? arguments = null, + CancellationToken ct = default); + + ValueTask RunHostCommandAsync( + string commandName, + IReadOnlyDictionary? arguments = null, + CancellationToken ct = default); + + ValueTask SaveRecordAsync(CancellationToken ct = default); + + ValueTask NewRecordAsync(CancellationToken ct = default); + + ValueTask RefreshRecordsAsync(CancellationToken ct = default); + + ValueTask OpenFormAsync( + string formName, + IReadOnlyDictionary? arguments = null, + CancellationToken ct = default); + + ValueTask CloseFormAsync(string? formName = null, CancellationToken ct = default); + + ValueTask ApplyFilterAsync( + string filter, + string target = "form", + IReadOnlyDictionary? arguments = null, + CancellationToken ct = default); + + ValueTask ClearFilterAsync(string target = "form", CancellationToken ct = default); +} + +public sealed record FormCommandApiResult(bool Succeeded, string? Message = null) +{ + public static FormCommandApiResult Success(string? message = null) => new(true, message); + + public static FormCommandApiResult Failure(string message) + { + ArgumentException.ThrowIfNullOrWhiteSpace(message); + return new(false, message); + } +} + +public sealed class FormModuleRecord : DynamicObject +{ + private readonly IDictionary _record; + private readonly Dictionary _fieldLookup; + + public FormModuleRecord(IDictionary record) + { + ArgumentNullException.ThrowIfNull(record); + _record = record; + _fieldLookup = record.Keys.ToDictionary(key => key, key => key, StringComparer.OrdinalIgnoreCase); + } + + public IReadOnlyDictionary Values => new ReadOnlyDictionary(_record); + + public object? this[string fieldName] + { + get => TryGetValue(fieldName, out object? value) ? value : throw UnknownField(fieldName); + set => Set(fieldName, value); + } + + public bool TryGetValue(string fieldName, out object? value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(fieldName); + if (_fieldLookup.TryGetValue(fieldName, out string? key)) + return _record.TryGetValue(key, out value); + + value = null; + return false; + } + + public void Set(string fieldName, object? value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(fieldName); + if (!_fieldLookup.TryGetValue(fieldName, out string? key)) + throw UnknownField(fieldName); + + _record[key] = value; + } + + public override bool TryGetMember(GetMemberBinder binder, out object? result) + => TryGetValue(binder.Name, out result); + + public override bool TrySetMember(SetMemberBinder binder, object? value) + { + Set(binder.Name, value); + return true; + } + + private static InvalidOperationException UnknownField(string fieldName) + => new($"Unknown form field '{fieldName}'. Code module writes must target fields that exist on the current form record."); +} + +internal static class EmptyObjectDictionary +{ + public static readonly IReadOnlyDictionary Instance = + new Dictionary(StringComparer.OrdinalIgnoreCase); +} + +internal static class EmptyStringDictionary +{ + public static readonly IReadOnlyDictionary Instance = + new Dictionary(StringComparer.OrdinalIgnoreCase); +} diff --git a/src/CSharpDB.CodeModules/Trust/CodeModuleTrustStore.cs b/src/CSharpDB.CodeModules/Trust/CodeModuleTrustStore.cs new file mode 100644 index 00000000..bfaf5fdb --- /dev/null +++ b/src/CSharpDB.CodeModules/Trust/CodeModuleTrustStore.cs @@ -0,0 +1,175 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace CSharpDB.CodeModules.Trust; + +public interface ICodeModuleTrustStore +{ + Task GetTrustStateAsync( + string databasePath, + string moduleSetHash, + CancellationToken ct = default); + + Task TrustAsync( + string databasePath, + string moduleSetHash, + CancellationToken ct = default); +} + +public sealed class FileCodeModuleTrustStore : ICodeModuleTrustStore +{ + private static readonly JsonSerializerOptions s_jsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = true, + }; + + private readonly string _storePath; + private readonly SemaphoreSlim _lock = new(1, 1); + + public FileCodeModuleTrustStore() + : this(DefaultStorePath()) + { + } + + public FileCodeModuleTrustStore(string storePath) + { + ArgumentException.ThrowIfNullOrWhiteSpace(storePath); + _storePath = storePath; + } + + public async Task GetTrustStateAsync( + string databasePath, + string moduleSetHash, + CancellationToken ct = default) + { + string normalizedPath = NormalizeDatabasePath(databasePath); + string normalizedHash = NormalizeHash(moduleSetHash); + await _lock.WaitAsync(ct); + try + { + TrustStoreDocument document = await ReadAsync(ct); + string key = BuildKey(normalizedPath, normalizedHash); + return document.Entries.TryGetValue(key, out TrustEntry? entry) + ? new CodeModuleTrustState(normalizedPath, normalizedHash, IsTrusted: true, entry.TrustedUtc) + : new CodeModuleTrustState(normalizedPath, normalizedHash, IsTrusted: false); + } + finally + { + _lock.Release(); + } + } + + public async Task TrustAsync( + string databasePath, + string moduleSetHash, + CancellationToken ct = default) + { + string normalizedPath = NormalizeDatabasePath(databasePath); + string normalizedHash = NormalizeHash(moduleSetHash); + await _lock.WaitAsync(ct); + try + { + TrustStoreDocument document = await ReadAsync(ct); + document.Entries[BuildKey(normalizedPath, normalizedHash)] = new TrustEntry( + normalizedPath, + normalizedHash, + DateTimeOffset.UtcNow); + await WriteAsync(document, ct); + } + finally + { + _lock.Release(); + } + } + + public static string NormalizeDatabasePath(string databasePath) + { + if (string.IsNullOrWhiteSpace(databasePath)) + return ""; + + try + { + return Path.GetFullPath(databasePath.Trim()).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar).ToUpperInvariant(); + } + catch (Exception) when (databasePath.Length > 0) + { + return databasePath.Trim().ToUpperInvariant(); + } + } + + private async Task ReadAsync(CancellationToken ct) + { + if (!File.Exists(_storePath)) + return new TrustStoreDocument(); + + await using FileStream stream = File.OpenRead(_storePath); + return await JsonSerializer.DeserializeAsync(stream, s_jsonOptions, ct) + ?? new TrustStoreDocument(); + } + + private async Task WriteAsync(TrustStoreDocument document, CancellationToken ct) + { + string? directory = Path.GetDirectoryName(_storePath); + if (!string.IsNullOrWhiteSpace(directory)) + Directory.CreateDirectory(directory); + + await using FileStream stream = File.Create(_storePath); + await JsonSerializer.SerializeAsync(stream, document, s_jsonOptions, ct); + } + + private static string BuildKey(string normalizedDatabasePath, string moduleSetHash) + => $"{normalizedDatabasePath}|{moduleSetHash}"; + + private static string NormalizeHash(string moduleSetHash) + { + ArgumentException.ThrowIfNullOrWhiteSpace(moduleSetHash); + return moduleSetHash.Trim().ToLowerInvariant(); + } + + private static string DefaultStorePath() + { + string root = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData); + if (string.IsNullOrWhiteSpace(root)) + root = AppContext.BaseDirectory; + + return Path.Combine(root, "CSharpDB", "code-module-trust.json"); + } + + private sealed class TrustStoreDocument + { + public Dictionary Entries { get; init; } = new(StringComparer.Ordinal); + } + + private sealed record TrustEntry(string DatabasePath, string ModuleSetHash, DateTimeOffset TrustedUtc); +} + +public sealed class InMemoryCodeModuleTrustStore : ICodeModuleTrustStore +{ + private readonly Dictionary _trusted = new(StringComparer.Ordinal); + + public Task GetTrustStateAsync( + string databasePath, + string moduleSetHash, + CancellationToken ct = default) + { + string normalizedPath = FileCodeModuleTrustStore.NormalizeDatabasePath(databasePath); + string normalizedHash = moduleSetHash.Trim().ToLowerInvariant(); + string key = $"{normalizedPath}|{normalizedHash}"; + return Task.FromResult(_trusted.TryGetValue(key, out DateTimeOffset trustedUtc) + ? new CodeModuleTrustState(normalizedPath, normalizedHash, IsTrusted: true, trustedUtc) + : new CodeModuleTrustState(normalizedPath, normalizedHash, IsTrusted: false)); + } + + public Task TrustAsync( + string databasePath, + string moduleSetHash, + CancellationToken ct = default) + { + string normalizedPath = FileCodeModuleTrustStore.NormalizeDatabasePath(databasePath); + string normalizedHash = moduleSetHash.Trim().ToLowerInvariant(); + _trusted[$"{normalizedPath}|{normalizedHash}"] = DateTimeOffset.UtcNow; + return Task.CompletedTask; + } +} diff --git a/src/CSharpDB/CSharpDB.csproj b/src/CSharpDB/CSharpDB.csproj index 8a9e740c..6bb95c37 100644 --- a/src/CSharpDB/CSharpDB.csproj +++ b/src/CSharpDB/CSharpDB.csproj @@ -18,6 +18,7 @@ + diff --git a/tests/CSharpDB.Admin.Forms.Tests/CSharpDB.Admin.Forms.Tests.csproj b/tests/CSharpDB.Admin.Forms.Tests/CSharpDB.Admin.Forms.Tests.csproj index 13a6b498..983dc350 100644 --- a/tests/CSharpDB.Admin.Forms.Tests/CSharpDB.Admin.Forms.Tests.csproj +++ b/tests/CSharpDB.Admin.Forms.Tests/CSharpDB.Admin.Forms.Tests.csproj @@ -12,6 +12,7 @@ + diff --git a/tests/CSharpDB.Admin.Forms.Tests/Services/FormCodeModuleEventTests.cs b/tests/CSharpDB.Admin.Forms.Tests/Services/FormCodeModuleEventTests.cs new file mode 100644 index 00000000..e050ed79 --- /dev/null +++ b/tests/CSharpDB.Admin.Forms.Tests/Services/FormCodeModuleEventTests.cs @@ -0,0 +1,247 @@ +using System.Text.Json; +using CSharpDB.Admin.Forms.Contracts; +using CSharpDB.Admin.Forms.Models; +using CSharpDB.Admin.Forms.Serialization; +using CSharpDB.Admin.Forms.Services; +using CSharpDB.Admin.Forms.Tests; +using CSharpDB.CodeModules; +using CSharpDB.CodeModules.Trust; +using CSharpDB.Primitives; + +namespace CSharpDB.Admin.Forms.Tests.Services; + +public sealed class FormCodeModuleEventTests +{ + [Fact] + public void FormAndControlEventBindings_WithCodeHandlers_RoundTripJson() + { + var form = CreateForm( + [ + new FormEventBinding( + FormEventKind.OnLoad, + string.Empty, + CodeHandler: new CodeModuleHandler("form:customers", "TestModules.CustomersModule", "OnLoad")), + ], + [ + new ControlDefinition( + "nameBox", + "text", + new Rect(0, 0, 100, 24), + new BindingDefinition("Name", "TwoWay"), + new PropertyBag(new Dictionary()), + null, + EventBindings: + [ + new ControlEventBinding( + ControlEventKind.OnChange, + string.Empty, + CodeHandler: new CodeModuleHandler("form:customers", "TestModules.CustomersModule", "nameBox_OnChange")), + ]), + ]); + + string json = JsonSerializer.Serialize(form, JsonDefaults.Options); + FormDefinition deserialized = JsonSerializer.Deserialize(json, JsonDefaults.Options)!; + + CodeModuleHandler formHandler = Assert.Single(deserialized.EventBindings!).CodeHandler!; + Assert.Equal("form:customers", formHandler.ModuleId); + Assert.Equal("OnLoad", formHandler.MethodName); + CodeModuleHandler controlHandler = Assert.Single(deserialized.Controls[0].EventBindings!).CodeHandler!; + Assert.Equal("nameBox_OnChange", controlHandler.MethodName); + } + + [Fact] + public async Task DispatchAsync_UntrustedModuleBlocksExecution() + { + CancellationToken ct = TestContext.Current.CancellationToken; + await using TestDatabaseScope db = await TestDatabaseScope.CreateAsync(); + var codeClient = new CSharpDbCodeModuleClient(db.Client, new InMemoryCodeModuleTrustStore()); + await codeClient.UpsertAsync(CreateModule(""" + using CSharpDB.CodeModules.Runtime; + namespace TestModules; + public sealed class CustomersModule : FormCodeModule + { + public void OnLoad(FormEventContext context) + { + Me.Name = "Bob"; + } + } + """), ct); + var dispatcher = CreateDispatcher(codeClient); + var record = new Dictionary { ["Name"] = "Alice" }; + + FormEventDispatchResult result = await dispatcher.DispatchAsync( + CreateForm([new FormEventBinding(FormEventKind.OnLoad, string.Empty, CodeHandler: Handler())]), + FormEventKind.OnLoad, + record, + ct); + + Assert.False(result.Succeeded); + Assert.Contains("not trusted", result.Message); + Assert.Equal("Alice", record["Name"]); + } + + [Fact] + public async Task DispatchAsync_CompileErrorBlocksExecutionWithDiagnostic() + { + CancellationToken ct = TestContext.Current.CancellationToken; + await using TestDatabaseScope db = await TestDatabaseScope.CreateAsync(); + var codeClient = new CSharpDbCodeModuleClient(db.Client, new InMemoryCodeModuleTrustStore()); + await codeClient.UpsertAsync(CreateModule("public sealed class Broken {"), ct); + var dispatcher = CreateDispatcher(codeClient); + + FormEventDispatchResult result = await dispatcher.DispatchAsync( + CreateForm([new FormEventBinding(FormEventKind.OnLoad, string.Empty, CodeHandler: Handler())]), + FormEventKind.OnLoad, + new Dictionary { ["Name"] = "Alice" }, + ct); + + Assert.False(result.Succeeded); + Assert.NotNull(result.Message); + } + + [Fact] + public async Task DispatchAsync_TrustedHandlerMutatesMe() + { + CancellationToken ct = TestContext.Current.CancellationToken; + await using TestDatabaseScope db = await TestDatabaseScope.CreateAsync(); + var trust = new InMemoryCodeModuleTrustStore(); + var codeClient = new CSharpDbCodeModuleClient(db.Client, trust); + await codeClient.UpsertAsync(CreateModule(""" + using CSharpDB.CodeModules.Runtime; + namespace TestModules; + public sealed class CustomersModule : FormCodeModule + { + public void OnLoad(FormEventContext context) + { + Me.Name = "Bob"; + } + } + """), ct); + await codeClient.TrustAsync(ct); + var dispatcher = CreateDispatcher(codeClient); + var record = new Dictionary { ["Name"] = "Alice" }; + + FormEventDispatchResult result = await dispatcher.DispatchAsync( + CreateForm([new FormEventBinding(FormEventKind.OnLoad, string.Empty, CodeHandler: Handler())]), + FormEventKind.OnLoad, + record, + ct); + + Assert.True(result.Succeeded); + Assert.Equal("Bob", record["Name"]); + } + + [Fact] + public async Task DispatchAsync_BeforeUpdateHandlerCanCancelSave() + { + CancellationToken ct = TestContext.Current.CancellationToken; + await using TestDatabaseScope db = await TestDatabaseScope.CreateAsync(); + var codeClient = new CSharpDbCodeModuleClient(db.Client, new InMemoryCodeModuleTrustStore()); + await codeClient.UpsertAsync(CreateModule(""" + using CSharpDB.CodeModules.Runtime; + namespace TestModules; + public sealed class CustomersModule : FormCodeModule + { + public void BeforeUpdate(FormBeforeEventContext context) + { + context.Cancel("Rejected by code."); + } + } + """), ct); + await codeClient.TrustAsync(ct); + var dispatcher = CreateDispatcher(codeClient); + + FormEventDispatchResult result = await dispatcher.DispatchAsync( + CreateForm([new FormEventBinding( + FormEventKind.BeforeUpdate, + string.Empty, + CodeHandler: Handler("BeforeUpdate"))]), + FormEventKind.BeforeUpdate, + new Dictionary { ["Name"] = "Alice" }, + ct); + + Assert.False(result.Succeeded); + Assert.Equal("Rejected by code.", result.Message); + } + + [Fact] + public async Task DispatchAsync_HandlerCanRunHostCommand() + { + CancellationToken ct = TestContext.Current.CancellationToken; + await using TestDatabaseScope db = await TestDatabaseScope.CreateAsync(); + var codeClient = new CSharpDbCodeModuleClient(db.Client, new InMemoryCodeModuleTrustStore()); + await codeClient.UpsertAsync(CreateModule(""" + using System.Collections.Generic; + using System.Threading.Tasks; + using CSharpDB.CodeModules.Runtime; + namespace TestModules; + public sealed class CustomersModule : FormCodeModule + { + public async Task OnLoad(FormEventContext context) + { + await DoCmd.RunHostCommandAsync("Audit", new Dictionary { ["source"] = "code" }); + } + } + """), ct); + await codeClient.TrustAsync(ct); + + DbCommandContext? captured = null; + DbCommandRegistry commands = DbCommandRegistry.Create(builder => + { + builder.AddCommand("Audit", context => + { + captured = context; + return DbCommandResult.Success(); + }); + }); + var dispatcher = CreateDispatcher(codeClient, commands); + + FormEventDispatchResult result = await dispatcher.DispatchAsync( + CreateForm([new FormEventBinding(FormEventKind.OnLoad, string.Empty, CodeHandler: Handler())]), + FormEventKind.OnLoad, + new Dictionary { ["Name"] = "Alice" }, + ct); + + Assert.True(result.Succeeded); + Assert.NotNull(captured); + Assert.Equal("code", captured!.Arguments["source"].AsText); + Assert.Equal("OnLoad", captured.Metadata["event"]); + } + + private static DefaultFormEventDispatcher CreateDispatcher( + CSharpDbCodeModuleClient codeClient, + DbCommandRegistry? commands = null) + => new( + commands ?? DbCommandRegistry.Empty, + DbExtensionPolicies.DefaultHostCallbackPolicy, + NullFormActionRuntime.Instance, + new CodeModuleFormEventDispatcher( + codeClient, + new CodeModuleRuntimeOptions { EnableInProcessExecution = true })); + + private static CodeModuleDefinition CreateModule(string source) + => new( + "form:customers", + "Customers", + CodeModuleKind.Form, + source, + OwnerKind: "Form", + OwnerId: "customers-form", + TypeName: "TestModules.CustomersModule"); + + private static CodeModuleHandler Handler(string methodName = "OnLoad") + => new("form:customers", "TestModules.CustomersModule", methodName); + + private static FormDefinition CreateForm( + IReadOnlyList eventBindings, + IReadOnlyList? controls = null) + => new( + "customers-form", + "Customers", + "Customers", + DefinitionVersion: 1, + SourceSchemaSignature: "customers:v1", + Layout: new LayoutDefinition("absolute", 8, SnapToGrid: false, []), + Controls: controls ?? [], + EventBindings: eventBindings); +} diff --git a/tests/CSharpDB.CodeModules.Tests/CSharpDB.CodeModules.Tests.csproj b/tests/CSharpDB.CodeModules.Tests/CSharpDB.CodeModules.Tests.csproj new file mode 100644 index 00000000..285b961e --- /dev/null +++ b/tests/CSharpDB.CodeModules.Tests/CSharpDB.CodeModules.Tests.csproj @@ -0,0 +1,23 @@ + + + + + + Exe + net10.0 + enable + enable + false + + + + + + + + + + + + + diff --git a/tests/CSharpDB.CodeModules.Tests/CSharpDbCodeModuleClientTests.cs b/tests/CSharpDB.CodeModules.Tests/CSharpDbCodeModuleClientTests.cs new file mode 100644 index 00000000..bb99c142 --- /dev/null +++ b/tests/CSharpDB.CodeModules.Tests/CSharpDbCodeModuleClientTests.cs @@ -0,0 +1,145 @@ +using CSharpDB.CodeModules.Trust; + +namespace CSharpDB.CodeModules.Tests; + +public sealed class CSharpDbCodeModuleClientTests +{ + [Fact] + public async Task ModuleCrud_PersistsMetadataAndStableHashes() + { + CancellationToken ct = TestContext.Current.CancellationToken; + await using TestDatabaseScope db = await TestDatabaseScope.CreateAsync(); + var client = new CSharpDbCodeModuleClient(db.Client, new InMemoryCodeModuleTrustStore()); + + CodeModuleDefinition saved = await client.UpsertAsync(new CodeModuleDefinition( + "form:customers", + "Customers", + CodeModuleKind.Form, + "public class Customers { }\r\n", + OwnerKind: "Form", + OwnerId: "customers-form", + TypeName: "Customers"), ct); + + Assert.Equal("form:customers", saved.ModuleId); + Assert.Equal(CodeModuleKind.Form, saved.Kind); + Assert.NotNull(saved.SourceHash); + Assert.Equal("public class Customers { }\n", saved.Source); + + CodeModuleDefinition? loaded = await client.GetAsync("form:customers", ct); + Assert.NotNull(loaded); + Assert.Equal(saved.SourceHash, loaded!.SourceHash); + + IReadOnlyList modules = await client.ListAsync(ct); + CodeModuleSummary summary = Assert.Single(modules); + Assert.Equal("customers-form", summary.OwnerId); + } + + [Fact] + public async Task ExportImport_RoundTripsAndDetectsBothChangedConflict() + { + CancellationToken ct = TestContext.Current.CancellationToken; + await using TestDatabaseScope sourceDb = await TestDatabaseScope.CreateAsync("code_modules_export"); + var source = new CSharpDbCodeModuleClient(sourceDb.Client, new InMemoryCodeModuleTrustStore()); + await source.UpsertAsync(new CodeModuleDefinition( + "module:helpers", + "Helpers", + CodeModuleKind.Standard, + "public static class Helpers { public static int A() => 1; }\n"), ct); + + string workspace = Path.Combine(Path.GetTempPath(), $"csharpdb_code_ws_{Guid.NewGuid():N}"); + CodeModuleExportResult export = await source.ExportAsync(workspace, ct); + Assert.Equal(1, export.ModuleCount); + string exportedSourcePath = Directory.EnumerateFiles(export.WorkspaceDirectory, "*.cs", SearchOption.AllDirectories).Single(); + + await source.UpsertAsync(new CodeModuleDefinition( + "module:helpers", + "Helpers", + CodeModuleKind.Standard, + "public static class Helpers { public static int A() => 2; }\n"), ct); + await File.WriteAllTextAsync(exportedSourcePath, "public static class Helpers { public static int A() => 3; }\n", ct); + + CodeModuleImportResult conflict = await source.ImportAsync(workspace, ct); + CodeModuleImportChange change = Assert.Single(conflict.Changes); + Assert.Equal(CodeModuleImportChangeKind.Conflict, change.Kind); + + await using TestDatabaseScope targetDb = await TestDatabaseScope.CreateAsync("code_modules_import"); + var target = new CSharpDbCodeModuleClient(targetDb.Client, new InMemoryCodeModuleTrustStore()); + CodeModuleImportResult imported = await target.ImportAsync(workspace, ct); + Assert.Equal(CodeModuleImportChangeKind.Added, Assert.Single(imported.Changes).Kind); + Assert.NotNull(await target.GetAsync("module:helpers", ct)); + + Directory.Delete(workspace, recursive: true); + } + + [Fact] + public async Task Build_ReturnsAssemblyBytesAndStructuredDiagnostics() + { + CancellationToken ct = TestContext.Current.CancellationToken; + await using TestDatabaseScope db = await TestDatabaseScope.CreateAsync(); + var client = new CSharpDbCodeModuleClient(db.Client, new InMemoryCodeModuleTrustStore()); + await client.UpsertAsync(new CodeModuleDefinition( + "form:orders", + "Orders", + CodeModuleKind.Form, + """ + using CSharpDB.CodeModules.Runtime; + + namespace TestModules; + + public sealed class OrdersModule : FormCodeModule + { + public void BeforeUpdate(FormBeforeEventContext context) + { + Me.Status = "Ready"; + } + } + """, + TypeName: "TestModules.OrdersModule"), ct); + + CodeModuleBuildResult build = await client.BuildAsync(ct); + + Assert.True(build.Succeeded); + Assert.NotNull(build.AssemblyBytes); + Assert.NotEmpty(build.AssemblyBytes!); + + await client.UpsertAsync(new CodeModuleDefinition( + "form:orders", + "Orders", + CodeModuleKind.Form, + "public sealed class Broken {", + TypeName: "Broken"), ct); + + CodeModuleBuildResult failed = await client.BuildAsync(ct); + Assert.False(failed.Succeeded); + CodeModuleDiagnostic diagnostic = failed.Diagnostics.First(d => d.Severity == CodeModuleDiagnosticSeverity.Error); + Assert.Equal("form:orders", diagnostic.ModuleId); + Assert.True(diagnostic.Line > 0); + } + + [Fact] + public async Task Trust_IsLocalAndInvalidatesWhenSourceChanges() + { + CancellationToken ct = TestContext.Current.CancellationToken; + await using TestDatabaseScope db = await TestDatabaseScope.CreateAsync(); + var trust = new InMemoryCodeModuleTrustStore(); + var client = new CSharpDbCodeModuleClient(db.Client, trust); + await client.UpsertAsync(new CodeModuleDefinition( + "module:helpers", + "Helpers", + CodeModuleKind.Standard, + "public static class Helpers { public static int A() => 1; }\n"), ct); + + CodeModuleBuildResult build = await client.BuildAsync(ct); + await client.TrustAsync(ct); + + Assert.True((await client.GetTrustStateAsync(build.ModuleSetHash, ct)).IsTrusted); + + await client.UpsertAsync(new CodeModuleDefinition( + "module:helpers", + "Helpers", + CodeModuleKind.Standard, + "public static class Helpers { public static int A() => 2; }\n"), ct); + + Assert.False((await client.GetTrustStateAsync(ct: ct)).IsTrusted); + } +} diff --git a/tests/CSharpDB.CodeModules.Tests/TestDatabaseScope.cs b/tests/CSharpDB.CodeModules.Tests/TestDatabaseScope.cs new file mode 100644 index 00000000..0811c864 --- /dev/null +++ b/tests/CSharpDB.CodeModules.Tests/TestDatabaseScope.cs @@ -0,0 +1,52 @@ +using CSharpDB.Client; +using CSharpDB.Client.Models; + +namespace CSharpDB.CodeModules.Tests; + +public sealed class TestDatabaseScope : IAsyncDisposable +{ + private readonly ICSharpDbClient _client; + + private TestDatabaseScope(string databasePath, ICSharpDbClient client) + { + DatabasePath = databasePath; + _client = client; + } + + public string DatabasePath { get; } + public ICSharpDbClient Client => _client; + + public static async Task CreateAsync(string? name = null) + { + string databasePath = Path.Combine( + Path.GetTempPath(), + $"{name ?? "csharpdb_code_modules"}_{Guid.NewGuid():N}.db"); + + ICSharpDbClient client = CSharpDbClient.Create(new CSharpDbClientOptions + { + DataSource = databasePath, + }); + + await client.GetInfoAsync(TestContext.Current.CancellationToken); + return new TestDatabaseScope(databasePath, client); + } + + public async Task ExecuteAsync(string sql) + { + SqlExecutionResult result = await _client.ExecuteSqlAsync(sql, TestContext.Current.CancellationToken); + Assert.Null(result.Error); + } + + public async ValueTask DisposeAsync() + { + await _client.DisposeAsync(); + TryDelete(DatabasePath); + TryDelete(DatabasePath + ".wal"); + } + + private static void TryDelete(string path) + { + if (File.Exists(path)) + File.Delete(path); + } +} diff --git a/www/blog/access-style-csharp-code-modules.html b/www/blog/access-style-csharp-code-modules.html new file mode 100644 index 00000000..11c033b0 --- /dev/null +++ b/www/blog/access-style-csharp-code-modules.html @@ -0,0 +1,477 @@ + + + + + + + + Build Access-Style C# Form Modules — Blog — CSharpDB + + + + + + + + + + + + + + +
+
+
+

Build Access-Style C# Form Modules

+
+ Guide + May 10, 2026 + · 10 min read +
+ +
+

Access form modules are useful because the code lives next to the form. A button click, a before-save rule, and a small helper method can travel with the database instead of being buried in a separate application host.

+

CSharpDB code modules bring that idea to Admin Forms with normal C# tooling. The database stores the source. Admin can generate event-handler stubs, export the modules to files, import changes, build them with Roslyn, show diagnostics, and require explicit local trust before anything runs.

+ +
+ Scenario: a warehouse team uses an Order Workbench Admin Form to manage outbound orders. Operators can edit an order, but the business has cross-field rules that are too specific for a generic required-field validator. A high-value expedited order needs manager approval. A hold needs a hold reason. A ship-ready order needs a carrier service and a valid required ship date. A supervisor also wants an Escalate button that marks the order for review and notifies the operations lead through a trusted host command. +
+ +

This post walks that scenario from setup to execution. The exact field names can be adapted to your own form, but the pattern is the important part: declarative form metadata handles layout, a host command owns external notification, and the form module owns local business rules that should travel with the database.

+ +

What Code Modules Add

+

Before code modules, Admin Forms already had trusted host commands and declarative action sequences. Those are still the right tools for many workflows. Code modules fill the gap where the logic is small, form-specific, and easier to express as C# than as a chain of declarative conditions.

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
UseBest ToolWhy
Set a field, show a message, open another form, apply a filterAction sequenceNo code, serializes cleanly with the form definition.
Send email, call an API, write to an application serviceTrusted host commandPrivileged integrations stay in the application host.
Cross-field validation, button-specific business logic, helper methodsC# code moduleThe logic travels with the database and builds like normal C#.
+ +

The Workflow

+
+
+
Step 1
+
Create handlers
+
Use the form designer to create a C# handler for BeforeUpdate and one for the escalation button's OnClick.
+
+
+
Step 2
+
Export files
+
Open the Code Modules tab and export the module workspace to a normal folder.
+
+
+
Step 3
+
Edit C#
+
Open the exported .cs file in your editor and add the form-specific rules.
+
+
+
Step 4
+
Import, build, trust
+
Import the files, build the module set, review diagnostics, then trust the current hash locally.
+
+
+ +

Host Setup

+

The packaged CSharpDB Admin host already wires the code-module services. If you host Admin Forms yourself, register both the code-module runtime and the form designer integration:

+ +
+ 
using CSharpDB.Admin.Forms.Services;
+using CSharpDB.CodeModules;
+
+builder.Services.AddCSharpDbCodeModules(options =>
+{
+    options.EnableInProcessExecution = true;
+});
+
+builder.Services.AddCSharpDbAdminForms();
+builder.Services.AddCSharpDbAdminFormCodeModules();
+
+ +

The explicit EnableInProcessExecution flag matters. A database can store code modules without being allowed to execute them. Runtime execution requires host opt-in, a successful build, and a local trust grant for the exact module-set hash.

+ +

Register the Host Command

+

The form module should not know how to send Slack messages, email, Teams notifications, or tickets. That belongs in a trusted host command. The form module can ask for NotifyOpsLead; the host decides what that means. In a self-hosted app, use the command overload below in place of the plain AddCSharpDbAdminForms() call from the setup snippet.

+ +
+ 
using CSharpDB.Primitives;
+
+builder.Services.AddCSharpDbAdminForms(commands =>
+{
+    commands.AddAsyncCommand(
+        "NotifyOpsLead",
+        new DbCommandOptions(
+            Description: "Notifies the operations lead that an order needs review.",
+            Timeout: TimeSpan.FromSeconds(5)),
+        async (context, ct) =>
+        {
+            string orderNumber = context.Arguments["orderNumber"].AsText;
+            string reason = context.Arguments["reason"].AsText;
+
+            await opsNotifier.NotifyAsync(orderNumber, reason, ct);
+
+            return DbCommandResult.Success("Ops lead notified.");
+        });
+});
+
+ +

In a real application, opsNotifier is your service. It can call an internal API, enqueue a message, or write to an audit table. The database-owned module only sees the safe command surface.

+ +

Create the Form Handlers

+

In Admin, open the Order Workbench form in the designer. Save the form first so it has a stable form id. Then add two event bindings:

+
    +
  1. At the form level, add BeforeUpdate and click Create in the C# handler row.
    2. +
  2. Select the escalation button, add OnClick, and click Create in the C# handler row.
    3. +
+

The designer creates or updates one form module and attaches CodeModuleHandler references to the event bindings. For a form named Order Workbench, the generated source looks like this:

+ +
+ 
using CSharpDB.CodeModules.Runtime;
+
+namespace CSharpDB.UserCode.Forms;
+
+public sealed class OrderWorkbenchModule : FormCodeModule
+{
+    public void OnBeforeUpdate(FormBeforeEventContext context)
+    {
+    }
+
+    public void EscalateOrderButton_OnClick(FormControlEventContext context)
+    {
+    }
+}
+
+ +

Export to Files

+

Open the Code Modules tab from the title bar or command palette. Choose a workspace folder and click Export. Admin creates a file workspace under .csharpdb-code:

+ +
+ 
.csharpdb-code/
+  csharpdb.codeproj.json
+  forms/
+    Order_Workbench.cs
+  modules/
+  classes/
+
+ +

The manifest records each module id, kind, owner, type name, file path, and source hash. On import, CSharpDB compares the manifest hash, current database hash, and file hash. If both the database and the exported file changed since export, import reports a conflict instead of overwriting one side.

+ +

Add the Business Rules

+

Open the exported form module in your normal C# editor and replace the generated methods with the order workflow. This example assumes the form record includes fields such as status, priority_code, carrier_service, required_ship_date, hold_reason, manager_approval_code, total_amount, notes, order_number, and last_reviewed_utc.

+ +
+ 
using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Threading.Tasks;
+using CSharpDB.CodeModules.Runtime;
+
+namespace CSharpDB.UserCode.Forms;
+
+public sealed class OrderWorkbenchModule : FormCodeModule
+{
+    private static readonly HashSet<string> ShipReadyStatuses = new(StringComparer.OrdinalIgnoreCase)
+    {
+        "ReadyToShip",
+        "PartiallyAllocated",
+    };
+
+    public void OnBeforeUpdate(FormBeforeEventContext context)
+    {
+        string status = Text("status");
+        string priority = Text("priority_code");
+        decimal amount = Money("total_amount");
+
+        if (ShipReadyStatuses.Contains(status) && string.IsNullOrWhiteSpace(Text("carrier_service")))
+        {
+            context.Cancel("Choose a carrier service before marking this order ready to ship.");
+            return;
+        }
+
+        DateTime? requiredShipDate = Date("required_ship_date");
+        if (status == "ReadyToShip" && requiredShipDate is not null && requiredShipDate.Value.Date < DateTime.Today)
+        {
+            context.Cancel("Required ship date is in the past. Update the date or put the order on hold.");
+            return;
+        }
+
+        if (status == "Hold" && string.IsNullOrWhiteSpace(Text("hold_reason")))
+        {
+            context.Cancel("Hold reason is required when an order is placed on hold.");
+            return;
+        }
+
+        if (priority == "EXPEDITE" && amount >= 10000m && string.IsNullOrWhiteSpace(Text("manager_approval_code")))
+        {
+            context.Cancel("High-value expedited orders require a manager approval code.");
+            return;
+        }
+
+        Me.last_reviewed_utc = DateTime.UtcNow;
+    }
+
+    public async Task EscalateOrderButton_OnClick(FormControlEventContext context)
+    {
+        string reason = context.Arguments.TryGetValue("reason", out object? value)
+            ? Convert.ToString(value, CultureInfo.InvariantCulture) ?? "Manual escalation"
+            : "Manual escalation";
+
+        Me.priority_code = "EXPEDITE";
+        Me.status = "NeedsReview";
+        Me.notes = AppendNote(Me.notes, $"Escalated at {DateTime.UtcNow:O}: {reason}");
+
+        var result = await DoCmd.RunHostCommandAsync(
+            "NotifyOpsLead",
+            new Dictionary<string, object?>
+            {
+                ["orderId"] = Me["id"],
+                ["orderNumber"] = Me["order_number"],
+                ["reason"] = reason,
+            });
+
+        await DoCmd.ShowMessageAsync(
+            result.Succeeded
+                ? "Order escalated and the operations lead was notified."
+                : result.Message ?? "Order was escalated, but the notification failed.");
+    }
+
+    private string Text(string fieldName)
+    {
+        object? value = Me[fieldName];
+        return Convert.ToString(value, CultureInfo.InvariantCulture) ?? string.Empty;
+    }
+
+    private decimal Money(string fieldName)
+    {
+        object? value = Me[fieldName];
+        return value is null ? 0m : Convert.ToDecimal(value, CultureInfo.InvariantCulture);
+    }
+
+    private DateTime? Date(string fieldName)
+    {
+        string text = Text(fieldName);
+        return DateTime.TryParse(
+            text,
+            CultureInfo.InvariantCulture,
+            DateTimeStyles.AllowWhiteSpaces,
+            out DateTime parsed)
+                ? parsed.Date
+                : null;
+    }
+
+    private static string AppendNote(object? existing, string note)
+    {
+        string current = Convert.ToString(existing, CultureInfo.InvariantCulture) ?? string.Empty;
+        return string.IsNullOrWhiteSpace(current)
+            ? note
+            : $"{current}{Environment.NewLine}{note}";
+    }
+}
+
+ +

The important detail is Me. It exposes the current form record as dynamic field access. Me.status = "NeedsReview" writes to the current record. Me["order_number"] reads a known field by name. Unknown fields throw so mistakes fail loudly instead of silently writing stray state.

+ +

Import, Build, and Trust

+

Return to Admin and open the Code Modules tab:

+
    +
  1. Click Import to read the changed files back into the database.
    2. +
  2. Click Build to compile all modules for the database into an in-memory assembly.
    3. +
  3. Review diagnostics in the tab. Diagnostics include module/path, line, column, severity, code, and message.
    4. +
  4. Click Trust after the build succeeds.
    5. +
+

Trust is local. It is keyed by normalized database path plus module-set hash and stored outside the database. Any source change produces a new module-set hash, which means you build and trust again before execution resumes.

+ +

Try the Workflow

+

Open the rendered Order Workbench form and test the rules with realistic records:

+
    +
  • Set status to ReadyToShip without a carrier service. Save should be canceled with a clear message.
    • +
  • Set status to Hold without a hold reason. Save should be canceled.
    • +
  • Set priority_code to EXPEDITE on a high-value order without approval. Save should be canceled.
    • +
  • Click the escalation button. The module should update the priority, status, and notes, then call the trusted host command.
    • +
+

This is the shape that makes code modules useful: the form definition still controls the UI, action sequences still handle simple declarative behavior, the host keeps ownership of external capabilities, and the database carries the small C# module that expresses form-specific business rules.

+ +

Where This Is Deliberately Narrow

+

This first slice is intentionally local and trusted. It does not add an in-browser IDE, a VS Code extension, file watching, debugger integration, report modules, remote execution through daemon transports, or sandboxing. That restraint is part of the safety model: execution is opt-in, build-gated, trust-gated, and limited to the runtime contracts Admin exposes.

+

For teams coming from Access, the mental model is familiar: form module code sits with the form, event handlers are created from the designer, and helper methods live nearby. The difference is that the edit loop uses normal C# files and the run loop refuses to execute changed code until the current module set is built and trusted locally.

+ +
+ ← All posts + Next: ASP.NET Core Identity → +
+
+
+
+
+ + + + + diff --git a/www/blog/index.html b/www/blog/index.html index fac21d7a..559b078e 100644 --- a/www/blog/index.html +++ b/www/blog/index.html @@ -118,6 +118,16 @@

Blog

Tutorials, guides, and best practices for building with CSharpDB.

+ +
+ Guide + May 10, 2026 +
+

Build Access-Style C# Form Modules

+

Build a real order-escalation workflow with database-owned C# form modules: generate handlers, export to files, edit in normal C# tools, import, build, trust, and run from Admin Forms.

+
Read article →
+ +
Guide diff --git a/www/css/style.css b/www/css/style.css index b7e46ed1..0f0a73f1 100644 --- a/www/css/style.css +++ b/www/css/style.css @@ -761,6 +761,16 @@ code { font-size: inherit; } +.code-block > pre, +.doc-content .code-block pre, +.blog-post-content .code-block pre { + margin: 0; + background: transparent; + border: 0; + border-radius: 0; + box-shadow: none; +} + .doc-content table { width: 100%; border-collapse: collapse; diff --git a/www/sitemap.xml b/www/sitemap.xml index 3614d2cb..b78e0105 100644 --- a/www/sitemap.xml +++ b/www/sitemap.xml @@ -296,10 +296,16 @@ https://csharpdb.com/blog/ - 2026-04-25 + 2026-05-10 weekly 0.7 + + https://csharpdb.com/blog/access-style-csharp-code-modules.html + 2026-05-10 + monthly + 0.6 + https://csharpdb.com/blog/fulfillment-hub-sample-walkthrough.html 2026-04-25 From 6e5cf4f65c933cbc47651e4320a59e303f37cd0d Mon Sep 17 00:00:00 2001 From: Maximum Code Date: Sun, 10 May 2026 10:47:29 -0700 Subject: [PATCH 2/8] Add native table archives and external tables --- .gitignore | 1 + CSharpDB.slnx | 2 + .../CSharpDB.Admin.ImportExport.csproj | 17 + .../Components/ImportExportProgress.razor | 78 ++ .../Components/ImportExportStatus.razor | 26 + .../ExternalTableRegistrationInfo.cs | 10 + .../ExternalTableRegistrationRequest.cs | 8 + .../Contracts/RestoreTableRequest.cs | 7 + .../Contracts/RestoreTableResult.cs | 7 + .../Contracts/TableExportDestination.cs | 7 + .../Contracts/TableExportProgress.cs | 17 + .../Contracts/TableExportRequest.cs | 8 + .../Contracts/TableExportResult.cs | 11 + .../Models/TableArchiveDownload.cs | 9 + .../Pages/ImportExport.razor | 461 +++++++++++ .../Services/ITableArchiveDownloadStore.cs | 9 + .../Services/ITableImportExportService.cs | 20 + ...ImportExportServiceCollectionExtensions.cs | 42 + .../Services/TableArchiveDownloadStore.cs | 58 ++ .../Services/TableImportExportService.cs | 606 ++++++++++++++ .../_Imports.razor | 6 + src/CSharpDB.Admin/CSharpDB.Admin.csproj | 1 + .../Components/Layout/CommandPalette.razor | 1 + .../Components/Layout/MainLayout.razor | 4 + .../Components/Layout/NavMenu.razor | 235 +++++- .../Components/Shared/DataGrid.razor | 127 ++- .../Components/Tabs/QueryTab.razor | 77 +- src/CSharpDB.Admin/Models/TabDescriptor.cs | 3 +- src/CSharpDB.Admin/Program.cs | 3 + .../Services/DatabaseClientHolder.cs | 18 +- .../Services/TabManagerService.cs | 13 + src/CSharpDB.Admin/_Imports.razor | 1 + src/CSharpDB.Admin/wwwroot/css/app.css | 313 +++++++ src/CSharpDB.Client/CSharpDB.Client.csproj | 1 + src/CSharpDB.Client/CSharpDbClient.cs | 19 +- .../ICSharpDbTableArchiveExporter.cs | 22 + .../Internal/EngineTransportClient.cs | 167 +++- .../Models/TableArchiveExportModels.cs | 19 + src/CSharpDB.Engine/Database.cs | 37 +- .../CSharpDB.Execution.csproj | 1 + src/CSharpDB.Execution/Operators.cs | 297 +++++++ src/CSharpDB.Execution/QueryPlanner.cs | 770 +++++++++++++++++- .../CSharpDB.ImportExport.csproj | 15 + .../Models/TableArchiveColumn.cs | 33 + .../Models/TableArchiveForeignKey.cs | 33 + .../Models/TableArchiveIndexManifest.cs | 11 + .../Models/TableArchiveManifest.cs | 14 + .../Models/TableArchiveRowLookupResult.cs | 5 + .../Models/TableArchiveSchema.cs | 27 + .../Serialization/TableArchiveJson.cs | 17 + .../Serialization/TableArchiveNativeFormat.cs | 191 +++++ .../TableArchivePrimaryKeyLookupReader.cs | 85 ++ .../TableArchives/TableArchiveReader.cs | 305 +++++++ .../TableArchives/TableArchiveWriter.cs | 393 +++++++++ src/CSharpDB.Sql/Ast.cs | 13 + src/CSharpDB.Sql/Parser.cs | 49 ++ src/CSharpDB.Sql/TokenType.cs | 1 + src/CSharpDB.Sql/Tokenizer.cs | 1 + .../Catalog/CatalogService.cs | 5 + src/CSharpDB.Storage/Catalog/SchemaCatalog.cs | 2 + .../Admin/TabManagerServiceTests.cs | 26 + tests/CSharpDB.Tests/CSharpDB.Tests.csproj | 1 + .../EngineTransportClientTests.cs | 136 ++++ tests/CSharpDB.Tests/ExternalTableTests.cs | 197 +++++ tests/CSharpDB.Tests/ParserTests.cs | 39 + tests/CSharpDB.Tests/TableArchiveTests.cs | 161 ++++ 66 files changed, 5198 insertions(+), 101 deletions(-) create mode 100644 src/CSharpDB.Admin.ImportExport/CSharpDB.Admin.ImportExport.csproj create mode 100644 src/CSharpDB.Admin.ImportExport/Components/ImportExportProgress.razor create mode 100644 src/CSharpDB.Admin.ImportExport/Components/ImportExportStatus.razor create mode 100644 src/CSharpDB.Admin.ImportExport/Contracts/ExternalTableRegistrationInfo.cs create mode 100644 src/CSharpDB.Admin.ImportExport/Contracts/ExternalTableRegistrationRequest.cs create mode 100644 src/CSharpDB.Admin.ImportExport/Contracts/RestoreTableRequest.cs create mode 100644 src/CSharpDB.Admin.ImportExport/Contracts/RestoreTableResult.cs create mode 100644 src/CSharpDB.Admin.ImportExport/Contracts/TableExportDestination.cs create mode 100644 src/CSharpDB.Admin.ImportExport/Contracts/TableExportProgress.cs create mode 100644 src/CSharpDB.Admin.ImportExport/Contracts/TableExportRequest.cs create mode 100644 src/CSharpDB.Admin.ImportExport/Contracts/TableExportResult.cs create mode 100644 src/CSharpDB.Admin.ImportExport/Models/TableArchiveDownload.cs create mode 100644 src/CSharpDB.Admin.ImportExport/Pages/ImportExport.razor create mode 100644 src/CSharpDB.Admin.ImportExport/Services/ITableArchiveDownloadStore.cs create mode 100644 src/CSharpDB.Admin.ImportExport/Services/ITableImportExportService.cs create mode 100644 src/CSharpDB.Admin.ImportExport/Services/ImportExportServiceCollectionExtensions.cs create mode 100644 src/CSharpDB.Admin.ImportExport/Services/TableArchiveDownloadStore.cs create mode 100644 src/CSharpDB.Admin.ImportExport/Services/TableImportExportService.cs create mode 100644 src/CSharpDB.Admin.ImportExport/_Imports.razor create mode 100644 src/CSharpDB.Client/ICSharpDbTableArchiveExporter.cs create mode 100644 src/CSharpDB.Client/Models/TableArchiveExportModels.cs create mode 100644 src/CSharpDB.ImportExport/CSharpDB.ImportExport.csproj create mode 100644 src/CSharpDB.ImportExport/Models/TableArchiveColumn.cs create mode 100644 src/CSharpDB.ImportExport/Models/TableArchiveForeignKey.cs create mode 100644 src/CSharpDB.ImportExport/Models/TableArchiveIndexManifest.cs create mode 100644 src/CSharpDB.ImportExport/Models/TableArchiveManifest.cs create mode 100644 src/CSharpDB.ImportExport/Models/TableArchiveRowLookupResult.cs create mode 100644 src/CSharpDB.ImportExport/Models/TableArchiveSchema.cs create mode 100644 src/CSharpDB.ImportExport/Serialization/TableArchiveJson.cs create mode 100644 src/CSharpDB.ImportExport/Serialization/TableArchiveNativeFormat.cs create mode 100644 src/CSharpDB.ImportExport/TableArchives/TableArchivePrimaryKeyLookupReader.cs create mode 100644 src/CSharpDB.ImportExport/TableArchives/TableArchiveReader.cs create mode 100644 src/CSharpDB.ImportExport/TableArchives/TableArchiveWriter.cs create mode 100644 tests/CSharpDB.Tests/ExternalTableTests.cs create mode 100644 tests/CSharpDB.Tests/TableArchiveTests.cs diff --git a/.gitignore b/.gitignore index 917b3e1b..a953c8cf 100644 --- a/.gitignore +++ b/.gitignore @@ -93,6 +93,7 @@ Desktop.ini ## CSharpDB runtime files *.cdb *.db +*.csdbtable *.db.journal *.wal diff --git a/CSharpDB.slnx b/CSharpDB.slnx index f7a6e573..aabb04c6 100644 --- a/CSharpDB.slnx +++ b/CSharpDB.slnx @@ -34,6 +34,7 @@ + @@ -42,6 +43,7 @@ + diff --git a/src/CSharpDB.Admin.ImportExport/CSharpDB.Admin.ImportExport.csproj b/src/CSharpDB.Admin.ImportExport/CSharpDB.Admin.ImportExport.csproj new file mode 100644 index 00000000..ed6ba885 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/CSharpDB.Admin.ImportExport.csproj @@ -0,0 +1,17 @@ + + + + net10.0 + enable + enable + CSharpDB.Admin.ImportExport + + + + + + + + + + diff --git a/src/CSharpDB.Admin.ImportExport/Components/ImportExportProgress.razor b/src/CSharpDB.Admin.ImportExport/Components/ImportExportProgress.razor new file mode 100644 index 00000000..4180eb5c --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Components/ImportExportProgress.razor @@ -0,0 +1,78 @@ +@namespace CSharpDB.Admin.ImportExport.Components + +@if (Progress is not null) +{ +
+
+
+
+ + @Progress.Operation +
+ @Progress.Stage +
+ @if (CanCancel) + { + + } +
+ +
+
+
+ +
+ @(Progress.Message ?? Progress.Stage) + @FormatRows() + @if (StartedAt is not null) + { + @FormatElapsed() + } +
+ + @if (!string.IsNullOrWhiteSpace(Progress.Path)) + { +
@Progress.Path
+ } +
+} + +@code { + [Parameter] public TableExportProgress? Progress { get; set; } + [Parameter] public DateTimeOffset? StartedAt { get; set; } + [Parameter] public bool CanCancel { get; set; } + [Parameter] public EventCallback OnCancel { get; set; } + + private string GetFillClass() => + Progress?.PercentComplete is int ? "import-export-progress-fill" : "import-export-progress-fill indeterminate"; + + private string GetFillStyle() => + Progress?.PercentComplete is int percent ? $"width: {percent}%;" : string.Empty; + + private string FormatRows() + { + if (Progress is null) + return string.Empty; + + string processed = Progress.RowsProcessed.ToString("N0", System.Globalization.CultureInfo.InvariantCulture); + return Progress.TotalRows is long total + ? $"{processed} / {total.ToString("N0", System.Globalization.CultureInfo.InvariantCulture)} rows" + : $"{processed} rows"; + } + + private string FormatElapsed() + { + if (StartedAt is null) + return string.Empty; + + TimeSpan elapsed = DateTimeOffset.UtcNow - StartedAt.Value; + return elapsed.TotalMinutes >= 1 + ? $"{(int)elapsed.TotalMinutes}m {elapsed.Seconds}s" + : $"{Math.Max(0, (int)elapsed.TotalSeconds)}s"; + } +} diff --git a/src/CSharpDB.Admin.ImportExport/Components/ImportExportStatus.razor b/src/CSharpDB.Admin.ImportExport/Components/ImportExportStatus.razor new file mode 100644 index 00000000..1193e02f --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Components/ImportExportStatus.razor @@ -0,0 +1,26 @@ +@namespace CSharpDB.Admin.ImportExport.Components + +@if (!string.IsNullOrWhiteSpace(Error)) +{ +
+ + @Error +
+} +@if (!string.IsNullOrWhiteSpace(Message)) +{ +
+ + @Message + @if (!string.IsNullOrWhiteSpace(DownloadUrl)) + { + Download + } +
+} + +@code { + [Parameter] public string? Message { get; set; } + [Parameter] public string? Error { get; set; } + [Parameter] public string? DownloadUrl { get; set; } +} diff --git a/src/CSharpDB.Admin.ImportExport/Contracts/ExternalTableRegistrationInfo.cs b/src/CSharpDB.Admin.ImportExport/Contracts/ExternalTableRegistrationInfo.cs new file mode 100644 index 00000000..476390fd --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Contracts/ExternalTableRegistrationInfo.cs @@ -0,0 +1,10 @@ +namespace CSharpDB.Admin.ImportExport.Contracts; + +public sealed class ExternalTableRegistrationInfo +{ + public required string TableName { get; init; } + public required string Path { get; init; } + public string? SourceTableName { get; init; } + public long RowCount { get; init; } + public DateTimeOffset? CreatedUtc { get; init; } +} diff --git a/src/CSharpDB.Admin.ImportExport/Contracts/ExternalTableRegistrationRequest.cs b/src/CSharpDB.Admin.ImportExport/Contracts/ExternalTableRegistrationRequest.cs new file mode 100644 index 00000000..a4dce1d0 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Contracts/ExternalTableRegistrationRequest.cs @@ -0,0 +1,8 @@ +namespace CSharpDB.Admin.ImportExport.Contracts; + +public sealed class ExternalTableRegistrationRequest +{ + public required string TableName { get; init; } + public required string ArchivePath { get; init; } + public bool ReplaceExisting { get; init; } = true; +} diff --git a/src/CSharpDB.Admin.ImportExport/Contracts/RestoreTableRequest.cs b/src/CSharpDB.Admin.ImportExport/Contracts/RestoreTableRequest.cs new file mode 100644 index 00000000..a213bd5d --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Contracts/RestoreTableRequest.cs @@ -0,0 +1,7 @@ +namespace CSharpDB.Admin.ImportExport.Contracts; + +public sealed class RestoreTableRequest +{ + public required string ArchivePath { get; init; } + public string? TargetTableName { get; init; } +} diff --git a/src/CSharpDB.Admin.ImportExport/Contracts/RestoreTableResult.cs b/src/CSharpDB.Admin.ImportExport/Contracts/RestoreTableResult.cs new file mode 100644 index 00000000..247aa978 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Contracts/RestoreTableResult.cs @@ -0,0 +1,7 @@ +namespace CSharpDB.Admin.ImportExport.Contracts; + +public sealed class RestoreTableResult +{ + public required string TableName { get; init; } + public long RowsInserted { get; init; } +} diff --git a/src/CSharpDB.Admin.ImportExport/Contracts/TableExportDestination.cs b/src/CSharpDB.Admin.ImportExport/Contracts/TableExportDestination.cs new file mode 100644 index 00000000..0e7bbba7 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Contracts/TableExportDestination.cs @@ -0,0 +1,7 @@ +namespace CSharpDB.Admin.ImportExport.Contracts; + +public enum TableExportDestination +{ + Download, + ServerPath, +} diff --git a/src/CSharpDB.Admin.ImportExport/Contracts/TableExportProgress.cs b/src/CSharpDB.Admin.ImportExport/Contracts/TableExportProgress.cs new file mode 100644 index 00000000..2739d93c --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Contracts/TableExportProgress.cs @@ -0,0 +1,17 @@ +namespace CSharpDB.Admin.ImportExport.Contracts; + +public sealed class TableExportProgress +{ + public required string Operation { get; init; } + public required string Stage { get; init; } + public string? Message { get; init; } + public string? TableName { get; init; } + public string? Path { get; init; } + public long RowsProcessed { get; init; } + public long? TotalRows { get; init; } + + public int? PercentComplete => + TotalRows is > 0 + ? (int)Math.Clamp(Math.Round((double)RowsProcessed / TotalRows.Value * 100), 0, 100) + : null; +} diff --git a/src/CSharpDB.Admin.ImportExport/Contracts/TableExportRequest.cs b/src/CSharpDB.Admin.ImportExport/Contracts/TableExportRequest.cs new file mode 100644 index 00000000..2257ef90 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Contracts/TableExportRequest.cs @@ -0,0 +1,8 @@ +namespace CSharpDB.Admin.ImportExport.Contracts; + +public sealed class TableExportRequest +{ + public required string TableName { get; init; } + public TableExportDestination Destination { get; init; } + public string? ServerPath { get; init; } +} diff --git a/src/CSharpDB.Admin.ImportExport/Contracts/TableExportResult.cs b/src/CSharpDB.Admin.ImportExport/Contracts/TableExportResult.cs new file mode 100644 index 00000000..31cf6dcf --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Contracts/TableExportResult.cs @@ -0,0 +1,11 @@ +namespace CSharpDB.Admin.ImportExport.Contracts; + +public sealed class TableExportResult +{ + public required string TableName { get; init; } + public required string FileName { get; init; } + public required string Path { get; init; } + public long RowCount { get; init; } + public string? DownloadUrl { get; init; } + public bool IsDownload { get; init; } +} diff --git a/src/CSharpDB.Admin.ImportExport/Models/TableArchiveDownload.cs b/src/CSharpDB.Admin.ImportExport/Models/TableArchiveDownload.cs new file mode 100644 index 00000000..4661af60 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Models/TableArchiveDownload.cs @@ -0,0 +1,9 @@ +namespace CSharpDB.Admin.ImportExport.Models; + +public sealed class TableArchiveDownload +{ + public required string Token { get; init; } + public required string Path { get; init; } + public required string FileName { get; init; } + public DateTimeOffset CreatedUtc { get; init; } = DateTimeOffset.UtcNow; +} diff --git a/src/CSharpDB.Admin.ImportExport/Pages/ImportExport.razor b/src/CSharpDB.Admin.ImportExport/Pages/ImportExport.razor new file mode 100644 index 00000000..ed7fdf67 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Pages/ImportExport.razor @@ -0,0 +1,461 @@ +@namespace CSharpDB.Admin.ImportExport.Pages +@inject ICSharpDbClient DbClient +@inject ITableImportExportService ImportExportService + +
+
+
+
+ tables + / + Import / Export +
+

Import / Export

+
+ +
+ +
+ + + +
+ + + + +
+ @switch (_mode) + { + case ImportExportMode.ExportTable: +
+ + +
+ Destination +
+ + +
+
+ + @if (_destination == TableExportDestination.ServerPath) + { + + } + +
+ +
+
+ break; + + case ImportExportMode.RegisterExternalTable: +
+ + +
+ +
+ + @if (_externalTables.Count > 0) + { +
+
+ Registered external tables + @_externalTables.Count +
+ @foreach (ExternalTableRegistrationInfo registration in _externalTables) + { +
+ + +
+ } +
+ } +
+ break; + + case ImportExportMode.RestoreTable: +
+ + +
+ +
+
+ break; + } +
+
+ +@code { + [Parameter] public string? InitialTableName { get; set; } + + private ImportExportMode _mode = ImportExportMode.ExportTable; + private TableExportDestination _destination = TableExportDestination.Download; + private IReadOnlyList _tables = Array.Empty(); + private IReadOnlyList _externalTables = Array.Empty(); + private string _selectedTable = string.Empty; + private string _serverPath = string.Empty; + private string _externalTableName = string.Empty; + private string _externalArchivePath = string.Empty; + private string _restoreArchivePath = string.Empty; + private string _restoreTableName = string.Empty; + private string? _message; + private string? _error; + private string? _downloadUrl; + private TableExportProgress? _progress; + private DateTimeOffset? _operationStartedAt; + private CancellationTokenSource? _operationCts; + private bool _busy; + + private bool CanExport => !_busy && !string.IsNullOrWhiteSpace(_selectedTable) && + (_destination == TableExportDestination.Download || !string.IsNullOrWhiteSpace(_serverPath)); + private bool CanRegisterExternal => !_busy && !string.IsNullOrWhiteSpace(_externalTableName) && !string.IsNullOrWhiteSpace(_externalArchivePath); + private bool CanRestore => !_busy && !string.IsNullOrWhiteSpace(_restoreArchivePath); + + protected override async Task OnInitializedAsync() + { + await RefreshAsync(); + } + + protected override async Task OnParametersSetAsync() + { + if (!string.IsNullOrWhiteSpace(InitialTableName) && + !string.Equals(_selectedTable, InitialTableName, StringComparison.OrdinalIgnoreCase)) + { + _selectedTable = InitialTableName; + _externalTableName = $"archived_{InitialTableName}"; + await RefreshDefaultServerPathAsync(); + } + } + + private async Task RefreshAsync() + { + await RunAsync(async ct => + { + _tables = (await DbClient.GetTableNamesAsync(ct)) + .Where(static name => !name.StartsWith("_", StringComparison.Ordinal)) + .OrderBy(static name => name, StringComparer.OrdinalIgnoreCase) + .ToArray(); + _externalTables = await ImportExportService.GetExternalTablesAsync(ct); + if (string.IsNullOrWhiteSpace(_selectedTable) || !_tables.Contains(_selectedTable, StringComparer.OrdinalIgnoreCase)) + _selectedTable = _tables.FirstOrDefault() ?? string.Empty; + + if (string.IsNullOrWhiteSpace(_externalTableName) && !string.IsNullOrWhiteSpace(_selectedTable)) + _externalTableName = $"archived_{_selectedTable}"; + + await RefreshDefaultServerPathAsync(ct); + }, clearMessage: false); + } + + private async Task OnSelectedTableChangedAsync() + { + _externalTableName = string.IsNullOrWhiteSpace(_selectedTable) ? string.Empty : $"archived_{_selectedTable}"; + await RefreshDefaultServerPathAsync(); + } + + private async Task RefreshDefaultServerPathAsync(CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(_selectedTable)) + return; + + _serverPath = await ImportExportService.GetDefaultServerExportPathAsync(_selectedTable, ct); + } + + private async Task ExportAsync() + { + await RunAsync(async ct => + { + SetProgress(new TableExportProgress + { + Operation = "Export table", + Stage = "Preparing", + Message = _destination == TableExportDestination.Download + ? "Preparing download export" + : "Preparing server-path export", + TableName = _selectedTable, + Path = _destination == TableExportDestination.ServerPath ? _serverPath : null, + }); + await InvokeAsync(StateHasChanged); + await Task.Yield(); + + var progress = CreateProgressReporter(ct); + var request = new TableExportRequest + { + TableName = _selectedTable, + Destination = _destination, + ServerPath = _destination == TableExportDestination.ServerPath ? _serverPath : null, + }; + var result = await Task.Run( + () => ImportExportService.ExportTableAsync(request, progress, ct), + ct); + + _downloadUrl = result.DownloadUrl; + _externalArchivePath = result.Path; + _restoreArchivePath = result.Path; + _message = result.IsDownload + ? $"Exported {result.RowCount} row(s) to {result.FileName}." + : $"Exported {result.RowCount} row(s) to {result.Path}."; + }); + } + + private async Task RegisterExternalTableAsync() + { + await RunAsync(async ct => + { + SetProgress(new TableExportProgress + { + Operation = "Register external table", + Stage = "Queued", + Message = "Waiting to register external table", + TableName = _externalTableName, + Path = _externalArchivePath, + }); + await InvokeAsync(StateHasChanged); + await Task.Yield(); + + var progress = CreateProgressReporter(ct); + var request = new ExternalTableRegistrationRequest + { + TableName = _externalTableName, + ArchivePath = _externalArchivePath, + }; + await Task.Run( + () => ImportExportService.RegisterExternalTableAsync(request, progress, ct), + ct); + _externalTables = await ImportExportService.GetExternalTablesAsync(ct); + _message = $"Registered external table {_externalTableName}."; + }); + } + + private async Task DropRegisteredExternalTableAsync(string tableName) + { + await RunAsync(async ct => + { + SetProgress(new TableExportProgress + { + Operation = "Drop external table", + Stage = "Dropping", + Message = "Removing external table registration", + TableName = tableName, + }); + + await Task.Run( + () => ImportExportService.DropExternalTableAsync(tableName, ct), + ct); + _externalTables = await ImportExportService.GetExternalTablesAsync(ct); + _message = $"Dropped external table {tableName}."; + }); + } + + private async Task RestoreAsync() + { + await RunAsync(async ct => + { + SetProgress(new TableExportProgress + { + Operation = "Restore table", + Stage = "Restoring", + Message = "Creating target table and inserting archived rows", + TableName = string.IsNullOrWhiteSpace(_restoreTableName) ? null : _restoreTableName, + Path = _restoreArchivePath, + }); + var request = new RestoreTableRequest + { + ArchivePath = _restoreArchivePath, + TargetTableName = _restoreTableName, + }; + var result = await Task.Run( + () => ImportExportService.RestoreTableAsync(request, ct), + ct); + _message = $"Restored {result.RowsInserted} row(s) into {result.TableName}."; + _tables = await DbClient.GetTableNamesAsync(ct); + }); + } + + private async Task RunAsync(Func action, bool clearMessage = true) + { + if (_busy) + return; + + _operationCts?.Dispose(); + _operationCts = new CancellationTokenSource(); + CancellationToken token = _operationCts.Token; + _busy = true; + _operationStartedAt = DateTimeOffset.UtcNow; + _progress = null; + _error = null; + if (clearMessage) + { + _message = null; + _downloadUrl = null; + } + + await InvokeAsync(StateHasChanged); + await Task.Yield(); + + try + { + await action(token); + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + _message = "Operation canceled."; + } + catch (Exception ex) + { + _error = ex.Message; + } + finally + { + _busy = false; + _progress = null; + _operationStartedAt = null; + _operationCts?.Dispose(); + _operationCts = null; + await InvokeAsync(StateHasChanged); + } + } + + private IProgress CreateProgressReporter(CancellationToken token) => + new Progress(progress => + { + _ = InvokeAsync(() => + { + if (!_busy || token.IsCancellationRequested) + return; + + _progress = progress; + StateHasChanged(); + }); + }); + + private void SetProgress(TableExportProgress progress) + { + if (!_busy) + return; + + _progress = progress; + StateHasChanged(); + } + + private void CancelOperation() + { + _operationCts?.Cancel(); + if (_progress is not null) + { + _progress = new TableExportProgress + { + Operation = _progress.Operation, + Stage = "Canceling", + Message = "Cancel requested", + TableName = _progress.TableName, + Path = _progress.Path, + RowsProcessed = _progress.RowsProcessed, + TotalRows = _progress.TotalRows, + }; + } + + _ = InvokeAsync(StateHasChanged); + } + + private void UseExternalTableRegistration(ExternalTableRegistrationInfo registration) + { + _externalTableName = registration.TableName; + _externalArchivePath = registration.Path; + } + + private void SetMode(ImportExportMode mode) + { + _mode = mode; + _error = null; + _message = null; + _downloadUrl = null; + } + + private void SetDestination(TableExportDestination destination) => _destination = destination; + + private string GetModeButtonClass(ImportExportMode mode) => _mode == mode ? "active" : string.Empty; + private string GetDestinationButtonClass(TableExportDestination destination) => _destination == destination ? "active" : string.Empty; + + private static string FormatExternalTableDetails(ExternalTableRegistrationInfo registration) + { + string source = string.IsNullOrWhiteSpace(registration.SourceTableName) + ? "external" + : registration.SourceTableName; + return $"{source} · {registration.RowCount:N0} rows"; + } + + private enum ImportExportMode + { + ExportTable, + RegisterExternalTable, + RestoreTable, + } +} diff --git a/src/CSharpDB.Admin.ImportExport/Services/ITableArchiveDownloadStore.cs b/src/CSharpDB.Admin.ImportExport/Services/ITableArchiveDownloadStore.cs new file mode 100644 index 00000000..3b8b45d9 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Services/ITableArchiveDownloadStore.cs @@ -0,0 +1,9 @@ +using CSharpDB.Admin.ImportExport.Models; + +namespace CSharpDB.Admin.ImportExport.Services; + +public interface ITableArchiveDownloadStore +{ + TableArchiveDownload Add(string path, string fileName); + bool TryTake(string token, out TableArchiveDownload download); +} diff --git a/src/CSharpDB.Admin.ImportExport/Services/ITableImportExportService.cs b/src/CSharpDB.Admin.ImportExport/Services/ITableImportExportService.cs new file mode 100644 index 00000000..0b880fe9 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Services/ITableImportExportService.cs @@ -0,0 +1,20 @@ +using CSharpDB.Admin.ImportExport.Contracts; + +namespace CSharpDB.Admin.ImportExport.Services; + +public interface ITableImportExportService +{ + Task GetDefaultServerExportPathAsync(string tableName, CancellationToken ct = default); + Task> GetExternalTablesAsync(CancellationToken ct = default); + Task ExportTableAsync( + TableExportRequest request, + IProgress? progress = null, + CancellationToken ct = default); + Task RegisterExternalTableAsync(ExternalTableRegistrationRequest request, CancellationToken ct = default); + Task RegisterExternalTableAsync( + ExternalTableRegistrationRequest request, + IProgress? progress, + CancellationToken ct = default); + Task DropExternalTableAsync(string tableName, CancellationToken ct = default); + Task RestoreTableAsync(RestoreTableRequest request, CancellationToken ct = default); +} diff --git a/src/CSharpDB.Admin.ImportExport/Services/ImportExportServiceCollectionExtensions.cs b/src/CSharpDB.Admin.ImportExport/Services/ImportExportServiceCollectionExtensions.cs new file mode 100644 index 00000000..49fe755b --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Services/ImportExportServiceCollectionExtensions.cs @@ -0,0 +1,42 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; +using Microsoft.Extensions.DependencyInjection; + +namespace CSharpDB.Admin.ImportExport.Services; + +public static class ImportExportServiceCollectionExtensions +{ + public static IServiceCollection AddCSharpDbAdminImportExport(this IServiceCollection services) + { + services.AddSingleton(); + services.AddScoped(); + return services; + } + + public static IEndpointRouteBuilder MapCSharpDbAdminImportExport(this IEndpointRouteBuilder endpoints) + { + endpoints.MapGet("/admin/import-export/download/{token}", ( + string token, + HttpContext httpContext, + ITableArchiveDownloadStore downloads) => + { + if (!downloads.TryTake(token, out var download) || !File.Exists(download.Path)) + return Results.NotFound(); + + httpContext.Response.OnCompleted(() => + { + TableArchiveDownloadStore.TryDelete(download.Path); + return Task.CompletedTask; + }); + + return Results.File( + download.Path, + "application/octet-stream", + download.FileName, + enableRangeProcessing: false); + }); + + return endpoints; + } +} diff --git a/src/CSharpDB.Admin.ImportExport/Services/TableArchiveDownloadStore.cs b/src/CSharpDB.Admin.ImportExport/Services/TableArchiveDownloadStore.cs new file mode 100644 index 00000000..be036dcb --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Services/TableArchiveDownloadStore.cs @@ -0,0 +1,58 @@ +using System.Collections.Concurrent; +using CSharpDB.Admin.ImportExport.Models; + +namespace CSharpDB.Admin.ImportExport.Services; + +public sealed class TableArchiveDownloadStore : ITableArchiveDownloadStore +{ + private readonly ConcurrentDictionary _downloads = new(StringComparer.Ordinal); + + public TableArchiveDownload Add(string path, string fileName) + { + CleanupExpiredDownloads(); + + var token = Convert.ToHexString(Guid.NewGuid().ToByteArray()).ToLowerInvariant(); + var download = new TableArchiveDownload + { + Token = token, + Path = path, + FileName = fileName, + }; + + _downloads[token] = download; + return download; + } + + public bool TryTake(string token, out TableArchiveDownload download) + { + bool found = _downloads.TryRemove(token, out var value); + download = value!; + return found; + } + + private void CleanupExpiredDownloads() + { + DateTimeOffset cutoff = DateTimeOffset.UtcNow.AddHours(-2); + foreach (var pair in _downloads) + { + if (pair.Value.CreatedUtc >= cutoff) + continue; + + if (_downloads.TryRemove(pair.Key, out var expired)) + TryDelete(expired.Path); + } + } + + internal static void TryDelete(string path) + { + try + { + if (File.Exists(path)) + File.Delete(path); + } + catch + { + // Best-effort cleanup for temporary download packages. + } + } +} diff --git a/src/CSharpDB.Admin.ImportExport/Services/TableImportExportService.cs b/src/CSharpDB.Admin.ImportExport/Services/TableImportExportService.cs new file mode 100644 index 00000000..0558fa22 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Services/TableImportExportService.cs @@ -0,0 +1,606 @@ +using System.Collections; +using System.Diagnostics; +using System.Globalization; +using System.Text; +using CSharpDB.Admin.ImportExport.Contracts; +using CSharpDB.Client; +using CSharpDB.ImportExport.TableArchives; +using TableArchiveExportProgress = CSharpDB.Client.Models.TableArchiveExportProgress; +using ClientColumnDefinition = CSharpDB.Client.Models.ColumnDefinition; +using ClientDbType = CSharpDB.Client.Models.DbType; +using ClientForeignKeyDefinition = CSharpDB.Client.Models.ForeignKeyDefinition; +using ClientForeignKeyOnDeleteAction = CSharpDB.Client.Models.ForeignKeyOnDeleteAction; +using ClientTableSchema = CSharpDB.Client.Models.TableSchema; +using PrimitiveColumnDefinition = CSharpDB.Primitives.ColumnDefinition; +using PrimitiveDbType = CSharpDB.Primitives.DbType; +using PrimitiveDbValue = CSharpDB.Primitives.DbValue; +using PrimitiveForeignKeyDefinition = CSharpDB.Primitives.ForeignKeyDefinition; +using PrimitiveForeignKeyOnDeleteAction = CSharpDB.Primitives.ForeignKeyOnDeleteAction; +using PrimitiveTableSchema = CSharpDB.Primitives.TableSchema; + +namespace CSharpDB.Admin.ImportExport.Services; + +public sealed class TableImportExportService( + ICSharpDbClient client, + ITableArchiveDownloadStore downloads) : ITableImportExportService +{ + private const int ExportPageSize = 1_000; + private const int RestoreInsertBatchSize = 100; + + public Task GetDefaultServerExportPathAsync(string tableName, CancellationToken ct = default) + { + string databaseFolder = ResolveDatabaseFolder(client.DataSource); + string fileName = $"{SanitizeFileName(tableName)}-{DateTime.Now:yyyyMMdd-HHmmss}.csdbtable"; + return Task.FromResult(Path.Combine(databaseFolder, "exports", fileName)); + } + + public async Task> GetExternalTablesAsync(CancellationToken ct = default) + { + var result = await client.ExecuteSqlAsync( + """ + SELECT table_name, path, source_table_name, row_count, created_utc + FROM sys.external_tables + ORDER BY table_name; + """, + ct); + + if (!string.IsNullOrWhiteSpace(result.Error)) + throw new InvalidOperationException(result.Error); + + if (result.Rows is not { Count: > 0 }) + return Array.Empty(); + + return result.Rows + .Select(MapExternalTableRegistration) + .ToArray(); + } + + public async Task ExportTableAsync( + TableExportRequest request, + IProgress? progress = null, + CancellationToken ct = default) + { + string tableName = RequireIdentifier(request.TableName, nameof(request.TableName)); + string path = request.Destination == TableExportDestination.Download + ? CreateTemporaryArchivePath(tableName) + : string.IsNullOrWhiteSpace(request.ServerPath) + ? await GetDefaultServerExportPathAsync(tableName, ct) + : request.ServerPath; + + ReportExportProgress( + progress, + tableName, + "Preparing", + "Preparing export target", + rowsProcessed: 0, + totalRows: null, + path); + + long rowCount; + if (client is ICSharpDbTableArchiveProgressExporter progressExporter && progressExporter.SupportsTableArchiveExport) + { + var archiveProgress = progress is null + ? null + : new Progress(p => ReportExportProgress( + progress, + p.TableName, + p.Stage, + p.Message ?? "Writing table archive", + p.RowsExported, + p.TotalRows, + p.Path ?? path)); + var archiveExport = await progressExporter.ExportTableArchiveAsync(tableName, path, archiveProgress, ct); + rowCount = archiveExport.RowCount; + } + else if (client is ICSharpDbTableArchiveExporter exporter && exporter.SupportsTableArchiveExport) + { + ReportExportProgress( + progress, + tableName, + "Exporting", + "Writing table archive", + rowsProcessed: 0, + totalRows: null, + path); + var archiveExport = await exporter.ExportTableArchiveAsync(tableName, path, ct); + rowCount = archiveExport.RowCount; + } + else + { + ClientTableSchema clientSchema = await client.GetTableSchemaAsync(tableName, ct) + ?? throw new InvalidOperationException($"Table '{tableName}' was not found."); + PrimitiveTableSchema schema = MapSchema(clientSchema); + var manifest = await TableArchiveWriter.WriteAsync( + path, + schema, + EnumerateRowsAsync(clientSchema, path, progress, ct), + ct); + rowCount = manifest.RowCount; + } + + string fileName = Path.GetFileName(path); + string? downloadUrl = null; + + if (request.Destination == TableExportDestination.Download) + { + ReportExportProgress( + progress, + tableName, + "Preparing download", + "Preparing one-time download link", + rowCount, + rowCount, + path); + var download = downloads.Add(path, fileName); + downloadUrl = $"/admin/import-export/download/{download.Token}"; + } + + ReportExportProgress( + progress, + tableName, + "Complete", + "Export complete", + rowCount, + rowCount, + path); + + return new TableExportResult + { + TableName = tableName, + FileName = fileName, + Path = path, + RowCount = rowCount, + DownloadUrl = downloadUrl, + IsDownload = request.Destination == TableExportDestination.Download, + }; + } + + public Task RegisterExternalTableAsync(ExternalTableRegistrationRequest request, CancellationToken ct = default) => + RegisterExternalTableAsync(request, progress: null, ct); + + public async Task RegisterExternalTableAsync( + ExternalTableRegistrationRequest request, + IProgress? progress, + CancellationToken ct = default) + { + string tableName = RequireIdentifier(request.TableName, nameof(request.TableName)); + if (string.IsNullOrWhiteSpace(request.ArchivePath)) + throw new ArgumentException("Archive path is required.", nameof(request.ArchivePath)); + + ReportExportProgress( + progress, + tableName, + "Validating", + "Reading archive manifest", + rowsProcessed: 0, + totalRows: 3, + request.ArchivePath); + await TableArchiveReader.ReadManifestAsync(ResolveArchivePath(request.ArchivePath), ct); + + if (request.ReplaceExisting) + { + ReportExportProgress( + progress, + tableName, + "Replacing", + "Dropping existing registration if present", + rowsProcessed: 1, + totalRows: 3, + request.ArchivePath); + await DropExternalTableAsync(tableName, ct); + } + + ReportExportProgress( + progress, + tableName, + "Registering", + "Writing external table registration", + rowsProcessed: 2, + totalRows: 3, + request.ArchivePath); + string sql = $"CREATE EXTERNAL TABLE {tableName} FROM {FormatStringLiteral(request.ArchivePath)};"; + await ExecuteCheckedAsync(sql, ct); + + ReportExportProgress( + progress, + tableName, + "Complete", + "External table registered", + rowsProcessed: 3, + totalRows: 3, + request.ArchivePath); + } + + public async Task DropExternalTableAsync(string tableName, CancellationToken ct = default) + { + string normalizedTableName = RequireIdentifier(tableName, nameof(tableName)); + string sql = $"DROP EXTERNAL TABLE IF EXISTS {normalizedTableName};"; + await ExecuteCheckedAsync(sql, ct); + } + + public async Task RestoreTableAsync(RestoreTableRequest request, CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(request.ArchivePath)) + throw new ArgumentException("Archive path is required.", nameof(request.ArchivePath)); + + PrimitiveTableSchema archiveSchema = await TableArchiveReader.ReadTableSchemaAsync(request.ArchivePath, ct: ct); + string targetTableName = string.IsNullOrWhiteSpace(request.TargetTableName) + ? RequireIdentifier(archiveSchema.TableName, nameof(request.TargetTableName)) + : RequireIdentifier(request.TargetTableName, nameof(request.TargetTableName)); + + var restoreSchema = new PrimitiveTableSchema + { + TableName = targetTableName, + Columns = archiveSchema.Columns, + ForeignKeys = archiveSchema.ForeignKeys, + NextRowId = archiveSchema.NextRowId, + }; + + await ExecuteCheckedAsync(BuildCreateTableSql(restoreSchema), ct); + + long inserted = 0; + var batch = new List(RestoreInsertBatchSize); + await foreach (PrimitiveDbValue[] row in TableArchiveReader.ReadRowsAsync(request.ArchivePath, ct)) + { + batch.Add(row); + if (batch.Count >= RestoreInsertBatchSize) + { + inserted += await InsertBatchAsync(targetTableName, archiveSchema.Columns, batch, ct); + batch.Clear(); + } + } + + if (batch.Count > 0) + inserted += await InsertBatchAsync(targetTableName, archiveSchema.Columns, batch, ct); + + return new RestoreTableResult + { + TableName = targetTableName, + RowsInserted = inserted, + }; + } + + private async IAsyncEnumerable EnumerateRowsAsync( + ClientTableSchema schema, + string path, + IProgress? progress, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct) + { + int page = 1; + int totalRows = int.MaxValue; + int seen = 0; + var interval = Stopwatch.StartNew(); + + while (seen < totalRows) + { + ct.ThrowIfCancellationRequested(); + var result = await client.BrowseTableAsync(schema.TableName, page, ExportPageSize, ct); + totalRows = result.TotalRows; + if (result.Rows.Count == 0) + yield break; + + foreach (object?[] row in result.Rows) + { + ct.ThrowIfCancellationRequested(); + yield return MapRow(schema, row); + seen++; + } + + if (seen >= totalRows || interval.ElapsedMilliseconds >= 500) + { + ReportExportProgress( + progress, + schema.TableName, + "Exporting", + "Writing table archive", + seen, + totalRows, + path); + interval.Restart(); + await Task.Yield(); + } + + page++; + } + + ReportExportProgress( + progress, + schema.TableName, + "Exporting", + "Writing table archive", + seen, + totalRows == int.MaxValue ? null : totalRows, + path); + } + + private static void ReportExportProgress( + IProgress? progress, + string tableName, + string stage, + string message, + long rowsProcessed, + long? totalRows, + string? path) + { + progress?.Report(new TableExportProgress + { + Operation = "Export table", + Stage = stage, + Message = message, + TableName = tableName, + Path = path, + RowsProcessed = rowsProcessed, + TotalRows = totalRows, + }); + } + + private async Task InsertBatchAsync( + string tableName, + IReadOnlyList columns, + IReadOnlyList rows, + CancellationToken ct) + { + if (rows.Count == 0) + return 0; + + var sql = new StringBuilder(); + sql.Append("INSERT INTO ").Append(tableName).Append(" ("); + for (int i = 0; i < columns.Count; i++) + { + if (i > 0) + sql.Append(", "); + sql.Append(columns[i].Name); + } + + sql.Append(") VALUES "); + for (int rowIndex = 0; rowIndex < rows.Count; rowIndex++) + { + if (rowIndex > 0) + sql.Append(", "); + sql.Append('('); + for (int columnIndex = 0; columnIndex < columns.Count; columnIndex++) + { + if (columnIndex > 0) + sql.Append(", "); + + PrimitiveDbValue value = columnIndex < rows[rowIndex].Length + ? rows[rowIndex][columnIndex] + : PrimitiveDbValue.Null; + sql.Append(FormatLiteral(value, columns[columnIndex].Type)); + } + + sql.Append(')'); + } + + sql.Append(';'); + var result = await client.ExecuteSqlAsync(sql.ToString(), ct); + if (!string.IsNullOrWhiteSpace(result.Error)) + throw new InvalidOperationException(result.Error); + + return result.RowsAffected; + } + + private static PrimitiveTableSchema MapSchema(ClientTableSchema schema) => new() + { + TableName = schema.TableName, + Columns = schema.Columns.Select(MapColumn).ToArray(), + ForeignKeys = schema.ForeignKeys.Select(MapForeignKey).ToArray(), + NextRowId = 1, + }; + + private static PrimitiveColumnDefinition MapColumn(ClientColumnDefinition column) => new() + { + Name = column.Name, + Type = column.Type switch + { + ClientDbType.Integer => PrimitiveDbType.Integer, + ClientDbType.Real => PrimitiveDbType.Real, + ClientDbType.Text => PrimitiveDbType.Text, + ClientDbType.Blob => PrimitiveDbType.Blob, + _ => throw new InvalidOperationException($"Unsupported column type '{column.Type}'."), + }, + Nullable = column.Nullable, + IsPrimaryKey = column.IsPrimaryKey, + IsIdentity = column.IsIdentity, + Collation = column.Collation, + }; + + private static PrimitiveForeignKeyDefinition MapForeignKey(ClientForeignKeyDefinition foreignKey) => new() + { + ConstraintName = foreignKey.ConstraintName, + ColumnName = foreignKey.ColumnName, + ReferencedTableName = foreignKey.ReferencedTableName, + ReferencedColumnName = foreignKey.ReferencedColumnName, + OnDelete = foreignKey.OnDelete == ClientForeignKeyOnDeleteAction.Cascade + ? PrimitiveForeignKeyOnDeleteAction.Cascade + : PrimitiveForeignKeyOnDeleteAction.Restrict, + SupportingIndexName = foreignKey.SupportingIndexName, + }; + + private static PrimitiveDbValue[] MapRow(ClientTableSchema schema, object?[] row) + { + var values = new PrimitiveDbValue[schema.Columns.Count]; + for (int i = 0; i < values.Length; i++) + { + object? value = i < row.Length ? row[i] : null; + values[i] = MapValue(schema.Columns[i].Type, value); + } + + return values; + } + + private static PrimitiveDbValue MapValue(ClientDbType columnType, object? value) + { + if (value is null) + return PrimitiveDbValue.Null; + + return columnType switch + { + ClientDbType.Integer => PrimitiveDbValue.FromInteger(Convert.ToInt64(value, CultureInfo.InvariantCulture)), + ClientDbType.Real => PrimitiveDbValue.FromReal(Convert.ToDouble(value, CultureInfo.InvariantCulture)), + ClientDbType.Text => PrimitiveDbValue.FromText(Convert.ToString(value, CultureInfo.InvariantCulture) ?? string.Empty), + ClientDbType.Blob => PrimitiveDbValue.FromBlob(ConvertToBytes(value)), + _ => throw new InvalidOperationException($"Unsupported column type '{columnType}'."), + }; + } + + private static byte[] ConvertToBytes(object value) + { + if (value is byte[] bytes) + return bytes; + + if (value is IEnumerable byteEnumerable) + return byteEnumerable.ToArray(); + + if (value is string text) + return Convert.FromBase64String(text); + + if (value is IEnumerable enumerable) + return enumerable.Cast().Select(item => Convert.ToByte(item, CultureInfo.InvariantCulture)).ToArray(); + + throw new InvalidOperationException($"Cannot convert value of type '{value.GetType().Name}' to BLOB."); + } + + private static ExternalTableRegistrationInfo MapExternalTableRegistration(object?[] row) + { + string createdText = row.Length > 4 ? Convert.ToString(row[4], CultureInfo.InvariantCulture) ?? string.Empty : string.Empty; + DateTimeOffset? createdUtc = DateTimeOffset.TryParse( + createdText, + CultureInfo.InvariantCulture, + DateTimeStyles.RoundtripKind, + out var parsed) + ? parsed + : null; + + return new ExternalTableRegistrationInfo + { + TableName = row.Length > 0 ? Convert.ToString(row[0], CultureInfo.InvariantCulture) ?? string.Empty : string.Empty, + Path = row.Length > 1 ? Convert.ToString(row[1], CultureInfo.InvariantCulture) ?? string.Empty : string.Empty, + SourceTableName = row.Length > 2 ? Convert.ToString(row[2], CultureInfo.InvariantCulture) : null, + RowCount = row.Length > 3 && row[3] is not null + ? Convert.ToInt64(row[3], CultureInfo.InvariantCulture) + : 0, + CreatedUtc = createdUtc, + }; + } + + private string ResolveArchivePath(string archivePath) + { + string trimmed = archivePath.Trim(); + if (Path.IsPathFullyQualified(trimmed)) + return trimmed; + + return Path.GetFullPath(Path.Combine(ResolveDatabaseFolder(client.DataSource), trimmed)); + } + + private static string BuildCreateTableSql(PrimitiveTableSchema schema) + { + var sql = new StringBuilder(); + sql.Append("CREATE TABLE ").Append(RequireIdentifier(schema.TableName, nameof(schema.TableName))).Append(" ("); + for (int i = 0; i < schema.Columns.Count; i++) + { + PrimitiveColumnDefinition column = schema.Columns[i]; + if (i > 0) + sql.Append(", "); + + sql.Append(RequireIdentifier(column.Name, nameof(column.Name))) + .Append(' ') + .Append(column.Type.ToString().ToUpperInvariant()); + + if (column.IsPrimaryKey) + sql.Append(" PRIMARY KEY"); + if (column.IsIdentity) + sql.Append(" IDENTITY"); + if (!column.Nullable && !column.IsPrimaryKey) + sql.Append(" NOT NULL"); + if (!string.IsNullOrWhiteSpace(column.Collation)) + sql.Append(" COLLATE ").Append(RequireIdentifier(column.Collation, nameof(column.Collation))); + } + + sql.Append(");"); + return sql.ToString(); + } + + private static string FormatLiteral(PrimitiveDbValue value, PrimitiveDbType columnType) + { + if (value.IsNull) + return "NULL"; + + return columnType switch + { + PrimitiveDbType.Integer => value.AsInteger.ToString(CultureInfo.InvariantCulture), + PrimitiveDbType.Real => value.AsReal.ToString("R", CultureInfo.InvariantCulture), + PrimitiveDbType.Blob => "X'" + Convert.ToHexString(value.AsBlob) + "'", + _ => FormatStringLiteral(value.Type == PrimitiveDbType.Text ? value.AsText : value.ToString()), + }; + } + + private static string FormatStringLiteral(string value) => "'" + value.Replace("'", "''", StringComparison.Ordinal) + "'"; + + private async Task ExecuteCheckedAsync(string sql, CancellationToken ct) + { + var result = await client.ExecuteSqlAsync(sql, ct); + if (!string.IsNullOrWhiteSpace(result.Error)) + throw new InvalidOperationException(result.Error); + } + + private static string RequireIdentifier(string value, string parameterName) + { + if (string.IsNullOrWhiteSpace(value)) + throw new ArgumentException("Identifier is required.", parameterName); + + string trimmed = value.Trim(); + if (!IsIdentifier(trimmed)) + throw new ArgumentException($"'{trimmed}' is not a valid CSharpDB identifier.", parameterName); + + return trimmed; + } + + private static bool IsIdentifier(string value) + { + if (value.Length == 0 || !(char.IsLetter(value[0]) || value[0] == '_')) + return false; + + for (int i = 1; i < value.Length; i++) + { + char c = value[i]; + if (!char.IsLetterOrDigit(c) && c != '_') + return false; + } + + return true; + } + + private static string ResolveDatabaseFolder(string dataSource) + { + if (string.IsNullOrWhiteSpace(dataSource) || + dataSource.StartsWith(":memory:", StringComparison.OrdinalIgnoreCase) || + (Uri.TryCreate(dataSource, UriKind.Absolute, out var uri) && !uri.IsFile)) + { + return Directory.GetCurrentDirectory(); + } + + string path = Uri.TryCreate(dataSource, UriKind.Absolute, out var fileUri) && fileUri.IsFile + ? fileUri.LocalPath + : Path.GetFullPath(dataSource); + string? directory = Path.GetDirectoryName(path); + return string.IsNullOrWhiteSpace(directory) ? Directory.GetCurrentDirectory() : directory; + } + + private static string CreateTemporaryArchivePath(string tableName) + { + string fileName = $"{SanitizeFileName(tableName)}-{DateTime.Now:yyyyMMdd-HHmmss}-{Guid.NewGuid():N}.csdbtable"; + string directory = Path.Combine(Path.GetTempPath(), "csharpdb-admin-exports"); + Directory.CreateDirectory(directory); + return Path.Combine(directory, fileName); + } + + private static string SanitizeFileName(string value) + { + char[] invalid = Path.GetInvalidFileNameChars(); + var builder = new StringBuilder(value.Length); + foreach (char c in value) + builder.Append(invalid.Contains(c) ? '_' : c); + return builder.Length == 0 ? "table" : builder.ToString(); + } +} diff --git a/src/CSharpDB.Admin.ImportExport/_Imports.razor b/src/CSharpDB.Admin.ImportExport/_Imports.razor new file mode 100644 index 00000000..45e9ef65 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/_Imports.razor @@ -0,0 +1,6 @@ +@using Microsoft.AspNetCore.Components +@using Microsoft.AspNetCore.Components.Web +@using CSharpDB.Admin.ImportExport.Contracts +@using CSharpDB.Admin.ImportExport.Components +@using CSharpDB.Admin.ImportExport.Services +@using CSharpDB.Client diff --git a/src/CSharpDB.Admin/CSharpDB.Admin.csproj b/src/CSharpDB.Admin/CSharpDB.Admin.csproj index bd5e62d0..a8efc399 100644 --- a/src/CSharpDB.Admin/CSharpDB.Admin.csproj +++ b/src/CSharpDB.Admin/CSharpDB.Admin.csproj @@ -13,6 +13,7 @@ + diff --git a/src/CSharpDB.Admin/Components/Layout/CommandPalette.razor b/src/CSharpDB.Admin/Components/Layout/CommandPalette.razor index 3423336d..a8842bc8 100644 --- a/src/CSharpDB.Admin/Components/Layout/CommandPalette.razor +++ b/src/CSharpDB.Admin/Components/Layout/CommandPalette.razor @@ -223,6 +223,7 @@ _items.Add(new PaletteItem("Command", "New Procedure", "Open procedure editor", "bi-gear-wide-connected", "icon-trigger", () => { TabManager.OpenNewProcedureTab(); return Task.CompletedTask; })); _items.Add(new PaletteItem("Command", "New Pipeline", "Open pipeline builder", "bi-diagram-3", "icon-view", () => { TabManager.OpenPipelineTab(); return Task.CompletedTask; })); _items.Add(new PaletteItem("Command", "Code Modules", "Open C# code modules", "bi-filetype-cs", "icon-system", () => { TabManager.OpenCodeModulesTab(); return Task.CompletedTask; })); + _items.Add(new PaletteItem("Command", "Import / Export", "Open table archive tools", "bi-arrow-left-right", "icon-system", () => { TabManager.OpenImportExportTab(); return Task.CompletedTask; })); _items.Add(new PaletteItem("Command", "Storage Inspector", "Open storage diagnostics", "bi-hdd-stack", "icon-system", () => { TabManager.OpenStorageTab(); return Task.CompletedTask; })); } diff --git a/src/CSharpDB.Admin/Components/Layout/MainLayout.razor b/src/CSharpDB.Admin/Components/Layout/MainLayout.razor index 3c02b127..68a438d5 100644 --- a/src/CSharpDB.Admin/Components/Layout/MainLayout.razor +++ b/src/CSharpDB.Admin/Components/Layout/MainLayout.razor @@ -86,6 +86,10 @@ break; + case TabKind.ImportExport: + + break; } } diff --git a/src/CSharpDB.Admin/Components/Layout/NavMenu.razor b/src/CSharpDB.Admin/Components/Layout/NavMenu.razor index 5ebae4d0..004efc46 100644 --- a/src/CSharpDB.Admin/Components/Layout/NavMenu.razor +++ b/src/CSharpDB.Admin/Components/Layout/NavMenu.razor @@ -38,6 +38,7 @@
+ @@ -47,6 +48,22 @@
@if (ShouldShowUtilityGroups) { +
+
+ + Tools + 1 +
+
+ +
+
+
@@ -206,6 +223,39 @@
} + @* External Tables *@ + @if (ShouldShowGroup("externalTables")) + { +
+
+ + + External Tables + @(_externalTables?.Count ?? 0) +
+ @if (_expandedGroups.Contains("externalTables") && _externalTables is not null) + { +
+ @foreach (var item in FilterExternalTables()) + { + var externalTable = item; +
+ + @externalTable.TableName + @FormatExternalTableHint(externalTable) +
+ } +
+ } +
+ } + @* Collections *@ @if (ShouldShowGroup("collections")) { @@ -639,6 +689,12 @@ private sealed record SystemCatalogItem(string Name, string Description, string Sql); + private sealed record ExternalTableExplorerItem( + string TableName, + string Path, + string SourceTableName, + long RowCount); + private static readonly IReadOnlyList s_systemCatalogItems = new[] { new SystemCatalogItem("sys.tables", "tables", "SELECT * FROM sys.tables ORDER BY table_name;"), @@ -653,10 +709,12 @@ new SystemCatalogItem("sys.triggers", "triggers", "SELECT * FROM sys.triggers ORDER BY trigger_name;"), new SystemCatalogItem("sys.objects", "all objects", "SELECT * FROM sys.objects ORDER BY object_type, object_name;"), new SystemCatalogItem("sys.saved_queries", "saved queries", "SELECT * FROM sys.saved_queries ORDER BY name;"), + new SystemCatalogItem("sys.external_tables", "external tables", "SELECT * FROM sys.external_tables ORDER BY table_name;"), }; private IReadOnlyCollection? _userTableNames; private IReadOnlyCollection? _systemTableNames; + private IReadOnlyList? _externalTables; private IReadOnlyCollection? _viewNames; private IReadOnlyCollection? _collectionNames; private IReadOnlyList? _indexes; @@ -667,7 +725,7 @@ private IReadOnlyList? _reports; private string _searchQuery = string.Empty; private string _objectFilter = "all"; - private readonly HashSet _expandedGroups = new() { "tables", "collections", "forms", "reports", "system", "views", "procedures", "callbacks" }; + private readonly HashSet _expandedGroups = new() { "tables", "externalTables", "collections", "forms", "reports", "system", "views", "procedures", "callbacks" }; private readonly HashSet _expandedTableNodes = new(StringComparer.OrdinalIgnoreCase); private readonly Dictionary _tableSchemas = new(StringComparer.OrdinalIgnoreCase); private readonly HashSet _tableSchemaLoadFailures = new(StringComparer.OrdinalIgnoreCase); @@ -749,6 +807,7 @@ var allTables = (await DbClient.GetTableNamesAsync()).OrderBy(n => n).ToList(); _userTableNames = allTables.Where(static n => !IsSystemTableName(n)).ToList(); _systemTableNames = allTables.Where(IsSystemTableName).ToList(); + _externalTables = await LoadExternalTablesAsync(); _viewNames = (await DbClient.GetViewNamesAsync()).OrderBy(n => n).ToList(); _collectionNames = (await DbClient.GetCollectionNamesAsync()).OrderBy(n => n, StringComparer.OrdinalIgnoreCase).ToList(); _indexes = (await DbClient.GetIndexesAsync()).OrderBy(i => i.IndexName).ToList(); @@ -767,6 +826,7 @@ { _userTableNames = null; _systemTableNames = null; + _externalTables = null; _viewNames = null; _collectionNames = null; _indexes = null; @@ -779,6 +839,39 @@ } } + private async Task> LoadExternalTablesAsync() + { + var result = await DbClient.ExecuteSqlAsync(""" + SELECT table_name, path, source_table_name, row_count + FROM sys.external_tables + ORDER BY table_name; + """); + + if (result.Error is not null) + return Array.Empty(); + + var items = new List(); + foreach (object?[] row in result.Rows ?? []) + { + if (row.Length < 4 || row[0] is null) + continue; + + string tableName = Convert.ToString(row[0], System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty; + if (string.IsNullOrWhiteSpace(tableName)) + continue; + + string path = Convert.ToString(row[1], System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty; + string sourceTableName = Convert.ToString(row[2], System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty; + long rowCount = row[3] is null + ? 0 + : Convert.ToInt64(row[3], System.Globalization.CultureInfo.InvariantCulture); + + items.Add(new ExternalTableExplorerItem(tableName, path, sourceTableName, rowCount)); + } + + return items; + } + private void ToggleGroup(string group) { if (!_expandedGroups.Remove(group)) @@ -813,7 +906,8 @@ => _objectFilter switch { "all" => true, - "tables" => group is "tables" or "views", + "tables" => group is "tables" or "externalTables" or "views", + "externalTables" => group == "externalTables", "collections" => group == "collections", "callbacks" => group == "callbacks", "forms" => group == "forms", @@ -836,6 +930,28 @@ return FilterItems(_collectionNames); } + private IEnumerable FilterExternalTables() + { + if (_externalTables is null) + return Enumerable.Empty(); + + if (string.IsNullOrWhiteSpace(_searchQuery)) + return _externalTables; + + return _externalTables.Where(item => + item.TableName.Contains(_searchQuery, StringComparison.OrdinalIgnoreCase) || + item.SourceTableName.Contains(_searchQuery, StringComparison.OrdinalIgnoreCase) || + item.Path.Contains(_searchQuery, StringComparison.OrdinalIgnoreCase)); + } + + private static string FormatExternalTableHint(ExternalTableExplorerItem table) + { + string source = string.IsNullOrWhiteSpace(table.SourceTableName) + ? "archive" + : table.SourceTableName; + return $"{source} · {table.RowCount:N0} rows"; + } + private IEnumerable FilterTableNames(IReadOnlyCollection tableNames) { if (string.IsNullOrWhiteSpace(_searchQuery)) @@ -1087,6 +1203,12 @@ private void OpenSystemCatalogTab(SystemCatalogItem item) => TabManager.OpenSystemCatalogTab(item.Name, item.Sql); + private void OpenExternalTable(string tableName) + => TabManager.OpenQueryTab($"SELECT * FROM {RequireSqlIdentifier(tableName)} LIMIT 50;"); + + private void OpenExternalTableMetadata(string tableName) + => TabManager.OpenQueryTab($"SELECT * FROM sys.external_tables WHERE table_name = {FormatSqlStringLiteral(tableName)};"); + private void OpenInternalFunction(FormulaFunctionDescriptor function) { TabDescriptor tab = TabManager.OpenCallbacksTab(); @@ -1140,6 +1262,17 @@ return activeTab.Id == $"{type}:{name}"; } + private bool IsActiveExternalTable(string tableName) + { + var activeTab = TabManager.ActiveTab; + return activeTab?.Kind == TabKind.Query + && activeTab.SqlText is { } sqlText + && sqlText.Contains(tableName, StringComparison.OrdinalIgnoreCase); + } + + private bool IsActiveImportExport() + => TabManager.ActiveTab?.Kind == TabKind.ImportExport; + private bool IsActiveSchema(string type, string name) { var activeTab = TabManager.ActiveTab; @@ -1243,6 +1376,17 @@ }); } + private void ShowExternalTablesGroupMenu(MouseEventArgs e) + { + ShowContextMenu(e, new List + { + new() { Label = "Register External Table...", Icon = "bi-box-arrow-up-right", OnClick = () => TabManager.OpenImportExportTab() }, + new() { Label = "View Metadata", Icon = "bi-hdd-stack", OnClick = () => OpenSystemCatalogTab(s_systemCatalogItems.First(item => item.Name == "sys.external_tables")) }, + ContextMenuItem.Separator(), + new() { Label = "Refresh", Icon = "bi-arrow-clockwise", OnClick = async () => { await LoadObjects(); await InvokeAsync(StateHasChanged); } }, + }); + } + private void ShowCollectionsGroupMenu(MouseEventArgs e) { ShowContextMenu(e, new List @@ -1274,6 +1418,7 @@ new() { Label = "Design", Icon = "bi-file-earmark-code", OnClick = () => TabManager.OpenTableSchemaTab(tableName) }, new() { Label = "Design Form...", Icon = "bi-ui-checks-grid", OnClick = () => TabManager.OpenFormDesignerTab(initialTableName: tableName, title: $"{tableName} Form") }, new() { Label = "Design Report...", Icon = "bi-file-earmark-richtext", OnClick = () => TabManager.OpenReportDesignerTab(sourceKind: ReportSourceKind.Table.ToString(), sourceName: tableName, title: $"{tableName} Report") }, + new() { Label = "Import / Export...", Icon = "bi-arrow-left-right", OnClick = () => TabManager.OpenImportExportTab(tableName) }, ContextMenuItem.Separator(), new() { Label = "Select Top 50", Icon = "bi-terminal", OnClick = () => TabManager.OpenQueryTab($"SELECT * FROM {tableName} LIMIT 50;") }, new() { Label = "Select Count", Icon = "bi-123", OnClick = () => TabManager.OpenQueryTab($"SELECT COUNT(*) AS row_count FROM {tableName};") }, @@ -1282,6 +1427,23 @@ }); } + private void ShowExternalTableItemMenu(MouseEventArgs e, ExternalTableExplorerItem table) + { + ShowContextMenu(e, new List + { + new() { Label = "Open", Icon = "bi-box-arrow-up-right", OnClick = () => OpenExternalTable(table.TableName) }, + new() { Label = "Open Metadata", Icon = "bi-hdd-stack", OnClick = () => OpenExternalTableMetadata(table.TableName) }, + ContextMenuItem.Separator(), + new() { Label = "Select Top 50", Icon = "bi-terminal", OnClick = () => TabManager.OpenQueryTab($"SELECT * FROM {RequireSqlIdentifier(table.TableName)} LIMIT 50;") }, + new() { Label = "Select Count", Icon = "bi-123", OnClick = () => TabManager.OpenQueryTab($"SELECT COUNT(*) AS row_count FROM {RequireSqlIdentifier(table.TableName)};") }, + ContextMenuItem.Separator(), + new() { Label = "Copy Name", Icon = "bi-clipboard", OnClick = async () => await CopyExternalTableTextAsync(table.TableName) }, + new() { Label = "Copy Path", Icon = "bi-clipboard-plus", OnClick = async () => await CopyExternalTableTextAsync(table.Path) }, + ContextMenuItem.Separator(), + new() { Label = "Drop External Table", Icon = "bi-trash3", IsDanger = true, OnClick = async () => await ConfirmDropExternalTableAsync(table.TableName) }, + }); + } + private void ShowFormsGroupMenu(MouseEventArgs e) { ShowContextMenu(e, new List @@ -1484,6 +1646,19 @@ } } + private async Task CopyExternalTableTextAsync(string text) + { + try + { + await JS.InvokeVoidAsync("clipboardInterop.writeText", text); + Toast.Success("Copied external table value."); + } + catch (Exception ex) + { + Toast.Error(ex.Message); + } + } + private async Task ConfirmDropTableAsync(string tableName) { var confirmed = await Modal.ConfirmAsync("Drop Table", $"Are you sure you want to drop table '{tableName}'? This action cannot be undone.", "Drop", isDanger: true); @@ -1498,6 +1673,32 @@ catch (Exception ex) { Toast.Error(ex.Message); } } + private async Task ConfirmDropExternalTableAsync(string tableName) + { + var confirmed = await Modal.ConfirmAsync( + "Drop External Table", + $"Are you sure you want to drop external table '{tableName}'? This removes the registration only; the archive file is not deleted.", + "Drop", + isDanger: true); + if (!confirmed) return; + + try + { + var result = await DbClient.ExecuteSqlAsync($"DROP EXTERNAL TABLE IF EXISTS {RequireSqlIdentifier(tableName)};"); + if (!string.IsNullOrWhiteSpace(result.Error)) + throw new InvalidOperationException(result.Error); + + Toast.Success($"Dropped external table '{tableName}'."); + Changes.NotifyChanged(); + await LoadObjects(); + await InvokeAsync(StateHasChanged); + } + catch (Exception ex) + { + Toast.Error(ex.Message); + } + } + private async Task ConfirmDropCollectionAsync(string collectionName) { var confirmed = await Modal.ConfirmAsync("Drop Collection", $"Are you sure you want to drop collection '{collectionName}'? All documents and collection indexes will be deleted. This action cannot be undone.", "Drop", isDanger: true); @@ -1609,6 +1810,36 @@ TabManager.StateChanged -= OnTabChanged; } + private static string RequireSqlIdentifier(string value) + { + if (string.IsNullOrWhiteSpace(value)) + throw new InvalidOperationException("Identifier is required."); + + string trimmed = value.Trim(); + if (!IsSqlIdentifier(trimmed)) + throw new InvalidOperationException($"'{trimmed}' is not a valid CSharpDB identifier."); + + return trimmed; + } + + private static bool IsSqlIdentifier(string value) + { + if (value.Length == 0 || !(char.IsLetter(value[0]) || value[0] == '_')) + return false; + + for (int i = 1; i < value.Length; i++) + { + char c = value[i]; + if (!char.IsLetterOrDigit(c) && c != '_') + return false; + } + + return true; + } + + private static string FormatSqlStringLiteral(string value) + => "'" + value.Replace("'", "''", StringComparison.Ordinal) + "'"; + private static bool IsSystemTableName(string name) => name.StartsWith("_", StringComparison.Ordinal); } diff --git a/src/CSharpDB.Admin/Components/Shared/DataGrid.razor b/src/CSharpDB.Admin/Components/Shared/DataGrid.razor index 2f65746b..2dd63353 100644 --- a/src/CSharpDB.Admin/Components/Shared/DataGrid.razor +++ b/src/CSharpDB.Admin/Components/Shared/DataGrid.razor @@ -13,8 +13,11 @@
@if (_loading) { -
+
+
} else if (_loadError is not null) @@ -232,6 +235,7 @@ private readonly List _viewCursorLookaheadRows = new(); private int _viewCursorNextPage; private bool _bypassViewCursorForNextLoad; + private CancellationTokenSource? _loadCts; private int TotalPages => _hasExactTotal ? Math.Max(1, (int)Math.Ceiling(_totalRows / (double)_pageSize)) @@ -282,47 +286,88 @@ public async Task LoadDataAsync() { + _loadCts?.Cancel(); + var loadCts = new CancellationTokenSource(); + _loadCts = loadCts; + CancellationToken ct = loadCts.Token; + _loading = true; _loadError = null; _editingCell = null; _selectedRows.Clear(); await NotifySelectionChangedAsync(); StateHasChanged(); + await Task.Yield(); try { if (ShouldLoadAllRows()) { await DisposeViewCursorAsync(); - await LoadAllRowsAsync(); + await LoadAllRowsAsync(ct); } else - await LoadCurrentPageAsync(); + await LoadCurrentPageAsync(ct); ApplyFiltersAndSort(); } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + await DisposeViewCursorAsync(); + if (ReferenceEquals(_loadCts, loadCts)) + _loadError = "Query canceled."; + } catch (Exception ex) { await DisposeViewCursorAsync(); - _allRows = new(); - _rows = new(); - _totalRows = 0; - _hasExactTotal = true; - _hasNextPage = false; - _loadError = ex.Message; - - if (QuerySql is null) - Toast.Error($"Failed to load data: {ex.Message}"); + if (ReferenceEquals(_loadCts, loadCts)) + { + _allRows = new(); + _rows = new(); + _totalRows = 0; + _hasExactTotal = true; + _hasNextPage = false; + _loadError = ex.Message; + + if (QuerySql is null) + Toast.Error($"Failed to load data: {ex.Message}"); + } } finally { - _loading = false; - await NotifyQueryStatusChangedAsync(); - await NotifyFiltersChangedAsync(); - StateHasChanged(); + bool isCurrentLoad = ReferenceEquals(_loadCts, loadCts); + if (isCurrentLoad) + { + _loadCts = null; + _loading = false; + } + + loadCts.Dispose(); + if (isCurrentLoad) + { + await NotifyQueryStatusChangedAsync(); + await NotifyFiltersChangedAsync(); + StateHasChanged(); + } } } + private void CancelLoad() + { + _loadCts?.Cancel(); + _loadError ??= "Canceling query..."; + StateHasChanged(); + } + + private Task ExecuteSqlAsync(string sql, CancellationToken ct) + => Task.Run(() => DbClient.ExecuteSqlAsync(sql, ct), ct); + + private Task GetTableSchemaAsync(string tableName, CancellationToken ct) + => Task.Run(() => DbClient.GetTableSchemaAsync(tableName, ct), ct); + + private Task TryOpenForwardOnlyQueryCursorAsync(string sql, CancellationToken ct) + => Task.Run(async () => await DbHolder.TryOpenForwardOnlyQueryCursorAsync(sql, ct), ct); + // ─── Sorting ──────────────────── private async Task SortColumn(int colIdx) { @@ -950,7 +995,7 @@ private bool ShouldLoadAllRows() => QuerySql is null && HasPendingChanges; - private async Task LoadCurrentPageAsync() + private async Task LoadCurrentPageAsync(CancellationToken ct) { _loadedAllRows = false; _lastQueryElapsed = null; @@ -959,18 +1004,18 @@ if (QuerySql is not null) { - await LoadQueryPageAsync(QuerySql); + await LoadQueryPageAsync(QuerySql, ct); return; } if (TableName is not null) { string tableName = RequireIdentifier(TableName); - var schema = await DbClient.GetTableSchemaAsync(tableName) + var schema = await GetTableSchemaAsync(tableName, ct) ?? throw new InvalidOperationException($"Table '{tableName}' was not found."); ApplyTableSchema(schema); - await LoadServerPageAsync(tableName); + await LoadServerPageAsync(tableName, ct); return; } @@ -978,7 +1023,7 @@ { string viewName = RequireIdentifier(ViewName); await DisposeViewCursorAsync(); - await LoadUnknownTotalSourcePageAsync(viewName, captureColumnNames: true); + await LoadUnknownTotalSourcePageAsync(viewName, captureColumnNames: true, ct: ct); Columns ??= Array.Empty(); PrimaryKeyColumn = null; @@ -990,9 +1035,9 @@ _totalRows = 0; } - private async Task LoadServerPageAsync(string sourceName) + private async Task LoadServerPageAsync(string sourceName, CancellationToken ct) { - var countResult = await DbClient.ExecuteSqlAsync(BuildCountSql(sourceName)); + var countResult = await ExecuteSqlAsync(BuildCountSql(sourceName), ct); if (countResult.Error is not null) throw new InvalidOperationException(countResult.Error); @@ -1000,7 +1045,7 @@ _hasExactTotal = true; ClampPageToAvailableRows(); - var queryResult = await DbClient.ExecuteSqlAsync(BuildSelectSql(sourceName)); + var queryResult = await ExecuteSqlAsync(BuildSelectSql(sourceName), ct); if (queryResult.Error is not null) throw new InvalidOperationException(queryResult.Error); @@ -1011,11 +1056,14 @@ Columns = queryResult.ColumnNames ?? Array.Empty(); } - private async Task LoadUnknownTotalSourcePageAsync(string sourceName, bool captureColumnNames = false) + private async Task LoadUnknownTotalSourcePageAsync( + string sourceName, + bool captureColumnNames = false, + CancellationToken ct = default) { while (true) { - var queryResult = await DbClient.ExecuteSqlAsync(BuildSelectSql(sourceName, _pageSize + 1)); + var queryResult = await ExecuteSqlAsync(BuildSelectSql(sourceName, _pageSize + 1), ct); if (queryResult.Error is not null) throw new InvalidOperationException(queryResult.Error); @@ -1037,7 +1085,7 @@ } } - private async Task TryLoadViewCursorPageAsync(string viewName) + private async Task TryLoadViewCursorPageAsync(string viewName, CancellationToken ct) { if (_bypassViewCursorForNextLoad) { @@ -1049,7 +1097,7 @@ if (_page == 1) { await DisposeViewCursorAsync(); - _viewCursor = await DbHolder.TryOpenForwardOnlyQueryCursorAsync(cursorSql); + _viewCursor = await TryOpenForwardOnlyQueryCursorAsync(cursorSql, ct); if (_viewCursor is null) return false; @@ -1068,11 +1116,11 @@ if (Columns is null || Columns.Length == 0) Columns = _viewCursor.ColumnNames; - await LoadViewCursorPageAsync(); + await LoadViewCursorPageAsync(ct); return true; } - private async Task LoadViewCursorPageAsync() + private async Task LoadViewCursorPageAsync(CancellationToken ct) { if (_viewCursor is null) throw new InvalidOperationException("View cursor is not available."); @@ -1086,7 +1134,7 @@ int remaining = (_pageSize + 1) - rows.Count; if (remaining > 0) - rows.AddRange(await _viewCursor.ReadNextAsync(remaining)); + rows.AddRange(await _viewCursor.ReadNextAsync(remaining, ct)); _hasNextPage = rows.Count > _pageSize; if (_hasNextPage) @@ -1103,7 +1151,7 @@ ApplyPageRows(rows, hasNextPage: _hasNextPage); } - private async Task LoadQueryPageAsync(string querySql) + private async Task LoadQueryPageAsync(string querySql, CancellationToken ct) { _queryPagingPlan ??= QueryPagingPlan.Parse(querySql); @@ -1114,8 +1162,9 @@ { string[] displayColumns = Columns ?? Array.Empty(); // Over-fetch one row so ad-hoc SQL paging can stay responsive without blocking on COUNT(*). - var pageResult = await DbClient.ExecuteSqlAsync( - _queryPagingPlan.BuildPageSql(_filters, _filterModes, _sortCol, _sortAsc, _pageSize + 1, _page, displayColumns)); + var pageResult = await ExecuteSqlAsync( + _queryPagingPlan.BuildPageSql(_filters, _filterModes, _sortCol, _sortAsc, _pageSize + 1, _page, displayColumns), + ct); if (pageResult.Error is not null) throw new InvalidOperationException(pageResult.Error); @@ -1144,7 +1193,7 @@ } } - private async Task LoadAllRowsAsync() + private async Task LoadAllRowsAsync(CancellationToken ct) { var pendingRows = _allRows .Where(row => row.State != RowState.Unmodified) @@ -1154,11 +1203,11 @@ if (TableName is not null) { string tableName = RequireIdentifier(TableName); - var schema = await DbClient.GetTableSchemaAsync(tableName) + var schema = await GetTableSchemaAsync(tableName, ct) ?? throw new InvalidOperationException($"Table '{tableName}' was not found."); ApplyTableSchema(schema); - var result = await DbClient.ExecuteSqlAsync($"SELECT * FROM {tableName}"); + var result = await ExecuteSqlAsync($"SELECT * FROM {tableName}", ct); if (result.Error is not null) throw new InvalidOperationException(result.Error); @@ -1167,7 +1216,7 @@ else if (ViewName is not null) { string viewName = RequireIdentifier(ViewName); - var result = await DbClient.ExecuteSqlAsync($"SELECT * FROM {viewName}"); + var result = await ExecuteSqlAsync($"SELECT * FROM {viewName}", ct); if (result.Error is not null) throw new InvalidOperationException(result.Error); @@ -1636,6 +1685,8 @@ public async ValueTask DisposeAsync() { + _loadCts?.Cancel(); + _loadCts?.Dispose(); await DisposeViewCursorAsync(); } } diff --git a/src/CSharpDB.Admin/Components/Tabs/QueryTab.razor b/src/CSharpDB.Admin/Components/Tabs/QueryTab.razor index f7db3340..41b90ec3 100644 --- a/src/CSharpDB.Admin/Components/Tabs/QueryTab.razor +++ b/src/CSharpDB.Admin/Components/Tabs/QueryTab.razor @@ -18,9 +18,15 @@
@if (_mode == QueryMode.Sql) { - + @if (_loading) + { + + }
+
+
+ + + + + + diff --git a/www/changelog.html b/www/changelog.html index dbad62ea..1350cb84 100644 --- a/www/changelog.html +++ b/www/changelog.html @@ -133,6 +133,74 @@

Changelog

Release announcements and project updates. For planned work, see the roadmap.

+
+
+ Release + May 2026 +
+

v3.8.0 — Native Table Archives & External Tables

+

Adds a real table-archive and external-table workflow: native .csdbtable files, fast snapshot exports, Admin Import / Export tooling, and read-only SQL access to archived tables:

+
    +
  • Native table archives — New CSharpDB.ImportExport project with native .csdbtable schema, manifest, row stream, BLOB/null/type metadata, and optional integer primary-key index sections.
    • +
  • Fast snapshot exports — Embedded client transport can export directly from a reader snapshot instead of paging through the UI, with progress callbacks and cancellation support for large tables.
    • +
  • Admin Import / Export — New Admin project area with export, register external table, restore table, one-time download endpoint, server-path exports, progress UI, cancellation, and Object Explorer access from both Tools and table context menus.
    • +
  • External table SQL — Added CREATE EXTERNAL TABLE ... FROM 'path.csdbtable' and DROP EXTERNAL TABLE, plus sys.external_tables metadata and relative-path resolution from the database folder.
    • +
  • Query planner integration — Registered archives are resolved as read-only table sources through ExternalTableScanOperator, can participate in normal SELECT queries and joins, use metadata fast paths for COUNT(*), and use embedded primary-key indexes for eligible point lookups.
    • +
  • Read-only safety — Writes, schema mutation, index creation, and trigger target usage against external tables are rejected with clear errors.
    • +
  • Changelog and blog docs — Added a table archives blog post and ignored generated *.csdbtable files so local exports do not become source artifacts.
    • +
+
+ +
+
+ Release + May 2026 +
+

v3.7.0 — Planner Observability, Adaptive Reoptimization & View Paging

+

Completes the optimizer and async I/O close-out work, exposes planner diagnostics publicly, and fixes the view/query paths found during the customer-filtered fulfillment investigation:

+
    +
  • Planner diagnostics — Added EXPLAIN ESTIMATE FOR SELECT coverage for select, WITH, and compound queries, plus sys.planner_* virtual catalogs for histograms, heavy hitters, and composite index prefix statistics.
    • +
  • Admin plan visibility — Query tabs can run estimates and show planner output in the Plan tab so users can inspect row estimates and plan choices without leaving Admin.
    • +
  • Opt-in adaptive reoptimization — New DatabaseOptions, embedded ADO.NET connection-string support, diagnostics, and fail-closed handling let eligible join plans correct bad cardinality assumptions before rows are emitted.
    • +
  • View/query fixes — Paged view browsing now issues bounded LIMIT/OFFSET SQL, simple view row-goal planning can reorder eligible join chains, and Query tab grids preserve scroll/pager layout.
    • +
  • Lookup join and result metadata polish — Improved indexed local predicate planning, residual right-side filters, index scan capacity hints, and SQL ColumnTypes propagation through engine, API, client, and Admin surfaces.
    • +
  • Benchmarks and DataGen — Closed the current generated collection fast-path and DataGen direct-load phase, refreshed release benchmark scorecards, added optimizer/async I/O close-out suites, and added a WAL point-read benchmark with latency-column guardrail support.
    • +
+
+ +
+
+ Release + May 2026 +
+

v3.6.0 — Trusted C# Callbacks, Built-In Functions & Admin Automation

+

Adds trusted in-process C# callbacks across SQL, formulas, procedures, forms, reports, and pipeline surfaces, then rounds the feature out with tableless scalar SQL, built-ins, and Admin discovery:

+
    +
  • Trusted scalar functions and commands — Host applications can register scalar functions through DbFunctionRegistry and commands through DbCommandRegistry, including async command support, timeouts, and shared argument conversion.
    • +
  • Automation surfaces — Admin Forms now support lifecycle events, selected-control events, command buttons, reusable action sequences, built-in record actions, conditional action steps, and generated automation metadata. Reports and pipelines gained trusted command hooks.
    • +
  • Tableless scalar SQL — Added SELECT without FROM using a single-row planner source for scalar expressions and callbacks that do not need row context.
    • +
  • Built-in scalar functions — Added common Access-style text, date/time, numeric, conversion, and null helpers for SQL and Admin formula use.
    • +
  • Callback catalog polish — Added callback metadata including CanRunWithoutFrom, SQL editor autocomplete for built-ins and tableless-safe callbacks, and an Internal/External split in the Admin callback catalog.
    • +
  • Release validation — Final tableless/callback stabilization passed with 1,877 tests, plus build, pack, daemon publish, and callback benchmark validation.
    • +
+
+ +
+
+ Release + April 2026 +
+

v3.5.0 — Generated Collection Payload Fast Path & Studio Mockups

+

Completes the opt-in generated collection binary payload fast path, refreshes release benchmark artifacts, and adds confirmed CSharpDB Studio Admin mockup documentation:

+
    +
  • Generated collection binary payloads — Source-generated collection models now encode/decode fixed field order, compact type/null metadata, and raw values while non-generated collection paths keep their existing JSON and binary behavior.
    • +
  • Direct payload decode improvements — Avoids duplicated direct payload header parsing, adds single-segment ReadOnlySpan<byte> top-level field lookups, and improves targeted UTF-8 span plumbing for text index/read/compare paths.
    • +
  • Collection benchmark coverage — Added generated collection codec benchmarks and expanded binary codec/generator tests. Focused investigation showed material gains in direct payload decode, field reads, field compares, and path-index lookups.
    • +
  • Release-core refresh — Refreshed benchmark README/history from the April 26 release-core artifacts; final guardrail compare passed with PASS=185, WARN=0, SKIP=0, FAIL=0.
    • +
  • Admin UI planning — Added CSharpDB Studio access-parity notes and static Admin UI mockups under www/admin-ui-mockups.
    • +
+
+
Release diff --git a/www/roadmap-reference.html b/www/roadmap-reference.html index a1b5c01a..3af1e8c6 100644 --- a/www/roadmap-reference.html +++ b/www/roadmap-reference.html @@ -64,7 +64,7 @@

On This Page

Source reference. This page preserves the original long-form markdown content that previously lived at docs/roadmap.md. For the shorter curated page, see Roadmap.

CSharpDB Roadmap

-

This document outlines the planned direction for CSharpDB, organized by timeframe and priority. Items are roughly ordered by expected impact within each tier, and statuses are intended to reflect the current v3.4.0 state of the repo.

+

This document outlines the planned direction for CSharpDB, organized by timeframe and priority. Items are roughly ordered by expected impact within each tier, and statuses are intended to reflect the current v3.8.0 state of the repo.

Near-Term

Recently completed improvements to query performance, storage/runtime behavior, maintenance workflows, and developer ergonomics.

@@ -188,6 +188,11 @@

Near-Term

Done + Native table archives and external tables + Native .csdbtable table snapshots with fast Admin Import / Export, download or server-path destinations, CREATE EXTERNAL TABLE / DROP EXTERNAL TABLE, sys.external_tables, read-only external table scans/joins, and embedded primary-key archive lookup indexes for eligible point reads + Done + + Older DB foreign-key retrofit migration Validate/apply maintenance workflow that rewrites existing child tables with persisted FK metadata across direct, HTTP, gRPC, CLI, and Admin Done @@ -207,8 +212,13 @@

Mid-Term

- User-defined functions - Broader built-in scalar function registry (UPPER, ABS, COALESCE, etc.), user-registered C# functions, native plugin extensions + User-defined functions and commands + Done for the trusted in-process model: host-registered C# scalar functions, common SQL/Admin built-ins, trusted commands, Admin Forms/Reports/pipeline hooks, declarative Admin Forms action sequences, and local Admin Forms C# code modules are implemented across the supported surfaces. Untrusted sandboxed UDF execution is intentionally out of scope + Done + + + Writable external tables + Planned opt-in writable external table registrations over mutable .csdbx files, backed by CSharpDB B+tree storage and limited to DML (INSERT, UPDATE, DELETE) in v1 while .csdbtable archives remain read-only Planned @@ -247,8 +257,13 @@

Mid-Term

Done - Remote host security - Add built-in authentication, authorization, and transport-security options for remote HTTP and gRPC access, including API keys, protected admin endpoints, and TLS/mTLS deployment support + Remote API-key protection + Opt-in API-key mode protects REST /api/* and daemon gRPC calls with constant-time key comparison while keeping default no-auth behavior for compatibility + Done + + + Remote host security hardening + Add authorization, protected admin endpoint scopes, JWT/RBAC options, and TLS/mTLS deployment helpers for remote HTTP and gRPC access Planned @@ -277,6 +292,16 @@

Mid-Term

Done + Admin Forms Access parity + Close the highest-impact Access-style form gaps: runtime responsive layouts, full inferred validation enforcement, richer record-source/filter/sort models, Layout View, form modes, broader action/event coverage, and broader control coverage; trusted command-backed form lifecycle events, command buttons, and selected control events are now started + Partial + + + Admin Reports Access parity + Close the highest-impact Access-style report gaps: bounded saved-query previews, full report rendering/export, parameter/filter prompts, richer grouping/totals options, Layout View, conditional formatting, subreports, and broader report controls; trusted command-backed report preview lifecycle events are now started + Partial + + Visual query designer Classic Admin query builder with source canvas, join editing, design grid, SQL preview, and saved designer layouts Done @@ -336,6 +361,11 @@

Long-Term

Done + External table index coverage + Follow writable .csdbx storage with broader external-table indexes, planner costing, and multi-column lookup/range support beyond the current archive primary-key point-lookup path + Planned + + Source-generated collection fast path Done for the current phase: opt-in generated collection models now provide GetGeneratedCollectionAsync<T>(...), generated field descriptors/index bindings, analyzer-packaged collection model/codecs, generated binary direct-payload encode/decode for supported document graphs, source-generated JSON fallback for unsupported shapes, trim/NativeAOT smoke coverage, and a dedicated sample Done @@ -367,13 +397,13 @@

Long-Term

Adaptive query re-optimization - Re-plan or adapt at runtime when observed cardinality diverges materially from persisted statistics, especially after stale stats or parameter-sensitive predicates - Planned + Done for the current phase: opt-in adaptive join execution can switch eligible index nested-loop joins to hash joins and can flip inner hash build sides at safe pre-emission boundaries when observed rows diverge materially from estimates. Broader EXPLAIN ANALYZE, runtime actual-row reporting, adaptive stats persistence, and arbitrary mid-plan reordering remain future work + Done Public planner histogram inspection - Expose histogram, heavy-hitter, composite-prefix, and estimate explanation diagnostics through a public surface; current histogram/prefix stats remain internal implementation details - Planned + Stable SQL-first diagnostics now expose sys.planner_histograms, sys.planner_heavy_hitters, sys.planner_index_prefix_stats, and EXPLAIN ESTIMATE FOR <query> while keeping raw histogram/prefix storage payloads internal + Done Async I/O batching @@ -397,8 +427,8 @@

Long-Term

Broader multi-writer insert optimization - Improve hot insert fan-in, row-id reservation, and other high-contention patterns beyond the current initial multi-writer path - Research + Opt-in ConcurrentWriteTransactions now reserves shared row-id ranges and rebases hot right-edge insert pages against pending WAL images, improving concurrent one-row auto-ID and explicit-ID insert fan-in while keeping serialized inserts as the default + Done API-level sharding @@ -410,11 +440,6 @@

Long-Term

Stream committed changes for read replicas or event-driven architectures Research - - WebAssembly sandboxed UDFs - Execute untrusted user-submitted functions in a WASM sandbox with resource limits (fuel, memory caps) via Wasmtime - Research -
@@ -429,8 +454,8 @@

Current Limitations

- Functions - Very limited scalar function surface today: built-in TEXT(expr) plus aggregate functions; no broader built-in function library or user-defined functions yet + Functions and automation + CSharpDB's UDF/command model is trusted and in-process by design. Current supported surfaces include host-registered scalar functions, common built-ins, trusted commands, form/report/pipeline hooks, declarative action sequences, and local Admin Forms C# modules; untrusted sandboxed execution is intentionally out of scope Query @@ -461,12 +486,24 @@

Current Limitations

FindByIndexAsync supports declared field-equality lookups; FindByPathAsync and FindByPathRangeAsync support path-based queries on indexed paths; FindAsync remains a full scan for unindexed predicates. Generated collections require registered descriptors for existing collection indexes; unsupported generated model shapes warn and use the source-generated JSON fallback instead of binary direct payloads + External Tables + Native .csdbtable archives can be registered and queried as read-only external tables. Writable external tables are planned as an opt-in .csdbx format; current archives remain read-only, and broader external indexes, range seeks, and deeper planner costing remain planned + + Networking CSharpDB.Daemon now hosts both REST and gRPC from one process; named pipes remain reserved but are not implemented end to end today Security - Remote HTTP and gRPC deployment still rely on external network controls or front-end TLS termination; built-in authentication, authorization, and TLS/mTLS support are still planned + Remote REST and daemon gRPC support opt-in API-key authentication, defaulting to None for backward compatibility. JWT, RBAC, mTLS helpers, TLS-specific configuration, and at-rest encryption are not implemented + + + Admin Forms + The Forms designer/runtime supports the core generated-form and data-entry path plus trusted command-backed automation, including lifecycle events, command buttons, selected-control events, conditional UI rules, domain formula helpers, and declarative action sequences for current record, form navigation, filtering, SQL/procedure, and control-property workflows. It still needs Access-parity work for responsive runtime rendering, complete inferred validation, richer form modes, additional events, advanced filtering/sorting, report/query/import/export actions, macro loops/on-error/temp vars, and broader controls + + + Admin Reports + The Reports designer/runtime supports the core banded preview path plus trusted command-backed preview lifecycle events, but still needs Access-parity work for bounded saved-query previews, full report output/export, parameters, richer grouping and totals semantics, conditional formatting, subreports, and broader controls Text / Multilingual @@ -478,7 +515,7 @@

Current Limitations

Storage - No page-level compression + No page-level compression; the compression SDK sample stores compressed payloads as ordinary application-managed BLOB values Storage @@ -494,7 +531,7 @@

Current Limitations

Query - Phase-2 cost-based planning is largely in place: ANALYZE, sys.table_stats, sys.column_stats, internal histograms/heavy hitters/prefix stats, and bounded small-chain join reordering now feed join/access-path costing; remaining work is adaptive re-optimization and public histogram/diagnostic surfacing rather than missing core stats-guided costing + Phase-2 cost-based planning is in place: ANALYZE, sys.table_stats, sys.column_stats, public planner-stat diagnostics, histogram/heavy-hitter/prefix estimates, and bounded small-chain join reordering now feed join/access-path costing. Opt-in adaptive join re-optimization can react to stale-stat or parameter-sensitive join cardinality misses, while broader runtime actuals, EXPLAIN ANALYZE, and full mid-plan reordering remain future work Query @@ -520,6 +557,8 @@

Completed Milestones

  • Ordered integer index range scans (<, <=, >, >=, BETWEEN) in the fast lookup path
  • ANALYZE, persisted sys.table_stats / sys.column_stats, and stale-aware column-stat refresh
  • Phase-2 cost-based query planning: statistics-guided access-path selection, join method choice, hash build-side choice, histogram/heavy-hitter/cardinality estimation, composite-prefix correlation modeling, and bounded small-chain inner-join reordering
    • +
  • Public planner diagnostics through EXPLAIN ESTIMATE FOR SELECT, WITH, compound queries, and sys.planner_* virtual catalogs
    • +
  • Opt-in adaptive join re-optimization behind DatabaseOptions.AdaptiveQueryReoptimization
  • SQL statement and SELECT plan caching
  • First-class IDENTITY / AUTOINCREMENT support for INTEGER PRIMARY KEY columns
  • Persisted table NextRowId high-water mark with compatibility fallback for legacy metadata
    • @@ -537,6 +576,7 @@

    Completed Milestones

  • Maintenance report, REINDEX, and VACUUM flows across client, CLI, API, and Admin UI
  • Dedicated CSharpDB.Daemon gRPC host for remote CSharpDB.Client access
  • Remote host consolidation in CSharpDB.Daemon, with REST /api and gRPC sharing the same warm hosted database client
    • +
  • Opt-in API-key protection for REST /api/* and daemon gRPC access
  • Storage tuning presets, bounded WAL read caching, memory-mapped main-file reads, and sliced background WAL auto-checkpointing
  • SQL executor/read-path fast paths for compact projections, broader join/index coverage, grouped aggregates, and correlated subquery filters
  • Batch-first SQL row-batch execution foundation with batch-aware scan/index/join roots, shared predicate/projection kernels, and batch-native generic aggregate paths
    • @@ -544,6 +584,7 @@

    Completed Milestones

  • REST API with 34 endpoints and OpenAPI/Scalar documentation
  • Blazor Server admin dashboard
  • Integrated Admin Forms and Reports designers with runtime preview/entry, database-backed metadata persistence, and print-ready report output
    • +
  • Trusted C# callbacks, commands, Admin automation hooks, and local Admin Forms C# code modules
  • B+tree delete rebalancing with underflow handling (borrow/merge + interior collapse path)
  • Reusable snapshot reader sessions for higher concurrent-read throughput
  • Comprehensive benchmark suite (micro, macro, stress, scaling, in-memory, shared-memory)
    • @@ -554,6 +595,7 @@

    Completed Milestones

  • Full-text search with tokenization, stemming, and relevance ranking
  • Hybrid storage mode with lazy-resident durable storage and gRPC tunable file-cache
  • Client-wide BackupAsync / RestoreAsync across direct, HTTP, gRPC, CLI, and Admin
    • +
  • Native .csdbtable table archives with Admin Import / Export, read-only external table registration, sys.external_tables, external table scans/joins, and embedded archive primary-key lookup indexes
  • ReplaceAsync for index stores
  • Package-driven ETL pipelines with validation, dry-run, execute/resume, persisted run history, and Admin visual designer support
    • @@ -570,9 +612,11 @@

    See Also

  • Database Encryption Plan — Encrypted storage format, key management, migration, and managed-surface rollout
  • Storage Engine Guide — CSharpDB.Storage API reference: device, pager, B+tree, WAL, indexing, serialization, and catalog
  • Native FFI Tutorials — Python and Node.js examples using the NativeAOT shared library
    • -
  • User-Defined Functions Plan — C# library functions callable by the database, native plugin extensions, and WASM sandboxing
    • +
  • User-Defined Functions Plan — Trusted C# functions, commands, form/report/pipeline hooks, and code modules
    • +
  • Writable External Tables Plan — Opt-in writable external tables backed by mutable B+tree external files
  • Pub/Sub Change Events Plan — Engine-level change events with channel-based delivery for real-time data subscriptions
  • Benchmark Suite — Performance data informing optimization priorities
    • +
  • Native Table Archives Blog — v3.8 table archive, external table, and Admin Import / Export overview
    • diff --git a/www/roadmap.html b/www/roadmap.html index 5ca9612d..7c97ec3b 100644 --- a/www/roadmap.html +++ b/www/roadmap.html @@ -131,7 +131,7 @@

    Roadmap

    -

    Planned direction for CSharpDB — organized by timeframe and priority. Reflects the current v3.4.0 state.

    +

    Planned direction for CSharpDB — organized by timeframe and priority. Reflects the current v3.8.0 state.

    Need the full source guide? The original long-form markdown version is preserved as Roadmap Source Reference.
    @@ -257,6 +257,13 @@

    Client Backup & Restore

    BackupAsync / RestoreAsync as first-class operations across direct, HTTP, gRPC, CLI, and Admin.

    +
    +
    +

    Native Table Archives & External Tables

    + Done +
    +

    Native .csdbtable snapshots with fast Admin Import / Export, download or server-path destinations, CREATE EXTERNAL TABLE, sys.external_tables, read-only scans/joins, and embedded primary-key lookup indexes.

    +

    Older DB Foreign-Key Retrofit Migration

    @@ -286,9 +293,16 @@

    User-Defined Functions and Commands

    - Partial + Done +
    +

    Done for the trusted in-process model: host-registered C# scalar functions, common SQL/Admin built-ins, trusted commands, Admin Forms/Reports/pipeline hooks, declarative form action sequences, and local Admin Forms C# code modules. Untrusted sandboxed UDF execution is intentionally out of scope.

    +
    +
    +
    +

    Writable External Tables

    + Planned
    -

    Trusted in-process C# scalar functions are implemented for SQL, triggers/procedures, direct clients, Admin Forms/Reports, and pipelines; trusted commands now back supported Admin automation surfaces. Aggregate/table-valued UDFs, richer macro flow, native plugins, database-owned code modules, and sandboxed UDFs remain future work.

    +

    Opt-in writable external table registrations over mutable .csdbx files, backed by CSharpDB B+tree storage and limited to INSERT, UPDATE, and DELETE in v1 while .csdbtable archives remain read-only.

    @@ -320,10 +334,17 @@

    Remote Host Consolidation

    -

    Remote Host Security

    +

    Remote API-Key Protection

    + Done +
    +

    Opt-in API-key mode protects REST /api/* and daemon gRPC calls with constant-time key comparison while keeping default no-auth behavior for compatibility.

    +
    +
    +
    +

    Remote Host Security Hardening

    Planned
    -

    Built-in authentication, authorization, API keys, protected admin endpoints, and TLS/mTLS deployment support.

    +

    Authorization, protected admin endpoint scopes, JWT/RBAC options, and TLS/mTLS deployment helpers for remote HTTP and gRPC access.

    @@ -414,6 +435,13 @@

    SQL Batched Row Transport

    Internal row-batch transport serves as the batch-first SQL execution foundation across batch-capable result boundaries, scans, joins, and generic aggregates.

    +
    +
    +

    External Table Index Coverage

    + Planned +
    +

    Follow writable .csdbx storage with broader external-table indexes, planner costing, and multi-column lookup/range support beyond the current archive primary-key point-lookup path.

    +

    Page-Level Compression

    @@ -438,16 +466,16 @@

    Cost-Based Query Optimizer

    Adaptive Query Re-Optimization

    - Planned + Done
    -

    Re-plan or adapt when observed cardinality diverges materially from persisted statistics, especially after stale stats or parameter-sensitive predicates.

    +

    Current phase is complete: opt-in adaptive join execution can switch eligible index nested-loop joins to hash joins and flip inner hash build sides at safe pre-emission boundaries.

    Public Planner Histogram Inspection

    - Planned + Done
    -

    Expose histogram, heavy-hitter, composite-prefix, and estimate diagnostics through a public surface while keeping current stats internals private.

    +

    Stable SQL-first diagnostics expose sys.planner_histograms, sys.planner_heavy_hitters, sys.planner_index_prefix_stats, and EXPLAIN ESTIMATE FOR <query>.

    @@ -480,9 +508,9 @@

    Initial Multi-Writer Support

    Broader Multi-Writer Optimization

    - Research + Done
    -

    Improve hot insert fan-in, row-id reservation, and other high-contention patterns beyond the current initial multi-writer path.

    +

    Opt-in concurrent write transactions now reserve shared row-id ranges and rebase hot right-edge insert pages against pending WAL images for improved insert fan-in.

    @@ -498,13 +526,6 @@

    Replication & Change Feed

    Stream committed changes for read replicas or event-driven architectures.

    -
    -
    -

    WebAssembly Sandboxed UDFs

    - Research -
    -

    Execute untrusted user-submitted functions in a WASM sandbox with resource limits via Wasmtime.

    -
    @@ -515,25 +536,26 @@

    Current Limitations

    - + - + + - - + + - + - +
    AreaLimitation
    Functions and automationTrusted in-process C# scalar functions are supported when registered by the host; Admin Forms, Admin Reports, and pipelines can invoke trusted host commands from supported event/hook surfaces; Admin Forms support declarative action sequences for run-command, set-field, show-message, and stop steps. Broader built-in scalar functions, aggregate/table-valued UDFs, stored C# modules, remote delegate serialization, additional Access-style control events, richer macro flow, and sandboxed UDFs are not implemented
    Functions and automationCSharpDB's UDF/command model is trusted and in-process by design. Current supported surfaces include host-registered scalar functions, common built-ins, trusted commands, form/report/pipeline hooks, declarative action sequences, and local Admin Forms C# modules; untrusted sandboxed execution is intentionally out of scope
    QueryScalar/IN/EXISTS subqueries are supported, including correlated cases in WHERE, non-aggregate projection, and UPDATE/DELETE expressions; correlated subqueries are not yet supported in JOIN ON, GROUP BY, HAVING, ORDER BY, or aggregate projections
    QueryUNION, INTERSECT, and EXCEPT are supported; UNION ALL is not implemented yet
    QueryNo window functions
    SchemaNo SQL DEFAULT column values or CHECK constraints yet. Foreign keys are currently v1 only: single-column, column-level REFERENCES with optional ON DELETE CASCADE; table-level/composite/deferred foreign keys and ON UPDATE actions are not implemented
    IndexesEquality lookups support current INTEGER/TEXT indexes, but ordered range-scan pushdown is still limited to single-column INTEGER index paths
    RowIdLegacy table schemas without persisted high-water metadata may pay a one-time key scan on first insert
    CollectionsFindByIndexAsync supports declared field-equality lookups; FindByPathAsync and FindByPathRangeAsync support path-based queries on indexed paths; FindAsync remains a full scan for unindexed predicates
    CollectionsFindByIndexAsync supports declared field-equality lookups; FindByPathAsync and FindByPathRangeAsync support path-based queries on indexed paths; FindAsync remains a full scan for unindexed predicates. Generated collections require registered descriptors for existing collection indexes; unsupported generated model shapes warn and use the source-generated JSON fallback instead of binary direct payloads
    External TablesNative .csdbtable archives can be registered and queried as read-only external tables. Writable external tables are planned as an opt-in .csdbx format; current archives remain read-only, and broader external indexes, range seeks, and deeper planner costing remain planned
    NetworkingCSharpDB.Daemon now hosts both REST and gRPC from one process; named pipes remain reserved but are not implemented end to end today
    SecurityRemote HTTP and gRPC deployment still rely on external network controls or front-end TLS termination; built-in authentication, authorization, and TLS/mTLS support are still planned
    Admin FormsThe Forms designer/runtime supports the core generated-form and data-entry path plus initial trusted command-backed automation, including lifecycle events, command buttons, selected control events, and declarative action sequences, but still needs Access-parity work for responsive runtime rendering, complete inferred validation, richer form modes, broader built-in actions, additional events, advanced filtering/sorting, and broader controls
    SecurityRemote REST and daemon gRPC support opt-in API-key authentication, defaulting to None for compatibility. JWT, RBAC, mTLS helpers, TLS-specific configuration, and at-rest encryption are not implemented
    Admin FormsThe Forms designer/runtime supports the core generated-form and data-entry path plus trusted command-backed automation, including lifecycle events, command buttons, selected-control events, conditional UI rules, domain formula helpers, declarative action sequences, and local C# code modules. It still needs Access-parity work for responsive runtime rendering, complete inferred validation, richer form modes, additional events, advanced filtering/sorting, report/query/import/export actions, macro loops/on-error/temp vars, and broader controls
    Admin ReportsThe Reports designer/runtime supports the core banded preview path plus trusted command-backed preview lifecycle events, but still needs Access-parity work for bounded saved-query previews, full report output/export, parameters, richer grouping and totals semantics, conditional formatting, subreports, and broader controls
    Text / MultilingualText is stored as UTF-8 and supports all Unicode languages; default semantics remain ordinal, but opt-in BINARY, NOCASE, NOCASE_AI, and ICU:<locale> collation are implemented for SQL and collection indexes. Dedicated ordered SQL text index optimization remains planned
    ConcurrencyPhysical WAL commit path is still serialized at the storage boundary. Initial multi-writer support is shipped, but observed gains depend on conflict shape and whether shared auto-commit INSERT is left on the default serialized path
    StorageNo page-level compression
    StorageNo page-level compression; the compression SDK sample stores compressed payloads as ordinary application-managed BLOB values
    StorageNo at-rest encryption for database/WAL files; on-disk storage is plaintext only
    StorageMemory-mapped reads are opt-in and currently apply only to clean main-file pages; WAL-backed reads still rely on the WAL/cache path
    StorageBy default, durable auto-commit single-row writes still pay a physical WAL flush per commit; opt-in UseDurableCommitBatchWindow(...) can trade some commit latency for higher throughput
    QueryPhase-2 cost-based planning is largely in place: ANALYZE, sys.table_stats, sys.column_stats, internal histograms/heavy hitters/prefix stats, and bounded small-chain join reordering now feed join/access-path costing; remaining work is adaptive re-optimization and public histogram/diagnostic surfacing rather than missing core stats-guided costing
    QueryPhase-2 cost-based planning is in place: ANALYZE, sys.table_stats, sys.column_stats, public planner-stat diagnostics, histogram/heavy-hitter/prefix estimates, and bounded small-chain join reordering now feed join/access-path costing. Opt-in adaptive join re-optimization can react to stale-stat or parameter-sensitive join cardinality misses, while broader runtime actuals, EXPLAIN ANALYZE, and full mid-plan reordering remain future work
    QueryInternal row-batch transport is now the default scan-heavy execution foundation across batch-capable scans, joins, aggregates, and result boundaries; remaining work is broader kernel specialization and optional SIMD-style tuning rather than missing core batch coverage
    @@ -557,6 +579,8 @@

    Completed Milestones

    ✓ Ordered integer index range scans in the fast lookup path
    ✓ ANALYZE with persisted table/column stats and stale-aware refresh
    ✓ Phase-2 cost-based query planning: statistics-guided access paths, join method/reordering, histogram/cardinality estimation
    +
    ✓ Public planner diagnostics with EXPLAIN ESTIMATE and sys.planner_* catalogs
    +
    ✓ Opt-in adaptive join re-optimization for eligible stale-stat and parameter-sensitive joins
    ✓ SELECT DISTINCT and DISTINCT aggregates
    ✓ SQL statement and SELECT plan caching
    ✓ First-class IDENTITY / AUTOINCREMENT support for INTEGER PRIMARY KEY columns
    @@ -577,15 +601,18 @@

    Completed Milestones

    ✓ Full-text search with tokenization, stemming, and relevance ranking
    ✓ Hybrid storage mode with lazy-resident durable storage and gRPC tunable file-cache
    ✓ Client-wide BackupAsync / RestoreAsync across direct, HTTP, gRPC, CLI, and Admin
    +
    ✓ Native .csdbtable table archives with Admin Import / Export and read-only external table registration
    ✓ ReplaceAsync for index stores
    ✓ Maintenance report, REINDEX, and VACUUM flows across client, CLI, API, and Admin UI
    ✓ Dedicated gRPC daemon host
    ✓ Remote host consolidation in CSharpDB.Daemon, with REST /api and gRPC sharing one warm hosted database client
    +
    ✓ Opt-in API-key protection for REST /api/* and daemon gRPC calls
    ✓ Daemon service packaging with self-contained archives and service install assets
    ✓ Storage tuning presets, bounded WAL read caching, memory-mapped reads, and sliced background checkpointing
    ✓ SQL executor/read-path fast paths for compact projections, broader join/index coverage, and correlated subquery filters
    ✓ REST API with 34+ endpoints and OpenAPI/Scalar documentation
    ✓ Blazor Server admin dashboard with Forms and Reports designers
    +
    ✓ Trusted C# callbacks, commands, Admin automation hooks, and local Admin Forms C# code modules
    ✓ Interactive CLI with meta-commands and file execution
    ✓ Package-driven ETL pipelines with validation, dry-run, execute/resume, and Admin visual designer
    ✓ VS Code extension with schema explorer
    diff --git a/www/sitemap.xml b/www/sitemap.xml index b78e0105..4af63a78 100644 --- a/www/sitemap.xml +++ b/www/sitemap.xml @@ -26,19 +26,19 @@ https://csharpdb.com/roadmap.html - 2026-04-20 + 2026-05-10 monthly 0.7 https://csharpdb.com/roadmap-reference.html - 2026-04-24 + 2026-05-10 monthly 0.5 https://csharpdb.com/changelog.html - 2026-04-21 + 2026-05-10 weekly 0.7 @@ -300,6 +300,12 @@ weekly 0.7 + + https://csharpdb.com/blog/table-archives.html + 2026-05-10 + monthly + 0.7 + https://csharpdb.com/blog/access-style-csharp-code-modules.html 2026-05-10 From d1b71c7c16bbeaf7526c714db1dab81646664a38 Mon Sep 17 00:00:00 2001 From: Maximum Code Date: Fri, 15 May 2026 18:57:34 -0700 Subject: [PATCH 4/8] Add persisted data model diagrams --- .../Services/DbReportSourceProvider.cs | 3 - .../Components/Layout/NavMenu.razor | 1 + .../Components/Shared/SchemaCanvas.razor | 18 +- .../Components/Tabs/DataModelTab.razor | 785 +++++++++++++++++- .../Components/Tabs/QueryTab.razor | 4 +- src/CSharpDB.Admin/Models/DataModelModels.cs | 51 ++ src/CSharpDB.Admin/Program.cs | 4 +- .../Services/DataModelGraphBuilder.cs | 138 +++ .../Services/DataModelService.cs | 533 +++++++++++- .../Services/HostCallbackCatalogService.cs | 3 +- src/CSharpDB.Admin/wwwroot/css/app.css | 90 ++ .../Internal/EngineTransportClient.cs | 4 +- .../DatabaseMaintenanceCoordinator.cs | 6 +- src/CSharpDB.Execution/QueryPlanner.cs | 163 +++- .../Services/DbReportSourceProviderTests.cs | 2 - .../DataModelDiagramServiceTests.cs | 194 +++++ .../DataModelGraphBuilderTests.cs | 56 ++ tests/CSharpDB.Tests/SystemCatalogTests.cs | 41 + 18 files changed, 1999 insertions(+), 97 deletions(-) create mode 100644 tests/CSharpDB.Tests/DataModelDiagramServiceTests.cs diff --git a/src/CSharpDB.Admin.Reports/Services/DbReportSourceProvider.cs b/src/CSharpDB.Admin.Reports/Services/DbReportSourceProvider.cs index b16c1642..003eb4d8 100644 --- a/src/CSharpDB.Admin.Reports/Services/DbReportSourceProvider.cs +++ b/src/CSharpDB.Admin.Reports/Services/DbReportSourceProvider.cs @@ -9,7 +9,6 @@ namespace CSharpDB.Admin.Reports.Services; public sealed class DbReportSourceProvider(ICSharpDbClient dbClient) : IReportSourceProvider { private const string DesignerLayoutPrefix = "__designer_layout:"; - private const string DataModelLayoutPrefix = "__data_model_layout:"; public async Task> ListSourceReferencesAsync() { @@ -46,8 +45,6 @@ internal static bool IsSupportedSavedQuery(SavedQueryDefinition savedQuery) { if (savedQuery.Name.StartsWith(DesignerLayoutPrefix, StringComparison.Ordinal)) return false; - if (savedQuery.Name.StartsWith(DataModelLayoutPrefix, StringComparison.Ordinal)) - return false; string sql = ReportSql.NormalizeSqlText(savedQuery.SqlText); if (!(sql.StartsWith("SELECT", StringComparison.OrdinalIgnoreCase) || sql.StartsWith("WITH", StringComparison.OrdinalIgnoreCase))) diff --git a/src/CSharpDB.Admin/Components/Layout/NavMenu.razor b/src/CSharpDB.Admin/Components/Layout/NavMenu.razor index 350e7d36..5ea9dbb9 100644 --- a/src/CSharpDB.Admin/Components/Layout/NavMenu.razor +++ b/src/CSharpDB.Admin/Components/Layout/NavMenu.razor @@ -716,6 +716,7 @@ new SystemCatalogItem("sys.objects", "all objects", "SELECT * FROM sys.objects ORDER BY object_type, object_name;"), new SystemCatalogItem("sys.saved_queries", "saved queries", "SELECT * FROM sys.saved_queries ORDER BY name;"), new SystemCatalogItem("sys.external_tables", "external tables", "SELECT * FROM sys.external_tables ORDER BY table_name;"), + new SystemCatalogItem("sys.diagrams", "diagrams", "SELECT * FROM sys.diagrams ORDER BY name;"), }; private IReadOnlyCollection? _userTableNames; diff --git a/src/CSharpDB.Admin/Components/Shared/SchemaCanvas.razor b/src/CSharpDB.Admin/Components/Shared/SchemaCanvas.razor index 673a2934..d269c656 100644 --- a/src/CSharpDB.Admin/Components/Shared/SchemaCanvas.razor +++ b/src/CSharpDB.Admin/Components/Shared/SchemaCanvas.razor @@ -65,7 +65,7 @@ { var current = node; bool selected = string.Equals(SelectedNodeName, current.Name, StringComparison.OrdinalIgnoreCase); -
    - @if (current.Kind == DataModelNodeKind.ExternalTable) - { - external - } - else - { + @if (current.Kind == DataModelNodeKind.ExternalTable) + { + external + } + else if (current.IsDraft) + { + pending create + } + else + { @current.IndexCount index@(current.IndexCount == 1 ? "" : "es") @current.TriggerCount trigger@(current.TriggerCount == 1 ? "" : "s") } diff --git a/src/CSharpDB.Admin/Components/Tabs/DataModelTab.razor b/src/CSharpDB.Admin/Components/Tabs/DataModelTab.razor index fcf64de8..8bcf6de1 100644 --- a/src/CSharpDB.Admin/Components/Tabs/DataModelTab.razor +++ b/src/CSharpDB.Admin/Components/Tabs/DataModelTab.razor @@ -1,5 +1,6 @@ @using CSharpDB.Admin.Models @inject IDataModelService DataModels +@inject IDataModelDiagramService Diagrams @inject TabManagerService TabManager @inject ToastService Toast @@ -44,6 +45,12 @@
    }
    + @@ -75,40 +82,54 @@
    + @bind="_diagramName" + placeholder="Diagram name..." />
    - @if (_showLoadPanel) {
    - @if (!_savedLayouts.Any()) + @if (!_savedDiagrams.Any()) { -
    No saved models
    +
    No saved diagrams
    } else { - @foreach (SavedQueryDefinition layout in _savedLayouts) + @foreach (DataModelDiagramSummary diagram in _savedDiagrams) { - var saved = layout; - } }
    }
    +
    +
    + @if (_showNewTablePanel) + { +
    + New Table + + + + + +
    + } + @if (_error is not null) {
    @_error
    @@ -154,7 +202,83 @@ { @SelectedNode.ArchivePath } + @if (SelectedNode.IsDraft) + { + Pending create table + }
    + + @if (SelectedNode.Kind == DataModelNodeKind.Table && !SelectedNode.IsDraft) + { +
    +
    + + +
    +
    + + + + +
    +
    + @foreach (DataModelColumn column in SelectedNode.Columns) + { + var currentColumn = column; +
    + + @currentColumn.Name + @currentColumn.TypeLabel + + + + +
    + } +
    + +
    + } } else if (SelectedRelationship is not null) { @@ -163,6 +287,17 @@ references @SelectedRelationship.RightTable.@SelectedRelationship.RightColumn @RelationshipKindLabel(SelectedRelationship.Kind)

    + @if (SelectedRelationship.Kind != DataModelRelationshipKind.ExternalArchiveForeignKey) + { +
    + +
    + } } else { @@ -189,6 +324,48 @@ } } + @if (MutableTableNodes.Count >= 2) + { +
    + + + + + + +
    + }
    @if (AllWarnings.Count > 0) @@ -202,15 +379,39 @@
    } +
    +

    Pending Changes

    + @if (_state.PendingOperations.Count == 0) + { +

    No staged schema changes.

    + } + else + { + @foreach (DataModelPendingOperation operation in _state.PendingOperations) + { + var current = operation; +
    + @current.Description + +
    + } + } +
    +
    -

    Preview DDL

    - 
    @PreviewSql
    +

    Pending SQL

    + 
    @PendingPreviewSql
    @code { + private const string DefaultDiagramName = "Default Diagram"; private const double MinZoom = 0.5; private const double MaxZoom = 2.0; private const double ZoomStep = 0.1; @@ -219,12 +420,26 @@ private DataModelState _state = new(); private IReadOnlyList _sourceOptions = []; - private IReadOnlyList _savedLayouts = []; + private IReadOnlyList _savedDiagrams = []; private string? _selectedNodeName; private string? _selectedRelationshipId; - private string _saveLayoutName = ""; + private string _diagramName = ""; + private string _newTableName = ""; + private string _newTablePrimaryKeyName = "Id"; + private string _newTablePrimaryKeyType = "INTEGER"; + private string _renameTableName = ""; + private string _newColumnName = ""; + private string _newColumnType = "TEXT"; + private bool _newColumnNotNull; + private readonly Dictionary _columnRenameNames = new(StringComparer.OrdinalIgnoreCase); + private string _relationshipLeftTable = ""; + private string _relationshipLeftColumn = ""; + private string _relationshipRightTable = ""; + private string _relationshipRightColumn = ""; + private string _relationshipOnDelete = "RESTRICT"; private bool _showAddSource; private bool _showLoadPanel; + private bool _showNewTablePanel; private bool _loading; private string? _error; @@ -241,18 +456,88 @@ ? null : _state.Relationships.FirstOrDefault(relationship => string.Equals(relationship.Id, _selectedRelationshipId, StringComparison.Ordinal)); + private IReadOnlyList MutableTableNodes => + _state.Nodes + .Where(static node => node.Kind == DataModelNodeKind.Table) + .OrderBy(static node => node.Name, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + private IReadOnlyList RelationshipLeftColumns => + FindNode(_relationshipLeftTable)?.Columns.OrderBy(static column => column.Name, StringComparer.OrdinalIgnoreCase).ToArray() ?? []; + + private IReadOnlyList RelationshipRightColumns => + FindNode(_relationshipRightTable)?.Columns.OrderBy(static column => column.Name, StringComparer.OrdinalIgnoreCase).ToArray() ?? []; + private IReadOnlyList AllWarnings => _state.Warnings .Concat(_state.Nodes.SelectMany(static node => node.Warnings.Select(warning => $"{node.Name}: {warning}"))) .Distinct(StringComparer.OrdinalIgnoreCase) .ToArray(); - private string PreviewSql => DataModels.BuildPreviewSql(_state); + private string PendingPreviewSql => DataModels.BuildPendingOperationsPreview(_state); private double CurrentScale => NormalizeScale(_state.Scale); private string ZoomLabel => $"{Math.Round(CurrentScale * 100):0}%"; + private bool CanStageNewTable => + !string.IsNullOrWhiteSpace(_newTableName) + && !string.IsNullOrWhiteSpace(_newTablePrimaryKeyName) + && !_state.Nodes.Any(node => string.Equals(node.Name, _newTableName.Trim(), StringComparison.OrdinalIgnoreCase)) + && !_sourceOptions.Any(option => string.Equals(option.Name, _newTableName.Trim(), StringComparison.OrdinalIgnoreCase)); + + private bool HasPendingDropForSelectedNode => + SelectedNode is not null && + _state.PendingOperations.Any(operation => + operation.Kind == DataModelPendingOperationKind.DropTable && + string.Equals(operation.TableName, SelectedNode.Name, StringComparison.OrdinalIgnoreCase)); + + private bool CanStageRenameSelectedTable => + SelectedNode is { Kind: DataModelNodeKind.Table, IsDraft: false } + && !HasPendingDropForSelectedNode + && !HasPendingTableRename(SelectedNode.Name) + && !string.IsNullOrWhiteSpace(_renameTableName) + && !string.Equals(_renameTableName.Trim(), SelectedNode.Name, StringComparison.OrdinalIgnoreCase) + && !_sourceOptions.Any(option => string.Equals(option.Name, _renameTableName.Trim(), StringComparison.OrdinalIgnoreCase)) + && !_state.Nodes.Any(node => string.Equals(node.Name, _renameTableName.Trim(), StringComparison.OrdinalIgnoreCase)); + + private bool CanStageAddColumn => + SelectedNode is { Kind: DataModelNodeKind.Table, IsDraft: false } + && !HasPendingDropForSelectedNode + && !HasPendingTableRename(SelectedNode.Name) + && !string.IsNullOrWhiteSpace(_newColumnName) + && !SelectedNode.Columns.Any(column => string.Equals(column.Name, _newColumnName.Trim(), StringComparison.OrdinalIgnoreCase)) + && !_state.PendingOperations.Any(operation => + operation.Kind == DataModelPendingOperationKind.AddColumn && + string.Equals(operation.TableName, SelectedNode.Name, StringComparison.OrdinalIgnoreCase) && + string.Equals(operation.ColumnName, _newColumnName.Trim(), StringComparison.OrdinalIgnoreCase)); + + private bool CanStageRelationship => + !string.IsNullOrWhiteSpace(_relationshipLeftTable) + && !string.IsNullOrWhiteSpace(_relationshipLeftColumn) + && !string.IsNullOrWhiteSpace(_relationshipRightTable) + && !string.IsNullOrWhiteSpace(_relationshipRightColumn) + && !string.Equals(_relationshipLeftTable, _relationshipRightTable, StringComparison.OrdinalIgnoreCase) + && FindNode(_relationshipLeftTable) is { Kind: DataModelNodeKind.Table } + && FindNode(_relationshipRightTable) is { Kind: DataModelNodeKind.Table } + && RelationshipLeftColumns.Any(column => string.Equals(column.Name, _relationshipLeftColumn, StringComparison.OrdinalIgnoreCase)) + && RelationshipRightColumns.Any(column => string.Equals(column.Name, _relationshipRightColumn, StringComparison.OrdinalIgnoreCase)) + && !_state.Relationships.Any(relationship => + string.Equals(relationship.LeftTable, _relationshipLeftTable, StringComparison.OrdinalIgnoreCase) && + string.Equals(relationship.LeftColumn, _relationshipLeftColumn, StringComparison.OrdinalIgnoreCase) && + string.Equals(relationship.RightTable, _relationshipRightTable, StringComparison.OrdinalIgnoreCase) && + string.Equals(relationship.RightColumn, _relationshipRightColumn, StringComparison.OrdinalIgnoreCase)); + + private bool HasPendingDropForSelectedRelationship => + SelectedRelationship is not null && + _state.PendingOperations.Any(operation => + operation.Kind == DataModelPendingOperationKind.DropForeignKey && + (string.Equals(operation.ConstraintName, SelectedRelationship.ConstraintName, StringComparison.OrdinalIgnoreCase) || + string.Equals(operation.ConstraintName, SelectedRelationship.Id, StringComparison.OrdinalIgnoreCase))); + + private bool UsesDefaultDiagram => + string.IsNullOrWhiteSpace(Tab?.InitialDataModelSourceName); + protected override async Task OnInitializedAsync() { await LoadInitialAsync(); @@ -269,6 +554,8 @@ { _state = saved; NormalizeZoom(); + EnsureDefaultDiagramName(); + _diagramName = _state.DiagramName ?? _state.SavedLayoutName ?? ""; } } catch @@ -284,7 +571,7 @@ try { _sourceOptions = await DataModels.GetSourceOptionsAsync(); - _savedLayouts = await DataModels.GetSavedLayoutsAsync(); + _savedDiagrams = await Diagrams.GetDiagramsAsync(); if (Tab?.DataModelStateJson is not null) { DataModelState? saved = DataModelGraphBuilder.DeserializeState(Tab.DataModelStateJson); @@ -292,12 +579,23 @@ { _state = saved; NormalizeZoom(); + EnsureDefaultDiagramName(); + _diagramName = _state.DiagramName ?? _state.SavedLayoutName ?? ""; + SaveState(); + await PersistActiveDiagramAsync(); } } else { - _state = await DataModels.BuildModelAsync(Tab?.InitialDataModelSourceName); + DataModelState? defaultDiagram = UsesDefaultDiagram + ? await Diagrams.LoadDiagramAsync(DefaultDiagramName) + : null; + _state = defaultDiagram ?? await DataModels.BuildModelAsync(Tab?.InitialDataModelSourceName); + NormalizeZoom(); + EnsureDefaultDiagramName(); + _diagramName = _state.DiagramName ?? ""; SaveState(); + await PersistActiveDiagramAsync(); } } catch (Exception ex) @@ -317,10 +615,29 @@ try { _sourceOptions = await DataModels.GetSourceOptionsAsync(); + if (!string.IsNullOrWhiteSpace(_state.DiagramName)) + { + DataModelState? saved = await Diagrams.LoadDiagramAsync(_state.DiagramName); + if (saved is not null) + { + _state = saved; + _diagramName = saved.DiagramName ?? _diagramName; + _selectedNodeName = null; + _selectedRelationshipId = null; + SaveState(); + return; + } + } + DataModelState refreshed = await DataModels.BuildModelAsync(Tab?.InitialDataModelSourceName); PreservePositions(refreshed); + refreshed.DiagramName = _state.DiagramName; + refreshed.SavedLayoutName = _state.SavedLayoutName; + refreshed.PendingOperations = _state.PendingOperations; _state = refreshed; + EnsureDefaultDiagramName(); SaveState(); + await PersistActiveDiagramAsync(); } catch (Exception ex) { @@ -341,9 +658,14 @@ double currentScale = CurrentScale; _state = await DataModels.BuildModelAsync(autoLayoutLimit: int.MaxValue); _state.Scale = currentScale; + _state.DiagramName = UsesDefaultDiagram && string.IsNullOrWhiteSpace(_diagramName) + ? DefaultDiagramName + : string.IsNullOrWhiteSpace(_diagramName) ? null : _diagramName.Trim(); + _state.SavedLayoutName = _state.DiagramName; _selectedNodeName = null; _selectedRelationshipId = null; SaveState(); + await PersistActiveDiagramAsync(); } catch (Exception ex) { @@ -367,6 +689,7 @@ _selectedNodeName = sourceName; _selectedRelationshipId = null; SaveState(); + await PersistActiveDiagramAsync(); } catch (Exception ex) { @@ -422,18 +745,26 @@ } } - private void ClearCanvas() + private async Task ClearCanvas() { - _state = new DataModelState(); + string? diagramName = _state.DiagramName; + _state = new DataModelState + { + DiagramName = diagramName, + SavedLayoutName = diagramName, + }; _selectedNodeName = null; _selectedRelationshipId = null; SaveState(); + await PersistActiveDiagramAsync(); } private void SelectNode(string name) { _selectedNodeName = name; _selectedRelationshipId = null; + _renameTableName = ""; + _newColumnName = ""; } private void SelectRelationship(string id) @@ -442,7 +773,50 @@ _selectedNodeName = null; } - private void RemoveNode(string name) + private DataModelNode? FindNode(string? name) => + string.IsNullOrWhiteSpace(name) + ? null + : _state.Nodes.FirstOrDefault(node => string.Equals(node.Name, name, StringComparison.OrdinalIgnoreCase)); + + private string ColumnRenameKey(string tableName, string columnName) => $"{tableName}.{columnName}"; + + private string GetColumnRenameValue(string tableName, string columnName) => + _columnRenameNames.TryGetValue(ColumnRenameKey(tableName, columnName), out string? value) ? value : ""; + + private void SetColumnRenameValue(string tableName, string columnName, string? value) + { + string key = ColumnRenameKey(tableName, columnName); + if (string.IsNullOrWhiteSpace(value)) + _columnRenameNames.Remove(key); + else + _columnRenameNames[key] = value; + } + + private bool CanStageRenameColumn(DataModelNode node, DataModelColumn column) + { + string value = GetColumnRenameValue(node.Name, column.Name).Trim(); + return !string.IsNullOrWhiteSpace(value) + && !HasPendingTableRename(node.Name) + && !string.Equals(value, column.Name, StringComparison.OrdinalIgnoreCase) + && !node.Columns.Any(existing => string.Equals(existing.Name, value, StringComparison.OrdinalIgnoreCase)) + && !_state.PendingOperations.Any(operation => + operation.Kind == DataModelPendingOperationKind.RenameColumn && + string.Equals(operation.TableName, node.Name, StringComparison.OrdinalIgnoreCase) && + string.Equals(operation.ColumnName, column.Name, StringComparison.OrdinalIgnoreCase)); + } + + private bool HasPendingColumnDrop(string tableName, string columnName) => + _state.PendingOperations.Any(operation => + operation.Kind == DataModelPendingOperationKind.DropColumn && + string.Equals(operation.TableName, tableName, StringComparison.OrdinalIgnoreCase) && + string.Equals(operation.ColumnName, columnName, StringComparison.OrdinalIgnoreCase)); + + private bool HasPendingTableRename(string tableName) => + _state.PendingOperations.Any(operation => + operation.Kind == DataModelPendingOperationKind.RenameTable && + string.Equals(operation.TableName, tableName, StringComparison.OrdinalIgnoreCase)); + + private async Task RemoveNode(string name) { _state.Nodes.RemoveAll(node => string.Equals(node.Name, name, StringComparison.OrdinalIgnoreCase)); _state.Relationships.RemoveAll(relationship => @@ -455,57 +829,91 @@ _selectedRelationshipId = null; SaveState(); + await PersistActiveDiagramAsync(); } - private Task OnNodeMoved(DataModelNodeMove move) + private async Task OnNodeMoved(DataModelNodeMove move) { + DataModelNode? node = FindNode(move.NodeName); + if (node is not null) + { + node.X = move.X; + node.Y = move.Y; + } + + EnsureDefaultDiagramName(); SaveState(); - return Task.CompletedTask; + await PersistActiveDiagramAsync(); } - private async Task SaveLayoutAsync() + private async Task SaveDiagramAsync() { try { - await DataModels.SaveLayoutAsync(_saveLayoutName, _state); - await RefreshSavedLayoutsAsync(); - Toast.Success($"Data model layout '{_saveLayoutName.Trim()}' saved."); + await Diagrams.SaveDiagramAsync(_diagramName, _state); + _diagramName = _state.DiagramName ?? _diagramName.Trim(); + await RefreshSavedDiagramsAsync(); + SaveState(); + Toast.Success($"Data model diagram '{_diagramName}' saved."); } catch (Exception ex) { - Toast.Error($"Failed to save data model layout: {ex.Message}"); + Toast.Error($"Failed to save data model diagram: {ex.Message}"); } } - private async Task LoadLayoutAsync(string fullName) + private async Task LoadDiagramAsync(string name) { _showLoadPanel = false; try { - DataModelState? loaded = await DataModels.LoadLayoutAsync(fullName); + DataModelState? loaded = await Diagrams.LoadDiagramAsync(name); if (loaded is null) { - Toast.Warning("Saved data model layout was not found."); + Toast.Warning("Saved data model diagram was not found."); return; } _state = loaded; NormalizeZoom(); + _diagramName = loaded.DiagramName ?? name; _selectedNodeName = null; _selectedRelationshipId = null; SaveState(); - Toast.Info($"Data model layout '{fullName[DataModelService.LayoutPrefix.Length..]}' loaded."); + Toast.Info($"Data model diagram '{_diagramName}' loaded."); + } + catch (Exception ex) + { + Toast.Error($"Failed to load data model diagram: {ex.Message}"); + } + } + + private async Task DeleteActiveDiagramAsync() + { + if (string.IsNullOrWhiteSpace(_state.DiagramName)) + return; + + string name = _state.DiagramName; + try + { + await Diagrams.DeleteDiagramAsync(name); + await RefreshSavedDiagramsAsync(); + _state.DiagramName = null; + _state.SavedLayoutName = null; + _diagramName = ""; + SaveState(); + Toast.Info($"Data model diagram '{name}' deleted."); } catch (Exception ex) { - Toast.Error($"Failed to load data model layout: {ex.Message}"); + Toast.Error($"Failed to delete data model diagram: {ex.Message}"); } } - private async Task RefreshSavedLayoutsAsync() + private async Task RefreshSavedDiagramsAsync() { - try { _savedLayouts = await DataModels.GetSavedLayoutsAsync(); } - catch { _savedLayouts = []; } + try { _savedDiagrams = await Diagrams.GetDiagramsAsync(); } + catch { _savedDiagrams = []; } } private void OpenInQueryDesigner() @@ -518,24 +926,292 @@ { _showAddSource = !_showAddSource; _showLoadPanel = false; + _showNewTablePanel = false; } private void ToggleLoadPanel() { _showLoadPanel = !_showLoadPanel; _showAddSource = false; + _showNewTablePanel = false; + } + + private void ToggleNewTablePanel() + { + _showNewTablePanel = !_showNewTablePanel; + _showAddSource = false; + _showLoadPanel = false; + } + + private async Task StageNewTableAsync() + { + string tableName = _newTableName.Trim(); + string pkName = _newTablePrimaryKeyName.Trim(); + var columns = new List + { + new() + { + Name = pkName, + TypeLabel = _newTablePrimaryKeyType, + IsPrimaryKey = true, + IsIdentity = string.Equals(_newTablePrimaryKeyType, "INTEGER", StringComparison.OrdinalIgnoreCase), + Nullable = false, + }, + }; + + var node = new DataModelNode + { + Name = tableName, + Kind = DataModelNodeKind.Table, + IsDraft = true, + X = _state.Nodes.Count == 0 ? 20 : _state.Nodes.Max(static node => node.X) + 250, + Y = _state.Nodes.Count == 0 ? 20 : _state.Nodes.Min(static node => node.Y), + Columns = columns, + Warnings = ["Pending create table"], + }; + + _state.Nodes.Add(node); + _state.PendingOperations.Add(new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.CreateTable, + TableName = tableName, + Columns = columns, + Description = $"Create table {tableName}", + }); + + _selectedNodeName = tableName; + _selectedRelationshipId = null; + _newTableName = ""; + _newTablePrimaryKeyName = "Id"; + _newTablePrimaryKeyType = "INTEGER"; + _showNewTablePanel = false; + SaveState(); + await PersistActiveDiagramAsync(); + } + + private async Task StageDropSelectedTableAsync() + { + if (SelectedNode is null || SelectedNode.Kind != DataModelNodeKind.Table || SelectedNode.IsDraft) + return; + + if (HasPendingDropForSelectedNode || HasPendingTableRename(SelectedNode.Name)) + return; + + _state.PendingOperations.Add(new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.DropTable, + TableName = SelectedNode.Name, + Description = $"Drop table {SelectedNode.Name}", + }); + SaveState(); + await PersistActiveDiagramAsync(); + } + + private async Task StageRenameSelectedTableAsync() + { + if (!CanStageRenameSelectedTable || SelectedNode is null) + return; + + string newName = _renameTableName.Trim(); + _state.PendingOperations.Add(new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.RenameTable, + TableName = SelectedNode.Name, + NewTableName = newName, + Description = $"Rename table {SelectedNode.Name} to {newName}", + }); + _renameTableName = ""; + SaveState(); + await PersistActiveDiagramAsync(); + } + + private async Task StageAddColumnAsync() + { + if (!CanStageAddColumn || SelectedNode is null) + return; + + string columnName = _newColumnName.Trim(); + _state.PendingOperations.Add(new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.AddColumn, + TableName = SelectedNode.Name, + ColumnName = columnName, + ColumnType = _newColumnType, + NotNull = _newColumnNotNull, + Description = $"Add column {SelectedNode.Name}.{columnName}", + }); + _newColumnName = ""; + _newColumnType = "TEXT"; + _newColumnNotNull = false; + SaveState(); + await PersistActiveDiagramAsync(); } - private void ZoomIn() => SetZoom(CurrentScale + ZoomStep); + private async Task StageDropColumnAsync(string tableName, string columnName) + { + if (HasPendingTableRename(tableName) || HasPendingColumnDrop(tableName, columnName)) + return; - private void ZoomOut() => SetZoom(CurrentScale - ZoomStep); + _state.PendingOperations.Add(new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.DropColumn, + TableName = tableName, + ColumnName = columnName, + Description = $"Drop column {tableName}.{columnName}", + }); + SaveState(); + await PersistActiveDiagramAsync(); + } - private void ResetZoom() => SetZoom(1); + private async Task StageRenameColumnAsync(string tableName, string columnName) + { + DataModelNode? node = FindNode(tableName); + DataModelColumn? column = node?.Columns.FirstOrDefault(candidate => string.Equals(candidate.Name, columnName, StringComparison.OrdinalIgnoreCase)); + if (node is null || column is null || !CanStageRenameColumn(node, column)) + return; - private void SetZoom(double scale) + string newName = GetColumnRenameValue(tableName, columnName).Trim(); + _state.PendingOperations.Add(new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.RenameColumn, + TableName = tableName, + ColumnName = columnName, + NewColumnName = newName, + Description = $"Rename column {tableName}.{columnName} to {newName}", + }); + _columnRenameNames.Remove(ColumnRenameKey(tableName, columnName)); + SaveState(); + await PersistActiveDiagramAsync(); + } + + private async Task StageRelationshipAsync() + { + if (!CanStageRelationship) + return; + + string relationshipId = Guid.NewGuid().ToString("N"); + var relationship = new DataModelRelationship + { + Id = relationshipId, + LeftTable = _relationshipLeftTable, + LeftColumn = _relationshipLeftColumn, + RightTable = _relationshipRightTable, + RightColumn = _relationshipRightColumn, + Kind = DataModelRelationshipKind.Draft, + OnDelete = _relationshipOnDelete, + }; + + _state.Relationships.Add(relationship); + _state.PendingOperations.Add(new DataModelPendingOperation + { + Id = relationshipId, + Kind = DataModelPendingOperationKind.AddForeignKey, + TableName = _relationshipLeftTable, + ColumnName = _relationshipLeftColumn, + ReferencedTableName = _relationshipRightTable, + ReferencedColumnName = _relationshipRightColumn, + OnDelete = _relationshipOnDelete, + Description = $"Add relationship {_relationshipLeftTable}.{_relationshipLeftColumn} -> {_relationshipRightTable}.{_relationshipRightColumn}", + }); + + _selectedRelationshipId = relationshipId; + _selectedNodeName = null; + _relationshipLeftTable = ""; + _relationshipLeftColumn = ""; + _relationshipRightTable = ""; + _relationshipRightColumn = ""; + _relationshipOnDelete = "RESTRICT"; + SaveState(); + await PersistActiveDiagramAsync(); + } + + private async Task StageDropSelectedRelationshipAsync() + { + if (SelectedRelationship is null || SelectedRelationship.Kind == DataModelRelationshipKind.ExternalArchiveForeignKey) + return; + + DataModelRelationship relationship = SelectedRelationship; + if (relationship.Kind == DataModelRelationshipKind.Draft) + { + _state.Relationships.Remove(relationship); + _state.PendingOperations.RemoveAll(operation => string.Equals(operation.Id, relationship.Id, StringComparison.Ordinal)); + _selectedRelationshipId = null; + SaveState(); + await PersistActiveDiagramAsync(); + return; + } + + string constraintName = relationship.ConstraintName ?? relationship.Id; + if (HasPendingDropForSelectedRelationship) + return; + + _state.PendingOperations.Add(new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.DropForeignKey, + TableName = relationship.LeftTable, + ConstraintName = constraintName, + Description = $"Drop relationship {constraintName}", + }); + SaveState(); + await PersistActiveDiagramAsync(); + } + + private async Task RemovePendingOperationAsync(string operationId) + { + DataModelPendingOperation? operation = _state.PendingOperations.FirstOrDefault(item => string.Equals(item.Id, operationId, StringComparison.Ordinal)); + if (operation is null) + return; + + _state.PendingOperations.Remove(operation); + if (operation.Kind == DataModelPendingOperationKind.CreateTable) + { + _state.Nodes.RemoveAll(node => node.IsDraft && string.Equals(node.Name, operation.TableName, StringComparison.OrdinalIgnoreCase)); + if (string.Equals(_selectedNodeName, operation.TableName, StringComparison.OrdinalIgnoreCase)) + _selectedNodeName = null; + } + else if (operation.Kind == DataModelPendingOperationKind.AddForeignKey) + { + _state.Relationships.RemoveAll(relationship => string.Equals(relationship.Id, operation.Id, StringComparison.Ordinal)); + if (string.Equals(_selectedRelationshipId, operation.Id, StringComparison.Ordinal)) + _selectedRelationshipId = null; + } + + SaveState(); + await PersistActiveDiagramAsync(); + } + + private async Task ApplyPendingChangesAsync() + { + _loading = true; + _error = null; + try + { + DataModelApplyResult result = await Diagrams.ApplyPendingOperationsAsync(_state); + Toast.Success($"Applied {result.Messages.Count} diagram change{(result.Messages.Count == 1 ? "" : "s")}."); + await RefreshAsync(); + } + catch (Exception ex) + { + _error = $"Failed to apply diagram changes: {ex.Message}"; + Toast.Error(_error); + } + finally + { + _loading = false; + } + } + + private Task ZoomIn() => SetZoom(CurrentScale + ZoomStep); + + private Task ZoomOut() => SetZoom(CurrentScale - ZoomStep); + + private Task ResetZoom() => SetZoom(1); + + private async Task SetZoom(double scale) { _state.Scale = Math.Round(NormalizeScale(scale), 2); SaveState(); + await PersistActiveDiagramAsync(); } private void NormalizeZoom() @@ -550,6 +1226,33 @@ Tab.DataModelStateJson = DataModelGraphBuilder.SerializeState(_state); } + private async Task PersistActiveDiagramAsync() + { + EnsureDefaultDiagramName(); + if (string.IsNullOrWhiteSpace(_state.DiagramName)) + return; + + try + { + await Diagrams.SaveDiagramAsync(_state.DiagramName, _state); + await RefreshSavedDiagramsAsync(); + } + catch (Exception ex) + { + Toast.Warning($"Diagram changes were not saved: {ex.Message}"); + } + } + + private void EnsureDefaultDiagramName() + { + if (!UsesDefaultDiagram || !string.IsNullOrWhiteSpace(_state.DiagramName)) + return; + + _state.DiagramName = DefaultDiagramName; + _state.SavedLayoutName = DefaultDiagramName; + _diagramName = DefaultDiagramName; + } + private static string RelationshipKindLabel(DataModelRelationshipKind kind) => kind switch { DataModelRelationshipKind.ExternalArchiveForeignKey => "Archive relationship", diff --git a/src/CSharpDB.Admin/Components/Tabs/QueryTab.razor b/src/CSharpDB.Admin/Components/Tabs/QueryTab.razor index cbe91cda..bf4ed7bd 100644 --- a/src/CSharpDB.Admin/Components/Tabs/QueryTab.razor +++ b/src/CSharpDB.Admin/Components/Tabs/QueryTab.razor @@ -423,6 +423,7 @@ "sys.objects", "sys.saved_queries", "sys.external_tables", + "sys.diagrams", ]; [Parameter] public TabDescriptor? Tab { get; set; } @@ -724,8 +725,7 @@ try { _savedQueries = (await DbClient.GetSavedQueriesAsync()) - .Where(q => !q.Name.StartsWith("__designer_layout:") - && !q.Name.StartsWith(DataModelService.LayoutPrefix, StringComparison.Ordinal)) + .Where(q => !q.Name.StartsWith("__designer_layout:")) .ToList(); if (!string.IsNullOrWhiteSpace(keepSelection) && _savedQueries.Any(q => string.Equals(q.Name, keepSelection, StringComparison.OrdinalIgnoreCase))) diff --git a/src/CSharpDB.Admin/Models/DataModelModels.cs b/src/CSharpDB.Admin/Models/DataModelModels.cs index 67beb06e..8bea4b07 100644 --- a/src/CSharpDB.Admin/Models/DataModelModels.cs +++ b/src/CSharpDB.Admin/Models/DataModelModels.cs @@ -2,10 +2,14 @@ namespace CSharpDB.Admin.Models; public sealed class DataModelState { + public int Version { get; set; } = 1; + public string? DiagramName { get; set; } public List Nodes { get; set; } = []; public List Relationships { get; set; } = []; + public List PendingOperations { get; set; } = []; public List Warnings { get; set; } = []; public string? SavedLayoutName { get; set; } + public string? SchemaSnapshotUtc { get; set; } public double ViewportX { get; set; } public double ViewportY { get; set; } public double Scale { get; set; } = 1; @@ -18,6 +22,7 @@ public sealed class DataModelNode public double X { get; set; } = 20; public double Y { get; set; } = 20; public bool IsCollapsed { get; set; } + public bool IsDraft { get; set; } public List Columns { get; set; } = []; public string? SourceTableName { get; set; } public string? ArchivePath { get; set; } @@ -67,6 +72,52 @@ public enum DataModelRelationshipKind Draft, } +public sealed class DataModelPendingOperation +{ + public string Id { get; set; } = Guid.NewGuid().ToString("N"); + public DataModelPendingOperationKind Kind { get; set; } + public string TableName { get; set; } = ""; + public string? NewTableName { get; set; } + public string? ColumnName { get; set; } + public string? NewColumnName { get; set; } + public string ColumnType { get; set; } = "TEXT"; + public bool NotNull { get; set; } + public List Columns { get; set; } = []; + public string? ReferencedTableName { get; set; } + public string? ReferencedColumnName { get; set; } + public string? ConstraintName { get; set; } + public string OnDelete { get; set; } = "RESTRICT"; + public string Description { get; set; } = ""; +} + +public enum DataModelPendingOperationKind +{ + CreateTable, + DropTable, + RenameTable, + AddColumn, + DropColumn, + RenameColumn, + AddForeignKey, + DropForeignKey, +} + +public sealed class DataModelDiagramSummary +{ + public long Id { get; init; } + public required string Name { get; init; } + public required string CreatedUtc { get; init; } + public required string UpdatedUtc { get; init; } + public int SourceCount { get; init; } + public int PendingOperationCount { get; init; } +} + +public sealed class DataModelApplyResult +{ + public bool Succeeded { get; init; } + public IReadOnlyList Messages { get; init; } = []; +} + public sealed class DataModelSourceOption { public required string Name { get; init; } diff --git a/src/CSharpDB.Admin/Program.cs b/src/CSharpDB.Admin/Program.cs index 9758618a..b37ea3fb 100644 --- a/src/CSharpDB.Admin/Program.cs +++ b/src/CSharpDB.Admin/Program.cs @@ -45,7 +45,9 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); -builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(sp => sp.GetRequiredService()); +builder.Services.AddScoped(sp => sp.GetRequiredService()); builder.Services.AddSingleton(); builder.Services.AddCSharpDbCodeModules(options => options.EnableInProcessExecution = true); builder.Services.AddCSharpDbAdminForms(); diff --git a/src/CSharpDB.Admin/Services/DataModelGraphBuilder.cs b/src/CSharpDB.Admin/Services/DataModelGraphBuilder.cs index 9657d63e..90a5a864 100644 --- a/src/CSharpDB.Admin/Services/DataModelGraphBuilder.cs +++ b/src/CSharpDB.Admin/Services/DataModelGraphBuilder.cs @@ -29,6 +29,74 @@ public static DataModelState Build( return state; } + public static DataModelState BuildFromDiagramState( + IReadOnlyList sources, + DataModelState savedState) + { + var orderedSources = sources + .OrderBy(static source => source.Kind) + .ThenBy(static source => source.TableName, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + var state = new DataModelState + { + Version = savedState.Version <= 0 ? 1 : savedState.Version, + DiagramName = savedState.DiagramName ?? savedState.SavedLayoutName, + SavedLayoutName = savedState.SavedLayoutName, + SchemaSnapshotUtc = savedState.SchemaSnapshotUtc, + ViewportX = savedState.ViewportX, + ViewportY = savedState.ViewportY, + Scale = savedState.Scale <= 0 ? 1 : savedState.Scale, + PendingOperations = savedState.PendingOperations.Select(CloneOperation).ToList(), + }; + + var selectedNames = new HashSet( + savedState.Nodes.Select(static node => node.Name), + StringComparer.OrdinalIgnoreCase); + + int index = 0; + foreach (DataModelNode savedNode in savedState.Nodes) + { + DataModelSourceMetadata? source = FindSource(orderedSources, savedNode.Name); + if (source is null) + { + if (savedNode.IsDraft || state.PendingOperations.Any(operation => + operation.Kind == DataModelPendingOperationKind.CreateTable && + string.Equals(operation.TableName, savedNode.Name, StringComparison.OrdinalIgnoreCase))) + { + state.Nodes.Add(CloneNode(savedNode)); + } + else + { + state.Warnings.Add($"Diagram source '{savedNode.Name}' no longer exists in the database."); + } + + continue; + } + + DataModelNode node = CreateNode(source, index++); + node.X = savedNode.X; + node.Y = savedNode.Y; + node.IsCollapsed = savedNode.IsCollapsed; + state.Nodes.Add(node); + } + + AddRelationships(orderedSources, state, selectedNames); + foreach (DataModelRelationship relationship in savedState.Relationships.Where(static relationship => relationship.Kind == DataModelRelationshipKind.Draft)) + { + if (!state.Relationships.Any(existing => string.Equals(existing.Id, relationship.Id, StringComparison.Ordinal))) + state.Relationships.Add(CloneRelationship(relationship)); + } + + foreach (string warning in savedState.Warnings) + { + if (!state.Warnings.Contains(warning, StringComparer.OrdinalIgnoreCase)) + state.Warnings.Add(warning); + } + + return state; + } + public static QueryDesignerState ToQueryDesignerState(DataModelState state) { var selectedNames = new HashSet( @@ -226,4 +294,74 @@ private static bool SourceMatches(DataModelSourceMetadata source, string sourceN || (!string.IsNullOrWhiteSpace(source.SourceTableName) && string.Equals(source.SourceTableName, sourceName, StringComparison.OrdinalIgnoreCase)); } + + private static DataModelNode CloneNode(DataModelNode node) => new() + { + Name = node.Name, + Kind = node.Kind, + X = node.X, + Y = node.Y, + IsCollapsed = node.IsCollapsed, + IsDraft = node.IsDraft, + SourceTableName = node.SourceTableName, + ArchivePath = node.ArchivePath, + ArchiveCreatedUtc = node.ArchiveCreatedUtc, + RowCount = node.RowCount, + IndexCount = node.IndexCount, + TriggerCount = node.TriggerCount, + Warnings = node.Warnings.ToList(), + Columns = node.Columns.Select(static column => new DataModelColumn + { + Name = column.Name, + TypeLabel = column.TypeLabel, + IsPrimaryKey = column.IsPrimaryKey, + IsIdentity = column.IsIdentity, + Nullable = column.Nullable, + Collation = column.Collation, + IsForeignKey = column.IsForeignKey, + IsIndexed = column.IsIndexed, + }).ToList(), + }; + + private static DataModelRelationship CloneRelationship(DataModelRelationship relationship) => new() + { + Id = relationship.Id, + LeftTable = relationship.LeftTable, + LeftColumn = relationship.LeftColumn, + RightTable = relationship.RightTable, + RightColumn = relationship.RightColumn, + Kind = relationship.Kind, + IsResolved = relationship.IsResolved, + ConstraintName = relationship.ConstraintName, + OnDelete = relationship.OnDelete, + Warning = relationship.Warning, + }; + + private static DataModelPendingOperation CloneOperation(DataModelPendingOperation operation) => new() + { + Id = operation.Id, + Kind = operation.Kind, + TableName = operation.TableName, + NewTableName = operation.NewTableName, + ColumnName = operation.ColumnName, + NewColumnName = operation.NewColumnName, + ColumnType = operation.ColumnType, + NotNull = operation.NotNull, + Columns = operation.Columns.Select(static column => new DataModelColumn + { + Name = column.Name, + TypeLabel = column.TypeLabel, + IsPrimaryKey = column.IsPrimaryKey, + IsIdentity = column.IsIdentity, + Nullable = column.Nullable, + Collation = column.Collation, + IsForeignKey = column.IsForeignKey, + IsIndexed = column.IsIndexed, + }).ToList(), + ReferencedTableName = operation.ReferencedTableName, + ReferencedColumnName = operation.ReferencedColumnName, + ConstraintName = operation.ConstraintName, + OnDelete = operation.OnDelete, + Description = operation.Description, + }; } diff --git a/src/CSharpDB.Admin/Services/DataModelService.cs b/src/CSharpDB.Admin/Services/DataModelService.cs index 6e11f560..374b4d9d 100644 --- a/src/CSharpDB.Admin/Services/DataModelService.cs +++ b/src/CSharpDB.Admin/Services/DataModelService.cs @@ -13,17 +13,25 @@ public interface IDataModelService { Task BuildModelAsync(string? seedSourceName = null, int autoLayoutLimit = DataModelGraphBuilder.DefaultAutoLayoutLimit, CancellationToken ct = default); Task> GetSourceOptionsAsync(CancellationToken ct = default); - Task> GetSavedLayoutsAsync(CancellationToken ct = default); - Task LoadLayoutAsync(string fullName, CancellationToken ct = default); - Task SaveLayoutAsync(string name, DataModelState state, CancellationToken ct = default); - Task DeleteLayoutAsync(string fullName, CancellationToken ct = default); string BuildPreviewSql(DataModelState state); + string BuildPendingOperationsPreview(DataModelState state); QueryDesignerState ToQueryDesignerState(DataModelState state); } -public sealed class DataModelService(ICSharpDbClient client) : IDataModelService +public interface IDataModelDiagramService { - public const string LayoutPrefix = "__data_model_layout:"; + Task> GetDiagramsAsync(CancellationToken ct = default); + Task LoadDiagramAsync(string name, CancellationToken ct = default); + Task SaveDiagramAsync(string name, DataModelState state, CancellationToken ct = default); + Task RenameDiagramAsync(string existingName, string newName, DataModelState state, CancellationToken ct = default); + Task DeleteDiagramAsync(string name, CancellationToken ct = default); + Task ApplyPendingOperationsAsync(DataModelState state, CancellationToken ct = default); +} + +public sealed class DataModelService(ICSharpDbClient client) : IDataModelService, IDataModelDiagramService +{ + private const string DiagramTableName = "__data_model_diagrams"; + private const string DiagramNameIndexName = "idx___data_model_diagrams_name"; public async Task BuildModelAsync( string? seedSourceName = null, @@ -31,7 +39,9 @@ public async Task BuildModelAsync( CancellationToken ct = default) { IReadOnlyList sources = await LoadSourcesAsync(ct); - return DataModelGraphBuilder.Build(sources, seedSourceName, autoLayoutLimit); + DataModelState state = DataModelGraphBuilder.Build(sources, seedSourceName, autoLayoutLimit); + state.SchemaSnapshotUtc = DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture); + return state; } public async Task> GetSourceOptionsAsync(CancellationToken ct = default) @@ -49,42 +59,148 @@ public async Task> GetSourceOptionsAsync(Ca .ToArray(); } - public async Task> GetSavedLayoutsAsync(CancellationToken ct = default) + public async Task> GetDiagramsAsync(CancellationToken ct = default) { - IReadOnlyList saved = await client.GetSavedQueriesAsync(ct); - return saved - .Where(static query => query.Name.StartsWith(LayoutPrefix, StringComparison.Ordinal)) - .OrderBy(static query => query.Name, StringComparer.OrdinalIgnoreCase) - .ToArray(); + await EnsureDiagramCatalogAsync(ct); + SqlExecutionResult result = await client.ExecuteSqlAsync( + $""" + SELECT id, name, diagram_json, created_utc, updated_utc + FROM {DiagramTableName} + ORDER BY name; + """, + ct); + + ThrowIfSqlError(result); + var diagrams = new List(); + foreach (object?[] row in result.Rows ?? []) + { + if (row.Length < 5 || row[1] is null) + continue; + + string name = Convert.ToString(row[1], CultureInfo.InvariantCulture) ?? string.Empty; + if (string.IsNullOrWhiteSpace(name)) + continue; + + string json = Convert.ToString(row[2], CultureInfo.InvariantCulture) ?? string.Empty; + int sourceCount = 0; + int pendingCount = 0; + try + { + DataModelState? state = DataModelGraphBuilder.DeserializeState(json); + sourceCount = state?.Nodes.Count ?? 0; + pendingCount = state?.PendingOperations.Count ?? 0; + } + catch + { + } + + diagrams.Add(new DataModelDiagramSummary + { + Id = row[0] is null ? 0 : Convert.ToInt64(row[0], CultureInfo.InvariantCulture), + Name = name, + CreatedUtc = Convert.ToString(row[3], CultureInfo.InvariantCulture) ?? string.Empty, + UpdatedUtc = Convert.ToString(row[4], CultureInfo.InvariantCulture) ?? string.Empty, + SourceCount = sourceCount, + PendingOperationCount = pendingCount, + }); + } + + return diagrams; } - public async Task LoadLayoutAsync(string fullName, CancellationToken ct = default) + public async Task LoadDiagramAsync(string name, CancellationToken ct = default) { - SavedQueryDefinition? saved = await client.GetSavedQueryAsync(fullName, ct); + await EnsureDiagramCatalogAsync(ct); + string normalized = NormalizeDiagramName(name); + SqlExecutionResult result = await client.ExecuteSqlAsync( + $""" + SELECT diagram_json + FROM {DiagramTableName} + WHERE name = {FormatSqlStringLiteral(normalized)} + LIMIT 1; + """, + ct); + + ThrowIfSqlError(result); + object?[]? row = result.Rows?.FirstOrDefault(); + if (row is null || row.Length == 0 || row[0] is null) + return null; + + string json = Convert.ToString(row[0], CultureInfo.InvariantCulture) ?? string.Empty; + DataModelState? saved = DataModelGraphBuilder.DeserializeState(json); if (saved is null) return null; - DataModelState? state = DataModelGraphBuilder.DeserializeState(saved.SqlText); - if (state is not null) - state.SavedLayoutName = fullName.StartsWith(LayoutPrefix, StringComparison.Ordinal) - ? fullName[LayoutPrefix.Length..] - : fullName; + saved.DiagramName = normalized; + saved.SavedLayoutName = normalized; + IReadOnlyList sources = await LoadSourcesAsync(ct); + return DataModelGraphBuilder.BuildFromDiagramState(sources, saved); + } - return state; + public async Task SaveDiagramAsync(string name, DataModelState state, CancellationToken ct = default) + { + await EnsureDiagramCatalogAsync(ct); + string normalized = NormalizeDiagramName(name); + state.Version = 1; + state.DiagramName = normalized; + state.SavedLayoutName = normalized; + state.SchemaSnapshotUtc ??= DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture); + string json = DataModelGraphBuilder.SerializeState(state); + string now = DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture); + + bool exists = await DiagramExistsAsync(normalized, ct); + SqlExecutionResult result = exists + ? await client.ExecuteSqlAsync( + $""" + UPDATE {DiagramTableName} + SET diagram_json = {FormatSqlStringLiteral(json)}, + updated_utc = {FormatSqlStringLiteral(now)} + WHERE name = {FormatSqlStringLiteral(normalized)}; + """, + ct) + : await client.ExecuteSqlAsync( + $""" + INSERT INTO {DiagramTableName} (name, diagram_json, created_utc, updated_utc) + VALUES ({FormatSqlStringLiteral(normalized)}, {FormatSqlStringLiteral(json)}, {FormatSqlStringLiteral(now)}, {FormatSqlStringLiteral(now)}); + """, + ct); + + ThrowIfSqlError(result); } - public async Task SaveLayoutAsync(string name, DataModelState state, CancellationToken ct = default) + public async Task RenameDiagramAsync(string existingName, string newName, DataModelState state, CancellationToken ct = default) { - string trimmed = name.Trim(); - if (string.IsNullOrWhiteSpace(trimmed)) - throw new ArgumentException("Layout name is required.", nameof(name)); + await EnsureDiagramCatalogAsync(ct); + string existing = NormalizeDiagramName(existingName); + string normalized = NormalizeDiagramName(newName); + if (!string.Equals(existing, normalized, StringComparison.OrdinalIgnoreCase) && await DiagramExistsAsync(normalized, ct)) + throw new InvalidOperationException($"A diagram named '{normalized}' already exists."); + + state.DiagramName = normalized; + state.SavedLayoutName = normalized; + string json = DataModelGraphBuilder.SerializeState(state); + string now = DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture); + SqlExecutionResult result = await client.ExecuteSqlAsync( + $""" + UPDATE {DiagramTableName} + SET name = {FormatSqlStringLiteral(normalized)}, + diagram_json = {FormatSqlStringLiteral(json)}, + updated_utc = {FormatSqlStringLiteral(now)} + WHERE name = {FormatSqlStringLiteral(existing)}; + """, + ct); - state.SavedLayoutName = trimmed; - await client.UpsertSavedQueryAsync(LayoutPrefix + trimmed, DataModelGraphBuilder.SerializeState(state), ct); + ThrowIfSqlError(result); } - public Task DeleteLayoutAsync(string fullName, CancellationToken ct = default) => - client.DeleteSavedQueryAsync(fullName, ct); + public async Task DeleteDiagramAsync(string name, CancellationToken ct = default) + { + await EnsureDiagramCatalogAsync(ct); + SqlExecutionResult result = await client.ExecuteSqlAsync( + $"DELETE FROM {DiagramTableName} WHERE name = {FormatSqlStringLiteral(NormalizeDiagramName(name))};", + ct); + ThrowIfSqlError(result); + } public QueryDesignerState ToQueryDesignerState(DataModelState state) => DataModelGraphBuilder.ToQueryDesignerState(state); @@ -164,6 +280,286 @@ public string BuildPreviewSql(DataModelState state) return sb.ToString().TrimEnd(); } + public string BuildPendingOperationsPreview(DataModelState state) + { + if (state.PendingOperations.Count == 0) + return "-- No pending schema changes."; + + var sb = new StringBuilder(); + sb.AppendLine("-- Pending schema changes. Apply from the diagram toolbar after review."); + foreach (DataModelPendingOperation operation in state.PendingOperations) + { + sb.AppendLine(); + sb.AppendLine("-- " + DescribeOperation(operation)); + sb.AppendLine(BuildOperationSql(operation)); + } + + return sb.ToString().TrimEnd(); + } + + public async Task ApplyPendingOperationsAsync(DataModelState state, CancellationToken ct = default) + { + if (state.PendingOperations.Count == 0) + return new DataModelApplyResult { Succeeded = true, Messages = ["No pending schema changes."] }; + + var messages = new List(); + foreach (DataModelPendingOperation operation in state.PendingOperations.ToArray()) + { + await ApplyOperationAsync(operation, ct); + ApplyOperationToState(state, operation); + messages.Add(DescribeOperation(operation)); + } + + state.PendingOperations.Clear(); + state.SchemaSnapshotUtc = DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture); + if (!string.IsNullOrWhiteSpace(state.DiagramName)) + await SaveDiagramAsync(state.DiagramName, state, ct); + + return new DataModelApplyResult + { + Succeeded = true, + Messages = messages, + }; + } + + private static void ApplyOperationToState(DataModelState state, DataModelPendingOperation operation) + { + switch (operation.Kind) + { + case DataModelPendingOperationKind.CreateTable: + foreach (DataModelNode node in state.Nodes.Where(node => string.Equals(node.Name, operation.TableName, StringComparison.OrdinalIgnoreCase))) + { + node.IsDraft = false; + node.Warnings.RemoveAll(static warning => warning.Contains("Pending create table", StringComparison.OrdinalIgnoreCase)); + } + break; + + case DataModelPendingOperationKind.DropTable: + state.Nodes.RemoveAll(node => string.Equals(node.Name, operation.TableName, StringComparison.OrdinalIgnoreCase)); + state.Relationships.RemoveAll(relationship => + string.Equals(relationship.LeftTable, operation.TableName, StringComparison.OrdinalIgnoreCase) || + string.Equals(relationship.RightTable, operation.TableName, StringComparison.OrdinalIgnoreCase)); + break; + + case DataModelPendingOperationKind.RenameTable: + string newTableName = RequireValue(operation.NewTableName, "new table name"); + foreach (DataModelNode node in state.Nodes.Where(node => string.Equals(node.Name, operation.TableName, StringComparison.OrdinalIgnoreCase))) + node.Name = newTableName; + foreach (DataModelRelationship relationship in state.Relationships) + { + if (string.Equals(relationship.LeftTable, operation.TableName, StringComparison.OrdinalIgnoreCase)) + relationship.LeftTable = newTableName; + if (string.Equals(relationship.RightTable, operation.TableName, StringComparison.OrdinalIgnoreCase)) + relationship.RightTable = newTableName; + } + break; + + case DataModelPendingOperationKind.AddColumn: + DataModelNode? addNode = FindStateNode(state, operation.TableName); + if (addNode is not null && !addNode.Columns.Any(column => string.Equals(column.Name, operation.ColumnName, StringComparison.OrdinalIgnoreCase))) + { + addNode.Columns.Add(new DataModelColumn + { + Name = RequireValue(operation.ColumnName, "column name"), + TypeLabel = NormalizeTypeLabel(operation.ColumnType), + Nullable = !operation.NotNull, + }); + } + break; + + case DataModelPendingOperationKind.DropColumn: + DataModelNode? dropNode = FindStateNode(state, operation.TableName); + if (dropNode is not null) + dropNode.Columns.RemoveAll(column => string.Equals(column.Name, operation.ColumnName, StringComparison.OrdinalIgnoreCase)); + state.Relationships.RemoveAll(relationship => + (string.Equals(relationship.LeftTable, operation.TableName, StringComparison.OrdinalIgnoreCase) && + string.Equals(relationship.LeftColumn, operation.ColumnName, StringComparison.OrdinalIgnoreCase)) || + (string.Equals(relationship.RightTable, operation.TableName, StringComparison.OrdinalIgnoreCase) && + string.Equals(relationship.RightColumn, operation.ColumnName, StringComparison.OrdinalIgnoreCase))); + break; + + case DataModelPendingOperationKind.RenameColumn: + DataModelNode? renameNode = FindStateNode(state, operation.TableName); + DataModelColumn? column = renameNode?.Columns.FirstOrDefault(column => string.Equals(column.Name, operation.ColumnName, StringComparison.OrdinalIgnoreCase)); + string newColumnName = RequireValue(operation.NewColumnName, "new column name"); + if (column is not null) + column.Name = newColumnName; + foreach (DataModelRelationship relationship in state.Relationships) + { + if (string.Equals(relationship.LeftTable, operation.TableName, StringComparison.OrdinalIgnoreCase) && + string.Equals(relationship.LeftColumn, operation.ColumnName, StringComparison.OrdinalIgnoreCase)) + relationship.LeftColumn = newColumnName; + if (string.Equals(relationship.RightTable, operation.TableName, StringComparison.OrdinalIgnoreCase) && + string.Equals(relationship.RightColumn, operation.ColumnName, StringComparison.OrdinalIgnoreCase)) + relationship.RightColumn = newColumnName; + } + break; + + case DataModelPendingOperationKind.AddForeignKey: + state.Relationships.RemoveAll(relationship => string.Equals(relationship.Id, operation.Id, StringComparison.Ordinal)); + break; + + case DataModelPendingOperationKind.DropForeignKey: + state.Relationships.RemoveAll(relationship => + !string.IsNullOrWhiteSpace(operation.ConstraintName) && + string.Equals(relationship.ConstraintName ?? relationship.Id, operation.ConstraintName, StringComparison.OrdinalIgnoreCase)); + break; + } + } + + private static DataModelNode? FindStateNode(DataModelState state, string tableName) => + state.Nodes.FirstOrDefault(node => string.Equals(node.Name, tableName, StringComparison.OrdinalIgnoreCase)); + + private async Task ApplyOperationAsync(DataModelPendingOperation operation, CancellationToken ct) + { + switch (operation.Kind) + { + case DataModelPendingOperationKind.CreateTable: + ThrowIfSqlError(await client.ExecuteSqlAsync(BuildOperationSql(operation), ct)); + break; + + case DataModelPendingOperationKind.DropTable: + await client.DropTableAsync(operation.TableName, ct); + break; + + case DataModelPendingOperationKind.RenameTable: + await client.RenameTableAsync(operation.TableName, RequireValue(operation.NewTableName, "new table name"), ct); + break; + + case DataModelPendingOperationKind.AddColumn: + await client.AddColumnAsync( + operation.TableName, + RequireValue(operation.ColumnName, "column name"), + ParseClientDbType(operation.ColumnType), + operation.NotNull, + ct); + break; + + case DataModelPendingOperationKind.DropColumn: + await client.DropColumnAsync(operation.TableName, RequireValue(operation.ColumnName, "column name"), ct); + break; + + case DataModelPendingOperationKind.RenameColumn: + await client.RenameColumnAsync( + operation.TableName, + RequireValue(operation.ColumnName, "column name"), + RequireValue(operation.NewColumnName, "new column name"), + ct); + break; + + case DataModelPendingOperationKind.AddForeignKey: + ForeignKeyMigrationResult migration = await client.MigrateForeignKeysAsync( + new ForeignKeyMigrationRequest + { + ValidateOnly = false, + Constraints = + [ + new ForeignKeyMigrationConstraintSpec + { + TableName = operation.TableName, + ColumnName = RequireValue(operation.ColumnName, "column name"), + ReferencedTableName = RequireValue(operation.ReferencedTableName, "referenced table"), + ReferencedColumnName = operation.ReferencedColumnName, + OnDelete = ParseOnDeleteAction(operation.OnDelete), + }, + ], + }, + ct); + if (!migration.Succeeded) + throw new InvalidOperationException($"Foreign key migration failed with {migration.ViolationCount} violation(s)."); + break; + + case DataModelPendingOperationKind.DropForeignKey: + ThrowIfSqlError(await client.ExecuteSqlAsync(BuildOperationSql(operation), ct)); + break; + + default: + throw new InvalidOperationException($"Unsupported diagram operation '{operation.Kind}'."); + } + } + + private static string BuildOperationSql(DataModelPendingOperation operation) => operation.Kind switch + { + DataModelPendingOperationKind.CreateTable => BuildCreateTableSql(operation), + DataModelPendingOperationKind.DropTable => $"DROP TABLE {FormatIdentifier(operation.TableName)};", + DataModelPendingOperationKind.RenameTable => $"ALTER TABLE {FormatIdentifier(operation.TableName)} RENAME TO {FormatIdentifier(RequireValue(operation.NewTableName, "new table name"))};", + DataModelPendingOperationKind.AddColumn => $"ALTER TABLE {FormatIdentifier(operation.TableName)} ADD COLUMN {BuildColumnSql(operation)};", + DataModelPendingOperationKind.DropColumn => $"ALTER TABLE {FormatIdentifier(operation.TableName)} DROP COLUMN {FormatIdentifier(RequireValue(operation.ColumnName, "column name"))};", + DataModelPendingOperationKind.RenameColumn => $"ALTER TABLE {FormatIdentifier(operation.TableName)} RENAME COLUMN {FormatIdentifier(RequireValue(operation.ColumnName, "column name"))} TO {FormatIdentifier(RequireValue(operation.NewColumnName, "new column name"))};", + DataModelPendingOperationKind.AddForeignKey => $"-- Uses validated foreign-key migration: {FormatIdentifier(operation.TableName)}.{FormatIdentifier(RequireValue(operation.ColumnName, "column name"))} -> {FormatIdentifier(RequireValue(operation.ReferencedTableName, "referenced table"))}.{FormatIdentifier(RequireValue(operation.ReferencedColumnName, "referenced column"))}", + DataModelPendingOperationKind.DropForeignKey => $"ALTER TABLE {FormatIdentifier(operation.TableName)} DROP CONSTRAINT {FormatIdentifier(RequireValue(operation.ConstraintName, "constraint name"))};", + _ => "-- Unsupported diagram operation.", + }; + + private static string BuildCreateTableSql(DataModelPendingOperation operation) + { + IReadOnlyList columns = operation.Columns.Count > 0 + ? operation.Columns + : + [ + new DataModelColumn + { + Name = "Id", + TypeLabel = "INTEGER", + IsPrimaryKey = true, + IsIdentity = true, + Nullable = false, + }, + ]; + + var sb = new StringBuilder() + .Append("CREATE TABLE ") + .Append(FormatIdentifier(operation.TableName)) + .AppendLine(" ("); + for (int i = 0; i < columns.Count; i++) + { + DataModelColumn column = columns[i]; + sb.Append(" ") + .Append(FormatIdentifier(column.Name)) + .Append(' ') + .Append(NormalizeTypeLabel(column.TypeLabel)); + if (column.IsPrimaryKey) + sb.Append(" PRIMARY KEY"); + if (column.IsIdentity) + sb.Append(" IDENTITY"); + if (!column.Nullable && !column.IsPrimaryKey) + sb.Append(" NOT NULL"); + if (!string.IsNullOrWhiteSpace(column.Collation)) + sb.Append(" COLLATE ").Append(column.Collation); + if (i < columns.Count - 1) + sb.Append(','); + sb.AppendLine(); + } + + sb.Append(");"); + return sb.ToString(); + } + + private static string BuildColumnSql(DataModelPendingOperation operation) + { + string column = $"{FormatIdentifier(RequireValue(operation.ColumnName, "column name"))} {NormalizeTypeLabel(operation.ColumnType)}"; + return operation.NotNull ? column + " NOT NULL" : column; + } + + private static string DescribeOperation(DataModelPendingOperation operation) + { + if (!string.IsNullOrWhiteSpace(operation.Description)) + return operation.Description; + + return operation.Kind switch + { + DataModelPendingOperationKind.CreateTable => $"Create table {operation.TableName}", + DataModelPendingOperationKind.DropTable => $"Drop table {operation.TableName}", + DataModelPendingOperationKind.RenameTable => $"Rename table {operation.TableName} to {operation.NewTableName}", + DataModelPendingOperationKind.AddColumn => $"Add column {operation.TableName}.{operation.ColumnName}", + DataModelPendingOperationKind.DropColumn => $"Drop column {operation.TableName}.{operation.ColumnName}", + DataModelPendingOperationKind.RenameColumn => $"Rename column {operation.TableName}.{operation.ColumnName} to {operation.NewColumnName}", + DataModelPendingOperationKind.AddForeignKey => $"Add relationship {operation.TableName}.{operation.ColumnName} -> {operation.ReferencedTableName}.{operation.ReferencedColumnName}", + DataModelPendingOperationKind.DropForeignKey => $"Drop relationship {operation.ConstraintName}", + _ => operation.Kind.ToString(), + }; + } + private async Task> LoadSourcesAsync(CancellationToken ct) { var sources = new List(); @@ -341,6 +737,85 @@ private static string FormatIdentifier(string identifier) private static string FormatSqlStringLiteral(string value) => "'" + value.Replace("'", "''", StringComparison.Ordinal) + "'"; + private async Task EnsureDiagramCatalogAsync(CancellationToken ct) + { + ThrowIfSqlError(await client.ExecuteSqlAsync( + $""" + CREATE TABLE IF NOT EXISTS {DiagramTableName} ( + id INTEGER PRIMARY KEY IDENTITY, + name TEXT NOT NULL, + diagram_json TEXT NOT NULL, + created_utc TEXT NOT NULL, + updated_utc TEXT NOT NULL + ); + """, + ct)); + + ThrowIfSqlError(await client.ExecuteSqlAsync( + $""" + CREATE UNIQUE INDEX IF NOT EXISTS {DiagramNameIndexName} + ON {DiagramTableName} (name); + """, + ct)); + } + + private async Task DiagramExistsAsync(string name, CancellationToken ct) + { + SqlExecutionResult result = await client.ExecuteSqlAsync( + $"SELECT COUNT(*) FROM {DiagramTableName} WHERE name = {FormatSqlStringLiteral(name)};", + ct); + ThrowIfSqlError(result); + object? value = result.Rows?.FirstOrDefault()?.FirstOrDefault(); + return value is not null && Convert.ToInt64(value, CultureInfo.InvariantCulture) > 0; + } + + private static void ThrowIfSqlError(SqlExecutionResult result) + { + if (!string.IsNullOrWhiteSpace(result.Error)) + throw new InvalidOperationException(result.Error); + } + + private static string NormalizeDiagramName(string name) + { + string trimmed = name.Trim(); + if (string.IsNullOrWhiteSpace(trimmed)) + throw new ArgumentException("Diagram name is required.", nameof(name)); + return trimmed; + } + + private static string RequireValue(string? value, string name) + { + if (string.IsNullOrWhiteSpace(value)) + throw new InvalidOperationException($"Diagram operation requires {name}."); + return value.Trim(); + } + + private static string NormalizeTypeLabel(string value) + { + string normalized = value.Trim().ToUpperInvariant(); + return normalized switch + { + "INT" => "INTEGER", + "INTEGER" or "REAL" or "TEXT" or "BLOB" => normalized, + _ => "TEXT", + }; + } + + private static DbType ParseClientDbType(string value) => NormalizeTypeLabel(value) switch + { + "INTEGER" => DbType.Integer, + "REAL" => DbType.Real, + "BLOB" => DbType.Blob, + _ => DbType.Text, + }; + + private static ForeignKeyOnDeleteAction ParseOnDeleteAction(string? value) + { + return string.Equals(value, "CASCADE", StringComparison.OrdinalIgnoreCase) + ? ForeignKeyOnDeleteAction.Cascade + : ForeignKeyOnDeleteAction.Restrict; + } + private sealed record ExternalTableRegistration( string TableName, string Path, diff --git a/src/CSharpDB.Admin/Services/HostCallbackCatalogService.cs b/src/CSharpDB.Admin/Services/HostCallbackCatalogService.cs index ba6bafbb..b041a712 100644 --- a/src/CSharpDB.Admin/Services/HostCallbackCatalogService.cs +++ b/src/CSharpDB.Admin/Services/HostCallbackCatalogService.cs @@ -276,8 +276,7 @@ private static async Task AddDatabaseReferencesAsync(List IReadOnlyList savedQueries = await dbClient.GetSavedQueriesAsync(); foreach (SavedQueryDefinition query in savedQueries) { - if (query.Name.StartsWith("__designer_layout:", StringComparison.Ordinal) - || query.Name.StartsWith(DataModelService.LayoutPrefix, StringComparison.Ordinal)) + if (query.Name.StartsWith("__designer_layout:", StringComparison.Ordinal)) { continue; } diff --git a/src/CSharpDB.Admin/wwwroot/css/app.css b/src/CSharpDB.Admin/wwwroot/css/app.css index 6fdc6e33..b39232f3 100644 --- a/src/CSharpDB.Admin/wwwroot/css/app.css +++ b/src/CSharpDB.Admin/wwwroot/css/app.css @@ -3102,6 +3102,18 @@ body { color: var(--accent-magenta); } +.schema-node.draft { + border-style: dashed; +} + +.schema-node.draft .designer-table-node-header { + background: rgba(224, 175, 104, 0.13); +} + +.schema-node.draft .designer-table-node-header .bi { + color: var(--accent-yellow); +} + .schema-node-warning { color: var(--accent-yellow); flex-shrink: 0; @@ -3239,6 +3251,84 @@ body { overflow-wrap: anywhere; } +.data-model-schema-actions { + margin-top: 8px; + display: flex; + flex-wrap: wrap; + gap: 6px; +} + +.data-model-inline-action { + width: 100%; + display: flex; + flex-wrap: wrap; + gap: 6px; + align-items: center; +} + +.data-model-inline-action .schema-input { + flex: 1 1 120px; + min-width: 0; +} + +.data-model-column-action-list, +.data-model-relationship-editor { + width: 100%; + display: flex; + flex-direction: column; + gap: 6px; +} + +.data-model-column-action-row { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(86px, 1fr) 32px 32px; + gap: 6px; + align-items: center; +} + +.data-model-column-action-row span { + min-width: 0; + display: flex; + flex-direction: column; +} + +.data-model-column-action-row strong, +.data-model-column-action-row small { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.data-model-relationship-editor { + margin-top: 10px; + padding-top: 10px; + border-top: 1px solid var(--border-color); +} + +.data-model-pending-row { + display: grid; + grid-template-columns: minmax(0, 1fr) 22px; + align-items: center; + gap: 8px; + padding: 6px 8px; + margin-bottom: 4px; + border: 1px solid var(--border-color); + border-radius: var(--radius-sm); + background: var(--bg-secondary); + color: var(--text-secondary); + font-size: 12px; +} + +.data-model-pending-row span { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.data-model-schema-dialog { + border-bottom: 1px solid var(--border-color); +} + .data-model-inspector-section.preview pre { max-height: 280px; overflow: auto; diff --git a/src/CSharpDB.Client/Internal/EngineTransportClient.cs b/src/CSharpDB.Client/Internal/EngineTransportClient.cs index 18b797a1..8d4fb4d5 100644 --- a/src/CSharpDB.Client/Internal/EngineTransportClient.cs +++ b/src/CSharpDB.Client/Internal/EngineTransportClient.cs @@ -25,6 +25,7 @@ internal sealed partial class EngineTransportClient : ICSharpDbClient, IEngineBa private const string ProcedureTableName = "__procedures"; private const string SavedQueryTableName = "__saved_queries"; private const string ExternalTablesTableName = "__external_tables"; + private const string DataModelDiagramsTableName = "__data_model_diagrams"; private static readonly Regex s_identifierPattern = new("^[A-Za-z_][A-Za-z0-9_]*$", RegexOptions.Compiled); private readonly string _databasePath; @@ -1151,7 +1152,8 @@ private static bool IsInternalTable(string tableName) => tableName.StartsWith(CollectionPrefix, StringComparison.Ordinal) || string.Equals(tableName, ProcedureTableName, StringComparison.OrdinalIgnoreCase) || string.Equals(tableName, SavedQueryTableName, StringComparison.OrdinalIgnoreCase) - || string.Equals(tableName, ExternalTablesTableName, StringComparison.OrdinalIgnoreCase); + || string.Equals(tableName, ExternalTablesTableName, StringComparison.OrdinalIgnoreCase) + || string.Equals(tableName, DataModelDiagramsTableName, StringComparison.OrdinalIgnoreCase); private static int NormalizePage(int page) => page < 1 ? 1 : page; diff --git a/src/CSharpDB.Engine/DatabaseMaintenanceCoordinator.cs b/src/CSharpDB.Engine/DatabaseMaintenanceCoordinator.cs index fbdbc3a9..3010e426 100644 --- a/src/CSharpDB.Engine/DatabaseMaintenanceCoordinator.cs +++ b/src/CSharpDB.Engine/DatabaseMaintenanceCoordinator.cs @@ -16,6 +16,7 @@ public static class DatabaseMaintenanceCoordinator private const string CollectionIndexPrefix = "_cidx_"; private const string ProcedureTableName = "__procedures"; private const string SavedQueryTableName = "__saved_queries"; + private const string DataModelDiagramsTableName = "__data_model_diagrams"; public static async ValueTask GetMaintenanceReportAsync( string databasePath, @@ -770,7 +771,7 @@ private static async ValueTask OpenStorageContextAsync(str private static IEnumerable GetTablesToCopy(SchemaCatalog catalog) { return catalog.GetTableNames() - .Concat([ProcedureTableName, SavedQueryTableName]) + .Concat([ProcedureTableName, SavedQueryTableName, DataModelDiagramsTableName]) .Distinct(StringComparer.OrdinalIgnoreCase) .OrderBy(name => name, StringComparer.OrdinalIgnoreCase); } @@ -807,7 +808,8 @@ private static async ValueTask CopyMetadataTableRowsAsync( private static bool IsClientMetadataTable(string tableName) { return string.Equals(tableName, ProcedureTableName, StringComparison.OrdinalIgnoreCase) || - string.Equals(tableName, SavedQueryTableName, StringComparison.OrdinalIgnoreCase); + string.Equals(tableName, SavedQueryTableName, StringComparison.OrdinalIgnoreCase) || + string.Equals(tableName, DataModelDiagramsTableName, StringComparison.OrdinalIgnoreCase); } private static void TryDeleteFile(string path) diff --git a/src/CSharpDB.Execution/QueryPlanner.cs b/src/CSharpDB.Execution/QueryPlanner.cs index fdb5f50e..fef91588 100644 --- a/src/CSharpDB.Execution/QueryPlanner.cs +++ b/src/CSharpDB.Execution/QueryPlanner.cs @@ -19,6 +19,7 @@ public sealed class QueryPlanner { private const string InternalSavedQueriesTableName = "__saved_queries"; private const string InternalExternalTablesTableName = "__external_tables"; + private const string InternalDataModelDiagramsTableName = "__data_model_diagrams"; private const int AnalyzeHistogramBucketCount = 16; private const int AnalyzeFrequentValueCount = 8; private const int MaxJoinReorderDpLeafCount = 6; @@ -350,6 +351,16 @@ public override int GetHashCode() => new ColumnDefinition { Name = "created_utc", Type = DbType.Text, Nullable = false }, ]; + private static readonly ColumnDefinition[] SystemDiagramsColumns = + [ + new ColumnDefinition { Name = "name", Type = DbType.Text, Nullable = false }, + new ColumnDefinition { Name = "created_utc", Type = DbType.Text, Nullable = false }, + new ColumnDefinition { Name = "updated_utc", Type = DbType.Text, Nullable = false }, + new ColumnDefinition { Name = "source_count", Type = DbType.Integer, Nullable = false }, + new ColumnDefinition { Name = "table_count", Type = DbType.Integer, Nullable = false }, + new ColumnDefinition { Name = "pending_operation_count", Type = DbType.Integer, Nullable = false }, + ]; + private static readonly ColumnDefinition[] DefaultCountStarOutputSchema = [ new ColumnDefinition @@ -402,6 +413,7 @@ public override int GetHashCode() => private List? _systemTriggersRowsCache; private List? _systemObjectsRowsCache; private List? _systemExternalTablesRowsCache; + private List? _systemDiagramsRowsCache; private IReadOnlyList? _externalTableRegistrationsCache; private long? _systemColumnsCountCache; private long? _systemIndexesCountCache; @@ -6605,6 +6617,7 @@ private bool TryBuildSimpleSystemCatalogCountStarQuery(SelectStatement stmt, out "sys.triggers" => _catalog.GetTriggers().Count, "sys.objects" => CountSystemObjects(), "sys.external_tables" => GetExternalTableRegistrations().Count, + "sys.diagrams" => CountSystemDiagrams(), "sys.table_stats" => _catalog.GetTableStatistics().Count, "sys.column_stats" => _catalog.GetColumnStatistics().Count, "sys.planner_histograms" => CountPlannerHistogramRows(), @@ -6637,6 +6650,9 @@ private long CountSystemColumns() long count = 0; foreach (string tableName in _catalog.GetTableNames()) { + if (IsHiddenInternalCatalogTable(tableName)) + continue; + var schema = _catalog.GetTable(tableName); if (schema != null) count += schema.Columns.Count; @@ -6649,6 +6665,9 @@ private long CountSystemColumns() private long CountVisibleUserTables() => _catalog.GetTableNames().LongCount(tableName => !IsHiddenInternalCatalogTable(tableName)); + private long CountSystemDiagrams() => + TryGetTableRowCount(InternalDataModelDiagramsTableName, out long diagramRows) ? diagramRows : 0; + private long CountSystemIndexes() { if (_systemIndexesCountCache.HasValue) @@ -6657,7 +6676,7 @@ private long CountSystemIndexes() long count = 0; foreach (var index in _catalog.GetIndexes()) { - if (index.Kind == IndexKind.ForeignKeyInternal) + if (index.Kind == IndexKind.ForeignKeyInternal || IsHiddenInternalCatalogTable(index.TableName)) continue; count += index.Columns.Count; @@ -6675,6 +6694,9 @@ private long CountSystemForeignKeys() long count = 0; foreach (string tableName in _catalog.GetTableNames()) { + if (IsHiddenInternalCatalogTable(tableName)) + continue; + TableSchema? schema = _catalog.GetTable(tableName); if (schema is not null) count += schema.ForeignKeys.Count; @@ -6686,11 +6708,12 @@ private long CountSystemForeignKeys() private long CountSystemObjects() => CountVisibleUserTables() - + _catalog.GetIndexes().Count(index => index.Kind != IndexKind.ForeignKeyInternal) + + _catalog.GetIndexes().Count(index => index.Kind != IndexKind.ForeignKeyInternal && !IsHiddenInternalCatalogTable(index.TableName)) + CountSystemForeignKeys() + _catalog.GetViewNames().Count + _catalog.GetTriggers().Count - + GetExternalTableRegistrations().Count; + + GetExternalTableRegistrations().Count + + CountSystemDiagrams(); private bool TryBuildSimpleScalarAggregateColumnQuery(SelectStatement stmt, out QueryResult result) { @@ -15591,7 +15614,8 @@ private static bool IsSystemCatalogTable(string tableName) => TryNormalizeSystemCatalogTableName(tableName, out _); private static bool IsHiddenInternalCatalogTable(string tableName) => - string.Equals(tableName, InternalExternalTablesTableName, StringComparison.OrdinalIgnoreCase); + string.Equals(tableName, InternalExternalTablesTableName, StringComparison.OrdinalIgnoreCase) + || string.Equals(tableName, InternalDataModelDiagramsTableName, StringComparison.OrdinalIgnoreCase); private static bool TryNormalizeSystemCatalogTableName(string tableName, out string normalized) { @@ -15658,6 +15682,13 @@ private static bool TryNormalizeSystemCatalogTableName(string tableName, out str return true; } + if (string.Equals(tableName, "sys.diagrams", StringComparison.OrdinalIgnoreCase) || + string.Equals(tableName, "sys_diagrams", StringComparison.OrdinalIgnoreCase)) + { + normalized = "sys.diagrams"; + return true; + } + if (string.Equals(tableName, "sys.table_stats", StringComparison.OrdinalIgnoreCase) || string.Equals(tableName, "sys_table_stats", StringComparison.OrdinalIgnoreCase)) { @@ -15770,6 +15801,11 @@ private bool TryBuildSystemCatalogSource(SimpleTableRef tableRef, out (IOperator rows = BuildSystemExternalTablesRows(); break; + case "sys.diagrams": + columns = SystemDiagramsColumns; + rows = BuildSystemDiagramsRows(); + break; + case "sys.table_stats": columns = SystemTableStatsColumns; rows = BuildSystemTableStatsRows(); @@ -15912,7 +15948,7 @@ private List BuildSystemIndexesRows() .OrderBy(i => i.TableName, StringComparer.OrdinalIgnoreCase) .ThenBy(i => i.IndexName, StringComparer.OrdinalIgnoreCase)) { - if (index.Kind == IndexKind.ForeignKeyInternal) + if (index.Kind == IndexKind.ForeignKeyInternal || IsHiddenInternalCatalogTable(index.TableName)) continue; var tableSchema = _catalog.GetTable(index.TableName); @@ -16028,11 +16064,12 @@ private List BuildSystemObjectsRows() return _systemObjectsRowsCache; int capacity = _catalog.GetTableNames().Count - + _catalog.GetIndexes().Count(index => index.Kind != IndexKind.ForeignKeyInternal) + + _catalog.GetIndexes().Count(index => index.Kind != IndexKind.ForeignKeyInternal && !IsHiddenInternalCatalogTable(index.TableName)) + (int)Math.Min(CountSystemForeignKeys(), int.MaxValue) + _catalog.GetViewNames().Count + _catalog.GetTriggers().Count - + GetExternalTableRegistrations().Count; + + GetExternalTableRegistrations().Count + + (int)Math.Min(CountSystemDiagrams(), int.MaxValue); var rows = new List(capacity); @@ -16115,6 +16152,16 @@ private List BuildSystemObjectsRows() ]); } + foreach (DbValue[] diagram in BuildSystemDiagramsRows().OrderBy(row => row[0].AsText, StringComparer.OrdinalIgnoreCase)) + { + rows.Add( + [ + diagram[0], + DbValue.FromText("DIAGRAM"), + DbValue.Null, + ]); + } + _systemObjectsRowsCache = rows; return rows; } @@ -16142,6 +16189,107 @@ private List BuildSystemExternalTablesRows() return rows; } + private List BuildSystemDiagramsRows() + { + if (_systemDiagramsRowsCache != null) + return _systemDiagramsRowsCache; + + if (_catalog.GetTable(InternalDataModelDiagramsTableName) is not { } schema) + { + _systemDiagramsRowsCache = []; + return _systemDiagramsRowsCache; + } + + int nameIndex = schema.GetColumnIndex("name"); + int jsonIndex = schema.GetColumnIndex("diagram_json"); + int createdIndex = schema.GetColumnIndex("created_utc"); + int updatedIndex = schema.GetColumnIndex("updated_utc"); + if (nameIndex < 0 || jsonIndex < 0 || createdIndex < 0 || updatedIndex < 0) + { + _systemDiagramsRowsCache = []; + return _systemDiagramsRowsCache; + } + + var tableTree = _catalog.GetTableTree(InternalDataModelDiagramsTableName, _pager); + var scan = new TableScanOperator( + tableTree, + schema, + GetReadSerializer(schema), + TryGetCachedTreeRowCountCapacityHint(tableTree)); + var rows = new List((int)Math.Min(CountSystemDiagrams(), int.MaxValue)); + + scan.OpenAsync(CancellationToken.None).AsTask().GetAwaiter().GetResult(); + try + { + while (scan.MoveNextAsync(CancellationToken.None).AsTask().GetAwaiter().GetResult()) + { + DbValue[] row = scan.Current; + if (nameIndex >= row.Length || jsonIndex >= row.Length || createdIndex >= row.Length || updatedIndex >= row.Length) + continue; + + DbValue name = row[nameIndex]; + DbValue json = row[jsonIndex]; + DbValue created = row[createdIndex]; + DbValue updated = row[updatedIndex]; + if (name.Type != DbType.Text || json.Type != DbType.Text || created.Type != DbType.Text || updated.Type != DbType.Text) + continue; + + var (sourceCount, tableCount, pendingOperationCount) = ReadDiagramCounts(json.AsText); + rows.Add( + [ + DbValue.FromText(name.AsText), + DbValue.FromText(created.AsText), + DbValue.FromText(updated.AsText), + DbValue.FromInteger(sourceCount), + DbValue.FromInteger(tableCount), + DbValue.FromInteger(pendingOperationCount), + ]); + } + } + finally + { + scan.DisposeAsync().AsTask().GetAwaiter().GetResult(); + } + + _systemDiagramsRowsCache = rows; + return rows; + } + + private static (long SourceCount, long TableCount, long PendingOperationCount) ReadDiagramCounts(string diagramJson) + { + try + { + using JsonDocument doc = JsonDocument.Parse(diagramJson); + JsonElement root = doc.RootElement; + long sourceCount = root.TryGetProperty("Nodes", out JsonElement nodes) && nodes.ValueKind == JsonValueKind.Array + ? nodes.GetArrayLength() + : 0; + long tableCount = 0; + if (nodes.ValueKind == JsonValueKind.Array) + { + foreach (JsonElement node in nodes.EnumerateArray()) + { + if (!node.TryGetProperty("Kind", out JsonElement kind) || + kind.ValueKind == JsonValueKind.String && + string.Equals(kind.GetString(), "Table", StringComparison.OrdinalIgnoreCase) || + kind.ValueKind == JsonValueKind.Number && kind.GetInt32() == 0) + { + tableCount++; + } + } + } + + long pendingOperationCount = root.TryGetProperty("PendingOperations", out JsonElement pending) && pending.ValueKind == JsonValueKind.Array + ? pending.GetArrayLength() + : 0; + return (sourceCount, tableCount, pendingOperationCount); + } + catch (JsonException) + { + return (0, 0, 0); + } + } + private List BuildSystemTableStatsRows() { var tableStats = _catalog.GetTableStatistics(); @@ -16208,6 +16356,7 @@ private long CountSystemCatalogRows(string normalized) "sys.triggers" => _catalog.GetTriggers().Count, "sys.objects" => CountSystemObjects(), "sys.external_tables" => GetExternalTableRegistrations().Count, + "sys.diagrams" => CountSystemDiagrams(), "sys.table_stats" => _catalog.GetTableStatistics().Count, "sys.column_stats" => _catalog.GetColumnStatistics().Count, "sys.planner_histograms" => CountPlannerHistogramRows(), diff --git a/tests/CSharpDB.Admin.Reports.Tests/Services/DbReportSourceProviderTests.cs b/tests/CSharpDB.Admin.Reports.Tests/Services/DbReportSourceProviderTests.cs index 62c73d29..d81cb7a2 100644 --- a/tests/CSharpDB.Admin.Reports.Tests/Services/DbReportSourceProviderTests.cs +++ b/tests/CSharpDB.Admin.Reports.Tests/Services/DbReportSourceProviderTests.cs @@ -68,7 +68,6 @@ public async Task ListSourceReferencesAsync_FiltersUnsupportedSavedQueriesAndSys await db.Client.UpsertSavedQueryAsync("cleanup_customers", "DELETE FROM Customers WHERE Total < 0;", TestContext.Current.CancellationToken); await db.Client.UpsertSavedQueryAsync("customer_by_id", "SELECT * FROM Customers WHERE Id = @id;", TestContext.Current.CancellationToken); await db.Client.UpsertSavedQueryAsync("__designer_layout:customers", "SELECT * FROM Customers;", TestContext.Current.CancellationToken); - await db.Client.UpsertSavedQueryAsync("__data_model_layout:customers", """{"nodes":[]}""", TestContext.Current.CancellationToken); var provider = new DbReportSourceProvider(db.Client); IReadOnlyList sources = await provider.ListSourceReferencesAsync(); @@ -80,7 +79,6 @@ public async Task ListSourceReferencesAsync_FiltersUnsupportedSavedQueriesAndSys Assert.DoesNotContain(sources, item => item.Name == "cleanup_customers"); Assert.DoesNotContain(sources, item => item.Name == "customer_by_id"); Assert.DoesNotContain(sources, item => item.Name == "__designer_layout:customers"); - Assert.DoesNotContain(sources, item => item.Name == "__data_model_layout:customers"); Assert.Null(await provider.GetSourceDefinitionAsync(new ReportSourceReference(ReportSourceKind.SavedQuery, "cleanup_customers"))); Assert.Null(await provider.GetSourceDefinitionAsync(new ReportSourceReference(ReportSourceKind.SavedQuery, "customer_by_id"))); diff --git a/tests/CSharpDB.Tests/DataModelDiagramServiceTests.cs b/tests/CSharpDB.Tests/DataModelDiagramServiceTests.cs new file mode 100644 index 00000000..46b2350c --- /dev/null +++ b/tests/CSharpDB.Tests/DataModelDiagramServiceTests.cs @@ -0,0 +1,194 @@ +using CSharpDB.Admin.Models; +using CSharpDB.Admin.Services; +using CSharpDB.Client; +using CSharpDB.Client.Models; + +namespace CSharpDB.Tests; + +public sealed class DataModelDiagramServiceTests : IAsyncLifetime +{ + private readonly string _dbPath = Path.Combine(Path.GetTempPath(), $"csharpdb_diagram_{Guid.NewGuid():N}.db"); + private ICSharpDbClient _client = null!; + private DataModelService _service = null!; + + public async ValueTask InitializeAsync() + { + _client = CSharpDbClient.Create(new CSharpDbClientOptions { DataSource = _dbPath }); + _service = new DataModelService(_client); + Assert.Null((await _client.ExecuteSqlAsync("CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT);")).Error); + Assert.Null((await _client.ExecuteSqlAsync("CREATE TABLE orders (id INTEGER PRIMARY KEY, customer_id INTEGER REFERENCES customers(id));")).Error); + } + + public async ValueTask DisposeAsync() + { + await _client.DisposeAsync(); + if (File.Exists(_dbPath)) + File.Delete(_dbPath); + if (File.Exists(_dbPath + ".wal")) + File.Delete(_dbPath + ".wal"); + } + + [Fact] + public async Task SaveLoadDiagram_RoundTripsTableMembershipAndPlacement() + { + DataModelState state = await _service.BuildModelAsync("customers", ct: TestContext.Current.CancellationToken); + state.Nodes.RemoveAll(node => node.Name == "orders"); + state.Relationships.RemoveAll(relationship => relationship.LeftTable == "orders" || relationship.RightTable == "orders"); + DataModelNode customers = Assert.Single(state.Nodes, node => node.Name == "customers"); + customers.X = 321; + customers.Y = 123; + customers.IsCollapsed = true; + + await _service.SaveDiagramAsync("Customer Diagram", state, TestContext.Current.CancellationToken); + + IReadOnlyList diagrams = await _service.GetDiagramsAsync(TestContext.Current.CancellationToken); + DataModelDiagramSummary summary = Assert.Single(diagrams); + Assert.Equal("Customer Diagram", summary.Name); + Assert.Equal(1, summary.SourceCount); + + DataModelState? loaded = await _service.LoadDiagramAsync("Customer Diagram", TestContext.Current.CancellationToken); + + Assert.NotNull(loaded); + DataModelNode loadedCustomers = Assert.Single(loaded.Nodes, node => node.Name == "customers"); + Assert.Equal(321, loadedCustomers.X); + Assert.Equal(123, loadedCustomers.Y); + Assert.True(loadedCustomers.IsCollapsed); + Assert.DoesNotContain(loaded.Nodes, node => node.Name == "orders"); + } + + [Fact] + public async Task SaveDiagram_CreatesHiddenInternalTableAndSysDiagramsMetadata() + { + DataModelState state = await _service.BuildModelAsync("customers", ct: TestContext.Current.CancellationToken); + state.Nodes.RemoveAll(node => node.Name == "orders"); + state.Relationships.RemoveAll(relationship => relationship.LeftTable == "orders" || relationship.RightTable == "orders"); + await _service.SaveDiagramAsync("Metadata", state, TestContext.Current.CancellationToken); + + SqlExecutionResult sys = await _client.ExecuteSqlAsync( + "SELECT name, source_count FROM sys.diagrams WHERE name = 'Metadata';", + TestContext.Current.CancellationToken); + Assert.Null(sys.Error); + object?[] row = Assert.Single(sys.Rows!); + Assert.Equal("Metadata", row[0]); + Assert.Equal(1L, Convert.ToInt64(row[1])); + + IReadOnlyList tables = await _client.GetTableNamesAsync(TestContext.Current.CancellationToken); + Assert.DoesNotContain("__data_model_diagrams", tables, StringComparer.OrdinalIgnoreCase); + } + + [Fact] + public async Task ApplyPendingOperations_CreatesStagedTableAndClearsPendingOperations() + { + var state = new DataModelState + { + DiagramName = "Pending", + Nodes = + [ + new DataModelNode + { + Name = "diagram_created", + IsDraft = true, + Columns = + [ + new DataModelColumn + { + Name = "id", + TypeLabel = "INTEGER", + IsPrimaryKey = true, + IsIdentity = true, + Nullable = false, + }, + ], + }, + ], + PendingOperations = + [ + new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.CreateTable, + TableName = "diagram_created", + Description = "Create table diagram_created", + Columns = + [ + new DataModelColumn + { + Name = "id", + TypeLabel = "INTEGER", + IsPrimaryKey = true, + IsIdentity = true, + Nullable = false, + }, + ], + }, + ], + }; + await _service.SaveDiagramAsync("Pending", state, TestContext.Current.CancellationToken); + + DataModelApplyResult result = await _service.ApplyPendingOperationsAsync(state, TestContext.Current.CancellationToken); + + Assert.True(result.Succeeded); + Assert.Empty(state.PendingOperations); + Assert.NotNull(await _client.GetTableSchemaAsync("diagram_created", TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task ApplyPendingOperations_RemovesDroppedTableFromSavedDiagram() + { + Assert.Null((await _client.ExecuteSqlAsync("CREATE TABLE diagram_drop (id INTEGER PRIMARY KEY);", TestContext.Current.CancellationToken)).Error); + DataModelState state = await _service.BuildModelAsync("diagram_drop", ct: TestContext.Current.CancellationToken); + state.DiagramName = "Drop Diagram Table"; + state.Relationships.Clear(); + state.PendingOperations.Add(new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.DropTable, + TableName = "diagram_drop", + Description = "Drop table diagram_drop", + }); + await _service.SaveDiagramAsync("Drop Diagram Table", state, TestContext.Current.CancellationToken); + + await _service.ApplyPendingOperationsAsync(state, TestContext.Current.CancellationToken); + DataModelState? loaded = await _service.LoadDiagramAsync("Drop Diagram Table", TestContext.Current.CancellationToken); + + Assert.Empty(state.PendingOperations); + Assert.DoesNotContain(state.Nodes, node => node.Name == "diagram_drop"); + Assert.NotNull(loaded); + Assert.DoesNotContain(loaded!.Nodes, node => node.Name == "diagram_drop"); + Assert.Null(await _client.GetTableSchemaAsync("diagram_drop", TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task ApplyPendingOperations_UpdatesSavedDiagramForRenameAndColumnChanges() + { + DataModelState state = await _service.BuildModelAsync("customers", ct: TestContext.Current.CancellationToken); + state.DiagramName = "Schema Edits"; + state.Nodes.RemoveAll(node => node.Name == "orders"); + state.Relationships.Clear(); + state.PendingOperations.Add(new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.RenameTable, + TableName = "customers", + NewTableName = "clients", + Description = "Rename table customers to clients", + }); + state.PendingOperations.Add(new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.AddColumn, + TableName = "clients", + ColumnName = "email", + ColumnType = "TEXT", + Description = "Add column clients.email", + }); + await _service.SaveDiagramAsync("Schema Edits", state, TestContext.Current.CancellationToken); + + await _service.ApplyPendingOperationsAsync(state, TestContext.Current.CancellationToken); + DataModelState? loaded = await _service.LoadDiagramAsync("Schema Edits", TestContext.Current.CancellationToken); + TableSchema? schema = await _client.GetTableSchemaAsync("clients", TestContext.Current.CancellationToken); + + Assert.NotNull(schema); + Assert.Contains(schema!.Columns, column => column.Name == "email"); + Assert.DoesNotContain(state.Nodes, node => node.Name == "customers"); + Assert.Contains(state.Nodes, node => node.Name == "clients"); + Assert.NotNull(loaded); + Assert.Contains(loaded!.Nodes, node => node.Name == "clients"); + } +} diff --git a/tests/CSharpDB.Tests/DataModelGraphBuilderTests.cs b/tests/CSharpDB.Tests/DataModelGraphBuilderTests.cs index 528bf9ff..0d16d35c 100644 --- a/tests/CSharpDB.Tests/DataModelGraphBuilderTests.cs +++ b/tests/CSharpDB.Tests/DataModelGraphBuilderTests.cs @@ -117,6 +117,62 @@ public void SerializeState_PreservesCanvasScale() Assert.Equal(1.4, roundTripped.Scale); } + [Fact] + public void BuildFromDiagramState_PreservesMembershipPlacementAndPendingOperations() + { + var saved = new DataModelState + { + DiagramName = "Fulfillment", + Scale = 1.25, + Nodes = + [ + new DataModelNode { Name = "Customers", X = 42, Y = 84, IsCollapsed = true }, + ], + PendingOperations = + [ + new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.DropTable, + TableName = "Orders", + Description = "Drop table Orders", + }, + ], + }; + + DataModelState restored = DataModelGraphBuilder.BuildFromDiagramState( + [ + Customers(), + Orders(), + ], + saved); + + DataModelNode node = Assert.Single(restored.Nodes); + Assert.Equal("Customers", node.Name); + Assert.Equal(42, node.X); + Assert.Equal(84, node.Y); + Assert.True(node.IsCollapsed); + Assert.Equal(1.25, restored.Scale); + Assert.Single(restored.PendingOperations); + Assert.Empty(restored.Relationships); + } + + [Fact] + public void BuildFromDiagramState_MissingSourceWarnsWithoutFailing() + { + var saved = new DataModelState + { + Nodes = + [ + new DataModelNode { Name = "MissingCustomers" }, + ], + }; + + DataModelState restored = DataModelGraphBuilder.BuildFromDiagramState([Customers()], saved); + + Assert.Empty(restored.Nodes); + Assert.Contains("MissingCustomers", Assert.Single(restored.Warnings)); + } + private static DataModelSourceMetadata Customers() => new() { TableName = "Customers", diff --git a/tests/CSharpDB.Tests/SystemCatalogTests.cs b/tests/CSharpDB.Tests/SystemCatalogTests.cs index a914ab3e..0fb3910e 100644 --- a/tests/CSharpDB.Tests/SystemCatalogTests.cs +++ b/tests/CSharpDB.Tests/SystemCatalogTests.cs @@ -286,6 +286,47 @@ INSERT INTO __saved_queries (name, sql_text, created_utc, updated_utc) Assert.Equal("2026-03-06T00:00:00Z", row[4].AsText); } + [Fact] + public async Task SystemCatalog_DiagramsVirtualTable_ProjectsBackingRowsAndHidesInternalTable() + { + var ct = TestContext.Current.CancellationToken; + + await _db.ExecuteAsync( + """ + CREATE TABLE __data_model_diagrams ( + id INTEGER PRIMARY KEY IDENTITY, + name TEXT NOT NULL, + diagram_json TEXT NOT NULL, + created_utc TEXT NOT NULL, + updated_utc TEXT NOT NULL + ) + """, + ct); + await _db.ExecuteAsync("CREATE UNIQUE INDEX idx___data_model_diagrams_name ON __data_model_diagrams(name)", ct); + await _db.ExecuteAsync( + """ + INSERT INTO __data_model_diagrams (name, diagram_json, created_utc, updated_utc) + VALUES ('Fulfillment', '{"Version":1,"Nodes":[{"Name":"customers","Kind":0},{"Name":"archived_customers","Kind":1}],"PendingOperations":[{"Kind":1}]}', '2026-05-12T00:00:00Z', '2026-05-12T01:00:00Z') + """, + ct); + + await using var diagrams = await _db.ExecuteAsync( + "SELECT name, created_utc, updated_utc, source_count, table_count, pending_operation_count FROM sys.diagrams", + ct); + var row = Assert.Single(await diagrams.ToListAsync(ct)); + Assert.Equal("Fulfillment", row[0].AsText); + Assert.Equal("2026-05-12T00:00:00Z", row[1].AsText); + Assert.Equal("2026-05-12T01:00:00Z", row[2].AsText); + Assert.Equal(2L, row[3].AsInteger); + Assert.Equal(1L, row[4].AsInteger); + Assert.Equal(1L, row[5].AsInteger); + + await using var hidden = await _db.ExecuteAsync( + "SELECT COUNT(*) FROM sys.tables WHERE table_name = '__data_model_diagrams'", + ct); + Assert.Equal(0L, Assert.Single(await hidden.ToListAsync(ct))[0].AsInteger); + } + [Fact] public async Task SystemCatalog_SchemaVersion_AdvancesOnDdlOnly() { From c34acabb05468814a99977c111020c4cdf4d4aed Mon Sep 17 00:00:00 2001 From: Maximum Code Date: Sat, 16 May 2026 15:47:43 -0700 Subject: [PATCH 5/8] docs: update v3.8 release notes --- RELEASE_NOTES.md | 258 ++++++++++++++----------------- docs/releases/v3.8.0-pr-notes.md | 121 +++++++++++++++ 2 files changed, 235 insertions(+), 144 deletions(-) create mode 100644 docs/releases/v3.8.0-pr-notes.md diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 288e03b5..73343e30 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,149 +1,119 @@ # What's New -## version3.7.0 - -version3.7.0 focuses on query-planner observability, opt-in adaptive join -reoptimization, faster paged view browsing, and the benchmark/documentation -close-out work around the current optimizer and async I/O roadmap phases. It -also carries smaller but important polish for SQL result metadata, fulfillment -sample lookup indexes, DataGen direct-load throughput, and benchmark regression -analysis. - -### Planner Observability - -- Added SQL-first planner diagnostics through `EXPLAIN ESTIMATE FOR SELECT`, - `EXPLAIN ESTIMATE FOR WITH`, and compound query estimate support. -- Added public `sys.planner_*` virtual catalogs for planner histograms, heavy - hitters, and composite index prefix statistics. -- Added bounded estimate diagnostics for stats freshness, lookup and filter - estimates, index choices, hash build-side selection, and join reordering. -- Added an Admin Query tab Estimate action and Plan tab rendering for planner - diagnostic rowsets. -- Documented how to read plan output, debug missing or stale stats, and spot - common query-planning red flags. - -### Adaptive Join Reoptimization - -- Added opt-in phase-one adaptive query reoptimization through - `DatabaseOptions.AdaptiveQueryReoptimization` and - `EnableAdaptiveQueryReoptimization(...)`. -- Added ADO.NET direct embedded connection-string support for - `Adaptive Query Reoptimization=true`; remote endpoint connections reject the - key so hosts enable the feature server-side. -- Added adaptive join wrappers that can switch eligible index nested-loop joins - to hash joins before rows are emitted when observed outer cardinality - diverges. -- Added adaptive hash join build-side flipping for eligible inner joins when - the planned build side is materially larger than estimated. -- Added internal diagnostics for eligible queries, attempts, successful - switches, rejected switches, divergence events, buffered rows, and fail-closed - fallback reasons. -- Kept default query behavior unchanged and suppressed adaptation for risky - shapes such as compound query children, correlated subqueries, cross/right - joins, and `SELECT *` cases where visible column order could change. - -### View And Lookup Planning - -- Taught row-goal planning to reorder eligible simple view join chains before - building the view operator tree, so bounded `LIMIT`/`OFFSET` view queries can - use the same streaming lookup plans as equivalent inline SQL. -- Updated the Admin DataGrid view path to page views with bounded - `LIMIT`/`OFFSET` instead of opening unbounded forward-only view cursors. -- Fixed the Query tab grid layout so the row grid is the only scroll container - and the pagination bar stays fixed below the rows. -- Improved lookup-join planning by using indexed local predicate estimates when - cardinality stats are unavailable or weaker. -- Preserved right-side local predicates as residual join filters for lookup - joins and passed estimated row counts into index scans as capacity hints. -- Added an `orders(customer_id)` lookup index to the fulfillment sample schema - for customer-filtered order views. - -### SQL Result Metadata - -- Propagated query column types through engine transport, HTTP API/client DTOs, - and the Admin DataGrid. -- View/query result metadata now preserves `ColumnTypes` across local and - remote SQL execution paths. -- Updated client SQL execution coverage so column names and column types are - both asserted for query results. -- Updated the API package reference for `Scalar.AspNetCore` from `2.14.10` to - `2.14.11`. - -### DataGen And Collection Fast Path Close-Out - -- Closed the current generated collection fast-path roadmap phase and split - package ergonomics and broader generator coverage into future roadmap items. -- Refreshed benchmark-facing docs with the current release-core snapshot and - generated collection codec diagnostics. -- Moved CSharpDB.DataGen direct loads onto the write-optimized storage preset. -- Reused SQL row buffers before `InsertBatch` copies. -- Documented the DataGen fast-path choices and capped direct collection - document target sizes to stay within the current inline collection payload - envelope. - -### Benchmarks And Roadmap Close-Out - -- Marked the current advanced cost-based query optimizer and async I/O batching - roadmap phases as completed current-phase work. -- Kept adaptive reoptimization and public histogram inspection documented as - separate planned/follow-up items where appropriate. -- Added optimizer close-out diagnostics for heavy-hitter equality, histogram - range estimates, composite-prefix correlation, and bounded join reordering. -- Added async I/O close-out diagnostics for save/backup/restore, vacuum and FK - logical rewrites, database inspector scans, and live WAL inspector scans. -- Refreshed roadmap, query/durable-write docs, async I/O audit notes, benchmark - catalog, README performance tables, and release-core manifest metadata with - the May 6 benchmark baseline. -- Added a BenchmarkDotNet WAL point-read benchmark for primary-key reads across - WAL-backed and checkpointed states at 100, 1k, 5k, and 10k target frames. -- Updated `Compare-Baseline.ps1` so performance checks can compare a - configurable time metric such as `P95` through `metricColumn` while keeping - `Mean` as the default. - -### Tests And Benchmarks - -- Added parser, catalog, planner, client, HTTP, and benchmark coverage for - planner diagnostics and `EXPLAIN ESTIMATE`. -- Added adaptive reoptimization engine/operator tests, ADO.NET option tests, - and the `AdaptiveReoptimizationBenchmark` diagnostic suite. -- Added regression coverage for bounded simple-view row-goal planning, late - unindexed detail joins, purchase-order style join chains, and Admin DataGrid - view paging SQL generation. -- Added tests for right-side join predicates, unique text-filter lookup joins, - and SQL result column type metadata. -- Added benchmark suites for optimizer close-out, async I/O close-out, planner - catalog diagnostics, and WAL point-read regression analysis. - -### Validation - -- `dotnet test .\CSharpDB.slnx -c Release` -- `dotnet build .\CSharpDB.slnx -c Release --no-restore` -- `dotnet test .\CSharpDB.slnx -c Release --no-build` -- `dotnet run -c Release --project .\tests\CSharpDB.Benchmarks\CSharpDB.Benchmarks.csproj -- --optimizer-closeout --repro` -- `dotnet run -c Release --project .\tests\CSharpDB.Benchmarks\CSharpDB.Benchmarks.csproj -- --async-io-closeout --repro` -- `pwsh -NoProfile .\tests\CSharpDB.Benchmarks\scripts\Run-Perf-Guardrails.ps1 -Mode release` - - Reported `PASS=187, WARN=0, SKIP=0, FAIL=0`. -- `dotnet build .\src\CSharpDB.Admin\CSharpDB.Admin.csproj -c Release` -- `dotnet test .\tests\CSharpDB.Admin.Forms.Tests\CSharpDB.Admin.Forms.Tests.csproj -c Release --filter FullyQualifiedName~DataGridTests` -- `dotnet test .\tests\CSharpDB.Tests\CSharpDB.Tests.csproj -c Release --filter FullyQualifiedName~SimpleViewLateUnindexedDetailJoinWithLimit|FullyQualifiedName~JoinChainWithLimit|FullyQualifiedName~PurchaseOrderLineJoinChainWithLimit` -- Targeted generated collection tests, trim smoke publish, DataGen Release - build, relational direct-load smoke, and document direct-load smoke passed. -- `dotnet build tests\CSharpDB.Benchmarks\CSharpDB.Benchmarks.csproj -c Release --no-restore` -- `dotnet run -c Release --project tests\CSharpDB.Benchmarks\CSharpDB.Benchmarks.csproj -- --micro --filter *WalPointReadBenchmarks* --job Dry` -- Focused lookup-join and SQL column type tests passed for: - `Join_WithWhereOnRightPrimaryKeyLookupSide_AppliesPredicate`, - `Join_WithUniqueTextFilterAndIndexedDependentSide_UsesLookupJoins`, and - `ExecuteSqlAsync_ReturnsQueryColumnTypes`. +## version3.8.0 + +version3.8.0 adds native table archives, read-only external tables, Admin +Import / Export workflows, and a SQL Server-style Data Model diagram surface. +It also captures the next planning tracks for writable external tables, data +hygiene, developer experience, advanced differentiators, and database DevOps +tooling. + +### Native Table Archives And Import / Export + +- Added the shared `CSharpDB.ImportExport` runtime project and a dedicated + `CSharpDB.Admin.ImportExport` Admin module. +- Added native `.csdbtable` archives using the `CSDBTBL3` seekable format with + JSON schema/manifest sections, length-prefixed encoded rows, row counts, + source table metadata, created timestamps, and optional integer primary-key + archive indexes. +- Added direct table snapshot export through the engine transport path so large + exports stream from a read snapshot instead of paging through the UI client. +- Added Admin Import / Export modes for table export, external table + registration, and table restore. +- Added export destinations for browser download and server/database-local + paths, including one-time server-side download packages so large archives do + not need to be loaded into JavaScript memory. +- Added progress reporting and cancellation for long-running export and + registration operations. +- Added `.csdbtable` to the ignore list so local table archive outputs do not + get picked up by source control. + +### External Tables + +- Added SQL support for: + ```sql + CREATE EXTERNAL TABLE archived_customers FROM 'exports/customers.csdbtable'; + DROP EXTERNAL TABLE archived_customers; + ``` +- Stored external table registrations in the internal `__external_tables` + metadata table and exposed read-only metadata through `sys.external_tables`. +- Resolved relative external archive paths from the database file directory. +- Made external `.csdbtable` tables usable in normal `SELECT` queries, + projections, filters, ordering, and joins. +- Added external table scan, external index nested-loop join, and fast integer + primary-key lookup paths over indexed archives. +- Kept external tables read-only in this release; writes, index creation, ALTER + operations, and trigger targets are rejected against external registrations. +- Added external tables to Admin Object Explorer and Query tab system-catalog + discovery. + +### Data Model Diagrams + +- Added a new Admin Data Model tab that visualizes user tables and external + tables on an ERD-style canvas. +- Reused the Query Designer canvas behavior for draggable schema nodes, + relationships, and zoom while keeping query-specific execution, filters, and + grid state separate. +- Added Object Explorer, command palette, table context-menu, and external + table context-menu entry points for the Data Model tab. +- Added saved diagram persistence through the internal + `__data_model_diagrams` table instead of saved-query layout records. +- Exposed diagram metadata through `sys.diagrams`, including source/table and + pending-operation counts, while hiding the internal table from normal table + lists. +- Added a default global diagram so table placement, zoom, and source + membership are remembered even before the user manually names a diagram. +- Added named diagram save/load/delete, table add/remove membership, node + placement persistence, collapsed state, zoom, and stale/missing source + warnings. +- Added preview-first staged schema operations for new table, drop table, + rename table, add/drop/rename column, and add/drop relationship workflows. + External tables remain display-only. + +### Planning Docs And Website + +- Added planning documents for writable external tables, the Data Hygiene + Engine, the Developer Experience adoption track, Advanced Differentiators, + and the Database DevOps Toolkit. +- Refreshed the roadmap to point at the new planning documents and to mark the + user-defined functions/commands work as completed with sandbox opt-out + coverage. +- Added the native table archives blog post and refreshed the website changelog, + roadmap pages, blog index, and sitemap. + +### Tests And Validation + +- Added archive round-trip tests for schema, values, blobs, empty tables, and + large table archive behavior. +- Added parser, engine, transport, and system-catalog coverage for external + table registration, querying, joining, filtering, ordering, read-only + enforcement, and metadata. +- Added Data Model graph and diagram service tests for placement persistence, + saved membership, pending operations, hidden internal tables, and + `sys.diagrams`. +- Added Admin tab manager and report-source tests for the new Data Model tab and + removal of the old data-model saved-query layout path. +- Focused validation run for this release note: + - `dotnet test tests\CSharpDB.Tests\CSharpDB.Tests.csproj --no-restore --filter "TableArchiveTests|ExternalTableTests|ParserTests|EngineTransportClientTests|DataModelGraphBuilderTests|DataModelDiagramServiceTests|SystemCatalogTests"` + - Passed: 170 tests. + - `dotnet test tests\CSharpDB.Admin.Reports.Tests\CSharpDB.Admin.Reports.Tests.csproj --no-restore --filter DbReportSourceProviderTests` + - Passed: 4 tests. + - `dotnet test tests\CSharpDB.Admin.Forms.Tests\CSharpDB.Admin.Forms.Tests.csproj --no-restore --filter TabManagerServiceTests` + - Passed: 17 tests. + - `dotnet build src\CSharpDB.Admin\CSharpDB.Admin.csproj` + - `git diff --check` + - Browser smoke test for opening Data Model and the New Table staging panel. ### Review Notes -- Adaptive query reoptimization is opt-in and intentionally leaves default - planning behavior unchanged. -- `EXPLAIN ESTIMATE` and `sys.planner_*` are diagnostic surfaces; normal select - planning should not depend on the diagnostic rowset materialization path. -- Simple view row-goal reordering is scoped to eligible join chains and bounded - paging shapes. -- The SQL result DTO column type addition is additive for API/client callers. -- The new WAL point-read benchmark provides a stable current signal, but it - needs a captured historical baseline before it can be used as a release - regression gate. +- `.csdbtable` external tables are intentionally read-only in this release. + Writable external tables are planned separately around an opt-in mutable + `.csdbx` file. +- The table archive format is native and seekable, not a ZIP of JSON rows. +- The Data Model tab stages schema edits and previews SQL before applying + changes; diagram membership and placement save independently. +- Drop-table actions still use the existing engine schema rules, so tables with + foreign-key-owned support indexes may need relationship cleanup first. +- An initial parallel validation attempt hit a transient compiler output lock; + the affected tests were rerun sequentially after shutting down the build + server and passed. diff --git a/docs/releases/v3.8.0-pr-notes.md b/docs/releases/v3.8.0-pr-notes.md new file mode 100644 index 00000000..932c51d4 --- /dev/null +++ b/docs/releases/v3.8.0-pr-notes.md @@ -0,0 +1,121 @@ +## Summary + +This `version3.8.0` release covers the commit range +`ed572e47f9d20a4ffa40400cd0467f664f717f3e..HEAD`. + +The release adds native `.csdbtable` table archives, Admin Import / Export +workflows, and read-only SQL external tables. Archives now use the native +`CSDBTBL3` seekable format with schema and manifest metadata, encoded rows, row +counts, source table metadata, created timestamps, and optional integer +primary-key archive indexes. Admin can export tables to browser downloads or +server/database-local paths, register archives as external tables, restore +archives to physical tables, show progress, and cancel long-running work. + +The SQL/runtime work adds `CREATE EXTERNAL TABLE ... FROM 'path.csdbtable'`, +`DROP EXTERNAL TABLE`, the internal `__external_tables` metadata table, +`sys.external_tables`, external table scans, indexed archive lookups, and joins +between live and archived tables. External `.csdbtable` registrations are +read-only in this release; writes, index creation, ALTER operations, and trigger +targets are rejected. + +The Admin release also adds a Data Model tab. It visualizes user tables and +external tables on a draggable ERD-style canvas, supports zoom, opens from +Object Explorer and the command palette, and can hand selected layouts to the +existing Query Designer. The follow-up diagram work adds a real +`__data_model_diagrams` system table, `sys.diagrams`, named diagram +save/load/delete, remembered table membership and placement, a default global +diagram, and preview-first staged schema operations for SQL Server-style diagram +editing. + +The docs/site portion adds planning documents for writable external tables, +Data Hygiene Engine, Developer Experience, Advanced Differentiators, and the +Database DevOps Toolkit. It also refreshes the roadmap, changelog, blog index, +and adds the v3.8 native table archives blog post. + +## Type of Change + +- [ ] Bug fix +- [x] New feature +- [ ] Breaking change +- [x] Documentation update +- [x] Refactor / maintenance +- [ ] Tests only + +## Related Issues + +No issue numbers were linked for this release branch. Included work: + +- Added `CSharpDB.ImportExport` and `CSharpDB.Admin.ImportExport`. +- Added native `.csdbtable` archive writing, reading, metadata, restore, and + integer primary-key archive lookup support. +- Added direct engine-transport table snapshot export for large table archives. +- Added Admin Import / Export progress UI, cancellation, download staging, + server-path exports, restore, and external table registration. +- Added `CREATE EXTERNAL TABLE` and `DROP EXTERNAL TABLE` parser/runtime + support. +- Added `__external_tables`, `sys.external_tables`, external table object + explorer entries, and query-tab catalog discovery. +- Added external table scan, external index nested-loop join, and fast archive + primary-key lookup planning. +- Enforced read-only external table behavior for write, index, ALTER, and + trigger-target operations. +- Added Admin Data Model tab, schema canvas extraction, object explorer and + command palette entry points, table/external table context entries, and Query + Designer handoff. +- Added `__data_model_diagrams`, `sys.diagrams`, hidden internal-table + filtering, named diagram save/load/delete, and default global diagram + placement persistence. +- Added staged diagram schema operations for new table, drop table, rename + table, add/drop/rename column, and add/drop relationship workflows. +- Removed the unreleased `__data_model_layout:` saved-query persistence path. +- Added planning docs for writable external tables, data hygiene, developer + experience, advanced differentiators, and database DevOps tooling. +- Refreshed roadmap/changelog/static website pages and added the native table + archives blog post. + +## Testing + +- [ ] `dotnet build CSharpDB.slnx` +- [x] Relevant tests executed +- [x] Failure-path tests executed (if applicable: cancellation, invalid/unsupported inputs, non-`DbException` paths) +- [x] Manual verification performed (if applicable) + +Validation performed: + +- `dotnet build src\CSharpDB.Admin\CSharpDB.Admin.csproj` +- `dotnet test tests\CSharpDB.Tests\CSharpDB.Tests.csproj --no-restore --filter "TableArchiveTests|ExternalTableTests|ParserTests|EngineTransportClientTests|DataModelGraphBuilderTests|DataModelDiagramServiceTests|SystemCatalogTests"` + - Passed: 170 tests. +- `dotnet test tests\CSharpDB.Admin.Reports.Tests\CSharpDB.Admin.Reports.Tests.csproj --no-restore --filter DbReportSourceProviderTests` + - Passed: 4 tests. +- `dotnet test tests\CSharpDB.Admin.Forms.Tests\CSharpDB.Admin.Forms.Tests.csproj --no-restore --filter TabManagerServiceTests` + - Passed: 17 tests. +- `git diff --check` +- Browser smoke test for opening the Admin Data Model tab and New Table staging + panel. + +## Checklist + +- [x] I followed the project style and conventions. +- [x] I added or updated tests for behavior changes. +- [x] I covered both success and failure paths for changed behavior. +- [x] I updated docs for user-facing changes. +- [x] I verified no sensitive data was added. + +## Notes for Reviewers + +- `.csdbtable` archives are native seekable packages, not CSV/JSON exports or a + ZIP of JSON rows. +- `.csdbtable` external tables are read-only by design in this release. Writable + external tables are planned separately with a mutable `.csdbx` file. +- External table registrations are stored in `__external_tables`; Data Model + diagrams are stored in `__data_model_diagrams`. Both internal tables are + hidden from normal table lists and exposed through `sys.*` views. +- The Data Model tab persists layout/table membership independently from schema + apply operations. Staged schema edits preview SQL first, then apply through + existing client/engine capabilities. +- Drop-table diagram actions still respect the existing engine FK rules. Tables + with foreign-key-owned support indexes may need relationship cleanup before a + drop can apply. +- An initial parallel validation attempt hit a transient compiler output lock + from `VBCSCompiler`; the affected tests were rerun sequentially after + `dotnet build-server shutdown` and passed. From 45d0cd5d7f6fb0ba85b760ac209b031e3a6f1162 Mon Sep 17 00:00:00 2001 From: Maximum Code Date: Sat, 16 May 2026 15:54:44 -0700 Subject: [PATCH 6/8] ignore benchmark test runs --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a953c8cf..e5684a7b 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,7 @@ coverage*/ !tests/CSharpDB.Benchmarks/baselines/focused-validation/20260303-180221/micro-results/CSharpDB.Benchmarks.Micro.TextIndexBenchmarks-report.csv tests/CSharpDB.Benchmarks/baselines/ tests/CSharpDB.Benchmarks/results/ +tests/CSharpDB.Benchmarks/run-logs/ !tests/CSharpDB.Benchmarks/baselines/ !tests/CSharpDB.Benchmarks/baselines/focused-validation/ !tests/CSharpDB.Benchmarks/baselines/focused-validation/20260326-123705/ From 55d1937fc654a4d9e60d76e9b6c24aac09c30335 Mon Sep 17 00:00:00 2001 From: Maximum Code Date: Sat, 16 May 2026 18:10:20 -0700 Subject: [PATCH 7/8] docs: clean planning markdown endings --- docs/advanced-differentiators/README.md | 1 - docs/data-hygiene-engine/README.md | 1 - docs/database-devops-toolkit/README.md | 1 - docs/developer-experience-adoption-engine/README.md | 1 - 4 files changed, 4 deletions(-) diff --git a/docs/advanced-differentiators/README.md b/docs/advanced-differentiators/README.md index ca562891..7821561c 100644 --- a/docs/advanced-differentiators/README.md +++ b/docs/advanced-differentiators/README.md @@ -245,4 +245,3 @@ Phase 4: full-text SQL ergonomics. - Client and Admin streaming API tests for reactive query delivery. - CLI and Admin docs examples for archive registration, change feed inspection, and subscription demos. - diff --git a/docs/data-hygiene-engine/README.md b/docs/data-hygiene-engine/README.md index 784e84fc..796a5701 100644 --- a/docs/data-hygiene-engine/README.md +++ b/docs/data-hygiene-engine/README.md @@ -198,4 +198,3 @@ Phase 4: product surfaces. `VALIDATE TABLE` violations. - Orphan detection from foreign-key metadata and explicit references. - Transaction rollback for failed dedup and merge operations. - diff --git a/docs/database-devops-toolkit/README.md b/docs/database-devops-toolkit/README.md index bdbc98d8..3a0c8d69 100644 --- a/docs/database-devops-toolkit/README.md +++ b/docs/database-devops-toolkit/README.md @@ -243,4 +243,3 @@ Phase 4: drift reports. and validation failures. - Admin service tests for target loading, diff preview, script preview, and destructive execution confirmation. - diff --git a/docs/developer-experience-adoption-engine/README.md b/docs/developer-experience-adoption-engine/README.md index 0c8df864..e1c97cd9 100644 --- a/docs/developer-experience-adoption-engine/README.md +++ b/docs/developer-experience-adoption-engine/README.md @@ -219,4 +219,3 @@ Phase 4: analytics helpers and adoption polish. and JSON output. - Surface tests for Admin, EF Core, MCP, and VS Code snippets as those phases are implemented. - From 378dde6641095f01253f3c01e2a079bea0b9159a Mon Sep 17 00:00:00 2001 From: Maximum Code Date: Sat, 16 May 2026 20:12:48 -0700 Subject: [PATCH 8/8] fix: constrain code module import paths --- .../CSharpDbCodeModuleClient.cs | 20 +++++++-- .../CSharpDbCodeModuleClientTests.cs | 42 +++++++++++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/src/CSharpDB.CodeModules/CSharpDbCodeModuleClient.cs b/src/CSharpDB.CodeModules/CSharpDbCodeModuleClient.cs index f413f58e..db1f8d1e 100644 --- a/src/CSharpDB.CodeModules/CSharpDbCodeModuleClient.cs +++ b/src/CSharpDB.CodeModules/CSharpDbCodeModuleClient.cs @@ -198,8 +198,7 @@ await File.ReadAllTextAsync(manifestPath, ct), var changes = new List(); foreach (CodeModuleWorkspaceManifestEntry entry in manifest.Modules) { - string relativePath = NormalizeRelativePath(entry.Path); - string absolutePath = Path.Combine(root, relativePath); + string absolutePath = ResolveWorkspaceSourcePath(root, entry.Path); if (!File.Exists(absolutePath)) { changes.Add(new CodeModuleImportChange(entry.ModuleId, entry.Path, CodeModuleImportChangeKind.Skipped, "Source file is missing.")); @@ -581,13 +580,26 @@ private static string BuildDiagnosticPath(CodeModuleDefinition module) _ => $"modules/{ToSafeFileName(module.Name)}.cs", }; - private static string NormalizeRelativePath(string path) + private static string ResolveWorkspaceSourcePath(string root, string path) { string normalized = path.Replace('/', Path.DirectorySeparatorChar).Replace('\\', Path.DirectorySeparatorChar); if (Path.IsPathRooted(normalized)) throw new InvalidOperationException("Code module workspace paths must be relative."); - return normalized; + if (normalized.Split(Path.DirectorySeparatorChar).Any(segment => segment == "..")) + throw new InvalidOperationException("Code module workspace paths must stay under the workspace root."); + + string absolutePath = Path.GetFullPath(Path.Combine(root, normalized)); + string relativeToRoot = Path.GetRelativePath(root, absolutePath); + if (Path.IsPathRooted(relativeToRoot) || + relativeToRoot == ".." || + relativeToRoot.StartsWith($"..{Path.DirectorySeparatorChar}", StringComparison.Ordinal) || + relativeToRoot.StartsWith($"..{Path.AltDirectorySeparatorChar}", StringComparison.Ordinal)) + { + throw new InvalidOperationException("Code module workspace paths must stay under the workspace root."); + } + + return absolutePath; } private static string ToSafeFileName(string value) diff --git a/tests/CSharpDB.CodeModules.Tests/CSharpDbCodeModuleClientTests.cs b/tests/CSharpDB.CodeModules.Tests/CSharpDbCodeModuleClientTests.cs index bb99c142..30718dc5 100644 --- a/tests/CSharpDB.CodeModules.Tests/CSharpDbCodeModuleClientTests.cs +++ b/tests/CSharpDB.CodeModules.Tests/CSharpDbCodeModuleClientTests.cs @@ -71,6 +71,48 @@ await source.UpsertAsync(new CodeModuleDefinition( Directory.Delete(workspace, recursive: true); } + [Fact] + public async Task Import_RejectsManifestPathEscapingWorkspaceRoot() + { + CancellationToken ct = TestContext.Current.CancellationToken; + await using TestDatabaseScope sourceDb = await TestDatabaseScope.CreateAsync("code_modules_export_escape"); + var source = new CSharpDbCodeModuleClient(sourceDb.Client, new InMemoryCodeModuleTrustStore()); + await source.UpsertAsync(new CodeModuleDefinition( + "module:helpers", + "Helpers", + CodeModuleKind.Standard, + "public static class Helpers { public static int A() => 1; }\n"), ct); + + string workspace = Path.Combine(Path.GetTempPath(), $"csharpdb_code_ws_{Guid.NewGuid():N}"); + try + { + CodeModuleExportResult export = await source.ExportAsync(workspace, ct); + string exportedSourcePath = Directory.EnumerateFiles(export.WorkspaceDirectory, "*.cs", SearchOption.AllDirectories).Single(); + string exportedRelativePath = Path.GetRelativePath(export.WorkspaceDirectory, exportedSourcePath).Replace('\\', '/'); + string outsidePath = Path.Combine(workspace, "outside.cs"); + await File.WriteAllTextAsync(outsidePath, "public static class Outside { }\n", ct); + + string manifestJson = await File.ReadAllTextAsync(export.ManifestPath, ct); + manifestJson = manifestJson.Replace( + $"\"path\": \"{exportedRelativePath}\"", + "\"path\": \"../outside.cs\"", + StringComparison.Ordinal); + await File.WriteAllTextAsync(export.ManifestPath, manifestJson, ct); + + await using TestDatabaseScope targetDb = await TestDatabaseScope.CreateAsync("code_modules_import_escape"); + var target = new CSharpDbCodeModuleClient(targetDb.Client, new InMemoryCodeModuleTrustStore()); + + InvalidOperationException ex = await Assert.ThrowsAsync(() => target.ImportAsync(workspace, ct)); + Assert.Contains("workspace root", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Null(await target.GetAsync("module:helpers", ct)); + } + finally + { + if (Directory.Exists(workspace)) + Directory.Delete(workspace, recursive: true); + } + } + [Fact] public async Task Build_ReturnsAssemblyBytesAndStructuredDiagnostics() {