diff --git a/.gitignore b/.gitignore index 917b3e1b..e5684a7b 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,7 @@ coverage*/ !tests/CSharpDB.Benchmarks/baselines/focused-validation/20260303-180221/micro-results/CSharpDB.Benchmarks.Micro.TextIndexBenchmarks-report.csv tests/CSharpDB.Benchmarks/baselines/ tests/CSharpDB.Benchmarks/results/ +tests/CSharpDB.Benchmarks/run-logs/ !tests/CSharpDB.Benchmarks/baselines/ !tests/CSharpDB.Benchmarks/baselines/focused-validation/ !tests/CSharpDB.Benchmarks/baselines/focused-validation/20260326-123705/ @@ -93,6 +94,7 @@ Desktop.ini ## CSharpDB runtime files *.cdb *.db +*.csdbtable *.db.journal *.wal diff --git a/CSharpDB.slnx b/CSharpDB.slnx index 691caea1..aabb04c6 100644 --- a/CSharpDB.slnx +++ b/CSharpDB.slnx @@ -29,10 +29,12 @@ + + @@ -41,6 +43,7 @@ + @@ -56,6 +59,7 @@ + diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 288e03b5..73343e30 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,149 +1,119 @@ # What's New -## version3.7.0 - -version3.7.0 focuses on query-planner observability, opt-in adaptive join -reoptimization, faster paged view browsing, and the benchmark/documentation -close-out work around the current optimizer and async I/O roadmap phases. It -also carries smaller but important polish for SQL result metadata, fulfillment -sample lookup indexes, DataGen direct-load throughput, and benchmark regression -analysis. - -### Planner Observability - -- Added SQL-first planner diagnostics through `EXPLAIN ESTIMATE FOR SELECT`, - `EXPLAIN ESTIMATE FOR WITH`, and compound query estimate support. -- Added public `sys.planner_*` virtual catalogs for planner histograms, heavy - hitters, and composite index prefix statistics. -- Added bounded estimate diagnostics for stats freshness, lookup and filter - estimates, index choices, hash build-side selection, and join reordering. -- Added an Admin Query tab Estimate action and Plan tab rendering for planner - diagnostic rowsets. -- Documented how to read plan output, debug missing or stale stats, and spot - common query-planning red flags. - -### Adaptive Join Reoptimization - -- Added opt-in phase-one adaptive query reoptimization through - `DatabaseOptions.AdaptiveQueryReoptimization` and - `EnableAdaptiveQueryReoptimization(...)`. -- Added ADO.NET direct embedded connection-string support for - `Adaptive Query Reoptimization=true`; remote endpoint connections reject the - key so hosts enable the feature server-side. -- Added adaptive join wrappers that can switch eligible index nested-loop joins - to hash joins before rows are emitted when observed outer cardinality - diverges. -- Added adaptive hash join build-side flipping for eligible inner joins when - the planned build side is materially larger than estimated. -- Added internal diagnostics for eligible queries, attempts, successful - switches, rejected switches, divergence events, buffered rows, and fail-closed - fallback reasons. -- Kept default query behavior unchanged and suppressed adaptation for risky - shapes such as compound query children, correlated subqueries, cross/right - joins, and `SELECT *` cases where visible column order could change. - -### View And Lookup Planning - -- Taught row-goal planning to reorder eligible simple view join chains before - building the view operator tree, so bounded `LIMIT`/`OFFSET` view queries can - use the same streaming lookup plans as equivalent inline SQL. -- Updated the Admin DataGrid view path to page views with bounded - `LIMIT`/`OFFSET` instead of opening unbounded forward-only view cursors. -- Fixed the Query tab grid layout so the row grid is the only scroll container - and the pagination bar stays fixed below the rows. -- Improved lookup-join planning by using indexed local predicate estimates when - cardinality stats are unavailable or weaker. -- Preserved right-side local predicates as residual join filters for lookup - joins and passed estimated row counts into index scans as capacity hints. -- Added an `orders(customer_id)` lookup index to the fulfillment sample schema - for customer-filtered order views. - -### SQL Result Metadata - -- Propagated query column types through engine transport, HTTP API/client DTOs, - and the Admin DataGrid. -- View/query result metadata now preserves `ColumnTypes` across local and - remote SQL execution paths. -- Updated client SQL execution coverage so column names and column types are - both asserted for query results. -- Updated the API package reference for `Scalar.AspNetCore` from `2.14.10` to - `2.14.11`. - -### DataGen And Collection Fast Path Close-Out - -- Closed the current generated collection fast-path roadmap phase and split - package ergonomics and broader generator coverage into future roadmap items. -- Refreshed benchmark-facing docs with the current release-core snapshot and - generated collection codec diagnostics. -- Moved CSharpDB.DataGen direct loads onto the write-optimized storage preset. -- Reused SQL row buffers before `InsertBatch` copies. -- Documented the DataGen fast-path choices and capped direct collection - document target sizes to stay within the current inline collection payload - envelope. - -### Benchmarks And Roadmap Close-Out - -- Marked the current advanced cost-based query optimizer and async I/O batching - roadmap phases as completed current-phase work. -- Kept adaptive reoptimization and public histogram inspection documented as - separate planned/follow-up items where appropriate. -- Added optimizer close-out diagnostics for heavy-hitter equality, histogram - range estimates, composite-prefix correlation, and bounded join reordering. -- Added async I/O close-out diagnostics for save/backup/restore, vacuum and FK - logical rewrites, database inspector scans, and live WAL inspector scans. -- Refreshed roadmap, query/durable-write docs, async I/O audit notes, benchmark - catalog, README performance tables, and release-core manifest metadata with - the May 6 benchmark baseline. -- Added a BenchmarkDotNet WAL point-read benchmark for primary-key reads across - WAL-backed and checkpointed states at 100, 1k, 5k, and 10k target frames. -- Updated `Compare-Baseline.ps1` so performance checks can compare a - configurable time metric such as `P95` through `metricColumn` while keeping - `Mean` as the default. - -### Tests And Benchmarks - -- Added parser, catalog, planner, client, HTTP, and benchmark coverage for - planner diagnostics and `EXPLAIN ESTIMATE`. -- Added adaptive reoptimization engine/operator tests, ADO.NET option tests, - and the `AdaptiveReoptimizationBenchmark` diagnostic suite. -- Added regression coverage for bounded simple-view row-goal planning, late - unindexed detail joins, purchase-order style join chains, and Admin DataGrid - view paging SQL generation. -- Added tests for right-side join predicates, unique text-filter lookup joins, - and SQL result column type metadata. -- Added benchmark suites for optimizer close-out, async I/O close-out, planner - catalog diagnostics, and WAL point-read regression analysis. - -### Validation - -- `dotnet test .\CSharpDB.slnx -c Release` -- `dotnet build .\CSharpDB.slnx -c Release --no-restore` -- `dotnet test .\CSharpDB.slnx -c Release --no-build` -- `dotnet run -c Release --project .\tests\CSharpDB.Benchmarks\CSharpDB.Benchmarks.csproj -- --optimizer-closeout --repro` -- `dotnet run -c Release --project .\tests\CSharpDB.Benchmarks\CSharpDB.Benchmarks.csproj -- --async-io-closeout --repro` -- `pwsh -NoProfile .\tests\CSharpDB.Benchmarks\scripts\Run-Perf-Guardrails.ps1 -Mode release` - - Reported `PASS=187, WARN=0, SKIP=0, FAIL=0`. -- `dotnet build .\src\CSharpDB.Admin\CSharpDB.Admin.csproj -c Release` -- `dotnet test .\tests\CSharpDB.Admin.Forms.Tests\CSharpDB.Admin.Forms.Tests.csproj -c Release --filter FullyQualifiedName~DataGridTests` -- `dotnet test .\tests\CSharpDB.Tests\CSharpDB.Tests.csproj -c Release --filter FullyQualifiedName~SimpleViewLateUnindexedDetailJoinWithLimit|FullyQualifiedName~JoinChainWithLimit|FullyQualifiedName~PurchaseOrderLineJoinChainWithLimit` -- Targeted generated collection tests, trim smoke publish, DataGen Release - build, relational direct-load smoke, and document direct-load smoke passed. -- `dotnet build tests\CSharpDB.Benchmarks\CSharpDB.Benchmarks.csproj -c Release --no-restore` -- `dotnet run -c Release --project tests\CSharpDB.Benchmarks\CSharpDB.Benchmarks.csproj -- --micro --filter *WalPointReadBenchmarks* --job Dry` -- Focused lookup-join and SQL column type tests passed for: - `Join_WithWhereOnRightPrimaryKeyLookupSide_AppliesPredicate`, - `Join_WithUniqueTextFilterAndIndexedDependentSide_UsesLookupJoins`, and - `ExecuteSqlAsync_ReturnsQueryColumnTypes`. +## version3.8.0 + +version3.8.0 adds native table archives, read-only external tables, Admin +Import / Export workflows, and a SQL Server-style Data Model diagram surface. +It also captures the next planning tracks for writable external tables, data +hygiene, developer experience, advanced differentiators, and database DevOps +tooling. + +### Native Table Archives And Import / Export + +- Added the shared `CSharpDB.ImportExport` runtime project and a dedicated + `CSharpDB.Admin.ImportExport` Admin module. +- Added native `.csdbtable` archives using the `CSDBTBL3` seekable format with + JSON schema/manifest sections, length-prefixed encoded rows, row counts, + source table metadata, created timestamps, and optional integer primary-key + archive indexes. +- Added direct table snapshot export through the engine transport path so large + exports stream from a read snapshot instead of paging through the UI client. +- Added Admin Import / Export modes for table export, external table + registration, and table restore. +- Added export destinations for browser download and server/database-local + paths, including one-time server-side download packages so large archives do + not need to be loaded into JavaScript memory. +- Added progress reporting and cancellation for long-running export and + registration operations. +- Added `.csdbtable` to the ignore list so local table archive outputs do not + get picked up by source control. + +### External Tables + +- Added SQL support for: + ```sql + CREATE EXTERNAL TABLE archived_customers FROM 'exports/customers.csdbtable'; + DROP EXTERNAL TABLE archived_customers; + ``` +- Stored external table registrations in the internal `__external_tables` + metadata table and exposed read-only metadata through `sys.external_tables`. +- Resolved relative external archive paths from the database file directory. +- Made external `.csdbtable` tables usable in normal `SELECT` queries, + projections, filters, ordering, and joins. +- Added external table scan, external index nested-loop join, and fast integer + primary-key lookup paths over indexed archives. +- Kept external tables read-only in this release; writes, index creation, ALTER + operations, and trigger targets are rejected against external registrations. +- Added external tables to Admin Object Explorer and Query tab system-catalog + discovery. + +### Data Model Diagrams + +- Added a new Admin Data Model tab that visualizes user tables and external + tables on an ERD-style canvas. +- Reused the Query Designer canvas behavior for draggable schema nodes, + relationships, and zoom while keeping query-specific execution, filters, and + grid state separate. +- Added Object Explorer, command palette, table context-menu, and external + table context-menu entry points for the Data Model tab. +- Added saved diagram persistence through the internal + `__data_model_diagrams` table instead of saved-query layout records. +- Exposed diagram metadata through `sys.diagrams`, including source/table and + pending-operation counts, while hiding the internal table from normal table + lists. +- Added a default global diagram so table placement, zoom, and source + membership are remembered even before the user manually names a diagram. +- Added named diagram save/load/delete, table add/remove membership, node + placement persistence, collapsed state, zoom, and stale/missing source + warnings. +- Added preview-first staged schema operations for new table, drop table, + rename table, add/drop/rename column, and add/drop relationship workflows. + External tables remain display-only. + +### Planning Docs And Website + +- Added planning documents for writable external tables, the Data Hygiene + Engine, the Developer Experience adoption track, Advanced Differentiators, + and the Database DevOps Toolkit. +- Refreshed the roadmap to point at the new planning documents and to mark the + user-defined functions/commands work as completed with sandbox opt-out + coverage. +- Added the native table archives blog post and refreshed the website changelog, + roadmap pages, blog index, and sitemap. + +### Tests And Validation + +- Added archive round-trip tests for schema, values, blobs, empty tables, and + large table archive behavior. +- Added parser, engine, transport, and system-catalog coverage for external + table registration, querying, joining, filtering, ordering, read-only + enforcement, and metadata. +- Added Data Model graph and diagram service tests for placement persistence, + saved membership, pending operations, hidden internal tables, and + `sys.diagrams`. +- Added Admin tab manager and report-source tests for the new Data Model tab and + removal of the old data-model saved-query layout path. +- Focused validation run for this release note: + - `dotnet test tests\CSharpDB.Tests\CSharpDB.Tests.csproj --no-restore --filter "TableArchiveTests|ExternalTableTests|ParserTests|EngineTransportClientTests|DataModelGraphBuilderTests|DataModelDiagramServiceTests|SystemCatalogTests"` + - Passed: 170 tests. + - `dotnet test tests\CSharpDB.Admin.Reports.Tests\CSharpDB.Admin.Reports.Tests.csproj --no-restore --filter DbReportSourceProviderTests` + - Passed: 4 tests. + - `dotnet test tests\CSharpDB.Admin.Forms.Tests\CSharpDB.Admin.Forms.Tests.csproj --no-restore --filter TabManagerServiceTests` + - Passed: 17 tests. + - `dotnet build src\CSharpDB.Admin\CSharpDB.Admin.csproj` + - `git diff --check` + - Browser smoke test for opening Data Model and the New Table staging panel. ### Review Notes -- Adaptive query reoptimization is opt-in and intentionally leaves default - planning behavior unchanged. -- `EXPLAIN ESTIMATE` and `sys.planner_*` are diagnostic surfaces; normal select - planning should not depend on the diagnostic rowset materialization path. -- Simple view row-goal reordering is scoped to eligible join chains and bounded - paging shapes. -- The SQL result DTO column type addition is additive for API/client callers. -- The new WAL point-read benchmark provides a stable current signal, but it - needs a captured historical baseline before it can be used as a release - regression gate. +- `.csdbtable` external tables are intentionally read-only in this release. + Writable external tables are planned separately around an opt-in mutable + `.csdbx` file. +- The table archive format is native and seekable, not a ZIP of JSON rows. +- The Data Model tab stages schema edits and previews SQL before applying + changes; diagram membership and placement save independently. +- Drop-table actions still use the existing engine schema rules, so tables with + foreign-key-owned support indexes may need relationship cleanup first. +- An initial parallel validation attempt hit a transient compiler output lock; + the affected tests were rerun sequentially after shutting down the build + server and passed. diff --git a/docs/admin-forms-access-parity/README.md b/docs/admin-forms-access-parity/README.md index f04845df..9cea97d1 100644 --- a/docs/admin-forms-access-parity/README.md +++ b/docs/admin-forms-access-parity/README.md @@ -29,7 +29,11 @@ The current forms surface already includes: - declarative form action sequences for run-command, reusable sequence, set-field, show-message, stop, built-in navigation, save, delete, refresh, and go-to-record steps +- open/close form, apply/clear filter, run SQL/procedure, and control-property + action steps with rendered-host policy gates where needed - conditional action steps and reusable named form action sequences +- conditional UI rules for visible, enabled, read-only, text, placeholder, and + value effects - visual designer editing for form and selected-control action sequences - generated automation metadata for export/import host callback requirements @@ -106,10 +110,10 @@ Expected fix: | Feature | Status | Notes | | --- | --- | --- | -| Command button control | Partial | Trusted command buttons can invoke host-registered C# commands, action-only click sequences, reusable action sequences, and built-in rendered-form navigation/save/delete actions; richer button styling and command presets remain future work. | -| Action model | Partial | Declarative action sequences support run-command, reusable sequence, set-field, show-message, stop, built-in navigation/save/delete/refresh/go-to, simple per-step conditions, visual designer editing, and generated automation metadata for form and selected-control events; open form, apply filter, clear filter, run SQL/procedure, loops, and conditional UI rules remain future work. | -| Event hooks | Partial | Form lifecycle events, command-button clicks, and selected control events can call trusted commands; additional Access-style events remain future work. | -| Conditional UI rules | Planned | Add visible/enabled/read-only expressions for controls. | +| Command button control | Partial | Trusted command buttons can invoke host-registered C# commands, action-only click sequences, reusable action sequences, and built-in rendered-form navigation/save/delete/filter/control actions; richer button styling and command presets remain future work. | +| Action model | Partial | Declarative action sequences support run-command, reusable sequence, set-field, show-message, stop, rendered record navigation/save/delete/refresh/go-to, open/close form, apply/clear filter, run SQL/procedure, control property changes, simple per-step conditions, visual designer editing, diagnostics, and generated automation metadata; database-owned C# form modules can now run trusted local handlers with explicit trust. Loops, on-error handling, temp/session variables, broader report/query/import/export actions, report modules, and sandboxed code remain future work. | +| Event hooks | Partial | Form lifecycle events, command-button clicks, and selected control events can call trusted commands, C# code-module handlers, or action sequences; additional Access-style events such as double-click, key, mouse, timer, dirty, and current events remain future work. | +| Conditional UI rules | Partial | Form-level rules can set rendered control properties such as visible, enabled, read-only, text, placeholder, and value; reusable rule presets and broader expression/event surfaces remain future work. | ### Phase 5: Broader Control and Property Coverage diff --git a/docs/admin-forms-access-parity/access-style-functions-and-macros.md b/docs/admin-forms-access-parity/access-style-functions-and-macros.md index 32f29b80..637ec46c 100644 --- a/docs/admin-forms-access-parity/access-style-functions-and-macros.md +++ b/docs/admin-forms-access-parity/access-style-functions-and-macros.md @@ -12,13 +12,15 @@ feel productive without requiring host code for every common task. - Keep user-facing form actions as declarative Admin Form actions where possible. - Route dangerous or host-specific actions through trusted callbacks, policy, and diagnostics. -- Treat database-owned C# code modules as the later `RunCode` target. +- Treat database-owned C# code modules as trusted event handlers first; a + declarative `RunCode` macro action can build on that runtime later. - Preserve the existing saved form wire shape by storing action and expression settings in metadata/property bags. Implementation note: Admin Forms formulas now include the expression functions -listed below as built-ins. Macro/action commands remain roadmap items unless -already covered by the existing form action model. +listed below as built-ins. The rendered Forms runtime also supports the current +declarative action model shown below; entries marked future remain roadmap +items. ## Included Expression Functions @@ -27,90 +29,89 @@ already covered by the existing form action model. These should be first because they appear constantly in form defaults, calculated controls, validation messages, and visibility/enabled rules. -| Function | Purpose | Priority | +| Function | Purpose | Status | | --- | --- | --- | -| `Nz(value, fallback)` | Replace null/empty values with a fallback. | V1 | -| `IsNull(value)` | Test for null. | V1 | -| `IsEmpty(value)` | Test for empty/unset values where applicable. | V1 | -| `IIf(condition, trueValue, falseValue)` | Inline conditional. | V1 | -| `Switch(condition1, value1, ...)` | Multi-branch conditional. | V2 | -| `Choose(index, value1, value2, ...)` | Pick from positional values. | V2 | +| `Nz(value, fallback)` | Replace null/empty values with a fallback. | Shipped | +| `IsNull(value)` | Test for null. | Shipped | +| `IsEmpty(value)` | Test for empty/unset values where applicable. | Shipped | +| `IIf(condition, trueValue, falseValue)` | Inline conditional. | Shipped | +| `Switch(condition1, value1, ...)` | Multi-branch conditional. | Shipped | +| `Choose(index, value1, value2, ...)` | Pick from positional values. | Shipped | ### Text -| Function | Purpose | Priority | +| Function | Purpose | Status | | --- | --- | --- | -| `Len(value)` | Text length. | V1 | -| `Left(value, count)` | Left substring. | V1 | -| `Right(value, count)` | Right substring. | V1 | -| `Mid(value, start, count)` | Middle substring. | V1 | -| `Trim(value)` | Trim both ends. | V1 | -| `LTrim(value)` | Trim left. | V2 | -| `RTrim(value)` | Trim right. | V2 | -| `UCase(value)` | Uppercase text. | V1 | -| `LCase(value)` | Lowercase text. | V1 | -| `InStr(value, search)` | Find substring position. | V1 | -| `Replace(value, search, replacement)` | Replace text. | V1 | -| `StrComp(left, right, comparison)` | Compare strings. | V2 | -| `Val(value)` | Parse leading numeric text. | V2 | +| `Len(value)` | Text length. | Shipped | +| `Left(value, count)` | Left substring. | Shipped | +| `Right(value, count)` | Right substring. | Shipped | +| `Mid(value, start, count)` | Middle substring. | Shipped | +| `Trim(value)` | Trim both ends. | Shipped | +| `LTrim(value)` | Trim left. | Shipped | +| `RTrim(value)` | Trim right. | Shipped | +| `UCase(value)` | Uppercase text. | Shipped | +| `LCase(value)` | Lowercase text. | Shipped | +| `InStr(value, search)` | Find substring position. | Shipped | +| `Replace(value, search, replacement)` | Replace text. | Shipped | +| `StrComp(left, right, comparison)` | Compare strings. | Shipped | +| `Val(value)` | Parse leading numeric text. | Shipped | ### Date and Time -| Function | Purpose | Priority | +| Function | Purpose | Status | | --- | --- | --- | -| `Date()` | Current date. | V1 | -| `Time()` | Current time. | V1 | -| `Now()` | Current date/time. | V1 | -| `Year(value)` | Extract year. | V1 | -| `Month(value)` | Extract month number. | V1 | -| `Day(value)` | Extract day of month. | V1 | -| `Hour(value)` | Extract hour. | V2 | -| `Minute(value)` | Extract minute. | V2 | -| `Second(value)` | Extract second. | V2 | -| `DateAdd(interval, amount, value)` | Add date/time interval. | V1 | -| `DateDiff(interval, start, end)` | Difference between dates. | V1 | -| `DatePart(interval, value)` | Extract date/time part. | V2 | -| `DateSerial(year, month, day)` | Construct date. | V2 | -| `TimeSerial(hour, minute, second)` | Construct time. | V2 | -| `Weekday(value)` | Day of week number. | V2 | -| `MonthName(month)` | Month display name. | V2 | +| `Date()` | Current date. | Shipped | +| `Time()` | Current time. | Shipped | +| `Now()` | Current date/time. | Shipped | +| `Year(value)` | Extract year. | Shipped | +| `Month(value)` | Extract month number. | Shipped | +| `Day(value)` | Extract day of month. | Shipped | +| `Hour(value)` | Extract hour. | Shipped | +| `Minute(value)` | Extract minute. | Shipped | +| `Second(value)` | Extract second. | Shipped | +| `DateAdd(interval, amount, value)` | Add date/time interval. | Shipped | +| `DateDiff(interval, start, end)` | Difference between dates. | Shipped | +| `DatePart(interval, value)` | Extract date/time part. | Shipped | +| `DateSerial(year, month, day)` | Construct date. | Shipped | +| `TimeSerial(hour, minute, second)` | Construct time. | Shipped | +| `Weekday(value)` | Day of week number. | Shipped | +| `MonthName(month)` | Month display name. | Shipped | ### Number and Conversion -| Function | Purpose | Priority | +| Function | Purpose | Status | | --- | --- | --- | -| `Abs(value)` | Absolute value. | V1 | -| `Round(value, digits)` | Round number. | V1 | -| `Int(value)` | Floor-like integer conversion. | V1 | -| `Fix(value)` | Truncate toward zero. | V2 | -| `Sgn(value)` | Sign of number. | V2 | -| `CStr(value)` | Convert to string. | V1 | -| `CInt(value)` | Convert to integer. | V1 | -| `CLng(value)` | Convert to long integer. | V2 | -| `CDbl(value)` | Convert to double. | V1 | -| `CBool(value)` | Convert to boolean. | V1 | -| `CDate(value)` | Convert to date/time. | V1 | -| `Format(value, format)` | Format date/number/text. | V2 | +| `Abs(value)` | Absolute value. | Shipped | +| `Round(value, digits)` | Round number. | Shipped | +| `Int(value)` | Floor-like integer conversion. | Shipped | +| `Fix(value)` | Truncate toward zero. | Shipped | +| `Sgn(value)` | Sign of number. | Shipped | +| `CStr(value)` | Convert to string. | Shipped | +| `CInt(value)` | Convert to integer. | Shipped | +| `CLng(value)` | Convert to long integer. | Shipped | +| `CDbl(value)` | Convert to double. | Shipped | +| `CBool(value)` | Convert to boolean. | Shipped | +| `CDate(value)` | Convert to date/time. | Shipped | +| `Format(value, format)` | Format date/number/text. | Shipped | ### Domain Aggregates -Domain aggregate functions are important for Access familiarity, but they need a -careful implementation because they read other rows/tables during form -evaluation. +Domain aggregate functions are important for Access familiarity. They are +available in Admin Forms formulas through the rendered Forms runtime, which +loads referenced domains with a row limit and evaluates criteria through the +Forms filter parser. -| Function | Purpose | Priority | +| Function | Purpose | Status | | --- | --- | --- | -| `DLookup(expr, domain, criteria)` | Read one value from a table/query. | V2 | -| `DCount(expr, domain, criteria)` | Count matching rows. | V2 | -| `DSum(expr, domain, criteria)` | Sum matching rows. | V2 | -| `DAvg(expr, domain, criteria)` | Average matching rows. | V2 | -| `DMin(expr, domain, criteria)` | Minimum matching value. | V2 | -| `DMax(expr, domain, criteria)` | Maximum matching value. | V2 | - -V2 should enforce query/table access through the same callback and diagnostics -boundary used for trusted extensions where relevant. These functions should also -have row limits and clear error handling so formula evaluation cannot become an -unbounded database workload. +| `DLookup(expr, domain, criteria)` | Read one value from a table/query. | Shipped in Admin Forms | +| `DCount(expr, domain, criteria)` | Count matching rows. | Shipped in Admin Forms | +| `DSum(expr, domain, criteria)` | Sum matching rows. | Shipped in Admin Forms | +| `DAvg(expr, domain, criteria)` | Average matching rows. | Shipped in Admin Forms | +| `DMin(expr, domain, criteria)` | Minimum matching value. | Shipped in Admin Forms | +| `DMax(expr, domain, criteria)` | Maximum matching value. | Shipped in Admin Forms | + +Future work can broaden these beyond rendered Admin Forms and add more +diagnostics for expensive domain reads. ## Included Macro and Action Commands @@ -119,115 +120,113 @@ unbounded database workload. These map cleanly to existing form runtime behavior and should remain declarative actions rather than host callbacks. -| Action | Purpose | Priority | +| Action | Purpose | Status | | --- | --- | --- | -| `NewRecord` | Start a new record. | V1 | -| `SaveRecord` | Save the current record. | V1 | -| `DeleteRecord` | Delete the current record. | V1 | -| `UndoRecord` | Revert unsaved edits. | V1 | -| `RefreshRecord` | Reload the current record. | V1 | -| `Requery` | Reload the form record source. | V1 | -| `GoToRecord` | Navigate to a specific record. | V1 | -| `FindRecord` | Search/navigate by criteria. | V2 | -| `NextRecord` | Navigate forward. | V1 | -| `PreviousRecord` | Navigate backward. | V1 | +| `NewRecord` | Start a new record. | Shipped | +| `SaveRecord` | Save the current record. | Shipped | +| `DeleteRecord` | Delete the current record. | Shipped | +| `UndoRecord` | Revert unsaved edits. | Future action | +| `RefreshRecords` | Reload the current record/page. | Shipped | +| `Requery` | Reload the form record source. | Covered by `RefreshRecords`; broader record-source requery is future | +| `GoToRecord` | Navigate to a specific record. | Shipped | +| `FindRecord` | Search/navigate by criteria. | Future action | +| `NextRecord` | Navigate forward. | Shipped | +| `PreviousRecord` | Navigate backward. | Shipped | ### Form, Window, and Report Actions -| Action | Purpose | Priority | +| Action | Purpose | Status | | --- | --- | --- | -| `OpenForm` | Open another saved form. | V1 | -| `CloseForm` | Close current or named form. | V1 | -| `OpenReport` | Open a saved report. | V2 | -| `CloseReport` | Close a report surface. | V2 | -| `PreviewReport` | Open report preview. | V2 | -| `PrintReport` | Print or export through report pipeline. | V2 | +| `OpenForm` | Open another saved form. | Shipped | +| `CloseForm` | Close current or named form. | Shipped | +| `OpenReport` | Open a saved report. | Future action | +| `CloseReport` | Close a report surface. | Future action | +| `PreviewReport` | Open report preview. | Future action | +| `PrintReport` | Print or export through report pipeline. | Future action | ### Filter and Sort Actions -| Action | Purpose | Priority | +| Action | Purpose | Status | | --- | --- | --- | -| `ApplyFilter` | Apply a form filter. | V1 | -| `ClearFilter` | Clear current filter. | V1 | -| `SetOrderBy` | Apply sort order. | V1 | -| `ClearOrderBy` | Clear sort order. | V1 | -| `SearchRecords` | Search over configured searchable fields. | V2 | +| `ApplyFilter` | Apply a form or data-grid filter. | Shipped | +| `ClearFilter` | Clear current form or data-grid filter. | Shipped | +| `SetOrderBy` | Apply sort order. | Future action | +| `ClearOrderBy` | Clear sort order. | Future action | +| `SearchRecords` | Search over configured searchable fields. | Future action | ### UI and Control Actions These are needed for Access-style command buttons and conditional workflows. -| Action | Purpose | Priority | +| Action | Purpose | Status | | --- | --- | --- | -| `SetValue` | Set a field/control value. | V1 | -| `SetProperty` | Set visible/enabled/read-only/text/style properties. | V1 | -| `SetFocus` | Move focus to a control. | V1 | -| `EnableControl` | Set enabled state. | V1 | -| `DisableControl` | Clear enabled state. | V1 | -| `ShowControl` | Set visible state. | V1 | -| `HideControl` | Clear visible state. | V1 | -| `LockControl` | Set read-only state. | V1 | -| `UnlockControl` | Clear read-only state. | V1 | -| `MsgBox` | Show a message dialog. | V1 | -| `InputBox` | Prompt for a value. | V2 | +| `SetFieldValue` | Set a current record field value. | Shipped | +| `SetControlProperty` | Set visible/enabled/read-only/text/placeholder/value properties. | Shipped | +| `SetFocus` | Move focus to a control. | Future action | +| `SetControlEnabled` | Set enabled state. | Shipped | +| `SetControlVisibility` | Set visible state. | Shipped | +| `SetControlReadOnly` | Set read-only state. | Shipped | +| `ShowMessage` | Show a message through the current Forms surface. | Shipped | +| `InputBox` | Prompt for a value. | Future action | ### Flow Actions -| Action | Purpose | Priority | +| Action | Purpose | Status | | --- | --- | --- | -| `If` / `Else` | Conditional action branching. | V1 | -| `StopMacro` | Stop the current action sequence. | V1 | -| `RunMacro` | Run a named action sequence. | V1 | -| `RunActionSequence` | Existing explicit reusable sequence action. | V1 | -| `OnError` | Configure failure handling. | V2 | +| Per-step `Condition` | Conditionally run or skip a step. | Shipped | +| `Stop` | Stop the current action sequence successfully. | Shipped | +| `RunActionSequence` | Run a named reusable sequence. | Shipped | +| `If` / `Else` blocks | Conditional action branching. | Future action | +| `OnError` | Configure failure handling. | Future action | ### Data and Query Actions These must be gated carefully because they can mutate data outside the current form record. -| Action | Purpose | Priority | +| Action | Purpose | Status | | --- | --- | --- | -| `OpenQuery` | Open a saved query result. | V2 | -| `RunQuery` | Execute a saved query. | V2 trusted | -| `RunProcedure` | Execute a saved procedure. | V2 trusted | -| `RunSQL` | Execute SQL text. | Later trusted | -| `ExportData` | Export records. | Later trusted | -| `ImportData` | Import records. | Later trusted | +| `OpenQuery` | Open a saved query result. | Future action | +| `RunQuery` | Execute a saved query. | Future trusted action | +| `RunProcedure` | Execute a saved procedure. | Shipped with host opt-in | +| `RunSql` | Execute SQL text. | Shipped with host opt-in | +| `ExportData` | Export records. | Future trusted action | +| `ImportData` | Import records. | Future trusted action | ### Code Bridge Actions -| Action | Purpose | Priority | +| Action | Purpose | Status | | --- | --- | --- | -| `RunCommand` | Invoke host-registered trusted command callback. | Existing | -| `RunCode` | Invoke database-owned C# code module function. | Later | +| `RunCommand` | Invoke host-registered trusted command callback. | Shipped | +| `RunCode` | Invoke database-owned C# code module function from a declarative action sequence. | Future action | -`RunCode` should wait for the database code modules work. It should compile and -execute database-owned C# through a trusted build/runtime model, not arbitrary -source text embedded directly inside form JSON. +Database-owned C# form modules now compile and execute through a trusted +build/runtime model for form and control event handlers. `RunCode` remains a +future macro action that should reuse that model rather than embedding arbitrary +source text directly inside form JSON. ### Temp and Session Variables -| Action | Purpose | Priority | +| Action | Purpose | Status | | --- | --- | --- | -| `SetTempVar` | Store a session-scoped value. | V2 | -| `RemoveTempVar` | Remove one session value. | V2 | -| `RemoveAllTempVars` | Clear session values. | V2 | +| `SetTempVar` | Store a session-scoped value. | Future action | +| `RemoveTempVar` | Remove one session value. | Future action | +| `RemoveAllTempVars` | Clear session values. | Future action | Temp variables should be scoped to the current user/session and be available to form expressions, filters, and action sequences. -## Recommended Implementation Order - -1. Add core formula functions: `Nz`, `IIf`, `Date`, `Now`, common text helpers, - numeric helpers, and conversions. -2. Add declarative form actions: `MsgBox`, `SetValue`, `SetProperty`, - `SetFocus`, `ApplyFilter`, `ClearFilter`, `Requery`, `OpenForm`, and - `CloseForm`. -3. Add domain aggregates with clear access checks, row limits, and diagnostics. -4. Add temp/session variables and make them visible to expressions and actions. -5. Add `RunCode` after database code modules exist. -6. Add import/export/file/app-launch style actions only as trusted operations. +## Recommended Remaining Implementation Order + +1. Add sort/search/find actions and any small aliases needed for Access naming + compatibility. +2. Add temp/session variables and make them visible to expressions, filters, + and action sequences. +3. Add richer macro flow such as `If` / `Else` blocks and `OnError`. +4. Add report/query/import/export actions behind explicit trusted boundaries. +5. Add `RunCode` after the code-module event-handler MVP is extended to macro + action invocation. +6. Add file/app-launch style actions only as trusted operations. ## Notes on Access Compatibility diff --git a/docs/advanced-differentiators/README.md b/docs/advanced-differentiators/README.md new file mode 100644 index 00000000..7821561c --- /dev/null +++ b/docs/advanced-differentiators/README.md @@ -0,0 +1,247 @@ +# Advanced Differentiators Plan + +This document captures the planned design for the feature group that makes +CSharpDB feel like a SQLite, Firebase, and SQL Server hybrid: archive-backed +time travel, change tracking feeds, reactive queries, and full-text search +integration. + +## Goal And Positioning + +The goal is to make CSharpDB useful not only as an embedded SQL database, but as +an application data platform that can answer operational questions: + +- What did this table look like at a point in time? +- What changed since my worker last checked? +- Can my UI react to database changes without polling? +- Can users search business text with database-native relevance? + +The first implementation should lean on existing CSharpDB strengths. Native +table archives already provide timestamped table snapshots. WAL snapshot +readers already provide point-in-time read mechanics for active sessions. +Full-text search already has storage, maintenance, tokenization, and ranking +infrastructure. This plan connects those pieces into a clear developer-facing +surface. + +## Current Foundation + +CSharpDB already has several foundations for this feature group: + +- Full-text indexes with tokenization, index maintenance, and ranked search + hits. +- Native `.csdbtable` table archives with a manifest `CreatedUtc` timestamp. +- Admin Import / Export and SQL external table registration over archive files. +- WAL-backed snapshot readers for consistent point-in-time reads while writers + continue. +- Roadmap research for replication and change feeds. + +The missing work is durable historical query routing, a retained change log, +subscription delivery, and SQL ergonomics around the existing full-text engine. + +## V1 SQL Contract + +Archive-backed time travel: + +```sql +SELECT * FROM Customers AS OF TIMESTAMP '2026-05-10T08:15:00Z'; +``` + +Change tracking feed: + +```sql +SELECT * FROM CHANGES(Customers); +``` + +Reactive query subscription: + +```sql +SUBSCRIBE TO SELECT * FROM Orders WHERE Status = 'Open'; +``` + +Full-text search SQL ergonomics: + +```sql +SELECT * FROM Customers +WHERE MATCH(Name, Notes) AGAINST ('priority support'); +``` + +Default semantics: + +- Time travel v1 is archive-backed. `AS OF TIMESTAMP` resolves to the newest + registered table archive at or before the requested timestamp. +- Archive manifests use the existing `CreatedUtc` timestamp. V1 does not add + row-version chains. +- Change feed v1 is commit-log backed and begins when change tracking is + enabled for a table. +- `CHANGES(table)` returns commit timestamp, operation, primary key or rowid, + changed columns, and current values. +- Reactive queries are exposed first through `CSharpDB.Client` and Admin + streaming APIs using SQL text as the subscription definition. +- `SUBSCRIBE TO SELECT ...` is the SQL contract, but v1 does not force ordinary + `ExecuteAsync` query cursors to become long-running streams. +- Full-text search builds on existing full-text indexes and adds SQL-level + `MATCH(...) AGAINST (...)` parsing and planning. + +## Time Travel Via Timestamped Table Archives + +The first time-travel implementation should use native table archives rather +than row-versioned storage. This keeps the feature aligned with the existing +export system and avoids changing the main table format before the use cases are +proven. + +Recommended behavior: + +- Register timestamped archives in internal metadata, keyed by source table and + archive `CreatedUtc`. +- Resolve `AS OF TIMESTAMP` by selecting the newest archive at or before the + requested timestamp. +- Execute the query against the resolved archive using the existing external + table scan and archive primary-key lookup paths. +- Return a clear error when no archive exists for the requested timestamp. +- Reject `AS OF` on mutating statements in v1. + +This is not true row-level temporal storage yet. It is point-in-time querying +over retained snapshots. + +## Change Tracking Feed + +Change tracking should use a retained commit log, not snapshot diffing. Tracking +starts only after it is enabled for a table, so v1 does not need to reconstruct +history that was never captured. + +Recommended SQL shape for enabling tracking: + +```sql +ALTER TABLE Customers ENABLE CHANGE TRACKING; +``` + +`CHANGES(Customers)` should expose a table-valued feed with at least: + +| Column | Meaning | +| --- | --- | +| `commit_timestamp` | UTC commit time for the change. | +| `commit_sequence` | Monotonic sequence for stable ordering. | +| `operation` | `INSERT`, `UPDATE`, or `DELETE`. | +| `table_name` | Source table. | +| `row_id` | Internal rowid when available. | +| `primary_key` | Primary-key value when available. | +| `changed_columns` | Column names changed by the operation. | +| `current_values` | Current row values for insert/update changes. | + +Deletes should include enough identity information to let consumers invalidate +or remove cached rows. Full before/after row images are future work. + +## Reactive Query Subscriptions + +Reactive queries should initially be a client and Admin streaming feature backed +by the same SQL text users already understand. + +Recommended behavior: + +- A subscription runs an initial query and emits the current result. +- Subsequent writes to referenced tables enqueue invalidation events. +- The subscription re-runs or incrementally refreshes the query based on the + supported plan shape. +- Backpressure, cancellation, and disconnects are explicit parts of the client + API contract. +- Admin can use the stream for live query tabs and dashboards without polling. + +The SQL statement documents intent: + +```sql +SUBSCRIBE TO SELECT * FROM Orders WHERE Status = 'Open'; +``` + +In v1, this statement should map to a subscription API rather than returning a +normal finite `QueryResult` from ordinary SQL execution. + +## Full-Text Search Integration + +Full-text search is already a shipped storage/indexing capability. The future +work here is SQL ergonomics and integration, not rebuilding the full-text index. + +Recommended behavior: + +- Parse `MATCH(column, ...) AGAINST ('query')` in `WHERE`. +- Resolve the expression to an existing compatible full-text index. +- Use the existing full-text reader for candidate rowids and scores. +- Expose score ordering through a system expression or projected score alias in + a later phase. +- Keep full-text index creation and maintenance on the existing infrastructure. + +## Metadata And Execution Model + +Time-travel archive metadata should be stored in internal tables and exposed via +system catalog views. The metadata should connect source table, archive path, +archive timestamp, row count, and schema fingerprint so query routing can fail +clearly when an archive no longer matches the requested table shape. + +Change tracking should use append-only internal storage with retention policy +metadata. A global commit sequence gives deterministic ordering across tables, +while per-table filtering keeps `CHANGES(table)` cheap. + +Reactive subscriptions should use table dependency analysis from parsed query +plans. V1 can invalidate and re-run the full query for broad correctness, then +add incremental delivery for simple primary-key and predicate shapes later. + +Full-text SQL planning should be a thin adapter from SQL syntax to the existing +full-text index reader and row fetch path. + +## Non-Goals + +- No row-version chain storage in v1 time travel. +- No `AS OF` support for writes in v1. +- No snapshot diffing as the primary change-feed mechanism. +- No full before/after image feed for every update/delete in v1. +- No requirement that ordinary `ExecuteAsync` query results become infinite + subscription cursors. +- No rebuild of the existing full-text indexing engine. +- No cross-database replication protocol in v1. + +## Phased Implementation Plan + +Phase 1: archive-backed time travel. + +- Add archive history metadata and system catalog exposure. +- Add `AS OF TIMESTAMP` parser and planner support for table references. +- Route eligible reads to the selected table archive. +- Add Admin and CLI affordances for listing and registering archive snapshots. + +Phase 2: commit-log change feed. + +- Add table-level change tracking enablement. +- Write append-only change records during committed inserts, updates, and + deletes for tracked tables. +- Add `CHANGES(table)` as a table-valued query source. +- Add retention and pruning controls. + +Phase 3: reactive subscriptions. + +- Add `CSharpDB.Client` subscription APIs and Admin live-query surfaces. +- Use parsed query dependency analysis to subscribe to referenced table changes. +- Deliver initial results, invalidation/update events, cancellation, reconnect, + and backpressure handling. +- Keep `SUBSCRIBE TO SELECT ...` as the SQL-facing contract for subscription + definitions. + +Phase 4: full-text SQL ergonomics. + +- Add parser and planner support for `MATCH(...) AGAINST (...)`. +- Resolve compatible full-text indexes and use existing full-text search hits. +- Add docs and examples that combine full-text search with reactive queries and + change feeds. + +## Future Test Plan + +- `AS OF TIMESTAMP` archive selection before, at, between, and after registered + archive timestamps. +- Missing archive behavior and schema mismatch handling. +- `CHANGES(table)` after insert, update, delete, checkpoint, reopen, and + retention pruning. +- Change feed ordering by commit timestamp and sequence. +- Subscription initial result delivery, later change notifications, + cancellation, reconnect, and backpressure. +- `MATCH(...) AGAINST (...)` parser coverage and equivalence to existing + full-text search APIs. +- Client and Admin streaming API tests for reactive query delivery. +- CLI and Admin docs examples for archive registration, change feed inspection, + and subscription demos. diff --git a/docs/data-hygiene-engine/README.md b/docs/data-hygiene-engine/README.md new file mode 100644 index 00000000..796a5701 --- /dev/null +++ b/docs/data-hygiene-engine/README.md @@ -0,0 +1,200 @@ +# Data Hygiene Engine Plan + +This document captures the planned design for a SQL-first Data Hygiene Engine in +CSharpDB. The goal is to make cleanup, validation, and relationship auditing +first-class database workflows instead of one-off scripts around the database. + +## Summary And Goals + +The Data Hygiene Engine should make CSharpDB feel like a smart database for +real-world data. It focuses on common operational problems that show up in +customer lists, booking systems, imports, and long-lived business databases: + +- Detect duplicate records before changing data. +- Deduplicate tables transactionally with deterministic survivor selection. +- Merge duplicate records conservatively. +- Store database-owned validation rules as SQL expressions. +- Find orphaned child rows from declared or explicit relationships. + +V1 is SQL-first. Admin UI, richer reporting, and pipeline integration should +build on the same engine behavior after the core commands are stable. + +## Current Related Capabilities + +CSharpDB already has pieces that this plan should build around: + +- `CSharpDB.Pipelines` includes a package-level `Deduplicate` transform for ETL + flows. +- SQL foreign keys and `sys.foreign_keys` metadata provide declared + parent/child relationships. +- Trusted validation callbacks exist for Admin Forms and host-owned business + logic. + +The Data Hygiene Engine is different because it is database-level SQL. Its +rules and commands should be portable with the database metadata, queryable +through system catalog surfaces, and usable without host-owned C# callbacks. + +## V1 SQL Contract + +Duplicate detection is a read-only preview: + +```sql +FIND DUPLICATES IN Customers ON Email; +``` + +Deduplication removes duplicate rows transactionally: + +```sql +DEDUP Customers ON Email KEEP FIRST; +``` + +Duplicate merge keeps one winner row, fills null fields from duplicate rows +where possible, and then removes the duplicate rows: + +```sql +MERGE DUPLICATES Customers ON Email; +``` + +Database-owned validation rules are SQL expressions: + +```sql +CREATE VALIDATION RULE ValidEmail +ON Customers.Email +AS Email LIKE '%@%' +MESSAGE 'Email must contain @'; +``` + +Rules are audit-only in v1. They are evaluated explicitly with `VALIDATE TABLE` +and do not block `INSERT` or `UPDATE` yet: + +```sql +VALIDATE TABLE Customers; +``` + +Orphan detection uses declared foreign keys by default: + +```sql +FIND ORPHANS IN Bookings; +``` + +When no foreign key exists, callers can provide an explicit relationship: + +```sql +FIND ORPHANS IN Bookings.BookId REFERENCES Books.Id; +``` + +Default semantics: + +- `KEEP FIRST` keeps the row with the lowest primary key when the table has one, + otherwise the lowest rowid. +- `KEEP LAST` keeps the row with the highest primary key when the table has one, + otherwise the highest rowid. +- `FIND` commands are read-only previews. +- `DEDUP` and `MERGE DUPLICATES` run through normal write transaction behavior + and return affected-row and affected-group summaries. +- `MERGE DUPLICATES` does not overwrite non-null winner values in v1. +- Validation rules are evaluated through `VALIDATE TABLE`; write enforcement is + future work. +- Orphan detection uses declared foreign keys first and explicit references when + supplied. + +## Metadata Model + +Validation rules should be stored in an internal metadata table rather than in a +new storage catalog sentinel. The metadata should be sufficient to rebuild and +evaluate the rule expression after reopening the database. + +Recommended metadata: + +| Column | Meaning | +| --- | --- | +| `rule_name` | Case-insensitive validation rule identifier. | +| `table_name` | Target table. | +| `column_name` | Optional target column; null for row-level rules. | +| `expression_sql` | SQL expression that must evaluate truthy for a valid row. | +| `message` | User-facing validation failure message. | +| `created_at` | Creation timestamp. | +| `is_enabled` | Whether `VALIDATE TABLE` should evaluate the rule. | + +Expose rules through `sys.validation_rules`. Hygiene commands should return +normal query results, so Admin, CLI, HTTP, gRPC, and ADO.NET surfaces can consume +the same output shape without a feature-specific transport. + +## Execution Model + +Duplicate detection groups rows by the requested key expressions. The result +should report enough information for users to preview the cleanup: duplicate +key values, duplicate group size, winner primary key or rowid, and duplicate +primary keys or rowids that would be removed. + +Deduplication uses the same grouping and winner selection as duplicate +detection, then deletes non-winner rows in one transaction. The command should +return a summary with duplicate groups found, rows deleted, and rows kept. + +Merge uses the same winner selection, then fills null winner columns from +duplicate rows when a non-null duplicate value is available. If multiple +duplicates provide different non-null values for the same empty winner column, +v1 should preserve the winner null and report the conflict rather than guessing. +After applying safe fill-null updates, it deletes the duplicate rows in the same +transaction. + +Validation rules reuse the SQL expression evaluator against each row scope. A +rule passes when the expression evaluates truthy. `FALSE` or `NULL` should be +reported as a validation violation with the configured message. + +Orphan detection builds anti-join style checks. With declared foreign keys, it +uses the FK metadata for the child and parent columns. With explicit references, +it validates the named child table/column and parent table/column before running +the check. + +## Non-Goals + +- No fuzzy matching in v1. +- No automatic write enforcement for validation rules in v1. +- No cross-table merge policies in v1. +- No Admin visual workspace in v1. +- No trusted C# validation callbacks as the primary implementation. +- No automatic repair of orphaned rows in v1. + +## Phased Implementation Plan + +Phase 1: read-only hygiene inspection. + +- Add parser and AST support for `FIND DUPLICATES`, `FIND ORPHANS`, and + `VALIDATE TABLE`. +- Add execution paths that return normal `QueryResult` objects. +- Add validation-rule metadata storage and `sys.validation_rules`. + +Phase 2: transactional deduplication. + +- Add parser and execution support for `DEDUP ... KEEP FIRST|LAST`. +- Reuse the duplicate detection grouping path for deterministic winner + selection. +- Delete duplicate rows in one write transaction and return a cleanup summary. + +Phase 3: conservative duplicate merge. + +- Add `MERGE DUPLICATES` execution over the duplicate grouping path. +- Fill null winner columns from duplicate rows where there is exactly one safe + non-null candidate value. +- Report unresolved merge conflicts without overwriting existing winner values. + +Phase 4: product surfaces. + +- Add an Admin hygiene workspace that previews duplicates, validation + violations, and orphans before applying mutations. +- Add pipeline integration that can call the same engine behaviors from ETL + packages. +- Add docs and examples for common import-cleanup workflows. + +## Future Test Plan + +- Parser tests for each new command and invalid syntax. +- Duplicate detection with nulls, case/collation behavior, composite keys, and + empty tables. +- Dedup winner selection by primary key and rowid. +- Merge fill-null behavior and conflict preservation. +- Validation rule creation, listing through `sys.validation_rules`, and + `VALIDATE TABLE` violations. +- Orphan detection from foreign-key metadata and explicit references. +- Transaction rollback for failed dedup and merge operations. diff --git a/docs/database-devops-toolkit/README.md b/docs/database-devops-toolkit/README.md new file mode 100644 index 00000000..3a0c8d69 --- /dev/null +++ b/docs/database-devops-toolkit/README.md @@ -0,0 +1,245 @@ +# Database DevOps Toolkit Plan + +This document captures the planned design for a first-party CSharpDB Database +DevOps Toolkit. The first implementation should focus on compare and deploy +workflows that help developers and operators understand differences between +databases, archives, and environments before applying changes. + +## Summary And Goals + +The toolkit should make database change management a normal CSharpDB workflow +instead of an external script collection. V1 should help users answer: + +- What schema changed between two databases? +- What rows differ between a live table and another database or archive? +- Has a database drifted from a known baseline? +- What SQL would synchronize the target with the source? +- Which changes are destructive and need explicit review? + +The goal is a scriptable CLI and visual Admin experience backed by the same +shared comparison services. + +## Current Foundation + +CSharpDB already has the primitives needed for a compare/deploy toolkit: + +- `ICSharpDbClient` exposes schema, table, index, view, trigger, procedure, + data, maintenance, backup, and inspection operations across direct and remote + transports. +- Client backup and restore can create database-level snapshots. +- Native `.csdbtable` archives can export table schema and rows with manifest + metadata. +- Admin already has object explorer, query tabs, storage/maintenance tooling, + Import / Export, and table browsing. +- The `csharpdb` CLI already has interactive SQL, maintenance, inspection, ETL, + backup, restore, and script execution commands. + +The toolkit should build on those surfaces instead of creating separate +database access paths. + +## V1 Toolkit Scope + +V1 is focused on compare/deploy: + +- Schema compare. +- Data compare. +- Drift reports. +- Generated schema deployment scripts. +- Generated data sync scripts. +- CLI commands for automation and CI. +- Admin Compare / Deploy workflow for visual review. + +V1 should not ship masking, cloning, monitoring, SQL linting, dependency graphs, +or SQL unit testing as finished features. Those remain future toolkit phases. + +## Compare Targets + +The first implementation should support these target types: + +| Target | Purpose | +| --- | --- | +| Live CSharpDB database | Compare an active database through `ICSharpDbClient`. | +| Backup database | Open or restore a database backup as a comparison target. | +| `.csdbtable` table archive | Compare a live table to an exported table snapshot. | +| SQL script target | Future source-control target for schema scripts and migrations. | + +Target adapters should normalize schema and row access behind shared toolkit +interfaces so CLI and Admin use the same comparison behavior. + +## Schema Compare + +Schema compare should identify structural differences between source and target +schemas. + +V1 should cover: + +- Added, removed, and changed tables. +- Added, removed, and changed columns. +- Column type, nullability, identity, primary-key, collation, and foreign-key + differences where metadata is available. +- Added, removed, and changed indexes. +- Added, removed, and changed views. +- Added, removed, and changed triggers. +- Added, removed, and changed procedures. + +The output should group changes by object type and include warnings for +destructive changes such as dropped tables, dropped columns, type changes, and +nullable-to-not-null changes. + +## Data Compare + +Data compare should identify row-level differences for selected tables. + +V1 should report: + +- Rows present in source but missing in target. +- Rows present in target but missing in source. +- Rows present in both with changed column values. +- Rows that cannot be matched because no stable key is available. + +Default key behavior: + +- Use the table primary key when one exists. +- Require explicit `--key ` when no primary key or stable key exists. +- Support composite keys when explicitly supplied. +- Treat `NULL`, text collation, and blobs consistently with CSharpDB value + comparison semantics. + +The comparison should stream rows where practical so large tables do not require +loading both sides fully into memory. + +## Drift Reports + +Drift reports compare a database against a known baseline and summarize whether +the current database still matches the expected state. + +V1 baseline sources: + +- Backup database. +- `.csdbtable` archive or archive manifest for table-level drift. +- Future SQL script/source-control target. + +The report should include: + +- Schema drift summary. +- Optional data drift summary for selected tables. +- Destructive or risky differences. +- Machine-readable JSON output for CI. +- Human-readable CLI and Admin summaries. + +## Deployment Script Generation + +Generated scripts should be preview-first. The toolkit should never silently +apply destructive changes. + +Schema deployment scripts should: + +- Use dependency-aware ordering where metadata allows. +- Create missing objects before dependent objects. +- Warn for destructive changes. +- Avoid pretending every destructive schema change is automatically safe. + +Data sync scripts should: + +- Use normal `INSERT`, `UPDATE`, and `DELETE` statements. +- Escape values correctly, including text, blobs, and nulls. +- Use primary key or explicit key predicates. +- Refuse script generation when no stable key exists. + +CLI execution of generated scripts should require an explicit apply option in a +future runtime implementation. Admin execution should require a confirmation +step after preview. + +## CLI And Admin Workflows + +CLI shapes: + +```powershell +csharpdb compare schema [--json] [--script-out ] +csharpdb compare data --table [--key ] [--json] [--script-out ] +csharpdb drift --baseline [--json] +``` + +Admin workflow: + +- Add a `Compare / Deploy` entry under Tools or Object Explorer. +- Let users pick source and target from live database, backup, or archive + targets. +- Show a schema diff view grouped by object type. +- Show a data diff view for selected tables with insert, update, and delete + counts plus row previews. +- Show a warnings panel for destructive or risky changes. +- Show generated deployment script previews. +- Require explicit confirmation before executing generated scripts. + +## Future Toolkit Phases + +After compare/deploy, the toolkit can expand into: + +- Data masking and anonymization. +- Lightweight clones for development and testing. +- Database monitoring and health history. +- SQL formatting and linting. +- Dependency and impact analysis for table/column changes. +- SQL-level unit tests. +- Source-control schema baselines and migration review. +- Single-row or table restore from archives. + +These should reuse the same target adapters, diff models, archive readers, and +Admin/CLI surfaces where possible. + +## Non-Goals + +- No masking or anonymization in V1. +- No lightweight clone feature in V1. +- No production monitoring dashboard in V1. +- No SQL formatter or linter in V1. +- No dependency graph or impact analyzer in V1. +- No SQL unit testing framework in V1. +- No SQL script/source-control target required for V1. +- No silent execution of destructive generated scripts. + +## Phased Implementation Plan + +Phase 1: shared compare foundation. + +- Add shared toolkit models and target adapters. +- Implement live database and `.csdbtable` archive target reads. +- Implement schema diff output for tables, columns, indexes, views, triggers, + and procedures. +- Add JSON result contracts for CLI and Admin. + +Phase 2: schema deploy scripts. + +- Generate schema deployment script previews from schema diffs. +- Add destructive-change warnings. +- Add CLI schema compare command with JSON and script output. +- Add Admin schema diff preview. + +Phase 3: data compare and sync scripts. + +- Implement keyed row comparison for selected tables. +- Generate data sync script previews. +- Add CLI data compare command. +- Add Admin data diff preview. + +Phase 4: drift reports. + +- Add baseline comparison against backups and table archives. +- Add CLI drift command. +- Add Admin drift summary. +- Add CI-oriented JSON output and failure codes. + +## Future Test Plan + +- Schema diff tests for tables, columns, indexes, views, triggers, and + procedures. +- Data diff tests for inserted, deleted, updated, null, blob, composite-key, + missing-key, and empty-table cases. +- Archive target tests comparing live table data to `.csdbtable` exports. +- Script generation tests for ordering, escaping, destructive warnings, and + invalid sync cases. +- CLI tests for schema compare, data compare, drift, JSON output, script output, + and validation failures. +- Admin service tests for target loading, diff preview, script preview, and + destructive execution confirmation. diff --git a/docs/developer-experience-adoption-engine/README.md b/docs/developer-experience-adoption-engine/README.md new file mode 100644 index 00000000..e1c97cd9 --- /dev/null +++ b/docs/developer-experience-adoption-engine/README.md @@ -0,0 +1,221 @@ +# Developer Experience Adoption Engine Plan + +This document captures the planned design for a Developer Experience Adoption +Engine in CSharpDB. The goal is to make common development tasks fast, +discoverable, and pleasant enough that developers want to use CSharpDB for new +projects, demos, tests, and local-first applications. + +## Summary And Goals + +The Adoption Engine should reduce setup friction and make CSharpDB feel helpful +during everyday development. V1 should focus on SQL-first capabilities that are +easy to explain and easy to surface through the existing CLI, Admin, EF Core, +MCP, and documentation paths: + +- Schema-aware data seeding. +- Explicit query result caching. +- Computed columns. +- Simple analytics helpers. +- CLI and tooling improvements that make these features discoverable. + +This plan builds on existing shipped work: the `csharpdb` CLI, VS Code +extension, EF Core provider, MCP server, generated collections, pipelines, and +Admin UI. The new work should connect those surfaces around developer workflows +instead of creating a parallel toolchain. + +## Current Foundation + +CSharpDB already has several adoption surfaces: + +- `CSharpDB.Cli` provides the `csharpdb` executable, interactive SQL shell, + meta-commands, file execution, maintenance commands, inspection commands, and + ETL pipeline commands. +- `CSharpDB.EntityFrameworkCore` supports file-backed EF Core runtime and + migrations, but computed columns are currently unsupported. +- `CSharpDB.Mcp` exposes schema, data, mutation, and SQL tools for local agent + integrations. +- `CSharpDB.Generators` improves typed collection ergonomics with source + generated collection descriptors. +- Admin and VS Code already provide interactive schema and query workflows. + +The Adoption Engine should make these pieces feel like one developer story: +create a database, seed it, run useful queries, cache known-expensive reads, +inspect schema, and export examples or fixtures. + +## V1 SQL Contract + +Data seeding inserts schema-aware sample rows: + +```sql +SEED Customers WITH 1000 ROWS; +``` + +Query result caching is explicit per query: + +```sql +SELECT * FROM Customers CACHE 60s; +``` + +Computed columns can be declared from SQL expressions: + +```sql +CREATE TABLE Customers ( + Id INTEGER PRIMARY KEY, + FirstName TEXT, + LastName TEXT, + FullName AS FirstName + ' ' + LastName +); +``` + +Simple analytics helpers provide beginner-friendly grouped aggregates: + +```sql +SELECT COUNT(*) BY Status FROM Bookings; +``` + +Default semantics: + +- `SEED` uses the target table schema, column names, types, nullability, + identity columns, and foreign keys to generate valid rows. +- `SEED` appends rows and returns inserted row count, skipped column count, and + generation warnings. +- `CACHE ` is opt-in and scoped to deterministic read-only `SELECT` + statements. +- Cached query keys include normalized SQL text, parameter values, database + identity, schema version, and table write versions. +- Computed columns are virtual in v1: values are evaluated on read and are not + physically stored. +- Computed columns are read-only targets for `INSERT` and `UPDATE`. +- `SELECT COUNT(*) BY Status FROM Bookings` is parser sugar for `SELECT Status, + COUNT(*) FROM Bookings GROUP BY Status`. + +## Additional Adoption Areas + +These areas came up while reviewing the existing repo and should be considered +part of the broader adoption story: + +- Project bootstrap: `csharpdb init` to create a database, starter schema, seed + script, and optional sample app. +- Fixture workflow: `csharpdb seed` and SQL `SEED` should support repeatable + deterministic test data so integration tests can recreate the same rows. +- Schema diff and migration preview: complement EF Core migrations with a + CSharpDB-native diff/preview path for SQL-first users. +- Query diagnostics: surface cache hits, seed timing, generated row counts, and + computed-column evaluation costs through CLI timing and Admin query details. +- Snippets and recipes: ship copyable examples for Admin, CLI, VS Code, MCP, + EF Core, and common app stacks. +- Agent/tooling flow: expose seeding, cache inspection, and schema summaries + through `CSharpDB.Mcp` so local agents can prepare realistic databases. +- Package ergonomics: make `dotnet tool install csharpdb` and package + references the recommended entry points when distribution is ready. + +## Metadata And Execution Model + +Data seeding should live in the SQL execution layer and reuse normal insert +paths. It should infer value generators from column metadata: + +- Integer identity and primary-key columns use normal engine identity behavior + unless an explicit generator is supplied in a future phase. +- Text columns use column-name-aware generators for common names such as + `Name`, `Email`, `Phone`, `Status`, `City`, and `Description`. +- Foreign-key columns choose from existing parent keys; if no parent rows exist, + v1 should report a generation warning rather than creating parent rows + implicitly. +- Blob columns are skipped by default unless a future explicit generator is + provided. + +Query result caching should be an engine-level cache, not only an Admin or CLI +cache. It should be invalidated by writes to referenced tables and by schema +changes. The cache should be bounded by entry count and approximate memory size, +and `CACHE 0s` should bypass storage while preserving parse compatibility. + +Computed columns should be stored as schema metadata with expression SQL, +result type, and dependency column names. V1 virtual computed columns should be +available in `SELECT`, `WHERE`, `ORDER BY`, Admin browsing, schema inspection, +and EF Core model validation. Indexing computed columns should remain future +work. + +Analytics helpers should lower into ordinary aggregate queries during parsing +or planning. They should not introduce a separate execution engine. The helper +syntax should remain intentionally small in v1: `COUNT(*) BY FROM + [WHERE ...] [ORDER BY ...] [LIMIT ...]`. + +## CLI And Tooling Deliverables + +The existing `csharpdb` CLI should become the primary adoption path: + +- Add `.seed
` as an interactive shortcut for SQL `SEED`. +- Add `csharpdb seed
--rows [--json]` for scripts and CI. +- Add `.cache [status|clear]` and non-interactive `csharpdb cache` inspection + commands once the engine cache exists. +- Add `.init` or `csharpdb init` for starter databases and example schemas. +- Add `.examples` to list local SQL snippets for seeding, computed columns, + analytics helpers, pipelines, and external tables. +- Make CLI output include concise timing, cache-hit, and inserted-row summaries. + +Other surfaces should use the same core behavior: + +- Admin should expose seeding from table context menus and show cache hit/timing + badges in query results. +- VS Code should add snippets for `SEED`, `CACHE`, computed columns, and + analytics helpers. +- EF Core should map virtual computed columns once the SQL engine supports + them. +- MCP should expose tools for seed execution and schema/example discovery. + +## Non-Goals + +- No automatic caching of every query in v1. +- No stored computed columns in v1. +- No indexing computed columns in v1. +- No faker dependency requirement in core engine v1. +- No automatic parent-row creation for foreign-key seeding in v1. +- No broad analytics language beyond simple grouped count helpers in v1. + +## Phased Implementation Plan + +Phase 1: CLI and seeding foundation. + +- Add SQL parser and AST support for `SEED
WITH ROWS`. +- Implement schema-aware seed execution through normal insert paths. +- Add CLI shortcuts and non-interactive seed command. +- Add tests for deterministic generation, identity columns, nullability, and + foreign-key warnings. + +Phase 2: computed columns. + +- Add computed-column schema metadata and parser support in `CREATE TABLE`. +- Evaluate virtual computed values during reads. +- Reject writes targeting computed columns. +- Update schema surfaces, Admin browse views, CLI `.schema`, EF Core validation, + and system catalog metadata. + +Phase 3: explicit query result caching. + +- Add parser support for `CACHE ` on read-only `SELECT` statements. +- Add bounded engine cache with table-write/schema invalidation. +- Surface cache hit/miss and expiration diagnostics through CLI and Admin. +- Add cache inspection and clear commands. + +Phase 4: analytics helpers and adoption polish. + +- Lower `SELECT COUNT(*) BY ... FROM ...` into ordinary grouped aggregates. +- Add snippets, examples, `csharpdb init`, and MCP discovery tools. +- Add docs that connect CLI, Admin, EF Core, MCP, and VS Code workflows. + +## Future Test Plan + +- Parser tests for `SEED`, `CACHE`, computed columns, and analytics helper + syntax, including invalid syntax. +- Seeding tests for all scalar types, identity columns, nullable columns, + foreign keys with and without parent rows, and deterministic repeatability. +- Computed-column tests for projection, filtering, ordering, schema metadata, + write rejection, and reopen persistence. +- Cache tests for TTL expiry, write invalidation, schema invalidation, + parameterized query keys, memory bounds, and read-only enforcement. +- Analytics helper tests that compare helper output to equivalent `GROUP BY` + SQL. +- CLI tests for `.seed`, non-interactive `seed`, cache inspection, examples, + and JSON output. +- Surface tests for Admin, EF Core, MCP, and VS Code snippets as those phases + are implemented. diff --git a/docs/releases/v3.8.0-pr-notes.md b/docs/releases/v3.8.0-pr-notes.md new file mode 100644 index 00000000..932c51d4 --- /dev/null +++ b/docs/releases/v3.8.0-pr-notes.md @@ -0,0 +1,121 @@ +## Summary + +This `version3.8.0` release covers the commit range +`ed572e47f9d20a4ffa40400cd0467f664f717f3e..HEAD`. + +The release adds native `.csdbtable` table archives, Admin Import / Export +workflows, and read-only SQL external tables. Archives now use the native +`CSDBTBL3` seekable format with schema and manifest metadata, encoded rows, row +counts, source table metadata, created timestamps, and optional integer +primary-key archive indexes. Admin can export tables to browser downloads or +server/database-local paths, register archives as external tables, restore +archives to physical tables, show progress, and cancel long-running work. + +The SQL/runtime work adds `CREATE EXTERNAL TABLE ... FROM 'path.csdbtable'`, +`DROP EXTERNAL TABLE`, the internal `__external_tables` metadata table, +`sys.external_tables`, external table scans, indexed archive lookups, and joins +between live and archived tables. External `.csdbtable` registrations are +read-only in this release; writes, index creation, ALTER operations, and trigger +targets are rejected. + +The Admin release also adds a Data Model tab. It visualizes user tables and +external tables on a draggable ERD-style canvas, supports zoom, opens from +Object Explorer and the command palette, and can hand selected layouts to the +existing Query Designer. The follow-up diagram work adds a real +`__data_model_diagrams` system table, `sys.diagrams`, named diagram +save/load/delete, remembered table membership and placement, a default global +diagram, and preview-first staged schema operations for SQL Server-style diagram +editing. + +The docs/site portion adds planning documents for writable external tables, +Data Hygiene Engine, Developer Experience, Advanced Differentiators, and the +Database DevOps Toolkit. It also refreshes the roadmap, changelog, blog index, +and adds the v3.8 native table archives blog post. + +## Type of Change + +- [ ] Bug fix +- [x] New feature +- [ ] Breaking change +- [x] Documentation update +- [x] Refactor / maintenance +- [ ] Tests only + +## Related Issues + +No issue numbers were linked for this release branch. Included work: + +- Added `CSharpDB.ImportExport` and `CSharpDB.Admin.ImportExport`. +- Added native `.csdbtable` archive writing, reading, metadata, restore, and + integer primary-key archive lookup support. +- Added direct engine-transport table snapshot export for large table archives. +- Added Admin Import / Export progress UI, cancellation, download staging, + server-path exports, restore, and external table registration. +- Added `CREATE EXTERNAL TABLE` and `DROP EXTERNAL TABLE` parser/runtime + support. +- Added `__external_tables`, `sys.external_tables`, external table object + explorer entries, and query-tab catalog discovery. +- Added external table scan, external index nested-loop join, and fast archive + primary-key lookup planning. +- Enforced read-only external table behavior for write, index, ALTER, and + trigger-target operations. +- Added Admin Data Model tab, schema canvas extraction, object explorer and + command palette entry points, table/external table context entries, and Query + Designer handoff. +- Added `__data_model_diagrams`, `sys.diagrams`, hidden internal-table + filtering, named diagram save/load/delete, and default global diagram + placement persistence. +- Added staged diagram schema operations for new table, drop table, rename + table, add/drop/rename column, and add/drop relationship workflows. +- Removed the unreleased `__data_model_layout:` saved-query persistence path. +- Added planning docs for writable external tables, data hygiene, developer + experience, advanced differentiators, and database DevOps tooling. +- Refreshed roadmap/changelog/static website pages and added the native table + archives blog post. + +## Testing + +- [ ] `dotnet build CSharpDB.slnx` +- [x] Relevant tests executed +- [x] Failure-path tests executed (if applicable: cancellation, invalid/unsupported inputs, non-`DbException` paths) +- [x] Manual verification performed (if applicable) + +Validation performed: + +- `dotnet build src\CSharpDB.Admin\CSharpDB.Admin.csproj` +- `dotnet test tests\CSharpDB.Tests\CSharpDB.Tests.csproj --no-restore --filter "TableArchiveTests|ExternalTableTests|ParserTests|EngineTransportClientTests|DataModelGraphBuilderTests|DataModelDiagramServiceTests|SystemCatalogTests"` + - Passed: 170 tests. +- `dotnet test tests\CSharpDB.Admin.Reports.Tests\CSharpDB.Admin.Reports.Tests.csproj --no-restore --filter DbReportSourceProviderTests` + - Passed: 4 tests. +- `dotnet test tests\CSharpDB.Admin.Forms.Tests\CSharpDB.Admin.Forms.Tests.csproj --no-restore --filter TabManagerServiceTests` + - Passed: 17 tests. +- `git diff --check` +- Browser smoke test for opening the Admin Data Model tab and New Table staging + panel. + +## Checklist + +- [x] I followed the project style and conventions. +- [x] I added or updated tests for behavior changes. +- [x] I covered both success and failure paths for changed behavior. +- [x] I updated docs for user-facing changes. +- [x] I verified no sensitive data was added. + +## Notes for Reviewers + +- `.csdbtable` archives are native seekable packages, not CSV/JSON exports or a + ZIP of JSON rows. +- `.csdbtable` external tables are read-only by design in this release. Writable + external tables are planned separately with a mutable `.csdbx` file. +- External table registrations are stored in `__external_tables`; Data Model + diagrams are stored in `__data_model_diagrams`. Both internal tables are + hidden from normal table lists and exposed through `sys.*` views. +- The Data Model tab persists layout/table membership independently from schema + apply operations. Staged schema edits preview SQL first, then apply through + existing client/engine capabilities. +- Drop-table diagram actions still respect the existing engine FK rules. Tables + with foreign-key-owned support indexes may need relationship cleanup before a + drop can apply. +- An initial parallel validation attempt hit a transient compiler output lock + from `VBCSCompiler`; the affected tests were rerun sequentially after + `dotnet build-server shutdown` and passed. diff --git a/docs/roadmap.md b/docs/roadmap.md index d47e660f..05659079 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,6 +1,6 @@ # CSharpDB Roadmap -This document outlines the planned direction for CSharpDB, organized by timeframe and priority. Items are roughly ordered by expected impact within each tier, and statuses are intended to reflect the current `v3.4.0` state of the repo. +This document outlines the planned direction for CSharpDB, organized by timeframe and priority. Items are roughly ordered by expected impact within each tier, and statuses are intended to reflect the current `v3.8.0` state of the repo. --- @@ -32,6 +32,7 @@ Recently completed improvements to query performance, storage/runtime behavior, | **Collection path indexes** | Nested scalar, array-element, nested array-object, Guid, temporal, and ordered text path indexes with `FindByPathAsync` / `FindByPathRangeAsync` | Done | | **Hybrid storage mode** | Lazy-resident durable storage with gRPC tunable file-cache configuration; Admin direct local hosting keeps a warm in-process database instance and uses hybrid incremental-durable options by default | Done | | **Client backup/restore** | `BackupAsync` / `RestoreAsync` as first-class `ICSharpDbClient` operations across direct, HTTP, gRPC, CLI, and Admin | Done | +| **Native table archives and external tables** | Native `.csdbtable` table snapshots with fast Admin Import / Export, download or server-path destinations, `CREATE EXTERNAL TABLE` / `DROP EXTERNAL TABLE`, `sys.external_tables`, read-only external table scans/joins, and embedded primary-key archive lookup indexes for eligible point reads | Done | | **Older DB foreign-key retrofit migration** | Validate/apply maintenance workflow that rewrites existing child tables with persisted FK metadata across direct, HTTP, gRPC, CLI, and Admin | Done | --- @@ -42,7 +43,8 @@ SQL feature parity, provider/tooling compatibility, and ecosystem expansion. | Feature | Description | Status | |---------|-------------|--------| -| **User-defined functions and commands** | Trusted in-process C# scalar functions are implemented for SQL, triggers/procedures, direct clients, Admin Forms/Reports, and pipelines; trusted commands now back Admin Forms lifecycle events, command-button clicks, selected control events, Admin Reports render lifecycle events, and pipeline run hooks; Admin Forms now have declarative action sequences for run-command, set-field, show-message, and stop steps. Broader built-in scalar functions, native plugin extensions, aggregate/table-valued UDFs, richer macro flow, additional Access-style control events, and sandboxed UDFs remain future work | Partial | +| **User-defined functions and commands** | Done for the trusted in-process model: host-registered C# scalar functions, common SQL/Admin built-ins, trusted commands, Admin Forms/Reports/pipeline hooks, declarative Admin Forms action sequences, and local Admin Forms C# code modules are implemented across the supported surfaces. Untrusted sandboxed UDF execution is intentionally out of scope | Done | +| **Writable external tables** | Planned opt-in writable external table registrations over mutable `.csdbx` files, backed by CSharpDB B+tree storage and limited to DML (`INSERT`, `UPDATE`, `DELETE`) in v1 while `.csdbtable` archives remain read-only | Planned | | **Subqueries** | Scalar subqueries, `IN (SELECT ...)`, `EXISTS (SELECT ...)`, including correlated evaluation in `WHERE`, non-aggregate projection, and `UPDATE`/`DELETE` expressions | Done | | **`UNION` / `INTERSECT` / `EXCEPT`** | Set operations across SELECT results, including use in top-level queries, views, and CTE query bodies | Done | | **Window functions** | `ROW_NUMBER()`, `RANK()`, `DENSE_RANK()`, `LEAD()`, `LAG()` | Planned | @@ -77,6 +79,7 @@ Advanced features and fundamental architecture enhancements. | **JSON path querying** | Query into JSON document fields in the Collection API (e.g., `$.address.city`) via `FindByPathAsync` / `FindByPathRangeAsync` | Done | | **Advanced collection storage path** | Binary direct-payload format with direct binary hydration, path-based field extraction, and richer expression/path indexes | Done | | **SQL batched row transport** | Internal row-batch transport serves as the batch-first SQL execution foundation across batch-capable result boundaries, scans, joins, and generic aggregates | Done | +| **External table index coverage** | Follow writable `.csdbx` storage with broader external-table indexes, planner costing, and multi-column lookup/range support beyond the current archive primary-key point-lookup path | Planned | | **Source-generated collection fast path** | Done for the current phase: opt-in generated collection models now provide `GetGeneratedCollectionAsync(...)`, generated field descriptors/index bindings, analyzer-packaged collection model/codecs, generated binary direct-payload encode/decode for supported document graphs, source-generated JSON fallback for unsupported shapes, trim/NativeAOT smoke coverage, and a dedicated sample | Done | | **Generated collection package ergonomics** | Streamline NuGet/analyzer packaging, templates, onboarding docs, and generated-collection setup so consumers can adopt the opt-in path with less project wiring | Planned | | **Broader generated collection model coverage** | Expand generator support beyond the current scalar, scalar collection, nested scalar, and nested collection-scalar shapes; unsupported shapes currently warn and fall back to source-generated JSON instead of binary direct payloads | Planned | @@ -91,8 +94,7 @@ Advanced features and fundamental architecture enhancements. | **Initial multi-writer support** | Explicit `WriteTransaction` conflict-detected retry flow, shared auto-commit non-insert isolation, and opt-in `ConcurrentWriteTransactions` for shared implicit inserts | Done | | **Broader multi-writer insert optimization** | Opt-in `ConcurrentWriteTransactions` now reserves shared row-id ranges and rebases hot right-edge insert pages against pending WAL images, improving concurrent one-row auto-ID and explicit-ID insert fan-in while keeping serialized inserts as the default | Done | | **API-level sharding** | Route API/daemon requests across multiple warm CSharpDB database files so independent tenants or shard keys can use separate WAL and commit paths, with v1 focused on single-shard writes and point reads | Research | -| **Replication / change feed** | Stream committed changes for read replicas or event-driven architectures | Research | -| **WebAssembly sandboxed UDFs** | Execute untrusted user-submitted functions in a WASM sandbox with resource limits (fuel, memory caps) via Wasmtime | Research | +| **Replication / change feed** | Retained commit-log change feeds and reactive query subscriptions for read replicas, live Admin views, and event-driven applications | Research | --- @@ -102,7 +104,7 @@ These are known simplifications in the current implementation: | Area | Limitation | |------|-----------| -| **Functions and automation** | Trusted in-process C# scalar functions are supported when registered by the host; Admin Forms, Admin Reports, and pipelines can invoke trusted host commands from supported event/hook surfaces; Admin Forms support declarative action sequences for run-command, set-field, show-message, and stop steps. Broader built-in scalar functions, aggregate/table-valued UDFs, stored C# modules, remote delegate serialization, additional Access-style control events, richer macro flow, and sandboxed UDFs are not implemented | +| **Functions and automation** | CSharpDB's UDF/command model is trusted and in-process by design. Current supported surfaces include host-registered scalar functions, common built-ins, trusted commands, form/report/pipeline hooks, declarative action sequences, and local Admin Forms C# modules; untrusted sandboxed execution is intentionally out of scope | | **Query** | Scalar/`IN`/`EXISTS` subqueries are supported, including correlated cases in `WHERE`, non-aggregate projection, and `UPDATE`/`DELETE` expressions; correlated subqueries are not yet supported in `JOIN ON`, `GROUP BY`, `HAVING`, `ORDER BY`, or aggregate projections | | **Query** | `UNION`, `INTERSECT`, and `EXCEPT` are supported; `UNION ALL` is not implemented yet | | **Query** | No window functions | @@ -110,9 +112,10 @@ These are known simplifications in the current implementation: | **Indexes** | Equality lookups support current `INTEGER`/`TEXT` indexes, but ordered range-scan pushdown is still limited to single-column `INTEGER` index paths | | **RowId** | Legacy table schemas without persisted high-water metadata may pay a one-time key scan on first insert | | **Collections** | `FindByIndexAsync` supports declared field-equality lookups; `FindByPathAsync` and `FindByPathRangeAsync` support path-based queries on indexed paths; `FindAsync` remains a full scan for unindexed predicates. Generated collections require registered descriptors for existing collection indexes; unsupported generated model shapes warn and use the source-generated JSON fallback instead of binary direct payloads | +| **External Tables** | Native `.csdbtable` archives can be registered and queried as read-only external tables. Writable external tables are planned as an opt-in `.csdbx` format; current archives remain read-only, and broader external indexes, range seeks, and deeper planner costing remain planned | | **Networking** | `CSharpDB.Daemon` now hosts both REST and gRPC from one process; named pipes remain reserved but are not implemented end to end today | | **Security** | Remote REST and daemon gRPC support opt-in API-key authentication, defaulting to `None` for backward compatibility. JWT, RBAC, mTLS helpers, TLS-specific configuration, and at-rest encryption are not implemented | -| **Admin Forms** | The Forms designer/runtime supports the core generated-form and data-entry path plus initial trusted command-backed automation, including lifecycle events, command buttons, selected control events, and declarative action sequences, but still needs Access-parity work for responsive runtime rendering, complete inferred validation, richer form modes, broader built-in actions, additional events, advanced filtering/sorting, and broader controls | +| **Admin Forms** | The Forms designer/runtime supports the core generated-form and data-entry path plus trusted command-backed automation, including lifecycle events, command buttons, selected-control events, conditional UI rules, domain formula helpers, and declarative action sequences for current record, form navigation, filtering, SQL/procedure, and control-property workflows. It still needs Access-parity work for responsive runtime rendering, complete inferred validation, richer form modes, additional events, advanced filtering/sorting, report/query/import/export actions, macro loops/on-error/temp vars, and broader controls | | **Admin Reports** | The Reports designer/runtime supports the core banded preview path plus trusted command-backed preview lifecycle events, but still needs Access-parity work for bounded saved-query previews, full report output/export, parameters, richer grouping and totals semantics, conditional formatting, subreports, and broader controls | | **Text / Multilingual** | Text is stored as UTF-8 and supports all Unicode languages; default semantics remain ordinal, but opt-in `BINARY`, `NOCASE`, `NOCASE_AI`, and `ICU:` collation are implemented for SQL and collection indexes. Dedicated ordered SQL text index optimization remains planned | | **Concurrency** | The physical WAL commit path is still serialized at the storage boundary. Initial multi-writer support is shipped, but observed gains still depend on conflict shape and whether shared auto-commit `INSERT` is left on the default serialized path | @@ -179,6 +182,7 @@ Major features already implemented: - Full-text search with tokenization, stemming, and relevance ranking - Hybrid storage mode with lazy-resident durable storage and gRPC tunable file-cache - Client-wide `BackupAsync` / `RestoreAsync` across direct, HTTP, gRPC, CLI, and Admin +- Native `.csdbtable` table archives with Admin Import / Export, read-only external table registration, `sys.external_tables`, external table scans/joins, and embedded archive primary-key lookup indexes - `ReplaceAsync` for index stores - Package-driven ETL pipelines with validation, dry-run, execute/resume, persisted run history, and Admin visual designer support @@ -197,8 +201,14 @@ Major features already implemented: - [Storage Engine Guide](storage/README.md) — CSharpDB.Storage API reference: device, pager, B+tree, WAL, indexing, serialization, and catalog - [Compression SDK Sample](../samples/compression-sdk/README.md) — Application-level payload compression helpers, benchmarks, and storage-format boundaries - [Native FFI Tutorials](tutorials/native-ffi/README.md) — Python and Node.js examples using the NativeAOT shared library -- [User-Defined Functions Plan](user-defined-functions/README.md) — C# library functions callable by the database, native plugin extensions, and WASM sandboxing +- [User-Defined Functions Plan](user-defined-functions/README.md) — Trusted C# functions, commands, form/report/pipeline hooks, and code modules +- [Data Hygiene Engine Plan](data-hygiene-engine/README.md) — SQL-first duplicate detection, deduplication, validation rules, and orphan detection +- [Developer Experience Adoption Engine Plan](developer-experience-adoption-engine/README.md) — SQL-first seeding, query caching, computed columns, analytics helpers, and CLI/tooling polish +- [Advanced Differentiators Plan](advanced-differentiators/README.md) — Archive-backed time travel, change feeds, reactive queries, and full-text SQL ergonomics +- [Database DevOps Toolkit Plan](database-devops-toolkit/README.md) — Schema compare, data compare, drift reports, and generated deployment scripts +- [Writable External Tables Plan](writable-external-tables/README.md) — Opt-in writable external tables backed by mutable B+tree external files - [Pub/Sub Change Events Plan](pub-sub-events/README.md) — Engine-level change events with channel-based delivery for real-time data subscriptions - [Admin Forms Access Parity Plan](admin-forms-access-parity/README.md) — Microsoft Access parity review findings and forms roadmap - [Admin Reports Access Parity Plan](admin-reports-access-parity/README.md) — Microsoft Access parity review findings and reports roadmap - [Benchmark Suite](../tests/CSharpDB.Benchmarks/README.md) — Performance data informing optimization priorities +- [Native Table Archives Blog](https://csharpdb.com/blog/table-archives.html) — v3.8 table archive, external table, and Admin Import / Export overview diff --git a/docs/trusted-csharp-functions/README.md b/docs/trusted-csharp-functions/README.md index d7df8bb6..b7e22d07 100644 --- a/docs/trusted-csharp-functions/README.md +++ b/docs/trusted-csharp-functions/README.md @@ -2,7 +2,7 @@ CSharpDB can call host-registered C# scalar functions from SQL and the embedded expression surfaces that sit on top of the engine. This is the CSharpDB equivalent of an Access-style application function integration: the application owns the C# code, registers it while opening or hosting the database, and users call the function by name in database expressions. -This feature is intentionally trusted and in-process. It does not store C# source code in the database, sandbox user code, load plugin assemblies from database files, or serialize delegates over HTTP or gRPC. +This callback feature is intentionally trusted and in-process. Host-registered callbacks do not store C# source code in the database, sandbox user code, load plugin assemblies from database files, or serialize delegates over HTTP or gRPC. For the newer database-owned Admin Forms C# module workflow, see [User-Defined Functions And Commands](../user-defined-functions/README.md). For an end-to-end app-builder walkthrough that combines Admin Forms, collections, macro actions, reports, trusted callbacks, and callback readiness, see the @@ -631,7 +631,7 @@ var shipButton = existingButton with }; ``` -The action set is intentionally small and form-focused: +The action set is intentionally form-focused: | Action | Behavior | | --- | --- | @@ -647,6 +647,16 @@ The action set is intentionally small and form-focused: | `PreviousRecord` | Moves the rendered form to the previous record. | | `NextRecord` | Moves the rendered form to the next record. | | `GoToRecord` | Navigates to a primary-key value from `Value`, `Arguments["value"]`, `Arguments["recordId"]`, `Arguments["primaryKey"]`, or the field named by `Target`. | +| `OpenForm` | Opens another saved form through the rendered form host. | +| `CloseForm` | Closes the current or named rendered form surface. | +| `ApplyFilter` | Applies a form or data-grid filter. | +| `ClearFilter` | Clears a form or data-grid filter. | +| `RunSql` | Executes SQL through the rendered host when SQL actions are enabled by policy. | +| `RunProcedure` | Executes a saved procedure through the rendered host when procedure actions are enabled by policy. | +| `SetControlProperty` | Overrides a rendered control property such as `visible`, `enabled`, `readOnly`, `text`, `placeholder`, or bound `value`. | +| `SetControlVisibility` | Short form for setting a rendered control's `visible` property. | +| `SetControlEnabled` | Short form for setting a rendered control's `enabled` property. | +| `SetControlReadOnly` | Short form for setting a rendered control's `readOnly` property. | Reusable action sequences are stored once on the form and invoked by name from form events, control events, or command buttons: @@ -709,9 +719,10 @@ var form = existingForm with The Admin Forms property inspector exposes action sequences with a visual editor on form-level and selected-control event bindings. Designers can add a -sequence, name it, add command, reusable-sequence, field, message, stop, and -built-in record steps, reorder or remove steps, choose registered commands or -reusable sequences when available, and set per-step conditions and +sequence, name it, add command, reusable-sequence, field, message, stop, +rendered record, form, filter, SQL/procedure, and control-property steps, +reorder or remove steps, choose registered commands or reusable sequences when +available, and set per-step conditions and `StopOnFailure`. The form-level property inspector also includes a reusable action-sequence library editor. JSON editing remains only for optional binding, `RunCommand`, or `RunActionSequence` argument payloads. @@ -753,16 +764,18 @@ backfilled when it is loaded. `BeforeInsert` and `BeforeUpdate`, and it can update the current rendered record from control events or command-button clicks. -Built-in record actions require a rendered Admin Forms data-entry runtime. -They are intended for command buttons and selected-control events. Headless -form lifecycle dispatch can still run `SetFieldValue`, `ShowMessage`, `Stop`, -and `RunCommand`, but it reports a failure if a sequence asks for rendered-form -navigation or save/delete actions. +Built-in record, form navigation, filter, SQL/procedure, and control-property +actions require a rendered Admin Forms data-entry runtime. They are intended for +command buttons and selected-control events. Headless form lifecycle dispatch +can still run `SetFieldValue`, `ShowMessage`, `Stop`, and `RunCommand`, but it +reports a failure if a sequence asks for actions that need the rendered form +instance. -Action sequences do not include loops, stored C# source, database-owned -plugins, or remote delegate serialization. Rendered Admin form runtimes support -direct SQL and procedure actions only when the host explicitly enables those -capabilities. +Action sequences do not include loops, a `RunCode` macro action, +database-owned plugins, or remote delegate serialization. Database-owned C# +form modules are handled as trusted event handlers through the separate code +module runtime. Rendered Admin form runtimes support direct SQL and procedure +actions only when the host explicitly enables those capabilities. --- @@ -1026,11 +1039,12 @@ V1 does not support: - Aggregate UDFs. - Table-valued UDFs. -- Stored C# source code or database-owned compiled modules. +- Database-owned C# modules beyond the local Admin Forms event-handler MVP, + such as report modules, procedure modules, or remote/daemon execution. - Sandboxed execution. - Async scalar delegates. - Passing a database handle into the function context. - Sending delegates over HTTP, gRPC, or pipeline package files. - Optimizer pushdown, expression indexes, generated columns, or constant folding based on custom function metadata. - Additional Access-style control events such as double-click, key, mouse, timer, and dirty/current events. -- Richer macro/action scripts with loops, reusable UI rule presets, additional event surfaces, or database-owned executable code. +- Richer macro/action scripts with loops, reusable UI rule presets, additional event surfaces, or `RunCode` action invocation. diff --git a/docs/user-defined-functions/README.md b/docs/user-defined-functions/README.md new file mode 100644 index 00000000..7969d657 --- /dev/null +++ b/docs/user-defined-functions/README.md @@ -0,0 +1,66 @@ +# User-Defined Functions And Commands Plan + +This page tracks the shipped and remaining work for CSharpDB function, +command, and automation extensibility. The implementation separates portable +database metadata from local execution trust: databases can reference functions, +commands, validation rules, declarative action sequences, and C# code modules, +while executable code runs only in trusted host contexts. + +## Shipped Capabilities + +- Trusted in-process C# scalar functions through `DbFunctionRegistry`, available + to SQL, triggers/procedures, direct clients, Admin Forms/Reports, and + pipelines. +- Common built-in scalar functions for SQL and Admin formulas, including text, + date/time, numeric, conversion, null, and conditional helpers. +- Admin Forms domain helper functions such as `DLookup`, `DCount`, `DSum`, + `DAvg`, `DMin`, and `DMax`, resolved by the rendered Forms runtime with + bounded row loading. +- Trusted command callbacks through `DbCommandRegistry`, used by Admin Forms, + Admin Reports, and pipeline lifecycle hooks. +- Admin Forms action sequences for host commands, reusable sequences, field + updates, messages, stop, rendered record navigation/save/delete/refresh/go-to, + open/close form, filter, SQL/procedure, control-property, conditional, and + rule workflows. +- Database-owned C# code modules for the local Admin Forms MVP: module source is + stored in `__code_modules`, build diagnostics are stored in + `__code_module_builds`, VS Code-friendly file sync exports/imports + `.csharpdb-code/csharpdb.codeproj.json` plus `.cs` files, and form/control + events can bind to trusted in-process handlers. +- Automation metadata and validation surfaces that let exported forms, reports, + and pipeline packages describe required host callbacks. + +## Current Boundaries + +- Scalar delegates are host-owned and in-process; delegates are not serialized + over HTTP, gRPC, or package files. +- Trusted callbacks are policy-mediated but not sandboxed. They run with the + permissions of the host process that registered them. +- Database-owned C# source execution requires host opt-in, successful Roslyn + build, and explicit local trust for the current module-set hash. Trust is not + stored in the database and source changes invalidate it. +- The current code-module runtime is limited to local Admin Forms form/control + handlers; reports, procedures, daemon/remote execution, in-browser editing, + and debugging integration remain outside this slice. +- Custom scalar functions are intentionally not used for optimizer pushdown, + expression indexes, generated columns, or constant folding. + +## Future Work + +- Aggregate and table-valued UDFs. +- Native/plugin extension loading with an explicit trust and packaging model. +- Report modules and broader database-owned code module surfaces beyond the + local Admin Forms MVP. +- Sandboxed UDF execution, including the WebAssembly/Wasmtime research track. +- Remote delegate or extension registration for daemon-hosted deployments. +- Additional Access-style form/control events, macro loops, on-error handling, + temp/session variables, and broader report/query/import/export actions. + +## Related Docs + +- [Trusted C# Callbacks](../trusted-csharp-functions/README.md) +- [Trusted Validation Rules](../trusted-csharp-functions/validation-rules.md) +- [Access-Style Macro Actions](../trusted-csharp-functions/access-style-macro-actions.md) +- [Access-Style Functions and Macros](../admin-forms-access-parity/access-style-functions-and-macros.md) +- [Database Code Modules With VS Code Sync Plan](../trusted-csharp-functions/database-code-modules-vscode-plan.md) +- [WebAssembly sandboxed UDFs](../roadmap.md#long-term) diff --git a/docs/writable-external-tables/README.md b/docs/writable-external-tables/README.md new file mode 100644 index 00000000..42ca1f26 --- /dev/null +++ b/docs/writable-external-tables/README.md @@ -0,0 +1,263 @@ +# Writable External Tables Plan + +This document captures the planned design for making CSharpDB external tables +writable. The current external table feature is intentionally read-only and is +based on native `.csdbtable` table archives. Writable external tables should be +a separate opt-in capability backed by a mutable B+tree external-table file. + +## Summary And Goals + +The goal is to let users keep a table outside the main database file while +still using normal SQL `SELECT`, `INSERT`, `UPDATE`, and `DELETE` statements +against it. + +The first implementation should: + +- Keep existing `.csdbtable` archives read-only. +- Add a mutable `.csdbx` external-table file format. +- Require explicit writable registration. +- Support DML only: `INSERT`, `UPDATE`, and `DELETE`. +- Reuse existing CSharpDB storage primitives where practical. +- Keep query behavior consistent with current external table scans and joins. + +## Current Read-Only Behavior + +Today, external tables are registrations over native `.csdbtable` archives: + +```sql +CREATE EXTERNAL TABLE archived_customers +FROM 'exports/customers.csdbtable'; +``` + +The archive is a table snapshot. Query planning can resolve it as a table source +for normal `SELECT` statements, including filters, projections, joins, ordering, +and `COUNT(*)` metadata fast paths. Eligible integer primary-key point lookups +can use the embedded archive index. + +The engine currently rejects mutating statements against external tables: + +- `INSERT` +- `UPDATE` +- `DELETE` +- `ALTER TABLE` +- `CREATE INDEX` +- Trigger target usage + +That behavior should remain the default for `.csdbtable` registrations. + +## Target SQL Contract + +Writable external tables must be opt-in at registration time: + +```sql +CREATE EXTERNAL TABLE customers_archive +FROM 'exports/customers.csdbx' +WITH (WRITABLE = TRUE); +``` + +Existing syntax remains read-only: + +```sql +CREATE EXTERNAL TABLE customers_archive +FROM 'exports/customers.csdbtable'; +``` + +Registration rules: + +- `WITH (WRITABLE = TRUE)` requires a mutable `.csdbx` file. +- `.csdbtable` archives cannot be made writable in place. +- If a user wants to write to an archive, Admin or a service API must convert it + to `.csdbx` first. +- `sys.external_tables` should expose whether the registration is writable and + which storage format backs it. + +Recommended metadata additions: + +| Column | Meaning | +| --- | --- | +| `is_writable` | `1` for writable external table registrations, otherwise `0`. | +| `storage_format` | `archive-native-v3` for `.csdbtable`, `mutable-btree-v1` for `.csdbx`. | +| `row_count` | Current row count for writable stores and manifest row count for archives. | + +## Mutable `.csdbx` Storage Model + +The `.csdbx` file should be a single-table mutable external store. It should not +be a full attached database in v1. + +Use existing storage primitives: + +- `Pager` for page management and WAL-backed durability. +- `SchemaCatalog` for table schema, row count, and `NextRowId`. +- `BTree` for the external table row store. +- Existing record serialization for row payloads. + +The external file should contain: + +- A file header identifying `CSDBEXT1`. +- One table schema. +- One table row B+tree. +- Persisted row count. +- Persisted `NextRowId`. +- Enough metadata to reopen the file without consulting the main database. + +The main database stores only the external table registration metadata. The +external file owns its table rows and table metadata. + +## DML Behavior + +Writable external tables should participate in existing SQL DML paths as closely +as possible while targeting the external store instead of the main database +catalog. + +### INSERT + +`INSERT` should: + +- Resolve column lists and values using the external table schema. +- Enforce nullability and integer primary-key identity behavior. +- Allocate `NextRowId` from the external file metadata. +- Insert into the external table B+tree. +- Update the external row count. + +### UPDATE + +`UPDATE` should: + +- Scan or seek the external table rows using existing predicate evaluation where + practical. +- Materialize rows to update before mutating the B+tree. +- Preserve primary-key uniqueness when the key changes. +- Update matching rows in the external B+tree. +- Keep external row count unchanged. + +### DELETE + +`DELETE` should: + +- Scan or seek matching external rows. +- Delete matching row IDs from the external B+tree. +- Decrement the external row count. + +## Non-Goals For V1 + +The first writable external table implementation should not include: + +- Making `.csdbtable` archives writable. +- `ALTER TABLE` support for writable external tables. +- `CREATE INDEX` or `DROP INDEX` on writable external tables. +- Trigger targets on writable external tables. +- Foreign-key enforcement across the main database and external files. +- Cross-store explicit transactions. +- Multi-table external files. +- Treating `.csdbx` as a full attached database. + +These restrictions keep the first version focused on the core writable table +contract and avoid introducing a distributed transaction problem. + +## Admin Workflow + +Admin should expose writable external tables through the existing Import / Export +surface. + +Recommended workflow: + +1. User opens Import / Export. +2. User selects Register External Table. +3. User chooses an archive or mutable external file path. +4. User enables `Writable external table`. +5. If the path is `.csdbtable`, Admin asks for a `.csdbx` conversion path. +6. Admin converts the archive to `.csdbx` using progress and cancellation. +7. Admin registers the `.csdbx` file with `WITH (WRITABLE = TRUE)`. + +Object Explorer should show writable external tables distinctly from read-only +archives. Data-grid editing should be enabled only for writable registrations. + +## Phased Implementation Plan + +### Phase 1 - Storage Foundation + +- Add the `.csdbx` file header and open/create APIs. +- Create a single-table external store wrapper over `Pager`, `SchemaCatalog`, + `BTree`, and existing row serialization. +- Implement archive-to-`.csdbx` conversion. +- Add tests for create, reopen, row count, schema, nulls, blobs, identity state, + and primary-key uniqueness. + +### Phase 2 - SQL Registration And Metadata + +- Extend parser support for `WITH (WRITABLE = TRUE)`. +- Extend external table metadata with `is_writable` and `storage_format`. +- Validate that writable registrations point at `.csdbx` files. +- Keep existing `.csdbtable` registrations read-only by default. +- Update `sys.external_tables`. + +### Phase 3 - Query Integration + +- Add a B+tree-backed external table scan operator. +- Resolve writable external tables to the new operator. +- Preserve existing filters, joins, projections, ordering, and aggregates. +- Keep current archive scan and primary-key lookup operators unchanged for + `.csdbtable`. + +### Phase 4 - DML Integration + +- Route writable external table `INSERT`, `UPDATE`, and `DELETE` to an external + mutation executor. +- Reuse existing row resolution and expression evaluation rules. +- Update external row count and `NextRowId` inside the `.csdbx` file. +- Reject unsupported DDL and explicit cross-store transaction usage with clear + errors. + +### Phase 5 - Admin Integration + +- Add writable registration controls. +- Add archive-to-`.csdbx` conversion progress and cancellation. +- Show writable/read-only state in registered external table lists. +- Enable data-grid editing only for writable external tables. + +## Test Plan + +Parser tests: + +- Parse writable external table registration. +- Reject malformed `WITH` options. +- Confirm current read-only syntax still parses. + +Storage tests: + +- Create and reopen `.csdbx`. +- Convert `.csdbtable` to `.csdbx`. +- Preserve schema, null values, integers, reals, text escaping, blobs, row count, + and identity state. +- Reject duplicate primary keys. + +Engine tests: + +- Register writable external table and select rows. +- Insert into writable external table. +- Update writable external table rows. +- Delete writable external table rows. +- Join physical tables to writable external tables after mutations. +- Reject writes to read-only `.csdbtable` external tables. +- Reject `ALTER`, `CREATE INDEX`, and trigger target usage against writable + external tables. +- Reject writable external table DML inside explicit cross-store transactions. +- Verify `sys.external_tables` exposes writable state and storage format. + +Admin tests: + +- Register read-only archive unchanged. +- Convert archive to writable external file. +- Register writable external table. +- Show progress and support cancellation during conversion. +- Show writable state in Object Explorer and Import / Export registration lists. + +## Acceptance Criteria + +- Existing read-only external table behavior is unchanged. +- Writable external tables require explicit opt-in. +- `.csdbtable` files are never mutated in place. +- `.csdbx` files can be reopened and queried after process restart. +- `INSERT`, `UPDATE`, and `DELETE` persist to the external file. +- Unsupported DDL fails with clear errors. +- Admin can convert and register writable external tables without blocking the UI. diff --git a/src/CSharpDB.Admin.Forms/CSharpDB.Admin.Forms.csproj b/src/CSharpDB.Admin.Forms/CSharpDB.Admin.Forms.csproj index 6b923cd2..c9e88ff8 100644 --- a/src/CSharpDB.Admin.Forms/CSharpDB.Admin.Forms.csproj +++ b/src/CSharpDB.Admin.Forms/CSharpDB.Admin.Forms.csproj @@ -9,6 +9,7 @@ + diff --git a/src/CSharpDB.Admin.Forms/Components/Designer/ControlEventBindingsEditor.razor b/src/CSharpDB.Admin.Forms/Components/Designer/ControlEventBindingsEditor.razor index 2f74b955..0638bf42 100644 --- a/src/CSharpDB.Admin.Forms/Components/Designer/ControlEventBindingsEditor.razor +++ b/src/CSharpDB.Admin.Forms/Components/Designer/ControlEventBindingsEditor.razor @@ -1,8 +1,10 @@ @using System.Text.Json @using CSharpDB.Admin.Forms.Models @using CSharpDB.Admin.Forms.Serialization +@using CSharpDB.CodeModules @using CSharpDB.Primitives @inject DbCommandRegistry CommandRegistry +@inject IFormCodeModuleDesignerService CodeModuleDesigner
@if (EventBindings.Count == 0) @@ -69,6 +71,28 @@ @onchange="@(e => UpdateArguments(idx, binding, e.Value?.ToString() ?? string.Empty))">
+
+ +
+ + +
+ + +
+
@_argumentError
} + @if (!string.IsNullOrWhiteSpace(_codeHandlerError)) + { +
@_codeHandlerError
+ } @@ -89,12 +117,15 @@ @code { [Parameter, EditorRequired] public IReadOnlyList EventBindings { get; set; } = []; [Parameter] public IReadOnlyList ActionSequences { get; set; } = []; + [Parameter] public ControlDefinition? Control { get; set; } [Parameter] public EventCallback> EventBindingsChanged { get; set; } private readonly Dictionary _argumentText = []; private string? _argumentError; + private string? _codeHandlerError; private static readonly ControlEventKind[] EventKinds = Enum.GetValues(); + [CascadingParameter] public DesignerState? State { get; set; } private IReadOnlyList RegisteredCommands => CommandRegistry.Commands.ToList(); @@ -168,6 +199,51 @@ private Task UpdateActionSequence(int index, ControlEventBinding binding, DbActionSequence? actionSequence) => ReplaceBinding(index, binding with { ActionSequence = actionSequence }); + private Task UpdateCodeHandler( + int index, + ControlEventBinding binding, + string? moduleId = null, + string? typeName = null, + string? methodName = null) + { + string resolvedModuleId = moduleId ?? binding.CodeHandler?.ModuleId ?? string.Empty; + string resolvedTypeName = typeName ?? binding.CodeHandler?.TypeName ?? string.Empty; + string resolvedMethodName = methodName ?? binding.CodeHandler?.MethodName ?? string.Empty; + CodeModuleHandler? handler = string.IsNullOrWhiteSpace(resolvedModuleId) && + string.IsNullOrWhiteSpace(resolvedTypeName) && + string.IsNullOrWhiteSpace(resolvedMethodName) + ? null + : new CodeModuleHandler(resolvedModuleId.Trim(), resolvedTypeName.Trim(), resolvedMethodName.Trim()); + return ReplaceBinding(index, binding with { CodeHandler = handler }); + } + + private async Task CreateCodeHandlerAsync(int index, ControlEventBinding binding) + { + _codeHandlerError = null; + if (State is null || Control is null) + { + _codeHandlerError = "Form designer state is unavailable."; + return; + } + + try + { + CodeModuleHandler handler = await CodeModuleDesigner.CreateHandlerAsync(new FormCodeModuleHandlerRequest( + State.FormId, + State.FormName, + State.TableName, + binding.Event.ToString(), + IsCancelable: false, + Control.ControlId, + Control.ControlType)); + await ReplaceBinding(index, binding with { CodeHandler = handler, CommandName = string.Empty }); + } + catch (Exception ex) + { + _codeHandlerError = ex.Message; + } + } + private async Task ReplaceBinding(int index, ControlEventBinding binding) { var updated = EventBindings.ToList(); @@ -199,6 +275,12 @@ => !string.IsNullOrWhiteSpace(commandName) && RegisteredCommands.All(command => !string.Equals(command.Name, commandName, StringComparison.OrdinalIgnoreCase)); + private static string GetCodeModuleId(ControlEventBinding binding) => binding.CodeHandler?.ModuleId ?? string.Empty; + + private static string GetCodeTypeName(ControlEventBinding binding) => binding.CodeHandler?.TypeName ?? string.Empty; + + private static string GetCodeMethodName(ControlEventBinding binding) => binding.CodeHandler?.MethodName ?? string.Empty; + private static string FormatArguments(IReadOnlyDictionary? arguments) => arguments is null || arguments.Count == 0 ? string.Empty diff --git a/src/CSharpDB.Admin.Forms/Components/Designer/FormEventBindingsEditor.razor b/src/CSharpDB.Admin.Forms/Components/Designer/FormEventBindingsEditor.razor index c1f14137..fc6fa01d 100644 --- a/src/CSharpDB.Admin.Forms/Components/Designer/FormEventBindingsEditor.razor +++ b/src/CSharpDB.Admin.Forms/Components/Designer/FormEventBindingsEditor.razor @@ -1,7 +1,9 @@ @using System.Text.Json @using CSharpDB.Admin.Forms.Serialization +@using CSharpDB.CodeModules @using CSharpDB.Primitives @inject DbCommandRegistry CommandRegistry +@inject IFormCodeModuleDesignerService CodeModuleDesigner
@if (EventBindings.Count == 0) @@ -68,6 +70,28 @@ @onchange="@(e => UpdateArguments(idx, binding, e.Value?.ToString() ?? string.Empty))">
+
+ +
+ + +
+ + +
+
@_argumentError
} + @if (!string.IsNullOrWhiteSpace(_codeHandlerError)) + { +
@_codeHandlerError
+ } @@ -92,8 +120,10 @@ private readonly Dictionary _argumentText = []; private string? _argumentError; + private string? _codeHandlerError; private static readonly FormEventKind[] EventKinds = Enum.GetValues(); + [CascadingParameter] public DesignerState? State { get; set; } private IReadOnlyList RegisteredCommands => CommandRegistry.Commands.ToList(); @@ -167,6 +197,49 @@ private Task UpdateActionSequence(int index, FormEventBinding binding, DbActionSequence? actionSequence) => ReplaceBinding(index, binding with { ActionSequence = actionSequence }); + private Task UpdateCodeHandler( + int index, + FormEventBinding binding, + string? moduleId = null, + string? typeName = null, + string? methodName = null) + { + string resolvedModuleId = moduleId ?? binding.CodeHandler?.ModuleId ?? string.Empty; + string resolvedTypeName = typeName ?? binding.CodeHandler?.TypeName ?? string.Empty; + string resolvedMethodName = methodName ?? binding.CodeHandler?.MethodName ?? string.Empty; + CodeModuleHandler? handler = string.IsNullOrWhiteSpace(resolvedModuleId) && + string.IsNullOrWhiteSpace(resolvedTypeName) && + string.IsNullOrWhiteSpace(resolvedMethodName) + ? null + : new CodeModuleHandler(resolvedModuleId.Trim(), resolvedTypeName.Trim(), resolvedMethodName.Trim()); + return ReplaceBinding(index, binding with { CodeHandler = handler }); + } + + private async Task CreateCodeHandlerAsync(int index, FormEventBinding binding) + { + _codeHandlerError = null; + if (State is null) + { + _codeHandlerError = "Form designer state is unavailable."; + return; + } + + try + { + CodeModuleHandler handler = await CodeModuleDesigner.CreateHandlerAsync(new FormCodeModuleHandlerRequest( + State.FormId, + State.FormName, + State.TableName, + binding.Event.ToString(), + IsCancelable(binding.Event))); + await ReplaceBinding(index, binding with { CodeHandler = handler, CommandName = string.Empty }); + } + catch (Exception ex) + { + _codeHandlerError = ex.Message; + } + } + private async Task ReplaceBinding(int index, FormEventBinding binding) { var updated = EventBindings.ToList(); @@ -198,6 +271,15 @@ => !string.IsNullOrWhiteSpace(commandName) && RegisteredCommands.All(command => !string.Equals(command.Name, commandName, StringComparison.OrdinalIgnoreCase)); + private static string GetCodeModuleId(FormEventBinding binding) => binding.CodeHandler?.ModuleId ?? string.Empty; + + private static string GetCodeTypeName(FormEventBinding binding) => binding.CodeHandler?.TypeName ?? string.Empty; + + private static string GetCodeMethodName(FormEventBinding binding) => binding.CodeHandler?.MethodName ?? string.Empty; + + private static bool IsCancelable(FormEventKind eventKind) + => eventKind is FormEventKind.BeforeInsert or FormEventKind.BeforeUpdate or FormEventKind.BeforeDelete; + private static string FormatArguments(IReadOnlyDictionary? arguments) => arguments is null || arguments.Count == 0 ? string.Empty diff --git a/src/CSharpDB.Admin.Forms/Components/Designer/FormRenderer.razor b/src/CSharpDB.Admin.Forms/Components/Designer/FormRenderer.razor index 64c761da..5c99f372 100644 --- a/src/CSharpDB.Admin.Forms/Components/Designer/FormRenderer.razor +++ b/src/CSharpDB.Admin.Forms/Components/Designer/FormRenderer.razor @@ -4,9 +4,11 @@ @using CSharpDB.Admin.Forms.Models @using CSharpDB.Admin.Forms.Pages @using CSharpDB.Admin.Forms.Services +@using CSharpDB.CodeModules @using CSharpDB.Primitives @inject DbCommandRegistry Commands @inject DbExtensionPolicy? CallbackPolicy +@inject ICodeModuleFormEventDispatcher CodeModules
@foreach (var control in GetControlsToRender()) @@ -1380,16 +1382,60 @@ return; } - if (binding.ActionSequence is null) + if (binding.CodeHandler is null && binding.ActionSequence is null) continue; } } - else if (binding.ActionSequence is null) + else if (binding.CodeHandler is null && binding.ActionSequence is null) { - await ReportCommandErrorAsync($"Control event '{eventKind}' has no command or action sequence."); + await ReportCommandErrorAsync($"Control event '{eventKind}' has no command, code handler, or action sequence."); return; } + if (binding.CodeHandler is not null) + { + var commandApi = new AdminFormCodeModuleCommandApi( + Form, + Commands, + EffectiveCallbackPolicy, + Record, + binding.Arguments, + runtimeArguments, + metadata, + Form.ActionSequences, + ActionRuntime ?? NullFormActionRuntime.Instance, + SetActionFieldValueAsync, + ReportCommandErrorAsync, + OnBuiltInAction); + CodeModuleFormDispatchResult codeResult = await CodeModules.DispatchAsync( + binding.CodeHandler, + new CodeModuleFormEventDispatchContext( + Form.FormId, + Form.Name, + Form.TableName, + eventKind.ToString(), + Record, + binding.Arguments, + runtimeArguments, + metadata, + commandApi, + IsCancelable: false, + control.ControlId, + control.ControlType)); + + if (!codeResult.Succeeded) + { + if (binding.StopOnFailure) + { + await ReportCommandErrorAsync(codeResult.Message ?? $"Control event '{eventKind}' code handler failed."); + return; + } + + if (binding.ActionSequence is null) + continue; + } + } + if (binding.ActionSequence is not null) { FormEventDispatchResult actionResult = await FormActionSequenceExecutor.ExecuteAsync( diff --git a/src/CSharpDB.Admin.Forms/Components/Designer/PropertyInspector.razor b/src/CSharpDB.Admin.Forms/Components/Designer/PropertyInspector.razor index ee78c33d..44228c13 100644 --- a/src/CSharpDB.Admin.Forms/Components/Designer/PropertyInspector.razor +++ b/src/CSharpDB.Admin.Forms/Components/Designer/PropertyInspector.razor @@ -92,6 +92,7 @@
Events
diff --git a/src/CSharpDB.Admin.Forms/Models/ControlEventBinding.cs b/src/CSharpDB.Admin.Forms/Models/ControlEventBinding.cs index c4f21be1..1fe64bd5 100644 --- a/src/CSharpDB.Admin.Forms/Models/ControlEventBinding.cs +++ b/src/CSharpDB.Admin.Forms/Models/ControlEventBinding.cs @@ -1,3 +1,4 @@ +using CSharpDB.CodeModules; using CSharpDB.Primitives; namespace CSharpDB.Admin.Forms.Models; @@ -15,4 +16,5 @@ public sealed record ControlEventBinding( string CommandName, IReadOnlyDictionary? Arguments = null, bool StopOnFailure = true, - DbActionSequence? ActionSequence = null); + DbActionSequence? ActionSequence = null, + CodeModuleHandler? CodeHandler = null); diff --git a/src/CSharpDB.Admin.Forms/Models/FormEventBinding.cs b/src/CSharpDB.Admin.Forms/Models/FormEventBinding.cs index 754b031c..bc3d63a8 100644 --- a/src/CSharpDB.Admin.Forms/Models/FormEventBinding.cs +++ b/src/CSharpDB.Admin.Forms/Models/FormEventBinding.cs @@ -1,3 +1,4 @@ +using CSharpDB.CodeModules; using CSharpDB.Primitives; namespace CSharpDB.Admin.Forms.Models; @@ -19,4 +20,5 @@ public sealed record FormEventBinding( string CommandName, IReadOnlyDictionary? Arguments = null, bool StopOnFailure = true, - DbActionSequence? ActionSequence = null); + DbActionSequence? ActionSequence = null, + CodeModuleHandler? CodeHandler = null); diff --git a/src/CSharpDB.Admin.Forms/README.md b/src/CSharpDB.Admin.Forms/README.md index 79126d1c..939cb819 100644 --- a/src/CSharpDB.Admin.Forms/README.md +++ b/src/CSharpDB.Admin.Forms/README.md @@ -18,6 +18,9 @@ This project is consumed by `CSharpDB.Admin`. It is not a standalone web host. - trusted command-backed form events and command buttons - trusted command-backed selected-control events - declarative action sequences for form and selected-control events +- Access-style rendered-form action steps for record navigation, save/delete, + open/close form, filtering, SQL/procedure, and control property workflows +- conditional action steps and conditional UI rules - generated automation metadata for import/export host callback requirements ## Main Components @@ -112,21 +115,30 @@ properties, optional validation overrides, optional renderer hints, and optional `ControlEventBinding` entries for selected control events such as `OnClick`, `OnChange`, `OnGotFocus`, and `OnLostFocus`. -Form and control event bindings can reference a trusted command name and can -optionally include a `DbActionSequence`. Forms can also store reusable named -action sequences in `ActionSequences`, and event/button sequences can invoke -them with `RunActionSequence`. Action sequences store declarative steps such as +Form and control event bindings can reference a trusted command name, a +`CodeModuleHandler`, and optionally include a `DbActionSequence`. Dispatch order +is trusted command, C# code-module handler, then declarative action sequence. +Forms can also store reusable named action sequences in `ActionSequences`, and +event/button sequences can invoke them with `RunActionSequence`. Action +sequences store declarative steps such as `RunCommand`, `RunActionSequence`, `SetFieldValue`, `ShowMessage`, `Stop`, `NewRecord`, `SaveRecord`, `DeleteRecord`, `RefreshRecords`, `PreviousRecord`, -`NextRecord`, and `GoToRecord`; they do not store C# source or serialized -delegates. The property inspector exposes a visual action-sequence editor on -form-level and selected-control event bindings plus a reusable action library -when editing form properties. JSON editing is limited to optional command or -nested-sequence argument payloads. - -The built-in record actions run only in the rendered Forms data-entry runtime. -Headless form event dispatch can still run command, field, message, and stop -steps, but navigation and save/delete actions require a rendered form instance. +`NextRecord`, `GoToRecord`, `OpenForm`, `CloseForm`, `ApplyFilter`, +`ClearFilter`, `RunSql`, `RunProcedure`, `SetControlProperty`, +`SetControlVisibility`, `SetControlEnabled`, and `SetControlReadOnly`; they do +not store serialized delegates. Database-owned C# source is stored separately +through `CSharpDB.CodeModules` and requires host opt-in, successful build, and +explicit local trust before execution. The property inspector exposes a +visual action-sequence editor on form-level and selected-control event bindings +plus a reusable action library when editing form properties. JSON editing is +limited to optional command or nested-sequence argument payloads. + +The built-in record, form navigation, filter, SQL/procedure, and control +property actions run only in the rendered Forms data-entry runtime. Headless +form event dispatch can still run command, field, message, stop, and other +runtime-independent steps, but actions that need a loaded rendered form instance +report a failure outside that surface. SQL and procedure actions also require +the rendered host to enable those capabilities explicitly. Every action step can also store a simple condition such as `Status = 'Ready'`, `Amount > 0`, or `IsActive`. False conditions skip that step; malformed diff --git a/src/CSharpDB.Admin.Forms/Services/AdminFormCodeModuleCommandApi.cs b/src/CSharpDB.Admin.Forms/Services/AdminFormCodeModuleCommandApi.cs new file mode 100644 index 00000000..28eb98fc --- /dev/null +++ b/src/CSharpDB.Admin.Forms/Services/AdminFormCodeModuleCommandApi.cs @@ -0,0 +1,190 @@ +using CSharpDB.Admin.Forms.Contracts; +using CSharpDB.Admin.Forms.Models; +using CSharpDB.CodeModules.Runtime; +using CSharpDB.Primitives; + +namespace CSharpDB.Admin.Forms.Services; + +internal sealed class AdminFormCodeModuleCommandApi( + FormDefinition form, + DbCommandRegistry commands, + DbExtensionPolicy callbackPolicy, + IReadOnlyDictionary? record, + IReadOnlyDictionary? bindingArguments, + IReadOnlyDictionary? runtimeArguments, + IReadOnlyDictionary metadata, + IReadOnlyList? reusableSequences, + IFormActionRuntime actionRuntime, + Func? setFieldValue = null, + Func? showMessage = null, + Func>? executeBuiltInFormAction = null) : IFormCommandApi +{ + public async ValueTask SetFieldAsync(string fieldName, object? value, CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(fieldName); + ct.ThrowIfCancellationRequested(); + + if (setFieldValue is not null) + { + await setFieldValue(fieldName, value); + return; + } + + if (record is not IDictionary mutable) + throw new InvalidOperationException($"The current form record is read-only and field '{fieldName}' cannot be changed."); + + string key = mutable.Keys.FirstOrDefault(candidate => string.Equals(candidate, fieldName, StringComparison.OrdinalIgnoreCase)) + ?? throw new InvalidOperationException($"Unknown form field '{fieldName}'."); + mutable[key] = value; + } + + public async ValueTask ShowMessageAsync(string message, CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(message); + ct.ThrowIfCancellationRequested(); + if (showMessage is not null) + await showMessage(message); + + return FormCommandApiResult.Success(message); + } + + public async ValueTask RunActionSequenceAsync( + string sequenceName, + IReadOnlyDictionary? arguments = null, + CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sequenceName); + FormEventDispatchResult result = await FormActionSequenceExecutor.ExecuteAsync( + new DbActionSequence( + [new DbActionStep(DbActionKind.RunActionSequence, SequenceName: sequenceName, Arguments: arguments)], + Name: "__CodeModule"), + commands, + record, + bindingArguments, + runtimeArguments, + metadata, + reusableSequences, + setFieldValue, + showMessage, + executeBuiltInFormAction, + actionRuntime, + callbackPolicy, + ct); + + return ToApiResult(result); + } + + public async ValueTask RunHostCommandAsync( + string commandName, + IReadOnlyDictionary? arguments = null, + CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(commandName); + if (!commands.TryGetCommand(commandName, out DbCommandDefinition definition)) + return DbCommandResult.Failure($"Unknown form command '{commandName}'."); + + Dictionary commandArguments = DbCommandArguments.FromObjectDictionaries(record, runtimeArguments, bindingArguments, arguments); + return await definition.InvokeAsync( + commandArguments, + metadata, + callbackPolicy, + DbExtensionHostMode.Embedded, + ct); + } + + public ValueTask SaveRecordAsync(CancellationToken ct = default) + => ExecuteRecordActionAsync(DbActionKind.SaveRecord, ct); + + public ValueTask NewRecordAsync(CancellationToken ct = default) + => ExecuteRecordActionAsync(DbActionKind.NewRecord, ct); + + public ValueTask RefreshRecordsAsync(CancellationToken ct = default) + => ExecuteRecordActionAsync(DbActionKind.RefreshRecords, ct); + + public async ValueTask OpenFormAsync( + string formName, + IReadOnlyDictionary? arguments = null, + CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(formName); + FormEventDispatchResult result = await actionRuntime.OpenFormAsync( + BuildRuntimeContext(null, arguments), + formName, + arguments ?? EmptyObjectDictionary.Instance, + ct); + return ToApiResult(result); + } + + public async ValueTask CloseFormAsync(string? formName = null, CancellationToken ct = default) + { + FormEventDispatchResult result = await actionRuntime.CloseFormAsync( + BuildRuntimeContext(null, null), + formName, + ct); + return ToApiResult(result); + } + + public async ValueTask ApplyFilterAsync( + string filter, + string target = "form", + IReadOnlyDictionary? arguments = null, + CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(filter); + string resolvedTarget = string.IsNullOrWhiteSpace(target) ? "form" : target; + FormEventDispatchResult result = await actionRuntime.ApplyFilterAsync( + BuildRuntimeContext(null, arguments), + resolvedTarget, + filter, + arguments ?? EmptyObjectDictionary.Instance, + ct); + return ToApiResult(result); + } + + public async ValueTask ClearFilterAsync(string target = "form", CancellationToken ct = default) + { + string resolvedTarget = string.IsNullOrWhiteSpace(target) ? "form" : target; + FormEventDispatchResult result = await actionRuntime.ClearFilterAsync( + BuildRuntimeContext(null, null), + resolvedTarget, + ct); + return ToApiResult(result); + } + + private async ValueTask ExecuteRecordActionAsync(DbActionKind kind, CancellationToken ct) + { + var step = new DbActionStep(kind); + FormEventDispatchResult result = executeBuiltInFormAction is not null && actionRuntime is NullFormActionRuntime + ? await executeBuiltInFormAction(step, ct) + : await actionRuntime.ExecuteRecordActionAsync(BuildRuntimeContext(step, null), step, ct); + + return ToApiResult(result); + } + + private FormActionRuntimeContext BuildRuntimeContext( + DbActionStep? step, + IReadOnlyDictionary? stepArguments) + => new( + form.FormId, + form.Name, + form.TableName, + metadata.TryGetValue("event", out string? eventName) ? eventName : null, + "__CodeModule", + 0, + record, + bindingArguments, + runtimeArguments, + stepArguments, + metadata); + + private static FormCommandApiResult ToApiResult(FormEventDispatchResult result) + => result.Succeeded + ? FormCommandApiResult.Success(result.Message) + : FormCommandApiResult.Failure(result.Message ?? "The form command failed."); + + private static class EmptyObjectDictionary + { + public static readonly IReadOnlyDictionary Instance = + new Dictionary(StringComparer.OrdinalIgnoreCase); + } +} diff --git a/src/CSharpDB.Admin.Forms/Services/AdminFormsServiceCollectionExtensions.cs b/src/CSharpDB.Admin.Forms/Services/AdminFormsServiceCollectionExtensions.cs index 4bea54cd..284a21af 100644 --- a/src/CSharpDB.Admin.Forms/Services/AdminFormsServiceCollectionExtensions.cs +++ b/src/CSharpDB.Admin.Forms/Services/AdminFormsServiceCollectionExtensions.cs @@ -1,4 +1,5 @@ using CSharpDB.Admin.Forms.Contracts; +using CSharpDB.CodeModules; using CSharpDB.Primitives; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; @@ -13,6 +14,8 @@ public static IServiceCollection AddCSharpDbAdminForms(this IServiceCollection s services.TryAddSingleton(DbValidationRuleRegistry.Empty); services.TryAddSingleton(DbExtensionPolicies.DefaultHostCallbackPolicy); services.TryAddSingleton(NullFormActionRuntime.Instance); + services.TryAddSingleton(NullCodeModuleFormEventDispatcher.Instance); + services.TryAddSingleton(NullFormCodeModuleDesignerService.Instance); services.TryAddFormControlRegistry(); services.AddScoped(); services.AddScoped(); @@ -33,6 +36,12 @@ public static IServiceCollection AddCSharpDbAdminForms( return services.AddCSharpDbAdminForms(); } + public static IServiceCollection AddCSharpDbAdminFormCodeModules(this IServiceCollection services) + { + services.AddScoped(); + return services; + } + public static IServiceCollection AddCSharpDbAdminFormValidationRules( this IServiceCollection services, Action configureRules) diff --git a/src/CSharpDB.Admin.Forms/Services/DefaultFormEventDispatcher.cs b/src/CSharpDB.Admin.Forms/Services/DefaultFormEventDispatcher.cs index 0db65f52..9e1f4740 100644 --- a/src/CSharpDB.Admin.Forms/Services/DefaultFormEventDispatcher.cs +++ b/src/CSharpDB.Admin.Forms/Services/DefaultFormEventDispatcher.cs @@ -1,5 +1,6 @@ using CSharpDB.Admin.Forms.Contracts; using CSharpDB.Admin.Forms.Models; +using CSharpDB.CodeModules; using CSharpDB.Primitives; namespace CSharpDB.Admin.Forms.Services; @@ -9,14 +10,15 @@ public sealed class DefaultFormEventDispatcher : IFormEventDispatcher private readonly DbCommandRegistry _commands; private readonly DbExtensionPolicy _callbackPolicy; private readonly IFormActionRuntime _actionRuntime; + private readonly ICodeModuleFormEventDispatcher _codeModules; public DefaultFormEventDispatcher(DbCommandRegistry commands) - : this(commands, DbExtensionPolicies.DefaultHostCallbackPolicy, NullFormActionRuntime.Instance) + : this(commands, DbExtensionPolicies.DefaultHostCallbackPolicy, NullFormActionRuntime.Instance, NullCodeModuleFormEventDispatcher.Instance) { } public DefaultFormEventDispatcher(DbCommandRegistry commands, IFormActionRuntime actionRuntime) - : this(commands, DbExtensionPolicies.DefaultHostCallbackPolicy, actionRuntime) + : this(commands, DbExtensionPolicies.DefaultHostCallbackPolicy, actionRuntime, NullCodeModuleFormEventDispatcher.Instance) { } @@ -24,14 +26,25 @@ public DefaultFormEventDispatcher( DbCommandRegistry commands, DbExtensionPolicy callbackPolicy, IFormActionRuntime actionRuntime) + : this(commands, callbackPolicy, actionRuntime, NullCodeModuleFormEventDispatcher.Instance) + { + } + + public DefaultFormEventDispatcher( + DbCommandRegistry commands, + DbExtensionPolicy callbackPolicy, + IFormActionRuntime actionRuntime, + ICodeModuleFormEventDispatcher codeModules) { ArgumentNullException.ThrowIfNull(commands); ArgumentNullException.ThrowIfNull(callbackPolicy); ArgumentNullException.ThrowIfNull(actionRuntime); + ArgumentNullException.ThrowIfNull(codeModules); _commands = commands; _callbackPolicy = callbackPolicy; _actionRuntime = actionRuntime; + _codeModules = codeModules; } public async Task DispatchAsync( @@ -100,13 +113,50 @@ public async Task DispatchAsync( if (binding.StopOnFailure) return FormEventDispatchResult.Failure(commandFailureMessage!); - if (binding.ActionSequence is null) + if (binding.CodeHandler is null && binding.ActionSequence is null) continue; } } - else if (binding.ActionSequence is null) + else if (binding.CodeHandler is null && binding.ActionSequence is null) { - return FormEventDispatchResult.Failure($"Form event '{eventKind}' has no command or action sequence."); + return FormEventDispatchResult.Failure($"Form event '{eventKind}' has no command, code handler, or action sequence."); + } + + if (binding.CodeHandler is not null) + { + var commandApi = new AdminFormCodeModuleCommandApi( + form, + _commands, + _callbackPolicy, + record, + binding.Arguments, + runtimeArguments: null, + metadata, + form.ActionSequences, + actionRuntime); + CodeModuleFormDispatchResult codeResult = await _codeModules.DispatchAsync( + binding.CodeHandler, + new CodeModuleFormEventDispatchContext( + form.FormId, + form.Name, + form.TableName, + eventKind.ToString(), + record, + binding.Arguments, + RuntimeArguments: null, + metadata, + commandApi, + IsCancelableEvent(eventKind)), + ct); + + if (!codeResult.Succeeded) + { + if (binding.StopOnFailure) + return FormEventDispatchResult.Failure(codeResult.Message ?? $"Form event '{eventKind}' code handler failed."); + + if (binding.ActionSequence is null) + continue; + } } if (binding.ActionSequence is not null) @@ -130,4 +180,7 @@ public async Task DispatchAsync( return FormEventDispatchResult.Success(); } + + private static bool IsCancelableEvent(FormEventKind eventKind) + => eventKind is FormEventKind.BeforeInsert or FormEventKind.BeforeUpdate or FormEventKind.BeforeDelete; } diff --git a/src/CSharpDB.Admin.Forms/Services/FormCodeModuleDesignerService.cs b/src/CSharpDB.Admin.Forms/Services/FormCodeModuleDesignerService.cs new file mode 100644 index 00000000..5f885120 --- /dev/null +++ b/src/CSharpDB.Admin.Forms/Services/FormCodeModuleDesignerService.cs @@ -0,0 +1,148 @@ +using System.Text; +using CSharpDB.Admin.Forms.Models; +using CSharpDB.CodeModules; +using CSharpDB.CodeModules.Runtime; + +namespace CSharpDB.Admin.Forms.Services; + +public interface IFormCodeModuleDesignerService +{ + Task CreateHandlerAsync( + FormCodeModuleHandlerRequest request, + CancellationToken ct = default); +} + +public sealed record FormCodeModuleHandlerRequest( + string FormId, + string FormName, + string TableName, + string EventName, + bool IsCancelable, + string? ControlId = null, + string? ControlType = null); + +public sealed class NullFormCodeModuleDesignerService : IFormCodeModuleDesignerService +{ + public static NullFormCodeModuleDesignerService Instance { get; } = new(); + + private NullFormCodeModuleDesignerService() + { + } + + public Task CreateHandlerAsync( + FormCodeModuleHandlerRequest request, + CancellationToken ct = default) + => Task.FromException( + new InvalidOperationException("C# code module designer support is not configured for this host.")); +} + +public sealed class CSharpDbFormCodeModuleDesignerService(CSharpDbCodeModuleClient codeModules) : IFormCodeModuleDesignerService +{ + private const string NamespaceName = "CSharpDB.UserCode.Forms"; + + public async Task CreateHandlerAsync( + FormCodeModuleHandlerRequest request, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(request); + if (string.IsNullOrWhiteSpace(request.FormId)) + throw new InvalidOperationException("Save the form before creating C# handlers."); + + string moduleId = $"form:{request.FormId}"; + string typeName = $"{NamespaceName}.{ToIdentifier(request.FormName, "Form")}Module"; + string methodName = string.IsNullOrWhiteSpace(request.ControlId) + ? $"On{ToIdentifier(request.EventName, "Event")}" + : $"{ToIdentifier(request.ControlId, "Control")}_{ToIdentifier(request.EventName, "Event")}"; + string contextType = string.IsNullOrWhiteSpace(request.ControlId) + ? request.IsCancelable ? nameof(FormBeforeEventContext) : nameof(FormEventContext) + : nameof(FormControlEventContext); + + CodeModuleDefinition? existing = await codeModules.GetAsync(moduleId, ct); + string source = existing is null + ? CreateFormModuleSource(typeName, methodName, contextType) + : EnsureMethodStub(existing.Source, methodName, contextType); + + await codeModules.UpsertAsync(new CodeModuleDefinition( + moduleId, + string.IsNullOrWhiteSpace(request.FormName) ? moduleId : request.FormName, + CodeModuleKind.Form, + source, + OwnerKind: "Form", + OwnerId: request.FormId, + TypeName: typeName, + Metadata: new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["tableName"] = request.TableName, + }), ct); + + return new CodeModuleHandler(moduleId, typeName, methodName); + } + + private static string CreateFormModuleSource(string typeName, string methodName, string contextType) + { + string shortTypeName = typeName.Split('.').Last(); + return $$""" + using CSharpDB.CodeModules.Runtime; + + namespace {{NamespaceName}}; + + public sealed class {{shortTypeName}} : FormCodeModule + { + public void {{methodName}}({{contextType}} context) + { + } + } + """; + } + + private static string EnsureMethodStub(string source, string methodName, string contextType) + { + if (source.Contains($" {methodName}(", StringComparison.Ordinal) || + source.Contains($"\t{methodName}(", StringComparison.Ordinal)) + { + return source; + } + + int insertAt = source.LastIndexOf('}'); + if (insertAt < 0) + return source + Environment.NewLine + CreateDetachedMethod(methodName, contextType); + + var builder = new StringBuilder(source.Length + 160); + builder.Append(source.AsSpan(0, insertAt)); + if (!source[..insertAt].EndsWith(Environment.NewLine, StringComparison.Ordinal)) + builder.AppendLine(); + + builder.AppendLine(); + builder.AppendLine($" public void {methodName}({contextType} context)"); + builder.AppendLine(" {"); + builder.AppendLine(" }"); + builder.Append(source.AsSpan(insertAt)); + return builder.ToString(); + } + + private static string CreateDetachedMethod(string methodName, string contextType) + => $$""" + + public void {{methodName}}({{contextType}} context) + { + } + """; + + private static string ToIdentifier(string? value, string fallback) + { + string text = string.IsNullOrWhiteSpace(value) ? fallback : value; + var builder = new StringBuilder(text.Length + 1); + foreach (char ch in text) + { + if (char.IsLetterOrDigit(ch) || ch == '_') + builder.Append(ch); + } + + if (builder.Length == 0) + builder.Append(fallback); + if (!char.IsLetter(builder[0]) && builder[0] != '_') + builder.Insert(0, '_'); + + return builder.ToString(); + } +} diff --git a/src/CSharpDB.Admin.ImportExport/CSharpDB.Admin.ImportExport.csproj b/src/CSharpDB.Admin.ImportExport/CSharpDB.Admin.ImportExport.csproj new file mode 100644 index 00000000..ed6ba885 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/CSharpDB.Admin.ImportExport.csproj @@ -0,0 +1,17 @@ + + + + net10.0 + enable + enable + CSharpDB.Admin.ImportExport + + + + + + + + + + diff --git a/src/CSharpDB.Admin.ImportExport/Components/ImportExportProgress.razor b/src/CSharpDB.Admin.ImportExport/Components/ImportExportProgress.razor new file mode 100644 index 00000000..4180eb5c --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Components/ImportExportProgress.razor @@ -0,0 +1,78 @@ +@namespace CSharpDB.Admin.ImportExport.Components + +@if (Progress is not null) +{ +
+
+
+
+ + @Progress.Operation +
+ @Progress.Stage +
+ @if (CanCancel) + { + + } +
+ +
+
+
+ +
+ @(Progress.Message ?? Progress.Stage) + @FormatRows() + @if (StartedAt is not null) + { + @FormatElapsed() + } +
+ + @if (!string.IsNullOrWhiteSpace(Progress.Path)) + { +
@Progress.Path
+ } +
+} + +@code { + [Parameter] public TableExportProgress? Progress { get; set; } + [Parameter] public DateTimeOffset? StartedAt { get; set; } + [Parameter] public bool CanCancel { get; set; } + [Parameter] public EventCallback OnCancel { get; set; } + + private string GetFillClass() => + Progress?.PercentComplete is int ? "import-export-progress-fill" : "import-export-progress-fill indeterminate"; + + private string GetFillStyle() => + Progress?.PercentComplete is int percent ? $"width: {percent}%;" : string.Empty; + + private string FormatRows() + { + if (Progress is null) + return string.Empty; + + string processed = Progress.RowsProcessed.ToString("N0", System.Globalization.CultureInfo.InvariantCulture); + return Progress.TotalRows is long total + ? $"{processed} / {total.ToString("N0", System.Globalization.CultureInfo.InvariantCulture)} rows" + : $"{processed} rows"; + } + + private string FormatElapsed() + { + if (StartedAt is null) + return string.Empty; + + TimeSpan elapsed = DateTimeOffset.UtcNow - StartedAt.Value; + return elapsed.TotalMinutes >= 1 + ? $"{(int)elapsed.TotalMinutes}m {elapsed.Seconds}s" + : $"{Math.Max(0, (int)elapsed.TotalSeconds)}s"; + } +} diff --git a/src/CSharpDB.Admin.ImportExport/Components/ImportExportStatus.razor b/src/CSharpDB.Admin.ImportExport/Components/ImportExportStatus.razor new file mode 100644 index 00000000..1193e02f --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Components/ImportExportStatus.razor @@ -0,0 +1,26 @@ +@namespace CSharpDB.Admin.ImportExport.Components + +@if (!string.IsNullOrWhiteSpace(Error)) +{ +
+ + @Error +
+} +@if (!string.IsNullOrWhiteSpace(Message)) +{ +
+ + @Message + @if (!string.IsNullOrWhiteSpace(DownloadUrl)) + { + Download + } +
+} + +@code { + [Parameter] public string? Message { get; set; } + [Parameter] public string? Error { get; set; } + [Parameter] public string? DownloadUrl { get; set; } +} diff --git a/src/CSharpDB.Admin.ImportExport/Contracts/ExternalTableRegistrationInfo.cs b/src/CSharpDB.Admin.ImportExport/Contracts/ExternalTableRegistrationInfo.cs new file mode 100644 index 00000000..476390fd --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Contracts/ExternalTableRegistrationInfo.cs @@ -0,0 +1,10 @@ +namespace CSharpDB.Admin.ImportExport.Contracts; + +public sealed class ExternalTableRegistrationInfo +{ + public required string TableName { get; init; } + public required string Path { get; init; } + public string? SourceTableName { get; init; } + public long RowCount { get; init; } + public DateTimeOffset? CreatedUtc { get; init; } +} diff --git a/src/CSharpDB.Admin.ImportExport/Contracts/ExternalTableRegistrationRequest.cs b/src/CSharpDB.Admin.ImportExport/Contracts/ExternalTableRegistrationRequest.cs new file mode 100644 index 00000000..a4dce1d0 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Contracts/ExternalTableRegistrationRequest.cs @@ -0,0 +1,8 @@ +namespace CSharpDB.Admin.ImportExport.Contracts; + +public sealed class ExternalTableRegistrationRequest +{ + public required string TableName { get; init; } + public required string ArchivePath { get; init; } + public bool ReplaceExisting { get; init; } = true; +} diff --git a/src/CSharpDB.Admin.ImportExport/Contracts/RestoreTableRequest.cs b/src/CSharpDB.Admin.ImportExport/Contracts/RestoreTableRequest.cs new file mode 100644 index 00000000..a213bd5d --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Contracts/RestoreTableRequest.cs @@ -0,0 +1,7 @@ +namespace CSharpDB.Admin.ImportExport.Contracts; + +public sealed class RestoreTableRequest +{ + public required string ArchivePath { get; init; } + public string? TargetTableName { get; init; } +} diff --git a/src/CSharpDB.Admin.ImportExport/Contracts/RestoreTableResult.cs b/src/CSharpDB.Admin.ImportExport/Contracts/RestoreTableResult.cs new file mode 100644 index 00000000..247aa978 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Contracts/RestoreTableResult.cs @@ -0,0 +1,7 @@ +namespace CSharpDB.Admin.ImportExport.Contracts; + +public sealed class RestoreTableResult +{ + public required string TableName { get; init; } + public long RowsInserted { get; init; } +} diff --git a/src/CSharpDB.Admin.ImportExport/Contracts/TableExportDestination.cs b/src/CSharpDB.Admin.ImportExport/Contracts/TableExportDestination.cs new file mode 100644 index 00000000..0e7bbba7 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Contracts/TableExportDestination.cs @@ -0,0 +1,7 @@ +namespace CSharpDB.Admin.ImportExport.Contracts; + +public enum TableExportDestination +{ + Download, + ServerPath, +} diff --git a/src/CSharpDB.Admin.ImportExport/Contracts/TableExportProgress.cs b/src/CSharpDB.Admin.ImportExport/Contracts/TableExportProgress.cs new file mode 100644 index 00000000..2739d93c --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Contracts/TableExportProgress.cs @@ -0,0 +1,17 @@ +namespace CSharpDB.Admin.ImportExport.Contracts; + +public sealed class TableExportProgress +{ + public required string Operation { get; init; } + public required string Stage { get; init; } + public string? Message { get; init; } + public string? TableName { get; init; } + public string? Path { get; init; } + public long RowsProcessed { get; init; } + public long? TotalRows { get; init; } + + public int? PercentComplete => + TotalRows is > 0 + ? (int)Math.Clamp(Math.Round((double)RowsProcessed / TotalRows.Value * 100), 0, 100) + : null; +} diff --git a/src/CSharpDB.Admin.ImportExport/Contracts/TableExportRequest.cs b/src/CSharpDB.Admin.ImportExport/Contracts/TableExportRequest.cs new file mode 100644 index 00000000..2257ef90 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Contracts/TableExportRequest.cs @@ -0,0 +1,8 @@ +namespace CSharpDB.Admin.ImportExport.Contracts; + +public sealed class TableExportRequest +{ + public required string TableName { get; init; } + public TableExportDestination Destination { get; init; } + public string? ServerPath { get; init; } +} diff --git a/src/CSharpDB.Admin.ImportExport/Contracts/TableExportResult.cs b/src/CSharpDB.Admin.ImportExport/Contracts/TableExportResult.cs new file mode 100644 index 00000000..31cf6dcf --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Contracts/TableExportResult.cs @@ -0,0 +1,11 @@ +namespace CSharpDB.Admin.ImportExport.Contracts; + +public sealed class TableExportResult +{ + public required string TableName { get; init; } + public required string FileName { get; init; } + public required string Path { get; init; } + public long RowCount { get; init; } + public string? DownloadUrl { get; init; } + public bool IsDownload { get; init; } +} diff --git a/src/CSharpDB.Admin.ImportExport/Models/TableArchiveDownload.cs b/src/CSharpDB.Admin.ImportExport/Models/TableArchiveDownload.cs new file mode 100644 index 00000000..4661af60 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Models/TableArchiveDownload.cs @@ -0,0 +1,9 @@ +namespace CSharpDB.Admin.ImportExport.Models; + +public sealed class TableArchiveDownload +{ + public required string Token { get; init; } + public required string Path { get; init; } + public required string FileName { get; init; } + public DateTimeOffset CreatedUtc { get; init; } = DateTimeOffset.UtcNow; +} diff --git a/src/CSharpDB.Admin.ImportExport/Pages/ImportExport.razor b/src/CSharpDB.Admin.ImportExport/Pages/ImportExport.razor new file mode 100644 index 00000000..ed7fdf67 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Pages/ImportExport.razor @@ -0,0 +1,461 @@ +@namespace CSharpDB.Admin.ImportExport.Pages +@inject ICSharpDbClient DbClient +@inject ITableImportExportService ImportExportService + +
+
+
+
+ tables + / + Import / Export +
+

Import / Export

+
+ +
+ +
+ + + +
+ + + + +
+ @switch (_mode) + { + case ImportExportMode.ExportTable: +
+ + +
+ Destination +
+ + +
+
+ + @if (_destination == TableExportDestination.ServerPath) + { + + } + +
+ +
+
+ break; + + case ImportExportMode.RegisterExternalTable: +
+ + +
+ +
+ + @if (_externalTables.Count > 0) + { +
+
+ Registered external tables + @_externalTables.Count +
+ @foreach (ExternalTableRegistrationInfo registration in _externalTables) + { +
+ + +
+ } +
+ } +
+ break; + + case ImportExportMode.RestoreTable: +
+ + +
+ +
+
+ break; + } +
+
+ +@code { + [Parameter] public string? InitialTableName { get; set; } + + private ImportExportMode _mode = ImportExportMode.ExportTable; + private TableExportDestination _destination = TableExportDestination.Download; + private IReadOnlyList _tables = Array.Empty(); + private IReadOnlyList _externalTables = Array.Empty(); + private string _selectedTable = string.Empty; + private string _serverPath = string.Empty; + private string _externalTableName = string.Empty; + private string _externalArchivePath = string.Empty; + private string _restoreArchivePath = string.Empty; + private string _restoreTableName = string.Empty; + private string? _message; + private string? _error; + private string? _downloadUrl; + private TableExportProgress? _progress; + private DateTimeOffset? _operationStartedAt; + private CancellationTokenSource? _operationCts; + private bool _busy; + + private bool CanExport => !_busy && !string.IsNullOrWhiteSpace(_selectedTable) && + (_destination == TableExportDestination.Download || !string.IsNullOrWhiteSpace(_serverPath)); + private bool CanRegisterExternal => !_busy && !string.IsNullOrWhiteSpace(_externalTableName) && !string.IsNullOrWhiteSpace(_externalArchivePath); + private bool CanRestore => !_busy && !string.IsNullOrWhiteSpace(_restoreArchivePath); + + protected override async Task OnInitializedAsync() + { + await RefreshAsync(); + } + + protected override async Task OnParametersSetAsync() + { + if (!string.IsNullOrWhiteSpace(InitialTableName) && + !string.Equals(_selectedTable, InitialTableName, StringComparison.OrdinalIgnoreCase)) + { + _selectedTable = InitialTableName; + _externalTableName = $"archived_{InitialTableName}"; + await RefreshDefaultServerPathAsync(); + } + } + + private async Task RefreshAsync() + { + await RunAsync(async ct => + { + _tables = (await DbClient.GetTableNamesAsync(ct)) + .Where(static name => !name.StartsWith("_", StringComparison.Ordinal)) + .OrderBy(static name => name, StringComparer.OrdinalIgnoreCase) + .ToArray(); + _externalTables = await ImportExportService.GetExternalTablesAsync(ct); + if (string.IsNullOrWhiteSpace(_selectedTable) || !_tables.Contains(_selectedTable, StringComparer.OrdinalIgnoreCase)) + _selectedTable = _tables.FirstOrDefault() ?? string.Empty; + + if (string.IsNullOrWhiteSpace(_externalTableName) && !string.IsNullOrWhiteSpace(_selectedTable)) + _externalTableName = $"archived_{_selectedTable}"; + + await RefreshDefaultServerPathAsync(ct); + }, clearMessage: false); + } + + private async Task OnSelectedTableChangedAsync() + { + _externalTableName = string.IsNullOrWhiteSpace(_selectedTable) ? string.Empty : $"archived_{_selectedTable}"; + await RefreshDefaultServerPathAsync(); + } + + private async Task RefreshDefaultServerPathAsync(CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(_selectedTable)) + return; + + _serverPath = await ImportExportService.GetDefaultServerExportPathAsync(_selectedTable, ct); + } + + private async Task ExportAsync() + { + await RunAsync(async ct => + { + SetProgress(new TableExportProgress + { + Operation = "Export table", + Stage = "Preparing", + Message = _destination == TableExportDestination.Download + ? "Preparing download export" + : "Preparing server-path export", + TableName = _selectedTable, + Path = _destination == TableExportDestination.ServerPath ? _serverPath : null, + }); + await InvokeAsync(StateHasChanged); + await Task.Yield(); + + var progress = CreateProgressReporter(ct); + var request = new TableExportRequest + { + TableName = _selectedTable, + Destination = _destination, + ServerPath = _destination == TableExportDestination.ServerPath ? _serverPath : null, + }; + var result = await Task.Run( + () => ImportExportService.ExportTableAsync(request, progress, ct), + ct); + + _downloadUrl = result.DownloadUrl; + _externalArchivePath = result.Path; + _restoreArchivePath = result.Path; + _message = result.IsDownload + ? $"Exported {result.RowCount} row(s) to {result.FileName}." + : $"Exported {result.RowCount} row(s) to {result.Path}."; + }); + } + + private async Task RegisterExternalTableAsync() + { + await RunAsync(async ct => + { + SetProgress(new TableExportProgress + { + Operation = "Register external table", + Stage = "Queued", + Message = "Waiting to register external table", + TableName = _externalTableName, + Path = _externalArchivePath, + }); + await InvokeAsync(StateHasChanged); + await Task.Yield(); + + var progress = CreateProgressReporter(ct); + var request = new ExternalTableRegistrationRequest + { + TableName = _externalTableName, + ArchivePath = _externalArchivePath, + }; + await Task.Run( + () => ImportExportService.RegisterExternalTableAsync(request, progress, ct), + ct); + _externalTables = await ImportExportService.GetExternalTablesAsync(ct); + _message = $"Registered external table {_externalTableName}."; + }); + } + + private async Task DropRegisteredExternalTableAsync(string tableName) + { + await RunAsync(async ct => + { + SetProgress(new TableExportProgress + { + Operation = "Drop external table", + Stage = "Dropping", + Message = "Removing external table registration", + TableName = tableName, + }); + + await Task.Run( + () => ImportExportService.DropExternalTableAsync(tableName, ct), + ct); + _externalTables = await ImportExportService.GetExternalTablesAsync(ct); + _message = $"Dropped external table {tableName}."; + }); + } + + private async Task RestoreAsync() + { + await RunAsync(async ct => + { + SetProgress(new TableExportProgress + { + Operation = "Restore table", + Stage = "Restoring", + Message = "Creating target table and inserting archived rows", + TableName = string.IsNullOrWhiteSpace(_restoreTableName) ? null : _restoreTableName, + Path = _restoreArchivePath, + }); + var request = new RestoreTableRequest + { + ArchivePath = _restoreArchivePath, + TargetTableName = _restoreTableName, + }; + var result = await Task.Run( + () => ImportExportService.RestoreTableAsync(request, ct), + ct); + _message = $"Restored {result.RowsInserted} row(s) into {result.TableName}."; + _tables = await DbClient.GetTableNamesAsync(ct); + }); + } + + private async Task RunAsync(Func action, bool clearMessage = true) + { + if (_busy) + return; + + _operationCts?.Dispose(); + _operationCts = new CancellationTokenSource(); + CancellationToken token = _operationCts.Token; + _busy = true; + _operationStartedAt = DateTimeOffset.UtcNow; + _progress = null; + _error = null; + if (clearMessage) + { + _message = null; + _downloadUrl = null; + } + + await InvokeAsync(StateHasChanged); + await Task.Yield(); + + try + { + await action(token); + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + _message = "Operation canceled."; + } + catch (Exception ex) + { + _error = ex.Message; + } + finally + { + _busy = false; + _progress = null; + _operationStartedAt = null; + _operationCts?.Dispose(); + _operationCts = null; + await InvokeAsync(StateHasChanged); + } + } + + private IProgress CreateProgressReporter(CancellationToken token) => + new Progress(progress => + { + _ = InvokeAsync(() => + { + if (!_busy || token.IsCancellationRequested) + return; + + _progress = progress; + StateHasChanged(); + }); + }); + + private void SetProgress(TableExportProgress progress) + { + if (!_busy) + return; + + _progress = progress; + StateHasChanged(); + } + + private void CancelOperation() + { + _operationCts?.Cancel(); + if (_progress is not null) + { + _progress = new TableExportProgress + { + Operation = _progress.Operation, + Stage = "Canceling", + Message = "Cancel requested", + TableName = _progress.TableName, + Path = _progress.Path, + RowsProcessed = _progress.RowsProcessed, + TotalRows = _progress.TotalRows, + }; + } + + _ = InvokeAsync(StateHasChanged); + } + + private void UseExternalTableRegistration(ExternalTableRegistrationInfo registration) + { + _externalTableName = registration.TableName; + _externalArchivePath = registration.Path; + } + + private void SetMode(ImportExportMode mode) + { + _mode = mode; + _error = null; + _message = null; + _downloadUrl = null; + } + + private void SetDestination(TableExportDestination destination) => _destination = destination; + + private string GetModeButtonClass(ImportExportMode mode) => _mode == mode ? "active" : string.Empty; + private string GetDestinationButtonClass(TableExportDestination destination) => _destination == destination ? "active" : string.Empty; + + private static string FormatExternalTableDetails(ExternalTableRegistrationInfo registration) + { + string source = string.IsNullOrWhiteSpace(registration.SourceTableName) + ? "external" + : registration.SourceTableName; + return $"{source} · {registration.RowCount:N0} rows"; + } + + private enum ImportExportMode + { + ExportTable, + RegisterExternalTable, + RestoreTable, + } +} diff --git a/src/CSharpDB.Admin.ImportExport/Services/ITableArchiveDownloadStore.cs b/src/CSharpDB.Admin.ImportExport/Services/ITableArchiveDownloadStore.cs new file mode 100644 index 00000000..3b8b45d9 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Services/ITableArchiveDownloadStore.cs @@ -0,0 +1,9 @@ +using CSharpDB.Admin.ImportExport.Models; + +namespace CSharpDB.Admin.ImportExport.Services; + +public interface ITableArchiveDownloadStore +{ + TableArchiveDownload Add(string path, string fileName); + bool TryTake(string token, out TableArchiveDownload download); +} diff --git a/src/CSharpDB.Admin.ImportExport/Services/ITableImportExportService.cs b/src/CSharpDB.Admin.ImportExport/Services/ITableImportExportService.cs new file mode 100644 index 00000000..0b880fe9 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Services/ITableImportExportService.cs @@ -0,0 +1,20 @@ +using CSharpDB.Admin.ImportExport.Contracts; + +namespace CSharpDB.Admin.ImportExport.Services; + +public interface ITableImportExportService +{ + Task GetDefaultServerExportPathAsync(string tableName, CancellationToken ct = default); + Task> GetExternalTablesAsync(CancellationToken ct = default); + Task ExportTableAsync( + TableExportRequest request, + IProgress? progress = null, + CancellationToken ct = default); + Task RegisterExternalTableAsync(ExternalTableRegistrationRequest request, CancellationToken ct = default); + Task RegisterExternalTableAsync( + ExternalTableRegistrationRequest request, + IProgress? progress, + CancellationToken ct = default); + Task DropExternalTableAsync(string tableName, CancellationToken ct = default); + Task RestoreTableAsync(RestoreTableRequest request, CancellationToken ct = default); +} diff --git a/src/CSharpDB.Admin.ImportExport/Services/ImportExportServiceCollectionExtensions.cs b/src/CSharpDB.Admin.ImportExport/Services/ImportExportServiceCollectionExtensions.cs new file mode 100644 index 00000000..49fe755b --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Services/ImportExportServiceCollectionExtensions.cs @@ -0,0 +1,42 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; +using Microsoft.Extensions.DependencyInjection; + +namespace CSharpDB.Admin.ImportExport.Services; + +public static class ImportExportServiceCollectionExtensions +{ + public static IServiceCollection AddCSharpDbAdminImportExport(this IServiceCollection services) + { + services.AddSingleton(); + services.AddScoped(); + return services; + } + + public static IEndpointRouteBuilder MapCSharpDbAdminImportExport(this IEndpointRouteBuilder endpoints) + { + endpoints.MapGet("/admin/import-export/download/{token}", ( + string token, + HttpContext httpContext, + ITableArchiveDownloadStore downloads) => + { + if (!downloads.TryTake(token, out var download) || !File.Exists(download.Path)) + return Results.NotFound(); + + httpContext.Response.OnCompleted(() => + { + TableArchiveDownloadStore.TryDelete(download.Path); + return Task.CompletedTask; + }); + + return Results.File( + download.Path, + "application/octet-stream", + download.FileName, + enableRangeProcessing: false); + }); + + return endpoints; + } +} diff --git a/src/CSharpDB.Admin.ImportExport/Services/TableArchiveDownloadStore.cs b/src/CSharpDB.Admin.ImportExport/Services/TableArchiveDownloadStore.cs new file mode 100644 index 00000000..be036dcb --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Services/TableArchiveDownloadStore.cs @@ -0,0 +1,58 @@ +using System.Collections.Concurrent; +using CSharpDB.Admin.ImportExport.Models; + +namespace CSharpDB.Admin.ImportExport.Services; + +public sealed class TableArchiveDownloadStore : ITableArchiveDownloadStore +{ + private readonly ConcurrentDictionary _downloads = new(StringComparer.Ordinal); + + public TableArchiveDownload Add(string path, string fileName) + { + CleanupExpiredDownloads(); + + var token = Convert.ToHexString(Guid.NewGuid().ToByteArray()).ToLowerInvariant(); + var download = new TableArchiveDownload + { + Token = token, + Path = path, + FileName = fileName, + }; + + _downloads[token] = download; + return download; + } + + public bool TryTake(string token, out TableArchiveDownload download) + { + bool found = _downloads.TryRemove(token, out var value); + download = value!; + return found; + } + + private void CleanupExpiredDownloads() + { + DateTimeOffset cutoff = DateTimeOffset.UtcNow.AddHours(-2); + foreach (var pair in _downloads) + { + if (pair.Value.CreatedUtc >= cutoff) + continue; + + if (_downloads.TryRemove(pair.Key, out var expired)) + TryDelete(expired.Path); + } + } + + internal static void TryDelete(string path) + { + try + { + if (File.Exists(path)) + File.Delete(path); + } + catch + { + // Best-effort cleanup for temporary download packages. + } + } +} diff --git a/src/CSharpDB.Admin.ImportExport/Services/TableImportExportService.cs b/src/CSharpDB.Admin.ImportExport/Services/TableImportExportService.cs new file mode 100644 index 00000000..0558fa22 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/Services/TableImportExportService.cs @@ -0,0 +1,606 @@ +using System.Collections; +using System.Diagnostics; +using System.Globalization; +using System.Text; +using CSharpDB.Admin.ImportExport.Contracts; +using CSharpDB.Client; +using CSharpDB.ImportExport.TableArchives; +using TableArchiveExportProgress = CSharpDB.Client.Models.TableArchiveExportProgress; +using ClientColumnDefinition = CSharpDB.Client.Models.ColumnDefinition; +using ClientDbType = CSharpDB.Client.Models.DbType; +using ClientForeignKeyDefinition = CSharpDB.Client.Models.ForeignKeyDefinition; +using ClientForeignKeyOnDeleteAction = CSharpDB.Client.Models.ForeignKeyOnDeleteAction; +using ClientTableSchema = CSharpDB.Client.Models.TableSchema; +using PrimitiveColumnDefinition = CSharpDB.Primitives.ColumnDefinition; +using PrimitiveDbType = CSharpDB.Primitives.DbType; +using PrimitiveDbValue = CSharpDB.Primitives.DbValue; +using PrimitiveForeignKeyDefinition = CSharpDB.Primitives.ForeignKeyDefinition; +using PrimitiveForeignKeyOnDeleteAction = CSharpDB.Primitives.ForeignKeyOnDeleteAction; +using PrimitiveTableSchema = CSharpDB.Primitives.TableSchema; + +namespace CSharpDB.Admin.ImportExport.Services; + +public sealed class TableImportExportService( + ICSharpDbClient client, + ITableArchiveDownloadStore downloads) : ITableImportExportService +{ + private const int ExportPageSize = 1_000; + private const int RestoreInsertBatchSize = 100; + + public Task GetDefaultServerExportPathAsync(string tableName, CancellationToken ct = default) + { + string databaseFolder = ResolveDatabaseFolder(client.DataSource); + string fileName = $"{SanitizeFileName(tableName)}-{DateTime.Now:yyyyMMdd-HHmmss}.csdbtable"; + return Task.FromResult(Path.Combine(databaseFolder, "exports", fileName)); + } + + public async Task> GetExternalTablesAsync(CancellationToken ct = default) + { + var result = await client.ExecuteSqlAsync( + """ + SELECT table_name, path, source_table_name, row_count, created_utc + FROM sys.external_tables + ORDER BY table_name; + """, + ct); + + if (!string.IsNullOrWhiteSpace(result.Error)) + throw new InvalidOperationException(result.Error); + + if (result.Rows is not { Count: > 0 }) + return Array.Empty(); + + return result.Rows + .Select(MapExternalTableRegistration) + .ToArray(); + } + + public async Task ExportTableAsync( + TableExportRequest request, + IProgress? progress = null, + CancellationToken ct = default) + { + string tableName = RequireIdentifier(request.TableName, nameof(request.TableName)); + string path = request.Destination == TableExportDestination.Download + ? CreateTemporaryArchivePath(tableName) + : string.IsNullOrWhiteSpace(request.ServerPath) + ? await GetDefaultServerExportPathAsync(tableName, ct) + : request.ServerPath; + + ReportExportProgress( + progress, + tableName, + "Preparing", + "Preparing export target", + rowsProcessed: 0, + totalRows: null, + path); + + long rowCount; + if (client is ICSharpDbTableArchiveProgressExporter progressExporter && progressExporter.SupportsTableArchiveExport) + { + var archiveProgress = progress is null + ? null + : new Progress(p => ReportExportProgress( + progress, + p.TableName, + p.Stage, + p.Message ?? "Writing table archive", + p.RowsExported, + p.TotalRows, + p.Path ?? path)); + var archiveExport = await progressExporter.ExportTableArchiveAsync(tableName, path, archiveProgress, ct); + rowCount = archiveExport.RowCount; + } + else if (client is ICSharpDbTableArchiveExporter exporter && exporter.SupportsTableArchiveExport) + { + ReportExportProgress( + progress, + tableName, + "Exporting", + "Writing table archive", + rowsProcessed: 0, + totalRows: null, + path); + var archiveExport = await exporter.ExportTableArchiveAsync(tableName, path, ct); + rowCount = archiveExport.RowCount; + } + else + { + ClientTableSchema clientSchema = await client.GetTableSchemaAsync(tableName, ct) + ?? throw new InvalidOperationException($"Table '{tableName}' was not found."); + PrimitiveTableSchema schema = MapSchema(clientSchema); + var manifest = await TableArchiveWriter.WriteAsync( + path, + schema, + EnumerateRowsAsync(clientSchema, path, progress, ct), + ct); + rowCount = manifest.RowCount; + } + + string fileName = Path.GetFileName(path); + string? downloadUrl = null; + + if (request.Destination == TableExportDestination.Download) + { + ReportExportProgress( + progress, + tableName, + "Preparing download", + "Preparing one-time download link", + rowCount, + rowCount, + path); + var download = downloads.Add(path, fileName); + downloadUrl = $"/admin/import-export/download/{download.Token}"; + } + + ReportExportProgress( + progress, + tableName, + "Complete", + "Export complete", + rowCount, + rowCount, + path); + + return new TableExportResult + { + TableName = tableName, + FileName = fileName, + Path = path, + RowCount = rowCount, + DownloadUrl = downloadUrl, + IsDownload = request.Destination == TableExportDestination.Download, + }; + } + + public Task RegisterExternalTableAsync(ExternalTableRegistrationRequest request, CancellationToken ct = default) => + RegisterExternalTableAsync(request, progress: null, ct); + + public async Task RegisterExternalTableAsync( + ExternalTableRegistrationRequest request, + IProgress? progress, + CancellationToken ct = default) + { + string tableName = RequireIdentifier(request.TableName, nameof(request.TableName)); + if (string.IsNullOrWhiteSpace(request.ArchivePath)) + throw new ArgumentException("Archive path is required.", nameof(request.ArchivePath)); + + ReportExportProgress( + progress, + tableName, + "Validating", + "Reading archive manifest", + rowsProcessed: 0, + totalRows: 3, + request.ArchivePath); + await TableArchiveReader.ReadManifestAsync(ResolveArchivePath(request.ArchivePath), ct); + + if (request.ReplaceExisting) + { + ReportExportProgress( + progress, + tableName, + "Replacing", + "Dropping existing registration if present", + rowsProcessed: 1, + totalRows: 3, + request.ArchivePath); + await DropExternalTableAsync(tableName, ct); + } + + ReportExportProgress( + progress, + tableName, + "Registering", + "Writing external table registration", + rowsProcessed: 2, + totalRows: 3, + request.ArchivePath); + string sql = $"CREATE EXTERNAL TABLE {tableName} FROM {FormatStringLiteral(request.ArchivePath)};"; + await ExecuteCheckedAsync(sql, ct); + + ReportExportProgress( + progress, + tableName, + "Complete", + "External table registered", + rowsProcessed: 3, + totalRows: 3, + request.ArchivePath); + } + + public async Task DropExternalTableAsync(string tableName, CancellationToken ct = default) + { + string normalizedTableName = RequireIdentifier(tableName, nameof(tableName)); + string sql = $"DROP EXTERNAL TABLE IF EXISTS {normalizedTableName};"; + await ExecuteCheckedAsync(sql, ct); + } + + public async Task RestoreTableAsync(RestoreTableRequest request, CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(request.ArchivePath)) + throw new ArgumentException("Archive path is required.", nameof(request.ArchivePath)); + + PrimitiveTableSchema archiveSchema = await TableArchiveReader.ReadTableSchemaAsync(request.ArchivePath, ct: ct); + string targetTableName = string.IsNullOrWhiteSpace(request.TargetTableName) + ? RequireIdentifier(archiveSchema.TableName, nameof(request.TargetTableName)) + : RequireIdentifier(request.TargetTableName, nameof(request.TargetTableName)); + + var restoreSchema = new PrimitiveTableSchema + { + TableName = targetTableName, + Columns = archiveSchema.Columns, + ForeignKeys = archiveSchema.ForeignKeys, + NextRowId = archiveSchema.NextRowId, + }; + + await ExecuteCheckedAsync(BuildCreateTableSql(restoreSchema), ct); + + long inserted = 0; + var batch = new List(RestoreInsertBatchSize); + await foreach (PrimitiveDbValue[] row in TableArchiveReader.ReadRowsAsync(request.ArchivePath, ct)) + { + batch.Add(row); + if (batch.Count >= RestoreInsertBatchSize) + { + inserted += await InsertBatchAsync(targetTableName, archiveSchema.Columns, batch, ct); + batch.Clear(); + } + } + + if (batch.Count > 0) + inserted += await InsertBatchAsync(targetTableName, archiveSchema.Columns, batch, ct); + + return new RestoreTableResult + { + TableName = targetTableName, + RowsInserted = inserted, + }; + } + + private async IAsyncEnumerable EnumerateRowsAsync( + ClientTableSchema schema, + string path, + IProgress? progress, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct) + { + int page = 1; + int totalRows = int.MaxValue; + int seen = 0; + var interval = Stopwatch.StartNew(); + + while (seen < totalRows) + { + ct.ThrowIfCancellationRequested(); + var result = await client.BrowseTableAsync(schema.TableName, page, ExportPageSize, ct); + totalRows = result.TotalRows; + if (result.Rows.Count == 0) + yield break; + + foreach (object?[] row in result.Rows) + { + ct.ThrowIfCancellationRequested(); + yield return MapRow(schema, row); + seen++; + } + + if (seen >= totalRows || interval.ElapsedMilliseconds >= 500) + { + ReportExportProgress( + progress, + schema.TableName, + "Exporting", + "Writing table archive", + seen, + totalRows, + path); + interval.Restart(); + await Task.Yield(); + } + + page++; + } + + ReportExportProgress( + progress, + schema.TableName, + "Exporting", + "Writing table archive", + seen, + totalRows == int.MaxValue ? null : totalRows, + path); + } + + private static void ReportExportProgress( + IProgress? progress, + string tableName, + string stage, + string message, + long rowsProcessed, + long? totalRows, + string? path) + { + progress?.Report(new TableExportProgress + { + Operation = "Export table", + Stage = stage, + Message = message, + TableName = tableName, + Path = path, + RowsProcessed = rowsProcessed, + TotalRows = totalRows, + }); + } + + private async Task InsertBatchAsync( + string tableName, + IReadOnlyList columns, + IReadOnlyList rows, + CancellationToken ct) + { + if (rows.Count == 0) + return 0; + + var sql = new StringBuilder(); + sql.Append("INSERT INTO ").Append(tableName).Append(" ("); + for (int i = 0; i < columns.Count; i++) + { + if (i > 0) + sql.Append(", "); + sql.Append(columns[i].Name); + } + + sql.Append(") VALUES "); + for (int rowIndex = 0; rowIndex < rows.Count; rowIndex++) + { + if (rowIndex > 0) + sql.Append(", "); + sql.Append('('); + for (int columnIndex = 0; columnIndex < columns.Count; columnIndex++) + { + if (columnIndex > 0) + sql.Append(", "); + + PrimitiveDbValue value = columnIndex < rows[rowIndex].Length + ? rows[rowIndex][columnIndex] + : PrimitiveDbValue.Null; + sql.Append(FormatLiteral(value, columns[columnIndex].Type)); + } + + sql.Append(')'); + } + + sql.Append(';'); + var result = await client.ExecuteSqlAsync(sql.ToString(), ct); + if (!string.IsNullOrWhiteSpace(result.Error)) + throw new InvalidOperationException(result.Error); + + return result.RowsAffected; + } + + private static PrimitiveTableSchema MapSchema(ClientTableSchema schema) => new() + { + TableName = schema.TableName, + Columns = schema.Columns.Select(MapColumn).ToArray(), + ForeignKeys = schema.ForeignKeys.Select(MapForeignKey).ToArray(), + NextRowId = 1, + }; + + private static PrimitiveColumnDefinition MapColumn(ClientColumnDefinition column) => new() + { + Name = column.Name, + Type = column.Type switch + { + ClientDbType.Integer => PrimitiveDbType.Integer, + ClientDbType.Real => PrimitiveDbType.Real, + ClientDbType.Text => PrimitiveDbType.Text, + ClientDbType.Blob => PrimitiveDbType.Blob, + _ => throw new InvalidOperationException($"Unsupported column type '{column.Type}'."), + }, + Nullable = column.Nullable, + IsPrimaryKey = column.IsPrimaryKey, + IsIdentity = column.IsIdentity, + Collation = column.Collation, + }; + + private static PrimitiveForeignKeyDefinition MapForeignKey(ClientForeignKeyDefinition foreignKey) => new() + { + ConstraintName = foreignKey.ConstraintName, + ColumnName = foreignKey.ColumnName, + ReferencedTableName = foreignKey.ReferencedTableName, + ReferencedColumnName = foreignKey.ReferencedColumnName, + OnDelete = foreignKey.OnDelete == ClientForeignKeyOnDeleteAction.Cascade + ? PrimitiveForeignKeyOnDeleteAction.Cascade + : PrimitiveForeignKeyOnDeleteAction.Restrict, + SupportingIndexName = foreignKey.SupportingIndexName, + }; + + private static PrimitiveDbValue[] MapRow(ClientTableSchema schema, object?[] row) + { + var values = new PrimitiveDbValue[schema.Columns.Count]; + for (int i = 0; i < values.Length; i++) + { + object? value = i < row.Length ? row[i] : null; + values[i] = MapValue(schema.Columns[i].Type, value); + } + + return values; + } + + private static PrimitiveDbValue MapValue(ClientDbType columnType, object? value) + { + if (value is null) + return PrimitiveDbValue.Null; + + return columnType switch + { + ClientDbType.Integer => PrimitiveDbValue.FromInteger(Convert.ToInt64(value, CultureInfo.InvariantCulture)), + ClientDbType.Real => PrimitiveDbValue.FromReal(Convert.ToDouble(value, CultureInfo.InvariantCulture)), + ClientDbType.Text => PrimitiveDbValue.FromText(Convert.ToString(value, CultureInfo.InvariantCulture) ?? string.Empty), + ClientDbType.Blob => PrimitiveDbValue.FromBlob(ConvertToBytes(value)), + _ => throw new InvalidOperationException($"Unsupported column type '{columnType}'."), + }; + } + + private static byte[] ConvertToBytes(object value) + { + if (value is byte[] bytes) + return bytes; + + if (value is IEnumerable byteEnumerable) + return byteEnumerable.ToArray(); + + if (value is string text) + return Convert.FromBase64String(text); + + if (value is IEnumerable enumerable) + return enumerable.Cast().Select(item => Convert.ToByte(item, CultureInfo.InvariantCulture)).ToArray(); + + throw new InvalidOperationException($"Cannot convert value of type '{value.GetType().Name}' to BLOB."); + } + + private static ExternalTableRegistrationInfo MapExternalTableRegistration(object?[] row) + { + string createdText = row.Length > 4 ? Convert.ToString(row[4], CultureInfo.InvariantCulture) ?? string.Empty : string.Empty; + DateTimeOffset? createdUtc = DateTimeOffset.TryParse( + createdText, + CultureInfo.InvariantCulture, + DateTimeStyles.RoundtripKind, + out var parsed) + ? parsed + : null; + + return new ExternalTableRegistrationInfo + { + TableName = row.Length > 0 ? Convert.ToString(row[0], CultureInfo.InvariantCulture) ?? string.Empty : string.Empty, + Path = row.Length > 1 ? Convert.ToString(row[1], CultureInfo.InvariantCulture) ?? string.Empty : string.Empty, + SourceTableName = row.Length > 2 ? Convert.ToString(row[2], CultureInfo.InvariantCulture) : null, + RowCount = row.Length > 3 && row[3] is not null + ? Convert.ToInt64(row[3], CultureInfo.InvariantCulture) + : 0, + CreatedUtc = createdUtc, + }; + } + + private string ResolveArchivePath(string archivePath) + { + string trimmed = archivePath.Trim(); + if (Path.IsPathFullyQualified(trimmed)) + return trimmed; + + return Path.GetFullPath(Path.Combine(ResolveDatabaseFolder(client.DataSource), trimmed)); + } + + private static string BuildCreateTableSql(PrimitiveTableSchema schema) + { + var sql = new StringBuilder(); + sql.Append("CREATE TABLE ").Append(RequireIdentifier(schema.TableName, nameof(schema.TableName))).Append(" ("); + for (int i = 0; i < schema.Columns.Count; i++) + { + PrimitiveColumnDefinition column = schema.Columns[i]; + if (i > 0) + sql.Append(", "); + + sql.Append(RequireIdentifier(column.Name, nameof(column.Name))) + .Append(' ') + .Append(column.Type.ToString().ToUpperInvariant()); + + if (column.IsPrimaryKey) + sql.Append(" PRIMARY KEY"); + if (column.IsIdentity) + sql.Append(" IDENTITY"); + if (!column.Nullable && !column.IsPrimaryKey) + sql.Append(" NOT NULL"); + if (!string.IsNullOrWhiteSpace(column.Collation)) + sql.Append(" COLLATE ").Append(RequireIdentifier(column.Collation, nameof(column.Collation))); + } + + sql.Append(");"); + return sql.ToString(); + } + + private static string FormatLiteral(PrimitiveDbValue value, PrimitiveDbType columnType) + { + if (value.IsNull) + return "NULL"; + + return columnType switch + { + PrimitiveDbType.Integer => value.AsInteger.ToString(CultureInfo.InvariantCulture), + PrimitiveDbType.Real => value.AsReal.ToString("R", CultureInfo.InvariantCulture), + PrimitiveDbType.Blob => "X'" + Convert.ToHexString(value.AsBlob) + "'", + _ => FormatStringLiteral(value.Type == PrimitiveDbType.Text ? value.AsText : value.ToString()), + }; + } + + private static string FormatStringLiteral(string value) => "'" + value.Replace("'", "''", StringComparison.Ordinal) + "'"; + + private async Task ExecuteCheckedAsync(string sql, CancellationToken ct) + { + var result = await client.ExecuteSqlAsync(sql, ct); + if (!string.IsNullOrWhiteSpace(result.Error)) + throw new InvalidOperationException(result.Error); + } + + private static string RequireIdentifier(string value, string parameterName) + { + if (string.IsNullOrWhiteSpace(value)) + throw new ArgumentException("Identifier is required.", parameterName); + + string trimmed = value.Trim(); + if (!IsIdentifier(trimmed)) + throw new ArgumentException($"'{trimmed}' is not a valid CSharpDB identifier.", parameterName); + + return trimmed; + } + + private static bool IsIdentifier(string value) + { + if (value.Length == 0 || !(char.IsLetter(value[0]) || value[0] == '_')) + return false; + + for (int i = 1; i < value.Length; i++) + { + char c = value[i]; + if (!char.IsLetterOrDigit(c) && c != '_') + return false; + } + + return true; + } + + private static string ResolveDatabaseFolder(string dataSource) + { + if (string.IsNullOrWhiteSpace(dataSource) || + dataSource.StartsWith(":memory:", StringComparison.OrdinalIgnoreCase) || + (Uri.TryCreate(dataSource, UriKind.Absolute, out var uri) && !uri.IsFile)) + { + return Directory.GetCurrentDirectory(); + } + + string path = Uri.TryCreate(dataSource, UriKind.Absolute, out var fileUri) && fileUri.IsFile + ? fileUri.LocalPath + : Path.GetFullPath(dataSource); + string? directory = Path.GetDirectoryName(path); + return string.IsNullOrWhiteSpace(directory) ? Directory.GetCurrentDirectory() : directory; + } + + private static string CreateTemporaryArchivePath(string tableName) + { + string fileName = $"{SanitizeFileName(tableName)}-{DateTime.Now:yyyyMMdd-HHmmss}-{Guid.NewGuid():N}.csdbtable"; + string directory = Path.Combine(Path.GetTempPath(), "csharpdb-admin-exports"); + Directory.CreateDirectory(directory); + return Path.Combine(directory, fileName); + } + + private static string SanitizeFileName(string value) + { + char[] invalid = Path.GetInvalidFileNameChars(); + var builder = new StringBuilder(value.Length); + foreach (char c in value) + builder.Append(invalid.Contains(c) ? '_' : c); + return builder.Length == 0 ? "table" : builder.ToString(); + } +} diff --git a/src/CSharpDB.Admin.ImportExport/_Imports.razor b/src/CSharpDB.Admin.ImportExport/_Imports.razor new file mode 100644 index 00000000..45e9ef65 --- /dev/null +++ b/src/CSharpDB.Admin.ImportExport/_Imports.razor @@ -0,0 +1,6 @@ +@using Microsoft.AspNetCore.Components +@using Microsoft.AspNetCore.Components.Web +@using CSharpDB.Admin.ImportExport.Contracts +@using CSharpDB.Admin.ImportExport.Components +@using CSharpDB.Admin.ImportExport.Services +@using CSharpDB.Client diff --git a/src/CSharpDB.Admin/CSharpDB.Admin.csproj b/src/CSharpDB.Admin/CSharpDB.Admin.csproj index 2c073bb4..6b2893d0 100644 --- a/src/CSharpDB.Admin/CSharpDB.Admin.csproj +++ b/src/CSharpDB.Admin/CSharpDB.Admin.csproj @@ -10,8 +10,11 @@ + + + diff --git a/src/CSharpDB.Admin/Components/Layout/CommandPalette.razor b/src/CSharpDB.Admin/Components/Layout/CommandPalette.razor index 40fb3611..383c7ee7 100644 --- a/src/CSharpDB.Admin/Components/Layout/CommandPalette.razor +++ b/src/CSharpDB.Admin/Components/Layout/CommandPalette.razor @@ -222,6 +222,9 @@ _items.Add(new PaletteItem("Command", "New Report", "Open report designer", "bi-file-earmark-richtext", "icon-report", () => { TabManager.OpenReportDesignerTab(); return Task.CompletedTask; })); _items.Add(new PaletteItem("Command", "New Procedure", "Open procedure editor", "bi-gear-wide-connected", "icon-trigger", () => { TabManager.OpenNewProcedureTab(); return Task.CompletedTask; })); _items.Add(new PaletteItem("Command", "New Pipeline", "Open pipeline builder", "bi-diagram-3", "icon-view", () => { TabManager.OpenPipelineTab(); return Task.CompletedTask; })); + _items.Add(new PaletteItem("Command", "Data Model", "Open visual data modeling canvas", "bi-diagram-3", "icon-view", () => { TabManager.OpenDataModelTab(); return Task.CompletedTask; })); + _items.Add(new PaletteItem("Command", "Code Modules", "Open C# code modules", "bi-filetype-cs", "icon-system", () => { TabManager.OpenCodeModulesTab(); return Task.CompletedTask; })); + _items.Add(new PaletteItem("Command", "Import / Export", "Open table archive tools", "bi-arrow-left-right", "icon-system", () => { TabManager.OpenImportExportTab(); return Task.CompletedTask; })); _items.Add(new PaletteItem("Command", "Storage Inspector", "Open storage diagnostics", "bi-hdd-stack", "icon-system", () => { TabManager.OpenStorageTab(); return Task.CompletedTask; })); } diff --git a/src/CSharpDB.Admin/Components/Layout/MainLayout.razor b/src/CSharpDB.Admin/Components/Layout/MainLayout.razor index 9b77d2d7..d274db53 100644 --- a/src/CSharpDB.Admin/Components/Layout/MainLayout.razor +++ b/src/CSharpDB.Admin/Components/Layout/MainLayout.razor @@ -51,6 +51,10 @@ break; + case TabKind.CodeModules: + + break; case TabKind.Procedure: @@ -82,6 +86,14 @@ break; + case TabKind.ImportExport: + + break; + case TabKind.DataModel: + + break; } } diff --git a/src/CSharpDB.Admin/Components/Layout/NavMenu.razor b/src/CSharpDB.Admin/Components/Layout/NavMenu.razor index 5ebae4d0..5ea9dbb9 100644 --- a/src/CSharpDB.Admin/Components/Layout/NavMenu.razor +++ b/src/CSharpDB.Admin/Components/Layout/NavMenu.razor @@ -38,6 +38,7 @@
+ @@ -47,6 +48,28 @@
@if (ShouldShowUtilityGroups) { +
+
+ + Tools + 2 +
+
+ + +
+
+
@@ -206,6 +229,39 @@
} + @* External Tables *@ + @if (ShouldShowGroup("externalTables")) + { +
+
+ + + External Tables + @(_externalTables?.Count ?? 0) +
+ @if (_expandedGroups.Contains("externalTables") && _externalTables is not null) + { +
+ @foreach (var item in FilterExternalTables()) + { + var externalTable = item; +
+ + @externalTable.TableName + @FormatExternalTableHint(externalTable) +
+ } +
+ } +
+ } + @* Collections *@ @if (ShouldShowGroup("collections")) { @@ -639,6 +695,12 @@ private sealed record SystemCatalogItem(string Name, string Description, string Sql); + private sealed record ExternalTableExplorerItem( + string TableName, + string Path, + string SourceTableName, + long RowCount); + private static readonly IReadOnlyList s_systemCatalogItems = new[] { new SystemCatalogItem("sys.tables", "tables", "SELECT * FROM sys.tables ORDER BY table_name;"), @@ -653,10 +715,13 @@ new SystemCatalogItem("sys.triggers", "triggers", "SELECT * FROM sys.triggers ORDER BY trigger_name;"), new SystemCatalogItem("sys.objects", "all objects", "SELECT * FROM sys.objects ORDER BY object_type, object_name;"), new SystemCatalogItem("sys.saved_queries", "saved queries", "SELECT * FROM sys.saved_queries ORDER BY name;"), + new SystemCatalogItem("sys.external_tables", "external tables", "SELECT * FROM sys.external_tables ORDER BY table_name;"), + new SystemCatalogItem("sys.diagrams", "diagrams", "SELECT * FROM sys.diagrams ORDER BY name;"), }; private IReadOnlyCollection? _userTableNames; private IReadOnlyCollection? _systemTableNames; + private IReadOnlyList? _externalTables; private IReadOnlyCollection? _viewNames; private IReadOnlyCollection? _collectionNames; private IReadOnlyList? _indexes; @@ -667,7 +732,7 @@ private IReadOnlyList? _reports; private string _searchQuery = string.Empty; private string _objectFilter = "all"; - private readonly HashSet _expandedGroups = new() { "tables", "collections", "forms", "reports", "system", "views", "procedures", "callbacks" }; + private readonly HashSet _expandedGroups = new() { "tables", "externalTables", "collections", "forms", "reports", "system", "views", "procedures", "callbacks" }; private readonly HashSet _expandedTableNodes = new(StringComparer.OrdinalIgnoreCase); private readonly Dictionary _tableSchemas = new(StringComparer.OrdinalIgnoreCase); private readonly HashSet _tableSchemaLoadFailures = new(StringComparer.OrdinalIgnoreCase); @@ -749,6 +814,7 @@ var allTables = (await DbClient.GetTableNamesAsync()).OrderBy(n => n).ToList(); _userTableNames = allTables.Where(static n => !IsSystemTableName(n)).ToList(); _systemTableNames = allTables.Where(IsSystemTableName).ToList(); + _externalTables = await LoadExternalTablesAsync(); _viewNames = (await DbClient.GetViewNamesAsync()).OrderBy(n => n).ToList(); _collectionNames = (await DbClient.GetCollectionNamesAsync()).OrderBy(n => n, StringComparer.OrdinalIgnoreCase).ToList(); _indexes = (await DbClient.GetIndexesAsync()).OrderBy(i => i.IndexName).ToList(); @@ -767,6 +833,7 @@ { _userTableNames = null; _systemTableNames = null; + _externalTables = null; _viewNames = null; _collectionNames = null; _indexes = null; @@ -779,6 +846,39 @@ } } + private async Task> LoadExternalTablesAsync() + { + var result = await DbClient.ExecuteSqlAsync(""" + SELECT table_name, path, source_table_name, row_count + FROM sys.external_tables + ORDER BY table_name; + """); + + if (result.Error is not null) + return Array.Empty(); + + var items = new List(); + foreach (object?[] row in result.Rows ?? []) + { + if (row.Length < 4 || row[0] is null) + continue; + + string tableName = Convert.ToString(row[0], System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty; + if (string.IsNullOrWhiteSpace(tableName)) + continue; + + string path = Convert.ToString(row[1], System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty; + string sourceTableName = Convert.ToString(row[2], System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty; + long rowCount = row[3] is null + ? 0 + : Convert.ToInt64(row[3], System.Globalization.CultureInfo.InvariantCulture); + + items.Add(new ExternalTableExplorerItem(tableName, path, sourceTableName, rowCount)); + } + + return items; + } + private void ToggleGroup(string group) { if (!_expandedGroups.Remove(group)) @@ -813,7 +913,8 @@ => _objectFilter switch { "all" => true, - "tables" => group is "tables" or "views", + "tables" => group is "tables" or "externalTables" or "views", + "externalTables" => group == "externalTables", "collections" => group == "collections", "callbacks" => group == "callbacks", "forms" => group == "forms", @@ -836,6 +937,28 @@ return FilterItems(_collectionNames); } + private IEnumerable FilterExternalTables() + { + if (_externalTables is null) + return Enumerable.Empty(); + + if (string.IsNullOrWhiteSpace(_searchQuery)) + return _externalTables; + + return _externalTables.Where(item => + item.TableName.Contains(_searchQuery, StringComparison.OrdinalIgnoreCase) || + item.SourceTableName.Contains(_searchQuery, StringComparison.OrdinalIgnoreCase) || + item.Path.Contains(_searchQuery, StringComparison.OrdinalIgnoreCase)); + } + + private static string FormatExternalTableHint(ExternalTableExplorerItem table) + { + string source = string.IsNullOrWhiteSpace(table.SourceTableName) + ? "archive" + : table.SourceTableName; + return $"{source} · {table.RowCount:N0} rows"; + } + private IEnumerable FilterTableNames(IReadOnlyCollection tableNames) { if (string.IsNullOrWhiteSpace(_searchQuery)) @@ -1087,6 +1210,12 @@ private void OpenSystemCatalogTab(SystemCatalogItem item) => TabManager.OpenSystemCatalogTab(item.Name, item.Sql); + private void OpenExternalTable(string tableName) + => TabManager.OpenQueryTab($"SELECT * FROM {RequireSqlIdentifier(tableName)} LIMIT 50;"); + + private void OpenExternalTableMetadata(string tableName) + => TabManager.OpenQueryTab($"SELECT * FROM sys.external_tables WHERE table_name = {FormatSqlStringLiteral(tableName)};"); + private void OpenInternalFunction(FormulaFunctionDescriptor function) { TabDescriptor tab = TabManager.OpenCallbacksTab(); @@ -1140,6 +1269,20 @@ return activeTab.Id == $"{type}:{name}"; } + private bool IsActiveExternalTable(string tableName) + { + var activeTab = TabManager.ActiveTab; + return activeTab?.Kind == TabKind.Query + && activeTab.SqlText is { } sqlText + && sqlText.Contains(tableName, StringComparison.OrdinalIgnoreCase); + } + + private bool IsActiveImportExport() + => TabManager.ActiveTab?.Kind == TabKind.ImportExport; + + private bool IsActiveDataModel() + => TabManager.ActiveTab?.Kind == TabKind.DataModel; + private bool IsActiveSchema(string type, string name) { var activeTab = TabManager.ActiveTab; @@ -1238,11 +1381,24 @@ { new() { Label = "New Table...", Icon = "bi-plus-lg", OnClick = () => TabManager.OpenTableDesignerTab() }, ContextMenuItem.Separator(), + new() { Label = "Data Model", Icon = "bi-diagram-3", OnClick = () => TabManager.OpenDataModelTab() }, new() { Label = "New Query", Icon = "bi-terminal", OnClick = () => TabManager.OpenQueryTab() }, new() { Label = "Refresh", Icon = "bi-arrow-clockwise", OnClick = async () => { await LoadObjects(); await InvokeAsync(StateHasChanged); } }, }); } + private void ShowExternalTablesGroupMenu(MouseEventArgs e) + { + ShowContextMenu(e, new List + { + new() { Label = "Register External Table...", Icon = "bi-box-arrow-up-right", OnClick = () => TabManager.OpenImportExportTab() }, + new() { Label = "Data Model", Icon = "bi-diagram-3", OnClick = () => TabManager.OpenDataModelTab() }, + new() { Label = "View Metadata", Icon = "bi-hdd-stack", OnClick = () => OpenSystemCatalogTab(s_systemCatalogItems.First(item => item.Name == "sys.external_tables")) }, + ContextMenuItem.Separator(), + new() { Label = "Refresh", Icon = "bi-arrow-clockwise", OnClick = async () => { await LoadObjects(); await InvokeAsync(StateHasChanged); } }, + }); + } + private void ShowCollectionsGroupMenu(MouseEventArgs e) { ShowContextMenu(e, new List @@ -1272,8 +1428,10 @@ { new() { Label = "Open", Icon = "bi-table", OnClick = () => TabManager.OpenTableTab(tableName) }, new() { Label = "Design", Icon = "bi-file-earmark-code", OnClick = () => TabManager.OpenTableSchemaTab(tableName) }, + new() { Label = "Show in Data Model", Icon = "bi-diagram-3", OnClick = () => TabManager.OpenDataModelTab(tableName) }, new() { Label = "Design Form...", Icon = "bi-ui-checks-grid", OnClick = () => TabManager.OpenFormDesignerTab(initialTableName: tableName, title: $"{tableName} Form") }, new() { Label = "Design Report...", Icon = "bi-file-earmark-richtext", OnClick = () => TabManager.OpenReportDesignerTab(sourceKind: ReportSourceKind.Table.ToString(), sourceName: tableName, title: $"{tableName} Report") }, + new() { Label = "Import / Export...", Icon = "bi-arrow-left-right", OnClick = () => TabManager.OpenImportExportTab(tableName) }, ContextMenuItem.Separator(), new() { Label = "Select Top 50", Icon = "bi-terminal", OnClick = () => TabManager.OpenQueryTab($"SELECT * FROM {tableName} LIMIT 50;") }, new() { Label = "Select Count", Icon = "bi-123", OnClick = () => TabManager.OpenQueryTab($"SELECT COUNT(*) AS row_count FROM {tableName};") }, @@ -1282,6 +1440,24 @@ }); } + private void ShowExternalTableItemMenu(MouseEventArgs e, ExternalTableExplorerItem table) + { + ShowContextMenu(e, new List + { + new() { Label = "Open", Icon = "bi-box-arrow-up-right", OnClick = () => OpenExternalTable(table.TableName) }, + new() { Label = "Open Metadata", Icon = "bi-hdd-stack", OnClick = () => OpenExternalTableMetadata(table.TableName) }, + new() { Label = "Show in Data Model", Icon = "bi-diagram-3", OnClick = () => TabManager.OpenDataModelTab(table.TableName) }, + ContextMenuItem.Separator(), + new() { Label = "Select Top 50", Icon = "bi-terminal", OnClick = () => TabManager.OpenQueryTab($"SELECT * FROM {RequireSqlIdentifier(table.TableName)} LIMIT 50;") }, + new() { Label = "Select Count", Icon = "bi-123", OnClick = () => TabManager.OpenQueryTab($"SELECT COUNT(*) AS row_count FROM {RequireSqlIdentifier(table.TableName)};") }, + ContextMenuItem.Separator(), + new() { Label = "Copy Name", Icon = "bi-clipboard", OnClick = async () => await CopyExternalTableTextAsync(table.TableName) }, + new() { Label = "Copy Path", Icon = "bi-clipboard-plus", OnClick = async () => await CopyExternalTableTextAsync(table.Path) }, + ContextMenuItem.Separator(), + new() { Label = "Drop External Table", Icon = "bi-trash3", IsDanger = true, OnClick = async () => await ConfirmDropExternalTableAsync(table.TableName) }, + }); + } + private void ShowFormsGroupMenu(MouseEventArgs e) { ShowContextMenu(e, new List @@ -1484,6 +1660,19 @@ } } + private async Task CopyExternalTableTextAsync(string text) + { + try + { + await JS.InvokeVoidAsync("clipboardInterop.writeText", text); + Toast.Success("Copied external table value."); + } + catch (Exception ex) + { + Toast.Error(ex.Message); + } + } + private async Task ConfirmDropTableAsync(string tableName) { var confirmed = await Modal.ConfirmAsync("Drop Table", $"Are you sure you want to drop table '{tableName}'? This action cannot be undone.", "Drop", isDanger: true); @@ -1498,6 +1687,32 @@ catch (Exception ex) { Toast.Error(ex.Message); } } + private async Task ConfirmDropExternalTableAsync(string tableName) + { + var confirmed = await Modal.ConfirmAsync( + "Drop External Table", + $"Are you sure you want to drop external table '{tableName}'? This removes the registration only; the archive file is not deleted.", + "Drop", + isDanger: true); + if (!confirmed) return; + + try + { + var result = await DbClient.ExecuteSqlAsync($"DROP EXTERNAL TABLE IF EXISTS {RequireSqlIdentifier(tableName)};"); + if (!string.IsNullOrWhiteSpace(result.Error)) + throw new InvalidOperationException(result.Error); + + Toast.Success($"Dropped external table '{tableName}'."); + Changes.NotifyChanged(); + await LoadObjects(); + await InvokeAsync(StateHasChanged); + } + catch (Exception ex) + { + Toast.Error(ex.Message); + } + } + private async Task ConfirmDropCollectionAsync(string collectionName) { var confirmed = await Modal.ConfirmAsync("Drop Collection", $"Are you sure you want to drop collection '{collectionName}'? All documents and collection indexes will be deleted. This action cannot be undone.", "Drop", isDanger: true); @@ -1609,6 +1824,36 @@ TabManager.StateChanged -= OnTabChanged; } + private static string RequireSqlIdentifier(string value) + { + if (string.IsNullOrWhiteSpace(value)) + throw new InvalidOperationException("Identifier is required."); + + string trimmed = value.Trim(); + if (!IsSqlIdentifier(trimmed)) + throw new InvalidOperationException($"'{trimmed}' is not a valid CSharpDB identifier."); + + return trimmed; + } + + private static bool IsSqlIdentifier(string value) + { + if (value.Length == 0 || !(char.IsLetter(value[0]) || value[0] == '_')) + return false; + + for (int i = 1; i < value.Length; i++) + { + char c = value[i]; + if (!char.IsLetterOrDigit(c) && c != '_') + return false; + } + + return true; + } + + private static string FormatSqlStringLiteral(string value) + => "'" + value.Replace("'", "''", StringComparison.Ordinal) + "'"; + private static bool IsSystemTableName(string name) => name.StartsWith("_", StringComparison.Ordinal); } diff --git a/src/CSharpDB.Admin/Components/Layout/TitleBar.razor b/src/CSharpDB.Admin/Components/Layout/TitleBar.razor index f2f00423..ab022eba 100644 --- a/src/CSharpDB.Admin/Components/Layout/TitleBar.razor +++ b/src/CSharpDB.Admin/Components/Layout/TitleBar.razor @@ -42,6 +42,9 @@ + diff --git a/src/CSharpDB.Admin/Components/Shared/DataGrid.razor b/src/CSharpDB.Admin/Components/Shared/DataGrid.razor index 2f65746b..2dd63353 100644 --- a/src/CSharpDB.Admin/Components/Shared/DataGrid.razor +++ b/src/CSharpDB.Admin/Components/Shared/DataGrid.razor @@ -13,8 +13,11 @@
@if (_loading) { -
+
+
} else if (_loadError is not null) @@ -232,6 +235,7 @@ private readonly List _viewCursorLookaheadRows = new(); private int _viewCursorNextPage; private bool _bypassViewCursorForNextLoad; + private CancellationTokenSource? _loadCts; private int TotalPages => _hasExactTotal ? Math.Max(1, (int)Math.Ceiling(_totalRows / (double)_pageSize)) @@ -282,47 +286,88 @@ public async Task LoadDataAsync() { + _loadCts?.Cancel(); + var loadCts = new CancellationTokenSource(); + _loadCts = loadCts; + CancellationToken ct = loadCts.Token; + _loading = true; _loadError = null; _editingCell = null; _selectedRows.Clear(); await NotifySelectionChangedAsync(); StateHasChanged(); + await Task.Yield(); try { if (ShouldLoadAllRows()) { await DisposeViewCursorAsync(); - await LoadAllRowsAsync(); + await LoadAllRowsAsync(ct); } else - await LoadCurrentPageAsync(); + await LoadCurrentPageAsync(ct); ApplyFiltersAndSort(); } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + await DisposeViewCursorAsync(); + if (ReferenceEquals(_loadCts, loadCts)) + _loadError = "Query canceled."; + } catch (Exception ex) { await DisposeViewCursorAsync(); - _allRows = new(); - _rows = new(); - _totalRows = 0; - _hasExactTotal = true; - _hasNextPage = false; - _loadError = ex.Message; - - if (QuerySql is null) - Toast.Error($"Failed to load data: {ex.Message}"); + if (ReferenceEquals(_loadCts, loadCts)) + { + _allRows = new(); + _rows = new(); + _totalRows = 0; + _hasExactTotal = true; + _hasNextPage = false; + _loadError = ex.Message; + + if (QuerySql is null) + Toast.Error($"Failed to load data: {ex.Message}"); + } } finally { - _loading = false; - await NotifyQueryStatusChangedAsync(); - await NotifyFiltersChangedAsync(); - StateHasChanged(); + bool isCurrentLoad = ReferenceEquals(_loadCts, loadCts); + if (isCurrentLoad) + { + _loadCts = null; + _loading = false; + } + + loadCts.Dispose(); + if (isCurrentLoad) + { + await NotifyQueryStatusChangedAsync(); + await NotifyFiltersChangedAsync(); + StateHasChanged(); + } } } + private void CancelLoad() + { + _loadCts?.Cancel(); + _loadError ??= "Canceling query..."; + StateHasChanged(); + } + + private Task ExecuteSqlAsync(string sql, CancellationToken ct) + => Task.Run(() => DbClient.ExecuteSqlAsync(sql, ct), ct); + + private Task GetTableSchemaAsync(string tableName, CancellationToken ct) + => Task.Run(() => DbClient.GetTableSchemaAsync(tableName, ct), ct); + + private Task TryOpenForwardOnlyQueryCursorAsync(string sql, CancellationToken ct) + => Task.Run(async () => await DbHolder.TryOpenForwardOnlyQueryCursorAsync(sql, ct), ct); + // ─── Sorting ──────────────────── private async Task SortColumn(int colIdx) { @@ -950,7 +995,7 @@ private bool ShouldLoadAllRows() => QuerySql is null && HasPendingChanges; - private async Task LoadCurrentPageAsync() + private async Task LoadCurrentPageAsync(CancellationToken ct) { _loadedAllRows = false; _lastQueryElapsed = null; @@ -959,18 +1004,18 @@ if (QuerySql is not null) { - await LoadQueryPageAsync(QuerySql); + await LoadQueryPageAsync(QuerySql, ct); return; } if (TableName is not null) { string tableName = RequireIdentifier(TableName); - var schema = await DbClient.GetTableSchemaAsync(tableName) + var schema = await GetTableSchemaAsync(tableName, ct) ?? throw new InvalidOperationException($"Table '{tableName}' was not found."); ApplyTableSchema(schema); - await LoadServerPageAsync(tableName); + await LoadServerPageAsync(tableName, ct); return; } @@ -978,7 +1023,7 @@ { string viewName = RequireIdentifier(ViewName); await DisposeViewCursorAsync(); - await LoadUnknownTotalSourcePageAsync(viewName, captureColumnNames: true); + await LoadUnknownTotalSourcePageAsync(viewName, captureColumnNames: true, ct: ct); Columns ??= Array.Empty(); PrimaryKeyColumn = null; @@ -990,9 +1035,9 @@ _totalRows = 0; } - private async Task LoadServerPageAsync(string sourceName) + private async Task LoadServerPageAsync(string sourceName, CancellationToken ct) { - var countResult = await DbClient.ExecuteSqlAsync(BuildCountSql(sourceName)); + var countResult = await ExecuteSqlAsync(BuildCountSql(sourceName), ct); if (countResult.Error is not null) throw new InvalidOperationException(countResult.Error); @@ -1000,7 +1045,7 @@ _hasExactTotal = true; ClampPageToAvailableRows(); - var queryResult = await DbClient.ExecuteSqlAsync(BuildSelectSql(sourceName)); + var queryResult = await ExecuteSqlAsync(BuildSelectSql(sourceName), ct); if (queryResult.Error is not null) throw new InvalidOperationException(queryResult.Error); @@ -1011,11 +1056,14 @@ Columns = queryResult.ColumnNames ?? Array.Empty(); } - private async Task LoadUnknownTotalSourcePageAsync(string sourceName, bool captureColumnNames = false) + private async Task LoadUnknownTotalSourcePageAsync( + string sourceName, + bool captureColumnNames = false, + CancellationToken ct = default) { while (true) { - var queryResult = await DbClient.ExecuteSqlAsync(BuildSelectSql(sourceName, _pageSize + 1)); + var queryResult = await ExecuteSqlAsync(BuildSelectSql(sourceName, _pageSize + 1), ct); if (queryResult.Error is not null) throw new InvalidOperationException(queryResult.Error); @@ -1037,7 +1085,7 @@ } } - private async Task TryLoadViewCursorPageAsync(string viewName) + private async Task TryLoadViewCursorPageAsync(string viewName, CancellationToken ct) { if (_bypassViewCursorForNextLoad) { @@ -1049,7 +1097,7 @@ if (_page == 1) { await DisposeViewCursorAsync(); - _viewCursor = await DbHolder.TryOpenForwardOnlyQueryCursorAsync(cursorSql); + _viewCursor = await TryOpenForwardOnlyQueryCursorAsync(cursorSql, ct); if (_viewCursor is null) return false; @@ -1068,11 +1116,11 @@ if (Columns is null || Columns.Length == 0) Columns = _viewCursor.ColumnNames; - await LoadViewCursorPageAsync(); + await LoadViewCursorPageAsync(ct); return true; } - private async Task LoadViewCursorPageAsync() + private async Task LoadViewCursorPageAsync(CancellationToken ct) { if (_viewCursor is null) throw new InvalidOperationException("View cursor is not available."); @@ -1086,7 +1134,7 @@ int remaining = (_pageSize + 1) - rows.Count; if (remaining > 0) - rows.AddRange(await _viewCursor.ReadNextAsync(remaining)); + rows.AddRange(await _viewCursor.ReadNextAsync(remaining, ct)); _hasNextPage = rows.Count > _pageSize; if (_hasNextPage) @@ -1103,7 +1151,7 @@ ApplyPageRows(rows, hasNextPage: _hasNextPage); } - private async Task LoadQueryPageAsync(string querySql) + private async Task LoadQueryPageAsync(string querySql, CancellationToken ct) { _queryPagingPlan ??= QueryPagingPlan.Parse(querySql); @@ -1114,8 +1162,9 @@ { string[] displayColumns = Columns ?? Array.Empty(); // Over-fetch one row so ad-hoc SQL paging can stay responsive without blocking on COUNT(*). - var pageResult = await DbClient.ExecuteSqlAsync( - _queryPagingPlan.BuildPageSql(_filters, _filterModes, _sortCol, _sortAsc, _pageSize + 1, _page, displayColumns)); + var pageResult = await ExecuteSqlAsync( + _queryPagingPlan.BuildPageSql(_filters, _filterModes, _sortCol, _sortAsc, _pageSize + 1, _page, displayColumns), + ct); if (pageResult.Error is not null) throw new InvalidOperationException(pageResult.Error); @@ -1144,7 +1193,7 @@ } } - private async Task LoadAllRowsAsync() + private async Task LoadAllRowsAsync(CancellationToken ct) { var pendingRows = _allRows .Where(row => row.State != RowState.Unmodified) @@ -1154,11 +1203,11 @@ if (TableName is not null) { string tableName = RequireIdentifier(TableName); - var schema = await DbClient.GetTableSchemaAsync(tableName) + var schema = await GetTableSchemaAsync(tableName, ct) ?? throw new InvalidOperationException($"Table '{tableName}' was not found."); ApplyTableSchema(schema); - var result = await DbClient.ExecuteSqlAsync($"SELECT * FROM {tableName}"); + var result = await ExecuteSqlAsync($"SELECT * FROM {tableName}", ct); if (result.Error is not null) throw new InvalidOperationException(result.Error); @@ -1167,7 +1216,7 @@ else if (ViewName is not null) { string viewName = RequireIdentifier(ViewName); - var result = await DbClient.ExecuteSqlAsync($"SELECT * FROM {viewName}"); + var result = await ExecuteSqlAsync($"SELECT * FROM {viewName}", ct); if (result.Error is not null) throw new InvalidOperationException(result.Error); @@ -1636,6 +1685,8 @@ public async ValueTask DisposeAsync() { + _loadCts?.Cancel(); + _loadCts?.Dispose(); await DisposeViewCursorAsync(); } } diff --git a/src/CSharpDB.Admin/Components/Shared/SchemaCanvas.razor b/src/CSharpDB.Admin/Components/Shared/SchemaCanvas.razor new file mode 100644 index 00000000..d269c656 --- /dev/null +++ b/src/CSharpDB.Admin/Components/Shared/SchemaCanvas.razor @@ -0,0 +1,265 @@ +@using CSharpDB.Admin.Models +@inject IJSRuntime JS +@implements IDisposable + +
+
+
+
+ + @foreach (DataModelRelationship relationship in State.Relationships.Where(ShouldRenderRelationship)) + { + var current = relationship; + var (x1, y1, x2, y2) = RelationshipPoints(current); + double cx1 = x1 + 80, cy1 = y1, cx2 = x2 - 80, cy2 = y2; + double mx = (x1 + x2) / 2, my = (y1 + y2) / 2; + bool selected = string.Equals(SelectedRelationshipId, current.Id, StringComparison.Ordinal); + + + + + + @RelationshipLabel(current) + + + } + + + @if (State.Nodes.Count == 0) + { +
+ + Add a table or load a layout to begin. +
+ } + + @foreach (DataModelNode node in State.Nodes) + { + var current = node; + bool selected = string.Equals(SelectedNodeName, current.Name, StringComparison.OrdinalIgnoreCase); +
+
+ + @current.Name + @if (current.Warnings.Count > 0) + { + + } + +
+ @if (!current.IsCollapsed) + { +
+ @if (current.Kind == DataModelNodeKind.ExternalTable) + { + external + } + else if (current.IsDraft) + { + pending create + } + else + { + @current.IndexCount index@(current.IndexCount == 1 ? "" : "es") + @current.TriggerCount trigger@(current.TriggerCount == 1 ? "" : "s") + } + @if (current.RowCount is not null) + { + @current.RowCount rows + } +
+ @foreach (DataModelColumn column in current.Columns) + { +
+ @column.TypeLabel + @column.Name + + @if (column.IsPrimaryKey) + { + + } + @if (column.IsForeignKey) + { + + } + @if (column.IsIndexed && !column.IsPrimaryKey) + { + + } + @if (!column.Nullable && !column.IsPrimaryKey) + { + + } + +
+ } + } +
+ } +
+
+
+
+ +@code { + private const int NodeWidth = 200; + private const int NodeHeaderHeight = 36; + private const int ColumnRowHeight = 28; + private const int NodeMetaHeight = 24; + + [Parameter] public DataModelState State { get; set; } = new(); + [Parameter] public string CanvasId { get; set; } = $"schema-canvas-{Guid.NewGuid():N}"; + [Parameter] public int HeightPx { get; set; } = 560; + [Parameter] public string? SelectedNodeName { get; set; } + [Parameter] public string? SelectedRelationshipId { get; set; } + [Parameter] public EventCallback OnNodeSelected { get; set; } + [Parameter] public EventCallback OnNodeRemoved { get; set; } + [Parameter] public EventCallback OnRelationshipSelected { get; set; } + [Parameter] public EventCallback OnNodeMoved { get; set; } + + private DotNetObjectReference? _dotNetRef; + private ElementReference _canvasRef; + + protected override async Task OnAfterRenderAsync(bool firstRender) + { + if (!firstRender) + return; + + _dotNetRef = DotNetObjectReference.Create(this); + try { await JS.InvokeVoidAsync("designerInterop.initDrag", _dotNetRef); } + catch { } + } + + [JSInvokable] + public async Task OnTableMoved(string tableName, double x, double y) + { + DataModelNode? node = State.Nodes.FirstOrDefault(n => string.Equals(n.Name, tableName, StringComparison.OrdinalIgnoreCase)); + if (node is null) + return; + + node.X = x; + node.Y = y; + await OnNodeMoved.InvokeAsync(new DataModelNodeMove(tableName, x, y)); + } + + private double SvgWidth => Math.Max(1200, State.Nodes.Select(static node => node.X + NodeWidth + 40).DefaultIfEmpty(1200).Max()); + + private double SvgHeight => Math.Max(600, State.Nodes.Select(node => node.Y + NodeHeight(node) + 40).DefaultIfEmpty(600).Max()); + + private double CanvasScale => NormalizeScale(State.Scale); + + private double ScaledSvgWidth => SvgWidth * CanvasScale; + + private double ScaledSvgHeight => SvgHeight * CanvasScale; + + private string ScaleText => + CanvasScale.ToString("0.###", System.Globalization.CultureInfo.InvariantCulture); + + private static int NodeHeight(DataModelNode node) => + node.IsCollapsed ? NodeHeaderHeight : NodeHeaderHeight + NodeMetaHeight + node.Columns.Count * ColumnRowHeight; + + private bool ShouldRenderRelationship(DataModelRelationship relationship) => + relationship.IsResolved + && State.Nodes.Any(node => string.Equals(node.Name, relationship.LeftTable, StringComparison.OrdinalIgnoreCase)) + && State.Nodes.Any(node => string.Equals(node.Name, relationship.RightTable, StringComparison.OrdinalIgnoreCase)); + + private (double x1, double y1, double x2, double y2) RelationshipPoints(DataModelRelationship relationship) + { + DataModelNode? left = State.Nodes.FirstOrDefault(node => string.Equals(node.Name, relationship.LeftTable, StringComparison.OrdinalIgnoreCase)); + DataModelNode? right = State.Nodes.FirstOrDefault(node => string.Equals(node.Name, relationship.RightTable, StringComparison.OrdinalIgnoreCase)); + double x1 = (left?.X ?? 0) + NodeWidth; + double y1 = (left?.Y ?? 0) + NodeHeaderHeight / 2.0; + double x2 = right?.X ?? 200; + double y2 = (right?.Y ?? 0) + NodeHeaderHeight / 2.0; + return (x1, y1, x2, y2); + } + + private static string RelationshipLabel(DataModelRelationship relationship) => relationship.Kind switch + { + DataModelRelationshipKind.ExternalArchiveForeignKey => "ARCHIVE", + DataModelRelationshipKind.Draft => "DRAFT", + _ => "FK", + }; + + private static string RelationshipClass(DataModelRelationship relationship) => relationship.Kind switch + { + DataModelRelationshipKind.ExternalArchiveForeignKey => "archive", + DataModelRelationshipKind.Draft => "draft", + _ => "physical", + }; + + private static string RelationshipDash(DataModelRelationship relationship) => relationship.Kind switch + { + DataModelRelationshipKind.ExternalArchiveForeignKey => "6 4", + DataModelRelationshipKind.Draft => "2 4", + _ => "", + }; + + private static string RelationshipStroke(DataModelRelationship relationship, bool selected) + { + if (selected) + return "var(--accent-blue)"; + + return relationship.Kind switch + { + DataModelRelationshipKind.ExternalArchiveForeignKey => "var(--accent-magenta)", + DataModelRelationshipKind.Draft => "var(--text-muted)", + _ => "var(--accent-green)", + }; + } + + private static string Px(double value) => + value.ToString("F1", System.Globalization.CultureInfo.InvariantCulture); + + private static double NormalizeScale(double scale) + { + if (double.IsNaN(scale) || double.IsInfinity(scale) || scale <= 0) + return 1; + + return Math.Clamp(scale, 0.5, 2.0); + } + + public void Dispose() + { + try { _ = JS.InvokeVoidAsync("designerInterop.dispose"); } catch { } + _dotNetRef?.Dispose(); + } +} diff --git a/src/CSharpDB.Admin/Components/Tabs/CodeModulesTab.razor b/src/CSharpDB.Admin/Components/Tabs/CodeModulesTab.razor new file mode 100644 index 00000000..16f7be65 --- /dev/null +++ b/src/CSharpDB.Admin/Components/Tabs/CodeModulesTab.razor @@ -0,0 +1,253 @@ +@using CSharpDB.CodeModules +@inject CSharpDbCodeModuleClient CodeModules +@inject ToastService Toast + +
+
+
+ + / + code modules +
+ +
+ + + + + +
+ +
+ +
+ +
+ @GetBuildText() + @GetTrustText() +
+
+ +
+
+ @if (_modules.Count == 0) + { +
No C# code modules are stored in this database.
+ } + else + { +
+ + + + + + + + + + @foreach (CodeModuleSummary module in _modules) + { + + + + + + + } + +
NameKindOwnerHash
@module.Name@module.Kind@FormatOwner(module)@ShortHash(module.SourceHash)
+ } + + +
+ @if (_selected is null) + { +
Select a module to preview source.
+ } + else + { +
+
+
@_selected.Kind
+

@_selected.Name

+
+ @ShortHash(_selected.SourceHash ?? string.Empty) +
+ + + + + + + + +
Module Id@_selected.ModuleId
Type@(_selected.TypeName ?? "-")
Owner@((_selected.OwnerKind ?? "-") + "/" + (_selected.OwnerId ?? "-"))
Updated@_selected.UpdatedUtc?.LocalDateTime.ToString("g")
+ +
+

Source Preview

+ 
@_selected.Source
+
+ } + +
+

Diagnostics

+ @if (_lastBuild is null) + { +
Build has not run in this tab.
+ } + else if (_lastBuild.Diagnostics.Count == 0) + { +
No diagnostics.
+ } + else + { + + + + + + + + + + + @foreach (CodeModuleDiagnostic diagnostic in _lastBuild.Diagnostics) + { + + + + + + + } + +
SeverityModuleLineMessage
@diagnostic.Severity@(diagnostic.ModuleId ?? diagnostic.Path ?? "-")@diagnostic.Line:@diagnostic.Column@diagnostic.Code @diagnostic.Message
+ } +
+ + @if (_lastImport is not null) + { +
+

Last Import

+ + + @foreach (CodeModuleImportChange change in _lastImport.Changes) + { + + } + +
@change.Kind@change.ModuleId - @(change.Message ?? change.Path)
+
+ } +
+ + + +@code { + [Parameter, EditorRequired] public TabDescriptor Tab { get; set; } = default!; + + private List _modules = []; + private CodeModuleDefinition? _selected; + private string? _selectedModuleId; + private CodeModuleBuildResult? _lastBuild; + private CodeModuleTrustState? _trustState; + private CodeModuleImportResult? _lastImport; + private string _workspacePath = string.Empty; + + private bool CanTrust => _lastBuild?.Succeeded == true && _trustState?.IsTrusted != true; + + protected override async Task OnInitializedAsync() + { + _workspacePath = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments); + await RefreshAsync(); + } + + private async Task RefreshAsync() + { + _modules = (await CodeModules.ListAsync()).ToList(); + _trustState = await CodeModules.GetTrustStateAsync(); + if (_selectedModuleId is not null) + _selected = await CodeModules.GetAsync(_selectedModuleId); + if (_selected is null && _modules.Count > 0) + await SelectModuleAsync(_modules[0].ModuleId); + } + + private async Task SelectModuleAsync(string moduleId) + { + _selectedModuleId = moduleId; + _selected = await CodeModules.GetAsync(moduleId); + } + + private async Task BuildAsync() + { + _lastBuild = await CodeModules.BuildAsync(); + _trustState = await CodeModules.GetTrustStateAsync(_lastBuild.ModuleSetHash); + Toast.Show(_lastBuild.Succeeded ? "Code modules built successfully." : "Code module build failed."); + } + + private async Task TrustAsync() + { + await CodeModules.TrustAsync(); + _lastBuild = await CodeModules.BuildAsync(); + _trustState = await CodeModules.GetTrustStateAsync(_lastBuild.ModuleSetHash); + Toast.Show("Current C# code module build is trusted locally."); + } + + private async Task ExportAsync() + { + CodeModuleExportResult result = await CodeModules.ExportAsync(GetWorkspacePath()); + Toast.Show($"Exported {result.ModuleCount} code module(s)."); + } + + private async Task ImportAsync() + { + _lastImport = await CodeModules.ImportAsync(GetWorkspacePath()); + await RefreshAsync(); + Toast.Show(_lastImport.HasConflicts ? "Code module import completed with conflicts." : "Code module import completed."); + } + + private string GetWorkspacePath() + => string.IsNullOrWhiteSpace(_workspacePath) + ? Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments) + : _workspacePath; + + private string GetBuildText() + => _lastBuild is null ? "Not built" : _lastBuild.Succeeded ? "Build passed" : "Build failed"; + + private string GetBuildBadgeClass() + => _lastBuild is null + ? "callbacks-policy-badge" + : _lastBuild.Succeeded ? "callbacks-policy-badge allowed" : "callbacks-policy-badge denied"; + + private string GetTrustText() + => _trustState?.IsTrusted == true ? "Trusted" : "Untrusted"; + + private string GetTrustBadgeClass() + => _trustState?.IsTrusted == true ? "callbacks-policy-badge allowed" : "callbacks-policy-badge denied"; + + private static string ShortHash(string hash) + => string.IsNullOrWhiteSpace(hash) ? "-" : hash[..Math.Min(12, hash.Length)]; + + private static string FormatOwner(CodeModuleSummary module) + => string.IsNullOrWhiteSpace(module.OwnerKind) && string.IsNullOrWhiteSpace(module.OwnerId) + ? "-" + : $"{module.OwnerKind ?? "-"}:{module.OwnerId ?? "-"}"; +} diff --git a/src/CSharpDB.Admin/Components/Tabs/DataModelTab.razor b/src/CSharpDB.Admin/Components/Tabs/DataModelTab.razor new file mode 100644 index 00000000..8bcf6de1 --- /dev/null +++ b/src/CSharpDB.Admin/Components/Tabs/DataModelTab.razor @@ -0,0 +1,1270 @@ +@using CSharpDB.Admin.Models +@inject IDataModelService DataModels +@inject IDataModelDiagramService Diagrams +@inject TabManagerService TabManager +@inject ToastService Toast + +
+
+ +
+ + @if (_showAddSource) + { +
+ @foreach (DataModelSourceOption option in AvailableSourceOptions) + { + var source = option; + + } + @if (!AvailableSourceOptions.Any()) + { +
No remaining sources
+ } +
+ } +
+ + + + +
+ + @ZoomLabel + + + +
+ + +
+ + @if (_showLoadPanel) + { +
+ @if (!_savedDiagrams.Any()) + { +
No saved diagrams
+ } + else + { + @foreach (DataModelDiagramSummary diagram in _savedDiagrams) + { + var saved = diagram; + + } + } +
+ } +
+ + +
+ + +
+
+ @_state.Nodes.Count source@(_state.Nodes.Count == 1 ? "" : "s") + | + @_state.Relationships.Count relationship@(_state.Relationships.Count == 1 ? "" : "s") +
+
+ + @if (_showNewTablePanel) + { +
+ New Table + + + + + +
+ } + + @if (_error is not null) + { +
@_error
+ } + +
+
+ +
+ +
+
+ +@code { + private const string DefaultDiagramName = "Default Diagram"; + private const double MinZoom = 0.5; + private const double MaxZoom = 2.0; + private const double ZoomStep = 0.1; + + [Parameter] public TabDescriptor Tab { get; set; } = default!; + + private DataModelState _state = new(); + private IReadOnlyList _sourceOptions = []; + private IReadOnlyList _savedDiagrams = []; + private string? _selectedNodeName; + private string? _selectedRelationshipId; + private string _diagramName = ""; + private string _newTableName = ""; + private string _newTablePrimaryKeyName = "Id"; + private string _newTablePrimaryKeyType = "INTEGER"; + private string _renameTableName = ""; + private string _newColumnName = ""; + private string _newColumnType = "TEXT"; + private bool _newColumnNotNull; + private readonly Dictionary _columnRenameNames = new(StringComparer.OrdinalIgnoreCase); + private string _relationshipLeftTable = ""; + private string _relationshipLeftColumn = ""; + private string _relationshipRightTable = ""; + private string _relationshipRightColumn = ""; + private string _relationshipOnDelete = "RESTRICT"; + private bool _showAddSource; + private bool _showLoadPanel; + private bool _showNewTablePanel; + private bool _loading; + private string? _error; + + private IEnumerable AvailableSourceOptions => + _sourceOptions.Where(option => !_state.Nodes.Any(node => string.Equals(node.Name, option.Name, StringComparison.OrdinalIgnoreCase))); + + private DataModelNode? SelectedNode => + string.IsNullOrWhiteSpace(_selectedNodeName) + ? null + : _state.Nodes.FirstOrDefault(node => string.Equals(node.Name, _selectedNodeName, StringComparison.OrdinalIgnoreCase)); + + private DataModelRelationship? SelectedRelationship => + string.IsNullOrWhiteSpace(_selectedRelationshipId) + ? null + : _state.Relationships.FirstOrDefault(relationship => string.Equals(relationship.Id, _selectedRelationshipId, StringComparison.Ordinal)); + + private IReadOnlyList MutableTableNodes => + _state.Nodes + .Where(static node => node.Kind == DataModelNodeKind.Table) + .OrderBy(static node => node.Name, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + private IReadOnlyList RelationshipLeftColumns => + FindNode(_relationshipLeftTable)?.Columns.OrderBy(static column => column.Name, StringComparer.OrdinalIgnoreCase).ToArray() ?? []; + + private IReadOnlyList RelationshipRightColumns => + FindNode(_relationshipRightTable)?.Columns.OrderBy(static column => column.Name, StringComparer.OrdinalIgnoreCase).ToArray() ?? []; + + private IReadOnlyList AllWarnings => + _state.Warnings + .Concat(_state.Nodes.SelectMany(static node => node.Warnings.Select(warning => $"{node.Name}: {warning}"))) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + private string PendingPreviewSql => DataModels.BuildPendingOperationsPreview(_state); + + private double CurrentScale => NormalizeScale(_state.Scale); + + private string ZoomLabel => $"{Math.Round(CurrentScale * 100):0}%"; + + private bool CanStageNewTable => + !string.IsNullOrWhiteSpace(_newTableName) + && !string.IsNullOrWhiteSpace(_newTablePrimaryKeyName) + && !_state.Nodes.Any(node => string.Equals(node.Name, _newTableName.Trim(), StringComparison.OrdinalIgnoreCase)) + && !_sourceOptions.Any(option => string.Equals(option.Name, _newTableName.Trim(), StringComparison.OrdinalIgnoreCase)); + + private bool HasPendingDropForSelectedNode => + SelectedNode is not null && + _state.PendingOperations.Any(operation => + operation.Kind == DataModelPendingOperationKind.DropTable && + string.Equals(operation.TableName, SelectedNode.Name, StringComparison.OrdinalIgnoreCase)); + + private bool CanStageRenameSelectedTable => + SelectedNode is { Kind: DataModelNodeKind.Table, IsDraft: false } + && !HasPendingDropForSelectedNode + && !HasPendingTableRename(SelectedNode.Name) + && !string.IsNullOrWhiteSpace(_renameTableName) + && !string.Equals(_renameTableName.Trim(), SelectedNode.Name, StringComparison.OrdinalIgnoreCase) + && !_sourceOptions.Any(option => string.Equals(option.Name, _renameTableName.Trim(), StringComparison.OrdinalIgnoreCase)) + && !_state.Nodes.Any(node => string.Equals(node.Name, _renameTableName.Trim(), StringComparison.OrdinalIgnoreCase)); + + private bool CanStageAddColumn => + SelectedNode is { Kind: DataModelNodeKind.Table, IsDraft: false } + && !HasPendingDropForSelectedNode + && !HasPendingTableRename(SelectedNode.Name) + && !string.IsNullOrWhiteSpace(_newColumnName) + && !SelectedNode.Columns.Any(column => string.Equals(column.Name, _newColumnName.Trim(), StringComparison.OrdinalIgnoreCase)) + && !_state.PendingOperations.Any(operation => + operation.Kind == DataModelPendingOperationKind.AddColumn && + string.Equals(operation.TableName, SelectedNode.Name, StringComparison.OrdinalIgnoreCase) && + string.Equals(operation.ColumnName, _newColumnName.Trim(), StringComparison.OrdinalIgnoreCase)); + + private bool CanStageRelationship => + !string.IsNullOrWhiteSpace(_relationshipLeftTable) + && !string.IsNullOrWhiteSpace(_relationshipLeftColumn) + && !string.IsNullOrWhiteSpace(_relationshipRightTable) + && !string.IsNullOrWhiteSpace(_relationshipRightColumn) + && !string.Equals(_relationshipLeftTable, _relationshipRightTable, StringComparison.OrdinalIgnoreCase) + && FindNode(_relationshipLeftTable) is { Kind: DataModelNodeKind.Table } + && FindNode(_relationshipRightTable) is { Kind: DataModelNodeKind.Table } + && RelationshipLeftColumns.Any(column => string.Equals(column.Name, _relationshipLeftColumn, StringComparison.OrdinalIgnoreCase)) + && RelationshipRightColumns.Any(column => string.Equals(column.Name, _relationshipRightColumn, StringComparison.OrdinalIgnoreCase)) + && !_state.Relationships.Any(relationship => + string.Equals(relationship.LeftTable, _relationshipLeftTable, StringComparison.OrdinalIgnoreCase) && + string.Equals(relationship.LeftColumn, _relationshipLeftColumn, StringComparison.OrdinalIgnoreCase) && + string.Equals(relationship.RightTable, _relationshipRightTable, StringComparison.OrdinalIgnoreCase) && + string.Equals(relationship.RightColumn, _relationshipRightColumn, StringComparison.OrdinalIgnoreCase)); + + private bool HasPendingDropForSelectedRelationship => + SelectedRelationship is not null && + _state.PendingOperations.Any(operation => + operation.Kind == DataModelPendingOperationKind.DropForeignKey && + (string.Equals(operation.ConstraintName, SelectedRelationship.ConstraintName, StringComparison.OrdinalIgnoreCase) || + string.Equals(operation.ConstraintName, SelectedRelationship.Id, StringComparison.OrdinalIgnoreCase))); + + private bool UsesDefaultDiagram => + string.IsNullOrWhiteSpace(Tab?.InitialDataModelSourceName); + + protected override async Task OnInitializedAsync() + { + await LoadInitialAsync(); + } + + protected override void OnParametersSet() + { + if (Tab?.DataModelStateJson is not null && _state.Nodes.Count == 0) + { + try + { + DataModelState? saved = DataModelGraphBuilder.DeserializeState(Tab.DataModelStateJson); + if (saved is not null) + { + _state = saved; + NormalizeZoom(); + EnsureDefaultDiagramName(); + _diagramName = _state.DiagramName ?? _state.SavedLayoutName ?? ""; + } + } + catch + { + } + } + } + + private async Task LoadInitialAsync() + { + _loading = true; + _error = null; + try + { + _sourceOptions = await DataModels.GetSourceOptionsAsync(); + _savedDiagrams = await Diagrams.GetDiagramsAsync(); + if (Tab?.DataModelStateJson is not null) + { + DataModelState? saved = DataModelGraphBuilder.DeserializeState(Tab.DataModelStateJson); + if (saved is not null) + { + _state = saved; + NormalizeZoom(); + EnsureDefaultDiagramName(); + _diagramName = _state.DiagramName ?? _state.SavedLayoutName ?? ""; + SaveState(); + await PersistActiveDiagramAsync(); + } + } + else + { + DataModelState? defaultDiagram = UsesDefaultDiagram + ? await Diagrams.LoadDiagramAsync(DefaultDiagramName) + : null; + _state = defaultDiagram ?? await DataModels.BuildModelAsync(Tab?.InitialDataModelSourceName); + NormalizeZoom(); + EnsureDefaultDiagramName(); + _diagramName = _state.DiagramName ?? ""; + SaveState(); + await PersistActiveDiagramAsync(); + } + } + catch (Exception ex) + { + _error = $"Failed to load data model: {ex.Message}"; + } + finally + { + _loading = false; + } + } + + private async Task RefreshAsync() + { + _loading = true; + _error = null; + try + { + _sourceOptions = await DataModels.GetSourceOptionsAsync(); + if (!string.IsNullOrWhiteSpace(_state.DiagramName)) + { + DataModelState? saved = await Diagrams.LoadDiagramAsync(_state.DiagramName); + if (saved is not null) + { + _state = saved; + _diagramName = saved.DiagramName ?? _diagramName; + _selectedNodeName = null; + _selectedRelationshipId = null; + SaveState(); + return; + } + } + + DataModelState refreshed = await DataModels.BuildModelAsync(Tab?.InitialDataModelSourceName); + PreservePositions(refreshed); + refreshed.DiagramName = _state.DiagramName; + refreshed.SavedLayoutName = _state.SavedLayoutName; + refreshed.PendingOperations = _state.PendingOperations; + _state = refreshed; + EnsureDefaultDiagramName(); + SaveState(); + await PersistActiveDiagramAsync(); + } + catch (Exception ex) + { + _error = $"Failed to refresh data model: {ex.Message}"; + } + finally + { + _loading = false; + } + } + + private async Task LoadAllAsync() + { + _loading = true; + _error = null; + try + { + double currentScale = CurrentScale; + _state = await DataModels.BuildModelAsync(autoLayoutLimit: int.MaxValue); + _state.Scale = currentScale; + _state.DiagramName = UsesDefaultDiagram && string.IsNullOrWhiteSpace(_diagramName) + ? DefaultDiagramName + : string.IsNullOrWhiteSpace(_diagramName) ? null : _diagramName.Trim(); + _state.SavedLayoutName = _state.DiagramName; + _selectedNodeName = null; + _selectedRelationshipId = null; + SaveState(); + await PersistActiveDiagramAsync(); + } + catch (Exception ex) + { + _error = $"Failed to load full data model: {ex.Message}"; + } + finally + { + _loading = false; + } + } + + private async Task AddSourceAsync(string sourceName) + { + _showAddSource = false; + _loading = true; + _error = null; + try + { + DataModelState addition = await DataModels.BuildModelAsync(sourceName); + Merge(addition); + _selectedNodeName = sourceName; + _selectedRelationshipId = null; + SaveState(); + await PersistActiveDiagramAsync(); + } + catch (Exception ex) + { + _error = $"Failed to add source '{sourceName}': {ex.Message}"; + } + finally + { + _loading = false; + } + } + + private void Merge(DataModelState addition) + { + double offsetX = _state.Nodes.Count == 0 ? 0 : _state.Nodes.Max(static node => node.X) + 250; + foreach (DataModelNode node in addition.Nodes) + { + if (_state.Nodes.Any(existing => string.Equals(existing.Name, node.Name, StringComparison.OrdinalIgnoreCase))) + continue; + + if (offsetX > 0) + node.X += offsetX; + _state.Nodes.Add(node); + } + + foreach (DataModelRelationship relationship in addition.Relationships) + { + if (!_state.Relationships.Any(existing => string.Equals(existing.Id, relationship.Id, StringComparison.Ordinal))) + _state.Relationships.Add(relationship); + } + + foreach (string warning in addition.Warnings) + { + if (!_state.Warnings.Contains(warning, StringComparer.OrdinalIgnoreCase)) + _state.Warnings.Add(warning); + } + } + + private void PreservePositions(DataModelState refreshed) + { + refreshed.ViewportX = _state.ViewportX; + refreshed.ViewportY = _state.ViewportY; + refreshed.Scale = CurrentScale; + + foreach (DataModelNode node in refreshed.Nodes) + { + DataModelNode? existing = _state.Nodes.FirstOrDefault(current => string.Equals(current.Name, node.Name, StringComparison.OrdinalIgnoreCase)); + if (existing is null) + continue; + + node.X = existing.X; + node.Y = existing.Y; + node.IsCollapsed = existing.IsCollapsed; + } + } + + private async Task ClearCanvas() + { + string? diagramName = _state.DiagramName; + _state = new DataModelState + { + DiagramName = diagramName, + SavedLayoutName = diagramName, + }; + _selectedNodeName = null; + _selectedRelationshipId = null; + SaveState(); + await PersistActiveDiagramAsync(); + } + + private void SelectNode(string name) + { + _selectedNodeName = name; + _selectedRelationshipId = null; + _renameTableName = ""; + _newColumnName = ""; + } + + private void SelectRelationship(string id) + { + _selectedRelationshipId = id; + _selectedNodeName = null; + } + + private DataModelNode? FindNode(string? name) => + string.IsNullOrWhiteSpace(name) + ? null + : _state.Nodes.FirstOrDefault(node => string.Equals(node.Name, name, StringComparison.OrdinalIgnoreCase)); + + private string ColumnRenameKey(string tableName, string columnName) => $"{tableName}.{columnName}"; + + private string GetColumnRenameValue(string tableName, string columnName) => + _columnRenameNames.TryGetValue(ColumnRenameKey(tableName, columnName), out string? value) ? value : ""; + + private void SetColumnRenameValue(string tableName, string columnName, string? value) + { + string key = ColumnRenameKey(tableName, columnName); + if (string.IsNullOrWhiteSpace(value)) + _columnRenameNames.Remove(key); + else + _columnRenameNames[key] = value; + } + + private bool CanStageRenameColumn(DataModelNode node, DataModelColumn column) + { + string value = GetColumnRenameValue(node.Name, column.Name).Trim(); + return !string.IsNullOrWhiteSpace(value) + && !HasPendingTableRename(node.Name) + && !string.Equals(value, column.Name, StringComparison.OrdinalIgnoreCase) + && !node.Columns.Any(existing => string.Equals(existing.Name, value, StringComparison.OrdinalIgnoreCase)) + && !_state.PendingOperations.Any(operation => + operation.Kind == DataModelPendingOperationKind.RenameColumn && + string.Equals(operation.TableName, node.Name, StringComparison.OrdinalIgnoreCase) && + string.Equals(operation.ColumnName, column.Name, StringComparison.OrdinalIgnoreCase)); + } + + private bool HasPendingColumnDrop(string tableName, string columnName) => + _state.PendingOperations.Any(operation => + operation.Kind == DataModelPendingOperationKind.DropColumn && + string.Equals(operation.TableName, tableName, StringComparison.OrdinalIgnoreCase) && + string.Equals(operation.ColumnName, columnName, StringComparison.OrdinalIgnoreCase)); + + private bool HasPendingTableRename(string tableName) => + _state.PendingOperations.Any(operation => + operation.Kind == DataModelPendingOperationKind.RenameTable && + string.Equals(operation.TableName, tableName, StringComparison.OrdinalIgnoreCase)); + + private async Task RemoveNode(string name) + { + _state.Nodes.RemoveAll(node => string.Equals(node.Name, name, StringComparison.OrdinalIgnoreCase)); + _state.Relationships.RemoveAll(relationship => + string.Equals(relationship.LeftTable, name, StringComparison.OrdinalIgnoreCase) + || string.Equals(relationship.RightTable, name, StringComparison.OrdinalIgnoreCase)); + + if (string.Equals(_selectedNodeName, name, StringComparison.OrdinalIgnoreCase)) + _selectedNodeName = null; + if (_selectedRelationshipId is not null && !_state.Relationships.Any(relationship => relationship.Id == _selectedRelationshipId)) + _selectedRelationshipId = null; + + SaveState(); + await PersistActiveDiagramAsync(); + } + + private async Task OnNodeMoved(DataModelNodeMove move) + { + DataModelNode? node = FindNode(move.NodeName); + if (node is not null) + { + node.X = move.X; + node.Y = move.Y; + } + + EnsureDefaultDiagramName(); + SaveState(); + await PersistActiveDiagramAsync(); + } + + private async Task SaveDiagramAsync() + { + try + { + await Diagrams.SaveDiagramAsync(_diagramName, _state); + _diagramName = _state.DiagramName ?? _diagramName.Trim(); + await RefreshSavedDiagramsAsync(); + SaveState(); + Toast.Success($"Data model diagram '{_diagramName}' saved."); + } + catch (Exception ex) + { + Toast.Error($"Failed to save data model diagram: {ex.Message}"); + } + } + + private async Task LoadDiagramAsync(string name) + { + _showLoadPanel = false; + try + { + DataModelState? loaded = await Diagrams.LoadDiagramAsync(name); + if (loaded is null) + { + Toast.Warning("Saved data model diagram was not found."); + return; + } + + _state = loaded; + NormalizeZoom(); + _diagramName = loaded.DiagramName ?? name; + _selectedNodeName = null; + _selectedRelationshipId = null; + SaveState(); + Toast.Info($"Data model diagram '{_diagramName}' loaded."); + } + catch (Exception ex) + { + Toast.Error($"Failed to load data model diagram: {ex.Message}"); + } + } + + private async Task DeleteActiveDiagramAsync() + { + if (string.IsNullOrWhiteSpace(_state.DiagramName)) + return; + + string name = _state.DiagramName; + try + { + await Diagrams.DeleteDiagramAsync(name); + await RefreshSavedDiagramsAsync(); + _state.DiagramName = null; + _state.SavedLayoutName = null; + _diagramName = ""; + SaveState(); + Toast.Info($"Data model diagram '{name}' deleted."); + } + catch (Exception ex) + { + Toast.Error($"Failed to delete data model diagram: {ex.Message}"); + } + } + + private async Task RefreshSavedDiagramsAsync() + { + try { _savedDiagrams = await Diagrams.GetDiagramsAsync(); } + catch { _savedDiagrams = []; } + } + + private void OpenInQueryDesigner() + { + QueryDesignerState designerState = DataModels.ToQueryDesignerState(_state); + TabManager.OpenQueryDesignerTab(designerState); + } + + private void ToggleAddSourcePanel() + { + _showAddSource = !_showAddSource; + _showLoadPanel = false; + _showNewTablePanel = false; + } + + private void ToggleLoadPanel() + { + _showLoadPanel = !_showLoadPanel; + _showAddSource = false; + _showNewTablePanel = false; + } + + private void ToggleNewTablePanel() + { + _showNewTablePanel = !_showNewTablePanel; + _showAddSource = false; + _showLoadPanel = false; + } + + private async Task StageNewTableAsync() + { + string tableName = _newTableName.Trim(); + string pkName = _newTablePrimaryKeyName.Trim(); + var columns = new List + { + new() + { + Name = pkName, + TypeLabel = _newTablePrimaryKeyType, + IsPrimaryKey = true, + IsIdentity = string.Equals(_newTablePrimaryKeyType, "INTEGER", StringComparison.OrdinalIgnoreCase), + Nullable = false, + }, + }; + + var node = new DataModelNode + { + Name = tableName, + Kind = DataModelNodeKind.Table, + IsDraft = true, + X = _state.Nodes.Count == 0 ? 20 : _state.Nodes.Max(static node => node.X) + 250, + Y = _state.Nodes.Count == 0 ? 20 : _state.Nodes.Min(static node => node.Y), + Columns = columns, + Warnings = ["Pending create table"], + }; + + _state.Nodes.Add(node); + _state.PendingOperations.Add(new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.CreateTable, + TableName = tableName, + Columns = columns, + Description = $"Create table {tableName}", + }); + + _selectedNodeName = tableName; + _selectedRelationshipId = null; + _newTableName = ""; + _newTablePrimaryKeyName = "Id"; + _newTablePrimaryKeyType = "INTEGER"; + _showNewTablePanel = false; + SaveState(); + await PersistActiveDiagramAsync(); + } + + private async Task StageDropSelectedTableAsync() + { + if (SelectedNode is null || SelectedNode.Kind != DataModelNodeKind.Table || SelectedNode.IsDraft) + return; + + if (HasPendingDropForSelectedNode || HasPendingTableRename(SelectedNode.Name)) + return; + + _state.PendingOperations.Add(new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.DropTable, + TableName = SelectedNode.Name, + Description = $"Drop table {SelectedNode.Name}", + }); + SaveState(); + await PersistActiveDiagramAsync(); + } + + private async Task StageRenameSelectedTableAsync() + { + if (!CanStageRenameSelectedTable || SelectedNode is null) + return; + + string newName = _renameTableName.Trim(); + _state.PendingOperations.Add(new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.RenameTable, + TableName = SelectedNode.Name, + NewTableName = newName, + Description = $"Rename table {SelectedNode.Name} to {newName}", + }); + _renameTableName = ""; + SaveState(); + await PersistActiveDiagramAsync(); + } + + private async Task StageAddColumnAsync() + { + if (!CanStageAddColumn || SelectedNode is null) + return; + + string columnName = _newColumnName.Trim(); + _state.PendingOperations.Add(new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.AddColumn, + TableName = SelectedNode.Name, + ColumnName = columnName, + ColumnType = _newColumnType, + NotNull = _newColumnNotNull, + Description = $"Add column {SelectedNode.Name}.{columnName}", + }); + _newColumnName = ""; + _newColumnType = "TEXT"; + _newColumnNotNull = false; + SaveState(); + await PersistActiveDiagramAsync(); + } + + private async Task StageDropColumnAsync(string tableName, string columnName) + { + if (HasPendingTableRename(tableName) || HasPendingColumnDrop(tableName, columnName)) + return; + + _state.PendingOperations.Add(new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.DropColumn, + TableName = tableName, + ColumnName = columnName, + Description = $"Drop column {tableName}.{columnName}", + }); + SaveState(); + await PersistActiveDiagramAsync(); + } + + private async Task StageRenameColumnAsync(string tableName, string columnName) + { + DataModelNode? node = FindNode(tableName); + DataModelColumn? column = node?.Columns.FirstOrDefault(candidate => string.Equals(candidate.Name, columnName, StringComparison.OrdinalIgnoreCase)); + if (node is null || column is null || !CanStageRenameColumn(node, column)) + return; + + string newName = GetColumnRenameValue(tableName, columnName).Trim(); + _state.PendingOperations.Add(new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.RenameColumn, + TableName = tableName, + ColumnName = columnName, + NewColumnName = newName, + Description = $"Rename column {tableName}.{columnName} to {newName}", + }); + _columnRenameNames.Remove(ColumnRenameKey(tableName, columnName)); + SaveState(); + await PersistActiveDiagramAsync(); + } + + private async Task StageRelationshipAsync() + { + if (!CanStageRelationship) + return; + + string relationshipId = Guid.NewGuid().ToString("N"); + var relationship = new DataModelRelationship + { + Id = relationshipId, + LeftTable = _relationshipLeftTable, + LeftColumn = _relationshipLeftColumn, + RightTable = _relationshipRightTable, + RightColumn = _relationshipRightColumn, + Kind = DataModelRelationshipKind.Draft, + OnDelete = _relationshipOnDelete, + }; + + _state.Relationships.Add(relationship); + _state.PendingOperations.Add(new DataModelPendingOperation + { + Id = relationshipId, + Kind = DataModelPendingOperationKind.AddForeignKey, + TableName = _relationshipLeftTable, + ColumnName = _relationshipLeftColumn, + ReferencedTableName = _relationshipRightTable, + ReferencedColumnName = _relationshipRightColumn, + OnDelete = _relationshipOnDelete, + Description = $"Add relationship {_relationshipLeftTable}.{_relationshipLeftColumn} -> {_relationshipRightTable}.{_relationshipRightColumn}", + }); + + _selectedRelationshipId = relationshipId; + _selectedNodeName = null; + _relationshipLeftTable = ""; + _relationshipLeftColumn = ""; + _relationshipRightTable = ""; + _relationshipRightColumn = ""; + _relationshipOnDelete = "RESTRICT"; + SaveState(); + await PersistActiveDiagramAsync(); + } + + private async Task StageDropSelectedRelationshipAsync() + { + if (SelectedRelationship is null || SelectedRelationship.Kind == DataModelRelationshipKind.ExternalArchiveForeignKey) + return; + + DataModelRelationship relationship = SelectedRelationship; + if (relationship.Kind == DataModelRelationshipKind.Draft) + { + _state.Relationships.Remove(relationship); + _state.PendingOperations.RemoveAll(operation => string.Equals(operation.Id, relationship.Id, StringComparison.Ordinal)); + _selectedRelationshipId = null; + SaveState(); + await PersistActiveDiagramAsync(); + return; + } + + string constraintName = relationship.ConstraintName ?? relationship.Id; + if (HasPendingDropForSelectedRelationship) + return; + + _state.PendingOperations.Add(new DataModelPendingOperation + { + Kind = DataModelPendingOperationKind.DropForeignKey, + TableName = relationship.LeftTable, + ConstraintName = constraintName, + Description = $"Drop relationship {constraintName}", + }); + SaveState(); + await PersistActiveDiagramAsync(); + } + + private async Task RemovePendingOperationAsync(string operationId) + { + DataModelPendingOperation? operation = _state.PendingOperations.FirstOrDefault(item => string.Equals(item.Id, operationId, StringComparison.Ordinal)); + if (operation is null) + return; + + _state.PendingOperations.Remove(operation); + if (operation.Kind == DataModelPendingOperationKind.CreateTable) + { + _state.Nodes.RemoveAll(node => node.IsDraft && string.Equals(node.Name, operation.TableName, StringComparison.OrdinalIgnoreCase)); + if (string.Equals(_selectedNodeName, operation.TableName, StringComparison.OrdinalIgnoreCase)) + _selectedNodeName = null; + } + else if (operation.Kind == DataModelPendingOperationKind.AddForeignKey) + { + _state.Relationships.RemoveAll(relationship => string.Equals(relationship.Id, operation.Id, StringComparison.Ordinal)); + if (string.Equals(_selectedRelationshipId, operation.Id, StringComparison.Ordinal)) + _selectedRelationshipId = null; + } + + SaveState(); + await PersistActiveDiagramAsync(); + } + + private async Task ApplyPendingChangesAsync() + { + _loading = true; + _error = null; + try + { + DataModelApplyResult result = await Diagrams.ApplyPendingOperationsAsync(_state); + Toast.Success($"Applied {result.Messages.Count} diagram change{(result.Messages.Count == 1 ? "" : "s")}."); + await RefreshAsync(); + } + catch (Exception ex) + { + _error = $"Failed to apply diagram changes: {ex.Message}"; + Toast.Error(_error); + } + finally + { + _loading = false; + } + } + + private Task ZoomIn() => SetZoom(CurrentScale + ZoomStep); + + private Task ZoomOut() => SetZoom(CurrentScale - ZoomStep); + + private Task ResetZoom() => SetZoom(1); + + private async Task SetZoom(double scale) + { + _state.Scale = Math.Round(NormalizeScale(scale), 2); + SaveState(); + await PersistActiveDiagramAsync(); + } + + private void NormalizeZoom() + { + _state.Scale = Math.Round(CurrentScale, 2); + } + + private void SaveState() + { + NormalizeZoom(); + if (Tab is not null) + Tab.DataModelStateJson = DataModelGraphBuilder.SerializeState(_state); + } + + private async Task PersistActiveDiagramAsync() + { + EnsureDefaultDiagramName(); + if (string.IsNullOrWhiteSpace(_state.DiagramName)) + return; + + try + { + await Diagrams.SaveDiagramAsync(_state.DiagramName, _state); + await RefreshSavedDiagramsAsync(); + } + catch (Exception ex) + { + Toast.Warning($"Diagram changes were not saved: {ex.Message}"); + } + } + + private void EnsureDefaultDiagramName() + { + if (!UsesDefaultDiagram || !string.IsNullOrWhiteSpace(_state.DiagramName)) + return; + + _state.DiagramName = DefaultDiagramName; + _state.SavedLayoutName = DefaultDiagramName; + _diagramName = DefaultDiagramName; + } + + private static string RelationshipKindLabel(DataModelRelationshipKind kind) => kind switch + { + DataModelRelationshipKind.ExternalArchiveForeignKey => "Archive relationship", + DataModelRelationshipKind.Draft => "Draft relationship", + _ => "Foreign key", + }; + + private static double NormalizeScale(double scale) + { + if (double.IsNaN(scale) || double.IsInfinity(scale) || scale <= 0) + return 1; + + return Math.Clamp(scale, MinZoom, MaxZoom); + } +} diff --git a/src/CSharpDB.Admin/Components/Tabs/QueryTab.razor b/src/CSharpDB.Admin/Components/Tabs/QueryTab.razor index f7db3340..bf4ed7bd 100644 --- a/src/CSharpDB.Admin/Components/Tabs/QueryTab.razor +++ b/src/CSharpDB.Admin/Components/Tabs/QueryTab.razor @@ -18,9 +18,15 @@
@if (_mode == QueryMode.Sql) { - + @if (_loading) + { + + }