From 10f30ba010708146e707f55e528475f11b1da3ee Mon Sep 17 00:00:00 2001 From: Lloyd Izard <76954858+LOCEANlloydizard@users.noreply.github.com> Date: Wed, 17 Jun 2026 22:10:20 -0700 Subject: [PATCH 1/3] Add Windows background service scripts --- README.md | 52 +++++++++++++++++++ .../deploy_prefect_server.windows.ps1 | 37 +++++++++++++ .../deploy_prefect_worker.windows.ps1 | 37 +++++++++++++ 3 files changed, 126 insertions(+) create mode 100644 src/echodataflow/services/deploy_prefect_server.windows.ps1 create mode 100644 src/echodataflow/services/deploy_prefect_worker.windows.ps1 diff --git a/README.md b/README.md index fef6dac..f2773d2 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,59 @@ Notes: - `database is locked` usually means SQLite write contention, not corruption. - For heavier multi-flow usage, move Prefect server DB to Postgres. +## Running Local Prefect Services on Windows (Task Scheduler) +To run a local Prefect server and worker as background services on Windows, you can +use PowerShell with Windows Task Scheduler and the provided script templates: + +- `src/echodataflow/services/deploy_prefect_server.windows.ps1` +- `src/echodataflow/services/deploy_prefect_worker.windows.ps1` + +1. Copy and customize the service environment file: + ```powershell + New-Item -ItemType Directory -Force "$HOME\.config\echodataflow" + New-Item -ItemType Directory -Force "$HOME\.local\var\log\echodataflow" + + Copy-Item src\echodataflow\services\services.env.example_local ` + "$HOME\.config\echodataflow\services.env" + ``` + +2. Edit `$HOME\.config\echodataflow\services.env` as needed: + - Adjust `ECHODATAFLOW_ENV` + - Adjust `ECHODATAFLOW_HOME` + - Adjust `ECHODATAFLOW_WORKDIR` + - Adjust `ECHODATAFLOW_LOG_DIR` + - Adjust `MAMBA_BIN` + - Adjust `PREFECT_POOL` + - Adjust `PREFECT_API_URL` + +3. Register the scheduled tasks: + ```powershell + schtasks /Create /TN "echodataflow-prefect-server" /SC ONLOGON /TR "powershell.exe -ExecutionPolicy Bypass -File C:\path\to\echodataflow\src\echodataflow\services\deploy_prefect_server.windows.ps1" + + schtasks /Create /TN "echodataflow-prefect-worker" /SC ONLOGON /TR "powershell.exe -ExecutionPolicy Bypass -File C:\path\to\echodataflow\src\echodataflow\services\deploy_prefect_worker.windows.ps1" + ``` + +4. Start the tasks: + ```powershell + schtasks /Run /TN "echodataflow-prefect-server" + schtasks /Run /TN "echodataflow-prefect-worker" + ``` + +5. Check task status: + ```powershell + schtasks /Query /TN "echodataflow-prefect-server" + schtasks /Query /TN "echodataflow-prefect-worker" + ``` + +6. Stop and delete the tasks: + ```powershell + schtasks /End /TN "echodataflow-prefect-worker" + schtasks /End /TN "echodataflow-prefect-server" + + schtasks /Delete /TN "echodataflow-prefect-worker" + schtasks /Delete /TN "echodataflow-prefect-server" + ``` ## License diff --git a/src/echodataflow/services/deploy_prefect_server.windows.ps1 b/src/echodataflow/services/deploy_prefect_server.windows.ps1 new file mode 100644 index 0000000..47dbe5e --- /dev/null +++ b/src/echodataflow/services/deploy_prefect_server.windows.ps1 @@ -0,0 +1,37 @@ +$EnvFile = "$HOME\.config\echodataflow\services.env" + +if (-not (Test-Path $EnvFile)) { + Write-Error "Missing services env file: $EnvFile" + exit 1 +} + +Get-Content $EnvFile | ForEach-Object { + if ($_ -match '^\s*([^#][^=]+)=(.*)$') { + $Name = $matches[1].Trim() + $Value = $matches[2].Trim() + Set-Item -Path "Env:$Name" -Value $Value + } +} + +$RequiredVars = @( + "ECHODATAFLOW_ENV", + "ECHODATAFLOW_WORKDIR", + "ECHODATAFLOW_LOG_DIR", + "MAMBA_BIN" +) + +foreach ($Var in $RequiredVars) { + if (-not (Get-Item -Path "Env:$Var" -ErrorAction SilentlyContinue)) { + Write-Error "Missing required environment variable: $Var" + exit 1 + } +} + +New-Item -ItemType Directory -Force -Path $env:ECHODATAFLOW_LOG_DIR | Out-Null + +Set-Location $env:ECHODATAFLOW_WORKDIR + +& $env:MAMBA_BIN run -n $env:ECHODATAFLOW_ENV ` + prefect server start ` + --host 127.0.0.1 ` + --port 4200 \ No newline at end of file diff --git a/src/echodataflow/services/deploy_prefect_worker.windows.ps1 b/src/echodataflow/services/deploy_prefect_worker.windows.ps1 new file mode 100644 index 0000000..c48a8eb --- /dev/null +++ b/src/echodataflow/services/deploy_prefect_worker.windows.ps1 @@ -0,0 +1,37 @@ +$EnvFile = "$HOME\.config\echodataflow\services.env" + +if (-not (Test-Path $EnvFile)) { + Write-Error "Missing services env file: $EnvFile" + exit 1 +} + +Get-Content $EnvFile | ForEach-Object { + if ($_ -match '^\s*([^#][^=]+)=(.*)$') { + $Name = $matches[1].Trim() + $Value = $matches[2].Trim() + Set-Item -Path "Env:$Name" -Value $Value + } +} + +$RequiredVars = @( + "ECHODATAFLOW_ENV", + "ECHODATAFLOW_WORKDIR", + "ECHODATAFLOW_LOG_DIR", + "MAMBA_BIN", + "PREFECT_POOL" +) + +foreach ($Var in $RequiredVars) { + if (-not (Get-Item -Path "Env:$Var" -ErrorAction SilentlyContinue)) { + Write-Error "Missing required environment variable: $Var" + exit 1 + } +} + +New-Item -ItemType Directory -Force -Path $env:ECHODATAFLOW_LOG_DIR | Out-Null + +Set-Location $env:ECHODATAFLOW_WORKDIR + +& $env:MAMBA_BIN run -n $env:ECHODATAFLOW_ENV ` + prefect worker start ` + --pool $env:PREFECT_POOL \ No newline at end of file From 3c536588e5a847d1511ff2f840a6250a3036ad13 Mon Sep 17 00:00:00 2001 From: Lloyd Izard <76954858+LOCEANlloydizard@users.noreply.github.com> Date: Thu, 18 Jun 2026 14:51:56 -0700 Subject: [PATCH 2/3] Add Windows Task Scheduler templates and update service docs --- README.md | 55 +++++++++++++++---- .../deploy_prefect_server.windows.task.xml | 41 ++++++++++++++ .../deploy_prefect_worker.windows.task.xml | 42 ++++++++++++++ 3 files changed, 127 insertions(+), 11 deletions(-) create mode 100644 src/echodataflow/services/deploy_prefect_server.windows.task.xml create mode 100644 src/echodataflow/services/deploy_prefect_worker.windows.task.xml diff --git a/README.md b/README.md index f2773d2..43d7ed7 100644 --- a/README.md +++ b/README.md @@ -174,10 +174,12 @@ Notes: ## Running Local Prefect Services on Windows (Task Scheduler) To run a local Prefect server and worker as background services on Windows, you can -use PowerShell with Windows Task Scheduler and the provided script templates: +use PowerShell with Windows Task Scheduler and the provided templates: - `src/echodataflow/services/deploy_prefect_server.windows.ps1` - `src/echodataflow/services/deploy_prefect_worker.windows.ps1` +- `src/echodataflow/services/deploy_prefect_server.windows.task.xml` +- `src/echodataflow/services/deploy_prefect_worker.windows.task.xml` 1. Copy and customize the service environment file: ```powershell @@ -197,32 +199,63 @@ use PowerShell with Windows Task Scheduler and the provided script templates: - Adjust `PREFECT_POOL` - Adjust `PREFECT_API_URL` -3. Register the scheduled tasks: +3. Copy and customize the Task Scheduler XML templates: ```powershell - schtasks /Create /TN "echodataflow-prefect-server" /SC ONLOGON /TR "powershell.exe -ExecutionPolicy Bypass -File C:\path\to\echodataflow\src\echodataflow\services\deploy_prefect_server.windows.ps1" + Copy-Item src\echodataflow\services\deploy_prefect_server.windows.task.xml ` + "$HOME\.config\echodataflow\prefect-server.task.xml" - schtasks /Create /TN "echodataflow-prefect-worker" /SC ONLOGON /TR "powershell.exe -ExecutionPolicy Bypass -File C:\path\to\echodataflow\src\echodataflow\services\deploy_prefect_worker.windows.ps1" + Copy-Item src\echodataflow\services\deploy_prefect_worker.windows.task.xml ` + "$HOME\.config\echodataflow\prefect-worker.task.xml" ``` -4. Start the tasks: +4. Edit both copied XML files and replace: + + ```text + C:\path\to\echodataflow + ``` + + with the absolute path to your local Echodataflow repository. + +5. Register the scheduled tasks: + ```powershell + schtasks /Create /TN "echodataflow-prefect-server" ` + /XML "$HOME\.config\echodataflow\prefect-server.task.xml" /F + + schtasks /Create /TN "echodataflow-prefect-worker" ` + /XML "$HOME\.config\echodataflow\prefect-worker.task.xml" /F + ``` + +6. Start the tasks: ```powershell schtasks /Run /TN "echodataflow-prefect-server" + Start-Sleep -Seconds 10 schtasks /Run /TN "echodataflow-prefect-worker" ``` -5. Check task status: +7. Check task status: ```powershell - schtasks /Query /TN "echodataflow-prefect-server" - schtasks /Query /TN "echodataflow-prefect-worker" + schtasks /Query /TN "echodataflow-prefect-server" /V /FO LIST + schtasks /Query /TN "echodataflow-prefect-worker" /V /FO LIST + ``` + +8. Verify the local Prefect server: + + Open: + + ```text + http://127.0.0.1:4200 ``` -6. Stop and delete the tasks: + The Prefect dashboard should load, and the worker should appear online under + **Work Pools**. + +9. To stop and delete the tasks: ```powershell schtasks /End /TN "echodataflow-prefect-worker" schtasks /End /TN "echodataflow-prefect-server" - schtasks /Delete /TN "echodataflow-prefect-worker" - schtasks /Delete /TN "echodataflow-prefect-server" + schtasks /Delete /TN "echodataflow-prefect-worker" /F + schtasks /Delete /TN "echodataflow-prefect-server" /F ``` ## License diff --git a/src/echodataflow/services/deploy_prefect_server.windows.task.xml b/src/echodataflow/services/deploy_prefect_server.windows.task.xml new file mode 100644 index 0000000..143aa03 --- /dev/null +++ b/src/echodataflow/services/deploy_prefect_server.windows.task.xml @@ -0,0 +1,41 @@ + + + + Run the local Echodataflow Prefect server at user logon. + + + + true + + + + + InteractiveToken + LeastPrivilege + + + + IgnoreNew + false + false + true + true + false + true + true + false + PT0S + 7 + + PT1M + 999 + + + + + powershell.exe + -NoProfile -ExecutionPolicy Bypass -File "C:\path\to\echodataflow\src\echodataflow\services\deploy_prefect_server.windows.ps1" + C:\path\to\echodataflow + + + \ No newline at end of file diff --git a/src/echodataflow/services/deploy_prefect_worker.windows.task.xml b/src/echodataflow/services/deploy_prefect_worker.windows.task.xml new file mode 100644 index 0000000..2206dea --- /dev/null +++ b/src/echodataflow/services/deploy_prefect_worker.windows.task.xml @@ -0,0 +1,42 @@ + + + + Run the local Echodataflow Prefect worker at user logon. + + + + true + PT1M + + + + + InteractiveToken + LeastPrivilege + + + + IgnoreNew + false + false + true + true + false + true + true + false + PT0S + 7 + + PT1M + 999 + + + + + powershell.exe + -NoProfile -ExecutionPolicy Bypass -File "C:\path\to\echodataflow\src\echodataflow\services\deploy_prefect_worker.windows.ps1" + C:\path\to\echodataflow + + + \ No newline at end of file From 4fa52b0cec407530249be8a39b82a9097488ebb4 Mon Sep 17 00:00:00 2001 From: Wu-Jung Lee Date: Wed, 24 Jun 2026 21:52:28 +0800 Subject: [PATCH 3/3] add powershell code into .xml directly --- README.md | 12 +--- .../deploy_prefect_server.windows.task.xml | 60 ++++++++++++++++++- .../deploy_prefect_worker.windows.task.xml | 60 ++++++++++++++++++- 3 files changed, 117 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 43d7ed7..feaac6f 100644 --- a/README.md +++ b/README.md @@ -176,8 +176,6 @@ Notes: To run a local Prefect server and worker as background services on Windows, you can use PowerShell with Windows Task Scheduler and the provided templates: -- `src/echodataflow/services/deploy_prefect_server.windows.ps1` -- `src/echodataflow/services/deploy_prefect_worker.windows.ps1` - `src/echodataflow/services/deploy_prefect_server.windows.task.xml` - `src/echodataflow/services/deploy_prefect_worker.windows.task.xml` @@ -208,13 +206,9 @@ use PowerShell with Windows Task Scheduler and the provided templates: "$HOME\.config\echodataflow\prefect-worker.task.xml" ``` -4. Edit both copied XML files and replace: - - ```text - C:\path\to\echodataflow - ``` - - with the absolute path to your local Echodataflow repository. +4. Both XML templates are self-contained and read their runtime configuration + from `$HOME\.config\echodataflow\services.env`, so they do not need separate + `deploy_prefect_*.windows.ps1` files or hard-coded repository script paths. 5. Register the scheduled tasks: ```powershell diff --git a/src/echodataflow/services/deploy_prefect_server.windows.task.xml b/src/echodataflow/services/deploy_prefect_server.windows.task.xml index 143aa03..d400b71 100644 --- a/src/echodataflow/services/deploy_prefect_server.windows.task.xml +++ b/src/echodataflow/services/deploy_prefect_server.windows.task.xml @@ -1,19 +1,26 @@ - + + Run the local Echodataflow Prefect server at user logon. + + true + + InteractiveToken LeastPrivilege + + IgnoreNew false @@ -31,11 +38,58 @@ 999 + + powershell.exe - -NoProfile -ExecutionPolicy Bypass -File "C:\path\to\echodataflow\src\echodataflow\services\deploy_prefect_server.windows.ps1" - C:\path\to\echodataflow + + + \ No newline at end of file diff --git a/src/echodataflow/services/deploy_prefect_worker.windows.task.xml b/src/echodataflow/services/deploy_prefect_worker.windows.task.xml index 2206dea..b6ec83e 100644 --- a/src/echodataflow/services/deploy_prefect_worker.windows.task.xml +++ b/src/echodataflow/services/deploy_prefect_worker.windows.task.xml @@ -1,20 +1,27 @@ - + + Run the local Echodataflow Prefect worker at user logon. + + true PT1M + + InteractiveToken LeastPrivilege + + IgnoreNew false @@ -32,11 +39,58 @@ 999 + + powershell.exe - -NoProfile -ExecutionPolicy Bypass -File "C:\path\to\echodataflow\src\echodataflow\services\deploy_prefect_worker.windows.ps1" - C:\path\to\echodataflow + + + \ No newline at end of file