diff --git a/.env.example b/.env.example index 75d1078..3c35491 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,9 @@ TIME_ZONE=UTC # Leave DATABASE_URL unset to use the default SQLite bootstrap. # Uncomment and adjust this value to develop against PostgreSQL. # DATABASE_URL=postgresql://postgres:postgres@127.0.0.1:5432/deep_workflow +# For hosted Neon + Vercel, keep DATABASE_URL as the runtime URL and optionally +# add DATABASE_ADMIN_URL as the direct admin URL for migrations and restore work. +# DATABASE_ADMIN_URL=postgresql://postgres:postgres@127.0.0.1:5432/deep_workflow # Optional tuning DJANGO_DB_CONN_MAX_AGE=60 diff --git a/AGENTS.md b/AGENTS.md index b6fa207..e34d0e2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,7 +10,7 @@ ## Technical Details -- Follow the roadmap in `plan.md`: Django, PostgreSQL, Django templates with HTMX and Alpine.js, Tailwind CSS, and Vercel for hosting. +- Follow the roadmap in `plan.md`: Django, PostgreSQL with Neon for hosted databases, Django templates with HTMX and Alpine.js, Tailwind CSS, and Vercel for hosting. - Keep the timer server-backed via timestamps so refreshes and device switching do not lose session state. - Prefer server-rendered flows and simple abstractions; only add client-side behavior where it materially improves the experience. - Cover core workflow logic and persistence with automated tests, and use integration or end-to-end tests for critical user flows. diff --git a/README.md b/README.md index 610987e..98bb811 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ The MVP is successful when: To keep the first version simple and fast to iterate on, the recommended stack is: - Django -- PostgreSQL +- PostgreSQL, with Neon as the recommended hosted provider - Django templates with HTMX and Alpine.js - Tailwind CSS - Vercel for production hosting and pull request preview deployments @@ -87,7 +87,7 @@ The implementation should land in small, reviewable PRs: This repository now includes the foundation plus roadmap slices 1 through 9: - Django project and `core` app wiring -- environment-based settings with `DATABASE_URL` support for PostgreSQL +- environment-based settings with `DATABASE_URL` support for PostgreSQL and Neon - login/logout with protected app routes - per-user timezone and default session duration settings - daily sheet and work session domain models with migrations, admin registration, and tests @@ -97,7 +97,7 @@ This repository now includes the foundation plus roadmap slices 1 through 9: - daily and weekly progress summaries with simple streak and completion indicators - mobile-first layout polish for the daily sheet, navigation, and touch targets - a manifest, service worker, and installable PWA shell -- production-hardened hosted settings, Vercel deployment config, readiness endpoints, and request-ID-aware logging +- production-hardened hosted settings, Vercel deployment config, readiness endpoints, request-ID-aware logging, and separate runtime/admin database URL support - PostgreSQL backup and restore documentation for hosted operations - Ruff linting/formatting, pytest-based tests, and GitHub Actions CI @@ -143,19 +143,37 @@ The repository now includes the files needed to host the app on Vercel productio - `wsgi.py` exposes the Django WSGI app through Vercel's Python runtime - `.python-version` pins Vercel to Python 3.12, matching the project's supported local Python 3.12.x runtime - `vercel.json` sets the Vercel build command and rewrites -- `scripts/vercel-build.sh` requires `DJANGO_SECRET_KEY` and `DATABASE_URL` for hosted builds, verifies database readiness, always runs `collectstatic`, and only runs migrations when `VERCEL_RUN_MIGRATIONS=1` +- `scripts/vercel-build.sh` requires `DJANGO_SECRET_KEY` and the runtime `DATABASE_URL` for hosted builds, verifies runtime database readiness, always runs `collectstatic`, and uses `DATABASE_ADMIN_URL` for migration-time checks and migrations when you provide it - hosted settings derive trusted hosts and CSRF origins from `APP_BASE_URL` plus Vercel's runtime URLs, then enable HTTPS redirects, secure cookies, conservative HSTS defaults, WhiteNoise static serving, and request-ID-aware logging +### Recommended Neon setup + +This repository's default hosted database path is **Neon PostgreSQL + Vercel**. + +The simplest safe setup is: + +1. Keep the Neon `main` branch for Production. +2. Create a Neon `preview` branch from `main` for all Vercel Preview deployments. +3. In Neon, open **Connect** for each branch and copy both connection strings: + - the **pooled** connection string for app runtime traffic + - the **direct** connection string for migrations, `pg_dump`, `pg_restore`, and one-off admin work +4. In Vercel, set: + - `DATABASE_URL` to the pooled connection string + - `DATABASE_ADMIN_URL` to the direct connection string + +This keeps the live app on Neon's pooled serverless-friendly path while leaving schema changes and restore work on the direct admin path. If you later want per-PR database branches, Neon supports that, but this repo currently recommends one dedicated `preview` branch because it is simpler to operate manually. + ### Required environment variables | Variable | Production | Preview | Notes | | --- | --- | --- | --- | | `DJANGO_SECRET_KEY` | Required | Required | Use a unique secret per environment. Hosted builds fail fast until this is set. | -| `DATABASE_URL` | Required | Required | Point previews at an isolated PostgreSQL database or branch database, not production. | +| `DATABASE_URL` | Required | Required | Use the Neon pooled runtime connection string. Point previews at an isolated PostgreSQL database or branch, not production. | +| `DATABASE_ADMIN_URL` | Recommended | Recommended | Use the Neon direct connection string for migrations, `pg_dump`, `pg_restore`, and SQLite-to-PostgreSQL transfers. | | `APP_BASE_URL` | `https://deep-workflow.vercel.app` | Optional | Sets the canonical production URL and anchors the production host/origin configuration. | | `DJANGO_ENABLE_HOSTED_SQLITE_FALLBACK` | Emergency only | Emergency only | Leave unset for normal deploys. Set to `1` only when intentionally activating the temporary hosted SQLite recovery mode. | | `DJANGO_SECURE_HSTS_PRELOAD` | Optional | Optional | Leave unset unless you intentionally want preload and already satisfy the preload requirements (`includeSubDomains` plus at least `31536000` seconds). | -| `VERCEL_RUN_MIGRATIONS` | Set to `1` only for deliberate schema rollouts | Set to `1` only for isolated preview databases | Build-time migrations are always opt-in. | +| `VERCEL_RUN_MIGRATIONS` | Set to `1` only for deliberate schema rollouts | Set to `1` only for isolated preview databases or the shared preview branch | Build-time migrations are always opt-in. | #### Generate and add `DJANGO_SECRET_KEY` @@ -178,9 +196,11 @@ Vercel injects `VERCEL_ENV`, `VERCEL_URL`, `VERCEL_BRANCH_URL`, and `VERCEL_PROJ 1. Connect the repository to Vercel and keep `main` as the production branch. 2. Set `APP_BASE_URL=https://deep-workflow.vercel.app` in the Production environment. -3. Set `DJANGO_SECRET_KEY` and `DATABASE_URL` in both Production and Preview. Use a separate preview database if you want preview deployments to run migrations. -4. Let pushes to non-`main` branches create Preview deployments automatically. Leave `VERCEL_RUN_MIGRATIONS` unset for normal deploys; only turn it on for isolated preview databases or a coordinated production schema rollout, then redeploy intentionally. -5. After each deployment, check `GET /health/ready/` for database readiness and `GET /health/live/` for the lightweight liveness probe. +3. In Neon, keep `main` for Production and create a separate `preview` branch for Preview deployments. +4. Set `DJANGO_SECRET_KEY`, `DATABASE_URL`, and `DATABASE_ADMIN_URL` in both Production and Preview. Production should use the Neon `main` branch; Preview should use the Neon `preview` branch. +5. Let pushes to non-`main` branches create Preview deployments automatically. Leave `VERCEL_RUN_MIGRATIONS` unset for normal deploys; only turn it on for the isolated preview branch or a coordinated production schema rollout, then redeploy intentionally. +6. After changing any Vercel environment variable, redeploy. Vercel only applies environment variable updates to new deployments. +7. After each deployment, check `GET /health/ready/` for database readiness and `GET /health/live/` for the lightweight liveness probe. #### Emergency hosted SQLite recovery @@ -200,6 +220,8 @@ This mode is for emergency recovery only. It moves sessions and flash messages t Take backups outside Vercel with PostgreSQL client tools and store them in durable storage that is separate from the app deployment. +With Neon, use the **direct** connection string for operational database work. In this repository, that means `DATABASE_ADMIN_URL` when you have it available. Keep `DATABASE_URL` as the pooled runtime URL for the app itself. + Create a compressed backup: ```bash @@ -209,7 +231,7 @@ pg_dump \ --no-owner \ --no-privileges \ --file "backups/deep_workflow_$(date +%F_%H%M%S).dump" \ - "$DATABASE_URL" + "$DATABASE_ADMIN_URL" ``` Verify a backup by restoring it into a disposable database first: @@ -239,6 +261,25 @@ pg_restore \ Prefer restoring into a fresh database, smoke-test the app with `/health/ready/`, and only then swap the production `DATABASE_URL`. That keeps rollback simple and avoids destructive restore work against the live database. +### Moving data from the emergency SQLite fallback into Neon + +If production is still running from the temporary hosted SQLite recovery path, use the included transfer script from a machine that has access to the SQLite file or SQLite `DATABASE_URL`. + +```bash +SOURCE_DATABASE_URL=sqlite:////absolute/path/to/recovery.sqlite3 \ +TARGET_DATABASE_URL="$DATABASE_ADMIN_URL" \ +./scripts/transfer-django-data.sh +``` + +The script: + +- verifies source and target connectivity +- exports Django model data from the source database +- applies migrations to the PostgreSQL target +- loads the exported data into the PostgreSQL target + +It intentionally excludes generated framework tables such as `contenttypes`, `auth.permission`, session rows, and admin log entries. Use a fresh Neon target branch or database when possible, then update Vercel to point Production and Preview back at Neon and redeploy. + ## Quality checks Run the foundational checks before opening a change: @@ -256,6 +297,7 @@ For a production-leaning settings smoke test, also run: ```bash . .venv/bin/activate APP_BASE_URL=https://deep-workflow.vercel.app \ +DATABASE_URL=postgresql://postgres:postgres@127.0.0.1:5432/deep_workflow \ DJANGO_DEBUG=False \ DJANGO_SECRET_KEY=local-deploy-check-only-1234567890-abcdefghijklmnopqrstuvwxyz \ python manage.py check --deploy diff --git a/core/tests/test_scripts.py b/core/tests/test_scripts.py new file mode 100644 index 0000000..2ebdac3 --- /dev/null +++ b/core/tests/test_scripts.py @@ -0,0 +1,116 @@ +import os +import subprocess +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +SOURCE_URL = "sqlite:////tmp/deep-workflow.sqlite3" +TARGET_URL = "postgresql://admin.example/deep_workflow" +RUNTIME_URL = "postgresql://runtime.example/deep_workflow" + + +def write_fake_python(tmp_path: Path) -> Path: + fake_python = tmp_path / "python" + fake_python.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + 'printf \'%s|%s\\n\' "${DATABASE_URL:-}" "$*" >> "$COMMAND_LOG"\n' + 'if [[ "${2:-}" == "dumpdata" ]]; then\n' + " printf '[]'\n" + "fi\n" + ) + fake_python.chmod(0o755) + return fake_python + + +def run_script( + tmp_path: Path, + script_name: str, + *, + extra_env: dict[str, str], +) -> list[str]: + write_fake_python(tmp_path) + command_log = tmp_path / "commands.log" + env = os.environ.copy() + env.update(extra_env) + env["COMMAND_LOG"] = str(command_log) + env["PATH"] = f"{tmp_path}:{env['PATH']}" + subprocess.run( + ["bash", str(REPO_ROOT / "scripts" / script_name)], + cwd=REPO_ROOT, + env=env, + check=True, + capture_output=True, + text=True, + ) + return command_log.read_text().splitlines() + + +def test_vercel_build_uses_runtime_database_for_checks_and_admin_url_for_migrate( + tmp_path: Path, +) -> None: + commands = run_script( + tmp_path, + "vercel-build.sh", + extra_env={ + "VERCEL_ENV": "production", + "VERCEL": "1", + "DJANGO_SECRET_KEY": "x" * 64, + "DATABASE_URL": RUNTIME_URL, + "DATABASE_ADMIN_URL": TARGET_URL, + "VERCEL_RUN_MIGRATIONS": "1", + }, + ) + + assert commands == [ + f"{RUNTIME_URL}|manage.py check_database", + f"{RUNTIME_URL}|manage.py collectstatic --noinput", + f"{TARGET_URL}|manage.py check_database", + f"{TARGET_URL}|manage.py migrate --noinput", + ] + + +def test_vercel_build_uses_runtime_database_for_migrate_without_admin_url( + tmp_path: Path, +) -> None: + commands = run_script( + tmp_path, + "vercel-build.sh", + extra_env={ + "VERCEL_ENV": "production", + "VERCEL": "1", + "DJANGO_SECRET_KEY": "x" * 64, + "DATABASE_URL": RUNTIME_URL, + "VERCEL_RUN_MIGRATIONS": "1", + }, + ) + + assert commands == [ + f"{RUNTIME_URL}|manage.py check_database", + f"{RUNTIME_URL}|manage.py collectstatic --noinput", + f"{RUNTIME_URL}|manage.py migrate --noinput", + ] + + +def test_transfer_django_data_moves_data_between_source_and_target_urls( + tmp_path: Path, +) -> None: + commands = run_script( + tmp_path, + "transfer-django-data.sh", + extra_env={ + "SOURCE_DATABASE_URL": SOURCE_URL, + "TARGET_DATABASE_URL": TARGET_URL, + }, + ) + + assert commands[0] == f"{SOURCE_URL}|manage.py check_database" + assert commands[1] == ( + f"{SOURCE_URL}|manage.py dumpdata --exclude admin.logentry " + "--exclude auth.permission --exclude contenttypes --exclude " + "sessions --natural-foreign --natural-primary" + ) + assert commands[2] == f"{TARGET_URL}|manage.py check_database" + assert commands[3] == f"{TARGET_URL}|manage.py migrate --noinput" + assert commands[4].startswith( + f"{TARGET_URL}|manage.py loaddata /tmp/deep-workflow-transfer." + ) diff --git a/plan.md b/plan.md index 101656d..620e57f 100644 --- a/plan.md +++ b/plan.md @@ -15,7 +15,7 @@ Build a hosted, modern, mobile-friendly web app that replaces a spreadsheet for Recommended stack for v1: - Django for the core web framework -- PostgreSQL for hosted persistence +- PostgreSQL with Neon for hosted persistence - Django templates + HTMX + Alpine.js for a modern UX without a heavy SPA - Tailwind CSS for responsive, polished UI - Vercel for production hosting and pull request preview deployments @@ -27,7 +27,7 @@ Why this stack: - Django gives auth, ORM, migrations, admin, forms, and deployment maturity out of the box - server-rendered pages keep the product simple while still feeling fast - HTMX/Alpine add interactivity where needed without creating a separate frontend app -- PostgreSQL makes hosted sync straightforward +- Neon-backed PostgreSQL makes hosted sync straightforward - Vercel gives a clear path for production releases and PR previews Assumptions for MVP: diff --git a/scripts/transfer-django-data.sh b/scripts/transfer-django-data.sh new file mode 100755 index 0000000..0a65e27 --- /dev/null +++ b/scripts/transfer-django-data.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ -z "${SOURCE_DATABASE_URL:-}" ]]; then + echo "SOURCE_DATABASE_URL is required. Point it at the current SQLite or PostgreSQL database you want to export." >&2 + exit 1 +fi + +if [[ -z "${TARGET_DATABASE_URL:-}" ]]; then + echo "TARGET_DATABASE_URL is required. Point it at the PostgreSQL database you want to import into." >&2 + exit 1 +fi + +fixture_file="$(mktemp "${TMPDIR:-/tmp}/deep-workflow-transfer.XXXXXX.json")" + +cleanup() { + rm -f "$fixture_file" +} + +trap cleanup EXIT + +echo "Checking source database connectivity." +DATABASE_URL="$SOURCE_DATABASE_URL" python manage.py check_database + +echo "Exporting Django data from source database." +DATABASE_URL="$SOURCE_DATABASE_URL" python manage.py dumpdata \ + --exclude admin.logentry \ + --exclude auth.permission \ + --exclude contenttypes \ + --exclude sessions \ + --natural-foreign \ + --natural-primary \ + > "$fixture_file" + +echo "Checking target database connectivity." +DATABASE_URL="$TARGET_DATABASE_URL" python manage.py check_database + +echo "Applying migrations to target database." +DATABASE_URL="$TARGET_DATABASE_URL" python manage.py migrate --noinput + +echo "Importing Django data into target database." +DATABASE_URL="$TARGET_DATABASE_URL" python manage.py loaddata "$fixture_file" diff --git a/scripts/vercel-build.sh b/scripts/vercel-build.sh index 511098c..0f11c73 100755 --- a/scripts/vercel-build.sh +++ b/scripts/vercel-build.sh @@ -3,6 +3,7 @@ set -euo pipefail vercel_environment="${VERCEL_ENV:-local}" vercel_runtime="${VERCEL:-}" +admin_database_url="${DATABASE_ADMIN_URL:-}" if [[ "$vercel_environment" == "preview" || "$vercel_environment" == "production" || "$vercel_environment" == "development" || "$vercel_runtime" == "1" || "$vercel_runtime" == "true" ]]; then if [[ -z "${DJANGO_SECRET_KEY:-}" ]]; then @@ -22,7 +23,13 @@ fi python manage.py collectstatic --noinput if [[ "${VERCEL_RUN_MIGRATIONS:-0}" == "1" ]]; then - python manage.py migrate --noinput + if [[ -n "$admin_database_url" ]]; then + echo "Using DATABASE_ADMIN_URL for migration-time database checks and migrations." + DATABASE_URL="$admin_database_url" python manage.py check_database + DATABASE_URL="$admin_database_url" python manage.py migrate --noinput + else + python manage.py migrate --noinput + fi else echo "Skipping migrations for ${VERCEL_ENV:-local} deployment. Set VERCEL_RUN_MIGRATIONS=1 to enable them explicitly." fi