diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 7aa9a8f..0000000 --- a/.dockerignore +++ /dev/null @@ -1,10 +0,0 @@ -.git -.github -.venv -__pycache__ -*.py[oc] -build -dist -*.egg-info -.dockerignore -Dockerfile diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml deleted file mode 100644 index dde661b..0000000 --- a/.github/workflows/build.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: Build image - -on: - - push - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - name: Install uv - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true - - - name: Install deps - run: make install-dev - - - name: Run codegen - run: make gen - - - name: Set up Docker runtime - uses: docker/setup-buildx-action@v3 - - - name: Build image - uses: docker/build-push-action@v6 - with: - context: . - push: false diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 0000000..b66843b --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,63 @@ +name: E2E wheel + +on: + push: + branches: [main, master] + pull_request: + +jobs: + build-wheel: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - run: corepack enable + + - name: Install Python deps + run: make install-dev + + - name: Run codegen + run: make gen + + - name: Build wheel + run: uv build + + - uses: actions/upload-artifact@v4 + with: + name: wheel + path: dist/*.whl + + verify: + needs: build-wheel + strategy: + matrix: + os: [ubuntu-22.04, ubuntu-24.04] + runs-on: ${{ matrix.os }} + steps: + - uses: astral-sh/setup-uv@v7 + with: + python-version: "3.13" + + - uses: actions/download-artifact@v4 + with: + name: wheel + path: dist + + - name: Verify uploader + run: | + set -e + WHEEL=$(ls dist/*.whl | head -n1) + uvx --python 3.13 --from "$WHEEL" uploader --no-browser & + PID=$! + sleep 8 + curl -sf http://127.0.0.1:8000/api/tasks >/dev/null + curl -sf http://127.0.0.1:8000/ >/dev/null + kill $PID diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0a55dd8..32b0d9e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,49 +1,41 @@ -name: Release image +name: Release wheel on: - workflow_dispatch: + push: + branches: + - master jobs: release: runs-on: ubuntu-latest permissions: - contents: read - packages: write + contents: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - - name: Install uv - uses: astral-sh/setup-uv@v7 + - uses: astral-sh/setup-uv@v7 with: enable-cache: true - - name: Install deps + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - run: corepack enable + + - name: Install Python deps run: make install-dev - name: Run codegen run: make gen - - name: Set up Docker runtime - uses: docker/setup-buildx-action@v3 - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set image name - run: echo "IMAGE_NAME=ghcr.io/${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV - - - name: Set short SHA - run: echo "SHORT_SHA=${GITHUB_SHA::7}" >> $GITHUB_ENV + - name: Build wheel + run: uv build - - name: Build and push image - uses: docker/build-push-action@v6 + - name: Publish GitHub release + uses: softprops/action-gh-release@v2 with: - context: . - push: true - tags: | - ${{ env.IMAGE_NAME }}:${{ env.SHORT_SHA }} - ${{ env.IMAGE_NAME }}:latest + tag_name: build-${{ github.sha }} + name: Wheel ${{ github.sha }} + files: dist/*.whl + generate_release_notes: true diff --git a/.gitignore b/.gitignore index 7db5ac0..4709427 100644 --- a/.gitignore +++ b/.gitignore @@ -174,6 +174,8 @@ pyrightconfig.json .vizier_cache +server/static/ + node_modules/ history.jsonl diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 80bc43f..0000000 --- a/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM python:3.12-slim AS builder -RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/* -COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/ -WORKDIR /app -COPY pyproject.toml uv.lock ./ -RUN uv sync --frozen --no-dev - -FROM python:3.12-slim AS runtime -WORKDIR /app -COPY --from=builder /app/.venv /app/.venv -ENV PATH="/app/.venv/bin:$PATH" -COPY . . diff --git a/app/sources/vizier.py b/app/sources/vizier.py index eaa45e2..524499b 100644 --- a/app/sources/vizier.py +++ b/app/sources/vizier.py @@ -12,10 +12,13 @@ from astroquery import vizier import app +import server.paths as paths from app.gen.client.adminapi import models, types VIZIER_URL = "https://vizier.cds.unistra.fr/viz-bin/votable/-tsv" +_DEFAULT_VIZIER_CACHE = str(paths.DATA_DIR / "vizier_cache") + def _coerce_row_to_schema( row_dict: dict[str, object], @@ -58,7 +61,7 @@ def __init__( self, catalog_name: str, table_name: str, - cache_path: str = ".vizier_cache/", + cache_path: str = _DEFAULT_VIZIER_CACHE, batch_size: int = 100, ): self.cache_path = cache_path diff --git a/app/sources/vizier_v2.py b/app/sources/vizier_v2.py index 9fb40bd..77291f0 100644 --- a/app/sources/vizier_v2.py +++ b/app/sources/vizier_v2.py @@ -6,8 +6,11 @@ from astroquery import vizier import app +import server.paths as paths from app.gen.client.adminapi import models, types +_DEFAULT_VIZIER_CACHE = str(paths.DATA_DIR / "vizier_cache") + def _sanitize_filename(string: str) -> str: return ( @@ -59,7 +62,7 @@ def __init__( table_name: str, index_column: str, *constraints: str, - cache_path: str = ".vizier_cache/", + cache_path: str = _DEFAULT_VIZIER_CACHE, batch_size: int = 1000, ): if len(constraints) % 3 != 0: diff --git a/hatch_build.py b/hatch_build.py new file mode 100644 index 0000000..f21135b --- /dev/null +++ b/hatch_build.py @@ -0,0 +1,21 @@ +import subprocess +from pathlib import Path + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + + +class CustomHook(BuildHookInterface): + PLUGIN_NAME = "custom" + + def initialize(self, version: str, build_data: dict) -> None: + frontend_dir = Path(self.root) / "frontend" + subprocess.run( + ["yarn", "install", "--frozen-lockfile"], + cwd=frontend_dir, + check=True, + ) + subprocess.run( + ["yarn", "build"], + cwd=frontend_dir, + check=True, + ) diff --git a/makefile b/makefile index 5c42b39..1125e71 100644 --- a/makefile +++ b/makefile @@ -68,11 +68,6 @@ fix-frontend: @output=$$(cd frontend && yarn run --silent prettier --write src 2>&1) || { echo "$$output"; exit 1; } @output=$$(cd frontend && yarn run --silent eslint --fix src 2>&1) || { echo "$$output"; exit 1; } -# only for mac as this is faster -build: - docker build . \ - --platform linux/arm64 - new-branch: @read -p "Branch name: " branch_name && \ branch_name=$${branch_name// /-} && \ diff --git a/pyproject.toml b/pyproject.toml index 71038dc..438169c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,9 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + [project] -name = "uploader" +name = "hyperleda-uploader" version = "0.1.0" description = "Add your description here" readme = "README.md" @@ -18,6 +22,18 @@ dependencies = [ "psycopg[binary]>=3.2.0", ] +[project.scripts] +uploader = "server.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["server", "app"] + +[tool.hatch.build.targets.wheel.force-include] +"frontend/dist" = "server/static" + +[tool.hatch.build.hooks.custom] +path = "hatch_build.py" + [dependency-groups] dev = [ "ruff~=0.15.0", @@ -121,6 +137,7 @@ reportImplicitAbstractClass = "error" exclude = [ "frontend/**", "app/gen/**", + "hatch_build.py", ".venv/**" ] diff --git a/server/cli.py b/server/cli.py new file mode 100644 index 0000000..3d48f22 --- /dev/null +++ b/server/cli.py @@ -0,0 +1,19 @@ +import argparse +import threading +import webbrowser + +import uvicorn + + +def main() -> None: + parser = argparse.ArgumentParser(description="HyperLEDA Uploader") + parser.add_argument("--port", type=int, default=8000) + parser.add_argument("--host", type=str, default="127.0.0.1") + parser.add_argument("--no-browser", action="store_true") + args = parser.parse_args() + + if not args.no_browser: + url = f"http://{args.host}:{args.port}/" + threading.Timer(1.5, webbrowser.open, args=[url]).start() + + uvicorn.run("server.main:app", host=args.host, port=args.port) diff --git a/server/credentials.py b/server/credentials.py index 4891a80..b3e301a 100644 --- a/server/credentials.py +++ b/server/credentials.py @@ -1,8 +1,8 @@ -import pathlib - import dotenv -ENV_PATH = pathlib.Path(".env") +import server.paths as paths + +ENV_PATH = paths.DATA_DIR / ".env" def save_credentials(db_user: str, db_password: str) -> None: diff --git a/server/forms/upload_vizier.py b/server/forms/upload_vizier.py index a4307fe..66bc016 100644 --- a/server/forms/upload_vizier.py +++ b/server/forms/upload_vizier.py @@ -5,16 +5,19 @@ from pydantic import BaseModel, Field import app.report as report +import server.paths as paths from app.endpoints import env_map from app.gen.client import adminapi from app.sources.vizier import VizierSource from app.upload import upload_for_web +_DEFAULT_VIZIER_CACHE = str(paths.DATA_DIR / "vizier_cache") + class UploadVizierForm(BaseModel): catalog_name: str = Field(..., title="VizieR catalog name") source_table_name: str = Field(..., title="VizieR table name") - cache_path: str = Field(default=".vizier_cache/", title="Cache path") + cache_path: str = Field(default=_DEFAULT_VIZIER_CACHE, title="Cache path") batch_size: int = Field(default=100, title="Batch size", ge=1) table_name: str = Field( default="", diff --git a/server/forms/upload_vizier_v2.py b/server/forms/upload_vizier_v2.py index 8aba0e4..666cb49 100644 --- a/server/forms/upload_vizier_v2.py +++ b/server/forms/upload_vizier_v2.py @@ -4,12 +4,15 @@ from pydantic import BaseModel, Field import app.report as report +import server.paths as paths from app.endpoints import env_map from app.gen.client import adminapi from app.sources.vizier_v2 import VizierV2Source from app.upload import upload_for_web from server.forms.upload_base import UploadBaseForm +_DEFAULT_VIZIER_CACHE = str(paths.DATA_DIR / "vizier_cache") + class UploadVizierV2Form(UploadBaseForm): catalog_name: str = Field(..., title="VizieR catalog name") @@ -20,7 +23,7 @@ class UploadVizierV2Form(UploadBaseForm): title="Constraints", description="Provide as repeating triples: column, operator, value.", ) - cache_path: str = Field(default=".vizier_cache/", title="Cache path") + cache_path: str = Field(default=_DEFAULT_VIZIER_CACHE, title="Cache path") batch_size: int = Field(default=1000, title="Batch size", ge=1) diff --git a/server/history.py b/server/history.py index 0c96f5e..228fcb0 100644 --- a/server/history.py +++ b/server/history.py @@ -1,10 +1,11 @@ import json import threading -from pathlib import Path from typing import Literal from pydantic import BaseModel +import server.paths as paths + HistoryStatus = Literal["success", "error"] @@ -17,7 +18,7 @@ class HistoryEntry(BaseModel): message: str -HISTORY_PATH = Path("history.jsonl") +HISTORY_PATH = paths.DATA_DIR / "history.jsonl" HISTORY_LOCK = threading.Lock() diff --git a/server/main.py b/server/main.py index 06699a7..4926611 100644 --- a/server/main.py +++ b/server/main.py @@ -1,11 +1,16 @@ import asyncio import json +import pathlib from typing import Any from fastapi import FastAPI, HTTPException from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import StreamingResponse from pydantic import ValidationError +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.responses import Response +from starlette.staticfiles import StaticFiles +from starlette.types import Scope from server.history import load_history from server.task_registry import register_all_tasks @@ -86,3 +91,18 @@ async def event_gen(): await asyncio.sleep(0.15) return StreamingResponse(event_gen(), media_type="text/event-stream") + + +class SPAStaticFiles(StaticFiles): + async def get_response(self, path: str, scope: Scope) -> Response: + try: + return await super().get_response(path, scope) + except StarletteHTTPException as exc: + if exc.status_code != 404: + raise + return await super().get_response("index.html", scope) + + +_static_dir = pathlib.Path(__file__).parent / "static" +if _static_dir.is_dir(): + app.mount("/", SPAStaticFiles(directory=_static_dir, html=True), name="static") diff --git a/server/paths.py b/server/paths.py new file mode 100644 index 0000000..6f8c2ba --- /dev/null +++ b/server/paths.py @@ -0,0 +1,4 @@ +from pathlib import Path + +DATA_DIR = Path.home() / ".hyperleda-uploader" +DATA_DIR.mkdir(parents=True, exist_ok=True) diff --git a/uv.lock b/uv.lock index c0fd920..0b16dfe 100644 --- a/uv.lock +++ b/uv.lock @@ -369,6 +369,55 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "hyperleda-uploader" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "astropy" }, + { name = "astroquery" }, + { name = "fastapi" }, + { name = "numpy" }, + { name = "openapi-python-client" }, + { name = "pandas" }, + { name = "psycopg", extra = ["binary"] }, + { name = "python-dotenv" }, + { name = "pyvo" }, + { name = "structlog" }, + { name = "uvicorn", extra = ["standard"] }, +] + +[package.dev-dependencies] +dev = [ + { name = "basedpyright" }, + { name = "pandas-stubs" }, + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "astropy", specifier = ">=7.1.0" }, + { name = "astroquery", git = "https://github.com/astropy/astroquery.git?branch=main" }, + { name = "fastapi", specifier = ">=0.115.0" }, + { name = "numpy", specifier = ">=2.3.4" }, + { name = "openapi-python-client", specifier = ">=0.27.1" }, + { name = "pandas", specifier = ">=2.3.3" }, + { name = "psycopg", extras = ["binary"], specifier = ">=3.2.0" }, + { name = "python-dotenv", specifier = ">=1.0.0" }, + { name = "pyvo", specifier = ">=1.8" }, + { name = "structlog", specifier = ">=25.3.0" }, + { name = "uvicorn", extras = ["standard"], specifier = ">=0.32.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "basedpyright", specifier = "~=1.38.3" }, + { name = "pandas-stubs", specifier = ">=2.2.3.250308" }, + { name = "pytest", specifier = "~=9.0.2" }, + { name = "ruff", specifier = "~=0.15.0" }, +] + [[package]] name = "idna" version = "3.11" @@ -1114,55 +1163,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1", size = 348521, upload-time = "2025-12-13T17:45:33.889Z" }, ] -[[package]] -name = "uploader" -version = "0.1.0" -source = { virtual = "." } -dependencies = [ - { name = "astropy" }, - { name = "astroquery" }, - { name = "fastapi" }, - { name = "numpy" }, - { name = "openapi-python-client" }, - { name = "pandas" }, - { name = "psycopg", extra = ["binary"] }, - { name = "python-dotenv" }, - { name = "pyvo" }, - { name = "structlog" }, - { name = "uvicorn", extra = ["standard"] }, -] - -[package.dev-dependencies] -dev = [ - { name = "basedpyright" }, - { name = "pandas-stubs" }, - { name = "pytest" }, - { name = "ruff" }, -] - -[package.metadata] -requires-dist = [ - { name = "astropy", specifier = ">=7.1.0" }, - { name = "astroquery", git = "https://github.com/astropy/astroquery.git?branch=main" }, - { name = "fastapi", specifier = ">=0.115.0" }, - { name = "numpy", specifier = ">=2.3.4" }, - { name = "openapi-python-client", specifier = ">=0.27.1" }, - { name = "pandas", specifier = ">=2.3.3" }, - { name = "psycopg", extras = ["binary"], specifier = ">=3.2.0" }, - { name = "python-dotenv", specifier = ">=1.0.0" }, - { name = "pyvo", specifier = ">=1.8" }, - { name = "structlog", specifier = ">=25.3.0" }, - { name = "uvicorn", extras = ["standard"], specifier = ">=0.32.0" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "basedpyright", specifier = "~=1.38.3" }, - { name = "pandas-stubs", specifier = ">=2.2.3.250308" }, - { name = "pytest", specifier = "~=9.0.2" }, - { name = "ruff", specifier = "~=0.15.0" }, -] - [[package]] name = "urllib3" version = "2.6.3"