From 9da5c2d93c76eda7068aefa0f2013c7a522c1101 Mon Sep 17 00:00:00 2001 From: krahd Date: Tue, 19 May 2026 01:13:54 -0300 Subject: [PATCH] Tighten model metadata inference and release docs --- STATUS.md | 9 ++++-- docs/RELEASE.md | 54 +++++++++++++++++++++++++++++++++++- modelito/__init__.py | 10 +++++++ modelito/model_metadata.py | 8 +++++- tests/test_model_metadata.py | 21 ++++++++++++++ 5 files changed, 97 insertions(+), 5 deletions(-) diff --git a/STATUS.md b/STATUS.md index 631a12b..9e5e367 100644 --- a/STATUS.md +++ b/STATUS.md @@ -1,6 +1,6 @@ # modelito – Project Status -Last updated: 2026-05-19 01:04 +Last updated: 2026-05-19 01:13 ## Project purpose @@ -143,11 +143,14 @@ python -m twine check dist/* - Python 3.10 test compatibility fixed by using `tomli` as fallback for `tomllib` (available from Python 3.11+). - Model metadata registry was made conservative and typed using a frozen dataclass, stale hardcoded entries were removed/downgraded, and modern model-family inference was added. - Provider APIs remain the source of truth for model capabilities; static metadata is now explicitly treated as best-effort fallback only. +- Model metadata inference no longer treats every `o...` model name as OpenAI; only `gpt-*` and known OpenAI reasoning prefixes (`o1*`, `o3*`, `o4*`) infer OpenAI. +- Model metadata helpers are now exported from the package root (`ModelMetadata`, `get_model_info`, `get_model_metadata`, `infer_model_metadata`). +- Release checklist now includes explicit trusted publishing requirements, tag/publish commands, and clean-environment install checks. - Current release line is `1.4.4`. - Current oMLX stack uses `OpenAICompatibleHTTPProvider` with strict-mode typed error handling. - Current provider typing includes `ChatProvider`, `MessageInput`, and `OpenAIMessageDict` exports, with `Client` chat-related methods accepting broadened message input types; provider protocols are aligned so `SyncProvider`, `AsyncProvider`, `StreamingProvider`, and `ChatProvider` all accept `Iterable[MessageInput]`. - `Client.chat_json()` now supports optional stronger schema validation via `strict_schema=True` using dataclass construction or Pydantic-style `model_validate`/`parse_obj` hooks, while preserving lightweight key-presence checks by default. -- Validation completed locally in this session: `python scripts/check_no_legacy_dicts.py` -> no literal dict-shaped message examples found in docs/examples; `ruff check .` clean; `mypy modelito --ignore-missing-imports` clean; `pytest -q` -> `249 passed, 2 skipped`. +- Validation completed locally in this session: `python scripts/check_no_legacy_dicts.py` -> no literal dict-shaped message examples found in docs/examples; `ruff check .` clean; `mypy modelito --ignore-missing-imports` clean; `pytest -q --ignore=tests/integration tests` -> `250 passed, 1 skipped`; `python -c "from modelito import get_model_metadata; print(get_model_metadata('gpt-4o-mini')['provider'])"` -> `openai`; optional checks: `python -m build` succeeded and `python -m twine check dist/*` passed. - Trusted publishing note: the workflow is configured for PyPI trusted publishing, but PyPI project-side trusted publisher settings must be verified before release, and the release tag must match `pyproject.toml`. - Historical release narratives are maintained in `CHANGELOG.md`; STATUS.md is kept as a current-state snapshot. @@ -184,4 +187,4 @@ python -m twine check dist/* --- -Last updated: 2026-05-19 01:04 +Last updated: 2026-05-19 01:13 diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 38dd13f..d833136 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -2,6 +2,12 @@ ## Before tagging +- Update `pyproject.toml` version. +- Update `CHANGELOG.md` if needed. +- Update `STATUS.md`. +- Ensure `STATUS.md` uses the timestamp line format `Last updated: YYYY-MM-DD HH:MM` at top and bottom. +- Confirm default CI intentionally excludes gated Ollama integration tests. + Run the release validation set: ```bash @@ -14,14 +20,60 @@ python -m twine check dist/* python -c "import modelito; print(modelito.__version__)" ``` +## Trusted publishing + +- PyPI publishing uses GitHub Actions trusted publishing, not a PyPI API token. +- Workflow: `.github/workflows/publish.yml`. +- Environment: `pypi`. +- PyPI project-side trusted publisher must match: + - repository: `krahd/modelito` + - workflow: `publish.yml` + - environment: `pypi` +- Release tags must use `vX.Y.Z`. +- The publish workflow validates tag version against `pyproject.toml`. + ## Before publishing - Confirm the tag version matches `pyproject.toml`. - Verify the PyPI project-side trusted publisher settings for `krahd/modelito`. - Confirm the GitHub Actions publish workflow is targeting the `pypi` environment. +## Tag and publish + +```bash +git status +git tag vX.Y.Z +git push origin vX.Y.Z +``` + ## After publishing - Check the uploaded release files on PyPI. - Confirm the release tag and published version match. -- Update `STATUS.md` if the release state changed. \ No newline at end of file +- Update `STATUS.md` if the release state changed. + +Run clean environment install verification: + +```bash +python -m venv /tmp/modelito-release-check +. /tmp/modelito-release-check/bin/activate +python -m pip install --upgrade pip +python -m pip install modelito +python -c "import modelito; print(modelito.__version__)" +``` + +Optional checks: + +```bash +python -m modelito doctor +modelito-doctor --help +python -m pip install "modelito[serve]" +modelito-serve --help +``` + +## Deferred release-adjacent work + +- Ollama raw passthrough/tool-call preservation remains deferred. +- Continue applying provider-addition policy in `docs/ARCHITECTURE.md`. +- Keep FastAPI/Uvicorn optional under `[serve]`. +- PyPI project-side trusted publisher settings must be verified externally before tagging. \ No newline at end of file diff --git a/modelito/__init__.py b/modelito/__init__.py index 62eb8d0..fd8db94 100644 --- a/modelito/__init__.py +++ b/modelito/__init__.py @@ -89,6 +89,12 @@ from .doctor import ProviderStatus, check_provider_ready, format_provider_status from .embeddings import Embedder, StubEmbeddingProvider, embed_texts from .messages import Message, Messages, Response, flatten_message_inputs +from .model_metadata import ( + ModelMetadata, + get_model_info, + get_model_metadata, + infer_model_metadata, +) from .normalization import normalize_models, normalize_metadata __all__ = [ @@ -130,6 +136,10 @@ "Messages", "Response", "flatten_message_inputs", + "ModelMetadata", + "get_model_info", + "get_model_metadata", + "infer_model_metadata", "normalize_models", "normalize_metadata", "load_config", diff --git a/modelito/model_metadata.py b/modelito/model_metadata.py index 980c573..0c185eb 100644 --- a/modelito/model_metadata.py +++ b/modelito/model_metadata.py @@ -75,7 +75,7 @@ def infer_model_metadata(model_name: str) -> ModelMetadata: normalized = str(model_name or "").strip().lower() metadata = ModelMetadata(id=str(model_name or "").strip() or "unknown") - if normalized.startswith("gpt-") or normalized.startswith("o"): + if normalized.startswith("gpt-") or _looks_like_openai_reasoning_model(normalized): metadata = _replace(metadata, provider="openai", supports_streaming=True) elif normalized.startswith("claude-"): metadata = _replace(metadata, provider="anthropic", supports_streaming=True) @@ -128,3 +128,9 @@ def _replace(metadata: ModelMetadata, **updates: Any) -> ModelMetadata: values = asdict(metadata) values.update(updates) return ModelMetadata(**values) + + +def _looks_like_openai_reasoning_model(normalized: str) -> bool: + return normalized in {"o1", "o3", "o4"} or normalized.startswith( + ("o1-", "o3-", "o4-") + ) diff --git a/tests/test_model_metadata.py b/tests/test_model_metadata.py index bb5145f..78be1a8 100644 --- a/tests/test_model_metadata.py +++ b/tests/test_model_metadata.py @@ -50,3 +50,24 @@ def test_backward_compatible_dict_return_shape_contains_legacy_aliases(): meta = get_model_metadata("gpt-4o-mini") assert "tools" in meta assert "functions" in meta + + +def test_infer_openai_reasoning_o_models(): + assert get_model_metadata("o1", infer=True)["provider"] == "openai" + assert get_model_metadata("o3-mini", infer=True)["provider"] == "openai" + assert get_model_metadata("o4-mini", infer=True)["provider"] == "openai" + + +def test_infer_does_not_map_arbitrary_o_names_to_openai(): + assert get_model_metadata("orca-mini", infer=True) == {} + assert get_model_metadata("openhermes", infer=True) == {} + assert get_model_metadata("ollama-local", infer=True) == {} + + +def test_root_exports_include_model_metadata_helpers(): + from modelito import ModelMetadata, get_model_metadata, infer_model_metadata + + assert get_model_metadata("gpt-4o-mini")["provider"] == "openai" + inferred = infer_model_metadata("o3-mini") + assert isinstance(inferred, ModelMetadata) + assert inferred.provider == "openai"