diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml index fa1b8480..6c8714c5 100644 --- a/.github/workflows/benchmarks.yaml +++ b/.github/workflows/benchmarks.yaml @@ -1,83 +1,48 @@ name: Benchmarks on: + push: + branches: + - main pull_request: types: - opened - synchronize + # `workflow_dispatch` allows CodSpeed to trigger backtest + # performance analysis in order to generate initial data. + workflow_dispatch: jobs: benchmark: - name: Benchmark tests - runs-on: ubuntu-latest + name: Run benchmarks + runs-on: codspeed-macro permissions: contents: read - pull-requests: write - strategy: - matrix: - python_version: [3.12] + id-token: write steps: - - name: Checkout branch + - name: Checkout uses: actions/checkout@v4 - with: - path: pr - - - name: Checkout main - uses: actions/checkout@v4 - with: - ref: main - path: main - name: Install python uses: actions/setup-python@v5 with: - python-version: ${{matrix.python_version}} + python-version: "3.12" - name: Install uv uses: astral-sh/setup-uv@v4 with: enable-cache: true - cache-dependency-glob: "main/uv.lock" - - - name: Setup benchmarks - run: | - echo "BASE_SHA=$(echo ${{ github.event.pull_request.base.sha }} | cut -c1-8)" >> $GITHUB_ENV - echo "HEAD_SHA=$(echo ${{ github.event.pull_request.head.sha }} | cut -c1-8)" >> $GITHUB_ENV - echo "PR_COMMENT=$(mktemp)" >> $GITHUB_ENV - - - name: Run benchmarks on PR - working-directory: ./pr - run: | - uv sync --group test - uv run pytest --benchmark-only --benchmark-save=pr - - - name: Run benchmarks on main - working-directory: ./main - continue-on-error: true - run: | - uv sync --group test - uv run pytest --benchmark-only --benchmark-save=base + cache-dependency-glob: "uv.lock" - - name: Compare results - continue-on-error: false - run: | - uvx pytest-benchmark compare **/.benchmarks/**/*.json | tee cmp_results + - name: Install project + run: uv sync --group test - echo 'Benchmark comparison for [`${{ env.BASE_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.base.sha }}) (base) vs [`${{ env.HEAD_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.head.sha }}) (PR)' >> pr_comment - echo '```' >> pr_comment - cat cmp_results >> pr_comment - echo '```' >> pr_comment - cat pr_comment > ${{ env.PR_COMMENT }} - - - name: Comment on PR - uses: actions/github-script@v7 + - name: Run benchmarks + uses: CodSpeedHQ/action@v4 + env: + RAY_ENABLE_UV_RUN_RUNTIME_ENV: 0 + PLUGBOARD_IO_READ_TIMEOUT: 5.0 with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: require('fs').readFileSync('${{ env.PR_COMMENT }}').toString() - }); + mode: walltime + run: uv run pytest tests/benchmark/ --codspeed diff --git a/README.md b/README.md index 7961dd36..d04af3b6 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,8 @@ CodeQL + + CodSpeed Badge
Docs diff --git a/pyproject.toml b/pyproject.toml index 3b5786c2..e7c09cfa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ test = [ "optuna>=3.0,<5", "pytest>=8.3,<10", "pytest-asyncio>=1.0,<2", - "pytest-benchmark>=5.1.0", + "pytest-codspeed>=4.3.0", "pytest-cases>=3.8,<4", "pytest-env>=1.1,<2", "pytest-rerunfailures>=15.0,<17", diff --git a/tests/benchmark/test_benchmarking.py b/tests/benchmark/test_benchmarking.py index 7554a7a0..023e94da 100644 --- a/tests/benchmark/test_benchmarking.py +++ b/tests/benchmark/test_benchmarking.py @@ -1,35 +1,49 @@ -"""Simple benchmark tests for Plugboard models.""" +"""Benchmark tests for Plugboard processes.""" -import asyncio +import pytest -from pytest_benchmark.fixture import BenchmarkFixture - -from plugboard.connector import AsyncioConnector -from plugboard.process import LocalProcess, Process +from plugboard.connector import AsyncioConnector, Connector, RayConnector, ZMQConnector +from plugboard.process import LocalProcess, Process, RayProcess from plugboard.schemas import ConnectorSpec from tests.integration.test_process_with_components_run import A, B -def _setup_process() -> tuple[tuple[Process], dict]: - comp_a = A(name="comp_a", iters=1000) +ITERS = 1000 + +CONNECTOR_PROCESS_PARAMS = [ + (AsyncioConnector, LocalProcess), + (ZMQConnector, LocalProcess), + (RayConnector, RayProcess), +] +CONNECTOR_PROCESS_IDS = ["asyncio", "zmq", "ray"] + + +def _build_process(connector_cls: type[Connector], process_cls: type[Process]) -> Process: + """Build a process with the given connector and process class.""" + comp_a = A(name="comp_a", iters=ITERS) comp_b1 = B(name="comp_b1", factor=1) comp_b2 = B(name="comp_b2", factor=2) components = [comp_a, comp_b1, comp_b2] connectors = [ - AsyncioConnector(spec=ConnectorSpec(source="comp_a.out_1", target="comp_b1.in_1")), - AsyncioConnector(spec=ConnectorSpec(source="comp_b1.out_1", target="comp_b2.in_1")), + connector_cls(spec=ConnectorSpec(source="comp_a.out_1", target="comp_b1.in_1")), + connector_cls(spec=ConnectorSpec(source="comp_b1.out_1", target="comp_b2.in_1")), ] - process = LocalProcess(components=components, connectors=connectors) - # Initialise process so that this is excluded from the benchmark timing - asyncio.run(process.init()) - # Return args and kwargs tuple for benchmark.pedantic - return (process,), {} - - -def _run_process(process: Process) -> None: - asyncio.run(process.run()) - - -def test_benchmark_process_run(benchmark: BenchmarkFixture) -> None: - """Benchmark the running of a Plugboard Process.""" - benchmark.pedantic(_run_process, setup=_setup_process, rounds=5) + return process_cls(components=components, connectors=connectors) + + +@pytest.mark.benchmark +@pytest.mark.parametrize( + "connector_cls, process_cls", + CONNECTOR_PROCESS_PARAMS, + ids=CONNECTOR_PROCESS_IDS, +) +@pytest.mark.asyncio +async def test_benchmark_process_lifecycle( + connector_cls: type[Connector], + process_cls: type[Process], + ray_ctx: None, +) -> None: + """Benchmark the full lifecycle (init, run, destroy) of a Plugboard Process.""" + process = _build_process(connector_cls, process_cls) + async with process: + await process.run() diff --git a/uv.lock b/uv.lock index bec1c8ea..6d72ce2e 100644 --- a/uv.lock +++ b/uv.lock @@ -3877,8 +3877,8 @@ all = [ { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-asyncio" }, - { name = "pytest-benchmark" }, { name = "pytest-cases" }, + { name = "pytest-codspeed" }, { name = "pytest-env" }, { name = "pytest-rerunfailures" }, { name = "radon" }, @@ -3923,8 +3923,8 @@ test = [ { name = "optuna" }, { name = "pytest" }, { name = "pytest-asyncio" }, - { name = "pytest-benchmark" }, { name = "pytest-cases" }, + { name = "pytest-codspeed" }, { name = "pytest-env" }, { name = "pytest-rerunfailures" }, { name = "ray", extra = ["default", "tune"] }, @@ -3993,8 +3993,8 @@ all = [ { name = "pre-commit", specifier = ">=3.8,<4" }, { name = "pytest", specifier = ">=8.3,<10" }, { name = "pytest-asyncio", specifier = ">=1.0,<2" }, - { name = "pytest-benchmark", specifier = ">=5.1.0" }, { name = "pytest-cases", specifier = ">=3.8,<4" }, + { name = "pytest-codspeed", specifier = ">=4.3.0" }, { name = "pytest-env", specifier = ">=1.1,<2" }, { name = "pytest-rerunfailures", specifier = ">=15.0,<17" }, { name = "radon", specifier = ">=6.0.1,<7" }, @@ -4039,8 +4039,8 @@ test = [ { name = "optuna", specifier = ">=3.0,<5" }, { name = "pytest", specifier = ">=8.3,<10" }, { name = "pytest-asyncio", specifier = ">=1.0,<2" }, - { name = "pytest-benchmark", specifier = ">=5.1.0" }, { name = "pytest-cases", specifier = ">=3.8,<4" }, + { name = "pytest-codspeed", specifier = ">=4.3.0" }, { name = "pytest-env", specifier = ">=1.1,<2" }, { name = "pytest-rerunfailures", specifier = ">=15.0,<17" }, { name = "ray", extras = ["default", "tune"], specifier = ">=2.40.0,<3" }, @@ -4276,15 +4276,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] -[[package]] -name = "py-cpuinfo" -version = "9.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, -] - [[package]] name = "py-partiql-parser" version = "0.6.3" @@ -4557,19 +4548,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, ] -[[package]] -name = "pytest-benchmark" -version = "5.2.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "py-cpuinfo" }, - { name = "pytest" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/24/34/9f732b76456d64faffbef6232f1f9dbec7a7c4999ff46282fa418bd1af66/pytest_benchmark-5.2.3.tar.gz", hash = "sha256:deb7317998a23c650fd4ff76e1230066a76cb45dcece0aca5607143c619e7779", size = 341340, upload-time = "2025-11-09T18:48:43.215Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/33/29/e756e715a48959f1c0045342088d7ca9762a2f509b945f362a316e9412b7/pytest_benchmark-5.2.3-py3-none-any.whl", hash = "sha256:bc839726ad20e99aaa0d11a127445457b4219bdb9e80a1afc4b51da7f96b0803", size = 45255, upload-time = "2025-11-09T18:48:39.765Z" }, -] - [[package]] name = "pytest-cases" version = "3.10.1" @@ -4585,6 +4563,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/61/f2/7a29fb0571562034b05c38dceabba48dcc622be5d6c5448db80779e55de7/pytest_cases-3.10.1-py2.py3-none-any.whl", hash = "sha256:0deb8a85b6132e44adbc1cfc57897c6a624ec23f48ab445a43c7d56a6b9315a4", size = 108870, upload-time = "2026-03-02T23:05:32.663Z" }, ] +[[package]] +name = "pytest-codspeed" +version = "4.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, + { name = "pytest" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/ab/eca41967d11c95392829a8b4bfa9220a51cffc4a33ec4653358000356918/pytest_codspeed-4.3.0.tar.gz", hash = "sha256:5230d9d65f39063a313ed1820df775166227ec5c20a1122968f85653d5efee48", size = 124745, upload-time = "2026-02-09T15:23:34.745Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/09/58/50df94e9a78e1c77818a492c90557eeb1309af025120c9a21e6375950c52/pytest_codspeed-4.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:527a3a02eaa3e4d4583adc4ba2327eef79628f3e1c682a4b959439551a72588e", size = 347395, upload-time = "2026-02-09T15:23:21.986Z" }, + { url = "https://files.pythonhosted.org/packages/e4/56/7dfbd3eefd112a14e6fb65f9ff31dacf2e9c381cb94b27332b81d2b13f8d/pytest_codspeed-4.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9858c2a6e1f391d5696757e7b6e9484749a7376c46f8b4dd9aebf093479a9667", size = 342625, upload-time = "2026-02-09T15:23:23.035Z" }, + { url = "https://files.pythonhosted.org/packages/7f/53/7255f6a25bc56ff1745b254b21545dfe0be2268f5b91ce78f7e8a908f0ad/pytest_codspeed-4.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34f2fd8497456eefbd325673f677ea80d93bb1bc08a578c1fa43a09cec3d1879", size = 347325, upload-time = "2026-02-09T15:23:23.998Z" }, + { url = "https://files.pythonhosted.org/packages/2e/f8/82ae570d8b9ad30f33c9d4002a7a1b2740de0e090540c69a28e4f711ebe2/pytest_codspeed-4.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df6a36a2a9da1406bc50428437f657f0bd8c842ae54bee5fb3ad30e01d50c0f5", size = 342558, upload-time = "2026-02-09T15:23:25.656Z" }, + { url = "https://files.pythonhosted.org/packages/b3/e1/55cfe9474f91d174c7a4b04d257b5fc6d4d06f3d3680f2da672ee59ccc10/pytest_codspeed-4.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bec30f4fc9c4973143cd80f0d33fa780e9fa3e01e4dbe8cedf229e72f1212c62", size = 347383, upload-time = "2026-02-09T15:23:26.68Z" }, + { url = "https://files.pythonhosted.org/packages/7f/3b/8fd781d959bbe789b3de8ce4c50d5706a684a0df377147dfb27b200c20c1/pytest_codspeed-4.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e6584e641cadf27d894ae90b87c50377232a97cbfd76ee0c7ecd0c056fa3f7f4", size = 342481, upload-time = "2026-02-09T15:23:27.686Z" }, + { url = "https://files.pythonhosted.org/packages/bb/0c/368045133c6effa2c665b1634b7b8a9c88b307f877fa31f1f8df47885b51/pytest_codspeed-4.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df0d1f6ea594f29b745c634d66d5f5f1caa1c3abd2af82fea49d656038e8fc77", size = 353680, upload-time = "2026-02-09T15:23:28.726Z" }, + { url = "https://files.pythonhosted.org/packages/59/21/e543abcd72244294e25ae88ec3a9311ade24d6913f8c8f42569d671700bc/pytest_codspeed-4.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2f5bb6d8898bea7db45e3c8b916ee48e36905b929477bb511b79c5a3ccacda4", size = 347888, upload-time = "2026-02-09T15:23:30.443Z" }, + { url = "https://files.pythonhosted.org/packages/55/d9/b8a53c20cf5b41042c205bb9d36d37da00418d30fd1a94bf9eb147820720/pytest_codspeed-4.3.0-py3-none-any.whl", hash = "sha256:05baff2a61dc9f3e92b92b9c2ab5fb45d9b802438f5373073f5766a91319ed7a", size = 125224, upload-time = "2026-02-09T15:23:33.774Z" }, +] + [[package]] name = "pytest-env" version = "1.6.0"