diff --git a/.github/workflows/formatting.yml b/.github/workflows/formatting.yml index 9d320fc..47ad016 100644 --- a/.github/workflows/formatting.yml +++ b/.github/workflows/formatting.yml @@ -14,9 +14,9 @@ jobs: ruff-format: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install ruff based off pyproject.toml - uses: astral-sh/ruff-action@4919ec5cf1f49eff0871dbcea0da843445b837e6 # v3.6.1 + uses: astral-sh/ruff-action@0ce1b0bf8b818ef400413f810f8a11cdbda0034b # v4.0.0 with: args: "format --check --diff" # only check formatting here version-file: "pyproject.toml" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 248b1c6..d1c3931 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,9 +29,9 @@ jobs: platform-version: "ubuntu-22.04" runs-on: ${{ matrix.platform-version }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -45,18 +45,18 @@ jobs: ruff: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install ruff based off pyproject.toml - uses: astral-sh/ruff-action@4919ec5cf1f49eff0871dbcea0da843445b837e6 # v3.6.1 + uses: astral-sh/ruff-action@0ce1b0bf8b818ef400413f810f8a11cdbda0034b # v4.0.0 with: version-file: "pyproject.toml" mypy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.x" - name: Install dependencies diff --git a/morgan/__init__.py b/morgan/__init__.py index bb72c79..30aac4a 100644 --- a/morgan/__init__.py +++ b/morgan/__init__.py @@ -2,6 +2,7 @@ import argparse import configparser +import gzip import hashlib import json import os @@ -147,35 +148,21 @@ def _mirror( # noqa: C901, PLR0912 if self._processed_pkgs.check(requirement): return None - # Check if requirement is relevant for any environment - if not is_requirement_relevant(requirement, self.envs.values()): - print(f"\tSkipping {requirement}, not relevant for any environment") - self._processed_pkgs.add(requirement) # Mark as processed - return None - + # Display the cause of 'Skipping...' + extras = None if required_by: + extras = required_by.extras print(f"[{required_by}]: {requirement}") else: print(f"{requirement}") - data: dict | None = None - - # get information about this package from the Simple API in JSON - # format as per PEP 691 - request = urllib.request.Request( # noqa: S310 - f"{self.index_url}{requirement.name}/", - headers={ - "Accept": "application/vnd.pypi.simple.v1+json", - }, - ) + # Check if requirement is relevant for any environment + if not is_requirement_relevant(requirement, self.envs.values(), extras=extras): + print("\tSkipping, not relevant for any environment") + self._processed_pkgs.add(requirement) # Mark as processed + return None - response_url = "" - with urllib.request.urlopen(request) as response: # noqa: S310 - data = json.load(response) - response_url = str(response.url) - if not data: - msg = f"Failed loading metadata: {response}" - raise RuntimeError(msg) + data, response_url = download_req(self.index_url, requirement.name) # check metadata version ~1.0 v_str = data["meta"]["api-version"] @@ -848,5 +835,32 @@ def my_url(arg): Mirrorer(args).copy_server() +def download_req(index_url: str, req_name: str) -> tuple[dict, str]: + # get information about this package from the Simple API in JSON + # format as per PEP 691 + url = index_url.rstrip("/") + request = urllib.request.Request( # noqa: S310 + f"{url}/{req_name}/", + headers={ + "Accept": "application/vnd.pypi.simple.v1+json", + "Accept-Encoding": "gzip", + }, + ) + + response_url = "" + with urllib.request.urlopen(request) as response: # noqa: S310 + # Check if response is gzip-encoded + if response.headers.get("Content-Encoding") == "gzip": + with gzip.GzipFile(fileobj=response) as gzip_response: + data = json.load(gzip_response) + else: + data = json.load(response) + response_url = str(response.url) + if data: + return data, response_url + msg = f"Failed loading metadata: {response}" + raise RuntimeError(msg) + + if __name__ == "__main__": main() diff --git a/tests/test_init.py b/tests/test_init.py index d7b65f4..56b9123 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -2,8 +2,10 @@ from __future__ import annotations import argparse +import contextlib import hashlib import os +import urllib.error import packaging.requirements import packaging.version @@ -376,6 +378,25 @@ def test_filter_files_with_latest_version_mirrored( assert self.extract_versions(filtered_files) == expected_versions + def test_skipping(self, make_mirrorer): + """Test that requirement not skipping.""" + mirrorer = make_mirrorer(mirror_all_versions=False) + required_by = packaging.requirements.Requirement("pyjwt[crypto]==2.10.1") + requirement = packaging.requirements.Requirement( + 'cryptography>=3.4.0; extra == "crypto"', + ) + + res = {} + with contextlib.suppress(urllib.error.HTTPError): + # if skipping then None + res = mirrorer._mirror( # noqa: SLF001 + requirement=requirement, + required_by=required_by, + ) + # if not skipping then HTTPError + + assert res is not None + @pytest.mark.parametrize( ("version_spec", "expected_versions"), [