Accept-Encoding: gzip

.github/workflows/formatting.yml

@@ -14,9 +14,9 @@ jobs:
   ruff-format:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
       - name: Install ruff based off pyproject.toml
-        uses: astral-sh/ruff-action@4919ec5cf1f49eff0871dbcea0da843445b837e6  # v3.6.1
+        uses: astral-sh/ruff-action@0ce1b0bf8b818ef400413f810f8a11cdbda0034b  # v4.0.0
         with:
           args: "format --check --diff"  # only check formatting here
           version-file: "pyproject.toml"

.github/workflows/tests.yml

@@ -29,9 +29,9 @@ jobs:
             platform-version: "ubuntu-22.04"
     runs-on: ${{ matrix.platform-version }}
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -45,18 +45,18 @@ jobs:
   ruff:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
       - name: Install ruff based off pyproject.toml
-        uses: astral-sh/ruff-action@4919ec5cf1f49eff0871dbcea0da843445b837e6  # v3.6.1
+        uses: astral-sh/ruff-action@0ce1b0bf8b818ef400413f810f8a11cdbda0034b  # v4.0.0
         with:
           version-file: "pyproject.toml"
 
   mypy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1  # v6.3.0
         with:
           python-version: "3.x"
       - name: Install dependencies

morgan/__init__.py

@@ -2,6 +2,7 @@
 
 import argparse
 import configparser
+import gzip
 import hashlib
 import json
 import os
@@ -147,35 +148,21 @@ def _mirror(  # noqa: C901, PLR0912
         if self._processed_pkgs.check(requirement):
             return None
 
-        # Check if requirement is relevant for any environment
-        if not is_requirement_relevant(requirement, self.envs.values()):
-            print(f"\tSkipping {requirement}, not relevant for any environment")
-            self._processed_pkgs.add(requirement)  # Mark as processed
-            return None
-
+        # Display the cause of 'Skipping...'
+        extras = None
         if required_by:
+            extras = required_by.extras
             print(f"[{required_by}]: {requirement}")
         else:
             print(f"{requirement}")
 
-        data: dict | None = None
-
-        # get information about this package from the Simple API in JSON
-        # format as per PEP 691
-        request = urllib.request.Request(  # noqa: S310
-            f"{self.index_url}{requirement.name}/",
-            headers={
-                "Accept": "application/vnd.pypi.simple.v1+json",
-            },
-        )
+        # Check if requirement is relevant for any environment
+        if not is_requirement_relevant(requirement, self.envs.values(), extras=extras):
+            print("\tSkipping, not relevant for any environment")
+            self._processed_pkgs.add(requirement)  # Mark as processed
+            return None
 
-        response_url = ""
-        with urllib.request.urlopen(request) as response:  # noqa: S310
-            data = json.load(response)
-            response_url = str(response.url)
-            if not data:
-                msg = f"Failed loading metadata: {response}"
-                raise RuntimeError(msg)
+        data, response_url = download_req(self.index_url, requirement.name)
 
         # check metadata version ~1.0
         v_str = data["meta"]["api-version"]
@@ -848,5 +835,32 @@ def my_url(arg):
         Mirrorer(args).copy_server()
 
 
+def download_req(index_url: str, req_name: str) -> tuple[dict, str]:
+    # get information about this package from the Simple API in JSON
+    # format as per PEP 691
+    url = index_url.rstrip("/")
+    request = urllib.request.Request(  # noqa: S310
+        f"{url}/{req_name}/",
+        headers={
+            "Accept": "application/vnd.pypi.simple.v1+json",
+            "Accept-Encoding": "gzip",
+        },
+    )
+
+    response_url = ""
+    with urllib.request.urlopen(request) as response:  # noqa: S310
+        # Check if response is gzip-encoded
+        if response.headers.get("Content-Encoding") == "gzip":
+            with gzip.GzipFile(fileobj=response) as gzip_response:
+                data = json.load(gzip_response)
+        else:
+            data = json.load(response)
+        response_url = str(response.url)
+        if data:
+            return data, response_url
+        msg = f"Failed loading metadata: {response}"
+        raise RuntimeError(msg)
+
+
 if __name__ == "__main__":
     main()

tests/test_init.py

@@ -2,8 +2,10 @@
 from __future__ import annotations
 
 import argparse
+import contextlib
 import hashlib
 import os
+import urllib.error
 
 import packaging.requirements
 import packaging.version
@@ -376,6 +378,25 @@ def test_filter_files_with_latest_version_mirrored(
 
         assert self.extract_versions(filtered_files) == expected_versions
 
+    def test_skipping(self, make_mirrorer):
+        """Test that requirement not skipping."""
+        mirrorer = make_mirrorer(mirror_all_versions=False)
+        required_by = packaging.requirements.Requirement("pyjwt[crypto]==2.10.1")
+        requirement = packaging.requirements.Requirement(
+            'cryptography>=3.4.0; extra == "crypto"',
+        )
+
+        res = {}
+        with contextlib.suppress(urllib.error.HTTPError):
+            # if skipping then None
+            res = mirrorer._mirror(  # noqa: SLF001
+                requirement=requirement,
+                required_by=required_by,
+            )
+            # if not skipping then HTTPError
+
+        assert res is not None
+
     @pytest.mark.parametrize(
         ("version_spec", "expected_versions"),
         [