Disallow new requests after ≥10 and ≥1% undocumented error responses

docs/use/api.rst

@@ -148,32 +148,32 @@ retries for :ref:`rate-limiting <zapi-rate-limit>` and :ref:`unsuccessful
 .. _default-retry-policy:
 
 The default retry policy, :data:`~zyte_api.zyte_api_retrying`, does the
-following:
+following for each request:
 
 -   Retries :ref:`rate-limiting responses <zapi-rate-limit>` forever.
 
--   Retries :ref:`temporary download errors
-    <zapi-temporary-download-errors>` up to 3 times.
+-   Retries :ref:`temporary download errors <zapi-temporary-download-errors>`
+    up to 3 times. :ref:`Permanent download errors
+    <zapi-permanent-download-errors>` also count towards this retry limit.
+
+-   Retries permanent download errors up to 3 times per request.
 
 -   Retries network errors until they have happened for 15 minutes straight.
 
+-   Retries error responses with an HTTP status code in the 500-599 range (503,
+    520 and 521 excluded) up to 3 times.
+
+-   Disallows new requests if undocumented error responses are more than 10
+    *and* more than 1% of all responses.
+
 All retries are done with an exponential backoff algorithm.
 
 .. _aggressive-retry-policy:
 
 If some :ref:`unsuccessful responses <zapi-unsuccessful-responses>` exceed
 maximum retries with the default retry policy, try using
-:data:`~zyte_api.aggressive_retrying` instead, which modifies the default retry
-policy as follows:
-
--   Temporary download error are retried 7 times. :ref:`Permanent download
-    errors <zapi-permanent-download-errors>` also count towards this retry
-    limit.
-
--   Retries permanent download errors up to 3 times.
-
--   Retries error responses with an HTTP status code in the 500-599 range (503,
-    520 and 521 excluded) up to 3 times.
+:data:`~zyte_api.aggressive_retrying` instead, which duplicates attempts for
+all retry scenarios.
 
 Alternatively, the reference documentation of :class:`~zyte_api.RetryFactory`
 and :class:`~zyte_api.AggressiveRetryFactory` features some examples of custom

pyproject.toml

@@ -4,3 +4,6 @@ multi_line_output = 3
 
 [tool.black]
 target-version = ["py39", "py310", "py311", "py312", "py313"]
+
+[tool.mypy]
+check_untyped_defs = true

tests/mockserver.py

@@ -6,10 +6,12 @@
 from base64 import b64encode
 from importlib import import_module
 from subprocess import PIPE, Popen
-from typing import Any, Dict
+from typing import Any, Dict, cast
 from urllib.parse import urlparse
 
 from twisted.internet import reactor
+from twisted.internet.defer import Deferred
+from twisted.internet.interfaces import IReactorTime
 from twisted.internet.task import deferLater
 from twisted.web.resource import Resource
 from twisted.web.server import NOT_DONE_YET, Site
@@ -40,7 +42,7 @@ def _cancelrequest(_):
             d.addErrback(lambda _: None)
             d.cancel()
 
-        d = deferLater(reactor, delay, f, *a, **kw)
+        d: Deferred = deferLater(cast(IReactorTime, reactor), delay, f, *a, **kw)
         request.notifyFinish().addErrback(_cancelrequest)
         return d
 
@@ -82,6 +84,7 @@ def render_POST(self, request):
 
         url = request_data["url"]
         domain = urlparse(url).netloc
+        response_data: Dict[str, Any]
         if domain == "e429.example":
             request.setResponseCode(429)
             response_data = {"status": 429, "type": "/limits/over-user-limit"}
@@ -119,7 +122,7 @@ def render_POST(self, request):
             request.setResponseCode(500)
             return b'["foo"]'
 
-        response_data: Dict[str, Any] = {
+        response_data = {
             "url": url,
         }
 

tests/test_async.py

@@ -3,7 +3,13 @@
 
 import pytest
 
-from zyte_api import AggressiveRetryFactory, AsyncZyteAPI, RequestError
+from zyte_api import (
+    AggressiveRetryFactory,
+    AsyncZyteAPI,
+    RequestError,
+    TooManyUndocumentedErrors,
+)
+from zyte_api._retry import ZyteAsyncRetrying
 from zyte_api.aio.client import AsyncClient
 from zyte_api.apikey import NoApiKey
 from zyte_api.errors import ParsedError
@@ -318,4 +324,18 @@ def test_retrying_class():
     """A descriptive exception is raised when creating a client with an
     AsyncRetrying subclass or similar instead of an instance of it."""
     with pytest.raises(ValueError):
-        AsyncZyteAPI(api_key="foo", retrying=AggressiveRetryFactory)
+        AsyncZyteAPI(api_key="foo", retrying=AggressiveRetryFactory)  # type: ignore[arg-type]
+
+
+@pytest.mark.asyncio
+async def test_too_many_undocumented_errors(mockserver):
+    ZyteAsyncRetrying._total_outcomes = 9
+    ZyteAsyncRetrying._total_undocumented_errors = 9
+
+    client = AsyncZyteAPI(api_key="a", api_url=mockserver.urljoin("/"))
+
+    await client.get({"url": "https://a.example", "httpResponseBody": True})
+    with pytest.raises(TooManyUndocumentedErrors):
+        await client.get({"url": "https://e500.example", "httpResponseBody": True})
+    with pytest.raises(TooManyUndocumentedErrors):
+        await client.get({"url": "https://a.example", "httpResponseBody": True})

tests/test_main.py

@@ -11,15 +11,6 @@
 from zyte_api.aio.errors import RequestError
 
 
-class MockRequestError(Exception):
-    @property
-    def parsed(self):
-        mock = Mock(
-            response_body=Mock(decode=Mock(return_value=forbidden_domain_response()))
-        )
-        return mock
-
-
 def get_json_content(file_object):
     if not file_object:
         return
@@ -53,7 +44,12 @@ def forbidden_domain_response():
 async def fake_exception(value=True):
     # Simulating an error condition
     if value:
-        raise MockRequestError()
+        raise RequestError(
+            query={"url": "https://example.com", "httpResponseBody": True},
+            response_content=json.dumps(forbidden_domain_response()).encode(),
+            request_info=None,
+            history=None,
+        )
 
     create_session_mock = AsyncMock()
     return await create_session_mock.coroutine()