From 7e17df8a028a2a3074c8e0c47312b2ef42c377d6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 22 Sep 2025 23:15:48 +0500 Subject: [PATCH 01/13] Typing improvements, 1st pass. --- .pre-commit-config.yaml | 2 +- pyproject.toml | 11 ++++++++ setup.py | 4 +++ tests/test_async.py | 8 +++++- tests/test_main.py | 3 ++- tests/test_retry.py | 4 +-- tests/test_sync.py | 1 + tox.ini | 10 ++++--- zyte_api/__main__.py | 14 +++++----- zyte_api/_async.py | 59 ++++++++++++++++++++++++----------------- zyte_api/_errors.py | 11 ++++---- zyte_api/_sync.py | 36 ++++++++++++++----------- zyte_api/_utils.py | 7 +++-- zyte_api/_x402.py | 10 +++---- zyte_api/py.typed | 0 zyte_api/stats.py | 43 +++++++++++++++++------------- zyte_api/utils.py | 8 +++--- 17 files changed, 141 insertions(+), 90 deletions(-) create mode 100644 zyte_api/py.typed diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4695580..5fd86f9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.6 + rev: v0.11.4 hooks: - id: ruff args: [ --fix ] diff --git a/pyproject.toml b/pyproject.toml index 72cae9a..b7f7ffb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,17 @@ exclude_also = [ "if TYPE_CHECKING:", ] +[tool.mypy] +#allow_untyped_defs = false + +[[tool.mypy.overrides]] +module = "runstats" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "tests.*" +allow_untyped_defs = true + [tool.pytest.ini_options] filterwarnings = [ "ignore:The zyte_api\\.aio module is deprecated:DeprecationWarning" diff --git a/setup.py b/setup.py index 0aa870f..d8b1381 100755 --- a/setup.py +++ b/setup.py @@ -12,6 +12,10 @@ author_email="opensource@zyte.com", url="https://github.com/zytedata/python-zyte-api", packages=find_packages(exclude=["tests", "examples"]), + package_data={ + "zyte_api": ["py.typed"], + }, + include_package_data=True, entry_points={ "console_scripts": ["zyte-api=zyte_api.__main__:_main"], }, diff --git a/tests/test_async.py b/tests/test_async.py index f54eff9..2e63007 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -1,4 +1,7 @@ +from __future__ import annotations + import asyncio +from typing import Any from unittest.mock import AsyncMock import pytest @@ -236,11 +239,13 @@ async def test_session_context_manager(mockserver): "httpResponseBody": "PGh0bWw+PGJvZHk+SGVsbG88aDE+V29ybGQhPC9oMT48L2JvZHk+PC9odG1sPg==", }, ] - actual_results = [] + actual_results: list[dict[str, Any] | Exception] = [] async with client.session() as session: + assert session._session.connector is not None assert session._session.connector.limit == client.n_conn actual_results.append(await session.get(queries[0])) for future in session.iter(queries[1:]): + result: dict[str, Any] | Exception try: result = await future except Exception as e: @@ -286,6 +291,7 @@ async def test_session_no_context_manager(mockserver): ] actual_results = [] session = client.session() + assert session._session.connector is not None assert session._session.connector.limit == client.n_conn actual_results.append(await session.get(queries[0])) for future in session.iter(queries[1:]): diff --git a/tests/test_main.py b/tests/test_main.py index a30cbdc..4af539f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -3,6 +3,7 @@ from json import JSONDecodeError from pathlib import Path from tempfile import NamedTemporaryFile +from typing import Any from unittest.mock import AsyncMock, Mock, patch import pytest @@ -41,7 +42,7 @@ def get_json_content(file_object): pass -def forbidden_domain_response(): +def forbidden_domain_response() -> dict[str, Any]: return { "type": "/download/temporary-error", "title": "Temporary Downloading Error", diff --git a/tests/test_retry.py b/tests/test_retry.py index 8bf8f7f..70d7df7 100644 --- a/tests/test_retry.py +++ b/tests/test_retry.py @@ -121,7 +121,7 @@ class CustomRetryFactory(retry_factory): ) -def mock_request_error(*, status=200): +def mock_request_error(*, status: int = 200) -> RequestError: return RequestError( history=None, request_info=None, @@ -136,7 +136,7 @@ def mock_request_error(*, status=200): class fast_forward: - def __init__(self, time): + def __init__(self, time: float): self.time = time diff --git a/tests/test_sync.py b/tests/test_sync.py index 8cb0d7a..86bc504 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -87,6 +87,7 @@ def test_session_context_manager(mockserver): ] actual_results = [] with client.session() as session: + assert session._session.connector is not None assert session._session.connector.limit == client._async_client.n_conn actual_results.append(session.get(queries[0])) actual_results.extend(session.iter(queries[1:])) diff --git a/tox.ini b/tox.ini index dce2440..8c54a61 100644 --- a/tox.ini +++ b/tox.ini @@ -43,12 +43,14 @@ deps = [testenv:mypy] deps = - mypy==1.12.0 + mypy==1.18.2 + eth-account==0.13.7 pytest==8.3.3 - Twisted==24.7.0 - types-tqdm==4.66.0.20240417 + Twisted==25.5.0 + types-tqdm==4.67.0.20250809 + x402==0.2.1 -commands = mypy --ignore-missing-imports \ +commands = mypy \ zyte_api \ tests diff --git a/zyte_api/__main__.py b/zyte_api/__main__.py index 060115a..c58cffd 100644 --- a/zyte_api/__main__.py +++ b/zyte_api/__main__.py @@ -34,14 +34,14 @@ async def run( queries, out, *, - n_conn, + n_conn: int, stop_on_errors=_UNSET, api_url: str | None, - api_key=None, - retry_errors=True, + api_key: str | None = None, + retry_errors: bool = True, store_errors=None, eth_key=None, -): +) -> None: if stop_on_errors is not _UNSET: warn( "The stop_on_errors parameter is deprecated.", @@ -51,7 +51,7 @@ async def run( else: stop_on_errors = False - def write_output(content): + def write_output(content) -> None: json.dump(content, out, ensure_ascii=False) out.write("\n") out.flush() @@ -118,7 +118,7 @@ def read_input(input_fp, intype): return records -def _get_argument_parser(program_name="zyte-api"): +def _get_argument_parser(program_name: str = "zyte-api") -> argparse.ArgumentParser: p = argparse.ArgumentParser( prog=program_name, description="Send Zyte API requests.", @@ -225,7 +225,7 @@ def _get_argument_parser(program_name="zyte-api"): return p -def _main(program_name="zyte-api"): +def _main(program_name: str = "zyte-api") -> None: """Process urls from input file through Zyte API""" p = _get_argument_parser(program_name=program_name) args = p.parse_args() diff --git a/zyte_api/_async.py b/zyte_api/_async.py index b19df17..32566c0 100644 --- a/zyte_api/_async.py +++ b/zyte_api/_async.py @@ -2,10 +2,9 @@ import asyncio import time -from asyncio import Future from functools import partial from os import environ -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast from warnings import warn import aiohttp @@ -23,12 +22,20 @@ from .utils import USER_AGENT, _process_query if TYPE_CHECKING: - from collections.abc import Iterator + from collections.abc import Awaitable, Callable, Iterator + from contextlib import AbstractAsyncContextManager - _ResponseFuture = Future[dict[str, Any]] + from eth_account.account import LocalAccount + # typing.Self requires Python 3.11 + from typing_extensions import Self -def _post_func(session): + _ResponseFuture = Awaitable[dict[str, Any]] + + +def _post_func( + session: aiohttp.ClientSession | None, +) -> Callable[..., AbstractAsyncContextManager[aiohttp.ClientResponse]]: """Return a function to send a POST request""" if session is None: return partial(aiohttp.request, method="POST", timeout=_AIO_API_TIMEOUT) @@ -36,17 +43,19 @@ def _post_func(session): class _AsyncSession: - def __init__(self, client, **session_kwargs): - self._client = client - self._session = create_session(client.n_conn, **session_kwargs) + def __init__(self, client: AsyncZyteAPI, **session_kwargs: Any): + self._client: AsyncZyteAPI = client + self._session: aiohttp.ClientSession = create_session( + client.n_conn, **session_kwargs + ) - async def __aenter__(self): + async def __aenter__(self) -> Self: return self - async def __aexit__(self, *exc_info): + async def __aexit__(self, *exc_info) -> None: await self._session.close() - async def close(self): + async def close(self) -> None: await self._session.close() async def get( @@ -54,9 +63,9 @@ async def get( query: dict, *, endpoint: str = "extract", - handle_retries=True, + handle_retries: bool = True, retrying: AsyncRetrying | None = None, - ): + ) -> dict[str, Any]: return await self._client.get( query=query, endpoint=endpoint, @@ -70,9 +79,9 @@ def iter( queries: list[dict], *, endpoint: str = "extract", - handle_retries=True, + handle_retries: bool = True, retrying: AsyncRetrying | None = None, - ) -> Iterator[Future]: + ) -> Iterator[_ResponseFuture]: return self._client.iter( queries=queries, endpoint=endpoint, @@ -91,7 +100,7 @@ def key(self) -> str: if isinstance(self._auth, str): return self._auth assert isinstance(self._auth, _x402Handler) - return self._auth.client.account.key.hex() + return cast("LocalAccount", self._auth.client.account).key.hex() @property def type(self) -> str: @@ -172,13 +181,13 @@ def api_key(self) -> str: async def get( self, - query: dict, + query: dict[str, Any], *, endpoint: str = "extract", - session=None, - handle_retries=True, + session: aiohttp.ClientSession | None = None, + handle_retries: bool = True, retrying: AsyncRetrying | None = None, - ) -> _ResponseFuture: + ) -> dict[str, Any]: """Asynchronous equivalent to :meth:`ZyteAPI.get`.""" retrying = retrying or self.retrying post = _post_func(session) @@ -204,7 +213,7 @@ async def get( response_stats = [] start_global = time.perf_counter() - async def request(): + async def request() -> dict[str, Any]: stats = ResponseStats.create(start_global) self.agg_stats.n_attempts += 1 @@ -233,7 +242,7 @@ async def request(): query=query, ) - response = await resp.json() + response = cast("dict[str, Any]", await resp.json()) stats.record_read(self.agg_stats) return response except Exception as e: @@ -263,7 +272,7 @@ def iter( *, endpoint: str = "extract", session: aiohttp.ClientSession | None = None, - handle_retries=True, + handle_retries: bool = True, retrying: AsyncRetrying | None = None, ) -> Iterator[_ResponseFuture]: """Asynchronous equivalent to :meth:`ZyteAPI.iter`. @@ -272,7 +281,7 @@ def iter( instead of only returning them. """ - def _request(query): + def _request(query: dict) -> _ResponseFuture: return self.get( query, endpoint=endpoint, @@ -283,7 +292,7 @@ def _request(query): return asyncio.as_completed([_request(query) for query in queries]) - def session(self, **kwargs): + def session(self, **kwargs: Any) -> _AsyncSession: """Asynchronous equivalent to :meth:`ZyteAPI.session`. You do not need to use :meth:`~AsyncZyteAPI.session` as an async diff --git a/zyte_api/_errors.py b/zyte_api/_errors.py index 5c13955..aab6b2b 100644 --- a/zyte_api/_errors.py +++ b/zyte_api/_errors.py @@ -15,7 +15,7 @@ class RequestError(ClientResponseError): ` or :ref:`unsuccessful ` response from Zyte API.""" - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any): #: Query sent to Zyte API. #: #: May be slightly different from the input query due to @@ -31,13 +31,14 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @property - def parsed(self): + def parsed(self) -> ParsedError: """Response as a :class:`ParsedError` object.""" - return ParsedError.from_body(self.response_content) + # TODO: self.response_content can be None but ParsedError doesn't expect it + return ParsedError.from_body(self.response_content) # type: ignore[arg-type] - def __str__(self): + def __str__(self) -> str: return ( f"RequestError: {self.status}, message={self.message}, " - f"headers={self.headers}, body={self.response_content}, " + f"headers={self.headers}, body={self.response_content!r}, " f"request_id={self.request_id}" ) diff --git a/zyte_api/_sync.py b/zyte_api/_sync.py index 5a318e4..7d40b3d 100644 --- a/zyte_api/_sync.py +++ b/zyte_api/_sync.py @@ -1,7 +1,8 @@ from __future__ import annotations import asyncio -from typing import TYPE_CHECKING +from asyncio import AbstractEventLoop +from typing import TYPE_CHECKING, Any from ._async import AsyncZyteAPI @@ -11,8 +12,11 @@ from aiohttp import ClientSession from tenacity import AsyncRetrying + # typing.Self requires Python 3.11 + from typing_extensions import Self -def _get_loop(): + +def _get_loop() -> AbstractEventLoop: try: return asyncio.get_event_loop() except RuntimeError: # pragma: no cover (tests always have a running loop) @@ -22,24 +26,24 @@ def _get_loop(): class _Session: - def __init__(self, client, **session_kwargs): - self._client = client + def __init__(self, client: ZyteAPI, **session_kwargs: Any): + self._client: ZyteAPI = client # https://github.com/aio-libs/aiohttp/pull/1468 - async def create_session(): + async def create_session() -> ClientSession: return client._async_client.session(**session_kwargs)._session loop = _get_loop() - self._session = loop.run_until_complete(create_session()) + self._session: ClientSession = loop.run_until_complete(create_session()) - def __enter__(self): + def __enter__(self) -> Self: return self - def __exit__(self, *exc_info): + def __exit__(self, *exc_info) -> None: loop = _get_loop() loop.run_until_complete(self._session.close()) - def close(self): + def close(self) -> None: loop = _get_loop() loop.run_until_complete(self._session.close()) @@ -48,9 +52,9 @@ def get( query: dict, *, endpoint: str = "extract", - handle_retries=True, + handle_retries: bool = True, retrying: AsyncRetrying | None = None, - ): + ) -> dict[str, Any]: return self._client.get( query=query, endpoint=endpoint, @@ -64,9 +68,9 @@ def iter( queries: list[dict], *, endpoint: str = "extract", - handle_retries=True, + handle_retries: bool = True, retrying: AsyncRetrying | None = None, - ) -> Generator[dict | Exception, None, None]: + ) -> Generator[dict[str, Any] | Exception, None, None]: return self._client.iter( queries=queries, endpoint=endpoint, @@ -131,7 +135,7 @@ def get( session: ClientSession | None = None, handle_retries: bool = True, retrying: AsyncRetrying | None = None, - ) -> dict: + ) -> dict[str, Any]: """Send *query* to Zyte API and return the result. *endpoint* is the Zyte API endpoint path relative to the client object @@ -165,7 +169,7 @@ def iter( session: ClientSession | None = None, handle_retries: bool = True, retrying: AsyncRetrying | None = None, - ) -> Generator[dict | Exception, None, None]: + ) -> Generator[dict[str, Any] | Exception, None, None]: """Send multiple *queries* to Zyte API in parallel and iterate over their results as they come. @@ -194,7 +198,7 @@ def iter( except Exception as exception: yield exception - def session(self, **kwargs): + def session(self, **kwargs: Any) -> _Session: """:ref:`Context manager ` to create a session. A session is an object that has the same API as the client object, diff --git a/zyte_api/_utils.py b/zyte_api/_utils.py index 025c598..06d7056 100644 --- a/zyte_api/_utils.py +++ b/zyte_api/_utils.py @@ -1,3 +1,4 @@ +from typing import Any from warnings import warn import aiohttp @@ -12,7 +13,7 @@ def deprecated_create_session( - connection_pool_size=100, **kwargs + connection_pool_size: int = 100, **kwargs: Any ) -> aiohttp.ClientSession: warn( ( @@ -25,7 +26,9 @@ def deprecated_create_session( return create_session(connection_pool_size=connection_pool_size, **kwargs) -def create_session(connection_pool_size=100, **kwargs) -> aiohttp.ClientSession: +def create_session( + connection_pool_size: int = 100, **kwargs: Any +) -> aiohttp.ClientSession: """Create a session with parameters suited for Zyte API""" kwargs.setdefault("timeout", _AIO_API_TIMEOUT) if "connector" not in kwargs: diff --git a/zyte_api/_x402.py b/zyte_api/_x402.py index b7d506c..49c3457 100644 --- a/zyte_api/_x402.py +++ b/zyte_api/_x402.py @@ -20,7 +20,7 @@ from zyte_api.stats import AggStats -CACHE: dict[bytes, tuple[Any, str]] = {} +CACHE: dict[bytes, tuple[Any, int]] = {} EXTRACT_KEYS = { "article", "articleList", @@ -131,7 +131,7 @@ async def get_headers( return self.get_headers_from_requirement_data(requirement_data) def get_headers_from_requirement_data( - self, requirement_data: tuple[Any, str] + self, requirement_data: tuple[Any, int] ) -> dict[str, str]: payment_header = self.client.create_payment_header(*requirement_data) return { @@ -145,7 +145,7 @@ async def get_requirement_data( query: dict[str, Any], headers: dict[str, str], post_fn: Callable[..., AbstractAsyncContextManager[ClientResponse]], - ) -> tuple[Any, str]: + ) -> tuple[Any, int]: if not MINIMIZE_REQUESTS: return await self.fetch_requirements(url, query, headers, post_fn) max_cost_hash = get_max_cost_hash(query) @@ -161,7 +161,7 @@ async def fetch_requirements( query: dict[str, Any], headers: dict[str, str], post_fn: Callable[..., AbstractAsyncContextManager[ClientResponse]], - ) -> tuple[Any, str]: + ) -> tuple[Any, int]: post_kwargs = {"url": url, "json": query, "headers": headers} async def request(): @@ -185,7 +185,7 @@ async def request(): data = await request() return self.parse_requirements(data) - def parse_requirements(self, data: dict[str, Any]) -> tuple[Any, str]: + def parse_requirements(self, data: dict[str, Any]) -> tuple[Any, int]: payment_response = self.x402PaymentRequiredResponse(**data) requirements = self.client.select_payment_requirements(payment_response.accepts) version = payment_response.x402_version diff --git a/zyte_api/py.typed b/zyte_api/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/zyte_api/stats.py b/zyte_api/stats.py index e52edfb..8ecdf5c 100644 --- a/zyte_api/stats.py +++ b/zyte_api/stats.py @@ -3,17 +3,24 @@ import functools import time from collections import Counter -from typing import Optional +from typing import TYPE_CHECKING, Callable, Optional import attr from runstats import Statistics from zyte_api.errors import ParsedError +if TYPE_CHECKING: + # typing.ParamSpec requires Python 3.10 + # typing.Self requires Python 3.11 + from typing_extensions import ParamSpec, Self -def zero_on_division_error(meth): + _P = ParamSpec("_P") + + +def zero_on_division_error(meth: Callable[_P, float]) -> Callable[_P, float]: @functools.wraps(meth) - def wrapper(*args, **kwargs): + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> float: try: return meth(*args, **kwargs) except ZeroDivisionError: @@ -23,7 +30,7 @@ def wrapper(*args, **kwargs): class AggStats: - def __init__(self): + def __init__(self) -> None: self.time_connect_stats = Statistics() self.time_total_stats = Statistics() @@ -39,11 +46,11 @@ def __init__(self): self.n_errors = 0 # number of errors, including errors which were retried self.n_402_req = 0 # requests for a 402 (payment required) response - self.status_codes = Counter() - self.exception_types = Counter() - self.api_error_types = Counter() + self.status_codes: Counter[int] = Counter() + self.exception_types: Counter[type] = Counter() + self.api_error_types: Counter[str | None] = Counter() - def __str__(self): + def __str__(self) -> str: return ( f"conn:{self.time_connect_stats.mean():0.2f}s, " f"resp:{self.time_total_stats.mean():0.2f}s, " @@ -52,7 +59,7 @@ def __str__(self): f"success:{self.n_success}/{self.n_processed}({self.success_ratio():.1%})" ) - def summary(self): + def summary(self) -> str: return ( "\n" "Summary\n" @@ -67,19 +74,19 @@ def summary(self): ) @zero_on_division_error - def throttle_ratio(self): + def throttle_ratio(self) -> float: return self.n_429 / self.n_attempts @zero_on_division_error - def error_ratio(self): + def error_ratio(self) -> float: return self.n_errors / self.n_attempts @zero_on_division_error - def success_ratio(self): + def success_ratio(self) -> float: return self.n_success / self.n_processed @property - def n_processed(self): + def n_processed(self) -> float: """Total number of processed URLs""" return self.n_success + self.n_fatal_errors @@ -114,33 +121,33 @@ class ResponseStats: exception: Optional[Exception] = attr.ib(default=None) @classmethod - def create(cls, start_global): + def create(cls, start_global: float) -> Self: start = time.perf_counter() return cls( start=start, time_delayed=start - start_global, ) - def record_connected(self, status: int, agg_stats: AggStats): + def record_connected(self, status: int, agg_stats: AggStats) -> None: self.status = status self.time_connect = time.perf_counter() - self._start agg_stats.time_connect_stats.push(self.time_connect) agg_stats.status_codes[self.status] += 1 - def record_read(self, agg_stats: AggStats | None = None): + def record_read(self, agg_stats: AggStats | None = None) -> None: now = time.perf_counter() self.time_total = now - self._start self.time_read = self.time_total - (self.time_connect or 0) if agg_stats: agg_stats.time_total_stats.push(self.time_total) - def record_exception(self, exception: Exception, agg_stats: AggStats): + def record_exception(self, exception: Exception, agg_stats: AggStats) -> None: self.time_exception = time.perf_counter() - self._start self.exception = exception agg_stats.status_codes[0] += 1 agg_stats.exception_types[exception.__class__] += 1 - def record_request_error(self, error_body: bytes, agg_stats: AggStats): + def record_request_error(self, error_body: bytes, agg_stats: AggStats) -> None: self.error = ParsedError.from_body(error_body) if self.status == 429: # XXX: status must be set already! diff --git a/zyte_api/utils.py b/zyte_api/utils.py index 3287f19..8681599 100644 --- a/zyte_api/utils.py +++ b/zyte_api/utils.py @@ -1,5 +1,7 @@ import re +from collections.abc import Sequence from pathlib import Path +from typing import Any from w3lib.url import safe_url_string @@ -8,7 +10,7 @@ USER_AGENT = f"python-zyte-api/{__version__}" -def _guess_intype(file_name, lines): +def _guess_intype(file_name: str, lines: Sequence[str]) -> str: extension = Path(file_name).suffix[1:] if extension in {"jl", "jsonl"}: return "jl" @@ -21,7 +23,7 @@ def _guess_intype(file_name, lines): return "txt" -def _process_query(query): +def _process_query(query: dict[str, Any]) -> dict[str, Any]: """Given a query to be sent to Zyte API, return a functionally-equivalent query that fixes any known issue. @@ -34,7 +36,7 @@ def _process_query(query): changes where needed, or a shallow copy of *query* with some common nested objects (e.g. shared ``actions`` list). """ - url = query.get("url", None) + url = query.get("url") if url is None: return query if not isinstance(url, str): From d8c9648af8a002e11263c4a8bac14dd2202418cf Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 22 Sep 2025 23:26:31 +0500 Subject: [PATCH 02/13] Bump ruff. --- .pre-commit-config.yaml | 4 ++-- pyproject.toml | 4 ++-- tests/conftest.py | 2 +- tests/test_retry.py | 10 +++++++--- tests/test_utils.py | 2 +- tests/test_x402.py | 2 +- zyte_api/_x402.py | 6 +++--- 7 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5fd86f9..af79841 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,8 +1,8 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.4 + rev: v0.13.1 hooks: - - id: ruff + - id: ruff-check args: [ --fix ] - id: ruff-format - repo: https://github.com/adamchainz/blacken-docs diff --git a/pyproject.toml b/pyproject.toml index b7f7ffb..e850863 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -162,8 +162,8 @@ ignore = [ "tests/*" = ["S"] "docs/**" = ["B006"] # Skip PEP 604 suggestions for files with attr classes -"zyte_api/errors.py" = ["UP007"] -"zyte_api/stats.py" = ["UP007"] +"zyte_api/errors.py" = ["UP007", "UP045"] +"zyte_api/stats.py" = ["UP007", "UP045"] [tool.ruff.lint.flake8-pytest-style] parametrize-values-type = "tuple" diff --git a/tests/conftest.py b/tests/conftest.py index db2b302..5ea1e5f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,7 @@ @pytest.fixture(scope="session") def mockserver(): - from .mockserver import MockServer + from .mockserver import MockServer # noqa: PLC0415 with MockServer() as server: yield server diff --git a/tests/test_retry.py b/tests/test_retry.py index 70d7df7..b3ed67c 100644 --- a/tests/test_retry.py +++ b/tests/test_retry.py @@ -19,9 +19,13 @@ def test_deprecated_imports(): - from zyte_api import RetryFactory, zyte_api_retrying - from zyte_api.aio.retry import RetryFactory as DeprecatedRetryFactory - from zyte_api.aio.retry import zyte_api_retrying as deprecated_zyte_api_retrying + from zyte_api import RetryFactory, zyte_api_retrying # noqa: PLC0415 + from zyte_api.aio.retry import ( # noqa: PLC0415 + RetryFactory as DeprecatedRetryFactory, + ) + from zyte_api.aio.retry import ( # noqa: PLC0415 + zyte_api_retrying as deprecated_zyte_api_retrying, + ) assert RetryFactory is DeprecatedRetryFactory assert zyte_api_retrying is deprecated_zyte_api_retrying diff --git a/tests/test_utils.py b/tests/test_utils.py index 01e59ed..a9092f7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -115,7 +115,7 @@ def test_process_query_bytes(): @pytest.mark.asyncio # https://github.com/aio-libs/aiohttp/pull/1468 async def test_deprecated_create_session(): - from zyte_api.aio.client import create_session as _create_session + from zyte_api.aio.client import create_session as _create_session # noqa: PLC0415 with pytest.warns( DeprecationWarning, diff --git a/tests/test_x402.py b/tests/test_x402.py index b309a83..c827ad1 100644 --- a/tests/test_x402.py +++ b/tests/test_x402.py @@ -50,7 +50,7 @@ def test_eth_key_short(): @contextlib.contextmanager def reset_x402_cache(): - from zyte_api import _x402 + from zyte_api import _x402 # noqa: PLC0415 try: yield _x402.CACHE diff --git a/zyte_api/_x402.py b/zyte_api/_x402.py index 49c3457..890c984 100644 --- a/zyte_api/_x402.py +++ b/zyte_api/_x402.py @@ -110,9 +110,9 @@ def __init__( semaphore: Semaphore, stats: AggStats, ): - from eth_account import Account - from x402.clients import x402Client - from x402.types import x402PaymentRequiredResponse + from eth_account import Account # noqa: PLC0415 + from x402.clients import x402Client # noqa: PLC0415 + from x402.types import x402PaymentRequiredResponse # noqa: PLC0415 account = Account.from_key(eth_key) self.client = x402Client(account=account) From 3ec4acd044828338d8625796e0b9befedbcab16f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 22 Sep 2025 23:41:18 +0500 Subject: [PATCH 03/13] Complete the typing. --- pyproject.toml | 2 +- zyte_api/__main__.py | 17 ++++++++++------- zyte_api/_async.py | 20 ++++++++++---------- zyte_api/_retry.py | 4 ++-- zyte_api/_sync.py | 2 +- zyte_api/_x402.py | 2 +- 6 files changed, 25 insertions(+), 22 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e850863..5aec0d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ exclude_also = [ ] [tool.mypy] -#allow_untyped_defs = false +allow_untyped_defs = false [[tool.mypy.overrides]] module = "runstats" diff --git a/zyte_api/__main__.py b/zyte_api/__main__.py index c58cffd..d6154a4 100644 --- a/zyte_api/__main__.py +++ b/zyte_api/__main__.py @@ -8,6 +8,7 @@ import logging import random import sys +from typing import Any, Literal, TextIO from warnings import warn import tqdm @@ -31,16 +32,16 @@ class DontRetryErrorsFactory(RetryFactory): async def run( - queries, - out, + queries: list[dict[str, Any]], + out: TextIO, *, n_conn: int, - stop_on_errors=_UNSET, + stop_on_errors: bool | object = _UNSET, api_url: str | None, api_key: str | None = None, retry_errors: bool = True, - store_errors=None, - eth_key=None, + store_errors: bool | None = None, + eth_key: str | None = None, ) -> None: if stop_on_errors is not _UNSET: warn( @@ -51,7 +52,7 @@ async def run( else: stop_on_errors = False - def write_output(content) -> None: + def write_output(content: Any) -> None: json.dump(content, out, ensure_ascii=False) out.write("\n") out.flush() @@ -99,7 +100,9 @@ def write_output(content) -> None: logger.info(f"\nException types:\n{client.agg_stats.exception_types.most_common()}") -def read_input(input_fp, intype): +def read_input( + input_fp: TextIO, intype: Literal["txt", "jl"] | object +) -> list[dict[str, Any]]: assert intype in {"txt", "jl", _UNSET} lines = input_fp.readlines() if not lines: diff --git a/zyte_api/_async.py b/zyte_api/_async.py index 32566c0..d4f8d65 100644 --- a/zyte_api/_async.py +++ b/zyte_api/_async.py @@ -52,7 +52,7 @@ def __init__(self, client: AsyncZyteAPI, **session_kwargs: Any): async def __aenter__(self) -> Self: return self - async def __aexit__(self, *exc_info) -> None: + async def __aexit__(self, *exc_info: object) -> None: await self._session.close() async def close(self) -> None: @@ -60,7 +60,7 @@ async def close(self) -> None: async def get( self, - query: dict, + query: dict[str, Any], *, endpoint: str = "extract", handle_retries: bool = True, @@ -76,7 +76,7 @@ async def get( def iter( self, - queries: list[dict], + queries: list[dict[str, Any]], *, endpoint: str = "extract", handle_retries: bool = True, @@ -92,21 +92,19 @@ def iter( class AuthInfo: - def __init__(self, *, _auth): - self._auth = _auth + def __init__(self, *, _auth: str | _x402Handler): + self._auth: str | _x402Handler = _auth @property def key(self) -> str: if isinstance(self._auth, str): return self._auth - assert isinstance(self._auth, _x402Handler) return cast("LocalAccount", self._auth.client.account).key.hex() @property def type(self) -> str: if isinstance(self._auth, str): return "zyte" - assert isinstance(self._auth, _x402Handler) return "eth" @@ -142,7 +140,9 @@ def __init__( self.api_url: str self._load_auth(api_key, eth_key, api_url) - def _load_auth(self, api_key: str | None, eth_key: str | None, api_url: str | None): + def _load_auth( + self, api_key: str | None, eth_key: str | None, api_url: str | None + ) -> None: if api_key: self._auth = api_key elif eth_key: @@ -268,7 +268,7 @@ async def request() -> dict[str, Any]: def iter( self, - queries: list[dict], + queries: list[dict[str, Any]], *, endpoint: str = "extract", session: aiohttp.ClientSession | None = None, @@ -281,7 +281,7 @@ def iter( instead of only returning them. """ - def _request(query: dict) -> _ResponseFuture: + def _request(query: dict[str, Any]) -> _ResponseFuture: return self.get( query, endpoint=endpoint, diff --git a/zyte_api/_retry.py b/zyte_api/_retry.py index 430d380..f0507d2 100644 --- a/zyte_api/_retry.py +++ b/zyte_api/_retry.py @@ -5,7 +5,7 @@ from collections import Counter from datetime import timedelta from itertools import count -from typing import Callable, Union +from typing import Any, Callable, Union from warnings import warn from aiohttp import client_exceptions @@ -165,7 +165,7 @@ def _402_error(exc: BaseException) -> bool: def _deprecated(message: str, callable: Callable) -> Callable: - def wrapper(factory, retry_state: RetryCallState): + def wrapper(factory: Any, retry_state: RetryCallState) -> Callable: warn(message, DeprecationWarning, stacklevel=3) return callable(retry_state=retry_state) diff --git a/zyte_api/_sync.py b/zyte_api/_sync.py index 7d40b3d..79ac816 100644 --- a/zyte_api/_sync.py +++ b/zyte_api/_sync.py @@ -39,7 +39,7 @@ async def create_session() -> ClientSession: def __enter__(self) -> Self: return self - def __exit__(self, *exc_info) -> None: + def __exit__(self, *exc_info: object) -> None: loop = _get_loop() loop.run_until_complete(self._session.close()) diff --git a/zyte_api/_x402.py b/zyte_api/_x402.py index 890c984..9fa1748 100644 --- a/zyte_api/_x402.py +++ b/zyte_api/_x402.py @@ -164,7 +164,7 @@ async def fetch_requirements( ) -> tuple[Any, int]: post_kwargs = {"url": url, "json": query, "headers": headers} - async def request(): + async def request() -> dict[str, Any]: self.stats.n_402_req += 1 async with self.semaphore, post_fn(**post_kwargs) as response: if response.status == 402: From f8a5ea62b2083750a55725649740b2f7b6371529 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 22 Sep 2025 23:49:31 +0500 Subject: [PATCH 04/13] More typing for tests. --- tests/mockserver.py | 3 ++- tests/test_async.py | 5 +++-- tests/test_sync.py | 8 ++++++-- zyte_api/__main__.py | 6 +++--- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index cf3fb11..aa3a602 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -107,6 +107,7 @@ def render_POST(self, request): ) request_data = json.loads(request.content.read()) + response_data: dict[str, Any] url = request_data["url"] domain = urlparse(url).netloc @@ -214,7 +215,7 @@ def render_POST(self, request): } return json.dumps(response_data).encode() - response_data: dict[str, Any] = { + response_data = { "url": url, } diff --git a/tests/test_async.py b/tests/test_async.py index 2e63007..08b4069 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -289,7 +289,8 @@ async def test_session_no_context_manager(mockserver): "httpResponseBody": "PGh0bWw+PGJvZHk+SGVsbG88aDE+V29ybGQhPC9oMT48L2JvZHk+PC9odG1sPg==", }, ] - actual_results = [] + actual_results: list[dict[str, Any] | Exception] = [] + result: dict[str, Any] | Exception session = client.session() assert session._session.connector is not None assert session._session.connector.limit == client.n_conn @@ -324,4 +325,4 @@ def test_retrying_class(): """A descriptive exception is raised when creating a client with an AsyncRetrying subclass or similar instead of an instance of it.""" with pytest.raises(ValueError, match="must be an instance of AsyncRetrying"): - AsyncZyteAPI(api_key="foo", retrying=AggressiveRetryFactory) + AsyncZyteAPI(api_key="foo", retrying=AggressiveRetryFactory) # type: ignore[arg-type] diff --git a/tests/test_sync.py b/tests/test_sync.py index 86bc504..f8e6145 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -1,4 +1,7 @@ +from __future__ import annotations + from types import GeneratorType +from typing import Any from unittest.mock import AsyncMock import pytest @@ -85,7 +88,7 @@ def test_session_context_manager(mockserver): "httpResponseBody": "PGh0bWw+PGJvZHk+SGVsbG88aDE+V29ybGQhPC9oMT48L2JvZHk+PC9odG1sPg==", }, ] - actual_results = [] + actual_results: list[dict[str, Any] | Exception] = [] with client.session() as session: assert session._session.connector is not None assert session._session.connector.limit == client._async_client.n_conn @@ -126,8 +129,9 @@ def test_session_no_context_manager(mockserver): "httpResponseBody": "PGh0bWw+PGJvZHk+SGVsbG88aDE+V29ybGQhPC9oMT48L2JvZHk+PC9odG1sPg==", }, ] - actual_results = [] + actual_results: list[dict[str, Any] | Exception] = [] session = client.session() + assert session._session.connector is not None assert session._session.connector.limit == client._async_client.n_conn actual_results.append(session.get(queries[0])) actual_results.extend(session.iter(queries[1:])) diff --git a/zyte_api/__main__.py b/zyte_api/__main__.py index d6154a4..382b48a 100644 --- a/zyte_api/__main__.py +++ b/zyte_api/__main__.py @@ -8,7 +8,7 @@ import logging import random import sys -from typing import Any, Literal, TextIO +from typing import IO, Any, Literal from warnings import warn import tqdm @@ -33,7 +33,7 @@ class DontRetryErrorsFactory(RetryFactory): async def run( queries: list[dict[str, Any]], - out: TextIO, + out: IO[str], *, n_conn: int, stop_on_errors: bool | object = _UNSET, @@ -101,7 +101,7 @@ def write_output(content: Any) -> None: def read_input( - input_fp: TextIO, intype: Literal["txt", "jl"] | object + input_fp: IO[str], intype: Literal["txt", "jl"] | object ) -> list[dict[str, Any]]: assert intype in {"txt", "jl", _UNSET} lines = input_fp.readlines() From 196354891d794d96d9a56aea1a7188e8a04dd798 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 23 Sep 2025 00:05:29 +0500 Subject: [PATCH 05/13] Disable implicit reexport. --- pyproject.toml | 1 + tests/test_main.py | 2 +- zyte_api/__init__.py | 15 +++++++++++++++ zyte_api/_async.py | 2 +- 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5aec0d3..27b6083 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ exclude_also = [ [tool.mypy] allow_untyped_defs = false +implicit_reexport = false [[tool.mypy.overrides]] module = "runstats" diff --git a/tests/test_main.py b/tests/test_main.py index 4af539f..993674c 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -8,8 +8,8 @@ import pytest +from zyte_api import RequestError from zyte_api.__main__ import run -from zyte_api.aio.errors import RequestError class MockRequestError(RequestError): diff --git a/zyte_api/__init__.py b/zyte_api/__init__.py index fab06e3..d47add2 100644 --- a/zyte_api/__init__.py +++ b/zyte_api/__init__.py @@ -23,3 +23,18 @@ #: :ref:`Aggresive retry policy `. aggressive_retrying = _aggressive_retrying + +__all__ = [ + "AggressiveRetryFactory", + "AsyncZyteAPI", + "AuthInfo", + "ParsedError", + "RequestError", + "RetryFactory", + "ZyteAPI", + "aggressive_retrying", + "stop_after_uninterrupted_delay", + "stop_on_count", + "stop_on_download_error", + "zyte_api_retrying", +] diff --git a/zyte_api/_async.py b/zyte_api/_async.py index d4f8d65..9f00f61 100644 --- a/zyte_api/_async.py +++ b/zyte_api/_async.py @@ -25,7 +25,7 @@ from collections.abc import Awaitable, Callable, Iterator from contextlib import AbstractAsyncContextManager - from eth_account.account import LocalAccount + from eth_account.signers.local import LocalAccount # typing.Self requires Python 3.11 from typing_extensions import Self From 18df9840daad58f2f250ef732f477d5177029d5f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 23 Sep 2025 00:15:41 +0500 Subject: [PATCH 06/13] Some more typing with --strict. --- tests/test_async.py | 9 ++++++--- tests/test_main.py | 13 +++++++++++-- tests/test_retry.py | 8 ++++---- tests/test_sync.py | 9 ++++++--- zyte_api/_sync.py | 8 ++++---- 5 files changed, 31 insertions(+), 16 deletions(-) diff --git a/tests/test_async.py b/tests/test_async.py index 08b4069..50e38b8 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -1,7 +1,7 @@ from __future__ import annotations import asyncio -from typing import Any +from typing import TYPE_CHECKING, Any from unittest.mock import AsyncMock import pytest @@ -12,6 +12,9 @@ from zyte_api.errors import ParsedError from zyte_api.utils import USER_AGENT +if TYPE_CHECKING: + from tests.mockserver import MockServer + @pytest.mark.parametrize( "client_cls", @@ -221,7 +224,7 @@ async def test_semaphore(client_cls, get_method, iter_method, mockserver): @pytest.mark.asyncio -async def test_session_context_manager(mockserver): +async def test_session_context_manager(mockserver: MockServer) -> None: client = AsyncZyteAPI(api_key="a", api_url=mockserver.urljoin("/")) queries = [ {"url": "https://a.example", "httpResponseBody": True}, @@ -271,7 +274,7 @@ async def test_session_context_manager(mockserver): @pytest.mark.asyncio -async def test_session_no_context_manager(mockserver): +async def test_session_no_context_manager(mockserver: MockServer) -> None: client = AsyncZyteAPI(api_key="a", api_url=mockserver.urljoin("/")) queries = [ {"url": "https://a.example", "httpResponseBody": True}, diff --git a/tests/test_main.py b/tests/test_main.py index 993674c..8e22b55 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,9 +1,11 @@ +from __future__ import annotations + import json import subprocess from json import JSONDecodeError from pathlib import Path from tempfile import NamedTemporaryFile -from typing import Any +from typing import TYPE_CHECKING, Any from unittest.mock import AsyncMock, Mock, patch import pytest @@ -11,6 +13,11 @@ from zyte_api import RequestError from zyte_api.__main__ import run +if TYPE_CHECKING: + from collections.abc import Iterable + + from tests.mockserver import MockServer + class MockRequestError(RequestError): def __init__(self, *args, **kwargs): @@ -178,7 +185,9 @@ async def test_run_stop_on_errors_true(mockserver): assert exc_info.value.query == query -def _run(*, input, mockserver, cli_params=None): +def _run( + *, input: str, mockserver: MockServer, cli_params: Iterable[str] | None = None +) -> subprocess.CompletedProcess[bytes]: cli_params = cli_params or () with NamedTemporaryFile("w") as url_list: url_list.write(input) diff --git a/tests/test_retry.py b/tests/test_retry.py index b3ed67c..22ba690 100644 --- a/tests/test_retry.py +++ b/tests/test_retry.py @@ -145,10 +145,10 @@ def __init__(self, time: float): class scale: - def __init__(self, factor): - self.factor = factor + def __init__(self, factor: float): + self.factor: float = factor - def __call__(self, number, add=0): + def __call__(self, number: float, add: int = 0) -> int: return int(number * self.factor) + add @@ -414,7 +414,7 @@ def wait(retry_state): retrying = copy(retrying) retrying.wait = wait - async def run(): + async def run() -> None: while True: try: outcome = outcomes.popleft() diff --git a/tests/test_sync.py b/tests/test_sync.py index f8e6145..4e3ca6b 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -1,7 +1,7 @@ from __future__ import annotations from types import GeneratorType -from typing import Any +from typing import TYPE_CHECKING, Any from unittest.mock import AsyncMock import pytest @@ -9,6 +9,9 @@ from zyte_api import ZyteAPI from zyte_api.apikey import NoApiKey +if TYPE_CHECKING: + from tests.mockserver import MockServer + def test_api_key(): ZyteAPI(api_key="a") @@ -70,7 +73,7 @@ def test_semaphore(mockserver): assert client._async_client._semaphore.__aexit__.call_count == len(queries) -def test_session_context_manager(mockserver): +def test_session_context_manager(mockserver: MockServer) -> None: client = ZyteAPI(api_key="a", api_url=mockserver.urljoin("/")) queries = [ {"url": "https://a.example", "httpResponseBody": True}, @@ -111,7 +114,7 @@ def test_session_context_manager(mockserver): assert actual_result in expected_results -def test_session_no_context_manager(mockserver): +def test_session_no_context_manager(mockserver: MockServer) -> None: client = ZyteAPI(api_key="a", api_url=mockserver.urljoin("/")) queries = [ {"url": "https://a.example", "httpResponseBody": True}, diff --git a/zyte_api/_sync.py b/zyte_api/_sync.py index 79ac816..95afb8d 100644 --- a/zyte_api/_sync.py +++ b/zyte_api/_sync.py @@ -49,7 +49,7 @@ def close(self) -> None: def get( self, - query: dict, + query: dict[str, Any], *, endpoint: str = "extract", handle_retries: bool = True, @@ -65,7 +65,7 @@ def get( def iter( self, - queries: list[dict], + queries: list[dict[str, Any]], *, endpoint: str = "extract", handle_retries: bool = True, @@ -129,7 +129,7 @@ def __init__( def get( self, - query: dict, + query: dict[str, Any], *, endpoint: str = "extract", session: ClientSession | None = None, @@ -163,7 +163,7 @@ def get( def iter( self, - queries: list[dict], + queries: list[dict[str, Any]], *, endpoint: str = "extract", session: ClientSession | None = None, From 98ecd3ad13f30cda46ad975c47acad620a35d6c7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 23 Sep 2025 00:24:39 +0500 Subject: [PATCH 07/13] More ruff rules. --- docs/conf.py | 2 +- pyproject.toml | 28 ++++++++++++++++++++++------ tests/mockserver.py | 13 ++++++++----- tests/test_main.py | 23 +++++++++++++---------- tests/test_utils.py | 6 +++--- tests/test_x402.py | 38 +++++++++++++++++++------------------- zyte_api/_retry.py | 4 ++-- zyte_api/aio/client.py | 4 ++-- zyte_api/aio/errors.py | 2 +- zyte_api/aio/retry.py | 2 +- 10 files changed, 72 insertions(+), 50 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index afa7667..7b92614 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,7 +16,7 @@ # -- Project information ----------------------------------------------------- project = "python-zyte-api" -copyright = "2021, Zyte Group Ltd" +project_copyright = "2021, Zyte Group Ltd" author = "Zyte Group Ltd" # The short X.Y version diff --git a/pyproject.toml b/pyproject.toml index 27b6083..34639b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,10 +46,16 @@ filterwarnings = [ [tool.ruff.lint] extend-select = [ + # flake8-builtins + "A", + # flake8-async + "ASYNC", # flake8-bugbear "B", # flake8-comprehensions "C4", + # flake8-commas + "COM", # pydocstyle "D", # flake8-future-annotations @@ -96,6 +102,8 @@ extend-select = [ "T10", # flake8-type-checking "TC", + # flake8-tidy-imports + "TID", # pyupgrade "UP", # pycodestyle warnings @@ -104,6 +112,8 @@ extend-select = [ "YTT", ] ignore = [ + # Trailing comma missing + "COM812", # Missing docstring in public module "D100", # Missing docstring in public class @@ -156,6 +166,18 @@ ignore = [ "S101", ] +[tool.ruff.lint.flake8-pytest-style] +parametrize-values-type = "tuple" + +[tool.ruff.lint.flake8-tidy-imports] +banned-module-level-imports = ["twisted.internet.reactor"] + +[tool.ruff.lint.flake8-type-checking] +runtime-evaluated-decorators = ["attr.s"] + +[tool.ruff.lint.isort] +split-on-trailing-comma = false + [tool.ruff.lint.per-file-ignores] "zyte_api/__init__.py" = ["F401"] "zyte_api/aio/errors.py" = ["F401"] @@ -166,11 +188,5 @@ ignore = [ "zyte_api/errors.py" = ["UP007", "UP045"] "zyte_api/stats.py" = ["UP007", "UP045"] -[tool.ruff.lint.flake8-pytest-style] -parametrize-values-type = "tuple" - -[tool.ruff.lint.flake8-type-checking] -runtime-evaluated-decorators = ["attr.s"] - [tool.ruff.lint.pydocstyle] convention = "pep257" diff --git a/tests/mockserver.py b/tests/mockserver.py index aa3a602..f6d1ec0 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -10,7 +10,6 @@ from typing import Any from urllib.parse import urlparse -from twisted.internet import reactor from twisted.internet.task import deferLater from twisted.web.resource import Resource from twisted.web.server import NOT_DONE_YET, Site @@ -22,11 +21,11 @@ # https://github.com/scrapy/scrapy/blob/02b97f98e74a994ad3e4d74e7ed55207e508a576/tests/mockserver.py#L27C1-L33C19 -def getarg(request, name, default=None, type=None): +def getarg(request, name, default=None, type_=None): if name in request.args: value = request.args[name][0] - if type is not None: - value = type(value) + if type_ is not None: + value = type_(value) return value return default @@ -41,6 +40,8 @@ class DropResource(Resource): isLeaf = True def deferRequest(self, request, delay, f, *a, **kw): + from twisted.internet import reactor + def _cancelrequest(_): # silence CancelledError d.addErrback(lambda _: None) @@ -56,7 +57,7 @@ def render_POST(self, request): return NOT_DONE_YET def _delayedRender(self, request): - abort = getarg(request, b"abort", 0, type=int) + abort = getarg(request, b"abort", 0, type_=int) request.write(b"this connection will be dropped\n") tr = request.channel.transport try: @@ -270,6 +271,8 @@ def urljoin(self, path): def main(): + from twisted.internet import reactor + parser = argparse.ArgumentParser() parser.add_argument("resource") parser.add_argument("--port", type=int) diff --git a/tests/test_main.py b/tests/test_main.py index 8e22b55..6c2c71e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -103,7 +103,7 @@ async def fake_exception(value=True): @pytest.mark.asyncio async def test_run(queries, expected_response, store_errors, exception): tmp_path = Path("temporary_file.jsonl") - temporary_file = tmp_path.open("w") + temporary_file = tmp_path.open("w") # noqa: ASYNC230 n_conn = 5 api_url = "https://example.com" api_key = "fake_key" @@ -186,11 +186,11 @@ async def test_run_stop_on_errors_true(mockserver): def _run( - *, input: str, mockserver: MockServer, cli_params: Iterable[str] | None = None + *, input_: str, mockserver: MockServer, cli_params: Iterable[str] | None = None ) -> subprocess.CompletedProcess[bytes]: cli_params = cli_params or () with NamedTemporaryFile("w") as url_list: - url_list.write(input) + url_list.write(input_) url_list.flush() # Note: Using “python -m zyte_api” instead of “zyte-api” enables # coverage tracking to work. @@ -212,14 +212,14 @@ def _run( def test_empty_input(mockserver): - result = _run(input="", mockserver=mockserver) + result = _run(input_="", mockserver=mockserver) assert result.returncode assert result.stdout == b"" assert result.stderr == b"No input queries found. Is the input file empty?\n" def test_intype_txt_implicit(mockserver): - result = _run(input="https://a.example", mockserver=mockserver) + result = _run(input_="https://a.example", mockserver=mockserver) assert not result.returncode assert ( result.stdout @@ -229,7 +229,9 @@ def test_intype_txt_implicit(mockserver): def test_intype_txt_explicit(mockserver): result = _run( - input="https://a.example", mockserver=mockserver, cli_params=["--intype", "txt"] + input_="https://a.example", + mockserver=mockserver, + cli_params=["--intype", "txt"], ) assert not result.returncode assert ( @@ -240,7 +242,8 @@ def test_intype_txt_explicit(mockserver): def test_intype_jsonl_implicit(mockserver): result = _run( - input='{"url": "https://a.example", "browserHtml": true}', mockserver=mockserver + input_='{"url": "https://a.example", "browserHtml": true}', + mockserver=mockserver, ) assert not result.returncode assert ( @@ -251,7 +254,7 @@ def test_intype_jsonl_implicit(mockserver): def test_intype_jsonl_explicit(mockserver): result = _run( - input='{"url": "https://a.example", "browserHtml": true}', + input_='{"url": "https://a.example", "browserHtml": true}', mockserver=mockserver, cli_params=["--intype", "jl"], ) @@ -265,7 +268,7 @@ def test_intype_jsonl_explicit(mockserver): @pytest.mark.flaky(reruns=16) def test_limit_and_shuffle(mockserver): result = _run( - input="https://a.example\nhttps://b.example", + input_="https://a.example\nhttps://b.example", mockserver=mockserver, cli_params=["--limit", "1", "--shuffle"], ) @@ -278,7 +281,7 @@ def test_limit_and_shuffle(mockserver): def test_run_non_json_response(mockserver): result = _run( - input="https://nonjson.example", + input_="https://nonjson.example", mockserver=mockserver, ) assert not result.returncode diff --git a/tests/test_utils.py b/tests/test_utils.py index a9092f7..b0d89eb 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -69,7 +69,7 @@ def test_guess_intype(file_name, first_line, expected): @pytest.mark.parametrize( - ("input", "output"), + ("input_", "output"), ( # Unsafe URLs in the url field are modified, while left untouched on # other fields. @@ -104,8 +104,8 @@ def test_guess_intype(file_name, first_line, expected): # the URL escaping logic exist upstream. ), ) -def test_process_query(input, output): - assert _process_query(input) == output +def test_process_query(input_, output): + assert _process_query(input_) == output def test_process_query_bytes(): diff --git a/tests/test_x402.py b/tests/test_x402.py index c827ad1..fd3e6ed 100644 --- a/tests/test_x402.py +++ b/tests/test_x402.py @@ -380,7 +380,7 @@ async def test_cache(scenario, mockserver): @mock.patch("zyte_api._x402.MINIMIZE_REQUESTS", False) async def test_no_cache(mockserver): client = AsyncZyteAPI(eth_key=KEY, api_url=mockserver.urljoin("/")) - input = {"url": "https://a.example", "httpResponseBody": True} + input_ = {"url": "https://a.example", "httpResponseBody": True} output = { "url": "https://a.example", "httpResponseBody": BODY, @@ -391,24 +391,24 @@ async def test_no_cache(mockserver): assert client.agg_stats.n_402_req == 0 # Initial request - actual_result = await client.get(input) + actual_result = await client.get(input_) assert actual_result == output assert len(cache) == 0 assert client.agg_stats.n_402_req == 1 # Identical request - actual_result = await client.get(input) + actual_result = await client.get(input_) assert actual_result == output assert len(cache) == 0 assert client.agg_stats.n_402_req == 2 # Request refresh - input = { + input_ = { "url": "https://a.example", "httpResponseBody": True, "echoData": "402-payment-retry-2", } - actual_result = await client.get(input) + actual_result = await client.get(input_) assert actual_result == output assert len(cache) == 0 assert client.agg_stats.n_402_req == 3 @@ -420,13 +420,13 @@ async def test_4xx(mockserver): """An unexpected status code lower than 500 raises RequestError immediately.""" client = AsyncZyteAPI(eth_key=KEY, api_url=mockserver.urljoin("/")) - input = {"url": "https://e404.example", "httpResponseBody": True} + input_ = {"url": "https://e404.example", "httpResponseBody": True} with reset_x402_cache() as cache: assert len(cache) == 0 assert client.agg_stats.n_402_req == 0 with pytest.raises(RequestError): - await client.get(input) + await client.get(input_) assert len(cache) == 0 assert client.agg_stats.n_402_req == 1 @@ -436,13 +436,13 @@ async def test_4xx(mockserver): async def test_5xx(mockserver): """An unexpected status code ≥ 500 gets retried once.""" client = AsyncZyteAPI(eth_key=KEY, api_url=mockserver.urljoin("/")) - input = {"url": "https://e500.example", "httpResponseBody": True} + input_ = {"url": "https://e500.example", "httpResponseBody": True} with reset_x402_cache() as cache: assert len(cache) == 0 assert client.agg_stats.n_402_req == 0 with pytest.raises(RequestError): - await client.get(input) + await client.get(input_) assert len(cache) == 0 assert client.agg_stats.n_402_req == 2 @@ -451,7 +451,7 @@ async def test_5xx(mockserver): @pytest.mark.asyncio async def test_payment_retry(mockserver): client = AsyncZyteAPI(eth_key=KEY, api_url=mockserver.urljoin("/")) - input = { + input_ = { "url": "https://a.example", "httpResponseBody": True, "echoData": "402-payment-retry", @@ -460,7 +460,7 @@ async def test_payment_retry(mockserver): with reset_x402_cache() as cache: assert len(cache) == 0 assert client.agg_stats.n_402_req == 0 - data = await client.get(input) + data = await client.get(input_) assert len(cache) == 1 assert data == {"httpResponseBody": BODY, "url": "https://a.example"} @@ -478,7 +478,7 @@ async def test_payment_retry(mockserver): @pytest.mark.asyncio async def test_payment_retry_exceeded(mockserver): client = AsyncZyteAPI(eth_key=KEY, api_url=mockserver.urljoin("/")) - input = { + input_ = { "url": "https://a.example", "httpResponseBody": True, "echoData": "402-payment-retry-exceeded", @@ -488,7 +488,7 @@ async def test_payment_retry_exceeded(mockserver): assert len(cache) == 0 assert client.agg_stats.n_402_req == 0 with pytest.raises(RequestError): - await client.get(input) + await client.get(input_) assert len(cache) == 1 assert client.agg_stats.n_success == 0 @@ -506,7 +506,7 @@ async def test_no_payment_retry(mockserver): """An HTTP 402 response received out of the context of the x402 protocol, as a response to a regular request using basic auth.""" client = AsyncZyteAPI(api_key="a", api_url=mockserver.urljoin("/")) - input = { + input_ = { "url": "https://a.example", "httpResponseBody": True, "echoData": "402-no-payment-retry", @@ -515,7 +515,7 @@ async def test_no_payment_retry(mockserver): with reset_x402_cache() as cache: assert len(cache) == 0 assert client.agg_stats.n_402_req == 0 - data = await client.get(input) + data = await client.get(input_) assert len(cache) == 0 assert data == {"httpResponseBody": BODY, "url": "https://a.example"} @@ -532,7 +532,7 @@ async def test_no_payment_retry(mockserver): @pytest.mark.asyncio async def test_no_payment_retry_exceeded(mockserver): client = AsyncZyteAPI(api_key="a", api_url=mockserver.urljoin("/")) - input = { + input_ = { "url": "https://a.example", "httpResponseBody": True, "echoData": "402-no-payment-retry-exceeded", @@ -542,7 +542,7 @@ async def test_no_payment_retry_exceeded(mockserver): assert len(cache) == 0 assert client.agg_stats.n_402_req == 0 with pytest.raises(RequestError): - await client.get(input) + await client.get(input_) assert len(cache) == 0 assert client.agg_stats.n_success == 0 @@ -558,7 +558,7 @@ async def test_no_payment_retry_exceeded(mockserver): @pytest.mark.asyncio async def test_long_error(mockserver): client = AsyncZyteAPI(api_key="a", api_url=mockserver.urljoin("/")) - input = { + input_ = { "url": "https://a.example", "httpResponseBody": True, "echoData": "402-long-error", @@ -568,7 +568,7 @@ async def test_long_error(mockserver): assert len(cache) == 0 assert client.agg_stats.n_402_req == 0 with pytest.raises(RequestError): - await client.get(input) + await client.get(input_) assert len(cache) == 0 assert client.agg_stats.n_success == 0 diff --git a/zyte_api/_retry.py b/zyte_api/_retry.py index f0507d2..fcf5259 100644 --- a/zyte_api/_retry.py +++ b/zyte_api/_retry.py @@ -164,10 +164,10 @@ def _402_error(exc: BaseException) -> bool: return isinstance(exc, RequestError) and exc.status == 402 -def _deprecated(message: str, callable: Callable) -> Callable: +def _deprecated(message: str, callable_: Callable) -> Callable: def wrapper(factory: Any, retry_state: RetryCallState) -> Callable: warn(message, DeprecationWarning, stacklevel=3) - return callable(retry_state=retry_state) + return callable_(retry_state=retry_state) return wrapper diff --git a/zyte_api/aio/client.py b/zyte_api/aio/client.py index 208cb38..ca861b8 100644 --- a/zyte_api/aio/client.py +++ b/zyte_api/aio/client.py @@ -1,5 +1,5 @@ -from .._async import AsyncZyteAPI -from .._utils import deprecated_create_session as create_session # noqa: F401 +from .._async import AsyncZyteAPI # noqa: TID252 +from .._utils import deprecated_create_session as create_session # noqa: F401, TID252 class AsyncClient(AsyncZyteAPI): diff --git a/zyte_api/aio/errors.py b/zyte_api/aio/errors.py index 987d8ba..b431d41 100644 --- a/zyte_api/aio/errors.py +++ b/zyte_api/aio/errors.py @@ -1 +1 @@ -from .._errors import RequestError +from .._errors import RequestError # noqa: TID252 diff --git a/zyte_api/aio/retry.py b/zyte_api/aio/retry.py index 5cd22a7..6a7b269 100644 --- a/zyte_api/aio/retry.py +++ b/zyte_api/aio/retry.py @@ -1 +1 @@ -from .._retry import RetryFactory, zyte_api_retrying +from .._retry import RetryFactory, zyte_api_retrying # noqa: TID252 From 24c4d686ef2eeb21c0b9a2541cfc14fb575b905b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 23 Sep 2025 00:50:45 +0500 Subject: [PATCH 08/13] Restore subprocess coverage. --- pyproject.toml | 3 +++ tox.ini | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 34639b9..4dae649 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,9 @@ filename = "zyte_api/__version__.py" [tool.coverage.run] branch = true +patch = [ + "subprocess", +] [tool.coverage.report] exclude_also = [ diff --git a/tox.ini b/tox.ini index 8c54a61..ff70eec 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ envlist = pre-commit,mypy,min,min-x402,py39,py310,py311,py312,py313,x402,docs,tw deps = pytest pytest-asyncio - pytest-cov + pytest-cov >= 7.0.0 pytest-rerunfailures pytest-twisted responses From f77236f88b189d9ff070f0789cf740af9ec6b6ed Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 23 Sep 2025 01:19:45 +0500 Subject: [PATCH 09/13] Add type hints for the RetryFactory attrs. --- zyte_api/_retry.py | 59 +++++++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/zyte_api/_retry.py b/zyte_api/_retry.py index fcf5259..c30afce 100644 --- a/zyte_api/_retry.py +++ b/zyte_api/_retry.py @@ -5,7 +5,7 @@ from collections import Counter from datetime import timedelta from itertools import count -from typing import Any, Callable, Union +from typing import TYPE_CHECKING, Any, Callable, Union, cast from warnings import warn from aiohttp import client_exceptions @@ -27,6 +27,9 @@ from ._errors import RequestError +if TYPE_CHECKING: + from tenacity.wait import wait_base + logger = logging.getLogger(__name__) _IDS = count() @@ -208,7 +211,7 @@ class CustomRetryFactory(RetryFactory): | retry_if_exception(_402_error) ) # throttling - throttling_wait = wait_chain( + throttling_wait: wait_base = wait_chain( # always wait 20-40s first wait_fixed(20) + wait_random(0, 20), # wait 20-40s again @@ -219,39 +222,47 @@ class CustomRetryFactory(RetryFactory): ) # connection errors, other client and server failures - network_error_stop = stop_after_uninterrupted_delay(15 * 60) - network_error_wait = ( + network_error_stop: stop_base = stop_after_uninterrupted_delay(15 * 60) + network_error_wait: wait_base = ( # wait from 3s to ~1m wait_random(3, 7) + wait_random_exponential(multiplier=1, max=55) ) - download_error_stop = stop_on_download_error(max_total=4, max_permanent=2) - download_error_wait = network_error_wait - - temporary_download_error_stop = _deprecated( - ( - "The zyte_api.RetryFactory.temporary_download_error_stop() method " - "is deprecated and will be removed in a future version. Use " - "download_error_stop() instead." + download_error_stop: stop_base = stop_on_download_error( + max_total=4, max_permanent=2 + ) + download_error_wait: wait_base = network_error_wait + + temporary_download_error_stop: stop_base = cast( + "stop_base", + _deprecated( + ( + "The zyte_api.RetryFactory.temporary_download_error_stop() method " + "is deprecated and will be removed in a future version. Use " + "download_error_stop() instead." + ), + download_error_stop, ), - download_error_stop, ) - temporary_download_error_wait = _deprecated( - ( - "The zyte_api.RetryFactory.temporary_download_error_wait() method " - "is deprecated and will be removed in a future version. Use " - "download_error_wait() instead." + temporary_download_error_wait: wait_base = cast( + "wait_base", + _deprecated( + ( + "The zyte_api.RetryFactory.temporary_download_error_wait() method " + "is deprecated and will be removed in a future version. Use " + "download_error_wait() instead." + ), + download_error_wait, ), - download_error_wait, ) - throttling_stop = stop_never + throttling_stop: stop_base = stop_never - undocumented_error_stop = stop_on_count(2) - undocumented_error_wait = network_error_wait + undocumented_error_stop: stop_base = stop_on_count(2) + undocumented_error_wait: wait_base = network_error_wait - x402_error_stop = stop_on_count(2) - x402_error_wait = wait_none() + x402_error_stop: stop_base = stop_on_count(2) + x402_error_wait: wait_base = wait_none() def wait(self, retry_state: RetryCallState) -> float: assert retry_state.outcome, "Unexpected empty outcome" From 1a91f97cb74d47b6becb017a772e1864694daea8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 24 Sep 2025 22:09:07 +0500 Subject: [PATCH 10/13] Add type hints for the AggressiveRetryFactory attrs. --- zyte_api/_retry.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zyte_api/_retry.py b/zyte_api/_retry.py index c30afce..d784d23 100644 --- a/zyte_api/_retry.py +++ b/zyte_api/_retry.py @@ -342,8 +342,10 @@ class CustomRetryFactory(AggressiveRetryFactory): CUSTOM_RETRY_POLICY = CustomRetryFactory().build() """ - download_error_stop = stop_on_download_error(max_total=8, max_permanent=4) - undocumented_error_stop = stop_on_count(4) + download_error_stop: stop_base = stop_on_download_error( + max_total=8, max_permanent=4 + ) + undocumented_error_stop: stop_base = stop_on_count(4) aggressive_retrying = AggressiveRetryFactory().build() From b5218564f8586c5499adcd240927ed07dfc129b2 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 Oct 2025 18:44:11 +0500 Subject: [PATCH 11/13] Fix RequestError.parsed(). --- zyte_api/_errors.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/zyte_api/_errors.py b/zyte_api/_errors.py index aab6b2b..b4296e8 100644 --- a/zyte_api/_errors.py +++ b/zyte_api/_errors.py @@ -33,8 +33,7 @@ def __init__(self, *args: Any, **kwargs: Any): @property def parsed(self) -> ParsedError: """Response as a :class:`ParsedError` object.""" - # TODO: self.response_content can be None but ParsedError doesn't expect it - return ParsedError.from_body(self.response_content) # type: ignore[arg-type] + return ParsedError.from_body(self.response_content or b"") def __str__(self) -> str: return ( From f75a52ce2a321b26a701641c7b64c7cd27a44479 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 Oct 2025 18:48:51 +0500 Subject: [PATCH 12/13] Bump tools. --- .github/workflows/publish.yml | 6 ++---- .github/workflows/test.yml | 12 ++++++------ .pre-commit-config.yaml | 6 +++--- .readthedocs.yml | 4 ++-- pyproject.toml | 5 ----- tox.ini | 6 +++--- 6 files changed, 16 insertions(+), 23 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f14bf84..a20b249 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,8 +12,8 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 with: python-version: 3.13 - run: | @@ -21,5 +21,3 @@ jobs: python -m build - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1ace900..6c916c0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,9 +30,9 @@ jobs: tox: x402 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -44,7 +44,7 @@ jobs: tox -e ${{ matrix.tox || 'py' }} - name: coverage if: ${{ success() }} - uses: codecov/codecov-action@v4.0.1 + uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -53,13 +53,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.12'] # Keep in sync with .readthedocs.yml + python-version: ["3.13"] # Keep in sync with .readthedocs.yml tox-job: ["mypy", "docs"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index af79841..d2e4108 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.13.1 + rev: v0.13.3 hooks: - id: ruff-check args: [ --fix ] - id: ruff-format - repo: https://github.com/adamchainz/blacken-docs - rev: 1.19.0 + rev: 1.20.0 hooks: - id: blacken-docs additional_dependencies: - - black==25.1.0 + - black==25.9.0 diff --git a/.readthedocs.yml b/.readthedocs.yml index f81f402..64e40c4 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,11 +3,11 @@ formats: all sphinx: configuration: docs/conf.py build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.12" # Keep in sync with .github/workflows/test.yml + python: "3.13" # Keep in sync with .github/workflows/test.yml python: install: - requirements: docs/requirements.txt diff --git a/pyproject.toml b/pyproject.toml index 4dae649..16449db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,11 +25,6 @@ patch = [ "subprocess", ] -[tool.coverage.report] -exclude_also = [ - "if TYPE_CHECKING:", -] - [tool.mypy] allow_untyped_defs = false implicit_reexport = false diff --git a/tox.ini b/tox.ini index ff70eec..7943eec 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ deps = responses twisted commands = - py.test \ + pytest \ --cov-report=term-missing --cov-report=html --cov-report=xml --cov=zyte_api \ --doctest-modules \ {posargs:zyte_api tests} @@ -68,8 +68,8 @@ commands = pre-commit run --all-files --show-diff-on-failure [testenv:twine] deps = - twine==6.1.0 - build==1.2.2.post1 + twine==6.2.0 + build==1.3.0 commands = python -m build --sdist twine check dist/* From cbbc848a347226e9bf104727beae4cf450f6ef63 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 Oct 2025 18:49:53 +0500 Subject: [PATCH 13/13] Add more linters. --- .pre-commit-config.yaml | 9 +++++++++ docs/Makefile | 2 +- docs/use/x402.rst | 2 +- tox.ini | 2 +- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d2e4108..c15bfa9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,3 +11,12 @@ repos: - id: blacken-docs additional_dependencies: - black==25.9.0 +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v1.0.0 + hooks: + - id: sphinx-lint diff --git a/docs/Makefile b/docs/Makefile index 298ea9e..5128596 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -16,4 +16,4 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/use/x402.rst b/docs/use/x402.rst index e87952c..34a21aa 100644 --- a/docs/use/x402.rst +++ b/docs/use/x402.rst @@ -4,7 +4,7 @@ x402 ==== -It is possible to use :ref:`Zyte API ` without a Zyte API account by +It is possible to use :ref:`Zyte API ` without a Zyte API account by using the x402_ protocol to handle payments: #. Read the `Zyte Terms of Service`_. By using Zyte API, you are accepting diff --git a/tox.ini b/tox.ini index 7943eec..18d31e6 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,7 @@ deps = {[min]deps} [testenv:min-x402] basepython = python3.10 -deps = +deps = {[min]deps} eth_account==0.13.7 x402==0.1.1