refactor!: Introduce fully typed clients [WIP]

.github/workflows/_tests.yaml

@@ -28,4 +28,4 @@ jobs:
       operating_systems: '["ubuntu-latest"]'
       python_version_for_codecov: "3.14"
       operating_system_for_codecov: ubuntu-latest
-      tests_concurrency: "1"
+      tests_concurrency: "16"

docs/01_overview/code/01_usage_async.py

@@ -16,6 +16,6 @@ async def main() -> None:
         return
 
     # Fetch results from the Actor run's default dataset.
-    dataset_client = apify_client.dataset(call_result['defaultDatasetId'])
+    dataset_client = apify_client.dataset(call_result.default_dataset_id)
     list_items_result = await dataset_client.list_items()
     print(f'Dataset: {list_items_result}')

docs/01_overview/code/01_usage_sync.py

@@ -16,6 +16,6 @@ def main() -> None:
         return
 
     # Fetch results from the Actor run's default dataset.
-    dataset_client = apify_client.dataset(call_result['defaultDatasetId'])
+    dataset_client = apify_client.dataset(call_result.default_dataset_id)
     list_items_result = dataset_client.list_items()
     print(f'Dataset: {list_items_result}')

docs/02_concepts/code/01_async_support.py

@@ -11,7 +11,7 @@ async def main() -> None:
 
     # Start the Actor and get the run ID
     run_result = await actor_client.start()
-    run_client = apify_client.run(run_result['id'])
+    run_client = apify_client.run(run_result.id)
     log_client = run_client.log()
 
     # Stream the logs

docs/03_examples/code/02_tasks_async.py

@@ -1,22 +1,22 @@
 import asyncio
 
 from apify_client import ApifyClientAsync
-from apify_client.clients.resource_clients import TaskClientAsync
+from apify_client._models import Run, Task
+from apify_client._resource_clients import TaskClientAsync
 
 TOKEN = 'MY-APIFY-TOKEN'
 HASHTAGS = ['zebra', 'lion', 'hippo']
 
 
-async def run_apify_task(client: TaskClientAsync) -> dict:
-    result = await client.call()
-    return result or {}
+async def run_apify_task(client: TaskClientAsync) -> Run | None:
+    return await client.call()
 
 
 async def main() -> None:
     apify_client = ApifyClientAsync(token=TOKEN)
 
     # Create Apify tasks
-    apify_tasks = list[dict]()
+    apify_tasks = list[Task]()
     apify_tasks_client = apify_client.tasks()
 
     for hashtag in HASHTAGS:
@@ -34,7 +34,7 @@ async def main() -> None:
     apify_task_clients = list[TaskClientAsync]()
 
     for apify_task in apify_tasks:
-        task_id = apify_task['id']
+        task_id = apify_task.id
         apify_task_client = apify_client.task(task_id)
         apify_task_clients.append(apify_task_client)
 

docs/03_examples/code/02_tasks_sync.py

@@ -1,20 +1,20 @@
 from apify_client import ApifyClient
-from apify_client.clients.resource_clients import TaskClient
+from apify_client._models import Run, Task
+from apify_client._resource_clients import TaskClient
 
 TOKEN = 'MY-APIFY-TOKEN'
 HASHTAGS = ['zebra', 'lion', 'hippo']
 
 
-def run_apify_task(client: TaskClient) -> dict:
-    result = client.call()
-    return result or {}
+def run_apify_task(client: TaskClient) -> Run | None:
+    return client.call()
 
 
 def main() -> None:
     apify_client = ApifyClient(token=TOKEN)
 
     # Create Apify tasks
-    apify_tasks = list[dict]()
+    apify_tasks = list[Task]()
     apify_tasks_client = apify_client.tasks()
 
     for hashtag in HASHTAGS:
@@ -32,18 +32,19 @@ def main() -> None:
     apify_task_clients = list[TaskClient]()
 
     for apify_task in apify_tasks:
-        task_id = apify_task['id']
+        task_id = apify_task.id
         apify_task_client = apify_client.task(task_id)
         apify_task_clients.append(apify_task_client)
 
     print('Task clients created:', apify_task_clients)
 
     # Execute Apify tasks
-    task_run_results = list[dict]()
+    task_run_results = list[Run]()
 
     for client in apify_task_clients:
         result = run_apify_task(client)
-        task_run_results.append(result)
+        if result is not None:
+            task_run_results.append(result)
 
     print('Task results:', task_run_results)
 

docs/03_examples/code/03_retrieve_async.py

@@ -19,11 +19,11 @@ async def main() -> None:
 
     for dataset_item in actor_datasets.items:
         # Dataset items can be handled here. Dataset items can be paginated
-        dataset_client = apify_client.dataset(dataset_item['id'])
+        dataset_client = apify_client.dataset(dataset_item.id)
         dataset_items = await dataset_client.list_items(limit=1000)
 
         # Items can be pushed to single dataset
-        merging_dataset_client = apify_client.dataset(merging_dataset['id'])
+        merging_dataset_client = apify_client.dataset(merging_dataset.id)
         await merging_dataset_client.push_items(dataset_items.items)
 
         # ...

docs/03_examples/code/03_retrieve_sync.py

@@ -17,11 +17,11 @@ def main() -> None:
 
     for dataset_item in actor_datasets.items:
         # Dataset items can be handled here. Dataset items can be paginated
-        dataset_client = apify_client.dataset(dataset_item['id'])
+        dataset_client = apify_client.dataset(dataset_item.id)
         dataset_items = dataset_client.list_items(limit=1000)
 
         # Items can be pushed to single dataset
-        merging_dataset_client = apify_client.dataset(merging_dataset['id'])
+        merging_dataset_client = apify_client.dataset(merging_dataset.id)
         merging_dataset_client.push_items(dataset_items.items)
 
         # ...

pyproject.toml

@@ -25,10 +25,10 @@ classifiers = [
 ]
 keywords = ["apify", "api", "client", "automation", "crawling", "scraping"]
 dependencies = [
-    "apify-shared>=2.1.0,<3.0.0",
     "colorama>=0.4.0",
     "impit>=0.9.2",
     "more_itertools>=10.0.0",
+    "pydantic[email]>=2.11.0",
 ]
 
 [project.urls]
@@ -64,6 +64,7 @@ dev = [
     "ty~=0.0.0",
     "types-colorama<0.5.0",
     "werkzeug<4.0.0", # Werkzeug is used by pytest-httpserver
+    "datamodel-code-generator[http,ruff]<1.0.0",
 ]
 
 [tool.hatch.build.targets.wheel]
@@ -144,6 +145,12 @@ indent-style = "space"
     "N999",   # Invalid module name
     "T201",   # print found
 ]
+"src/apify_client/_models.py" = [
+    "D",      # Everything from the pydocstyle
+    "E501",   # Line too long
+    "ERA001", # Commented-out code
+    "TC003",  # Move standard library import into a type-checking block
+]
 
 [tool.ruff.lint.flake8-quotes]
 docstring-quotes = "double"
@@ -171,10 +178,7 @@ python-version = "3.10"
 include = ["src", "tests", "scripts", "docs", "website"]
 
 [[tool.ty.overrides]]
-include = [
-    "docs/**/*.py",
-    "website/**/*.py",
-]
+include = ["docs/**/*.py", "website/**/*.py"]
 
 [tool.ty.overrides.rules]
 unresolved-import = "ignore"
@@ -220,3 +224,22 @@ cwd = "website"
 [tool.poe.tasks.run-docs]
 shell = "./build_api_reference.sh && corepack enable && yarn && uv run yarn start"
 cwd = "website"
+
+# https://koxudaxi.github.io/datamodel-code-generator/
+[tool.datamodel-codegen]
+# url = "https://docs.apify.com/api/openapi.json"
+input = "../apify-docs/static/api/openapi.json"
+input_file_type = "openapi"
+output = "src/apify_client/_models.py"
+target_python_version = "3.10"
+output_model_type = "pydantic_v2.BaseModel"
+use_schema_description = true
+use_field_description = true
+use_union_operator = true
+capitalise_enum_members = true
+collapse_root_models = true
+set_default_enum_member = true
+use_annotated = true
+wrap_string_literal = true
+snake_case_field = true
+formatters = ["ruff-check", "ruff-format"]

scripts/utils.py

@@ -25,7 +25,7 @@ def get_current_package_version() -> str:
 # It replaces the version number on the line with the format `version = "1.2.3"`
 def set_current_package_version(version: str) -> None:
     with open(PYPROJECT_TOML_FILE_PATH, 'r+', encoding='utf-8') as pyproject_toml_file:
-        updated_pyproject_toml_file_lines = []
+        updated_pyproject_toml_file_lines = list[str]()
         version_string_found = False
         for line in pyproject_toml_file:
             line_processed = line
@@ -59,8 +59,8 @@ def get_published_package_versions() -> list:
         package_data = json.load(urlopen(package_info_url))  # noqa: S310
         published_versions = list(package_data['releases'].keys())
     # If the URL returns 404, it means the package has no releases yet (which is okay in our case)
-    except HTTPError as e:
-        if e.code != 404:
+    except HTTPError as exc:
+        if exc.code != 404:
             raise
         published_versions = []
     return published_versions

src/apify_client/__init__.py

@@ -1,6 +1,6 @@
 from importlib import metadata
 
-from .client import ApifyClient, ApifyClientAsync
+from ._apify_client import ApifyClient, ApifyClientAsync
 
 __version__ = metadata.version('apify-client')