feat: add pagination support for file metadata retrieval

README.md

@@ -508,6 +508,13 @@ all_content = result.get_content()
 all_content = await result.aget_content()
 ```
 
+or access response metadata:
+
+```python
+headers = result.headers
+content_type = result.content_type
+```
+
 or write it to the file:
 
 ```python
@@ -564,14 +571,32 @@ Both methods return `None` on success. `source` and `destination` must point to
 
 #### Accessing Metadata
 
-Use `metadata()` to access metadata of a file:
+Use `get_metadata()` to access metadata of a file or folder:
 
 ```python
-metadata = await async_client.files.metadata(
+# Sync client
+metadata = sync_client.files.get_metadata(
+    url=sync_client.my_files_home() / "relative_folder/my-file.txt"
+)
+
+# Async client
+metadata = await async_client.files.get_metadata(
     url=await async_client.my_files_home() / "relative_folder/my-file.txt"
 )
 ```
 
+Folder metadata can be paginated with `limit` and `token`:
+
+```python
+metadata = await async_client.files.get_metadata(
+    url=await async_client.my_files_home() / "relative_folder/",
+    limit=100,
+    token=next_token,
+)
+next_token = metadata.next_token
+items = metadata.items
+```
+
 Example of metadata:
 
 ```python
@@ -584,6 +609,7 @@ FileMetadata(
     resource_type="FILE",
     content_length=12,
     content_type="application/octet-stream",
+    next_token=None,
     items=None,
     updatedAt=1724836248936,
     etag="9749fad13d6e7092a6337c4af9d83764",

aidial_client/resources/files.py

@@ -149,10 +149,18 @@ def copy_to(
             on_http_error=_files_error_processor,
         )
 
-    def get_metadata(self, url: Union[str, PurePosixPath]) -> FileMetadata:
+    def get_metadata(
+        self,
+        url: Union[str, PurePosixPath],
+        *,
+        limit: Optional[int] = None,
+        token: Optional[str] = None,
+    ) -> FileMetadata:
         return self.metadata.get(
             resource="files",
             relative_url=self.get_api_path(str(url)),
+            limit=limit,
+            token=token,
         )
 
 
@@ -269,9 +277,15 @@ async def copy_to(
         )
 
     async def get_metadata(
-        self, url: Union[str, PurePosixPath]
+        self,
+        url: Union[str, PurePosixPath],
+        *,
+        limit: Optional[int] = None,
+        token: Optional[str] = None,
     ) -> FileMetadata:
         return await self.metadata.get(
             resource="files",
             relative_url=self.get_api_path(str(url)),
+            limit=limit,
+            token=token,
         )

aidial_client/resources/metadata.py

@@ -1,10 +1,11 @@
-from typing import Literal, Type, Union, overload
+from typing import Literal, Optional, Type, Union, overload
 from urllib.parse import urljoin
 
 from typing_extensions import assert_never
 
 from aidial_client._constants import METADATA_PREFIX
 from aidial_client._internal_types._http_request import FinalRequestOptions
+from aidial_client._utils._dict import remove_none
 from aidial_client.helpers.storage_resource import StorageResourceType
 from aidial_client.resources.base import AsyncResource, Resource
 from aidial_client.types.metadata import (
@@ -32,58 +33,96 @@ def _get_cast_to(
 class Metadata(Resource):
     @overload
     def get(
-        self, resource: Literal["files"], relative_url: str
+        self,
+        resource: Literal["files"],
+        relative_url: str,
+        *,
+        limit: Optional[int] = None,
+        token: Optional[str] = None,
     ) -> FileMetadata: ...
 
     @overload
     def get(
-        self, resource: Literal["conversations"], relative_url: str
+        self,
+        resource: Literal["conversations"],
+        relative_url: str,
+        *,
+        limit: Optional[int] = None,
+        token: Optional[str] = None,
     ) -> ConversationMetadata: ...
 
     @overload
     def get(
-        self, resource: Literal["prompts"], relative_url: str
+        self,
+        resource: Literal["prompts"],
+        relative_url: str,
+        *,
+        limit: Optional[int] = None,
+        token: Optional[str] = None,
     ) -> PromptMetadata: ...
 
     def get(
         self,
         resource: StorageResourceType,
         relative_url: str,
+        *,
+        limit: Optional[int] = None,
+        token: Optional[str] = None,
     ) -> Union[FileMetadata, ConversationMetadata, PromptMetadata]:
         return self.http_client.request(
             cast_to=_get_cast_to(resource),
             options=FinalRequestOptions(
                 method="GET",
                 url=urljoin(METADATA_PREFIX, relative_url),
+                params=remove_none({"limit": limit, "token": token}),
             ),
         )
 
 
 class AsyncMetadata(AsyncResource):
     @overload
     async def get(
-        self, resource: Literal["files"], relative_url: str
+        self,
+        resource: Literal["files"],
+        relative_url: str,
+        *,
+        limit: Optional[int] = None,
+        token: Optional[str] = None,
     ) -> FileMetadata: ...
 
     @overload
     async def get(
-        self, resource: Literal["conversations"], relative_url: str
+        self,
+        resource: Literal["conversations"],
+        relative_url: str,
+        *,
+        limit: Optional[int] = None,
+        token: Optional[str] = None,
     ) -> ConversationMetadata: ...
 
     @overload
     async def get(
-        self, resource: Literal["prompts"], relative_url: str
+        self,
+        resource: Literal["prompts"],
+        relative_url: str,
+        *,
+        limit: Optional[int] = None,
+        token: Optional[str] = None,
     ) -> PromptMetadata: ...
 
     async def get(
         self,
         resource: StorageResourceType,
         relative_url: str,
+        *,
+        limit: Optional[int] = None,
+        token: Optional[str] = None,
     ) -> Union[FileMetadata, ConversationMetadata, PromptMetadata]:
         return await self.http_client.request(
             cast_to=_get_cast_to(resource),
             options=FinalRequestOptions(
                 method="GET",
                 url=urljoin(METADATA_PREFIX, relative_url),
+                params=remove_none({"limit": limit, "token": token}),
             ),
         )

aidial_client/types/file.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Union
+from typing import Optional, Union
 
 import aiofiles
 import httpx
@@ -43,3 +43,11 @@ async def aget_content(self) -> bytes:
     @property
     def filename(self) -> str:
         return self._filename
+
+    @property
+    def headers(self) -> httpx.Headers:
+        return self._response.headers
+
+    @property
+    def content_type(self) -> Optional[str]:
+        return self.headers.get("content-type")

aidial_client/types/metadata.py

@@ -37,6 +37,7 @@ class FileMetadata(BaseMetadata):
     resource_type: Literal["FILE"]
     content_length: Optional[int] = None
     content_type: Optional[str] = None
+    next_token: Optional[str] = None
     items: Optional[List[FileItem]] = None
     etag: Optional[str] = None
 

tests/resources/files/test_metadata.py

@@ -1,7 +1,11 @@
+from typing import Any, List
 from unittest.mock import AsyncMock, Mock
 
+import httpx
 import pytest
 
+from aidial_client import Dial
+from aidial_client._client import AsyncDial
 from aidial_client.types.metadata import FileMetadata
 from tests.client_mock import get_async_client_mock, get_client_mock
 
@@ -24,9 +28,44 @@
             "contentType": "image/png",
         }
     ],
+    "nextToken": "next-page-token",
 }
 
 
+def _make_capturing_client(captured: List[httpx.Request]) -> Dial:
+    client = Dial(api_key="dummy", base_url="http://dial.core")
+
+    def send_mock(request: httpx.Request, **_: Any) -> httpx.Response:
+        captured.append(request)
+        response = httpx.Response(
+            status_code=200, request=request, json=METADATA_RESPONSE_MOCK
+        )
+        response.request = request
+        return response
+
+    client._http_client._internal_http_client.send = send_mock
+    client._get_my_bucket = Mock(return_value="test-bucket")
+    return client
+
+
+def _make_async_capturing_client(
+    captured: List[httpx.Request],
+) -> AsyncDial:
+    client = AsyncDial(api_key="dummy", base_url="http://dial.core")
+
+    async def send_mock(request: httpx.Request, **_: Any) -> httpx.Response:
+        captured.append(request)
+        response = httpx.Response(
+            status_code=200, request=request, json=METADATA_RESPONSE_MOCK
+        )
+        response.request = request
+        return response
+
+    client._http_client._internal_http_client.send = send_mock
+    client._get_my_bucket = AsyncMock(return_value="test-bucket")
+    return client
+
+
 def test_get_metadata():
     client = get_client_mock(status_code=200, json_mock=METADATA_RESPONSE_MOCK)
     client._get_my_bucket = Mock(return_value="test-bucket")
@@ -44,6 +83,26 @@ def test_get_metadata():
         assert r.bucket == "test-bucket"
         assert r.items and len(r.items) == 1
         assert r.items[0].node_type == "ITEM"
+        assert r.next_token == "next-page-token"
+
+
+def test_get_metadata_sends_pagination_params():
+    captured: List[httpx.Request] = []
+    client = _make_capturing_client(captured)
+
+    result = client.files.get_metadata(
+        url=client.my_files_home() / "folder1/folder2/",
+        limit=100,
+        token="page-token",
+    )
+
+    assert isinstance(result, FileMetadata)
+    assert len(captured) == 1
+    request = captured[0]
+    assert request.method == "GET"
+    assert request.url.path == "/v1/metadata/files/test-bucket/folder1/folder2"
+    assert request.url.params["limit"] == "100"
+    assert request.url.params["token"] == "page-token"
 
 
 @pytest.mark.asyncio
@@ -66,3 +125,24 @@ async def test_get_metadata_async():
         assert r.bucket == "test-bucket"
         assert r.items and len(r.items) == 1
         assert r.items[0].node_type == "ITEM"
+        assert r.next_token == "next-page-token"
+
+
+@pytest.mark.asyncio
+async def test_get_metadata_async_sends_pagination_params():
+    captured: List[httpx.Request] = []
+    client = _make_async_capturing_client(captured)
+
+    result = await client.files.get_metadata(
+        url=await client.my_files_home() / "folder1/folder2/",
+        limit=100,
+        token="page-token",
+    )
+
+    assert isinstance(result, FileMetadata)
+    assert len(captured) == 1
+    request = captured[0]
+    assert request.method == "GET"
+    assert request.url.path == "/v1/metadata/files/test-bucket/folder1/folder2"
+    assert request.url.params["limit"] == "100"
+    assert request.url.params["token"] == "page-token"

tests/test_types.py

@@ -1,7 +1,9 @@
+import httpx
 import pytest
 from pydantic import ValidationError
 
 from aidial_client.types.chat.response import Attachment
+from aidial_client.types.file import FileDownloadResponse
 from aidial_client.types.metadata import BaseMetadata
 
 
@@ -54,3 +56,27 @@ def test_metadata_population():
         assert getattr(metadata_by_name, field) == getattr(
             metadata_by_alias, field
         )
+
+
+def test_file_download_response_metadata():
+    response = httpx.Response(
+        200,
+        content=b"test content",
+        headers={"content-type": "text/plain", "x-test-header": "test"},
+    )
+
+    download_response = FileDownloadResponse(
+        response=response, filename="test.txt"
+    )
+
+    assert download_response.headers["x-test-header"] == "test"
+    assert download_response.content_type == "text/plain"
+
+
+def test_file_download_response_metadata_without_content_type():
+    response = httpx.Response(200, content=b"test content")
+    download_response = FileDownloadResponse(
+        response=response, filename="test.txt"
+    )
+
+    assert download_response.content_type is None