Validate info object schemas during unit and integration tests (#7752)

lambdas/indexer/openapi.json

@@ -1100,7 +1100,7 @@
                 ]
             }
         },
-        "/schemas/{facility}/{schema_name}/{version_and_extension}": {
+        "/schemas/{facility}/{name}/{version_and_extension}": {
             "get": {
                 "responses": {
                     "400": {
@@ -1128,21 +1128,44 @@
                                 "schema": {
                                     "type": "object",
                                     "properties": {
-                                        "schema": {
-                                            "type": "string"
+                                        "$schema": {
+                                            "type": "string",
+                                            "format": "url"
                                         },
-                                        "id": {
-                                            "type": "string"
+                                        "$id": {
+                                            "type": "string",
+                                            "format": "url"
                                         },
                                         "type": {
-                                            "type": "string"
+                                            "type": "object"
                                         }
                                     },
-                                    "required": [
-                                        "schema",
-                                        "id",
-                                        "type"
-                                    ],
+                                    "example": {
+                                        "$schema": "https://json-schema.org/draft/2020-12/schema",
+                                        "title": "info",
+                                        "description": "Information describing a file mirrored by Azul",
+                                        "type": "object",
+                                        "properties": {
+                                            "content-type": {
+                                                "type": "array",
+                                                "items": {
+                                                    "type": "string"
+                                                },
+                                                "description": "Content types associated with the mirrored file, as defined for the HTTP response header of the same name"
+                                            },
+                                            "$schema": {
+                                                "type": "string",
+                                                "format": "uri",
+                                                "pattern": "^https?://.*/info/v[1-9][0-9]*\\.json$",
+                                                "description": "URL of a JSON schema the document containing this property is valid against"
+                                            }
+                                        },
+                                        "required": [
+                                            "content-type",
+                                            "$schema"
+                                        ],
+                                        "$id": "http://localhost/schemas/mirror/info/v2.json"
+                                    },
                                     "additionalProperties": true
                                 }
                             }
@@ -1160,24 +1183,27 @@
                         "required": true,
                         "schema": {
                             "type": "string"
-                        }
+                        },
+                        "example": "mirror"
                     },
                     {
-                        "name": "schema_name",
+                        "name": "name",
                         "in": "path",
                         "required": true,
                         "schema": {
                             "type": "string"
-                        }
+                        },
+                        "example": "info"
                     },
                     {
                         "name": "version_and_extension",
                         "in": "path",
                         "required": true,
                         "schema": {
                             "type": "string",
-                            "pattern": "v\\d+\\.json"
-                        }
+                            "pattern": "v([1-9][0-9]*)\\.json"
+                        },
+                        "example": "v2.json"
                     }
                 ],
                 "description": "\n[JSON Schemas](https://json-schema.org/docs) for various Azul facilities.\n"

resources/static/schemas/mirror/info/v1.json

@@ -11,12 +11,12 @@
         "$schema": {
             "type": "string",
             "format": "uri",
-            "pattern": "^https?://.*/info/v\\d+\\.json$",
-            "description": "URL of a JSON schema the JSON containing this property is valid against"
+            "pattern": "^https?://.*/info/v[1-9][0-9]*\\.json$",
+            "description": "URL of a JSON schema the document containing this property is valid against"
         }
     },
     "required": [
         "content-type",
         "$schema"
     ]
-}
+}

resources/static/schemas/mirror/info/v2.json

@@ -14,8 +14,8 @@
         "$schema": {
             "type": "string",
             "format": "uri",
-            "pattern": "^https?://.*/info/v\\d+\\.json$",
-            "description": "URL of a JSON schema the JSON containing this property is valid against"
+            "pattern": "^https?://.*/info/v[1-9][0-9]*\\.json$",
+            "description": "URL of a JSON schema the document containing this property is valid against"
         }
     },
     "required": [

scripts/generate_openapi_document.py

@@ -61,7 +61,8 @@ def main():
         lambda_endpoint = furl('http://localhost')
         with patch.object(target=AzulChaliceApp,
                           attribute='base_url',
-                          new=lambda_endpoint):
+                          new_callable=PropertyMock,
+                          side_effect=lambda_endpoint.copy):
             app_module = load_app_module(app_name)
             assert app_module.app.base_url == lambda_endpoint
             app_spec = app_module.app.spec()

src/azul/indexer/mirror_service.py

@@ -89,6 +89,7 @@
 )
 from azul.types import (
     JSON,
+    MutableJSON,
     json_element_strings,
 )
 
@@ -190,7 +191,7 @@ def update(self,
 
 class SchemaUrlFunc(Protocol):
 
-    def __call__(self, *, schema_name: str, version: int) -> mutable_furl: ...
+    def __call__(self, *, name: str, version: int) -> mutable_furl: ...
 
 
 @attrs.frozen(kw_only=True)
@@ -332,6 +333,8 @@ class BaseMirrorService:
 
     catalog: CatalogName
 
+    info_schema_version = 2
+
     @cached_property
     def _queues(self) -> Queues:
         return Queues()
@@ -481,6 +484,9 @@ def mirror_url(self, file: File) -> str:
                                                file_name=file.name,
                                                content_type=file.content_type)
 
+    def info(self, file: File) -> MutableJSON:
+        return json.loads(self._storage.get_object(self._info_object_key(file)))
+
     def info_exists(self, file: File) -> bool:
         return self._storage.object_exists(self._info_object_key(file))
 
@@ -717,7 +723,8 @@ def _info(self, file: File, old_info: JSON | None = None) -> JSON:
             content_types.add(file.content_type)
         return {
             content_type: sorted(content_types),
-            '$schema': str(self._schema_url_func(schema_name='info', version=2))
+            '$schema': str(self._schema_url_func(name='info',
+                                                 version=self.info_schema_version)),
         }
 
     def _update_info(self, file: File):

src/azul/openapi/schema.py

@@ -263,6 +263,14 @@ def pattern(regex: str | re.Pattern, _type: Form = str) -> JSON:
     }
 
 
+def format(name: str, **kwargs) -> JSON:
+    return {
+        **schema(str),
+        'format': name,
+        **kwargs
+    }
+
+
 def default(default: PrimitiveJSON, /, form: Form = None) -> JSON:
     """
     Add a documented default value to the type schema.

src/azul/schemas.py

@@ -1,8 +1,13 @@
 import json
+import re
 from typing import (
     Any,
 )
 
+from chalice import (
+    BadRequestError,
+)
+
 from azul import (
     format_description as fd,
     mutable_furl,
@@ -24,17 +29,24 @@ class SchemaController(AppController):
     """
     A controller for serving JSON schemas relating to an Azul facility
     """
-    schema_url_path = '/schemas/{facility}/{schema_name}/{version_and_extension}'
+    _schema_route = '/schemas/{facility}/{name}/{version_and_extension}'
+
+    version_and_extension_re = re.compile(r'v([1-9][0-9]*)\.json')
 
-    def schema_url(self,
-                   *,
-                   facility: str,
-                   schema_name: str,
-                   version: int
-                   ) -> mutable_furl:
-        path = self.schema_url_path.format(facility=facility,
-                                           schema_name=schema_name,
-                                           version_and_extension=f'v{version}.json')
+    def _parse_version(self, version_and_extension: str):
+        match = self.version_and_extension_re.match(version_and_extension)
+        if match:
+            return match.group(1)
+        else:
+            raise BadRequestError('Invalid version and extension', version_and_extension)
+
+    def _format_version(self, version: int) -> str:
+        return f'v{version}.json'
+
+    def schema_url(self, facility: str, name: str, version: int) -> mutable_furl:
+        path = self._schema_route.format(facility=facility,
+                                         name=name,
+                                         version_and_extension=self._format_version(version))
         return self.app.base_url.set(path=path)
 
     def handlers(self) -> dict[str, Any]:
@@ -44,16 +56,18 @@ def handlers(self) -> dict[str, Any]:
         """
 
         @self.app.route(
-            self.schema_url_path,
+            self._schema_route,
             methods=['GET'],
             cors=True,
             spec={
                 'summary': 'Retrieve JSON schemas',
                 'tags': ['Auxiliary'],
                 'parameters': [
-                    params.path('facility', str),
-                    params.path('schema_name', str),
-                    params.path('version_and_extension', schema.pattern(r'v\d+\.json')),
+                    params.path('facility', str, example='mirror'),
+                    params.path('name', str, example='info'),
+                    params.path('version_and_extension',
+                                schema.pattern(self.version_and_extension_re.pattern),
+                                example='v2.json'),
                 ],
                 'description': fd(
                     '''
@@ -65,23 +79,27 @@ def handlers(self) -> dict[str, Any]:
                         'description': 'Contents of the schema',
                         **responses.json_content(
                             schema.object(
-                                schema=str,
-                                id=str,
-                                type=str,
-                                additionalProperties=True
+                                properties={
+                                    '$schema': schema.format('url'),
+                                    '$id': schema.format('url'),
+                                    'type': schema.schema(JSON)
+                                },
+                                additionalProperties=True,
+                                example=self.get_schema('mirror', 'info', 2)
                             )
                         )
                     }
                 }
             }
         )
-        def get_schema(facility: str,
-                       schema_name: str,
-                       version_and_extension: str
-                       ) -> JSON:
-            path = 'schemas', facility, schema_name, version_and_extension
-            schema = json.loads(self.app.load_static_resource(*path))
-            schema['$id'] = str(self.app.self_url)
-            return schema
+        def get_schema(facility: str, name: str, version_and_extension: str) -> JSON:
+            version = self._parse_version(version_and_extension)
+            return self.get_schema(facility, name, version)
 
         return locals()
+
+    def get_schema(self, facility: str, name: str, version: int) -> JSON:
+        path = 'schemas', facility, name, self._format_version(version)
+        schema = json.loads(self.app.load_static_resource(*path))
+        schema['$id'] = str(self.schema_url(facility, name, version))
+        return schema

test/indexer/test_mirror_controller.py

@@ -19,6 +19,7 @@
 )
 from azul import (
     R,
+    cached_property,
     config,
 )
 from azul.deployment import (
@@ -212,6 +213,8 @@ def _test_mirror_file(self, file, file_message):
         with patch.object(MirrorService, '_download', return_value=self._file_contents):
             self.mirror_controller.mirror(event)
         self._validate_file_contents(file, self._file_contents)
+        content_types = self._get_content_types_from_info_object(file)
+        self.assertEqual([file.content_type], content_types)
 
     def _test_corrupted_download(self, file_message):
         event = self._mirror_event(file_message)
@@ -250,23 +253,56 @@ def _test_content_type_update(self, file, file_message):
             else:
                 self.assertIn(content_type, new_content_types)
 
+    @cached_property
+    def _info_schema(self) -> JSON:
+        version = self.service.info_schema_version
+        schema = self.mirror_controller.get_schema('mirror', 'info', version)
+        return schema
+
     def _get_content_types_from_info_object(self, file) -> list[str]:
         service = self.service
         info = json.loads(service._storage.get_object(service._info_object_key(file)))
+        jsonschema.validate(info, self._info_schema)
         content_types = info['content-type']
         self.assertIsInstance(content_types, list)
         self.assertEqual(sorted(set(content_types)), content_types)
         return content_types
 
-    def test_info_schema(self):
+    def test_info_schema_response(self):
         client = http_client(log)
         file = MagicMock(content_type='text/plain')
         info = self.service._info(file)
-        response = client.request('GET', info['$schema'])
+        schema_url = info['$schema']
+        response = client.request('GET', schema_url)
         self.assertEqual(200, response.status, response.data)
         schema = json.loads(response.data)
+        self.assertEqual(self._info_schema, schema)
         jsonschema.validate(info, schema)
 
+    def test_info_schema(self):
+        schema = self._info_schema
+        instance = {
+            'content-type': ['application/binary'],
+            '$schema': 'https://localhost/schemas/mirror/info/v2.json'
+        }
+        jsonschema.validate(instance, schema)
+        invalid_instances = [
+            {
+                'content-type': ['application/binary'],
+                '$schema': 'https://localhost/schemas/mirror/info/v0.json'
+            },
+            {
+                'content-type': 'application/binary',
+                '$schema': 'https://localhost/schemas/mirror/info/v3.json'
+            },
+            {
+                '$schema': 'https://localhost/schemas/mirror/info/v4.json'
+            }
+        ]
+        for instance in invalid_instances:
+            with self.assertRaises(jsonschema.exceptions.ValidationError):
+                jsonschema.validate(instance, schema)
+
     def test_files_not_mirrored(self):
         self._create_mock_queues(config.mirror_queue_names)