From c05eb2376ca87c1bdded7251471aec7bff6d0d21 Mon Sep 17 00:00:00 2001 From: "Thomas F. Hagelien" Date: Tue, 27 Feb 2024 11:08:20 +0100 Subject: [PATCH 1/8] Added IRIs as extra field properties in datamodels --- oteapi/models/datacacheconfig.py | 13 +++++---- oteapi/models/filterconfig.py | 15 ++++++++--- oteapi/models/mappingconfig.py | 3 +++ oteapi/models/parserconfig.py | 12 +++++++-- oteapi/models/resourceconfig.py | 11 +++++++- oteapi/models/secretconfig.py | 16 ++++++----- oteapi/models/transformationconfig.py | 30 ++++++++++++++++----- oteapi/strategies/parse/application_json.py | 1 + 8 files changed, 75 insertions(+), 26 deletions(-) diff --git a/oteapi/models/datacacheconfig.py b/oteapi/models/datacacheconfig.py index b36753cf4..68aad6bb0 100644 --- a/oteapi/models/datacacheconfig.py +++ b/oteapi/models/datacacheconfig.py @@ -5,18 +5,18 @@ from pydantic import Field -from oteapi.models.genericconfig import AttrDict - -class DataCacheConfig(AttrDict): +class DataCacheConfig: """DataCache Configuration. - This class should not be used directly as a configuration object for a strategy object, but only as a configuration field inside a configuration object. """ - cacheDir: Path = Field(Path("oteapi"), description="Cache directory.") + cacheDir: Path = Field( + Path("oteapi"), + description="Cache directory.", + ) accessKey: Optional[str] = Field( None, description="Key with which the downloaded content can be accessed. " @@ -36,6 +36,5 @@ class DataCacheConfig(AttrDict): tag: Optional[str] = Field( None, description="Tag assigned to the downloaded content, typically " - "identifying a session. Used with the `evict()` method to clean up a " - "all cache entries with a given tag.", + "identifying a session. Used with the `evict()` method to clean up all cache entries with a given tag.", ) diff --git a/oteapi/models/filterconfig.py b/oteapi/models/filterconfig.py index cf5931c8c..564e011e7 100644 --- a/oteapi/models/filterconfig.py +++ b/oteapi/models/filterconfig.py @@ -11,13 +11,22 @@ class FilterConfig(GenericConfig): """Filter Strategy Data Configuration.""" filterType: str = Field( - ..., description="Type of registered filter strategy. E.g., `filter/sql`." + ..., + description="Type of registered filter strategy. E.g., `filter/sql`.", + IRI="http://purl.org/dc/terms/type", # type: ignore + ) + query: Optional[str] = Field( + None, + description="Define a query operation.", + IRI="http://www.w3.org/1999/02/22-rdf-syntax-ns#Statement", # type: ignore ) - query: Optional[str] = Field(None, description="Define a query operation.") condition: Optional[str] = Field( None, description="Logical statement indicating when a filter should be applied.", + IRI="http://www.w3.org/2000/01/rdf-schema#comment", # type: ignore ) limit: Optional[int] = Field( - None, description="Number of items remaining after a filter expression." + None, + description="Number of items remaining after a filter expression.", + IRI="http://schema.org/Integer", # type: ignore ) diff --git a/oteapi/models/mappingconfig.py b/oteapi/models/mappingconfig.py index 105178932..c4296bda5 100644 --- a/oteapi/models/mappingconfig.py +++ b/oteapi/models/mappingconfig.py @@ -15,6 +15,7 @@ class MappingConfig(GenericConfig): mappingType: str = Field( ..., description="Type of registered mapping strategy.", + IRI="http://purl.org/dc/terms/type", # type: ignore ) prefixes: Optional[Dict[str, str]] = Field( None, @@ -22,8 +23,10 @@ class MappingConfig(GenericConfig): "Dictionary of shortnames that expands to an IRI given as local " "value/IRI-expansion-pairs." ), + IRI="http://www.w3.org/2004/02/skos/core#notation", # type: ignore ) triples: Optional[Set[RDFTriple]] = Field( None, description="Set of RDF triples given as (subject, predicate, object).", + IRI="http://www.w3.org/1999/02/22-rdf-syntax-ns#Statement", # type: ignore ) diff --git a/oteapi/models/parserconfig.py b/oteapi/models/parserconfig.py index 852de9fa5..aefdc9991 100644 --- a/oteapi/models/parserconfig.py +++ b/oteapi/models/parserconfig.py @@ -8,5 +8,13 @@ class ParserConfig(GenericConfig): """Parser Strategy Data Configuration.""" - parserType: str = Field(..., description="Type of registered parser strategy.") - entity: AnyHttpUrl = Field(..., description="IRI to the entity or collection.") + parserType: str = Field( + ..., + description="Type of registered parser strategy.", + IRI="http://purl.org/dc/terms/type", + ) # type: ignore + entity: AnyHttpUrl = Field( + ..., + description="IRI to the metadata (entity) or collection of entities.", + IRI="http://schema.org/URL", + ) # type: ignore diff --git a/oteapi/models/resourceconfig.py b/oteapi/models/resourceconfig.py index 521e5e5c8..e11e4d860 100644 --- a/oteapi/models/resourceconfig.py +++ b/oteapi/models/resourceconfig.py @@ -21,7 +21,9 @@ class ResourceConfig(GenericConfig, SecretConfig): """ resourceType: Optional[str] = Field( - None, description="Type of registered resource strategy." + None, + description="Type of registered resource strategy.", + IRI="http://purl.org/dc/terms/type", # type: ignore ) downloadUrl: Optional[HostlessAnyUrl] = Field( @@ -32,6 +34,7 @@ class ResourceConfig(GenericConfig, SecretConfig): " which this distribution is available directly, typically through a HTTPS" " GET request or SFTP." ), + IRI="http://www.w3.org/ns/dcat#downloadURL", # type: ignore ) mediaType: Optional[str] = Field( None, @@ -42,6 +45,7 @@ class ResourceConfig(GenericConfig, SecretConfig): " type of the distribution is defined in IANA " "[[IANA-MEDIA-TYPES](https://www.w3.org/TR/vocab-dcat-2/#bib-iana-media-types)]." ), + IRI="http://www.w3.org/ns/dcat#mediaType", # type: ignore ) accessUrl: Optional[HostlessAnyUrl] = Field( None, @@ -53,28 +57,33 @@ class ResourceConfig(GenericConfig, SecretConfig): "query or API call.\n`downloadURL` is preferred for direct links to " "downloadable resources." ), + IRI="http://www.w3.org/ns/dcat#accessURL", # type: ignore ) accessService: Optional[str] = Field( None, description=( "A data service that gives access to the distribution of the dataset." ), + IRI="http://www.w3.org/ns/dcat#accessService", # type: ignore ) license: Optional[str] = Field( None, description=( "A legal document under which the distribution is made available." ), + IRI="http://purl.org/dc/terms/license", # type: ignore ) accessRights: Optional[str] = Field( None, description=( "A rights statement that concerns how the distribution is accessed." ), + IRI="http://purl.org/dc/terms/accessRights", # type: ignore ) publisher: Optional[str] = Field( None, description="The entity responsible for making the resource/item available.", + IRI="http://purl.org/dc/terms/publisher", # type: ignore ) @model_validator(mode="after") diff --git a/oteapi/models/secretconfig.py b/oteapi/models/secretconfig.py index 524e143ab..75aebdfd5 100644 --- a/oteapi/models/secretconfig.py +++ b/oteapi/models/secretconfig.py @@ -23,20 +23,22 @@ class SecretConfig(BaseModel): """Simple model for handling secret in other config-models.""" user: Optional[TogglableSecretStr] = Field( - None, description="User name for authentication." + None, + description="User name for authentication.", ) password: Optional[TogglableSecretStr] = Field( - None, description="Password for authentication." + None, + description="Password for authentication.", ) token: Optional[TogglableSecretStr] = Field( None, - description=( - "An access token for providing access and meta data to an application." - ), + description="An access token for providing access and meta data to an application.", ) client_id: Optional[TogglableSecretStr] = Field( - None, description="Client ID for an OAUTH2 client." + None, + description="Client ID for an OAUTH2 client.", ) client_secret: Optional[TogglableSecretStr] = Field( - None, description="Client secret for an OAUTH2 client." + None, + description="Client secret for an OAUTH2 client.", ) diff --git a/oteapi/models/transformationconfig.py b/oteapi/models/transformationconfig.py index 16eaab741..78b3a6886 100644 --- a/oteapi/models/transformationconfig.py +++ b/oteapi/models/transformationconfig.py @@ -39,9 +39,12 @@ class TransformationConfig(GenericConfig, SecretConfig): description=( "Type of registered transformation strategy. E.g., `celery/remote`." ), + IRI="http://purl.org/dc/terms/type", # type: ignore ) name: Optional[str] = Field( - None, description="Human-readable name of the transformation strategy." + None, + description="Human-readable name of the transformation strategy.", + IRI="http://purl.org/dc/elements/1.1/title", # type: ignore ) due: Optional[datetime] = Field( None, @@ -49,30 +52,45 @@ class TransformationConfig(GenericConfig, SecretConfig): "Optional field to indicate a due data/time for when a transformation " "should finish." ), + IRI="http://purl.org/dc/terms/date", # type: ignore ) priority: Optional[ProcessPriority] = Field( ProcessPriority.MEDIUM, description="Define the process priority of the transformation execution.", + IRI="http://www.w3.org/ns/adms#status", # type: ignore ) class TransformationStatus(BaseModel): """Return from transformation status.""" - id: str = Field(..., description="ID for the given transformation process.") + id: str = Field( + ..., + description="ID for the given transformation process.", + IRI="http://purl.org/dc/terms/identifier", # type: ignore + ) status: Optional[str] = Field( - None, description="Status for the transformation process." + None, + description="Status for the transformation process.", + IRI="http://www.w3.org/ns/adms#status", # type: ignore ) messages: Optional[List[str]] = Field( - None, description="Messages related to the transformation process." + None, + description="Messages related to the transformation process.", + IRI="http://purl.org/dc/terms/description", # type: ignore ) created: Optional[datetime] = Field( None, description="Time of creation for the transformation process. Given in UTC.", + IRI="http://purl.org/dc/terms/created", # type: ignore ) startTime: Optional[datetime] = Field( - None, description="Time when the transformation process started. Given in UTC." + None, + description="Time when the transformation process started. Given in UTC.", + IRI="http://purl.org/dc/terms/date", # type: ignore ) finishTime: Optional[datetime] = Field( - None, description="Time when the tranformation process finished. Given in UTC." + None, + description="Time when the tranformation process finished. Given in UTC.", + IRI="http://purl.org/dc/terms/date", # type: ignore ) diff --git a/oteapi/strategies/parse/application_json.py b/oteapi/strategies/parse/application_json.py index 96f4c52f8..ea19fd3a5 100644 --- a/oteapi/strategies/parse/application_json.py +++ b/oteapi/strategies/parse/application_json.py @@ -45,6 +45,7 @@ class JSONParserConfig(ParserConfig): "parser/json", description=ParserConfig.model_fields["parserType"].description, ) + configuration: JSONConfig = Field( ..., description="JSON parse strategy-specific configuration." ) From 8fb9de381e05984c92b61a4fb304f4033a4baa4b Mon Sep 17 00:00:00 2001 From: "Thomas F. Hagelien" Date: Tue, 27 Feb 2024 11:11:25 +0100 Subject: [PATCH 2/8] reset DataCacheConfig signature --- oteapi/models/datacacheconfig.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/oteapi/models/datacacheconfig.py b/oteapi/models/datacacheconfig.py index 68aad6bb0..12d0d21ca 100644 --- a/oteapi/models/datacacheconfig.py +++ b/oteapi/models/datacacheconfig.py @@ -5,8 +5,10 @@ from pydantic import Field +from oteapi.models.genericconfig import AttrDict -class DataCacheConfig: + +class DataCacheConfig(AttrDict): """DataCache Configuration. This class should not be used directly as a configuration object for a strategy object, but only as a configuration field inside From 70c113b3f5b1e5927fe487612a896551b0ddfdc8 Mon Sep 17 00:00:00 2001 From: Thomas Hagelien Date: Thu, 29 Feb 2024 13:00:59 +0100 Subject: [PATCH 3/8] Update oteapi/models/filterconfig.py Co-authored-by: Casper Welzel Andersen <43357585+CasperWA@users.noreply.github.com> --- oteapi/models/filterconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oteapi/models/filterconfig.py b/oteapi/models/filterconfig.py index 564e011e7..abfd584a8 100644 --- a/oteapi/models/filterconfig.py +++ b/oteapi/models/filterconfig.py @@ -18,7 +18,7 @@ class FilterConfig(GenericConfig): query: Optional[str] = Field( None, description="Define a query operation.", - IRI="http://www.w3.org/1999/02/22-rdf-syntax-ns#Statement", # type: ignore + json_schema_extra={"IRI": "http://www.w3.org/1999/02/22-rdf-syntax-ns#Statement"}, ) condition: Optional[str] = Field( None, From c7d19747961afb31d25a757621017cba434dfd91 Mon Sep 17 00:00:00 2001 From: Thomas Hagelien Date: Thu, 29 Feb 2024 13:01:10 +0100 Subject: [PATCH 4/8] Update oteapi/models/datacacheconfig.py Co-authored-by: Casper Welzel Andersen <43357585+CasperWA@users.noreply.github.com> --- oteapi/models/datacacheconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/oteapi/models/datacacheconfig.py b/oteapi/models/datacacheconfig.py index 12d0d21ca..8a5a5b9f5 100644 --- a/oteapi/models/datacacheconfig.py +++ b/oteapi/models/datacacheconfig.py @@ -10,6 +10,7 @@ class DataCacheConfig(AttrDict): """DataCache Configuration. + This class should not be used directly as a configuration object for a strategy object, but only as a configuration field inside a configuration object. From f00b31e70bee8e798b8978b44521638f21a19203 Mon Sep 17 00:00:00 2001 From: Thomas Hagelien Date: Thu, 29 Feb 2024 13:01:20 +0100 Subject: [PATCH 5/8] Update oteapi/models/filterconfig.py Co-authored-by: Casper Welzel Andersen <43357585+CasperWA@users.noreply.github.com> --- oteapi/models/filterconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oteapi/models/filterconfig.py b/oteapi/models/filterconfig.py index abfd584a8..1193b14b5 100644 --- a/oteapi/models/filterconfig.py +++ b/oteapi/models/filterconfig.py @@ -13,7 +13,7 @@ class FilterConfig(GenericConfig): filterType: str = Field( ..., description="Type of registered filter strategy. E.g., `filter/sql`.", - IRI="http://purl.org/dc/terms/type", # type: ignore + json_schema_extra={"IRI": "http://purl.org/dc/terms/type"}, ) query: Optional[str] = Field( None, From 37551c7961f9d2fa758c484e4c72b7455e2f61ae Mon Sep 17 00:00:00 2001 From: Thomas Hagelien Date: Thu, 29 Feb 2024 13:01:27 +0100 Subject: [PATCH 6/8] Update oteapi/models/filterconfig.py Co-authored-by: Casper Welzel Andersen <43357585+CasperWA@users.noreply.github.com> --- oteapi/models/filterconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oteapi/models/filterconfig.py b/oteapi/models/filterconfig.py index 1193b14b5..b0fcc4bb6 100644 --- a/oteapi/models/filterconfig.py +++ b/oteapi/models/filterconfig.py @@ -23,7 +23,7 @@ class FilterConfig(GenericConfig): condition: Optional[str] = Field( None, description="Logical statement indicating when a filter should be applied.", - IRI="http://www.w3.org/2000/01/rdf-schema#comment", # type: ignore + json_schema_extra={"IRI": "http://www.w3.org/2000/01/rdf-schema#comment"}, ) limit: Optional[int] = Field( None, From 216d71735a4a05d115b1bc7bad094129cc9ae21c Mon Sep 17 00:00:00 2001 From: Thomas Hagelien Date: Thu, 29 Feb 2024 13:01:44 +0100 Subject: [PATCH 7/8] Update oteapi/models/filterconfig.py Co-authored-by: Casper Welzel Andersen <43357585+CasperWA@users.noreply.github.com> --- oteapi/models/filterconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oteapi/models/filterconfig.py b/oteapi/models/filterconfig.py index b0fcc4bb6..99e29ad85 100644 --- a/oteapi/models/filterconfig.py +++ b/oteapi/models/filterconfig.py @@ -28,5 +28,5 @@ class FilterConfig(GenericConfig): limit: Optional[int] = Field( None, description="Number of items remaining after a filter expression.", - IRI="http://schema.org/Integer", # type: ignore + json_schema_extra={"IRI": "http://schema.org/Integer"}, ) From 3f9b721eab0c7c135899b6bf07884d0534f4aa74 Mon Sep 17 00:00:00 2001 From: Thomas Hagelien Date: Thu, 29 Feb 2024 13:02:07 +0100 Subject: [PATCH 8/8] Update oteapi/models/transformationconfig.py Co-authored-by: Jesper Friis --- oteapi/models/transformationconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oteapi/models/transformationconfig.py b/oteapi/models/transformationconfig.py index 78b3a6886..f7be6b69b 100644 --- a/oteapi/models/transformationconfig.py +++ b/oteapi/models/transformationconfig.py @@ -44,7 +44,7 @@ class TransformationConfig(GenericConfig, SecretConfig): name: Optional[str] = Field( None, description="Human-readable name of the transformation strategy.", - IRI="http://purl.org/dc/elements/1.1/title", # type: ignore + IRI="http://purl.org/dc/terms/title", # type: ignore ) due: Optional[datetime] = Field( None,