From 8e48a8f3cabaec056cee600728564113736c2934 Mon Sep 17 00:00:00 2001 From: Amund Isaksen Date: Thu, 5 Mar 2026 07:16:48 +0100 Subject: [PATCH 01/13] Add bufr encoding --- api/formatters/__init__.py | 2 ++ api/formatters/bufr.py | 12 ++++++++++++ api/requirements.in | 1 + api/requirements.txt | 19 +++++++++---------- 4 files changed, 24 insertions(+), 10 deletions(-) create mode 100644 api/formatters/bufr.py diff --git a/api/formatters/__init__.py b/api/formatters/__init__.py index 14768c23..1be8b61d 100644 --- a/api/formatters/__init__.py +++ b/api/formatters/__init__.py @@ -3,6 +3,7 @@ from . import covjson from . import geojson +from . import bufr logger = logging.getLogger(__name__) @@ -17,5 +18,6 @@ class Metadata_Formats(str, Enum): formatters = { "CoverageJSON": covjson.convert_to_covjson, + "bufr": bufr.convert_to_bufr, } # observations metadata_formatters = {"GeoJSON": geojson.convert_to_geojson} # metadata diff --git a/api/formatters/bufr.py b/api/formatters/bufr.py new file mode 100644 index 00000000..f3e805cc --- /dev/null +++ b/api/formatters/bufr.py @@ -0,0 +1,12 @@ +import json + +from . import covjson + +from bufr_tools import covjson2bufr + + +def convert_to_bufr(raw_data): + cov_json = covjson.convert_to_covjson(raw_data) + bufr_content = covjson2bufr(json.dumps(cov_json.model_dump(mode="json"))) + + return bufr_content diff --git a/api/requirements.in b/api/requirements.in index a100962f..fe58547f 100644 --- a/api/requirements.in +++ b/api/requirements.in @@ -16,3 +16,4 @@ jinja2~=3.1 isodate~=0.7.2 prometheus-fastapi-instrumentator~=7.0.0 rdflib~=7.1.4 +rodeo-bufr-tools~=0.4.2 diff --git a/api/requirements.txt b/api/requirements.txt index ad778ee3..a00fdd61 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,9 +1,5 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --no-emit-index-url requirements.in -# +# This file was autogenerated by uv via the following command: +# uv pip compile requirements.in -o requirements.txt annotated-types==0.7.0 # via pydantic anyio==4.9.0 @@ -52,6 +48,8 @@ prometheus-fastapi-instrumentator==7.0.2 # via -r requirements.in protobuf==5.29.4 # via grpcio-tools +pybind11==2.11.2 + # via rodeo-bufr-tools pydantic==2.11.4 # via # covjson-pydantic @@ -68,6 +66,10 @@ pyyaml==6.0.2 # via uvicorn rdflib==7.1.4 # via -r requirements.in +rodeo-bufr-tools==0.4.2 + # via -r requirements.in +setuptools==82.0.0 + # via grpcio-tools shapely==2.1.1 # via -r requirements.in sniffio==1.3.1 @@ -87,7 +89,7 @@ typing-extensions==4.13.2 # typing-inspection typing-inspection==0.4.0 # via pydantic -uvicorn[standard]==0.32.1 +uvicorn==0.32.1 # via -r requirements.in uvloop==0.21.0 # via uvicorn @@ -95,6 +97,3 @@ watchfiles==1.0.5 # via uvicorn websockets==15.0.1 # via uvicorn - -# The following packages are considered to be unsafe in a requirements file: -# setuptools From 981bd4193e9042e76e6a70fd58174c28bf0f3130 Mon Sep 17 00:00:00 2001 From: Amund Isaksen Date: Thu, 5 Mar 2026 07:18:30 +0100 Subject: [PATCH 02/13] Add uv lock file to ignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 453230cc..c91eba0e 100644 --- a/.gitignore +++ b/.gitignore @@ -220,3 +220,6 @@ qudt_units.json api/test/output/ datastore/load-test/output/ ingest/test/output/ + +# Ignore uv lock file +*uv.lock From 860644875738c475821593cfdb629de2a534277c Mon Sep 17 00:00:00 2001 From: Amund Isaksen Date: Mon, 9 Mar 2026 13:59:14 +0100 Subject: [PATCH 03/13] Add bufr library dep --- api/Dockerfile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index 3b644dd0..284cf98f 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -5,26 +5,26 @@ SHELL ["/bin/bash", "-eux", "-o", "pipefail", "-c"] ENV DOCKER_PATH="/app" RUN apt-get update \ - && apt-get -y upgrade \ - && apt-get install -y --no-install-recommends git curl \ - # Cleanup - && rm -rf /usr/tmp \ - && apt-get autoremove -y \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + && apt-get -y upgrade \ + && apt-get install -y --no-install-recommends git curl libeccodes-data \ + # Cleanup + && rm -rf /usr/tmp \ + && apt-get autoremove -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* COPY "./protobuf/datastore.proto" "/protobuf/datastore.proto" COPY "./requirements.txt" "${DOCKER_PATH}/requirements.txt" # hadolint ignore=DL3013 RUN pip install --no-cache-dir --upgrade pip \ - && pip install --no-cache-dir --upgrade -r "${DOCKER_PATH}/requirements.txt" + && pip install --no-cache-dir --upgrade -r "${DOCKER_PATH}/requirements.txt" # Compiling the protobuf file RUN python -m grpc_tools.protoc \ - --proto_path="protobuf" "protobuf/datastore.proto" \ - --python_out="${DOCKER_PATH}" \ - --grpc_python_out="${DOCKER_PATH}" + --proto_path="protobuf" "protobuf/datastore.proto" \ + --python_out="${DOCKER_PATH}" \ + --grpc_python_out="${DOCKER_PATH}" COPY "." "${DOCKER_PATH}/" From 91ce1b03e68cd49e24a07f16c65dfd9d5ad96b59 Mon Sep 17 00:00:00 2001 From: Amund Isaksen Date: Mon, 9 Mar 2026 13:59:57 +0100 Subject: [PATCH 04/13] Add new bufr formatter --- api/formatters/__init__.py | 11 +++++++++-- api/formatters/bufr.py | 8 ++++++-- api/formatters/covjson.py | 13 ++++++++++--- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/api/formatters/__init__.py b/api/formatters/__init__.py index 1be8b61d..3aa6d2bd 100644 --- a/api/formatters/__init__.py +++ b/api/formatters/__init__.py @@ -10,6 +10,7 @@ class Formats(str, Enum): covjson = "CoverageJSON" # According to EDR spec + bufr = "bufr" class Metadata_Formats(str, Enum): @@ -17,7 +18,13 @@ class Metadata_Formats(str, Enum): formatters = { - "CoverageJSON": covjson.convert_to_covjson, - "bufr": bufr.convert_to_bufr, + "CoverageJSON": { + "format_function": covjson.convert_to_covjson, + "response_format": "application/prs.coverage+json", + }, + "bufr": { + "format_function": bufr.convert_to_bufr, + "response_format": "application/bufr", + }, } # observations metadata_formatters = {"GeoJSON": geojson.convert_to_geojson} # metadata diff --git a/api/formatters/bufr.py b/api/formatters/bufr.py index f3e805cc..0e500104 100644 --- a/api/formatters/bufr.py +++ b/api/formatters/bufr.py @@ -5,8 +5,12 @@ from bufr_tools import covjson2bufr -def convert_to_bufr(raw_data): +def convert_to_bufr(raw_data: str): cov_json = covjson.convert_to_covjson(raw_data) - bufr_content = covjson2bufr(json.dumps(cov_json.model_dump(mode="json"))) + print(cov_json) + bufr_content = covjson2bufr.covjson2bufr(json.dumps(cov_json)) + print(bufr_content) + if not bufr_content and not cov_json: + raise ValueError("No content") return bufr_content diff --git a/api/formatters/covjson.py b/api/formatters/covjson.py index 2456a472..eabf8ec3 100644 --- a/api/formatters/covjson.py +++ b/api/formatters/covjson.py @@ -77,7 +77,10 @@ def convert_to_covjson(observations): referencing = [ ReferenceSystemConnectionObject( coordinates=["x", "y"], - system=ReferenceSystem(type="GeographicCRS", id="http://www.opengis.net/def/crs/OGC/1.3/CRS84"), + system=ReferenceSystem( + type="GeographicCRS", + id="http://www.opengis.net/def/crs/OGC/1.3/CRS84", + ), ), ReferenceSystemConnectionObject( coordinates=["t"], @@ -106,7 +109,9 @@ def convert_to_covjson(observations): parameters[parameter_id] = make_parameter(data.ts_mdata) ranges[parameter_id] = NdArrayFloat( - values=values_no_nan, axisNames=["t", "x", "y"], shape=[len(values_no_nan), 1, 1] + values=values_no_nan, + axisNames=["t", "x", "y"], + shape=[len(values_no_nan), 1, 1], ) custom_fields = {"metocean:wigosId": data.ts_mdata.platform} @@ -118,7 +123,9 @@ def convert_to_covjson(observations): return coverages[0] else: parameter_union = reduce(operator.ior, (c.parameters.root for c in coverages), {}) - return CoverageCollection(coverages=coverages, parameters=dict(sorted(parameter_union.items()))) + return CoverageCollection( + coverages=coverages, parameters=dict(sorted(parameter_union.items())) + ).model_dump_json() # (mode="json") def _collect_data(ts_mdata, obs_mdata): From 25611dc152d5971842e6a6c7686887ee9c0b65d6 Mon Sep 17 00:00:00 2001 From: Amund Isaksen Date: Mon, 9 Mar 2026 14:00:19 +0100 Subject: [PATCH 05/13] Add support for bufr output --- api/routers/edr.py | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/api/routers/edr.py b/api/routers/edr.py index 85c9aff5..8efc3bc1 100644 --- a/api/routers/edr.py +++ b/api/routers/edr.py @@ -17,6 +17,7 @@ from fastapi import Path from fastapi import Query from fastapi import Request +from fastapi.responses import Response from formatters.covjson import make_parameter from geojson_pydantic import Feature from geojson_pydantic import Point @@ -174,7 +175,11 @@ async def get_locations( ) async def get_data_location_id( location_id: Annotated[ - str, Path(description=edr_query_parameter_descriptions.wigos_id, openapi_examples=openapi_examples.wigos_id) + str, + Path( + description=edr_query_parameter_descriptions.wigos_id, + openapi_examples=openapi_examples.wigos_id, + ), ], parameter_name: Annotated[ str | None, @@ -186,7 +191,10 @@ async def get_data_location_id( ] = None, datetime: Annotated[ str | None, - Query(description=edr_query_parameter_descriptions.datetime, openapi_examples=openapi_examples.datetime), + Query( + description=edr_query_parameter_descriptions.datetime, + openapi_examples=openapi_examples.datetime, + ), ] = None, f: Annotated[ formatters.Formats, Query(description=edr_query_parameter_descriptions.format) @@ -231,9 +239,9 @@ async def get_data_location_id( grpc_response = await get_obs_request(request) observations = grpc_response.observations - response = formatters.formatters[f](observations) + response = formatters.formatters[f]["format_function"](observations) - return response + return Response(content=response, media_type=formatters.formatters[f]["response_format"]) @router.get( @@ -245,7 +253,11 @@ async def get_data_location_id( ) async def get_data_position( coords: Annotated[ - str, Query(description=edr_query_parameter_descriptions.point, openapi_examples=openapi_examples.point) + str, + Query( + description=edr_query_parameter_descriptions.point, + openapi_examples=openapi_examples.point, + ), ], parameter_name: Annotated[ str | None, @@ -324,9 +336,9 @@ async def get_data_position( grpc_response = await get_obs_request(request) observations = grpc_response.observations - response = formatters.formatters[f](observations) + response = formatters.formatters[f]["format_function"](observations) - return response + return Response(content=response, media_type=formatters.formatters[f]["response_format"]) @router.get( @@ -338,7 +350,11 @@ async def get_data_position( ) async def get_data_area( coords: Annotated[ - str, Query(description=edr_query_parameter_descriptions.area, openapi_examples=openapi_examples.polygon) + str, + Query( + description=edr_query_parameter_descriptions.area, + openapi_examples=openapi_examples.polygon, + ), ], parameter_name: Annotated[ str | None, @@ -350,7 +366,10 @@ async def get_data_area( ] = None, datetime: Annotated[ str | None, - Query(description=edr_query_parameter_descriptions.datetime, openapi_examples=openapi_examples.datetime), + Query( + description=edr_query_parameter_descriptions.datetime, + openapi_examples=openapi_examples.datetime, + ), ] = None, f: Annotated[ formatters.Formats, Query(description=edr_query_parameter_descriptions.format) @@ -415,6 +434,6 @@ async def get_data_area( grpc_response = await get_obs_request(request) observations = grpc_response.observations - response = formatters.formatters[f](observations) + response = formatters.formatters[f]["format_function"](observations) - return response + return Response(content=response, media_type=formatters.formatters[f]["response_format"]) From d5d5b56dbca10be3056a480b1d21a0ace5b9fc8b Mon Sep 17 00:00:00 2001 From: Amund Isaksen Date: Mon, 9 Mar 2026 14:51:27 +0100 Subject: [PATCH 06/13] Convert covjson to json string --- api/formatters/bufr.py | 7 ++----- api/formatters/covjson.py | 4 +++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/api/formatters/bufr.py b/api/formatters/bufr.py index 0e500104..781a0b5a 100644 --- a/api/formatters/bufr.py +++ b/api/formatters/bufr.py @@ -1,5 +1,3 @@ -import json - from . import covjson from bufr_tools import covjson2bufr @@ -7,9 +5,8 @@ def convert_to_bufr(raw_data: str): cov_json = covjson.convert_to_covjson(raw_data) - print(cov_json) - bufr_content = covjson2bufr.covjson2bufr(json.dumps(cov_json)) - print(bufr_content) + print(type(cov_json)) + bufr_content = covjson2bufr.covjson2bufr(cov_json) if not bufr_content and not cov_json: raise ValueError("No content") diff --git a/api/formatters/covjson.py b/api/formatters/covjson.py index eabf8ec3..25450b7e 100644 --- a/api/formatters/covjson.py +++ b/api/formatters/covjson.py @@ -125,7 +125,9 @@ def convert_to_covjson(observations): parameter_union = reduce(operator.ior, (c.parameters.root for c in coverages), {}) return CoverageCollection( coverages=coverages, parameters=dict(sorted(parameter_union.items())) - ).model_dump_json() # (mode="json") + ).model_dump_json( + exclude_none=True + ) # (mode="json") def _collect_data(ts_mdata, obs_mdata): From 0ce437c84d5b25d72f0ec98d03b22ee84240fe42 Mon Sep 17 00:00:00 2001 From: Amund Isaksen Date: Wed, 11 Mar 2026 15:41:47 +0100 Subject: [PATCH 07/13] Bump reodeo-bufr-tools --- api/requirements.in | 2 +- api/requirements.txt | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/api/requirements.in b/api/requirements.in index fe58547f..c44909bf 100644 --- a/api/requirements.in +++ b/api/requirements.in @@ -16,4 +16,4 @@ jinja2~=3.1 isodate~=0.7.2 prometheus-fastapi-instrumentator~=7.0.0 rdflib~=7.1.4 -rodeo-bufr-tools~=0.4.2 +rodeo-bufr-tools~=0.4.5 diff --git a/api/requirements.txt b/api/requirements.txt index a00fdd61..19a792db 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,5 +1,9 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile requirements.in -o requirements.txt +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --no-emit-index-url requirements.in +# annotated-types==0.7.0 # via pydantic anyio==4.9.0 @@ -66,10 +70,8 @@ pyyaml==6.0.2 # via uvicorn rdflib==7.1.4 # via -r requirements.in -rodeo-bufr-tools==0.4.2 +rodeo-bufr-tools==0.4.5 # via -r requirements.in -setuptools==82.0.0 - # via grpcio-tools shapely==2.1.1 # via -r requirements.in sniffio==1.3.1 @@ -89,7 +91,7 @@ typing-extensions==4.13.2 # typing-inspection typing-inspection==0.4.0 # via pydantic -uvicorn==0.32.1 +uvicorn[standard]==0.32.1 # via -r requirements.in uvloop==0.21.0 # via uvicorn @@ -97,3 +99,6 @@ watchfiles==1.0.5 # via uvicorn websockets==15.0.1 # via uvicorn + +# The following packages are considered to be unsafe in a requirements file: +# setuptools From 9ad239f416a0e9be762306e8a7bbb46e4ba19e7d Mon Sep 17 00:00:00 2001 From: Amund Isaksen Date: Wed, 11 Mar 2026 16:08:15 +0100 Subject: [PATCH 08/13] Dump singel coverage to json str --- api/formatters/covjson.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/formatters/covjson.py b/api/formatters/covjson.py index 25450b7e..6e3a29e9 100644 --- a/api/formatters/covjson.py +++ b/api/formatters/covjson.py @@ -120,7 +120,7 @@ def convert_to_covjson(observations): if len(coverages) == 0: raise HTTPException(status_code=404, detail="Requested data not found.") elif len(coverages) == 1: - return coverages[0] + return coverages[0].model_dump_json(exclude_none=True) else: parameter_union = reduce(operator.ior, (c.parameters.root for c in coverages), {}) return CoverageCollection( From 1a877c06df6d9df3f3f42a4ca10dadff3ddeeba7 Mon Sep 17 00:00:00 2001 From: istvans Date: Mon, 23 Mar 2026 09:32:35 +0100 Subject: [PATCH 09/13] BUFR output sequence description --- api/BUFR_out.md | 62 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 api/BUFR_out.md diff --git a/api/BUFR_out.md b/api/BUFR_out.md new file mode 100644 index 00000000..6ad54df7 --- /dev/null +++ b/api/BUFR_out.md @@ -0,0 +1,62 @@ +# E-SOH BUFR Output Format Sequence + +E-SOH provides a BUFR output file when you select the BUFR format from the dropdown menu. The following table describes the BUFR sequence. + +| Descriptor | Extended/Repeated | Name | Data Source | Note | +|------------|-------------------|------|-------------|------| +||| +||| **STATION IDENTIFICATION** +301150 || WIGOS identifier | metocean:wigosId | Mandatory +|| 0 01 125 | WIGOS identifier series +|| 0 01 126 | WIGOS issuer of identifier +|| 0 01 127 | WIGOS issue number +|| 0 01 128 | WIGOS local identifier (character) +301090 || Surface station identification || Mandatory +|| 3 01 004 | Surface station identification +|| 3 01 011 | Year, month, day +|| 3 01 012 | Hour, minute +|| 3 01 021 | Latitude/longitude (high accuracy) +|| 0 07 030 | Height of station ground above mean sea level +|| 0 07 031 | Height of barometer above mean sea level +||| **BASIC SURFACE OBSERVATIONS** +302031 || Pressure information | air_pressure, air_pressure_at_mean_sea_level | Optional +302032 || Temperature and humidity data | air_temperature, dew_point_temperature, relative_humidity | Optional +302033 || Visibility data || Optional +||| +||| **EXTREME TEMPERATURES** (Optional) +105000 || Replication descriptor +031001 || Delayed descriptor replication factor +|| 007032 | Height of sensor above local ground +|| 004025 | Time period or displacement +|| 012111 | Maximum temperature, at height and over period specified | air_temperature +|| 004025 | Time period or displacement +|| 012112 | Minimum temperature, at height and over period specified | air_temperature +||| +||| **WIND DATA** (Optional) +302042 ||| wind_speed, wind_from_direction, wind_speed_of_gust, wind_gust_from_direction +||| +||| **RADIATION** (Optional) +106000 || Replication descriptor +031101 || Delayed descriptor replication factor +|| 007032 | Height of sensor above local ground +|| 004025 | Time period or displacement +|| 014002 | Long-wave radiation, integrated over period specified | integral_wrt_time_of_surface_downwelling_longwave_flux_in_air +|| 014004 | Short-wave radiation, integrated over period specified | integral_wrt_time_of_surface_downwelling_shortwave_flux_in_air +|| 014012 | Net long-wave radiation, integrated over period specified | integral_wrt_time_of_surface_net_downward_longwave_flux +|| 014014 | Net short-wave radiation, integrated over period specified | integral_wrt_time_of_surface_net_downward_shortwave_flux +||| +||| **RADIATION** (Optional) +106000 || Replication descriptor +031101 || Delayed descriptor replication factor +|| 007032 | Height of sensor above local ground +|| 014002 | Downward long-wave radiation, integrated over period specified | integral_wrt_time_of_surface_net_downward_longwave_flux | Positive +|| 014002 | Upward long-wave radiation, integrated over period specified | integral_wrt_time_of_surface_net_downward_longwave_flux | Negative +|| 014004 | Downward short-wave radiation, integrated over period specified | integral_wrt_time_of_surface_downwelling_shortwave_flux_in_air | Positive +|| 014004 | Upward short-wave radiation, integrated over period specified | integral_wrt_time_of_surface_downwelling_shortwave_flux_in_air | Negative +||| +||| **PRECIPITATION** (Optional) +103000 || Replication descriptor +031001 || Delayed descriptor replication factor +|| 007032 | Height of sensor above local ground +|| 004025 | Time period or displacement +|| 013011 | Total precipitation/total, water equivalent | precipitation_amount \ No newline at end of file From 0930f2a75a89a7ce361659f64ebe5893f46661d0 Mon Sep 17 00:00:00 2001 From: istvans Date: Mon, 23 Mar 2026 09:40:05 +0100 Subject: [PATCH 10/13] Removed whitespaces --- api/BUFR_out.md | 112 ++++++++++++++++++++++++------------------------ 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/api/BUFR_out.md b/api/BUFR_out.md index 6ad54df7..820c6098 100644 --- a/api/BUFR_out.md +++ b/api/BUFR_out.md @@ -4,59 +4,59 @@ E-SOH provides a BUFR output file when you select the BUFR format from the dropd | Descriptor | Extended/Repeated | Name | Data Source | Note | |------------|-------------------|------|-------------|------| -||| -||| **STATION IDENTIFICATION** -301150 || WIGOS identifier | metocean:wigosId | Mandatory -|| 0 01 125 | WIGOS identifier series -|| 0 01 126 | WIGOS issuer of identifier -|| 0 01 127 | WIGOS issue number -|| 0 01 128 | WIGOS local identifier (character) -301090 || Surface station identification || Mandatory -|| 3 01 004 | Surface station identification -|| 3 01 011 | Year, month, day -|| 3 01 012 | Hour, minute -|| 3 01 021 | Latitude/longitude (high accuracy) -|| 0 07 030 | Height of station ground above mean sea level -|| 0 07 031 | Height of barometer above mean sea level -||| **BASIC SURFACE OBSERVATIONS** -302031 || Pressure information | air_pressure, air_pressure_at_mean_sea_level | Optional -302032 || Temperature and humidity data | air_temperature, dew_point_temperature, relative_humidity | Optional -302033 || Visibility data || Optional -||| -||| **EXTREME TEMPERATURES** (Optional) -105000 || Replication descriptor -031001 || Delayed descriptor replication factor -|| 007032 | Height of sensor above local ground -|| 004025 | Time period or displacement -|| 012111 | Maximum temperature, at height and over period specified | air_temperature -|| 004025 | Time period or displacement -|| 012112 | Minimum temperature, at height and over period specified | air_temperature -||| -||| **WIND DATA** (Optional) -302042 ||| wind_speed, wind_from_direction, wind_speed_of_gust, wind_gust_from_direction -||| -||| **RADIATION** (Optional) -106000 || Replication descriptor -031101 || Delayed descriptor replication factor -|| 007032 | Height of sensor above local ground -|| 004025 | Time period or displacement -|| 014002 | Long-wave radiation, integrated over period specified | integral_wrt_time_of_surface_downwelling_longwave_flux_in_air -|| 014004 | Short-wave radiation, integrated over period specified | integral_wrt_time_of_surface_downwelling_shortwave_flux_in_air -|| 014012 | Net long-wave radiation, integrated over period specified | integral_wrt_time_of_surface_net_downward_longwave_flux -|| 014014 | Net short-wave radiation, integrated over period specified | integral_wrt_time_of_surface_net_downward_shortwave_flux -||| -||| **RADIATION** (Optional) -106000 || Replication descriptor -031101 || Delayed descriptor replication factor -|| 007032 | Height of sensor above local ground -|| 014002 | Downward long-wave radiation, integrated over period specified | integral_wrt_time_of_surface_net_downward_longwave_flux | Positive -|| 014002 | Upward long-wave radiation, integrated over period specified | integral_wrt_time_of_surface_net_downward_longwave_flux | Negative -|| 014004 | Downward short-wave radiation, integrated over period specified | integral_wrt_time_of_surface_downwelling_shortwave_flux_in_air | Positive -|| 014004 | Upward short-wave radiation, integrated over period specified | integral_wrt_time_of_surface_downwelling_shortwave_flux_in_air | Negative -||| -||| **PRECIPITATION** (Optional) -103000 || Replication descriptor -031001 || Delayed descriptor replication factor -|| 007032 | Height of sensor above local ground -|| 004025 | Time period or displacement -|| 013011 | Total precipitation/total, water equivalent | precipitation_amount \ No newline at end of file +||| +||| **STATION IDENTIFICATION** +301150 || WIGOS identifier | metocean:wigosId | Mandatory +|| 0 01 125 | WIGOS identifier series +|| 0 01 126 | WIGOS issuer of identifier +|| 0 01 127 | WIGOS issue number +|| 0 01 128 | WIGOS local identifier (character) +301090 || Surface station identification || Mandatory +|| 3 01 004 | Surface station identification +|| 3 01 011 | Year, month, day +|| 3 01 012 | Hour, minute +|| 3 01 021 | Latitude/longitude (high accuracy) +|| 0 07 030 | Height of station ground above mean sea level +|| 0 07 031 | Height of barometer above mean sea level +||| **BASIC SURFACE OBSERVATIONS** +302031 || Pressure information | air_pressure, air_pressure_at_mean_sea_level | Optional +302032 || Temperature and humidity data | air_temperature, dew_point_temperature, relative_humidity | Optional +302033 || Visibility data || Optional +||| +||| **EXTREME TEMPERATURES** (Optional) +105000 || Replication descriptor +031001 || Delayed descriptor replication factor +|| 007032 | Height of sensor above local ground +|| 004025 | Time period or displacement +|| 012111 | Maximum temperature, at height and over period specified | air_temperature +|| 004025 | Time period or displacement +|| 012112 | Minimum temperature, at height and over period specified | air_temperature +||| +||| **WIND DATA** (Optional) +302042 ||| wind_speed, wind_from_direction, wind_speed_of_gust, wind_gust_from_direction +||| +||| **RADIATION** (Optional) +106000 || Replication descriptor +031101 || Delayed descriptor replication factor +|| 007032 | Height of sensor above local ground +|| 004025 | Time period or displacement +|| 014002 | Long-wave radiation, integrated over period specified | integral_wrt_time_of_surface_downwelling_longwave_flux_in_air +|| 014004 | Short-wave radiation, integrated over period specified | integral_wrt_time_of_surface_downwelling_shortwave_flux_in_air +|| 014012 | Net long-wave radiation, integrated over period specified | integral_wrt_time_of_surface_net_downward_longwave_flux +|| 014014 | Net short-wave radiation, integrated over period specified | integral_wrt_time_of_surface_net_downward_shortwave_flux +||| +||| **RADIATION** (Optional) +106000 || Replication descriptor +031101 || Delayed descriptor replication factor +|| 007032 | Height of sensor above local ground +|| 014002 | Downward long-wave radiation, integrated over period specified | integral_wrt_time_of_surface_net_downward_longwave_flux | Positive +|| 014002 | Upward long-wave radiation, integrated over period specified | integral_wrt_time_of_surface_net_downward_longwave_flux | Negative +|| 014004 | Downward short-wave radiation, integrated over period specified | integral_wrt_time_of_surface_downwelling_shortwave_flux_in_air | Positive +|| 014004 | Upward short-wave radiation, integrated over period specified | integral_wrt_time_of_surface_downwelling_shortwave_flux_in_air | Negative +||| +||| **PRECIPITATION** (Optional) +103000 || Replication descriptor +031001 || Delayed descriptor replication factor +|| 007032 | Height of sensor above local ground +|| 004025 | Time period or displacement +|| 013011 | Total precipitation/total, water equivalent | precipitation_amount From 8ffa8739ab15bbda78b8ede7d713ca3df4bfb4a7 Mon Sep 17 00:00:00 2001 From: istvans Date: Tue, 24 Mar 2026 14:52:43 +0100 Subject: [PATCH 11/13] rodeo-bufr-toos v0.4.8 --- api/dev_requirements.txt | 62 ++++++++++++++++++++----------------- api/requirements.in | 2 +- api/requirements.txt | 57 +++++++++++++++++----------------- ingest/pyproject.toml | 2 +- ingest/requirements-dev.txt | 2 +- ingest/requirements.txt | 10 +++--- 6 files changed, 69 insertions(+), 66 deletions(-) diff --git a/api/dev_requirements.txt b/api/dev_requirements.txt index bbbe96d5..20cededd 100644 --- a/api/dev_requirements.txt +++ b/api/dev_requirements.txt @@ -8,23 +8,23 @@ annotated-types==0.7.0 # via # -r requirements.txt # pydantic -anyio==4.9.0 +anyio==4.12.1 # via # -r requirements.txt # httpx # starlette # watchfiles -brotli==1.1.0 +brotli==1.2.0 # via # -r requirements.txt # brotli-asgi -brotli-asgi==1.4.0 +brotli-asgi==1.6.0 # via -r requirements.txt certifi==2025.4.26 # via # httpcore # httpx -click==8.2.0 +click==8.3.1 # via # -r requirements.txt # uvicorn @@ -36,15 +36,15 @@ deepdiff==7.0.1 # via -r dev_requirements.in edr-pydantic==0.7.0 # via -r requirements.txt -fastapi==0.115.12 +fastapi==0.115.14 # via -r requirements.txt geojson-pydantic==1.2.0 # via -r requirements.txt -grpcio==1.71.0 +grpcio==1.78.0 # via # -r requirements.txt # grpcio-tools -grpcio-tools==1.71.0 +grpcio-tools==1.78.0 # via -r requirements.txt gunicorn==23.0.0 # via -r requirements.txt @@ -55,13 +55,13 @@ h11==0.16.0 # uvicorn httpcore==1.0.9 # via httpx -httptools==0.6.4 +httptools==0.7.1 # via # -r requirements.txt # uvicorn httpx==0.27.2 # via -r dev_requirements.in -idna==3.10 +idna==3.11 # via # -r requirements.txt # anyio @@ -72,45 +72,49 @@ isodate==0.7.2 # via -r requirements.txt jinja2==3.1.6 # via -r requirements.txt -markupsafe==3.0.2 +markupsafe==3.0.3 # via # -r requirements.txt # jinja2 -numpy==2.2.6 +numpy==2.4.3 # via # -r requirements.txt # shapely ordered-set==4.1.0 # via deepdiff -packaging==25.0 +packaging==26.0 # via # -r requirements.txt # gunicorn # pytest pluggy==1.6.0 # via pytest -prometheus-client==0.22.0 +prometheus-client==0.24.1 # via # -r requirements.txt # prometheus-fastapi-instrumentator prometheus-fastapi-instrumentator==7.0.2 # via -r requirements.txt -protobuf==5.29.4 +protobuf==6.33.6 # via # -r requirements.txt # grpcio-tools -pydantic==2.11.4 +pybind11==2.11.2 + # via + # -r requirements.txt + # rodeo-bufr-tools +pydantic==2.12.5 # via # -r requirements.txt # covjson-pydantic # edr-pydantic # fastapi # geojson-pydantic -pydantic-core==2.33.2 +pydantic-core==2.41.5 # via # -r requirements.txt # pydantic -pyparsing==3.2.3 +pyparsing==3.3.2 # via # -r requirements.txt # rdflib @@ -123,53 +127,53 @@ pytest-cov==5.0.0 # via -r dev_requirements.in pytest-timeout==2.4.0 # via -r dev_requirements.in -python-dotenv==1.1.0 +python-dotenv==1.2.2 # via # -r requirements.txt # uvicorn -pyyaml==6.0.2 +pyyaml==6.0.3 # via # -r requirements.txt # uvicorn rdflib==7.1.4 # via -r requirements.txt -shapely==2.1.1 +rodeo-bufr-tools==0.4.8 + # via -r requirements.txt +shapely==2.1.2 # via -r requirements.txt sniffio==1.3.1 - # via - # -r requirements.txt - # anyio - # httpx + # via httpx starlette==0.46.2 # via # -r requirements.txt # brotli-asgi # fastapi # prometheus-fastapi-instrumentator -typing-extensions==4.13.2 +typing-extensions==4.15.0 # via # -r requirements.txt # anyio # edr-pydantic # fastapi + # grpcio # pydantic # pydantic-core # typing-inspection -typing-inspection==0.4.0 +typing-inspection==0.4.2 # via # -r requirements.txt # pydantic uvicorn[standard]==0.32.1 # via -r requirements.txt -uvloop==0.21.0 +uvloop==0.22.1 # via # -r requirements.txt # uvicorn -watchfiles==1.0.5 +watchfiles==1.1.1 # via # -r requirements.txt # uvicorn -websockets==15.0.1 +websockets==16.0 # via # -r requirements.txt # uvicorn diff --git a/api/requirements.in b/api/requirements.in index c44909bf..644cbdb6 100644 --- a/api/requirements.in +++ b/api/requirements.in @@ -16,4 +16,4 @@ jinja2~=3.1 isodate~=0.7.2 prometheus-fastapi-instrumentator~=7.0.0 rdflib~=7.1.4 -rodeo-bufr-tools~=0.4.5 +rodeo-bufr-tools~=0.4.8 diff --git a/api/requirements.txt b/api/requirements.txt index 19a792db..ca4d5131 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -2,102 +2,101 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --no-emit-index-url requirements.in +# pip-compile --no-emit-index-url # annotated-types==0.7.0 # via pydantic -anyio==4.9.0 +anyio==4.12.1 # via # starlette # watchfiles -brotli==1.1.0 +brotli==1.2.0 # via brotli-asgi -brotli-asgi==1.4.0 +brotli-asgi==1.6.0 # via -r requirements.in -click==8.2.0 +click==8.3.1 # via uvicorn covjson-pydantic==0.7.0 # via -r requirements.in edr-pydantic==0.7.0 # via -r requirements.in -fastapi==0.115.12 +fastapi==0.115.14 # via -r requirements.in geojson-pydantic==1.2.0 # via -r requirements.in -grpcio==1.71.0 +grpcio==1.78.0 # via grpcio-tools -grpcio-tools==1.71.0 +grpcio-tools==1.78.0 # via -r requirements.in gunicorn==23.0.0 # via -r requirements.in h11==0.16.0 # via uvicorn -httptools==0.6.4 +httptools==0.7.1 # via uvicorn -idna==3.10 +idna==3.11 # via anyio isodate==0.7.2 # via -r requirements.in jinja2==3.1.6 # via -r requirements.in -markupsafe==3.0.2 +markupsafe==3.0.3 # via jinja2 -numpy==2.2.6 +numpy==2.4.3 # via shapely -packaging==25.0 +packaging==26.0 # via gunicorn -prometheus-client==0.22.0 +prometheus-client==0.24.1 # via prometheus-fastapi-instrumentator prometheus-fastapi-instrumentator==7.0.2 # via -r requirements.in -protobuf==5.29.4 +protobuf==6.33.6 # via grpcio-tools pybind11==2.11.2 # via rodeo-bufr-tools -pydantic==2.11.4 +pydantic==2.12.5 # via # covjson-pydantic # edr-pydantic # fastapi # geojson-pydantic -pydantic-core==2.33.2 +pydantic-core==2.41.5 # via pydantic -pyparsing==3.2.3 +pyparsing==3.3.2 # via rdflib -python-dotenv==1.1.0 +python-dotenv==1.2.2 # via uvicorn -pyyaml==6.0.2 +pyyaml==6.0.3 # via uvicorn rdflib==7.1.4 # via -r requirements.in -rodeo-bufr-tools==0.4.5 +rodeo-bufr-tools==0.4.8 # via -r requirements.in -shapely==2.1.1 +shapely==2.1.2 # via -r requirements.in -sniffio==1.3.1 - # via anyio starlette==0.46.2 # via # brotli-asgi # fastapi # prometheus-fastapi-instrumentator -typing-extensions==4.13.2 +typing-extensions==4.15.0 # via # anyio # edr-pydantic # fastapi + # grpcio # pydantic # pydantic-core # typing-inspection -typing-inspection==0.4.0 +typing-inspection==0.4.2 # via pydantic uvicorn[standard]==0.32.1 # via -r requirements.in -uvloop==0.21.0 +uvloop==0.22.1 # via uvicorn -watchfiles==1.0.5 +watchfiles==1.1.1 # via uvicorn -websockets==15.0.1 +websockets==16.0 # via uvicorn # The following packages are considered to be unsafe in a requirements file: diff --git a/ingest/pyproject.toml b/ingest/pyproject.toml index d99159c4..74514951 100644 --- a/ingest/pyproject.toml +++ b/ingest/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ "grpcio ~= 1.78.0", "grpcio-tools ~= 1.78.0", "geojson-pydantic ~= 2.0.0", - "rodeo-bufr-tools ~= 0.4.5", + "rodeo-bufr-tools ~= 0.4.8", ] name = "esoh-ingest" description = "This project is made for parsing and publishing metadata to the E-SOH project." diff --git a/ingest/requirements-dev.txt b/ingest/requirements-dev.txt index 29918a1f..8194903c 100644 --- a/ingest/requirements-dev.txt +++ b/ingest/requirements-dev.txt @@ -146,7 +146,7 @@ pyyaml==6.0.3 # via pre-commit requests==2.32.5 # via esoh-ingest (pyproject.toml) -rodeo-bufr-tools==0.4.5 +rodeo-bufr-tools==0.4.8 # via esoh-ingest (pyproject.toml) six==1.17.0 # via python-dateutil diff --git a/ingest/requirements.txt b/ingest/requirements.txt index 6731aa75..c837e21f 100644 --- a/ingest/requirements.txt +++ b/ingest/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --no-emit-index-url --output-file=requirements.txt pyproject.toml +# pip-compile pyproject.toml # annotated-doc==0.0.4 # via fastapi @@ -12,11 +12,11 @@ anyio==4.12.1 # via starlette certifi==2026.2.25 # via requests -charset-normalizer==3.4.5 +charset-normalizer==3.4.6 # via requests click==8.3.1 # via uvicorn -fastapi==0.135.1 +fastapi==0.135.2 # via esoh-ingest (pyproject.toml) geojson-pydantic==2.0.0 # via esoh-ingest (pyproject.toml) @@ -46,7 +46,7 @@ prometheus-client==0.24.1 # via prometheus-fastapi-instrumentator prometheus-fastapi-instrumentator==7.0.2 # via esoh-ingest (pyproject.toml) -protobuf==6.33.5 +protobuf==6.33.6 # via # esoh-ingest (pyproject.toml) # grpcio-tools @@ -66,7 +66,7 @@ python-multipart==0.0.22 # via esoh-ingest (pyproject.toml) requests==2.32.5 # via esoh-ingest (pyproject.toml) -rodeo-bufr-tools==0.4.5 +rodeo-bufr-tools==0.4.8 # via esoh-ingest (pyproject.toml) six==1.17.0 # via python-dateutil From a4c5648f6b4b0bc83cdfecce84d970ace4ef506b Mon Sep 17 00:00:00 2001 From: istvans Date: Wed, 25 Mar 2026 08:26:44 +0100 Subject: [PATCH 12/13] EDR main merge fix --- api/routers/edr.py | 6 +++--- datastore/data-loader/client_knmi_station_ingest.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/routers/edr.py b/api/routers/edr.py index 8c2bacba..436ef76b 100644 --- a/api/routers/edr.py +++ b/api/routers/edr.py @@ -438,7 +438,7 @@ async def get_data_area( observations = grpc_response.observations response = formatters.formatters[f]["format_function"](observations) - return response + return Response(content=response, media_type=formatters.formatters[f]["response_format"]) @router.get( @@ -548,6 +548,6 @@ async def get_data_radius( grpc_response = await get_obs_request(request) observations = grpc_response.observations - response = formatters.formatters[f](observations) + response = formatters.formatters[f]["format_function"](observations) - return response + return Response(content=response, media_type=formatters.formatters[f]["response_format"]) diff --git a/datastore/data-loader/client_knmi_station_ingest.py b/datastore/data-loader/client_knmi_station_ingest.py index 02c0e775..90de74a6 100755 --- a/datastore/data-loader/client_knmi_station_ingest.py +++ b/datastore/data-loader/client_knmi_station_ingest.py @@ -119,7 +119,7 @@ def netcdf_file_to_requests(file_path: Path | str) -> Tuple[List, List]: def send_request_to_ingest(msg, url): try: - response = requests.post(url, json=msg) + response = requests.post(url, data=json.dumps(msg)) response.raise_for_status() return response.status_code, response.json() except requests.RequestException as e: From f9f20aeaad6f29892402d24b17c3b399abe288d7 Mon Sep 17 00:00:00 2001 From: istvans Date: Wed, 25 Mar 2026 08:29:47 +0100 Subject: [PATCH 13/13] missing import --- datastore/data-loader/client_knmi_station_ingest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datastore/data-loader/client_knmi_station_ingest.py b/datastore/data-loader/client_knmi_station_ingest.py index 90de74a6..f2ee89ef 100755 --- a/datastore/data-loader/client_knmi_station_ingest.py +++ b/datastore/data-loader/client_knmi_station_ingest.py @@ -3,6 +3,7 @@ import math import os import requests +import json from multiprocessing import cpu_count, Pool from pathlib import Path from time import perf_counter