diff --git a/api/BUFR_out.md b/api/BUFR_out.md new file mode 100644 index 00000000..820c6098 --- /dev/null +++ b/api/BUFR_out.md @@ -0,0 +1,62 @@ +# E-SOH BUFR Output Format Sequence + +E-SOH provides a BUFR output file when you select the BUFR format from the dropdown menu. The following table describes the BUFR sequence. + +| Descriptor | Extended/Repeated | Name | Data Source | Note | +|------------|-------------------|------|-------------|------| +||| +||| **STATION IDENTIFICATION** +301150 || WIGOS identifier | metocean:wigosId | Mandatory +|| 0 01 125 | WIGOS identifier series +|| 0 01 126 | WIGOS issuer of identifier +|| 0 01 127 | WIGOS issue number +|| 0 01 128 | WIGOS local identifier (character) +301090 || Surface station identification || Mandatory +|| 3 01 004 | Surface station identification +|| 3 01 011 | Year, month, day +|| 3 01 012 | Hour, minute +|| 3 01 021 | Latitude/longitude (high accuracy) +|| 0 07 030 | Height of station ground above mean sea level +|| 0 07 031 | Height of barometer above mean sea level +||| **BASIC SURFACE OBSERVATIONS** +302031 || Pressure information | air_pressure, air_pressure_at_mean_sea_level | Optional +302032 || Temperature and humidity data | air_temperature, dew_point_temperature, relative_humidity | Optional +302033 || Visibility data || Optional +||| +||| **EXTREME TEMPERATURES** (Optional) +105000 || Replication descriptor +031001 || Delayed descriptor replication factor +|| 007032 | Height of sensor above local ground +|| 004025 | Time period or displacement +|| 012111 | Maximum temperature, at height and over period specified | air_temperature +|| 004025 | Time period or displacement +|| 012112 | Minimum temperature, at height and over period specified | air_temperature +||| +||| **WIND DATA** (Optional) +302042 ||| wind_speed, wind_from_direction, wind_speed_of_gust, wind_gust_from_direction +||| +||| **RADIATION** (Optional) +106000 || Replication descriptor +031101 || Delayed descriptor replication factor +|| 007032 | Height of sensor above local ground +|| 004025 | Time period or displacement +|| 014002 | Long-wave radiation, integrated over period specified | integral_wrt_time_of_surface_downwelling_longwave_flux_in_air +|| 014004 | Short-wave radiation, integrated over period specified | integral_wrt_time_of_surface_downwelling_shortwave_flux_in_air +|| 014012 | Net long-wave radiation, integrated over period specified | integral_wrt_time_of_surface_net_downward_longwave_flux +|| 014014 | Net short-wave radiation, integrated over period specified | integral_wrt_time_of_surface_net_downward_shortwave_flux +||| +||| **RADIATION** (Optional) +106000 || Replication descriptor +031101 || Delayed descriptor replication factor +|| 007032 | Height of sensor above local ground +|| 014002 | Downward long-wave radiation, integrated over period specified | integral_wrt_time_of_surface_net_downward_longwave_flux | Positive +|| 014002 | Upward long-wave radiation, integrated over period specified | integral_wrt_time_of_surface_net_downward_longwave_flux | Negative +|| 014004 | Downward short-wave radiation, integrated over period specified | integral_wrt_time_of_surface_downwelling_shortwave_flux_in_air | Positive +|| 014004 | Upward short-wave radiation, integrated over period specified | integral_wrt_time_of_surface_downwelling_shortwave_flux_in_air | Negative +||| +||| **PRECIPITATION** (Optional) +103000 || Replication descriptor +031001 || Delayed descriptor replication factor +|| 007032 | Height of sensor above local ground +|| 004025 | Time period or displacement +|| 013011 | Total precipitation/total, water equivalent | precipitation_amount diff --git a/api/Dockerfile b/api/Dockerfile index 3b644dd0..284cf98f 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -5,26 +5,26 @@ SHELL ["/bin/bash", "-eux", "-o", "pipefail", "-c"] ENV DOCKER_PATH="/app" RUN apt-get update \ - && apt-get -y upgrade \ - && apt-get install -y --no-install-recommends git curl \ - # Cleanup - && rm -rf /usr/tmp \ - && apt-get autoremove -y \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + && apt-get -y upgrade \ + && apt-get install -y --no-install-recommends git curl libeccodes-data \ + # Cleanup + && rm -rf /usr/tmp \ + && apt-get autoremove -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* COPY "./protobuf/datastore.proto" "/protobuf/datastore.proto" COPY "./requirements.txt" "${DOCKER_PATH}/requirements.txt" # hadolint ignore=DL3013 RUN pip install --no-cache-dir --upgrade pip \ - && pip install --no-cache-dir --upgrade -r "${DOCKER_PATH}/requirements.txt" + && pip install --no-cache-dir --upgrade -r "${DOCKER_PATH}/requirements.txt" # Compiling the protobuf file RUN python -m grpc_tools.protoc \ - --proto_path="protobuf" "protobuf/datastore.proto" \ - --python_out="${DOCKER_PATH}" \ - --grpc_python_out="${DOCKER_PATH}" + --proto_path="protobuf" "protobuf/datastore.proto" \ + --python_out="${DOCKER_PATH}" \ + --grpc_python_out="${DOCKER_PATH}" COPY "." "${DOCKER_PATH}/" diff --git a/api/dev_requirements.txt b/api/dev_requirements.txt index bbbe96d5..20cededd 100644 --- a/api/dev_requirements.txt +++ b/api/dev_requirements.txt @@ -8,23 +8,23 @@ annotated-types==0.7.0 # via # -r requirements.txt # pydantic -anyio==4.9.0 +anyio==4.12.1 # via # -r requirements.txt # httpx # starlette # watchfiles -brotli==1.1.0 +brotli==1.2.0 # via # -r requirements.txt # brotli-asgi -brotli-asgi==1.4.0 +brotli-asgi==1.6.0 # via -r requirements.txt certifi==2025.4.26 # via # httpcore # httpx -click==8.2.0 +click==8.3.1 # via # -r requirements.txt # uvicorn @@ -36,15 +36,15 @@ deepdiff==7.0.1 # via -r dev_requirements.in edr-pydantic==0.7.0 # via -r requirements.txt -fastapi==0.115.12 +fastapi==0.115.14 # via -r requirements.txt geojson-pydantic==1.2.0 # via -r requirements.txt -grpcio==1.71.0 +grpcio==1.78.0 # via # -r requirements.txt # grpcio-tools -grpcio-tools==1.71.0 +grpcio-tools==1.78.0 # via -r requirements.txt gunicorn==23.0.0 # via -r requirements.txt @@ -55,13 +55,13 @@ h11==0.16.0 # uvicorn httpcore==1.0.9 # via httpx -httptools==0.6.4 +httptools==0.7.1 # via # -r requirements.txt # uvicorn httpx==0.27.2 # via -r dev_requirements.in -idna==3.10 +idna==3.11 # via # -r requirements.txt # anyio @@ -72,45 +72,49 @@ isodate==0.7.2 # via -r requirements.txt jinja2==3.1.6 # via -r requirements.txt -markupsafe==3.0.2 +markupsafe==3.0.3 # via # -r requirements.txt # jinja2 -numpy==2.2.6 +numpy==2.4.3 # via # -r requirements.txt # shapely ordered-set==4.1.0 # via deepdiff -packaging==25.0 +packaging==26.0 # via # -r requirements.txt # gunicorn # pytest pluggy==1.6.0 # via pytest -prometheus-client==0.22.0 +prometheus-client==0.24.1 # via # -r requirements.txt # prometheus-fastapi-instrumentator prometheus-fastapi-instrumentator==7.0.2 # via -r requirements.txt -protobuf==5.29.4 +protobuf==6.33.6 # via # -r requirements.txt # grpcio-tools -pydantic==2.11.4 +pybind11==2.11.2 + # via + # -r requirements.txt + # rodeo-bufr-tools +pydantic==2.12.5 # via # -r requirements.txt # covjson-pydantic # edr-pydantic # fastapi # geojson-pydantic -pydantic-core==2.33.2 +pydantic-core==2.41.5 # via # -r requirements.txt # pydantic -pyparsing==3.2.3 +pyparsing==3.3.2 # via # -r requirements.txt # rdflib @@ -123,53 +127,53 @@ pytest-cov==5.0.0 # via -r dev_requirements.in pytest-timeout==2.4.0 # via -r dev_requirements.in -python-dotenv==1.1.0 +python-dotenv==1.2.2 # via # -r requirements.txt # uvicorn -pyyaml==6.0.2 +pyyaml==6.0.3 # via # -r requirements.txt # uvicorn rdflib==7.1.4 # via -r requirements.txt -shapely==2.1.1 +rodeo-bufr-tools==0.4.8 + # via -r requirements.txt +shapely==2.1.2 # via -r requirements.txt sniffio==1.3.1 - # via - # -r requirements.txt - # anyio - # httpx + # via httpx starlette==0.46.2 # via # -r requirements.txt # brotli-asgi # fastapi # prometheus-fastapi-instrumentator -typing-extensions==4.13.2 +typing-extensions==4.15.0 # via # -r requirements.txt # anyio # edr-pydantic # fastapi + # grpcio # pydantic # pydantic-core # typing-inspection -typing-inspection==0.4.0 +typing-inspection==0.4.2 # via # -r requirements.txt # pydantic uvicorn[standard]==0.32.1 # via -r requirements.txt -uvloop==0.21.0 +uvloop==0.22.1 # via # -r requirements.txt # uvicorn -watchfiles==1.0.5 +watchfiles==1.1.1 # via # -r requirements.txt # uvicorn -websockets==15.0.1 +websockets==16.0 # via # -r requirements.txt # uvicorn diff --git a/api/formatters/__init__.py b/api/formatters/__init__.py index 14768c23..3aa6d2bd 100644 --- a/api/formatters/__init__.py +++ b/api/formatters/__init__.py @@ -3,12 +3,14 @@ from . import covjson from . import geojson +from . import bufr logger = logging.getLogger(__name__) class Formats(str, Enum): covjson = "CoverageJSON" # According to EDR spec + bufr = "bufr" class Metadata_Formats(str, Enum): @@ -16,6 +18,13 @@ class Metadata_Formats(str, Enum): formatters = { - "CoverageJSON": covjson.convert_to_covjson, + "CoverageJSON": { + "format_function": covjson.convert_to_covjson, + "response_format": "application/prs.coverage+json", + }, + "bufr": { + "format_function": bufr.convert_to_bufr, + "response_format": "application/bufr", + }, } # observations metadata_formatters = {"GeoJSON": geojson.convert_to_geojson} # metadata diff --git a/api/formatters/bufr.py b/api/formatters/bufr.py new file mode 100644 index 00000000..781a0b5a --- /dev/null +++ b/api/formatters/bufr.py @@ -0,0 +1,13 @@ +from . import covjson + +from bufr_tools import covjson2bufr + + +def convert_to_bufr(raw_data: str): + cov_json = covjson.convert_to_covjson(raw_data) + print(type(cov_json)) + bufr_content = covjson2bufr.covjson2bufr(cov_json) + + if not bufr_content and not cov_json: + raise ValueError("No content") + return bufr_content diff --git a/api/formatters/covjson.py b/api/formatters/covjson.py index 2456a472..6e3a29e9 100644 --- a/api/formatters/covjson.py +++ b/api/formatters/covjson.py @@ -77,7 +77,10 @@ def convert_to_covjson(observations): referencing = [ ReferenceSystemConnectionObject( coordinates=["x", "y"], - system=ReferenceSystem(type="GeographicCRS", id="http://www.opengis.net/def/crs/OGC/1.3/CRS84"), + system=ReferenceSystem( + type="GeographicCRS", + id="http://www.opengis.net/def/crs/OGC/1.3/CRS84", + ), ), ReferenceSystemConnectionObject( coordinates=["t"], @@ -106,7 +109,9 @@ def convert_to_covjson(observations): parameters[parameter_id] = make_parameter(data.ts_mdata) ranges[parameter_id] = NdArrayFloat( - values=values_no_nan, axisNames=["t", "x", "y"], shape=[len(values_no_nan), 1, 1] + values=values_no_nan, + axisNames=["t", "x", "y"], + shape=[len(values_no_nan), 1, 1], ) custom_fields = {"metocean:wigosId": data.ts_mdata.platform} @@ -115,10 +120,14 @@ def convert_to_covjson(observations): if len(coverages) == 0: raise HTTPException(status_code=404, detail="Requested data not found.") elif len(coverages) == 1: - return coverages[0] + return coverages[0].model_dump_json(exclude_none=True) else: parameter_union = reduce(operator.ior, (c.parameters.root for c in coverages), {}) - return CoverageCollection(coverages=coverages, parameters=dict(sorted(parameter_union.items()))) + return CoverageCollection( + coverages=coverages, parameters=dict(sorted(parameter_union.items())) + ).model_dump_json( + exclude_none=True + ) # (mode="json") def _collect_data(ts_mdata, obs_mdata): diff --git a/api/requirements.in b/api/requirements.in index a100962f..644cbdb6 100644 --- a/api/requirements.in +++ b/api/requirements.in @@ -16,3 +16,4 @@ jinja2~=3.1 isodate~=0.7.2 prometheus-fastapi-instrumentator~=7.0.0 rdflib~=7.1.4 +rodeo-bufr-tools~=0.4.8 diff --git a/api/requirements.txt b/api/requirements.txt index ad778ee3..ca4d5131 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -2,98 +2,101 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --no-emit-index-url requirements.in +# pip-compile --no-emit-index-url # annotated-types==0.7.0 # via pydantic -anyio==4.9.0 +anyio==4.12.1 # via # starlette # watchfiles -brotli==1.1.0 +brotli==1.2.0 # via brotli-asgi -brotli-asgi==1.4.0 +brotli-asgi==1.6.0 # via -r requirements.in -click==8.2.0 +click==8.3.1 # via uvicorn covjson-pydantic==0.7.0 # via -r requirements.in edr-pydantic==0.7.0 # via -r requirements.in -fastapi==0.115.12 +fastapi==0.115.14 # via -r requirements.in geojson-pydantic==1.2.0 # via -r requirements.in -grpcio==1.71.0 +grpcio==1.78.0 # via grpcio-tools -grpcio-tools==1.71.0 +grpcio-tools==1.78.0 # via -r requirements.in gunicorn==23.0.0 # via -r requirements.in h11==0.16.0 # via uvicorn -httptools==0.6.4 +httptools==0.7.1 # via uvicorn -idna==3.10 +idna==3.11 # via anyio isodate==0.7.2 # via -r requirements.in jinja2==3.1.6 # via -r requirements.in -markupsafe==3.0.2 +markupsafe==3.0.3 # via jinja2 -numpy==2.2.6 +numpy==2.4.3 # via shapely -packaging==25.0 +packaging==26.0 # via gunicorn -prometheus-client==0.22.0 +prometheus-client==0.24.1 # via prometheus-fastapi-instrumentator prometheus-fastapi-instrumentator==7.0.2 # via -r requirements.in -protobuf==5.29.4 +protobuf==6.33.6 # via grpcio-tools -pydantic==2.11.4 +pybind11==2.11.2 + # via rodeo-bufr-tools +pydantic==2.12.5 # via # covjson-pydantic # edr-pydantic # fastapi # geojson-pydantic -pydantic-core==2.33.2 +pydantic-core==2.41.5 # via pydantic -pyparsing==3.2.3 +pyparsing==3.3.2 # via rdflib -python-dotenv==1.1.0 +python-dotenv==1.2.2 # via uvicorn -pyyaml==6.0.2 +pyyaml==6.0.3 # via uvicorn rdflib==7.1.4 # via -r requirements.in -shapely==2.1.1 +rodeo-bufr-tools==0.4.8 + # via -r requirements.in +shapely==2.1.2 # via -r requirements.in -sniffio==1.3.1 - # via anyio starlette==0.46.2 # via # brotli-asgi # fastapi # prometheus-fastapi-instrumentator -typing-extensions==4.13.2 +typing-extensions==4.15.0 # via # anyio # edr-pydantic # fastapi + # grpcio # pydantic # pydantic-core # typing-inspection -typing-inspection==0.4.0 +typing-inspection==0.4.2 # via pydantic uvicorn[standard]==0.32.1 # via -r requirements.in -uvloop==0.21.0 +uvloop==0.22.1 # via uvicorn -watchfiles==1.0.5 +watchfiles==1.1.1 # via uvicorn -websockets==15.0.1 +websockets==16.0 # via uvicorn # The following packages are considered to be unsafe in a requirements file: diff --git a/api/routers/edr.py b/api/routers/edr.py index 164e63d2..436ef76b 100644 --- a/api/routers/edr.py +++ b/api/routers/edr.py @@ -19,6 +19,7 @@ from fastapi import Path from fastapi import Query from fastapi import Request +from fastapi.responses import Response from formatters.covjson import make_parameter from geojson_pydantic import Feature from geojson_pydantic import Point @@ -240,9 +241,9 @@ async def get_data_location_id( grpc_response = await get_obs_request(request) observations = grpc_response.observations - response = formatters.formatters[f](observations) + response = formatters.formatters[f]["format_function"](observations) - return response + return Response(content=response, media_type=formatters.formatters[f]["response_format"]) @router.get( @@ -337,9 +338,9 @@ async def get_data_position( grpc_response = await get_obs_request(request) observations = grpc_response.observations - response = formatters.formatters[f](observations) + response = formatters.formatters[f]["format_function"](observations) - return response + return Response(content=response, media_type=formatters.formatters[f]["response_format"]) @router.get( @@ -435,9 +436,9 @@ async def get_data_area( grpc_response = await get_obs_request(request) observations = grpc_response.observations - response = formatters.formatters[f](observations) + response = formatters.formatters[f]["format_function"](observations) - return response + return Response(content=response, media_type=formatters.formatters[f]["response_format"]) @router.get( @@ -547,6 +548,6 @@ async def get_data_radius( grpc_response = await get_obs_request(request) observations = grpc_response.observations - response = formatters.formatters[f](observations) + response = formatters.formatters[f]["format_function"](observations) - return response + return Response(content=response, media_type=formatters.formatters[f]["response_format"]) diff --git a/datastore/data-loader/client_knmi_station_ingest.py b/datastore/data-loader/client_knmi_station_ingest.py index 02c0e775..f2ee89ef 100755 --- a/datastore/data-loader/client_knmi_station_ingest.py +++ b/datastore/data-loader/client_knmi_station_ingest.py @@ -3,6 +3,7 @@ import math import os import requests +import json from multiprocessing import cpu_count, Pool from pathlib import Path from time import perf_counter @@ -119,7 +120,7 @@ def netcdf_file_to_requests(file_path: Path | str) -> Tuple[List, List]: def send_request_to_ingest(msg, url): try: - response = requests.post(url, json=msg) + response = requests.post(url, data=json.dumps(msg)) response.raise_for_status() return response.status_code, response.json() except requests.RequestException as e: diff --git a/ingest/pyproject.toml b/ingest/pyproject.toml index d99159c4..74514951 100644 --- a/ingest/pyproject.toml +++ b/ingest/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ "grpcio ~= 1.78.0", "grpcio-tools ~= 1.78.0", "geojson-pydantic ~= 2.0.0", - "rodeo-bufr-tools ~= 0.4.5", + "rodeo-bufr-tools ~= 0.4.8", ] name = "esoh-ingest" description = "This project is made for parsing and publishing metadata to the E-SOH project." diff --git a/ingest/requirements-dev.txt b/ingest/requirements-dev.txt index 29918a1f..8194903c 100644 --- a/ingest/requirements-dev.txt +++ b/ingest/requirements-dev.txt @@ -146,7 +146,7 @@ pyyaml==6.0.3 # via pre-commit requests==2.32.5 # via esoh-ingest (pyproject.toml) -rodeo-bufr-tools==0.4.5 +rodeo-bufr-tools==0.4.8 # via esoh-ingest (pyproject.toml) six==1.17.0 # via python-dateutil diff --git a/ingest/requirements.txt b/ingest/requirements.txt index 6731aa75..c837e21f 100644 --- a/ingest/requirements.txt +++ b/ingest/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --no-emit-index-url --output-file=requirements.txt pyproject.toml +# pip-compile pyproject.toml # annotated-doc==0.0.4 # via fastapi @@ -12,11 +12,11 @@ anyio==4.12.1 # via starlette certifi==2026.2.25 # via requests -charset-normalizer==3.4.5 +charset-normalizer==3.4.6 # via requests click==8.3.1 # via uvicorn -fastapi==0.135.1 +fastapi==0.135.2 # via esoh-ingest (pyproject.toml) geojson-pydantic==2.0.0 # via esoh-ingest (pyproject.toml) @@ -46,7 +46,7 @@ prometheus-client==0.24.1 # via prometheus-fastapi-instrumentator prometheus-fastapi-instrumentator==7.0.2 # via esoh-ingest (pyproject.toml) -protobuf==6.33.5 +protobuf==6.33.6 # via # esoh-ingest (pyproject.toml) # grpcio-tools @@ -66,7 +66,7 @@ python-multipart==0.0.22 # via esoh-ingest (pyproject.toml) requests==2.32.5 # via esoh-ingest (pyproject.toml) -rodeo-bufr-tools==0.4.5 +rodeo-bufr-tools==0.4.8 # via esoh-ingest (pyproject.toml) six==1.17.0 # via python-dateutil