Use new (misc) keys in MTG2 actions

src/multio/action/average-rate/AverageRate.h

@@ -36,16 +36,24 @@ struct AverageRateKeys {
     dm::EntryType_t<decltype(dm::PARAM)> param;
     dm::EntryType_t<decltype(dm::TIMESPAN)> timespan;
     dm::EntryType_t<decltype(dm::STATTYPE)> stattype;
+    dm::EntryType_t<decltype(dm::BitmapPresent)> bitmapPresent;
     dm::EntryType_t<decltype(dm::MissingValue)> missingValue;
 
     static constexpr std::string_view record_name_ = "average-rate";
     static constexpr auto record_entries_ = std::make_tuple(
-        dm::PARAM,
-        dm::TIMESPAN.tagRequired(),
-        dm::STATTYPE,
-        dm::MissingValue
+        dm::PARAM,                   // access: read/write
+        dm::TIMESPAN.tagRequired(),  // access: read only
+        dm::STATTYPE,                // access: read only (must be unset)
+        dm::BitmapPresent,           // access: read only
+        dm::MissingValue             // access: read/write (will be unset if bitmapPresent false)
     );
 
+    static void applyDefaults(AverageRateKeys& k) {
+        if (!k.bitmapPresent.get()) {
+            k.missingValue.unset();
+        }
+    }
+
     static void validate(const AverageRateKeys& k) {
         if (k.timespan.get().toSeconds() == 0) {
             throw eckit::SeriousBug(
@@ -60,6 +68,13 @@ struct AverageRateKeys {
                 Here()
             );
         }
+
+        if (k.bitmapPresent.get() && !k.missingValue.isSet()) {
+            throw eckit::SeriousBug(
+                "Value for missingValue is required if bitmapPresent is true!",
+                Here()
+            );
+        }
     }
 };
 

src/multio/action/encode-mtg2/EncodeMtg2.cc

@@ -81,18 +81,16 @@ void EncodeMtg2::executeImpl(Message msg) {
 
     auto& md = msg.metadata();
 
-    // Read and set unscoped mars keys
+    // Read mars and misc keys from the message metadata
     auto marsRec = dm::readRecord<dm::FullMarsRecord>(md);
-
-    // Read scoped misc keys
-    auto scopedMiscRec = dm::scopeRecord(dm::MiscRecord{});
-    dm::readRecord(scopedMiscRec, md);
-    // Write unscoped misc keys
-    auto miscRec = dm::unscopeRecord(std::move(scopedMiscRec));
+    auto miscRec = dm::readRecord<dm::MiscRecord>(md);
 
     // Apply mappings
     auto mappingResult = mars2mars::applyMappings(mars2mars::allRules(), marsRec, miscRec);
 
+    // Dump (mapped) mars and misc keys to local configurations
+    const auto mars = dm::dumpRecord<eckit::LocalConfiguration>(marsRec);
+    const auto misc = dm::dumpUnscopedRecord<eckit::LocalConfiguration>(miscRec);
 
     executeNext(dispatchPrecisionTag(msg.precision(), [&](auto pt) {
         using Precision = typename decltype(pt)::type;

src/multio/action/scale/Scale.h

@@ -17,13 +17,30 @@ namespace cf = multio::util::config;
 
 struct ScaleMetadataKeys {
     dm::EntryType_t<decltype(dm::PARAM)> param;
+    dm::EntryType_t<decltype(dm::BitmapPresent)> bitmapPresent;
     dm::EntryType_t<decltype(dm::MissingValue)> missingValue;
 
     static constexpr std::string_view record_name_ = "scale-action-metadata";
     static constexpr auto record_entries_ = std::make_tuple(
-        dm::PARAM,        // access: read/write
-        dm::MissingValue  // access: read only
+        dm::PARAM,          // access: read/write
+        dm::BitmapPresent,  // access: read only
+        dm::MissingValue    // access: read/write (will be unset if bitmapPresent false)
     );
+
+    static void applyDefaults(ScaleMetadataKeys& k) {
+        if (!k.bitmapPresent.get()) {
+            k.missingValue.unset();
+        }
+    }
+
+    static void validate(const ScaleMetadataKeys& k) {
+        if (k.bitmapPresent.get() && !k.missingValue.isSet()) {
+            throw eckit::SeriousBug(
+                "Value for missingValue is required if bitmapPresent is true!",
+                Here()
+            );
+        }
+    }
 };
 
 //------------------------ Action Configuration Keys ------------------------//

src/multio/action/statistics-mtg2/CMakeLists.txt

@@ -39,6 +39,8 @@ list( APPEND _statistics_sources
     TemporalStatistics.h
     Statistics.cc
     Statistics.h
+    MetadataKeys.cc
+    MetadataKeys.h
     # SynopticCollection.cc
     # SynopticCollection.h
     # SynopticFilters.cc

src/multio/action/statistics-mtg2/MetadataKeys.cc

@@ -0,0 +1,62 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation
+ * nor does it submit to any jurisdiction.
+ */
+
+#include "MetadataKeys.h"
+#include <unordered_map>
+#include "multio/datamod/core/DataModellingException.h"
+
+using FlushKind = multio::action::statistics_mtg2::FlushKind;
+
+std::string multio::datamod::DumpType<FlushKind>::dump(FlushKind v) {
+    switch (v) {
+        case FlushKind::Default:
+            return "default";
+        case FlushKind::FirstStep:
+            return "first-step";
+        case FlushKind::StepAndRestart:
+            return "step-and-restart";
+        case FlushKind::LastStep:
+            return "last-step";
+        case FlushKind::EndOfSimulation:
+            return "end-of-simulation";
+        case FlushKind::CloseConnection:
+            return "close-connection";
+        default:
+            throw multio::datamod::DataModellingException(
+                "DumpType<FlushKind>::dump: Unexpected enum value " + std::to_string(static_cast<std::size_t>(v)),
+                Here());
+    }
+}
+
+FlushKind multio::datamod::ParseType<FlushKind>::parse(const std::string& s) {
+    static const std::unordered_map<std::string, FlushKind> map{
+        {"default", FlushKind::Default},
+        {"first-step", FlushKind::FirstStep},
+        {"step-and-restart", FlushKind::StepAndRestart},
+        {"last-step", FlushKind::LastStep},
+        {"end-of-simulation", FlushKind::EndOfSimulation},
+        {"close-connection", FlushKind::CloseConnection}};
+    if (auto it = map.find(s); it != map.end()) {
+        return it->second;
+    }
+    throw multio::datamod::DataModellingException("ParseType<FlushKind>::parse: Unknown FlushKind string: " + s,
+                                                  Here());
+}
+
+FlushKind multio::datamod::ParseType<FlushKind>::parse(std::int64_t val) {
+    static const std::unordered_map<std::int64_t, FlushKind> map{
+        {0, FlushKind::FirstStep},     {1, FlushKind::Default},          {2, FlushKind::StepAndRestart},
+        {3, FlushKind::LastStep},      {4, FlushKind::EndOfSimulation},  {5, FlushKind::CloseConnection}};
+    if (auto it = map.find(val); it != map.end()) {
+        return it->second;
+    }
+    throw multio::datamod::DataModellingException(
+        "ParseType<FlushKind>::parse: Unknown FlushKind integer: " + std::to_string(val), Here());
+}

src/multio/action/statistics-mtg2/MetadataKeys.h

@@ -0,0 +1,118 @@
+#pragma once
+
+#include "multio/datamod/core/EntryDef.h"
+#include "multio/datamod/GribKeys.h"
+#include "multio/datamod/MarsMiscGeo.h"
+#include "multio/datamod/MarsKeys.h"
+
+namespace multio::action::statistics_mtg2 {
+
+//----------------------------- FlushKind Enum ------------------------------//
+
+enum class FlushKind : std::size_t
+{
+    Default,
+    FirstStep,
+    StepAndRestart,
+    LastStep,
+    EndOfSimulation,
+    CloseConnection
+};
+
+}  // namespace multio::action::statistics_mtg2
+
+//-------------- FlushKind type parsing/dumping specializations --------------//
+
+template <>
+struct multio::datamod::DumpType<multio::action::statistics_mtg2::FlushKind> {
+    static std::string dump(multio::action::statistics_mtg2::FlushKind v);
+};
+
+template <>
+struct multio::datamod::ParseType<multio::action::statistics_mtg2::FlushKind> {
+    static multio::action::statistics_mtg2::FlushKind parse(const std::string& s);
+    static multio::action::statistics_mtg2::FlushKind parse(std::int64_t val);
+};
+
+//--------------------------------- Records ---------------------------------//
+
+namespace multio::action::statistics_mtg2 {
+
+namespace dm = multio::datamod;
+
+//-------------------------- Flush-local EntryDefs --------------------------//
+
+constexpr auto FLUSH_KIND =
+    dm::EntryDef<FlushKind>{"flushKind"}
+        .withDefault(FlushKind::Default)
+        .withAccessor([](auto&& v) { return &v.flushKind; });
+
+constexpr auto RESTART_DATE_TIME =
+    dm::EntryDef<std::string>{"restartDateTime"}
+        .tagOptional()
+        .withAccessor([](auto&& v) { return &v.restartDateTime; });
+
+constexpr auto SERVER_RANK =
+    dm::EntryDef<std::int64_t>{"serverRank"}
+        .tagOptional()
+        .withAccessor([](auto&& v) { return &v.serverRank; });
+
+//----------------------- Field Metadata Keys Record ------------------------//
+
+struct FieldMetadataKeys {
+    dm::EntryType_t<decltype(dm::DATE)> date;
+    dm::EntryType_t<decltype(dm::TIME)> time;
+    dm::EntryType_t<decltype(dm::STEP)> step;
+    dm::EntryType_t<decltype(dm::TIMESPAN)> timespan;
+    dm::EntryType_t<decltype(dm::PARAM)> param;
+    dm::EntryType_t<decltype(dm::STREAM)> stream;
+    dm::EntryType_t<decltype(dm::LEVTYPE)> levtype;
+    dm::EntryType_t<decltype(dm::LEVELIST)> levelist;
+    dm::EntryType_t<decltype(dm::GRID)> grid;
+    dm::EntryType_t<decltype(dm::TRUNCATION)> truncation;
+    dm::EntryType_t<decltype(dm::TimeIncrementInSeconds)> timeIncrementInSeconds;
+    dm::EntryType_t<decltype(dm::BitmapPresent)> bitmapPresent;
+    dm::EntryType_t<decltype(dm::MissingValue)> missingValue;
+
+    static constexpr std::string_view record_name_ = "statistics-mtg2-field";
+    static constexpr auto record_entries_ = std::make_tuple(
+        dm::DATE,
+        dm::TIME,
+        dm::STEP.tagRequired(),
+        dm::TIMESPAN,
+        dm::PARAM,
+        dm::STREAM.tagOptional(),
+        dm::LEVTYPE.tagRequired(),
+        dm::LEVELIST,
+        dm::GRID,
+        dm::TRUNCATION,
+        dm::TimeIncrementInSeconds,
+        dm::BitmapPresent,
+        dm::MissingValue
+    );
+};
+
+//------------------------ Flush Metadata Keys Record -----------------------//
+
+struct FlushMetadataKeys {
+    dm::EntryType_t<decltype(dm::DATE)> date;
+    dm::EntryType_t<decltype(dm::TIME)> time;
+    dm::EntryType_t<decltype(dm::STEP)> step;
+    dm::EntryType_t<decltype(FLUSH_KIND)> flushKind;
+    dm::EntryType_t<decltype(RESTART_DATE_TIME)> restartDateTime;
+    dm::EntryType_t<decltype(SERVER_RANK)> serverRank;
+
+    static constexpr std::string_view record_name_ = "statistics-mtg2-flush";
+    static constexpr auto record_entries_ = std::make_tuple(
+        dm::DATE.tagOptional(),
+        dm::TIME.tagOptional(),
+        dm::STEP,
+        FLUSH_KIND,
+        RESTART_DATE_TIME,
+        SERVER_RANK
+    );
+};
+
+//---------------------------------------------------------------------------//
+
+}  // namespace multio::action::statistics_mtg2