diff --git a/src/multio/action/average-rate/AverageRate.h b/src/multio/action/average-rate/AverageRate.h index 69e0ec1cf..5b1c91ba8 100644 --- a/src/multio/action/average-rate/AverageRate.h +++ b/src/multio/action/average-rate/AverageRate.h @@ -36,16 +36,24 @@ struct AverageRateKeys { dm::EntryType_t param; dm::EntryType_t timespan; dm::EntryType_t stattype; + dm::EntryType_t bitmapPresent; dm::EntryType_t missingValue; static constexpr std::string_view record_name_ = "average-rate"; static constexpr auto record_entries_ = std::make_tuple( - dm::PARAM, - dm::TIMESPAN.tagRequired(), - dm::STATTYPE, - dm::MissingValue + dm::PARAM, // access: read/write + dm::TIMESPAN.tagRequired(), // access: read only + dm::STATTYPE, // access: read only (must be unset) + dm::BitmapPresent, // access: read only + dm::MissingValue // access: read/write (will be unset if bitmapPresent false) ); + static void applyDefaults(AverageRateKeys& k) { + if (!k.bitmapPresent.get()) { + k.missingValue.unset(); + } + } + static void validate(const AverageRateKeys& k) { if (k.timespan.get().toSeconds() == 0) { throw eckit::SeriousBug( @@ -60,6 +68,13 @@ struct AverageRateKeys { Here() ); } + + if (k.bitmapPresent.get() && !k.missingValue.isSet()) { + throw eckit::SeriousBug( + "Value for missingValue is required if bitmapPresent is true!", + Here() + ); + } } }; diff --git a/src/multio/action/encode-mtg2/EncodeMtg2.cc b/src/multio/action/encode-mtg2/EncodeMtg2.cc index 9cd6b9e06..a15d3a56f 100644 --- a/src/multio/action/encode-mtg2/EncodeMtg2.cc +++ b/src/multio/action/encode-mtg2/EncodeMtg2.cc @@ -81,18 +81,16 @@ void EncodeMtg2::executeImpl(Message msg) { auto& md = msg.metadata(); - // Read and set unscoped mars keys + // Read mars and misc keys from the message metadata auto marsRec = dm::readRecord(md); - - // Read scoped misc keys - auto scopedMiscRec = dm::scopeRecord(dm::MiscRecord{}); - dm::readRecord(scopedMiscRec, md); - // Write unscoped misc keys - auto miscRec = dm::unscopeRecord(std::move(scopedMiscRec)); + auto miscRec = dm::readRecord(md); // Apply mappings auto mappingResult = mars2mars::applyMappings(mars2mars::allRules(), marsRec, miscRec); + // Dump (mapped) mars and misc keys to local configurations + const auto mars = dm::dumpRecord(marsRec); + const auto misc = dm::dumpUnscopedRecord(miscRec); executeNext(dispatchPrecisionTag(msg.precision(), [&](auto pt) { using Precision = typename decltype(pt)::type; diff --git a/src/multio/action/scale/Scale.h b/src/multio/action/scale/Scale.h index 7eee7fcf7..661537666 100644 --- a/src/multio/action/scale/Scale.h +++ b/src/multio/action/scale/Scale.h @@ -17,13 +17,30 @@ namespace cf = multio::util::config; struct ScaleMetadataKeys { dm::EntryType_t param; + dm::EntryType_t bitmapPresent; dm::EntryType_t missingValue; static constexpr std::string_view record_name_ = "scale-action-metadata"; static constexpr auto record_entries_ = std::make_tuple( - dm::PARAM, // access: read/write - dm::MissingValue // access: read only + dm::PARAM, // access: read/write + dm::BitmapPresent, // access: read only + dm::MissingValue // access: read/write (will be unset if bitmapPresent false) ); + + static void applyDefaults(ScaleMetadataKeys& k) { + if (!k.bitmapPresent.get()) { + k.missingValue.unset(); + } + } + + static void validate(const ScaleMetadataKeys& k) { + if (k.bitmapPresent.get() && !k.missingValue.isSet()) { + throw eckit::SeriousBug( + "Value for missingValue is required if bitmapPresent is true!", + Here() + ); + } + } }; //------------------------ Action Configuration Keys ------------------------// diff --git a/src/multio/action/statistics-mtg2/CMakeLists.txt b/src/multio/action/statistics-mtg2/CMakeLists.txt index 88d5de53b..eda794322 100644 --- a/src/multio/action/statistics-mtg2/CMakeLists.txt +++ b/src/multio/action/statistics-mtg2/CMakeLists.txt @@ -39,6 +39,8 @@ list( APPEND _statistics_sources TemporalStatistics.h Statistics.cc Statistics.h + MetadataKeys.cc + MetadataKeys.h # SynopticCollection.cc # SynopticCollection.h # SynopticFilters.cc diff --git a/src/multio/action/statistics-mtg2/MetadataKeys.cc b/src/multio/action/statistics-mtg2/MetadataKeys.cc new file mode 100644 index 000000000..00c4a3ee6 --- /dev/null +++ b/src/multio/action/statistics-mtg2/MetadataKeys.cc @@ -0,0 +1,62 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include "MetadataKeys.h" +#include +#include "multio/datamod/core/DataModellingException.h" + +using FlushKind = multio::action::statistics_mtg2::FlushKind; + +std::string multio::datamod::DumpType::dump(FlushKind v) { + switch (v) { + case FlushKind::Default: + return "default"; + case FlushKind::FirstStep: + return "first-step"; + case FlushKind::StepAndRestart: + return "step-and-restart"; + case FlushKind::LastStep: + return "last-step"; + case FlushKind::EndOfSimulation: + return "end-of-simulation"; + case FlushKind::CloseConnection: + return "close-connection"; + default: + throw multio::datamod::DataModellingException( + "DumpType::dump: Unexpected enum value " + std::to_string(static_cast(v)), + Here()); + } +} + +FlushKind multio::datamod::ParseType::parse(const std::string& s) { + static const std::unordered_map map{ + {"default", FlushKind::Default}, + {"first-step", FlushKind::FirstStep}, + {"step-and-restart", FlushKind::StepAndRestart}, + {"last-step", FlushKind::LastStep}, + {"end-of-simulation", FlushKind::EndOfSimulation}, + {"close-connection", FlushKind::CloseConnection}}; + if (auto it = map.find(s); it != map.end()) { + return it->second; + } + throw multio::datamod::DataModellingException("ParseType::parse: Unknown FlushKind string: " + s, + Here()); +} + +FlushKind multio::datamod::ParseType::parse(std::int64_t val) { + static const std::unordered_map map{ + {0, FlushKind::FirstStep}, {1, FlushKind::Default}, {2, FlushKind::StepAndRestart}, + {3, FlushKind::LastStep}, {4, FlushKind::EndOfSimulation}, {5, FlushKind::CloseConnection}}; + if (auto it = map.find(val); it != map.end()) { + return it->second; + } + throw multio::datamod::DataModellingException( + "ParseType::parse: Unknown FlushKind integer: " + std::to_string(val), Here()); +} diff --git a/src/multio/action/statistics-mtg2/MetadataKeys.h b/src/multio/action/statistics-mtg2/MetadataKeys.h new file mode 100644 index 000000000..aa871da10 --- /dev/null +++ b/src/multio/action/statistics-mtg2/MetadataKeys.h @@ -0,0 +1,118 @@ +#pragma once + +#include "multio/datamod/core/EntryDef.h" +#include "multio/datamod/GribKeys.h" +#include "multio/datamod/MarsMiscGeo.h" +#include "multio/datamod/MarsKeys.h" + +namespace multio::action::statistics_mtg2 { + +//----------------------------- FlushKind Enum ------------------------------// + +enum class FlushKind : std::size_t +{ + Default, + FirstStep, + StepAndRestart, + LastStep, + EndOfSimulation, + CloseConnection +}; + +} // namespace multio::action::statistics_mtg2 + +//-------------- FlushKind type parsing/dumping specializations --------------// + +template <> +struct multio::datamod::DumpType { + static std::string dump(multio::action::statistics_mtg2::FlushKind v); +}; + +template <> +struct multio::datamod::ParseType { + static multio::action::statistics_mtg2::FlushKind parse(const std::string& s); + static multio::action::statistics_mtg2::FlushKind parse(std::int64_t val); +}; + +//--------------------------------- Records ---------------------------------// + +namespace multio::action::statistics_mtg2 { + +namespace dm = multio::datamod; + +//-------------------------- Flush-local EntryDefs --------------------------// + +constexpr auto FLUSH_KIND = + dm::EntryDef{"flushKind"} + .withDefault(FlushKind::Default) + .withAccessor([](auto&& v) { return &v.flushKind; }); + +constexpr auto RESTART_DATE_TIME = + dm::EntryDef{"restartDateTime"} + .tagOptional() + .withAccessor([](auto&& v) { return &v.restartDateTime; }); + +constexpr auto SERVER_RANK = + dm::EntryDef{"serverRank"} + .tagOptional() + .withAccessor([](auto&& v) { return &v.serverRank; }); + +//----------------------- Field Metadata Keys Record ------------------------// + +struct FieldMetadataKeys { + dm::EntryType_t date; + dm::EntryType_t time; + dm::EntryType_t step; + dm::EntryType_t timespan; + dm::EntryType_t param; + dm::EntryType_t stream; + dm::EntryType_t levtype; + dm::EntryType_t levelist; + dm::EntryType_t grid; + dm::EntryType_t truncation; + dm::EntryType_t timeIncrementInSeconds; + dm::EntryType_t bitmapPresent; + dm::EntryType_t missingValue; + + static constexpr std::string_view record_name_ = "statistics-mtg2-field"; + static constexpr auto record_entries_ = std::make_tuple( + dm::DATE, + dm::TIME, + dm::STEP.tagRequired(), + dm::TIMESPAN, + dm::PARAM, + dm::STREAM.tagOptional(), + dm::LEVTYPE.tagRequired(), + dm::LEVELIST, + dm::GRID, + dm::TRUNCATION, + dm::TimeIncrementInSeconds, + dm::BitmapPresent, + dm::MissingValue + ); +}; + +//------------------------ Flush Metadata Keys Record -----------------------// + +struct FlushMetadataKeys { + dm::EntryType_t date; + dm::EntryType_t time; + dm::EntryType_t step; + dm::EntryType_t flushKind; + dm::EntryType_t restartDateTime; + dm::EntryType_t serverRank; + + static constexpr std::string_view record_name_ = "statistics-mtg2-flush"; + static constexpr auto record_entries_ = std::make_tuple( + dm::DATE.tagOptional(), + dm::TIME.tagOptional(), + dm::STEP, + FLUSH_KIND, + RESTART_DATE_TIME, + SERVER_RANK + ); +}; + +//---------------------------------------------------------------------------// + +} // namespace multio::action::statistics_mtg2 diff --git a/src/multio/action/statistics-mtg2/OperationWindow.cc b/src/multio/action/statistics-mtg2/OperationWindow.cc index 590c2a9f8..ebd44edfc 100644 --- a/src/multio/action/statistics-mtg2/OperationWindow.cc +++ b/src/multio/action/statistics-mtg2/OperationWindow.cc @@ -70,7 +70,7 @@ OperationWindow make_window(const std::unique_ptr& periodUpdater, eckit::DateTime startPoint{periodUpdater->computeWinStartTime(epoch + deltaStart)}; eckit::DateTime creationPoint{periodUpdater->computeWinCreationTime(epoch + deltaStart)}; eckit::DateTime endPoint{periodUpdater->computeWinEndTime(startPoint)}; - return OperationWindow{epochPoint, startPoint, creationPoint, endPoint, cfg.timeStep(), cfg.options().windowType()}; + return OperationWindow{epochPoint, startPoint, creationPoint, endPoint, cfg.timeIncrementInSeconds(), cfg.options().windowType()}; }; OperationWindow load_window(std::shared_ptr& IOmanager, const StatisticsOptions& opt) { @@ -92,7 +92,7 @@ OperationWindow::OperationWindow(std::shared_ptr& IOmanager, const prevPoint_{eckit::Date{0}, eckit::Time{0}}, endPoint_{eckit::Date{0}, eckit::Time{0}}, lastFlush_{eckit::Date{0}, eckit::Time{0}}, - timeStepInSeconds_{0}, + timeIncrementInSeconds_{0}, count_{0}, counts_{}, windowType_{WindowType::ForwardOffset} { @@ -102,7 +102,7 @@ OperationWindow::OperationWindow(std::shared_ptr& IOmanager, const OperationWindow::OperationWindow(const eckit::DateTime& epochPoint, const eckit::DateTime& startPoint, const eckit::DateTime& creationPoint, const eckit::DateTime& endPoint, - long timeStepInSeconds, WindowType windowType) : + long timeIncrementInSeconds, WindowType windowType) : epochPoint_{epochPoint}, startPoint_{startPoint}, creationPoint_{creationPoint}, @@ -110,7 +110,7 @@ OperationWindow::OperationWindow(const eckit::DateTime& epochPoint, const eckit: prevPoint_{creationPoint}, endPoint_{endPoint}, lastFlush_{epochPoint}, - timeStepInSeconds_{timeStepInSeconds}, + timeIncrementInSeconds_{timeIncrementInSeconds}, count_{0}, counts_{}, windowType_{windowType} {} @@ -241,7 +241,7 @@ long OperationWindow::timeSpanInSeconds() const { } long OperationWindow::timeSpanInSteps() const { - return timeSpanInSeconds() / timeStepInSeconds_; + return timeSpanInSeconds() / timeIncrementInSeconds_; } long OperationWindow::lastPointsDiffInSeconds() const { @@ -297,23 +297,23 @@ long OperationWindow::prevPointInHours() const { long OperationWindow::startPointInSteps() const { - return startPointInSeconds() / timeStepInSeconds_; + return startPointInSeconds() / timeIncrementInSeconds_; } long OperationWindow::creationPointInSteps() const { - return creationPointInSeconds() / timeStepInSeconds_; + return creationPointInSeconds() / timeIncrementInSeconds_; } long OperationWindow::endPointInSteps() const { - return endPointInSeconds() / timeStepInSeconds_; + return endPointInSeconds() / timeIncrementInSeconds_; } long OperationWindow::currPointInSteps() const { - return currPointInSeconds() / timeStepInSeconds_; + return currPointInSeconds() / timeIncrementInSeconds_; } long OperationWindow::prevPointInSteps() const { - return prevPointInSeconds() / timeStepInSeconds_; + return prevPointInSeconds() / timeIncrementInSeconds_; } long OperationWindow::startPointInSeconds(const eckit::DateTime& refPoint) const { @@ -359,23 +359,23 @@ long OperationWindow::prevPointInHours(const eckit::DateTime& refPoint) const { long OperationWindow::startPointInSteps(const eckit::DateTime& refPoint) const { - return startPointInSeconds(refPoint) / timeStepInSeconds_; + return startPointInSeconds(refPoint) / timeIncrementInSeconds_; } long OperationWindow::creationPointInSteps(const eckit::DateTime& refPoint) const { - return creationPointInSeconds(refPoint) / timeStepInSeconds_; + return creationPointInSeconds(refPoint) / timeIncrementInSeconds_; } long OperationWindow::endPointInSteps(const eckit::DateTime& refPoint) const { - return endPointInSeconds(refPoint) / timeStepInSeconds_; + return endPointInSeconds(refPoint) / timeIncrementInSeconds_; } long OperationWindow::currPointInSteps(const eckit::DateTime& refPoint) const { - return currPointInSeconds(refPoint) / timeStepInSeconds_; + return currPointInSeconds(refPoint) / timeIncrementInSeconds_; } long OperationWindow::prevPointInSteps(const eckit::DateTime& refPoint) const { - return prevPointInSeconds(refPoint) / timeStepInSeconds_; + return prevPointInSeconds(refPoint) / timeIncrementInSeconds_; } eckit::DateTime OperationWindow::epochPoint() const { @@ -432,7 +432,7 @@ void OperationWindow::updateFlush() { } long OperationWindow::lastFlushInSteps() const { - return (lastFlush_ - epochPoint_) / timeStepInSeconds_; + return (lastFlush_ - epochPoint_) / timeIncrementInSeconds_; } void OperationWindow::initCountsLazy(size_t size) const { @@ -460,7 +460,7 @@ void OperationWindow::serialize(IOBuffer& currState, const std::string& fname, c outFile << "prevPoint_ :: " << prevPoint_ << std::endl; outFile << "currPoint_ :: " << currPoint_ << std::endl; outFile << "lastFlush_ :: " << lastFlush_ << std::endl; - outFile << "timeStepInSeconds_ :: " << timeStepInSeconds_ << std::endl; + outFile << "timeIncrementInSeconds_ :: " << timeIncrementInSeconds_ << std::endl; outFile << "count_ :: " << count_ << std::endl; outFile << "counts_.size() :: " << counts_.size() << std::endl; outFile << "windowType_ :: " << (windowType_ == WindowType::ForwardOffset ? "forward-offset" : "backward-offset") << std::endl; @@ -488,7 +488,7 @@ void OperationWindow::serialize(IOBuffer& currState, const std::string& fname, c currState[12] = static_cast(lastFlush_.date().yyyymmdd()); currState[13] = static_cast(lastFlush_.time().hhmmss()); - currState[14] = static_cast(timeStepInSeconds_); + currState[14] = static_cast(timeIncrementInSeconds_); currState[15] = static_cast(count_); currState[16] = static_cast(windowType_); @@ -513,7 +513,7 @@ void OperationWindow::deserialize(const IOBuffer& currState, const std::string& prevPoint_ = yyyymmdd_hhmmss2DateTime(static_cast(currState[8]), static_cast(currState[9])); currPoint_ = yyyymmdd_hhmmss2DateTime(static_cast(currState[10]), static_cast(currState[11])); lastFlush_ = yyyymmdd_hhmmss2DateTime(static_cast(currState[12]), static_cast(currState[13])); - timeStepInSeconds_ = static_cast(currState[14]); + timeIncrementInSeconds_ = static_cast(currState[14]); count_ = static_cast(currState[15]); windowType_ = static_cast(currState[16]); @@ -532,7 +532,7 @@ void OperationWindow::deserialize(const IOBuffer& currState, const std::string& outFile << "prevPoint_ :: " << prevPoint_ << std::endl; outFile << "currPoint_ :: " << currPoint_ << std::endl; outFile << "lastFlush_ :: " << lastFlush_ << std::endl; - outFile << "timeStepInSeconds_ :: " << timeStepInSeconds_ << std::endl; + outFile << "timeIncrementInSeconds_ :: " << timeIncrementInSeconds_ << std::endl; outFile << "count_ :: " << count_ << std::endl; outFile << "counts_.size() :: " << counts_.size() << std::endl; outFile << "windowType_ :: " << (windowType_ == WindowType::ForwardOffset ? "forward-offset" : "backward-offset") << std::endl; diff --git a/src/multio/action/statistics-mtg2/OperationWindow.h b/src/multio/action/statistics-mtg2/OperationWindow.h index 6295f3f76..dec857409 100644 --- a/src/multio/action/statistics-mtg2/OperationWindow.h +++ b/src/multio/action/statistics-mtg2/OperationWindow.h @@ -21,7 +21,7 @@ class OperationWindow { OperationWindow(const eckit::DateTime& epochPoint, const eckit::DateTime& startPoint, const eckit::DateTime& creationPoint, const eckit::DateTime& endPoint, - long timeStepInSeconds, WindowType windowType); + long timeIncrementInSeconds, WindowType windowType); long count() const; const std::vector& counts() const; @@ -85,7 +85,7 @@ class OperationWindow { long prevPointInSteps(const eckit::DateTime& refPoint) const; long endPointInSteps(const eckit::DateTime& refPoint) const; - long timeStepInSeconds() const; + long timeIncrementInSeconds() const; eckit::DateTime epochPoint() const; @@ -115,7 +115,7 @@ class OperationWindow { eckit::DateTime endPoint_; eckit::DateTime lastFlush_; - long timeStepInSeconds_; + long timeIncrementInSeconds_; long count_; mutable std::vector counts_; WindowType windowType_; diff --git a/src/multio/action/statistics-mtg2/PeriodUpdaters.h b/src/multio/action/statistics-mtg2/PeriodUpdaters.h index 574f122bd..358123a90 100644 --- a/src/multio/action/statistics-mtg2/PeriodUpdaters.h +++ b/src/multio/action/statistics-mtg2/PeriodUpdaters.h @@ -9,7 +9,6 @@ #include "multio/action/statistics-mtg2/period-updaters/PeriodUpdater.h" #include "eckit/types/DateTime.h" -#include "multio/action/statistics-mtg2/TimeUtils.h" #include "multio/message/Message.h" #include "OperationWindow.h" diff --git a/src/multio/action/statistics-mtg2/Statistics.cc b/src/multio/action/statistics-mtg2/Statistics.cc index c56b0d054..c85cb08f1 100644 --- a/src/multio/action/statistics-mtg2/Statistics.cc +++ b/src/multio/action/statistics-mtg2/Statistics.cc @@ -12,7 +12,6 @@ #include //currently needed because ECKIT PathName only has hardlinks for for the directories we need a symlink #include -#include #include "TemporalStatistics.h" @@ -21,8 +20,9 @@ #include "multio/LibMultio.h" #include "multio/action/statistics-mtg2/cfg/StatisticsOptions.h" #include "multio/datamod/ContainerInterop.h" -#include "multio/datamod/Glossary.h" +#include "multio/datamod/MarsKeys.h" #include "multio/datamod/MarsMiscGeo.h" +#include "multio/datamod/core/EntryParser.h" #include "multio/datamod/types/StatType.h" #include "multio/message/Message.h" #include "multio/util/Timing.h" @@ -45,28 +45,20 @@ Statistics::Statistics(const ComponentConfiguration& compConf) : operationMapping_{StatisticsOperationMapping::makeStatisticsOperationMapping()}, IOmanager_{StatisticsIOFactory::instance().build(opt_.restartLib(), opt_.restartPath(), opt_.restartPrefix())} {} -std::string Statistics::generateRestartNameFromFlush(const message::Message& msg) const { +std::string Statistics::generateRestartNameFromFlush(const message::Message& msg, const FlushMetadataKeys& flush) const { std::string folderName; - // Restart flush directly provides the folderName - auto restartDateTime = msg.metadata().getOpt("restartDateTime"); - auto step = msg.metadata().getOpt(dm::legacy::Step); - auto timeStep = msg.metadata().getOpt(dm::legacy::TimeStep); - auto date = msg.metadata().getOpt(dm::legacy::Date); - auto time = msg.metadata().getOpt(dm::legacy::Time); - - if (restartDateTime) { - folderName = *restartDateTime; + if (flush.restartDateTime.isSet()) { + folderName = flush.restartDateTime.get(); } - // Restart flush provides the step, timeStep, date and time - else if (step && timeStep && date && time) { + // Restart flush provides the step, date and time + else if (flush.step.isSet() && flush.date.isSet() && flush.time.isSet()) { - std::int64_t flushStep = *step; - std::int64_t flushTimeStep = *timeStep; - std::int64_t flushDate = *date; - std::int64_t flushTime = *time; + std::int64_t flushStepInSeconds = flush.step.get().toSeconds(); + std::int64_t flushDate = flush.date.get(); + std::int64_t flushTime = flush.time.get(); // Compute the date and time from the step // TODO: Remove this multiple times repeated code @@ -89,7 +81,7 @@ std::string Statistics::generateRestartNameFromFlush(const message::Message& msg // } // } - eckit::DateTime dt = epoch + static_cast(flushStep * flushTimeStep); + eckit::DateTime dt = epoch + static_cast(flushStepInSeconds); std::ostringstream tmp; tmp << std::setw(8) << std::setfill('0') << dt.date().yyyymmdd() << "-" << std::setw(6) << std::setfill('0') << dt.time().hhmmss(); @@ -97,11 +89,11 @@ std::string Statistics::generateRestartNameFromFlush(const message::Message& msg } // Restart flush provides the step - else if (step) { - std::int64_t flushStep = *step; + else if (flush.step.isSet()) { + std::int64_t flushStepInHours = flush.step.get().toHours(); std::ostringstream tmp; tmp << lastDateTime_ << "-"; - tmp << std::setw(6) << std::setfill('0') << flushStep; + tmp << std::setw(6) << std::setfill('0') << flushStepInHours; folderName = tmp.str(); } @@ -160,59 +152,8 @@ void Statistics::DumpTemporalStatistics() { return; } -enum class FlushKind : std::size_t -{ - Default, - FirstStep, - StepAndRestart, - LastStep, - EndOfSimulation, - CloseConnection -}; - -FlushKind parseFlushKind(const std::string& str) { - static const std::unordered_map map{{"first-step", FlushKind::FirstStep}, - {"default", FlushKind::Default}, - {"step-and-restart", FlushKind::StepAndRestart}, - {"last-step", FlushKind::LastStep}, - {"end-of-simulation", FlushKind::EndOfSimulation}, - {"close-connection", FlushKind::CloseConnection}}; - if (auto search = map.find(str); search != map.end()) { - return search->second; - } - throw message::MetadataException(std::string("Unknown FlushKind: ") + str, Here()); -} - - -FlushKind parseFlushKind(std::int64_t val) { - static const std::unordered_map map{ - {0, FlushKind::FirstStep}, {1, FlushKind::Default}, {2, FlushKind::StepAndRestart}, - {3, FlushKind::LastStep}, {4, FlushKind::EndOfSimulation}, {5, FlushKind::CloseConnection}}; - if (auto search = map.find(val); search != map.end()) { - return search->second; - } - throw message::MetadataException(std::string("Unknown FlushKind: ") + std::to_string(val), Here()); -} - -FlushKind parseFlushKind(const message::Message& msg) { - if (msg.tag() != message::Message::Tag::Flush) { - throw message::MetadataException("Message is not a flush.", Here()); - } - FlushKind flushKind{FlushKind::Default}; - if (auto search = msg.metadata().find("flushKind"); search != msg.metadata().end()) { - search->second.visit(eckit::Overloaded{[&](const std::string& str) { flushKind = parseFlushKind(str); }, - [&](const std::int64_t val) { flushKind = parseFlushKind(val); }, - [&](const auto&) { - throw message::MetadataException( - "FlushKind needs to be either string or integer.", Here()); - }}); - } - return flushKind; -} - - -void Statistics::TryDumpRestart(const message::Message& msg) { - if (parseFlushKind(msg) != FlushKind::StepAndRestart) { +void Statistics::TryDumpRestart(const message::Message& msg, const FlushMetadataKeys& flush) { + if (flush.flushKind.get() != FlushKind::StepAndRestart) { return; } @@ -225,10 +166,10 @@ void Statistics::TryDumpRestart(const message::Message& msg) { // NOTE: This is a bit of a hack, in case no serverRank is provided, we // assume that all processors are "master" and rely on atomicity of // filesystem operation to avoid race conditions - auto is_master = msg.metadata().getOpt("serverRank").value_or(true); + auto is_master = !flush.serverRank.isSet() || flush.serverRank.get() != 0; // Generate name of the main restart directory - std::string restartFolderName = generateRestartNameFromFlush(msg); + std::string restartFolderName = generateRestartNameFromFlush(msg, flush); // Log for Dump operation LOG_DEBUG_LIB(LibMultio) << "Performing a Dump :: Writing to " << restartFolderName << std::endl; @@ -300,22 +241,6 @@ bool Statistics::HasRestartKey(const std::string& key) { } -message::Metadata Statistics::outputMetadata(const message::Metadata& inputMetadata, const StatisticsConfiguration& cfg, - const std::string& key) const { - auto& win = fieldStats_.at(key)->cwin(); - // if (win.endPointInSeconds() % 3600 != 0L) { - // std::ostringstream os; - // os << "Step in seconds needs to be a multiple of 3600 :: " << fieldStats_.at(key)->win().endPointInSeconds() - // << std::endl; - // throw eckit::SeriousBug(os.str(), Here()); - // } - auto md = inputMetadata; - - md.set(dm::legacy::StartDate, win.epochPoint().date().yyyymmdd()); - md.set(dm::legacy::StartTime, win.epochPoint().time().hhmmss()); - - return md; -} void Statistics::updateLatestDateTime(const StatisticsConfiguration& cfg) { @@ -334,10 +259,10 @@ void Statistics::executeImpl(message::Message msg) { // Handle flush if (msg.tag() == message::Message::Tag::Flush) { - TryDumpRestart(msg); + const auto flush = dm::readRecord(msg.metadata()); + TryDumpRestart(msg, flush); - FlushKind flushKind = parseFlushKind(msg); - if (parseFlushKind(msg) == FlushKind::LastStep) { + if (flush.flushKind.get() == FlushKind::LastStep) { emitAllStatistics(msg.source(), msg.destination()); } @@ -454,8 +379,6 @@ dm::StatTypeOperation operationNameToStatTypeOperation(std::string_view opName) } -const std::map opname_to_stattype{ - {"average", "av"}, {"maximum", "mx"}, {"minimum", "mn"}, {"stddev", "st"}}; } // namespace void Statistics::emitStatistics(TemporalStatistics& ts, message::Peer source, message::Peer destination) { @@ -529,7 +452,7 @@ void Statistics::emitStatistics(TemporalStatistics& ts, message::Peer source, me switch (cfg.outputTimeReference()) { case OutputTimeReference::StartOfForecast: { const std::int64_t step = ts.win().currPointInHours(); - md.set(dm::legacy::Step, step); + dm::dumpEntry(dm::STEP, dm::STEP.makeEntry(step), md); break; } case OutputTimeReference::StartOfWindow: { @@ -543,7 +466,7 @@ void Statistics::emitStatistics(TemporalStatistics& ts, message::Peer source, me lengthOfWindow.set(ts.win().currPointInHours() - ts.win().creationPointInHours()); } - md.set(dm::legacy::Step, lengthOfWindow.get().toHours()); + dm::dumpEntry(dm::STEP, dm::STEP.makeEntry(lengthOfWindow.get().toHours()), md); // We explicitly take the creation point - alternative would be the start point. // The start point may be different for the first window, i.e. if the simulation starts in the mid of a month. // To not confuse the output, we explicitly just output the window for which data has been received. @@ -551,9 +474,8 @@ void Statistics::emitStatistics(TemporalStatistics& ts, message::Peer source, me // Some additional mechanism has to make sure that these do not occur in the output (i.e. additional action). auto dt = ts.win().creationPoint(); - md.set(dm::legacy::Date, dt.date().yyyymmdd()); - md.set(dm::legacy::Time, - dt.time().hhmmss()); // Official MARS time is in hhmm, in multio hhmmss is used + dm::dumpEntry(dm::DATE, dm::DATE.makeEntry(dt.date().yyyymmdd()), md); + dm::dumpEntry(dm::TIME, dm::TIME.makeEntry(dt.time().hhmmss()), md); // Official MARS time is in hhmm, in multio hhmmss is used break; } } diff --git a/src/multio/action/statistics-mtg2/Statistics.h b/src/multio/action/statistics-mtg2/Statistics.h index 874aafc33..c2e0f8aa9 100644 --- a/src/multio/action/statistics-mtg2/Statistics.h +++ b/src/multio/action/statistics-mtg2/Statistics.h @@ -16,6 +16,7 @@ #pragma once +#include "MetadataKeys.h" #include "PeriodUpdaters.h" #include "StatisticsIO.h" #include "multio/action/ChainedAction.h" @@ -35,14 +36,12 @@ class Statistics : public ChainedAction { public: explicit Statistics(const ComponentConfiguration& compConf); void executeImpl(message::Message msg) override; - message::Metadata outputMetadata(const message::Metadata& inputMetadata, const StatisticsConfiguration& opt, - const std::string& key) const; private: bool needRestart_; std::string lastDateTime_; - void TryDumpRestart(const message::Message& msg); - std::string generateRestartNameFromFlush(const message::Message& msg) const; + void TryDumpRestart(const message::Message& msg, const FlushMetadataKeys& flush); + std::string generateRestartNameFromFlush(const message::Message& msg, const FlushMetadataKeys& flush) const; void DeleteLatestSymLink(); void CreateLatestSymLink(); void CreateMainRestartDirectory(const std::string& restartFolderName, bool is_master); diff --git a/src/multio/action/statistics-mtg2/StatisticsIO.cc b/src/multio/action/statistics-mtg2/StatisticsIO.cc index 22a13c96e..c2fa3e2c7 100644 --- a/src/multio/action/statistics-mtg2/StatisticsIO.cc +++ b/src/multio/action/statistics-mtg2/StatisticsIO.cc @@ -220,18 +220,6 @@ void StatisticsIO::createCurrentDir() const { return; }; -void StatisticsIO::createDateTimeDir() const { - if (!hasValidDateTime_) { - std::ostringstream os; - os << "ERROR : no valid datetime found"; - throw eckit::SeriousBug{os.str(), Here()}; - } - std::ostringstream dir; - dir << basePath_ << "/" << uniqueID_ << "/" << dateTime_; - eckit::PathName{dir.str()}.mkdir(); - return; -}; - std::string StatisticsIO::generateCurrFileName(const std::string& name) const { std::ostringstream os; os << getCurrentDir() << "/" << name << "." << ext_; diff --git a/src/multio/action/statistics-mtg2/StatisticsIO.h b/src/multio/action/statistics-mtg2/StatisticsIO.h index 8f387d940..f32ad24d6 100644 --- a/src/multio/action/statistics-mtg2/StatisticsIO.h +++ b/src/multio/action/statistics-mtg2/StatisticsIO.h @@ -61,7 +61,6 @@ class StatisticsIO { std::string getUniqueRestartDir() const; bool currentDirExists() const; void createCurrentDir() const; - void createDateTimeDir() const; IOBuffer getBuffer(std::size_t size); std::vector getFiles(); diff --git a/src/multio/action/statistics-mtg2/SynopticFilters.cc b/src/multio/action/statistics-mtg2/SynopticFilters.cc index a9c253983..61804444c 100644 --- a/src/multio/action/statistics-mtg2/SynopticFilters.cc +++ b/src/multio/action/statistics-mtg2/SynopticFilters.cc @@ -6,7 +6,6 @@ #include "eckit/exception/Exceptions.h" #include "eckit/types/DateTime.h" -#include "multio/action/statistics-mtg2/TimeUtils.h" #include "multio/config/ComponentConfiguration.h" #include "multio/action/statistics-mtg2/synoptic-filters/AllTimesFilter.h" diff --git a/src/multio/action/statistics-mtg2/TimeUtils.cc b/src/multio/action/statistics-mtg2/TimeUtils.cc index e45f55fe2..7f7b0456c 100644 --- a/src/multio/action/statistics-mtg2/TimeUtils.cc +++ b/src/multio/action/statistics-mtg2/TimeUtils.cc @@ -12,62 +12,7 @@ eckit::DateTime epochDateTime(const message::Message& msg, const StatisticsConfi eckit::DateTime currentDateTime(const message::Message& msg, const StatisticsConfiguration& cfg) { - return epochDateTime(msg, cfg) + static_cast(cfg.step() * cfg.timeStep()); -} - - -bool isBeginningOfYear(const message::Message& msg, const StatisticsConfiguration& cfg) { - - // Get the current time - eckit::DateTime now = currentDateTime(msg, cfg); - - long month = now.date().month(); - long day = now.date().day(); - long hour = now.time().hours(); - long min = now.time().minutes(); - long sec = now.time().seconds(); - - return month == 1 && day == 1 && hour == 0 && min == 0 && sec == 0; -} - - -bool isBeginningOfMonth(const message::Message& msg, const StatisticsConfiguration& cfg) { - - // Get the current time - eckit::DateTime now = currentDateTime(msg, cfg); - - - long day = now.date().day(); - long hour = now.time().hours(); - long min = now.time().minutes(); - long sec = now.time().seconds(); - - return day == 1 && hour == 0 && min == 0 && sec == 0; -} - - -bool isBeginningOfDay(const message::Message& msg, const StatisticsConfiguration& cfg) { - - // Get the current time - eckit::DateTime now = currentDateTime(msg, cfg); - - long hour = now.time().hours(); - long min = now.time().minutes(); - long sec = now.time().seconds(); - - return hour == 0 && min == 0 && sec == 0; -} - - -bool isBeginningOfHour(const message::Message& msg, const StatisticsConfiguration& cfg) { - - // Get the current time - eckit::DateTime now = currentDateTime(msg, cfg); - - long min = now.time().minutes(); - long sec = now.time().seconds(); - - return min == 0 && sec == 0; + return epochDateTime(msg, cfg) + static_cast(cfg.step() * cfg.timeIncrementInSeconds()); } diff --git a/src/multio/action/statistics-mtg2/TimeUtils.h b/src/multio/action/statistics-mtg2/TimeUtils.h index 1961a88cd..bab0729d1 100644 --- a/src/multio/action/statistics-mtg2/TimeUtils.h +++ b/src/multio/action/statistics-mtg2/TimeUtils.h @@ -10,9 +10,4 @@ namespace multio::action::statistics_mtg2 { eckit::DateTime epochDateTime(const message::Message& msg, const StatisticsConfiguration& cfg); eckit::DateTime currentDateTime(const message::Message& msg, const StatisticsConfiguration& cfg); -bool isBeginningOfYear(const message::Message& msg, const StatisticsConfiguration& cfg); -bool isBeginningOfMonth(const message::Message& msg, const StatisticsConfiguration& cfg); -bool isBeginningOfDay(const message::Message& msg, const StatisticsConfiguration& cfg); -bool isBeginningOfHour(const message::Message& msg, const StatisticsConfiguration& cfg); - } // namespace multio::action::statistics_mtg2 \ No newline at end of file diff --git a/src/multio/action/statistics-mtg2/cfg/StatisticsConfiguration.cc b/src/multio/action/statistics-mtg2/cfg/StatisticsConfiguration.cc index 98d7ed379..f74708a4e 100644 --- a/src/multio/action/statistics-mtg2/cfg/StatisticsConfiguration.cc +++ b/src/multio/action/statistics-mtg2/cfg/StatisticsConfiguration.cc @@ -11,119 +11,64 @@ #include "multio/action/statistics-mtg2/cfg/StatisticsOptions.h" #include "multio/datamod/ContainerInterop.h" -#include "multio/datamod/Glossary.h" -#include "multio/datamod/MarsKeys.h" #include "multio/datamod/core/EntryParser.h" +#include "multio/datamod/core/TypeParserDumper.h" namespace multio::action::statistics_mtg2 { namespace dm = multio::datamod; -std::int64_t readDate(const message::Metadata& md) { - // NOTE: Currently no support for messages without step (from analysis)! - ASSERT(md.getOpt(dm::legacy::Step).has_value()); - const auto dateVal = md.getOpt(dm::legacy::Date); - ASSERT(dateVal.has_value()); - return *dateVal; -} - -std::int64_t readTime(const message::Metadata& md) { - // NOTE: Currently no support for messages without step (from analysis)! - ASSERT(md.getOpt(dm::legacy::Step).has_value()); - const auto timeVal = md.getOpt(dm::legacy::Time); - ASSERT(timeVal.has_value()); - return *timeVal; -} - -std::int64_t readLevel(const message::Metadata& md) { - if (auto level = md.getOpt(dm::legacy::Level); level) { - return *level; +std::int64_t deriveLevel(const FieldMetadataKeys& md) { + if (md.levelist.isSet()) { + return md.levelist.get(); } - else if (auto levelist = md.getOpt(dm::legacy::Levelist); levelist) { - return *levelist; - } - // if none of the above metadata options are present the default levtype is 0 return 0; } -std::int64_t readTimeStep(const message::Metadata& md, int64_t defaultTimeStep) { - return md.getOpt(dm::legacy::TimeStep).value_or(defaultTimeStep); -} - -std::int64_t readStep(const message::Metadata& md) { - // NOTE: Currently no support for messages without step (from analysis)! - const auto stepVal = md.getOpt(dm::legacy::Step); - ASSERT(stepVal.has_value()); - return *stepVal; +std::string deriveLevType(const FieldMetadataKeys& md) { + return dm::DumpType::dump(md.levtype.get()); } -std::optional readTimespan(const message::Metadata& md) { - if (const auto& timespan = dm::parseEntry(dm::TIMESPAN, md); timespan.isSet()) { - return timespan.get().toHours(); +std::string deriveGridType(const FieldMetadataKeys& md) { + if (md.grid.isSet()) { + return md.grid.get(); } - return std::nullopt; -} - -std::int64_t readParam(const message::Metadata& md) { - // TODO: use whole validated keyset... - if (const auto& param = dm::parseEntry(dm::PARAM, md); param.isSet()) { - return param.get().id(); - } - else if (auto paramId = md.getOpt(dm::legacy::ParamId); paramId) { - return *paramId; - } - throw eckit::SeriousBug{"Param metadata not present", Here()}; -} - -std::string readLevType(const message::Metadata& md) { - if (auto levType = md.getOpt(dm::legacy::Levtype); levType) { - return *levType; - } - throw message::MetadataException("Levtype missing in metadata", Here()); -} - -std::string readGridType(const message::Metadata& md) { - // TODO use whole validated keyset... - if (const auto& grid = dm::parseEntry(dm::GRID, md); grid.isSet()) { - return grid.get(); - } - // Truncation is always present when we are dealing with Spherical Harmonics - if (dm::parseEntry(dm::TRUNCATION, md).isSet()) { + if (md.truncation.isSet()) { return "none"; } - std::ostringstream os; - os << "Cannot find grid or truncation in metadata : " << md; + os << "Cannot find grid or truncation in metadata"; throw eckit::SeriousBug{os.str(), Here()}; } -std::string readPrecision(const message::Metadata& md) { - if (auto precision = md.getOpt(dm::legacy::Precision); precision) { +// TODO: add a proper EntryDef for precision +std::string derivePrecision(const message::Metadata& rawMd) { + if (auto precision = rawMd.getOpt("misc-precision"); precision) { return *precision; } throw eckit::SeriousBug{"precision metadata not present", Here()}; } -std::optional readMissingValue(const message::Metadata& md) { - const auto missingVal = md.getOpt(dm::legacy::MissingValue); - const auto bitMapPresent = md.getOpt(dm::legacy::BitmapPresent); - - if (missingVal && bitMapPresent && *bitMapPresent) { - return missingVal; +std::optional deriveMissingValue(const FieldMetadataKeys& md) { + if (md.missingValue.isSet() && md.bitmapPresent.isSet() && md.bitmapPresent.get()) { + return md.missingValue.get(); } return std::nullopt; } -OutputTimeReference readOutputTimeReference(const message::Metadata& md, const StatisticsOptions& opt) { +OutputTimeReference readOutputTimeReference(const FieldMetadataKeys& md, const StatisticsOptions& opt) { // Check if output-time-reference has been explicitly overwritten if (auto parsedOutRef = opt.outputTimeReference(); parsedOutRef) { return *parsedOutRef; } // Look up for stream in metadata or in configuration - std::optional stream = md.getOpt("stream"); + std::optional stream; + if (md.stream.isSet()) { + stream = md.stream.get(); + } if (!stream) { // Look for stream in options const auto& omd = opt.setMetadata(); @@ -155,22 +100,17 @@ OutputTimeReference readOutputTimeReference(const message::Metadata& md, const S StatisticsConfiguration::StatisticsConfiguration(const message::Metadata& md, const message::Peer& src, const StatisticsOptions& opt) : + md_{dm::readRecord(md)}, opt_{opt}, - date_{readDate(md)}, - time_{readTime(md)}, - level_{readLevel(md)}, - timeStep_{readTimeStep(md, opt.timeStep())}, - step_{readStep(md)}, - timespan_{readTimespan(md)}, - param_{readParam(md)}, - levType_{readLevType(md)}, - gridType_{readGridType(md)}, - precision_{readPrecision(md)}, - missingValue_{readMissingValue(md)}, + level_{deriveLevel(md_)}, + levType_{deriveLevType(md_)}, + gridType_{deriveGridType(md_)}, + precision_{derivePrecision(md)}, + missingValue_{deriveMissingValue(md_)}, key_{generateKey(src)}, epoch_{computeEpoch()}, curr_{computeCurr()}, - outputTimeReference_{readOutputTimeReference(md, opt)} {} + outputTimeReference_{readOutputTimeReference(md_, opt)} {} StatisticsConfiguration::StatisticsConfiguration(const message::Message& msg, const StatisticsOptions& opt) : StatisticsConfiguration(msg.metadata(), msg.source(), opt) {}; @@ -178,21 +118,21 @@ StatisticsConfiguration::StatisticsConfiguration(const message::Message& msg, co std::string StatisticsConfiguration::generateKey(const message::Peer& src) const { std::ostringstream os; - os << param_ << "-" << level_ << "-" << levType_ << "-" << gridType_ << "-" << precision_ << "-" << src.group() + os << param() << "-" << level_ << "-" << levType_ << "-" << gridType_ << "-" << precision_ << "-" << src.group() << "_" << src.id(); return os.str(); } eckit::DateTime StatisticsConfiguration::computeEpoch() const { - eckit::Date date{date_}; - const auto hour = time_ / 10000; - const auto minute = (time_ % 10000) / 100; - return eckit::DateTime{date, eckit::Time{hour, minute, 0}}; + eckit::Date d{date()}; + const auto hour = time() / 10000; + const auto minute = (time() % 10000) / 100; + return eckit::DateTime{d, eckit::Time{hour, minute, 0}}; } eckit::DateTime StatisticsConfiguration::computeCurr() const { - return epoch() + static_cast(std::max(step_, static_cast(0)) * timeStep_); + return epoch() + static_cast(std::max(step(), static_cast(0)) * timeIncrementInSeconds()); } @@ -201,32 +141,26 @@ const StatisticsOptions& StatisticsConfiguration::options() const { } std::int64_t StatisticsConfiguration::date() const { - return date_; + return md_.date.get(); } std::int64_t StatisticsConfiguration::time() const { - return time_; -} -std::int64_t StatisticsConfiguration::level() const { - return level_; + return md_.time.get(); } -std::int64_t StatisticsConfiguration::timeStep() const { - return timeStep_; +std::int64_t StatisticsConfiguration::timeIncrementInSeconds() const { + return md_.timeIncrementInSeconds.get(); } std::int64_t StatisticsConfiguration::step() const { - return step_; + return md_.step.get().toHours(); } std::optional StatisticsConfiguration::timespan() const { - return timespan_; + if (md_.timespan.isSet()) { + return md_.timespan.get().toHours(); + } + return std::nullopt; } int64_t StatisticsConfiguration::param() const { - return param_; -} -const std::string& StatisticsConfiguration::levType() const { - return levType_; -} -const std::string& StatisticsConfiguration::gridType() const { - return gridType_; + return md_.param.get().id(); } const std::string& StatisticsConfiguration::precision() const { return precision_; diff --git a/src/multio/action/statistics-mtg2/cfg/StatisticsConfiguration.h b/src/multio/action/statistics-mtg2/cfg/StatisticsConfiguration.h index 689d09cc9..79972d59f 100644 --- a/src/multio/action/statistics-mtg2/cfg/StatisticsConfiguration.h +++ b/src/multio/action/statistics-mtg2/cfg/StatisticsConfiguration.h @@ -3,6 +3,7 @@ #include "StatisticsOptions.h" #include "eckit/types/DateTime.h" +#include "multio/action/statistics-mtg2/MetadataKeys.h" #include "multio/message/Message.h" namespace multio::action::statistics_mtg2 { @@ -10,18 +11,16 @@ namespace multio::action::statistics_mtg2 { class StatisticsConfiguration { private: + // Metadata record parsed from message + const FieldMetadataKeys md_; + // Options const StatisticsOptions& opt_; - // Metadata to be extracted from the message - const std::int64_t date_; - const std::int64_t time_; + // Derived scalar fields const std::int64_t level_; - const std::int64_t timeStep_; - const std::int64_t step_; - const std::optional timespan_; - const std::int64_t param_; + // Derived string fields const std::string levType_; const std::string gridType_; const std::string precision_; @@ -35,7 +34,7 @@ class StatisticsConfiguration { // Timing utils const eckit::DateTime epoch_; const eckit::DateTime curr_; - + const OutputTimeReference outputTimeReference_; // --------------------------------------------------------------------------------------------- @@ -54,14 +53,11 @@ class StatisticsConfiguration { std::int64_t date() const; std::int64_t time() const; - std::int64_t level() const; - std::int64_t timeStep() const; + std::int64_t timeIncrementInSeconds() const; std::int64_t step() const; std::optional timespan() const; std::int64_t param() const; - const std::string& levType() const; - const std::string& gridType() const; const std::string& precision() const; // Handle missing values @@ -72,7 +68,7 @@ class StatisticsConfiguration { const eckit::DateTime& epoch() const; const eckit::DateTime& curr() const; - + OutputTimeReference outputTimeReference() const; }; diff --git a/src/multio/action/statistics-mtg2/cfg/StatisticsOptions.cc b/src/multio/action/statistics-mtg2/cfg/StatisticsOptions.cc index 985d859af..fc216b24b 100644 --- a/src/multio/action/statistics-mtg2/cfg/StatisticsOptions.cc +++ b/src/multio/action/statistics-mtg2/cfg/StatisticsOptions.cc @@ -7,11 +7,6 @@ namespace multio::action::statistics_mtg2 { -std::int64_t parseTimeStep(const eckit::LocalConfiguration& cfg) { - // How many seconds in a timestep - return cfg.getLong("time-step", 3600L); -} - bool parseInitialConditionPresent(const eckit::LocalConfiguration& cfg) { // Used to determine if the solver emit the initial condition. // This is a relevant information for statistics computations. @@ -159,7 +154,6 @@ std::vector> parseSetMetadata(const eckit::L StatisticsOptions::StatisticsOptions(const eckit::LocalConfiguration& cfg) : - timeStep_{parseTimeStep(cfg)}, initialConditionPresent_{parseInitialConditionPresent(cfg)}, readRestart_{parseReadRestart(cfg)}, writeRestart_{parseWriteRestart(cfg)}, @@ -177,9 +171,6 @@ StatisticsOptions::StatisticsOptions(const eckit::LocalConfiguration& cfg) : outputTimeReference_{parseOutputTimeRef(cfg)} {} -std::int64_t StatisticsOptions::timeStep() const { - return timeStep_; -} bool StatisticsOptions::initialConditionPresent() const { return initialConditionPresent_; } diff --git a/src/multio/action/statistics-mtg2/cfg/StatisticsOptions.h b/src/multio/action/statistics-mtg2/cfg/StatisticsOptions.h index b06a15fd1..a3467154b 100644 --- a/src/multio/action/statistics-mtg2/cfg/StatisticsOptions.h +++ b/src/multio/action/statistics-mtg2/cfg/StatisticsOptions.h @@ -28,7 +28,6 @@ enum class OutputTimeReference : std::int64_t */ class StatisticsOptions { private: - const std::int64_t timeStep_; const bool initialConditionPresent_; const bool readRestart_; @@ -53,7 +52,6 @@ class StatisticsOptions { public: StatisticsOptions(const eckit::LocalConfiguration& cfg); - std::int64_t timeStep() const; bool initialConditionPresent() const; bool readRestart() const; diff --git a/src/multio/action/statistics-mtg2/mappings/StatisticsParamMapping.cc b/src/multio/action/statistics-mtg2/mappings/StatisticsParamMapping.cc index d271bacce..7285be609 100644 --- a/src/multio/action/statistics-mtg2/mappings/StatisticsParamMapping.cc +++ b/src/multio/action/statistics-mtg2/mappings/StatisticsParamMapping.cc @@ -1,7 +1,9 @@ #include "StatisticsParamMapping.h" #include "multio/LibMultio.h" -#include "multio/datamod/Glossary.h" +#include "multio/datamod/ContainerInterop.h" +#include "multio/datamod/MarsKeys.h" +#include "multio/datamod/core/EntryParser.h" #include "eckit/config/LocalConfiguration.h" #include "eckit/config/YAMLConfiguration.h" @@ -41,10 +43,12 @@ std::optional StatisticsParamMapping::getMapping(std::int64_t para } void StatisticsParamMapping::applyMapping(message::Metadata& metadata, std::int64_t typeOfStatisticalProcessing, bool strict) const { - if (auto paramOld = metadata.getOpt(dm::legacy::Param); paramOld) { - auto paramNew = getMapping(*paramOld, typeOfStatisticalProcessing); + const auto& param = dm::parseEntry(dm::PARAM, metadata); + if (param.isSet()) { + auto paramOld = param.get().id(); + auto paramNew = getMapping(paramOld, typeOfStatisticalProcessing); if (paramNew.has_value()) { - metadata.set(dm::legacy::Param, *paramNew); + dm::dumpEntry(dm::PARAM, dm::PARAM.makeEntry(dm::Param{*paramNew}), metadata); return; } if (!strict) { @@ -52,7 +56,7 @@ void StatisticsParamMapping::applyMapping(message::Metadata& metadata, std::int6 } std::ostringstream os; - os << "Mapping for param=" << *paramOld << " and typeOfStatisticalProcessing=" << typeOfStatisticalProcessing << " is undefined!" << std::endl; + os << "Mapping for param=" << paramOld << " and typeOfStatisticalProcessing=" << typeOfStatisticalProcessing << " is undefined!" << std::endl; throw eckit::SeriousBug(os.str() , Here()); } diff --git a/src/multio/action/statistics-mtg2/period-updaters/PeriodUpdater.h b/src/multio/action/statistics-mtg2/period-updaters/PeriodUpdater.h index 0beaba56b..2776234ee 100644 --- a/src/multio/action/statistics-mtg2/period-updaters/PeriodUpdater.h +++ b/src/multio/action/statistics-mtg2/period-updaters/PeriodUpdater.h @@ -4,7 +4,6 @@ #include "eckit/types/DateTime.h" #include "multio/action/statistics-mtg2/StatisticsIO.h" -#include "multio/action/statistics-mtg2/TimeUtils.h" #include "multio/action/statistics-mtg2/cfg/StatisticsConfiguration.h" #include "multio/message/Message.h" diff --git a/src/multio/datamod/GribKeys.h b/src/multio/datamod/GribKeys.h index f8b2bc81a..f67e32869 100644 --- a/src/multio/datamod/GribKeys.h +++ b/src/multio/datamod/GribKeys.h @@ -32,33 +32,33 @@ namespace multio::datamod { //----------------------------------------------------------------------------- constexpr auto Discipline = - EntryDef{"discipline"} + EntryDef{"misc-discipline"} .withDefault(0) .withAccessor([](auto&& v) { return &v.discipline; }); // Section1 constexpr auto TablesVersion = - EntryDef{"tablesVersion"} + EntryDef{"misc-tablesVersion"} .tagOptional() .withAccessor([](auto&& v) { return &v.tablesVersion; }); - + constexpr auto LocalTablesVersion = - EntryDef{"localTablesVersion"} + EntryDef{"misc-localTablesVersion"} .tagOptional() .withAccessor([](auto&& v) { return &v.localTablesVersion; }); - + constexpr auto ProductionStatusOfProcessedData = - EntryDef{"productionStatusOfProcessedData"} + EntryDef{"misc-productionStatusOfProcessedData"} .tagOptional() .withAccessor([](auto&& v) { return &v.productionStatusOfProcessedData; }); constexpr auto SubCentre = - EntryDef{"subCentre"} + EntryDef{"misc-subCentre"} .tagOptional() .withAccessor([](auto&& v) { return &v.subCentre; }); constexpr auto TypeOfProcessedDataEntry = - EntryDef{"typeOfProcessedData"} + EntryDef{"misc-typeOfProcessedData"} .tagOptional() .withAccessor([](auto&& v) { return &v.typeOfProcessedData; }); @@ -66,158 +66,158 @@ constexpr auto TypeOfProcessedDataEntry = // Section3 (more to be moved here from MarsMiscGeo.h) constexpr auto ShapeOfTheEarth = - EntryDef{"shapeOfTheEarth"} + EntryDef{"misc-shapeOfTheEarth"} .withDefault(6) .withAccessor([](auto&& v) { return &v.shapeOfTheEarth; }); // Section 3 - GG constexpr auto TruncateDegrees = - EntryDef{"truncateDegrees"} + EntryDef{"misc-truncateDegrees"} .tagOptional() .withAccessor([](auto&& v) { return &v.truncateDegrees; }); constexpr auto NumberOfPointsAlongAMeridian = - EntryDef{"numberOfPointsAlongAMeridian"} + EntryDef{"misc-numberOfPointsAlongAMeridian"} .tagOptional() .withAccessor([](auto&& v) { return &v.numberOfPointsAlongAMeridian; }); constexpr auto NumberOfParallelsBetweenAPoleAndTheEquator = - EntryDef{"numberOfParallelsBetweenAPoleAndTheEquator"} + EntryDef{"misc-numberOfParallelsBetweenAPoleAndTheEquator"} .withAccessor([](auto&& v) { return &v.numberOfParallelsBetweenAPoleAndTheEquator; }); constexpr auto LatitudeOfFirstGridPointInDegrees = - EntryDef{"latitudeOfFirstGridPointInDegrees"} + EntryDef{"misc-latitudeOfFirstGridPointInDegrees"} .withAccessor([](auto&& v) { return &v.latitudeOfFirstGridPointInDegrees; }); constexpr auto LongitudeOfFirstGridPointInDegrees = - EntryDef{"longitudeOfFirstGridPointInDegrees"} + EntryDef{"misc-longitudeOfFirstGridPointInDegrees"} .withAccessor([](auto&& v) { return &v.longitudeOfFirstGridPointInDegrees; }); constexpr auto LatitudeOfLastGridPointInDegrees = - EntryDef{"latitudeOfLastGridPointInDegrees"} + EntryDef{"misc-latitudeOfLastGridPointInDegrees"} .withAccessor([](auto&& v) { return &v.latitudeOfLastGridPointInDegrees; }); constexpr auto LongitudeOfLastGridPointInDegrees = - EntryDef{"longitudeOfLastGridPointInDegrees"} + EntryDef{"misc-longitudeOfLastGridPointInDegrees"} .withAccessor([](auto&& v) { return &v.longitudeOfLastGridPointInDegrees; }); constexpr auto IDirectionIncrementInDegrees = - EntryDef{"iDirectionIncrementInDegrees"} + EntryDef{"misc-iDirectionIncrementInDegrees"} .withAccessor([](auto&& v) { return &v.iDirectionIncrementInDegrees; }); constexpr auto JDirectionIncrementInDegrees = - EntryDef{"jDirectionIncrementInDegrees"} + EntryDef{"misc-jDirectionIncrementInDegrees"} .withAccessor([](auto&& v) { return &v.jDirectionIncrementInDegrees; }); constexpr auto Pl = - EntryDef>{"pl"} + EntryDef>{"misc-pl"} .tagOptional() .withAccessor([](auto&& v) { return &v.pl; }); - - + + // Section 3 - LL constexpr auto NumberOfPointsAlongAParallel = - EntryDef{"numberOfPointsAlongAParallel"} + EntryDef{"misc-numberOfPointsAlongAParallel"} .tagOptional() .withAccessor([](auto&& v) { return &v.numberOfPointsAlongAParallel; }); // Section 3 - SH -constexpr auto PentagonalResolutionParameterJ = - EntryDef{"pentagonalResolutionParameterJ"}.withAccessor( +constexpr auto PentagonalResolutionParameterJ = + EntryDef{"misc-pentagonalResolutionParameterJ"}.withAccessor( [](auto&& v) { return &v.pentagonalResolutionParameterJ; }); -constexpr auto PentagonalResolutionParameterK = EntryDef{"pentagonalResolutionParameterK"}.withAccessor( +constexpr auto PentagonalResolutionParameterK = EntryDef{"misc-pentagonalResolutionParameterK"}.withAccessor( [](auto&& v) { return &v.pentagonalResolutionParameterK; }); -constexpr auto PentagonalResolutionParameterM = EntryDef{"pentagonalResolutionParameterM"}.withAccessor( +constexpr auto PentagonalResolutionParameterM = EntryDef{"misc-pentagonalResolutionParameterM"}.withAccessor( [](auto&& v) { return &v.pentagonalResolutionParameterM; }); // Section 3 - HEALpix constexpr auto NSide = - EntryDef{"nside"} + EntryDef{"misc-nside"} .withAccessor([](auto&& v) { return &v.nside; }); constexpr auto OrderingConvention = - EntryDef{"orderingConvention"} + EntryDef{"misc-orderingConvention"} .tagOptional() .withAccessor([](auto&& v) { return &v.orderingConvention; }); // Section4 constexpr auto ProductDefinitionTemplateNumber = - EntryDef{"productDefinitionTemplateNumber"} + EntryDef{"misc-productDefinitionTemplateNumber"} .tagOptional() - .withAccessor([](auto&& v) { return &v.productDefinitionTemplateNumber; }); + .withAccessor([](auto&& v) { return &v.productDefinitionTemplateNumber; }); constexpr auto GeneratingProcessIdentifier = - EntryDef{"generatingProcessIdentifier"} + EntryDef{"misc-generatingProcessIdentifier"} .tagOptional() .withAccessor([](auto&& v) { return &v.generatingProcessIdentifier; }); constexpr auto TypeOfLevelEntry = - EntryDef{"typeOfLevel"} + EntryDef{"misc-typeOfLevel"} .tagOptional() .withAccessor([](auto&& v) { return &v.typeOfLevel; }); constexpr auto TypeOfStatisticalProcessingEntry = - EntryDef{"typeOfStatisticalProcessing"} + EntryDef{"misc-typeOfStatisticalProcessing"} .tagOptional() - .withAccessor([](auto&& v) { return &v.typeOfStatisticalProcessing; }); + .withAccessor([](auto&& v) { return &v.typeOfStatisticalProcessing; }); constexpr auto ScaleFactorOfCentralWaveNumber = - EntryDef{"scaleFactorOfCentralWaveNumber"} + EntryDef{"misc-scaleFactorOfCentralWaveNumber"} .tagOptional() .withAccessor([](auto&& v) { return &v.scaleFactorOfCentralWaveNumber; }); constexpr auto ScaledValueOfCentralWaveNumber = - EntryDef{"scaledValueOfCentralWaveNumber"} + EntryDef{"misc-scaledValueOfCentralWaveNumber"} .tagOptional() .withAccessor([](auto&& v) { return &v.scaledValueOfCentralWaveNumber; }); // Ensemble constexpr auto TypeOfEnsembleForecast = - EntryDef{"typeOfEnsembleForecast"} + EntryDef{"misc-typeOfEnsembleForecast"} .tagOptional() .withAccessor([](auto&& v) { return &v.typeOfEnsembleForecast; }); constexpr auto NumberOfForecastsInEnsemble = - EntryDef{"numberOfForecastsInEnsemble"} + EntryDef{"misc-numberOfForecastsInEnsemble"} .tagOptional() .withAccessor([](auto&& v) { return &v.numberOfForecastsInEnsemble; }); -// Satellite +// Satellite constexpr auto SatelliteSeries = - EntryDef{"satelliteSeries"} + EntryDef{"misc-satelliteSeries"} .tagOptional() .withAccessor([](auto&& v) { return &v.satelliteSeries; }); // Horizontal Keys constexpr auto PressureUnits - = EntryDef{"pressureUnits"} + = EntryDef{"misc-pressureUnits"} .tagOptional() .withAccessor([](auto&& v) { return &v.pressureUnits; }); constexpr auto TypeOfFirstFixedSurface - = EntryDef{"typeOfFirstFixedSurface"} + = EntryDef{"misc-typeOfFirstFixedSurface"} .tagOptional() .withAccessor([](auto&& v) { return &v.typeOfFirstFixedSurface; }); constexpr auto TypeOfSecondFixedSurface - = EntryDef{"typeOfSecondFixedSurface"} + = EntryDef{"misc-typeOfSecondFixedSurface"} .tagOptional() .withAccessor([](auto&& v) { return &v.typeOfSecondFixedSurface; }); constexpr auto ScaledValueOfFirstFixedSurface - = EntryDef{"scaledValueOfFirstFixedSurface"} + = EntryDef{"misc-scaledValueOfFirstFixedSurface"} .tagOptional() .withAccessor([](auto&& v) { return &v.scaledValueOfFirstFixedSurface; }); constexpr auto ScaledValueOfSecondFixedSurface - = EntryDef{"scaledValueOfSecondFixedSurface"} + = EntryDef{"misc-scaledValueOfSecondFixedSurface"} .tagOptional() .withAccessor([](auto&& v) { return &v.scaledValueOfSecondFixedSurface; }); constexpr auto ScaleFactorOfFirstFixedSurface - = EntryDef{"scaleFactorOfFirstFixedSurface"} + = EntryDef{"misc-scaleFactorOfFirstFixedSurface"} .tagOptional() .withAccessor([](auto&& v) { return &v.scaleFactorOfFirstFixedSurface; }); constexpr auto ScaleFactorOfSecondFixedSurface - = EntryDef{"scaleFactorOfSecondFixedSurface"} + = EntryDef{"misc-scaleFactorOfSecondFixedSurface"} .tagOptional() .withAccessor([](auto&& v) { return &v.scaleFactorOfSecondFixedSurface; }); @@ -246,21 +246,22 @@ struct HorizontalGribKeys { // Data Repres constexpr auto BitmapPresent = - EntryDef{"bitmapPresent"} - .tagOptional() + EntryDef{"misc-bitmapPresent"} + .withDefault(false) .withAccessor([](auto&& v) { return &v.bitmapPresent; }); constexpr auto MissingValue = - EntryDef{"missingValue"} + EntryDef{"misc-missingValue"} .tagOptional() .withAccessor([](auto&& v) { return &v.missingValue; }); + constexpr auto BitsPerValue = - EntryDef{"bitsPerValue"} + EntryDef{"misc-bitsPerValue"} .tagOptional() .withAccessor([](auto&& v) { return &v.bitsPerValue; }); constexpr auto LaplacianOperator = - EntryDef{"laplacianOperator"} + EntryDef{"misc-laplacianOperator"} .tagOptional() .withAccessor([](auto&& v) { return &v.laplacianOperator; }); @@ -269,12 +270,12 @@ constexpr auto LaplacianOperator = constexpr auto PVPresent = - EntryDef{"PVPresent"} + EntryDef{"misc-PVPresent"} .tagOptional() .withAccessor([](auto&& v) { return &v.pvPresent; }); constexpr auto Pv = - EntryDef>{"pv"} + EntryDef>{"misc-pv"} .tagOptional() .withAccessor([](auto&& v) { return &v.pv; }); diff --git a/src/multio/datamod/MarsMiscGeo.h b/src/multio/datamod/MarsMiscGeo.h index 7ef05d74e..99f7c195c 100644 --- a/src/multio/datamod/MarsMiscGeo.h +++ b/src/multio/datamod/MarsMiscGeo.h @@ -252,22 +252,22 @@ struct FullMarsRecord : ComposedRecord{"initialStep"} // + EntryDef{"misc-initialStep"} // .withDefault(0) .withAccessor([](auto&& v) { return &v.initialStep; }); constexpr auto TimeIncrementInSeconds = // - EntryDef{"timeIncrementInSeconds"} // + EntryDef{"misc-timeIncrementInSeconds"} // .withDefault(3600) .withAccessor([](auto&& v) { return &v.timeIncrementInSeconds; }); constexpr auto LengthOfTimeWindow = // - EntryDef{"lengthOfTimeWindow"} // + EntryDef{"misc-lengthOfTimeWindow"} // .tagOptional() .withAccessor([](auto&& v) { return &v.lengthOfTimeWindow; }); constexpr auto LengthOfTimeWindowInSeconds = // - EntryDef{"lengthOfTimeWindowInSeconds"} // + EntryDef{"misc-lengthOfTimeWindowInSeconds"} // .tagOptional() .withAccessor([](auto&& v) { return &v.lengthOfTimeWindowInSeconds; }); @@ -288,22 +288,22 @@ constexpr auto LengthOfTimeWindowInSeconds = // // Pv defined in GribKeys constexpr auto ScaleFactorOfWaveDirections = // - EntryDef{"scaleFactorOfWaveDirections"} // + EntryDef{"misc-scaleFactorOfWaveDirections"} // .tagOptional() .withAccessor([](auto&& v) { return &v.scaleFactorOfWaveDirections; }); constexpr auto ScaleFactorOfWaveFrequencies = // - EntryDef{"scaleFactorOfWaveFrequencies"} // + EntryDef{"misc-scaleFactorOfWaveFrequencies"} // .tagOptional() .withAccessor([](auto&& v) { return &v.scaleFactorOfWaveFrequencies; }); constexpr auto WaveDirections = // - EntryDef>{"waveDirections"} // + EntryDef>{"misc-waveDirections"} // .tagOptional() .withAccessor([](auto&& v) { return &v.waveDirections; }); constexpr auto WaveFrequencies = // - EntryDef>{"waveFrequencies"} // + EntryDef>{"misc-waveFrequencies"} // .tagOptional() .withAccessor([](auto&& v) { return &v.waveFrequencies; }); diff --git a/src/multio/datamod/core/EntryDumper.h b/src/multio/datamod/core/EntryDumper.h index af62373b4..81e279c72 100644 --- a/src/multio/datamod/core/EntryDumper.h +++ b/src/multio/datamod/core/EntryDumper.h @@ -106,6 +106,34 @@ Container dumpRecord(RecordType&& rec, const DumpOptions& opts = DumpOptions{}) } +//----------------------------------------------------------------------------- +// Dumping a record with the record-name prefix stripped from keys +//----------------------------------------------------------------------------- + +template >, bool> = true> +void dumpUnscopedRecord(RecordType&& rec, Container& cont, const DumpOptions& opts = DumpOptions{}) { + const std::string prefix = std::string(RecordName_v>) + "-"; + std::apply( + [&](const auto&... entryDef) { + auto stripPrefix = [&](const auto& ed) -> std::string { + std::string k{ed.key()}; + return (k.compare(0, prefix.size(), prefix) == 0) ? k.substr(prefix.size()) : k; + }; + (dumpEntry(scopedEntryDef(entryDef, stripPrefix(entryDef)), + entryDef.get(std::forward(rec)), cont, opts), ...); + }, + recordEntries(rec)); +} + +template , bool> = true> +Container dumpUnscopedRecord(RecordType&& rec, const DumpOptions& opts = DumpOptions{}) { + Container ret; + dumpUnscopedRecord(std::forward(rec), ret, opts); + return ret; +} + + //----------------------------------------------------------------------------- } // namespace multio::datamod diff --git a/tests/multio/action/scale/Scale.cc b/tests/multio/action/scale/Scale.cc index c4b18f22f..8a2a4d54d 100644 --- a/tests/multio/action/scale/Scale.cc +++ b/tests/multio/action/scale/Scale.cc @@ -165,7 +165,8 @@ CASE("local-to-wmo mapping with missing value") { auto md = Metadata({ {"param", 228}, {"misc-precision", "double"}, - {"missingValue", 999.0} + {"misc-bitmapPresent", true}, + {"misc-missingValue", 999.0} }); std::vector values = {0.0, 999.0, 1.0}; auto pl = eckit::Buffer(values.data(), values.size() * sizeof(double)); diff --git a/tests/multio/action/statistics-mtg2/ParamMapping.cc b/tests/multio/action/statistics-mtg2/ParamMapping.cc index f9ac1fe73..d40217db3 100644 --- a/tests/multio/action/statistics-mtg2/ParamMapping.cc +++ b/tests/multio/action/statistics-mtg2/ParamMapping.cc @@ -94,8 +94,8 @@ std::int64_t testParameterMapping(std::int64_t param, std::string op) { for (int64_t step = 0; step <= 1; ++step) { auto md = Metadata({ {"param", param}, - {"levtype", "none"}, - {"grid", "none"}, + {"levtype", "sfc"}, + {"grid", "O80"}, {"date", 20200721}, {"time", 0000}, {"step", step},