diff --git a/.github/workflows/rust-dev.yml b/.github/workflows/rust-dev.yml deleted file mode 100644 index 0b45977..0000000 --- a/.github/workflows/rust-dev.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: cargodev - -on: - push: - branches-ignore: - - main - pull_request: - branches: - - "**" - -env: - CARGO_TERM_COLOR: always - -jobs: - pre_job: - continue-on-error: true - runs-on: ubuntu-latest - outputs: - should_skip: ${{ steps.skip_check.outputs.should_skip }} - steps: - - id: skip_check - uses: fkirc/skip-duplicate-actions@v5 - with: - # All of these options are optional, so you can remove them if you are happy with the defaults - concurrent_skipping: 'same_content_newer' - skip_after_successful_duplicate: 'true' - paths_ignore: '["**/README.md"]' - do_not_skip: '["pull_request", "workflow_dispatch", "schedule"]' - - build-linux: - - name: Build on Linux - needs: pre_job - if: needs.pre_job.outputs.should_skip != 'true' - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Prepare environment - run: | - sudo apt-get update - sudo apt-get install clang - sudo apt-get install libclang1 - sudo apt-get install libeccodes-dev - rustup update stable - cargo clean - - name: Build release - run: | - cargo build --release --features "experimental_index, message_ndarray" - - name: Check with clippy - run: | - cargo clippy --features "experimental_index, message_ndarray" -- -D warnings - - name: Test with cargo - run: | - cargo clean - RUST_BACKTRACE=full cargo test --features "experimental_index, message_ndarray" -- --include-ignored - - build-macos: - - name: Build on MacOS - needs: pre_job - if: needs.pre_job.outputs.should_skip != 'true' - runs-on: macos-latest - - steps: - - uses: actions/checkout@v4 - - name: Prepare environment - run: | - brew install eccodes - rustup update stable - cargo clean - - name: Build release - run: | - cargo build --release --features "experimental_index, message_ndarray" - - name: Check with clippy - run: | - cargo clippy --features "experimental_index, message_ndarray" -- -D warnings - - name: Test with cargo - run: | - cargo clean - RUST_BACKTRACE=full cargo test --features "experimental_index, message_ndarray" -- --include-ignored diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 368ea84..9da914e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,6 +1,6 @@ name: cargo -on: +on: push: branches: - main @@ -22,72 +22,68 @@ jobs: uses: fkirc/skip-duplicate-actions@v5 with: # All of these options are optional, so you can remove them if you are happy with the defaults - concurrent_skipping: 'same_content_newer' - skip_after_successful_duplicate: 'true' + concurrent_skipping: "same_content_newer" + skip_after_successful_duplicate: "true" paths_ignore: '["**/README.md"]' do_not_skip: '["pull_request", "workflow_dispatch", "schedule"]' build: - name: Build on Ubuntu needs: pre_job if: needs.pre_job.outputs.should_skip != 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - name: Prepare environment - run: | - sudo apt-get update - sudo apt-get install clang - sudo apt-get install libclang1 - sudo apt-get install libeccodes-dev - rustup update stable - cargo install cargo-criterion - cargo clean - - name: Build with cargo - run: | - cargo build --release --features "experimental_index, message_ndarray" - cargo clean - - name: Test with cargo - run: | - cargo test --no-default-features - cargo test --features "message_ndarray" - cargo test --features "experimental_index" - cargo test --features "experimental_index, message_ndarray" - cargo clean - - name: Benchmark with criterion - run: | - cargo criterion - cargo clean + - uses: actions/checkout@v4 + - name: Prepare environment + run: | + sudo apt-get update + sudo apt-get install clang + sudo apt-get install libclang1 + sudo apt-get install libeccodes-dev + rustup update stable + cargo install cargo-criterion + cargo clean + - name: Check release build + run: | + cargo build --release --features "ndarray" + - name: Check with clippy + run: | + cargo clippy --features "ndarray" -- -D warnings + - name: Check tests + run: | + cargo test --no-default-features + cargo test --features "ndarray" + - name: Benchmark with criterion + run: | + cargo criterion + cargo clean build-macos: - name: Build on MacOS needs: pre_job if: needs.pre_job.outputs.should_skip != 'true' runs-on: macos-latest steps: - - uses: actions/checkout@v4 - - name: Prepare environment - run: | - brew install eccodes - rustup update stable - cargo install cargo-criterion - cargo clean - - name: Build with cargo - run: | - cargo build --release --features "experimental_index, message_ndarray" - cargo clean - - name: Test with cargo - run: | - cargo test --no-default-features - cargo test --features "message_ndarray" - cargo test --features "experimental_index" - cargo test --features "experimental_index, message_ndarray" - cargo clean - - name: Benchmark with criterion - run: | - cargo criterion - cargo clean + - uses: actions/checkout@v4 + - name: Prepare environment + run: | + brew install eccodes + rustup update stable + cargo install cargo-criterion + cargo clean + - name: Check release build + run: | + cargo build --release --features --features "ndarray" + - name: Check with clippy + run: | + cargo clippy --features "ndarray" -- -D warnings + - name: Check tests + run: | + cargo test --no-default-features + cargo test --features "ndarray" + - name: Benchmark with criterion + run: | + cargo criterion + cargo clean diff --git a/Cargo.toml b/Cargo.toml index bc8c758..f7874d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "eccodes" description = "Unofficial high-level Rust bindings of the latest ecCodes release" repository = "https://github.com/ScaleWeather/eccodes" -version = "0.13.4" +version = "0.14.0-alpha.1" readme = "README.md" authors = ["Jakub Lewandowski "] keywords = ["eccodes", "grib", "bufr", "meteorology", "weather"] @@ -14,40 +14,41 @@ categories = [ "science", ] license = "Apache-2.0" -edition = "2021" +edition = "2024" exclude = [".github/*", ".vscode/*", ".idea/*", "data/*"] -rust-version = "1.82.0" +rust-version = "1.85.1" [dependencies] -eccodes-sys = { version = "0.6.0", default-features = false } +eccodes-sys = { version = "0.7.0", default-features = false } libc = { version = "0.2", default-features = false } thiserror = { version = "2.0", default-features = false } -log = { version = "0.4", default-features = false } errno = { version = "0.3", default-features = false } num-derive = { version = "0.4", default-features = false } num-traits = { version = "0.2", default-features = false } fallible-iterator = { version = "0.3", default-features = false } -fallible-streaming-iterator = { version = "0.1.9", default-features = false } -ndarray = { version = "0.16", default-features = false, optional = true, features = [ +ndarray = { version = "0.17", default-features = false, optional = true, features = [ "std", ] } +tracing = { version = "0.1", default-features = false, features = [ + "std", + "attributes", + "log", +] } [dev-dependencies] -reqwest = { version = "0.12", features = ["rustls-tls"] } -criterion = "0.5" -testing_logger = "0.1" -rand = "0.8" +reqwest = { version = "0.13", features = ["rustls"] } +criterion = "0.8" +rand = "0.9" anyhow = { version = "1.0", features = ["backtrace"] } float-cmp = "0.10" [features] -default = ["message_ndarray", "experimental_index"] +default = ["ndarray"] docs = ["eccodes-sys/docs"] -experimental_index = [] -message_ndarray = ["dep:ndarray"] +ndarray = ["dep:ndarray"] [package.metadata.docs.rs] -features = ["docs", "experimental_index", "message_ndarray"] +features = ["docs", "ndarray"] [[bench]] name = "main" diff --git a/README.md b/README.md index 3d0e479..a1d7ce2 100644 --- a/README.md +++ b/README.md @@ -60,58 +60,72 @@ export LD_LIBRARY_PATH=/lib ### Working with GRIB files -To access a GRIB file you need to create `CodesHandle` with one of provided constructors. +To access a GRIB file you need to create `CodesFile` with one of the provided constructors. -GRIB files consist of messages which represent data fields at specific time and level. -Messages are represented by the `KeyedMessage` structure. +ecCodes represents GRIB files as a set of separate messages, each containing data fields at specific time and level. +Messages are represented here by a generic `CodesMessage` structure, but you shouldn't use it directly. +Instead use `RefMessage`, `ArcMessage` or `BufMessage` to operations - check the docs for more information when to use each. -`CodesHandle` implements `FallibleStreamingIterator` -which allows you to iterate over messages in the file. The iterator returns `&KeyedMessage` which valid until next iteration. -`KeyedMessage` implements several methods to access the data as needed, most of those can be called directly on `&KeyedMessage`. -You can also use `try_clone()` to clone the message and prolong its lifetime. +To obtain `CodesMessage` from `CodesFile` you need to create an instance of `RefMessageIter` or `ArcMessageIter` using +`CodesFile::ref_message_iter()` or `CodesFile::arc_message_iter()`. +Those structures implement `FallibleIterator`, please check its documentation if you are not familiar with it. -Data defining and contained by `KeyedMessage` is represented by `Key`s. -You can read them directly with `read_key()`, use `KeysIterator` -to iterate over them or use `CodesNearest` to get the values of four nearest gridpoints for given coordinates. +`CodesMessage` implements several methods to access the data as needed, most of those can be called directly. +Almost all methods can be called on any `CodesMessage`, except for `KeyWrite` operations, which can be called only on `BufMessage` +to avoid confusion if written keys are save to file or not. -You can also modify the message with `write_key()` and write it to a new file with `write_to_file()`. +Data contained by `CodesMessage` is represented as *keys* (like in dictionary). +Keys can be read with static types using `read_key()` or with dynamic types +using `read_key_dynamic()`. +To discover what keys are present in a message use `KeysIterator`. -#### Example +With `ndarray` feature (enabled by default) you can also read `CodesMessage` into `ndarray` using `to_ndarray()` +and `to_lons_lats_values()`. -```rust -// We are reading the mean sea level pressure for 4 gridpoints -// nearest to Reykjavik (64.13N, -21.89E) for 1st June 2021 00:00 UTC -// from ERA5 Climate Reanalysis +You can use `CodesNearest` to get the data values of four nearest gridpoints for given coordinates. + +To modify keys within the message use `write_key_unchecked()`. +To save that modified message use `write_to_file()`. -use eccodes::{ProductKind, CodesHandle, KeyType}; -use eccodes::FallibleStreamingIterator; +#### Example 1 - Reading GRIB file -// Open the GRIB file and create the CodesHandle -let file_path = Path::new("./data/iceland.grib"); -let product_kind = ProductKind::GRIB; -let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; +In this example we are reading mean sea level pressure for 4 gridpoints nearest to +Reykjavik (64.13N, -21.89E) for 1st June 2021 00:00 UTC from ERA5 Climate Reanalysis. + +```rust +use eccodes::{CodesFile, FallibleIterator, KeyRead, ProductKind}; +// Open the GRIB file and create the CodesFile +let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; // Use iterator to find a message with shortName "msl" and typeOfLevel "surface" // We can use while let or for_each() to iterate over the messages -while let Some(msg) = handle.next()? { - if msg.read_key("shortName")?.value == KeyType::Str("msl".to_string()) - && msg.read_key("typeOfLevel")?.value == KeyType::Str("surface".to_string()) { - +while let Some(msg) = handle.ref_message_iter().next()? { + // We need to specify the type of key we read + let short_name: String = msg.read_key("shortName")?; + let type_of_level: String = msg.read_key("typeOfLevel")?; + if short_name == "msl" && type_of_level == "surface" { // Create CodesNearest for given message - let nearest_gridpoints = msg.codes_nearest()? + let nearest_gridpoints = msg + .codes_nearest()? // Find the nearest gridpoints to Reykjavik .find_nearest(64.13, -21.89)?; // Print value and distance of the nearest gridpoint - println!("value: {}, distance: {}", - nearest_gridpoints[3].value, - nearest_gridpoints[3].distance); + println!( + "value: {}, distance: {}", + nearest_gridpoints[3].value, nearest_gridpoints[3].distance + ); } } ``` +### Multithreading + +Since v0.14 messages inside a file can be read as `ArcMessage` which +allows to move and share the message across threads. + ### Writing GRIB files -The crate provides a basic support for setting `KeyedMessage` keys +The crate provides a basic support for setting `CodesMessage` keys and writing GRIB files. The easiest (and safest) way to create a new custom message is to copy existing one from other GRIB file, modify the keys and write to new file. @@ -125,11 +139,11 @@ This crate aims to return error whenever possible, even if the error is caused b As ecCodes is often used in scientific applications with long and extensive jobs, this allows the user to handle the error in the way that suits them best and not risk crashes. -All error descriptions are provided in the `errors` module. -Destructors, which cannot panic, report errors through the `log` crate. +All error descriptions are provided in the [`errors`] module. +Destructors, which cannot panic, report errors through `tracing` and `log` crate. None of the functions in this crate explicitly panics. -However, users should not that dependencies might panic in some edge cases. +However, users should be aware that dependencies (eg. `ndarray`) might panic in some edge cases. ## Safety @@ -139,19 +153,17 @@ Moreover, pointers are always checked for null before being dereferenced. That said, neither main developer nor contributors have expertise in unsafe Rust and bugs might have slipped through. We are also not responsible for bugs in the ecCodes library. -If you find a bug or have a suggestion, feel free to discuss it on Github. +**For critical applications always perform extensive testing before using this crate in production.** -## Features +If you find a bug or have a suggestion, feel free to discuss it on Github. -- `message_ndarray` - enables support for converting `KeyedMessage` to `ndarray::Array`. -This feature is enabled by default. It is currently tested only with simple lat-lon grids. +## Feature Flags -- `experimental_index` - enables support for creating and using index files for GRIB files. -This feature experimental and disabled by default. If you want to use it, please read -the information provided in `codes_index` documentation. +- `ndarray` - enables support for converting [`CodesMessage`](codes_message::CodesMessage) to [`ndarray::Array`]. + This feature is enabled by default. It is currently tested only with simple lat-lon grids. - `docs` - builds the crate without linking ecCodes, particularly useful when building the documentation -on [docs.rs](https://docs.rs/). For more details check documentation of [eccodes-sys](https://crates.io/crates/eccodes-sys). + on [docs.rs](https://docs.rs/). For more details check documentation of [eccodes-sys](https://crates.io/crates/eccodes-sys). To build your own crate with this crate as dependency on docs.rs without linking ecCodes add following lines to your `Cargo.toml` diff --git a/benches/main.rs b/benches/main.rs index 0b99ed9..9fc90ff 100644 --- a/benches/main.rs +++ b/benches/main.rs @@ -1,17 +1,17 @@ -use eccodes::FallibleStreamingIterator; +use criterion::{Criterion, criterion_group, criterion_main}; +use eccodes::FallibleIterator; +use eccodes::codes_file::{CodesFile, ProductKind}; +use std::hint::black_box; use std::path::Path; -use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use eccodes::codes_handle::{CodesHandle, ProductKind}; - pub fn key_reading(c: &mut Criterion) { //prepare the variables for benchmark let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); + let mut handle = CodesFile::new_from_file(file_path, product_kind).unwrap(); - let msg = handle.next().unwrap().unwrap(); + let msg = handle.ref_message_iter().next().unwrap().unwrap(); c.bench_function("long reading", |b| { b.iter(|| msg.read_key_dynamic(black_box("dataDate")).unwrap()) diff --git a/data/iceland-levels-bad-path.grib.idx b/data/iceland-levels-bad-path.grib.idx deleted file mode 100644 index cbef1ea..0000000 Binary files a/data/iceland-levels-bad-path.grib.idx and /dev/null differ diff --git a/data/iceland-surface.grib.idx b/data/iceland-surface.grib.idx deleted file mode 100644 index 2aae480..0000000 Binary files a/data/iceland-surface.grib.idx and /dev/null differ diff --git a/src/codes_file/iterator.rs b/src/codes_file/iterator.rs new file mode 100644 index 0000000..e04c39c --- /dev/null +++ b/src/codes_file/iterator.rs @@ -0,0 +1,398 @@ +use fallible_iterator::FallibleIterator; + +use crate::{ArcMessage, CodesFile, RefMessage, errors::CodesError}; +use std::fmt::Debug; +use std::sync::{Arc, Mutex}; + +/// Iterator over messages in `CodesFile` which returns [`RefMessage`] with lifetime tied to the `CodesFile`. +/// +/// This structure implements [`FallibleIterator`] which allows you to iterate over messages in the file. +/// The iterator returns [`RefMessage`] with lifetime tied to the lifetime of `CodesFile`, that is `RefMessage` +/// cannot outlive the `CodesFile` it was generated from. +/// +/// Creating this iter requires `CodesFile` to be mutable. +/// +/// If you need to share the message(s) across threads use [`ArcMessageIter`]. +/// +/// If you need a longer lifetime or want to modify the message, use [`try_clone()`](RefMessage::try_clone). +/// +/// ## Example +/// +/// ``` +/// use eccodes::{CodesFile, ProductKind, FallibleIterator}; +/// # +/// # fn main() -> anyhow::Result<()> { +/// // Open the file +/// let mut file = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; +/// +/// // Create RefMessageIter +/// let mut msg_iter = file.ref_message_iter(); +/// +/// // Now we can access the first message with next() +/// // Note that FallibleIterator must be in scope +/// let _msg = msg_iter.next()?; +/// +/// // Note we cannot drop the file here, because we first need to drop the message +/// // drop(file); +/// # Ok(()) +/// # } +/// ``` +#[derive(Debug)] +pub struct RefMessageIter<'a, D: Debug> { + codes_file: &'a mut CodesFile, +} + +impl CodesFile { + /// Generates [`RefMessageIter`] that allows to access messages as references to their parent file. + pub const fn ref_message_iter(&mut self) -> RefMessageIter<'_, D> { + RefMessageIter { codes_file: self } + } +} + +impl<'ch, D: Debug> FallibleIterator for RefMessageIter<'ch, D> { + type Item = RefMessage<'ch>; + type Error = CodesError; + + /// # Errors + /// + /// The method will return [`CodesInternal`](crate::errors::CodesInternal) + /// when internal ecCodes function returns non-zero code. + fn next(&mut self) -> Result, Self::Error> { + let eccodes_handle = self.codes_file.generate_codes_handle()?; + + if eccodes_handle.is_null() { + Ok(None) + } else { + Ok(Some(RefMessage::new(eccodes_handle))) + } + } +} + +/// Iterator over messages in `CodesFile` which returns [`ArcMessage`] which can be shared across threads. +/// +/// `ArcMessage` implements `Send + Sync` so it can be both moved to thread (for example, to read messages in parallel) +/// or shared across threads (when wrapped in [`Arc`]). +/// +/// This structure implements [`FallibleIterator`] - see the documentation for information how that differs from a standard `Iter`. +/// +/// Creating this iter does not require `CodesFile` to be mutable, because it takes ownership over the `CodesFile`. +/// +/// If you don't need to share the message, use [`RefMessageIter`] to avoid the performance overhead of [`Arc`]. +/// +/// If you want to modify the message, use [`try_clone()`](RefMessage::try_clone). +/// +/// ## Example +/// +/// See the second example in the main crate description for example usage of `ArcMessageIter`. +#[derive(Debug)] +pub struct ArcMessageIter { + codes_file: Arc>>, +} +impl CodesFile { + /// Generates [`ArcMessageIter`] that allows to access messages that can be shared across threads. + pub fn arc_message_iter(self) -> ArcMessageIter { + ArcMessageIter { + codes_file: Arc::new(Mutex::new(self)), + } + } +} + +impl FallibleIterator for ArcMessageIter { + type Item = ArcMessage; + + type Error = CodesError; + + /// # Errors + /// + /// The method will return [`CodesInternal`](crate::errors::CodesInternal) + /// when internal ecCodes function returns non-zero code. + /// + /// # Panics + /// + /// This method internally uses a Mutex to access `CodesFile`, which can panic when poisoned, + /// but thers is no path in which you can get to the state of poisoned mutex, while still able to access this method. + fn next(&mut self) -> Result, Self::Error> { + let eccodes_handle = self + .codes_file + .lock() + // This mutex can be poisoned only when thread that holds ArcMessageIter panics, which would make using the mutex impossible") + .expect("The mutex inside ArcMessageIter got poisoned") + .generate_codes_handle()?; + + if eccodes_handle.is_null() { + Ok(None) + } else { + Ok(Some(ArcMessage::new(eccodes_handle, &self.codes_file))) + } + } +} + +#[cfg(test)] +mod tests { + use crate::{ + FallibleIterator, + codes_file::{CodesFile, ProductKind}, + codes_message::DynamicKeyType, + }; + use anyhow::{Context, Ok, Result}; + use float_cmp::assert_approx_eq; + use std::{ + path::Path, + sync::{Arc, Barrier}, + }; + + #[test] + fn iterator_lifetimes() -> Result<()> { + let file_path = Path::new("./data/iceland-levels.grib"); + let product_kind = ProductKind::GRIB; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + + let msg1 = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + let key1 = msg1.read_key_dynamic("typeOfLevel")?; + drop(msg1); + + let msg2 = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + let key2 = msg2.read_key_dynamic("typeOfLevel")?; + drop(msg2); + + let msg3 = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + let key3 = msg3.read_key_dynamic("typeOfLevel")?; + drop(msg3); + + assert_eq!(key1, DynamicKeyType::Str("isobaricInhPa".to_string())); + assert_eq!(key2, DynamicKeyType::Str("isobaricInhPa".to_string())); + assert_eq!(key3, DynamicKeyType::Str("isobaricInhPa".to_string())); + + Ok(()) + } + + #[test] + fn message_lifetime_safety() -> Result<()> { + let file_path = Path::new("./data/iceland-levels.grib"); + let product_kind = ProductKind::GRIB; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let msg2; + let msg4; + + { + let mut mgen = handle.ref_message_iter(); + + let msg1 = mgen.next()?.context("Message not some")?; + drop(msg1); + msg2 = mgen.next()?.context("Message not some")?; + let msg3 = mgen.next()?.context("Message not some")?; + drop(msg3); + msg4 = mgen.next()?.context("Message not some")?; + let msg5 = mgen.next()?.context("Message not some")?; + drop(msg5); + } + // drop(handle); <- this is not allowed + + let key2 = msg2.read_key_dynamic("typeOfLevel")?; + let key4 = msg4.read_key_dynamic("typeOfLevel")?; + + assert_eq!(key2, DynamicKeyType::Str("isobaricInhPa".to_string())); + assert_eq!(key4, DynamicKeyType::Str("isobaricInhPa".to_string())); + + Ok(()) + } + + #[test] + fn iterator_fn() -> Result<()> { + let file_path = Path::new("./data/iceland-surface.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + + while let Some(msg) = handle.ref_message_iter().next()? { + let key = msg.read_key_dynamic("shortName")?; + + match key { + DynamicKeyType::Str(_) => {} + _ => panic!("Incorrect variant of string key"), + } + } + + Ok(()) + } + + #[test] + fn iterator_collected() -> Result<()> { + let file_path = Path::new("./data/iceland-surface.grib"); + let product_kind = ProductKind::GRIB; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + + let mut handle_collected = vec![]; + + while let Some(msg) = handle.ref_message_iter().next()? { + handle_collected.push(msg.try_clone()?); + } + + for msg in handle_collected { + let key: DynamicKeyType = msg.read_key_dynamic("name")?; + match key { + DynamicKeyType::Str(_) => {} + _ => panic!("Incorrect variant of string key"), + } + } + + Ok(()) + } + + #[test] + fn iterator_return() -> Result<()> { + let file_path = Path::new("./data/iceland-surface.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + + assert!(!current_message.message_handle.is_null()); + + Ok(()) + } + + #[test] + fn iterator_beyond_none() -> Result<()> { + let file_path = Path::new("./data/iceland-surface.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let mut mgen = handle.ref_message_iter(); + + assert!(mgen.next()?.is_some()); + assert!(mgen.next()?.is_some()); + assert!(mgen.next()?.is_some()); + assert!(mgen.next()?.is_some()); + assert!(mgen.next()?.is_some()); + + assert!(mgen.next()?.is_none()); + assert!(mgen.next()?.is_none()); + assert!(mgen.next()?.is_none()); + assert!(mgen.next()?.is_none()); + + Ok(()) + } + + #[test] + fn iterator_filter() -> Result<()> { + let file_path = Path::new("./data/iceland.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + + // Use iterator to get a Keyed message with shortName "msl" and typeOfLevel "surface" + // First, filter and collect the messages to get those that we want + let mut level = vec![]; + + while let Some(msg) = handle.ref_message_iter().next()? { + if msg.read_key_dynamic("shortName")? == DynamicKeyType::Str("msl".to_string()) + && msg.read_key_dynamic("typeOfLevel")? + == DynamicKeyType::Str("surface".to_string()) + { + level.push(msg.try_clone()?); + } + } + + // Now unwrap and access the first and only element of resulting vector + // Find nearest modifies internal CodesMessage fields so we need mutable reference + let level = &level[0]; + + let short_name = level.read_key_dynamic("shortName")?; + assert_eq!(short_name, DynamicKeyType::Str("msl".into())); + + // Get the four nearest gridpoints of Reykjavik + let nearest_gridpoints = level.codes_nearest()?.find_nearest(64.13, -21.89)?; + let value = nearest_gridpoints[3].value; + let distance = nearest_gridpoints[3].distance; + + assert_approx_eq!(f64, value, 100557.9375); + assert_approx_eq!(f64, distance, 14.358879960775498); + + Ok(()) + } + + #[test] + fn thread_safety_messsage_wise() -> Result<()> { + let file_path = Path::new("./data/iceland-levels.grib"); + let product_kind = ProductKind::GRIB; + + let handle = CodesFile::new_from_file(file_path, product_kind)?; + let mut mgen = handle.arc_message_iter(); + // let _ = handle.atomic_message_generator(); <- not allowed due to ownership + + let barrier = Arc::new(Barrier::new(10)); + + let mut v = vec![]; + + for _ in 0..10 { + let msg = mgen.next()?.context("No more messages")?; + let b = barrier.clone(); + + let t = std::thread::spawn(move || { + for _ in 0..10 { + b.wait(); + for _ in 0..100 { + let _ = msg.read_key_dynamic("shortName").unwrap(); + } + } + }); + + v.push(t); + } + + for th in v { + th.join().unwrap(); + } + + Ok(()) + } + + #[test] + fn thread_safety_within_message() -> Result<()> { + let file_path = Path::new("./data/iceland-levels.grib"); + let product_kind = ProductKind::GRIB; + + let handle = CodesFile::new_from_file(file_path, product_kind)?; + let mut mgen = handle.arc_message_iter(); + let msg = Arc::new(mgen.next()?.context("No more messages")?); + + let barrier = Arc::new(Barrier::new(10)); + + let mut v = vec![]; + + for _ in 0..10 { + let msg_inner = msg.clone(); + let b = barrier.clone(); + + let t = std::thread::spawn(move || { + for _ in 0..10 { + b.wait(); + for _ in 0..100 { + let _ = msg_inner.read_key_dynamic("shortName").unwrap(); + } + } + }); + + v.push(t); + } + + for th in v { + th.join().unwrap(); + } + + Ok(()) + } +} diff --git a/src/codes_file/mod.rs b/src/codes_file/mod.rs new file mode 100644 index 0000000..2e00520 --- /dev/null +++ b/src/codes_file/mod.rs @@ -0,0 +1,289 @@ +//! Definition and constructors of `CodesFile` +//! used for accessing GRIB files + +use crate::{CodesError, intermediate_bindings::codes_handle_new_from_file, pointer_guard}; +use eccodes_sys::{ProductKind_PRODUCT_GRIB, codes_handle}; +use errno::errno; +use libc::{FILE, size_t}; +use std::{ + fmt::Debug, + fs::{File, OpenOptions}, + os::unix::prelude::AsRawFd, + path::Path, +}; +use tracing::instrument; + +pub use iterator::{ArcMessageIter, RefMessageIter}; + +mod iterator; + +/// Structure providing access to the GRIB file which takes a full ownership of the accessed file. +/// +/// It can be constructed from: +/// +/// - File path using [`new_from_file()`](CodesFile::new_from_file) +/// - From memory buffer using [`new_from_memory()`](CodesFile::new_from_memory) +/// +/// Destructor for this structure does not panic, but some internal functions may rarely fail +/// leading to bugs. Errors encountered in the destructor are logged with [`tracing`]. +/// +/// To access GRIB messages the ecCodes library uses a method similar to a C-style iterator. +/// It digests the `* FILE` multiple times, each time returning the `*mut codes_handle` +/// to a message inside the file. Therefore in this crate access messages from `CodesFile` +/// use [`ref_message_iter()`](CodesFile::ref_message_iter) or [`arc_message_iter()`](CodesFile::arc_message_iter). +#[derive(Debug)] +pub struct CodesFile { + // fields are dropped from top + pointer: *mut FILE, + product_kind: ProductKind, + _data: D, +} + +// 2024-07-26 +// Previously CodesFile had implemented Drop which called libc::fclose() +// but that closed the file descriptor and interfered with rust's fs::file destructor. +// +// To my best understanding the purpose of destructor is to clear memory and remove +// any pointers that would be dangling. +// +// The only pointer that is handed out of CodesFile is &CodesMessage, which is tied +// to CodesFile through lifetimes, so if we destruct CodesFile that pointer is first +// destructed as well. Source pointer is only used internally so we don't need to worry about it. +// +// Clearing the memory is handled on ecCodes side by CodesMessage/CodesIndex destructors +// and on rust side by destructors of data_container we own. + +impl CodesFile { + fn generate_codes_handle(&mut self) -> Result<*mut codes_handle, CodesError> { + unsafe { codes_handle_new_from_file(self.pointer, self.product_kind) } + } +} + +/// Enum representing the kind of product (file type) inside handled file. +/// Used to indicate to ecCodes how it should decode/encode messages. +#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)] +pub enum ProductKind { + #[allow(missing_docs)] + GRIB = ProductKind_PRODUCT_GRIB as isize, +} + +impl CodesFile { + /// Opens file at given [`Path`] as selected [`ProductKind`] and contructs `CodesFile`. + /// + /// ## Example + /// + /// ``` + /// # use eccodes::{ProductKind, CodesFile}; + /// # use std::path::Path; + /// # fn main() -> anyhow::Result<()> { + /// let handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; + /// # Ok(()) + /// # } + /// ``` + /// + /// The function creates [`fs::File`](std::fs::File) from provided path and utilises + /// [`fdopen()`](https://man7.org/linux/man-pages/man3/fdopen.3.html) + /// to associate [`io::RawFd`](`std::os::unix::io::RawFd`) + /// with a stream represented by [`libc::FILE`](https://docs.rs/libc/0.2.101/libc/enum.FILE.html) pointer. + /// + /// The constructor takes as argument a [`path`](Path) instead of [`File`] + /// to ensure that `fdopen()` uses the same mode as [`File`]. + /// + /// The file stream and [`File`] are safely closed when `CodesFile` is dropped. + /// + /// ## Errors + /// Returns [`CodesError::FileHandlingInterrupted`] with [`io::Error`](std::io::Error) + /// when the file cannot be opened. + /// + /// Returns [`CodesError::LibcNonZero`] with [`errno`](errno::Errno) information + /// when the stream cannot be created from the file descriptor. + /// + /// Returns [`CodesError::Internal`] with error code + /// when internal [`codes_handle`] cannot be created. + #[instrument(level = "trace")] + pub fn new_from_file + Debug>( + file_path: P, + product_kind: ProductKind, + ) -> Result { + let file = OpenOptions::new().read(true).open(file_path)?; + let file_pointer = open_with_fdopen(&file)?; + + Ok(Self { + _data: file, + pointer: file_pointer, + product_kind, + }) + } +} +impl CodesFile> { + /// Opens data in provided buffer as selected [`ProductKind`] and contructs `CodesFile`. + /// + /// ## Example + /// + /// ``` + /// # async fn run() -> anyhow::Result<()> { + /// # use eccodes::{ProductKind, CodesFile}; + /// # + /// let product_kind = ProductKind::GRIB; + /// let file_data = + /// reqwest::get("https://github.com/ScaleWeather/eccodes/blob/main/data/iceland.grib?raw=true") + /// .await? + /// .bytes() + /// .await? + /// .to_vec(); + /// + /// let handle = CodesFile::new_from_memory(file_data, product_kind)?; + /// # Ok(()) + /// # } + /// ``` + /// + /// The function associates data in memory with a stream + /// represented by [`libc::FILE`](https://docs.rs/libc/0.2.101/libc/enum.FILE.html) pointer + /// using [`fmemopen()`](https://man7.org/linux/man-pages/man3/fmemopen.3.html). + /// + /// The constructor takes full ownership of the data inside buffer, + /// which is safely dropped during the [`CodesFile`] drop. + /// + /// ## Errors + /// Returns [`CodesError::LibcNonZero`] with [`errno`](errno::Errno) information + /// when the file stream cannot be created. + /// + /// Returns [`CodesError::Internal`] with error code + /// when internal [`codes_handle`] cannot be created. + #[instrument(level = "trace")] + pub fn new_from_memory( + mut file_data: Vec, + product_kind: ProductKind, + ) -> Result { + let file_pointer = open_with_fmemopen(&mut file_data)?; + + Ok(Self { + _data: file_data, + product_kind, + pointer: file_pointer, + }) + } +} + +#[instrument(level = "trace")] +fn open_with_fdopen(file: &File) -> Result<*mut FILE, CodesError> { + let file_ptr = unsafe { libc::fdopen(file.as_raw_fd(), "r".as_ptr().cast()) }; + + if file_ptr.is_null() { + let error_val = errno(); + let error_code = error_val.0; + return Err(CodesError::LibcNonZero(error_code, error_val)); + } + + Ok(file_ptr) +} + +#[instrument(level = "trace")] +fn open_with_fmemopen(file_data: &mut [u8]) -> Result<*mut FILE, CodesError> { + let file_data_ptr = file_data.as_mut_ptr(); + pointer_guard::non_null!(file_data_ptr); + + let file_ptr; + unsafe { + file_ptr = libc::fmemopen( + file_data_ptr.cast(), + file_data.len() as size_t, + "r".as_ptr().cast(), + ); + } + + if file_ptr.is_null() { + let error_val = errno(); + let error_code = error_val.0; + return Err(CodesError::LibcNonZero(error_code, error_val)); + } + + Ok(file_ptr) +} + +#[cfg(test)] +mod tests { + use crate::codes_file::{CodesFile, ProductKind}; + use anyhow::{Context, Result}; + use eccodes_sys::ProductKind_PRODUCT_GRIB; + use fallible_iterator::FallibleIterator; + use std::{fs::File, io::Read, path::Path}; + + #[test] + fn file_constructor() -> Result<()> { + let file_path = Path::new("./data/iceland.grib"); + let product_kind = ProductKind::GRIB; + + let codes_file = CodesFile::new_from_file(file_path, product_kind)?; + + assert!(!codes_file.pointer.is_null()); + assert_eq!(codes_file.product_kind as u32, { ProductKind_PRODUCT_GRIB }); + + codes_file._data.metadata()?; + + Ok(()) + } + + #[test] + fn memory_constructor() -> Result<()> { + let product_kind = ProductKind::GRIB; + + let mut f = File::open(Path::new("./data/iceland.grib"))?; + let mut buf = Vec::new(); + f.read_to_end(&mut buf)?; + + let codes_file = CodesFile::new_from_memory(buf, product_kind)?; + assert!(!codes_file.pointer.is_null()); + assert_eq!(codes_file.product_kind as u32, { ProductKind_PRODUCT_GRIB }); + + assert!(!codes_file._data.is_empty()); + + Ok(()) + } + + #[test] + fn codes_handle_drop_file() -> Result<()> { + let file_path = Path::new("./data/iceland-surface.grib"); + let product_kind = ProductKind::GRIB; + + let handle = CodesFile::new_from_file(file_path, product_kind)?; + drop(handle); + + Ok(()) + } + + #[test] + fn codes_handle_drop_mem() -> Result<()> { + let product_kind = ProductKind::GRIB; + + let mut f = File::open(Path::new("./data/iceland.grib"))?; + let mut buf = Vec::new(); + f.read_to_end(&mut buf)?; + + let handle = CodesFile::new_from_memory(buf, product_kind)?; + drop(handle); + + Ok(()) + } + + #[test] + fn multiple_drops() -> Result<()> { + { + let file_path = Path::new("./data/iceland-surface.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + + let _ref_msg = handle.ref_message_iter().next()?.context("no message")?; + let mut clone_msg = _ref_msg.try_clone()?; + drop(_ref_msg); + let _oth_ref = handle.ref_message_iter().next()?.context("no message")?; + + let _nrst = clone_msg.codes_nearest()?; + drop(_nrst); + let _kiter = clone_msg.default_keys_iterator()?; + } + + Ok(()) + } +} diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs deleted file mode 100644 index d29b89c..0000000 --- a/src/codes_handle/iterator.rs +++ /dev/null @@ -1,180 +0,0 @@ -use crate::{codes_handle::HandleGenerator, errors::CodesError, CodesHandle, KeyedMessage}; -use fallible_streaming_iterator::FallibleStreamingIterator; -use std::fmt::Debug; - -/// # Errors -/// -/// The `advance()` and `next()` methods will return [`CodesInternal`](crate::errors::CodesInternal) -/// when internal ecCodes function returns non-zero code. -impl FallibleStreamingIterator for CodesHandle { - type Item = KeyedMessage; - - type Error = CodesError; - - fn advance(&mut self) -> Result<(), Self::Error> { - // destructor of KeyedMessage calls ecCodes - - let new_eccodes_handle = self.source.gen_codes_handle()?; - - self.current_message = if new_eccodes_handle.is_null() { - None - } else { - Some(KeyedMessage { - message_handle: new_eccodes_handle, - }) - }; - - Ok(()) - } - - fn get(&self) -> Option<&Self::Item> { - self.current_message.as_ref() - } -} - -#[cfg(test)] -mod tests { - use crate::{ - codes_handle::{CodesHandle, ProductKind}, - DynamicKeyType, - }; - use anyhow::{Context, Ok, Result}; - use fallible_streaming_iterator::FallibleStreamingIterator; - use std::path::Path; - - #[test] - fn iterator_lifetimes() -> Result<()> { - let file_path = Path::new("./data/iceland-levels.grib"); - let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - - let msg1 = handle.next()?.context("Message not some")?; - let key1 = msg1.read_key_dynamic("typeOfLevel")?; - - let msg2 = handle.next()?.context("Message not some")?; - let key2 = msg2.read_key_dynamic("typeOfLevel")?; - - let msg3 = handle.next()?.context("Message not some")?; - let key3 = msg3.read_key_dynamic("typeOfLevel")?; - - assert_eq!(key1, DynamicKeyType::Str("isobaricInhPa".to_string())); - assert_eq!(key2, DynamicKeyType::Str("isobaricInhPa".to_string())); - assert_eq!(key3, DynamicKeyType::Str("isobaricInhPa".to_string())); - - Ok(()) - } - - #[test] - fn iterator_fn() -> Result<()> { - let file_path = Path::new("./data/iceland-surface.grib"); - let product_kind = ProductKind::GRIB; - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - - while let Some(msg) = handle.next()? { - let key = msg.read_key_dynamic("shortName")?; - - match key { - DynamicKeyType::Str(_) => {} - _ => panic!("Incorrect variant of string key"), - } - } - - Ok(()) - } - - #[test] - fn iterator_collected() -> Result<()> { - let file_path = Path::new("./data/iceland-surface.grib"); - let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - - let mut handle_collected = vec![]; - - while let Some(msg) = handle.next()? { - handle_collected.push(msg.try_clone()?); - } - - for msg in handle_collected { - let key: DynamicKeyType = msg.read_key_dynamic("name")?; - match key { - DynamicKeyType::Str(_) => {} - _ => panic!("Incorrect variant of string key"), - } - } - - Ok(()) - } - - #[test] - fn iterator_return() -> Result<()> { - let file_path = Path::new("./data/iceland-surface.grib"); - let product_kind = ProductKind::GRIB; - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; - - assert!(!current_message.message_handle.is_null()); - - Ok(()) - } - - #[test] - fn iterator_beyond_none() -> Result<()> { - let file_path = Path::new("./data/iceland-surface.grib"); - let product_kind = ProductKind::GRIB; - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - - assert!(handle.next()?.is_some()); - assert!(handle.next()?.is_some()); - assert!(handle.next()?.is_some()); - assert!(handle.next()?.is_some()); - assert!(handle.next()?.is_some()); - - assert!(handle.next()?.is_none()); - assert!(handle.next()?.is_none()); - assert!(handle.next()?.is_none()); - assert!(handle.next()?.is_none()); - - Ok(()) - } - - #[test] - fn iterator_filter() -> Result<()> { - let file_path = Path::new("./data/iceland.grib"); - let product_kind = ProductKind::GRIB; - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - - // Use iterator to get a Keyed message with shortName "msl" and typeOfLevel "surface" - // First, filter and collect the messages to get those that we want - let mut level = vec![]; - - while let Some(msg) = handle.next()? { - if msg.read_key_dynamic("shortName")? == DynamicKeyType::Str("msl".to_string()) - && msg.read_key_dynamic("typeOfLevel")? - == DynamicKeyType::Str("surface".to_string()) - { - level.push(msg.try_clone()?); - } - } - - // Now unwrap and access the first and only element of resulting vector - // Find nearest modifies internal KeyedMessage fields so we need mutable reference - let level = &level[0]; - - println!("{:?}", level.read_key_dynamic("shortName")); - - // Get the four nearest gridpoints of Reykjavik - let nearest_gridpoints = level.codes_nearest()?.find_nearest(64.13, -21.89)?; - - // Print value and distance of the nearest gridpoint - println!( - "value: {}, distance: {}", - nearest_gridpoints[3].value, nearest_gridpoints[3].distance - ); - - Ok(()) - } -} diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs deleted file mode 100644 index 51cc977..0000000 --- a/src/codes_handle/mod.rs +++ /dev/null @@ -1,561 +0,0 @@ -//! Definition and constructors of `CodesHandle` -//! used for accessing GRIB files - -#[cfg(feature = "experimental_index")] -use crate::codes_index::CodesIndex; -use crate::{ - intermediate_bindings::codes_handle_new_from_file, pointer_guard, CodesError, KeyedMessage, -}; -use eccodes_sys::{codes_handle, ProductKind_PRODUCT_GRIB}; -use errno::errno; -use libc::{c_void, size_t, FILE}; -use std::{ - fmt::Debug, - fs::{File, OpenOptions}, - os::unix::prelude::AsRawFd, - path::Path, -}; - -mod iterator; - -/// This is an internal structure used to access provided file by `CodesHandle`. -/// It also allows to differentiate between `CodesHandle` created from file and from index. -/// It is not intended to be used directly by the user. -#[doc(hidden)] -#[derive(Debug)] -pub struct CodesFile { - // fields dropped from top - pointer: *mut FILE, - product_kind: ProductKind, - _data: D, -} - -/// Internal trait implemented for types that can be called to generate `*mut codes_handle`. -#[doc(hidden)] -pub trait HandleGenerator { - fn gen_codes_handle(&self) -> Result<*mut codes_handle, CodesError>; -} - -impl HandleGenerator for CodesFile { - fn gen_codes_handle(&self) -> Result<*mut codes_handle, CodesError> { - unsafe { codes_handle_new_from_file(self.pointer, self.product_kind) } - } -} - -/// Structure providing access to the GRIB file which takes a full ownership of the accessed file. -/// -/// It can be constructed from: -/// -/// - File path using [`new_from_file()`](CodesHandle::new_from_file) -/// - From memory buffer using [`new_from_memory()`](CodesHandle::new_from_memory) -/// - From GRIB index using [`new_from_index()`](CodesHandle::new_from_index) (with `experimental_index` feature enabled) -/// -/// Destructor for this structure does not panic, but some internal functions may rarely fail -/// leading to bugs. Errors encountered in the destructor are logged with [`log`]. -/// -/// # `FallibleStreamingIterator` -/// -/// This structure implements [`FallibleStreamingIterator`](crate::FallibleStreamingIterator) trait which allows to access GRIB messages. -/// -/// To access GRIB messages the ecCodes library uses a method similar to a C-style iterator. -/// It digests the `* FILE` multiple times, each time returning the `*mut codes_handle` -/// to a message inside the file. The behavior of previous `*mut codes_handle` after next one is generated is undefined -/// and we assume here that it is unsafe to use "old" `*mut codes_handle`. -/// -/// In Rust, such pattern is best represented by a streaming iterator which returns a reference to the message, -/// that is valid only until the next iteration. If you need to prolong the lifetime of the message, you can clone it. -/// Internal ecCodes functions can fail, necessitating the streaming iterator to be implemented with -/// [`FallibleStreamingIterator`](crate::FallibleStreamingIterator) trait. -/// -/// As of `0.10` release, none of the available streaming iterator crates utilises already stabilized GATs. -/// This unfortunately significantly limits the number of methods available for `CodesHandle` iterator. -/// Therefore the probably most versatile way to iterate over the messages is to use `while let` loop. -/// -/// ``` -/// use eccodes::{ProductKind, CodesHandle, KeyRead}; -/// # use std::path::Path; -/// // FallibleStreamingIterator must be in scope to use it -/// use eccodes::FallibleStreamingIterator; -/// # -/// # fn main() -> anyhow::Result<(), eccodes::errors::CodesError> { -/// let file_path = Path::new("./data/iceland-surface.grib"); -/// let product_kind = ProductKind::GRIB; -/// -/// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; -/// -/// // Print names of messages in the file -/// while let Some(message) = handle.next()? { -/// // The message must be unwraped as internal next() can fail -/// let key: String = message.read_key("name")?; -/// println!("{key}"); -/// -/// } -/// # Ok(()) -/// # } -/// ``` -/// -/// You can also manually collect the messages into a vector to use them later. -/// -/// ``` -/// use eccodes::{ProductKind, CodesHandle, KeyedMessage}; -/// # use eccodes::errors::CodesError; -/// # use std::path::Path; -/// use eccodes::FallibleStreamingIterator; -/// # -/// # fn main() -> anyhow::Result<(), eccodes::errors::CodesError> { -/// let file_path = Path::new("./data/iceland-surface.grib"); -/// let product_kind = ProductKind::GRIB; -/// -/// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; -/// -/// let mut handle_collected = vec![]; -/// -/// while let Some(msg) = handle.next()? { -/// handle_collected.push(msg.try_clone()?); -/// } -/// # Ok(()) -/// # } -/// ``` -/// -/// All available methods for `CodesHandle` iterator can be found in [`FallibleStreamingIterator`](crate::FallibleStreamingIterator) trait. -#[derive(Debug)] -pub struct CodesHandle { - // fields are dropped from top to bottom - current_message: Option, - source: S, -} - -// 2024-07-26 -// Previously CodesHandle had implemented Drop which called libc::fclose() -// but that closed the file descriptor and interfered with rust's fs::file destructor. -// -// To my best understanding the purpose of destructor is to clear memory and remove -// any pointers that would be dangling. -// -// The only pointer that is handed out of CodesHandle is &KeyedMessage, which is tied -// to CodesHandle through lifetimes, so if we destruct CodesHandle that pointer is first -// destructed as well. Source pointer is only used internally so we don't need to worry about it. -// -// Clearing the memory is handled on ecCodes side by KeyedMessage/CodesIndex destructors -// and on rust side by destructors of data_container we own. - -/// Enum representing the kind of product (file type) inside handled file. -/// Used to indicate to ecCodes how it should decode/encode messages. -#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)] -pub enum ProductKind { - #[allow(missing_docs)] - GRIB = ProductKind_PRODUCT_GRIB as isize, -} - -impl CodesHandle> { - ///Opens file at given [`Path`] as selected [`ProductKind`] and contructs `CodesHandle`. - /// - ///## Example - /// - ///``` - ///# use eccodes::codes_handle::{ProductKind, CodesHandle}; - ///# use std::path::Path; - ///# fn main() -> anyhow::Result<()> { - ///let file_path = Path::new("./data/iceland.grib"); - ///let product_kind = ProductKind::GRIB; - /// - ///let handle = CodesHandle::new_from_file(file_path, product_kind)?; - /// # Ok(()) - /// # } - ///``` - /// - ///The function creates [`fs::File`](std::fs::File) from provided path and utilises - ///[`fdopen()`](https://man7.org/linux/man-pages/man3/fdopen.3.html) - ///to associate [`io::RawFd`](`std::os::unix::io::RawFd`) - ///with a stream represented by [`libc::FILE`](https://docs.rs/libc/0.2.101/libc/enum.FILE.html) pointer. - /// - ///The constructor takes as argument a [`path`](Path) instead of [`File`] - ///to ensure that `fdopen()` uses the same mode as [`File`]. - /// - /// The file stream and [`File`] are safely closed when `CodesHandle` is dropped. - /// - ///## Errors - ///Returns [`CodesError::FileHandlingInterrupted`] with [`io::Error`](std::io::Error) - ///when the file cannot be opened. - /// - ///Returns [`CodesError::LibcNonZero`] with [`errno`](errno::Errno) information - ///when the stream cannot be created from the file descriptor. - /// - ///Returns [`CodesError::Internal`] with error code - ///when internal [`codes_handle`] cannot be created. - pub fn new_from_file>( - file_path: P, - product_kind: ProductKind, - ) -> Result { - let file = OpenOptions::new().read(true).open(file_path)?; - let file_pointer = open_with_fdopen(&file)?; - - Ok(Self { - source: CodesFile { - _data: file, - pointer: file_pointer, - product_kind, - }, - current_message: None, - }) - } -} -impl CodesHandle>> { - ///Opens data in provided buffer as selected [`ProductKind`] and contructs `CodesHandle`. - /// - ///## Example - /// - ///``` - ///# async fn run() -> anyhow::Result<()> { - ///# use eccodes::{ProductKind, CodesHandle}; - ///# - ///let product_kind = ProductKind::GRIB; - ///let file_data = - /// reqwest::get("https://github.com/ScaleWeather/eccodes/blob/main/data/iceland.grib?raw=true") - /// .await? - /// .bytes() - /// .await? - /// .to_vec(); - /// - ///let handle = CodesHandle::new_from_memory(file_data, product_kind)?; - /// # Ok(()) - ///# } - ///``` - /// - ///The function associates data in memory with a stream - ///represented by [`libc::FILE`](https://docs.rs/libc/0.2.101/libc/enum.FILE.html) pointer - ///using [`fmemopen()`](https://man7.org/linux/man-pages/man3/fmemopen.3.html). - /// - ///The constructor takes full ownership of the data inside buffer, - ///which is safely dropped during the [`CodesHandle`] drop. - /// - ///## Errors - ///Returns [`CodesError::LibcNonZero`] with [`errno`](errno::Errno) information - ///when the file stream cannot be created. - /// - ///Returns [`CodesError::Internal`] with error code - ///when internal [`codes_handle`] cannot be created. - pub fn new_from_memory( - file_data: Vec, - product_kind: ProductKind, - ) -> Result { - let file_pointer = open_with_fmemopen(&file_data)?; - - Ok(Self { - source: CodesFile { - _data: file_data, - product_kind, - pointer: file_pointer, - }, - current_message: None, - }) - } -} - -#[cfg(feature = "experimental_index")] -#[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] -impl CodesHandle { - /// Creates [`CodesHandle`] for provided [`CodesIndex`]. - /// - /// ## Example - /// - /// ``` - /// # fn run() -> anyhow::Result<()> { - /// # use eccodes::{CodesHandle, CodesIndex}; - /// # - /// let index = CodesIndex::new_from_keys(&vec!["shortName", "typeOfLevel", "level"])?; - /// let handle = CodesHandle::new_from_index(index)?; - /// - /// Ok(()) - /// # } - /// ``` - /// - /// The function takes ownership of the provided [`CodesIndex`] which owns - /// the GRIB data. [`CodesHandle`] created from [`CodesIndex`] is of different type - /// than the one created from file or memory buffer, because it internally uses - /// different functions to access messages. But it can be used in the same way. - /// - /// ⚠️ Warning: This function may interfere with other functions in concurrent context, - /// due to ecCodes issues with thread-safety for indexes. More information can be found - /// in [`codes_index`](crate::codes_index) module documentation. - /// - /// ## Errors - /// - /// Returns [`CodesError::Internal`] with error code - /// when internal [`codes_handle`] cannot be created. - pub fn new_from_index(index: CodesIndex) -> Result { - let new_handle = CodesHandle { - source: index, - current_message: None, - }; - - Ok(new_handle) - } -} - -fn open_with_fdopen(file: &File) -> Result<*mut FILE, CodesError> { - let file_ptr = unsafe { libc::fdopen(file.as_raw_fd(), "r".as_ptr().cast::<_>()) }; - - if file_ptr.is_null() { - let error_val = errno(); - let error_code = error_val.0; - return Err(CodesError::LibcNonZero(error_code, error_val)); - } - - Ok(file_ptr) -} - -fn open_with_fmemopen(file_data: &[u8]) -> Result<*mut FILE, CodesError> { - let file_data_ptr = file_data.as_ptr() as *mut c_void; - pointer_guard::non_null!(file_data_ptr); - - let file_ptr; - unsafe { - file_ptr = libc::fmemopen( - file_data_ptr, - file_data.len() as size_t, - "r".as_ptr().cast::<_>(), - ); - } - - if file_ptr.is_null() { - let error_val = errno(); - let error_code = error_val.0; - return Err(CodesError::LibcNonZero(error_code, error_val)); - } - - Ok(file_ptr) -} - -#[cfg(test)] -mod tests { - use crate::codes_handle::{CodesHandle, ProductKind}; - #[cfg(feature = "experimental_index")] - use crate::codes_index::{CodesIndex, Select}; - use anyhow::{Context, Result}; - use eccodes_sys::ProductKind_PRODUCT_GRIB; - use fallible_streaming_iterator::FallibleStreamingIterator; - use std::{fs::File, io::Read, path::Path}; - - #[test] - fn file_constructor() -> Result<()> { - let file_path = Path::new("./data/iceland.grib"); - let product_kind = ProductKind::GRIB; - - let handle = CodesHandle::new_from_file(file_path, product_kind)?; - - assert!(!handle.source.pointer.is_null()); - assert!(handle.current_message.is_none()); - assert_eq!(handle.source.product_kind as u32, { - ProductKind_PRODUCT_GRIB - }); - - handle.source._data.metadata()?; - - Ok(()) - } - - #[test] - fn memory_constructor() -> Result<()> { - let product_kind = ProductKind::GRIB; - - let mut f = File::open(Path::new("./data/iceland.grib"))?; - let mut buf = Vec::new(); - f.read_to_end(&mut buf)?; - - let handle = CodesHandle::new_from_memory(buf, product_kind)?; - assert!(!handle.source.pointer.is_null()); - assert!(handle.current_message.is_none()); - assert_eq!(handle.source.product_kind as u32, { - ProductKind_PRODUCT_GRIB - }); - - assert!(!handle.source._data.is_empty()); - - Ok(()) - } - - #[test] - #[cfg(feature = "experimental_index")] - fn index_constructor_and_destructor() -> Result<()> { - use anyhow::Ok; - - let file_path = Path::new("./data/iceland-surface.grib.idx"); - let index = CodesIndex::read_from_file(file_path)? - .select("shortName", "2t")? - .select("typeOfLevel", "surface")? - .select("level", 0)? - .select("stepType", "instant")?; - - let i_ptr = index.pointer.clone(); - - let handle = CodesHandle::new_from_index(index)?; - - assert_eq!(handle.source.pointer, i_ptr); - assert!(handle.current_message.is_none()); - - Ok(()) - } - - #[test] - fn codes_handle_drop_file() -> Result<()> { - testing_logger::setup(); - - let file_path = Path::new("./data/iceland-surface.grib"); - let product_kind = ProductKind::GRIB; - - let handle = CodesHandle::new_from_file(file_path, product_kind)?; - drop(handle); - - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 0); - }); - - Ok(()) - } - - #[test] - fn codes_handle_drop_mem() -> Result<()> { - testing_logger::setup(); - - let product_kind = ProductKind::GRIB; - - let mut f = File::open(Path::new("./data/iceland.grib"))?; - let mut buf = Vec::new(); - f.read_to_end(&mut buf)?; - - let handle = CodesHandle::new_from_memory(buf, product_kind)?; - drop(handle); - - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 0); - }); - - Ok(()) - } - - #[test] - fn multiple_drops() -> Result<()> { - testing_logger::setup(); - { - let file_path = Path::new("./data/iceland-surface.grib"); - let product_kind = ProductKind::GRIB; - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - - let _ref_msg = handle.next()?.context("no message")?; - let clone_msg = _ref_msg.try_clone()?; - let _oth_ref = handle.next()?.context("no message")?; - - let _nrst = clone_msg.codes_nearest()?; - let _kiter = clone_msg.default_keys_iterator()?; - } - - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 5); - - let expected_logs = vec![ - ("codes_handle_delete", log::Level::Trace), - ("codes_keys_iterator_delete", log::Level::Trace), - ("codes_grib_nearest_delete", log::Level::Trace), - ("codes_handle_delete", log::Level::Trace), - ("codes_handle_delete", log::Level::Trace), - ]; - - captured_logs - .iter() - .zip(expected_logs) - .for_each(|(clg, elg)| { - assert_eq!(clg.body, elg.0); - assert_eq!(clg.level, elg.1) - }); - }); - - Ok(()) - } - - #[test] - #[cfg(feature = "experimental_index")] - fn codes_handle_drop_index() -> Result<()> { - testing_logger::setup(); - - let file_path = Path::new("./data/iceland-surface.grib.idx"); - let index = CodesIndex::read_from_file(file_path)?; - assert!(!index.pointer.is_null()); - - let handle = CodesHandle::new_from_index(index)?; - drop(handle); - - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 1); - assert_eq!(captured_logs[0].body, "codes_index_delete"); - assert_eq!(captured_logs[0].level, log::Level::Trace); - }); - - Ok(()) - } - - #[test] - #[cfg(feature = "experimental_index")] - fn empty_index_constructor() -> Result<()> { - let index = - CodesIndex::new_from_keys(&vec!["shortName", "typeOfLevel", "level", "stepType"])?; - - let mut handle = CodesHandle::new_from_index(index)?; - - assert!(!handle.source.pointer.is_null()); - assert!(handle.current_message.is_none()); - - let msg = handle.next()?; - - assert!(!msg.is_some()); - - Ok(()) - } - - #[test] - #[cfg(feature = "experimental_index")] - fn multiple_drops_with_index() -> Result<()> { - testing_logger::setup(); - { - let keys = vec!["typeOfLevel", "level"]; - let index = CodesIndex::new_from_keys(&keys)?; - let grib_path = Path::new("./data/iceland-levels.grib"); - let index = index - .add_grib_file(grib_path)? - .select("typeOfLevel", "isobaricInhPa")? - .select("level", 600)?; - - let mut handle = CodesHandle::new_from_index(index)?; - let _ref_msg = handle.next()?.context("no message")?; - let clone_msg = _ref_msg.try_clone()?; - let _oth_ref = handle.next()?.context("no message")?; - - let _nrst = clone_msg.codes_nearest()?; - let _kiter = clone_msg.default_keys_iterator()?; - } - - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 6); - - let expected_logs = vec![ - ("codes_handle_delete", log::Level::Trace), - ("codes_keys_iterator_delete", log::Level::Trace), - ("codes_grib_nearest_delete", log::Level::Trace), - ("codes_handle_delete", log::Level::Trace), - ("codes_handle_delete", log::Level::Trace), - ("codes_index_delete", log::Level::Trace), - ]; - - captured_logs - .iter() - .zip(expected_logs) - .for_each(|(clg, elg)| { - assert_eq!(clg.body, elg.0); - assert_eq!(clg.level, elg.1) - }); - }); - - Ok(()) - } -} diff --git a/src/codes_index.rs b/src/codes_index.rs index 3c07cf7..670ccfc 100644 --- a/src/codes_index.rs +++ b/src/codes_index.rs @@ -25,13 +25,42 @@ //! //! The issues have been partially mitigated by implementing global mutex for `codes_index` operations. //! Please not that mutex is used only for `codes_index` functions to not affect performance of other not-problematic functions in this crate. -//! This solution, eliminated tsegfaults in tests, but occasional non-zero return codes still appear. However, this is +//! This solution, eliminated segfaults in tests, but occasional non-zero return codes still appear. However, this is //! not a guarantee and possbility of safety issues is non-zero! //! //! To avoid the memory issues altogether, do not use this feature at all. If you want to use it, take care to use `CodesIndex` in entirely //! non-concurrent environment. //! //! If you have any suggestions or ideas how to improve the safety of this feature, please open an issue or a pull request. +//! +//! ## Why functions are not marked unsafe? +//! +//! Consider this example: +//! +//! ```ignore +//! thread::spawn(|| -> Result<()> { +//! let file_path = Path::new("./data/iceland-surface.grib.idx"); +//! let mut index_op = CodesIndex::read_from_file(file_path)?; +//! +//! loop { +//! index_op = index_op +//! .select("shortName", "2t")? +//! .select("typeOfLevel", "surface")? +//! .select("level", 0)? +//! .select("stepType", "instant")?; +//! } +//! }); +//! +//! let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; +//! let grib_path = Path::new("./data/iceland-surface.grib"); +//! let index = CodesIndex::new_from_keys(&keys)?.add_grib_file(grib_path); +//! ``` +//! +//! Each of the used functions is memory-safe when used sequentially, but when used as above memory bugs may appear. +//! Safety issues arising from using `CodesIndex` also extend beyond this module. For example, in code above, if we +//! created `CodesHandle` from `iceland-surface.grib` file, operations in that handle could also result in memory bugs. +//! Thus, it is not possible to mark only some functions as `unsafe`, because use of `CodesIndex` *poisons* the whole crate. +//! use crate::{ codes_handle::HandleGenerator, @@ -43,7 +72,8 @@ use crate::{ }, }; use eccodes_sys::{codes_handle, codes_index}; -use std::{path::Path, ptr::null_mut}; +use std::{fmt::Debug, path::Path, ptr::null_mut}; +use tracing::instrument; #[derive(Debug)] #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] @@ -138,6 +168,7 @@ impl CodesIndex { /// /// This function will return [`CodesError::Internal`] if the index cannot be created. #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] + #[instrument(level = "trace")] pub fn new_from_keys(keys: &[&str]) -> Result { let keys = keys.join(","); @@ -173,7 +204,10 @@ impl CodesIndex { /// This function will return [`CodesError::Internal`] if the index file is not valid or /// the GRIB file is not present in the same relative path as during the index file creation. #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] - pub fn read_from_file>(index_file_path: P) -> Result { + #[instrument(level = "trace")] + pub fn read_from_file + Debug>( + index_file_path: P, + ) -> Result { let index_file_path: &Path = index_file_path.as_ref(); let file_path = index_file_path.to_str().ok_or_else(|| { std::io::Error::new(std::io::ErrorKind::InvalidData, "Path is not valid utf8") @@ -271,6 +305,7 @@ impl HandleGenerator for CodesIndex { #[doc(hidden)] impl Drop for CodesIndex { + #[instrument(level = "trace")] fn drop(&mut self) { unsafe { codes_index_delete(self.pointer) } self.pointer = null_mut(); @@ -279,13 +314,13 @@ impl Drop for CodesIndex { #[cfg(test)] mod tests { - use anyhow::{bail, Context, Result}; - use fallible_streaming_iterator::FallibleStreamingIterator; + use anyhow::{Context, Result, bail}; + use fallible_iterator::FallibleIterator; use crate::{ + CodesError, CodesHandle, codes_index::{CodesIndex, Select}, errors::CodesInternal, - CodesError, CodesHandle, }; use std::path::Path; #[test] @@ -307,15 +342,7 @@ mod tests { #[test] fn index_destructor() -> Result<()> { let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; - let index = CodesIndex::new_from_keys(&keys)?; - - drop(index); - - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 1); - assert_eq!(captured_logs[0].body, "codes_index_delete"); - assert_eq!(captured_logs[0].level, log::Level::Trace); - }); + let _index = CodesIndex::new_from_keys(&keys)?; Ok(()) } @@ -359,36 +386,19 @@ mod tests { #[test] fn handle_from_index_destructor() -> Result<()> { - testing_logger::setup(); - { - let keys = vec!["typeOfLevel", "level"]; - let index = CodesIndex::new_from_keys(&keys)?; - let grib_path = Path::new("./data/iceland-levels.grib"); - let index = index - .add_grib_file(grib_path)? - .select("typeOfLevel", "isobaricInhPa")? - .select("level", 600)?; - - let mut handle = CodesHandle::new_from_index(index)?; - let _ref_msg = handle.next()?.context("no message")?; - } - - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 2); - - let expected_logs = vec![ - ("codes_handle_delete", log::Level::Trace), - ("codes_index_delete", log::Level::Trace), - ]; - - captured_logs - .iter() - .zip(expected_logs) - .for_each(|(clg, elg)| { - assert_eq!(clg.body, elg.0); - assert_eq!(clg.level, elg.1) - }); - }); + let keys = vec!["typeOfLevel", "level"]; + let index = CodesIndex::new_from_keys(&keys)?; + let grib_path = Path::new("./data/iceland-levels.grib"); + let index = index + .add_grib_file(grib_path)? + .select("typeOfLevel", "isobaricInhPa")? + .select("level", 600)?; + + let mut handle = CodesHandle::new_from_index(index)?; + let _ref_msg = handle + .ref_message_generator() + .next()? + .context("no message")?; Ok(()) } diff --git a/src/codes_message/clone.rs b/src/codes_message/clone.rs new file mode 100644 index 0000000..1043dd6 --- /dev/null +++ b/src/codes_message/clone.rs @@ -0,0 +1,51 @@ +use crate::{ + BufMessage, CodesError, codes_message::CodesMessage, intermediate_bindings::codes_handle_clone, +}; +use std::fmt::Debug; + +impl CodesMessage

{ + /// Custom function to clone the `CodesMessage`. + /// + /// **Be careful of the memory overhead!** ecCodes (when reading from file) defers reading the data into memory + /// if possible. Simply creating `CodesMessage` or even reading some keys will use only a little of memory. + /// This function **will** read the whole message into the memory, which can be of a significant size for big grids. + /// + /// # Errors + /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to clone the message. + pub fn try_clone(&self) -> Result { + Ok(BufMessage::new(unsafe { + codes_handle_clone(self.message_handle)? + })) + } +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use anyhow::{Context, Result}; + use fallible_iterator::FallibleIterator; + + use crate::{CodesFile, ProductKind}; + + #[test] + fn check_clone_safety() -> Result<()> { + let file_path = Path::new("./data/iceland-levels.grib"); + let product_kind = ProductKind::GRIB; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + + let msg1 = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + let key1 = msg1.read_key_dynamic("typeOfLevel")?; + + let msg_clone = msg1.try_clone()?; + drop(msg1); + drop(handle); + let key1_clone = msg_clone.read_key_dynamic("typeOfLevel")?; + assert_eq!(key1, key1_clone); + + Ok(()) + } +} diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs new file mode 100644 index 0000000..e10bc32 --- /dev/null +++ b/src/codes_message/mod.rs @@ -0,0 +1,267 @@ +//! Definition of `CodesMessage` and its associated functions +//! used for reading and writing data of given variable from GRIB file + +mod clone; +#[cfg(feature = "ndarray")] +#[cfg_attr(docsrs, doc(cfg(feature = "ndarray")))] +mod ndarray; +mod read; +mod write; + +#[cfg_attr(docsrs, doc(cfg(feature = "ndarray")))] +pub use ndarray::RustyCodesMessage; +pub use read::{DynamicKeyType, KeyPropertiesRead, KeyRead}; +pub use write::KeyWrite; + +use eccodes_sys::codes_handle; +use std::{ + fmt::Debug, + hash::Hash, + marker::PhantomData, + ptr::null_mut, + sync::{Arc, Mutex}, +}; +use tracing::{Level, event, instrument}; + +use crate::{CodesFile, intermediate_bindings::codes_handle_delete}; + +/// Base structure for [`RefMessage`], [`ArcMessage`] and [`BufMessage`] that provides access to the data +/// contained in the GRIB file, which directly corresponds to the message in the GRIB file. +/// +/// `RefMessage` comes with no performance overhead but has its liftime tied to its parent `CodesFile`. +/// +/// `ArcMessage` can be moved and shared across threads. +/// +/// `BufMessage` is independent of its parent and can be edited with [`KeyWrite`] methods. +/// +/// **Usage examples are provided in documentation of implemented methods.** +/// +/// You can think about the message as a container of data corresponding to a single variable +/// at given date, time and level. In ecCodes the message is represented as a collection of unique +/// key-value pairs. +/// +/// You can read a `Key` with static types using [`read_key()`](KeyRead::read_key()) or with [`DynamicKeyType`] using[`read_key_dynamic()`](CodesMessage::read_key_dynamic()) +/// To iterate over all key names use [`KeysIterator`](crate::KeysIterator). You can also modify the message using +/// [`write_key_unchecked()`](KeyWrite::write_key_unchecked()) (only available for `BufMessage`). +/// +/// [`CodesNearest`](crate::CodesNearest) can be used to find nearest gridpoints for given coordinates in the `CodesMessage`. +/// +/// To write the message to file use [`write_to_file()`](CodesMessage::write_to_file). +/// +/// If you are interested only in getting data values from the message you can use +/// [`to_ndarray()`](CodesMessage::to_ndarray) available in `ndarray` feature. +/// +/// Some of the useful keys are: `validityDate`, `validityTime`, `level`, `typeOfLevel`, `shortName`, `units` and `values`. +/// +/// Note that names, types and availability of some keys can vary between platforms and ecCodes versions. You should test +/// your code whenever changing the environment. +/// +/// Destructor for this structure does not panic, but some internal functions may rarely fail +/// leading to bugs. Errors encountered in desctructor the are reported via [`tracing`]. +#[derive(Debug)] +pub struct CodesMessage { + pub(crate) _parent: P, + pub(crate) message_handle: *mut codes_handle, +} + +// This is a little unintuitive, but we use `()` here to not unnecessarily pollute +// CodesMessage and derived types with generics, because `PhantomData` is needed +// only for lifetime restriction and we tightly control how `CodesMessage` is created. +#[derive(Debug, Hash, PartialEq, Eq, PartialOrd)] +#[doc(hidden)] +pub struct RefParent<'ch>(PhantomData<&'ch ()>); + +#[derive(Debug, Hash, PartialEq, Eq, PartialOrd)] +#[doc(hidden)] +pub struct BufParent(); + +#[derive(Debug)] +#[doc(hidden)] +pub struct ArcParent { + _arc_handle: Arc>>, +} + +/// [`CodesMessage`] that has its liftime tied to its parent `CodesFile`. +pub type RefMessage<'ch> = CodesMessage>; + +/// [`CodesMessage`] that can be moved and shared across threads. +pub type ArcMessage = CodesMessage>; + +unsafe impl Send for ArcMessage {} +unsafe impl Sync for ArcMessage {} + +/// [`CodesMessage`] that is independent of its parent and can be edited with [`KeyWrite`] methods. +pub type BufMessage = CodesMessage; + +unsafe impl Send for BufMessage {} +unsafe impl Sync for BufMessage {} + +impl RefMessage<'_> { + pub(crate) const fn new(handle: *mut codes_handle) -> Self { + RefMessage { + _parent: RefParent(PhantomData), + message_handle: handle, + } + } +} + +impl ArcMessage { + pub(crate) fn new(handle: *mut codes_handle, parent: &Arc>>) -> Self { + ArcMessage { + _parent: ArcParent { + _arc_handle: parent.clone(), + }, + message_handle: handle, + } + } +} + +impl BufMessage { + pub(crate) const fn new(handle: *mut codes_handle) -> Self { + BufMessage { + _parent: BufParent(), + message_handle: handle, + } + } +} + +impl Drop for CodesMessage

{ + /// Executes the destructor for this type. + /// This method calls destructor functions from ecCodes library. + /// In some edge cases these functions can return non-zero code. + /// In such case all pointers and file descriptors are safely deleted. + /// However memory leaks can still occur. + /// + /// If any function called in the destructor returns an error warning will appear in log/tracing. + /// If bugs occur during `CodesMessage` drop please enable log output and post issue on [Github](https://github.com/ScaleWeather/eccodes). + /// + /// Technical note: delete functions in ecCodes can only fail with [`CodesInternalError`](crate::errors::CodesInternal::CodesInternalError) + /// when other functions corrupt the inner memory of pointer, in that case memory leak is possible. + /// In case of corrupt pointer segmentation fault will occur. + /// The pointers are cleared at the end of drop as they are not functional regardless of result of delete functions. + /// + /// # Panics + /// + /// In debug mode when error is encountered. + #[instrument(level = "trace")] + fn drop(&mut self) { + unsafe { + codes_handle_delete(self.message_handle).unwrap_or_else(|error| { + event!( + Level::ERROR, + "codes_handle_delete() returned an error: {:?}", + &error + ); + debug_assert!(false, "Error in CodesMessage::drop"); + }); + } + + self.message_handle = null_mut(); + } +} + +#[cfg(test)] +mod tests { + use crate::codes_file::{CodesFile, ProductKind}; + use anyhow::{Context, Result}; + use fallible_iterator::FallibleIterator; + use std::path::Path; + + #[test] + fn check_docs_keys() -> Result<()> { + let file_path = Path::new("./data/iceland.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + + let _ = current_message.read_key_dynamic("validityDate")?; + let _ = current_message.read_key_dynamic("validityTime")?; + let _ = current_message.read_key_dynamic("level")?; + let _ = current_message.read_key_dynamic("shortName")?; + let _ = current_message.read_key_dynamic("units")?; + let _ = current_message.read_key_dynamic("values")?; + let _ = current_message.read_key_dynamic("typeOfLevel")?; + + Ok(()) + } + + #[test] + fn message_clone_1() -> Result<()> { + let file_path = Path::new("./data/iceland.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + let cloned_message = current_message.try_clone()?; + + assert_ne!( + current_message.message_handle, + cloned_message.message_handle + ); + + Ok(()) + } + + #[test] + fn message_clone_2() -> Result<()> { + let file_path = Path::new("./data/iceland.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let mut mgen = handle.ref_message_iter(); + let msg = mgen.next()?.context("Message not some")?.try_clone()?; + let _ = mgen.next()?; + + drop(handle); + + let _ = msg.read_key_dynamic("dataDate")?; + let _ = msg.read_key_dynamic("jDirectionIncrementInDegrees")?; + let _ = msg.read_key_dynamic("values")?; + let _ = msg.read_key_dynamic("name")?; + let _ = msg.read_key_dynamic("section1Padding")?; + let _ = msg.read_key_dynamic("experimentVersionNumber")?; + + Ok(()) + } + + #[test] + fn message_clone_drop() -> Result<()> { + let file_path = Path::new("./data/iceland.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let _msg_ref = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + let _msg_clone = _msg_ref.try_clone()?; + + drop(_msg_ref); + drop(handle); + drop(_msg_clone); + + Ok(()) + } + + #[test] + fn ref_message_drop_null() -> Result<()> { + let file_path = Path::new("./data/iceland.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let mut current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + current_message.message_handle = std::ptr::null_mut(); + + Ok(()) + } +} diff --git a/src/message_ndarray.rs b/src/codes_message/ndarray.rs similarity index 89% rename from src/message_ndarray.rs rename to src/codes_message/ndarray.rs index e60119f..13a66cd 100644 --- a/src/message_ndarray.rs +++ b/src/codes_message/ndarray.rs @@ -1,14 +1,15 @@ -#![cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] -//! Definitions for converting a `KeyedMessage` to ndarray +//! Definitions for converting a `CodesMessage` to ndarray -use ndarray::{s, Array2, Array3}; +use std::fmt::Debug; -use crate::{errors::MessageNdarrayError, CodesError, KeyRead, KeyedMessage}; +use ndarray::{Array2, Array3, s}; -/// Struct returned by [`KeyedMessage::to_lons_lats_values()`] method. +use crate::{CodesError, KeyRead, codes_message::CodesMessage, errors::MessageNdarrayError}; + +/// Struct returned by [`CodesMessage::to_lons_lats_values()`] method. /// The arrays are collocated, meaning that `longitudes[i, j]` and `latitudes[i, j]` are the coordinates of `values[i, j]`. #[derive(Clone, PartialEq, Debug, Default)] -#[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] +#[cfg_attr(docsrs, doc(cfg(feature = "ndarray")))] pub struct RustyCodesMessage { /// Longitudes in degrees pub longitudes: Array2, @@ -18,7 +19,7 @@ pub struct RustyCodesMessage { pub values: Array2, } -impl KeyedMessage { +impl CodesMessage

{ /// Converts the message to a 2D ndarray. /// /// Returns ndarray where first dimension represents y coordinates and second dimension represents x coordinates, @@ -40,7 +41,7 @@ impl KeyedMessage { /// /// - When the required keys are not present or if their values are not of the expected type /// - When the number of values mismatch with the `Ni` and `Nj` keys - #[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] + #[cfg_attr(docsrs, doc(cfg(feature = "ndarray")))] pub fn to_ndarray(&self) -> Result, CodesError> { let ni: i64 = self.read_key("Ni")?; let ni = usize::try_from(ni).map_err(MessageNdarrayError::from)?; @@ -74,7 +75,7 @@ impl KeyedMessage { } } - /// Same as [`KeyedMessage::to_ndarray()`] but returns the longitudes and latitudes alongside values. + /// Same as [`CodesMessage::to_ndarray()`] but returns the longitudes and latitudes alongside values. /// Fields are returned as separate arrays in [`RustyCodesMessage`]. /// /// Compared to `to_ndarray` this method has performance overhead as returned arrays may be cloned. @@ -85,7 +86,7 @@ impl KeyedMessage { /// /// - When the required keys are not present or if their values are not of the expected type /// - When the number of values mismatch with the `Ni` and `Nj` keys - #[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] + #[cfg_attr(docsrs, doc(cfg(feature = "ndarray")))] pub fn to_lons_lats_values(&self) -> Result { let ni: i64 = self.read_key("Ni")?; let ni = usize::try_from(ni).map_err(MessageNdarrayError::from)?; @@ -140,21 +141,21 @@ impl KeyedMessage { #[cfg(test)] mod tests { + use fallible_iterator::FallibleIterator; use float_cmp::assert_approx_eq; use super::*; - use crate::codes_handle::CodesHandle; - use crate::DynamicKeyType; - use crate::FallibleStreamingIterator; use crate::ProductKind; + use crate::codes_file::CodesFile; + use crate::codes_message::DynamicKeyType; use std::path::Path; #[test] fn test_to_ndarray() -> Result<(), CodesError> { let file_path = Path::new("./data/iceland-surface.grib"); - let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; + let mut handle = CodesFile::new_from_file(file_path, ProductKind::GRIB)?; - while let Some(msg) = handle.next()? { + while let Some(msg) = handle.ref_message_iter().next()? { if msg.read_key_dynamic("shortName")? == DynamicKeyType::Str("2d".to_string()) { let ndarray = msg.to_ndarray()?; @@ -178,9 +179,9 @@ mod tests { #[test] fn test_lons_lats() -> Result<(), CodesError> { let file_path = Path::new("./data/iceland-surface.grib"); - let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; + let mut handle = CodesFile::new_from_file(file_path, ProductKind::GRIB)?; - while let Some(msg) = handle.next()? { + while let Some(msg) = handle.ref_message_iter().next()? { if msg.read_key_dynamic("shortName")? == DynamicKeyType::Str("2d".to_string()) { let rmsg = msg.to_lons_lats_values()?; diff --git a/src/codes_message/read.rs b/src/codes_message/read.rs new file mode 100644 index 0000000..e469c1a --- /dev/null +++ b/src/codes_message/read.rs @@ -0,0 +1,380 @@ +use std::{cmp::Ordering, fmt::Debug}; + +use crate::{ + codes_message::CodesMessage, + errors::CodesError, + intermediate_bindings::{ + NativeKeyType, codes_get_bytes, codes_get_double, codes_get_double_array, codes_get_long, + codes_get_long_array, codes_get_native_type, codes_get_size, codes_get_string, + }, +}; + +/// Provides GRIB key reading capabilites. Implemented by [`CodesMessage`] for all possible key types. +pub trait KeyRead { + /// Tries to read a key of given name from [`CodesMessage`]. This function checks if key native type + /// matches the requested type (ie. you cannot read integer as string, or array as a number). + /// + /// # Example + /// + /// ``` + /// # use eccodes::{CodesFile, KeyRead, FallibleIterator, ProductKind}; + /// # use anyhow::Context; + /// # + /// # fn main() -> anyhow::Result<()> { + /// # + /// let mut file = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; + /// let message = file.ref_message_iter().next()?.context("no message")?; + /// let short_name: String = message.read_key("shortName")?; + /// + /// assert_eq!(short_name, "msl"); + /// # Ok(()) + /// # } + /// ``` + /// + /// # Errors + /// + /// Returns [`WrongRequestedKeySize`](CodesError::WrongRequestedKeyType) when trying to read key in non-native type (use [`unchecked`](KeyRead::read_key_unchecked) instead). + /// + /// Returns [`WrongRequestedKeySize`](CodesError::WrongRequestedKeySize) when trying to read array as integer. + /// + /// Returns [`IncorrectKeySize`](CodesError::IncorrectKeySize) when key size is 0. This can indicate corrupted data. + /// + /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to read the key. + fn read_key(&self, name: &str) -> Result; + + /// Skips all the checks provided by [`read_key`](KeyRead::read_key) and directly calls ecCodes, ensuring only memory and type safety. + /// + /// This function has better perfomance than [`read_key`](KeyRead::read_key) but all error handling and (possible) + /// type conversions are performed directly by ecCodes. + /// + /// This function is also useful for (not usually used) keys that return incorrect native type. + /// + /// # Errors + /// + /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to read the key. + fn read_key_unchecked(&self, name: &str) -> Result; +} + +#[doc(hidden)] +pub trait KeyPropertiesRead { + fn get_key_size(&self, key_name: &str) -> Result; + fn get_key_native_type(&self, key_name: &str) -> Result; +} + +impl KeyPropertiesRead for CodesMessage

{ + fn get_key_size(&self, key_name: &str) -> Result { + unsafe { codes_get_size(self.message_handle, key_name) } + } + + fn get_key_native_type(&self, key_name: &str) -> Result { + unsafe { codes_get_native_type(self.message_handle, key_name) } + } +} + +macro_rules! impl_key_read { + ($key_sizing:ident, $ec_func:ident, $key_variant:path, $gen_type:ty) => { + impl KeyRead<$gen_type> for CodesMessage

{ + fn read_key_unchecked(&self, key_name: &str) -> Result<$gen_type, CodesError> { + unsafe { $ec_func(self.message_handle, key_name) } + } + + fn read_key(&self, key_name: &str) -> Result<$gen_type, CodesError> { + match self.get_key_native_type(key_name)? { + $key_variant => (), + _ => return Err(CodesError::WrongRequestedKeyType), + } + + let key_size = self.get_key_size(key_name)?; + + key_size_check!($key_sizing, key_size); + + self.read_key_unchecked(key_name) + } + } + }; +} + +macro_rules! key_size_check { + // size_var is needed because of macro hygiene + (scalar, $size_var:ident) => { + match $size_var.cmp(&1) { + Ordering::Greater => return Err(CodesError::WrongRequestedKeySize), + Ordering::Less => return Err(CodesError::IncorrectKeySize), + Ordering::Equal => (), + } + }; + + (array, $size_var:ident) => { + if $size_var < 1 { + return Err(CodesError::IncorrectKeySize); + } + }; +} + +impl_key_read!(scalar, codes_get_long, NativeKeyType::Long, i64); +impl_key_read!(scalar, codes_get_double, NativeKeyType::Double, f64); +impl_key_read!(array, codes_get_string, NativeKeyType::Str, String); +impl_key_read!(array, codes_get_bytes, NativeKeyType::Bytes, Vec); +impl_key_read!(array, codes_get_long_array, NativeKeyType::Long, Vec); +impl_key_read!( + array, + codes_get_double_array, + NativeKeyType::Double, + Vec +); + +/// Enum of types GRIB key can have. +/// +/// Messages inside GRIB files can contain keys of arbitrary types, which are known only at runtime (after being checked). +/// ecCodes can return several different types of key, which are represented by this enum +/// and each variant contains the respective data type. +#[derive(Clone, Debug, PartialEq)] +pub enum DynamicKeyType { + #[allow(missing_docs)] + Float(f64), + #[allow(missing_docs)] + Int(i64), + #[allow(missing_docs)] + FloatArray(Vec), + #[allow(missing_docs)] + IntArray(Vec), + #[allow(missing_docs)] + Str(String), + #[allow(missing_docs)] + Bytes(Vec), +} + +impl CodesMessage

{ + /// Method to get a value of given key with [`DynamicKeyType`] from the `CodesMessage`, if it exists. + /// + /// In most cases you should use [`read_key()`](KeyRead::read_key) due to more predictive behaviour + /// and simpler interface. + /// + /// This function exists for backwards compatibility and user convienience. + /// + /// This function checks the type of requested key and tries to read it as the native type. + /// That flow adds performance overhead, but makes the function highly unlikely to fail. + /// + /// This function will try to retrieve the key of native string type as string even + /// when the nul byte is not positioned at the end of key value. + /// + /// If retrieving the key value in native type fails this function will try to read + /// the requested key as bytes. + /// + /// # Example + /// + /// ``` + /// use eccodes::{ProductKind, CodesFile, DynamicKeyType, FallibleIterator}; + /// # use anyhow::Context; + /// # + /// # fn main() -> anyhow::Result<()> { + /// # + /// let mut file = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; + /// let message = file.ref_message_iter().next()?.context("no message")?; + /// let message_short_name = message.read_key_dynamic("shortName")?; + /// let expected_short_name = DynamicKeyType::Str("msl".to_string()); + /// + /// assert_eq!(message_short_name, expected_short_name); + /// # Ok(()) + /// # } + /// ``` + /// + /// # Errors + /// + /// Returns [`CodesNotFound`](crate::errors::CodesInternal::CodesNotFound) + /// when a key of given name has not been found in the message. + /// + /// Returns [`CodesError::MissingKey`] when a given key does not have a specified type. + /// + /// Returns [`CodesError::Internal`] when one of internal ecCodes functions to read the key fails. + /// + /// Returns [`CodesError::CstrUTF8`] and [`CodesError::NulChar`] when the string returned by ecCodes + /// library cannot be parsed as valid UTF8 Rust string. + /// + /// Returns [`CodesError::IncorrectKeySize`] when the size of given key is lower than 1. This indicates corrupted data file, + /// bug in the crate or bug in the ecCodes library. If you encounter this error please check + /// if your file is correct and report it on Github. + pub fn read_key_dynamic(&self, key_name: &str) -> Result { + let key_type = self.get_key_native_type(key_name)?; + let key_size = self.get_key_size(key_name)?; + + match key_type { + NativeKeyType::Long => { + if key_size == 1 { + self.read_key_unchecked(key_name).map(DynamicKeyType::Int) + } else if key_size >= 2 { + self.read_key_unchecked(key_name) + .map(DynamicKeyType::IntArray) + } else { + return Err(CodesError::IncorrectKeySize); + } + } + NativeKeyType::Double => { + if key_size == 1 { + self.read_key_unchecked(key_name).map(DynamicKeyType::Float) + } else if key_size >= 2 { + self.read_key_unchecked(key_name) + .map(DynamicKeyType::FloatArray) + } else { + return Err(CodesError::IncorrectKeySize); + } + } + NativeKeyType::Bytes => self.read_key_unchecked(key_name).map(DynamicKeyType::Bytes), + NativeKeyType::Missing => return Err(CodesError::MissingKey), + _ => self.read_key_unchecked(key_name).map(DynamicKeyType::Str), + } + .or_else(|_| self.read_key_unchecked(key_name).map(DynamicKeyType::Bytes)) + } +} + +#[cfg(test)] +mod tests { + use anyhow::{Context, Result}; + + use crate::codes_file::{CodesFile, ProductKind}; + use crate::{CodesError, KeyRead}; + use crate::{FallibleIterator, codes_message::DynamicKeyType}; + use std::path::Path; + + #[test] + fn key_reader() -> Result<()> { + let file_path = Path::new("./data/iceland.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + + let str_key = current_message.read_key_dynamic("name")?; + match str_key { + DynamicKeyType::Str(_) => {} + _ => panic!("Incorrect variant of string key"), + } + + let double_key = current_message.read_key_dynamic("jDirectionIncrementInDegrees")?; + match double_key { + DynamicKeyType::Float(_) => {} + _ => panic!("Incorrect variant of double key"), + } + + let long_key = current_message.read_key_dynamic("numberOfPointsAlongAParallel")?; + match long_key { + DynamicKeyType::Int(_) => {} + _ => panic!("Incorrect variant of long key"), + } + + let double_arr_key = current_message.read_key_dynamic("values")?; + match double_arr_key { + DynamicKeyType::FloatArray(_) => {} + _ => panic!("Incorrect variant of double array key"), + } + + Ok(()) + } + + #[test] + fn era5_keys_dynamic() -> Result<()> { + let file_path = Path::new("./data/iceland.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let mut current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + let key_names = current_message + .default_keys_iterator()? + .collect::>()?; + + for key_name in &key_names { + assert!(!key_name.is_empty()); + assert!(current_message.read_key_dynamic(key_name).is_ok()); + } + + Ok(()) + } + + #[test] + fn gfs_keys_dynamic() -> Result<()> { + let file_path = Path::new("./data/gfs.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let mut current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + let key_names = current_message + .default_keys_iterator()? + .collect::>()?; + + for key_name in &key_names { + assert!(!key_name.is_empty()); + assert!(current_message.read_key_dynamic(key_name).is_ok()); + } + + Ok(()) + } + + #[test] + fn missing_key() -> Result<()> { + let file_path = Path::new("./data/iceland.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + + let missing_key = current_message.read_key_dynamic("doesNotExist"); + + assert!(missing_key.is_err()); + + Ok(()) + } + + #[test] + fn incorrect_key_type() -> Result<()> { + let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + + let missing_key: Result = current_message.read_key("shortName"); + + assert!(missing_key.is_err()); + + Ok(()) + } + + #[test] + // checks if we can read keys that are used in benchmarks + fn benchmark_keys() -> Result<()> { + let file_path = Path::new("./data/iceland.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + + let msg = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + + let _ = msg.read_key_dynamic("dataDate")?; + let _ = msg.read_key_dynamic("jDirectionIncrementInDegrees")?; + let _ = msg.read_key_dynamic("values")?; + let _ = msg.read_key_dynamic("name")?; + let _ = msg.read_key_dynamic("section1Padding")?; + let _ = msg.read_key_dynamic("experimentVersionNumber")?; + let _ = msg + .read_key_dynamic("zero") + .unwrap_or_else(|_| msg.read_key_dynamic("zeros").unwrap()); + + Ok(()) + } +} diff --git a/src/codes_message/write.rs b/src/codes_message/write.rs new file mode 100644 index 0000000..8a00774 --- /dev/null +++ b/src/codes_message/write.rs @@ -0,0 +1,285 @@ +use std::{fmt::Debug, fs::OpenOptions, io::Write, path::Path, slice}; + +use crate::{ + codes_message::{BufMessage, CodesMessage}, + errors::CodesError, + intermediate_bindings::{ + codes_get_message, codes_set_bytes, codes_set_double, codes_set_double_array, + codes_set_long, codes_set_long_array, codes_set_string, + }, +}; + +/// Provides GRIB key writing capabilites. Implemented by [`CodesMessage`] for all possible key types. +pub trait KeyWrite { + /// Writes key with given name and value to [`CodesMessage`] overwriting existing value, unless + /// the key is read-only. + /// + /// This function directly calls ecCodes ensuring only type and memory safety. + /// Unchecked doesn't mean it's unsafe - just that there are no checks on Rust side in comparison to + /// `read_key` which has such checks. + /// + /// # Example + /// + /// ``` + /// # use anyhow::Context; + /// # use eccodes::{CodesFile, FallibleIterator, KeyWrite, ProductKind}; + /// # + /// # fn main() -> anyhow::Result<()> { + /// let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; + /// + /// // CodesFile iterator returns immutable messages. + /// // To edit a message it must be cloned. + /// let mut message = handle + /// .ref_message_iter() + /// .next()? + /// .context("no message")? + /// .try_clone()?; + /// message.write_key_unchecked("level", 1)?; + /// # Ok(()) + /// # } + /// ``` + /// + /// # Errors + /// + /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to write the key. + fn write_key_unchecked(&mut self, name: &str, value: T) -> Result<&mut Self, CodesError>; +} + +macro_rules! impl_key_write { + ($ec_func:ident, $gen_type:ty) => { + impl KeyWrite<$gen_type> for BufMessage { + fn write_key_unchecked( + &mut self, + name: &str, + value: $gen_type, + ) -> Result<&mut Self, CodesError> { + unsafe { + $ec_func(self.message_handle, name, value)?; + } + Ok(self) + } + } + }; +} + +impl_key_write!(codes_set_long, i64); +impl_key_write!(codes_set_double, f64); +impl_key_write!(codes_set_long_array, &[i64]); +impl_key_write!(codes_set_double_array, &[f64]); +impl_key_write!(codes_set_bytes, &[u8]); +impl_key_write!(codes_set_string, &str); + +impl CodesMessage { + /// Function to write given `CodesMessage` to a file at provided path. + /// If file does not exists it will be created. + /// If `append` is set to `true` file will be opened in append mode + /// and no data will be overwritten (useful when writing mutiple messages to one file). + /// + /// # Example + /// + /// ``` + /// # use eccodes::{CodesError, CodesFile, FallibleIterator, KeyRead, ProductKind}; + /// # use std::fs::remove_file; + /// # fn main() -> anyhow::Result<(), CodesError> { + /// let mut handle = CodesFile::new_from_file("./data/iceland-levels.grib", ProductKind::GRIB)?; + /// + /// while let Some(msg) = handle.ref_message_iter().next()? { + /// let level: i64 = msg.read_key("level")?; + /// if level == 800 { + /// msg.write_to_file("./data/iceland-800hPa.grib", true)?; + /// } + /// } + /// # remove_file("./data/iceland-800hPa.grib")?; + /// # Ok(()) + /// } + /// ``` + /// + /// # Errors + /// + /// Returns [`CodesError::FileHandlingInterrupted`] when the file cannot be opened, + /// created or correctly written. + /// + /// Returns [`CodesInternal`](crate::errors::CodesInternal) + /// when internal ecCodes function returns non-zero code. + pub fn write_to_file>( + &self, + file_path: P, + append: bool, + ) -> Result<(), CodesError> { + let msg = unsafe { codes_get_message(self.message_handle)? }; + let buf = unsafe { slice::from_raw_parts(msg.0.cast(), msg.1) }; + let mut file = OpenOptions::new() + .write(true) + .create(true) + .append(append) + .open(file_path)?; + + file.write_all(buf)?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use anyhow::{Context, Result}; + use fallible_iterator::FallibleIterator; + + use crate::{ + KeyRead, + codes_file::{CodesFile, ProductKind}, + codes_message::{DynamicKeyType, KeyWrite}, + }; + use std::{fs::remove_file, path::Path}; + + #[test] + fn write_message_ref() -> Result<()> { + let file_path = Path::new("./data/iceland.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + let out_path = Path::new("./data/iceland_write.grib"); + current_message.write_to_file(out_path, false)?; + + remove_file(out_path)?; + + Ok(()) + } + + #[test] + fn write_message_clone() -> Result<()> { + let file_path = Path::new("./data/iceland.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")? + .try_clone()?; + + drop(handle); + + let out_path = Path::new("./data/iceland_write_clone.grib"); + current_message.write_to_file(out_path, false)?; + + remove_file(out_path)?; + + Ok(()) + } + + #[test] + fn append_message() -> Result<()> { + let product_kind = ProductKind::GRIB; + let out_path = Path::new("./data/iceland_append.grib"); + + let file_path = Path::new("./data/iceland-surface.grib"); + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + current_message.write_to_file(out_path, false)?; + + let file_path = Path::new("./data/iceland-levels.grib"); + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + current_message.write_to_file(out_path, true)?; + + remove_file(out_path)?; + + Ok(()) + } + + #[test] + fn write_key() -> Result<()> { + let product_kind = ProductKind::GRIB; + let file_path = Path::new("./data/iceland.grib"); + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + + let old_key = current_message.read_key_dynamic("centre")?; + + let mut cloned_message = current_message.try_clone()?; + cloned_message.write_key_unchecked("centre", "cnmc")?; + + let read_key = cloned_message.read_key_dynamic("centre")?; + + assert_ne!(old_key, read_key); + assert_eq!(read_key, DynamicKeyType::Str("cnmc".into())); + + Ok(()) + } + + #[test] + fn write_key_types() -> Result<()> { + let product_kind = ProductKind::GRIB; + let file_path = Path::new("./data/iceland.grib"); + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let mut message = handle + .ref_message_iter() + .next()? + .context("Message not some")? + .try_clone()?; + + let mut values_array: Vec = message.read_key("values")?; + values_array[0] = 0.0; + + message.write_key_unchecked("centre", "cnmc")?; // str + message.write_key_unchecked("day", 3)?; // int + message.write_key_unchecked("latitudeOfFirstGridPointInDegrees", 7.0)?; // float + message.write_key_unchecked("values", values_array.as_slice())?; //float array + + // int array and bytes cannot be tested because written key must exist and must be writable + Ok(()) + } + + #[test] + fn edit_keys_and_save() -> Result<()> { + let product_kind = ProductKind::GRIB; + let file_path = Path::new("./data/iceland.grib"); + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + + let old_key = current_message.read_key_dynamic("centre")?; + + let mut cloned_message = current_message.try_clone()?; + cloned_message.write_key_unchecked("centre", "cnmc")?; + + cloned_message.write_to_file(Path::new("./data/iceland_edit.grib"), false)?; + + let file_path = Path::new("./data/iceland_edit.grib"); + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + + let read_key = current_message.read_key_dynamic("centre")?; + + assert_ne!(old_key, read_key); + assert_eq!(read_key, DynamicKeyType::Str("cnmc".into())); + + remove_file(Path::new("./data/iceland_edit.grib"))?; + + Ok(()) + } +} diff --git a/src/codes_nearest.rs b/src/codes_nearest.rs index ef083f3..ace5c73 100644 --- a/src/codes_nearest.rs +++ b/src/codes_nearest.rs @@ -1,23 +1,24 @@ //! Definition and associated functions of `CodesNearest` -//! used for finding nearest gridpoints in `KeyedMessage` +//! used for finding nearest gridpoints in `CodesMessage` -use std::ptr::null_mut; +use std::{fmt::Debug, ptr::null_mut}; use eccodes_sys::codes_nearest; -use log::error; +use tracing::{Level, event, instrument}; use crate::{ + CodesError, + codes_message::CodesMessage, intermediate_bindings::{ codes_grib_nearest_delete, codes_grib_nearest_find, codes_grib_nearest_new, }, - CodesError, KeyedMessage, }; -/// The structure used to find nearest gridpoints in `KeyedMessage`. +/// The structure used to find nearest gridpoints in `CodesMessage`. #[derive(Debug)] -pub struct CodesNearest<'a> { +pub struct CodesNearest<'a, P: Debug> { nearest_handle: *mut codes_nearest, - parent_message: &'a KeyedMessage, + parent_message: &'a CodesMessage

, } /// The structure returned by [`CodesNearest::find_nearest()`]. @@ -32,20 +33,20 @@ pub struct NearestGridpoint { pub lon: f64, /// Distance between requested point and this gridpoint in kilometers pub distance: f64, - ///Value of parameter at this gridpoint contained by `KeyedMessage` in corresponding units + ///Value of parameter at this gridpoint contained by `CodesMessage` in corresponding units pub value: f64, } -impl KeyedMessage { - /// Creates a new instance of [`CodesNearest`] for the `KeyedMessage`. - /// [`CodesNearest`] can be used to find nearest gridpoints for given coordinates in the `KeyedMessage` +impl CodesMessage

{ + /// Creates a new instance of [`CodesNearest`] for the `CodesMessage`. + /// [`CodesNearest`] can be used to find nearest gridpoints for given coordinates in the `CodesMessage` /// by calling [`find_nearest()`](crate::CodesNearest::find_nearest). /// /// # Errors /// /// This function returns [`CodesInternal`](crate::errors::CodesInternal) when /// internal nearest handle cannot be created. - pub fn codes_nearest(&self) -> Result, CodesError> { + pub fn codes_nearest(&self) -> Result, CodesError> { let nearest_handle = unsafe { codes_grib_nearest_new(self.message_handle)? }; Ok(CodesNearest { @@ -55,7 +56,7 @@ impl KeyedMessage { } } -impl CodesNearest<'_> { +impl CodesNearest<'_, P> { ///Function to get four [`NearestGridpoint`]s of a point represented by requested coordinates. /// ///The inputs are latitude and longitude of requested point in respectively degrees north and @@ -64,20 +65,15 @@ impl CodesNearest<'_> { ///### Example /// ///``` - /// use eccodes::{ProductKind, CodesHandle, KeyedMessage, KeysIteratorFlags}; - /// # use std::path::Path; - /// use eccodes::FallibleStreamingIterator; /// # use anyhow::Context; + /// # use eccodes::{CodesFile, FallibleIterator, ProductKind}; /// # fn main() -> anyhow::Result<()> { - /// let file_path = Path::new("./data/iceland.grib"); - /// let product_kind = ProductKind::GRIB; + /// let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; + /// let msg = handle.ref_message_iter().next()?.context("no message")?; /// - /// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - /// let msg = handle.next()?.context("no message")?; - /// - /// let c_nearest = msg.codes_nearest()?; + /// let mut c_nearest = msg.codes_nearest()?; /// let out = c_nearest.find_nearest(64.13, -21.89)?; - /// # Ok(()) + /// # Ok(()) /// # } ///``` /// @@ -85,7 +81,11 @@ impl CodesNearest<'_> { /// ///This function returns [`CodesInternal`](crate::errors::CodesInternal) when ///one of ecCodes function returns the non-zero code. - pub fn find_nearest(&self, lat: f64, lon: f64) -> Result<[NearestGridpoint; 4], CodesError> { + pub fn find_nearest( + &mut self, + lat: f64, + lon: f64, + ) -> Result<[NearestGridpoint; 4], CodesError> { let output_points; unsafe { @@ -101,15 +101,20 @@ impl CodesNearest<'_> { } } -#[doc(hidden)] -impl Drop for CodesNearest<'_> { +impl Drop for CodesNearest<'_, P> { + /// # Panics + /// + /// In debug when error is encountered + #[instrument(level = "trace")] fn drop(&mut self) { unsafe { codes_grib_nearest_delete(self.nearest_handle).unwrap_or_else(|error| { - error!( + event!( + Level::ERROR, "codes_grib_nearest_delete() returned an error: {:?}", &error ); + debug_assert!(false, "Error in CodesNearest::drop"); }); } @@ -122,9 +127,9 @@ mod tests { use std::path::Path; use anyhow::{Context, Result}; - use fallible_streaming_iterator::FallibleStreamingIterator; + use fallible_iterator::FallibleIterator; - use crate::{CodesHandle, ProductKind}; + use crate::{CodesFile, ProductKind}; #[test] fn find_nearest() -> Result<()> { @@ -132,20 +137,26 @@ mod tests { let file_path2 = Path::new("./data/iceland-surface.grib"); let product_kind = ProductKind::GRIB; - let mut handle1 = CodesHandle::new_from_file(file_path1, product_kind)?; - let msg1 = handle1.next()?.context("Message not some")?; - let nrst1 = msg1.codes_nearest()?; + let mut handle1 = CodesFile::new_from_file(file_path1, product_kind)?; + let msg1 = handle1 + .ref_message_iter() + .next()? + .context("Message not some")?; + let mut nrst1 = msg1.codes_nearest()?; let out1 = nrst1.find_nearest(64.13, -21.89)?; - let mut handle2 = CodesHandle::new_from_file(file_path2, product_kind)?; - let msg2 = handle2.next()?.context("Message not some")?; - let nrst2 = msg2.codes_nearest()?; + let mut handle2 = CodesFile::new_from_file(file_path2, product_kind)?; + let msg2 = handle2 + .ref_message_iter() + .next()? + .context("Message not some")?; + let mut nrst2 = msg2.codes_nearest()?; let out2 = nrst2.find_nearest(64.13, -21.89)?; assert!(out1[0].value > 10000.0); assert!(out2[3].index == 551); - assert!(out1[1].lat == 64.0); - assert!(out2[2].lon == -21.75); + assert!((out1[1].lat - 64.0).abs() < f64::EPSILON); + assert!((out2[2].lon - -21.75).abs() < f64::EPSILON); assert!(out1[0].distance > 15.0); Ok(()) @@ -156,26 +167,30 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; let _nrst = current_message.codes_nearest()?; - drop(_nrst); + Ok(()) + } - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 1); - assert_eq!(captured_logs[0].body, "codes_grib_nearest_delete"); - assert_eq!(captured_logs[0].level, log::Level::Trace); - }); + #[test] + fn destructor_null() -> Result<()> { + let file_path = Path::new("./data/iceland.grib"); + let product_kind = ProductKind::GRIB; - drop(handle); + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 1); - assert_eq!(captured_logs[0].body, "codes_handle_delete"); - assert_eq!(captured_logs[0].level, log::Level::Trace); - }); + let mut nrst = current_message.codes_nearest()?; + nrst.nearest_handle = std::ptr::null_mut(); Ok(()) } diff --git a/src/errors.rs b/src/errors.rs index 4d8ec8f..a3d7b66 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -74,14 +74,24 @@ pub enum CodesError { /// Returned when function in `message_ndarray` module cannot convert /// the message to ndarray. Check [`MessageNdarrayError`] for more details. - #[cfg(feature = "message_ndarray")] - #[error("error occured while converting KeyedMessage to ndarray {0}")] + #[cfg(feature = "ndarray")] + #[error("error occured while converting CodesMessage to ndarray {0}")] NdarrayConvert(#[from] MessageNdarrayError), + + /// eccodes functions return errors as error codes and it is technically possible + /// that the library might return an error code that does not appear in [`CodesInternal`] enum. + #[error("eccodes returned unrecognized error code: {0}")] + UnrecognizedErrorCode(i32), + + /// Similarly to error codes, eccodes return key type as i32, so + /// it's technically possible that this code does not appear in internal native key types. + #[error("Unrecognized native key type code: {0}")] + UnrecognizedKeyTypeCode(i32), } /// Errors returned by the `message_ndarray` module. -#[cfg(feature = "message_ndarray")] -#[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] +#[cfg(feature = "ndarray")] +#[cfg_attr(docsrs, doc(cfg(feature = "ndarray")))] #[derive(PartialEq, Clone, Error, Debug)] pub enum MessageNdarrayError { /// Returned when functions converting to ndarray cannot correctly diff --git a/src/intermediate_bindings/codes_get.rs b/src/intermediate_bindings/codes_get.rs index dc71054..82ae4f7 100644 --- a/src/intermediate_bindings/codes_get.rs +++ b/src/intermediate_bindings/codes_get.rs @@ -7,10 +7,7 @@ use eccodes_sys::codes_handle; use libc::c_void; use num_traits::FromPrimitive; -use crate::{ - errors::{CodesError, CodesInternal}, - pointer_guard, -}; +use crate::{errors::CodesError, intermediate_bindings::error_code_to_result, pointer_guard}; use super::NativeKeyType; @@ -18,243 +15,224 @@ pub unsafe fn codes_get_native_type( handle: *const codes_handle, key: &str, ) -> Result { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); - let mut key_type: i32 = 0; + let key = CString::new(key).unwrap(); + let mut key_type: i32 = 0; - let error_code = eccodes_sys::codes_get_native_type(handle, key.as_ptr(), &raw mut key_type); + let error_code = + eccodes_sys::codes_get_native_type(handle, key.as_ptr(), &raw mut key_type); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + FromPrimitive::from_i32(key_type).ok_or(CodesError::UnrecognizedKeyTypeCode(key_type)) } - - Ok(FromPrimitive::from_i32(key_type).unwrap()) } pub unsafe fn codes_get_size(handle: *const codes_handle, key: &str) -> Result { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); - let mut key_size: usize = 0; + let key = CString::new(key).unwrap(); + let mut key_size: usize = 0; - let error_code = eccodes_sys::codes_get_size(handle, key.as_ptr(), &raw mut key_size); + let error_code = eccodes_sys::codes_get_size(handle, key.as_ptr(), &raw mut key_size); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(key_size) } - - Ok(key_size) } pub unsafe fn codes_get_long(handle: *const codes_handle, key: &str) -> Result { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); - let mut key_value: i64 = 0; + let key = CString::new(key).unwrap(); + let mut key_value: i64 = 0; - let error_code = eccodes_sys::codes_get_long(handle, key.as_ptr(), &raw mut key_value); + let error_code = eccodes_sys::codes_get_long(handle, key.as_ptr(), &raw mut key_value); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(key_value) } - - Ok(key_value) } pub unsafe fn codes_get_double(handle: *const codes_handle, key: &str) -> Result { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); - let mut key_value: f64 = 0.0; + let key = CString::new(key).unwrap(); + let mut key_value: f64 = 0.0; - let error_code = eccodes_sys::codes_get_double(handle, key.as_ptr(), &raw mut key_value); + let error_code = eccodes_sys::codes_get_double(handle, key.as_ptr(), &raw mut key_value); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(key_value) } - - Ok(key_value) } pub unsafe fn codes_get_double_array( handle: *const codes_handle, key: &str, ) -> Result, CodesError> { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let mut key_size = codes_get_size(handle, key)?; - let key = CString::new(key).unwrap(); + let mut key_size = codes_get_size(handle, key)?; + let key = CString::new(key).unwrap(); - let mut key_values: Vec = vec![0.0; key_size]; + let mut key_values: Vec = vec![0.0; key_size]; - let error_code = eccodes_sys::codes_get_double_array( - handle, - key.as_ptr(), - key_values.as_mut_ptr().cast::<_>(), - &raw mut key_size, - ); + let error_code = eccodes_sys::codes_get_double_array( + handle, + key.as_ptr(), + key_values.as_mut_ptr().cast(), + &raw mut key_size, + ); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(key_values) } - - Ok(key_values) } pub unsafe fn codes_get_long_array( handle: *const codes_handle, key: &str, ) -> Result, CodesError> { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let mut key_size = codes_get_size(handle, key)?; - let key = CString::new(key).unwrap(); + let mut key_size = codes_get_size(handle, key)?; + let key = CString::new(key).unwrap(); - let mut key_values: Vec = vec![0; key_size]; + let mut key_values: Vec = vec![0; key_size]; - let error_code = eccodes_sys::codes_get_long_array( - handle, - key.as_ptr(), - key_values.as_mut_ptr().cast::<_>(), - &raw mut key_size, - ); + let error_code = eccodes_sys::codes_get_long_array( + handle, + key.as_ptr(), + key_values.as_mut_ptr().cast(), + &raw mut key_size, + ); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(key_values) } - - Ok(key_values) } pub unsafe fn codes_get_length( handle: *const codes_handle, key: &str, ) -> Result { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); - let mut key_length: usize = 0; + let key = CString::new(key).unwrap(); + let mut key_length: usize = 0; - let error_code = eccodes_sys::codes_get_length(handle, key.as_ptr(), &raw mut key_length); + let error_code = eccodes_sys::codes_get_length(handle, key.as_ptr(), &raw mut key_length); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(key_length) } - - Ok(key_length) } pub unsafe fn codes_get_string( handle: *const codes_handle, key: &str, ) -> Result { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let mut key_length = codes_get_length(handle, key)?; - let key = CString::new(key).unwrap(); + let mut key_length = codes_get_length(handle, key)?; + let key = CString::new(key).unwrap(); - let mut key_message: Vec = vec![0; key_length]; + let mut key_message: Vec = vec![0; key_length]; - let error_code = eccodes_sys::codes_get_string( - handle, - key.as_ptr(), - key_message.as_mut_ptr().cast::<_>(), - &raw mut key_length, - ); + let error_code = eccodes_sys::codes_get_string( + handle, + key.as_ptr(), + key_message.as_mut_ptr().cast(), + &raw mut key_length, + ); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + key_message.truncate(key_length); + let key_message_result = CStr::from_bytes_with_nul(key_message.as_ref()); - key_message.truncate(key_length); - let key_message_result = CStr::from_bytes_with_nul(key_message.as_ref()); + let key_message_cstr = if let Ok(msg) = key_message_result { + msg + } else { + key_message.push(0); + CStr::from_bytes_with_nul(key_message.as_ref())? + }; - let key_message_cstr = if let Ok(msg) = key_message_result { - msg - } else { - key_message.push(0); - CStr::from_bytes_with_nul(key_message.as_ref())? - }; + let key_message_string = key_message_cstr.to_str()?.to_string(); - let key_message_string = key_message_cstr.to_str()?.to_string(); - - Ok(key_message_string) + Ok(key_message_string) + } } pub unsafe fn codes_get_bytes( handle: *const codes_handle, key: &str, ) -> Result, CodesError> { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let mut key_size = codes_get_length(handle, key)?; - let key = CString::new(key).unwrap(); + let mut key_size = codes_get_length(handle, key)?; + let key = CString::new(key).unwrap(); - let mut buffer: Vec = vec![0; key_size]; + let mut buffer: Vec = vec![0; key_size]; - let error_code = eccodes_sys::codes_get_bytes( - handle, - key.as_ptr(), - buffer.as_mut_ptr().cast::<_>(), - &raw mut key_size, - ); + let error_code = eccodes_sys::codes_get_bytes( + handle, + key.as_ptr(), + buffer.as_mut_ptr().cast(), + &raw mut key_size, + ); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(buffer) } - - Ok(buffer) } pub unsafe fn codes_get_message_size(handle: *const codes_handle) -> Result { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let mut size: usize = 0; + let mut size: usize = 0; - let error_code = eccodes_sys::codes_get_message_size(handle, &raw mut size); + let error_code = eccodes_sys::codes_get_message_size(handle, &raw mut size); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(size) } - - Ok(size) } +/// Can panic in debug pub unsafe fn codes_get_message( handle: *const codes_handle, ) -> Result<(*const c_void, usize), CodesError> { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let buffer_size = codes_get_message_size(handle)?; + let buffer_size = codes_get_message_size(handle)?; - let buffer: Vec = vec![0; buffer_size]; - let mut buffer_ptr = buffer.as_ptr().cast::<_>(); + let buffer: Vec = vec![0; buffer_size]; + let mut buffer_ptr = buffer.as_ptr().cast(); - let mut message_size: usize = 0; + let mut message_size: usize = 0; - let error_code = eccodes_sys::codes_get_message(handle, &raw mut buffer_ptr, &raw mut message_size); + let error_code = + eccodes_sys::codes_get_message(handle, &raw mut buffer_ptr, &raw mut message_size); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } - - assert!( - buffer_size == message_size, - "Buffer and message sizes ar not equal in codes_get_message! + debug_assert!( + buffer_size == message_size, + "Buffer and message sizes ar not equal in codes_get_message! Please report this panic on Github." - ); + ); - Ok((buffer_ptr, message_size)) + Ok((buffer_ptr, message_size)) + } } diff --git a/src/intermediate_bindings/codes_handle.rs b/src/intermediate_bindings/codes_handle.rs index 2913e1a..9c72b04 100644 --- a/src/intermediate_bindings/codes_handle.rs +++ b/src/intermediate_bindings/codes_handle.rs @@ -4,82 +4,50 @@ use std::ptr::{self}; use eccodes_sys::{codes_context, codes_handle}; -#[cfg(feature = "experimental_index")] -use eccodes_sys::{codes_index, CODES_LOCK}; use libc::FILE; -use num_traits::FromPrimitive; +use tracing::instrument; use crate::{ - codes_handle::ProductKind, - errors::{CodesError, CodesInternal}, + codes_file::ProductKind, errors::CodesError, intermediate_bindings::error_code_to_result, pointer_guard, }; +#[instrument(level = "trace")] pub unsafe fn codes_handle_new_from_file( file_pointer: *mut FILE, product_kind: ProductKind, ) -> Result<*mut codes_handle, CodesError> { - pointer_guard::non_null!(file_pointer); + unsafe { + pointer_guard::non_null!(file_pointer); - let context: *mut codes_context = ptr::null_mut(); //default context + let context: *mut codes_context = ptr::null_mut(); //default context - let mut error_code: i32 = 0; + let mut error_code: i32 = 0; - let file_handle = eccodes_sys::codes_handle_new_from_file( - context, - file_pointer.cast::<_>(), - product_kind as u32, - &raw mut error_code, - ); + let file_handle = eccodes_sys::codes_handle_new_from_file( + context, + file_pointer.cast(), + product_kind as u32, + &raw mut error_code, + ); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(file_handle) } - - Ok(file_handle) } +#[instrument(level = "trace")] pub unsafe fn codes_handle_delete(handle: *mut codes_handle) -> Result<(), CodesError> { - #[cfg(test)] - log::trace!("codes_handle_delete"); - - if handle.is_null() { - return Ok(()); - } - - let error_code = eccodes_sys::codes_handle_delete(handle); + unsafe { + if handle.is_null() { + return Ok(()); + } - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } - - Ok(()) -} - -#[cfg(feature = "experimental_index")] -pub unsafe fn codes_handle_new_from_index( - index: *mut codes_index, -) -> Result<*mut codes_handle, CodesError> { - pointer_guard::non_null!(index); - - let mut error_code: i32 = 0; - - let _g = CODES_LOCK.lock().unwrap(); - let codes_handle = eccodes_sys::codes_handle_new_from_index(index, &raw mut error_code); - - // special case! codes_handle_new_from_index returns -43 when there are no messages left in the index - // this is also indicated by a null pointer, which is handled upstream - if error_code == -43 { - return Ok(codes_handle); - } + let error_code = eccodes_sys::codes_handle_delete(handle); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(()) } - Ok(codes_handle) } pub unsafe fn codes_handle_clone( diff --git a/src/intermediate_bindings/codes_index.rs b/src/intermediate_bindings/codes_index.rs deleted file mode 100644 index 097b7e9..0000000 --- a/src/intermediate_bindings/codes_index.rs +++ /dev/null @@ -1,132 +0,0 @@ -#![allow(non_camel_case_types)] -#![allow(clippy::module_name_repetitions)] - -use eccodes_sys::{codes_context, codes_index, CODES_LOCK}; -use num_traits::FromPrimitive; -use std::{ffi::CString, ptr}; - -use crate::{ - errors::{CodesError, CodesInternal}, - pointer_guard, -}; - -// all index functions are safeguarded by a lock -// because there are random errors appearing when using the index functions concurrently - -pub unsafe fn codes_index_new(keys: &str) -> Result<*mut codes_index, CodesError> { - let context: *mut codes_context = ptr::null_mut(); //default context - let mut error_code: i32 = 0; - let keys = CString::new(keys).unwrap(); - - let _g = CODES_LOCK.lock().unwrap(); - let codes_index = eccodes_sys::codes_index_new(context, keys.as_ptr(), &raw mut error_code); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } - Ok(codes_index) -} - -pub unsafe fn codes_index_read(filename: &str) -> Result<*mut codes_index, CodesError> { - let filename = CString::new(filename).unwrap(); - let context: *mut codes_context = ptr::null_mut(); //default context - let mut error_code: i32 = 0; - - let _g = CODES_LOCK.lock().unwrap(); - let codes_index = eccodes_sys::codes_index_read(context, filename.as_ptr(), &raw mut error_code); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } - Ok(codes_index) -} - -pub unsafe fn codes_index_delete(index: *mut codes_index) { - #[cfg(test)] - log::trace!("codes_index_delete"); - - if index.is_null() { - return; - } - - let _g = CODES_LOCK.lock().unwrap(); - eccodes_sys::codes_index_delete(index); -} - -pub unsafe fn codes_index_add_file( - index: *mut codes_index, - filename: &str, -) -> Result<(), CodesError> { - pointer_guard::non_null!(index); - - let filename = CString::new(filename).unwrap(); - - let _g = CODES_LOCK.lock().unwrap(); - let error_code = eccodes_sys::codes_index_add_file(index, filename.as_ptr()); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } - Ok(()) -} - -pub unsafe fn codes_index_select_long( - index: *mut codes_index, - key: &str, - value: i64, -) -> Result<(), CodesError> { - pointer_guard::non_null!(index); - - let key = CString::new(key).unwrap(); - - let _g = CODES_LOCK.lock().unwrap(); - let error_code = eccodes_sys::codes_index_select_long(index, key.as_ptr(), value); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } - Ok(()) -} - -pub unsafe fn codes_index_select_double( - index: *mut codes_index, - key: &str, - value: f64, -) -> Result<(), CodesError> { - pointer_guard::non_null!(index); - - let key = CString::new(key).unwrap(); - - let _g = CODES_LOCK.lock().unwrap(); - let error_code = eccodes_sys::codes_index_select_double(index, key.as_ptr(), value); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } - Ok(()) -} - -pub unsafe fn codes_index_select_string( - index: *mut codes_index, - key: &str, - value: &str, -) -> Result<(), CodesError> { - pointer_guard::non_null!(index); - - let key = CString::new(key).unwrap(); - let value = CString::new(value).unwrap(); - - let _g = CODES_LOCK.lock().unwrap(); - let error_code = eccodes_sys::codes_index_select_string(index, key.as_ptr(), value.as_ptr()); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } - Ok(()) -} diff --git a/src/intermediate_bindings/codes_keys.rs b/src/intermediate_bindings/codes_keys.rs index 5ade02f..20db605 100644 --- a/src/intermediate_bindings/codes_keys.rs +++ b/src/intermediate_bindings/codes_keys.rs @@ -5,71 +5,72 @@ use std::ffi::{CStr, CString}; use eccodes_sys::{codes_handle, codes_keys_iterator}; -use num_traits::FromPrimitive; +use tracing::instrument; -use crate::{ - errors::{CodesError, CodesInternal}, - pointer_guard, -}; +use crate::{errors::CodesError, intermediate_bindings::error_code_to_result, pointer_guard}; +#[instrument(level = "trace")] pub unsafe fn codes_keys_iterator_new( handle: *mut codes_handle, flags: u32, namespace: &str, ) -> Result<*mut codes_keys_iterator, CodesError> { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let namespace = CString::new(namespace).unwrap(); + let namespace = CString::new(namespace).unwrap(); - let kiter = eccodes_sys::codes_keys_iterator_new(handle, u64::from(flags), namespace.as_ptr()); + let kiter = + eccodes_sys::codes_keys_iterator_new(handle, u64::from(flags), namespace.as_ptr()); - if kiter.is_null() { - return Err(CodesError::KeysIteratorFailed); - } + if kiter.is_null() { + return Err(CodesError::KeysIteratorFailed); + } - Ok(kiter) + Ok(kiter) + } } +#[instrument(level = "trace")] pub unsafe fn codes_keys_iterator_delete( keys_iterator: *mut codes_keys_iterator, ) -> Result<(), CodesError> { - #[cfg(test)] - log::trace!("codes_keys_iterator_delete"); - - if keys_iterator.is_null() { - return Ok(()); - } + unsafe { + if keys_iterator.is_null() { + return Ok(()); + } - let error_code = eccodes_sys::codes_keys_iterator_delete(keys_iterator); + let error_code = eccodes_sys::codes_keys_iterator_delete(keys_iterator); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(()) } - - Ok(()) } pub unsafe fn codes_keys_iterator_next( keys_iterator: *mut codes_keys_iterator, ) -> Result { - pointer_guard::non_null!(keys_iterator); + unsafe { + pointer_guard::non_null!(keys_iterator); - let next_item_exists = eccodes_sys::codes_keys_iterator_next(keys_iterator); + let next_item_exists = eccodes_sys::codes_keys_iterator_next(keys_iterator); - Ok(next_item_exists == 1) + Ok(next_item_exists == 1) + } } pub unsafe fn codes_keys_iterator_get_name( keys_iterator: *mut codes_keys_iterator, ) -> Result { - pointer_guard::non_null!(keys_iterator); + unsafe { + pointer_guard::non_null!(keys_iterator); - let name_pointer = eccodes_sys::codes_keys_iterator_get_name(keys_iterator); + let name_pointer = eccodes_sys::codes_keys_iterator_get_name(keys_iterator); - let name_c_str = CStr::from_ptr(name_pointer); - let name_str = name_c_str.to_str()?; - let name_string = name_str.to_owned(); + let name_c_str = CStr::from_ptr(name_pointer); + let name_str = name_c_str.to_str()?; + let name_string = name_str.to_owned(); - Ok(name_string) + Ok(name_string) + } } diff --git a/src/intermediate_bindings/codes_set.rs b/src/intermediate_bindings/codes_set.rs index 0ff0cce..fc763aa 100644 --- a/src/intermediate_bindings/codes_set.rs +++ b/src/intermediate_bindings/codes_set.rs @@ -5,30 +5,23 @@ use std::ffi::CString; use eccodes_sys::codes_handle; -use num_traits::FromPrimitive; - -use crate::{ - errors::{CodesError, CodesInternal}, - pointer_guard, -}; +use crate::{errors::CodesError, intermediate_bindings::error_code_to_result, pointer_guard}; pub unsafe fn codes_set_long( handle: *mut codes_handle, key: &str, value: i64, ) -> Result<(), CodesError> { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); + let key = CString::new(key).unwrap(); - let error_code = eccodes_sys::codes_set_long(handle, key.as_ptr(), value); + let error_code = eccodes_sys::codes_set_long(handle, key.as_ptr(), value); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(()) } - - Ok(()) } pub unsafe fn codes_set_double( @@ -36,18 +29,16 @@ pub unsafe fn codes_set_double( key: &str, value: f64, ) -> Result<(), CodesError> { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); + let key = CString::new(key).unwrap(); - let error_code = eccodes_sys::codes_set_double(handle, key.as_ptr(), value); + let error_code = eccodes_sys::codes_set_double(handle, key.as_ptr(), value); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(()) } - - Ok(()) } pub unsafe fn codes_set_long_array( @@ -55,25 +46,19 @@ pub unsafe fn codes_set_long_array( key: &str, values: &[i64], ) -> Result<(), CodesError> { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); + let key = CString::new(key).unwrap(); - let length = values.len(); + let length = values.len(); - let error_code = eccodes_sys::codes_set_long_array( - handle, - key.as_ptr(), - values.as_ptr().cast::<_>(), - length, - ); + let error_code = + eccodes_sys::codes_set_long_array(handle, key.as_ptr(), values.as_ptr().cast(), length); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(()) } - - Ok(()) } pub unsafe fn codes_set_double_array( @@ -81,25 +66,23 @@ pub unsafe fn codes_set_double_array( key: &str, values: &[f64], ) -> Result<(), CodesError> { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); + let key = CString::new(key).unwrap(); - let length = values.len(); + let length = values.len(); - let error_code = eccodes_sys::codes_set_double_array( - handle, - key.as_ptr(), - values.as_ptr().cast::<_>(), - length, - ); + let error_code = eccodes_sys::codes_set_double_array( + handle, + key.as_ptr(), + values.as_ptr().cast(), + length, + ); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(()) } - - Ok(()) } pub unsafe fn codes_set_string( @@ -107,21 +90,19 @@ pub unsafe fn codes_set_string( key: &str, value: &str, ) -> Result<(), CodesError> { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); - let mut length = value.len(); - let value = CString::new(value).unwrap(); + let key = CString::new(key).unwrap(); + let mut length = value.len(); + let value = CString::new(value).unwrap(); - let error_code = - eccodes_sys::codes_set_string(handle, key.as_ptr(), value.as_ptr(), &raw mut length); + let error_code = + eccodes_sys::codes_set_string(handle, key.as_ptr(), value.as_ptr(), &raw mut length); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(()) } - - Ok(()) } pub unsafe fn codes_set_bytes( @@ -129,23 +110,21 @@ pub unsafe fn codes_set_bytes( key: &str, values: &[u8], ) -> Result<(), CodesError> { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); + let key = CString::new(key).unwrap(); - let mut length = values.len(); + let mut length = values.len(); - let error_code = eccodes_sys::codes_set_bytes( - handle, - key.as_ptr(), - values.as_ptr().cast::<_>(), - &raw mut length, - ); + let error_code = eccodes_sys::codes_set_bytes( + handle, + key.as_ptr(), + values.as_ptr().cast(), + &raw mut length, + ); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(()) } - - Ok(()) } diff --git a/src/intermediate_bindings/grib_nearest.rs b/src/intermediate_bindings/grib_nearest.rs index 129cb83..8c51265 100644 --- a/src/intermediate_bindings/grib_nearest.rs +++ b/src/intermediate_bindings/grib_nearest.rs @@ -3,48 +3,43 @@ use std::ptr::addr_of_mut; -use eccodes_sys::{codes_handle, codes_nearest, CODES_NEAREST_SAME_DATA, CODES_NEAREST_SAME_GRID}; +use eccodes_sys::{CODES_NEAREST_SAME_DATA, CODES_NEAREST_SAME_GRID, codes_handle, codes_nearest}; -use num_traits::FromPrimitive; +use tracing::instrument; use crate::{ - errors::{CodesError, CodesInternal}, - pointer_guard, NearestGridpoint, + NearestGridpoint, errors::CodesError, intermediate_bindings::error_code_to_result, + pointer_guard, }; +#[instrument(level = "trace")] pub unsafe fn codes_grib_nearest_new( handle: *const codes_handle, ) -> Result<*mut codes_nearest, CodesError> { - pointer_guard::non_null!(handle); + unsafe { + pointer_guard::non_null!(handle); - let mut error_code: i32 = 0; + let mut error_code: i32 = 0; - let nearest = eccodes_sys::codes_grib_nearest_new(handle, &raw mut error_code); + let nearest = eccodes_sys::codes_grib_nearest_new(handle, &raw mut error_code); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(nearest) } - - Ok(nearest) } +#[instrument(level = "trace")] pub unsafe fn codes_grib_nearest_delete(nearest: *mut codes_nearest) -> Result<(), CodesError> { - #[cfg(test)] - log::trace!("codes_grib_nearest_delete"); - - if nearest.is_null() { - return Ok(()); - } + unsafe { + if nearest.is_null() { + return Ok(()); + } - let error_code = eccodes_sys::codes_grib_nearest_delete(nearest); + let error_code = eccodes_sys::codes_grib_nearest_delete(nearest); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(()) } - - Ok(()) } pub unsafe fn codes_grib_nearest_find( @@ -53,49 +48,47 @@ pub unsafe fn codes_grib_nearest_find( lat: f64, lon: f64, ) -> Result<[NearestGridpoint; 4], CodesError> { - pointer_guard::non_null!(handle); - pointer_guard::non_null!(nearest); - - // such flags are set because find nearest for given nearest is always - // called on the same grib message - let flags = CODES_NEAREST_SAME_GRID + CODES_NEAREST_SAME_DATA; - - let mut output_lats = [0_f64; 4]; - let mut output_lons = [0_f64; 4]; - let mut output_values = [0_f64; 4]; - let mut output_distances = [0_f64; 4]; - let mut output_indexes = [0_i32; 4]; - - let mut length: usize = 4; - - let error_code = eccodes_sys::codes_grib_nearest_find( - nearest, - handle, - lat, - lon, - u64::from(flags), - addr_of_mut!(output_lats[0]), - addr_of_mut!(output_lons[0]), - addr_of_mut!(output_values[0]), - addr_of_mut!(output_distances[0]), - addr_of_mut!(output_indexes[0]), - &raw mut length, - ); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + unsafe { + pointer_guard::non_null!(handle); + pointer_guard::non_null!(nearest); + + // such flags are set because find nearest for given nearest is always + // called on the same grib message + let flags = CODES_NEAREST_SAME_GRID + CODES_NEAREST_SAME_DATA; + + let mut output_lats = [0_f64; 4]; + let mut output_lons = [0_f64; 4]; + let mut output_values = [0_f64; 4]; + let mut output_distances = [0_f64; 4]; + let mut output_indexes = [0_i32; 4]; + + let mut length: usize = 4; + + let error_code = eccodes_sys::codes_grib_nearest_find( + nearest, + handle, + lat, + lon, + u64::from(flags), + addr_of_mut!(output_lats[0]), + addr_of_mut!(output_lons[0]), + addr_of_mut!(output_values[0]), + addr_of_mut!(output_distances[0]), + addr_of_mut!(output_indexes[0]), + &raw mut length, + ); + error_code_to_result(error_code)?; + + let mut output = [NearestGridpoint::default(); 4]; + + for i in 0..4 { + output[i].lat = output_lats[i]; + output[i].lon = output_lons[i]; + output[i].distance = output_distances[i]; + output[i].index = output_indexes[i]; + output[i].value = output_values[i]; + } + + Ok(output) } - - let mut output = [NearestGridpoint::default(); 4]; - - for i in 0..4 { - output[i].lat = output_lats[i]; - output[i].lon = output_lons[i]; - output[i].distance = output_distances[i]; - output[i].index = output_indexes[i]; - output[i].value = output_values[i]; - } - - Ok(output) } diff --git a/src/intermediate_bindings/mod.rs b/src/intermediate_bindings/mod.rs index a90e336..0338c1b 100644 --- a/src/intermediate_bindings/mod.rs +++ b/src/intermediate_bindings/mod.rs @@ -9,8 +9,6 @@ mod codes_get; mod codes_handle; -#[cfg(feature = "experimental_index")] -mod codes_index; mod codes_keys; mod codes_set; mod grib_nearest; @@ -32,14 +30,7 @@ pub use codes_get::{ codes_get_long_array, codes_get_message, codes_get_native_type, codes_get_size, codes_get_string, }; -#[cfg(feature = "experimental_index")] -pub use codes_handle::codes_handle_new_from_index; pub use codes_handle::{codes_handle_clone, codes_handle_delete, codes_handle_new_from_file}; -#[cfg(feature = "experimental_index")] -pub use codes_index::{ - codes_index_add_file, codes_index_delete, codes_index_new, codes_index_read, - codes_index_select_double, codes_index_select_long, codes_index_select_string, -}; pub use codes_keys::{ codes_keys_iterator_delete, codes_keys_iterator_get_name, codes_keys_iterator_new, codes_keys_iterator_next, @@ -51,3 +42,38 @@ pub use codes_set::{ pub use grib_nearest::{ codes_grib_nearest_delete, codes_grib_nearest_find, codes_grib_nearest_new, }; +use num_traits::FromPrimitive; + +use crate::{CodesError, errors::CodesInternal}; + +fn error_code_to_result(error_code: i32) -> Result<(), CodesError> { + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code) + .ok_or(CodesError::UnrecognizedErrorCode(error_code))?; + return Err(err.into()); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::{CodesError, errors::CodesInternal, intermediate_bindings::error_code_to_result}; + + #[test] + fn error_codes() { + let ok_result = error_code_to_result(0); + let defined_error = error_code_to_result(-2).unwrap_err(); // codes internal + let undefined_error = error_code_to_result(4).unwrap_err(); + + assert!(ok_result.is_ok()); + assert!(std::matches!( + defined_error, + CodesError::Internal(CodesInternal::CodesInternalError) + )); + assert!(std::matches!( + undefined_error, + CodesError::UnrecognizedErrorCode(4) + )); + } +} diff --git a/src/keyed_message/mod.rs b/src/keyed_message/mod.rs deleted file mode 100644 index 3dd81cc..0000000 --- a/src/keyed_message/mod.rs +++ /dev/null @@ -1,324 +0,0 @@ -//! Definition of `KeyedMessage` and its associated functions -//! used for reading and writing data of given variable from GRIB file - -mod read; -mod write; - -use eccodes_sys::codes_handle; -use log::error; -use std::ptr::null_mut; - -use crate::{ - intermediate_bindings::{ - codes_get_native_type, codes_get_size, codes_handle_clone, codes_handle_delete, - NativeKeyType, - }, - CodesError, -}; - -/// Structure that provides access to the data contained in the GRIB file, which directly corresponds to the message in the GRIB file -/// -/// **Usage examples are provided in documentation of each method.** -/// -/// You can think about the message as a container of data corresponding to a single variable -/// at given date, time and level. In ecCodes the message is represented as a collection of unique -/// key-value pairs. -/// -/// You can read a `Key` with static types using [`read_key()`](KeyRead::read_key()) or with [`DynamicKeyType`] using[`read_key_dynamic()`](KeyedMessage::read_key_dynamic()) -/// To iterate over all key names use [`KeysIterator`](crate::KeysIterator). You can also modify the message using -/// [`write_key()`](KeyWrite::write_key()). This crate can successfully read all keys from ERA5 and GFS files. -/// -/// If you are interested only in getting data values from the message you can use -/// [`to_ndarray()`](KeyedMessage::to_ndarray) from the [`message_ndarray`](crate::message_ndarray) module. -/// -/// Some of the useful keys are: `validityDate`, `validityTime`, `level`, `typeOfLevel`, `shortName`, `units` and `values`. -/// -/// Note that names, types and availability of some keys can vary between platforms and ecCodes versions. You should test -/// your code whenever changing the environment. -/// -/// [`CodesNearest`](crate::CodesNearest) can be used to find nearest gridpoints for given coordinates in the `KeyedMessage`. -/// -/// Most of `KeyedMessage` methods (except for writing) can be used directly with `&KeyedMessage` -/// returned by `CodesHandle` iterator, which provides the best performance. -/// When mutable access or longer liftime is needed the message can be cloned with [`try_clone`](KeyedMessage::try_clone) -/// Note that cloning comes with a performance and memory overhead. -/// You should take care that your system has enough memory before cloning. -/// -/// Destructor for this structure does not panic, but some internal functions may rarely fail -/// leading to bugs. Errors encountered in desctructor the are logged with [`log`]. -#[derive(Hash, Debug)] -pub struct KeyedMessage { - pub(crate) message_handle: *mut codes_handle, -} - -/// Provides GRIB key reading capabilites. Implemented by [`KeyedMessage`] for all possible key types. -pub trait KeyRead { - /// Tries to read a key of given name from [`KeyedMessage`]. This function checks if key native type - /// matches the requested type (ie. you cannot read integer as string, or array as a number). - /// - /// # Example - /// - /// ``` - /// # use eccodes::{ProductKind, CodesHandle, KeyRead}; - /// # use std::path::Path; - /// # use anyhow::Context; - /// # use eccodes::FallibleStreamingIterator; - /// # - /// # fn main() -> anyhow::Result<()> { - /// # let file_path = Path::new("./data/iceland.grib"); - /// # let product_kind = ProductKind::GRIB; - /// # - /// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - /// let message = handle.next()?.context("no message")?; - /// let short_name: String = message.read_key("shortName")?; - /// - /// assert_eq!(short_name, "msl"); - /// # Ok(()) - /// # } - /// ``` - /// - /// # Errors - /// - /// Returns [`WrongRequestedKeySize`](CodesError::WrongRequestedKeyType) when trying to read key in non-native type (use [`unchecked`](KeyRead::read_key_unchecked) instead). - /// - /// Returns [`WrongRequestedKeySize`](CodesError::WrongRequestedKeySize) when trying to read array as integer. - /// - /// Returns [`IncorrectKeySize`](CodesError::IncorrectKeySize) when key size is 0. This can indicate corrupted data. - /// - /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to read the key. - fn read_key(&self, name: &str) -> Result; - - /// Skips all the checks provided by [`read_key`](KeyRead::read_key) and directly calls ecCodes, ensuring only memory and type safety. - /// - /// This function has better perfomance than [`read_key`](KeyRead::read_key) but all error handling and (possible) - /// type conversions are performed directly by ecCodes. - /// - /// This function is also useful for (not usually used) keys that return incorrect native type. - /// - /// # Example - /// - /// ``` - /// # use eccodes::{ProductKind, CodesHandle, KeyRead}; - /// # use std::path::Path; - /// # use anyhow::Context; - /// # use eccodes::FallibleStreamingIterator; - /// # - /// # fn main() -> anyhow::Result<()> { - /// # let file_path = Path::new("./data/iceland.grib"); - /// # let product_kind = ProductKind::GRIB; - /// # - /// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - /// let message = handle.next()?.context("no message")?; - /// let short_name: String = message.read_key_unchecked("shortName")?; - /// - /// assert_eq!(short_name, "msl"); - /// # Ok(()) - /// # } - /// ``` - /// - /// # Errors - /// - /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to read the key. - fn read_key_unchecked(&self, name: &str) -> Result; -} - -/// Provides GRIB key writing capabilites. Implemented by [`KeyedMessage`] for all possible key types. -pub trait KeyWrite { - /// Writes key with given name and value to [`KeyedMessage`] overwriting existing value, unless - /// the key is read-only. This function directly calls ecCodes ensuring only type and memory safety. - /// - /// # Example - /// - /// ``` - /// # use eccodes::{ProductKind, CodesHandle, KeyWrite}; - /// # use std::path::Path; - /// # use anyhow::Context; - /// # use eccodes::FallibleStreamingIterator; - /// # - /// # fn main() -> anyhow::Result<()> { - /// # let file_path = Path::new("./data/iceland.grib"); - /// # let product_kind = ProductKind::GRIB; - /// # - /// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - /// - /// // CodesHandle iterator returns immutable messages. - /// // To edit a message it must be cloned. - /// let mut message = handle.next()?.context("no message")?.try_clone()?; - /// message.write_key("level", 1)?; - /// # Ok(()) - /// # } - /// ``` - /// - /// # Errors - /// - /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to write the key. - fn write_key(&mut self, name: &str, value: T) -> Result<(), CodesError>; -} - -/// Enum of types GRIB key can have. -/// -/// Messages inside GRIB files can contain keys of arbitrary types, which are known only at runtime (after being checked). -/// ecCodes can return several different types of key, which are represented by this enum -/// and each variant contains the respective data type. -#[derive(Clone, Debug, PartialEq)] -pub enum DynamicKeyType { - #[allow(missing_docs)] - Float(f64), - #[allow(missing_docs)] - Int(i64), - #[allow(missing_docs)] - FloatArray(Vec), - #[allow(missing_docs)] - IntArray(Vec), - #[allow(missing_docs)] - Str(String), - #[allow(missing_docs)] - Bytes(Vec), -} - -impl KeyedMessage { - /// Custom function to clone the `KeyedMessage`. This function comes with memory overhead. - /// - /// # Errors - /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to clone the message. - pub fn try_clone(&self) -> Result { - let new_handle = unsafe { codes_handle_clone(self.message_handle)? }; - - Ok(Self { - message_handle: new_handle, - }) - } - - fn get_key_size(&self, key_name: &str) -> Result { - unsafe { codes_get_size(self.message_handle, key_name) } - } - - fn get_key_native_type(&self, key_name: &str) -> Result { - unsafe { codes_get_native_type(self.message_handle, key_name) } - } -} - -#[doc(hidden)] -impl Drop for KeyedMessage { - /// Executes the destructor for this type. - /// This method calls destructor functions from ecCodes library. - /// In some edge cases these functions can return non-zero code. - /// In such case all pointers and file descriptors are safely deleted. - /// However memory leaks can still occur. - /// - /// If any function called in the destructor returns an error warning will appear in log. - /// If bugs occur during `CodesHandle` drop please enable log output and post issue on [Github](https://github.com/ScaleWeather/eccodes). - /// - /// Technical note: delete functions in ecCodes can only fail with [`CodesInternalError`](crate::errors::CodesInternal::CodesInternalError) - /// when other functions corrupt the inner memory of pointer, in that case memory leak is possible. - /// In case of corrupt pointer segmentation fault will occur. - /// The pointers are cleared at the end of drop as they are not functional regardless of result of delete functions. - fn drop(&mut self) { - unsafe { - codes_handle_delete(self.message_handle).unwrap_or_else(|error| { - error!("codes_handle_delete() returned an error: {:?}", &error); - }); - } - - self.message_handle = null_mut(); - } -} - -#[cfg(test)] -mod tests { - use crate::codes_handle::{CodesHandle, ProductKind}; - use crate::FallibleStreamingIterator; - use anyhow::{Context, Result}; - use std::path::Path; - use testing_logger; - - #[test] - fn check_docs_keys() -> Result<()> { - let file_path = Path::new("./data/iceland.grib"); - let product_kind = ProductKind::GRIB; - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; - - let _ = current_message.read_key_dynamic("validityDate")?; - let _ = current_message.read_key_dynamic("validityTime")?; - let _ = current_message.read_key_dynamic("level")?; - let _ = current_message.read_key_dynamic("shortName")?; - let _ = current_message.read_key_dynamic("units")?; - let _ = current_message.read_key_dynamic("values")?; - let _ = current_message.read_key_dynamic("typeOfLevel")?; - - Ok(()) - } - - #[test] - fn message_clone_1() -> Result<()> { - let file_path = Path::new("./data/iceland.grib"); - let product_kind = ProductKind::GRIB; - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; - let cloned_message = current_message.try_clone()?; - - assert_ne!( - current_message.message_handle, - cloned_message.message_handle - ); - - Ok(()) - } - - #[test] - fn message_clone_2() -> Result<()> { - let file_path = Path::new("./data/iceland.grib"); - let product_kind = ProductKind::GRIB; - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let msg = handle.next()?.context("Message not some")?.try_clone()?; - let _ = handle.next()?; - - drop(handle); - - let _ = msg.read_key_dynamic("dataDate")?; - let _ = msg.read_key_dynamic("jDirectionIncrementInDegrees")?; - let _ = msg.read_key_dynamic("values")?; - let _ = msg.read_key_dynamic("name")?; - let _ = msg.read_key_dynamic("section1Padding")?; - let _ = msg.read_key_dynamic("experimentVersionNumber")?; - - Ok(()) - } - - #[test] - fn message_clone_drop() -> Result<()> { - testing_logger::setup(); - let file_path = Path::new("./data/iceland.grib"); - let product_kind = ProductKind::GRIB; - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let _msg_ref = handle.next()?.context("Message not some")?; - let _msg_clone = _msg_ref.try_clone()?; - - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 0); - }); - - drop(_msg_clone); - - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 1); - assert_eq!(captured_logs[0].body, "codes_handle_delete"); - assert_eq!(captured_logs[0].level, log::Level::Trace); - }); - - drop(handle); - - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 1); - assert_eq!(captured_logs[0].body, "codes_handle_delete"); - assert_eq!(captured_logs[0].level, log::Level::Trace); - }); - - Ok(()) - } -} diff --git a/src/keyed_message/read.rs b/src/keyed_message/read.rs deleted file mode 100644 index 2187ef7..0000000 --- a/src/keyed_message/read.rs +++ /dev/null @@ -1,416 +0,0 @@ -use std::cmp::Ordering; - -use crate::{ - errors::CodesError, - intermediate_bindings::{ - codes_get_bytes, codes_get_double, codes_get_double_array, codes_get_long, - codes_get_long_array, codes_get_native_type, codes_get_size, codes_get_string, - NativeKeyType, - }, - DynamicKeyType, KeyRead, KeyedMessage, -}; - -impl KeyRead for KeyedMessage { - fn read_key(&self, key_name: &str) -> Result { - match self.get_key_native_type(key_name)? { - NativeKeyType::Long => (), - _ => return Err(CodesError::WrongRequestedKeyType), - } - - let key_size = self.get_key_size(key_name)?; - - match key_size.cmp(&1) { - Ordering::Greater => return Err(CodesError::WrongRequestedKeySize), - Ordering::Less => return Err(CodesError::IncorrectKeySize), - Ordering::Equal => (), - } - - self.read_key_unchecked(key_name) - } - - fn read_key_unchecked(&self, key_name: &str) -> Result { - unsafe { codes_get_long(self.message_handle, key_name) } - } -} - -impl KeyRead for KeyedMessage { - fn read_key(&self, key_name: &str) -> Result { - match self.get_key_native_type(key_name)? { - NativeKeyType::Double => (), - _ => return Err(CodesError::WrongRequestedKeyType), - } - - let key_size = self.get_key_size(key_name)?; - - match key_size.cmp(&1) { - Ordering::Greater => return Err(CodesError::WrongRequestedKeySize), - Ordering::Less => return Err(CodesError::IncorrectKeySize), - Ordering::Equal => (), - } - - self.read_key_unchecked(key_name) - } - - fn read_key_unchecked(&self, key_name: &str) -> Result { - unsafe { codes_get_double(self.message_handle, key_name) } - } -} - -impl KeyRead for KeyedMessage { - fn read_key(&self, key_name: &str) -> Result { - match self.get_key_native_type(key_name)? { - NativeKeyType::Str => (), - _ => return Err(CodesError::WrongRequestedKeyType), - } - - let key_size = self.get_key_size(key_name)?; - - if key_size < 1 { - return Err(CodesError::IncorrectKeySize); - } - - self.read_key_unchecked(key_name) - } - - fn read_key_unchecked(&self, key_name: &str) -> Result { - unsafe { codes_get_string(self.message_handle, key_name) } - } -} - -impl KeyRead> for KeyedMessage { - fn read_key(&self, key_name: &str) -> Result, CodesError> { - match self.get_key_native_type(key_name)? { - NativeKeyType::Long => (), - _ => return Err(CodesError::WrongRequestedKeyType), - } - - let key_size = self.get_key_size(key_name)?; - - if key_size < 1 { - return Err(CodesError::IncorrectKeySize); - } - - self.read_key_unchecked(key_name) - } - - fn read_key_unchecked(&self, key_name: &str) -> Result, CodesError> { - unsafe { codes_get_long_array(self.message_handle, key_name) } - } -} - -impl KeyRead> for KeyedMessage { - fn read_key(&self, key_name: &str) -> Result, CodesError> { - match self.get_key_native_type(key_name)? { - NativeKeyType::Double => (), - _ => return Err(CodesError::WrongRequestedKeyType), - } - - let key_size = self.get_key_size(key_name)?; - - if key_size < 1 { - return Err(CodesError::IncorrectKeySize); - } - - self.read_key_unchecked(key_name) - } - - fn read_key_unchecked(&self, key_name: &str) -> Result, CodesError> { - unsafe { codes_get_double_array(self.message_handle, key_name) } - } -} - -impl KeyRead> for KeyedMessage { - fn read_key(&self, key_name: &str) -> Result, CodesError> { - match self.get_key_native_type(key_name)? { - NativeKeyType::Bytes => (), - _ => return Err(CodesError::WrongRequestedKeyType), - } - - let key_size = self.get_key_size(key_name)?; - - if key_size < 1 { - return Err(CodesError::IncorrectKeySize); - } - - self.read_key_unchecked(key_name) - } - - fn read_key_unchecked(&self, key_name: &str) -> Result, CodesError> { - unsafe { codes_get_bytes(self.message_handle, key_name) } - } -} - -impl KeyedMessage { - /// Method to get a value of given key with [`DynamicKeyType`] from the `KeyedMessage`, if it exists. - /// - /// In most cases you should use [`read_key()`](KeyRead::read_key) due to more predictive behaviour - /// and simpler interface. - /// - /// This function exists for backwards compatibility and user convienience. - /// - /// This function checks the type of requested key and tries to read it as the native type. - /// That flow adds performance overhead, but makes the function highly unlikely to fail. - /// - /// This function will try to retrieve the key of native string type as string even - /// when the nul byte is not positioned at the end of key value. - /// - /// If retrieving the key value in native type fails this function will try to read - /// the requested key as bytes. - /// - /// # Example - /// - /// ``` - /// use eccodes::{ProductKind, CodesHandle, DynamicKeyType}; - /// # use std::path::Path; - /// # use anyhow::Context; - /// use eccodes::FallibleStreamingIterator; - /// # - /// # fn main() -> anyhow::Result<()> { - /// let file_path = Path::new("./data/iceland.grib"); - /// let product_kind = ProductKind::GRIB; - /// - /// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - /// let message = handle.next()?.context("no message")?; - /// let message_short_name = message.read_key_dynamic("shortName")?; - /// let expected_short_name = DynamicKeyType::Str("msl".to_string()); - /// - /// assert_eq!(message_short_name, expected_short_name); - /// # Ok(()) - /// # } - /// ``` - /// - /// # Errors - /// - /// Returns [`CodesNotFound`](crate::errors::CodesInternal::CodesNotFound) - /// when a key of given name has not been found in the message. - /// - /// Returns [`CodesError::MissingKey`] when a given key does not have a specified type. - /// - /// Returns [`CodesError::Internal`] when one of internal ecCodes functions to read the key fails. - /// - /// Returns [`CodesError::CstrUTF8`] and [`CodesError::NulChar`] when the string returned by ecCodes - /// library cannot be parsed as valid UTF8 Rust string. - /// - /// Returns [`CodesError::IncorrectKeySize`] when the size of given key is lower than 1. This indicates corrupted data file, - /// bug in the crate or bug in the ecCodes library. If you encounter this error please check - /// if your file is correct and report it on Github. - pub fn read_key_dynamic(&self, key_name: &str) -> Result { - let key_type; - - unsafe { - key_type = codes_get_native_type(self.message_handle, key_name)?; - } - - let key_value = match key_type { - NativeKeyType::Long => { - let key_size; - unsafe { key_size = codes_get_size(self.message_handle, key_name)? } - - if key_size == 1 { - let value; - unsafe { - value = codes_get_long(self.message_handle, key_name); - } - - match value { - Ok(val) => Ok(DynamicKeyType::Int(val)), - Err(err) => Err(err), - } - } else if key_size >= 2 { - let value; - unsafe { - value = codes_get_long_array(self.message_handle, key_name); - } - - match value { - Ok(val) => Ok(DynamicKeyType::IntArray(val)), - Err(err) => Err(err), - } - } else { - return Err(CodesError::IncorrectKeySize); - } - } - NativeKeyType::Double => { - let key_size; - unsafe { key_size = codes_get_size(self.message_handle, key_name)? } - - if key_size == 1 { - let value; - unsafe { - value = codes_get_double(self.message_handle, key_name); - } - - match value { - Ok(val) => Ok(DynamicKeyType::Float(val)), - Err(err) => Err(err), - } - } else if key_size >= 2 { - let value; - unsafe { - value = codes_get_double_array(self.message_handle, key_name); - } - - match value { - Ok(val) => Ok(DynamicKeyType::FloatArray(val)), - Err(err) => Err(err), - } - } else { - return Err(CodesError::IncorrectKeySize); - } - } - NativeKeyType::Bytes => { - let value; - unsafe { - value = codes_get_bytes(self.message_handle, key_name); - } - - match value { - Ok(val) => Ok(DynamicKeyType::Bytes(val)), - Err(err) => Err(err), - } - } - NativeKeyType::Missing => return Err(CodesError::MissingKey), - _ => { - let value; - unsafe { - value = codes_get_string(self.message_handle, key_name); - } - - match value { - Ok(val) => Ok(DynamicKeyType::Str(val)), - Err(err) => Err(err), - } - } - }; - - if let Ok(value) = key_value { - Ok(value) - } else { - let value; - unsafe { - value = codes_get_bytes(self.message_handle, key_name)?; - } - - Ok(DynamicKeyType::Bytes(value)) - } - } -} - -#[cfg(test)] -mod tests { - use anyhow::{Context, Result}; - - use crate::codes_handle::{CodesHandle, ProductKind}; - use crate::{DynamicKeyType, FallibleIterator, FallibleStreamingIterator}; - use std::path::Path; - - #[test] - fn key_reader() -> Result<()> { - let file_path = Path::new("./data/iceland.grib"); - let product_kind = ProductKind::GRIB; - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - - let current_message = handle.next()?.context("Message not some")?; - - let str_key = current_message.read_key_dynamic("name")?; - - match str_key { - DynamicKeyType::Str(_) => {} - _ => panic!("Incorrect variant of string key"), - } - - let double_key = current_message.read_key_dynamic("jDirectionIncrementInDegrees")?; - match double_key { - DynamicKeyType::Float(_) => {} - _ => panic!("Incorrect variant of double key"), - } - - let long_key = current_message.read_key_dynamic("numberOfPointsAlongAParallel")?; - - match long_key { - DynamicKeyType::Int(_) => {} - _ => panic!("Incorrect variant of long key"), - } - - let double_arr_key = current_message.read_key_dynamic("values")?; - - match double_arr_key { - DynamicKeyType::FloatArray(_) => {} - _ => panic!("Incorrect variant of double array key"), - } - - Ok(()) - } - - #[test] - fn era5_keys_dynamic() -> Result<()> { - let file_path = Path::new("./data/iceland.grib"); - let product_kind = ProductKind::GRIB; - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; - let mut kiter = current_message.default_keys_iterator()?; - - while let Some(key_name) = kiter.next()? { - assert!(!key_name.is_empty()); - assert!(current_message.read_key_dynamic(&key_name).is_ok()) - } - - Ok(()) - } - - #[test] - fn gfs_keys_dynamic() -> Result<()> { - let file_path = Path::new("./data/gfs.grib"); - let product_kind = ProductKind::GRIB; - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; - let mut kiter = current_message.default_keys_iterator()?; - - while let Some(key_name) = kiter.next()? { - assert!(!key_name.is_empty()); - assert!(current_message.read_key_dynamic(&key_name).is_ok()) - } - - Ok(()) - } - - #[test] - fn missing_key() -> Result<()> { - let file_path = Path::new("./data/iceland.grib"); - let product_kind = ProductKind::GRIB; - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; - - let missing_key = current_message.read_key_dynamic("doesNotExist"); - - assert!(missing_key.is_err()); - - Ok(()) - } - - #[test] - fn benchmark_keys() -> Result<()> { - let file_path = Path::new("./data/iceland.grib"); - let product_kind = ProductKind::GRIB; - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - - let msg = handle.next()?.context("Message not some")?; - - let _ = msg.read_key_dynamic("dataDate")?; - let _ = msg.read_key_dynamic("jDirectionIncrementInDegrees")?; - let _ = msg.read_key_dynamic("values")?; - let _ = msg.read_key_dynamic("name")?; - let _ = msg.read_key_dynamic("section1Padding")?; - let _ = msg.read_key_dynamic("experimentVersionNumber")?; - let _ = msg - .read_key_dynamic("zero") - .unwrap_or_else(|_| msg.read_key_dynamic("zeros").unwrap()); - - Ok(()) - } -} diff --git a/src/keyed_message/write.rs b/src/keyed_message/write.rs deleted file mode 100644 index 31cffba..0000000 --- a/src/keyed_message/write.rs +++ /dev/null @@ -1,244 +0,0 @@ -use std::{fs::OpenOptions, io::Write, path::Path, slice}; - -use crate::{ - errors::CodesError, - intermediate_bindings::{ - codes_get_message, codes_set_bytes, codes_set_double, codes_set_double_array, - codes_set_long, codes_set_long_array, codes_set_string, - }, - KeyedMessage, -}; - -use super::KeyWrite; - -impl KeyWrite for KeyedMessage { - fn write_key(&mut self, name: &str, value: i64) -> Result<(), CodesError> { - unsafe { codes_set_long(self.message_handle, name, value) } - } -} - -impl KeyWrite for KeyedMessage { - fn write_key(&mut self, name: &str, value: f64) -> Result<(), CodesError> { - unsafe { codes_set_double(self.message_handle, name, value) } - } -} - -impl KeyWrite<&[i64]> for KeyedMessage { - fn write_key(&mut self, name: &str, value: &[i64]) -> Result<(), CodesError> { - unsafe { codes_set_long_array(self.message_handle, name, value) } - } -} - -impl KeyWrite<&[f64]> for KeyedMessage { - fn write_key(&mut self, name: &str, value: &[f64]) -> Result<(), CodesError> { - unsafe { codes_set_double_array(self.message_handle, name, value) } - } -} - -impl KeyWrite<&[u8]> for KeyedMessage { - fn write_key(&mut self, name: &str, value: &[u8]) -> Result<(), CodesError> { - unsafe { codes_set_bytes(self.message_handle, name, value) } - } -} - -impl KeyWrite<&Vec> for KeyedMessage { - fn write_key(&mut self, name: &str, value: &Vec) -> Result<(), CodesError> { - unsafe { codes_set_long_array(self.message_handle, name, value) } - } -} - -impl KeyWrite<&Vec> for KeyedMessage { - fn write_key(&mut self, name: &str, value: &Vec) -> Result<(), CodesError> { - unsafe { codes_set_double_array(self.message_handle, name, value) } - } -} - -impl KeyWrite<&Vec> for KeyedMessage { - fn write_key(&mut self, name: &str, value: &Vec) -> Result<(), CodesError> { - unsafe { codes_set_bytes(self.message_handle, name, value) } - } -} - -impl KeyWrite<&str> for KeyedMessage { - fn write_key(&mut self, name: &str, value: &str) -> Result<(), CodesError> { - unsafe { codes_set_string(self.message_handle, name, value) } - } -} - -impl KeyWrite<&String> for KeyedMessage { - fn write_key(&mut self, name: &str, value: &String) -> Result<(), CodesError> { - unsafe { codes_set_string(self.message_handle, name, value) } - } -} - -impl KeyedMessage { - /// Function to write given `KeyedMessage` to a file at provided path. - /// If file does not exists it will be created. - /// If `append` is set to `true` file will be opened in append mode - /// and no data will be overwritten (useful when writing mutiple messages to one file). - /// - /// # Example - /// - /// ``` - /// use eccodes::{CodesHandle, KeyRead, ProductKind}; - /// # use eccodes::errors::CodesError; - /// use eccodes::FallibleStreamingIterator; - /// # use std::path::Path; - /// # use std::fs::remove_file; - /// # - /// # fn main() -> anyhow::Result<(), CodesError> { - /// let in_path = Path::new("./data/iceland-levels.grib"); - /// let out_path = Path::new("./data/iceland-800hPa.grib"); - /// - /// let mut handle = CodesHandle::new_from_file(in_path, ProductKind::GRIB)?; - /// - /// while let Some(msg) = handle.next()? { - /// let level: i64 = msg.read_key("level")?; - /// if level == 800 { - /// msg.write_to_file(out_path, true)?; - /// } - /// } - /// # remove_file(out_path)?; - /// # Ok(()) - /// # } - /// ``` - /// - /// # Errors - /// - /// Returns [`CodesError::FileHandlingInterrupted`] when the file cannot be opened, - /// created or correctly written. - /// - /// Returns [`CodesInternal`](crate::errors::CodesInternal) - /// when internal ecCodes function returns non-zero code. - pub fn write_to_file>( - &self, - file_path: P, - append: bool, - ) -> Result<(), CodesError> { - let msg = unsafe { codes_get_message(self.message_handle)? }; - let buf = unsafe { slice::from_raw_parts(msg.0.cast::<_>(), msg.1) }; - let mut file = OpenOptions::new() - .write(true) - .create(true) - .append(append) - .open(file_path)?; - - file.write_all(buf)?; - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use anyhow::{Context, Result}; - - use crate::{ - codes_handle::{CodesHandle, ProductKind}, - DynamicKeyType, FallibleStreamingIterator, KeyWrite, - }; - use std::{fs::remove_file, path::Path}; - - #[test] - fn write_message_ref() -> Result<()> { - let file_path = Path::new("./data/iceland.grib"); - let product_kind = ProductKind::GRIB; - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - - let current_message = handle.next()?.context("Message not some")?; - let out_path = Path::new("./data/iceland_write.grib"); - current_message.write_to_file(out_path, false)?; - - remove_file(out_path)?; - - Ok(()) - } - - #[test] - fn write_message_clone() -> Result<()> { - let file_path = Path::new("./data/iceland.grib"); - let product_kind = ProductKind::GRIB; - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?.try_clone()?; - - drop(handle); - - let out_path = Path::new("./data/iceland_write_clone.grib"); - current_message.write_to_file(out_path, false)?; - - remove_file(out_path)?; - - Ok(()) - } - - #[test] - fn append_message() -> Result<()> { - let product_kind = ProductKind::GRIB; - let out_path = Path::new("./data/iceland_append.grib"); - - let file_path = Path::new("./data/iceland-surface.grib"); - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; - current_message.write_to_file(out_path, false)?; - - let file_path = Path::new("./data/iceland-levels.grib"); - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; - current_message.write_to_file(out_path, true)?; - - remove_file(out_path)?; - - Ok(()) - } - - #[test] - fn write_key() -> Result<()> { - let product_kind = ProductKind::GRIB; - let file_path = Path::new("./data/iceland.grib"); - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let mut current_message = handle.next()?.context("Message not some")?.try_clone()?; - - let old_key = current_message.read_key_dynamic("centre")?; - - current_message.write_key("centre", "cnmc")?; - - let read_key = current_message.read_key_dynamic("centre")?; - - assert_ne!(old_key, read_key); - assert_eq!(read_key, DynamicKeyType::Str("cnmc".into())); - - Ok(()) - } - - #[test] - fn edit_keys_and_save() -> Result<()> { - let product_kind = ProductKind::GRIB; - let file_path = Path::new("./data/iceland.grib"); - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let mut current_message = handle.next()?.context("Message not some")?.try_clone()?; - - let old_key = current_message.read_key_dynamic("centre")?; - - current_message.write_key("centre", "cnmc")?; - - current_message.write_to_file(Path::new("./data/iceland_edit.grib"), false)?; - - let file_path = Path::new("./data/iceland_edit.grib"); - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; - - let read_key = current_message.read_key_dynamic("centre")?; - - assert_ne!(old_key, read_key); - assert_eq!(read_key, DynamicKeyType::Str("cnmc".into())); - - remove_file(Path::new("./data/iceland_edit.grib"))?; - - Ok(()) - } -} diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index 51ca4a8..9079716 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -1,20 +1,20 @@ -//! Definition of `KeysIterator` used for iterating through keys in `KeyedMessage` +//! Definition of `KeysIterator` used for iterating through keys in `CodesMessage` use eccodes_sys::codes_keys_iterator; use fallible_iterator::FallibleIterator; -use log::error; -use std::{marker::PhantomData, ptr::null_mut}; +use std::{fmt::Debug, marker::PhantomData, ptr::null_mut}; +use tracing::{Level, event, instrument}; use crate::{ + codes_message::CodesMessage, errors::CodesError, intermediate_bindings::{ codes_keys_iterator_delete, codes_keys_iterator_get_name, codes_keys_iterator_new, codes_keys_iterator_next, }, - KeyedMessage, }; -/// Structure to iterate through key names in [`KeyedMessage`]. +/// Structure to iterate through key names in [`CodesMessage`]. /// /// Mainly useful to discover what keys are present inside the message. /// @@ -28,26 +28,19 @@ use crate::{ /// ## Example /// /// ``` -/// use eccodes::{ProductKind, CodesHandle, KeyedMessage, KeysIteratorFlags}; -/// # use std::path::Path; -/// # use anyhow::Context; -/// use eccodes::{FallibleIterator, FallibleStreamingIterator}; -/// # -/// # fn main() -> anyhow::Result<()> { -/// # -/// let file_path = Path::new("./data/iceland.grib"); -/// let product_kind = ProductKind::GRIB; -/// -/// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; -/// let current_message = handle.next()?.context("no message")?; -/// -/// let mut keys_iter = current_message.default_keys_iterator()?; -/// -/// while let Some(key_name) = keys_iter.next()? { -/// println!("{key_name}"); -/// } -/// # Ok(()) -/// # } +/// # use anyhow::Context; +/// # use eccodes::{CodesFile, FallibleIterator, ProductKind}; +/// # fn main() -> anyhow::Result<()> { +/// let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; +/// let mut current_message = handle.ref_message_iter().next()?.context("no message")?; +/// +/// let mut keys_iter = current_message.default_keys_iterator()?; +/// +/// while let Some(key_name) = keys_iter.next()? { +/// println!("{key_name}"); +/// } +/// # Ok(()) +/// # } /// ``` /// /// ## Errors @@ -57,7 +50,8 @@ use crate::{ #[allow(clippy::module_name_repetitions)] #[derive(Debug)] pub struct KeysIterator<'a> { - parent_message: PhantomData<&'a KeyedMessage>, + /// Same trick as in `RefMessage` + parent_message: PhantomData<&'a ()>, iterator_handle: *mut codes_keys_iterator, next_item_exists: bool, } @@ -87,30 +81,26 @@ pub enum KeysIteratorFlags { SkipEditionSpecific = eccodes_sys::CODES_KEYS_ITERATOR_SKIP_EDITION_SPECIFIC as isize, } -impl KeyedMessage { +impl CodesMessage

{ /// Creates new [`KeysIterator`] for the message with specified flags and namespace. /// /// The flags are set by providing any combination of [`KeysIteratorFlags`] /// inside a slice. Check the documentation for the details of each flag meaning. /// /// Namespace is set simply as string, eg. `"ls"`, `"time"`, `"parameter"`, `"geography"`, `"statistics"`. + /// Empty string "" will return all keys. /// Invalid namespace will result in empty iterator. /// /// # Example /// /// ``` - /// use eccodes::{ProductKind, CodesHandle, KeyedMessage, KeysIteratorFlags}; - /// # use std::path::Path; + /// use eccodes::{ProductKind, CodesFile, KeysIteratorFlags, FallibleIterator}; /// # use anyhow::Context; - /// use eccodes::{FallibleIterator, FallibleStreamingIterator}; /// # /// # fn main() -> anyhow::Result<()> { /// # - /// let file_path = Path::new("./data/iceland.grib"); - /// let product_kind = ProductKind::GRIB; - /// - /// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - /// let current_message = handle.next()?.context("no message")?; + /// let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; + /// let mut current_message = handle.ref_message_iter().next()?.context("no message")?; /// /// let flags = [ /// KeysIteratorFlags::AllKeys, @@ -134,8 +124,9 @@ impl KeyedMessage { /// /// This function returns [`CodesInternal`](crate::errors::CodesInternal) when /// internal ecCodes function returns non-zero code. + #[instrument(level = "trace")] pub fn new_keys_iterator<'a>( - &'a self, + &'a mut self, flags: &[KeysIteratorFlags], namespace: &str, ) -> Result, CodesError> { @@ -152,7 +143,7 @@ impl KeyedMessage { }) } - /// Same as [`new_keys_iterator()`](KeyedMessage::new_keys_iterator) but with default + /// Same as [`new_keys_iterator()`](CodesMessage::new_keys_iterator) but with default /// parameters: [`AllKeys`](KeysIteratorFlags::AllKeys) flag and `""` namespace, /// yeilding iterator over all keys in the message. /// @@ -160,7 +151,7 @@ impl KeyedMessage { /// /// This function returns [`CodesInternal`](crate::errors::CodesInternal) when /// internal ecCodes function returns non-zero code. - pub fn default_keys_iterator(&self) -> Result, CodesError> { + pub fn default_keys_iterator(&mut self) -> Result, CodesError> { let iterator_handle = unsafe { codes_keys_iterator_new(self.message_handle, 0, "")? }; let next_item_exists = unsafe { codes_keys_iterator_next(iterator_handle)? }; @@ -197,13 +188,17 @@ impl FallibleIterator for KeysIterator<'_> { #[doc(hidden)] impl Drop for KeysIterator<'_> { + #[instrument(level = "trace")] fn drop(&mut self) { unsafe { codes_keys_iterator_delete(self.iterator_handle).unwrap_or_else(|error| { - error!( + event!( + Level::ERROR, "codes_keys_iterator_delete() returned an error: {:?}", &error ); + #[cfg(test)] + panic!("Error in KeysIterator::drop"); }); } @@ -215,8 +210,8 @@ impl Drop for KeysIterator<'_> { mod tests { use anyhow::{Context, Result}; - use crate::codes_handle::{CodesHandle, ProductKind}; - use crate::{FallibleIterator, FallibleStreamingIterator}; + use crate::FallibleIterator; + use crate::codes_file::{CodesFile, ProductKind}; use std::path::Path; use super::KeysIteratorFlags; @@ -226,8 +221,11 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let mut current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; let flags = [ KeysIteratorFlags::AllKeys, //0 @@ -251,8 +249,11 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let mut current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; let flags = vec![ KeysIteratorFlags::AllKeys, //0 @@ -274,26 +275,30 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let mut current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; let _kiter = current_message.default_keys_iterator()?; - drop(_kiter); + Ok(()) + } - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 1); - assert_eq!(captured_logs[0].body, "codes_keys_iterator_delete"); - assert_eq!(captured_logs[0].level, log::Level::Trace); - }); + #[test] + fn destructor_null() -> Result<()> { + let file_path = Path::new("./data/iceland.grib"); + let product_kind = ProductKind::GRIB; - drop(handle); + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let mut current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 1); - assert_eq!(captured_logs[0].body, "codes_handle_delete"); - assert_eq!(captured_logs[0].level, log::Level::Trace); - }); + let mut kiter = current_message.default_keys_iterator()?; + kiter.iterator_handle = std::ptr::null_mut(); Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index 65d4f16..34df109 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,9 +17,11 @@ //! //! This crate contains (mostly) safe high-level bindings for ecCodes library. //! Bindings can be considered safe mainly because all crate structures -//! will take ownership of the data in memory before passing the raw pointer to ecCodes. +//! take ownership of the data in memory before passing the raw pointer to the ecCodes. //! -//! **Currently only reading of GRIB files is supported.** +//! **Currently only operations on GRIB files are supported.** +//! +//! **Version 0.14 introduces breaking changes, [check them below](#changes-in-version-014)!** //! //! Because of the ecCodes library API characteristics theses bindings are //! rather thick wrapper to make this crate safe and convenient to use. @@ -40,10 +42,10 @@ //! this allows the user to handle the error in the way that suits them best and not risk crashes. //! //! All error descriptions are provided in the [`errors`] module. -//! Destructors, which cannot panic, report errors through the `log` crate. +//! Destructors, which cannot panic, report errors through `tracing` and `log` crate. //! //! None of the functions in this crate explicitly panics. -//! However, users should be aware that dependencies might panic in some edge cases. +//! However, users should be aware that dependencies (eg. `ndarray`) might panic in some edge cases. //! //! ## Safety //! @@ -53,183 +55,219 @@ //! That said, neither main developer nor contributors have expertise in unsafe Rust and bugs might have //! slipped through. We are also not responsible for bugs in the ecCodes library. //! -//! If you find a bug or have a suggestion, feel free to discuss it on Github. -//! -//! ## Features -//! -//! - `message_ndarray` - enables support for converting [`KeyedMessage`] to [`ndarray::Array`]. -//! This feature is enabled by default. It is currently tested only with simple lat-lon grids. -//! -//! - `experimental_index` - enables support for creating and using index files for GRIB files. -//! **This feature is experimental** and disabled by default. If you want to use it, please read -//! the information provided in [`codes_index`] documentation. +//! **For critical applications always perform extensive testing before using this crate in production.** //! -//! - `docs` - builds the crate without linking ecCodes, particularly useful when building the documentation -//! on [docs.rs](https://docs.rs/). For more details check documentation of [eccodes-sys](https://crates.io/crates/eccodes-sys). +//! If you find a bug or have a suggestion, feel free to discuss it on Github. //! -//! To build your own crate with this crate as dependency on docs.rs without linking ecCodes add following lines to your `Cargo.toml` +//! ## Usage //! -//! ```text -//! [package.metadata.docs.rs] -//! features = ["eccodes/docs"] -//! ``` +//! To access a GRIB file you need to create [`CodesFile`] with one of the provided constructors. //! -//! ## Usage +//! ecCodes represents GRIB files as a set of separate messages, each containing data fields at specific time and level. +//! Messages are represented here by a generic [`CodesMessage`](codes_message::CodesMessage) structure, but you shouldn't use it directly. +//! Instead use [`RefMessage`], [`ArcMessage`] or [`BufMessage`] to operations - check the docs for more information when to use each. //! -//! To access a GRIB file you need to create [`CodesHandle`] with one of provided constructors. +//! To obtain `CodesMessage` from `CodesFile` you need to create an instance of [`RefMessageIter`] or [`ArcMessageIter`] using +//! [`CodesFile::ref_message_iter()`] or [`CodesFile::arc_message_iter()`]. +//! Those structures implement [`FallibleIterator`], please check its documentation if you are not familiar with it. //! -//! GRIB files consist of messages which represent data fields at specific time and level. -//! Messages are represented by the [`KeyedMessage`] structure. +//! `CodesMessage` implements several methods to access the data as needed, most of those can be called directly. +//! Almost all methods can be called on any `CodesMessage`, except for [`KeyWrite`] operations, which can be called only on [`BufMessage`] +//! to avoid confusion if written keys are save to file or not. //! -//! [`CodesHandle`] implements [`FallibleStreamingIterator`](CodesHandle#impl-FallibleStreamingIterator-for-CodesHandle) -//! which allows you to iterate over messages in the file. The iterator returns `&KeyedMessage` which valid is until next iteration. -//! `KeyedMessage` implements several methods to access the data as needed, most of those can be called directly on `&KeyedMessage`. -//! You can also use [`try_clone()`](KeyedMessage::try_clone) to clone the message and prolong its lifetime. +//! Data contained by `CodesMessage` is represented as *keys* (like in dictionary). +//! Keys can be read with static types using [`read_key()`](KeyRead::read_key) or with [dynamic types](codes_message::DynamicKeyType) +//! using [`read_key_dynamic()`](codes_message::CodesMessage::read_key_dynamic). +//! To discover what keys are present in a message use [`KeysIterator`](KeysIterator). //! -//! Data contained by `KeyedMessage` is represented as *keys* (like in dictionary). -//! Keys can be read with static types using [`read_key()`](KeyedMessage::read_key) or with [dynamic types](`DynamicKeyType`) -//! using [`read_key_dynamic()`](KeyedMessage::read_key_dynamic). To discover what keys are present in a message use [`KeysIterator`](KeyedMessage). +//! With `ndarray` feature (enabled by default) you can also read `CodesMessage` into `ndarray` using [`to_ndarray()`](codes_message::CodesMessage::to_ndarray) +//! and [`to_lons_lats_values()`](codes_message::CodesMessage::to_lons_lats_values). //! //! You can use [`CodesNearest`] to get the data values of four nearest gridpoints for given coordinates. //! -//! You can also modify the message with [`write_key()`](KeyedMessage::write_key) and write -//! it to a new file with [`write_to_file()`](KeyedMessage::write_to_file). +//! To modify keys within the message use [`write_key_unchecked()`](KeyWrite::write_key_unchecked). +//! To save that modified message use [`write_to_file()`](codes_message::CodesMessage::write_to_file). //! //! #### Example 1 - Reading GRIB file //! -//! ``` -//! // We are reading the mean sea level pressure for 4 gridpoints -//! // nearest to Reykjavik (64.13N, -21.89E) for 1st June 2021 00:00 UTC -//! // from ERA5 Climate Reanalysis -//! -//! use eccodes::{ProductKind, CodesHandle, KeyRead}; -//! # use std::path::Path; -//! use eccodes::FallibleStreamingIterator; -//! # -//! # fn main() -> anyhow::Result<()> { +//! In this example we are reading mean sea level pressure for 4 gridpoints nearest to +//! Reykjavik (64.13N, -21.89E) for 1st June 2021 00:00 UTC from ERA5 Climate Reanalysis. //! -//! // Open the GRIB file and create the CodesHandle -//! let file_path = Path::new("./data/iceland.grib"); -//! let product_kind = ProductKind::GRIB; -//! let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; +//! ``` +//! use eccodes::{CodesFile, FallibleIterator, KeyRead, ProductKind}; //! +//! # fn main() -> anyhow::Result<()> { +//! // Open the GRIB file and create the CodesFile +//! let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; //! // Use iterator to find a message with shortName "msl" and typeOfLevel "surface" //! // We can use while let or for_each() to iterate over the messages -//! while let Some(msg) = handle.next()? { +//! while let Some(msg) = handle.ref_message_iter().next()? { //! // We need to specify the type of key we read //! let short_name: String = msg.read_key("shortName")?; //! let type_of_level: String = msg.read_key("typeOfLevel")?; -//! //! if short_name == "msl" && type_of_level == "surface" { //! // Create CodesNearest for given message -//! let nearest_gridpoints = msg.codes_nearest()? +//! let nearest_gridpoints = msg +//! .codes_nearest()? //! // Find the nearest gridpoints to Reykjavik //! .find_nearest(64.13, -21.89)?; -//! //! // Print value and distance of the nearest gridpoint -//! println!("value: {}, distance: {}", -//! nearest_gridpoints[3].value, -//! nearest_gridpoints[3].distance); +//! println!( +//! "value: {}, distance: {}", +//! nearest_gridpoints[3].value, nearest_gridpoints[3].distance +//! ); //! } //! } -//! # Ok(()) +//! # Ok(()) //! # } //! ``` //! -//! #### Example 2 - Writing GRIB files +//! #### (New in 0.14) Example 2 - Concurrent read //! -//! ```rust -//! // The crate provides basic support for setting `KeyedMessage` keys -//! // and writing GRIB files. The easiests (and safest) way to create a -//! // new custom message is to copy exisitng one from other GRIB file, -//! // modify the keys and write to new file. +//! This example shows how `ArcMessage` can be used to do concurrent operations +//! on different message within one file and on the same message as well. //! -//! // Here we are computing the temperature at 850hPa as an average -//! // of 900hPa and 800hPa and writing it to a new file. +//! ``` +//! use eccodes::{CodesError, CodesFile, FallibleIterator, KeyRead, ProductKind}; +//! use std::sync::Arc; //! -//! use eccodes::FallibleStreamingIterator; -//! use eccodes::{CodesHandle, KeyRead, KeyWrite, ProductKind}; -//! # use std::{fs::remove_file, path::Path}; -//! //! # fn main() -> anyhow::Result<()> { -//! // Start by opening the file and creating CodesHandle -//! let file_path = Path::new("./data/iceland-levels.grib"); -//! let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; +//! // We open the file as before +//! let handle = CodesFile::new_from_file("./data/iceland-levels.grib", ProductKind::GRIB)?; //! -//! // We need a message to edit, in this case we can use -//! // temperature at 700hPa, which is similar to our result -//! let mut new_msg = vec![]; +//! // Note different mutability - RefMessageIter required the handle to be mutable +//! let mut arc_msg_gen = handle.arc_message_iter(); //! -//! // Get data values of temperatures at 800hPa and 900hPa -//! let mut t800: Vec = vec![]; -//! let mut t900: Vec = vec![]; +//! let mut join_handles = vec![]; //! -//! // Iterate over the messages and collect the data to defined vectors -//! while let Some(msg) = handle.next()? { -//! let short_name: String = msg.read_key("shortName")?; +//! while let Some(msg) = arc_msg_gen.next()? { +//! // ArcMessage is Send+Sync +//! let msg1 = Arc::new(msg); +//! let msg2 = msg1.clone(); //! -//! if short_name == "t" { -//! let level: i64 = msg.read_key("level")?; +//! // For each message we spawn two threads then we do operations simulataneously +//! join_handles.push(std::thread::spawn(move || msg1.read_key("shortName"))); +//! join_handles.push(std::thread::spawn(move || msg2.read_key("typeOfLevel"))); +//! } //! -//! if level == 700 { -//! // To use message outside of the iterator we need to clone it -//! new_msg.push(msg.try_clone()?); -//! } +//! // Now we collect and print the results +//! let read_results = join_handles +//! .into_iter() +//! // In production you should probably handle this join error +//! .map(|jh| jh.join().unwrap()) +//! .collect::, CodesError>>()?; //! -//! if level == 800 { -//! t800 = msg.read_key("values")?; -//! } +//! println!("{read_results:?}"); //! -//! if level == 900 { -//! t900 = msg.read_key("values")?; -//! } -//! } +//! # Ok(()) +//! # } +//! ``` +//! +//! #### Example 3 - Writing GRIB files and indexing +//! +//! The crate provides basic support for setting `CodesMessage` keys +//! and writing GRIB files. To create a new custom message we need to copy +//! existing one from other GRIB file, modify the keys and write to new file. +//! +//! In this example we are computing the temperature at 850hPa as an average +//! of 900hPa and 800hPa and writing it to a new file. This example also shows +//! how you can create a simple index for a file to get messages we need. +//! To read values from a message you can also use [`to_ndarray()`](codes_message::CodesMessage::to_ndarray). +//! +//! ``` +//! use anyhow::Context; +//! use eccodes::{CodesFile, FallibleIterator, KeyRead, KeyWrite, ProductKind}; +//! use std::{collections::HashMap, fs::remove_file, path::Path}; +//! +//! # fn main() -> anyhow::Result<()> { +//! // Start by opening the file and creating CodesFile +//! let file_path = Path::new("./data/iceland-levels.grib"); +//! let mut handle = CodesFile::new_from_file(file_path, ProductKind::GRIB)?; +//! +//! // To build the index we need to collect all messages +//! let messages = handle.ref_message_iter().collect::>()?; +//! let mut msg_index = HashMap::new(); +//! msg_index.reserve(messages.len()); +//! +//! // Now we can put the messages into a hashmap and index them by shortName and level +//! for msg in messages.into_iter() { +//! // all messages in this grib are on the same level type +//! let short_name: String = msg.read_key("shortName")?; +//! let level: i64 = msg.read_key("level")?; +//! +//! msg_index.insert((short_name, level), msg); //! } //! -//! // This converts the vector to a single message -//! let mut new_msg = new_msg.remove(0); +//! // Now we can get the values from messages we need +//! let t_800: Vec = msg_index +//! .get(&("t".to_string(), 800)) +//! .context("message missing")? // we use anyhow context for convienience +//! .read_key("values")?; +//! let t_900: Vec = msg_index +//! .get(&("t".to_string(), 800)) +//! .context("message missing")? +//! .read_key("values")?; +//! +//! // We will also clone t at 700hPa to edit it +//! let mut t_850_msg = msg_index +//! .get(&("t".to_string(), 700)) +//! .context("message missing")? +//! .try_clone()?; //! //! // Compute temperature at 850hPa -//! let t850: Vec = t800 +//! let t_850_values: Vec = t_800 //! .iter() -//! .zip(t900.iter()) +//! .zip(t_900.iter()) //! .map(|t| (t.0 + t.1) / 2.0) //! .collect(); //! -//! // Edit appropriate keys in the editable message -//! new_msg.write_key("level", 850)?; -//! new_msg.write_key("values", &t850)?; +//! // Edit appropriate keys in the cloned (editable) message +//! t_850_msg.write_key_unchecked("level", 850)?; +//! t_850_msg.write_key_unchecked("values", t_850_values.as_slice())?; //! //! // Save the message to a new file without appending -//! new_msg.write_to_file(Path::new("iceland-850.grib"), false)?; -//! -//! # remove_file(Path::new("iceland-850.grib")).unwrap(); -//! # Ok(()) +//! t_850_msg.write_to_file(Path::new("iceland-850.grib"), false)?; +//! # remove_file(Path::new("iceland-850.grib")).unwrap(); +//! # Ok(()) //! # } //! ``` //! +//! ## Changes in version 0.14 +//! +//! 1. `experimental_index` feature has been removed - users are encouraged to create their own indexes as shown above or use iterator filtering +//! 2. `message_ndarray` feature has been renamed to `ndarray` +//! 3. `CodesHandle` has been renamed to `CodesFile` +//! 4. `KeyedMessage` has been replaced with generic `CodesMessage` - `RefMessage` has the most similar behaviour to `CodesMessage`] +//! 5. `write_key` is now `write_key_unchecked` +//! 6. Dependency on `FallibleStreamingIterator` has been removed +//! +//! ## Feature Flags +//! +//! - `ndarray` - enables support for converting [`CodesMessage`](codes_message::CodesMessage) to [`ndarray::Array`]. +//! This feature is enabled by default. It is currently tested only with simple lat-lon grids. +//! +//! - `docs` - builds the crate without linking ecCodes, particularly useful when building the documentation +//! on [docs.rs](https://docs.rs/). For more details check documentation of [eccodes-sys](https://crates.io/crates/eccodes-sys). +//! +//! To build your own crate with this crate as dependency on docs.rs without linking ecCodes add following lines to your `Cargo.toml` +//! +//! ```text +//! [package.metadata.docs.rs] +//! features = ["eccodes/docs"] +//! ``` -pub mod codes_handle; -#[cfg(feature = "experimental_index")] -#[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] -pub mod codes_index; +pub mod codes_file; +pub mod codes_message; pub mod codes_nearest; pub mod errors; mod intermediate_bindings; -pub mod keyed_message; pub mod keys_iterator; -#[cfg(feature = "message_ndarray")] -#[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] -pub mod message_ndarray; + mod pointer_guard; -pub use codes_handle::{CodesHandle, ProductKind}; -#[cfg(feature = "experimental_index")] -#[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] -pub use codes_index::CodesIndex; +pub use codes_file::{ArcMessageIter, CodesFile, ProductKind, RefMessageIter}; +pub use codes_message::{ArcMessage, BufMessage, DynamicKeyType, KeyRead, KeyWrite, RefMessage}; pub use codes_nearest::{CodesNearest, NearestGridpoint}; pub use errors::CodesError; pub use fallible_iterator::{FallibleIterator, IntoFallibleIterator}; -pub use fallible_streaming_iterator::FallibleStreamingIterator; -pub use keyed_message::{DynamicKeyType, KeyRead, KeyWrite, KeyedMessage}; pub use keys_iterator::{KeysIterator, KeysIteratorFlags}; diff --git a/src/pointer_guard.rs b/src/pointer_guard.rs index ba8c4a0..5dfeb3b 100644 --- a/src/pointer_guard.rs +++ b/src/pointer_guard.rs @@ -1,6 +1,7 @@ macro_rules! non_null { ($ptr:expr) => { if $ptr.is_null() { + debug_assert!(false, "Null pointer encountered"); return Err(CodesError::NullPtr); } }; @@ -10,10 +11,10 @@ pub(crate) use non_null; #[cfg(test)] mod tests { use crate::errors::CodesError; - use crate::pointer_guard::non_null; use std::ptr; #[test] + #[should_panic = "Null pointer encountered"] fn test_non_null() { let ptr: *mut i32 = ptr::null_mut(); let result = simulated_function(ptr); @@ -24,14 +25,14 @@ mod tests { match result { CodesError::NullPtr => (), - _ => panic!("Incorrect error type: {:?}", result), + _ => panic!("Incorrect error type: {result:?}"), } } #[test] fn test_non_null_ok() { let mut x = 42_i32; - let ptr = &mut x as *mut i32; + let ptr = &raw mut x; let result = simulated_function(ptr); diff --git a/tests/example.rs b/tests/example.rs new file mode 100644 index 0000000..8d443c3 --- /dev/null +++ b/tests/example.rs @@ -0,0 +1,13 @@ +use anyhow::Context; +use eccodes::{CodesFile, FallibleIterator, ProductKind}; +fn main() -> anyhow::Result<()> { + let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; + let mut current_message = handle.ref_message_iter().next()?.context("no message")?; + + let mut keys_iter = current_message.default_keys_iterator()?; + + while let Some(key_name) = keys_iter.next()? { + println!("{key_name}"); + } + Ok(()) +} diff --git a/tests/handle.rs b/tests/handle.rs index 19a6377..191fca5 100644 --- a/tests/handle.rs +++ b/tests/handle.rs @@ -1,11 +1,11 @@ use std::{path::Path, thread}; use anyhow::{Context, Result}; -use eccodes::{CodesHandle, DynamicKeyType, FallibleStreamingIterator, ProductKind}; +use eccodes::{CodesFile, FallibleIterator, ProductKind, codes_message::DynamicKeyType}; #[test] fn thread_safety() { - // errors are fine + // errors are fine, segfaults are not thread_safety_core().unwrap_or(()); } @@ -14,8 +14,11 @@ fn thread_safety_core() -> Result<()> { loop { let file_path = Path::new("./data/iceland.grib"); - let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; - let current_message = handle.next()?.context("Message not some")?; + let mut handle = CodesFile::new_from_file(file_path, ProductKind::GRIB)?; + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; for _ in 0..100 { let _ = current_message.read_key_dynamic("name")?; @@ -27,16 +30,17 @@ fn thread_safety_core() -> Result<()> { _ => panic!("Incorrect variant of string key"), } } - - drop(handle); } }); for _ in 0..1000 { let file_path = Path::new("./data/iceland.grib"); - let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; - let current_message = handle.next()?.context("Message not some")?; + let mut handle = CodesFile::new_from_file(file_path, ProductKind::GRIB)?; + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; let long_key = current_message.read_key_dynamic("numberOfPointsAlongAParallel")?; @@ -44,33 +48,7 @@ fn thread_safety_core() -> Result<()> { DynamicKeyType::Int(_) => {} _ => panic!("Incorrect variant of long key"), } - - drop(handle); } Ok(()) } - -#[test] -fn check_no_testing_logs() -> Result<()> { - testing_logger::setup(); - { - let file_path = Path::new("./data/iceland-surface.grib"); - let product_kind = ProductKind::GRIB; - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - - let _ref_msg = handle.next()?.context("no message")?; - let clone_msg = _ref_msg.try_clone()?; - let _oth_ref = handle.next()?.context("no message")?; - - let _nrst = clone_msg.codes_nearest()?; - let _kiter = clone_msg.default_keys_iterator()?; - } - - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 0); - }); - - Ok(()) -} diff --git a/tests/index.rs b/tests/index.rs deleted file mode 100644 index e58358e..0000000 --- a/tests/index.rs +++ /dev/null @@ -1,249 +0,0 @@ -#![cfg(feature = "experimental_index")] - -use std::{ - path::Path, - sync::{Arc, Barrier}, - thread, -}; - -use anyhow::{Context, Result}; -use eccodes::{ - codes_index::Select, CodesError, CodesHandle, CodesIndex, FallibleStreamingIterator, KeyRead, - ProductKind, -}; -use rand::Rng; - -#[test] -fn iterate_handle_from_index() -> Result<()> { - let file_path = Path::new("./data/iceland-surface.grib.idx"); - let index = CodesIndex::read_from_file(file_path)? - .select("shortName", "2t")? - .select("typeOfLevel", "surface")? - .select("level", 0)? - .select("stepType", "instant")?; - - let handle = CodesHandle::new_from_index(index)?; - - let counter = handle.count()?; - - assert_eq!(counter, 1); - - Ok(()) -} - -#[test] -fn read_index_messages() -> Result<()> { - let file_path = Path::new("./data/iceland-surface.grib.idx"); - let index = CodesIndex::read_from_file(file_path)? - .select("shortName", "2t")? - .select("typeOfLevel", "surface")? - .select("level", 0)? - .select("stepType", "instant")?; - - let mut handle = CodesHandle::new_from_index(index)?; - let current_message = handle.next()?.context("Message not some")?; - - { - let short_name: String = current_message.read_key("shortName")?; - assert_eq!(short_name, "2t"); - } - { - let level: i64 = current_message.read_key("level")?; - assert_eq!(level, 0); - } - - Ok(()) -} - -#[test] -fn collect_index_iterator() -> Result<()> { - let keys = vec!["typeOfLevel", "level"]; - let index = CodesIndex::new_from_keys(&keys)?; - let grib_path = Path::new("./data/iceland-levels.grib"); - - let index = index - .add_grib_file(grib_path)? - .select("typeOfLevel", "isobaricInhPa")? - .select("level", 700)?; - - let mut handle = CodesHandle::new_from_index(index)?; - - let mut levels = vec![]; - - while let Some(msg) = handle.next()? { - levels.push(msg.try_clone()?); - } - - assert_eq!(levels.len(), 5); - - Ok(()) -} - -#[test] -fn add_file_error() -> Result<()> { - thread::spawn(|| -> Result<()> { - let grib_path = Path::new("./data/iceland-levels.grib"); - let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; - let mut index_op = CodesIndex::new_from_keys(&keys)?; - - loop { - index_op = index_op.add_grib_file(grib_path)?; - } - }); - - thread::sleep(std::time::Duration::from_millis(250)); - - let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; - let wrong_path = Path::new("./data/xxx.grib"); - let index = CodesIndex::new_from_keys(&keys)?.add_grib_file(wrong_path); - - assert!(index.is_err()); - - Ok(()) -} - -#[test] -fn index_panic() -> Result<()> { - thread::spawn(|| -> Result<()> { - let grib_path = Path::new("./data/iceland-levels.grib"); - let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; - let mut index_op = CodesIndex::new_from_keys(&keys)?; - - loop { - index_op = index_op.add_grib_file(grib_path)?; - } - }); - - thread::sleep(std::time::Duration::from_millis(250)); - - let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; - let wrong_path = Path::new("./data/xxx.grib"); - let index = CodesIndex::new_from_keys(&keys)?; - - let result = std::panic::catch_unwind(|| index.add_grib_file(wrong_path).unwrap()); - - assert!(result.is_err()); - - Ok(()) -} - -#[test] -#[ignore = "for releases, indexing is experimental"] -fn add_file_while_index_open() -> Result<()> { - thread::spawn(|| -> Result<()> { - let file_path = Path::new("./data/iceland-surface.grib.idx"); - let mut index_op = CodesIndex::read_from_file(file_path)?; - - loop { - index_op = index_op - .select("shortName", "2t")? - .select("typeOfLevel", "surface")? - .select("level", 0)? - .select("stepType", "instant")?; - } - }); - - let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; - let grib_path = Path::new("./data/iceland-surface.grib"); - let index = CodesIndex::new_from_keys(&keys)?.add_grib_file(grib_path); - - assert!(index.is_ok()); - - Ok(()) -} - -#[test] -fn add_file_to_read_index() -> Result<()> { - let file_path = Path::new("./data/iceland-surface.grib.idx"); - let grib_path = Path::new("./data/iceland-surface.grib"); - - let _index = CodesIndex::read_from_file(file_path)? - .add_grib_file(grib_path)? - .select("shortName", "2t")? - .select("typeOfLevel", "surface")? - .select("level", 0)? - .select("stepType", "instant")?; - - Ok(()) -} - -#[test] -#[ignore = "for releases, indexing is experimental"] -fn simulatenous_index_destructors() -> Result<()> { - let barrier = Arc::new(Barrier::new(2)); - let b1 = barrier.clone(); - let b2 = barrier.clone(); - - let h1 = thread::spawn(move || -> anyhow::Result<(), CodesError> { - let file_path = Path::new("./data/iceland-surface.grib.idx"); - - for _ in 0..100 { - let index_op = CodesIndex::read_from_file(file_path)? - .select("shortName", "2t")? - .select("typeOfLevel", "surface")? - .select("level", 0)? - .select("stepType", "instant")?; - - b1.wait(); - drop(index_op); - } - - Ok(()) - }); - - let h2 = thread::spawn(move || -> anyhow::Result<(), CodesError> { - let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; - let grib_path = Path::new("./data/iceland-surface.grib"); - - for _ in 0..100 { - let index = CodesIndex::new_from_keys(&keys)? - .add_grib_file(grib_path)? - .select("shortName", "2t")? - .select("typeOfLevel", "surface")? - .select("level", 0)? - .select("stepType", "instant")?; - - b2.wait(); - drop(index); - } - - Ok(()) - }); - - // errors are fine - h1.join().unwrap().unwrap_or(()); - h2.join().unwrap().unwrap_or(()); - - Ok(()) -} - -#[test] -#[ignore = "for releases, indexing is experimental"] -fn index_handle_interference() -> Result<()> { - thread::spawn(|| -> Result<()> { - let file_path = Path::new("./data/iceland.grib"); - - loop { - let handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB); - - assert!(handle.is_ok()); - } - }); - - let mut rng = rand::thread_rng(); - let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; - let grib_path = Path::new("./data/iceland.grib"); - - for _ in 0..10 { - let sleep_time = rng.gen_range(1..42); // randomizing sleep time to hopefully catch segfaults - - let index = CodesIndex::new_from_keys(&keys)?.add_grib_file(grib_path)?; - let i_handle = CodesHandle::new_from_index(index); - - assert!(i_handle.is_ok()); - - thread::sleep(std::time::Duration::from_millis(sleep_time)); - } - - Ok(()) -}