From 0ccadb801a4bebfbefcb4d613830085a248249d6 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 8 Sep 2025 19:54:55 +0200 Subject: [PATCH 01/65] test if CodesHandle is thread safe --- src/codes_handle/mod.rs | 73 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 3 deletions(-) diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index 51cc977..9a8fc65 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -329,13 +329,18 @@ fn open_with_fmemopen(file_data: &[u8]) -> Result<*mut FILE, CodesError> { #[cfg(test)] mod tests { - use crate::codes_handle::{CodesHandle, ProductKind}; + use crate::codes_handle::{CodesFile, CodesHandle, ProductKind}; #[cfg(feature = "experimental_index")] use crate::codes_index::{CodesIndex, Select}; use anyhow::{Context, Result}; - use eccodes_sys::ProductKind_PRODUCT_GRIB; + use eccodes_sys::{grib_handle, ProductKind_PRODUCT_GRIB}; use fallible_streaming_iterator::FallibleStreamingIterator; - use std::{fs::File, io::Read, path::Path}; + use std::{ + fs::File, + io::Read, + path::Path, + sync::{Arc, Barrier, Mutex}, + }; #[test] fn file_constructor() -> Result<()> { @@ -355,6 +360,68 @@ mod tests { Ok(()) } + #[derive(Debug)] + struct AtomicContainer { + _parent: Arc>>, + pointer: Mutex<*mut grib_handle>, + } + unsafe impl Send for AtomicContainer {} + unsafe impl Sync for AtomicContainer {} + + impl AtomicContainer { + fn new(_parent: Arc>>, pointer: *mut grib_handle) -> Self { + Self { + _parent, + pointer: Mutex::new(pointer), + } + } + } + + #[test] + fn handle_thread_safety() -> Result<()> { + let file_path = Path::new("./data/iceland-levels.grib"); + let product_kind = ProductKind::GRIB; + + let handle = CodesHandle::new_from_file(file_path, product_kind)?; + + let handle = Arc::new(handle); + let barrier = Arc::new(Barrier::new(10)); + + let mut v = vec![]; + + for _ in 0..10 { + let fhndl = handle.clone(); + let b = barrier.clone(); + + let msg_hndl = unsafe { + crate::intermediate_bindings::codes_handle_new_from_file( + fhndl.source.pointer, + handle.source.product_kind, + ) + .unwrap() + }; + let cntri = Arc::new(AtomicContainer::new(fhndl, msg_hndl)); + + let t = std::thread::spawn(move || { + let p = cntri.pointer.lock().unwrap(); + + for _ in 0..1000 { + b.wait(); + let _ = unsafe { + crate::intermediate_bindings::codes_get_size(*p, "shortName").unwrap() + }; + } + }); + + v.push(t); + } + + v.into_iter().for_each(|th| th.join().unwrap()); + + assert!(false); + Ok(()) + } + #[test] fn memory_constructor() -> Result<()> { let product_kind = ProductKind::GRIB; From 00033848da4b7e4b898009021c195615ca56c539 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sat, 20 Sep 2025 11:06:58 +0200 Subject: [PATCH 02/65] remove mutex --- src/codes_handle/mod.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index 9a8fc65..420c588 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -339,7 +339,7 @@ mod tests { fs::File, io::Read, path::Path, - sync::{Arc, Barrier, Mutex}, + sync::{Arc, Barrier}, }; #[test] @@ -363,7 +363,7 @@ mod tests { #[derive(Debug)] struct AtomicContainer { _parent: Arc>>, - pointer: Mutex<*mut grib_handle>, + pointer: *mut grib_handle, } unsafe impl Send for AtomicContainer {} unsafe impl Sync for AtomicContainer {} @@ -372,12 +372,14 @@ mod tests { fn new(_parent: Arc>>, pointer: *mut grib_handle) -> Self { Self { _parent, - pointer: Mutex::new(pointer), + // Use the mutable methods to avoid the performance hit of Mutex + pointer, } } } #[test] + #[ignore] fn handle_thread_safety() -> Result<()> { let file_path = Path::new("./data/iceland-levels.grib"); let product_kind = ProductKind::GRIB; @@ -403,12 +405,11 @@ mod tests { let cntri = Arc::new(AtomicContainer::new(fhndl, msg_hndl)); let t = std::thread::spawn(move || { - let p = cntri.pointer.lock().unwrap(); - for _ in 0..1000 { b.wait(); let _ = unsafe { - crate::intermediate_bindings::codes_get_size(*p, "shortName").unwrap() + crate::intermediate_bindings::codes_get_size(cntri.pointer, "shortName") + .unwrap() }; } }); @@ -418,6 +419,7 @@ mod tests { v.into_iter().for_each(|th| th.join().unwrap()); + todo!("Implement tracing to follow contructors and drops"); assert!(false); Ok(()) } From af59b26d36c3bc29f53dba55c7e02f3aa052ba4e Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sat, 20 Sep 2025 11:08:30 +0200 Subject: [PATCH 03/65] update cargo.toml --- Cargo.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bc8c758..be10aa3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,9 +14,9 @@ categories = [ "science", ] license = "Apache-2.0" -edition = "2021" +edition = "2024" exclude = [".github/*", ".vscode/*", ".idea/*", "data/*"] -rust-version = "1.82.0" +rust-version = "1.85.0" [dependencies] eccodes-sys = { version = "0.6.0", default-features = false } @@ -34,14 +34,14 @@ ndarray = { version = "0.16", default-features = false, optional = true, feature [dev-dependencies] reqwest = { version = "0.12", features = ["rustls-tls"] } -criterion = "0.5" +criterion = "0.7" testing_logger = "0.1" -rand = "0.8" +rand = "0.9" anyhow = { version = "1.0", features = ["backtrace"] } float-cmp = "0.10" [features] -default = ["message_ndarray", "experimental_index"] +default = ["message_ndarray"] docs = ["eccodes-sys/docs"] experimental_index = [] message_ndarray = ["dep:ndarray"] From 1cccf4b6c999fd50db99254daabfa24771c09a2a Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sat, 20 Sep 2025 11:10:31 +0200 Subject: [PATCH 04/65] fixes for 2024 edition --- src/intermediate_bindings/codes_get.rs | 44 +++++++++++------------ src/intermediate_bindings/codes_handle.rs | 8 ++--- src/intermediate_bindings/codes_keys.rs | 16 ++++----- src/intermediate_bindings/codes_set.rs | 24 ++++++------- src/intermediate_bindings/grib_nearest.rs | 12 +++---- 5 files changed, 52 insertions(+), 52 deletions(-) diff --git a/src/intermediate_bindings/codes_get.rs b/src/intermediate_bindings/codes_get.rs index dc71054..0f580be 100644 --- a/src/intermediate_bindings/codes_get.rs +++ b/src/intermediate_bindings/codes_get.rs @@ -17,7 +17,7 @@ use super::NativeKeyType; pub unsafe fn codes_get_native_type( handle: *const codes_handle, key: &str, -) -> Result { +) -> Result { unsafe { pointer_guard::non_null!(handle); let key = CString::new(key).unwrap(); @@ -31,9 +31,9 @@ pub unsafe fn codes_get_native_type( } Ok(FromPrimitive::from_i32(key_type).unwrap()) -} +}} -pub unsafe fn codes_get_size(handle: *const codes_handle, key: &str) -> Result { +pub unsafe fn codes_get_size(handle: *const codes_handle, key: &str) -> Result { unsafe { pointer_guard::non_null!(handle); let key = CString::new(key).unwrap(); @@ -47,9 +47,9 @@ pub unsafe fn codes_get_size(handle: *const codes_handle, key: &str) -> Result Result { +pub unsafe fn codes_get_long(handle: *const codes_handle, key: &str) -> Result { unsafe { pointer_guard::non_null!(handle); let key = CString::new(key).unwrap(); @@ -63,9 +63,9 @@ pub unsafe fn codes_get_long(handle: *const codes_handle, key: &str) -> Result Result { +pub unsafe fn codes_get_double(handle: *const codes_handle, key: &str) -> Result { unsafe { pointer_guard::non_null!(handle); let key = CString::new(key).unwrap(); @@ -79,12 +79,12 @@ pub unsafe fn codes_get_double(handle: *const codes_handle, key: &str) -> Result } Ok(key_value) -} +}} pub unsafe fn codes_get_double_array( handle: *const codes_handle, key: &str, -) -> Result, CodesError> { +) -> Result, CodesError> { unsafe { pointer_guard::non_null!(handle); let mut key_size = codes_get_size(handle, key)?; @@ -105,12 +105,12 @@ pub unsafe fn codes_get_double_array( } Ok(key_values) -} +}} pub unsafe fn codes_get_long_array( handle: *const codes_handle, key: &str, -) -> Result, CodesError> { +) -> Result, CodesError> { unsafe { pointer_guard::non_null!(handle); let mut key_size = codes_get_size(handle, key)?; @@ -131,12 +131,12 @@ pub unsafe fn codes_get_long_array( } Ok(key_values) -} +}} pub unsafe fn codes_get_length( handle: *const codes_handle, key: &str, -) -> Result { +) -> Result { unsafe { pointer_guard::non_null!(handle); let key = CString::new(key).unwrap(); @@ -150,12 +150,12 @@ pub unsafe fn codes_get_length( } Ok(key_length) -} +}} pub unsafe fn codes_get_string( handle: *const codes_handle, key: &str, -) -> Result { +) -> Result { unsafe { pointer_guard::non_null!(handle); let mut key_length = codes_get_length(handle, key)?; @@ -188,12 +188,12 @@ pub unsafe fn codes_get_string( let key_message_string = key_message_cstr.to_str()?.to_string(); Ok(key_message_string) -} +}} pub unsafe fn codes_get_bytes( handle: *const codes_handle, key: &str, -) -> Result, CodesError> { +) -> Result, CodesError> { unsafe { pointer_guard::non_null!(handle); let mut key_size = codes_get_length(handle, key)?; @@ -214,9 +214,9 @@ pub unsafe fn codes_get_bytes( } Ok(buffer) -} +}} -pub unsafe fn codes_get_message_size(handle: *const codes_handle) -> Result { +pub unsafe fn codes_get_message_size(handle: *const codes_handle) -> Result { unsafe { pointer_guard::non_null!(handle); let mut size: usize = 0; @@ -229,11 +229,11 @@ pub unsafe fn codes_get_message_size(handle: *const codes_handle) -> Result Result<(*const c_void, usize), CodesError> { +) -> Result<(*const c_void, usize), CodesError> { unsafe { pointer_guard::non_null!(handle); let buffer_size = codes_get_message_size(handle)?; @@ -257,4 +257,4 @@ pub unsafe fn codes_get_message( ); Ok((buffer_ptr, message_size)) -} +}} diff --git a/src/intermediate_bindings/codes_handle.rs b/src/intermediate_bindings/codes_handle.rs index 2913e1a..b4104bc 100644 --- a/src/intermediate_bindings/codes_handle.rs +++ b/src/intermediate_bindings/codes_handle.rs @@ -18,7 +18,7 @@ use crate::{ pub unsafe fn codes_handle_new_from_file( file_pointer: *mut FILE, product_kind: ProductKind, -) -> Result<*mut codes_handle, CodesError> { +) -> Result<*mut codes_handle, CodesError> { unsafe { pointer_guard::non_null!(file_pointer); let context: *mut codes_context = ptr::null_mut(); //default context @@ -38,9 +38,9 @@ pub unsafe fn codes_handle_new_from_file( } Ok(file_handle) -} +}} -pub unsafe fn codes_handle_delete(handle: *mut codes_handle) -> Result<(), CodesError> { +pub unsafe fn codes_handle_delete(handle: *mut codes_handle) -> Result<(), CodesError> { unsafe { #[cfg(test)] log::trace!("codes_handle_delete"); @@ -56,7 +56,7 @@ pub unsafe fn codes_handle_delete(handle: *mut codes_handle) -> Result<(), Codes } Ok(()) -} +}} #[cfg(feature = "experimental_index")] pub unsafe fn codes_handle_new_from_index( diff --git a/src/intermediate_bindings/codes_keys.rs b/src/intermediate_bindings/codes_keys.rs index 5ade02f..c5c7874 100644 --- a/src/intermediate_bindings/codes_keys.rs +++ b/src/intermediate_bindings/codes_keys.rs @@ -16,7 +16,7 @@ pub unsafe fn codes_keys_iterator_new( handle: *mut codes_handle, flags: u32, namespace: &str, -) -> Result<*mut codes_keys_iterator, CodesError> { +) -> Result<*mut codes_keys_iterator, CodesError> { unsafe { pointer_guard::non_null!(handle); let namespace = CString::new(namespace).unwrap(); @@ -28,11 +28,11 @@ pub unsafe fn codes_keys_iterator_new( } Ok(kiter) -} +}} pub unsafe fn codes_keys_iterator_delete( keys_iterator: *mut codes_keys_iterator, -) -> Result<(), CodesError> { +) -> Result<(), CodesError> { unsafe { #[cfg(test)] log::trace!("codes_keys_iterator_delete"); @@ -48,21 +48,21 @@ pub unsafe fn codes_keys_iterator_delete( } Ok(()) -} +}} pub unsafe fn codes_keys_iterator_next( keys_iterator: *mut codes_keys_iterator, -) -> Result { +) -> Result { unsafe { pointer_guard::non_null!(keys_iterator); let next_item_exists = eccodes_sys::codes_keys_iterator_next(keys_iterator); Ok(next_item_exists == 1) -} +}} pub unsafe fn codes_keys_iterator_get_name( keys_iterator: *mut codes_keys_iterator, -) -> Result { +) -> Result { unsafe { pointer_guard::non_null!(keys_iterator); let name_pointer = eccodes_sys::codes_keys_iterator_get_name(keys_iterator); @@ -72,4 +72,4 @@ pub unsafe fn codes_keys_iterator_get_name( let name_string = name_str.to_owned(); Ok(name_string) -} +}} diff --git a/src/intermediate_bindings/codes_set.rs b/src/intermediate_bindings/codes_set.rs index 0ff0cce..442bf5f 100644 --- a/src/intermediate_bindings/codes_set.rs +++ b/src/intermediate_bindings/codes_set.rs @@ -16,7 +16,7 @@ pub unsafe fn codes_set_long( handle: *mut codes_handle, key: &str, value: i64, -) -> Result<(), CodesError> { +) -> Result<(), CodesError> { unsafe { pointer_guard::non_null!(handle); let key = CString::new(key).unwrap(); @@ -29,13 +29,13 @@ pub unsafe fn codes_set_long( } Ok(()) -} +}} pub unsafe fn codes_set_double( handle: *mut codes_handle, key: &str, value: f64, -) -> Result<(), CodesError> { +) -> Result<(), CodesError> { unsafe { pointer_guard::non_null!(handle); let key = CString::new(key).unwrap(); @@ -48,13 +48,13 @@ pub unsafe fn codes_set_double( } Ok(()) -} +}} pub unsafe fn codes_set_long_array( handle: *mut codes_handle, key: &str, values: &[i64], -) -> Result<(), CodesError> { +) -> Result<(), CodesError> { unsafe { pointer_guard::non_null!(handle); let key = CString::new(key).unwrap(); @@ -74,13 +74,13 @@ pub unsafe fn codes_set_long_array( } Ok(()) -} +}} pub unsafe fn codes_set_double_array( handle: *mut codes_handle, key: &str, values: &[f64], -) -> Result<(), CodesError> { +) -> Result<(), CodesError> { unsafe { pointer_guard::non_null!(handle); let key = CString::new(key).unwrap(); @@ -100,13 +100,13 @@ pub unsafe fn codes_set_double_array( } Ok(()) -} +}} pub unsafe fn codes_set_string( handle: *mut codes_handle, key: &str, value: &str, -) -> Result<(), CodesError> { +) -> Result<(), CodesError> { unsafe { pointer_guard::non_null!(handle); let key = CString::new(key).unwrap(); @@ -122,13 +122,13 @@ pub unsafe fn codes_set_string( } Ok(()) -} +}} pub unsafe fn codes_set_bytes( handle: *mut codes_handle, key: &str, values: &[u8], -) -> Result<(), CodesError> { +) -> Result<(), CodesError> { unsafe { pointer_guard::non_null!(handle); let key = CString::new(key).unwrap(); @@ -148,4 +148,4 @@ pub unsafe fn codes_set_bytes( } Ok(()) -} +}} diff --git a/src/intermediate_bindings/grib_nearest.rs b/src/intermediate_bindings/grib_nearest.rs index 129cb83..bd92833 100644 --- a/src/intermediate_bindings/grib_nearest.rs +++ b/src/intermediate_bindings/grib_nearest.rs @@ -14,7 +14,7 @@ use crate::{ pub unsafe fn codes_grib_nearest_new( handle: *const codes_handle, -) -> Result<*mut codes_nearest, CodesError> { +) -> Result<*mut codes_nearest, CodesError> { unsafe { pointer_guard::non_null!(handle); let mut error_code: i32 = 0; @@ -27,9 +27,9 @@ pub unsafe fn codes_grib_nearest_new( } Ok(nearest) -} +}} -pub unsafe fn codes_grib_nearest_delete(nearest: *mut codes_nearest) -> Result<(), CodesError> { +pub unsafe fn codes_grib_nearest_delete(nearest: *mut codes_nearest) -> Result<(), CodesError> { unsafe { #[cfg(test)] log::trace!("codes_grib_nearest_delete"); @@ -45,14 +45,14 @@ pub unsafe fn codes_grib_nearest_delete(nearest: *mut codes_nearest) -> Result<( } Ok(()) -} +}} pub unsafe fn codes_grib_nearest_find( handle: *const codes_handle, nearest: *mut codes_nearest, lat: f64, lon: f64, -) -> Result<[NearestGridpoint; 4], CodesError> { +) -> Result<[NearestGridpoint; 4], CodesError> { unsafe { pointer_guard::non_null!(handle); pointer_guard::non_null!(nearest); @@ -98,4 +98,4 @@ pub unsafe fn codes_grib_nearest_find( } Ok(output) -} +}} From b6e3d6a65f947f5d7f2fd91170441a8290160a53 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sat, 20 Sep 2025 11:48:28 +0200 Subject: [PATCH 05/65] (wip - failing tests) replace log with tracing, trace constructors and destructors --- Cargo.toml | 7 ++++++- src/codes_handle/mod.rs | 12 ++++++++---- src/codes_index.rs | 3 +++ src/codes_nearest.rs | 5 +++-- src/intermediate_bindings/codes_handle.rs | 13 ++++++++++--- src/intermediate_bindings/codes_index.rs | 2 ++ src/intermediate_bindings/codes_keys.rs | 5 +++-- src/intermediate_bindings/grib_nearest.rs | 6 +++--- src/keyed_message/mod.rs | 5 +++-- src/keys_iterator.rs | 6 ++++-- 10 files changed, 45 insertions(+), 19 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index be10aa3..d053c9a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,6 @@ rust-version = "1.85.0" eccodes-sys = { version = "0.6.0", default-features = false } libc = { version = "0.2", default-features = false } thiserror = { version = "2.0", default-features = false } -log = { version = "0.4", default-features = false } errno = { version = "0.3", default-features = false } num-derive = { version = "0.4", default-features = false } num-traits = { version = "0.2", default-features = false } @@ -31,11 +30,17 @@ fallible-streaming-iterator = { version = "0.1.9", default-features = false } ndarray = { version = "0.16", default-features = false, optional = true, features = [ "std", ] } +tracing = { version = "0.1", default-features = false, features = [ + "std", + "attributes", + "log", +] } [dev-dependencies] reqwest = { version = "0.12", features = ["rustls-tls"] } criterion = "0.7" testing_logger = "0.1" +log = "*" rand = "0.9" anyhow = { version = "1.0", features = ["backtrace"] } float-cmp = "0.10" diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index 420c588..d9b16d8 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -8,7 +8,8 @@ use crate::{ }; use eccodes_sys::{codes_handle, ProductKind_PRODUCT_GRIB}; use errno::errno; -use libc::{c_void, size_t, FILE}; +use libc::{c_char, c_void, size_t, FILE}; +use tracing::instrument; use std::{ fmt::Debug, fs::{File, OpenOptions}, @@ -183,7 +184,8 @@ impl CodesHandle> { /// ///Returns [`CodesError::Internal`] with error code ///when internal [`codes_handle`] cannot be created. - pub fn new_from_file>( + #[instrument(level = "trace")] + pub fn new_from_file + Debug>( file_path: P, product_kind: ProductKind, ) -> Result { @@ -235,6 +237,7 @@ impl CodesHandle>> { /// ///Returns [`CodesError::Internal`] with error code ///when internal [`codes_handle`] cannot be created. + #[instrument(level = "trace")] pub fn new_from_memory( file_data: Vec, product_kind: ProductKind, @@ -283,6 +286,7 @@ impl CodesHandle { /// /// Returns [`CodesError::Internal`] with error code /// when internal [`codes_handle`] cannot be created. + #[instrument(level = "trace")] pub fn new_from_index(index: CodesIndex) -> Result { let new_handle = CodesHandle { source: index, @@ -293,6 +297,7 @@ impl CodesHandle { } } +#[instrument(level = "trace")] fn open_with_fdopen(file: &File) -> Result<*mut FILE, CodesError> { let file_ptr = unsafe { libc::fdopen(file.as_raw_fd(), "r".as_ptr().cast::<_>()) }; @@ -305,6 +310,7 @@ fn open_with_fdopen(file: &File) -> Result<*mut FILE, CodesError> { Ok(file_ptr) } +#[instrument(level = "trace")] fn open_with_fmemopen(file_data: &[u8]) -> Result<*mut FILE, CodesError> { let file_data_ptr = file_data.as_ptr() as *mut c_void; pointer_guard::non_null!(file_data_ptr); @@ -419,8 +425,6 @@ mod tests { v.into_iter().for_each(|th| th.join().unwrap()); - todo!("Implement tracing to follow contructors and drops"); - assert!(false); Ok(()) } diff --git a/src/codes_index.rs b/src/codes_index.rs index 3c07cf7..5fc7ddb 100644 --- a/src/codes_index.rs +++ b/src/codes_index.rs @@ -138,6 +138,7 @@ impl CodesIndex { /// /// This function will return [`CodesError::Internal`] if the index cannot be created. #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] + #[instrument(level = "trace")] pub fn new_from_keys(keys: &[&str]) -> Result { let keys = keys.join(","); @@ -173,6 +174,7 @@ impl CodesIndex { /// This function will return [`CodesError::Internal`] if the index file is not valid or /// the GRIB file is not present in the same relative path as during the index file creation. #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] + #[instrument(level = "trace")] pub fn read_from_file>(index_file_path: P) -> Result { let index_file_path: &Path = index_file_path.as_ref(); let file_path = index_file_path.to_str().ok_or_else(|| { @@ -271,6 +273,7 @@ impl HandleGenerator for CodesIndex { #[doc(hidden)] impl Drop for CodesIndex { + #[instrument(level = "trace")] fn drop(&mut self) { unsafe { codes_index_delete(self.pointer) } self.pointer = null_mut(); diff --git a/src/codes_nearest.rs b/src/codes_nearest.rs index ef083f3..105adab 100644 --- a/src/codes_nearest.rs +++ b/src/codes_nearest.rs @@ -4,7 +4,7 @@ use std::ptr::null_mut; use eccodes_sys::codes_nearest; -use log::error; +use tracing::{event, instrument, Level}; use crate::{ intermediate_bindings::{ @@ -103,10 +103,11 @@ impl CodesNearest<'_> { #[doc(hidden)] impl Drop for CodesNearest<'_> { + #[instrument(level = "trace")] fn drop(&mut self) { unsafe { codes_grib_nearest_delete(self.nearest_handle).unwrap_or_else(|error| { - error!( + event!(Level::ERROR, "codes_grib_nearest_delete() returned an error: {:?}", &error ); diff --git a/src/intermediate_bindings/codes_handle.rs b/src/intermediate_bindings/codes_handle.rs index b4104bc..75bf94b 100644 --- a/src/intermediate_bindings/codes_handle.rs +++ b/src/intermediate_bindings/codes_handle.rs @@ -8,6 +8,7 @@ use eccodes_sys::{codes_context, codes_handle}; use eccodes_sys::{codes_index, CODES_LOCK}; use libc::FILE; use num_traits::FromPrimitive; +use tracing::instrument; use crate::{ codes_handle::ProductKind, @@ -15,6 +16,13 @@ use crate::{ pointer_guard, }; +#[cfg(target_os = "macos")] +type _SYS_IO_FILE = eccodes_sys::__sFILE; + +#[cfg(not(target_os = "macos"))] +type _SYS_IO_FILE = eccodes_sys::_IO_FILE; + +#[instrument(level = "trace")] pub unsafe fn codes_handle_new_from_file( file_pointer: *mut FILE, product_kind: ProductKind, @@ -40,10 +48,8 @@ pub unsafe fn codes_handle_new_from_file( Ok(file_handle) }} +#[instrument(level = "trace")] pub unsafe fn codes_handle_delete(handle: *mut codes_handle) -> Result<(), CodesError> { unsafe { - #[cfg(test)] - log::trace!("codes_handle_delete"); - if handle.is_null() { return Ok(()); } @@ -59,6 +65,7 @@ pub unsafe fn codes_handle_delete(handle: *mut codes_handle) -> Result<(), Codes }} #[cfg(feature = "experimental_index")] +#[instrument(level = "trace")] pub unsafe fn codes_handle_new_from_index( index: *mut codes_index, ) -> Result<*mut codes_handle, CodesError> { diff --git a/src/intermediate_bindings/codes_index.rs b/src/intermediate_bindings/codes_index.rs index 097b7e9..4620c99 100644 --- a/src/intermediate_bindings/codes_index.rs +++ b/src/intermediate_bindings/codes_index.rs @@ -13,6 +13,7 @@ use crate::{ // all index functions are safeguarded by a lock // because there are random errors appearing when using the index functions concurrently +#[instrument(level = "trace")] pub unsafe fn codes_index_new(keys: &str) -> Result<*mut codes_index, CodesError> { let context: *mut codes_context = ptr::null_mut(); //default context let mut error_code: i32 = 0; @@ -43,6 +44,7 @@ pub unsafe fn codes_index_read(filename: &str) -> Result<*mut codes_index, Codes Ok(codes_index) } +#[instrument(level = "trace")] pub unsafe fn codes_index_delete(index: *mut codes_index) { #[cfg(test)] log::trace!("codes_index_delete"); diff --git a/src/intermediate_bindings/codes_keys.rs b/src/intermediate_bindings/codes_keys.rs index c5c7874..f3521dd 100644 --- a/src/intermediate_bindings/codes_keys.rs +++ b/src/intermediate_bindings/codes_keys.rs @@ -6,12 +6,14 @@ use std::ffi::{CStr, CString}; use eccodes_sys::{codes_handle, codes_keys_iterator}; use num_traits::FromPrimitive; +use tracing::instrument; use crate::{ errors::{CodesError, CodesInternal}, pointer_guard, }; +#[instrument(level = "trace")] pub unsafe fn codes_keys_iterator_new( handle: *mut codes_handle, flags: u32, @@ -30,11 +32,10 @@ pub unsafe fn codes_keys_iterator_new( Ok(kiter) }} +#[instrument(level = "trace")] pub unsafe fn codes_keys_iterator_delete( keys_iterator: *mut codes_keys_iterator, ) -> Result<(), CodesError> { unsafe { - #[cfg(test)] - log::trace!("codes_keys_iterator_delete"); if keys_iterator.is_null() { return Ok(()); diff --git a/src/intermediate_bindings/grib_nearest.rs b/src/intermediate_bindings/grib_nearest.rs index bd92833..2fb9d13 100644 --- a/src/intermediate_bindings/grib_nearest.rs +++ b/src/intermediate_bindings/grib_nearest.rs @@ -6,12 +6,14 @@ use std::ptr::addr_of_mut; use eccodes_sys::{codes_handle, codes_nearest, CODES_NEAREST_SAME_DATA, CODES_NEAREST_SAME_GRID}; use num_traits::FromPrimitive; +use tracing::instrument; use crate::{ errors::{CodesError, CodesInternal}, pointer_guard, NearestGridpoint, }; +#[instrument(level = "trace")] pub unsafe fn codes_grib_nearest_new( handle: *const codes_handle, ) -> Result<*mut codes_nearest, CodesError> { unsafe { @@ -29,10 +31,8 @@ pub unsafe fn codes_grib_nearest_new( Ok(nearest) }} +#[instrument(level = "trace")] pub unsafe fn codes_grib_nearest_delete(nearest: *mut codes_nearest) -> Result<(), CodesError> { unsafe { - #[cfg(test)] - log::trace!("codes_grib_nearest_delete"); - if nearest.is_null() { return Ok(()); } diff --git a/src/keyed_message/mod.rs b/src/keyed_message/mod.rs index 3dd81cc..3859881 100644 --- a/src/keyed_message/mod.rs +++ b/src/keyed_message/mod.rs @@ -5,7 +5,7 @@ mod read; mod write; use eccodes_sys::codes_handle; -use log::error; +use tracing::{event, instrument, Level}; use std::ptr::null_mut; use crate::{ @@ -213,10 +213,11 @@ impl Drop for KeyedMessage { /// when other functions corrupt the inner memory of pointer, in that case memory leak is possible. /// In case of corrupt pointer segmentation fault will occur. /// The pointers are cleared at the end of drop as they are not functional regardless of result of delete functions. + #[instrument(level = "trace")] fn drop(&mut self) { unsafe { codes_handle_delete(self.message_handle).unwrap_or_else(|error| { - error!("codes_handle_delete() returned an error: {:?}", &error); + event!(Level::ERROR, "codes_handle_delete() returned an error: {:?}", &error); }); } diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index 51ca4a8..58afc90 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -2,7 +2,7 @@ use eccodes_sys::codes_keys_iterator; use fallible_iterator::FallibleIterator; -use log::error; +use tracing::{event, instrument, Level}; use std::{marker::PhantomData, ptr::null_mut}; use crate::{ @@ -134,6 +134,7 @@ impl KeyedMessage { /// /// This function returns [`CodesInternal`](crate::errors::CodesInternal) when /// internal ecCodes function returns non-zero code. + #[instrument(level = "trace")] pub fn new_keys_iterator<'a>( &'a self, flags: &[KeysIteratorFlags], @@ -197,10 +198,11 @@ impl FallibleIterator for KeysIterator<'_> { #[doc(hidden)] impl Drop for KeysIterator<'_> { + #[instrument(level = "trace")] fn drop(&mut self) { unsafe { codes_keys_iterator_delete(self.iterator_handle).unwrap_or_else(|error| { - error!( + event!(Level::ERROR, "codes_keys_iterator_delete() returned an error: {:?}", &error ); From bf5b7e20f75fa9bd6d5ccae30772900481b1206f Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sat, 20 Sep 2025 11:50:21 +0200 Subject: [PATCH 06/65] fix criterion --- benches/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benches/main.rs b/benches/main.rs index 0b99ed9..724e193 100644 --- a/benches/main.rs +++ b/benches/main.rs @@ -1,7 +1,7 @@ use eccodes::FallibleStreamingIterator; use std::path::Path; - -use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use std::hint::black_box; +use criterion::{criterion_group, criterion_main, Criterion}; use eccodes::codes_handle::{CodesHandle, ProductKind}; pub fn key_reading(c: &mut Criterion) { From 83a87e13f74c54913696cc02d5f3b69e70677241 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sat, 20 Sep 2025 12:06:59 +0200 Subject: [PATCH 07/65] rustfmt --- benches/main.rs | 6 +- src/codes_handle/iterator.rs | 4 +- src/codes_handle/mod.rs | 10 +- src/codes_index.rs | 4 +- src/codes_nearest.rs | 7 +- src/intermediate_bindings/codes_get.rs | 332 ++++++++++++---------- src/intermediate_bindings/codes_handle.rs | 62 ++-- src/intermediate_bindings/codes_index.rs | 2 +- src/intermediate_bindings/codes_keys.rs | 74 ++--- src/intermediate_bindings/codes_set.rs | 174 ++++++------ src/intermediate_bindings/grib_nearest.rs | 145 +++++----- src/keyed_message/mod.rs | 16 +- src/keyed_message/read.rs | 5 +- src/keyed_message/write.rs | 4 +- src/keys_iterator.rs | 7 +- src/message_ndarray.rs | 6 +- tests/index.rs | 4 +- 17 files changed, 460 insertions(+), 402 deletions(-) diff --git a/benches/main.rs b/benches/main.rs index 724e193..5dd9c35 100644 --- a/benches/main.rs +++ b/benches/main.rs @@ -1,8 +1,8 @@ +use criterion::{Criterion, criterion_group, criterion_main}; use eccodes::FallibleStreamingIterator; -use std::path::Path; -use std::hint::black_box; -use criterion::{criterion_group, criterion_main, Criterion}; use eccodes::codes_handle::{CodesHandle, ProductKind}; +use std::hint::black_box; +use std::path::Path; pub fn key_reading(c: &mut Criterion) { //prepare the variables for benchmark diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index d29b89c..17e817f 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -1,4 +1,4 @@ -use crate::{codes_handle::HandleGenerator, errors::CodesError, CodesHandle, KeyedMessage}; +use crate::{CodesHandle, KeyedMessage, codes_handle::HandleGenerator, errors::CodesError}; use fallible_streaming_iterator::FallibleStreamingIterator; use std::fmt::Debug; @@ -35,8 +35,8 @@ impl FallibleStreamingIterator for CodesHandle { #[cfg(test)] mod tests { use crate::{ - codes_handle::{CodesHandle, ProductKind}, DynamicKeyType, + codes_handle::{CodesHandle, ProductKind}, }; use anyhow::{Context, Ok, Result}; use fallible_streaming_iterator::FallibleStreamingIterator; diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index d9b16d8..9e4c131 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -4,18 +4,18 @@ #[cfg(feature = "experimental_index")] use crate::codes_index::CodesIndex; use crate::{ - intermediate_bindings::codes_handle_new_from_file, pointer_guard, CodesError, KeyedMessage, + CodesError, KeyedMessage, intermediate_bindings::codes_handle_new_from_file, pointer_guard, }; -use eccodes_sys::{codes_handle, ProductKind_PRODUCT_GRIB}; +use eccodes_sys::{ProductKind_PRODUCT_GRIB, codes_handle}; use errno::errno; -use libc::{c_char, c_void, size_t, FILE}; -use tracing::instrument; +use libc::{FILE, c_char, c_void, size_t}; use std::{ fmt::Debug, fs::{File, OpenOptions}, os::unix::prelude::AsRawFd, path::Path, }; +use tracing::instrument; mod iterator; @@ -339,7 +339,7 @@ mod tests { #[cfg(feature = "experimental_index")] use crate::codes_index::{CodesIndex, Select}; use anyhow::{Context, Result}; - use eccodes_sys::{grib_handle, ProductKind_PRODUCT_GRIB}; + use eccodes_sys::{ProductKind_PRODUCT_GRIB, grib_handle}; use fallible_streaming_iterator::FallibleStreamingIterator; use std::{ fs::File, diff --git a/src/codes_index.rs b/src/codes_index.rs index 5fc7ddb..37572ed 100644 --- a/src/codes_index.rs +++ b/src/codes_index.rs @@ -282,13 +282,13 @@ impl Drop for CodesIndex { #[cfg(test)] mod tests { - use anyhow::{bail, Context, Result}; + use anyhow::{Context, Result, bail}; use fallible_streaming_iterator::FallibleStreamingIterator; use crate::{ + CodesError, CodesHandle, codes_index::{CodesIndex, Select}, errors::CodesInternal, - CodesError, CodesHandle, }; use std::path::Path; #[test] diff --git a/src/codes_nearest.rs b/src/codes_nearest.rs index 105adab..f7a86ef 100644 --- a/src/codes_nearest.rs +++ b/src/codes_nearest.rs @@ -4,13 +4,13 @@ use std::ptr::null_mut; use eccodes_sys::codes_nearest; -use tracing::{event, instrument, Level}; +use tracing::{Level, event, instrument}; use crate::{ + CodesError, KeyedMessage, intermediate_bindings::{ codes_grib_nearest_delete, codes_grib_nearest_find, codes_grib_nearest_new, }, - CodesError, KeyedMessage, }; /// The structure used to find nearest gridpoints in `KeyedMessage`. @@ -107,7 +107,8 @@ impl Drop for CodesNearest<'_> { fn drop(&mut self) { unsafe { codes_grib_nearest_delete(self.nearest_handle).unwrap_or_else(|error| { - event!(Level::ERROR, + event!( + Level::ERROR, "codes_grib_nearest_delete() returned an error: {:?}", &error ); diff --git a/src/intermediate_bindings/codes_get.rs b/src/intermediate_bindings/codes_get.rs index 0f580be..545b5b7 100644 --- a/src/intermediate_bindings/codes_get.rs +++ b/src/intermediate_bindings/codes_get.rs @@ -17,244 +17,266 @@ use super::NativeKeyType; pub unsafe fn codes_get_native_type( handle: *const codes_handle, key: &str, -) -> Result { unsafe { - pointer_guard::non_null!(handle); +) -> Result { + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); - let mut key_type: i32 = 0; + let key = CString::new(key).unwrap(); + let mut key_type: i32 = 0; - let error_code = eccodes_sys::codes_get_native_type(handle, key.as_ptr(), &raw mut key_type); + let error_code = eccodes_sys::codes_get_native_type(handle, key.as_ptr(), &mut key_type); - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } + + Ok(FromPrimitive::from_i32(key_type).unwrap()) } +} - Ok(FromPrimitive::from_i32(key_type).unwrap()) -}} +pub unsafe fn codes_get_size(handle: *const codes_handle, key: &str) -> Result { + unsafe { + pointer_guard::non_null!(handle); -pub unsafe fn codes_get_size(handle: *const codes_handle, key: &str) -> Result { unsafe { - pointer_guard::non_null!(handle); + let key = CString::new(key).unwrap(); + let mut key_size: usize = 0; - let key = CString::new(key).unwrap(); - let mut key_size: usize = 0; + let error_code = eccodes_sys::codes_get_size(handle, key.as_ptr(), &mut key_size); - let error_code = eccodes_sys::codes_get_size(handle, key.as_ptr(), &raw mut key_size); + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(key_size) } +} - Ok(key_size) -}} +pub unsafe fn codes_get_long(handle: *const codes_handle, key: &str) -> Result { + unsafe { + pointer_guard::non_null!(handle); -pub unsafe fn codes_get_long(handle: *const codes_handle, key: &str) -> Result { unsafe { - pointer_guard::non_null!(handle); + let key = CString::new(key).unwrap(); + let mut key_value: i64 = 0; - let key = CString::new(key).unwrap(); - let mut key_value: i64 = 0; + let error_code = eccodes_sys::codes_get_long(handle, key.as_ptr(), &mut key_value); - let error_code = eccodes_sys::codes_get_long(handle, key.as_ptr(), &raw mut key_value); + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(key_value) } +} - Ok(key_value) -}} +pub unsafe fn codes_get_double(handle: *const codes_handle, key: &str) -> Result { + unsafe { + pointer_guard::non_null!(handle); -pub unsafe fn codes_get_double(handle: *const codes_handle, key: &str) -> Result { unsafe { - pointer_guard::non_null!(handle); + let key = CString::new(key).unwrap(); + let mut key_value: f64 = 0.0; - let key = CString::new(key).unwrap(); - let mut key_value: f64 = 0.0; + let error_code = eccodes_sys::codes_get_double(handle, key.as_ptr(), &mut key_value); - let error_code = eccodes_sys::codes_get_double(handle, key.as_ptr(), &raw mut key_value); + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(key_value) } - - Ok(key_value) -}} +} pub unsafe fn codes_get_double_array( handle: *const codes_handle, key: &str, -) -> Result, CodesError> { unsafe { - pointer_guard::non_null!(handle); +) -> Result, CodesError> { + unsafe { + pointer_guard::non_null!(handle); - let mut key_size = codes_get_size(handle, key)?; - let key = CString::new(key).unwrap(); + let mut key_size = codes_get_size(handle, key)?; + let key = CString::new(key).unwrap(); - let mut key_values: Vec = vec![0.0; key_size]; + let mut key_values: Vec = vec![0.0; key_size]; - let error_code = eccodes_sys::codes_get_double_array( - handle, - key.as_ptr(), - key_values.as_mut_ptr().cast::<_>(), - &raw mut key_size, - ); + let error_code = eccodes_sys::codes_get_double_array( + handle, + key.as_ptr(), + key_values.as_mut_ptr().cast::(), + &mut key_size, + ); - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - Ok(key_values) -}} + Ok(key_values) + } +} pub unsafe fn codes_get_long_array( handle: *const codes_handle, key: &str, -) -> Result, CodesError> { unsafe { - pointer_guard::non_null!(handle); +) -> Result, CodesError> { + unsafe { + pointer_guard::non_null!(handle); - let mut key_size = codes_get_size(handle, key)?; - let key = CString::new(key).unwrap(); + let mut key_size = codes_get_size(handle, key)?; + let key = CString::new(key).unwrap(); - let mut key_values: Vec = vec![0; key_size]; + let mut key_values: Vec = vec![0; key_size]; - let error_code = eccodes_sys::codes_get_long_array( - handle, - key.as_ptr(), - key_values.as_mut_ptr().cast::<_>(), - &raw mut key_size, - ); + let error_code = eccodes_sys::codes_get_long_array( + handle, + key.as_ptr(), + key_values.as_mut_ptr().cast::(), + &mut key_size, + ); - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - Ok(key_values) -}} + Ok(key_values) + } +} pub unsafe fn codes_get_length( handle: *const codes_handle, key: &str, -) -> Result { unsafe { - pointer_guard::non_null!(handle); +) -> Result { + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); - let mut key_length: usize = 0; + let key = CString::new(key).unwrap(); + let mut key_length: usize = 0; - let error_code = eccodes_sys::codes_get_length(handle, key.as_ptr(), &raw mut key_length); + let error_code = eccodes_sys::codes_get_length(handle, key.as_ptr(), &mut key_length); - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - Ok(key_length) -}} + Ok(key_length) + } +} pub unsafe fn codes_get_string( handle: *const codes_handle, key: &str, -) -> Result { unsafe { - pointer_guard::non_null!(handle); +) -> Result { + unsafe { + pointer_guard::non_null!(handle); - let mut key_length = codes_get_length(handle, key)?; - let key = CString::new(key).unwrap(); + let mut key_length = codes_get_length(handle, key)?; + let key = CString::new(key).unwrap(); - let mut key_message: Vec = vec![0; key_length]; + let mut key_message: Vec = vec![0; key_length]; - let error_code = eccodes_sys::codes_get_string( - handle, - key.as_ptr(), - key_message.as_mut_ptr().cast::<_>(), - &raw mut key_length, - ); + let error_code = eccodes_sys::codes_get_string( + handle, + key.as_ptr(), + key_message.as_mut_ptr().cast::(), + &mut key_length, + ); - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - key_message.truncate(key_length); - let key_message_result = CStr::from_bytes_with_nul(key_message.as_ref()); + key_message.truncate(key_length); + let key_message_result = CStr::from_bytes_with_nul(key_message.as_ref()); - let key_message_cstr = if let Ok(msg) = key_message_result { - msg - } else { - key_message.push(0); - CStr::from_bytes_with_nul(key_message.as_ref())? - }; + let key_message_cstr = if let Ok(msg) = key_message_result { + msg + } else { + key_message.push(0); + CStr::from_bytes_with_nul(key_message.as_ref())? + }; - let key_message_string = key_message_cstr.to_str()?.to_string(); + let key_message_string = key_message_cstr.to_str()?.to_string(); - Ok(key_message_string) -}} + Ok(key_message_string) + } +} pub unsafe fn codes_get_bytes( handle: *const codes_handle, key: &str, -) -> Result, CodesError> { unsafe { - pointer_guard::non_null!(handle); +) -> Result, CodesError> { + unsafe { + pointer_guard::non_null!(handle); + + let mut key_size = codes_get_length(handle, key)?; + let key = CString::new(key).unwrap(); - let mut key_size = codes_get_length(handle, key)?; - let key = CString::new(key).unwrap(); + let mut buffer: Vec = vec![0; key_size]; - let mut buffer: Vec = vec![0; key_size]; + let error_code = eccodes_sys::codes_get_bytes( + handle, + key.as_ptr(), + buffer.as_mut_ptr().cast::(), + &mut key_size, + ); - let error_code = eccodes_sys::codes_get_bytes( - handle, - key.as_ptr(), - buffer.as_mut_ptr().cast::<_>(), - &raw mut key_size, - ); + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(buffer) } +} - Ok(buffer) -}} +pub unsafe fn codes_get_message_size(handle: *const codes_handle) -> Result { + unsafe { + pointer_guard::non_null!(handle); -pub unsafe fn codes_get_message_size(handle: *const codes_handle) -> Result { unsafe { - pointer_guard::non_null!(handle); + let mut size: usize = 0; - let mut size: usize = 0; + let error_code = eccodes_sys::codes_get_message_size(handle, &mut size); - let error_code = eccodes_sys::codes_get_message_size(handle, &raw mut size); + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(size) } - - Ok(size) -}} +} pub unsafe fn codes_get_message( handle: *const codes_handle, -) -> Result<(*const c_void, usize), CodesError> { unsafe { - pointer_guard::non_null!(handle); +) -> Result<(*const c_void, usize), CodesError> { + unsafe { + pointer_guard::non_null!(handle); - let buffer_size = codes_get_message_size(handle)?; + let buffer_size = codes_get_message_size(handle)?; - let buffer: Vec = vec![0; buffer_size]; - let mut buffer_ptr = buffer.as_ptr().cast::<_>(); + let buffer: Vec = vec![0; buffer_size]; + let mut buffer_ptr = buffer.as_ptr().cast::(); - let mut message_size: usize = 0; + let mut message_size: usize = 0; - let error_code = eccodes_sys::codes_get_message(handle, &raw mut buffer_ptr, &raw mut message_size); + let error_code = eccodes_sys::codes_get_message(handle, &mut buffer_ptr, &mut message_size); - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - assert!( - buffer_size == message_size, - "Buffer and message sizes ar not equal in codes_get_message! + assert!( + buffer_size == message_size, + "Buffer and message sizes ar not equal in codes_get_message! Please report this panic on Github." - ); + ); - Ok((buffer_ptr, message_size)) -}} + Ok((buffer_ptr, message_size)) + } +} diff --git a/src/intermediate_bindings/codes_handle.rs b/src/intermediate_bindings/codes_handle.rs index 75bf94b..d242837 100644 --- a/src/intermediate_bindings/codes_handle.rs +++ b/src/intermediate_bindings/codes_handle.rs @@ -3,9 +3,9 @@ use std::ptr::{self}; -use eccodes_sys::{codes_context, codes_handle}; #[cfg(feature = "experimental_index")] -use eccodes_sys::{codes_index, CODES_LOCK}; +use eccodes_sys::{CODES_LOCK, codes_index}; +use eccodes_sys::{codes_context, codes_handle}; use libc::FILE; use num_traits::FromPrimitive; use tracing::instrument; @@ -26,43 +26,47 @@ type _SYS_IO_FILE = eccodes_sys::_IO_FILE; pub unsafe fn codes_handle_new_from_file( file_pointer: *mut FILE, product_kind: ProductKind, -) -> Result<*mut codes_handle, CodesError> { unsafe { - pointer_guard::non_null!(file_pointer); +) -> Result<*mut codes_handle, CodesError> { + unsafe { + pointer_guard::non_null!(file_pointer); - let context: *mut codes_context = ptr::null_mut(); //default context + let context: *mut codes_context = ptr::null_mut(); //default context - let mut error_code: i32 = 0; + let mut error_code: i32 = 0; - let file_handle = eccodes_sys::codes_handle_new_from_file( - context, - file_pointer.cast::<_>(), - product_kind as u32, - &raw mut error_code, - ); + let file_handle = eccodes_sys::codes_handle_new_from_file( + context, + file_pointer.cast::<_SYS_IO_FILE>(), + product_kind as u32, + &mut error_code, + ); - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - Ok(file_handle) -}} + Ok(file_handle) + } +} #[instrument(level = "trace")] -pub unsafe fn codes_handle_delete(handle: *mut codes_handle) -> Result<(), CodesError> { unsafe { - if handle.is_null() { - return Ok(()); - } +pub unsafe fn codes_handle_delete(handle: *mut codes_handle) -> Result<(), CodesError> { + unsafe { + if handle.is_null() { + return Ok(()); + } - let error_code = eccodes_sys::codes_handle_delete(handle); + let error_code = eccodes_sys::codes_handle_delete(handle); - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - Ok(()) -}} + Ok(()) + } +} #[cfg(feature = "experimental_index")] #[instrument(level = "trace")] diff --git a/src/intermediate_bindings/codes_index.rs b/src/intermediate_bindings/codes_index.rs index 4620c99..6d5bd14 100644 --- a/src/intermediate_bindings/codes_index.rs +++ b/src/intermediate_bindings/codes_index.rs @@ -1,7 +1,7 @@ #![allow(non_camel_case_types)] #![allow(clippy::module_name_repetitions)] -use eccodes_sys::{codes_context, codes_index, CODES_LOCK}; +use eccodes_sys::{CODES_LOCK, codes_context, codes_index}; use num_traits::FromPrimitive; use std::{ffi::CString, ptr}; diff --git a/src/intermediate_bindings/codes_keys.rs b/src/intermediate_bindings/codes_keys.rs index f3521dd..f267658 100644 --- a/src/intermediate_bindings/codes_keys.rs +++ b/src/intermediate_bindings/codes_keys.rs @@ -18,59 +18,67 @@ pub unsafe fn codes_keys_iterator_new( handle: *mut codes_handle, flags: u32, namespace: &str, -) -> Result<*mut codes_keys_iterator, CodesError> { unsafe { - pointer_guard::non_null!(handle); +) -> Result<*mut codes_keys_iterator, CodesError> { + unsafe { + pointer_guard::non_null!(handle); - let namespace = CString::new(namespace).unwrap(); + let namespace = CString::new(namespace).unwrap(); - let kiter = eccodes_sys::codes_keys_iterator_new(handle, u64::from(flags), namespace.as_ptr()); + let kiter = + eccodes_sys::codes_keys_iterator_new(handle, u64::from(flags), namespace.as_ptr()); - if kiter.is_null() { - return Err(CodesError::KeysIteratorFailed); - } + if kiter.is_null() { + return Err(CodesError::KeysIteratorFailed); + } - Ok(kiter) -}} + Ok(kiter) + } +} #[instrument(level = "trace")] pub unsafe fn codes_keys_iterator_delete( keys_iterator: *mut codes_keys_iterator, -) -> Result<(), CodesError> { unsafe { +) -> Result<(), CodesError> { + unsafe { + if keys_iterator.is_null() { + return Ok(()); + } - if keys_iterator.is_null() { - return Ok(()); - } + let error_code = eccodes_sys::codes_keys_iterator_delete(keys_iterator); - let error_code = eccodes_sys::codes_keys_iterator_delete(keys_iterator); + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); + Ok(()) } - - Ok(()) -}} +} pub unsafe fn codes_keys_iterator_next( keys_iterator: *mut codes_keys_iterator, -) -> Result { unsafe { - pointer_guard::non_null!(keys_iterator); +) -> Result { + unsafe { + pointer_guard::non_null!(keys_iterator); - let next_item_exists = eccodes_sys::codes_keys_iterator_next(keys_iterator); + let next_item_exists = eccodes_sys::codes_keys_iterator_next(keys_iterator); - Ok(next_item_exists == 1) -}} + Ok(next_item_exists == 1) + } +} pub unsafe fn codes_keys_iterator_get_name( keys_iterator: *mut codes_keys_iterator, -) -> Result { unsafe { - pointer_guard::non_null!(keys_iterator); +) -> Result { + unsafe { + pointer_guard::non_null!(keys_iterator); - let name_pointer = eccodes_sys::codes_keys_iterator_get_name(keys_iterator); + let name_pointer = eccodes_sys::codes_keys_iterator_get_name(keys_iterator); - let name_c_str = CStr::from_ptr(name_pointer); - let name_str = name_c_str.to_str()?; - let name_string = name_str.to_owned(); + let name_c_str = CStr::from_ptr(name_pointer); + let name_str = name_c_str.to_str()?; + let name_string = name_str.to_owned(); - Ok(name_string) -}} + Ok(name_string) + } +} diff --git a/src/intermediate_bindings/codes_set.rs b/src/intermediate_bindings/codes_set.rs index 442bf5f..80f7117 100644 --- a/src/intermediate_bindings/codes_set.rs +++ b/src/intermediate_bindings/codes_set.rs @@ -16,136 +16,148 @@ pub unsafe fn codes_set_long( handle: *mut codes_handle, key: &str, value: i64, -) -> Result<(), CodesError> { unsafe { - pointer_guard::non_null!(handle); +) -> Result<(), CodesError> { + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); + let key = CString::new(key).unwrap(); - let error_code = eccodes_sys::codes_set_long(handle, key.as_ptr(), value); + let error_code = eccodes_sys::codes_set_long(handle, key.as_ptr(), value); - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - Ok(()) -}} + Ok(()) + } +} pub unsafe fn codes_set_double( handle: *mut codes_handle, key: &str, value: f64, -) -> Result<(), CodesError> { unsafe { - pointer_guard::non_null!(handle); +) -> Result<(), CodesError> { + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); + let key = CString::new(key).unwrap(); - let error_code = eccodes_sys::codes_set_double(handle, key.as_ptr(), value); + let error_code = eccodes_sys::codes_set_double(handle, key.as_ptr(), value); - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - Ok(()) -}} + Ok(()) + } +} pub unsafe fn codes_set_long_array( handle: *mut codes_handle, key: &str, values: &[i64], -) -> Result<(), CodesError> { unsafe { - pointer_guard::non_null!(handle); +) -> Result<(), CodesError> { + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); + let key = CString::new(key).unwrap(); - let length = values.len(); + let length = values.len(); - let error_code = eccodes_sys::codes_set_long_array( - handle, - key.as_ptr(), - values.as_ptr().cast::<_>(), - length, - ); + let error_code = eccodes_sys::codes_set_long_array( + handle, + key.as_ptr(), + values.as_ptr().cast::(), + length, + ); - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - Ok(()) -}} + Ok(()) + } +} pub unsafe fn codes_set_double_array( handle: *mut codes_handle, key: &str, values: &[f64], -) -> Result<(), CodesError> { unsafe { - pointer_guard::non_null!(handle); +) -> Result<(), CodesError> { + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); + let key = CString::new(key).unwrap(); - let length = values.len(); + let length = values.len(); - let error_code = eccodes_sys::codes_set_double_array( - handle, - key.as_ptr(), - values.as_ptr().cast::<_>(), - length, - ); + let error_code = eccodes_sys::codes_set_double_array( + handle, + key.as_ptr(), + values.as_ptr().cast::(), + length, + ); - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - Ok(()) -}} + Ok(()) + } +} pub unsafe fn codes_set_string( handle: *mut codes_handle, key: &str, value: &str, -) -> Result<(), CodesError> { unsafe { - pointer_guard::non_null!(handle); +) -> Result<(), CodesError> { + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); - let mut length = value.len(); - let value = CString::new(value).unwrap(); + let key = CString::new(key).unwrap(); + let mut length = value.len(); + let value = CString::new(value).unwrap(); - let error_code = - eccodes_sys::codes_set_string(handle, key.as_ptr(), value.as_ptr(), &raw mut length); + let error_code = + eccodes_sys::codes_set_string(handle, key.as_ptr(), value.as_ptr(), &mut length); - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - Ok(()) -}} + Ok(()) + } +} pub unsafe fn codes_set_bytes( handle: *mut codes_handle, key: &str, values: &[u8], -) -> Result<(), CodesError> { unsafe { - pointer_guard::non_null!(handle); +) -> Result<(), CodesError> { + unsafe { + pointer_guard::non_null!(handle); - let key = CString::new(key).unwrap(); + let key = CString::new(key).unwrap(); - let mut length = values.len(); + let mut length = values.len(); - let error_code = eccodes_sys::codes_set_bytes( - handle, - key.as_ptr(), - values.as_ptr().cast::<_>(), - &raw mut length, - ); + let error_code = eccodes_sys::codes_set_bytes( + handle, + key.as_ptr(), + values.as_ptr().cast::(), + &mut length, + ); - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - Ok(()) -}} + Ok(()) + } +} diff --git a/src/intermediate_bindings/grib_nearest.rs b/src/intermediate_bindings/grib_nearest.rs index 2fb9d13..a02cc66 100644 --- a/src/intermediate_bindings/grib_nearest.rs +++ b/src/intermediate_bindings/grib_nearest.rs @@ -3,99 +3,106 @@ use std::ptr::addr_of_mut; -use eccodes_sys::{codes_handle, codes_nearest, CODES_NEAREST_SAME_DATA, CODES_NEAREST_SAME_GRID}; +use eccodes_sys::{CODES_NEAREST_SAME_DATA, CODES_NEAREST_SAME_GRID, codes_handle, codes_nearest}; use num_traits::FromPrimitive; use tracing::instrument; use crate::{ + NearestGridpoint, errors::{CodesError, CodesInternal}, - pointer_guard, NearestGridpoint, + pointer_guard, }; #[instrument(level = "trace")] pub unsafe fn codes_grib_nearest_new( handle: *const codes_handle, -) -> Result<*mut codes_nearest, CodesError> { unsafe { - pointer_guard::non_null!(handle); +) -> Result<*mut codes_nearest, CodesError> { + unsafe { + pointer_guard::non_null!(handle); - let mut error_code: i32 = 0; + let mut error_code: i32 = 0; - let nearest = eccodes_sys::codes_grib_nearest_new(handle, &raw mut error_code); + let nearest = eccodes_sys::codes_grib_nearest_new(handle, &mut error_code); - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - Ok(nearest) -}} + Ok(nearest) + } +} #[instrument(level = "trace")] -pub unsafe fn codes_grib_nearest_delete(nearest: *mut codes_nearest) -> Result<(), CodesError> { unsafe { - if nearest.is_null() { - return Ok(()); - } +pub unsafe fn codes_grib_nearest_delete(nearest: *mut codes_nearest) -> Result<(), CodesError> { + unsafe { + if nearest.is_null() { + return Ok(()); + } - let error_code = eccodes_sys::codes_grib_nearest_delete(nearest); + let error_code = eccodes_sys::codes_grib_nearest_delete(nearest); - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } - Ok(()) -}} + Ok(()) + } +} pub unsafe fn codes_grib_nearest_find( handle: *const codes_handle, nearest: *mut codes_nearest, lat: f64, lon: f64, -) -> Result<[NearestGridpoint; 4], CodesError> { unsafe { - pointer_guard::non_null!(handle); - pointer_guard::non_null!(nearest); - - // such flags are set because find nearest for given nearest is always - // called on the same grib message - let flags = CODES_NEAREST_SAME_GRID + CODES_NEAREST_SAME_DATA; - - let mut output_lats = [0_f64; 4]; - let mut output_lons = [0_f64; 4]; - let mut output_values = [0_f64; 4]; - let mut output_distances = [0_f64; 4]; - let mut output_indexes = [0_i32; 4]; - - let mut length: usize = 4; - - let error_code = eccodes_sys::codes_grib_nearest_find( - nearest, - handle, - lat, - lon, - u64::from(flags), - addr_of_mut!(output_lats[0]), - addr_of_mut!(output_lons[0]), - addr_of_mut!(output_values[0]), - addr_of_mut!(output_distances[0]), - addr_of_mut!(output_indexes[0]), - &raw mut length, - ); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); +) -> Result<[NearestGridpoint; 4], CodesError> { + unsafe { + pointer_guard::non_null!(handle); + pointer_guard::non_null!(nearest); + + // such flags are set because find nearest for given nearest is always + // called on the same grib message + let flags = CODES_NEAREST_SAME_GRID + CODES_NEAREST_SAME_DATA; + + let mut output_lats = [0_f64; 4]; + let mut output_lons = [0_f64; 4]; + let mut output_values = [0_f64; 4]; + let mut output_distances = [0_f64; 4]; + let mut output_indexes = [0_i32; 4]; + + let mut length: usize = 4; + + let error_code = eccodes_sys::codes_grib_nearest_find( + nearest, + handle, + lat, + lon, + u64::from(flags), + addr_of_mut!(output_lats[0]), + addr_of_mut!(output_lons[0]), + addr_of_mut!(output_values[0]), + addr_of_mut!(output_distances[0]), + addr_of_mut!(output_indexes[0]), + &mut length, + ); + + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); + return Err(err.into()); + } + + let mut output = [NearestGridpoint::default(); 4]; + + for i in 0..4 { + output[i].lat = output_lats[i]; + output[i].lon = output_lons[i]; + output[i].distance = output_distances[i]; + output[i].index = output_indexes[i]; + output[i].value = output_values[i]; + } + + Ok(output) } - - let mut output = [NearestGridpoint::default(); 4]; - - for i in 0..4 { - output[i].lat = output_lats[i]; - output[i].lon = output_lons[i]; - output[i].distance = output_distances[i]; - output[i].index = output_indexes[i]; - output[i].value = output_values[i]; - } - - Ok(output) -}} +} diff --git a/src/keyed_message/mod.rs b/src/keyed_message/mod.rs index 3859881..516e7de 100644 --- a/src/keyed_message/mod.rs +++ b/src/keyed_message/mod.rs @@ -5,15 +5,15 @@ mod read; mod write; use eccodes_sys::codes_handle; -use tracing::{event, instrument, Level}; use std::ptr::null_mut; +use tracing::{Level, event, instrument}; use crate::{ + CodesError, intermediate_bindings::{ - codes_get_native_type, codes_get_size, codes_handle_clone, codes_handle_delete, - NativeKeyType, + NativeKeyType, codes_get_native_type, codes_get_size, codes_handle_clone, + codes_handle_delete, }, - CodesError, }; /// Structure that provides access to the data contained in the GRIB file, which directly corresponds to the message in the GRIB file @@ -217,7 +217,11 @@ impl Drop for KeyedMessage { fn drop(&mut self) { unsafe { codes_handle_delete(self.message_handle).unwrap_or_else(|error| { - event!(Level::ERROR, "codes_handle_delete() returned an error: {:?}", &error); + event!( + Level::ERROR, + "codes_handle_delete() returned an error: {:?}", + &error + ); }); } @@ -227,8 +231,8 @@ impl Drop for KeyedMessage { #[cfg(test)] mod tests { - use crate::codes_handle::{CodesHandle, ProductKind}; use crate::FallibleStreamingIterator; + use crate::codes_handle::{CodesHandle, ProductKind}; use anyhow::{Context, Result}; use std::path::Path; use testing_logger; diff --git a/src/keyed_message/read.rs b/src/keyed_message/read.rs index 2187ef7..d46f8b3 100644 --- a/src/keyed_message/read.rs +++ b/src/keyed_message/read.rs @@ -1,13 +1,12 @@ use std::cmp::Ordering; use crate::{ + DynamicKeyType, KeyRead, KeyedMessage, errors::CodesError, intermediate_bindings::{ - codes_get_bytes, codes_get_double, codes_get_double_array, codes_get_long, + NativeKeyType, codes_get_bytes, codes_get_double, codes_get_double_array, codes_get_long, codes_get_long_array, codes_get_native_type, codes_get_size, codes_get_string, - NativeKeyType, }, - DynamicKeyType, KeyRead, KeyedMessage, }; impl KeyRead for KeyedMessage { diff --git a/src/keyed_message/write.rs b/src/keyed_message/write.rs index 31cffba..e639974 100644 --- a/src/keyed_message/write.rs +++ b/src/keyed_message/write.rs @@ -1,12 +1,12 @@ use std::{fs::OpenOptions, io::Write, path::Path, slice}; use crate::{ + KeyedMessage, errors::CodesError, intermediate_bindings::{ codes_get_message, codes_set_bytes, codes_set_double, codes_set_double_array, codes_set_long, codes_set_long_array, codes_set_string, }, - KeyedMessage, }; use super::KeyWrite; @@ -134,8 +134,8 @@ mod tests { use anyhow::{Context, Result}; use crate::{ - codes_handle::{CodesHandle, ProductKind}, DynamicKeyType, FallibleStreamingIterator, KeyWrite, + codes_handle::{CodesHandle, ProductKind}, }; use std::{fs::remove_file, path::Path}; diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index 58afc90..8d504f6 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -2,16 +2,16 @@ use eccodes_sys::codes_keys_iterator; use fallible_iterator::FallibleIterator; -use tracing::{event, instrument, Level}; use std::{marker::PhantomData, ptr::null_mut}; +use tracing::{Level, event, instrument}; use crate::{ + KeyedMessage, errors::CodesError, intermediate_bindings::{ codes_keys_iterator_delete, codes_keys_iterator_get_name, codes_keys_iterator_new, codes_keys_iterator_next, }, - KeyedMessage, }; /// Structure to iterate through key names in [`KeyedMessage`]. @@ -202,7 +202,8 @@ impl Drop for KeysIterator<'_> { fn drop(&mut self) { unsafe { codes_keys_iterator_delete(self.iterator_handle).unwrap_or_else(|error| { - event!(Level::ERROR, + event!( + Level::ERROR, "codes_keys_iterator_delete() returned an error: {:?}", &error ); diff --git a/src/message_ndarray.rs b/src/message_ndarray.rs index e60119f..d68889d 100644 --- a/src/message_ndarray.rs +++ b/src/message_ndarray.rs @@ -1,9 +1,9 @@ #![cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] //! Definitions for converting a `KeyedMessage` to ndarray -use ndarray::{s, Array2, Array3}; +use ndarray::{Array2, Array3, s}; -use crate::{errors::MessageNdarrayError, CodesError, KeyRead, KeyedMessage}; +use crate::{CodesError, KeyRead, KeyedMessage, errors::MessageNdarrayError}; /// Struct returned by [`KeyedMessage::to_lons_lats_values()`] method. /// The arrays are collocated, meaning that `longitudes[i, j]` and `latitudes[i, j]` are the coordinates of `values[i, j]`. @@ -143,10 +143,10 @@ mod tests { use float_cmp::assert_approx_eq; use super::*; - use crate::codes_handle::CodesHandle; use crate::DynamicKeyType; use crate::FallibleStreamingIterator; use crate::ProductKind; + use crate::codes_handle::CodesHandle; use std::path::Path; #[test] diff --git a/tests/index.rs b/tests/index.rs index e58358e..d3674c0 100644 --- a/tests/index.rs +++ b/tests/index.rs @@ -8,8 +8,8 @@ use std::{ use anyhow::{Context, Result}; use eccodes::{ - codes_index::Select, CodesError, CodesHandle, CodesIndex, FallibleStreamingIterator, KeyRead, - ProductKind, + CodesError, CodesHandle, CodesIndex, FallibleStreamingIterator, KeyRead, ProductKind, + codes_index::Select, }; use rand::Rng; From b4adbb01ad43ca985db10eea176480e3dfd36ee0 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sat, 20 Sep 2025 12:37:43 +0200 Subject: [PATCH 08/65] stop testing destructors with log - instead we have null guards and we early fail test with infalliable drop fails; codes index is just special... --- Cargo.toml | 2 -- src/codes_handle/mod.rs | 33 --------------------------------- src/codes_nearest.rs | 15 ++------------- src/keyed_message/mod.rs | 23 +++-------------------- src/keys_iterator.rs | 15 ++------------- tests/handle.rs | 24 ------------------------ 6 files changed, 7 insertions(+), 105 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d053c9a..449dad3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,8 +39,6 @@ tracing = { version = "0.1", default-features = false, features = [ [dev-dependencies] reqwest = { version = "0.12", features = ["rustls-tls"] } criterion = "0.7" -testing_logger = "0.1" -log = "*" rand = "0.9" anyhow = { version = "1.0", features = ["backtrace"] } float-cmp = "0.10" diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index 9e4c131..4c726a9 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -472,25 +472,17 @@ mod tests { #[test] fn codes_handle_drop_file() -> Result<()> { - testing_logger::setup(); - let file_path = Path::new("./data/iceland-surface.grib"); let product_kind = ProductKind::GRIB; let handle = CodesHandle::new_from_file(file_path, product_kind)?; drop(handle); - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 0); - }); - Ok(()) } #[test] fn codes_handle_drop_mem() -> Result<()> { - testing_logger::setup(); - let product_kind = ProductKind::GRIB; let mut f = File::open(Path::new("./data/iceland.grib"))?; @@ -500,16 +492,11 @@ mod tests { let handle = CodesHandle::new_from_memory(buf, product_kind)?; drop(handle); - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 0); - }); - Ok(()) } #[test] fn multiple_drops() -> Result<()> { - testing_logger::setup(); { let file_path = Path::new("./data/iceland-surface.grib"); let product_kind = ProductKind::GRIB; @@ -524,26 +511,6 @@ mod tests { let _kiter = clone_msg.default_keys_iterator()?; } - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 5); - - let expected_logs = vec![ - ("codes_handle_delete", log::Level::Trace), - ("codes_keys_iterator_delete", log::Level::Trace), - ("codes_grib_nearest_delete", log::Level::Trace), - ("codes_handle_delete", log::Level::Trace), - ("codes_handle_delete", log::Level::Trace), - ]; - - captured_logs - .iter() - .zip(expected_logs) - .for_each(|(clg, elg)| { - assert_eq!(clg.body, elg.0); - assert_eq!(clg.level, elg.1) - }); - }); - Ok(()) } diff --git a/src/codes_nearest.rs b/src/codes_nearest.rs index f7a86ef..3131ae0 100644 --- a/src/codes_nearest.rs +++ b/src/codes_nearest.rs @@ -112,6 +112,8 @@ impl Drop for CodesNearest<'_> { "codes_grib_nearest_delete() returned an error: {:?}", &error ); + #[cfg(test)] + assert!(false, "Error in CodesNearest::drop") }); } @@ -164,21 +166,8 @@ mod tests { let _nrst = current_message.codes_nearest()?; drop(_nrst); - - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 1); - assert_eq!(captured_logs[0].body, "codes_grib_nearest_delete"); - assert_eq!(captured_logs[0].level, log::Level::Trace); - }); - drop(handle); - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 1); - assert_eq!(captured_logs[0].body, "codes_handle_delete"); - assert_eq!(captured_logs[0].level, log::Level::Trace); - }); - Ok(()) } } diff --git a/src/keyed_message/mod.rs b/src/keyed_message/mod.rs index 516e7de..6a59c84 100644 --- a/src/keyed_message/mod.rs +++ b/src/keyed_message/mod.rs @@ -222,6 +222,8 @@ impl Drop for KeyedMessage { "codes_handle_delete() returned an error: {:?}", &error ); + #[cfg(test)] + assert!(false, "Error in KeyedMessage::drop") }); } @@ -235,7 +237,6 @@ mod tests { use crate::codes_handle::{CodesHandle, ProductKind}; use anyhow::{Context, Result}; use std::path::Path; - use testing_logger; #[test] fn check_docs_keys() -> Result<()> { @@ -296,7 +297,6 @@ mod tests { #[test] fn message_clone_drop() -> Result<()> { - testing_logger::setup(); let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; @@ -304,25 +304,8 @@ mod tests { let _msg_ref = handle.next()?.context("Message not some")?; let _msg_clone = _msg_ref.try_clone()?; - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 0); - }); - - drop(_msg_clone); - - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 1); - assert_eq!(captured_logs[0].body, "codes_handle_delete"); - assert_eq!(captured_logs[0].level, log::Level::Trace); - }); - drop(handle); - - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 1); - assert_eq!(captured_logs[0].body, "codes_handle_delete"); - assert_eq!(captured_logs[0].level, log::Level::Trace); - }); + drop(_msg_clone); Ok(()) } diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index 8d504f6..1b3b350 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -207,6 +207,8 @@ impl Drop for KeysIterator<'_> { "codes_keys_iterator_delete() returned an error: {:?}", &error ); + #[cfg(test)] + assert!(false, "Error in KeysIterator::drop") }); } @@ -283,21 +285,8 @@ mod tests { let _kiter = current_message.default_keys_iterator()?; drop(_kiter); - - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 1); - assert_eq!(captured_logs[0].body, "codes_keys_iterator_delete"); - assert_eq!(captured_logs[0].level, log::Level::Trace); - }); - drop(handle); - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 1); - assert_eq!(captured_logs[0].body, "codes_handle_delete"); - assert_eq!(captured_logs[0].level, log::Level::Trace); - }); - Ok(()) } } diff --git a/tests/handle.rs b/tests/handle.rs index 19a6377..2fed63c 100644 --- a/tests/handle.rs +++ b/tests/handle.rs @@ -50,27 +50,3 @@ fn thread_safety_core() -> Result<()> { Ok(()) } - -#[test] -fn check_no_testing_logs() -> Result<()> { - testing_logger::setup(); - { - let file_path = Path::new("./data/iceland-surface.grib"); - let product_kind = ProductKind::GRIB; - - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - - let _ref_msg = handle.next()?.context("no message")?; - let clone_msg = _ref_msg.try_clone()?; - let _oth_ref = handle.next()?.context("no message")?; - - let _nrst = clone_msg.codes_nearest()?; - let _kiter = clone_msg.default_keys_iterator()?; - } - - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 0); - }); - - Ok(()) -} From 4f21d4524fb927414fd85bd87f52fd15645c53d6 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sat, 20 Sep 2025 18:28:00 +0200 Subject: [PATCH 09/65] remove fallible streaming iterator for using keyedmessage with phantomdata, all tests pass except doctests --- Cargo.toml | 1 - benches/main.rs | 4 +- src/codes_handle/iterator.rs | 94 +++++++++++++++++++++--------------- src/codes_handle/mod.rs | 43 ++++++----------- src/codes_nearest.rs | 26 ++++++---- src/keyed_message/mod.rs | 37 +++++++++----- src/keyed_message/read.rs | 26 +++++----- src/keyed_message/write.rs | 43 +++++++++-------- src/keys_iterator.rs | 25 ++++++---- src/lib.rs | 1 - src/message_ndarray.rs | 8 +-- tests/handle.rs | 16 +++--- 12 files changed, 178 insertions(+), 146 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 449dad3..0925d2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,6 @@ errno = { version = "0.3", default-features = false } num-derive = { version = "0.4", default-features = false } num-traits = { version = "0.2", default-features = false } fallible-iterator = { version = "0.3", default-features = false } -fallible-streaming-iterator = { version = "0.1.9", default-features = false } ndarray = { version = "0.16", default-features = false, optional = true, features = [ "std", ] } diff --git a/benches/main.rs b/benches/main.rs index 5dd9c35..0eaee7b 100644 --- a/benches/main.rs +++ b/benches/main.rs @@ -1,5 +1,5 @@ use criterion::{Criterion, criterion_group, criterion_main}; -use eccodes::FallibleStreamingIterator; +use eccodes::FallibleIterator; use eccodes::codes_handle::{CodesHandle, ProductKind}; use std::hint::black_box; use std::path::Path; @@ -11,7 +11,7 @@ pub fn key_reading(c: &mut Criterion) { let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - let msg = handle.next().unwrap().unwrap(); + let msg = handle.message_generator().next().unwrap().unwrap(); c.bench_function("long reading", |b| { b.iter(|| msg.read_key_dynamic(black_box("dataDate")).unwrap()) diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index 17e817f..a27f48a 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -1,45 +1,48 @@ +use fallible_iterator::FallibleIterator; + use crate::{CodesHandle, KeyedMessage, codes_handle::HandleGenerator, errors::CodesError}; -use fallible_streaming_iterator::FallibleStreamingIterator; -use std::fmt::Debug; +use std::marker::PhantomData; + +#[derive(Debug)] +pub struct KeyedMessageGenerator<'a, S: HandleGenerator> { + codes_handle: &'a mut CodesHandle, +} + +impl CodesHandle { + pub fn message_generator<'a>(&'a mut self) -> KeyedMessageGenerator<'a, S> { + KeyedMessageGenerator { codes_handle: self } + } +} /// # Errors /// -/// The `advance()` and `next()` methods will return [`CodesInternal`](crate::errors::CodesInternal) +/// The `next()` will return [`CodesInternal`](crate::errors::CodesInternal) /// when internal ecCodes function returns non-zero code. -impl FallibleStreamingIterator for CodesHandle { - type Item = KeyedMessage; - +impl<'ch, S: HandleGenerator> FallibleIterator for KeyedMessageGenerator<'ch, S> { + type Item = KeyedMessage<'ch>; type Error = CodesError; - fn advance(&mut self) -> Result<(), Self::Error> { - // destructor of KeyedMessage calls ecCodes + fn next(&mut self) -> Result, Self::Error> { + let new_eccodes_handle = self.codes_handle.source.gen_codes_handle()?; - let new_eccodes_handle = self.source.gen_codes_handle()?; - - self.current_message = if new_eccodes_handle.is_null() { - None + if new_eccodes_handle.is_null() { + Ok(None) } else { - Some(KeyedMessage { + Ok(Some(KeyedMessage { + parent_message: PhantomData, message_handle: new_eccodes_handle, - }) - }; - - Ok(()) - } - - fn get(&self) -> Option<&Self::Item> { - self.current_message.as_ref() + })) + } } } #[cfg(test)] mod tests { use crate::{ - DynamicKeyType, + DynamicKeyType, FallibleIterator, codes_handle::{CodesHandle, ProductKind}, }; use anyhow::{Context, Ok, Result}; - use fallible_streaming_iterator::FallibleStreamingIterator; use std::path::Path; #[test] @@ -48,14 +51,26 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let msg1 = handle.next()?.context("Message not some")?; + let msg1 = handle + .message_generator() + .next()? + .context("Message not some")?; let key1 = msg1.read_key_dynamic("typeOfLevel")?; + drop(msg1); - let msg2 = handle.next()?.context("Message not some")?; + let msg2 = handle + .message_generator() + .next()? + .context("Message not some")?; let key2 = msg2.read_key_dynamic("typeOfLevel")?; + drop(msg2); - let msg3 = handle.next()?.context("Message not some")?; + let msg3 = handle + .message_generator() + .next()? + .context("Message not some")?; let key3 = msg3.read_key_dynamic("typeOfLevel")?; + drop(msg3); assert_eq!(key1, DynamicKeyType::Str("isobaricInhPa".to_string())); assert_eq!(key2, DynamicKeyType::Str("isobaricInhPa".to_string())); @@ -71,7 +86,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - while let Some(msg) = handle.next()? { + while let Some(msg) = handle.message_generator().next()? { let key = msg.read_key_dynamic("shortName")?; match key { @@ -91,7 +106,7 @@ mod tests { let mut handle_collected = vec![]; - while let Some(msg) = handle.next()? { + while let Some(msg) = handle.message_generator().next()? { handle_collected.push(msg.try_clone()?); } @@ -112,7 +127,7 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let current_message = handle.message_generator().next()?.context("Message not some")?; assert!(!current_message.message_handle.is_null()); @@ -125,17 +140,18 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut mgen = handle.message_generator(); - assert!(handle.next()?.is_some()); - assert!(handle.next()?.is_some()); - assert!(handle.next()?.is_some()); - assert!(handle.next()?.is_some()); - assert!(handle.next()?.is_some()); + assert!(mgen.next()?.is_some()); + assert!(mgen.next()?.is_some()); + assert!(mgen.next()?.is_some()); + assert!(mgen.next()?.is_some()); + assert!(mgen.next()?.is_some()); - assert!(handle.next()?.is_none()); - assert!(handle.next()?.is_none()); - assert!(handle.next()?.is_none()); - assert!(handle.next()?.is_none()); + assert!(mgen.next()?.is_none()); + assert!(mgen.next()?.is_none()); + assert!(mgen.next()?.is_none()); + assert!(mgen.next()?.is_none()); Ok(()) } @@ -151,7 +167,7 @@ mod tests { // First, filter and collect the messages to get those that we want let mut level = vec![]; - while let Some(msg) = handle.next()? { + while let Some(msg) = handle.message_generator().next()? { if msg.read_key_dynamic("shortName")? == DynamicKeyType::Str("msl".to_string()) && msg.read_key_dynamic("typeOfLevel")? == DynamicKeyType::Str("surface".to_string()) diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index 4c726a9..1b8826e 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -3,9 +3,7 @@ #[cfg(feature = "experimental_index")] use crate::codes_index::CodesIndex; -use crate::{ - CodesError, KeyedMessage, intermediate_bindings::codes_handle_new_from_file, pointer_guard, -}; +use crate::{CodesError, intermediate_bindings::codes_handle_new_from_file, pointer_guard}; use eccodes_sys::{ProductKind_PRODUCT_GRIB, codes_handle}; use errno::errno; use libc::{FILE, c_char, c_void, size_t}; @@ -24,7 +22,7 @@ mod iterator; /// It is not intended to be used directly by the user. #[doc(hidden)] #[derive(Debug)] -pub struct CodesFile { +pub struct CodesFile { // fields dropped from top pointer: *mut FILE, product_kind: ProductKind, @@ -33,11 +31,11 @@ pub struct CodesFile { /// Internal trait implemented for types that can be called to generate `*mut codes_handle`. #[doc(hidden)] -pub trait HandleGenerator { +pub trait HandleGenerator: Debug { fn gen_codes_handle(&self) -> Result<*mut codes_handle, CodesError>; } -impl HandleGenerator for CodesFile { +impl HandleGenerator for CodesFile { fn gen_codes_handle(&self) -> Result<*mut codes_handle, CodesError> { unsafe { codes_handle_new_from_file(self.pointer, self.product_kind) } } @@ -52,25 +50,17 @@ impl HandleGenerator for CodesFile { /// - From GRIB index using [`new_from_index()`](CodesHandle::new_from_index) (with `experimental_index` feature enabled) /// /// Destructor for this structure does not panic, but some internal functions may rarely fail -/// leading to bugs. Errors encountered in the destructor are logged with [`log`]. +/// leading to bugs. Errors encountered in the destructor are logged with [`tracing`]. /// -/// # `FallibleStreamingIterator` +/// # `FallibleIterator` /// -/// This structure implements [`FallibleStreamingIterator`](crate::FallibleStreamingIterator) trait which allows to access GRIB messages. +/// This structure implements [`FallibleIterator`](crate::FallibleStreamingIterator) trait which allows to access GRIB messages. /// /// To access GRIB messages the ecCodes library uses a method similar to a C-style iterator. /// It digests the `* FILE` multiple times, each time returning the `*mut codes_handle` -/// to a message inside the file. The behavior of previous `*mut codes_handle` after next one is generated is undefined -/// and we assume here that it is unsafe to use "old" `*mut codes_handle`. -/// -/// In Rust, such pattern is best represented by a streaming iterator which returns a reference to the message, -/// that is valid only until the next iteration. If you need to prolong the lifetime of the message, you can clone it. -/// Internal ecCodes functions can fail, necessitating the streaming iterator to be implemented with -/// [`FallibleStreamingIterator`](crate::FallibleStreamingIterator) trait. +/// to a message inside the file. /// -/// As of `0.10` release, none of the available streaming iterator crates utilises already stabilized GATs. -/// This unfortunately significantly limits the number of methods available for `CodesHandle` iterator. -/// Therefore the probably most versatile way to iterate over the messages is to use `while let` loop. +/// This behaviour is represented in this crate by `FallibleIterator`, because generating `KeyedMessage` can fail. /// /// ``` /// use eccodes::{ProductKind, CodesHandle, KeyRead}; @@ -120,9 +110,7 @@ impl HandleGenerator for CodesFile { /// /// All available methods for `CodesHandle` iterator can be found in [`FallibleStreamingIterator`](crate::FallibleStreamingIterator) trait. #[derive(Debug)] -pub struct CodesHandle { - // fields are dropped from top to bottom - current_message: Option, +pub struct CodesHandle { source: S, } @@ -198,7 +186,6 @@ impl CodesHandle> { pointer: file_pointer, product_kind, }, - current_message: None, }) } } @@ -250,7 +237,6 @@ impl CodesHandle>> { product_kind, pointer: file_pointer, }, - current_message: None, }) } } @@ -340,7 +326,7 @@ mod tests { use crate::codes_index::{CodesIndex, Select}; use anyhow::{Context, Result}; use eccodes_sys::{ProductKind_PRODUCT_GRIB, grib_handle}; - use fallible_streaming_iterator::FallibleStreamingIterator; + use fallible_iterator::FallibleIterator; use std::{ fs::File, io::Read, @@ -356,7 +342,6 @@ mod tests { let handle = CodesHandle::new_from_file(file_path, product_kind)?; assert!(!handle.source.pointer.is_null()); - assert!(handle.current_message.is_none()); assert_eq!(handle.source.product_kind as u32, { ProductKind_PRODUCT_GRIB }); @@ -438,7 +423,6 @@ mod tests { let handle = CodesHandle::new_from_memory(buf, product_kind)?; assert!(!handle.source.pointer.is_null()); - assert!(handle.current_message.is_none()); assert_eq!(handle.source.product_kind as u32, { ProductKind_PRODUCT_GRIB }); @@ -503,9 +487,10 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let _ref_msg = handle.next()?.context("no message")?; + let _ref_msg = handle.message_generator().next()?.context("no message")?; let clone_msg = _ref_msg.try_clone()?; - let _oth_ref = handle.next()?.context("no message")?; + drop(_ref_msg); + let _oth_ref = handle.message_generator().next()?.context("no message")?; let _nrst = clone_msg.codes_nearest()?; let _kiter = clone_msg.default_keys_iterator()?; diff --git a/src/codes_nearest.rs b/src/codes_nearest.rs index 3131ae0..a86dd1f 100644 --- a/src/codes_nearest.rs +++ b/src/codes_nearest.rs @@ -1,7 +1,7 @@ //! Definition and associated functions of `CodesNearest` //! used for finding nearest gridpoints in `KeyedMessage` -use std::ptr::null_mut; +use std::{fmt::Debug, ptr::null_mut}; use eccodes_sys::codes_nearest; use tracing::{Level, event, instrument}; @@ -17,7 +17,7 @@ use crate::{ #[derive(Debug)] pub struct CodesNearest<'a> { nearest_handle: *mut codes_nearest, - parent_message: &'a KeyedMessage, + parent_message: &'a KeyedMessage<'a>, } /// The structure returned by [`CodesNearest::find_nearest()`]. @@ -36,7 +36,7 @@ pub struct NearestGridpoint { pub value: f64, } -impl KeyedMessage { +impl KeyedMessage<'_> { /// Creates a new instance of [`CodesNearest`] for the `KeyedMessage`. /// [`CodesNearest`] can be used to find nearest gridpoints for given coordinates in the `KeyedMessage` /// by calling [`find_nearest()`](crate::CodesNearest::find_nearest). @@ -126,7 +126,7 @@ mod tests { use std::path::Path; use anyhow::{Context, Result}; - use fallible_streaming_iterator::FallibleStreamingIterator; + use fallible_iterator::FallibleIterator; use crate::{CodesHandle, ProductKind}; @@ -137,12 +137,18 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle1 = CodesHandle::new_from_file(file_path1, product_kind)?; - let msg1 = handle1.next()?.context("Message not some")?; + let msg1 = handle1 + .message_generator() + .next()? + .context("Message not some")?; let nrst1 = msg1.codes_nearest()?; let out1 = nrst1.find_nearest(64.13, -21.89)?; let mut handle2 = CodesHandle::new_from_file(file_path2, product_kind)?; - let msg2 = handle2.next()?.context("Message not some")?; + let msg2 = handle2 + .message_generator() + .next()? + .context("Message not some")?; let nrst2 = msg2.codes_nearest()?; let out2 = nrst2.find_nearest(64.13, -21.89)?; @@ -161,13 +167,13 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let current_message = handle + .message_generator() + .next()? + .context("Message not some")?; let _nrst = current_message.codes_nearest()?; - drop(_nrst); - drop(handle); - Ok(()) } } diff --git a/src/keyed_message/mod.rs b/src/keyed_message/mod.rs index 6a59c84..00de774 100644 --- a/src/keyed_message/mod.rs +++ b/src/keyed_message/mod.rs @@ -5,7 +5,7 @@ mod read; mod write; use eccodes_sys::codes_handle; -use std::ptr::null_mut; +use std::{fmt::Debug, marker::PhantomData, ptr::null_mut}; use tracing::{Level, event, instrument}; use crate::{ @@ -47,7 +47,11 @@ use crate::{ /// Destructor for this structure does not panic, but some internal functions may rarely fail /// leading to bugs. Errors encountered in desctructor the are logged with [`log`]. #[derive(Hash, Debug)] -pub struct KeyedMessage { +pub struct KeyedMessage<'ch> { + /// This is a little unintuitive, but we use `()` here to not unnecessarily pollute + /// KeyedMessage and derived types with generics, because `PhantomData` is needed + /// only for lifetime restriction and we tightly control how `KeyedMessage` is created. + pub(crate) parent_message: PhantomData<&'ch ()>, pub(crate) message_handle: *mut codes_handle, } @@ -176,15 +180,16 @@ pub enum DynamicKeyType { Bytes(Vec), } -impl KeyedMessage { +impl KeyedMessage<'_> { /// Custom function to clone the `KeyedMessage`. This function comes with memory overhead. /// /// # Errors /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to clone the message. - pub fn try_clone(&self) -> Result { + pub fn try_clone(&self) -> Result, CodesError> { let new_handle = unsafe { codes_handle_clone(self.message_handle)? }; - Ok(Self { + Ok(KeyedMessage { + parent_message: PhantomData, message_handle: new_handle, }) } @@ -199,7 +204,7 @@ impl KeyedMessage { } #[doc(hidden)] -impl Drop for KeyedMessage { +impl Drop for KeyedMessage<'_> { /// Executes the destructor for this type. /// This method calls destructor functions from ecCodes library. /// In some edge cases these functions can return non-zero code. @@ -233,9 +238,9 @@ impl Drop for KeyedMessage { #[cfg(test)] mod tests { - use crate::FallibleStreamingIterator; use crate::codes_handle::{CodesHandle, ProductKind}; use anyhow::{Context, Result}; + use fallible_iterator::FallibleIterator; use std::path::Path; #[test] @@ -244,7 +249,10 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let current_message = handle + .message_generator() + .next()? + .context("Message not some")?; let _ = current_message.read_key_dynamic("validityDate")?; let _ = current_message.read_key_dynamic("validityTime")?; @@ -263,7 +271,10 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let current_message = handle + .message_generator() + .next()? + .context("Message not some")?; let cloned_message = current_message.try_clone()?; assert_ne!( @@ -280,8 +291,9 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let msg = handle.next()?.context("Message not some")?.try_clone()?; - let _ = handle.next()?; + let mut mgen = handle.message_generator(); + let msg = mgen.next()?.context("Message not some")?.try_clone()?; + let _ = mgen.next()?; drop(handle); @@ -301,9 +313,10 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let _msg_ref = handle.next()?.context("Message not some")?; + let _msg_ref = handle.message_generator().next()?.context("Message not some")?; let _msg_clone = _msg_ref.try_clone()?; + drop(_msg_ref); drop(handle); drop(_msg_clone); diff --git a/src/keyed_message/read.rs b/src/keyed_message/read.rs index d46f8b3..3b6f457 100644 --- a/src/keyed_message/read.rs +++ b/src/keyed_message/read.rs @@ -9,7 +9,7 @@ use crate::{ }, }; -impl KeyRead for KeyedMessage { +impl KeyRead for KeyedMessage<'_> { fn read_key(&self, key_name: &str) -> Result { match self.get_key_native_type(key_name)? { NativeKeyType::Long => (), @@ -32,7 +32,7 @@ impl KeyRead for KeyedMessage { } } -impl KeyRead for KeyedMessage { +impl KeyRead for KeyedMessage<'_> { fn read_key(&self, key_name: &str) -> Result { match self.get_key_native_type(key_name)? { NativeKeyType::Double => (), @@ -55,7 +55,7 @@ impl KeyRead for KeyedMessage { } } -impl KeyRead for KeyedMessage { +impl KeyRead for KeyedMessage<'_> { fn read_key(&self, key_name: &str) -> Result { match self.get_key_native_type(key_name)? { NativeKeyType::Str => (), @@ -76,7 +76,7 @@ impl KeyRead for KeyedMessage { } } -impl KeyRead> for KeyedMessage { +impl KeyRead> for KeyedMessage<'_> { fn read_key(&self, key_name: &str) -> Result, CodesError> { match self.get_key_native_type(key_name)? { NativeKeyType::Long => (), @@ -97,7 +97,7 @@ impl KeyRead> for KeyedMessage { } } -impl KeyRead> for KeyedMessage { +impl KeyRead> for KeyedMessage<'_> { fn read_key(&self, key_name: &str) -> Result, CodesError> { match self.get_key_native_type(key_name)? { NativeKeyType::Double => (), @@ -118,7 +118,7 @@ impl KeyRead> for KeyedMessage { } } -impl KeyRead> for KeyedMessage { +impl KeyRead> for KeyedMessage<'_> { fn read_key(&self, key_name: &str) -> Result, CodesError> { match self.get_key_native_type(key_name)? { NativeKeyType::Bytes => (), @@ -139,7 +139,7 @@ impl KeyRead> for KeyedMessage { } } -impl KeyedMessage { +impl KeyedMessage<'_> { /// Method to get a value of given key with [`DynamicKeyType`] from the `KeyedMessage`, if it exists. /// /// In most cases you should use [`read_key()`](KeyRead::read_key) due to more predictive behaviour @@ -300,7 +300,7 @@ mod tests { use anyhow::{Context, Result}; use crate::codes_handle::{CodesHandle, ProductKind}; - use crate::{DynamicKeyType, FallibleIterator, FallibleStreamingIterator}; + use crate::{DynamicKeyType, FallibleIterator}; use std::path::Path; #[test] @@ -310,7 +310,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let current_message = handle.message_generator().next()?.context("Message not some")?; let str_key = current_message.read_key_dynamic("name")?; @@ -348,7 +348,7 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let current_message = handle.message_generator().next()?.context("Message not some")?; let mut kiter = current_message.default_keys_iterator()?; while let Some(key_name) = kiter.next()? { @@ -365,7 +365,7 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let current_message = handle.message_generator().next()?.context("Message not some")?; let mut kiter = current_message.default_keys_iterator()?; while let Some(key_name) = kiter.next()? { @@ -382,7 +382,7 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let current_message = handle.message_generator().next()?.context("Message not some")?; let missing_key = current_message.read_key_dynamic("doesNotExist"); @@ -398,7 +398,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let msg = handle.next()?.context("Message not some")?; + let msg = handle.message_generator().next()?.context("Message not some")?; let _ = msg.read_key_dynamic("dataDate")?; let _ = msg.read_key_dynamic("jDirectionIncrementInDegrees")?; diff --git a/src/keyed_message/write.rs b/src/keyed_message/write.rs index e639974..3eb118f 100644 --- a/src/keyed_message/write.rs +++ b/src/keyed_message/write.rs @@ -11,67 +11,67 @@ use crate::{ use super::KeyWrite; -impl KeyWrite for KeyedMessage { +impl KeyWrite for KeyedMessage<'_> { fn write_key(&mut self, name: &str, value: i64) -> Result<(), CodesError> { unsafe { codes_set_long(self.message_handle, name, value) } } } -impl KeyWrite for KeyedMessage { +impl KeyWrite for KeyedMessage<'_> { fn write_key(&mut self, name: &str, value: f64) -> Result<(), CodesError> { unsafe { codes_set_double(self.message_handle, name, value) } } } -impl KeyWrite<&[i64]> for KeyedMessage { +impl KeyWrite<&[i64]> for KeyedMessage<'_> { fn write_key(&mut self, name: &str, value: &[i64]) -> Result<(), CodesError> { unsafe { codes_set_long_array(self.message_handle, name, value) } } } -impl KeyWrite<&[f64]> for KeyedMessage { +impl KeyWrite<&[f64]> for KeyedMessage<'_> { fn write_key(&mut self, name: &str, value: &[f64]) -> Result<(), CodesError> { unsafe { codes_set_double_array(self.message_handle, name, value) } } } -impl KeyWrite<&[u8]> for KeyedMessage { +impl KeyWrite<&[u8]> for KeyedMessage<'_> { fn write_key(&mut self, name: &str, value: &[u8]) -> Result<(), CodesError> { unsafe { codes_set_bytes(self.message_handle, name, value) } } } -impl KeyWrite<&Vec> for KeyedMessage { +impl KeyWrite<&Vec> for KeyedMessage<'_> { fn write_key(&mut self, name: &str, value: &Vec) -> Result<(), CodesError> { unsafe { codes_set_long_array(self.message_handle, name, value) } } } -impl KeyWrite<&Vec> for KeyedMessage { +impl KeyWrite<&Vec> for KeyedMessage<'_> { fn write_key(&mut self, name: &str, value: &Vec) -> Result<(), CodesError> { unsafe { codes_set_double_array(self.message_handle, name, value) } } } -impl KeyWrite<&Vec> for KeyedMessage { +impl KeyWrite<&Vec> for KeyedMessage<'_> { fn write_key(&mut self, name: &str, value: &Vec) -> Result<(), CodesError> { unsafe { codes_set_bytes(self.message_handle, name, value) } } } -impl KeyWrite<&str> for KeyedMessage { +impl KeyWrite<&str> for KeyedMessage<'_> { fn write_key(&mut self, name: &str, value: &str) -> Result<(), CodesError> { unsafe { codes_set_string(self.message_handle, name, value) } } } -impl KeyWrite<&String> for KeyedMessage { +impl KeyWrite<&String> for KeyedMessage<'_> { fn write_key(&mut self, name: &str, value: &String) -> Result<(), CodesError> { unsafe { codes_set_string(self.message_handle, name, value) } } } -impl KeyedMessage { +impl KeyedMessage<'_> { /// Function to write given `KeyedMessage` to a file at provided path. /// If file does not exists it will be created. /// If `append` is set to `true` file will be opened in append mode @@ -132,9 +132,10 @@ impl KeyedMessage { #[cfg(test)] mod tests { use anyhow::{Context, Result}; + use fallible_iterator::FallibleIterator; use crate::{ - DynamicKeyType, FallibleStreamingIterator, KeyWrite, + DynamicKeyType, KeyWrite, codes_handle::{CodesHandle, ProductKind}, }; use std::{fs::remove_file, path::Path}; @@ -146,7 +147,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let current_message = handle.message_generator().next()?.context("Message not some")?; let out_path = Path::new("./data/iceland_write.grib"); current_message.write_to_file(out_path, false)?; @@ -161,7 +162,11 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?.try_clone()?; + let current_message = handle + .message_generator() + .next()? + .context("Message not some")? + .try_clone()?; drop(handle); @@ -180,12 +185,12 @@ mod tests { let file_path = Path::new("./data/iceland-surface.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let current_message = handle.message_generator().next()?.context("Message not some")?; current_message.write_to_file(out_path, false)?; let file_path = Path::new("./data/iceland-levels.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let current_message = handle.message_generator().next()?.context("Message not some")?; current_message.write_to_file(out_path, true)?; remove_file(out_path)?; @@ -199,7 +204,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let mut current_message = handle.next()?.context("Message not some")?.try_clone()?; + let mut current_message = handle.message_generator().next()?.context("Message not some")?.try_clone()?; let old_key = current_message.read_key_dynamic("centre")?; @@ -219,7 +224,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let mut current_message = handle.next()?.context("Message not some")?.try_clone()?; + let mut current_message = handle.message_generator().next()?.context("Message not some")?.try_clone()?; let old_key = current_message.read_key_dynamic("centre")?; @@ -230,7 +235,7 @@ mod tests { let file_path = Path::new("./data/iceland_edit.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let current_message = handle.message_generator().next()?.context("Message not some")?; let read_key = current_message.read_key_dynamic("centre")?; diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index 1b3b350..0818cc5 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -57,7 +57,8 @@ use crate::{ #[allow(clippy::module_name_repetitions)] #[derive(Debug)] pub struct KeysIterator<'a> { - parent_message: PhantomData<&'a KeyedMessage>, + /// Same trick as in `KeyedMessage` + parent_message: PhantomData<&'a ()>, iterator_handle: *mut codes_keys_iterator, next_item_exists: bool, } @@ -87,7 +88,7 @@ pub enum KeysIteratorFlags { SkipEditionSpecific = eccodes_sys::CODES_KEYS_ITERATOR_SKIP_EDITION_SPECIFIC as isize, } -impl KeyedMessage { +impl KeyedMessage<'_> { /// Creates new [`KeysIterator`] for the message with specified flags and namespace. /// /// The flags are set by providing any combination of [`KeysIteratorFlags`] @@ -220,8 +221,8 @@ impl Drop for KeysIterator<'_> { mod tests { use anyhow::{Context, Result}; + use crate::FallibleIterator; use crate::codes_handle::{CodesHandle, ProductKind}; - use crate::{FallibleIterator, FallibleStreamingIterator}; use std::path::Path; use super::KeysIteratorFlags; @@ -232,7 +233,10 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let current_message = handle + .message_generator() + .next()? + .context("Message not some")?; let flags = [ KeysIteratorFlags::AllKeys, //0 @@ -257,7 +261,10 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let current_message = handle + .message_generator() + .next()? + .context("Message not some")?; let flags = vec![ KeysIteratorFlags::AllKeys, //0 @@ -280,13 +287,13 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.next()?.context("Message not some")?; + let current_message = handle + .message_generator() + .next()? + .context("Message not some")?; let _kiter = current_message.default_keys_iterator()?; - drop(_kiter); - drop(handle); - Ok(()) } } diff --git a/src/lib.rs b/src/lib.rs index 65d4f16..9e731f7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -230,6 +230,5 @@ pub use codes_index::CodesIndex; pub use codes_nearest::{CodesNearest, NearestGridpoint}; pub use errors::CodesError; pub use fallible_iterator::{FallibleIterator, IntoFallibleIterator}; -pub use fallible_streaming_iterator::FallibleStreamingIterator; pub use keyed_message::{DynamicKeyType, KeyRead, KeyWrite, KeyedMessage}; pub use keys_iterator::{KeysIterator, KeysIteratorFlags}; diff --git a/src/message_ndarray.rs b/src/message_ndarray.rs index d68889d..0fee85f 100644 --- a/src/message_ndarray.rs +++ b/src/message_ndarray.rs @@ -18,7 +18,7 @@ pub struct RustyCodesMessage { pub values: Array2, } -impl KeyedMessage { +impl KeyedMessage<'_> { /// Converts the message to a 2D ndarray. /// /// Returns ndarray where first dimension represents y coordinates and second dimension represents x coordinates, @@ -140,11 +140,11 @@ impl KeyedMessage { #[cfg(test)] mod tests { + use fallible_iterator::FallibleIterator; use float_cmp::assert_approx_eq; use super::*; use crate::DynamicKeyType; - use crate::FallibleStreamingIterator; use crate::ProductKind; use crate::codes_handle::CodesHandle; use std::path::Path; @@ -154,7 +154,7 @@ mod tests { let file_path = Path::new("./data/iceland-surface.grib"); let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; - while let Some(msg) = handle.next()? { + while let Some(msg) = handle.message_generator().next()? { if msg.read_key_dynamic("shortName")? == DynamicKeyType::Str("2d".to_string()) { let ndarray = msg.to_ndarray()?; @@ -180,7 +180,7 @@ mod tests { let file_path = Path::new("./data/iceland-surface.grib"); let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; - while let Some(msg) = handle.next()? { + while let Some(msg) = handle.message_generator().next()? { if msg.read_key_dynamic("shortName")? == DynamicKeyType::Str("2d".to_string()) { let rmsg = msg.to_lons_lats_values()?; diff --git a/tests/handle.rs b/tests/handle.rs index 2fed63c..2101c1b 100644 --- a/tests/handle.rs +++ b/tests/handle.rs @@ -1,7 +1,7 @@ use std::{path::Path, thread}; use anyhow::{Context, Result}; -use eccodes::{CodesHandle, DynamicKeyType, FallibleStreamingIterator, ProductKind}; +use eccodes::{CodesHandle, DynamicKeyType, FallibleIterator, ProductKind}; #[test] fn thread_safety() { @@ -15,7 +15,10 @@ fn thread_safety_core() -> Result<()> { let file_path = Path::new("./data/iceland.grib"); let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; - let current_message = handle.next()?.context("Message not some")?; + let current_message = handle + .message_generator() + .next()? + .context("Message not some")?; for _ in 0..100 { let _ = current_message.read_key_dynamic("name")?; @@ -27,8 +30,6 @@ fn thread_safety_core() -> Result<()> { _ => panic!("Incorrect variant of string key"), } } - - drop(handle); } }); @@ -36,7 +37,10 @@ fn thread_safety_core() -> Result<()> { let file_path = Path::new("./data/iceland.grib"); let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; - let current_message = handle.next()?.context("Message not some")?; + let current_message = handle + .message_generator() + .next()? + .context("Message not some")?; let long_key = current_message.read_key_dynamic("numberOfPointsAlongAParallel")?; @@ -44,8 +48,6 @@ fn thread_safety_core() -> Result<()> { DynamicKeyType::Int(_) => {} _ => panic!("Incorrect variant of long key"), } - - drop(handle); } Ok(()) From 9061ce3292fae82e94b34bf4c38983a5c5705460 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sat, 20 Sep 2025 19:03:03 +0200 Subject: [PATCH 10/65] start implementing atomic interface --- src/codes_handle/atomic_iterator.rs | 95 +++++++++++++++++++++++++++++ src/codes_handle/iterator.rs | 33 +++++++++- src/codes_handle/mod.rs | 81 ++++-------------------- 3 files changed, 138 insertions(+), 71 deletions(-) create mode 100644 src/codes_handle/atomic_iterator.rs diff --git a/src/codes_handle/atomic_iterator.rs b/src/codes_handle/atomic_iterator.rs new file mode 100644 index 0000000..27c26d3 --- /dev/null +++ b/src/codes_handle/atomic_iterator.rs @@ -0,0 +1,95 @@ +#![allow(unused)] + +use std::sync::Arc; + +use eccodes_sys::codes_handle; +use fallible_iterator::FallibleIterator; + +use crate::{CodesError, CodesHandle, codes_handle::ThreadSafeHandle}; + +#[derive(Debug)] +pub struct AtomicMessageGenerator { + codes_handle: Arc>, +} +impl CodesHandle { + pub fn atomic_message_generator(self) -> AtomicMessageGenerator { + AtomicMessageGenerator { + codes_handle: Arc::new(self), + } + } +} + +impl FallibleIterator for AtomicMessageGenerator { + type Item = AtomicMessage; + + type Error = CodesError; + + fn next(&mut self) -> Result, Self::Error> { + let new_eccodes_handle = self.codes_handle.source.gen_codes_handle()?; + + if new_eccodes_handle.is_null() { + Ok(None) + } else { + Ok(Some(AtomicMessage { + _parent: self.codes_handle.clone(), + pointer: new_eccodes_handle, + })) + } + } +} + +#[derive(Debug)] +pub struct AtomicMessage { + _parent: Arc>, + pointer: *mut codes_handle, +} +unsafe impl Send for AtomicMessage {} +unsafe impl Sync for AtomicMessage {} + +#[cfg(test)] +mod tests { + use std::{ + path::Path, + sync::{Arc, Barrier}, + }; + + use anyhow::{Context, Result}; + use fallible_iterator::FallibleIterator; + + use crate::{CodesHandle, ProductKind}; + + #[test] + fn atomic_thread_safety() -> Result<()> { + let file_path = Path::new("./data/iceland-levels.grib"); + let product_kind = ProductKind::GRIB; + + let handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut mgen = handle.atomic_message_generator(); + // let _ = handle.atomic_message_generator(); <- not allowed due to ownership + + let barrier = Arc::new(Barrier::new(10)); + + let mut v = vec![]; + + for _ in 0..10 { + let msg = Arc::new(mgen.next()?.context("No more messages")?); + let b = barrier.clone(); + + let t = std::thread::spawn(move || { + for _ in 0..1000 { + b.wait(); + let _ = unsafe { + crate::intermediate_bindings::codes_get_size(msg.pointer, "shortName") + .unwrap() + }; + } + }); + + v.push(t); + } + + v.into_iter().for_each(|th| th.join().unwrap()); + + Ok(()) + } +} diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index a27f48a..b185c74 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -79,6 +79,34 @@ mod tests { Ok(()) } + #[test] + fn message_lifetime_safety() -> Result<()> { + let file_path = Path::new("./data/iceland-levels.grib"); + let product_kind = ProductKind::GRIB; + let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut mgen = handle.message_generator(); + + let msg1 = mgen.next()?.context("Message not some")?; + drop(msg1); + let msg2 = mgen.next()?.context("Message not some")?; + let msg3 = mgen.next()?.context("Message not some")?; + drop(msg3); + let msg4 = mgen.next()?.context("Message not some")?; + let msg5 = mgen.next()?.context("Message not some")?; + drop(msg5); + + drop(mgen); + // drop(handle); <- this is not allowed + + let key2 = msg2.read_key_dynamic("typeOfLevel")?; + let key4 = msg4.read_key_dynamic("typeOfLevel")?; + + assert_eq!(key2, DynamicKeyType::Str("isobaricInhPa".to_string())); + assert_eq!(key4, DynamicKeyType::Str("isobaricInhPa".to_string())); + + Ok(()) + } + #[test] fn iterator_fn() -> Result<()> { let file_path = Path::new("./data/iceland-surface.grib"); @@ -127,7 +155,10 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.message_generator().next()?.context("Message not some")?; + let current_message = handle + .message_generator() + .next()? + .context("Message not some")?; assert!(!current_message.message_handle.is_null()); diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index 1b8826e..c39ce22 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -15,6 +15,7 @@ use std::{ }; use tracing::instrument; +mod atomic_iterator; mod iterator; /// This is an internal structure used to access provided file by `CodesHandle`. @@ -29,6 +30,13 @@ pub struct CodesFile { _data: D, } +/// Marker trait to differentiate between `CodesHandle` created from index and file/buffer. +#[doc(hidden)] +pub trait ThreadSafeHandle: HandleGenerator {} + +impl ThreadSafeHandle for CodesFile> {} +impl ThreadSafeHandle for CodesFile {} + /// Internal trait implemented for types that can be called to generate `*mut codes_handle`. #[doc(hidden)] pub trait HandleGenerator: Debug { @@ -321,18 +329,13 @@ fn open_with_fmemopen(file_data: &[u8]) -> Result<*mut FILE, CodesError> { #[cfg(test)] mod tests { - use crate::codes_handle::{CodesFile, CodesHandle, ProductKind}; + use crate::codes_handle::{CodesHandle, ProductKind}; #[cfg(feature = "experimental_index")] use crate::codes_index::{CodesIndex, Select}; use anyhow::{Context, Result}; - use eccodes_sys::{ProductKind_PRODUCT_GRIB, grib_handle}; + use eccodes_sys::ProductKind_PRODUCT_GRIB; use fallible_iterator::FallibleIterator; - use std::{ - fs::File, - io::Read, - path::Path, - sync::{Arc, Barrier}, - }; + use std::{fs::File, io::Read, path::Path}; #[test] fn file_constructor() -> Result<()> { @@ -351,68 +354,6 @@ mod tests { Ok(()) } - #[derive(Debug)] - struct AtomicContainer { - _parent: Arc>>, - pointer: *mut grib_handle, - } - unsafe impl Send for AtomicContainer {} - unsafe impl Sync for AtomicContainer {} - - impl AtomicContainer { - fn new(_parent: Arc>>, pointer: *mut grib_handle) -> Self { - Self { - _parent, - // Use the mutable methods to avoid the performance hit of Mutex - pointer, - } - } - } - - #[test] - #[ignore] - fn handle_thread_safety() -> Result<()> { - let file_path = Path::new("./data/iceland-levels.grib"); - let product_kind = ProductKind::GRIB; - - let handle = CodesHandle::new_from_file(file_path, product_kind)?; - - let handle = Arc::new(handle); - let barrier = Arc::new(Barrier::new(10)); - - let mut v = vec![]; - - for _ in 0..10 { - let fhndl = handle.clone(); - let b = barrier.clone(); - - let msg_hndl = unsafe { - crate::intermediate_bindings::codes_handle_new_from_file( - fhndl.source.pointer, - handle.source.product_kind, - ) - .unwrap() - }; - let cntri = Arc::new(AtomicContainer::new(fhndl, msg_hndl)); - - let t = std::thread::spawn(move || { - for _ in 0..1000 { - b.wait(); - let _ = unsafe { - crate::intermediate_bindings::codes_get_size(cntri.pointer, "shortName") - .unwrap() - }; - } - }); - - v.push(t); - } - - v.into_iter().for_each(|th| th.join().unwrap()); - - Ok(()) - } - #[test] fn memory_constructor() -> Result<()> { let product_kind = ProductKind::GRIB; From 8e06902a19fe2e2b11ccfdda08fbd18fda0e7c53 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sat, 20 Sep 2025 19:48:22 +0200 Subject: [PATCH 11/65] read implement in progress --- src/atomic_message/mod.rs | 24 +++++ src/atomic_message/read.rs | 110 +++++++++++++++++++++++ src/codes_handle/atomic_iterator.rs | 19 ++-- src/codes_handle/iterator.rs | 2 +- src/keyed_message/read.rs | 4 +- src/keyed_message/write.rs | 2 +- src/lib.rs | 4 +- src/message_ndarray.rs | 130 +++++++++++++++++++++++++++- tests/handle.rs | 2 +- 9 files changed, 275 insertions(+), 22 deletions(-) create mode 100644 src/atomic_message/mod.rs create mode 100644 src/atomic_message/read.rs diff --git a/src/atomic_message/mod.rs b/src/atomic_message/mod.rs new file mode 100644 index 0000000..dfd97ae --- /dev/null +++ b/src/atomic_message/mod.rs @@ -0,0 +1,24 @@ +mod read; + +use std::sync::Arc; + +use eccodes_sys::codes_handle; + +use crate::{CodesHandle, codes_handle::ThreadSafeHandle}; + +pub use read::{KeyRead, ArrayKeyRead, ScalarKeyRead}; + +/// Because standard `KeyedMessage` is not Copy or Clone it can provide access methods without +/// requiring `&mut self`. As `AtomicMessage` implements `Send + Sync` this exclusive method access is not +/// guaranteed with just `&self`. `AtomicMessage` also implements a minimal subset of functionalities +/// to limit the risk of some internal ecCodes functions not being thread-safe. +/// +/// Right now `AtomicMessage` is also not clonable +#[derive(Debug)] +pub struct AtomicMessage { + pub(crate) _parent: Arc>, + pub(crate) message_handle: *mut codes_handle, +} + +unsafe impl Send for AtomicMessage {} +unsafe impl Sync for AtomicMessage {} diff --git a/src/atomic_message/read.rs b/src/atomic_message/read.rs new file mode 100644 index 0000000..4948c54 --- /dev/null +++ b/src/atomic_message/read.rs @@ -0,0 +1,110 @@ +use std::cmp::Ordering; + +use crate::{ + CodesError, + atomic_message::AtomicMessage, + codes_handle::ThreadSafeHandle, + intermediate_bindings::{ + NativeKeyType, codes_get_bytes, codes_get_double, codes_get_double_array, codes_get_long, + codes_get_long_array, codes_get_native_type, codes_get_size, codes_get_string, + }, +}; + +#[doc(hidden)] +pub trait KeyReadHelpers { + fn get_key_size(&mut self, key_name: &str) -> Result; + fn get_key_native_type(&mut self, key_name: &str) -> Result; +} +pub trait KeyRead: KeyReadHelpers { + fn read_key_unchecked(&mut self, name: &str) -> Result; +} + +pub trait ArrayKeyRead: KeyRead { + fn read_key(&mut self, key_name: &str) -> Result { + match self.get_key_native_type(key_name)? { + NativeKeyType::Bytes => (), + _ => return Err(CodesError::WrongRequestedKeyType), + } + + let key_size = self.get_key_size(key_name)?; + + if key_size < 1 { + return Err(CodesError::IncorrectKeySize); + } + + self.read_key_unchecked(key_name) + } +} + +pub trait ScalarKeyRead: KeyRead { + fn read_key(&mut self, key_name: &str) -> Result { + match self.get_key_native_type(key_name)? { + NativeKeyType::Long => (), + _ => return Err(CodesError::WrongRequestedKeyType), + } + + let key_size = self.get_key_size(key_name)?; + + match key_size.cmp(&1) { + Ordering::Greater => return Err(CodesError::WrongRequestedKeySize), + Ordering::Less => return Err(CodesError::IncorrectKeySize), + Ordering::Equal => (), + } + + self.read_key_unchecked(key_name) + } +} + +impl KeyReadHelpers for AtomicMessage { + fn get_key_size(&mut self, key_name: &str) -> Result { + unsafe { codes_get_size(self.message_handle, key_name) } + } + + fn get_key_native_type(&mut self, key_name: &str) -> Result { + unsafe { codes_get_native_type(self.message_handle, key_name) } + } +} + +impl KeyRead for AtomicMessage { + fn read_key_unchecked(&mut self, key_name: &str) -> Result { + unsafe { codes_get_long(self.message_handle, key_name) } + } +} + +impl KeyRead for AtomicMessage { + fn read_key_unchecked(&mut self, key_name: &str) -> Result { + unsafe { codes_get_double(self.message_handle, key_name) } + } +} + +impl KeyRead for AtomicMessage { + fn read_key_unchecked(&mut self, key_name: &str) -> Result { + unsafe { codes_get_string(self.message_handle, key_name) } + } +} + +impl KeyRead> for AtomicMessage { + fn read_key_unchecked(&mut self, key_name: &str) -> Result, CodesError> { + unsafe { codes_get_long_array(self.message_handle, key_name) } + } +} + +impl KeyRead> for AtomicMessage { + fn read_key_unchecked(&mut self, key_name: &str) -> Result, CodesError> { + unsafe { codes_get_double_array(self.message_handle, key_name) } + } +} + +impl KeyRead> for AtomicMessage { + fn read_key_unchecked(&mut self, key_name: &str) -> Result, CodesError> { + unsafe { codes_get_bytes(self.message_handle, key_name) } + } +} + +impl ScalarKeyRead for AtomicMessage {} +impl ScalarKeyRead for AtomicMessage {} + +impl ArrayKeyRead for AtomicMessage {} +impl ArrayKeyRead> for AtomicMessage {} +impl ArrayKeyRead> for AtomicMessage {} +impl ArrayKeyRead> for AtomicMessage {} diff --git a/src/codes_handle/atomic_iterator.rs b/src/codes_handle/atomic_iterator.rs index 27c26d3..5e2cf13 100644 --- a/src/codes_handle/atomic_iterator.rs +++ b/src/codes_handle/atomic_iterator.rs @@ -1,11 +1,10 @@ -#![allow(unused)] - use std::sync::Arc; -use eccodes_sys::codes_handle; use fallible_iterator::FallibleIterator; -use crate::{CodesError, CodesHandle, codes_handle::ThreadSafeHandle}; +use crate::{ + CodesError, CodesHandle, atomic_message::AtomicMessage, codes_handle::ThreadSafeHandle, +}; #[derive(Debug)] pub struct AtomicMessageGenerator { @@ -32,20 +31,12 @@ impl FallibleIterator for AtomicMessageGenerator { } else { Ok(Some(AtomicMessage { _parent: self.codes_handle.clone(), - pointer: new_eccodes_handle, + message_handle: new_eccodes_handle, })) } } } -#[derive(Debug)] -pub struct AtomicMessage { - _parent: Arc>, - pointer: *mut codes_handle, -} -unsafe impl Send for AtomicMessage {} -unsafe impl Sync for AtomicMessage {} - #[cfg(test)] mod tests { use std::{ @@ -79,7 +70,7 @@ mod tests { for _ in 0..1000 { b.wait(); let _ = unsafe { - crate::intermediate_bindings::codes_get_size(msg.pointer, "shortName") + crate::intermediate_bindings::codes_get_size(msg.message_handle, "shortName") .unwrap() }; } diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index b185c74..bd402eb 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -39,7 +39,7 @@ impl<'ch, S: HandleGenerator> FallibleIterator for KeyedMessageGenerator<'ch, S> #[cfg(test)] mod tests { use crate::{ - DynamicKeyType, FallibleIterator, + keyed_message::DynamicKeyType, FallibleIterator, codes_handle::{CodesHandle, ProductKind}, }; use anyhow::{Context, Ok, Result}; diff --git a/src/keyed_message/read.rs b/src/keyed_message/read.rs index 3b6f457..50b30e3 100644 --- a/src/keyed_message/read.rs +++ b/src/keyed_message/read.rs @@ -1,7 +1,7 @@ use std::cmp::Ordering; use crate::{ - DynamicKeyType, KeyRead, KeyedMessage, + keyed_message::DynamicKeyType, keyed_message::KeyRead, KeyedMessage, errors::CodesError, intermediate_bindings::{ NativeKeyType, codes_get_bytes, codes_get_double, codes_get_double_array, codes_get_long, @@ -300,7 +300,7 @@ mod tests { use anyhow::{Context, Result}; use crate::codes_handle::{CodesHandle, ProductKind}; - use crate::{DynamicKeyType, FallibleIterator}; + use crate::{keyed_message::DynamicKeyType, FallibleIterator}; use std::path::Path; #[test] diff --git a/src/keyed_message/write.rs b/src/keyed_message/write.rs index 3eb118f..fe7bec7 100644 --- a/src/keyed_message/write.rs +++ b/src/keyed_message/write.rs @@ -135,7 +135,7 @@ mod tests { use fallible_iterator::FallibleIterator; use crate::{ - DynamicKeyType, KeyWrite, + keyed_message::DynamicKeyType, keyed_message::KeyWrite, codes_handle::{CodesHandle, ProductKind}, }; use std::{fs::remove_file, path::Path}; diff --git a/src/lib.rs b/src/lib.rs index 9e731f7..5402b70 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -222,6 +222,7 @@ pub mod keys_iterator; #[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] pub mod message_ndarray; mod pointer_guard; +pub mod atomic_message; pub use codes_handle::{CodesHandle, ProductKind}; #[cfg(feature = "experimental_index")] @@ -230,5 +231,6 @@ pub use codes_index::CodesIndex; pub use codes_nearest::{CodesNearest, NearestGridpoint}; pub use errors::CodesError; pub use fallible_iterator::{FallibleIterator, IntoFallibleIterator}; -pub use keyed_message::{DynamicKeyType, KeyRead, KeyWrite, KeyedMessage}; +pub use keyed_message::{KeyedMessage}; pub use keys_iterator::{KeysIterator, KeysIteratorFlags}; +pub use atomic_message::{AtomicMessage}; diff --git a/src/message_ndarray.rs b/src/message_ndarray.rs index 0fee85f..8c645a5 100644 --- a/src/message_ndarray.rs +++ b/src/message_ndarray.rs @@ -3,7 +3,13 @@ use ndarray::{Array2, Array3, s}; -use crate::{CodesError, KeyRead, KeyedMessage, errors::MessageNdarrayError}; +use crate::{ + AtomicMessage, CodesError, KeyedMessage, + atomic_message::{ArrayKeyRead, ScalarKeyRead}, + codes_handle::ThreadSafeHandle, + errors::MessageNdarrayError, + keyed_message::KeyRead, +}; /// Struct returned by [`KeyedMessage::to_lons_lats_values()`] method. /// The arrays are collocated, meaning that `longitudes[i, j]` and `latitudes[i, j]` are the coordinates of `values[i, j]`. @@ -138,15 +144,135 @@ impl KeyedMessage<'_> { } } +impl AtomicMessage { + /// Converts the message to a 2D ndarray. + /// + /// Returns ndarray where first dimension represents y coordinates and second dimension represents x coordinates, + /// ie. `[lat, lon]`. + /// + /// Common convention for grib files on regular lon-lat grid assumes that: + /// index `[0, 0]` is the top-left corner of the grid: + /// x coordinates are increasing with the i index, + /// y coordinates are decreasing with the j index. + /// + /// This convention can be checked with `iScansNegatively` and `jScansPositively` keys - + /// if both are false, the above convention is used. + /// + /// Requires the keys `Ni`, `Nj` and `values` to be present in the message. + /// + /// Tested only with simple lat-lon grids. + /// + /// # Errors + /// + /// - When the required keys are not present or if their values are not of the expected type + /// - When the number of values mismatch with the `Ni` and `Nj` keys + #[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] + pub fn to_ndarray(&self) -> Result, CodesError> { + let ni: i64 = self.read_key("Ni")?; + let ni = usize::try_from(ni).map_err(MessageNdarrayError::from)?; + + let nj: i64 = self.read_key("Nj")?; + let nj = usize::try_from(nj).map_err(MessageNdarrayError::from)?; + + let vals: Vec = self.read_key("values")?; + if vals.len() != (ni * nj) { + return Err(MessageNdarrayError::UnexpectedValuesLength(vals.len(), ni * nj).into()); + } + + let j_scanning: i64 = self.read_key("jPointsAreConsecutive")?; + + if ![0, 1].contains(&j_scanning) { + return Err(MessageNdarrayError::UnexpectedKeyValue( + "jPointsAreConsecutive".to_owned(), + ) + .into()); + } + + let j_scanning = j_scanning != 0; + + let shape = if j_scanning { (ni, nj) } else { (nj, ni) }; + let vals = Array2::from_shape_vec(shape, vals).map_err(MessageNdarrayError::from)?; + + if j_scanning { + Ok(vals.reversed_axes()) + } else { + Ok(vals) + } + } + + /// Same as [`KeyedMessage::to_ndarray()`] but returns the longitudes and latitudes alongside values. + /// Fields are returned as separate arrays in [`RustyCodesMessage`]. + /// + /// Compared to `to_ndarray` this method has performance overhead as returned arrays may be cloned. + /// + /// This method requires the `latLonValues`, `Ni` and `Nj` keys to be present in the message. + /// + /// # Errors + /// + /// - When the required keys are not present or if their values are not of the expected type + /// - When the number of values mismatch with the `Ni` and `Nj` keys + #[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] + pub fn to_lons_lats_values(&self) -> Result { + let ni: i64 = self.read_key("Ni")?; + let ni = usize::try_from(ni).map_err(MessageNdarrayError::from)?; + + let nj: i64 = self.read_key("Nj")?; + let nj = usize::try_from(nj).map_err(MessageNdarrayError::from)?; + + let latlonvals: Vec = self.read_key("latLonValues")?; + + if latlonvals.len() != (ni * nj * 3) { + return Err( + MessageNdarrayError::UnexpectedValuesLength(latlonvals.len(), ni * nj * 3).into(), + ); + } + + let j_scanning: i64 = self.read_key("jPointsAreConsecutive")?; + + if ![0, 1].contains(&j_scanning) { + return Err(MessageNdarrayError::UnexpectedKeyValue( + "jPointsAreConsecutive".to_owned(), + ) + .into()); + } + + let j_scanning = j_scanning != 0; + + let shape = if j_scanning { + (ni, nj, 3_usize) + } else { + (nj, ni, 3_usize) + }; + + let mut latlonvals = + Array3::from_shape_vec(shape, latlonvals).map_err(MessageNdarrayError::from)?; + + if j_scanning { + latlonvals.swap_axes(0, 1); + } + + let (lats, lons, vals) = + latlonvals + .view_mut() + .multi_slice_move((s![.., .., 0], s![.., .., 1], s![.., .., 2])); + + Ok(RustyCodesMessage { + longitudes: lons.into_owned(), + latitudes: lats.into_owned(), + values: vals.into_owned(), + }) + } +} + #[cfg(test)] mod tests { use fallible_iterator::FallibleIterator; use float_cmp::assert_approx_eq; use super::*; - use crate::DynamicKeyType; use crate::ProductKind; use crate::codes_handle::CodesHandle; + use crate::keyed_message::DynamicKeyType; use std::path::Path; #[test] diff --git a/tests/handle.rs b/tests/handle.rs index 2101c1b..dffdbda 100644 --- a/tests/handle.rs +++ b/tests/handle.rs @@ -1,7 +1,7 @@ use std::{path::Path, thread}; use anyhow::{Context, Result}; -use eccodes::{CodesHandle, DynamicKeyType, FallibleIterator, ProductKind}; +use eccodes::{CodesHandle, keyed_message::DynamicKeyType, FallibleIterator, ProductKind}; #[test] fn thread_safety() { From 6477acac8729ead5c0b3f7145c52844911d167c0 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 21 Sep 2025 10:06:48 +0200 Subject: [PATCH 12/65] implement key_read with macros --- src/atomic_message/mod.rs | 2 +- src/atomic_message/read.rs | 108 ++++++++++++++----------------------- 2 files changed, 42 insertions(+), 68 deletions(-) diff --git a/src/atomic_message/mod.rs b/src/atomic_message/mod.rs index dfd97ae..659c1e0 100644 --- a/src/atomic_message/mod.rs +++ b/src/atomic_message/mod.rs @@ -6,7 +6,7 @@ use eccodes_sys::codes_handle; use crate::{CodesHandle, codes_handle::ThreadSafeHandle}; -pub use read::{KeyRead, ArrayKeyRead, ScalarKeyRead}; +pub use read::{KeyRead}; /// Because standard `KeyedMessage` is not Copy or Clone it can provide access methods without /// requiring `&mut self`. As `AtomicMessage` implements `Send + Sync` this exclusive method access is not diff --git a/src/atomic_message/read.rs b/src/atomic_message/read.rs index 4948c54..a94767d 100644 --- a/src/atomic_message/read.rs +++ b/src/atomic_message/read.rs @@ -15,44 +15,50 @@ pub trait KeyReadHelpers { fn get_key_size(&mut self, key_name: &str) -> Result; fn get_key_native_type(&mut self, key_name: &str) -> Result; } + pub trait KeyRead: KeyReadHelpers { - fn read_key_unchecked(&mut self, name: &str) -> Result; + fn read_key(&mut self, key_name: &str) -> Result; + fn read_key_unchecked(&mut self, key_name: &str) -> Result; } -pub trait ArrayKeyRead: KeyRead { - fn read_key(&mut self, key_name: &str) -> Result { - match self.get_key_native_type(key_name)? { - NativeKeyType::Bytes => (), - _ => return Err(CodesError::WrongRequestedKeyType), - } +macro_rules! impl_key_read { + ($key_sizing:ident, $ec_func:ident, $key_variant:path, $gen_type:ty) => { + impl KeyRead<$gen_type> for AtomicMessage { + fn read_key_unchecked(&mut self, key_name: &str) -> Result<$gen_type, CodesError> { + unsafe { $ec_func(self.message_handle, key_name) } + } - let key_size = self.get_key_size(key_name)?; + fn read_key(&mut self, key_name: &str) -> Result<$gen_type, CodesError> { + match self.get_key_native_type(key_name)? { + $key_variant => (), + _ => return Err(CodesError::WrongRequestedKeyType), + } - if key_size < 1 { - return Err(CodesError::IncorrectKeySize); - } + let key_size = self.get_key_size(key_name)?; - self.read_key_unchecked(key_name) - } -} + key_size_check!($key_sizing, key_size); -pub trait ScalarKeyRead: KeyRead { - fn read_key(&mut self, key_name: &str) -> Result { - match self.get_key_native_type(key_name)? { - NativeKeyType::Long => (), - _ => return Err(CodesError::WrongRequestedKeyType), + self.read_key_unchecked(key_name) + } } + }; +} - let key_size = self.get_key_size(key_name)?; - - match key_size.cmp(&1) { +macro_rules! key_size_check { + // size_var is needed because of macro hygiene + (scalar, $size_var:ident) => { + match $size_var.cmp(&1) { Ordering::Greater => return Err(CodesError::WrongRequestedKeySize), Ordering::Less => return Err(CodesError::IncorrectKeySize), Ordering::Equal => (), } + }; - self.read_key_unchecked(key_name) - } + (array, $size_var:ident) => { + if $size_var < 1 { + return Err(CodesError::IncorrectKeySize); + } + }; } impl KeyReadHelpers for AtomicMessage { @@ -65,46 +71,14 @@ impl KeyReadHelpers for AtomicMessage { } } -impl KeyRead for AtomicMessage { - fn read_key_unchecked(&mut self, key_name: &str) -> Result { - unsafe { codes_get_long(self.message_handle, key_name) } - } -} - -impl KeyRead for AtomicMessage { - fn read_key_unchecked(&mut self, key_name: &str) -> Result { - unsafe { codes_get_double(self.message_handle, key_name) } - } -} - -impl KeyRead for AtomicMessage { - fn read_key_unchecked(&mut self, key_name: &str) -> Result { - unsafe { codes_get_string(self.message_handle, key_name) } - } -} - -impl KeyRead> for AtomicMessage { - fn read_key_unchecked(&mut self, key_name: &str) -> Result, CodesError> { - unsafe { codes_get_long_array(self.message_handle, key_name) } - } -} - -impl KeyRead> for AtomicMessage { - fn read_key_unchecked(&mut self, key_name: &str) -> Result, CodesError> { - unsafe { codes_get_double_array(self.message_handle, key_name) } - } -} - -impl KeyRead> for AtomicMessage { - fn read_key_unchecked(&mut self, key_name: &str) -> Result, CodesError> { - unsafe { codes_get_bytes(self.message_handle, key_name) } - } -} - -impl ScalarKeyRead for AtomicMessage {} -impl ScalarKeyRead for AtomicMessage {} - -impl ArrayKeyRead for AtomicMessage {} -impl ArrayKeyRead> for AtomicMessage {} -impl ArrayKeyRead> for AtomicMessage {} -impl ArrayKeyRead> for AtomicMessage {} +impl_key_read!(scalar, codes_get_long, NativeKeyType::Long, i64); +impl_key_read!(scalar, codes_get_double, NativeKeyType::Double, f64); +impl_key_read!(array, codes_get_string, NativeKeyType::Str, String); +impl_key_read!(array, codes_get_bytes, NativeKeyType::Bytes, Vec); +impl_key_read!(array, codes_get_long_array, NativeKeyType::Long, Vec); +impl_key_read!( + array, + codes_get_double_array, + NativeKeyType::Double, + Vec +); From 8d7789e0baca509f754d8d3e247d8f6b35f6cbde Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 21 Sep 2025 10:13:55 +0200 Subject: [PATCH 13/65] implement ndarray for atomic message --- src/atomic_message/mod.rs | 2 +- src/atomic_message/read.rs | 4 ++-- src/message_ndarray.rs | 11 ++++------- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/atomic_message/mod.rs b/src/atomic_message/mod.rs index 659c1e0..e015caf 100644 --- a/src/atomic_message/mod.rs +++ b/src/atomic_message/mod.rs @@ -6,7 +6,7 @@ use eccodes_sys::codes_handle; use crate::{CodesHandle, codes_handle::ThreadSafeHandle}; -pub use read::{KeyRead}; +pub use read::{AtomicKeyRead}; /// Because standard `KeyedMessage` is not Copy or Clone it can provide access methods without /// requiring `&mut self`. As `AtomicMessage` implements `Send + Sync` this exclusive method access is not diff --git a/src/atomic_message/read.rs b/src/atomic_message/read.rs index a94767d..52b6c42 100644 --- a/src/atomic_message/read.rs +++ b/src/atomic_message/read.rs @@ -16,14 +16,14 @@ pub trait KeyReadHelpers { fn get_key_native_type(&mut self, key_name: &str) -> Result; } -pub trait KeyRead: KeyReadHelpers { +pub trait AtomicKeyRead: KeyReadHelpers { fn read_key(&mut self, key_name: &str) -> Result; fn read_key_unchecked(&mut self, key_name: &str) -> Result; } macro_rules! impl_key_read { ($key_sizing:ident, $ec_func:ident, $key_variant:path, $gen_type:ty) => { - impl KeyRead<$gen_type> for AtomicMessage { + impl AtomicKeyRead<$gen_type> for AtomicMessage { fn read_key_unchecked(&mut self, key_name: &str) -> Result<$gen_type, CodesError> { unsafe { $ec_func(self.message_handle, key_name) } } diff --git a/src/message_ndarray.rs b/src/message_ndarray.rs index 8c645a5..342a0cd 100644 --- a/src/message_ndarray.rs +++ b/src/message_ndarray.rs @@ -4,11 +4,8 @@ use ndarray::{Array2, Array3, s}; use crate::{ - AtomicMessage, CodesError, KeyedMessage, - atomic_message::{ArrayKeyRead, ScalarKeyRead}, - codes_handle::ThreadSafeHandle, - errors::MessageNdarrayError, - keyed_message::KeyRead, + AtomicMessage, CodesError, KeyedMessage, atomic_message::AtomicKeyRead, + codes_handle::ThreadSafeHandle, errors::MessageNdarrayError, keyed_message::KeyRead, }; /// Struct returned by [`KeyedMessage::to_lons_lats_values()`] method. @@ -167,7 +164,7 @@ impl AtomicMessage { /// - When the required keys are not present or if their values are not of the expected type /// - When the number of values mismatch with the `Ni` and `Nj` keys #[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] - pub fn to_ndarray(&self) -> Result, CodesError> { + pub fn to_ndarray(&mut self) -> Result, CodesError> { let ni: i64 = self.read_key("Ni")?; let ni = usize::try_from(ni).map_err(MessageNdarrayError::from)?; @@ -212,7 +209,7 @@ impl AtomicMessage { /// - When the required keys are not present or if their values are not of the expected type /// - When the number of values mismatch with the `Ni` and `Nj` keys #[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] - pub fn to_lons_lats_values(&self) -> Result { + pub fn to_lons_lats_values(&mut self) -> Result { let ni: i64 = self.read_key("Ni")?; let ni = usize::try_from(ni).map_err(MessageNdarrayError::from)?; From ac4f2e6e315c826a3e6722da9eb06c6beed75313 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 21 Sep 2025 12:15:10 +0200 Subject: [PATCH 14/65] update eccodes-sys, some fixes to codes_index and start doing docs --- Cargo.toml | 6 +- src/codes_handle/mod.rs | 2 + src/codes_index.rs | 32 ++++++++- src/intermediate_bindings/codes_index.rs | 1 + src/keyed_message/mod.rs | 6 +- src/keyed_message/write.rs | 4 +- src/lib.rs | 82 +++++++++++++++--------- 7 files changed, 96 insertions(+), 37 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0925d2f..f091062 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,11 @@ name = "eccodes" description = "Unofficial high-level Rust bindings of the latest ecCodes release" repository = "https://github.com/ScaleWeather/eccodes" +<<<<<<< HEAD version = "0.13.4" +======= +version = "0.14.0" +>>>>>>> 205d4a9 (update eccodes-sys, some fixes to codes_index and start doing docs) readme = "README.md" authors = ["Jakub Lewandowski "] keywords = ["eccodes", "grib", "bufr", "meteorology", "weather"] @@ -19,7 +23,7 @@ exclude = [".github/*", ".vscode/*", ".idea/*", "data/*"] rust-version = "1.85.0" [dependencies] -eccodes-sys = { version = "0.6.0", default-features = false } +eccodes-sys = { version = "0.7.0", default-features = false } libc = { version = "0.2", default-features = false } thiserror = { version = "2.0", default-features = false } errno = { version = "0.3", default-features = false } diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index c39ce22..7a71333 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -15,6 +15,8 @@ use std::{ }; use tracing::instrument; +pub use iterator::KeyedMessageGenerator; + mod atomic_iterator; mod iterator; diff --git a/src/codes_index.rs b/src/codes_index.rs index 37572ed..883076a 100644 --- a/src/codes_index.rs +++ b/src/codes_index.rs @@ -25,13 +25,42 @@ //! //! The issues have been partially mitigated by implementing global mutex for `codes_index` operations. //! Please not that mutex is used only for `codes_index` functions to not affect performance of other not-problematic functions in this crate. -//! This solution, eliminated tsegfaults in tests, but occasional non-zero return codes still appear. However, this is +//! This solution, eliminated segfaults in tests, but occasional non-zero return codes still appear. However, this is //! not a guarantee and possbility of safety issues is non-zero! //! //! To avoid the memory issues altogether, do not use this feature at all. If you want to use it, take care to use `CodesIndex` in entirely //! non-concurrent environment. //! //! If you have any suggestions or ideas how to improve the safety of this feature, please open an issue or a pull request. +//! +//! ## Why functions are not marked unsafe? +//! +//! Consider this example: +//! +//! ```ignore +//! thread::spawn(|| -> Result<()> { +//! let file_path = Path::new("./data/iceland-surface.grib.idx"); +//! let mut index_op = CodesIndex::read_from_file(file_path)?; +//! +//! loop { +//! index_op = index_op +//! .select("shortName", "2t")? +//! .select("typeOfLevel", "surface")? +//! .select("level", 0)? +//! .select("stepType", "instant")?; +//! } +//! }); +//! +//! let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; +//! let grib_path = Path::new("./data/iceland-surface.grib"); +//! let index = CodesIndex::new_from_keys(&keys)?.add_grib_file(grib_path); +//! ``` +//! +//! Each of the used functions is memory-safe when used sequentially, but when used as above memory bugs may appear. +//! Safety issues arising from using `CodesIndex` also extend beyond this module. For example, in code above, if we +//! created `CodesHandle` from `iceland-surface.grib` file, operations in that handle could also result in memory bugs. +//! Thus, it is not possible to mark only some functions as `unsafe`, because use of `CodesIndex` *poisons* the whole crate. +//! use crate::{ codes_handle::HandleGenerator, @@ -44,6 +73,7 @@ use crate::{ }; use eccodes_sys::{codes_handle, codes_index}; use std::{path::Path, ptr::null_mut}; +use tracing::instrument; #[derive(Debug)] #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] diff --git a/src/intermediate_bindings/codes_index.rs b/src/intermediate_bindings/codes_index.rs index 6d5bd14..8b85cc7 100644 --- a/src/intermediate_bindings/codes_index.rs +++ b/src/intermediate_bindings/codes_index.rs @@ -4,6 +4,7 @@ use eccodes_sys::{CODES_LOCK, codes_context, codes_index}; use num_traits::FromPrimitive; use std::{ffi::CString, ptr}; +use tracing::instrument; use crate::{ errors::{CodesError, CodesInternal}, diff --git a/src/keyed_message/mod.rs b/src/keyed_message/mod.rs index 00de774..ee1b42a 100644 --- a/src/keyed_message/mod.rs +++ b/src/keyed_message/mod.rs @@ -181,7 +181,11 @@ pub enum DynamicKeyType { } impl KeyedMessage<'_> { - /// Custom function to clone the `KeyedMessage`. This function comes with memory overhead. + /// Custom function to clone the `KeyedMessage`. + /// + /// **Be careful of the memory overhead!** ecCodes (when reading from file) defers reading the data into memory + /// if possible. Simply creating `KeyedMessage` or even reading some keys will use only a little of memory. + /// This function **will** read the whole message into the memory, which can be of a significant size for big grids. /// /// # Errors /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to clone the message. diff --git a/src/keyed_message/write.rs b/src/keyed_message/write.rs index fe7bec7..89d9914 100644 --- a/src/keyed_message/write.rs +++ b/src/keyed_message/write.rs @@ -204,7 +204,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let mut current_message = handle.message_generator().next()?.context("Message not some")?.try_clone()?; + let mut current_message = handle.message_generator().next()?.context("Message not some")?; let old_key = current_message.read_key_dynamic("centre")?; @@ -224,7 +224,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let mut current_message = handle.message_generator().next()?.context("Message not some")?.try_clone()?; + let mut current_message = handle.message_generator().next()?.context("Message not some")?; let old_key = current_message.read_key_dynamic("centre")?; diff --git a/src/lib.rs b/src/lib.rs index 5402b70..0c34eaa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,9 +17,11 @@ //! //! This crate contains (mostly) safe high-level bindings for ecCodes library. //! Bindings can be considered safe mainly because all crate structures -//! will take ownership of the data in memory before passing the raw pointer to ecCodes. +//! take ownership of the data in memory before passing the raw pointer to the ecCodes. //! -//! **Currently only reading of GRIB files is supported.** +//! **Currently only operations on GRIB files are supported.** +//! +//! **Version 0.14 introduces some breaking changes, [check them below](#changes-in-version-014)!** //! //! Because of the ecCodes library API characteristics theses bindings are //! rather thick wrapper to make this crate safe and convenient to use. @@ -40,10 +42,10 @@ //! this allows the user to handle the error in the way that suits them best and not risk crashes. //! //! All error descriptions are provided in the [`errors`] module. -//! Destructors, which cannot panic, report errors through the `log` crate. +//! Destructors, which cannot panic, report errors through `tracing` and `log` crate. //! //! None of the functions in this crate explicitly panics. -//! However, users should be aware that dependencies might panic in some edge cases. +//! However, users should be aware that dependencies (eg. `ndarray`) might panic in some edge cases. //! //! ## Safety //! @@ -53,36 +55,25 @@ //! That said, neither main developer nor contributors have expertise in unsafe Rust and bugs might have //! slipped through. We are also not responsible for bugs in the ecCodes library. //! -//! If you find a bug or have a suggestion, feel free to discuss it on Github. -//! -//! ## Features -//! -//! - `message_ndarray` - enables support for converting [`KeyedMessage`] to [`ndarray::Array`]. -//! This feature is enabled by default. It is currently tested only with simple lat-lon grids. -//! -//! - `experimental_index` - enables support for creating and using index files for GRIB files. -//! **This feature is experimental** and disabled by default. If you want to use it, please read -//! the information provided in [`codes_index`] documentation. -//! -//! - `docs` - builds the crate without linking ecCodes, particularly useful when building the documentation -//! on [docs.rs](https://docs.rs/). For more details check documentation of [eccodes-sys](https://crates.io/crates/eccodes-sys). +//! **For critical applications always perform extensive testing before using this crate in production.** //! -//! To build your own crate with this crate as dependency on docs.rs without linking ecCodes add following lines to your `Cargo.toml` -//! -//! ```text -//! [package.metadata.docs.rs] -//! features = ["eccodes/docs"] -//! ``` +//! If you find a bug or have a suggestion, feel free to discuss it on Github. //! //! ## Usage //! //! To access a GRIB file you need to create [`CodesHandle`] with one of provided constructors. //! -//! GRIB files consist of messages which represent data fields at specific time and level. -//! Messages are represented by the [`KeyedMessage`] structure. -//! -//! [`CodesHandle`] implements [`FallibleStreamingIterator`](CodesHandle#impl-FallibleStreamingIterator-for-CodesHandle) -//! which allows you to iterate over messages in the file. The iterator returns `&KeyedMessage` which valid is until next iteration. +//! ecCodes represents GRIB files as a set of separate messages, each representing data fields at specific time and level. +//! Messages are represented here by the [`KeyedMessage`] structure. +//! +//! To obtain `KeyedMessage`(s) from `CodesHandle` you need to create an instance of [`KeyedMessageGenerator`](codes_handle::KeyedMessageGenerator) +//! with [`CodesHandle::message_generator()`]. This is an analogous interface to `IterMut` and `iter_mut()` in [`std::slice`]. +//! +//! `KeyedMessageGenerator` implements [`FallibleIterator`](codes_handle::KeyedMessageGenerator#impl-FallibleIterator-for-KeyedMessageGenerator%3C'ch,+S%3E) +//! which allows you to iterate over messages in the file. The iterator returns `KeyedMessage` with lifetime tied to the lifetime of `CodesHandle`, +//! that is `KeyedMessage` cannot outlive the `CodesHandle` it was generated from. If you need to prolong its lifetime, you can use +//! [`try_clone()`](KeyedMessage::try_clone), but that comes with performance and memory overhead. +//! //! `KeyedMessage` implements several methods to access the data as needed, most of those can be called directly on `&KeyedMessage`. //! You can also use [`try_clone()`](KeyedMessage::try_clone) to clone the message and prolong its lifetime. //! @@ -136,7 +127,10 @@ //! # } //! ``` //! -//! #### Example 2 - Writing GRIB files +//! #### **New in 0.14** Example 3: Concurrent read +//! +//! +//! #### Example 3 - Writing GRIB files //! //! ```rust //! // The crate provides basic support for setting `KeyedMessage` keys @@ -207,8 +201,33 @@ //! # Ok(()) //! # } //! ``` +//! + +//! +//! +//! ## Changes in version 0.14 +//! +//! +//! ## Feature Flags +//! +//! - `message_ndarray` - enables support for converting [`KeyedMessage`] to [`ndarray::Array`]. +//! This feature is enabled by default. It is currently tested only with simple lat-lon grids. //! +//! - `experimental_index` - **⚠️ This feature is experimental and might be unsafe in some contexts ⚠️** +//! This flag enables support for creating and using index files for GRIB files. +//! If you want to use it, please read the information provided in [`codes_index`] documentation. +//! +//! - `docs` - builds the crate without linking ecCodes, particularly useful when building the documentation +//! on [docs.rs](https://docs.rs/). For more details check documentation of [eccodes-sys](https://crates.io/crates/eccodes-sys). +//! +//! To build your own crate with this crate as dependency on docs.rs without linking ecCodes add following lines to your `Cargo.toml` +//! +//! ```text +//! [package.metadata.docs.rs] +//! features = ["eccodes/docs"] +//! ``` +pub mod atomic_message; pub mod codes_handle; #[cfg(feature = "experimental_index")] #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] @@ -222,8 +241,8 @@ pub mod keys_iterator; #[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] pub mod message_ndarray; mod pointer_guard; -pub mod atomic_message; +pub use atomic_message::{AtomicKeyRead, AtomicMessage}; pub use codes_handle::{CodesHandle, ProductKind}; #[cfg(feature = "experimental_index")] #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] @@ -231,6 +250,5 @@ pub use codes_index::CodesIndex; pub use codes_nearest::{CodesNearest, NearestGridpoint}; pub use errors::CodesError; pub use fallible_iterator::{FallibleIterator, IntoFallibleIterator}; -pub use keyed_message::{KeyedMessage}; +pub use keyed_message::{KeyRead, KeyWrite, KeyedMessage}; pub use keys_iterator::{KeysIterator, KeysIteratorFlags}; -pub use atomic_message::{AtomicMessage}; From 0c8051bcaad8f1d08b87b308d0251c452fcd1b58 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 21 Sep 2025 13:17:19 +0200 Subject: [PATCH 15/65] (wip) follow ndarray idea --- src/keyed_message/clone.rs | 32 +++++ src/keyed_message/mod.rs | 194 +++++---------------------- src/keyed_message/read.rs | 263 ++++++++++++++++++++++--------------- src/keyed_message/write.rs | 33 ++++- 4 files changed, 252 insertions(+), 270 deletions(-) create mode 100644 src/keyed_message/clone.rs diff --git a/src/keyed_message/clone.rs b/src/keyed_message/clone.rs new file mode 100644 index 0000000..cad7aee --- /dev/null +++ b/src/keyed_message/clone.rs @@ -0,0 +1,32 @@ +use crate::{ + AtomicMessage, CodesError, KeyedMessage, codes_handle::ThreadSafeHandle, + intermediate_bindings::codes_handle_clone, keyed_message::ClonedMessage, +}; + +pub trait TryClone { + /// Custom function to clone the `KeyedMessage` and `AtomicMessage`. + /// + /// **Be careful of the memory overhead!** ecCodes (when reading from file) defers reading the data into memory + /// if possible. Simply creating `KeyedMessage` or even reading some keys will use only a little of memory. + /// This function **will** read the whole message into the memory, which can be of a significant size for big grids. + /// + /// # Errors + /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to clone the message. + fn try_clone(&self) -> Result; +} + +impl TryClone for KeyedMessage<'_> { + fn try_clone(&self) -> Result { + Ok(ClonedMessage { + message_handle: unsafe { codes_handle_clone(self.message_handle)? }, + }) + } +} + +impl TryClone for AtomicMessage { + fn try_clone(&self) -> Result { + Ok(ClonedMessage { + message_handle: unsafe { codes_handle_clone(self.message_handle)? }, + }) + } +} diff --git a/src/keyed_message/mod.rs b/src/keyed_message/mod.rs index ee1b42a..e62a86d 100644 --- a/src/keyed_message/mod.rs +++ b/src/keyed_message/mod.rs @@ -1,19 +1,16 @@ //! Definition of `KeyedMessage` and its associated functions //! used for reading and writing data of given variable from GRIB file +mod clone; mod read; mod write; use eccodes_sys::codes_handle; -use std::{fmt::Debug, marker::PhantomData, ptr::null_mut}; +use std::{fmt::Debug, hash::Hash, marker::PhantomData, ptr::null_mut, sync::Arc}; use tracing::{Level, event, instrument}; use crate::{ - CodesError, - intermediate_bindings::{ - NativeKeyType, codes_get_native_type, codes_get_size, codes_handle_clone, - codes_handle_delete, - }, + CodesHandle, codes_handle::ThreadSafeHandle, intermediate_bindings::codes_handle_delete, }; /// Structure that provides access to the data contained in the GRIB file, which directly corresponds to the message in the GRIB file @@ -46,169 +43,43 @@ use crate::{ /// /// Destructor for this structure does not panic, but some internal functions may rarely fail /// leading to bugs. Errors encountered in desctructor the are logged with [`log`]. -#[derive(Hash, Debug)] -pub struct KeyedMessage<'ch> { - /// This is a little unintuitive, but we use `()` here to not unnecessarily pollute - /// KeyedMessage and derived types with generics, because `PhantomData` is needed - /// only for lifetime restriction and we tightly control how `KeyedMessage` is created. - pub(crate) parent_message: PhantomData<&'ch ()>, - pub(crate) message_handle: *mut codes_handle, -} +type KeyedMessage<'ch> = CodesMessage>; -/// Provides GRIB key reading capabilites. Implemented by [`KeyedMessage`] for all possible key types. -pub trait KeyRead { - /// Tries to read a key of given name from [`KeyedMessage`]. This function checks if key native type - /// matches the requested type (ie. you cannot read integer as string, or array as a number). - /// - /// # Example - /// - /// ``` - /// # use eccodes::{ProductKind, CodesHandle, KeyRead}; - /// # use std::path::Path; - /// # use anyhow::Context; - /// # use eccodes::FallibleStreamingIterator; - /// # - /// # fn main() -> anyhow::Result<()> { - /// # let file_path = Path::new("./data/iceland.grib"); - /// # let product_kind = ProductKind::GRIB; - /// # - /// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - /// let message = handle.next()?.context("no message")?; - /// let short_name: String = message.read_key("shortName")?; - /// - /// assert_eq!(short_name, "msl"); - /// # Ok(()) - /// # } - /// ``` - /// - /// # Errors - /// - /// Returns [`WrongRequestedKeySize`](CodesError::WrongRequestedKeyType) when trying to read key in non-native type (use [`unchecked`](KeyRead::read_key_unchecked) instead). - /// - /// Returns [`WrongRequestedKeySize`](CodesError::WrongRequestedKeySize) when trying to read array as integer. - /// - /// Returns [`IncorrectKeySize`](CodesError::IncorrectKeySize) when key size is 0. This can indicate corrupted data. - /// - /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to read the key. - fn read_key(&self, name: &str) -> Result; +/// Because standard `KeyedMessage` is not Copy or Clone it can provide access methods without +/// requiring `&mut self`. As `AtomicMessage` implements `Send + Sync` this exclusive method access is not +/// guaranteed with just `&self`. `AtomicMessage` also implements a minimal subset of functionalities +/// to limit the risk of some internal ecCodes functions not being thread-safe. +type AtomicMessage = CodesMessage>; - /// Skips all the checks provided by [`read_key`](KeyRead::read_key) and directly calls ecCodes, ensuring only memory and type safety. - /// - /// This function has better perfomance than [`read_key`](KeyRead::read_key) but all error handling and (possible) - /// type conversions are performed directly by ecCodes. - /// - /// This function is also useful for (not usually used) keys that return incorrect native type. - /// - /// # Example - /// - /// ``` - /// # use eccodes::{ProductKind, CodesHandle, KeyRead}; - /// # use std::path::Path; - /// # use anyhow::Context; - /// # use eccodes::FallibleStreamingIterator; - /// # - /// # fn main() -> anyhow::Result<()> { - /// # let file_path = Path::new("./data/iceland.grib"); - /// # let product_kind = ProductKind::GRIB; - /// # - /// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - /// let message = handle.next()?.context("no message")?; - /// let short_name: String = message.read_key_unchecked("shortName")?; - /// - /// assert_eq!(short_name, "msl"); - /// # Ok(()) - /// # } - /// ``` - /// - /// # Errors - /// - /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to read the key. - fn read_key_unchecked(&self, name: &str) -> Result; -} +unsafe impl Send for AtomicMessage {} +unsafe impl Sync for AtomicMessage {} -/// Provides GRIB key writing capabilites. Implemented by [`KeyedMessage`] for all possible key types. -pub trait KeyWrite { - /// Writes key with given name and value to [`KeyedMessage`] overwriting existing value, unless - /// the key is read-only. This function directly calls ecCodes ensuring only type and memory safety. - /// - /// # Example - /// - /// ``` - /// # use eccodes::{ProductKind, CodesHandle, KeyWrite}; - /// # use std::path::Path; - /// # use anyhow::Context; - /// # use eccodes::FallibleStreamingIterator; - /// # - /// # fn main() -> anyhow::Result<()> { - /// # let file_path = Path::new("./data/iceland.grib"); - /// # let product_kind = ProductKind::GRIB; - /// # - /// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - /// - /// // CodesHandle iterator returns immutable messages. - /// // To edit a message it must be cloned. - /// let mut message = handle.next()?.context("no message")?.try_clone()?; - /// message.write_key("level", 1)?; - /// # Ok(()) - /// # } - /// ``` - /// - /// # Errors - /// - /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to write the key. - fn write_key(&mut self, name: &str, value: T) -> Result<(), CodesError>; -} +type ClonedMessage = CodesMessage; -/// Enum of types GRIB key can have. -/// -/// Messages inside GRIB files can contain keys of arbitrary types, which are known only at runtime (after being checked). -/// ecCodes can return several different types of key, which are represented by this enum -/// and each variant contains the respective data type. -#[derive(Clone, Debug, PartialEq)] -pub enum DynamicKeyType { - #[allow(missing_docs)] - Float(f64), - #[allow(missing_docs)] - Int(i64), - #[allow(missing_docs)] - FloatArray(Vec), - #[allow(missing_docs)] - IntArray(Vec), - #[allow(missing_docs)] - Str(String), - #[allow(missing_docs)] - Bytes(Vec), +unsafe impl Send for ClonedMessage {} +unsafe impl Sync for ClonedMessage {} + +/// All messages use this struct for operations. +#[derive(Debug)] +struct CodesMessage { + pub(crate) _parent: P, + pub(crate) message_handle: *mut codes_handle, } -impl KeyedMessage<'_> { - /// Custom function to clone the `KeyedMessage`. - /// - /// **Be careful of the memory overhead!** ecCodes (when reading from file) defers reading the data into memory - /// if possible. Simply creating `KeyedMessage` or even reading some keys will use only a little of memory. - /// This function **will** read the whole message into the memory, which can be of a significant size for big grids. - /// - /// # Errors - /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to clone the message. - pub fn try_clone(&self) -> Result, CodesError> { - let new_handle = unsafe { codes_handle_clone(self.message_handle)? }; - - Ok(KeyedMessage { - parent_message: PhantomData, - message_handle: new_handle, - }) - } +#[derive(Debug, Hash, PartialEq, PartialOrd)] +struct OwnedRepr(); - fn get_key_size(&self, key_name: &str) -> Result { - unsafe { codes_get_size(self.message_handle, key_name) } - } +#[derive(Debug)] +struct ArcRepr(Arc>); - fn get_key_native_type(&self, key_name: &str) -> Result { - unsafe { codes_get_native_type(self.message_handle, key_name) } - } -} +/// This is a little unintuitive, but we use `()` here to not unnecessarily pollute +/// KeyedMessage and derived types with generics, because `PhantomData` is needed +/// only for lifetime restriction and we tightly control how `KeyedMessage` is created. +#[derive(Debug, Hash, PartialEq, PartialOrd)] +struct RefRepr<'ch>(PhantomData<&'ch ()>); #[doc(hidden)] -impl Drop for KeyedMessage<'_> { +impl Drop for CodesMessage

{ /// Executes the destructor for this type. /// This method calls destructor functions from ecCodes library. /// In some edge cases these functions can return non-zero code. @@ -317,7 +188,10 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let _msg_ref = handle.message_generator().next()?.context("Message not some")?; + let _msg_ref = handle + .message_generator() + .next()? + .context("Message not some")?; let _msg_clone = _msg_ref.try_clone()?; drop(_msg_ref); diff --git a/src/keyed_message/read.rs b/src/keyed_message/read.rs index 50b30e3..f59c700 100644 --- a/src/keyed_message/read.rs +++ b/src/keyed_message/read.rs @@ -1,142 +1,172 @@ use std::cmp::Ordering; use crate::{ - keyed_message::DynamicKeyType, keyed_message::KeyRead, KeyedMessage, + KeyedMessage, + codes_handle::ThreadSafeHandle, errors::CodesError, intermediate_bindings::{ NativeKeyType, codes_get_bytes, codes_get_double, codes_get_double_array, codes_get_long, codes_get_long_array, codes_get_native_type, codes_get_size, codes_get_string, }, + keyed_message::AtomicMessage, }; -impl KeyRead for KeyedMessage<'_> { - fn read_key(&self, key_name: &str) -> Result { - match self.get_key_native_type(key_name)? { - NativeKeyType::Long => (), - _ => return Err(CodesError::WrongRequestedKeyType), - } - - let key_size = self.get_key_size(key_name)?; - - match key_size.cmp(&1) { - Ordering::Greater => return Err(CodesError::WrongRequestedKeySize), - Ordering::Less => return Err(CodesError::IncorrectKeySize), - Ordering::Equal => (), - } - - self.read_key_unchecked(key_name) - } +/// Provides GRIB key reading capabilites. Implemented by [`KeyedMessage`] for all possible key types. +pub trait KeyRead { + /// Tries to read a key of given name from [`KeyedMessage`]. This function checks if key native type + /// matches the requested type (ie. you cannot read integer as string, or array as a number). + /// + /// # Example + /// + /// ``` + /// # use eccodes::{ProductKind, CodesHandle, KeyRead}; + /// # use std::path::Path; + /// # use anyhow::Context; + /// # use eccodes::FallibleStreamingIterator; + /// # + /// # fn main() -> anyhow::Result<()> { + /// # let file_path = Path::new("./data/iceland.grib"); + /// # let product_kind = ProductKind::GRIB; + /// # + /// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + /// let message = handle.next()?.context("no message")?; + /// let short_name: String = message.read_key("shortName")?; + /// + /// assert_eq!(short_name, "msl"); + /// # Ok(()) + /// # } + /// ``` + /// + /// # Errors + /// + /// Returns [`WrongRequestedKeySize`](CodesError::WrongRequestedKeyType) when trying to read key in non-native type (use [`unchecked`](KeyRead::read_key_unchecked) instead). + /// + /// Returns [`WrongRequestedKeySize`](CodesError::WrongRequestedKeySize) when trying to read array as integer. + /// + /// Returns [`IncorrectKeySize`](CodesError::IncorrectKeySize) when key size is 0. This can indicate corrupted data. + /// + /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to read the key. + fn read_key(&self, name: &str) -> Result; - fn read_key_unchecked(&self, key_name: &str) -> Result { - unsafe { codes_get_long(self.message_handle, key_name) } - } + /// Skips all the checks provided by [`read_key`](KeyRead::read_key) and directly calls ecCodes, ensuring only memory and type safety. + /// + /// This function has better perfomance than [`read_key`](KeyRead::read_key) but all error handling and (possible) + /// type conversions are performed directly by ecCodes. + /// + /// This function is also useful for (not usually used) keys that return incorrect native type. + /// + /// # Example + /// + /// ``` + /// # use eccodes::{ProductKind, CodesHandle, KeyRead}; + /// # use std::path::Path; + /// # use anyhow::Context; + /// # use eccodes::FallibleStreamingIterator; + /// # + /// # fn main() -> anyhow::Result<()> { + /// # let file_path = Path::new("./data/iceland.grib"); + /// # let product_kind = ProductKind::GRIB; + /// # + /// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + /// let message = handle.next()?.context("no message")?; + /// let short_name: String = message.read_key_unchecked("shortName")?; + /// + /// assert_eq!(short_name, "msl"); + /// # Ok(()) + /// # } + /// ``` + /// + /// # Errors + /// + /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to read the key. + fn read_key_unchecked(&self, name: &str) -> Result; } -impl KeyRead for KeyedMessage<'_> { - fn read_key(&self, key_name: &str) -> Result { - match self.get_key_native_type(key_name)? { - NativeKeyType::Double => (), - _ => return Err(CodesError::WrongRequestedKeyType), - } - - let key_size = self.get_key_size(key_name)?; - - match key_size.cmp(&1) { - Ordering::Greater => return Err(CodesError::WrongRequestedKeySize), - Ordering::Less => return Err(CodesError::IncorrectKeySize), - Ordering::Equal => (), - } - - self.read_key_unchecked(key_name) - } - - fn read_key_unchecked(&self, key_name: &str) -> Result { - unsafe { codes_get_double(self.message_handle, key_name) } - } +#[doc(hidden)] +pub trait KeyReadHelpers { + fn get_key_size(&mut self, key_name: &str) -> Result; + fn get_key_native_type(&mut self, key_name: &str) -> Result; } -impl KeyRead for KeyedMessage<'_> { - fn read_key(&self, key_name: &str) -> Result { - match self.get_key_native_type(key_name)? { - NativeKeyType::Str => (), - _ => return Err(CodesError::WrongRequestedKeyType), - } - - let key_size = self.get_key_size(key_name)?; - - if key_size < 1 { - return Err(CodesError::IncorrectKeySize); - } - - self.read_key_unchecked(key_name) +impl KeyReadHelpers for KeyedMessage<'_> { + fn get_key_size(&mut self, key_name: &str) -> Result { + unsafe { codes_get_size(self.message_handle, key_name) } } - fn read_key_unchecked(&self, key_name: &str) -> Result { - unsafe { codes_get_string(self.message_handle, key_name) } + fn get_key_native_type(&mut self, key_name: &str) -> Result { + unsafe { codes_get_native_type(self.message_handle, key_name) } } } -impl KeyRead> for KeyedMessage<'_> { - fn read_key(&self, key_name: &str) -> Result, CodesError> { - match self.get_key_native_type(key_name)? { - NativeKeyType::Long => (), - _ => return Err(CodesError::WrongRequestedKeyType), - } - - let key_size = self.get_key_size(key_name)?; - - if key_size < 1 { - return Err(CodesError::IncorrectKeySize); - } - - self.read_key_unchecked(key_name) +impl KeyReadHelpers for AtomicMessage { + fn get_key_size(&mut self, key_name: &str) -> Result { + unsafe { codes_get_size(self.message_handle, key_name) } } - fn read_key_unchecked(&self, key_name: &str) -> Result, CodesError> { - unsafe { codes_get_long_array(self.message_handle, key_name) } + fn get_key_native_type(&mut self, key_name: &str) -> Result { + unsafe { codes_get_native_type(self.message_handle, key_name) } } } -impl KeyRead> for KeyedMessage<'_> { - fn read_key(&self, key_name: &str) -> Result, CodesError> { - match self.get_key_native_type(key_name)? { - NativeKeyType::Double => (), - _ => return Err(CodesError::WrongRequestedKeyType), - } +macro_rules! impl_key_read { + ($key_sizing:ident, $ec_func:ident, $key_variant:path, $gen_type:ty) => { + impl AtomicKeyRead<$gen_type> for AtomicMessage { + fn read_key_unchecked(&mut self, key_name: &str) -> Result<$gen_type, CodesError> { + unsafe { $ec_func(self.message_handle, key_name) } + } - let key_size = self.get_key_size(key_name)?; + fn read_key(&mut self, key_name: &str) -> Result<$gen_type, CodesError> { + match self.get_key_native_type(key_name)? { + $key_variant => (), + _ => return Err(CodesError::WrongRequestedKeyType), + } - if key_size < 1 { - return Err(CodesError::IncorrectKeySize); - } + let key_size = self.get_key_size(key_name)?; - self.read_key_unchecked(key_name) - } + key_size_check!($key_sizing, key_size); - fn read_key_unchecked(&self, key_name: &str) -> Result, CodesError> { - unsafe { codes_get_double_array(self.message_handle, key_name) } - } + self.read_key_unchecked(key_name) + } + } + }; } -impl KeyRead> for KeyedMessage<'_> { - fn read_key(&self, key_name: &str) -> Result, CodesError> { - match self.get_key_native_type(key_name)? { - NativeKeyType::Bytes => (), - _ => return Err(CodesError::WrongRequestedKeyType), +macro_rules! key_size_check { + // size_var is needed because of macro hygiene + (scalar, $size_var:ident) => { + match $size_var.cmp(&1) { + Ordering::Greater => return Err(CodesError::WrongRequestedKeySize), + Ordering::Less => return Err(CodesError::IncorrectKeySize), + Ordering::Equal => (), } + }; - let key_size = self.get_key_size(key_name)?; - - if key_size < 1 { + (array, $size_var:ident) => { + if $size_var < 1 { return Err(CodesError::IncorrectKeySize); } + }; +} - self.read_key_unchecked(key_name) - } - - fn read_key_unchecked(&self, key_name: &str) -> Result, CodesError> { - unsafe { codes_get_bytes(self.message_handle, key_name) } - } +/// Enum of types GRIB key can have. +/// +/// Messages inside GRIB files can contain keys of arbitrary types, which are known only at runtime (after being checked). +/// ecCodes can return several different types of key, which are represented by this enum +/// and each variant contains the respective data type. +#[derive(Clone, Debug, PartialEq)] +pub enum DynamicKeyType { + #[allow(missing_docs)] + Float(f64), + #[allow(missing_docs)] + Int(i64), + #[allow(missing_docs)] + FloatArray(Vec), + #[allow(missing_docs)] + IntArray(Vec), + #[allow(missing_docs)] + Str(String), + #[allow(missing_docs)] + Bytes(Vec), } impl KeyedMessage<'_> { @@ -300,7 +330,7 @@ mod tests { use anyhow::{Context, Result}; use crate::codes_handle::{CodesHandle, ProductKind}; - use crate::{keyed_message::DynamicKeyType, FallibleIterator}; + use crate::{FallibleIterator, keyed_message::DynamicKeyType}; use std::path::Path; #[test] @@ -310,7 +340,10 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.message_generator().next()?.context("Message not some")?; + let current_message = handle + .message_generator() + .next()? + .context("Message not some")?; let str_key = current_message.read_key_dynamic("name")?; @@ -348,7 +381,10 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.message_generator().next()?.context("Message not some")?; + let current_message = handle + .message_generator() + .next()? + .context("Message not some")?; let mut kiter = current_message.default_keys_iterator()?; while let Some(key_name) = kiter.next()? { @@ -365,7 +401,10 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.message_generator().next()?.context("Message not some")?; + let current_message = handle + .message_generator() + .next()? + .context("Message not some")?; let mut kiter = current_message.default_keys_iterator()?; while let Some(key_name) = kiter.next()? { @@ -382,7 +421,10 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.message_generator().next()?.context("Message not some")?; + let current_message = handle + .message_generator() + .next()? + .context("Message not some")?; let missing_key = current_message.read_key_dynamic("doesNotExist"); @@ -398,7 +440,10 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let msg = handle.message_generator().next()?.context("Message not some")?; + let msg = handle + .message_generator() + .next()? + .context("Message not some")?; let _ = msg.read_key_dynamic("dataDate")?; let _ = msg.read_key_dynamic("jDirectionIncrementInDegrees")?; diff --git a/src/keyed_message/write.rs b/src/keyed_message/write.rs index 89d9914..f256a20 100644 --- a/src/keyed_message/write.rs +++ b/src/keyed_message/write.rs @@ -9,7 +9,38 @@ use crate::{ }, }; -use super::KeyWrite; +/// Provides GRIB key writing capabilites. Implemented by [`KeyedMessage`] for all possible key types. +pub trait KeyWrite { + /// Writes key with given name and value to [`KeyedMessage`] overwriting existing value, unless + /// the key is read-only. This function directly calls ecCodes ensuring only type and memory safety. + /// + /// # Example + /// + /// ``` + /// # use eccodes::{ProductKind, CodesHandle, KeyWrite}; + /// # use std::path::Path; + /// # use anyhow::Context; + /// # use eccodes::FallibleStreamingIterator; + /// # + /// # fn main() -> anyhow::Result<()> { + /// # let file_path = Path::new("./data/iceland.grib"); + /// # let product_kind = ProductKind::GRIB; + /// # + /// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + /// + /// // CodesHandle iterator returns immutable messages. + /// // To edit a message it must be cloned. + /// let mut message = handle.next()?.context("no message")?.try_clone()?; + /// message.write_key("level", 1)?; + /// # Ok(()) + /// # } + /// ``` + /// + /// # Errors + /// + /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to write the key. + fn write_key(&mut self, name: &str, value: T) -> Result<(), CodesError>; +} impl KeyWrite for KeyedMessage<'_> { fn write_key(&mut self, name: &str, value: i64) -> Result<(), CodesError> { From 9f87d8d130d90a4b544cf2768d5d3ffb6f579dc6 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 28 Sep 2025 09:20:01 +0200 Subject: [PATCH 16/65] wip - checkpoint --- benches/main.rs | 2 +- src/atomic_message/mod.rs | 24 ---- src/codes_handle/atomic_iterator.rs | 86 --------------- src/codes_handle/iterator.rs | 104 +++++++++++++++--- src/codes_handle/mod.rs | 7 +- src/{keyed_message => codes_message}/clone.rs | 12 +- src/{keyed_message => codes_message}/mod.rs | 42 +++---- src/{keyed_message => codes_message}/read.rs | 14 +-- src/{keyed_message => codes_message}/write.rs | 14 +-- src/codes_nearest.rs | 6 +- src/intermediate_bindings/codes_get.rs | 3 +- src/keys_iterator.rs | 6 +- src/lib.rs | 6 +- src/message_ndarray.rs | 4 +- tests/handle.rs | 4 +- 15 files changed, 147 insertions(+), 187 deletions(-) delete mode 100644 src/atomic_message/mod.rs delete mode 100644 src/codes_handle/atomic_iterator.rs rename src/{keyed_message => codes_message}/clone.rs (75%) rename src/{keyed_message => codes_message}/mod.rs (88%) rename src/{keyed_message => codes_message}/read.rs (98%) rename src/{keyed_message => codes_message}/write.rs (94%) diff --git a/benches/main.rs b/benches/main.rs index 0eaee7b..ecb995a 100644 --- a/benches/main.rs +++ b/benches/main.rs @@ -11,7 +11,7 @@ pub fn key_reading(c: &mut Criterion) { let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); - let msg = handle.message_generator().next().unwrap().unwrap(); + let msg = handle.ref_message_generator().next().unwrap().unwrap(); c.bench_function("long reading", |b| { b.iter(|| msg.read_key_dynamic(black_box("dataDate")).unwrap()) diff --git a/src/atomic_message/mod.rs b/src/atomic_message/mod.rs deleted file mode 100644 index e015caf..0000000 --- a/src/atomic_message/mod.rs +++ /dev/null @@ -1,24 +0,0 @@ -mod read; - -use std::sync::Arc; - -use eccodes_sys::codes_handle; - -use crate::{CodesHandle, codes_handle::ThreadSafeHandle}; - -pub use read::{AtomicKeyRead}; - -/// Because standard `KeyedMessage` is not Copy or Clone it can provide access methods without -/// requiring `&mut self`. As `AtomicMessage` implements `Send + Sync` this exclusive method access is not -/// guaranteed with just `&self`. `AtomicMessage` also implements a minimal subset of functionalities -/// to limit the risk of some internal ecCodes functions not being thread-safe. -/// -/// Right now `AtomicMessage` is also not clonable -#[derive(Debug)] -pub struct AtomicMessage { - pub(crate) _parent: Arc>, - pub(crate) message_handle: *mut codes_handle, -} - -unsafe impl Send for AtomicMessage {} -unsafe impl Sync for AtomicMessage {} diff --git a/src/codes_handle/atomic_iterator.rs b/src/codes_handle/atomic_iterator.rs deleted file mode 100644 index 5e2cf13..0000000 --- a/src/codes_handle/atomic_iterator.rs +++ /dev/null @@ -1,86 +0,0 @@ -use std::sync::Arc; - -use fallible_iterator::FallibleIterator; - -use crate::{ - CodesError, CodesHandle, atomic_message::AtomicMessage, codes_handle::ThreadSafeHandle, -}; - -#[derive(Debug)] -pub struct AtomicMessageGenerator { - codes_handle: Arc>, -} -impl CodesHandle { - pub fn atomic_message_generator(self) -> AtomicMessageGenerator { - AtomicMessageGenerator { - codes_handle: Arc::new(self), - } - } -} - -impl FallibleIterator for AtomicMessageGenerator { - type Item = AtomicMessage; - - type Error = CodesError; - - fn next(&mut self) -> Result, Self::Error> { - let new_eccodes_handle = self.codes_handle.source.gen_codes_handle()?; - - if new_eccodes_handle.is_null() { - Ok(None) - } else { - Ok(Some(AtomicMessage { - _parent: self.codes_handle.clone(), - message_handle: new_eccodes_handle, - })) - } - } -} - -#[cfg(test)] -mod tests { - use std::{ - path::Path, - sync::{Arc, Barrier}, - }; - - use anyhow::{Context, Result}; - use fallible_iterator::FallibleIterator; - - use crate::{CodesHandle, ProductKind}; - - #[test] - fn atomic_thread_safety() -> Result<()> { - let file_path = Path::new("./data/iceland-levels.grib"); - let product_kind = ProductKind::GRIB; - - let handle = CodesHandle::new_from_file(file_path, product_kind)?; - let mut mgen = handle.atomic_message_generator(); - // let _ = handle.atomic_message_generator(); <- not allowed due to ownership - - let barrier = Arc::new(Barrier::new(10)); - - let mut v = vec![]; - - for _ in 0..10 { - let msg = Arc::new(mgen.next()?.context("No more messages")?); - let b = barrier.clone(); - - let t = std::thread::spawn(move || { - for _ in 0..1000 { - b.wait(); - let _ = unsafe { - crate::intermediate_bindings::codes_get_size(msg.message_handle, "shortName") - .unwrap() - }; - } - }); - - v.push(t); - } - - v.into_iter().for_each(|th| th.join().unwrap()); - - Ok(()) - } -} diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index bd402eb..edc2a92 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -1,16 +1,16 @@ use fallible_iterator::FallibleIterator; -use crate::{CodesHandle, KeyedMessage, codes_handle::HandleGenerator, errors::CodesError}; +use crate::{CodesHandle, codes_handle::HandleGenerator, errors::CodesError}; use std::marker::PhantomData; #[derive(Debug)] -pub struct KeyedMessageGenerator<'a, S: HandleGenerator> { +pub struct RefMessageGenerator<'a, S: HandleGenerator> { codes_handle: &'a mut CodesHandle, } impl CodesHandle { - pub fn message_generator<'a>(&'a mut self) -> KeyedMessageGenerator<'a, S> { - KeyedMessageGenerator { codes_handle: self } + pub fn ref_message_generator<'a>(&'a mut self) -> RefMessageGenerator<'a, S> { + RefMessageGenerator { codes_handle: self } } } @@ -18,8 +18,8 @@ impl CodesHandle { /// /// The `next()` will return [`CodesInternal`](crate::errors::CodesInternal) /// when internal ecCodes function returns non-zero code. -impl<'ch, S: HandleGenerator> FallibleIterator for KeyedMessageGenerator<'ch, S> { - type Item = KeyedMessage<'ch>; +impl<'ch, S: HandleGenerator> FallibleIterator for RefMessageGenerator<'ch, S> { + type Item = RefMessage<'ch>; type Error = CodesError; fn next(&mut self) -> Result, Self::Error> { @@ -28,7 +28,7 @@ impl<'ch, S: HandleGenerator> FallibleIterator for KeyedMessageGenerator<'ch, S> if new_eccodes_handle.is_null() { Ok(None) } else { - Ok(Some(KeyedMessage { + Ok(Some(RefMessage { parent_message: PhantomData, message_handle: new_eccodes_handle, })) @@ -36,11 +36,43 @@ impl<'ch, S: HandleGenerator> FallibleIterator for KeyedMessageGenerator<'ch, S> } } +#[derive(Debug)] +pub struct AtomicMessageGenerator { + codes_handle: Arc>, +} +impl CodesHandle { + pub fn atomic_message_generator(self) -> AtomicMessageGenerator { + AtomicMessageGenerator { + codes_handle: Arc::new(self), + } + } +} + +impl FallibleIterator for AtomicMessageGenerator { + type Item = AtomicMessage; + + type Error = CodesError; + + fn next(&mut self) -> Result, Self::Error> { + let new_eccodes_handle = self.codes_handle.source.gen_codes_handle()?; + + if new_eccodes_handle.is_null() { + Ok(None) + } else { + Ok(Some(AtomicMessage { + _parent: self.codes_handle.clone(), + message_handle: new_eccodes_handle, + })) + } + } +} + #[cfg(test)] mod tests { use crate::{ - keyed_message::DynamicKeyType, FallibleIterator, + FallibleIterator, codes_handle::{CodesHandle, ProductKind}, + keyed_message::DynamicKeyType, }; use anyhow::{Context, Ok, Result}; use std::path::Path; @@ -52,21 +84,21 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; let msg1 = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; let key1 = msg1.read_key_dynamic("typeOfLevel")?; drop(msg1); let msg2 = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; let key2 = msg2.read_key_dynamic("typeOfLevel")?; drop(msg2); let msg3 = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; let key3 = msg3.read_key_dynamic("typeOfLevel")?; @@ -84,7 +116,7 @@ mod tests { let file_path = Path::new("./data/iceland-levels.grib"); let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let mut mgen = handle.message_generator(); + let mut mgen = handle.ref_message_generator(); let msg1 = mgen.next()?.context("Message not some")?; drop(msg1); @@ -114,7 +146,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - while let Some(msg) = handle.message_generator().next()? { + while let Some(msg) = handle.ref_message_generator().next()? { let key = msg.read_key_dynamic("shortName")?; match key { @@ -134,7 +166,7 @@ mod tests { let mut handle_collected = vec![]; - while let Some(msg) = handle.message_generator().next()? { + while let Some(msg) = handle.ref_message_generator().next()? { handle_collected.push(msg.try_clone()?); } @@ -156,7 +188,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; let current_message = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; @@ -171,7 +203,7 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let mut mgen = handle.message_generator(); + let mut mgen = handle.ref_message_generator(); assert!(mgen.next()?.is_some()); assert!(mgen.next()?.is_some()); @@ -198,7 +230,7 @@ mod tests { // First, filter and collect the messages to get those that we want let mut level = vec![]; - while let Some(msg) = handle.message_generator().next()? { + while let Some(msg) = handle.ref_message_generator().next()? { if msg.read_key_dynamic("shortName")? == DynamicKeyType::Str("msl".to_string()) && msg.read_key_dynamic("typeOfLevel")? == DynamicKeyType::Str("surface".to_string()) @@ -224,4 +256,42 @@ mod tests { Ok(()) } + + #[test] + fn atomic_thread_safety() -> Result<()> { + let file_path = Path::new("./data/iceland-levels.grib"); + let product_kind = ProductKind::GRIB; + + let handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut mgen = handle.atomic_message_generator(); + // let _ = handle.atomic_message_generator(); <- not allowed due to ownership + + let barrier = Arc::new(Barrier::new(10)); + + let mut v = vec![]; + + for _ in 0..10 { + let msg = Arc::new(mgen.next()?.context("No more messages")?); + let b = barrier.clone(); + + let t = std::thread::spawn(move || { + for _ in 0..1000 { + b.wait(); + let _ = unsafe { + crate::intermediate_bindings::codes_get_size( + msg.message_handle, + "shortName", + ) + .unwrap() + }; + } + }); + + v.push(t); + } + + v.into_iter().for_each(|th| th.join().unwrap()); + + Ok(()) + } } diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index 7a71333..8896121 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -15,9 +15,8 @@ use std::{ }; use tracing::instrument; -pub use iterator::KeyedMessageGenerator; +pub use iterator::RefMessageGenerator; -mod atomic_iterator; mod iterator; /// This is an internal structure used to access provided file by `CodesHandle`. @@ -430,10 +429,10 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let _ref_msg = handle.message_generator().next()?.context("no message")?; + let _ref_msg = handle.ref_message_generator().next()?.context("no message")?; let clone_msg = _ref_msg.try_clone()?; drop(_ref_msg); - let _oth_ref = handle.message_generator().next()?.context("no message")?; + let _oth_ref = handle.ref_message_generator().next()?.context("no message")?; let _nrst = clone_msg.codes_nearest()?; let _kiter = clone_msg.default_keys_iterator()?; diff --git a/src/keyed_message/clone.rs b/src/codes_message/clone.rs similarity index 75% rename from src/keyed_message/clone.rs rename to src/codes_message/clone.rs index cad7aee..7fb0d94 100644 --- a/src/keyed_message/clone.rs +++ b/src/codes_message/clone.rs @@ -1,6 +1,6 @@ use crate::{ AtomicMessage, CodesError, KeyedMessage, codes_handle::ThreadSafeHandle, - intermediate_bindings::codes_handle_clone, keyed_message::ClonedMessage, + intermediate_bindings::codes_handle_clone, keyed_message::BufMessage, }; pub trait TryClone { @@ -12,20 +12,20 @@ pub trait TryClone { /// /// # Errors /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to clone the message. - fn try_clone(&self) -> Result; + fn try_clone(&self) -> Result; } impl TryClone for KeyedMessage<'_> { - fn try_clone(&self) -> Result { - Ok(ClonedMessage { + fn try_clone(&self) -> Result { + Ok(BufMessage { message_handle: unsafe { codes_handle_clone(self.message_handle)? }, }) } } impl TryClone for AtomicMessage { - fn try_clone(&self) -> Result { - Ok(ClonedMessage { + fn try_clone(&self) -> Result { + Ok(BufMessage { message_handle: unsafe { codes_handle_clone(self.message_handle)? }, }) } diff --git a/src/keyed_message/mod.rs b/src/codes_message/mod.rs similarity index 88% rename from src/keyed_message/mod.rs rename to src/codes_message/mod.rs index e62a86d..8f8686e 100644 --- a/src/keyed_message/mod.rs +++ b/src/codes_message/mod.rs @@ -43,42 +43,41 @@ use crate::{ /// /// Destructor for this structure does not panic, but some internal functions may rarely fail /// leading to bugs. Errors encountered in desctructor the are logged with [`log`]. -type KeyedMessage<'ch> = CodesMessage>; +pub type RefMessage<'ch> = CodesMessage>; /// Because standard `KeyedMessage` is not Copy or Clone it can provide access methods without /// requiring `&mut self`. As `AtomicMessage` implements `Send + Sync` this exclusive method access is not /// guaranteed with just `&self`. `AtomicMessage` also implements a minimal subset of functionalities /// to limit the risk of some internal ecCodes functions not being thread-safe. -type AtomicMessage = CodesMessage>; +pub type ArcMessage = CodesMessage>; -unsafe impl Send for AtomicMessage {} -unsafe impl Sync for AtomicMessage {} +unsafe impl Send for ArcMessage {} +unsafe impl Sync for ArcMessage {} -type ClonedMessage = CodesMessage; +pub type BufMessage = CodesMessage; -unsafe impl Send for ClonedMessage {} -unsafe impl Sync for ClonedMessage {} +unsafe impl Send for BufMessage {} +unsafe impl Sync for BufMessage {} /// All messages use this struct for operations. #[derive(Debug)] -struct CodesMessage { +pub struct CodesMessage { pub(crate) _parent: P, pub(crate) message_handle: *mut codes_handle, } #[derive(Debug, Hash, PartialEq, PartialOrd)] -struct OwnedRepr(); +struct BufParent(); #[derive(Debug)] -struct ArcRepr(Arc>); +struct ArcParent(Arc>); /// This is a little unintuitive, but we use `()` here to not unnecessarily pollute /// KeyedMessage and derived types with generics, because `PhantomData` is needed /// only for lifetime restriction and we tightly control how `KeyedMessage` is created. #[derive(Debug, Hash, PartialEq, PartialOrd)] -struct RefRepr<'ch>(PhantomData<&'ch ()>); +struct RefParent<'ch>(PhantomData<&'ch ()>); -#[doc(hidden)] impl Drop for CodesMessage

{ /// Executes the destructor for this type. /// This method calls destructor functions from ecCodes library. @@ -86,13 +85,17 @@ impl Drop for CodesMessage

{ /// In such case all pointers and file descriptors are safely deleted. /// However memory leaks can still occur. /// - /// If any function called in the destructor returns an error warning will appear in log. - /// If bugs occur during `CodesHandle` drop please enable log output and post issue on [Github](https://github.com/ScaleWeather/eccodes). + /// If any function called in the destructor returns an error warning will appear in log/tracing. + /// If bugs occur during `CodesMessage` drop please enable log output and post issue on [Github](https://github.com/ScaleWeather/eccodes). /// /// Technical note: delete functions in ecCodes can only fail with [`CodesInternalError`](crate::errors::CodesInternal::CodesInternalError) /// when other functions corrupt the inner memory of pointer, in that case memory leak is possible. /// In case of corrupt pointer segmentation fault will occur. /// The pointers are cleared at the end of drop as they are not functional regardless of result of delete functions. + /// + /// # Panics + /// + /// In debug #[instrument(level = "trace")] fn drop(&mut self) { unsafe { @@ -102,8 +105,7 @@ impl Drop for CodesMessage

{ "codes_handle_delete() returned an error: {:?}", &error ); - #[cfg(test)] - assert!(false, "Error in KeyedMessage::drop") + debug_assert!(false, "Error in KeyedMessage::drop") }); } @@ -125,7 +127,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; let current_message = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; @@ -147,7 +149,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; let current_message = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; let cloned_message = current_message.try_clone()?; @@ -166,7 +168,7 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let mut mgen = handle.message_generator(); + let mut mgen = handle.ref_message_generator(); let msg = mgen.next()?.context("Message not some")?.try_clone()?; let _ = mgen.next()?; @@ -189,7 +191,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; let _msg_ref = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; let _msg_clone = _msg_ref.try_clone()?; diff --git a/src/keyed_message/read.rs b/src/codes_message/read.rs similarity index 98% rename from src/keyed_message/read.rs rename to src/codes_message/read.rs index f59c700..31f67cc 100644 --- a/src/keyed_message/read.rs +++ b/src/codes_message/read.rs @@ -8,7 +8,7 @@ use crate::{ NativeKeyType, codes_get_bytes, codes_get_double, codes_get_double_array, codes_get_long, codes_get_long_array, codes_get_native_type, codes_get_size, codes_get_string, }, - keyed_message::AtomicMessage, + keyed_message::ArcMessage, }; /// Provides GRIB key reading capabilites. Implemented by [`KeyedMessage`] for all possible key types. @@ -98,7 +98,7 @@ impl KeyReadHelpers for KeyedMessage<'_> { } } -impl KeyReadHelpers for AtomicMessage { +impl KeyReadHelpers for ArcMessage { fn get_key_size(&mut self, key_name: &str) -> Result { unsafe { codes_get_size(self.message_handle, key_name) } } @@ -341,7 +341,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; let current_message = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; @@ -382,7 +382,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; let current_message = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; let mut kiter = current_message.default_keys_iterator()?; @@ -402,7 +402,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; let current_message = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; let mut kiter = current_message.default_keys_iterator()?; @@ -422,7 +422,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; let current_message = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; @@ -441,7 +441,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; let msg = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; diff --git a/src/keyed_message/write.rs b/src/codes_message/write.rs similarity index 94% rename from src/keyed_message/write.rs rename to src/codes_message/write.rs index f256a20..cc6ebfb 100644 --- a/src/keyed_message/write.rs +++ b/src/codes_message/write.rs @@ -178,7 +178,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.message_generator().next()?.context("Message not some")?; + let current_message = handle.ref_message_generator().next()?.context("Message not some")?; let out_path = Path::new("./data/iceland_write.grib"); current_message.write_to_file(out_path, false)?; @@ -194,7 +194,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; let current_message = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")? .try_clone()?; @@ -216,12 +216,12 @@ mod tests { let file_path = Path::new("./data/iceland-surface.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.message_generator().next()?.context("Message not some")?; + let current_message = handle.ref_message_generator().next()?.context("Message not some")?; current_message.write_to_file(out_path, false)?; let file_path = Path::new("./data/iceland-levels.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.message_generator().next()?.context("Message not some")?; + let current_message = handle.ref_message_generator().next()?.context("Message not some")?; current_message.write_to_file(out_path, true)?; remove_file(out_path)?; @@ -235,7 +235,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let mut current_message = handle.message_generator().next()?.context("Message not some")?; + let mut current_message = handle.ref_message_generator().next()?.context("Message not some")?; let old_key = current_message.read_key_dynamic("centre")?; @@ -255,7 +255,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let mut current_message = handle.message_generator().next()?.context("Message not some")?; + let mut current_message = handle.ref_message_generator().next()?.context("Message not some")?; let old_key = current_message.read_key_dynamic("centre")?; @@ -266,7 +266,7 @@ mod tests { let file_path = Path::new("./data/iceland_edit.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.message_generator().next()?.context("Message not some")?; + let current_message = handle.ref_message_generator().next()?.context("Message not some")?; let read_key = current_message.read_key_dynamic("centre")?; diff --git a/src/codes_nearest.rs b/src/codes_nearest.rs index a86dd1f..29dc179 100644 --- a/src/codes_nearest.rs +++ b/src/codes_nearest.rs @@ -138,7 +138,7 @@ mod tests { let mut handle1 = CodesHandle::new_from_file(file_path1, product_kind)?; let msg1 = handle1 - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; let nrst1 = msg1.codes_nearest()?; @@ -146,7 +146,7 @@ mod tests { let mut handle2 = CodesHandle::new_from_file(file_path2, product_kind)?; let msg2 = handle2 - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; let nrst2 = msg2.codes_nearest()?; @@ -168,7 +168,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; let current_message = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; diff --git a/src/intermediate_bindings/codes_get.rs b/src/intermediate_bindings/codes_get.rs index 545b5b7..dbeae42 100644 --- a/src/intermediate_bindings/codes_get.rs +++ b/src/intermediate_bindings/codes_get.rs @@ -251,6 +251,7 @@ pub unsafe fn codes_get_message_size(handle: *const codes_handle) -> Result Result<(*const c_void, usize), CodesError> { @@ -271,7 +272,7 @@ pub unsafe fn codes_get_message( return Err(err.into()); } - assert!( + debug_assert!( buffer_size == message_size, "Buffer and message sizes ar not equal in codes_get_message! Please report this panic on Github." diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index 0818cc5..aa8605e 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -234,7 +234,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; let current_message = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; @@ -262,7 +262,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; let current_message = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; @@ -288,7 +288,7 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; let current_message = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; diff --git a/src/lib.rs b/src/lib.rs index 0c34eaa..a2b141f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -227,7 +227,6 @@ //! features = ["eccodes/docs"] //! ``` -pub mod atomic_message; pub mod codes_handle; #[cfg(feature = "experimental_index")] #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] @@ -235,14 +234,13 @@ pub mod codes_index; pub mod codes_nearest; pub mod errors; mod intermediate_bindings; -pub mod keyed_message; +pub mod codes_message; pub mod keys_iterator; #[cfg(feature = "message_ndarray")] #[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] pub mod message_ndarray; mod pointer_guard; -pub use atomic_message::{AtomicKeyRead, AtomicMessage}; pub use codes_handle::{CodesHandle, ProductKind}; #[cfg(feature = "experimental_index")] #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] @@ -250,5 +248,5 @@ pub use codes_index::CodesIndex; pub use codes_nearest::{CodesNearest, NearestGridpoint}; pub use errors::CodesError; pub use fallible_iterator::{FallibleIterator, IntoFallibleIterator}; -pub use keyed_message::{KeyRead, KeyWrite, KeyedMessage}; +pub use codes_message::{RefMessage, ArcMessage, BufMessage}; pub use keys_iterator::{KeysIterator, KeysIteratorFlags}; diff --git a/src/message_ndarray.rs b/src/message_ndarray.rs index 342a0cd..1d002db 100644 --- a/src/message_ndarray.rs +++ b/src/message_ndarray.rs @@ -277,7 +277,7 @@ mod tests { let file_path = Path::new("./data/iceland-surface.grib"); let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; - while let Some(msg) = handle.message_generator().next()? { + while let Some(msg) = handle.ref_message_generator().next()? { if msg.read_key_dynamic("shortName")? == DynamicKeyType::Str("2d".to_string()) { let ndarray = msg.to_ndarray()?; @@ -303,7 +303,7 @@ mod tests { let file_path = Path::new("./data/iceland-surface.grib"); let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; - while let Some(msg) = handle.message_generator().next()? { + while let Some(msg) = handle.ref_message_generator().next()? { if msg.read_key_dynamic("shortName")? == DynamicKeyType::Str("2d".to_string()) { let rmsg = msg.to_lons_lats_values()?; diff --git a/tests/handle.rs b/tests/handle.rs index dffdbda..6bf3549 100644 --- a/tests/handle.rs +++ b/tests/handle.rs @@ -16,7 +16,7 @@ fn thread_safety_core() -> Result<()> { let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; let current_message = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; @@ -38,7 +38,7 @@ fn thread_safety_core() -> Result<()> { let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; let current_message = handle - .message_generator() + .ref_message_generator() .next()? .context("Message not some")?; From 56a64e4a1807c3a4e61a639ba69da15b880cce68 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 28 Sep 2025 09:27:10 +0200 Subject: [PATCH 17/65] key read with macros --- src/atomic_message/read.rs | 84 -------------------------------------- src/codes_message/read.rs | 46 ++++++++++----------- 2 files changed, 23 insertions(+), 107 deletions(-) delete mode 100644 src/atomic_message/read.rs diff --git a/src/atomic_message/read.rs b/src/atomic_message/read.rs deleted file mode 100644 index 52b6c42..0000000 --- a/src/atomic_message/read.rs +++ /dev/null @@ -1,84 +0,0 @@ -use std::cmp::Ordering; - -use crate::{ - CodesError, - atomic_message::AtomicMessage, - codes_handle::ThreadSafeHandle, - intermediate_bindings::{ - NativeKeyType, codes_get_bytes, codes_get_double, codes_get_double_array, codes_get_long, - codes_get_long_array, codes_get_native_type, codes_get_size, codes_get_string, - }, -}; - -#[doc(hidden)] -pub trait KeyReadHelpers { - fn get_key_size(&mut self, key_name: &str) -> Result; - fn get_key_native_type(&mut self, key_name: &str) -> Result; -} - -pub trait AtomicKeyRead: KeyReadHelpers { - fn read_key(&mut self, key_name: &str) -> Result; - fn read_key_unchecked(&mut self, key_name: &str) -> Result; -} - -macro_rules! impl_key_read { - ($key_sizing:ident, $ec_func:ident, $key_variant:path, $gen_type:ty) => { - impl AtomicKeyRead<$gen_type> for AtomicMessage { - fn read_key_unchecked(&mut self, key_name: &str) -> Result<$gen_type, CodesError> { - unsafe { $ec_func(self.message_handle, key_name) } - } - - fn read_key(&mut self, key_name: &str) -> Result<$gen_type, CodesError> { - match self.get_key_native_type(key_name)? { - $key_variant => (), - _ => return Err(CodesError::WrongRequestedKeyType), - } - - let key_size = self.get_key_size(key_name)?; - - key_size_check!($key_sizing, key_size); - - self.read_key_unchecked(key_name) - } - } - }; -} - -macro_rules! key_size_check { - // size_var is needed because of macro hygiene - (scalar, $size_var:ident) => { - match $size_var.cmp(&1) { - Ordering::Greater => return Err(CodesError::WrongRequestedKeySize), - Ordering::Less => return Err(CodesError::IncorrectKeySize), - Ordering::Equal => (), - } - }; - - (array, $size_var:ident) => { - if $size_var < 1 { - return Err(CodesError::IncorrectKeySize); - } - }; -} - -impl KeyReadHelpers for AtomicMessage { - fn get_key_size(&mut self, key_name: &str) -> Result { - unsafe { codes_get_size(self.message_handle, key_name) } - } - - fn get_key_native_type(&mut self, key_name: &str) -> Result { - unsafe { codes_get_native_type(self.message_handle, key_name) } - } -} - -impl_key_read!(scalar, codes_get_long, NativeKeyType::Long, i64); -impl_key_read!(scalar, codes_get_double, NativeKeyType::Double, f64); -impl_key_read!(array, codes_get_string, NativeKeyType::Str, String); -impl_key_read!(array, codes_get_bytes, NativeKeyType::Bytes, Vec); -impl_key_read!(array, codes_get_long_array, NativeKeyType::Long, Vec); -impl_key_read!( - array, - codes_get_double_array, - NativeKeyType::Double, - Vec -); diff --git a/src/codes_message/read.rs b/src/codes_message/read.rs index 31f67cc..e5cea64 100644 --- a/src/codes_message/read.rs +++ b/src/codes_message/read.rs @@ -1,14 +1,12 @@ -use std::cmp::Ordering; +use std::{cmp::Ordering, fmt::Debug}; use crate::{ - KeyedMessage, - codes_handle::ThreadSafeHandle, + codes_message::CodesMessage, errors::CodesError, intermediate_bindings::{ NativeKeyType, codes_get_bytes, codes_get_double, codes_get_double_array, codes_get_long, codes_get_long_array, codes_get_native_type, codes_get_size, codes_get_string, }, - keyed_message::ArcMessage, }; /// Provides GRIB key reading capabilites. Implemented by [`KeyedMessage`] for all possible key types. @@ -84,38 +82,28 @@ pub trait KeyRead { #[doc(hidden)] pub trait KeyReadHelpers { - fn get_key_size(&mut self, key_name: &str) -> Result; - fn get_key_native_type(&mut self, key_name: &str) -> Result; + fn get_key_size(&self, key_name: &str) -> Result; + fn get_key_native_type(&self, key_name: &str) -> Result; } -impl KeyReadHelpers for KeyedMessage<'_> { - fn get_key_size(&mut self, key_name: &str) -> Result { +impl KeyReadHelpers for CodesMessage

{ + fn get_key_size(&self, key_name: &str) -> Result { unsafe { codes_get_size(self.message_handle, key_name) } } - fn get_key_native_type(&mut self, key_name: &str) -> Result { - unsafe { codes_get_native_type(self.message_handle, key_name) } - } -} - -impl KeyReadHelpers for ArcMessage { - fn get_key_size(&mut self, key_name: &str) -> Result { - unsafe { codes_get_size(self.message_handle, key_name) } - } - - fn get_key_native_type(&mut self, key_name: &str) -> Result { + fn get_key_native_type(&self, key_name: &str) -> Result { unsafe { codes_get_native_type(self.message_handle, key_name) } } } macro_rules! impl_key_read { ($key_sizing:ident, $ec_func:ident, $key_variant:path, $gen_type:ty) => { - impl AtomicKeyRead<$gen_type> for AtomicMessage { - fn read_key_unchecked(&mut self, key_name: &str) -> Result<$gen_type, CodesError> { + impl KeyRead<$gen_type> for CodesMessage

{ + fn read_key_unchecked(&self, key_name: &str) -> Result<$gen_type, CodesError> { unsafe { $ec_func(self.message_handle, key_name) } } - fn read_key(&mut self, key_name: &str) -> Result<$gen_type, CodesError> { + fn read_key(&self, key_name: &str) -> Result<$gen_type, CodesError> { match self.get_key_native_type(key_name)? { $key_variant => (), _ => return Err(CodesError::WrongRequestedKeyType), @@ -148,6 +136,18 @@ macro_rules! key_size_check { }; } +impl_key_read!(scalar, codes_get_long, NativeKeyType::Long, i64); +impl_key_read!(scalar, codes_get_double, NativeKeyType::Double, f64); +impl_key_read!(array, codes_get_string, NativeKeyType::Str, String); +impl_key_read!(array, codes_get_bytes, NativeKeyType::Bytes, Vec); +impl_key_read!(array, codes_get_long_array, NativeKeyType::Long, Vec); +impl_key_read!( + array, + codes_get_double_array, + NativeKeyType::Double, + Vec +); + /// Enum of types GRIB key can have. /// /// Messages inside GRIB files can contain keys of arbitrary types, which are known only at runtime (after being checked). @@ -169,7 +169,7 @@ pub enum DynamicKeyType { Bytes(Vec), } -impl KeyedMessage<'_> { +impl CodesMessage

{ /// Method to get a value of given key with [`DynamicKeyType`] from the `KeyedMessage`, if it exists. /// /// In most cases you should use [`read_key()`](KeyRead::read_key) due to more predictive behaviour From 5046213d7f6d5866b6548b374d14393bdac2f283 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 28 Sep 2025 09:29:39 +0200 Subject: [PATCH 18/65] cleanup clone --- src/codes_message/clone.rs | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/src/codes_message/clone.rs b/src/codes_message/clone.rs index 7fb0d94..9ba252f 100644 --- a/src/codes_message/clone.rs +++ b/src/codes_message/clone.rs @@ -1,9 +1,9 @@ use crate::{ - AtomicMessage, CodesError, KeyedMessage, codes_handle::ThreadSafeHandle, - intermediate_bindings::codes_handle_clone, keyed_message::BufMessage, + BufMessage, CodesError, codes_message::CodesMessage, intermediate_bindings::codes_handle_clone, }; +use std::fmt::Debug; -pub trait TryClone { +impl CodesMessage

{ /// Custom function to clone the `KeyedMessage` and `AtomicMessage`. /// /// **Be careful of the memory overhead!** ecCodes (when reading from file) defers reading the data into memory @@ -12,20 +12,9 @@ pub trait TryClone { /// /// # Errors /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to clone the message. - fn try_clone(&self) -> Result; -} - -impl TryClone for KeyedMessage<'_> { - fn try_clone(&self) -> Result { - Ok(BufMessage { - message_handle: unsafe { codes_handle_clone(self.message_handle)? }, - }) - } -} - -impl TryClone for AtomicMessage { fn try_clone(&self) -> Result { Ok(BufMessage { + _parent: (), message_handle: unsafe { codes_handle_clone(self.message_handle)? }, }) } From e8d78853b030a0aca913ddb6c42da1f0f249af1f Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 28 Sep 2025 09:49:20 +0200 Subject: [PATCH 19/65] key write with macros --- src/codes_message/write.rs | 118 ++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 68 deletions(-) diff --git a/src/codes_message/write.rs b/src/codes_message/write.rs index cc6ebfb..8a501ea 100644 --- a/src/codes_message/write.rs +++ b/src/codes_message/write.rs @@ -1,7 +1,7 @@ -use std::{fs::OpenOptions, io::Write, path::Path, slice}; +use std::{fmt::Debug, fs::OpenOptions, io::Write, path::Path, slice}; use crate::{ - KeyedMessage, + codes_message::CodesMessage, errors::CodesError, intermediate_bindings::{ codes_get_message, codes_set_bytes, codes_set_double, codes_set_double_array, @@ -11,6 +11,8 @@ use crate::{ /// Provides GRIB key writing capabilites. Implemented by [`KeyedMessage`] for all possible key types. pub trait KeyWrite { + /// Unchecked doesn't mean it's unsafe - just that there are no checks on Rust side in comparison to + /// `read_key` which has such checks. /// Writes key with given name and value to [`KeyedMessage`] overwriting existing value, unless /// the key is read-only. This function directly calls ecCodes ensuring only type and memory safety. /// @@ -39,70 +41,31 @@ pub trait KeyWrite { /// # Errors /// /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to write the key. - fn write_key(&mut self, name: &str, value: T) -> Result<(), CodesError>; + fn write_key_unchecked(&mut self, name: &str, value: T) -> Result<(), CodesError>; } -impl KeyWrite for KeyedMessage<'_> { - fn write_key(&mut self, name: &str, value: i64) -> Result<(), CodesError> { - unsafe { codes_set_long(self.message_handle, name, value) } - } -} - -impl KeyWrite for KeyedMessage<'_> { - fn write_key(&mut self, name: &str, value: f64) -> Result<(), CodesError> { - unsafe { codes_set_double(self.message_handle, name, value) } - } -} - -impl KeyWrite<&[i64]> for KeyedMessage<'_> { - fn write_key(&mut self, name: &str, value: &[i64]) -> Result<(), CodesError> { - unsafe { codes_set_long_array(self.message_handle, name, value) } - } -} - -impl KeyWrite<&[f64]> for KeyedMessage<'_> { - fn write_key(&mut self, name: &str, value: &[f64]) -> Result<(), CodesError> { - unsafe { codes_set_double_array(self.message_handle, name, value) } - } -} - -impl KeyWrite<&[u8]> for KeyedMessage<'_> { - fn write_key(&mut self, name: &str, value: &[u8]) -> Result<(), CodesError> { - unsafe { codes_set_bytes(self.message_handle, name, value) } - } -} - -impl KeyWrite<&Vec> for KeyedMessage<'_> { - fn write_key(&mut self, name: &str, value: &Vec) -> Result<(), CodesError> { - unsafe { codes_set_long_array(self.message_handle, name, value) } - } -} - -impl KeyWrite<&Vec> for KeyedMessage<'_> { - fn write_key(&mut self, name: &str, value: &Vec) -> Result<(), CodesError> { - unsafe { codes_set_double_array(self.message_handle, name, value) } - } -} - -impl KeyWrite<&Vec> for KeyedMessage<'_> { - fn write_key(&mut self, name: &str, value: &Vec) -> Result<(), CodesError> { - unsafe { codes_set_bytes(self.message_handle, name, value) } - } -} - -impl KeyWrite<&str> for KeyedMessage<'_> { - fn write_key(&mut self, name: &str, value: &str) -> Result<(), CodesError> { - unsafe { codes_set_string(self.message_handle, name, value) } - } +macro_rules! impl_key_write { + ($ec_func:ident, $gen_type:ty) => { + impl KeyWrite<$gen_type> for CodesMessage

{ + fn write_key_unchecked( + &mut self, + name: &str, + value: $gen_type, + ) -> Result<(), CodesError> { + unsafe { $ec_func(self.message_handle, name, value) } + } + } + }; } -impl KeyWrite<&String> for KeyedMessage<'_> { - fn write_key(&mut self, name: &str, value: &String) -> Result<(), CodesError> { - unsafe { codes_set_string(self.message_handle, name, value) } - } -} +impl_key_write!(codes_set_long, i64); +impl_key_write!(codes_set_double, f64); +impl_key_write!(codes_set_long_array, &[i64]); +impl_key_write!(codes_set_double_array, &[f64]); +impl_key_write!(codes_set_bytes, &[u8]); +impl_key_write!(codes_set_string, &str); -impl KeyedMessage<'_> { +impl CodesMessage { /// Function to write given `KeyedMessage` to a file at provided path. /// If file does not exists it will be created. /// If `append` is set to `true` file will be opened in append mode @@ -166,8 +129,9 @@ mod tests { use fallible_iterator::FallibleIterator; use crate::{ - keyed_message::DynamicKeyType, keyed_message::KeyWrite, codes_handle::{CodesHandle, ProductKind}, + keyed_message::DynamicKeyType, + keyed_message::KeyWrite, }; use std::{fs::remove_file, path::Path}; @@ -178,7 +142,10 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.ref_message_generator().next()?.context("Message not some")?; + let current_message = handle + .ref_message_generator() + .next()? + .context("Message not some")?; let out_path = Path::new("./data/iceland_write.grib"); current_message.write_to_file(out_path, false)?; @@ -216,12 +183,18 @@ mod tests { let file_path = Path::new("./data/iceland-surface.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.ref_message_generator().next()?.context("Message not some")?; + let current_message = handle + .ref_message_generator() + .next()? + .context("Message not some")?; current_message.write_to_file(out_path, false)?; let file_path = Path::new("./data/iceland-levels.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.ref_message_generator().next()?.context("Message not some")?; + let current_message = handle + .ref_message_generator() + .next()? + .context("Message not some")?; current_message.write_to_file(out_path, true)?; remove_file(out_path)?; @@ -235,7 +208,10 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let mut current_message = handle.ref_message_generator().next()?.context("Message not some")?; + let mut current_message = handle + .ref_message_generator() + .next()? + .context("Message not some")?; let old_key = current_message.read_key_dynamic("centre")?; @@ -255,7 +231,10 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let mut current_message = handle.ref_message_generator().next()?.context("Message not some")?; + let mut current_message = handle + .ref_message_generator() + .next()? + .context("Message not some")?; let old_key = current_message.read_key_dynamic("centre")?; @@ -266,7 +245,10 @@ mod tests { let file_path = Path::new("./data/iceland_edit.grib"); let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle.ref_message_generator().next()?.context("Message not some")?; + let current_message = handle + .ref_message_generator() + .next()? + .context("Message not some")?; let read_key = current_message.read_key_dynamic("centre")?; From 8b02c1ddf6ee6352e277b9c88f9dcfd7411bb8b5 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 28 Sep 2025 10:01:30 +0200 Subject: [PATCH 20/65] implement generators --- src/codes_handle/iterator.rs | 31 ++++++++++++++++--------------- src/codes_message/mod.rs | 22 ++++++++++++++++++++-- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index edc2a92..85cd9e0 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -1,7 +1,11 @@ use fallible_iterator::FallibleIterator; -use crate::{CodesHandle, codes_handle::HandleGenerator, errors::CodesError}; -use std::marker::PhantomData; +use crate::{ + ArcMessage, CodesHandle, RefMessage, + codes_handle::{HandleGenerator, ThreadSafeHandle}, + errors::CodesError, +}; +use std::sync::Arc; #[derive(Debug)] pub struct RefMessageGenerator<'a, S: HandleGenerator> { @@ -28,28 +32,25 @@ impl<'ch, S: HandleGenerator> FallibleIterator for RefMessageGenerator<'ch, S> { if new_eccodes_handle.is_null() { Ok(None) } else { - Ok(Some(RefMessage { - parent_message: PhantomData, - message_handle: new_eccodes_handle, - })) + Ok(Some(RefMessage::new_from_gen(new_eccodes_handle))) } } } #[derive(Debug)] -pub struct AtomicMessageGenerator { +pub struct ArcMessageGenerator { codes_handle: Arc>, } impl CodesHandle { - pub fn atomic_message_generator(self) -> AtomicMessageGenerator { - AtomicMessageGenerator { + pub fn arc_message_generator(self) -> ArcMessageGenerator { + ArcMessageGenerator { codes_handle: Arc::new(self), } } } -impl FallibleIterator for AtomicMessageGenerator { - type Item = AtomicMessage; +impl FallibleIterator for ArcMessageGenerator { + type Item = ArcMessage; type Error = CodesError; @@ -59,10 +60,10 @@ impl FallibleIterator for AtomicMessageGenerator { if new_eccodes_handle.is_null() { Ok(None) } else { - Ok(Some(AtomicMessage { - _parent: self.codes_handle.clone(), - message_handle: new_eccodes_handle, - })) + Ok(Some(ArcMessage::new_from_gen( + new_eccodes_handle, + &self.codes_handle, + ))) } } } diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index 8f8686e..81f7aa0 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -72,6 +72,24 @@ struct BufParent(); #[derive(Debug)] struct ArcParent(Arc>); +impl RefMessage<'_> { + pub(crate) fn new_from_gen(handle: *mut codes_handle) -> Self { + RefMessage { + _parent: RefParent(PhantomData), + message_handle: handle, + } + } +} + +impl ArcMessage { + pub(crate) fn new_from_gen(handle: *mut codes_handle, parent: &Arc>) -> Self { + ArcMessage { + _parent: ArcParent(parent.clone()), + message_handle: handle, + } + } +} + /// This is a little unintuitive, but we use `()` here to not unnecessarily pollute /// KeyedMessage and derived types with generics, because `PhantomData` is needed /// only for lifetime restriction and we tightly control how `KeyedMessage` is created. @@ -92,9 +110,9 @@ impl Drop for CodesMessage

{ /// when other functions corrupt the inner memory of pointer, in that case memory leak is possible. /// In case of corrupt pointer segmentation fault will occur. /// The pointers are cleared at the end of drop as they are not functional regardless of result of delete functions. - /// + /// /// # Panics - /// + /// /// In debug #[instrument(level = "trace")] fn drop(&mut self) { From 4d69473a0297524bcc382ba8df7ab789769e2aea Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 28 Sep 2025 10:04:30 +0200 Subject: [PATCH 21/65] fix clone --- src/codes_message/clone.rs | 7 +++---- src/codes_message/mod.rs | 10 ++++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/codes_message/clone.rs b/src/codes_message/clone.rs index 9ba252f..b83bcce 100644 --- a/src/codes_message/clone.rs +++ b/src/codes_message/clone.rs @@ -13,9 +13,8 @@ impl CodesMessage

{ /// # Errors /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to clone the message. fn try_clone(&self) -> Result { - Ok(BufMessage { - _parent: (), - message_handle: unsafe { codes_handle_clone(self.message_handle)? }, - }) + Ok(BufMessage::new(unsafe { + codes_handle_clone(self.message_handle)? + })) } } diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index 81f7aa0..1f76bea 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -90,6 +90,16 @@ impl ArcMessage { } } +impl BufMessage { + /// This could be a From, but that would be less idiomatic and would expose interface that we don't want exposed.I + pub(crate) fn new(handle: *mut codes_handle) -> Self { + BufMessage { + _parent: BufParent(), + message_handle: handle, + } + } +} + /// This is a little unintuitive, but we use `()` here to not unnecessarily pollute /// KeyedMessage and derived types with generics, because `PhantomData` is needed /// only for lifetime restriction and we tightly control how `KeyedMessage` is created. From 3004394924d197b3e6d9cd53706833c9c34f618a Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 28 Sep 2025 10:11:21 +0200 Subject: [PATCH 22/65] fix ndarray --- src/codes_message/mod.rs | 3 + src/codes_message/read.rs | 5 +- src/lib.rs | 2 +- src/message_ndarray.rs | 129 ++------------------------------------ 4 files changed, 10 insertions(+), 129 deletions(-) diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index 1f76bea..348b0ae 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -9,6 +9,9 @@ use eccodes_sys::codes_handle; use std::{fmt::Debug, hash::Hash, marker::PhantomData, ptr::null_mut, sync::Arc}; use tracing::{Level, event, instrument}; +pub use read::KeyRead; +pub use write::KeyWrite; + use crate::{ CodesHandle, codes_handle::ThreadSafeHandle, intermediate_bindings::codes_handle_delete, }; diff --git a/src/codes_message/read.rs b/src/codes_message/read.rs index e5cea64..b3f6887 100644 --- a/src/codes_message/read.rs +++ b/src/codes_message/read.rs @@ -80,13 +80,12 @@ pub trait KeyRead { fn read_key_unchecked(&self, name: &str) -> Result; } -#[doc(hidden)] -pub trait KeyReadHelpers { +pub trait KeyPropertiesRead { fn get_key_size(&self, key_name: &str) -> Result; fn get_key_native_type(&self, key_name: &str) -> Result; } -impl KeyReadHelpers for CodesMessage

{ +impl KeyPropertiesRead for CodesMessage

{ fn get_key_size(&self, key_name: &str) -> Result { unsafe { codes_get_size(self.message_handle, key_name) } } diff --git a/src/lib.rs b/src/lib.rs index a2b141f..784d081 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -248,5 +248,5 @@ pub use codes_index::CodesIndex; pub use codes_nearest::{CodesNearest, NearestGridpoint}; pub use errors::CodesError; pub use fallible_iterator::{FallibleIterator, IntoFallibleIterator}; -pub use codes_message::{RefMessage, ArcMessage, BufMessage}; +pub use codes_message::{RefMessage, ArcMessage, BufMessage, KeyRead, KeyWrite}; pub use keys_iterator::{KeysIterator, KeysIteratorFlags}; diff --git a/src/message_ndarray.rs b/src/message_ndarray.rs index 1d002db..f10c698 100644 --- a/src/message_ndarray.rs +++ b/src/message_ndarray.rs @@ -1,12 +1,11 @@ #![cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] //! Definitions for converting a `KeyedMessage` to ndarray +use std::fmt::Debug; + use ndarray::{Array2, Array3, s}; -use crate::{ - AtomicMessage, CodesError, KeyedMessage, atomic_message::AtomicKeyRead, - codes_handle::ThreadSafeHandle, errors::MessageNdarrayError, keyed_message::KeyRead, -}; +use crate::{codes_message::CodesMessage, errors::MessageNdarrayError, CodesError, KeyRead}; /// Struct returned by [`KeyedMessage::to_lons_lats_values()`] method. /// The arrays are collocated, meaning that `longitudes[i, j]` and `latitudes[i, j]` are the coordinates of `values[i, j]`. @@ -21,7 +20,7 @@ pub struct RustyCodesMessage { pub values: Array2, } -impl KeyedMessage<'_> { +impl CodesMessage

{ /// Converts the message to a 2D ndarray. /// /// Returns ndarray where first dimension represents y coordinates and second dimension represents x coordinates, @@ -141,126 +140,6 @@ impl KeyedMessage<'_> { } } -impl AtomicMessage { - /// Converts the message to a 2D ndarray. - /// - /// Returns ndarray where first dimension represents y coordinates and second dimension represents x coordinates, - /// ie. `[lat, lon]`. - /// - /// Common convention for grib files on regular lon-lat grid assumes that: - /// index `[0, 0]` is the top-left corner of the grid: - /// x coordinates are increasing with the i index, - /// y coordinates are decreasing with the j index. - /// - /// This convention can be checked with `iScansNegatively` and `jScansPositively` keys - - /// if both are false, the above convention is used. - /// - /// Requires the keys `Ni`, `Nj` and `values` to be present in the message. - /// - /// Tested only with simple lat-lon grids. - /// - /// # Errors - /// - /// - When the required keys are not present or if their values are not of the expected type - /// - When the number of values mismatch with the `Ni` and `Nj` keys - #[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] - pub fn to_ndarray(&mut self) -> Result, CodesError> { - let ni: i64 = self.read_key("Ni")?; - let ni = usize::try_from(ni).map_err(MessageNdarrayError::from)?; - - let nj: i64 = self.read_key("Nj")?; - let nj = usize::try_from(nj).map_err(MessageNdarrayError::from)?; - - let vals: Vec = self.read_key("values")?; - if vals.len() != (ni * nj) { - return Err(MessageNdarrayError::UnexpectedValuesLength(vals.len(), ni * nj).into()); - } - - let j_scanning: i64 = self.read_key("jPointsAreConsecutive")?; - - if ![0, 1].contains(&j_scanning) { - return Err(MessageNdarrayError::UnexpectedKeyValue( - "jPointsAreConsecutive".to_owned(), - ) - .into()); - } - - let j_scanning = j_scanning != 0; - - let shape = if j_scanning { (ni, nj) } else { (nj, ni) }; - let vals = Array2::from_shape_vec(shape, vals).map_err(MessageNdarrayError::from)?; - - if j_scanning { - Ok(vals.reversed_axes()) - } else { - Ok(vals) - } - } - - /// Same as [`KeyedMessage::to_ndarray()`] but returns the longitudes and latitudes alongside values. - /// Fields are returned as separate arrays in [`RustyCodesMessage`]. - /// - /// Compared to `to_ndarray` this method has performance overhead as returned arrays may be cloned. - /// - /// This method requires the `latLonValues`, `Ni` and `Nj` keys to be present in the message. - /// - /// # Errors - /// - /// - When the required keys are not present or if their values are not of the expected type - /// - When the number of values mismatch with the `Ni` and `Nj` keys - #[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] - pub fn to_lons_lats_values(&mut self) -> Result { - let ni: i64 = self.read_key("Ni")?; - let ni = usize::try_from(ni).map_err(MessageNdarrayError::from)?; - - let nj: i64 = self.read_key("Nj")?; - let nj = usize::try_from(nj).map_err(MessageNdarrayError::from)?; - - let latlonvals: Vec = self.read_key("latLonValues")?; - - if latlonvals.len() != (ni * nj * 3) { - return Err( - MessageNdarrayError::UnexpectedValuesLength(latlonvals.len(), ni * nj * 3).into(), - ); - } - - let j_scanning: i64 = self.read_key("jPointsAreConsecutive")?; - - if ![0, 1].contains(&j_scanning) { - return Err(MessageNdarrayError::UnexpectedKeyValue( - "jPointsAreConsecutive".to_owned(), - ) - .into()); - } - - let j_scanning = j_scanning != 0; - - let shape = if j_scanning { - (ni, nj, 3_usize) - } else { - (nj, ni, 3_usize) - }; - - let mut latlonvals = - Array3::from_shape_vec(shape, latlonvals).map_err(MessageNdarrayError::from)?; - - if j_scanning { - latlonvals.swap_axes(0, 1); - } - - let (lats, lons, vals) = - latlonvals - .view_mut() - .multi_slice_move((s![.., .., 0], s![.., .., 1], s![.., .., 2])); - - Ok(RustyCodesMessage { - longitudes: lons.into_owned(), - latitudes: lats.into_owned(), - values: vals.into_owned(), - }) - } -} - #[cfg(test)] mod tests { use fallible_iterator::FallibleIterator; From 286c6d45c53744c34615c40d3f36db52fe57ce49 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 28 Sep 2025 10:41:54 +0200 Subject: [PATCH 23/65] fix visibility and warnings --- src/codes_message/clone.rs | 2 +- src/codes_message/mod.rs | 20 +++++++++++--------- src/codes_nearest.rs | 24 +++++++++++++----------- src/keys_iterator.rs | 12 ++++++------ 4 files changed, 31 insertions(+), 27 deletions(-) diff --git a/src/codes_message/clone.rs b/src/codes_message/clone.rs index b83bcce..9931b1b 100644 --- a/src/codes_message/clone.rs +++ b/src/codes_message/clone.rs @@ -12,7 +12,7 @@ impl CodesMessage

{ /// /// # Errors /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to clone the message. - fn try_clone(&self) -> Result { + pub fn try_clone(&self) -> Result { Ok(BufMessage::new(unsafe { codes_handle_clone(self.message_handle)? })) diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index 348b0ae..5a21da2 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -52,7 +52,7 @@ pub type RefMessage<'ch> = CodesMessage>; /// requiring `&mut self`. As `AtomicMessage` implements `Send + Sync` this exclusive method access is not /// guaranteed with just `&self`. `AtomicMessage` also implements a minimal subset of functionalities /// to limit the risk of some internal ecCodes functions not being thread-safe. -pub type ArcMessage = CodesMessage>; +pub type ArcMessage = CodesMessage>; unsafe impl Send for ArcMessage {} unsafe impl Sync for ArcMessage {} @@ -69,11 +69,17 @@ pub struct CodesMessage { pub(crate) message_handle: *mut codes_handle, } +/// This is a little unintuitive, but we use `()` here to not unnecessarily pollute +/// KeyedMessage and derived types with generics, because `PhantomData` is needed +/// only for lifetime restriction and we tightly control how `KeyedMessage` is created. +#[derive(Debug, Hash, PartialEq, PartialOrd)] +pub struct RefParent<'ch>(PhantomData<&'ch ()>); + #[derive(Debug, Hash, PartialEq, PartialOrd)] -struct BufParent(); +pub struct BufParent(); #[derive(Debug)] -struct ArcParent(Arc>); +pub struct ArcParent{_arc_handle: Arc>} impl RefMessage<'_> { pub(crate) fn new_from_gen(handle: *mut codes_handle) -> Self { @@ -87,7 +93,7 @@ impl RefMessage<'_> { impl ArcMessage { pub(crate) fn new_from_gen(handle: *mut codes_handle, parent: &Arc>) -> Self { ArcMessage { - _parent: ArcParent(parent.clone()), + _parent: ArcParent{_arc_handle: parent.clone()}, message_handle: handle, } } @@ -103,11 +109,7 @@ impl BufMessage { } } -/// This is a little unintuitive, but we use `()` here to not unnecessarily pollute -/// KeyedMessage and derived types with generics, because `PhantomData` is needed -/// only for lifetime restriction and we tightly control how `KeyedMessage` is created. -#[derive(Debug, Hash, PartialEq, PartialOrd)] -struct RefParent<'ch>(PhantomData<&'ch ()>); + impl Drop for CodesMessage

{ /// Executes the destructor for this type. diff --git a/src/codes_nearest.rs b/src/codes_nearest.rs index 29dc179..23617b2 100644 --- a/src/codes_nearest.rs +++ b/src/codes_nearest.rs @@ -7,7 +7,8 @@ use eccodes_sys::codes_nearest; use tracing::{Level, event, instrument}; use crate::{ - CodesError, KeyedMessage, + CodesError, + codes_message::CodesMessage, intermediate_bindings::{ codes_grib_nearest_delete, codes_grib_nearest_find, codes_grib_nearest_new, }, @@ -15,9 +16,9 @@ use crate::{ /// The structure used to find nearest gridpoints in `KeyedMessage`. #[derive(Debug)] -pub struct CodesNearest<'a> { +pub struct CodesNearest<'a, P: Debug> { nearest_handle: *mut codes_nearest, - parent_message: &'a KeyedMessage<'a>, + parent_message: &'a CodesMessage

, } /// The structure returned by [`CodesNearest::find_nearest()`]. @@ -36,7 +37,7 @@ pub struct NearestGridpoint { pub value: f64, } -impl KeyedMessage<'_> { +impl CodesMessage

{ /// Creates a new instance of [`CodesNearest`] for the `KeyedMessage`. /// [`CodesNearest`] can be used to find nearest gridpoints for given coordinates in the `KeyedMessage` /// by calling [`find_nearest()`](crate::CodesNearest::find_nearest). @@ -45,7 +46,7 @@ impl KeyedMessage<'_> { /// /// This function returns [`CodesInternal`](crate::errors::CodesInternal) when /// internal nearest handle cannot be created. - pub fn codes_nearest(&self) -> Result, CodesError> { + pub fn codes_nearest<'a>(&'a self) -> Result, CodesError> { let nearest_handle = unsafe { codes_grib_nearest_new(self.message_handle)? }; Ok(CodesNearest { @@ -55,7 +56,7 @@ impl KeyedMessage<'_> { } } -impl CodesNearest<'_> { +impl CodesNearest<'_, P> { ///Function to get four [`NearestGridpoint`]s of a point represented by requested coordinates. /// ///The inputs are latitude and longitude of requested point in respectively degrees north and @@ -85,7 +86,7 @@ impl CodesNearest<'_> { /// ///This function returns [`CodesInternal`](crate::errors::CodesInternal) when ///one of ecCodes function returns the non-zero code. - pub fn find_nearest(&self, lat: f64, lon: f64) -> Result<[NearestGridpoint; 4], CodesError> { + pub fn find_nearest(&mut self, lat: f64, lon: f64) -> Result<[NearestGridpoint; 4], CodesError> { let output_points; unsafe { @@ -101,8 +102,10 @@ impl CodesNearest<'_> { } } -#[doc(hidden)] -impl Drop for CodesNearest<'_> { +impl Drop for CodesNearest<'_, P> { + /// # Panics + /// + /// In debug #[instrument(level = "trace")] fn drop(&mut self) { unsafe { @@ -112,8 +115,7 @@ impl Drop for CodesNearest<'_> { "codes_grib_nearest_delete() returned an error: {:?}", &error ); - #[cfg(test)] - assert!(false, "Error in CodesNearest::drop") + debug_assert!(false, "Error in CodesNearest::drop") }); } diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index aa8605e..5628c56 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -2,11 +2,11 @@ use eccodes_sys::codes_keys_iterator; use fallible_iterator::FallibleIterator; -use std::{marker::PhantomData, ptr::null_mut}; +use std::{fmt::Debug, marker::PhantomData, ptr::null_mut}; use tracing::{Level, event, instrument}; use crate::{ - KeyedMessage, + codes_message::CodesMessage, errors::CodesError, intermediate_bindings::{ codes_keys_iterator_delete, codes_keys_iterator_get_name, codes_keys_iterator_new, @@ -57,7 +57,7 @@ use crate::{ #[allow(clippy::module_name_repetitions)] #[derive(Debug)] pub struct KeysIterator<'a> { - /// Same trick as in `KeyedMessage` + /// Same trick as in `RefMessage` parent_message: PhantomData<&'a ()>, iterator_handle: *mut codes_keys_iterator, next_item_exists: bool, @@ -88,7 +88,7 @@ pub enum KeysIteratorFlags { SkipEditionSpecific = eccodes_sys::CODES_KEYS_ITERATOR_SKIP_EDITION_SPECIFIC as isize, } -impl KeyedMessage<'_> { +impl CodesMessage

{ /// Creates new [`KeysIterator`] for the message with specified flags and namespace. /// /// The flags are set by providing any combination of [`KeysIteratorFlags`] @@ -137,7 +137,7 @@ impl KeyedMessage<'_> { /// internal ecCodes function returns non-zero code. #[instrument(level = "trace")] pub fn new_keys_iterator<'a>( - &'a self, + &'a mut self, flags: &[KeysIteratorFlags], namespace: &str, ) -> Result, CodesError> { @@ -162,7 +162,7 @@ impl KeyedMessage<'_> { /// /// This function returns [`CodesInternal`](crate::errors::CodesInternal) when /// internal ecCodes function returns non-zero code. - pub fn default_keys_iterator(&self) -> Result, CodesError> { + pub fn default_keys_iterator<'a>(&'a mut self) -> Result, CodesError> { let iterator_handle = unsafe { codes_keys_iterator_new(self.message_handle, 0, "")? }; let next_item_exists = unsafe { codes_keys_iterator_next(iterator_handle)? }; From afa5af0052a9ea2db6319fa8860e9d952e01915a Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 28 Sep 2025 10:56:28 +0200 Subject: [PATCH 24/65] some doc cleanup --- src/codes_handle/mod.rs | 12 +++++++++--- src/codes_message/mod.rs | 22 +++++++++++++++------- src/lib.rs | 24 ++++++++++++------------ src/message_ndarray.rs | 5 ++--- 4 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index 8896121..2b23eda 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -15,7 +15,7 @@ use std::{ }; use tracing::instrument; -pub use iterator::RefMessageGenerator; +pub use iterator::{ArcMessageGenerator, RefMessageGenerator}; mod iterator; @@ -429,10 +429,16 @@ mod tests { let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let _ref_msg = handle.ref_message_generator().next()?.context("no message")?; + let _ref_msg = handle + .ref_message_generator() + .next()? + .context("no message")?; let clone_msg = _ref_msg.try_clone()?; drop(_ref_msg); - let _oth_ref = handle.ref_message_generator().next()?.context("no message")?; + let _oth_ref = handle + .ref_message_generator() + .next()? + .context("no message")?; let _nrst = clone_msg.codes_nearest()?; let _kiter = clone_msg.default_keys_iterator()?; diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index 5a21da2..914b613 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -5,13 +5,15 @@ mod clone; mod read; mod write; +#[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] +pub use crate::message_ndarray::RustyCodesMessage; +pub use read::{DynamicKeyType, KeyPropertiesRead, KeyRead}; +pub use write::KeyWrite; + use eccodes_sys::codes_handle; use std::{fmt::Debug, hash::Hash, marker::PhantomData, ptr::null_mut, sync::Arc}; use tracing::{Level, event, instrument}; -pub use read::KeyRead; -pub use write::KeyWrite; - use crate::{ CodesHandle, codes_handle::ThreadSafeHandle, intermediate_bindings::codes_handle_delete, }; @@ -63,6 +65,7 @@ unsafe impl Send for BufMessage {} unsafe impl Sync for BufMessage {} /// All messages use this struct for operations. +#[doc(hidden)] #[derive(Debug)] pub struct CodesMessage { pub(crate) _parent: P, @@ -73,13 +76,18 @@ pub struct CodesMessage { /// KeyedMessage and derived types with generics, because `PhantomData` is needed /// only for lifetime restriction and we tightly control how `KeyedMessage` is created. #[derive(Debug, Hash, PartialEq, PartialOrd)] +#[doc(hidden)] pub struct RefParent<'ch>(PhantomData<&'ch ()>); #[derive(Debug, Hash, PartialEq, PartialOrd)] +#[doc(hidden)] pub struct BufParent(); #[derive(Debug)] -pub struct ArcParent{_arc_handle: Arc>} +#[doc(hidden)] +pub struct ArcParent { + _arc_handle: Arc>, +} impl RefMessage<'_> { pub(crate) fn new_from_gen(handle: *mut codes_handle) -> Self { @@ -93,7 +101,9 @@ impl RefMessage<'_> { impl ArcMessage { pub(crate) fn new_from_gen(handle: *mut codes_handle, parent: &Arc>) -> Self { ArcMessage { - _parent: ArcParent{_arc_handle: parent.clone()}, + _parent: ArcParent { + _arc_handle: parent.clone(), + }, message_handle: handle, } } @@ -109,8 +119,6 @@ impl BufMessage { } } - - impl Drop for CodesMessage

{ /// Executes the destructor for this type. /// This method calls destructor functions from ecCodes library. diff --git a/src/lib.rs b/src/lib.rs index 784d081..ce1ae77 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -65,7 +65,7 @@ //! //! ecCodes represents GRIB files as a set of separate messages, each representing data fields at specific time and level. //! Messages are represented here by the [`KeyedMessage`] structure. -//! +//! //! To obtain `KeyedMessage`(s) from `CodesHandle` you need to create an instance of [`KeyedMessageGenerator`](codes_handle::KeyedMessageGenerator) //! with [`CodesHandle::message_generator()`]. This is an analogous interface to `IterMut` and `iter_mut()` in [`std::slice`]. //! @@ -73,7 +73,7 @@ //! which allows you to iterate over messages in the file. The iterator returns `KeyedMessage` with lifetime tied to the lifetime of `CodesHandle`, //! that is `KeyedMessage` cannot outlive the `CodesHandle` it was generated from. If you need to prolong its lifetime, you can use //! [`try_clone()`](KeyedMessage::try_clone), but that comes with performance and memory overhead. -//! +//! //! `KeyedMessage` implements several methods to access the data as needed, most of those can be called directly on `&KeyedMessage`. //! You can also use [`try_clone()`](KeyedMessage::try_clone) to clone the message and prolong its lifetime. //! @@ -128,8 +128,8 @@ //! ``` //! //! #### **New in 0.14** Example 3: Concurrent read -//! -//! +//! +//! //! #### Example 3 - Writing GRIB files //! //! ```rust @@ -201,13 +201,13 @@ //! # Ok(()) //! # } //! ``` -//! +//! -//! -//! +//! +//! //! ## Changes in version 0.14 -//! -//! +//! +//! //! ## Feature Flags //! //! - `message_ndarray` - enables support for converting [`KeyedMessage`] to [`ndarray::Array`]. @@ -231,22 +231,22 @@ pub mod codes_handle; #[cfg(feature = "experimental_index")] #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] pub mod codes_index; +pub mod codes_message; pub mod codes_nearest; pub mod errors; mod intermediate_bindings; -pub mod codes_message; pub mod keys_iterator; #[cfg(feature = "message_ndarray")] #[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] -pub mod message_ndarray; +mod message_ndarray; mod pointer_guard; pub use codes_handle::{CodesHandle, ProductKind}; #[cfg(feature = "experimental_index")] #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] pub use codes_index::CodesIndex; +pub use codes_message::{ArcMessage, BufMessage, RefMessage, KeyRead, KeyWrite}; pub use codes_nearest::{CodesNearest, NearestGridpoint}; pub use errors::CodesError; pub use fallible_iterator::{FallibleIterator, IntoFallibleIterator}; -pub use codes_message::{RefMessage, ArcMessage, BufMessage, KeyRead, KeyWrite}; pub use keys_iterator::{KeysIterator, KeysIteratorFlags}; diff --git a/src/message_ndarray.rs b/src/message_ndarray.rs index f10c698..48be7f4 100644 --- a/src/message_ndarray.rs +++ b/src/message_ndarray.rs @@ -1,11 +1,10 @@ -#![cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] //! Definitions for converting a `KeyedMessage` to ndarray use std::fmt::Debug; use ndarray::{Array2, Array3, s}; -use crate::{codes_message::CodesMessage, errors::MessageNdarrayError, CodesError, KeyRead}; +use crate::{CodesError, KeyRead, codes_message::CodesMessage, errors::MessageNdarrayError}; /// Struct returned by [`KeyedMessage::to_lons_lats_values()`] method. /// The arrays are collocated, meaning that `longitudes[i, j]` and `latitudes[i, j]` are the coordinates of `values[i, j]`. @@ -148,7 +147,7 @@ mod tests { use super::*; use crate::ProductKind; use crate::codes_handle::CodesHandle; - use crate::keyed_message::DynamicKeyType; + use crate::codes_message::DynamicKeyType; use std::path::Path; #[test] From 94ee798c1a9f831af2c98facc524b58be3a48227 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 28 Sep 2025 16:26:06 +0200 Subject: [PATCH 25/65] fix all warnings in lib except missing docs --- Cargo.toml | 4 ---- src/codes_handle/mod.rs | 1 - src/codes_index.rs | 4 ++-- src/intermediate_bindings/codes_handle.rs | 2 +- src/intermediate_bindings/codes_index.rs | 20 ++++++++++---------- 5 files changed, 13 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f091062..378b9dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,11 +2,7 @@ name = "eccodes" description = "Unofficial high-level Rust bindings of the latest ecCodes release" repository = "https://github.com/ScaleWeather/eccodes" -<<<<<<< HEAD -version = "0.13.4" -======= version = "0.14.0" ->>>>>>> 205d4a9 (update eccodes-sys, some fixes to codes_index and start doing docs) readme = "README.md" authors = ["Jakub Lewandowski "] keywords = ["eccodes", "grib", "bufr", "meteorology", "weather"] diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index 2b23eda..f05b097 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -285,7 +285,6 @@ impl CodesHandle { pub fn new_from_index(index: CodesIndex) -> Result { let new_handle = CodesHandle { source: index, - current_message: None, }; Ok(new_handle) diff --git a/src/codes_index.rs b/src/codes_index.rs index 883076a..fa4bc6b 100644 --- a/src/codes_index.rs +++ b/src/codes_index.rs @@ -72,7 +72,7 @@ use crate::{ }, }; use eccodes_sys::{codes_handle, codes_index}; -use std::{path::Path, ptr::null_mut}; +use std::{path::Path, ptr::null_mut, fmt::Debug}; use tracing::instrument; #[derive(Debug)] @@ -205,7 +205,7 @@ impl CodesIndex { /// the GRIB file is not present in the same relative path as during the index file creation. #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] #[instrument(level = "trace")] - pub fn read_from_file>(index_file_path: P) -> Result { + pub fn read_from_file + Debug>(index_file_path: P) -> Result { let index_file_path: &Path = index_file_path.as_ref(); let file_path = index_file_path.to_str().ok_or_else(|| { std::io::Error::new(std::io::ErrorKind::InvalidData, "Path is not valid utf8") diff --git a/src/intermediate_bindings/codes_handle.rs b/src/intermediate_bindings/codes_handle.rs index d242837..12cddba 100644 --- a/src/intermediate_bindings/codes_handle.rs +++ b/src/intermediate_bindings/codes_handle.rs @@ -78,7 +78,7 @@ pub unsafe fn codes_handle_new_from_index( let mut error_code: i32 = 0; let _g = CODES_LOCK.lock().unwrap(); - let codes_handle = eccodes_sys::codes_handle_new_from_index(index, &raw mut error_code); + let codes_handle = unsafe { eccodes_sys::codes_handle_new_from_index(index, &mut error_code) }; // special case! codes_handle_new_from_index returns -43 when there are no messages left in the index // this is also indicated by a null pointer, which is handled upstream diff --git a/src/intermediate_bindings/codes_index.rs b/src/intermediate_bindings/codes_index.rs index 8b85cc7..8087993 100644 --- a/src/intermediate_bindings/codes_index.rs +++ b/src/intermediate_bindings/codes_index.rs @@ -21,7 +21,8 @@ pub unsafe fn codes_index_new(keys: &str) -> Result<*mut codes_index, CodesError let keys = CString::new(keys).unwrap(); let _g = CODES_LOCK.lock().unwrap(); - let codes_index = eccodes_sys::codes_index_new(context, keys.as_ptr(), &raw mut error_code); + let codes_index = + unsafe { eccodes_sys::codes_index_new(context, keys.as_ptr(), &mut error_code) }; if error_code != 0 { let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); @@ -36,7 +37,8 @@ pub unsafe fn codes_index_read(filename: &str) -> Result<*mut codes_index, Codes let mut error_code: i32 = 0; let _g = CODES_LOCK.lock().unwrap(); - let codes_index = eccodes_sys::codes_index_read(context, filename.as_ptr(), &raw mut error_code); + let codes_index = + unsafe { eccodes_sys::codes_index_read(context, filename.as_ptr(), &mut error_code) }; if error_code != 0 { let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); @@ -47,15 +49,12 @@ pub unsafe fn codes_index_read(filename: &str) -> Result<*mut codes_index, Codes #[instrument(level = "trace")] pub unsafe fn codes_index_delete(index: *mut codes_index) { - #[cfg(test)] - log::trace!("codes_index_delete"); - if index.is_null() { return; } let _g = CODES_LOCK.lock().unwrap(); - eccodes_sys::codes_index_delete(index); + unsafe { eccodes_sys::codes_index_delete(index) }; } pub unsafe fn codes_index_add_file( @@ -67,7 +66,7 @@ pub unsafe fn codes_index_add_file( let filename = CString::new(filename).unwrap(); let _g = CODES_LOCK.lock().unwrap(); - let error_code = eccodes_sys::codes_index_add_file(index, filename.as_ptr()); + let error_code = unsafe { eccodes_sys::codes_index_add_file(index, filename.as_ptr()) }; if error_code != 0 { let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); @@ -86,7 +85,7 @@ pub unsafe fn codes_index_select_long( let key = CString::new(key).unwrap(); let _g = CODES_LOCK.lock().unwrap(); - let error_code = eccodes_sys::codes_index_select_long(index, key.as_ptr(), value); + let error_code = unsafe { eccodes_sys::codes_index_select_long(index, key.as_ptr(), value) }; if error_code != 0 { let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); @@ -105,7 +104,7 @@ pub unsafe fn codes_index_select_double( let key = CString::new(key).unwrap(); let _g = CODES_LOCK.lock().unwrap(); - let error_code = eccodes_sys::codes_index_select_double(index, key.as_ptr(), value); + let error_code = unsafe { eccodes_sys::codes_index_select_double(index, key.as_ptr(), value) }; if error_code != 0 { let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); @@ -125,7 +124,8 @@ pub unsafe fn codes_index_select_string( let value = CString::new(value).unwrap(); let _g = CODES_LOCK.lock().unwrap(); - let error_code = eccodes_sys::codes_index_select_string(index, key.as_ptr(), value.as_ptr()); + let error_code = + unsafe { eccodes_sys::codes_index_select_string(index, key.as_ptr(), value.as_ptr()) }; if error_code != 0 { let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); From af5f4b408f9da1fd9238bb863b693475945ce590 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 28 Sep 2025 16:29:23 +0200 Subject: [PATCH 26/65] fix test compile errors in codes_handle::mod --- src/codes_handle/mod.rs | 49 ++++++++--------------------------------- 1 file changed, 9 insertions(+), 40 deletions(-) diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index f05b097..5d0f034 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -283,9 +283,7 @@ impl CodesHandle { /// when internal [`codes_handle`] cannot be created. #[instrument(level = "trace")] pub fn new_from_index(index: CodesIndex) -> Result { - let new_handle = CodesHandle { - source: index, - }; + let new_handle = CodesHandle { source: index }; Ok(new_handle) } @@ -390,7 +388,6 @@ mod tests { let handle = CodesHandle::new_from_index(index)?; assert_eq!(handle.source.pointer, i_ptr); - assert!(handle.current_message.is_none()); Ok(()) } @@ -432,7 +429,7 @@ mod tests { .ref_message_generator() .next()? .context("no message")?; - let clone_msg = _ref_msg.try_clone()?; + let mut clone_msg = _ref_msg.try_clone()?; drop(_ref_msg); let _oth_ref = handle .ref_message_generator() @@ -440,6 +437,7 @@ mod tests { .context("no message")?; let _nrst = clone_msg.codes_nearest()?; + drop(_nrst); let _kiter = clone_msg.default_keys_iterator()?; } @@ -449,8 +447,6 @@ mod tests { #[test] #[cfg(feature = "experimental_index")] fn codes_handle_drop_index() -> Result<()> { - testing_logger::setup(); - let file_path = Path::new("./data/iceland-surface.grib.idx"); let index = CodesIndex::read_from_file(file_path)?; assert!(!index.pointer.is_null()); @@ -458,12 +454,6 @@ mod tests { let handle = CodesHandle::new_from_index(index)?; drop(handle); - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 1); - assert_eq!(captured_logs[0].body, "codes_index_delete"); - assert_eq!(captured_logs[0].level, log::Level::Trace); - }); - Ok(()) } @@ -476,9 +466,8 @@ mod tests { let mut handle = CodesHandle::new_from_index(index)?; assert!(!handle.source.pointer.is_null()); - assert!(handle.current_message.is_none()); - let msg = handle.next()?; + let msg = handle.ref_message_generator().next()?; assert!(!msg.is_some()); @@ -488,7 +477,6 @@ mod tests { #[test] #[cfg(feature = "experimental_index")] fn multiple_drops_with_index() -> Result<()> { - testing_logger::setup(); { let keys = vec!["typeOfLevel", "level"]; let index = CodesIndex::new_from_keys(&keys)?; @@ -499,35 +487,16 @@ mod tests { .select("level", 600)?; let mut handle = CodesHandle::new_from_index(index)?; - let _ref_msg = handle.next()?.context("no message")?; - let clone_msg = _ref_msg.try_clone()?; - let _oth_ref = handle.next()?.context("no message")?; + let mut mgen = handle.ref_message_generator(); + let _ref_msg = mgen.next()?.context("no message")?; + let mut clone_msg = _ref_msg.try_clone()?; + let _oth_ref = mgen.next()?.context("no message")?; let _nrst = clone_msg.codes_nearest()?; + drop(_nrst); let _kiter = clone_msg.default_keys_iterator()?; } - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 6); - - let expected_logs = vec![ - ("codes_handle_delete", log::Level::Trace), - ("codes_keys_iterator_delete", log::Level::Trace), - ("codes_grib_nearest_delete", log::Level::Trace), - ("codes_handle_delete", log::Level::Trace), - ("codes_handle_delete", log::Level::Trace), - ("codes_index_delete", log::Level::Trace), - ]; - - captured_logs - .iter() - .zip(expected_logs) - .for_each(|(clg, elg)| { - assert_eq!(clg.body, elg.0); - assert_eq!(clg.level, elg.1) - }); - }); - Ok(()) } } From f952969d69b804f7585c14d1509e2637845e1a55 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 28 Sep 2025 16:33:07 +0200 Subject: [PATCH 27/65] fix more unit tests compile errors --- src/codes_handle/iterator.rs | 6 +++--- src/codes_message/write.rs | 7 +++---- src/codes_nearest.rs | 4 ++-- src/keys_iterator.rs | 6 +++--- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index 85cd9e0..9f25d20 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -73,10 +73,10 @@ mod tests { use crate::{ FallibleIterator, codes_handle::{CodesHandle, ProductKind}, - keyed_message::DynamicKeyType, + codes_message::DynamicKeyType, }; use anyhow::{Context, Ok, Result}; - use std::path::Path; + use std::{path::Path, sync::{Arc, Barrier}}; #[test] fn iterator_lifetimes() -> Result<()> { @@ -264,7 +264,7 @@ mod tests { let product_kind = ProductKind::GRIB; let handle = CodesHandle::new_from_file(file_path, product_kind)?; - let mut mgen = handle.atomic_message_generator(); + let mut mgen = handle.arc_message_generator(); // let _ = handle.atomic_message_generator(); <- not allowed due to ownership let barrier = Arc::new(Barrier::new(10)); diff --git a/src/codes_message/write.rs b/src/codes_message/write.rs index 8a501ea..d0c56b2 100644 --- a/src/codes_message/write.rs +++ b/src/codes_message/write.rs @@ -130,8 +130,7 @@ mod tests { use crate::{ codes_handle::{CodesHandle, ProductKind}, - keyed_message::DynamicKeyType, - keyed_message::KeyWrite, + codes_message::{DynamicKeyType, KeyWrite}, }; use std::{fs::remove_file, path::Path}; @@ -215,7 +214,7 @@ mod tests { let old_key = current_message.read_key_dynamic("centre")?; - current_message.write_key("centre", "cnmc")?; + current_message.write_key_unchecked("centre", "cnmc")?; let read_key = current_message.read_key_dynamic("centre")?; @@ -238,7 +237,7 @@ mod tests { let old_key = current_message.read_key_dynamic("centre")?; - current_message.write_key("centre", "cnmc")?; + current_message.write_key_unchecked("centre", "cnmc")?; current_message.write_to_file(Path::new("./data/iceland_edit.grib"), false)?; diff --git a/src/codes_nearest.rs b/src/codes_nearest.rs index 23617b2..e6b7359 100644 --- a/src/codes_nearest.rs +++ b/src/codes_nearest.rs @@ -143,7 +143,7 @@ mod tests { .ref_message_generator() .next()? .context("Message not some")?; - let nrst1 = msg1.codes_nearest()?; + let mut nrst1 = msg1.codes_nearest()?; let out1 = nrst1.find_nearest(64.13, -21.89)?; let mut handle2 = CodesHandle::new_from_file(file_path2, product_kind)?; @@ -151,7 +151,7 @@ mod tests { .ref_message_generator() .next()? .context("Message not some")?; - let nrst2 = msg2.codes_nearest()?; + let mut nrst2 = msg2.codes_nearest()?; let out2 = nrst2.find_nearest(64.13, -21.89)?; assert!(out1[0].value > 10000.0); diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index 5628c56..c0e23d0 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -233,7 +233,7 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle + let mut current_message = handle .ref_message_generator() .next()? .context("Message not some")?; @@ -261,7 +261,7 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle + let mut current_message = handle .ref_message_generator() .next()? .context("Message not some")?; @@ -287,7 +287,7 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle + let mut current_message = handle .ref_message_generator() .next()? .context("Message not some")?; From 06dff95cc9ac32aa12bf2d6e3c9588a51c8baa7b Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 28 Sep 2025 16:37:30 +0200 Subject: [PATCH 28/65] fix compile errors in unit tests --- src/codes_index.rs | 61 ++++++++++++--------------------------- src/codes_message/read.rs | 22 ++++++++------ 2 files changed, 32 insertions(+), 51 deletions(-) diff --git a/src/codes_index.rs b/src/codes_index.rs index fa4bc6b..670ccfc 100644 --- a/src/codes_index.rs +++ b/src/codes_index.rs @@ -72,7 +72,7 @@ use crate::{ }, }; use eccodes_sys::{codes_handle, codes_index}; -use std::{path::Path, ptr::null_mut, fmt::Debug}; +use std::{fmt::Debug, path::Path, ptr::null_mut}; use tracing::instrument; #[derive(Debug)] @@ -205,7 +205,9 @@ impl CodesIndex { /// the GRIB file is not present in the same relative path as during the index file creation. #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] #[instrument(level = "trace")] - pub fn read_from_file + Debug>(index_file_path: P) -> Result { + pub fn read_from_file + Debug>( + index_file_path: P, + ) -> Result { let index_file_path: &Path = index_file_path.as_ref(); let file_path = index_file_path.to_str().ok_or_else(|| { std::io::Error::new(std::io::ErrorKind::InvalidData, "Path is not valid utf8") @@ -313,7 +315,7 @@ impl Drop for CodesIndex { #[cfg(test)] mod tests { use anyhow::{Context, Result, bail}; - use fallible_streaming_iterator::FallibleStreamingIterator; + use fallible_iterator::FallibleIterator; use crate::{ CodesError, CodesHandle, @@ -340,15 +342,7 @@ mod tests { #[test] fn index_destructor() -> Result<()> { let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; - let index = CodesIndex::new_from_keys(&keys)?; - - drop(index); - - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 1); - assert_eq!(captured_logs[0].body, "codes_index_delete"); - assert_eq!(captured_logs[0].level, log::Level::Trace); - }); + let _index = CodesIndex::new_from_keys(&keys)?; Ok(()) } @@ -392,36 +386,19 @@ mod tests { #[test] fn handle_from_index_destructor() -> Result<()> { - testing_logger::setup(); - { - let keys = vec!["typeOfLevel", "level"]; - let index = CodesIndex::new_from_keys(&keys)?; - let grib_path = Path::new("./data/iceland-levels.grib"); - let index = index - .add_grib_file(grib_path)? - .select("typeOfLevel", "isobaricInhPa")? - .select("level", 600)?; - - let mut handle = CodesHandle::new_from_index(index)?; - let _ref_msg = handle.next()?.context("no message")?; - } - - testing_logger::validate(|captured_logs| { - assert_eq!(captured_logs.len(), 2); - - let expected_logs = vec![ - ("codes_handle_delete", log::Level::Trace), - ("codes_index_delete", log::Level::Trace), - ]; - - captured_logs - .iter() - .zip(expected_logs) - .for_each(|(clg, elg)| { - assert_eq!(clg.body, elg.0); - assert_eq!(clg.level, elg.1) - }); - }); + let keys = vec!["typeOfLevel", "level"]; + let index = CodesIndex::new_from_keys(&keys)?; + let grib_path = Path::new("./data/iceland-levels.grib"); + let index = index + .add_grib_file(grib_path)? + .select("typeOfLevel", "isobaricInhPa")? + .select("level", 600)?; + + let mut handle = CodesHandle::new_from_index(index)?; + let _ref_msg = handle + .ref_message_generator() + .next()? + .context("no message")?; Ok(()) } diff --git a/src/codes_message/read.rs b/src/codes_message/read.rs index b3f6887..438d4a3 100644 --- a/src/codes_message/read.rs +++ b/src/codes_message/read.rs @@ -329,7 +329,7 @@ mod tests { use anyhow::{Context, Result}; use crate::codes_handle::{CodesHandle, ProductKind}; - use crate::{FallibleIterator, keyed_message::DynamicKeyType}; + use crate::{FallibleIterator, codes_message::DynamicKeyType}; use std::path::Path; #[test] @@ -380,16 +380,18 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle + let mut current_message = handle .ref_message_generator() .next()? .context("Message not some")?; - let mut kiter = current_message.default_keys_iterator()?; + let key_names = current_message + .default_keys_iterator()? + .collect::>()?; - while let Some(key_name) = kiter.next()? { + key_names.iter().for_each(|key_name| { assert!(!key_name.is_empty()); assert!(current_message.read_key_dynamic(&key_name).is_ok()) - } + }); Ok(()) } @@ -400,16 +402,18 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - let current_message = handle + let mut current_message = handle .ref_message_generator() .next()? .context("Message not some")?; - let mut kiter = current_message.default_keys_iterator()?; + let key_names = current_message + .default_keys_iterator()? + .collect::>()?; - while let Some(key_name) = kiter.next()? { + key_names.iter().for_each(|key_name| { assert!(!key_name.is_empty()); assert!(current_message.read_key_dynamic(&key_name).is_ok()) - } + }); Ok(()) } From 90fbfd5f5f4615a863218dc5a2ff0626ec0e2a52 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 28 Sep 2025 16:41:30 +0200 Subject: [PATCH 29/65] fix compile errors in integration tests --- tests/handle.rs | 2 +- tests/index.rs | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/handle.rs b/tests/handle.rs index 6bf3549..0539465 100644 --- a/tests/handle.rs +++ b/tests/handle.rs @@ -1,7 +1,7 @@ use std::{path::Path, thread}; use anyhow::{Context, Result}; -use eccodes::{CodesHandle, keyed_message::DynamicKeyType, FallibleIterator, ProductKind}; +use eccodes::{CodesHandle, FallibleIterator, ProductKind, codes_message::DynamicKeyType}; #[test] fn thread_safety() { diff --git a/tests/index.rs b/tests/index.rs index d3674c0..a8fd1be 100644 --- a/tests/index.rs +++ b/tests/index.rs @@ -8,7 +8,7 @@ use std::{ use anyhow::{Context, Result}; use eccodes::{ - CodesError, CodesHandle, CodesIndex, FallibleStreamingIterator, KeyRead, ProductKind, + CodesError, CodesHandle, CodesIndex, FallibleIterator, KeyRead, ProductKind, codes_index::Select, }; use rand::Rng; @@ -22,9 +22,9 @@ fn iterate_handle_from_index() -> Result<()> { .select("level", 0)? .select("stepType", "instant")?; - let handle = CodesHandle::new_from_index(index)?; + let mut handle = CodesHandle::new_from_index(index)?; - let counter = handle.count()?; + let counter = handle.ref_message_generator().count()?; assert_eq!(counter, 1); @@ -41,7 +41,10 @@ fn read_index_messages() -> Result<()> { .select("stepType", "instant")?; let mut handle = CodesHandle::new_from_index(index)?; - let current_message = handle.next()?.context("Message not some")?; + let current_message = handle + .ref_message_generator() + .next()? + .context("Message not some")?; { let short_name: String = current_message.read_key("shortName")?; @@ -68,11 +71,10 @@ fn collect_index_iterator() -> Result<()> { let mut handle = CodesHandle::new_from_index(index)?; - let mut levels = vec![]; - - while let Some(msg) = handle.next()? { - levels.push(msg.try_clone()?); - } + let levels = handle + .ref_message_generator() + .map(|msg| msg.try_clone()) + .collect::>()?; assert_eq!(levels.len(), 5); @@ -230,12 +232,12 @@ fn index_handle_interference() -> Result<()> { } }); - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; let grib_path = Path::new("./data/iceland.grib"); for _ in 0..10 { - let sleep_time = rng.gen_range(1..42); // randomizing sleep time to hopefully catch segfaults + let sleep_time = rng.random_range(1..42); // randomizing sleep time to hopefully catch segfaults let index = CodesIndex::new_from_keys(&keys)?.add_grib_file(grib_path)?; let i_handle = CodesHandle::new_from_index(index); From 3977b5496cc01bfa158ec978e50e26b54d1f144e Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Sun, 9 Nov 2025 15:59:46 +0100 Subject: [PATCH 30/65] small test fixes --- src/pointer_guard.rs | 2 ++ tests/index.rs | 1 + 2 files changed, 3 insertions(+) diff --git a/src/pointer_guard.rs b/src/pointer_guard.rs index ba8c4a0..c1b2c0d 100644 --- a/src/pointer_guard.rs +++ b/src/pointer_guard.rs @@ -1,6 +1,7 @@ macro_rules! non_null { ($ptr:expr) => { if $ptr.is_null() { + debug_assert!(false, "Null pointer encountered"); return Err(CodesError::NullPtr); } }; @@ -14,6 +15,7 @@ mod tests { use std::ptr; #[test] + #[should_panic] fn test_non_null() { let ptr: *mut i32 = ptr::null_mut(); let result = simulated_function(ptr); diff --git a/tests/index.rs b/tests/index.rs index a8fd1be..6898356 100644 --- a/tests/index.rs +++ b/tests/index.rs @@ -155,6 +155,7 @@ fn add_file_while_index_open() -> Result<()> { } #[test] +#[ignore = "for releases, indexing is experimental"] fn add_file_to_read_index() -> Result<()> { let file_path = Path::new("./data/iceland-surface.grib.idx"); let grib_path = Path::new("./data/iceland-surface.grib"); From 453782ea59513c2fbaf138e6f66b6dcfd7779dc2 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 5 Jan 2026 13:05:27 +0100 Subject: [PATCH 31/65] update dependencies --- Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 378b9dc..754f78e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ errno = { version = "0.3", default-features = false } num-derive = { version = "0.4", default-features = false } num-traits = { version = "0.2", default-features = false } fallible-iterator = { version = "0.3", default-features = false } -ndarray = { version = "0.16", default-features = false, optional = true, features = [ +ndarray = { version = "0.17", default-features = false, optional = true, features = [ "std", ] } tracing = { version = "0.1", default-features = false, features = [ @@ -36,8 +36,8 @@ tracing = { version = "0.1", default-features = false, features = [ ] } [dev-dependencies] -reqwest = { version = "0.12", features = ["rustls-tls"] } -criterion = "0.7" +reqwest = { version = "0.13", features = ["rustls"] } +criterion = "0.8" rand = "0.9" anyhow = { version = "1.0", features = ["backtrace"] } float-cmp = "0.10" From 346598a4d214fa8c4156a473741b936591406401 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 5 Jan 2026 13:12:17 +0100 Subject: [PATCH 32/65] rename CodesHandle to CodesFile to avoid confusion with eccodes API; this required to rename CodesFile to CodesFileSource --- benches/main.rs | 4 ++-- src/codes_handle/iterator.rs | 28 ++++++++++++------------ src/codes_handle/mod.rs | 42 ++++++++++++++++++------------------ src/codes_message/mod.rs | 16 +++++++------- src/codes_message/read.rs | 12 +++++------ src/codes_message/write.rs | 16 +++++++------- src/codes_nearest.rs | 8 +++---- src/keys_iterator.rs | 8 +++---- src/lib.rs | 4 ++-- src/message_ndarray.rs | 6 +++--- src/pointer_guard.rs | 1 - tests/handle.rs | 6 +++--- tests/index.rs | 2 +- 13 files changed, 76 insertions(+), 77 deletions(-) diff --git a/benches/main.rs b/benches/main.rs index ecb995a..3c3afc4 100644 --- a/benches/main.rs +++ b/benches/main.rs @@ -1,6 +1,6 @@ use criterion::{Criterion, criterion_group, criterion_main}; use eccodes::FallibleIterator; -use eccodes::codes_handle::{CodesHandle, ProductKind}; +use eccodes::codes_handle::{CodesFile, ProductKind}; use std::hint::black_box; use std::path::Path; @@ -9,7 +9,7 @@ pub fn key_reading(c: &mut Criterion) { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind).unwrap(); + let mut handle = CodesFile::new_from_file(file_path, product_kind).unwrap(); let msg = handle.ref_message_generator().next().unwrap().unwrap(); diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index 9f25d20..c8fa268 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -1,7 +1,7 @@ use fallible_iterator::FallibleIterator; use crate::{ - ArcMessage, CodesHandle, RefMessage, + ArcMessage, CodesFile, RefMessage, codes_handle::{HandleGenerator, ThreadSafeHandle}, errors::CodesError, }; @@ -9,10 +9,10 @@ use std::sync::Arc; #[derive(Debug)] pub struct RefMessageGenerator<'a, S: HandleGenerator> { - codes_handle: &'a mut CodesHandle, + codes_handle: &'a mut CodesFile, } -impl CodesHandle { +impl CodesFile { pub fn ref_message_generator<'a>(&'a mut self) -> RefMessageGenerator<'a, S> { RefMessageGenerator { codes_handle: self } } @@ -39,9 +39,9 @@ impl<'ch, S: HandleGenerator> FallibleIterator for RefMessageGenerator<'ch, S> { #[derive(Debug)] pub struct ArcMessageGenerator { - codes_handle: Arc>, + codes_handle: Arc>, } -impl CodesHandle { +impl CodesFile { pub fn arc_message_generator(self) -> ArcMessageGenerator { ArcMessageGenerator { codes_handle: Arc::new(self), @@ -72,7 +72,7 @@ impl FallibleIterator for ArcMessageGenerator { mod tests { use crate::{ FallibleIterator, - codes_handle::{CodesHandle, ProductKind}, + codes_handle::{CodesFile, ProductKind}, codes_message::DynamicKeyType, }; use anyhow::{Context, Ok, Result}; @@ -82,7 +82,7 @@ mod tests { fn iterator_lifetimes() -> Result<()> { let file_path = Path::new("./data/iceland-levels.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let msg1 = handle .ref_message_generator() @@ -116,7 +116,7 @@ mod tests { fn message_lifetime_safety() -> Result<()> { let file_path = Path::new("./data/iceland-levels.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let mut mgen = handle.ref_message_generator(); let msg1 = mgen.next()?.context("Message not some")?; @@ -145,7 +145,7 @@ mod tests { let file_path = Path::new("./data/iceland-surface.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; while let Some(msg) = handle.ref_message_generator().next()? { let key = msg.read_key_dynamic("shortName")?; @@ -163,7 +163,7 @@ mod tests { fn iterator_collected() -> Result<()> { let file_path = Path::new("./data/iceland-surface.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let mut handle_collected = vec![]; @@ -187,7 +187,7 @@ mod tests { let file_path = Path::new("./data/iceland-surface.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle .ref_message_generator() .next()? @@ -203,7 +203,7 @@ mod tests { let file_path = Path::new("./data/iceland-surface.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let mut mgen = handle.ref_message_generator(); assert!(mgen.next()?.is_some()); @@ -225,7 +225,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; // Use iterator to get a Keyed message with shortName "msl" and typeOfLevel "surface" // First, filter and collect the messages to get those that we want @@ -263,7 +263,7 @@ mod tests { let file_path = Path::new("./data/iceland-levels.grib"); let product_kind = ProductKind::GRIB; - let handle = CodesHandle::new_from_file(file_path, product_kind)?; + let handle = CodesFile::new_from_file(file_path, product_kind)?; let mut mgen = handle.arc_message_generator(); // let _ = handle.atomic_message_generator(); <- not allowed due to ownership diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index 5d0f034..fd6a556 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -24,7 +24,7 @@ mod iterator; /// It is not intended to be used directly by the user. #[doc(hidden)] #[derive(Debug)] -pub struct CodesFile { +pub struct CodesFileSource { // fields dropped from top pointer: *mut FILE, product_kind: ProductKind, @@ -35,8 +35,8 @@ pub struct CodesFile { #[doc(hidden)] pub trait ThreadSafeHandle: HandleGenerator {} -impl ThreadSafeHandle for CodesFile> {} -impl ThreadSafeHandle for CodesFile {} +impl ThreadSafeHandle for CodesFileSource> {} +impl ThreadSafeHandle for CodesFileSource {} /// Internal trait implemented for types that can be called to generate `*mut codes_handle`. #[doc(hidden)] @@ -44,7 +44,7 @@ pub trait HandleGenerator: Debug { fn gen_codes_handle(&self) -> Result<*mut codes_handle, CodesError>; } -impl HandleGenerator for CodesFile { +impl HandleGenerator for CodesFileSource { fn gen_codes_handle(&self) -> Result<*mut codes_handle, CodesError> { unsafe { codes_handle_new_from_file(self.pointer, self.product_kind) } } @@ -119,7 +119,7 @@ impl HandleGenerator for CodesFile { /// /// All available methods for `CodesHandle` iterator can be found in [`FallibleStreamingIterator`](crate::FallibleStreamingIterator) trait. #[derive(Debug)] -pub struct CodesHandle { +pub struct CodesFile { source: S, } @@ -145,7 +145,7 @@ pub enum ProductKind { GRIB = ProductKind_PRODUCT_GRIB as isize, } -impl CodesHandle> { +impl CodesFile> { ///Opens file at given [`Path`] as selected [`ProductKind`] and contructs `CodesHandle`. /// ///## Example @@ -190,7 +190,7 @@ impl CodesHandle> { let file_pointer = open_with_fdopen(&file)?; Ok(Self { - source: CodesFile { + source: CodesFileSource { _data: file, pointer: file_pointer, product_kind, @@ -198,7 +198,7 @@ impl CodesHandle> { }) } } -impl CodesHandle>> { +impl CodesFile>> { ///Opens data in provided buffer as selected [`ProductKind`] and contructs `CodesHandle`. /// ///## Example @@ -241,7 +241,7 @@ impl CodesHandle>> { let file_pointer = open_with_fmemopen(&file_data)?; Ok(Self { - source: CodesFile { + source: CodesFileSource { _data: file_data, product_kind, pointer: file_pointer, @@ -252,7 +252,7 @@ impl CodesHandle>> { #[cfg(feature = "experimental_index")] #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] -impl CodesHandle { +impl CodesFile { /// Creates [`CodesHandle`] for provided [`CodesIndex`]. /// /// ## Example @@ -283,7 +283,7 @@ impl CodesHandle { /// when internal [`codes_handle`] cannot be created. #[instrument(level = "trace")] pub fn new_from_index(index: CodesIndex) -> Result { - let new_handle = CodesHandle { source: index }; + let new_handle = CodesFile { source: index }; Ok(new_handle) } @@ -327,7 +327,7 @@ fn open_with_fmemopen(file_data: &[u8]) -> Result<*mut FILE, CodesError> { #[cfg(test)] mod tests { - use crate::codes_handle::{CodesHandle, ProductKind}; + use crate::codes_handle::{CodesFile, ProductKind}; #[cfg(feature = "experimental_index")] use crate::codes_index::{CodesIndex, Select}; use anyhow::{Context, Result}; @@ -340,7 +340,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let handle = CodesHandle::new_from_file(file_path, product_kind)?; + let handle = CodesFile::new_from_file(file_path, product_kind)?; assert!(!handle.source.pointer.is_null()); assert_eq!(handle.source.product_kind as u32, { @@ -360,7 +360,7 @@ mod tests { let mut buf = Vec::new(); f.read_to_end(&mut buf)?; - let handle = CodesHandle::new_from_memory(buf, product_kind)?; + let handle = CodesFile::new_from_memory(buf, product_kind)?; assert!(!handle.source.pointer.is_null()); assert_eq!(handle.source.product_kind as u32, { ProductKind_PRODUCT_GRIB @@ -385,7 +385,7 @@ mod tests { let i_ptr = index.pointer.clone(); - let handle = CodesHandle::new_from_index(index)?; + let handle = CodesFile::new_from_index(index)?; assert_eq!(handle.source.pointer, i_ptr); @@ -397,7 +397,7 @@ mod tests { let file_path = Path::new("./data/iceland-surface.grib"); let product_kind = ProductKind::GRIB; - let handle = CodesHandle::new_from_file(file_path, product_kind)?; + let handle = CodesFile::new_from_file(file_path, product_kind)?; drop(handle); Ok(()) @@ -411,7 +411,7 @@ mod tests { let mut buf = Vec::new(); f.read_to_end(&mut buf)?; - let handle = CodesHandle::new_from_memory(buf, product_kind)?; + let handle = CodesFile::new_from_memory(buf, product_kind)?; drop(handle); Ok(()) @@ -423,7 +423,7 @@ mod tests { let file_path = Path::new("./data/iceland-surface.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let _ref_msg = handle .ref_message_generator() @@ -451,7 +451,7 @@ mod tests { let index = CodesIndex::read_from_file(file_path)?; assert!(!index.pointer.is_null()); - let handle = CodesHandle::new_from_index(index)?; + let handle = CodesFile::new_from_index(index)?; drop(handle); Ok(()) @@ -463,7 +463,7 @@ mod tests { let index = CodesIndex::new_from_keys(&vec!["shortName", "typeOfLevel", "level", "stepType"])?; - let mut handle = CodesHandle::new_from_index(index)?; + let mut handle = CodesFile::new_from_index(index)?; assert!(!handle.source.pointer.is_null()); @@ -486,7 +486,7 @@ mod tests { .select("typeOfLevel", "isobaricInhPa")? .select("level", 600)?; - let mut handle = CodesHandle::new_from_index(index)?; + let mut handle = CodesFile::new_from_index(index)?; let mut mgen = handle.ref_message_generator(); let _ref_msg = mgen.next()?.context("no message")?; let mut clone_msg = _ref_msg.try_clone()?; diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index 914b613..3c35454 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -15,7 +15,7 @@ use std::{fmt::Debug, hash::Hash, marker::PhantomData, ptr::null_mut, sync::Arc} use tracing::{Level, event, instrument}; use crate::{ - CodesHandle, codes_handle::ThreadSafeHandle, intermediate_bindings::codes_handle_delete, + CodesFile, codes_handle::ThreadSafeHandle, intermediate_bindings::codes_handle_delete, }; /// Structure that provides access to the data contained in the GRIB file, which directly corresponds to the message in the GRIB file @@ -86,7 +86,7 @@ pub struct BufParent(); #[derive(Debug)] #[doc(hidden)] pub struct ArcParent { - _arc_handle: Arc>, + _arc_handle: Arc>, } impl RefMessage<'_> { @@ -99,7 +99,7 @@ impl RefMessage<'_> { } impl ArcMessage { - pub(crate) fn new_from_gen(handle: *mut codes_handle, parent: &Arc>) -> Self { + pub(crate) fn new_from_gen(handle: *mut codes_handle, parent: &Arc>) -> Self { ArcMessage { _parent: ArcParent { _arc_handle: parent.clone(), @@ -156,7 +156,7 @@ impl Drop for CodesMessage

{ #[cfg(test)] mod tests { - use crate::codes_handle::{CodesHandle, ProductKind}; + use crate::codes_handle::{CodesFile, ProductKind}; use anyhow::{Context, Result}; use fallible_iterator::FallibleIterator; use std::path::Path; @@ -166,7 +166,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle .ref_message_generator() .next()? @@ -188,7 +188,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle .ref_message_generator() .next()? @@ -208,7 +208,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let mut mgen = handle.ref_message_generator(); let msg = mgen.next()?.context("Message not some")?.try_clone()?; let _ = mgen.next()?; @@ -230,7 +230,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let _msg_ref = handle .ref_message_generator() .next()? diff --git a/src/codes_message/read.rs b/src/codes_message/read.rs index 438d4a3..e24236d 100644 --- a/src/codes_message/read.rs +++ b/src/codes_message/read.rs @@ -328,7 +328,7 @@ impl CodesMessage

{ mod tests { use anyhow::{Context, Result}; - use crate::codes_handle::{CodesHandle, ProductKind}; + use crate::codes_handle::{CodesFile, ProductKind}; use crate::{FallibleIterator, codes_message::DynamicKeyType}; use std::path::Path; @@ -337,7 +337,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle .ref_message_generator() @@ -379,7 +379,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let mut current_message = handle .ref_message_generator() .next()? @@ -401,7 +401,7 @@ mod tests { let file_path = Path::new("./data/gfs.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let mut current_message = handle .ref_message_generator() .next()? @@ -423,7 +423,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle .ref_message_generator() .next()? @@ -441,7 +441,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let msg = handle .ref_message_generator() diff --git a/src/codes_message/write.rs b/src/codes_message/write.rs index d0c56b2..2cc3d8a 100644 --- a/src/codes_message/write.rs +++ b/src/codes_message/write.rs @@ -129,7 +129,7 @@ mod tests { use fallible_iterator::FallibleIterator; use crate::{ - codes_handle::{CodesHandle, ProductKind}, + codes_handle::{CodesFile, ProductKind}, codes_message::{DynamicKeyType, KeyWrite}, }; use std::{fs::remove_file, path::Path}; @@ -139,7 +139,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle .ref_message_generator() @@ -158,7 +158,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle .ref_message_generator() .next()? @@ -181,7 +181,7 @@ mod tests { let out_path = Path::new("./data/iceland_append.grib"); let file_path = Path::new("./data/iceland-surface.grib"); - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle .ref_message_generator() .next()? @@ -189,7 +189,7 @@ mod tests { current_message.write_to_file(out_path, false)?; let file_path = Path::new("./data/iceland-levels.grib"); - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle .ref_message_generator() .next()? @@ -206,7 +206,7 @@ mod tests { let product_kind = ProductKind::GRIB; let file_path = Path::new("./data/iceland.grib"); - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let mut current_message = handle .ref_message_generator() .next()? @@ -229,7 +229,7 @@ mod tests { let product_kind = ProductKind::GRIB; let file_path = Path::new("./data/iceland.grib"); - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let mut current_message = handle .ref_message_generator() .next()? @@ -243,7 +243,7 @@ mod tests { let file_path = Path::new("./data/iceland_edit.grib"); - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle .ref_message_generator() .next()? diff --git a/src/codes_nearest.rs b/src/codes_nearest.rs index e6b7359..5e03c72 100644 --- a/src/codes_nearest.rs +++ b/src/codes_nearest.rs @@ -130,7 +130,7 @@ mod tests { use anyhow::{Context, Result}; use fallible_iterator::FallibleIterator; - use crate::{CodesHandle, ProductKind}; + use crate::{CodesFile, ProductKind}; #[test] fn find_nearest() -> Result<()> { @@ -138,7 +138,7 @@ mod tests { let file_path2 = Path::new("./data/iceland-surface.grib"); let product_kind = ProductKind::GRIB; - let mut handle1 = CodesHandle::new_from_file(file_path1, product_kind)?; + let mut handle1 = CodesFile::new_from_file(file_path1, product_kind)?; let msg1 = handle1 .ref_message_generator() .next()? @@ -146,7 +146,7 @@ mod tests { let mut nrst1 = msg1.codes_nearest()?; let out1 = nrst1.find_nearest(64.13, -21.89)?; - let mut handle2 = CodesHandle::new_from_file(file_path2, product_kind)?; + let mut handle2 = CodesFile::new_from_file(file_path2, product_kind)?; let msg2 = handle2 .ref_message_generator() .next()? @@ -168,7 +168,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle .ref_message_generator() .next()? diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index c0e23d0..de0c343 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -222,7 +222,7 @@ mod tests { use anyhow::{Context, Result}; use crate::FallibleIterator; - use crate::codes_handle::{CodesHandle, ProductKind}; + use crate::codes_handle::{CodesFile, ProductKind}; use std::path::Path; use super::KeysIteratorFlags; @@ -232,7 +232,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let mut current_message = handle .ref_message_generator() .next()? @@ -260,7 +260,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let mut current_message = handle .ref_message_generator() .next()? @@ -286,7 +286,7 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let mut current_message = handle .ref_message_generator() .next()? diff --git a/src/lib.rs b/src/lib.rs index ce1ae77..773834d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,6 @@ #![warn(clippy::pedantic)] #![allow(clippy::cast_possible_wrap)] -#![warn(missing_docs)] +// #![warn(missing_docs)] #![warn(clippy::cargo)] #![warn(clippy::perf)] #![warn(clippy::doc_lazy_continuation)] @@ -241,7 +241,7 @@ pub mod keys_iterator; mod message_ndarray; mod pointer_guard; -pub use codes_handle::{CodesHandle, ProductKind}; +pub use codes_handle::{CodesFile, ProductKind}; #[cfg(feature = "experimental_index")] #[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] pub use codes_index::CodesIndex; diff --git a/src/message_ndarray.rs b/src/message_ndarray.rs index 48be7f4..d7db30c 100644 --- a/src/message_ndarray.rs +++ b/src/message_ndarray.rs @@ -146,14 +146,14 @@ mod tests { use super::*; use crate::ProductKind; - use crate::codes_handle::CodesHandle; + use crate::codes_handle::CodesFile; use crate::codes_message::DynamicKeyType; use std::path::Path; #[test] fn test_to_ndarray() -> Result<(), CodesError> { let file_path = Path::new("./data/iceland-surface.grib"); - let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; + let mut handle = CodesFile::new_from_file(file_path, ProductKind::GRIB)?; while let Some(msg) = handle.ref_message_generator().next()? { if msg.read_key_dynamic("shortName")? == DynamicKeyType::Str("2d".to_string()) { @@ -179,7 +179,7 @@ mod tests { #[test] fn test_lons_lats() -> Result<(), CodesError> { let file_path = Path::new("./data/iceland-surface.grib"); - let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; + let mut handle = CodesFile::new_from_file(file_path, ProductKind::GRIB)?; while let Some(msg) = handle.ref_message_generator().next()? { if msg.read_key_dynamic("shortName")? == DynamicKeyType::Str("2d".to_string()) { diff --git a/src/pointer_guard.rs b/src/pointer_guard.rs index c1b2c0d..d3cef1b 100644 --- a/src/pointer_guard.rs +++ b/src/pointer_guard.rs @@ -11,7 +11,6 @@ pub(crate) use non_null; #[cfg(test)] mod tests { use crate::errors::CodesError; - use crate::pointer_guard::non_null; use std::ptr; #[test] diff --git a/tests/handle.rs b/tests/handle.rs index 0539465..ed4087d 100644 --- a/tests/handle.rs +++ b/tests/handle.rs @@ -1,7 +1,7 @@ use std::{path::Path, thread}; use anyhow::{Context, Result}; -use eccodes::{CodesHandle, FallibleIterator, ProductKind, codes_message::DynamicKeyType}; +use eccodes::{CodesFile, FallibleIterator, ProductKind, codes_message::DynamicKeyType}; #[test] fn thread_safety() { @@ -14,7 +14,7 @@ fn thread_safety_core() -> Result<()> { loop { let file_path = Path::new("./data/iceland.grib"); - let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; + let mut handle = CodesFile::new_from_file(file_path, ProductKind::GRIB)?; let current_message = handle .ref_message_generator() .next()? @@ -36,7 +36,7 @@ fn thread_safety_core() -> Result<()> { for _ in 0..1000 { let file_path = Path::new("./data/iceland.grib"); - let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; + let mut handle = CodesFile::new_from_file(file_path, ProductKind::GRIB)?; let current_message = handle .ref_message_generator() .next()? diff --git a/tests/index.rs b/tests/index.rs index 6898356..b201d0f 100644 --- a/tests/index.rs +++ b/tests/index.rs @@ -8,7 +8,7 @@ use std::{ use anyhow::{Context, Result}; use eccodes::{ - CodesError, CodesHandle, CodesIndex, FallibleIterator, KeyRead, ProductKind, + CodesError, CodesFile, CodesIndex, FallibleIterator, KeyRead, ProductKind, codes_index::Select, }; use rand::Rng; From 13f056d2c2cec6fb408b02adacf68386eb3acb28 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 5 Jan 2026 13:17:11 +0100 Subject: [PATCH 33/65] rename Generators to Iters --- benches/main.rs | 2 +- src/codes_handle/iterator.rs | 36 ++++++++++++++++++------------------ src/codes_handle/mod.rs | 6 +++--- src/codes_message/mod.rs | 8 ++++---- src/codes_message/read.rs | 10 +++++----- src/codes_message/write.rs | 14 +++++++------- src/codes_nearest.rs | 6 +++--- src/keys_iterator.rs | 6 +++--- src/message_ndarray.rs | 4 ++-- tests/handle.rs | 4 ++-- 10 files changed, 48 insertions(+), 48 deletions(-) diff --git a/benches/main.rs b/benches/main.rs index 3c3afc4..12b821a 100644 --- a/benches/main.rs +++ b/benches/main.rs @@ -11,7 +11,7 @@ pub fn key_reading(c: &mut Criterion) { let mut handle = CodesFile::new_from_file(file_path, product_kind).unwrap(); - let msg = handle.ref_message_generator().next().unwrap().unwrap(); + let msg = handle.ref_message_iter().next().unwrap().unwrap(); c.bench_function("long reading", |b| { b.iter(|| msg.read_key_dynamic(black_box("dataDate")).unwrap()) diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index c8fa268..f8acc46 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -8,13 +8,13 @@ use crate::{ use std::sync::Arc; #[derive(Debug)] -pub struct RefMessageGenerator<'a, S: HandleGenerator> { +pub struct RefMessageIter<'a, S: HandleGenerator> { codes_handle: &'a mut CodesFile, } impl CodesFile { - pub fn ref_message_generator<'a>(&'a mut self) -> RefMessageGenerator<'a, S> { - RefMessageGenerator { codes_handle: self } + pub fn ref_message_iter<'a>(&'a mut self) -> RefMessageIter<'a, S> { + RefMessageIter { codes_handle: self } } } @@ -22,7 +22,7 @@ impl CodesFile { /// /// The `next()` will return [`CodesInternal`](crate::errors::CodesInternal) /// when internal ecCodes function returns non-zero code. -impl<'ch, S: HandleGenerator> FallibleIterator for RefMessageGenerator<'ch, S> { +impl<'ch, S: HandleGenerator> FallibleIterator for RefMessageIter<'ch, S> { type Item = RefMessage<'ch>; type Error = CodesError; @@ -38,18 +38,18 @@ impl<'ch, S: HandleGenerator> FallibleIterator for RefMessageGenerator<'ch, S> { } #[derive(Debug)] -pub struct ArcMessageGenerator { +pub struct ArcMessageIter { codes_handle: Arc>, } impl CodesFile { - pub fn arc_message_generator(self) -> ArcMessageGenerator { - ArcMessageGenerator { + pub fn arc_message_iter(self) -> ArcMessageIter { + ArcMessageIter { codes_handle: Arc::new(self), } } } -impl FallibleIterator for ArcMessageGenerator { +impl FallibleIterator for ArcMessageIter { type Item = ArcMessage; type Error = CodesError; @@ -85,21 +85,21 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let msg1 = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; let key1 = msg1.read_key_dynamic("typeOfLevel")?; drop(msg1); let msg2 = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; let key2 = msg2.read_key_dynamic("typeOfLevel")?; drop(msg2); let msg3 = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; let key3 = msg3.read_key_dynamic("typeOfLevel")?; @@ -117,7 +117,7 @@ mod tests { let file_path = Path::new("./data/iceland-levels.grib"); let product_kind = ProductKind::GRIB; let mut handle = CodesFile::new_from_file(file_path, product_kind)?; - let mut mgen = handle.ref_message_generator(); + let mut mgen = handle.ref_message_iter(); let msg1 = mgen.next()?.context("Message not some")?; drop(msg1); @@ -147,7 +147,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; - while let Some(msg) = handle.ref_message_generator().next()? { + while let Some(msg) = handle.ref_message_iter().next()? { let key = msg.read_key_dynamic("shortName")?; match key { @@ -167,7 +167,7 @@ mod tests { let mut handle_collected = vec![]; - while let Some(msg) = handle.ref_message_generator().next()? { + while let Some(msg) = handle.ref_message_iter().next()? { handle_collected.push(msg.try_clone()?); } @@ -189,7 +189,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; @@ -204,7 +204,7 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesFile::new_from_file(file_path, product_kind)?; - let mut mgen = handle.ref_message_generator(); + let mut mgen = handle.ref_message_iter(); assert!(mgen.next()?.is_some()); assert!(mgen.next()?.is_some()); @@ -231,7 +231,7 @@ mod tests { // First, filter and collect the messages to get those that we want let mut level = vec![]; - while let Some(msg) = handle.ref_message_generator().next()? { + while let Some(msg) = handle.ref_message_iter().next()? { if msg.read_key_dynamic("shortName")? == DynamicKeyType::Str("msl".to_string()) && msg.read_key_dynamic("typeOfLevel")? == DynamicKeyType::Str("surface".to_string()) @@ -264,7 +264,7 @@ mod tests { let product_kind = ProductKind::GRIB; let handle = CodesFile::new_from_file(file_path, product_kind)?; - let mut mgen = handle.arc_message_generator(); + let mut mgen = handle.arc_message_iter(); // let _ = handle.atomic_message_generator(); <- not allowed due to ownership let barrier = Arc::new(Barrier::new(10)); diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index fd6a556..d9b9d1f 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -15,7 +15,7 @@ use std::{ }; use tracing::instrument; -pub use iterator::{ArcMessageGenerator, RefMessageGenerator}; +pub use iterator::{ArcMessageIter, RefMessageIter}; mod iterator; @@ -426,13 +426,13 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let _ref_msg = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("no message")?; let mut clone_msg = _ref_msg.try_clone()?; drop(_ref_msg); let _oth_ref = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("no message")?; diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index 3c35454..3e6016e 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -168,7 +168,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; @@ -190,7 +190,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; let cloned_message = current_message.try_clone()?; @@ -209,7 +209,7 @@ mod tests { let product_kind = ProductKind::GRIB; let mut handle = CodesFile::new_from_file(file_path, product_kind)?; - let mut mgen = handle.ref_message_generator(); + let mut mgen = handle.ref_message_iter(); let msg = mgen.next()?.context("Message not some")?.try_clone()?; let _ = mgen.next()?; @@ -232,7 +232,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let _msg_ref = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; let _msg_clone = _msg_ref.try_clone()?; diff --git a/src/codes_message/read.rs b/src/codes_message/read.rs index e24236d..7402b5e 100644 --- a/src/codes_message/read.rs +++ b/src/codes_message/read.rs @@ -340,7 +340,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; @@ -381,7 +381,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let mut current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; let key_names = current_message @@ -403,7 +403,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let mut current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; let key_names = current_message @@ -425,7 +425,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; @@ -444,7 +444,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let msg = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; diff --git a/src/codes_message/write.rs b/src/codes_message/write.rs index 2cc3d8a..1a283b8 100644 --- a/src/codes_message/write.rs +++ b/src/codes_message/write.rs @@ -142,7 +142,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; let out_path = Path::new("./data/iceland_write.grib"); @@ -160,7 +160,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")? .try_clone()?; @@ -183,7 +183,7 @@ mod tests { let file_path = Path::new("./data/iceland-surface.grib"); let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; current_message.write_to_file(out_path, false)?; @@ -191,7 +191,7 @@ mod tests { let file_path = Path::new("./data/iceland-levels.grib"); let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; current_message.write_to_file(out_path, true)?; @@ -208,7 +208,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let mut current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; @@ -231,7 +231,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let mut current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; @@ -245,7 +245,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; diff --git a/src/codes_nearest.rs b/src/codes_nearest.rs index 5e03c72..8b14b1f 100644 --- a/src/codes_nearest.rs +++ b/src/codes_nearest.rs @@ -140,7 +140,7 @@ mod tests { let mut handle1 = CodesFile::new_from_file(file_path1, product_kind)?; let msg1 = handle1 - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; let mut nrst1 = msg1.codes_nearest()?; @@ -148,7 +148,7 @@ mod tests { let mut handle2 = CodesFile::new_from_file(file_path2, product_kind)?; let msg2 = handle2 - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; let mut nrst2 = msg2.codes_nearest()?; @@ -170,7 +170,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index de0c343..9d1d93d 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -234,7 +234,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let mut current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; @@ -262,7 +262,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let mut current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; @@ -288,7 +288,7 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; let mut current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; diff --git a/src/message_ndarray.rs b/src/message_ndarray.rs index d7db30c..06fe47a 100644 --- a/src/message_ndarray.rs +++ b/src/message_ndarray.rs @@ -155,7 +155,7 @@ mod tests { let file_path = Path::new("./data/iceland-surface.grib"); let mut handle = CodesFile::new_from_file(file_path, ProductKind::GRIB)?; - while let Some(msg) = handle.ref_message_generator().next()? { + while let Some(msg) = handle.ref_message_iter().next()? { if msg.read_key_dynamic("shortName")? == DynamicKeyType::Str("2d".to_string()) { let ndarray = msg.to_ndarray()?; @@ -181,7 +181,7 @@ mod tests { let file_path = Path::new("./data/iceland-surface.grib"); let mut handle = CodesFile::new_from_file(file_path, ProductKind::GRIB)?; - while let Some(msg) = handle.ref_message_generator().next()? { + while let Some(msg) = handle.ref_message_iter().next()? { if msg.read_key_dynamic("shortName")? == DynamicKeyType::Str("2d".to_string()) { let rmsg = msg.to_lons_lats_values()?; diff --git a/tests/handle.rs b/tests/handle.rs index ed4087d..99a6128 100644 --- a/tests/handle.rs +++ b/tests/handle.rs @@ -16,7 +16,7 @@ fn thread_safety_core() -> Result<()> { let mut handle = CodesFile::new_from_file(file_path, ProductKind::GRIB)?; let current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; @@ -38,7 +38,7 @@ fn thread_safety_core() -> Result<()> { let mut handle = CodesFile::new_from_file(file_path, ProductKind::GRIB)?; let current_message = handle - .ref_message_generator() + .ref_message_iter() .next()? .context("Message not some")?; From 997ef79ba94c0c0f8d187c89d08f1f3e3a0d0dba Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 5 Jan 2026 13:42:43 +0100 Subject: [PATCH 34/65] some minor stuff --- src/codes_handle/iterator.rs | 17 +++++++---------- src/codes_handle/mod.rs | 2 +- src/codes_message/clone.rs | 31 +++++++++++++++++++++++++++++++ src/codes_message/mod.rs | 4 ++-- 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index f8acc46..f2619d1 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -32,7 +32,7 @@ impl<'ch, S: HandleGenerator> FallibleIterator for RefMessageIter<'ch, S> { if new_eccodes_handle.is_null() { Ok(None) } else { - Ok(Some(RefMessage::new_from_gen(new_eccodes_handle))) + Ok(Some(RefMessage::new(new_eccodes_handle))) } } } @@ -60,7 +60,7 @@ impl FallibleIterator for ArcMessageIter { if new_eccodes_handle.is_null() { Ok(None) } else { - Ok(Some(ArcMessage::new_from_gen( + Ok(Some(ArcMessage::new( new_eccodes_handle, &self.codes_handle, ))) @@ -76,7 +76,10 @@ mod tests { codes_message::DynamicKeyType, }; use anyhow::{Context, Ok, Result}; - use std::{path::Path, sync::{Arc, Barrier}}; + use std::{ + path::Path, + sync::{Arc, Barrier}, + }; #[test] fn iterator_lifetimes() -> Result<()> { @@ -278,13 +281,7 @@ mod tests { let t = std::thread::spawn(move || { for _ in 0..1000 { b.wait(); - let _ = unsafe { - crate::intermediate_bindings::codes_get_size( - msg.message_handle, - "shortName", - ) - .unwrap() - }; + let _ = msg.read_key_dynamic("shortName").unwrap(); } }); diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index d9b9d1f..6240867 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -20,7 +20,7 @@ pub use iterator::{ArcMessageIter, RefMessageIter}; mod iterator; /// This is an internal structure used to access provided file by `CodesHandle`. -/// It also allows to differentiate between `CodesHandle` created from file and from index. +/// It also allows to differentiate between `CodesFile` created from file and from index. /// It is not intended to be used directly by the user. #[doc(hidden)] #[derive(Debug)] diff --git a/src/codes_message/clone.rs b/src/codes_message/clone.rs index 9931b1b..630011a 100644 --- a/src/codes_message/clone.rs +++ b/src/codes_message/clone.rs @@ -18,3 +18,34 @@ impl CodesMessage

{ })) } } + +#[cfg(test)] +mod tests { + use std::path::Path; + + use anyhow::{Context, Result}; + use fallible_iterator::FallibleIterator; + + use crate::{CodesFile, ProductKind}; + + #[test] + fn check_clone_safety() -> Result<()> { + let file_path = Path::new("./data/iceland-levels.grib"); + let product_kind = ProductKind::GRIB; + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + + let msg1 = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + let key1 = msg1.read_key_dynamic("typeOfLevel")?; + + let msg_clone = msg1.try_clone()?; + drop(msg1); + drop(handle); + let key1_clone = msg_clone.read_key_dynamic("typeOfLevel")?; + assert_eq!(key1, key1_clone); + + Ok(()) + } +} diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index 3e6016e..dd2fba9 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -90,7 +90,7 @@ pub struct ArcParent { } impl RefMessage<'_> { - pub(crate) fn new_from_gen(handle: *mut codes_handle) -> Self { + pub(crate) fn new(handle: *mut codes_handle) -> Self { RefMessage { _parent: RefParent(PhantomData), message_handle: handle, @@ -99,7 +99,7 @@ impl RefMessage<'_> { } impl ArcMessage { - pub(crate) fn new_from_gen(handle: *mut codes_handle, parent: &Arc>) -> Self { + pub(crate) fn new(handle: *mut codes_handle, parent: &Arc>) -> Self { ArcMessage { _parent: ArcParent { _arc_handle: parent.clone(), From 304b452dcd148147a53f326df05e1be47e3c1fbf Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 5 Jan 2026 14:45:54 +0100 Subject: [PATCH 35/65] minor comment --- tests/handle.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/handle.rs b/tests/handle.rs index 99a6128..191fca5 100644 --- a/tests/handle.rs +++ b/tests/handle.rs @@ -5,7 +5,7 @@ use eccodes::{CodesFile, FallibleIterator, ProductKind, codes_message::DynamicKe #[test] fn thread_safety() { - // errors are fine + // errors are fine, segfaults are not thread_safety_core().unwrap_or(()); } From c304930d53ff5a25e4cc2b45dbdce2dc2ac74014 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 5 Jan 2026 15:12:49 +0100 Subject: [PATCH 36/65] fix: gen_codes_handle() should take &mut self as argument, because it operates on mutable pointer. That requires us to introduce Mutex into ArcMessage, but it is unused, so the performance should be basically unaffected. --- src/codes_handle/iterator.rs | 14 ++++++++++---- src/codes_handle/mod.rs | 4 ++-- src/codes_message/mod.rs | 6 +++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index f2619d1..ca26e98 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -5,7 +5,7 @@ use crate::{ codes_handle::{HandleGenerator, ThreadSafeHandle}, errors::CodesError, }; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; #[derive(Debug)] pub struct RefMessageIter<'a, S: HandleGenerator> { @@ -39,12 +39,12 @@ impl<'ch, S: HandleGenerator> FallibleIterator for RefMessageIter<'ch, S> { #[derive(Debug)] pub struct ArcMessageIter { - codes_handle: Arc>, + codes_handle: Arc>>, } impl CodesFile { pub fn arc_message_iter(self) -> ArcMessageIter { ArcMessageIter { - codes_handle: Arc::new(self), + codes_handle: Arc::new(Mutex::new(self)), } } } @@ -55,7 +55,13 @@ impl FallibleIterator for ArcMessageIter { type Error = CodesError; fn next(&mut self) -> Result, Self::Error> { - let new_eccodes_handle = self.codes_handle.source.gen_codes_handle()?; + let new_eccodes_handle = self + .codes_handle + .lock() + // This mutex can be poisoned only when thread that holds ArcMessageIter panics, which would make using the mutex impossible") + .unwrap() + .source + .gen_codes_handle()?; if new_eccodes_handle.is_null() { Ok(None) diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index 6240867..5a18b7a 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -41,11 +41,11 @@ impl ThreadSafeHandle for CodesFileSource {} /// Internal trait implemented for types that can be called to generate `*mut codes_handle`. #[doc(hidden)] pub trait HandleGenerator: Debug { - fn gen_codes_handle(&self) -> Result<*mut codes_handle, CodesError>; + fn gen_codes_handle(&mut self) -> Result<*mut codes_handle, CodesError>; } impl HandleGenerator for CodesFileSource { - fn gen_codes_handle(&self) -> Result<*mut codes_handle, CodesError> { + fn gen_codes_handle(&mut self) -> Result<*mut codes_handle, CodesError> { unsafe { codes_handle_new_from_file(self.pointer, self.product_kind) } } } diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index dd2fba9..67ea609 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -11,7 +11,7 @@ pub use read::{DynamicKeyType, KeyPropertiesRead, KeyRead}; pub use write::KeyWrite; use eccodes_sys::codes_handle; -use std::{fmt::Debug, hash::Hash, marker::PhantomData, ptr::null_mut, sync::Arc}; +use std::{fmt::Debug, hash::Hash, marker::PhantomData, ptr::null_mut, sync::{Arc, Mutex}}; use tracing::{Level, event, instrument}; use crate::{ @@ -86,7 +86,7 @@ pub struct BufParent(); #[derive(Debug)] #[doc(hidden)] pub struct ArcParent { - _arc_handle: Arc>, + _arc_handle: Arc>>, } impl RefMessage<'_> { @@ -99,7 +99,7 @@ impl RefMessage<'_> { } impl ArcMessage { - pub(crate) fn new(handle: *mut codes_handle, parent: &Arc>) -> Self { + pub(crate) fn new(handle: *mut codes_handle, parent: &Arc>>) -> Self { ArcMessage { _parent: ArcParent { _arc_handle: parent.clone(), From 03ed81a1ced9789194d3229ad8b9d23704be05b5 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 5 Jan 2026 15:16:15 +0100 Subject: [PATCH 37/65] allow writing only for BufMessages to not suggest persistent grib file edition --- src/codes_message/write.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/codes_message/write.rs b/src/codes_message/write.rs index 1a283b8..ca8ba02 100644 --- a/src/codes_message/write.rs +++ b/src/codes_message/write.rs @@ -1,7 +1,7 @@ use std::{fmt::Debug, fs::OpenOptions, io::Write, path::Path, slice}; use crate::{ - codes_message::CodesMessage, + codes_message::{BufMessage, CodesMessage}, errors::CodesError, intermediate_bindings::{ codes_get_message, codes_set_bytes, codes_set_double, codes_set_double_array, @@ -46,7 +46,7 @@ pub trait KeyWrite { macro_rules! impl_key_write { ($ec_func:ident, $gen_type:ty) => { - impl KeyWrite<$gen_type> for CodesMessage

{ + impl KeyWrite<$gen_type> for BufMessage { fn write_key_unchecked( &mut self, name: &str, @@ -207,14 +207,16 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let mut handle = CodesFile::new_from_file(file_path, product_kind)?; - let mut current_message = handle + let current_message = handle .ref_message_iter() .next()? .context("Message not some")?; let old_key = current_message.read_key_dynamic("centre")?; - current_message.write_key_unchecked("centre", "cnmc")?; + current_message + .try_clone()? + .write_key_unchecked("centre", "cnmc")?; let read_key = current_message.read_key_dynamic("centre")?; @@ -230,14 +232,16 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let mut handle = CodesFile::new_from_file(file_path, product_kind)?; - let mut current_message = handle + let current_message = handle .ref_message_iter() .next()? .context("Message not some")?; let old_key = current_message.read_key_dynamic("centre")?; - current_message.write_key_unchecked("centre", "cnmc")?; + current_message + .try_clone()? + .write_key_unchecked("centre", "cnmc")?; current_message.write_to_file(Path::new("./data/iceland_edit.grib"), false)?; From 6ad607d041957358a5b2e7a8f3395d6a9b4c1352 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Thu, 8 Jan 2026 10:06:20 +0100 Subject: [PATCH 38/65] Remove experimental indexing feature With new message implementation users can fairly easily create their own indexes and this removal allows a significant simiplifcation of code and overall improvement of memory safety --- Cargo.toml | 3 +- src/codes_handle/mod.rs | 130 +---------- src/intermediate_bindings/codes_handle.rs | 27 --- src/intermediate_bindings/codes_index.rs | 135 ------------ src/intermediate_bindings/mod.rs | 9 - src/lib.rs | 8 +- tests/index.rs | 252 ---------------------- 7 files changed, 4 insertions(+), 560 deletions(-) delete mode 100644 src/intermediate_bindings/codes_index.rs delete mode 100644 tests/index.rs diff --git a/Cargo.toml b/Cargo.toml index 754f78e..dc97a48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,11 +45,10 @@ float-cmp = "0.10" [features] default = ["message_ndarray"] docs = ["eccodes-sys/docs"] -experimental_index = [] message_ndarray = ["dep:ndarray"] [package.metadata.docs.rs] -features = ["docs", "experimental_index", "message_ndarray"] +features = ["docs", "message_ndarray"] [[bench]] name = "main" diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index 5a18b7a..d2b6ec1 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -1,8 +1,6 @@ //! Definition and constructors of `CodesHandle` //! used for accessing GRIB files -#[cfg(feature = "experimental_index")] -use crate::codes_index::CodesIndex; use crate::{CodesError, intermediate_bindings::codes_handle_new_from_file, pointer_guard}; use eccodes_sys::{ProductKind_PRODUCT_GRIB, codes_handle}; use errno::errno; @@ -250,45 +248,6 @@ impl CodesFile>> { } } -#[cfg(feature = "experimental_index")] -#[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] -impl CodesFile { - /// Creates [`CodesHandle`] for provided [`CodesIndex`]. - /// - /// ## Example - /// - /// ``` - /// # fn run() -> anyhow::Result<()> { - /// # use eccodes::{CodesHandle, CodesIndex}; - /// # - /// let index = CodesIndex::new_from_keys(&vec!["shortName", "typeOfLevel", "level"])?; - /// let handle = CodesHandle::new_from_index(index)?; - /// - /// Ok(()) - /// # } - /// ``` - /// - /// The function takes ownership of the provided [`CodesIndex`] which owns - /// the GRIB data. [`CodesHandle`] created from [`CodesIndex`] is of different type - /// than the one created from file or memory buffer, because it internally uses - /// different functions to access messages. But it can be used in the same way. - /// - /// ⚠️ Warning: This function may interfere with other functions in concurrent context, - /// due to ecCodes issues with thread-safety for indexes. More information can be found - /// in [`codes_index`](crate::codes_index) module documentation. - /// - /// ## Errors - /// - /// Returns [`CodesError::Internal`] with error code - /// when internal [`codes_handle`] cannot be created. - #[instrument(level = "trace")] - pub fn new_from_index(index: CodesIndex) -> Result { - let new_handle = CodesFile { source: index }; - - Ok(new_handle) - } -} - #[instrument(level = "trace")] fn open_with_fdopen(file: &File) -> Result<*mut FILE, CodesError> { let file_ptr = unsafe { libc::fdopen(file.as_raw_fd(), "r".as_ptr().cast::<_>()) }; @@ -328,8 +287,6 @@ fn open_with_fmemopen(file_data: &[u8]) -> Result<*mut FILE, CodesError> { #[cfg(test)] mod tests { use crate::codes_handle::{CodesFile, ProductKind}; - #[cfg(feature = "experimental_index")] - use crate::codes_index::{CodesIndex, Select}; use anyhow::{Context, Result}; use eccodes_sys::ProductKind_PRODUCT_GRIB; use fallible_iterator::FallibleIterator; @@ -371,27 +328,6 @@ mod tests { Ok(()) } - #[test] - #[cfg(feature = "experimental_index")] - fn index_constructor_and_destructor() -> Result<()> { - use anyhow::Ok; - - let file_path = Path::new("./data/iceland-surface.grib.idx"); - let index = CodesIndex::read_from_file(file_path)? - .select("shortName", "2t")? - .select("typeOfLevel", "surface")? - .select("level", 0)? - .select("stepType", "instant")?; - - let i_ptr = index.pointer.clone(); - - let handle = CodesFile::new_from_index(index)?; - - assert_eq!(handle.source.pointer, i_ptr); - - Ok(()) - } - #[test] fn codes_handle_drop_file() -> Result<()> { let file_path = Path::new("./data/iceland-surface.grib"); @@ -425,72 +361,10 @@ mod tests { let mut handle = CodesFile::new_from_file(file_path, product_kind)?; - let _ref_msg = handle - .ref_message_iter() - .next()? - .context("no message")?; + let _ref_msg = handle.ref_message_iter().next()?.context("no message")?; let mut clone_msg = _ref_msg.try_clone()?; drop(_ref_msg); - let _oth_ref = handle - .ref_message_iter() - .next()? - .context("no message")?; - - let _nrst = clone_msg.codes_nearest()?; - drop(_nrst); - let _kiter = clone_msg.default_keys_iterator()?; - } - - Ok(()) - } - - #[test] - #[cfg(feature = "experimental_index")] - fn codes_handle_drop_index() -> Result<()> { - let file_path = Path::new("./data/iceland-surface.grib.idx"); - let index = CodesIndex::read_from_file(file_path)?; - assert!(!index.pointer.is_null()); - - let handle = CodesFile::new_from_index(index)?; - drop(handle); - - Ok(()) - } - - #[test] - #[cfg(feature = "experimental_index")] - fn empty_index_constructor() -> Result<()> { - let index = - CodesIndex::new_from_keys(&vec!["shortName", "typeOfLevel", "level", "stepType"])?; - - let mut handle = CodesFile::new_from_index(index)?; - - assert!(!handle.source.pointer.is_null()); - - let msg = handle.ref_message_generator().next()?; - - assert!(!msg.is_some()); - - Ok(()) - } - - #[test] - #[cfg(feature = "experimental_index")] - fn multiple_drops_with_index() -> Result<()> { - { - let keys = vec!["typeOfLevel", "level"]; - let index = CodesIndex::new_from_keys(&keys)?; - let grib_path = Path::new("./data/iceland-levels.grib"); - let index = index - .add_grib_file(grib_path)? - .select("typeOfLevel", "isobaricInhPa")? - .select("level", 600)?; - - let mut handle = CodesFile::new_from_index(index)?; - let mut mgen = handle.ref_message_generator(); - let _ref_msg = mgen.next()?.context("no message")?; - let mut clone_msg = _ref_msg.try_clone()?; - let _oth_ref = mgen.next()?.context("no message")?; + let _oth_ref = handle.ref_message_iter().next()?.context("no message")?; let _nrst = clone_msg.codes_nearest()?; drop(_nrst); diff --git a/src/intermediate_bindings/codes_handle.rs b/src/intermediate_bindings/codes_handle.rs index 12cddba..7034a20 100644 --- a/src/intermediate_bindings/codes_handle.rs +++ b/src/intermediate_bindings/codes_handle.rs @@ -3,8 +3,6 @@ use std::ptr::{self}; -#[cfg(feature = "experimental_index")] -use eccodes_sys::{CODES_LOCK, codes_index}; use eccodes_sys::{codes_context, codes_handle}; use libc::FILE; use num_traits::FromPrimitive; @@ -68,31 +66,6 @@ pub unsafe fn codes_handle_delete(handle: *mut codes_handle) -> Result<(), Codes } } -#[cfg(feature = "experimental_index")] -#[instrument(level = "trace")] -pub unsafe fn codes_handle_new_from_index( - index: *mut codes_index, -) -> Result<*mut codes_handle, CodesError> { - pointer_guard::non_null!(index); - - let mut error_code: i32 = 0; - - let _g = CODES_LOCK.lock().unwrap(); - let codes_handle = unsafe { eccodes_sys::codes_handle_new_from_index(index, &mut error_code) }; - - // special case! codes_handle_new_from_index returns -43 when there are no messages left in the index - // this is also indicated by a null pointer, which is handled upstream - if error_code == -43 { - return Ok(codes_handle); - } - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } - Ok(codes_handle) -} - pub unsafe fn codes_handle_clone( source_handle: *const codes_handle, ) -> Result<*mut codes_handle, CodesError> { diff --git a/src/intermediate_bindings/codes_index.rs b/src/intermediate_bindings/codes_index.rs deleted file mode 100644 index 8087993..0000000 --- a/src/intermediate_bindings/codes_index.rs +++ /dev/null @@ -1,135 +0,0 @@ -#![allow(non_camel_case_types)] -#![allow(clippy::module_name_repetitions)] - -use eccodes_sys::{CODES_LOCK, codes_context, codes_index}; -use num_traits::FromPrimitive; -use std::{ffi::CString, ptr}; -use tracing::instrument; - -use crate::{ - errors::{CodesError, CodesInternal}, - pointer_guard, -}; - -// all index functions are safeguarded by a lock -// because there are random errors appearing when using the index functions concurrently - -#[instrument(level = "trace")] -pub unsafe fn codes_index_new(keys: &str) -> Result<*mut codes_index, CodesError> { - let context: *mut codes_context = ptr::null_mut(); //default context - let mut error_code: i32 = 0; - let keys = CString::new(keys).unwrap(); - - let _g = CODES_LOCK.lock().unwrap(); - let codes_index = - unsafe { eccodes_sys::codes_index_new(context, keys.as_ptr(), &mut error_code) }; - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } - Ok(codes_index) -} - -pub unsafe fn codes_index_read(filename: &str) -> Result<*mut codes_index, CodesError> { - let filename = CString::new(filename).unwrap(); - let context: *mut codes_context = ptr::null_mut(); //default context - let mut error_code: i32 = 0; - - let _g = CODES_LOCK.lock().unwrap(); - let codes_index = - unsafe { eccodes_sys::codes_index_read(context, filename.as_ptr(), &mut error_code) }; - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } - Ok(codes_index) -} - -#[instrument(level = "trace")] -pub unsafe fn codes_index_delete(index: *mut codes_index) { - if index.is_null() { - return; - } - - let _g = CODES_LOCK.lock().unwrap(); - unsafe { eccodes_sys::codes_index_delete(index) }; -} - -pub unsafe fn codes_index_add_file( - index: *mut codes_index, - filename: &str, -) -> Result<(), CodesError> { - pointer_guard::non_null!(index); - - let filename = CString::new(filename).unwrap(); - - let _g = CODES_LOCK.lock().unwrap(); - let error_code = unsafe { eccodes_sys::codes_index_add_file(index, filename.as_ptr()) }; - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } - Ok(()) -} - -pub unsafe fn codes_index_select_long( - index: *mut codes_index, - key: &str, - value: i64, -) -> Result<(), CodesError> { - pointer_guard::non_null!(index); - - let key = CString::new(key).unwrap(); - - let _g = CODES_LOCK.lock().unwrap(); - let error_code = unsafe { eccodes_sys::codes_index_select_long(index, key.as_ptr(), value) }; - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } - Ok(()) -} - -pub unsafe fn codes_index_select_double( - index: *mut codes_index, - key: &str, - value: f64, -) -> Result<(), CodesError> { - pointer_guard::non_null!(index); - - let key = CString::new(key).unwrap(); - - let _g = CODES_LOCK.lock().unwrap(); - let error_code = unsafe { eccodes_sys::codes_index_select_double(index, key.as_ptr(), value) }; - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } - Ok(()) -} - -pub unsafe fn codes_index_select_string( - index: *mut codes_index, - key: &str, - value: &str, -) -> Result<(), CodesError> { - pointer_guard::non_null!(index); - - let key = CString::new(key).unwrap(); - let value = CString::new(value).unwrap(); - - let _g = CODES_LOCK.lock().unwrap(); - let error_code = - unsafe { eccodes_sys::codes_index_select_string(index, key.as_ptr(), value.as_ptr()) }; - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } - Ok(()) -} diff --git a/src/intermediate_bindings/mod.rs b/src/intermediate_bindings/mod.rs index a90e336..e0ef58f 100644 --- a/src/intermediate_bindings/mod.rs +++ b/src/intermediate_bindings/mod.rs @@ -9,8 +9,6 @@ mod codes_get; mod codes_handle; -#[cfg(feature = "experimental_index")] -mod codes_index; mod codes_keys; mod codes_set; mod grib_nearest; @@ -32,14 +30,7 @@ pub use codes_get::{ codes_get_long_array, codes_get_message, codes_get_native_type, codes_get_size, codes_get_string, }; -#[cfg(feature = "experimental_index")] -pub use codes_handle::codes_handle_new_from_index; pub use codes_handle::{codes_handle_clone, codes_handle_delete, codes_handle_new_from_file}; -#[cfg(feature = "experimental_index")] -pub use codes_index::{ - codes_index_add_file, codes_index_delete, codes_index_new, codes_index_read, - codes_index_select_double, codes_index_select_long, codes_index_select_string, -}; pub use codes_keys::{ codes_keys_iterator_delete, codes_keys_iterator_get_name, codes_keys_iterator_new, codes_keys_iterator_next, diff --git a/src/lib.rs b/src/lib.rs index 773834d..3dbcc6b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -228,9 +228,6 @@ //! ``` pub mod codes_handle; -#[cfg(feature = "experimental_index")] -#[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] -pub mod codes_index; pub mod codes_message; pub mod codes_nearest; pub mod errors; @@ -242,10 +239,7 @@ mod message_ndarray; mod pointer_guard; pub use codes_handle::{CodesFile, ProductKind}; -#[cfg(feature = "experimental_index")] -#[cfg_attr(docsrs, doc(cfg(feature = "experimental_index")))] -pub use codes_index::CodesIndex; -pub use codes_message::{ArcMessage, BufMessage, RefMessage, KeyRead, KeyWrite}; +pub use codes_message::{ArcMessage, BufMessage, KeyRead, KeyWrite, RefMessage}; pub use codes_nearest::{CodesNearest, NearestGridpoint}; pub use errors::CodesError; pub use fallible_iterator::{FallibleIterator, IntoFallibleIterator}; diff --git a/tests/index.rs b/tests/index.rs deleted file mode 100644 index b201d0f..0000000 --- a/tests/index.rs +++ /dev/null @@ -1,252 +0,0 @@ -#![cfg(feature = "experimental_index")] - -use std::{ - path::Path, - sync::{Arc, Barrier}, - thread, -}; - -use anyhow::{Context, Result}; -use eccodes::{ - CodesError, CodesFile, CodesIndex, FallibleIterator, KeyRead, ProductKind, - codes_index::Select, -}; -use rand::Rng; - -#[test] -fn iterate_handle_from_index() -> Result<()> { - let file_path = Path::new("./data/iceland-surface.grib.idx"); - let index = CodesIndex::read_from_file(file_path)? - .select("shortName", "2t")? - .select("typeOfLevel", "surface")? - .select("level", 0)? - .select("stepType", "instant")?; - - let mut handle = CodesHandle::new_from_index(index)?; - - let counter = handle.ref_message_generator().count()?; - - assert_eq!(counter, 1); - - Ok(()) -} - -#[test] -fn read_index_messages() -> Result<()> { - let file_path = Path::new("./data/iceland-surface.grib.idx"); - let index = CodesIndex::read_from_file(file_path)? - .select("shortName", "2t")? - .select("typeOfLevel", "surface")? - .select("level", 0)? - .select("stepType", "instant")?; - - let mut handle = CodesHandle::new_from_index(index)?; - let current_message = handle - .ref_message_generator() - .next()? - .context("Message not some")?; - - { - let short_name: String = current_message.read_key("shortName")?; - assert_eq!(short_name, "2t"); - } - { - let level: i64 = current_message.read_key("level")?; - assert_eq!(level, 0); - } - - Ok(()) -} - -#[test] -fn collect_index_iterator() -> Result<()> { - let keys = vec!["typeOfLevel", "level"]; - let index = CodesIndex::new_from_keys(&keys)?; - let grib_path = Path::new("./data/iceland-levels.grib"); - - let index = index - .add_grib_file(grib_path)? - .select("typeOfLevel", "isobaricInhPa")? - .select("level", 700)?; - - let mut handle = CodesHandle::new_from_index(index)?; - - let levels = handle - .ref_message_generator() - .map(|msg| msg.try_clone()) - .collect::>()?; - - assert_eq!(levels.len(), 5); - - Ok(()) -} - -#[test] -fn add_file_error() -> Result<()> { - thread::spawn(|| -> Result<()> { - let grib_path = Path::new("./data/iceland-levels.grib"); - let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; - let mut index_op = CodesIndex::new_from_keys(&keys)?; - - loop { - index_op = index_op.add_grib_file(grib_path)?; - } - }); - - thread::sleep(std::time::Duration::from_millis(250)); - - let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; - let wrong_path = Path::new("./data/xxx.grib"); - let index = CodesIndex::new_from_keys(&keys)?.add_grib_file(wrong_path); - - assert!(index.is_err()); - - Ok(()) -} - -#[test] -fn index_panic() -> Result<()> { - thread::spawn(|| -> Result<()> { - let grib_path = Path::new("./data/iceland-levels.grib"); - let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; - let mut index_op = CodesIndex::new_from_keys(&keys)?; - - loop { - index_op = index_op.add_grib_file(grib_path)?; - } - }); - - thread::sleep(std::time::Duration::from_millis(250)); - - let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; - let wrong_path = Path::new("./data/xxx.grib"); - let index = CodesIndex::new_from_keys(&keys)?; - - let result = std::panic::catch_unwind(|| index.add_grib_file(wrong_path).unwrap()); - - assert!(result.is_err()); - - Ok(()) -} - -#[test] -#[ignore = "for releases, indexing is experimental"] -fn add_file_while_index_open() -> Result<()> { - thread::spawn(|| -> Result<()> { - let file_path = Path::new("./data/iceland-surface.grib.idx"); - let mut index_op = CodesIndex::read_from_file(file_path)?; - - loop { - index_op = index_op - .select("shortName", "2t")? - .select("typeOfLevel", "surface")? - .select("level", 0)? - .select("stepType", "instant")?; - } - }); - - let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; - let grib_path = Path::new("./data/iceland-surface.grib"); - let index = CodesIndex::new_from_keys(&keys)?.add_grib_file(grib_path); - - assert!(index.is_ok()); - - Ok(()) -} - -#[test] -#[ignore = "for releases, indexing is experimental"] -fn add_file_to_read_index() -> Result<()> { - let file_path = Path::new("./data/iceland-surface.grib.idx"); - let grib_path = Path::new("./data/iceland-surface.grib"); - - let _index = CodesIndex::read_from_file(file_path)? - .add_grib_file(grib_path)? - .select("shortName", "2t")? - .select("typeOfLevel", "surface")? - .select("level", 0)? - .select("stepType", "instant")?; - - Ok(()) -} - -#[test] -#[ignore = "for releases, indexing is experimental"] -fn simulatenous_index_destructors() -> Result<()> { - let barrier = Arc::new(Barrier::new(2)); - let b1 = barrier.clone(); - let b2 = barrier.clone(); - - let h1 = thread::spawn(move || -> anyhow::Result<(), CodesError> { - let file_path = Path::new("./data/iceland-surface.grib.idx"); - - for _ in 0..100 { - let index_op = CodesIndex::read_from_file(file_path)? - .select("shortName", "2t")? - .select("typeOfLevel", "surface")? - .select("level", 0)? - .select("stepType", "instant")?; - - b1.wait(); - drop(index_op); - } - - Ok(()) - }); - - let h2 = thread::spawn(move || -> anyhow::Result<(), CodesError> { - let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; - let grib_path = Path::new("./data/iceland-surface.grib"); - - for _ in 0..100 { - let index = CodesIndex::new_from_keys(&keys)? - .add_grib_file(grib_path)? - .select("shortName", "2t")? - .select("typeOfLevel", "surface")? - .select("level", 0)? - .select("stepType", "instant")?; - - b2.wait(); - drop(index); - } - - Ok(()) - }); - - // errors are fine - h1.join().unwrap().unwrap_or(()); - h2.join().unwrap().unwrap_or(()); - - Ok(()) -} - -#[test] -#[ignore = "for releases, indexing is experimental"] -fn index_handle_interference() -> Result<()> { - thread::spawn(|| -> Result<()> { - let file_path = Path::new("./data/iceland.grib"); - - loop { - let handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB); - - assert!(handle.is_ok()); - } - }); - - let mut rng = rand::rng(); - let keys = vec!["shortName", "typeOfLevel", "level", "stepType"]; - let grib_path = Path::new("./data/iceland.grib"); - - for _ in 0..10 { - let sleep_time = rng.random_range(1..42); // randomizing sleep time to hopefully catch segfaults - - let index = CodesIndex::new_from_keys(&keys)?.add_grib_file(grib_path)?; - let i_handle = CodesHandle::new_from_index(index); - - assert!(i_handle.is_ok()); - - thread::sleep(std::time::Duration::from_millis(sleep_time)); - } - - Ok(()) -} From 01cc62a4bef6baad5182d75dd3827708bdfaf26c Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Thu, 8 Jan 2026 10:18:33 +0100 Subject: [PATCH 39/65] Remove unnecessary files and code leftover from indexing feature --- data/iceland-levels-bad-path.grib.idx | Bin 1381 -> 0 bytes data/iceland-surface.grib.idx | Bin 399 -> 0 bytes src/codes_handle/iterator.rs | 53 ++++++++--------- src/codes_handle/mod.rs | 80 +++++++++----------------- src/codes_message/mod.rs | 26 +++++---- 5 files changed, 65 insertions(+), 94 deletions(-) delete mode 100644 data/iceland-levels-bad-path.grib.idx delete mode 100644 data/iceland-surface.grib.idx diff --git a/data/iceland-levels-bad-path.grib.idx b/data/iceland-levels-bad-path.grib.idx deleted file mode 100644 index cbef1ea63e2bb77c8cb0c47ee8f0adb4a09b66ab..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1381 zcmaKsze~eF6vr<_ZABE9f`fzLB7!v?)H+#2)Pg^XbSSu()@TDYO`B`{F^G%c;2^k& zlWyH~5*NY2L2z+&bae7xc;20lM4|U?cm(eAeeXU8m&DS>e14&n0Z64M-HKDQz3qvb z6FRjZz1wulj4|j9s`Vycv-cbmj3Y1_U~n*6V1x|%_@VEt@2tp#3H@$RFWXJGlJ}|w zn?bkgWPmwkSzu1fG9yb?mRV7{GL~gANCw>Ti=vYW3D*m_?Qt=eiDB%2OJJ%Mh-2aU z?!k}0K{ZN1W23*DogsM?0TmW0NFG%{2QCFk6#?Cx6OdAs)a#v=Qqm2*bTXlhikp%5 zEBagG6ok^HRSH7s6Q6=mx_wDNk@Wk6mQvDOFFiP*jmFa13{S diff --git a/data/iceland-surface.grib.idx b/data/iceland-surface.grib.idx deleted file mode 100644 index 2aae48060180a4cd90960b69ab9e79cdb91b6737..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 399 zcmajb-3r1W7zW^PNo^5CCqYmL-9Y>45+aBo3ZjEfZF4nnYfLxjrn`kNEu#ae1V!Y1 zUPX7cU#yqM2yyHO36(U+V#aBa%tT$KG-m#}%1!`)U2(Zqa!YeYkVIjvmj+w&L`{h# z7aXuBZzbEM8`dyR&@~l+ZLUuobV5K+NLC(HP6e_;NLonM4$n$>69gwjRM(nu@0iVp h7}TJJJogp;g7lLuGwA;@(l(>vPouxYM$^d;1P{RAb6@}f diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index ca26e98..56c10e5 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -1,20 +1,17 @@ use fallible_iterator::FallibleIterator; -use crate::{ - ArcMessage, CodesFile, RefMessage, - codes_handle::{HandleGenerator, ThreadSafeHandle}, - errors::CodesError, -}; +use crate::{ArcMessage, CodesFile, RefMessage, errors::CodesError}; +use std::fmt::Debug; use std::sync::{Arc, Mutex}; #[derive(Debug)] -pub struct RefMessageIter<'a, S: HandleGenerator> { - codes_handle: &'a mut CodesFile, +pub struct RefMessageIter<'a, D: Debug> { + codes_file: &'a mut CodesFile, } -impl CodesFile { - pub fn ref_message_iter<'a>(&'a mut self) -> RefMessageIter<'a, S> { - RefMessageIter { codes_handle: self } +impl CodesFile { + pub fn ref_message_iter<'a>(&'a mut self) -> RefMessageIter<'a, D> { + RefMessageIter { codes_file: self } } } @@ -22,54 +19,50 @@ impl CodesFile { /// /// The `next()` will return [`CodesInternal`](crate::errors::CodesInternal) /// when internal ecCodes function returns non-zero code. -impl<'ch, S: HandleGenerator> FallibleIterator for RefMessageIter<'ch, S> { +impl<'ch, D: Debug> FallibleIterator for RefMessageIter<'ch, D> { type Item = RefMessage<'ch>; type Error = CodesError; fn next(&mut self) -> Result, Self::Error> { - let new_eccodes_handle = self.codes_handle.source.gen_codes_handle()?; + let eccodes_handle = self.codes_file.generate_codes_handle()?; - if new_eccodes_handle.is_null() { + if eccodes_handle.is_null() { Ok(None) } else { - Ok(Some(RefMessage::new(new_eccodes_handle))) + Ok(Some(RefMessage::new(eccodes_handle))) } } } #[derive(Debug)] -pub struct ArcMessageIter { - codes_handle: Arc>>, +pub struct ArcMessageIter { + codes_file: Arc>>, } -impl CodesFile { - pub fn arc_message_iter(self) -> ArcMessageIter { +impl CodesFile { + pub fn arc_message_iter(self) -> ArcMessageIter { ArcMessageIter { - codes_handle: Arc::new(Mutex::new(self)), + codes_file: Arc::new(Mutex::new(self)), } } } -impl FallibleIterator for ArcMessageIter { - type Item = ArcMessage; +impl FallibleIterator for ArcMessageIter { + type Item = ArcMessage; type Error = CodesError; fn next(&mut self) -> Result, Self::Error> { - let new_eccodes_handle = self - .codes_handle + let eccodes_handle = self + .codes_file .lock() // This mutex can be poisoned only when thread that holds ArcMessageIter panics, which would make using the mutex impossible") .unwrap() - .source - .gen_codes_handle()?; + .generate_codes_handle()?; - if new_eccodes_handle.is_null() { + if eccodes_handle.is_null() { Ok(None) } else { - Ok(Some(ArcMessage::new( - new_eccodes_handle, - &self.codes_handle, - ))) + Ok(Some(ArcMessage::new(eccodes_handle, &self.codes_file))) } } } diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index d2b6ec1..8a8f1ea 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -17,37 +17,6 @@ pub use iterator::{ArcMessageIter, RefMessageIter}; mod iterator; -/// This is an internal structure used to access provided file by `CodesHandle`. -/// It also allows to differentiate between `CodesFile` created from file and from index. -/// It is not intended to be used directly by the user. -#[doc(hidden)] -#[derive(Debug)] -pub struct CodesFileSource { - // fields dropped from top - pointer: *mut FILE, - product_kind: ProductKind, - _data: D, -} - -/// Marker trait to differentiate between `CodesHandle` created from index and file/buffer. -#[doc(hidden)] -pub trait ThreadSafeHandle: HandleGenerator {} - -impl ThreadSafeHandle for CodesFileSource> {} -impl ThreadSafeHandle for CodesFileSource {} - -/// Internal trait implemented for types that can be called to generate `*mut codes_handle`. -#[doc(hidden)] -pub trait HandleGenerator: Debug { - fn gen_codes_handle(&mut self) -> Result<*mut codes_handle, CodesError>; -} - -impl HandleGenerator for CodesFileSource { - fn gen_codes_handle(&mut self) -> Result<*mut codes_handle, CodesError> { - unsafe { codes_handle_new_from_file(self.pointer, self.product_kind) } - } -} - /// Structure providing access to the GRIB file which takes a full ownership of the accessed file. /// /// It can be constructed from: @@ -117,8 +86,11 @@ impl HandleGenerator for CodesFileSource { /// /// All available methods for `CodesHandle` iterator can be found in [`FallibleStreamingIterator`](crate::FallibleStreamingIterator) trait. #[derive(Debug)] -pub struct CodesFile { - source: S, +pub struct CodesFile { + // fields are dropped from top + pointer: *mut FILE, + product_kind: ProductKind, + _data: D, } // 2024-07-26 @@ -135,6 +107,12 @@ pub struct CodesFile { // Clearing the memory is handled on ecCodes side by KeyedMessage/CodesIndex destructors // and on rust side by destructors of data_container we own. +impl CodesFile { + fn generate_codes_handle(&mut self) -> Result<*mut codes_handle, CodesError> { + unsafe { codes_handle_new_from_file(self.pointer, self.product_kind) } + } +} + /// Enum representing the kind of product (file type) inside handled file. /// Used to indicate to ecCodes how it should decode/encode messages. #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)] @@ -143,7 +121,7 @@ pub enum ProductKind { GRIB = ProductKind_PRODUCT_GRIB as isize, } -impl CodesFile> { +impl CodesFile { ///Opens file at given [`Path`] as selected [`ProductKind`] and contructs `CodesHandle`. /// ///## Example @@ -188,15 +166,13 @@ impl CodesFile> { let file_pointer = open_with_fdopen(&file)?; Ok(Self { - source: CodesFileSource { - _data: file, - pointer: file_pointer, - product_kind, - }, + _data: file, + pointer: file_pointer, + product_kind, }) } } -impl CodesFile>> { +impl CodesFile> { ///Opens data in provided buffer as selected [`ProductKind`] and contructs `CodesHandle`. /// ///## Example @@ -239,11 +215,9 @@ impl CodesFile>> { let file_pointer = open_with_fmemopen(&file_data)?; Ok(Self { - source: CodesFileSource { - _data: file_data, - product_kind, - pointer: file_pointer, - }, + _data: file_data, + product_kind, + pointer: file_pointer, }) } } @@ -297,14 +271,14 @@ mod tests { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; - let handle = CodesFile::new_from_file(file_path, product_kind)?; + let codes_file = CodesFile::new_from_file(file_path, product_kind)?; - assert!(!handle.source.pointer.is_null()); - assert_eq!(handle.source.product_kind as u32, { + assert!(!codes_file.pointer.is_null()); + assert_eq!(codes_file.product_kind as u32, { ProductKind_PRODUCT_GRIB }); - handle.source._data.metadata()?; + codes_file._data.metadata()?; Ok(()) } @@ -317,13 +291,13 @@ mod tests { let mut buf = Vec::new(); f.read_to_end(&mut buf)?; - let handle = CodesFile::new_from_memory(buf, product_kind)?; - assert!(!handle.source.pointer.is_null()); - assert_eq!(handle.source.product_kind as u32, { + let codes_file = CodesFile::new_from_memory(buf, product_kind)?; + assert!(!codes_file.pointer.is_null()); + assert_eq!(codes_file.product_kind as u32, { ProductKind_PRODUCT_GRIB }); - assert!(!handle.source._data.is_empty()); + assert!(!codes_file._data.is_empty()); Ok(()) } diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index 67ea609..78490d1 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -11,12 +11,16 @@ pub use read::{DynamicKeyType, KeyPropertiesRead, KeyRead}; pub use write::KeyWrite; use eccodes_sys::codes_handle; -use std::{fmt::Debug, hash::Hash, marker::PhantomData, ptr::null_mut, sync::{Arc, Mutex}}; +use std::{ + fmt::Debug, + hash::Hash, + marker::PhantomData, + ptr::null_mut, + sync::{Arc, Mutex}, +}; use tracing::{Level, event, instrument}; -use crate::{ - CodesFile, codes_handle::ThreadSafeHandle, intermediate_bindings::codes_handle_delete, -}; +use crate::{CodesFile, intermediate_bindings::codes_handle_delete}; /// Structure that provides access to the data contained in the GRIB file, which directly corresponds to the message in the GRIB file /// @@ -54,10 +58,10 @@ pub type RefMessage<'ch> = CodesMessage>; /// requiring `&mut self`. As `AtomicMessage` implements `Send + Sync` this exclusive method access is not /// guaranteed with just `&self`. `AtomicMessage` also implements a minimal subset of functionalities /// to limit the risk of some internal ecCodes functions not being thread-safe. -pub type ArcMessage = CodesMessage>; +pub type ArcMessage = CodesMessage>; -unsafe impl Send for ArcMessage {} -unsafe impl Sync for ArcMessage {} +unsafe impl Send for ArcMessage {} +unsafe impl Sync for ArcMessage {} pub type BufMessage = CodesMessage; @@ -85,8 +89,8 @@ pub struct BufParent(); #[derive(Debug)] #[doc(hidden)] -pub struct ArcParent { - _arc_handle: Arc>>, +pub struct ArcParent { + _arc_handle: Arc>>, } impl RefMessage<'_> { @@ -98,8 +102,8 @@ impl RefMessage<'_> { } } -impl ArcMessage { - pub(crate) fn new(handle: *mut codes_handle, parent: &Arc>>) -> Self { +impl ArcMessage { + pub(crate) fn new(handle: *mut codes_handle, parent: &Arc>>) -> Self { ArcMessage { _parent: ArcParent { _arc_handle: parent.clone(), From a315abc89b5b431109754440db24aa9b0e50a9b9 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Thu, 8 Jan 2026 10:44:33 +0100 Subject: [PATCH 40/65] Fix write tests and add better key write interface --- src/codes_message/write.rs | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/codes_message/write.rs b/src/codes_message/write.rs index ca8ba02..7d23296 100644 --- a/src/codes_message/write.rs +++ b/src/codes_message/write.rs @@ -41,7 +41,7 @@ pub trait KeyWrite { /// # Errors /// /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to write the key. - fn write_key_unchecked(&mut self, name: &str, value: T) -> Result<(), CodesError>; + fn write_key_unchecked(&mut self, name: &str, value: T) -> Result<&mut Self, CodesError>; } macro_rules! impl_key_write { @@ -51,8 +51,11 @@ macro_rules! impl_key_write { &mut self, name: &str, value: $gen_type, - ) -> Result<(), CodesError> { - unsafe { $ec_func(self.message_handle, name, value) } + ) -> Result<&mut Self, CodesError> { + unsafe { + $ec_func(self.message_handle, name, value)?; + } + Ok(self) } } }; @@ -214,11 +217,11 @@ mod tests { let old_key = current_message.read_key_dynamic("centre")?; - current_message - .try_clone()? - .write_key_unchecked("centre", "cnmc")?; + let mut cloned_message = current_message + .try_clone()?; + cloned_message.write_key_unchecked("centre", "cnmc")?; - let read_key = current_message.read_key_dynamic("centre")?; + let read_key = cloned_message.read_key_dynamic("centre")?; assert_ne!(old_key, read_key); assert_eq!(read_key, DynamicKeyType::Str("cnmc".into())); @@ -239,11 +242,11 @@ mod tests { let old_key = current_message.read_key_dynamic("centre")?; - current_message - .try_clone()? - .write_key_unchecked("centre", "cnmc")?; + let mut cloned_message = current_message + .try_clone()?; + cloned_message.write_key_unchecked("centre", "cnmc")?; - current_message.write_to_file(Path::new("./data/iceland_edit.grib"), false)?; + cloned_message.write_to_file(Path::new("./data/iceland_edit.grib"), false)?; let file_path = Path::new("./data/iceland_edit.grib"); From e86529bc1655c7021f5e19cf4e3eb8eafc0324fb Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Thu, 8 Jan 2026 12:00:44 +0100 Subject: [PATCH 41/65] Apply clippy fixes --- src/codes_handle/iterator.rs | 2 +- src/codes_message/mod.rs | 2 +- src/codes_nearest.rs | 4 ++-- src/intermediate_bindings/codes_get.rs | 22 +++++++++++----------- src/intermediate_bindings/codes_handle.rs | 2 +- src/intermediate_bindings/codes_set.rs | 4 ++-- src/intermediate_bindings/grib_nearest.rs | 4 ++-- src/keys_iterator.rs | 2 +- src/lib.rs | 8 ++++---- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index 56c10e5..e34525d 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -10,7 +10,7 @@ pub struct RefMessageIter<'a, D: Debug> { } impl CodesFile { - pub fn ref_message_iter<'a>(&'a mut self) -> RefMessageIter<'a, D> { + pub fn ref_message_iter(&mut self) -> RefMessageIter<'_, D> { RefMessageIter { codes_file: self } } } diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index 78490d1..6103b29 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -150,7 +150,7 @@ impl Drop for CodesMessage

{ "codes_handle_delete() returned an error: {:?}", &error ); - debug_assert!(false, "Error in KeyedMessage::drop") + debug_assert!(false, "Error in KeyedMessage::drop"); }); } diff --git a/src/codes_nearest.rs b/src/codes_nearest.rs index 8b14b1f..42140ba 100644 --- a/src/codes_nearest.rs +++ b/src/codes_nearest.rs @@ -46,7 +46,7 @@ impl CodesMessage

{ /// /// This function returns [`CodesInternal`](crate::errors::CodesInternal) when /// internal nearest handle cannot be created. - pub fn codes_nearest<'a>(&'a self) -> Result, CodesError> { + pub fn codes_nearest(&self) -> Result, CodesError> { let nearest_handle = unsafe { codes_grib_nearest_new(self.message_handle)? }; Ok(CodesNearest { @@ -115,7 +115,7 @@ impl Drop for CodesNearest<'_, P> { "codes_grib_nearest_delete() returned an error: {:?}", &error ); - debug_assert!(false, "Error in CodesNearest::drop") + debug_assert!(false, "Error in CodesNearest::drop"); }); } diff --git a/src/intermediate_bindings/codes_get.rs b/src/intermediate_bindings/codes_get.rs index dbeae42..3521233 100644 --- a/src/intermediate_bindings/codes_get.rs +++ b/src/intermediate_bindings/codes_get.rs @@ -24,7 +24,7 @@ pub unsafe fn codes_get_native_type( let key = CString::new(key).unwrap(); let mut key_type: i32 = 0; - let error_code = eccodes_sys::codes_get_native_type(handle, key.as_ptr(), &mut key_type); + let error_code = eccodes_sys::codes_get_native_type(handle, key.as_ptr(), &raw mut key_type); if error_code != 0 { let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); @@ -42,7 +42,7 @@ pub unsafe fn codes_get_size(handle: *const codes_handle, key: &str) -> Result Result Result let key = CString::new(key).unwrap(); let mut key_value: f64 = 0.0; - let error_code = eccodes_sys::codes_get_double(handle, key.as_ptr(), &mut key_value); + let error_code = eccodes_sys::codes_get_double(handle, key.as_ptr(), &raw mut key_value); if error_code != 0 { let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); @@ -105,7 +105,7 @@ pub unsafe fn codes_get_double_array( handle, key.as_ptr(), key_values.as_mut_ptr().cast::(), - &mut key_size, + &raw mut key_size, ); if error_code != 0 { @@ -133,7 +133,7 @@ pub unsafe fn codes_get_long_array( handle, key.as_ptr(), key_values.as_mut_ptr().cast::(), - &mut key_size, + &raw mut key_size, ); if error_code != 0 { @@ -155,7 +155,7 @@ pub unsafe fn codes_get_length( let key = CString::new(key).unwrap(); let mut key_length: usize = 0; - let error_code = eccodes_sys::codes_get_length(handle, key.as_ptr(), &mut key_length); + let error_code = eccodes_sys::codes_get_length(handle, key.as_ptr(), &raw mut key_length); if error_code != 0 { let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); @@ -182,7 +182,7 @@ pub unsafe fn codes_get_string( handle, key.as_ptr(), key_message.as_mut_ptr().cast::(), - &mut key_length, + &raw mut key_length, ); if error_code != 0 { @@ -222,7 +222,7 @@ pub unsafe fn codes_get_bytes( handle, key.as_ptr(), buffer.as_mut_ptr().cast::(), - &mut key_size, + &raw mut key_size, ); if error_code != 0 { @@ -240,7 +240,7 @@ pub unsafe fn codes_get_message_size(handle: *const codes_handle) -> Result(), product_kind as u32, - &mut error_code, + &raw mut error_code, ); if error_code != 0 { diff --git a/src/intermediate_bindings/codes_set.rs b/src/intermediate_bindings/codes_set.rs index 80f7117..1f7f558 100644 --- a/src/intermediate_bindings/codes_set.rs +++ b/src/intermediate_bindings/codes_set.rs @@ -123,7 +123,7 @@ pub unsafe fn codes_set_string( let value = CString::new(value).unwrap(); let error_code = - eccodes_sys::codes_set_string(handle, key.as_ptr(), value.as_ptr(), &mut length); + eccodes_sys::codes_set_string(handle, key.as_ptr(), value.as_ptr(), &raw mut length); if error_code != 0 { let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); @@ -150,7 +150,7 @@ pub unsafe fn codes_set_bytes( handle, key.as_ptr(), values.as_ptr().cast::(), - &mut length, + &raw mut length, ); if error_code != 0 { diff --git a/src/intermediate_bindings/grib_nearest.rs b/src/intermediate_bindings/grib_nearest.rs index a02cc66..ad066f8 100644 --- a/src/intermediate_bindings/grib_nearest.rs +++ b/src/intermediate_bindings/grib_nearest.rs @@ -23,7 +23,7 @@ pub unsafe fn codes_grib_nearest_new( let mut error_code: i32 = 0; - let nearest = eccodes_sys::codes_grib_nearest_new(handle, &mut error_code); + let nearest = eccodes_sys::codes_grib_nearest_new(handle, &raw mut error_code); if error_code != 0 { let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); @@ -85,7 +85,7 @@ pub unsafe fn codes_grib_nearest_find( addr_of_mut!(output_values[0]), addr_of_mut!(output_distances[0]), addr_of_mut!(output_indexes[0]), - &mut length, + &raw mut length, ); if error_code != 0 { diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index 9d1d93d..b902a62 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -162,7 +162,7 @@ impl CodesMessage

{ /// /// This function returns [`CodesInternal`](crate::errors::CodesInternal) when /// internal ecCodes function returns non-zero code. - pub fn default_keys_iterator<'a>(&'a mut self) -> Result, CodesError> { + pub fn default_keys_iterator(&mut self) -> Result, CodesError> { let iterator_handle = unsafe { codes_keys_iterator_new(self.message_handle, 0, "")? }; let next_item_exists = unsafe { codes_keys_iterator_next(iterator_handle)? }; diff --git a/src/lib.rs b/src/lib.rs index 3dbcc6b..24e2e40 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -211,14 +211,14 @@ //! ## Feature Flags //! //! - `message_ndarray` - enables support for converting [`KeyedMessage`] to [`ndarray::Array`]. -//! This feature is enabled by default. It is currently tested only with simple lat-lon grids. +//! This feature is enabled by default. It is currently tested only with simple lat-lon grids. //! //! - `experimental_index` - **⚠️ This feature is experimental and might be unsafe in some contexts ⚠️** -//! This flag enables support for creating and using index files for GRIB files. -//! If you want to use it, please read the information provided in [`codes_index`] documentation. +//! This flag enables support for creating and using index files for GRIB files. +//! If you want to use it, please read the information provided in [`codes_index`] documentation. //! //! - `docs` - builds the crate without linking ecCodes, particularly useful when building the documentation -//! on [docs.rs](https://docs.rs/). For more details check documentation of [eccodes-sys](https://crates.io/crates/eccodes-sys). +//! on [docs.rs](https://docs.rs/). For more details check documentation of [eccodes-sys](https://crates.io/crates/eccodes-sys). //! //! To build your own crate with this crate as dependency on docs.rs without linking ecCodes add following lines to your `Cargo.toml` //! From ff85edbf2ace6bdf02250be88cebef3cee782af2 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Thu, 8 Jan 2026 12:37:59 +0100 Subject: [PATCH 42/65] Apply more clippy fixes (mostly in tests) --- src/codes_handle/iterator.rs | 29 ++++++++++++++++------------- src/codes_message/read.rs | 12 ++++++------ src/codes_nearest.rs | 10 +++++++--- src/keys_iterator.rs | 2 +- src/pointer_guard.rs | 6 +++--- 5 files changed, 33 insertions(+), 26 deletions(-) diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index e34525d..4fd92fc 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -119,18 +119,21 @@ mod tests { let file_path = Path::new("./data/iceland-levels.grib"); let product_kind = ProductKind::GRIB; let mut handle = CodesFile::new_from_file(file_path, product_kind)?; - let mut mgen = handle.ref_message_iter(); - - let msg1 = mgen.next()?.context("Message not some")?; - drop(msg1); - let msg2 = mgen.next()?.context("Message not some")?; - let msg3 = mgen.next()?.context("Message not some")?; - drop(msg3); - let msg4 = mgen.next()?.context("Message not some")?; - let msg5 = mgen.next()?.context("Message not some")?; - drop(msg5); - - drop(mgen); + let msg2; + let msg4; + + { + let mut mgen = handle.ref_message_iter(); + + let msg1 = mgen.next()?.context("Message not some")?; + drop(msg1); + msg2 = mgen.next()?.context("Message not some")?; + let msg3 = mgen.next()?.context("Message not some")?; + drop(msg3); + msg4 = mgen.next()?.context("Message not some")?; + let msg5 = mgen.next()?.context("Message not some")?; + drop(msg5); + } // drop(handle); <- this is not allowed let key2 = msg2.read_key_dynamic("typeOfLevel")?; @@ -287,7 +290,7 @@ mod tests { v.push(t); } - v.into_iter().for_each(|th| th.join().unwrap()); + for th in v { th.join().unwrap(); } Ok(()) } diff --git a/src/codes_message/read.rs b/src/codes_message/read.rs index 7402b5e..744648a 100644 --- a/src/codes_message/read.rs +++ b/src/codes_message/read.rs @@ -388,10 +388,10 @@ mod tests { .default_keys_iterator()? .collect::>()?; - key_names.iter().for_each(|key_name| { + for key_name in &key_names { assert!(!key_name.is_empty()); - assert!(current_message.read_key_dynamic(&key_name).is_ok()) - }); + assert!(current_message.read_key_dynamic(key_name).is_ok()); + } Ok(()) } @@ -410,10 +410,10 @@ mod tests { .default_keys_iterator()? .collect::>()?; - key_names.iter().for_each(|key_name| { + for key_name in &key_names { assert!(!key_name.is_empty()); - assert!(current_message.read_key_dynamic(&key_name).is_ok()) - }); + assert!(current_message.read_key_dynamic(key_name).is_ok()); + } Ok(()) } diff --git a/src/codes_nearest.rs b/src/codes_nearest.rs index 42140ba..b96efe7 100644 --- a/src/codes_nearest.rs +++ b/src/codes_nearest.rs @@ -86,7 +86,11 @@ impl CodesNearest<'_, P> { /// ///This function returns [`CodesInternal`](crate::errors::CodesInternal) when ///one of ecCodes function returns the non-zero code. - pub fn find_nearest(&mut self, lat: f64, lon: f64) -> Result<[NearestGridpoint; 4], CodesError> { + pub fn find_nearest( + &mut self, + lat: f64, + lon: f64, + ) -> Result<[NearestGridpoint; 4], CodesError> { let output_points; unsafe { @@ -156,8 +160,8 @@ mod tests { assert!(out1[0].value > 10000.0); assert!(out2[3].index == 551); - assert!(out1[1].lat == 64.0); - assert!(out2[2].lon == -21.75); + assert!((out1[1].lat - 64.0).abs() < f64::EPSILON); + assert!((out2[2].lon - -21.75).abs() < f64::EPSILON); assert!(out1[0].distance > 15.0); Ok(()) diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index b902a62..8699940 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -209,7 +209,7 @@ impl Drop for KeysIterator<'_> { &error ); #[cfg(test)] - assert!(false, "Error in KeysIterator::drop") + panic!("Error in KeysIterator::drop"); }); } diff --git a/src/pointer_guard.rs b/src/pointer_guard.rs index d3cef1b..5dfeb3b 100644 --- a/src/pointer_guard.rs +++ b/src/pointer_guard.rs @@ -14,7 +14,7 @@ mod tests { use std::ptr; #[test] - #[should_panic] + #[should_panic = "Null pointer encountered"] fn test_non_null() { let ptr: *mut i32 = ptr::null_mut(); let result = simulated_function(ptr); @@ -25,14 +25,14 @@ mod tests { match result { CodesError::NullPtr => (), - _ => panic!("Incorrect error type: {:?}", result), + _ => panic!("Incorrect error type: {result:?}"), } } #[test] fn test_non_null_ok() { let mut x = 42_i32; - let ptr = &mut x as *mut i32; + let ptr = &raw mut x; let result = simulated_function(ptr); From 6ef3aec43ee6823fdfa89414dcac50490e3c9210 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Thu, 8 Jan 2026 15:12:28 +0100 Subject: [PATCH 43/65] Add tests for null catchers in destructors --- src/codes_message/mod.rs | 15 +++++++++++++++ src/codes_nearest.rs | 17 +++++++++++++++++ src/keys_iterator.rs | 17 +++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index 6103b29..943a77d 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -247,4 +247,19 @@ mod tests { Ok(()) } + + #[test] + fn ref_message_drop_null() -> Result<()> { + let file_path = Path::new("./data/iceland.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let mut current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + current_message.message_handle = std::ptr::null_mut(); + + Ok(()) + } } diff --git a/src/codes_nearest.rs b/src/codes_nearest.rs index b96efe7..3a32b24 100644 --- a/src/codes_nearest.rs +++ b/src/codes_nearest.rs @@ -182,4 +182,21 @@ mod tests { Ok(()) } + + #[test] + fn destructor_null() -> Result<()> { + let file_path = Path::new("./data/iceland.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + + let mut nrst = current_message.codes_nearest()?; + nrst.nearest_handle = std::ptr::null_mut(); + + Ok(()) + } } diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index 8699940..cad656c 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -296,4 +296,21 @@ mod tests { Ok(()) } + + #[test] + fn destructor_null() -> Result<()> { + let file_path = Path::new("./data/iceland.grib"); + let product_kind = ProductKind::GRIB; + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let mut current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + + let mut kiter = current_message.default_keys_iterator()?; + kiter.iterator_handle = std::ptr::null_mut(); + + Ok(()) + } } From 0af56509bc85648481152181d659e95992674d7c Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Thu, 8 Jan 2026 15:37:12 +0100 Subject: [PATCH 44/65] Remove unsafe block from dynamic key read and make the whole function more idiomatic, thus easier to comprehend --- src/codes_message/read.rs | 102 ++++++++------------------------------ 1 file changed, 21 insertions(+), 81 deletions(-) diff --git a/src/codes_message/read.rs b/src/codes_message/read.rs index 744648a..6b26f89 100644 --- a/src/codes_message/read.rs +++ b/src/codes_message/read.rs @@ -223,104 +223,44 @@ impl CodesMessage

{ /// bug in the crate or bug in the ecCodes library. If you encounter this error please check /// if your file is correct and report it on Github. pub fn read_key_dynamic(&self, key_name: &str) -> Result { - let key_type; + let key_type = self.get_key_native_type(key_name)?; + let key_size = self.get_key_size(key_name)?; - unsafe { - key_type = codes_get_native_type(self.message_handle, key_name)?; - } - - let key_value = match key_type { + match key_type { NativeKeyType::Long => { - let key_size; - unsafe { key_size = codes_get_size(self.message_handle, key_name)? } - if key_size == 1 { - let value; - unsafe { - value = codes_get_long(self.message_handle, key_name); - } - - match value { - Ok(val) => Ok(DynamicKeyType::Int(val)), - Err(err) => Err(err), - } + self.read_key_unchecked(key_name) + .map(|v| DynamicKeyType::Int(v)) } else if key_size >= 2 { - let value; - unsafe { - value = codes_get_long_array(self.message_handle, key_name); - } - - match value { - Ok(val) => Ok(DynamicKeyType::IntArray(val)), - Err(err) => Err(err), - } + self.read_key_unchecked(key_name) + .map(|v| DynamicKeyType::IntArray(v)) } else { return Err(CodesError::IncorrectKeySize); } } NativeKeyType::Double => { - let key_size; - unsafe { key_size = codes_get_size(self.message_handle, key_name)? } - if key_size == 1 { - let value; - unsafe { - value = codes_get_double(self.message_handle, key_name); - } - - match value { - Ok(val) => Ok(DynamicKeyType::Float(val)), - Err(err) => Err(err), - } + self.read_key_unchecked(key_name) + .map(|v| DynamicKeyType::Float(v)) } else if key_size >= 2 { - let value; - unsafe { - value = codes_get_double_array(self.message_handle, key_name); - } - - match value { - Ok(val) => Ok(DynamicKeyType::FloatArray(val)), - Err(err) => Err(err), - } + self.read_key_unchecked(key_name) + .map(|v| DynamicKeyType::FloatArray(v)) } else { return Err(CodesError::IncorrectKeySize); } } - NativeKeyType::Bytes => { - let value; - unsafe { - value = codes_get_bytes(self.message_handle, key_name); - } - - match value { - Ok(val) => Ok(DynamicKeyType::Bytes(val)), - Err(err) => Err(err), - } - } + NativeKeyType::Bytes => self + .read_key_unchecked(key_name) + .map(|v| DynamicKeyType::Bytes(v)), NativeKeyType::Missing => return Err(CodesError::MissingKey), - _ => { - let value; - unsafe { - value = codes_get_string(self.message_handle, key_name); - } - - match value { - Ok(val) => Ok(DynamicKeyType::Str(val)), - Err(err) => Err(err), - } - } - }; - - if let Ok(value) = key_value { - Ok(value) - } else { - let value; - unsafe { - value = codes_get_bytes(self.message_handle, key_name)?; - } - - Ok(DynamicKeyType::Bytes(value)) + _ => self + .read_key_unchecked(key_name) + .map(|v| DynamicKeyType::Str(v)), } + .or_else(|_| { + self.read_key_unchecked(key_name) + .map(|v| DynamicKeyType::Bytes(v)) + }) } } From 1c7cc3600ab8df11d288309111a9d8cd4e4669f2 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Thu, 8 Jan 2026 16:03:37 +0100 Subject: [PATCH 45/65] Test writing of more key types --- src/codes_message/write.rs | 31 +++++++++++++++++++++++++++---- src/keys_iterator.rs | 1 + 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/codes_message/write.rs b/src/codes_message/write.rs index 7d23296..ef34571 100644 --- a/src/codes_message/write.rs +++ b/src/codes_message/write.rs @@ -132,6 +132,7 @@ mod tests { use fallible_iterator::FallibleIterator; use crate::{ + KeyRead, codes_handle::{CodesFile, ProductKind}, codes_message::{DynamicKeyType, KeyWrite}, }; @@ -217,8 +218,7 @@ mod tests { let old_key = current_message.read_key_dynamic("centre")?; - let mut cloned_message = current_message - .try_clone()?; + let mut cloned_message = current_message.try_clone()?; cloned_message.write_key_unchecked("centre", "cnmc")?; let read_key = cloned_message.read_key_dynamic("centre")?; @@ -229,6 +229,30 @@ mod tests { Ok(()) } + #[test] + fn write_key_types() -> Result<()> { + let product_kind = ProductKind::GRIB; + let file_path = Path::new("./data/iceland.grib"); + + let mut handle = CodesFile::new_from_file(file_path, product_kind)?; + let mut message = handle + .ref_message_iter() + .next()? + .context("Message not some")? + .try_clone()?; + + let mut values_array: Vec = message.read_key("values")?; + values_array[0] = 0.0; + + message.write_key_unchecked("centre", "cnmc")?; // str + message.write_key_unchecked("day", 3)?; // int + message.write_key_unchecked("latitudeOfFirstGridPointInDegrees", 7.0)?; // float + message.write_key_unchecked("values", values_array.as_slice())?; //float array + + // int array and bytes cannot be tested because written key must exist and must be writable + Ok(()) + } + #[test] fn edit_keys_and_save() -> Result<()> { let product_kind = ProductKind::GRIB; @@ -242,8 +266,7 @@ mod tests { let old_key = current_message.read_key_dynamic("centre")?; - let mut cloned_message = current_message - .try_clone()?; + let mut cloned_message = current_message.try_clone()?; cloned_message.write_key_unchecked("centre", "cnmc")?; cloned_message.write_to_file(Path::new("./data/iceland_edit.grib"), false)?; diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index cad656c..c198892 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -95,6 +95,7 @@ impl CodesMessage

{ /// inside a slice. Check the documentation for the details of each flag meaning. /// /// Namespace is set simply as string, eg. `"ls"`, `"time"`, `"parameter"`, `"geography"`, `"statistics"`. + /// Empty string "" will return all keys. /// Invalid namespace will result in empty iterator. /// /// # Example From 1c9bf8986c69b625f8747029a044206d92a0ae38 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Fri, 9 Jan 2026 08:13:22 +0100 Subject: [PATCH 46/65] Add test to checks if incorrect key type is captured --- src/codes_message/read.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/codes_message/read.rs b/src/codes_message/read.rs index 6b26f89..e5e093c 100644 --- a/src/codes_message/read.rs +++ b/src/codes_message/read.rs @@ -269,6 +269,7 @@ mod tests { use anyhow::{Context, Result}; use crate::codes_handle::{CodesFile, ProductKind}; + use crate::{CodesError, KeyRead}; use crate::{FallibleIterator, codes_message::DynamicKeyType}; use std::path::Path; @@ -285,7 +286,6 @@ mod tests { .context("Message not some")?; let str_key = current_message.read_key_dynamic("name")?; - match str_key { DynamicKeyType::Str(_) => {} _ => panic!("Incorrect variant of string key"), @@ -298,14 +298,12 @@ mod tests { } let long_key = current_message.read_key_dynamic("numberOfPointsAlongAParallel")?; - match long_key { DynamicKeyType::Int(_) => {} _ => panic!("Incorrect variant of long key"), } let double_arr_key = current_message.read_key_dynamic("values")?; - match double_arr_key { DynamicKeyType::FloatArray(_) => {} _ => panic!("Incorrect variant of double array key"), @@ -377,6 +375,22 @@ mod tests { } #[test] + fn incorrect_key_type() -> Result<()> { + let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; + let current_message = handle + .ref_message_iter() + .next()? + .context("Message not some")?; + + let missing_key: Result = current_message.read_key("shortName"); + + assert!(missing_key.is_err()); + + Ok(()) + } + + #[test] + // checks if we can read keys that are used in benchmarks fn benchmark_keys() -> Result<()> { let file_path = Path::new("./data/iceland.grib"); let product_kind = ProductKind::GRIB; From 69642f43d677b1d4b2ba77615a8961f4c155b1ae Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Fri, 9 Jan 2026 08:35:29 +0100 Subject: [PATCH 47/65] Remove unnecessary code repetition in intermediate bindings by converting error codes in a dedicated function. The side effect is that now not defined error codes won't cause panic. Also removed unwrap during conversion of native key type code. --- src/errors.rs | 10 +++ src/intermediate_bindings/codes_get.rs | 79 +++++------------------ src/intermediate_bindings/codes_handle.rs | 16 +---- src/intermediate_bindings/codes_keys.rs | 12 +--- src/intermediate_bindings/codes_set.rs | 43 ++---------- src/intermediate_bindings/grib_nearest.rs | 22 ++----- src/intermediate_bindings/mod.rs | 47 ++++++++++++-- 7 files changed, 85 insertions(+), 144 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index 4d8ec8f..30241c2 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -77,6 +77,16 @@ pub enum CodesError { #[cfg(feature = "message_ndarray")] #[error("error occured while converting KeyedMessage to ndarray {0}")] NdarrayConvert(#[from] MessageNdarrayError), + + /// eccodes functions return errors as error codes and it is technically possible + /// that the library might return an error code that does not appear in [`CodesInternal`] enum. + #[error("eccodes returned unrecognized error code: {0}")] + UnrecognizedErrorCode(i32), + + /// Similarly to error codes, eccodes return key type as i32, so + /// it's technically possible that this code does not appear in [`NativeKeyType`](crate::intermediate_bindings::NativeKeyType) + #[error("Unrecognized native key type code: {0}")] + UnrecognizedKeyTypeCode(i32), } /// Errors returned by the `message_ndarray` module. diff --git a/src/intermediate_bindings/codes_get.rs b/src/intermediate_bindings/codes_get.rs index 3521233..465d8a8 100644 --- a/src/intermediate_bindings/codes_get.rs +++ b/src/intermediate_bindings/codes_get.rs @@ -7,10 +7,7 @@ use eccodes_sys::codes_handle; use libc::c_void; use num_traits::FromPrimitive; -use crate::{ - errors::{CodesError, CodesInternal}, - pointer_guard, -}; +use crate::{errors::CodesError, intermediate_bindings::error_code_to_result, pointer_guard}; use super::NativeKeyType; @@ -24,14 +21,11 @@ pub unsafe fn codes_get_native_type( let key = CString::new(key).unwrap(); let mut key_type: i32 = 0; - let error_code = eccodes_sys::codes_get_native_type(handle, key.as_ptr(), &raw mut key_type); + let error_code = + eccodes_sys::codes_get_native_type(handle, key.as_ptr(), &raw mut key_type); + error_code_to_result(error_code)?; - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } - - Ok(FromPrimitive::from_i32(key_type).unwrap()) + FromPrimitive::from_i32(key_type).ok_or(CodesError::UnrecognizedKeyTypeCode(key_type)) } } @@ -43,11 +37,7 @@ pub unsafe fn codes_get_size(handle: *const codes_handle, key: &str) -> Result Result Result let mut key_value: f64 = 0.0; let error_code = eccodes_sys::codes_get_double(handle, key.as_ptr(), &raw mut key_value); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + error_code_to_result(error_code)?; Ok(key_value) } @@ -107,11 +89,7 @@ pub unsafe fn codes_get_double_array( key_values.as_mut_ptr().cast::(), &raw mut key_size, ); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + error_code_to_result(error_code)?; Ok(key_values) } @@ -135,11 +113,7 @@ pub unsafe fn codes_get_long_array( key_values.as_mut_ptr().cast::(), &raw mut key_size, ); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + error_code_to_result(error_code)?; Ok(key_values) } @@ -156,11 +130,7 @@ pub unsafe fn codes_get_length( let mut key_length: usize = 0; let error_code = eccodes_sys::codes_get_length(handle, key.as_ptr(), &raw mut key_length); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + error_code_to_result(error_code)?; Ok(key_length) } @@ -184,11 +154,7 @@ pub unsafe fn codes_get_string( key_message.as_mut_ptr().cast::(), &raw mut key_length, ); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + error_code_to_result(error_code)?; key_message.truncate(key_length); let key_message_result = CStr::from_bytes_with_nul(key_message.as_ref()); @@ -224,11 +190,7 @@ pub unsafe fn codes_get_bytes( buffer.as_mut_ptr().cast::(), &raw mut key_size, ); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + error_code_to_result(error_code)?; Ok(buffer) } @@ -241,11 +203,7 @@ pub unsafe fn codes_get_message_size(handle: *const codes_handle) -> Result Result<(), Codes } let error_code = eccodes_sys::codes_handle_delete(handle); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + error_code_to_result(error_code)?; Ok(()) } diff --git a/src/intermediate_bindings/codes_keys.rs b/src/intermediate_bindings/codes_keys.rs index f267658..20db605 100644 --- a/src/intermediate_bindings/codes_keys.rs +++ b/src/intermediate_bindings/codes_keys.rs @@ -5,13 +5,9 @@ use std::ffi::{CStr, CString}; use eccodes_sys::{codes_handle, codes_keys_iterator}; -use num_traits::FromPrimitive; use tracing::instrument; -use crate::{ - errors::{CodesError, CodesInternal}, - pointer_guard, -}; +use crate::{errors::CodesError, intermediate_bindings::error_code_to_result, pointer_guard}; #[instrument(level = "trace")] pub unsafe fn codes_keys_iterator_new( @@ -45,11 +41,7 @@ pub unsafe fn codes_keys_iterator_delete( } let error_code = eccodes_sys::codes_keys_iterator_delete(keys_iterator); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + error_code_to_result(error_code)?; Ok(()) } diff --git a/src/intermediate_bindings/codes_set.rs b/src/intermediate_bindings/codes_set.rs index 1f7f558..7ab75a4 100644 --- a/src/intermediate_bindings/codes_set.rs +++ b/src/intermediate_bindings/codes_set.rs @@ -5,12 +5,7 @@ use std::ffi::CString; use eccodes_sys::codes_handle; -use num_traits::FromPrimitive; - -use crate::{ - errors::{CodesError, CodesInternal}, - pointer_guard, -}; +use crate::{errors::CodesError, intermediate_bindings::error_code_to_result, pointer_guard}; pub unsafe fn codes_set_long( handle: *mut codes_handle, @@ -23,11 +18,7 @@ pub unsafe fn codes_set_long( let key = CString::new(key).unwrap(); let error_code = eccodes_sys::codes_set_long(handle, key.as_ptr(), value); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + error_code_to_result(error_code)?; Ok(()) } @@ -44,11 +35,7 @@ pub unsafe fn codes_set_double( let key = CString::new(key).unwrap(); let error_code = eccodes_sys::codes_set_double(handle, key.as_ptr(), value); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + error_code_to_result(error_code)?; Ok(()) } @@ -72,11 +59,7 @@ pub unsafe fn codes_set_long_array( values.as_ptr().cast::(), length, ); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + error_code_to_result(error_code)?; Ok(()) } @@ -100,11 +83,7 @@ pub unsafe fn codes_set_double_array( values.as_ptr().cast::(), length, ); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + error_code_to_result(error_code)?; Ok(()) } @@ -124,11 +103,7 @@ pub unsafe fn codes_set_string( let error_code = eccodes_sys::codes_set_string(handle, key.as_ptr(), value.as_ptr(), &raw mut length); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + error_code_to_result(error_code)?; Ok(()) } @@ -152,11 +127,7 @@ pub unsafe fn codes_set_bytes( values.as_ptr().cast::(), &raw mut length, ); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + error_code_to_result(error_code)?; Ok(()) } diff --git a/src/intermediate_bindings/grib_nearest.rs b/src/intermediate_bindings/grib_nearest.rs index ad066f8..8c51265 100644 --- a/src/intermediate_bindings/grib_nearest.rs +++ b/src/intermediate_bindings/grib_nearest.rs @@ -5,12 +5,10 @@ use std::ptr::addr_of_mut; use eccodes_sys::{CODES_NEAREST_SAME_DATA, CODES_NEAREST_SAME_GRID, codes_handle, codes_nearest}; -use num_traits::FromPrimitive; use tracing::instrument; use crate::{ - NearestGridpoint, - errors::{CodesError, CodesInternal}, + NearestGridpoint, errors::CodesError, intermediate_bindings::error_code_to_result, pointer_guard, }; @@ -24,11 +22,7 @@ pub unsafe fn codes_grib_nearest_new( let mut error_code: i32 = 0; let nearest = eccodes_sys::codes_grib_nearest_new(handle, &raw mut error_code); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + error_code_to_result(error_code)?; Ok(nearest) } @@ -42,11 +36,7 @@ pub unsafe fn codes_grib_nearest_delete(nearest: *mut codes_nearest) -> Result<( } let error_code = eccodes_sys::codes_grib_nearest_delete(nearest); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + error_code_to_result(error_code)?; Ok(()) } @@ -87,11 +77,7 @@ pub unsafe fn codes_grib_nearest_find( addr_of_mut!(output_indexes[0]), &raw mut length, ); - - if error_code != 0 { - let err: CodesInternal = FromPrimitive::from_i32(error_code).unwrap(); - return Err(err.into()); - } + error_code_to_result(error_code)?; let mut output = [NearestGridpoint::default(); 4]; diff --git a/src/intermediate_bindings/mod.rs b/src/intermediate_bindings/mod.rs index e0ef58f..8714d7a 100644 --- a/src/intermediate_bindings/mod.rs +++ b/src/intermediate_bindings/mod.rs @@ -25,20 +25,57 @@ pub enum NativeKeyType { Missing = eccodes_sys::CODES_TYPE_MISSING as isize, } -pub use codes_get::{ +pub(crate) use codes_get::{ codes_get_bytes, codes_get_double, codes_get_double_array, codes_get_long, codes_get_long_array, codes_get_message, codes_get_native_type, codes_get_size, codes_get_string, }; -pub use codes_handle::{codes_handle_clone, codes_handle_delete, codes_handle_new_from_file}; -pub use codes_keys::{ +pub(crate) use codes_handle::{ + codes_handle_clone, codes_handle_delete, codes_handle_new_from_file, +}; +pub(crate) use codes_keys::{ codes_keys_iterator_delete, codes_keys_iterator_get_name, codes_keys_iterator_new, codes_keys_iterator_next, }; -pub use codes_set::{ +pub(crate) use codes_set::{ codes_set_bytes, codes_set_double, codes_set_double_array, codes_set_long, codes_set_long_array, codes_set_string, }; -pub use grib_nearest::{ +pub(crate) use grib_nearest::{ codes_grib_nearest_delete, codes_grib_nearest_find, codes_grib_nearest_new, }; +use num_traits::FromPrimitive; + +use crate::{CodesError, errors::CodesInternal}; + +fn error_code_to_result(error_code: i32) -> Result<(), CodesError> { + if error_code != 0 { + let err: CodesInternal = FromPrimitive::from_i32(error_code) + .ok_or(CodesError::UnrecognizedErrorCode(error_code))?; + return Err(err.into()); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::{CodesError, errors::CodesInternal, intermediate_bindings::error_code_to_result}; + + #[test] + fn error_codes() { + let ok_result = error_code_to_result(0); + let defined_error = error_code_to_result(-2).unwrap_err(); // codes internal + let undefined_error = error_code_to_result(4).unwrap_err(); + + assert!(ok_result.is_ok()); + assert!(std::matches!( + defined_error, + CodesError::Internal(CodesInternal::CodesInternalError) + )); + assert!(std::matches!( + undefined_error, + CodesError::UnrecognizedErrorCode(4) + )); + } +} From b425c6118ff44fc76b258d7fd6aa5aa8bd970a58 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Fri, 9 Jan 2026 08:50:07 +0100 Subject: [PATCH 48/65] remove prints from test --- src/codes_handle/iterator.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index 4fd92fc..f4d9fb3 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -75,6 +75,7 @@ mod tests { codes_message::DynamicKeyType, }; use anyhow::{Context, Ok, Result}; + use float_cmp::assert_approx_eq; use std::{ path::Path, sync::{Arc, Barrier}, @@ -249,16 +250,16 @@ mod tests { // Find nearest modifies internal KeyedMessage fields so we need mutable reference let level = &level[0]; - println!("{:?}", level.read_key_dynamic("shortName")); + let short_name = level.read_key_dynamic("shortName")?; + assert_eq!(short_name, DynamicKeyType::Str("msl".into())); // Get the four nearest gridpoints of Reykjavik let nearest_gridpoints = level.codes_nearest()?.find_nearest(64.13, -21.89)?; + let value = nearest_gridpoints[3].value; + let distance = nearest_gridpoints[3].distance; - // Print value and distance of the nearest gridpoint - println!( - "value: {}, distance: {}", - nearest_gridpoints[3].value, nearest_gridpoints[3].distance - ); + assert_approx_eq!(f64, value, 100557.9375); + assert_approx_eq!(f64, distance, 14.358879960775498); Ok(()) } @@ -290,7 +291,9 @@ mod tests { v.push(t); } - for th in v { th.join().unwrap(); } + for th in v { + th.join().unwrap(); + } Ok(()) } From 8d442c5d6917c81148d133a9922ecc401a46c422 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Fri, 9 Jan 2026 08:50:52 +0100 Subject: [PATCH 49/65] rename message_ndarray feature to ndarray --- .github/workflows/rust-dev.yml | 12 ++++++------ .github/workflows/rust.yml | 12 ++++++------ Cargo.toml | 10 +++++++--- README.md | 2 +- src/codes_message/mod.rs | 2 +- src/errors.rs | 6 +++--- src/lib.rs | 6 +++--- src/message_ndarray.rs | 6 +++--- 8 files changed, 30 insertions(+), 26 deletions(-) diff --git a/.github/workflows/rust-dev.yml b/.github/workflows/rust-dev.yml index 0b45977..3d00230 100644 --- a/.github/workflows/rust-dev.yml +++ b/.github/workflows/rust-dev.yml @@ -46,14 +46,14 @@ jobs: cargo clean - name: Build release run: | - cargo build --release --features "experimental_index, message_ndarray" + cargo build --release --features "experimental_index, ndarray" - name: Check with clippy run: | - cargo clippy --features "experimental_index, message_ndarray" -- -D warnings + cargo clippy --features "experimental_index, ndarray" -- -D warnings - name: Test with cargo run: | cargo clean - RUST_BACKTRACE=full cargo test --features "experimental_index, message_ndarray" -- --include-ignored + RUST_BACKTRACE=full cargo test --features "experimental_index, ndarray" -- --include-ignored build-macos: @@ -71,11 +71,11 @@ jobs: cargo clean - name: Build release run: | - cargo build --release --features "experimental_index, message_ndarray" + cargo build --release --features "experimental_index, ndarray" - name: Check with clippy run: | - cargo clippy --features "experimental_index, message_ndarray" -- -D warnings + cargo clippy --features "experimental_index, ndarray" -- -D warnings - name: Test with cargo run: | cargo clean - RUST_BACKTRACE=full cargo test --features "experimental_index, message_ndarray" -- --include-ignored + RUST_BACKTRACE=full cargo test --features "experimental_index, ndarray" -- --include-ignored diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 368ea84..3da12a3 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -47,14 +47,14 @@ jobs: cargo clean - name: Build with cargo run: | - cargo build --release --features "experimental_index, message_ndarray" + cargo build --release --features "experimental_index, ndarray" cargo clean - name: Test with cargo run: | cargo test --no-default-features - cargo test --features "message_ndarray" + cargo test --features "ndarray" cargo test --features "experimental_index" - cargo test --features "experimental_index, message_ndarray" + cargo test --features "experimental_index, ndarray" cargo clean - name: Benchmark with criterion run: | @@ -78,14 +78,14 @@ jobs: cargo clean - name: Build with cargo run: | - cargo build --release --features "experimental_index, message_ndarray" + cargo build --release --features "experimental_index, ndarray" cargo clean - name: Test with cargo run: | cargo test --no-default-features - cargo test --features "message_ndarray" + cargo test --features "ndarray" cargo test --features "experimental_index" - cargo test --features "experimental_index, message_ndarray" + cargo test --features "experimental_index, ndarray" cargo clean - name: Benchmark with criterion run: | diff --git a/Cargo.toml b/Cargo.toml index dc97a48..31ce732 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,13 +43,17 @@ anyhow = { version = "1.0", features = ["backtrace"] } float-cmp = "0.10" [features] -default = ["message_ndarray"] +default = ["ndarray"] docs = ["eccodes-sys/docs"] -message_ndarray = ["dep:ndarray"] +ndarray = ["dep:ndarray"] [package.metadata.docs.rs] -features = ["docs", "message_ndarray"] +features = ["docs", "ndarray"] [[bench]] name = "main" harness = false + +# that's temporary until I fix the docs +[lib] +doctest = false diff --git a/README.md b/README.md index 3d0e479..91aad85 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ If you find a bug or have a suggestion, feel free to discuss it on Github. ## Features -- `message_ndarray` - enables support for converting `KeyedMessage` to `ndarray::Array`. +- `ndarray` - enables support for converting `KeyedMessage` to `ndarray::Array`. This feature is enabled by default. It is currently tested only with simple lat-lon grids. - `experimental_index` - enables support for creating and using index files for GRIB files. diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index 943a77d..f6ded43 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -5,7 +5,7 @@ mod clone; mod read; mod write; -#[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] +#[cfg_attr(docsrs, doc(cfg(feature = "ndarray")))] pub use crate::message_ndarray::RustyCodesMessage; pub use read::{DynamicKeyType, KeyPropertiesRead, KeyRead}; pub use write::KeyWrite; diff --git a/src/errors.rs b/src/errors.rs index 30241c2..61e7c99 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -74,7 +74,7 @@ pub enum CodesError { /// Returned when function in `message_ndarray` module cannot convert /// the message to ndarray. Check [`MessageNdarrayError`] for more details. - #[cfg(feature = "message_ndarray")] + #[cfg(feature = "ndarray")] #[error("error occured while converting KeyedMessage to ndarray {0}")] NdarrayConvert(#[from] MessageNdarrayError), @@ -90,8 +90,8 @@ pub enum CodesError { } /// Errors returned by the `message_ndarray` module. -#[cfg(feature = "message_ndarray")] -#[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] +#[cfg(feature = "ndarray")] +#[cfg_attr(docsrs, doc(cfg(feature = "ndarray")))] #[derive(PartialEq, Clone, Error, Debug)] pub enum MessageNdarrayError { /// Returned when functions converting to ndarray cannot correctly diff --git a/src/lib.rs b/src/lib.rs index 24e2e40..fcf5b30 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -210,7 +210,7 @@ //! //! ## Feature Flags //! -//! - `message_ndarray` - enables support for converting [`KeyedMessage`] to [`ndarray::Array`]. +//! - `ndarray` - enables support for converting [`KeyedMessage`] to [`ndarray::Array`]. //! This feature is enabled by default. It is currently tested only with simple lat-lon grids. //! //! - `experimental_index` - **⚠️ This feature is experimental and might be unsafe in some contexts ⚠️** @@ -233,8 +233,8 @@ pub mod codes_nearest; pub mod errors; mod intermediate_bindings; pub mod keys_iterator; -#[cfg(feature = "message_ndarray")] -#[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] +#[cfg(feature = "ndarray")] +#[cfg_attr(docsrs, doc(cfg(feature = "ndarray")))] mod message_ndarray; mod pointer_guard; diff --git a/src/message_ndarray.rs b/src/message_ndarray.rs index 06fe47a..8b0df6d 100644 --- a/src/message_ndarray.rs +++ b/src/message_ndarray.rs @@ -9,7 +9,7 @@ use crate::{CodesError, KeyRead, codes_message::CodesMessage, errors::MessageNda /// Struct returned by [`KeyedMessage::to_lons_lats_values()`] method. /// The arrays are collocated, meaning that `longitudes[i, j]` and `latitudes[i, j]` are the coordinates of `values[i, j]`. #[derive(Clone, PartialEq, Debug, Default)] -#[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] +#[cfg_attr(docsrs, doc(cfg(feature = "ndarray")))] pub struct RustyCodesMessage { /// Longitudes in degrees pub longitudes: Array2, @@ -41,7 +41,7 @@ impl CodesMessage

{ /// /// - When the required keys are not present or if their values are not of the expected type /// - When the number of values mismatch with the `Ni` and `Nj` keys - #[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] + #[cfg_attr(docsrs, doc(cfg(feature = "ndarray")))] pub fn to_ndarray(&self) -> Result, CodesError> { let ni: i64 = self.read_key("Ni")?; let ni = usize::try_from(ni).map_err(MessageNdarrayError::from)?; @@ -86,7 +86,7 @@ impl CodesMessage

{ /// /// - When the required keys are not present or if their values are not of the expected type /// - When the number of values mismatch with the `Ni` and `Nj` keys - #[cfg_attr(docsrs, doc(cfg(feature = "message_ndarray")))] + #[cfg_attr(docsrs, doc(cfg(feature = "ndarray")))] pub fn to_lons_lats_values(&self) -> Result { let ni: i64 = self.read_key("Ni")?; let ni = usize::try_from(ni).map_err(MessageNdarrayError::from)?; From 4af0ae8acd66c91a23ba3b2509e2ee84c60eddde Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Fri, 9 Jan 2026 08:53:33 +0100 Subject: [PATCH 50/65] move and rename the ndarray module --- src/codes_message/mod.rs | 5 ++++- src/{message_ndarray.rs => codes_message/ndarray.rs} | 0 src/lib.rs | 4 +--- 3 files changed, 5 insertions(+), 4 deletions(-) rename src/{message_ndarray.rs => codes_message/ndarray.rs} (100%) diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index f6ded43..c3cdb64 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -4,9 +4,12 @@ mod clone; mod read; mod write; +#[cfg(feature = "ndarray")] +#[cfg_attr(docsrs, doc(cfg(feature = "ndarray")))] +mod ndarray; #[cfg_attr(docsrs, doc(cfg(feature = "ndarray")))] -pub use crate::message_ndarray::RustyCodesMessage; +pub use ndarray::RustyCodesMessage; pub use read::{DynamicKeyType, KeyPropertiesRead, KeyRead}; pub use write::KeyWrite; diff --git a/src/message_ndarray.rs b/src/codes_message/ndarray.rs similarity index 100% rename from src/message_ndarray.rs rename to src/codes_message/ndarray.rs diff --git a/src/lib.rs b/src/lib.rs index fcf5b30..95dd51e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -233,9 +233,7 @@ pub mod codes_nearest; pub mod errors; mod intermediate_bindings; pub mod keys_iterator; -#[cfg(feature = "ndarray")] -#[cfg_attr(docsrs, doc(cfg(feature = "ndarray")))] -mod message_ndarray; + mod pointer_guard; pub use codes_handle::{CodesFile, ProductKind}; From 0fdedd2644fb06e54bbb54adb7c7a19e3975a407 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Fri, 9 Jan 2026 08:56:14 +0100 Subject: [PATCH 51/65] Apply clippy fixes and rustfmt --- src/codes_handle/mod.rs | 8 ++------ src/codes_message/mod.rs | 4 ++-- src/codes_message/read.rs | 23 +++++++---------------- 3 files changed, 11 insertions(+), 24 deletions(-) diff --git a/src/codes_handle/mod.rs b/src/codes_handle/mod.rs index 8a8f1ea..b8c3294 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_handle/mod.rs @@ -274,9 +274,7 @@ mod tests { let codes_file = CodesFile::new_from_file(file_path, product_kind)?; assert!(!codes_file.pointer.is_null()); - assert_eq!(codes_file.product_kind as u32, { - ProductKind_PRODUCT_GRIB - }); + assert_eq!(codes_file.product_kind as u32, { ProductKind_PRODUCT_GRIB }); codes_file._data.metadata()?; @@ -293,9 +291,7 @@ mod tests { let codes_file = CodesFile::new_from_memory(buf, product_kind)?; assert!(!codes_file.pointer.is_null()); - assert_eq!(codes_file.product_kind as u32, { - ProductKind_PRODUCT_GRIB - }); + assert_eq!(codes_file.product_kind as u32, { ProductKind_PRODUCT_GRIB }); assert!(!codes_file._data.is_empty()); diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index c3cdb64..860f120 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -2,11 +2,11 @@ //! used for reading and writing data of given variable from GRIB file mod clone; -mod read; -mod write; #[cfg(feature = "ndarray")] #[cfg_attr(docsrs, doc(cfg(feature = "ndarray")))] mod ndarray; +mod read; +mod write; #[cfg_attr(docsrs, doc(cfg(feature = "ndarray")))] pub use ndarray::RustyCodesMessage; diff --git a/src/codes_message/read.rs b/src/codes_message/read.rs index e5e093c..6b65652 100644 --- a/src/codes_message/read.rs +++ b/src/codes_message/read.rs @@ -229,38 +229,29 @@ impl CodesMessage

{ match key_type { NativeKeyType::Long => { if key_size == 1 { - self.read_key_unchecked(key_name) - .map(|v| DynamicKeyType::Int(v)) + self.read_key_unchecked(key_name).map(DynamicKeyType::Int) } else if key_size >= 2 { self.read_key_unchecked(key_name) - .map(|v| DynamicKeyType::IntArray(v)) + .map(DynamicKeyType::IntArray) } else { return Err(CodesError::IncorrectKeySize); } } NativeKeyType::Double => { if key_size == 1 { - self.read_key_unchecked(key_name) - .map(|v| DynamicKeyType::Float(v)) + self.read_key_unchecked(key_name).map(DynamicKeyType::Float) } else if key_size >= 2 { self.read_key_unchecked(key_name) - .map(|v| DynamicKeyType::FloatArray(v)) + .map(DynamicKeyType::FloatArray) } else { return Err(CodesError::IncorrectKeySize); } } - NativeKeyType::Bytes => self - .read_key_unchecked(key_name) - .map(|v| DynamicKeyType::Bytes(v)), + NativeKeyType::Bytes => self.read_key_unchecked(key_name).map(DynamicKeyType::Bytes), NativeKeyType::Missing => return Err(CodesError::MissingKey), - _ => self - .read_key_unchecked(key_name) - .map(|v| DynamicKeyType::Str(v)), + _ => self.read_key_unchecked(key_name).map(DynamicKeyType::Str), } - .or_else(|_| { - self.read_key_unchecked(key_name) - .map(|v| DynamicKeyType::Bytes(v)) - }) + .or_else(|_| self.read_key_unchecked(key_name).map(DynamicKeyType::Bytes)) } } From 48de739513cd7c41dd9c1fb6c93cd85d22db11a7 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Fri, 9 Jan 2026 09:03:04 +0100 Subject: [PATCH 52/65] Increase the chance of clashes in test for arc mesage --- src/codes_handle/iterator.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index f4d9fb3..e8d4026 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -282,9 +282,11 @@ mod tests { let b = barrier.clone(); let t = std::thread::spawn(move || { - for _ in 0..1000 { + for _ in 0..10 { b.wait(); - let _ = msg.read_key_dynamic("shortName").unwrap(); + for _ in 0..100 { + let _ = msg.read_key_dynamic("shortName").unwrap(); + } } }); From d9f1416b54a3a46f6bc2871987162513716eea35 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Fri, 9 Jan 2026 10:24:23 +0100 Subject: [PATCH 53/65] Test ArcMessage thread safety more comprehensively --- src/codes_handle/iterator.rs | 44 ++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/src/codes_handle/iterator.rs b/src/codes_handle/iterator.rs index e8d4026..4de723c 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_handle/iterator.rs @@ -4,6 +4,10 @@ use crate::{ArcMessage, CodesFile, RefMessage, errors::CodesError}; use std::fmt::Debug; use std::sync::{Arc, Mutex}; +/// `KeyedMessageGenerator` implements [`FallibleIterator`](codes_handle::KeyedMessageGenerator#impl-FallibleIterator-for-KeyedMessageGenerator%3C'ch,+S%3E) +/// which allows you to iterate over messages in the file. The iterator returns `KeyedMessage` with lifetime tied to the lifetime of `CodesHandle`, +/// that is `KeyedMessage` cannot outlive the `CodesHandle` it was generated from. If you need to prolong its lifetime, you can use +/// [`try_clone()`](KeyedMessage::try_clone), but that comes with performance and memory overhead. #[derive(Debug)] pub struct RefMessageIter<'a, D: Debug> { codes_file: &'a mut CodesFile, @@ -265,7 +269,7 @@ mod tests { } #[test] - fn atomic_thread_safety() -> Result<()> { + fn thread_safety_messsage_wise() -> Result<()> { let file_path = Path::new("./data/iceland-levels.grib"); let product_kind = ProductKind::GRIB; @@ -278,7 +282,7 @@ mod tests { let mut v = vec![]; for _ in 0..10 { - let msg = Arc::new(mgen.next()?.context("No more messages")?); + let msg = mgen.next()?.context("No more messages")?; let b = barrier.clone(); let t = std::thread::spawn(move || { @@ -299,4 +303,40 @@ mod tests { Ok(()) } + + #[test] + fn thread_safety_within_message() -> Result<()> { + let file_path = Path::new("./data/iceland-levels.grib"); + let product_kind = ProductKind::GRIB; + + let handle = CodesFile::new_from_file(file_path, product_kind)?; + let mut mgen = handle.arc_message_iter(); + let msg = Arc::new(mgen.next()?.context("No more messages")?); + + let barrier = Arc::new(Barrier::new(10)); + + let mut v = vec![]; + + for _ in 0..10 { + let msg_inner = msg.clone(); + let b = barrier.clone(); + + let t = std::thread::spawn(move || { + for _ in 0..10 { + b.wait(); + for _ in 0..100 { + let _ = msg_inner.read_key_dynamic("shortName").unwrap(); + } + } + }); + + v.push(t); + } + + for th in v { + th.join().unwrap(); + } + + Ok(()) + } } From 32474331b7d1af9e9d8d32ed8e887ca0c566ada1 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Fri, 9 Jan 2026 11:27:39 +0100 Subject: [PATCH 54/65] wip docs --- Cargo.toml | 4 - src/codes_message/mod.rs | 1 - src/lib.rs | 223 +++++++++++++++++++++------------------ 3 files changed, 123 insertions(+), 105 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 31ce732..c46816c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,3 @@ features = ["docs", "ndarray"] [[bench]] name = "main" harness = false - -# that's temporary until I fix the docs -[lib] -doctest = false diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index 860f120..88d3fd4 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -72,7 +72,6 @@ unsafe impl Send for BufMessage {} unsafe impl Sync for BufMessage {} /// All messages use this struct for operations. -#[doc(hidden)] #[derive(Debug)] pub struct CodesMessage { pub(crate) _parent: P, diff --git a/src/lib.rs b/src/lib.rs index 95dd51e..0135833 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,7 +21,7 @@ //! //! **Currently only operations on GRIB files are supported.** //! -//! **Version 0.14 introduces some breaking changes, [check them below](#changes-in-version-014)!** +//! **Version 0.14 introduces breaking changes, [check them below](#changes-in-version-014)!** //! //! Because of the ecCodes library API characteristics theses bindings are //! rather thick wrapper to make this crate safe and convenient to use. @@ -61,162 +61,185 @@ //! //! ## Usage //! -//! To access a GRIB file you need to create [`CodesHandle`] with one of provided constructors. +//! To access a GRIB file you need to create [`CodesFile`] with one of the provided constructors. //! -//! ecCodes represents GRIB files as a set of separate messages, each representing data fields at specific time and level. -//! Messages are represented here by the [`KeyedMessage`] structure. +//! ecCodes represents GRIB files as a set of separate messages, each containing data fields at specific time and level. +//! Messages are represented here by a generic [`CodesMessage`](codes_message::CodesMessage) structure, but you shouldn't use it directly. +//! Instead use [`RefMessage`], [`ArcMessage`] or [`BufMessage`] to operations - check the docs for more information when to use each. //! -//! To obtain `KeyedMessage`(s) from `CodesHandle` you need to create an instance of [`KeyedMessageGenerator`](codes_handle::KeyedMessageGenerator) -//! with [`CodesHandle::message_generator()`]. This is an analogous interface to `IterMut` and `iter_mut()` in [`std::slice`]. +//! To obtain `CodesMessage` from `CodesFile` you need to create an instance of [`RefMessageIter`] or [`ArcMessageIter`] using +//! [`CodesFile::ref_message_iter()`] or [`CodesFile::arc_message_iter()`]. +//! Those structures implement [`FallibleIterator`], please check its documentation if you are not familiar with it. //! -//! `KeyedMessageGenerator` implements [`FallibleIterator`](codes_handle::KeyedMessageGenerator#impl-FallibleIterator-for-KeyedMessageGenerator%3C'ch,+S%3E) -//! which allows you to iterate over messages in the file. The iterator returns `KeyedMessage` with lifetime tied to the lifetime of `CodesHandle`, -//! that is `KeyedMessage` cannot outlive the `CodesHandle` it was generated from. If you need to prolong its lifetime, you can use -//! [`try_clone()`](KeyedMessage::try_clone), but that comes with performance and memory overhead. -//! -//! `KeyedMessage` implements several methods to access the data as needed, most of those can be called directly on `&KeyedMessage`. -//! You can also use [`try_clone()`](KeyedMessage::try_clone) to clone the message and prolong its lifetime. +//! `CodesMessage` implements several methods to access the data as needed, most of those can be called directly. +//! Almost all methods can be called on any `CodesMessage`, except for [`KeyWrite`] operations, which can be called only on [`BufMessage`] +//! to avoid confusion if written keys are save to file or not. //! //! Data contained by `KeyedMessage` is represented as *keys* (like in dictionary). -//! Keys can be read with static types using [`read_key()`](KeyedMessage::read_key) or with [dynamic types](`DynamicKeyType`) -//! using [`read_key_dynamic()`](KeyedMessage::read_key_dynamic). To discover what keys are present in a message use [`KeysIterator`](KeyedMessage). +//! Keys can be read with static types using [`read_key()`](KeyRead::read_key) or with [dynamic types](codes_message::DynamicKeyType) +//! using [`read_key_dynamic()`](codes_message::CodesMessage::read_key_dynamic). +//! To discover what keys are present in a message use [`KeysIterator`](KeysIterator). +//! +//! With `ndarray` feature (enabled by default) you can also read `CodesMessage` into `ndarray` using [`to_ndarray()`](codes_message::CodesMessage::to_ndarray) +//! and [`to_lons_lats_values()`](codes_message::CodesMessage::to_lons_lats_values). //! //! You can use [`CodesNearest`] to get the data values of four nearest gridpoints for given coordinates. //! -//! You can also modify the message with [`write_key()`](KeyedMessage::write_key) and write -//! it to a new file with [`write_to_file()`](KeyedMessage::write_to_file). +//! To modify keys within the message use [`write_key_unchecked()`](KeyWrite::write_key_unchecked). +//! To save that modified message use [`write_to_file()`](codes_message::CodesMessage::write_to_file). //! //! #### Example 1 - Reading GRIB file //! +//! In this example we are reading mean sea level pressure for 4 gridpoints nearest to +//! Reykjavik (64.13N, -21.89E) for 1st June 2021 00:00 UTC from ERA5 Climate Reanalysis. +//! //! ``` -//! // We are reading the mean sea level pressure for 4 gridpoints -//! // nearest to Reykjavik (64.13N, -21.89E) for 1st June 2021 00:00 UTC -//! // from ERA5 Climate Reanalysis -//! -//! use eccodes::{ProductKind, CodesHandle, KeyRead}; -//! # use std::path::Path; -//! use eccodes::FallibleStreamingIterator; -//! # -//! # fn main() -> anyhow::Result<()> { +//! use eccodes::{CodesFile, FallibleIterator, KeyRead, ProductKind}; //! +//! # fn main() -> anyhow::Result<()> { //! // Open the GRIB file and create the CodesHandle -//! let file_path = Path::new("./data/iceland.grib"); -//! let product_kind = ProductKind::GRIB; -//! let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; -//! +//! let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; //! // Use iterator to find a message with shortName "msl" and typeOfLevel "surface" //! // We can use while let or for_each() to iterate over the messages -//! while let Some(msg) = handle.next()? { +//! while let Some(msg) = handle.ref_message_iter().next()? { //! // We need to specify the type of key we read //! let short_name: String = msg.read_key("shortName")?; //! let type_of_level: String = msg.read_key("typeOfLevel")?; -//! //! if short_name == "msl" && type_of_level == "surface" { //! // Create CodesNearest for given message -//! let nearest_gridpoints = msg.codes_nearest()? +//! let nearest_gridpoints = msg +//! .codes_nearest()? //! // Find the nearest gridpoints to Reykjavik //! .find_nearest(64.13, -21.89)?; -//! //! // Print value and distance of the nearest gridpoint -//! println!("value: {}, distance: {}", -//! nearest_gridpoints[3].value, -//! nearest_gridpoints[3].distance); +//! println!( +//! "value: {}, distance: {}", +//! nearest_gridpoints[3].value, nearest_gridpoints[3].distance +//! ); //! } //! } -//! # Ok(()) +//! # Ok(()) //! # } //! ``` //! -//! #### **New in 0.14** Example 3: Concurrent read +//! #### (New in 0.14) Example 2 - Concurrent read //! +//! This example shows how `ArcMessage` can be used to do concurrent operations +//! on different message within one file and on the same message as well. //! -//! #### Example 3 - Writing GRIB files +//! ``` +//! use eccodes::{CodesError, CodesFile, FallibleIterator, KeyRead, ProductKind}; +//! use std::sync::Arc; //! -//! ```rust -//! // The crate provides basic support for setting `KeyedMessage` keys -//! // and writing GRIB files. The easiests (and safest) way to create a -//! // new custom message is to copy exisitng one from other GRIB file, -//! // modify the keys and write to new file. +//! # fn main() -> anyhow::Result<()> { +//! // We open the file as before +//! let handle = CodesFile::new_from_file("./data/iceland-levels.grib", ProductKind::GRIB)?; //! -//! // Here we are computing the temperature at 850hPa as an average -//! // of 900hPa and 800hPa and writing it to a new file. +//! // Note different mutability - RefMessageIter required the handle to be mutable +//! let mut arc_msg_gen = handle.arc_message_iter(); //! -//! use eccodes::FallibleStreamingIterator; -//! use eccodes::{CodesHandle, KeyRead, KeyWrite, ProductKind}; -//! # use std::{fs::remove_file, path::Path}; -//! -//! # fn main() -> anyhow::Result<()> { -//! // Start by opening the file and creating CodesHandle -//! let file_path = Path::new("./data/iceland-levels.grib"); -//! let mut handle = CodesHandle::new_from_file(file_path, ProductKind::GRIB)?; +//! let mut join_handles = vec![]; //! -//! // We need a message to edit, in this case we can use -//! // temperature at 700hPa, which is similar to our result -//! let mut new_msg = vec![]; +//! while let Some(msg) = arc_msg_gen.next()? { +//! // ArcMessage is Send+Sync +//! let msg1 = Arc::new(msg); +//! let msg2 = msg1.clone(); //! -//! // Get data values of temperatures at 800hPa and 900hPa -//! let mut t800: Vec = vec![]; -//! let mut t900: Vec = vec![]; +//! // For each message we spawn two threads then we do operations simulataneously +//! join_handles.push(std::thread::spawn(move || msg1.read_key("shortName"))); +//! join_handles.push(std::thread::spawn(move || msg2.read_key("typeOfLevel"))); +//! } //! -//! // Iterate over the messages and collect the data to defined vectors -//! while let Some(msg) = handle.next()? { -//! let short_name: String = msg.read_key("shortName")?; +//! // Now we collect and print the results +//! let read_results = join_handles +//! .into_iter() +//! // In production you should probably handle this join error +//! .map(|jh| jh.join().unwrap()) +//! .collect::, CodesError>>()?; //! -//! if short_name == "t" { -//! let level: i64 = msg.read_key("level")?; +//! println!("{read_results:?}"); //! -//! if level == 700 { -//! // To use message outside of the iterator we need to clone it -//! new_msg.push(msg.try_clone()?); -//! } +//! # Ok(()) +//! # } +//! ``` //! -//! if level == 800 { -//! t800 = msg.read_key("values")?; -//! } +//! #### Example 3 - Writing GRIB files and indexing //! -//! if level == 900 { -//! t900 = msg.read_key("values")?; -//! } -//! } -//! } +//! The crate provides basic support for setting `KeyedMessage` keys +//! and writing GRIB files. To create a new custom message we need to copy +//! existing one from other GRIB file, modify the keys and write to new file. //! -//! // This converts the vector to a single message -//! let mut new_msg = new_msg.remove(0); +//! In this example we are computing the temperature at 850hPa as an average +//! of 900hPa and 800hPa and writing it to a new file. This example also shows +//! how you can create a simple index for a file to get messages we need. +//! To read values from a message you can also use [`to_ndarray()`](codes_message::CodesMessage::to_ndarray). //! +//! ``` +//! use anyhow::Context; +//! use eccodes::{CodesFile, FallibleIterator, KeyRead, KeyWrite, ProductKind}; +//! use std::{collections::HashMap, fs::remove_file, path::Path}; +//! +//! # fn main() -> anyhow::Result<()> { +//! // Start by opening the file and creating CodesHandle +//! let file_path = Path::new("./data/iceland-levels.grib"); +//! let mut handle = CodesFile::new_from_file(file_path, ProductKind::GRIB)?; +//! +//! // To build the index we need to collect all messages +//! let messages = handle.ref_message_iter().collect::>()?; +//! let mut msg_index = HashMap::new(); +//! msg_index.reserve(messages.len()); +//! +//! // Now we can put the messages into a hashmap and index them by shortName and level +//! for msg in messages.into_iter() { +//! // all messages in this grib are on the same level type +//! let short_name: String = msg.read_key("shortName")?; +//! let level: i64 = msg.read_key("level")?; +//! +//! msg_index.insert((short_name, level), msg); +//! } +//! +//! // Now we can get the values from messages we need +//! let t_800: Vec = msg_index +//! .get(&("t".to_string(), 800)) +//! .context("message missing")? // we use anyhow context for convienience +//! .read_key("values")?; +//! let t_900: Vec = msg_index +//! .get(&("t".to_string(), 800)) +//! .context("message missing")? +//! .read_key("values")?; +//! +//! // We will also clone t at 700hPa to edit it +//! let mut t_850_msg = msg_index +//! .get(&("t".to_string(), 700)) +//! .context("message missing")? +//! .try_clone()?; +//! //! // Compute temperature at 850hPa -//! let t850: Vec = t800 +//! let t_850_values: Vec = t_800 //! .iter() -//! .zip(t900.iter()) +//! .zip(t_900.iter()) //! .map(|t| (t.0 + t.1) / 2.0) //! .collect(); -//! -//! // Edit appropriate keys in the editable message -//! new_msg.write_key("level", 850)?; -//! new_msg.write_key("values", &t850)?; -//! +//! +//! // Edit appropriate keys in the cloned (editable) message +//! t_850_msg.write_key_unchecked("level", 850)?; +//! t_850_msg.write_key_unchecked("values", t_850_values.as_slice())?; +//! //! // Save the message to a new file without appending -//! new_msg.write_to_file(Path::new("iceland-850.grib"), false)?; -//! -//! # remove_file(Path::new("iceland-850.grib")).unwrap(); -//! # Ok(()) +//! t_850_msg.write_to_file(Path::new("iceland-850.grib"), false)?; +//! # remove_file(Path::new("iceland-850.grib")).unwrap(); +//! # Ok(()) //! # } //! ``` //! - -//! -//! //! ## Changes in version 0.14 //! //! //! ## Feature Flags //! -//! - `ndarray` - enables support for converting [`KeyedMessage`] to [`ndarray::Array`]. +//! - `ndarray` - enables support for converting [`CodesMessage`](codes_message::CodesMessage) to [`ndarray::Array`]. //! This feature is enabled by default. It is currently tested only with simple lat-lon grids. //! -//! - `experimental_index` - **⚠️ This feature is experimental and might be unsafe in some contexts ⚠️** -//! This flag enables support for creating and using index files for GRIB files. -//! If you want to use it, please read the information provided in [`codes_index`] documentation. -//! //! - `docs` - builds the crate without linking ecCodes, particularly useful when building the documentation //! on [docs.rs](https://docs.rs/). For more details check documentation of [eccodes-sys](https://crates.io/crates/eccodes-sys). //! @@ -236,7 +259,7 @@ pub mod keys_iterator; mod pointer_guard; -pub use codes_handle::{CodesFile, ProductKind}; +pub use codes_handle::{ArcMessageIter, CodesFile, ProductKind, RefMessageIter}; pub use codes_message::{ArcMessage, BufMessage, KeyRead, KeyWrite, RefMessage}; pub use codes_nearest::{CodesNearest, NearestGridpoint}; pub use errors::CodesError; From 99ac939b46a577c70e22326f06e977eebf040252 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 12 Jan 2026 09:17:12 +0100 Subject: [PATCH 55/65] finish fixing docs in lib.rs --- src/lib.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 0135833..5b968b5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -233,7 +233,13 @@ //! ``` //! //! ## Changes in version 0.14 -//! +//! +//! 1. `experimental_index` feature has been removed - users are encouraged to create their own indexes as shown above or use iterator filtering +//! 2. `message_ndarray` feature has been renamed to `ndarray` +//! 3. `CodesHandle` has been renamed to `CodesFile` +//! 4. `KeyedMessage` has been replaced with generic `CodesMessage` - `RefMessage` has the most similar behaviour to `KeyedMessage`] +//! 5. `write_key` is now `write_key_unchecked` +//! 6. Dependency on `FallibleStreamingIterator` has been removed //! //! ## Feature Flags //! From c515d2605f31937d20a132ed472f1eda80d60dd5 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 12 Jan 2026 10:13:15 +0100 Subject: [PATCH 56/65] rename codes_handle module to codes_file and finish docs for that module --- benches/main.rs | 2 +- src/{codes_handle => codes_file}/iterator.rs | 76 ++++++- src/{codes_handle => codes_file}/mod.rs | 201 +++++++------------ src/codes_message/mod.rs | 2 +- src/codes_message/ndarray.rs | 2 +- src/codes_message/read.rs | 2 +- src/codes_message/write.rs | 2 +- src/intermediate_bindings/codes_handle.rs | 2 +- src/keys_iterator.rs | 2 +- src/lib.rs | 6 +- 10 files changed, 148 insertions(+), 149 deletions(-) rename src/{codes_handle => codes_file}/iterator.rs (76%) rename src/{codes_handle => codes_file}/mod.rs (57%) diff --git a/benches/main.rs b/benches/main.rs index 12b821a..9fc90ff 100644 --- a/benches/main.rs +++ b/benches/main.rs @@ -1,6 +1,6 @@ use criterion::{Criterion, criterion_group, criterion_main}; use eccodes::FallibleIterator; -use eccodes::codes_handle::{CodesFile, ProductKind}; +use eccodes::codes_file::{CodesFile, ProductKind}; use std::hint::black_box; use std::path::Path; diff --git a/src/codes_handle/iterator.rs b/src/codes_file/iterator.rs similarity index 76% rename from src/codes_handle/iterator.rs rename to src/codes_file/iterator.rs index 4de723c..fa57554 100644 --- a/src/codes_handle/iterator.rs +++ b/src/codes_file/iterator.rs @@ -4,29 +4,59 @@ use crate::{ArcMessage, CodesFile, RefMessage, errors::CodesError}; use std::fmt::Debug; use std::sync::{Arc, Mutex}; -/// `KeyedMessageGenerator` implements [`FallibleIterator`](codes_handle::KeyedMessageGenerator#impl-FallibleIterator-for-KeyedMessageGenerator%3C'ch,+S%3E) -/// which allows you to iterate over messages in the file. The iterator returns `KeyedMessage` with lifetime tied to the lifetime of `CodesHandle`, -/// that is `KeyedMessage` cannot outlive the `CodesHandle` it was generated from. If you need to prolong its lifetime, you can use -/// [`try_clone()`](KeyedMessage::try_clone), but that comes with performance and memory overhead. +/// Iterator over messages in `CodesFile` which returns [`RefMessage`] with lifetime tied to the `CodesFile`. +/// +/// This structure implements [`FallibleIterator`] which allows you to iterate over messages in the file. +/// The iterator returns [`RefMessage`] with lifetime tied to the lifetime of `CodesFile`, that is `RefMessage` +/// cannot outlive the `CodesFile` it was generated from. +/// +/// Creating this iter requires `CodesFile` to be mutable. +/// +/// If you need to share the message(s) across threads use [`ArcMessageIter`]. +/// +/// If you need a longer lifetime or want to modify the message, use [`try_clone()`](RefMessage::try_clone). +/// +/// ## Example +/// +/// ``` +/// use eccodes::{CodesFile, ProductKind, FallibleIterator}; +/// # +/// # fn main() -> anyhow::Result<()> { +/// // Open the file +/// let mut file = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; +/// +/// // Create RefMessageIter +/// let mut msg_iter = file.ref_message_iter(); +/// +/// // Now we can access the first message with next() +/// // Note that FallibleIterator must be in scope +/// let _msg = msg_iter.next()?; +/// +/// // Note we cannot drop the file here, because we first need to drop the message +/// // drop(file); +/// # Ok(()) +/// # } +/// ``` #[derive(Debug)] pub struct RefMessageIter<'a, D: Debug> { codes_file: &'a mut CodesFile, } impl CodesFile { + /// Generates [`RefMessageIter`] that allows to access messages as references to their parent file. pub fn ref_message_iter(&mut self) -> RefMessageIter<'_, D> { RefMessageIter { codes_file: self } } } -/// # Errors -/// -/// The `next()` will return [`CodesInternal`](crate::errors::CodesInternal) -/// when internal ecCodes function returns non-zero code. impl<'ch, D: Debug> FallibleIterator for RefMessageIter<'ch, D> { type Item = RefMessage<'ch>; type Error = CodesError; + /// # Errors + /// + /// The method will return [`CodesInternal`](crate::errors::CodesInternal) + /// when internal ecCodes function returns non-zero code. fn next(&mut self) -> Result, Self::Error> { let eccodes_handle = self.codes_file.generate_codes_handle()?; @@ -38,11 +68,28 @@ impl<'ch, D: Debug> FallibleIterator for RefMessageIter<'ch, D> { } } +/// Iterator over messages in `CodesFile` which returns [`ArcMessage`] which can be shared across threads. +/// +/// `ArcMessage` implements `Send + Sync` so it can be both moved to thread (for example, to read messages in parallel) +/// or shared across threads (when wrapped in [`Arc`]). +/// +/// This structure implements [`FallibleIterator`] - see the documentation for information how that differs from a standard `Iter``. +/// +/// Creating this iter does not require `CodesFile` to be mutable, because it takes ownership over the `CodesFile`. +/// +/// If you don't need to share the message, use [`RefMessageIter`] to avoid the performance overhead of [`Arc`]. +/// +/// If you want to modify the message, use [`try_clone()`](RefMessage::try_clone). +/// +/// ## Example +/// +/// See the second example in the main crate description for example usage of `ArcMessageIter`. #[derive(Debug)] pub struct ArcMessageIter { codes_file: Arc>>, } impl CodesFile { + /// Generates [`ArcMessageIter`] that allows to access messages that can be shared across threads. pub fn arc_message_iter(self) -> ArcMessageIter { ArcMessageIter { codes_file: Arc::new(Mutex::new(self)), @@ -55,12 +102,21 @@ impl FallibleIterator for ArcMessageIter { type Error = CodesError; + /// # Errors + /// + /// The method will return [`CodesInternal`](crate::errors::CodesInternal) + /// when internal ecCodes function returns non-zero code. + /// + /// # Panics + /// + /// This method internally uses a Mutex to access `CodesFile`, which can panic when poisoned, + /// but thers is no path in which you can get to the state of poisoned mutex, while still able to access this method. fn next(&mut self) -> Result, Self::Error> { let eccodes_handle = self .codes_file .lock() // This mutex can be poisoned only when thread that holds ArcMessageIter panics, which would make using the mutex impossible") - .unwrap() + .expect("The mutex inside ArcMessageIter got poisoned") .generate_codes_handle()?; if eccodes_handle.is_null() { @@ -75,7 +131,7 @@ impl FallibleIterator for ArcMessageIter { mod tests { use crate::{ FallibleIterator, - codes_handle::{CodesFile, ProductKind}, + codes_file::{CodesFile, ProductKind}, codes_message::DynamicKeyType, }; use anyhow::{Context, Ok, Result}; diff --git a/src/codes_handle/mod.rs b/src/codes_file/mod.rs similarity index 57% rename from src/codes_handle/mod.rs rename to src/codes_file/mod.rs index b8c3294..0b943d1 100644 --- a/src/codes_handle/mod.rs +++ b/src/codes_file/mod.rs @@ -21,70 +21,16 @@ mod iterator; /// /// It can be constructed from: /// -/// - File path using [`new_from_file()`](CodesHandle::new_from_file) -/// - From memory buffer using [`new_from_memory()`](CodesHandle::new_from_memory) -/// - From GRIB index using [`new_from_index()`](CodesHandle::new_from_index) (with `experimental_index` feature enabled) +/// - File path using [`new_from_file()`](CodesFile::new_from_file) +/// - From memory buffer using [`new_from_memory()`](CodesFile::new_from_memory) /// /// Destructor for this structure does not panic, but some internal functions may rarely fail /// leading to bugs. Errors encountered in the destructor are logged with [`tracing`]. -/// -/// # `FallibleIterator` -/// -/// This structure implements [`FallibleIterator`](crate::FallibleStreamingIterator) trait which allows to access GRIB messages. -/// +/// /// To access GRIB messages the ecCodes library uses a method similar to a C-style iterator. /// It digests the `* FILE` multiple times, each time returning the `*mut codes_handle` -/// to a message inside the file. -/// -/// This behaviour is represented in this crate by `FallibleIterator`, because generating `KeyedMessage` can fail. -/// -/// ``` -/// use eccodes::{ProductKind, CodesHandle, KeyRead}; -/// # use std::path::Path; -/// // FallibleStreamingIterator must be in scope to use it -/// use eccodes::FallibleStreamingIterator; -/// # -/// # fn main() -> anyhow::Result<(), eccodes::errors::CodesError> { -/// let file_path = Path::new("./data/iceland-surface.grib"); -/// let product_kind = ProductKind::GRIB; -/// -/// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; -/// -/// // Print names of messages in the file -/// while let Some(message) = handle.next()? { -/// // The message must be unwraped as internal next() can fail -/// let key: String = message.read_key("name")?; -/// println!("{key}"); -/// -/// } -/// # Ok(()) -/// # } -/// ``` -/// -/// You can also manually collect the messages into a vector to use them later. -/// -/// ``` -/// use eccodes::{ProductKind, CodesHandle, KeyedMessage}; -/// # use eccodes::errors::CodesError; -/// # use std::path::Path; -/// use eccodes::FallibleStreamingIterator; -/// # -/// # fn main() -> anyhow::Result<(), eccodes::errors::CodesError> { -/// let file_path = Path::new("./data/iceland-surface.grib"); -/// let product_kind = ProductKind::GRIB; -/// -/// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; -/// -/// let mut handle_collected = vec![]; -/// -/// while let Some(msg) = handle.next()? { -/// handle_collected.push(msg.try_clone()?); -/// } -/// # Ok(()) -/// # } -/// ``` -/// -/// All available methods for `CodesHandle` iterator can be found in [`FallibleStreamingIterator`](crate::FallibleStreamingIterator) trait. +/// to a message inside the file. Therefore in this crate access messages from `CodesFile` +/// use [`ref_message_iter()`](CodesFile::ref_message_iter) or [`arc_message_iter()`](CodesFile::arc_message_iter). #[derive(Debug)] pub struct CodesFile { // fields are dropped from top @@ -122,41 +68,38 @@ pub enum ProductKind { } impl CodesFile { - ///Opens file at given [`Path`] as selected [`ProductKind`] and contructs `CodesHandle`. - /// - ///## Example - /// - ///``` - ///# use eccodes::codes_handle::{ProductKind, CodesHandle}; - ///# use std::path::Path; - ///# fn main() -> anyhow::Result<()> { - ///let file_path = Path::new("./data/iceland.grib"); - ///let product_kind = ProductKind::GRIB; - /// - ///let handle = CodesHandle::new_from_file(file_path, product_kind)?; - /// # Ok(()) - /// # } - ///``` - /// - ///The function creates [`fs::File`](std::fs::File) from provided path and utilises - ///[`fdopen()`](https://man7.org/linux/man-pages/man3/fdopen.3.html) - ///to associate [`io::RawFd`](`std::os::unix::io::RawFd`) - ///with a stream represented by [`libc::FILE`](https://docs.rs/libc/0.2.101/libc/enum.FILE.html) pointer. - /// - ///The constructor takes as argument a [`path`](Path) instead of [`File`] - ///to ensure that `fdopen()` uses the same mode as [`File`]. - /// - /// The file stream and [`File`] are safely closed when `CodesHandle` is dropped. - /// - ///## Errors - ///Returns [`CodesError::FileHandlingInterrupted`] with [`io::Error`](std::io::Error) - ///when the file cannot be opened. - /// - ///Returns [`CodesError::LibcNonZero`] with [`errno`](errno::Errno) information - ///when the stream cannot be created from the file descriptor. - /// - ///Returns [`CodesError::Internal`] with error code - ///when internal [`codes_handle`] cannot be created. + /// Opens file at given [`Path`] as selected [`ProductKind`] and contructs `CodesFile`. + /// + /// ## Example + /// + /// ``` + /// # use eccodes::{ProductKind, CodesFile}; + /// # use std::path::Path; + /// # fn main() -> anyhow::Result<()> { + /// let handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; + /// # Ok(()) + /// # } + /// ``` + /// + /// The function creates [`fs::File`](std::fs::File) from provided path and utilises + /// [`fdopen()`](https://man7.org/linux/man-pages/man3/fdopen.3.html) + /// to associate [`io::RawFd`](`std::os::unix::io::RawFd`) + /// with a stream represented by [`libc::FILE`](https://docs.rs/libc/0.2.101/libc/enum.FILE.html) pointer. + /// + /// The constructor takes as argument a [`path`](Path) instead of [`File`] + /// to ensure that `fdopen()` uses the same mode as [`File`]. + /// + /// The file stream and [`File`] are safely closed when `CodesFile` is dropped. + /// + /// ## Errors + /// Returns [`CodesError::FileHandlingInterrupted`] with [`io::Error`](std::io::Error) + /// when the file cannot be opened. + /// + /// Returns [`CodesError::LibcNonZero`] with [`errno`](errno::Errno) information + /// when the stream cannot be created from the file descriptor. + /// + /// Returns [`CodesError::Internal`] with error code + /// when internal [`codes_handle`] cannot be created. #[instrument(level = "trace")] pub fn new_from_file + Debug>( file_path: P, @@ -173,40 +116,40 @@ impl CodesFile { } } impl CodesFile> { - ///Opens data in provided buffer as selected [`ProductKind`] and contructs `CodesHandle`. - /// - ///## Example - /// - ///``` - ///# async fn run() -> anyhow::Result<()> { - ///# use eccodes::{ProductKind, CodesHandle}; - ///# - ///let product_kind = ProductKind::GRIB; - ///let file_data = - /// reqwest::get("https://github.com/ScaleWeather/eccodes/blob/main/data/iceland.grib?raw=true") - /// .await? - /// .bytes() - /// .await? - /// .to_vec(); - /// - ///let handle = CodesHandle::new_from_memory(file_data, product_kind)?; - /// # Ok(()) - ///# } - ///``` - /// - ///The function associates data in memory with a stream - ///represented by [`libc::FILE`](https://docs.rs/libc/0.2.101/libc/enum.FILE.html) pointer - ///using [`fmemopen()`](https://man7.org/linux/man-pages/man3/fmemopen.3.html). - /// - ///The constructor takes full ownership of the data inside buffer, - ///which is safely dropped during the [`CodesHandle`] drop. - /// - ///## Errors - ///Returns [`CodesError::LibcNonZero`] with [`errno`](errno::Errno) information - ///when the file stream cannot be created. - /// - ///Returns [`CodesError::Internal`] with error code - ///when internal [`codes_handle`] cannot be created. + /// Opens data in provided buffer as selected [`ProductKind`] and contructs `CodesFile`. + /// + /// ## Example + /// + /// ``` + /// # async fn run() -> anyhow::Result<()> { + /// # use eccodes::{ProductKind, CodesFile}; + /// # + /// let product_kind = ProductKind::GRIB; + /// let file_data = + /// reqwest::get("https://github.com/ScaleWeather/eccodes/blob/main/data/iceland.grib?raw=true") + /// .await? + /// .bytes() + /// .await? + /// .to_vec(); + /// + /// let handle = CodesFile::new_from_memory(file_data, product_kind)?; + /// # Ok(()) + /// # } + /// ``` + /// + /// The function associates data in memory with a stream + /// represented by [`libc::FILE`](https://docs.rs/libc/0.2.101/libc/enum.FILE.html) pointer + /// using [`fmemopen()`](https://man7.org/linux/man-pages/man3/fmemopen.3.html). + /// + /// The constructor takes full ownership of the data inside buffer, + /// which is safely dropped during the [`CodesFile`] drop. + /// + /// ## Errors + /// Returns [`CodesError::LibcNonZero`] with [`errno`](errno::Errno) information + /// when the file stream cannot be created. + /// + /// Returns [`CodesError::Internal`] with error code + /// when internal [`codes_handle`] cannot be created. #[instrument(level = "trace")] pub fn new_from_memory( file_data: Vec, @@ -260,7 +203,7 @@ fn open_with_fmemopen(file_data: &[u8]) -> Result<*mut FILE, CodesError> { #[cfg(test)] mod tests { - use crate::codes_handle::{CodesFile, ProductKind}; + use crate::codes_file::{CodesFile, ProductKind}; use anyhow::{Context, Result}; use eccodes_sys::ProductKind_PRODUCT_GRIB; use fallible_iterator::FallibleIterator; diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index 88d3fd4..55d4ff9 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -162,7 +162,7 @@ impl Drop for CodesMessage

{ #[cfg(test)] mod tests { - use crate::codes_handle::{CodesFile, ProductKind}; + use crate::codes_file::{CodesFile, ProductKind}; use anyhow::{Context, Result}; use fallible_iterator::FallibleIterator; use std::path::Path; diff --git a/src/codes_message/ndarray.rs b/src/codes_message/ndarray.rs index 8b0df6d..0f684ce 100644 --- a/src/codes_message/ndarray.rs +++ b/src/codes_message/ndarray.rs @@ -146,7 +146,7 @@ mod tests { use super::*; use crate::ProductKind; - use crate::codes_handle::CodesFile; + use crate::codes_file::CodesFile; use crate::codes_message::DynamicKeyType; use std::path::Path; diff --git a/src/codes_message/read.rs b/src/codes_message/read.rs index 6b65652..5a2e74a 100644 --- a/src/codes_message/read.rs +++ b/src/codes_message/read.rs @@ -259,7 +259,7 @@ impl CodesMessage

{ mod tests { use anyhow::{Context, Result}; - use crate::codes_handle::{CodesFile, ProductKind}; + use crate::codes_file::{CodesFile, ProductKind}; use crate::{CodesError, KeyRead}; use crate::{FallibleIterator, codes_message::DynamicKeyType}; use std::path::Path; diff --git a/src/codes_message/write.rs b/src/codes_message/write.rs index ef34571..b64c9b0 100644 --- a/src/codes_message/write.rs +++ b/src/codes_message/write.rs @@ -133,7 +133,7 @@ mod tests { use crate::{ KeyRead, - codes_handle::{CodesFile, ProductKind}, + codes_file::{CodesFile, ProductKind}, codes_message::{DynamicKeyType, KeyWrite}, }; use std::{fs::remove_file, path::Path}; diff --git a/src/intermediate_bindings/codes_handle.rs b/src/intermediate_bindings/codes_handle.rs index 124f2c1..bb92acd 100644 --- a/src/intermediate_bindings/codes_handle.rs +++ b/src/intermediate_bindings/codes_handle.rs @@ -8,7 +8,7 @@ use libc::FILE; use tracing::instrument; use crate::{ - codes_handle::ProductKind, errors::CodesError, intermediate_bindings::error_code_to_result, + codes_file::ProductKind, errors::CodesError, intermediate_bindings::error_code_to_result, pointer_guard, }; diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index c198892..1cc47ab 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -223,7 +223,7 @@ mod tests { use anyhow::{Context, Result}; use crate::FallibleIterator; - use crate::codes_handle::{CodesFile, ProductKind}; + use crate::codes_file::{CodesFile, ProductKind}; use std::path::Path; use super::KeysIteratorFlags; diff --git a/src/lib.rs b/src/lib.rs index 5b968b5..67ef053 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,6 @@ #![warn(clippy::pedantic)] #![allow(clippy::cast_possible_wrap)] -// #![warn(missing_docs)] +#![warn(missing_docs)] #![warn(clippy::cargo)] #![warn(clippy::perf)] #![warn(clippy::doc_lazy_continuation)] @@ -256,7 +256,7 @@ //! features = ["eccodes/docs"] //! ``` -pub mod codes_handle; +pub mod codes_file; pub mod codes_message; pub mod codes_nearest; pub mod errors; @@ -265,7 +265,7 @@ pub mod keys_iterator; mod pointer_guard; -pub use codes_handle::{ArcMessageIter, CodesFile, ProductKind, RefMessageIter}; +pub use codes_file::{ArcMessageIter, CodesFile, ProductKind, RefMessageIter}; pub use codes_message::{ArcMessage, BufMessage, KeyRead, KeyWrite, RefMessage}; pub use codes_nearest::{CodesNearest, NearestGridpoint}; pub use errors::CodesError; From 508adef11701e070bd85f731f71cf6a670f23f56 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 12 Jan 2026 10:44:48 +0100 Subject: [PATCH 57/65] finish docs for codes_message module --- src/codes_file/iterator.rs | 2 +- src/codes_file/mod.rs | 10 ++--- src/codes_message/clone.rs | 4 +- src/codes_message/mod.rs | 76 ++++++++++++++++++------------------ src/codes_message/ndarray.rs | 6 +-- src/codes_message/read.rs | 50 ++++++------------------ src/codes_message/write.rs | 74 ++++++++++++++++------------------- src/codes_nearest.rs | 14 +++---- src/errors.rs | 4 +- src/keys_iterator.rs | 14 +++---- src/lib.rs | 12 +++--- tests/example.rs | 14 +++++++ 12 files changed, 130 insertions(+), 150 deletions(-) create mode 100644 tests/example.rs diff --git a/src/codes_file/iterator.rs b/src/codes_file/iterator.rs index fa57554..79a949a 100644 --- a/src/codes_file/iterator.rs +++ b/src/codes_file/iterator.rs @@ -307,7 +307,7 @@ mod tests { } // Now unwrap and access the first and only element of resulting vector - // Find nearest modifies internal KeyedMessage fields so we need mutable reference + // Find nearest modifies internal CodesMessage fields so we need mutable reference let level = &level[0]; let short_name = level.read_key_dynamic("shortName")?; diff --git a/src/codes_file/mod.rs b/src/codes_file/mod.rs index 0b943d1..1d3cf3f 100644 --- a/src/codes_file/mod.rs +++ b/src/codes_file/mod.rs @@ -1,4 +1,4 @@ -//! Definition and constructors of `CodesHandle` +//! Definition and constructors of `CodesFile` //! used for accessing GRIB files use crate::{CodesError, intermediate_bindings::codes_handle_new_from_file, pointer_guard}; @@ -40,17 +40,17 @@ pub struct CodesFile { } // 2024-07-26 -// Previously CodesHandle had implemented Drop which called libc::fclose() +// Previously CodesFile had implemented Drop which called libc::fclose() // but that closed the file descriptor and interfered with rust's fs::file destructor. // // To my best understanding the purpose of destructor is to clear memory and remove // any pointers that would be dangling. // -// The only pointer that is handed out of CodesHandle is &KeyedMessage, which is tied -// to CodesHandle through lifetimes, so if we destruct CodesHandle that pointer is first +// The only pointer that is handed out of CodesFile is &CodesMessage, which is tied +// to CodesFile through lifetimes, so if we destruct CodesFile that pointer is first // destructed as well. Source pointer is only used internally so we don't need to worry about it. // -// Clearing the memory is handled on ecCodes side by KeyedMessage/CodesIndex destructors +// Clearing the memory is handled on ecCodes side by CodesMessage/CodesIndex destructors // and on rust side by destructors of data_container we own. impl CodesFile { diff --git a/src/codes_message/clone.rs b/src/codes_message/clone.rs index 630011a..1043dd6 100644 --- a/src/codes_message/clone.rs +++ b/src/codes_message/clone.rs @@ -4,10 +4,10 @@ use crate::{ use std::fmt::Debug; impl CodesMessage

{ - /// Custom function to clone the `KeyedMessage` and `AtomicMessage`. + /// Custom function to clone the `CodesMessage`. /// /// **Be careful of the memory overhead!** ecCodes (when reading from file) defers reading the data into memory - /// if possible. Simply creating `KeyedMessage` or even reading some keys will use only a little of memory. + /// if possible. Simply creating `CodesMessage` or even reading some keys will use only a little of memory. /// This function **will** read the whole message into the memory, which can be of a significant size for big grids. /// /// # Errors diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index 55d4ff9..cdb1dd3 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -1,4 +1,4 @@ -//! Definition of `KeyedMessage` and its associated functions +//! Definition of `CodesMessage` and its associated functions //! used for reading and writing data of given variable from GRIB file mod clone; @@ -25,62 +25,48 @@ use tracing::{Level, event, instrument}; use crate::{CodesFile, intermediate_bindings::codes_handle_delete}; -/// Structure that provides access to the data contained in the GRIB file, which directly corresponds to the message in the GRIB file +/// Base structure for [`RefMessage`], [`ArcMessage`] and [`BufMessage`] that provides access to the data +/// contained in the GRIB file, which directly corresponds to the message in the GRIB file. /// -/// **Usage examples are provided in documentation of each method.** +/// `RefMessage` comes with no performance overhead but has its liftime tied to its parent `CodesFile`. +/// +/// `ArcMessage` can be moved and shared across threads. +/// +/// `BufMessage` is independent of its parent and can be edited with [`KeyWrite`] methods. +/// +/// **Usage examples are provided in documentation of implemented methods.** /// /// You can think about the message as a container of data corresponding to a single variable /// at given date, time and level. In ecCodes the message is represented as a collection of unique /// key-value pairs. /// -/// You can read a `Key` with static types using [`read_key()`](KeyRead::read_key()) or with [`DynamicKeyType`] using[`read_key_dynamic()`](KeyedMessage::read_key_dynamic()) +/// You can read a `Key` with static types using [`read_key()`](KeyRead::read_key()) or with [`DynamicKeyType`] using[`read_key_dynamic()`](CodesMessage::read_key_dynamic()) /// To iterate over all key names use [`KeysIterator`](crate::KeysIterator). You can also modify the message using -/// [`write_key()`](KeyWrite::write_key()). This crate can successfully read all keys from ERA5 and GFS files. +/// [`write_key_unchecked()`](KeyWrite::write_key_unchecked()) (only available for `BufMessage`). +/// +/// [`CodesNearest`](crate::CodesNearest) can be used to find nearest gridpoints for given coordinates in the `CodesMessage`. +/// +/// To write the message to file use [`write_to_file()`](CodesMessage::write_to_file). /// /// If you are interested only in getting data values from the message you can use -/// [`to_ndarray()`](KeyedMessage::to_ndarray) from the [`message_ndarray`](crate::message_ndarray) module. +/// [`to_ndarray()`](CodesMessage::to_ndarray) available in `ndarray` feature. /// /// Some of the useful keys are: `validityDate`, `validityTime`, `level`, `typeOfLevel`, `shortName`, `units` and `values`. /// /// Note that names, types and availability of some keys can vary between platforms and ecCodes versions. You should test /// your code whenever changing the environment. /// -/// [`CodesNearest`](crate::CodesNearest) can be used to find nearest gridpoints for given coordinates in the `KeyedMessage`. -/// -/// Most of `KeyedMessage` methods (except for writing) can be used directly with `&KeyedMessage` -/// returned by `CodesHandle` iterator, which provides the best performance. -/// When mutable access or longer liftime is needed the message can be cloned with [`try_clone`](KeyedMessage::try_clone) -/// Note that cloning comes with a performance and memory overhead. -/// You should take care that your system has enough memory before cloning. -/// /// Destructor for this structure does not panic, but some internal functions may rarely fail -/// leading to bugs. Errors encountered in desctructor the are logged with [`log`]. -pub type RefMessage<'ch> = CodesMessage>; - -/// Because standard `KeyedMessage` is not Copy or Clone it can provide access methods without -/// requiring `&mut self`. As `AtomicMessage` implements `Send + Sync` this exclusive method access is not -/// guaranteed with just `&self`. `AtomicMessage` also implements a minimal subset of functionalities -/// to limit the risk of some internal ecCodes functions not being thread-safe. -pub type ArcMessage = CodesMessage>; - -unsafe impl Send for ArcMessage {} -unsafe impl Sync for ArcMessage {} - -pub type BufMessage = CodesMessage; - -unsafe impl Send for BufMessage {} -unsafe impl Sync for BufMessage {} - -/// All messages use this struct for operations. +/// leading to bugs. Errors encountered in desctructor the are reported via [`tracing`]. #[derive(Debug)] pub struct CodesMessage { pub(crate) _parent: P, pub(crate) message_handle: *mut codes_handle, } -/// This is a little unintuitive, but we use `()` here to not unnecessarily pollute -/// KeyedMessage and derived types with generics, because `PhantomData` is needed -/// only for lifetime restriction and we tightly control how `KeyedMessage` is created. +// This is a little unintuitive, but we use `()` here to not unnecessarily pollute +// CodesMessage and derived types with generics, because `PhantomData` is needed +// only for lifetime restriction and we tightly control how `CodesMessage` is created. #[derive(Debug, Hash, PartialEq, PartialOrd)] #[doc(hidden)] pub struct RefParent<'ch>(PhantomData<&'ch ()>); @@ -95,6 +81,21 @@ pub struct ArcParent { _arc_handle: Arc>>, } +/// [`CodesMessage`] that has its liftime tied to its parent `CodesFile`. +pub type RefMessage<'ch> = CodesMessage>; + +/// [`CodesMessage`] that can be moved and shared across threads. +pub type ArcMessage = CodesMessage>; + +unsafe impl Send for ArcMessage {} +unsafe impl Sync for ArcMessage {} + +/// [`CodesMessage`] that is independent of its parent and can be edited with [`KeyWrite`] methods. +pub type BufMessage = CodesMessage; + +unsafe impl Send for BufMessage {} +unsafe impl Sync for BufMessage {} + impl RefMessage<'_> { pub(crate) fn new(handle: *mut codes_handle) -> Self { RefMessage { @@ -116,7 +117,6 @@ impl ArcMessage { } impl BufMessage { - /// This could be a From, but that would be less idiomatic and would expose interface that we don't want exposed.I pub(crate) fn new(handle: *mut codes_handle) -> Self { BufMessage { _parent: BufParent(), @@ -142,7 +142,7 @@ impl Drop for CodesMessage

{ /// /// # Panics /// - /// In debug + /// In debug mode when error is encountered. #[instrument(level = "trace")] fn drop(&mut self) { unsafe { @@ -152,7 +152,7 @@ impl Drop for CodesMessage

{ "codes_handle_delete() returned an error: {:?}", &error ); - debug_assert!(false, "Error in KeyedMessage::drop"); + debug_assert!(false, "Error in CodesMessage::drop"); }); } diff --git a/src/codes_message/ndarray.rs b/src/codes_message/ndarray.rs index 0f684ce..13a66cd 100644 --- a/src/codes_message/ndarray.rs +++ b/src/codes_message/ndarray.rs @@ -1,4 +1,4 @@ -//! Definitions for converting a `KeyedMessage` to ndarray +//! Definitions for converting a `CodesMessage` to ndarray use std::fmt::Debug; @@ -6,7 +6,7 @@ use ndarray::{Array2, Array3, s}; use crate::{CodesError, KeyRead, codes_message::CodesMessage, errors::MessageNdarrayError}; -/// Struct returned by [`KeyedMessage::to_lons_lats_values()`] method. +/// Struct returned by [`CodesMessage::to_lons_lats_values()`] method. /// The arrays are collocated, meaning that `longitudes[i, j]` and `latitudes[i, j]` are the coordinates of `values[i, j]`. #[derive(Clone, PartialEq, Debug, Default)] #[cfg_attr(docsrs, doc(cfg(feature = "ndarray")))] @@ -75,7 +75,7 @@ impl CodesMessage

{ } } - /// Same as [`KeyedMessage::to_ndarray()`] but returns the longitudes and latitudes alongside values. + /// Same as [`CodesMessage::to_ndarray()`] but returns the longitudes and latitudes alongside values. /// Fields are returned as separate arrays in [`RustyCodesMessage`]. /// /// Compared to `to_ndarray` this method has performance overhead as returned arrays may be cloned. diff --git a/src/codes_message/read.rs b/src/codes_message/read.rs index 5a2e74a..e469c1a 100644 --- a/src/codes_message/read.rs +++ b/src/codes_message/read.rs @@ -9,25 +9,21 @@ use crate::{ }, }; -/// Provides GRIB key reading capabilites. Implemented by [`KeyedMessage`] for all possible key types. +/// Provides GRIB key reading capabilites. Implemented by [`CodesMessage`] for all possible key types. pub trait KeyRead { - /// Tries to read a key of given name from [`KeyedMessage`]. This function checks if key native type + /// Tries to read a key of given name from [`CodesMessage`]. This function checks if key native type /// matches the requested type (ie. you cannot read integer as string, or array as a number). /// /// # Example /// /// ``` - /// # use eccodes::{ProductKind, CodesHandle, KeyRead}; - /// # use std::path::Path; + /// # use eccodes::{CodesFile, KeyRead, FallibleIterator, ProductKind}; /// # use anyhow::Context; - /// # use eccodes::FallibleStreamingIterator; /// # /// # fn main() -> anyhow::Result<()> { - /// # let file_path = Path::new("./data/iceland.grib"); - /// # let product_kind = ProductKind::GRIB; /// # - /// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - /// let message = handle.next()?.context("no message")?; + /// let mut file = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; + /// let message = file.ref_message_iter().next()?.context("no message")?; /// let short_name: String = message.read_key("shortName")?; /// /// assert_eq!(short_name, "msl"); @@ -53,33 +49,13 @@ pub trait KeyRead { /// /// This function is also useful for (not usually used) keys that return incorrect native type. /// - /// # Example - /// - /// ``` - /// # use eccodes::{ProductKind, CodesHandle, KeyRead}; - /// # use std::path::Path; - /// # use anyhow::Context; - /// # use eccodes::FallibleStreamingIterator; - /// # - /// # fn main() -> anyhow::Result<()> { - /// # let file_path = Path::new("./data/iceland.grib"); - /// # let product_kind = ProductKind::GRIB; - /// # - /// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - /// let message = handle.next()?.context("no message")?; - /// let short_name: String = message.read_key_unchecked("shortName")?; - /// - /// assert_eq!(short_name, "msl"); - /// # Ok(()) - /// # } - /// ``` - /// /// # Errors /// /// This function will return [`CodesInternal`](crate::errors::CodesInternal) if ecCodes fails to read the key. fn read_key_unchecked(&self, name: &str) -> Result; } +#[doc(hidden)] pub trait KeyPropertiesRead { fn get_key_size(&self, key_name: &str) -> Result; fn get_key_native_type(&self, key_name: &str) -> Result; @@ -169,7 +145,7 @@ pub enum DynamicKeyType { } impl CodesMessage

{ - /// Method to get a value of given key with [`DynamicKeyType`] from the `KeyedMessage`, if it exists. + /// Method to get a value of given key with [`DynamicKeyType`] from the `CodesMessage`, if it exists. /// /// In most cases you should use [`read_key()`](KeyRead::read_key) due to more predictive behaviour /// and simpler interface. @@ -188,17 +164,13 @@ impl CodesMessage

{ /// # Example /// /// ``` - /// use eccodes::{ProductKind, CodesHandle, DynamicKeyType}; - /// # use std::path::Path; + /// use eccodes::{ProductKind, CodesFile, DynamicKeyType, FallibleIterator}; /// # use anyhow::Context; - /// use eccodes::FallibleStreamingIterator; /// # /// # fn main() -> anyhow::Result<()> { - /// let file_path = Path::new("./data/iceland.grib"); - /// let product_kind = ProductKind::GRIB; - /// - /// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; - /// let message = handle.next()?.context("no message")?; + /// # + /// let mut file = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; + /// let message = file.ref_message_iter().next()?.context("no message")?; /// let message_short_name = message.read_key_dynamic("shortName")?; /// let expected_short_name = DynamicKeyType::Str("msl".to_string()); /// diff --git a/src/codes_message/write.rs b/src/codes_message/write.rs index b64c9b0..43a483f 100644 --- a/src/codes_message/write.rs +++ b/src/codes_message/write.rs @@ -9,33 +9,34 @@ use crate::{ }, }; -/// Provides GRIB key writing capabilites. Implemented by [`KeyedMessage`] for all possible key types. +/// Provides GRIB key writing capabilites. Implemented by [`CodesMessage`] for all possible key types. pub trait KeyWrite { + /// Writes key with given name and value to [`CodesMessage`] overwriting existing value, unless + /// the key is read-only. + /// + /// This function directly calls ecCodes ensuring only type and memory safety. /// Unchecked doesn't mean it's unsafe - just that there are no checks on Rust side in comparison to /// `read_key` which has such checks. - /// Writes key with given name and value to [`KeyedMessage`] overwriting existing value, unless - /// the key is read-only. This function directly calls ecCodes ensuring only type and memory safety. /// /// # Example /// /// ``` - /// # use eccodes::{ProductKind, CodesHandle, KeyWrite}; - /// # use std::path::Path; - /// # use anyhow::Context; - /// # use eccodes::FallibleStreamingIterator; - /// # - /// # fn main() -> anyhow::Result<()> { - /// # let file_path = Path::new("./data/iceland.grib"); - /// # let product_kind = ProductKind::GRIB; - /// # - /// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + /// # use anyhow::Context; + /// # use eccodes::{CodesFile, FallibleIterator, KeyWrite, ProductKind}; + /// # + /// # fn main() -> anyhow::Result<()> { + /// let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; /// - /// // CodesHandle iterator returns immutable messages. - /// // To edit a message it must be cloned. - /// let mut message = handle.next()?.context("no message")?.try_clone()?; - /// message.write_key("level", 1)?; - /// # Ok(()) - /// # } + /// // CodesFile iterator returns immutable messages. + /// // To edit a message it must be cloned. + /// let mut message = handle + /// .ref_message_iter() + /// .next()? + /// .context("no message")? + /// .try_clone()?; + /// message.write_key_unchecked("level", 1)?; + /// # Ok(()) + /// # } /// ``` /// /// # Errors @@ -69,7 +70,7 @@ impl_key_write!(codes_set_bytes, &[u8]); impl_key_write!(codes_set_string, &str); impl CodesMessage { - /// Function to write given `KeyedMessage` to a file at provided path. + /// Function to write given `CodesMessage` to a file at provided path. /// If file does not exists it will be created. /// If `append` is set to `true` file will be opened in append mode /// and no data will be overwritten (useful when writing mutiple messages to one file). @@ -77,27 +78,20 @@ impl CodesMessage { /// # Example /// /// ``` - /// use eccodes::{CodesHandle, KeyRead, ProductKind}; - /// # use eccodes::errors::CodesError; - /// use eccodes::FallibleStreamingIterator; - /// # use std::path::Path; - /// # use std::fs::remove_file; - /// # - /// # fn main() -> anyhow::Result<(), CodesError> { - /// let in_path = Path::new("./data/iceland-levels.grib"); - /// let out_path = Path::new("./data/iceland-800hPa.grib"); - /// - /// let mut handle = CodesHandle::new_from_file(in_path, ProductKind::GRIB)?; + /// # use eccodes::{CodesError, CodesFile, FallibleIterator, KeyRead, ProductKind}; + /// # use std::fs::remove_file; + /// # fn main() -> anyhow::Result<(), CodesError> { + /// let mut handle = CodesFile::new_from_file("./data/iceland-levels.grib", ProductKind::GRIB)?; /// - /// while let Some(msg) = handle.next()? { - /// let level: i64 = msg.read_key("level")?; - /// if level == 800 { - /// msg.write_to_file(out_path, true)?; - /// } - /// } - /// # remove_file(out_path)?; - /// # Ok(()) - /// # } + /// while let Some(msg) = handle.ref_message_iter().next()? { + /// let level: i64 = msg.read_key("level")?; + /// if level == 800 { + /// msg.write_to_file("./data/iceland-800hPa.grib", true)?; + /// } + /// } + /// # remove_file("./data/iceland-800hPa.grib")?; + /// # Ok(()) + /// } /// ``` /// /// # Errors diff --git a/src/codes_nearest.rs b/src/codes_nearest.rs index 3a32b24..ed58fbd 100644 --- a/src/codes_nearest.rs +++ b/src/codes_nearest.rs @@ -1,5 +1,5 @@ //! Definition and associated functions of `CodesNearest` -//! used for finding nearest gridpoints in `KeyedMessage` +//! used for finding nearest gridpoints in `CodesMessage` use std::{fmt::Debug, ptr::null_mut}; @@ -14,7 +14,7 @@ use crate::{ }, }; -/// The structure used to find nearest gridpoints in `KeyedMessage`. +/// The structure used to find nearest gridpoints in `CodesMessage`. #[derive(Debug)] pub struct CodesNearest<'a, P: Debug> { nearest_handle: *mut codes_nearest, @@ -33,13 +33,13 @@ pub struct NearestGridpoint { pub lon: f64, /// Distance between requested point and this gridpoint in kilometers pub distance: f64, - ///Value of parameter at this gridpoint contained by `KeyedMessage` in corresponding units + ///Value of parameter at this gridpoint contained by `CodesMessage` in corresponding units pub value: f64, } impl CodesMessage

{ - /// Creates a new instance of [`CodesNearest`] for the `KeyedMessage`. - /// [`CodesNearest`] can be used to find nearest gridpoints for given coordinates in the `KeyedMessage` + /// Creates a new instance of [`CodesNearest`] for the `CodesMessage`. + /// [`CodesNearest`] can be used to find nearest gridpoints for given coordinates in the `CodesMessage` /// by calling [`find_nearest()`](crate::CodesNearest::find_nearest). /// /// # Errors @@ -65,7 +65,7 @@ impl CodesNearest<'_, P> { ///### Example /// ///``` - /// use eccodes::{ProductKind, CodesHandle, KeyedMessage, KeysIteratorFlags}; + /// use eccodes::{ProductKind, CodesFile, CodesMessage, KeysIteratorFlags}; /// # use std::path::Path; /// use eccodes::FallibleStreamingIterator; /// # use anyhow::Context; @@ -73,7 +73,7 @@ impl CodesNearest<'_, P> { /// let file_path = Path::new("./data/iceland.grib"); /// let product_kind = ProductKind::GRIB; /// - /// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + /// let mut handle = CodesFile::new_from_file(file_path, product_kind)?; /// let msg = handle.next()?.context("no message")?; /// /// let c_nearest = msg.codes_nearest()?; diff --git a/src/errors.rs b/src/errors.rs index 61e7c99..a3d7b66 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -75,7 +75,7 @@ pub enum CodesError { /// Returned when function in `message_ndarray` module cannot convert /// the message to ndarray. Check [`MessageNdarrayError`] for more details. #[cfg(feature = "ndarray")] - #[error("error occured while converting KeyedMessage to ndarray {0}")] + #[error("error occured while converting CodesMessage to ndarray {0}")] NdarrayConvert(#[from] MessageNdarrayError), /// eccodes functions return errors as error codes and it is technically possible @@ -84,7 +84,7 @@ pub enum CodesError { UnrecognizedErrorCode(i32), /// Similarly to error codes, eccodes return key type as i32, so - /// it's technically possible that this code does not appear in [`NativeKeyType`](crate::intermediate_bindings::NativeKeyType) + /// it's technically possible that this code does not appear in internal native key types. #[error("Unrecognized native key type code: {0}")] UnrecognizedKeyTypeCode(i32), } diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index 1cc47ab..25f844b 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -1,4 +1,4 @@ -//! Definition of `KeysIterator` used for iterating through keys in `KeyedMessage` +//! Definition of `KeysIterator` used for iterating through keys in `CodesMessage` use eccodes_sys::codes_keys_iterator; use fallible_iterator::FallibleIterator; @@ -14,7 +14,7 @@ use crate::{ }, }; -/// Structure to iterate through key names in [`KeyedMessage`]. +/// Structure to iterate through key names in [`CodesMessage`]. /// /// Mainly useful to discover what keys are present inside the message. /// @@ -28,7 +28,7 @@ use crate::{ /// ## Example /// /// ``` -/// use eccodes::{ProductKind, CodesHandle, KeyedMessage, KeysIteratorFlags}; +/// use eccodes::{ProductKind, CodesFile, CodesMessage, KeysIteratorFlags}; /// # use std::path::Path; /// # use anyhow::Context; /// use eccodes::{FallibleIterator, FallibleStreamingIterator}; @@ -38,7 +38,7 @@ use crate::{ /// let file_path = Path::new("./data/iceland.grib"); /// let product_kind = ProductKind::GRIB; /// -/// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; +/// let mut handle = CodesFile::new_from_file(file_path, product_kind)?; /// let current_message = handle.next()?.context("no message")?; /// /// let mut keys_iter = current_message.default_keys_iterator()?; @@ -101,7 +101,7 @@ impl CodesMessage

{ /// # Example /// /// ``` - /// use eccodes::{ProductKind, CodesHandle, KeyedMessage, KeysIteratorFlags}; + /// use eccodes::{ProductKind, CodesFile, CodesMessage, KeysIteratorFlags}; /// # use std::path::Path; /// # use anyhow::Context; /// use eccodes::{FallibleIterator, FallibleStreamingIterator}; @@ -111,7 +111,7 @@ impl CodesMessage

{ /// let file_path = Path::new("./data/iceland.grib"); /// let product_kind = ProductKind::GRIB; /// - /// let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; + /// let mut handle = CodesFile::new_from_file(file_path, product_kind)?; /// let current_message = handle.next()?.context("no message")?; /// /// let flags = [ @@ -155,7 +155,7 @@ impl CodesMessage

{ }) } - /// Same as [`new_keys_iterator()`](KeyedMessage::new_keys_iterator) but with default + /// Same as [`new_keys_iterator()`](CodesMessage::new_keys_iterator) but with default /// parameters: [`AllKeys`](KeysIteratorFlags::AllKeys) flag and `""` namespace, /// yeilding iterator over all keys in the message. /// diff --git a/src/lib.rs b/src/lib.rs index 67ef053..409930d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -75,7 +75,7 @@ //! Almost all methods can be called on any `CodesMessage`, except for [`KeyWrite`] operations, which can be called only on [`BufMessage`] //! to avoid confusion if written keys are save to file or not. //! -//! Data contained by `KeyedMessage` is represented as *keys* (like in dictionary). +//! Data contained by `CodesMessage` is represented as *keys* (like in dictionary). //! Keys can be read with static types using [`read_key()`](KeyRead::read_key) or with [dynamic types](codes_message::DynamicKeyType) //! using [`read_key_dynamic()`](codes_message::CodesMessage::read_key_dynamic). //! To discover what keys are present in a message use [`KeysIterator`](KeysIterator). @@ -97,7 +97,7 @@ //! use eccodes::{CodesFile, FallibleIterator, KeyRead, ProductKind}; //! //! # fn main() -> anyhow::Result<()> { -//! // Open the GRIB file and create the CodesHandle +//! // Open the GRIB file and create the CodesFile //! let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; //! // Use iterator to find a message with shortName "msl" and typeOfLevel "surface" //! // We can use while let or for_each() to iterate over the messages @@ -165,7 +165,7 @@ //! //! #### Example 3 - Writing GRIB files and indexing //! -//! The crate provides basic support for setting `KeyedMessage` keys +//! The crate provides basic support for setting `CodesMessage` keys //! and writing GRIB files. To create a new custom message we need to copy //! existing one from other GRIB file, modify the keys and write to new file. //! @@ -180,7 +180,7 @@ //! use std::{collections::HashMap, fs::remove_file, path::Path}; //! //! # fn main() -> anyhow::Result<()> { -//! // Start by opening the file and creating CodesHandle +//! // Start by opening the file and creating CodesFile //! let file_path = Path::new("./data/iceland-levels.grib"); //! let mut handle = CodesFile::new_from_file(file_path, ProductKind::GRIB)?; //! @@ -237,7 +237,7 @@ //! 1. `experimental_index` feature has been removed - users are encouraged to create their own indexes as shown above or use iterator filtering //! 2. `message_ndarray` feature has been renamed to `ndarray` //! 3. `CodesHandle` has been renamed to `CodesFile` -//! 4. `KeyedMessage` has been replaced with generic `CodesMessage` - `RefMessage` has the most similar behaviour to `KeyedMessage`] +//! 4. `KeyedMessage` has been replaced with generic `CodesMessage` - `RefMessage` has the most similar behaviour to `CodesMessage`] //! 5. `write_key` is now `write_key_unchecked` //! 6. Dependency on `FallibleStreamingIterator` has been removed //! @@ -266,7 +266,7 @@ pub mod keys_iterator; mod pointer_guard; pub use codes_file::{ArcMessageIter, CodesFile, ProductKind, RefMessageIter}; -pub use codes_message::{ArcMessage, BufMessage, KeyRead, KeyWrite, RefMessage}; +pub use codes_message::{ArcMessage, BufMessage, KeyRead, KeyWrite, RefMessage, DynamicKeyType}; pub use codes_nearest::{CodesNearest, NearestGridpoint}; pub use errors::CodesError; pub use fallible_iterator::{FallibleIterator, IntoFallibleIterator}; diff --git a/tests/example.rs b/tests/example.rs new file mode 100644 index 0000000..d3be78c --- /dev/null +++ b/tests/example.rs @@ -0,0 +1,14 @@ +use eccodes::{CodesError, CodesFile, FallibleIterator, KeyRead, ProductKind}; +use std::fs::remove_file; +fn main() -> anyhow::Result<(), CodesError> { + let mut handle = CodesFile::new_from_file("./data/iceland-levels.grib", ProductKind::GRIB)?; + + while let Some(msg) = handle.ref_message_iter().next()? { + let level: i64 = msg.read_key("level")?; + if level == 800 { + msg.write_to_file("./data/iceland-800hPa.grib", true)?; + } + } + remove_file("./data/iceland-800hPa.grib")?; + Ok(()) +} From ed7e594177ffcabff14ea670a440cb3a89425b13 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 12 Jan 2026 11:08:49 +0100 Subject: [PATCH 58/65] finish miscellaneous docs --- src/codes_file/iterator.rs | 2 +- src/codes_nearest.rs | 17 ++++++--------- src/keys_iterator.rs | 44 ++++++++++++++------------------------ tests/example.rs | 23 ++++++++++---------- 4 files changed, 34 insertions(+), 52 deletions(-) diff --git a/src/codes_file/iterator.rs b/src/codes_file/iterator.rs index 79a949a..185f1f9 100644 --- a/src/codes_file/iterator.rs +++ b/src/codes_file/iterator.rs @@ -73,7 +73,7 @@ impl<'ch, D: Debug> FallibleIterator for RefMessageIter<'ch, D> { /// `ArcMessage` implements `Send + Sync` so it can be both moved to thread (for example, to read messages in parallel) /// or shared across threads (when wrapped in [`Arc`]). /// -/// This structure implements [`FallibleIterator`] - see the documentation for information how that differs from a standard `Iter``. +/// This structure implements [`FallibleIterator`] - see the documentation for information how that differs from a standard `Iter`. /// /// Creating this iter does not require `CodesFile` to be mutable, because it takes ownership over the `CodesFile`. /// diff --git a/src/codes_nearest.rs b/src/codes_nearest.rs index ed58fbd..ace5c73 100644 --- a/src/codes_nearest.rs +++ b/src/codes_nearest.rs @@ -65,20 +65,15 @@ impl CodesNearest<'_, P> { ///### Example /// ///``` - /// use eccodes::{ProductKind, CodesFile, CodesMessage, KeysIteratorFlags}; - /// # use std::path::Path; - /// use eccodes::FallibleStreamingIterator; /// # use anyhow::Context; + /// # use eccodes::{CodesFile, FallibleIterator, ProductKind}; /// # fn main() -> anyhow::Result<()> { - /// let file_path = Path::new("./data/iceland.grib"); - /// let product_kind = ProductKind::GRIB; + /// let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; + /// let msg = handle.ref_message_iter().next()?.context("no message")?; /// - /// let mut handle = CodesFile::new_from_file(file_path, product_kind)?; - /// let msg = handle.next()?.context("no message")?; - /// - /// let c_nearest = msg.codes_nearest()?; + /// let mut c_nearest = msg.codes_nearest()?; /// let out = c_nearest.find_nearest(64.13, -21.89)?; - /// # Ok(()) + /// # Ok(()) /// # } ///``` /// @@ -109,7 +104,7 @@ impl CodesNearest<'_, P> { impl Drop for CodesNearest<'_, P> { /// # Panics /// - /// In debug + /// In debug when error is encountered #[instrument(level = "trace")] fn drop(&mut self) { unsafe { diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index 25f844b..399cf04 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -28,26 +28,19 @@ use crate::{ /// ## Example /// /// ``` -/// use eccodes::{ProductKind, CodesFile, CodesMessage, KeysIteratorFlags}; -/// # use std::path::Path; -/// # use anyhow::Context; -/// use eccodes::{FallibleIterator, FallibleStreamingIterator}; -/// # -/// # fn main() -> anyhow::Result<()> { -/// # -/// let file_path = Path::new("./data/iceland.grib"); -/// let product_kind = ProductKind::GRIB; -/// -/// let mut handle = CodesFile::new_from_file(file_path, product_kind)?; -/// let current_message = handle.next()?.context("no message")?; -/// -/// let mut keys_iter = current_message.default_keys_iterator()?; -/// -/// while let Some(key_name) = keys_iter.next()? { -/// println!("{key_name}"); -/// } -/// # Ok(()) -/// # } +/// # use anyhow::Context; +/// # use eccodes::{CodesFile, FallibleIterator, ProductKind}; +/// # fn main() -> anyhow::Result<()> { +/// let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; +/// let mut current_message = handle.ref_message_iter().next()?.context("no message")?; +/// +/// let mut keys_iter = current_message.default_keys_iterator()?; +/// +/// while let Some(key_name) = keys_iter.next()? { +/// println!("{key_name}"); +/// } +/// # Ok(()) +/// # } /// ``` /// /// ## Errors @@ -101,18 +94,13 @@ impl CodesMessage

{ /// # Example /// /// ``` - /// use eccodes::{ProductKind, CodesFile, CodesMessage, KeysIteratorFlags}; - /// # use std::path::Path; + /// use eccodes::{ProductKind, CodesFile, KeysIteratorFlags, FallibleIterator}; /// # use anyhow::Context; - /// use eccodes::{FallibleIterator, FallibleStreamingIterator}; /// # /// # fn main() -> anyhow::Result<()> { /// # - /// let file_path = Path::new("./data/iceland.grib"); - /// let product_kind = ProductKind::GRIB; - /// - /// let mut handle = CodesFile::new_from_file(file_path, product_kind)?; - /// let current_message = handle.next()?.context("no message")?; + /// let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; + /// let mut current_message = handle.ref_message_iter().next()?.context("no message")?; /// /// let flags = [ /// KeysIteratorFlags::AllKeys, diff --git a/tests/example.rs b/tests/example.rs index d3be78c..dd89bf0 100644 --- a/tests/example.rs +++ b/tests/example.rs @@ -1,14 +1,13 @@ -use eccodes::{CodesError, CodesFile, FallibleIterator, KeyRead, ProductKind}; -use std::fs::remove_file; -fn main() -> anyhow::Result<(), CodesError> { - let mut handle = CodesFile::new_from_file("./data/iceland-levels.grib", ProductKind::GRIB)?; +use anyhow::Context; +use eccodes::{CodesFile, FallibleIterator, ProductKind}; +fn main() -> anyhow::Result<()> { +let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; +let mut current_message = handle.ref_message_iter().next()?.context("no message")?; - while let Some(msg) = handle.ref_message_iter().next()? { - let level: i64 = msg.read_key("level")?; - if level == 800 { - msg.write_to_file("./data/iceland-800hPa.grib", true)?; - } - } - remove_file("./data/iceland-800hPa.grib")?; - Ok(()) +let mut keys_iter = current_message.default_keys_iterator()?; + +while let Some(key_name) = keys_iter.next()? { + println!("{key_name}"); +} +Ok(()) } From b72f916a9d3d8cfc41e86cfbc5a38e97c6954d98 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 12 Jan 2026 11:17:33 +0100 Subject: [PATCH 59/65] finish readme --- README.md | 96 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 54 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 91aad85..a1d7ce2 100644 --- a/README.md +++ b/README.md @@ -60,58 +60,72 @@ export LD_LIBRARY_PATH=/lib ### Working with GRIB files -To access a GRIB file you need to create `CodesHandle` with one of provided constructors. +To access a GRIB file you need to create `CodesFile` with one of the provided constructors. -GRIB files consist of messages which represent data fields at specific time and level. -Messages are represented by the `KeyedMessage` structure. +ecCodes represents GRIB files as a set of separate messages, each containing data fields at specific time and level. +Messages are represented here by a generic `CodesMessage` structure, but you shouldn't use it directly. +Instead use `RefMessage`, `ArcMessage` or `BufMessage` to operations - check the docs for more information when to use each. -`CodesHandle` implements `FallibleStreamingIterator` -which allows you to iterate over messages in the file. The iterator returns `&KeyedMessage` which valid until next iteration. -`KeyedMessage` implements several methods to access the data as needed, most of those can be called directly on `&KeyedMessage`. -You can also use `try_clone()` to clone the message and prolong its lifetime. +To obtain `CodesMessage` from `CodesFile` you need to create an instance of `RefMessageIter` or `ArcMessageIter` using +`CodesFile::ref_message_iter()` or `CodesFile::arc_message_iter()`. +Those structures implement `FallibleIterator`, please check its documentation if you are not familiar with it. -Data defining and contained by `KeyedMessage` is represented by `Key`s. -You can read them directly with `read_key()`, use `KeysIterator` -to iterate over them or use `CodesNearest` to get the values of four nearest gridpoints for given coordinates. +`CodesMessage` implements several methods to access the data as needed, most of those can be called directly. +Almost all methods can be called on any `CodesMessage`, except for `KeyWrite` operations, which can be called only on `BufMessage` +to avoid confusion if written keys are save to file or not. -You can also modify the message with `write_key()` and write it to a new file with `write_to_file()`. +Data contained by `CodesMessage` is represented as *keys* (like in dictionary). +Keys can be read with static types using `read_key()` or with dynamic types +using `read_key_dynamic()`. +To discover what keys are present in a message use `KeysIterator`. -#### Example +With `ndarray` feature (enabled by default) you can also read `CodesMessage` into `ndarray` using `to_ndarray()` +and `to_lons_lats_values()`. -```rust -// We are reading the mean sea level pressure for 4 gridpoints -// nearest to Reykjavik (64.13N, -21.89E) for 1st June 2021 00:00 UTC -// from ERA5 Climate Reanalysis +You can use `CodesNearest` to get the data values of four nearest gridpoints for given coordinates. + +To modify keys within the message use `write_key_unchecked()`. +To save that modified message use `write_to_file()`. -use eccodes::{ProductKind, CodesHandle, KeyType}; -use eccodes::FallibleStreamingIterator; +#### Example 1 - Reading GRIB file -// Open the GRIB file and create the CodesHandle -let file_path = Path::new("./data/iceland.grib"); -let product_kind = ProductKind::GRIB; -let mut handle = CodesHandle::new_from_file(file_path, product_kind)?; +In this example we are reading mean sea level pressure for 4 gridpoints nearest to +Reykjavik (64.13N, -21.89E) for 1st June 2021 00:00 UTC from ERA5 Climate Reanalysis. + +```rust +use eccodes::{CodesFile, FallibleIterator, KeyRead, ProductKind}; +// Open the GRIB file and create the CodesFile +let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; // Use iterator to find a message with shortName "msl" and typeOfLevel "surface" // We can use while let or for_each() to iterate over the messages -while let Some(msg) = handle.next()? { - if msg.read_key("shortName")?.value == KeyType::Str("msl".to_string()) - && msg.read_key("typeOfLevel")?.value == KeyType::Str("surface".to_string()) { - +while let Some(msg) = handle.ref_message_iter().next()? { + // We need to specify the type of key we read + let short_name: String = msg.read_key("shortName")?; + let type_of_level: String = msg.read_key("typeOfLevel")?; + if short_name == "msl" && type_of_level == "surface" { // Create CodesNearest for given message - let nearest_gridpoints = msg.codes_nearest()? + let nearest_gridpoints = msg + .codes_nearest()? // Find the nearest gridpoints to Reykjavik .find_nearest(64.13, -21.89)?; // Print value and distance of the nearest gridpoint - println!("value: {}, distance: {}", - nearest_gridpoints[3].value, - nearest_gridpoints[3].distance); + println!( + "value: {}, distance: {}", + nearest_gridpoints[3].value, nearest_gridpoints[3].distance + ); } } ``` +### Multithreading + +Since v0.14 messages inside a file can be read as `ArcMessage` which +allows to move and share the message across threads. + ### Writing GRIB files -The crate provides a basic support for setting `KeyedMessage` keys +The crate provides a basic support for setting `CodesMessage` keys and writing GRIB files. The easiest (and safest) way to create a new custom message is to copy existing one from other GRIB file, modify the keys and write to new file. @@ -125,11 +139,11 @@ This crate aims to return error whenever possible, even if the error is caused b As ecCodes is often used in scientific applications with long and extensive jobs, this allows the user to handle the error in the way that suits them best and not risk crashes. -All error descriptions are provided in the `errors` module. -Destructors, which cannot panic, report errors through the `log` crate. +All error descriptions are provided in the [`errors`] module. +Destructors, which cannot panic, report errors through `tracing` and `log` crate. None of the functions in this crate explicitly panics. -However, users should not that dependencies might panic in some edge cases. +However, users should be aware that dependencies (eg. `ndarray`) might panic in some edge cases. ## Safety @@ -139,19 +153,17 @@ Moreover, pointers are always checked for null before being dereferenced. That said, neither main developer nor contributors have expertise in unsafe Rust and bugs might have slipped through. We are also not responsible for bugs in the ecCodes library. -If you find a bug or have a suggestion, feel free to discuss it on Github. +**For critical applications always perform extensive testing before using this crate in production.** -## Features +If you find a bug or have a suggestion, feel free to discuss it on Github. -- `ndarray` - enables support for converting `KeyedMessage` to `ndarray::Array`. -This feature is enabled by default. It is currently tested only with simple lat-lon grids. +## Feature Flags -- `experimental_index` - enables support for creating and using index files for GRIB files. -This feature experimental and disabled by default. If you want to use it, please read -the information provided in `codes_index` documentation. +- `ndarray` - enables support for converting [`CodesMessage`](codes_message::CodesMessage) to [`ndarray::Array`]. + This feature is enabled by default. It is currently tested only with simple lat-lon grids. - `docs` - builds the crate without linking ecCodes, particularly useful when building the documentation -on [docs.rs](https://docs.rs/). For more details check documentation of [eccodes-sys](https://crates.io/crates/eccodes-sys). + on [docs.rs](https://docs.rs/). For more details check documentation of [eccodes-sys](https://crates.io/crates/eccodes-sys). To build your own crate with this crate as dependency on docs.rs without linking ecCodes add following lines to your `Cargo.toml` From 48e616361064cd73938736ef765325069399cd42 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 12 Jan 2026 17:04:09 +0100 Subject: [PATCH 60/65] Apply: clippy::ptr_as_ptr, clippy::as_ptr_cast_mut, clippy::redundant_pub_crate, clippy::derive_partial_eq_without_eq, clippy::missing_const_for_fn --- src/codes_file/iterator.rs | 2 +- src/codes_file/mod.rs | 40 ++++++++++++++++---------------- src/codes_message/mod.rs | 8 +++---- src/intermediate_bindings/mod.rs | 10 ++++---- 4 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/codes_file/iterator.rs b/src/codes_file/iterator.rs index 185f1f9..6babcc9 100644 --- a/src/codes_file/iterator.rs +++ b/src/codes_file/iterator.rs @@ -44,7 +44,7 @@ pub struct RefMessageIter<'a, D: Debug> { impl CodesFile { /// Generates [`RefMessageIter`] that allows to access messages as references to their parent file. - pub fn ref_message_iter(&mut self) -> RefMessageIter<'_, D> { + pub const fn ref_message_iter(&mut self) -> RefMessageIter<'_, D> { RefMessageIter { codes_file: self } } } diff --git a/src/codes_file/mod.rs b/src/codes_file/mod.rs index 1d3cf3f..356f426 100644 --- a/src/codes_file/mod.rs +++ b/src/codes_file/mod.rs @@ -26,7 +26,7 @@ mod iterator; /// /// Destructor for this structure does not panic, but some internal functions may rarely fail /// leading to bugs. Errors encountered in the destructor are logged with [`tracing`]. -/// +/// /// To access GRIB messages the ecCodes library uses a method similar to a C-style iterator. /// It digests the `* FILE` multiple times, each time returning the `*mut codes_handle` /// to a message inside the file. Therefore in this crate access messages from `CodesFile` @@ -69,9 +69,9 @@ pub enum ProductKind { impl CodesFile { /// Opens file at given [`Path`] as selected [`ProductKind`] and contructs `CodesFile`. - /// + /// /// ## Example - /// + /// /// ``` /// # use eccodes::{ProductKind, CodesFile}; /// # use std::path::Path; @@ -80,24 +80,24 @@ impl CodesFile { /// # Ok(()) /// # } /// ``` - /// + /// /// The function creates [`fs::File`](std::fs::File) from provided path and utilises /// [`fdopen()`](https://man7.org/linux/man-pages/man3/fdopen.3.html) /// to associate [`io::RawFd`](`std::os::unix::io::RawFd`) /// with a stream represented by [`libc::FILE`](https://docs.rs/libc/0.2.101/libc/enum.FILE.html) pointer. - /// + /// /// The constructor takes as argument a [`path`](Path) instead of [`File`] /// to ensure that `fdopen()` uses the same mode as [`File`]. - /// + /// /// The file stream and [`File`] are safely closed when `CodesFile` is dropped. - /// + /// /// ## Errors /// Returns [`CodesError::FileHandlingInterrupted`] with [`io::Error`](std::io::Error) /// when the file cannot be opened. - /// + /// /// Returns [`CodesError::LibcNonZero`] with [`errno`](errno::Errno) information /// when the stream cannot be created from the file descriptor. - /// + /// /// Returns [`CodesError::Internal`] with error code /// when internal [`codes_handle`] cannot be created. #[instrument(level = "trace")] @@ -117,9 +117,9 @@ impl CodesFile { } impl CodesFile> { /// Opens data in provided buffer as selected [`ProductKind`] and contructs `CodesFile`. - /// + /// /// ## Example - /// + /// /// ``` /// # async fn run() -> anyhow::Result<()> { /// # use eccodes::{ProductKind, CodesFile}; @@ -131,31 +131,31 @@ impl CodesFile> { /// .bytes() /// .await? /// .to_vec(); - /// + /// /// let handle = CodesFile::new_from_memory(file_data, product_kind)?; /// # Ok(()) /// # } /// ``` - /// + /// /// The function associates data in memory with a stream /// represented by [`libc::FILE`](https://docs.rs/libc/0.2.101/libc/enum.FILE.html) pointer /// using [`fmemopen()`](https://man7.org/linux/man-pages/man3/fmemopen.3.html). - /// + /// /// The constructor takes full ownership of the data inside buffer, /// which is safely dropped during the [`CodesFile`] drop. - /// + /// /// ## Errors /// Returns [`CodesError::LibcNonZero`] with [`errno`](errno::Errno) information /// when the file stream cannot be created. - /// + /// /// Returns [`CodesError::Internal`] with error code /// when internal [`codes_handle`] cannot be created. #[instrument(level = "trace")] pub fn new_from_memory( - file_data: Vec, + mut file_data: Vec, product_kind: ProductKind, ) -> Result { - let file_pointer = open_with_fmemopen(&file_data)?; + let file_pointer = open_with_fmemopen(&mut file_data)?; Ok(Self { _data: file_data, @@ -179,8 +179,8 @@ fn open_with_fdopen(file: &File) -> Result<*mut FILE, CodesError> { } #[instrument(level = "trace")] -fn open_with_fmemopen(file_data: &[u8]) -> Result<*mut FILE, CodesError> { - let file_data_ptr = file_data.as_ptr() as *mut c_void; +fn open_with_fmemopen(file_data: &mut [u8]) -> Result<*mut FILE, CodesError> { + let file_data_ptr = file_data.as_mut_ptr().cast::(); pointer_guard::non_null!(file_data_ptr); let file_ptr; diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index cdb1dd3..9713596 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -67,11 +67,11 @@ pub struct CodesMessage { // This is a little unintuitive, but we use `()` here to not unnecessarily pollute // CodesMessage and derived types with generics, because `PhantomData` is needed // only for lifetime restriction and we tightly control how `CodesMessage` is created. -#[derive(Debug, Hash, PartialEq, PartialOrd)] +#[derive(Debug, Hash, PartialEq, Eq, PartialOrd)] #[doc(hidden)] pub struct RefParent<'ch>(PhantomData<&'ch ()>); -#[derive(Debug, Hash, PartialEq, PartialOrd)] +#[derive(Debug, Hash, PartialEq, Eq, PartialOrd)] #[doc(hidden)] pub struct BufParent(); @@ -97,7 +97,7 @@ unsafe impl Send for BufMessage {} unsafe impl Sync for BufMessage {} impl RefMessage<'_> { - pub(crate) fn new(handle: *mut codes_handle) -> Self { + pub(crate) const fn new(handle: *mut codes_handle) -> Self { RefMessage { _parent: RefParent(PhantomData), message_handle: handle, @@ -117,7 +117,7 @@ impl ArcMessage { } impl BufMessage { - pub(crate) fn new(handle: *mut codes_handle) -> Self { + pub(crate) const fn new(handle: *mut codes_handle) -> Self { BufMessage { _parent: BufParent(), message_handle: handle, diff --git a/src/intermediate_bindings/mod.rs b/src/intermediate_bindings/mod.rs index 8714d7a..7416ab6 100644 --- a/src/intermediate_bindings/mod.rs +++ b/src/intermediate_bindings/mod.rs @@ -25,23 +25,23 @@ pub enum NativeKeyType { Missing = eccodes_sys::CODES_TYPE_MISSING as isize, } -pub(crate) use codes_get::{ +pub use codes_get::{ codes_get_bytes, codes_get_double, codes_get_double_array, codes_get_long, codes_get_long_array, codes_get_message, codes_get_native_type, codes_get_size, codes_get_string, }; -pub(crate) use codes_handle::{ +pub use codes_handle::{ codes_handle_clone, codes_handle_delete, codes_handle_new_from_file, }; -pub(crate) use codes_keys::{ +pub use codes_keys::{ codes_keys_iterator_delete, codes_keys_iterator_get_name, codes_keys_iterator_new, codes_keys_iterator_next, }; -pub(crate) use codes_set::{ +pub use codes_set::{ codes_set_bytes, codes_set_double, codes_set_double_array, codes_set_long, codes_set_long_array, codes_set_string, }; -pub(crate) use grib_nearest::{ +pub use grib_nearest::{ codes_grib_nearest_delete, codes_grib_nearest_find, codes_grib_nearest_new, }; use num_traits::FromPrimitive; From 8365f9c0d83346dabdc370f3c43656c952abfd0f Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 12 Jan 2026 17:33:41 +0100 Subject: [PATCH 61/65] switch to alpha versioning before final publish --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index c46816c..3c78a3a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "eccodes" description = "Unofficial high-level Rust bindings of the latest ecCodes release" repository = "https://github.com/ScaleWeather/eccodes" -version = "0.14.0" +version = "0.14.0-alpha.1" readme = "README.md" authors = ["Jakub Lewandowski "] keywords = ["eccodes", "grib", "bufr", "meteorology", "weather"] From fd9036f9bb8f05eb7a4b7373a900fb2fa0a684ef Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 12 Jan 2026 17:33:47 +0100 Subject: [PATCH 62/65] fix and update CI --- .github/workflows/rust-dev.yml | 81 -------------------------- .github/workflows/rust.yml | 100 ++++++++++++++++----------------- 2 files changed, 48 insertions(+), 133 deletions(-) delete mode 100644 .github/workflows/rust-dev.yml diff --git a/.github/workflows/rust-dev.yml b/.github/workflows/rust-dev.yml deleted file mode 100644 index 3d00230..0000000 --- a/.github/workflows/rust-dev.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: cargodev - -on: - push: - branches-ignore: - - main - pull_request: - branches: - - "**" - -env: - CARGO_TERM_COLOR: always - -jobs: - pre_job: - continue-on-error: true - runs-on: ubuntu-latest - outputs: - should_skip: ${{ steps.skip_check.outputs.should_skip }} - steps: - - id: skip_check - uses: fkirc/skip-duplicate-actions@v5 - with: - # All of these options are optional, so you can remove them if you are happy with the defaults - concurrent_skipping: 'same_content_newer' - skip_after_successful_duplicate: 'true' - paths_ignore: '["**/README.md"]' - do_not_skip: '["pull_request", "workflow_dispatch", "schedule"]' - - build-linux: - - name: Build on Linux - needs: pre_job - if: needs.pre_job.outputs.should_skip != 'true' - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Prepare environment - run: | - sudo apt-get update - sudo apt-get install clang - sudo apt-get install libclang1 - sudo apt-get install libeccodes-dev - rustup update stable - cargo clean - - name: Build release - run: | - cargo build --release --features "experimental_index, ndarray" - - name: Check with clippy - run: | - cargo clippy --features "experimental_index, ndarray" -- -D warnings - - name: Test with cargo - run: | - cargo clean - RUST_BACKTRACE=full cargo test --features "experimental_index, ndarray" -- --include-ignored - - build-macos: - - name: Build on MacOS - needs: pre_job - if: needs.pre_job.outputs.should_skip != 'true' - runs-on: macos-latest - - steps: - - uses: actions/checkout@v4 - - name: Prepare environment - run: | - brew install eccodes - rustup update stable - cargo clean - - name: Build release - run: | - cargo build --release --features "experimental_index, ndarray" - - name: Check with clippy - run: | - cargo clippy --features "experimental_index, ndarray" -- -D warnings - - name: Test with cargo - run: | - cargo clean - RUST_BACKTRACE=full cargo test --features "experimental_index, ndarray" -- --include-ignored diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 3da12a3..9da914e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,6 +1,6 @@ name: cargo -on: +on: push: branches: - main @@ -22,72 +22,68 @@ jobs: uses: fkirc/skip-duplicate-actions@v5 with: # All of these options are optional, so you can remove them if you are happy with the defaults - concurrent_skipping: 'same_content_newer' - skip_after_successful_duplicate: 'true' + concurrent_skipping: "same_content_newer" + skip_after_successful_duplicate: "true" paths_ignore: '["**/README.md"]' do_not_skip: '["pull_request", "workflow_dispatch", "schedule"]' build: - name: Build on Ubuntu needs: pre_job if: needs.pre_job.outputs.should_skip != 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - name: Prepare environment - run: | - sudo apt-get update - sudo apt-get install clang - sudo apt-get install libclang1 - sudo apt-get install libeccodes-dev - rustup update stable - cargo install cargo-criterion - cargo clean - - name: Build with cargo - run: | - cargo build --release --features "experimental_index, ndarray" - cargo clean - - name: Test with cargo - run: | - cargo test --no-default-features - cargo test --features "ndarray" - cargo test --features "experimental_index" - cargo test --features "experimental_index, ndarray" - cargo clean - - name: Benchmark with criterion - run: | - cargo criterion - cargo clean + - uses: actions/checkout@v4 + - name: Prepare environment + run: | + sudo apt-get update + sudo apt-get install clang + sudo apt-get install libclang1 + sudo apt-get install libeccodes-dev + rustup update stable + cargo install cargo-criterion + cargo clean + - name: Check release build + run: | + cargo build --release --features "ndarray" + - name: Check with clippy + run: | + cargo clippy --features "ndarray" -- -D warnings + - name: Check tests + run: | + cargo test --no-default-features + cargo test --features "ndarray" + - name: Benchmark with criterion + run: | + cargo criterion + cargo clean build-macos: - name: Build on MacOS needs: pre_job if: needs.pre_job.outputs.should_skip != 'true' runs-on: macos-latest steps: - - uses: actions/checkout@v4 - - name: Prepare environment - run: | - brew install eccodes - rustup update stable - cargo install cargo-criterion - cargo clean - - name: Build with cargo - run: | - cargo build --release --features "experimental_index, ndarray" - cargo clean - - name: Test with cargo - run: | - cargo test --no-default-features - cargo test --features "ndarray" - cargo test --features "experimental_index" - cargo test --features "experimental_index, ndarray" - cargo clean - - name: Benchmark with criterion - run: | - cargo criterion - cargo clean + - uses: actions/checkout@v4 + - name: Prepare environment + run: | + brew install eccodes + rustup update stable + cargo install cargo-criterion + cargo clean + - name: Check release build + run: | + cargo build --release --features --features "ndarray" + - name: Check with clippy + run: | + cargo clippy --features "ndarray" -- -D warnings + - name: Check tests + run: | + cargo test --no-default-features + cargo test --features "ndarray" + - name: Benchmark with criterion + run: | + cargo criterion + cargo clean From e24184e19d0da8b188e959926d85066c86eb8af8 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 12 Jan 2026 17:36:11 +0100 Subject: [PATCH 63/65] use type inference in casts wherever possible, this also fixes compilation for different targets --- src/codes_file/mod.rs | 10 +++++----- src/codes_message/write.rs | 2 +- src/intermediate_bindings/codes_get.rs | 10 +++++----- src/intermediate_bindings/codes_handle.rs | 8 +------- src/intermediate_bindings/codes_set.rs | 6 +++--- 5 files changed, 15 insertions(+), 21 deletions(-) diff --git a/src/codes_file/mod.rs b/src/codes_file/mod.rs index 356f426..2e00520 100644 --- a/src/codes_file/mod.rs +++ b/src/codes_file/mod.rs @@ -4,7 +4,7 @@ use crate::{CodesError, intermediate_bindings::codes_handle_new_from_file, pointer_guard}; use eccodes_sys::{ProductKind_PRODUCT_GRIB, codes_handle}; use errno::errno; -use libc::{FILE, c_char, c_void, size_t}; +use libc::{FILE, size_t}; use std::{ fmt::Debug, fs::{File, OpenOptions}, @@ -167,7 +167,7 @@ impl CodesFile> { #[instrument(level = "trace")] fn open_with_fdopen(file: &File) -> Result<*mut FILE, CodesError> { - let file_ptr = unsafe { libc::fdopen(file.as_raw_fd(), "r".as_ptr().cast::<_>()) }; + let file_ptr = unsafe { libc::fdopen(file.as_raw_fd(), "r".as_ptr().cast()) }; if file_ptr.is_null() { let error_val = errno(); @@ -180,15 +180,15 @@ fn open_with_fdopen(file: &File) -> Result<*mut FILE, CodesError> { #[instrument(level = "trace")] fn open_with_fmemopen(file_data: &mut [u8]) -> Result<*mut FILE, CodesError> { - let file_data_ptr = file_data.as_mut_ptr().cast::(); + let file_data_ptr = file_data.as_mut_ptr(); pointer_guard::non_null!(file_data_ptr); let file_ptr; unsafe { file_ptr = libc::fmemopen( - file_data_ptr, + file_data_ptr.cast(), file_data.len() as size_t, - "r".as_ptr().cast::<_>(), + "r".as_ptr().cast(), ); } diff --git a/src/codes_message/write.rs b/src/codes_message/write.rs index 43a483f..8a00774 100644 --- a/src/codes_message/write.rs +++ b/src/codes_message/write.rs @@ -107,7 +107,7 @@ impl CodesMessage { append: bool, ) -> Result<(), CodesError> { let msg = unsafe { codes_get_message(self.message_handle)? }; - let buf = unsafe { slice::from_raw_parts(msg.0.cast::<_>(), msg.1) }; + let buf = unsafe { slice::from_raw_parts(msg.0.cast(), msg.1) }; let mut file = OpenOptions::new() .write(true) .create(true) diff --git a/src/intermediate_bindings/codes_get.rs b/src/intermediate_bindings/codes_get.rs index 465d8a8..82ae4f7 100644 --- a/src/intermediate_bindings/codes_get.rs +++ b/src/intermediate_bindings/codes_get.rs @@ -86,7 +86,7 @@ pub unsafe fn codes_get_double_array( let error_code = eccodes_sys::codes_get_double_array( handle, key.as_ptr(), - key_values.as_mut_ptr().cast::(), + key_values.as_mut_ptr().cast(), &raw mut key_size, ); error_code_to_result(error_code)?; @@ -110,7 +110,7 @@ pub unsafe fn codes_get_long_array( let error_code = eccodes_sys::codes_get_long_array( handle, key.as_ptr(), - key_values.as_mut_ptr().cast::(), + key_values.as_mut_ptr().cast(), &raw mut key_size, ); error_code_to_result(error_code)?; @@ -151,7 +151,7 @@ pub unsafe fn codes_get_string( let error_code = eccodes_sys::codes_get_string( handle, key.as_ptr(), - key_message.as_mut_ptr().cast::(), + key_message.as_mut_ptr().cast(), &raw mut key_length, ); error_code_to_result(error_code)?; @@ -187,7 +187,7 @@ pub unsafe fn codes_get_bytes( let error_code = eccodes_sys::codes_get_bytes( handle, key.as_ptr(), - buffer.as_mut_ptr().cast::(), + buffer.as_mut_ptr().cast(), &raw mut key_size, ); error_code_to_result(error_code)?; @@ -219,7 +219,7 @@ pub unsafe fn codes_get_message( let buffer_size = codes_get_message_size(handle)?; let buffer: Vec = vec![0; buffer_size]; - let mut buffer_ptr = buffer.as_ptr().cast::(); + let mut buffer_ptr = buffer.as_ptr().cast(); let mut message_size: usize = 0; diff --git a/src/intermediate_bindings/codes_handle.rs b/src/intermediate_bindings/codes_handle.rs index bb92acd..9c72b04 100644 --- a/src/intermediate_bindings/codes_handle.rs +++ b/src/intermediate_bindings/codes_handle.rs @@ -12,12 +12,6 @@ use crate::{ pointer_guard, }; -#[cfg(target_os = "macos")] -type _SYS_IO_FILE = eccodes_sys::__sFILE; - -#[cfg(not(target_os = "macos"))] -type _SYS_IO_FILE = eccodes_sys::_IO_FILE; - #[instrument(level = "trace")] pub unsafe fn codes_handle_new_from_file( file_pointer: *mut FILE, @@ -32,7 +26,7 @@ pub unsafe fn codes_handle_new_from_file( let file_handle = eccodes_sys::codes_handle_new_from_file( context, - file_pointer.cast::<_SYS_IO_FILE>(), + file_pointer.cast(), product_kind as u32, &raw mut error_code, ); diff --git a/src/intermediate_bindings/codes_set.rs b/src/intermediate_bindings/codes_set.rs index 7ab75a4..fcf0447 100644 --- a/src/intermediate_bindings/codes_set.rs +++ b/src/intermediate_bindings/codes_set.rs @@ -56,7 +56,7 @@ pub unsafe fn codes_set_long_array( let error_code = eccodes_sys::codes_set_long_array( handle, key.as_ptr(), - values.as_ptr().cast::(), + values.as_ptr().cast(), length, ); error_code_to_result(error_code)?; @@ -80,7 +80,7 @@ pub unsafe fn codes_set_double_array( let error_code = eccodes_sys::codes_set_double_array( handle, key.as_ptr(), - values.as_ptr().cast::(), + values.as_ptr().cast(), length, ); error_code_to_result(error_code)?; @@ -124,7 +124,7 @@ pub unsafe fn codes_set_bytes( let error_code = eccodes_sys::codes_set_bytes( handle, key.as_ptr(), - values.as_ptr().cast::(), + values.as_ptr().cast(), &raw mut length, ); error_code_to_result(error_code)?; From f07ebe6a4eb99e3430244f5ec5885c4cff054c7c Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 12 Jan 2026 17:37:11 +0100 Subject: [PATCH 64/65] rustfmt --- src/codes_file/iterator.rs | 12 ++++++------ src/codes_message/mod.rs | 4 ++-- src/intermediate_bindings/codes_set.rs | 8 ++------ src/intermediate_bindings/mod.rs | 4 +--- src/keys_iterator.rs | 4 ++-- src/lib.rs | 22 +++++++++++----------- tests/example.rs | 14 +++++++------- 7 files changed, 31 insertions(+), 37 deletions(-) diff --git a/src/codes_file/iterator.rs b/src/codes_file/iterator.rs index 6babcc9..e04c39c 100644 --- a/src/codes_file/iterator.rs +++ b/src/codes_file/iterator.rs @@ -72,17 +72,17 @@ impl<'ch, D: Debug> FallibleIterator for RefMessageIter<'ch, D> { /// /// `ArcMessage` implements `Send + Sync` so it can be both moved to thread (for example, to read messages in parallel) /// or shared across threads (when wrapped in [`Arc`]). -/// +/// /// This structure implements [`FallibleIterator`] - see the documentation for information how that differs from a standard `Iter`. /// /// Creating this iter does not require `CodesFile` to be mutable, because it takes ownership over the `CodesFile`. /// /// If you don't need to share the message, use [`RefMessageIter`] to avoid the performance overhead of [`Arc`]. -/// +/// /// If you want to modify the message, use [`try_clone()`](RefMessage::try_clone). -/// +/// /// ## Example -/// +/// /// See the second example in the main crate description for example usage of `ArcMessageIter`. #[derive(Debug)] pub struct ArcMessageIter { @@ -106,9 +106,9 @@ impl FallibleIterator for ArcMessageIter { /// /// The method will return [`CodesInternal`](crate::errors::CodesInternal) /// when internal ecCodes function returns non-zero code. - /// + /// /// # Panics - /// + /// /// This method internally uses a Mutex to access `CodesFile`, which can panic when poisoned, /// but thers is no path in which you can get to the state of poisoned mutex, while still able to access this method. fn next(&mut self) -> Result, Self::Error> { diff --git a/src/codes_message/mod.rs b/src/codes_message/mod.rs index 9713596..e10bc32 100644 --- a/src/codes_message/mod.rs +++ b/src/codes_message/mod.rs @@ -43,9 +43,9 @@ use crate::{CodesFile, intermediate_bindings::codes_handle_delete}; /// You can read a `Key` with static types using [`read_key()`](KeyRead::read_key()) or with [`DynamicKeyType`] using[`read_key_dynamic()`](CodesMessage::read_key_dynamic()) /// To iterate over all key names use [`KeysIterator`](crate::KeysIterator). You can also modify the message using /// [`write_key_unchecked()`](KeyWrite::write_key_unchecked()) (only available for `BufMessage`). -/// +/// /// [`CodesNearest`](crate::CodesNearest) can be used to find nearest gridpoints for given coordinates in the `CodesMessage`. -/// +/// /// To write the message to file use [`write_to_file()`](CodesMessage::write_to_file). /// /// If you are interested only in getting data values from the message you can use diff --git a/src/intermediate_bindings/codes_set.rs b/src/intermediate_bindings/codes_set.rs index fcf0447..fc763aa 100644 --- a/src/intermediate_bindings/codes_set.rs +++ b/src/intermediate_bindings/codes_set.rs @@ -53,12 +53,8 @@ pub unsafe fn codes_set_long_array( let length = values.len(); - let error_code = eccodes_sys::codes_set_long_array( - handle, - key.as_ptr(), - values.as_ptr().cast(), - length, - ); + let error_code = + eccodes_sys::codes_set_long_array(handle, key.as_ptr(), values.as_ptr().cast(), length); error_code_to_result(error_code)?; Ok(()) diff --git a/src/intermediate_bindings/mod.rs b/src/intermediate_bindings/mod.rs index 7416ab6..0338c1b 100644 --- a/src/intermediate_bindings/mod.rs +++ b/src/intermediate_bindings/mod.rs @@ -30,9 +30,7 @@ pub use codes_get::{ codes_get_long_array, codes_get_message, codes_get_native_type, codes_get_size, codes_get_string, }; -pub use codes_handle::{ - codes_handle_clone, codes_handle_delete, codes_handle_new_from_file, -}; +pub use codes_handle::{codes_handle_clone, codes_handle_delete, codes_handle_new_from_file}; pub use codes_keys::{ codes_keys_iterator_delete, codes_keys_iterator_get_name, codes_keys_iterator_new, codes_keys_iterator_next, diff --git a/src/keys_iterator.rs b/src/keys_iterator.rs index 399cf04..9079716 100644 --- a/src/keys_iterator.rs +++ b/src/keys_iterator.rs @@ -33,9 +33,9 @@ use crate::{ /// # fn main() -> anyhow::Result<()> { /// let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; /// let mut current_message = handle.ref_message_iter().next()?.context("no message")?; -/// +/// /// let mut keys_iter = current_message.default_keys_iterator()?; -/// +/// /// while let Some(key_name) = keys_iter.next()? { /// println!("{key_name}"); /// } diff --git a/src/lib.rs b/src/lib.rs index 409930d..34df109 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -178,26 +178,26 @@ //! use anyhow::Context; //! use eccodes::{CodesFile, FallibleIterator, KeyRead, KeyWrite, ProductKind}; //! use std::{collections::HashMap, fs::remove_file, path::Path}; -//! +//! //! # fn main() -> anyhow::Result<()> { //! // Start by opening the file and creating CodesFile //! let file_path = Path::new("./data/iceland-levels.grib"); //! let mut handle = CodesFile::new_from_file(file_path, ProductKind::GRIB)?; -//! +//! //! // To build the index we need to collect all messages //! let messages = handle.ref_message_iter().collect::>()?; //! let mut msg_index = HashMap::new(); //! msg_index.reserve(messages.len()); -//! +//! //! // Now we can put the messages into a hashmap and index them by shortName and level //! for msg in messages.into_iter() { //! // all messages in this grib are on the same level type //! let short_name: String = msg.read_key("shortName")?; //! let level: i64 = msg.read_key("level")?; -//! +//! //! msg_index.insert((short_name, level), msg); //! } -//! +//! //! // Now we can get the values from messages we need //! let t_800: Vec = msg_index //! .get(&("t".to_string(), 800)) @@ -207,24 +207,24 @@ //! .get(&("t".to_string(), 800)) //! .context("message missing")? //! .read_key("values")?; -//! +//! //! // We will also clone t at 700hPa to edit it //! let mut t_850_msg = msg_index //! .get(&("t".to_string(), 700)) //! .context("message missing")? //! .try_clone()?; -//! +//! //! // Compute temperature at 850hPa //! let t_850_values: Vec = t_800 //! .iter() //! .zip(t_900.iter()) //! .map(|t| (t.0 + t.1) / 2.0) //! .collect(); -//! +//! //! // Edit appropriate keys in the cloned (editable) message //! t_850_msg.write_key_unchecked("level", 850)?; //! t_850_msg.write_key_unchecked("values", t_850_values.as_slice())?; -//! +//! //! // Save the message to a new file without appending //! t_850_msg.write_to_file(Path::new("iceland-850.grib"), false)?; //! # remove_file(Path::new("iceland-850.grib")).unwrap(); @@ -233,7 +233,7 @@ //! ``` //! //! ## Changes in version 0.14 -//! +//! //! 1. `experimental_index` feature has been removed - users are encouraged to create their own indexes as shown above or use iterator filtering //! 2. `message_ndarray` feature has been renamed to `ndarray` //! 3. `CodesHandle` has been renamed to `CodesFile` @@ -266,7 +266,7 @@ pub mod keys_iterator; mod pointer_guard; pub use codes_file::{ArcMessageIter, CodesFile, ProductKind, RefMessageIter}; -pub use codes_message::{ArcMessage, BufMessage, KeyRead, KeyWrite, RefMessage, DynamicKeyType}; +pub use codes_message::{ArcMessage, BufMessage, DynamicKeyType, KeyRead, KeyWrite, RefMessage}; pub use codes_nearest::{CodesNearest, NearestGridpoint}; pub use errors::CodesError; pub use fallible_iterator::{FallibleIterator, IntoFallibleIterator}; diff --git a/tests/example.rs b/tests/example.rs index dd89bf0..8d443c3 100644 --- a/tests/example.rs +++ b/tests/example.rs @@ -1,13 +1,13 @@ use anyhow::Context; use eccodes::{CodesFile, FallibleIterator, ProductKind}; fn main() -> anyhow::Result<()> { -let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; -let mut current_message = handle.ref_message_iter().next()?.context("no message")?; + let mut handle = CodesFile::new_from_file("./data/iceland.grib", ProductKind::GRIB)?; + let mut current_message = handle.ref_message_iter().next()?.context("no message")?; -let mut keys_iter = current_message.default_keys_iterator()?; + let mut keys_iter = current_message.default_keys_iterator()?; -while let Some(key_name) = keys_iter.next()? { - println!("{key_name}"); -} -Ok(()) + while let Some(key_name) = keys_iter.next()? { + println!("{key_name}"); + } + Ok(()) } From b02eca090c86e20029b9870d2c6997136887baf3 Mon Sep 17 00:00:00 2001 From: Quba1 <22771850+Quba1@users.noreply.github.com> Date: Mon, 12 Jan 2026 17:51:45 +0100 Subject: [PATCH 65/65] update msrv --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 3c78a3a..f7874d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ categories = [ license = "Apache-2.0" edition = "2024" exclude = [".github/*", ".vscode/*", ".idea/*", "data/*"] -rust-version = "1.85.0" +rust-version = "1.85.1" [dependencies] eccodes-sys = { version = "0.7.0", default-features = false }