From 70c4356425a0d1e77d299836f3f20b9f89823d78 Mon Sep 17 00:00:00 2001 From: hude Date: Wed, 6 May 2026 13:26:24 +0900 Subject: [PATCH 1/4] Use u16 memory indices --- stable-fs/src/fs_tests.rs | 4 +-- stable-fs/src/storage/stable.rs | 28 +++++++++---------- stable-fs/src/test_utils.rs | 4 +-- .../src/canister_initial_backend/src/lib.rs | 2 +- .../src/canister_upgraded_backend/src/lib.rs | 2 +- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/stable-fs/src/fs_tests.rs b/stable-fs/src/fs_tests.rs index 600ba1b..e323a5f 100644 --- a/stable-fs/src/fs_tests.rs +++ b/stable-fs/src/fs_tests.rs @@ -838,7 +838,7 @@ mod tests { let memory_manager = MemoryManager::init(new_vector_memory()); let memory = memory_manager.get(MemoryId::new(1)); - let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210); + let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210u16); let mut fs = FileSystem::new(Box::new(storage)).unwrap(); fs.mount_memory_file( "test.txt", @@ -852,7 +852,7 @@ mod tests { write_text_file(&mut fs, root_fd, "test.txt", content, 2).unwrap(); // imitate canister upgrade (we keep the memory manager but recreate the file system with the same virtual memories) - let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210); + let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210u16); let mut fs = FileSystem::new(Box::new(storage)).unwrap(); fs.mount_memory_file( "test.txt", diff --git a/stable-fs/src/storage/stable.rs b/stable-fs/src/storage/stable.rs index f9ad4bf..7e186e5 100644 --- a/stable-fs/src/storage/stable.rs +++ b/stable-fs/src/storage/stable.rs @@ -38,13 +38,13 @@ use super::{ pub const ROOT_NODE: Node = 0; const FS_VERSION: u32 = 1; -const DEFAULT_FIRST_MEMORY_INDEX: u8 = 229; +const DEFAULT_FIRST_MEMORY_INDEX: u16 = 229; /// the maximum index accepted as the end range -const MAX_MEMORY_INDEX: u8 = 254; +const MAX_MEMORY_INDEX: u16 = 254; /// the number of memory indices used by the file system (currently 10) -const MEMORY_INDEX_COUNT: u8 = 10; +const MEMORY_INDEX_COUNT: u16 = 10; /// index containing cached metadata (deprecated) const MOUNTED_META_PTR: u64 = 16; @@ -152,7 +152,7 @@ impl StableStorage { pub fn new_with_memory_manager( memory_manager: &MemoryManager, - memory_indices: Range, + memory_indices: Range, ) -> StableStorage { if memory_indices.end - memory_indices.start < MEMORY_INDEX_COUNT { panic!("The memory index range must include at least {MEMORY_INDEX_COUNT} incides"); @@ -163,36 +163,36 @@ impl StableStorage { } let header_memory = memory_manager.get(MemoryId::new( - memory_indices.start + StorageMemoryIdx::Header as u8, + memory_indices.start + StorageMemoryIdx::Header as u16, )); let metadata_memory = memory_manager.get(MemoryId::new( - memory_indices.start + StorageMemoryIdx::Metadata as u8, + memory_indices.start + StorageMemoryIdx::Metadata as u16, )); let direntry_memory = memory_manager.get(MemoryId::new( - memory_indices.start + StorageMemoryIdx::DirEntries as u8, + memory_indices.start + StorageMemoryIdx::DirEntries as u16, )); let filechunk_memory = memory_manager.get(MemoryId::new( - memory_indices.start + StorageMemoryIdx::FileChunksV1 as u8, + memory_indices.start + StorageMemoryIdx::FileChunksV1 as u16, )); let mounted_meta_memory = memory_manager.get(MemoryId::new( - memory_indices.start + StorageMemoryIdx::MountedMetadata as u8, + memory_indices.start + StorageMemoryIdx::MountedMetadata as u16, )); let v2_chunk_ptr_memory = memory_manager.get(MemoryId::new( - memory_indices.start + StorageMemoryIdx::FileChunksV2 as u8, + memory_indices.start + StorageMemoryIdx::FileChunksV2 as u16, )); let v2_allocator_memory = memory_manager.get(MemoryId::new( - memory_indices.start + StorageMemoryIdx::ChunkAllocatorV2 as u8, + memory_indices.start + StorageMemoryIdx::ChunkAllocatorV2 as u16, )); let v2_chunks_memory = memory_manager.get(MemoryId::new( - memory_indices.start + StorageMemoryIdx::FileChunksMemoryV2 as u8, + memory_indices.start + StorageMemoryIdx::FileChunksMemoryV2 as u16, )); let cache_journal = memory_manager.get(MemoryId::new( - memory_indices.start + StorageMemoryIdx::CacheJournal as u8, + memory_indices.start + StorageMemoryIdx::CacheJournal as u16, )); let direntry_lookup_memory = memory_manager.get(MemoryId::new( - memory_indices.start + StorageMemoryIdx::DirEntryLookup as u8, + memory_indices.start + StorageMemoryIdx::DirEntryLookup as u16, )); let memories = StorageMemories { diff --git a/stable-fs/src/test_utils.rs b/stable-fs/src/test_utils.rs index e1b95de..83b1cf2 100644 --- a/stable-fs/src/test_utils.rs +++ b/stable-fs/src/test_utils.rs @@ -216,9 +216,9 @@ mod test_env { let memory_manager = m.borrow(); //v0.4 - //let storage = StableStorage::new_with_memory_manager(&memory_manager, 200u8); + //let storage = StableStorage::new_with_memory_manager(&memory_manager, 200u16); //v0.5, v0.6 ... - let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210u8); + let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210u16); // set chunk version to V1 //storage.set_chunk_type(storage::stable::ChunkType::V1); diff --git a/test_canisters/canister_initial/src/canister_initial_backend/src/lib.rs b/test_canisters/canister_initial/src/canister_initial_backend/src/lib.rs index 5e85136..69f8e6d 100644 --- a/test_canisters/canister_initial/src/canister_initial_backend/src/lib.rs +++ b/test_canisters/canister_initial/src/canister_initial_backend/src/lib.rs @@ -53,7 +53,7 @@ thread_local! { MEMORY_MANAGER.with(|m| { let memory_manager = m.borrow(); - let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210u8); + let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210u16); let fs = RefCell::new( FileSystem::new(Box::new(storage)).unwrap() diff --git a/test_canisters/canister_upgraded/src/canister_upgraded_backend/src/lib.rs b/test_canisters/canister_upgraded/src/canister_upgraded_backend/src/lib.rs index beab5fb..5ed3021 100644 --- a/test_canisters/canister_upgraded/src/canister_upgraded_backend/src/lib.rs +++ b/test_canisters/canister_upgraded/src/canister_upgraded_backend/src/lib.rs @@ -53,7 +53,7 @@ thread_local! { MEMORY_MANAGER.with(|m| { let memory_manager = m.borrow(); - let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210u8); + let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210u16); let fs = RefCell::new( FileSystem::new(Box::new(storage)).unwrap() From 9cbfa96c8932e49d322abeccd97c9abe86008a26 Mon Sep 17 00:00:00 2001 From: hude Date: Wed, 6 May 2026 13:38:44 +0900 Subject: [PATCH 2/4] Patch stable structures for u16 memory ids --- Cargo.lock | 3 +- Cargo.toml | 2 + stable-fs/src/storage/stable.rs | 66 +++++++++++++++++++++++++++++++-- 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3080fc0..f278f7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1324,8 +1324,7 @@ dependencies = [ [[package]] name = "ic-stable-structures" version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee3372ddc0cf2a747fc26ce2d075a240ed6bfab151e63bc70109e8967f7ce6f" +source = "git+https://github.com/humandebri/stable-structures?branch=feature%2Fmemory-manager-u16-v2#a24a7d7572e36eda104abd8280946aeb7ac4e060" dependencies = [ "ic_principal", ] diff --git a/Cargo.toml b/Cargo.toml index 635a428..2746d94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,3 +27,5 @@ stable-fs = { path = "stable-fs" } ic-test = "0.4" +[patch.crates-io] +ic-stable-structures = { git = "https://github.com/humandebri/stable-structures", branch = "feature/memory-manager-u16-v2" } diff --git a/stable-fs/src/storage/stable.rs b/stable-fs/src/storage/stable.rs index 7e186e5..550f76a 100644 --- a/stable-fs/src/storage/stable.rs +++ b/stable-fs/src/storage/stable.rs @@ -40,8 +40,9 @@ const FS_VERSION: u32 = 1; const DEFAULT_FIRST_MEMORY_INDEX: u16 = 229; -/// the maximum index accepted as the end range -const MAX_MEMORY_INDEX: u16 = 254; +/// The maximum memory index accepted as the end range. +/// `u16::MAX` is reserved by `ic-stable-structures`. +const MAX_MEMORY_INDEX: u16 = u16::MAX - 1; /// the number of memory indices used by the file system (currently 10) const MEMORY_INDEX_COUNT: u16 = 10; @@ -154,11 +155,15 @@ impl StableStorage { memory_manager: &MemoryManager, memory_indices: Range, ) -> StableStorage { + if memory_indices.end <= memory_indices.start { + panic!("The memory index range must not be empty"); + } + if memory_indices.end - memory_indices.start < MEMORY_INDEX_COUNT { panic!("The memory index range must include at least {MEMORY_INDEX_COUNT} incides"); } - if memory_indices.end > MAX_MEMORY_INDEX { + if memory_indices.end - 1 > MAX_MEMORY_INDEX { panic!("Last memory index must be less than or equal to {MAX_MEMORY_INDEX}"); } @@ -1137,6 +1142,61 @@ mod tests { use super::*; + #[test] + fn new_with_memory_manager_accepts_u16_memory_ids() { + let memory_manager = MemoryManager::init(DefaultMemoryImpl::default()); + let mut storage = StableStorage::new_with_memory_manager(&memory_manager, 1000..1010u16); + + let node = storage.new_node(); + storage + .put_metadata( + node, + &Metadata { + node, + file_type: FileType::RegularFile, + link_count: 1, + size: 0, + times: Times::default(), + chunk_type: Some(storage.chunk_type()), + maximum_size_allowed: None, + first_dir_entry: None, + last_dir_entry: None, + }, + ) + .unwrap(); + + assert_eq!(storage.get_metadata(node).unwrap().node, node); + } + + #[test] + fn new_with_memory_manager_accepts_highest_non_reserved_memory_id() { + let memory_manager = MemoryManager::init(DefaultMemoryImpl::default()); + let mut storage = StableStorage::new_with_memory_manager( + &memory_manager, + (u16::MAX - MEMORY_INDEX_COUNT)..u16::MAX, + ); + + let node = storage.new_node(); + storage + .put_metadata( + node, + &Metadata { + node, + file_type: FileType::RegularFile, + link_count: 1, + size: 0, + times: Times::default(), + chunk_type: Some(storage.chunk_type()), + maximum_size_allowed: None, + first_dir_entry: None, + last_dir_entry: None, + }, + ) + .unwrap(); + + assert_eq!(storage.get_metadata(node).unwrap().node, node); + } + #[test] fn read_and_write_filechunk() { let mut storage = StableStorage::new(DefaultMemoryImpl::default()); From 677e54471d683a6a988234dc1675b499ebed23d9 Mon Sep 17 00:00:00 2001 From: hude Date: Wed, 6 May 2026 13:50:45 +0900 Subject: [PATCH 3/4] Update tests for current memory layout --- stable-fs/src/fs_tests.rs | 58 ++++++++++++--------------------------- 1 file changed, 17 insertions(+), 41 deletions(-) diff --git a/stable-fs/src/fs_tests.rs b/stable-fs/src/fs_tests.rs index e323a5f..abd221c 100644 --- a/stable-fs/src/fs_tests.rs +++ b/stable-fs/src/fs_tests.rs @@ -1859,41 +1859,24 @@ mod tests { Ok(()) } - fn new_vector_memory_init(v: Vec) -> VectorMemory { - use std::{cell::RefCell, rc::Rc}; - - Rc::new(RefCell::new(v)) - } - - /* - #[test] - fn test_reading_structure() { - let v = std::fs::read("./test_canisters/res/memory-v0_4-op35_1000.bin").unwrap(); - let memory = new_vector_memory_init(v); - - let v_files = std::fs::read("./test_canisters/res/structure-v0_4-op35_1000.txt").unwrap(); - let files_old = std::str::from_utf8(&v_files).unwrap(); - - let storage = StableStorage::new(memory); - - let mut fs = FileSystem::new(Box::new(storage)).unwrap(); - let files = list_all_files_as_string(&mut fs).unwrap(); - - assert_eq!(files, files_old); - } - */ - #[test] - fn test_file_content_upgrade_from_stable_fs_v0_4() { + fn test_file_content_after_filesystem_reopen() { let file_name = "some_folder/some_file.txt"; let old_content = "some content"; let new_content = "other content"; - // read old version - let v = std::fs::read("../test_canisters/res/memory-v0.4-some_file_content.bin").unwrap(); - let memory = new_vector_memory_init(v); - let storage = StableStorage::new(memory); + let memory = new_vector_memory(); + let storage = StableStorage::new(memory.clone()); + let mut fs = FileSystem::new(Box::new(storage)).unwrap(); + let dir_fd = fs + .create_open_directory(fs.root_fd(), "some_folder", FdStat::default(), 0) + .unwrap(); + fs.close(dir_fd).unwrap(); + let root_fd = fs.root_fd(); + write_text_file(&mut fs, root_fd, file_name, old_content, 1).unwrap(); + + let storage = StableStorage::new(memory); let mut fs = FileSystem::new(Box::new(storage)).unwrap(); let fd = fs @@ -1949,18 +1932,9 @@ mod tests { } #[test] - fn test_generate_structure_v4_with_current_version() { - // read old version - let v = std::fs::read("../test_canisters/res/memory-v0_4-op35_1000.bin").unwrap(); - let memory = new_vector_memory_init(v); - let storage = StableStorage::new(memory); - - let mut fs = FileSystem::new(Box::new(storage)).unwrap(); - let files_v4 = list_all_files_as_string(&mut fs).unwrap(); - - // generate new version + fn test_generate_structure_matches_current_fixture() { let memory = new_vector_memory(); - let storage = StableStorage::new(memory.clone()); + let storage = StableStorage::new(memory); let mut fs = FileSystem::new(Box::new(storage)).unwrap(); let root_fd = fs .create_open_directory(fs.root_fd(), "root_dir", FdStat::default(), 0) @@ -1971,8 +1945,10 @@ mod tests { fs.close(root_fd).unwrap(); let files_v7 = list_all_files_as_string(&mut fs).unwrap(); + let fixture = + std::fs::read_to_string("../test_canisters/res/structure-v0_7-op35_1000.txt").unwrap(); - assert_eq!(files_v4, files_v7); + assert_eq!(fixture, files_v7); } #[test] From 42f1c6f547fd664b47eff489cedf416693df76cd Mon Sep 17 00:00:00 2001 From: hude Date: Wed, 6 May 2026 14:03:45 +0900 Subject: [PATCH 4/4] Pin u16 stable structures dependency --- Cargo.lock | 2 +- Cargo.toml | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f278f7e..31be19a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1324,7 +1324,7 @@ dependencies = [ [[package]] name = "ic-stable-structures" version = "0.7.2" -source = "git+https://github.com/humandebri/stable-structures?branch=feature%2Fmemory-manager-u16-v2#a24a7d7572e36eda104abd8280946aeb7ac4e060" +source = "git+https://github.com/humandebri/stable-structures?rev=a24a7d7572e36eda104abd8280946aeb7ac4e060#a24a7d7572e36eda104abd8280946aeb7ac4e060" dependencies = [ "ic_principal", ] diff --git a/Cargo.toml b/Cargo.toml index 2746d94..43a8c9e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ bitflags = "2.10.0" ciborium = "0.2.2" ic-cdk = "0.19.0" -ic-stable-structures = "0.7.2" +ic-stable-structures = { git = "https://github.com/humandebri/stable-structures", rev = "a24a7d7572e36eda104abd8280946aeb7ac4e060" } serde = "1.0.219" serde_bytes = "0.11.19" @@ -26,6 +26,3 @@ convert_case = "0.10.0" stable-fs = { path = "stable-fs" } ic-test = "0.4" - -[patch.crates-io] -ic-stable-structures = { git = "https://github.com/humandebri/stable-structures", branch = "feature/memory-manager-u16-v2" }