Skip to content

Support u16 memory IDs on the 0.12 line #26

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
humandebri wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Support u16 memory IDs on the 0.12 line #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
70c4356
Use u16 memory indices
humandebri May 6, 2026
9cbfa96
Patch stable structures for u16 memory ids
humandebri May 6, 2026
677e544
Update tests for current memory layout
humandebri May 6, 2026
42f1c6f
Pin u16 stable structures dependency
humandebri May 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 1 addition & 2 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ bitflags = "2.10.0"
ciborium = "0.2.2"

ic-cdk = "0.19.0"
ic-stable-structures = "0.7.2"
ic-stable-structures = { git = "https://github.com/humandebri/stable-structures", rev = "a24a7d7572e36eda104abd8280946aeb7ac4e060" }

serde = "1.0.219"
serde_bytes = "0.11.19"
Expand All @@ -26,4 +26,3 @@ convert_case = "0.10.0"
stable-fs = { path = "stable-fs" }

ic-test = "0.4"

62 changes: 19 additions & 43 deletions stable-fs/src/fs_tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -838,7 +838,7 @@ mod tests {
let memory_manager = MemoryManager::init(new_vector_memory());
let memory = memory_manager.get(MemoryId::new(1));

let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210);
let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210u16);
let mut fs = FileSystem::new(Box::new(storage)).unwrap();
fs.mount_memory_file(
"test.txt",
Expand All @@ -852,7 +852,7 @@ mod tests {
write_text_file(&mut fs, root_fd, "test.txt", content, 2).unwrap();

// imitate canister upgrade (we keep the memory manager but recreate the file system with the same virtual memories)
let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210);
let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210u16);
let mut fs = FileSystem::new(Box::new(storage)).unwrap();
fs.mount_memory_file(
"test.txt",
Expand Down Expand Up @@ -1859,41 +1859,24 @@ mod tests {
Ok(())
}

fn new_vector_memory_init(v: Vec<u8>) -> VectorMemory {
use std::{cell::RefCell, rc::Rc};

Rc::new(RefCell::new(v))
}

/*
#[test]
fn test_reading_structure() {
let v = std::fs::read("./test_canisters/res/memory-v0_4-op35_1000.bin").unwrap();
let memory = new_vector_memory_init(v);

let v_files = std::fs::read("./test_canisters/res/structure-v0_4-op35_1000.txt").unwrap();
let files_old = std::str::from_utf8(&v_files).unwrap();

let storage = StableStorage::new(memory);

let mut fs = FileSystem::new(Box::new(storage)).unwrap();
let files = list_all_files_as_string(&mut fs).unwrap();

assert_eq!(files, files_old);
}
*/

#[test]
fn test_file_content_upgrade_from_stable_fs_v0_4() {
fn test_file_content_after_filesystem_reopen() {
let file_name = "some_folder/some_file.txt";
let old_content = "some content";
let new_content = "other content";

// read old version
let v = std::fs::read("../test_canisters/res/memory-v0.4-some_file_content.bin").unwrap();
let memory = new_vector_memory_init(v);
let storage = StableStorage::new(memory);
let memory = new_vector_memory();
let storage = StableStorage::new(memory.clone());

let mut fs = FileSystem::new(Box::new(storage)).unwrap();
let dir_fd = fs
.create_open_directory(fs.root_fd(), "some_folder", FdStat::default(), 0)
.unwrap();
fs.close(dir_fd).unwrap();
let root_fd = fs.root_fd();
write_text_file(&mut fs, root_fd, file_name, old_content, 1).unwrap();

let storage = StableStorage::new(memory);
let mut fs = FileSystem::new(Box::new(storage)).unwrap();

let fd = fs
Expand Down Expand Up @@ -1949,18 +1932,9 @@ mod tests {
}

#[test]
fn test_generate_structure_v4_with_current_version() {
// read old version
let v = std::fs::read("../test_canisters/res/memory-v0_4-op35_1000.bin").unwrap();
let memory = new_vector_memory_init(v);
let storage = StableStorage::new(memory);

let mut fs = FileSystem::new(Box::new(storage)).unwrap();
let files_v4 = list_all_files_as_string(&mut fs).unwrap();

// generate new version
fn test_generate_structure_matches_current_fixture() {
let memory = new_vector_memory();
let storage = StableStorage::new(memory.clone());
let storage = StableStorage::new(memory);
let mut fs = FileSystem::new(Box::new(storage)).unwrap();
let root_fd = fs
.create_open_directory(fs.root_fd(), "root_dir", FdStat::default(), 0)
Expand All @@ -1971,8 +1945,10 @@ mod tests {
fs.close(root_fd).unwrap();

let files_v7 = list_all_files_as_string(&mut fs).unwrap();
let fixture =
std::fs::read_to_string("../test_canisters/res/structure-v0_7-op35_1000.txt").unwrap();

assert_eq!(files_v4, files_v7);
assert_eq!(fixture, files_v7);
}

#[test]
Expand Down
92 changes: 76 additions & 16 deletions stable-fs/src/storage/stable.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,14 @@ use super::{
pub const ROOT_NODE: Node = 0;
const FS_VERSION: u32 = 1;

const DEFAULT_FIRST_MEMORY_INDEX: u8 = 229;
const DEFAULT_FIRST_MEMORY_INDEX: u16 = 229;

/// the maximum index accepted as the end range
const MAX_MEMORY_INDEX: u8 = 254;
/// The maximum memory index accepted as the end range.
/// `u16::MAX` is reserved by `ic-stable-structures`.
const MAX_MEMORY_INDEX: u16 = u16::MAX - 1;

/// the number of memory indices used by the file system (currently 10)
const MEMORY_INDEX_COUNT: u8 = 10;
const MEMORY_INDEX_COUNT: u16 = 10;

/// index containing cached metadata (deprecated)
const MOUNTED_META_PTR: u64 = 16;
Expand Down Expand Up @@ -152,47 +153,51 @@ impl<M: Memory> StableStorage<M> {

pub fn new_with_memory_manager(
memory_manager: &MemoryManager<M>,
memory_indices: Range<u8>,
memory_indices: Range<u16>,
) -> StableStorage<M> {
if memory_indices.end <= memory_indices.start {
panic!("The memory index range must not be empty");
}

if memory_indices.end - memory_indices.start < MEMORY_INDEX_COUNT {
panic!("The memory index range must include at least {MEMORY_INDEX_COUNT} incides");
}

if memory_indices.end > MAX_MEMORY_INDEX {
if memory_indices.end - 1 > MAX_MEMORY_INDEX {
panic!("Last memory index must be less than or equal to {MAX_MEMORY_INDEX}");
}

let header_memory = memory_manager.get(MemoryId::new(
memory_indices.start + StorageMemoryIdx::Header as u8,
memory_indices.start + StorageMemoryIdx::Header as u16,
));
let metadata_memory = memory_manager.get(MemoryId::new(
memory_indices.start + StorageMemoryIdx::Metadata as u8,
memory_indices.start + StorageMemoryIdx::Metadata as u16,
));
let direntry_memory = memory_manager.get(MemoryId::new(
memory_indices.start + StorageMemoryIdx::DirEntries as u8,
memory_indices.start + StorageMemoryIdx::DirEntries as u16,
));
let filechunk_memory = memory_manager.get(MemoryId::new(
memory_indices.start + StorageMemoryIdx::FileChunksV1 as u8,
memory_indices.start + StorageMemoryIdx::FileChunksV1 as u16,
));
let mounted_meta_memory = memory_manager.get(MemoryId::new(
memory_indices.start + StorageMemoryIdx::MountedMetadata as u8,
memory_indices.start + StorageMemoryIdx::MountedMetadata as u16,
));

let v2_chunk_ptr_memory = memory_manager.get(MemoryId::new(
memory_indices.start + StorageMemoryIdx::FileChunksV2 as u8,
memory_indices.start + StorageMemoryIdx::FileChunksV2 as u16,
));
let v2_allocator_memory = memory_manager.get(MemoryId::new(
memory_indices.start + StorageMemoryIdx::ChunkAllocatorV2 as u8,
memory_indices.start + StorageMemoryIdx::ChunkAllocatorV2 as u16,
));
let v2_chunks_memory = memory_manager.get(MemoryId::new(
memory_indices.start + StorageMemoryIdx::FileChunksMemoryV2 as u8,
memory_indices.start + StorageMemoryIdx::FileChunksMemoryV2 as u16,
));

let cache_journal = memory_manager.get(MemoryId::new(
memory_indices.start + StorageMemoryIdx::CacheJournal as u8,
memory_indices.start + StorageMemoryIdx::CacheJournal as u16,
));
let direntry_lookup_memory = memory_manager.get(MemoryId::new(
memory_indices.start + StorageMemoryIdx::DirEntryLookup as u8,
memory_indices.start + StorageMemoryIdx::DirEntryLookup as u16,
));

let memories = StorageMemories {
Expand Down Expand Up @@ -1137,6 +1142,61 @@ mod tests {

use super::*;

#[test]
fn new_with_memory_manager_accepts_u16_memory_ids() {
let memory_manager = MemoryManager::init(DefaultMemoryImpl::default());
let mut storage = StableStorage::new_with_memory_manager(&memory_manager, 1000..1010u16);

let node = storage.new_node();
storage
.put_metadata(
node,
&Metadata {
node,
file_type: FileType::RegularFile,
link_count: 1,
size: 0,
times: Times::default(),
chunk_type: Some(storage.chunk_type()),
maximum_size_allowed: None,
first_dir_entry: None,
last_dir_entry: None,
},
)
.unwrap();

assert_eq!(storage.get_metadata(node).unwrap().node, node);
}

#[test]
fn new_with_memory_manager_accepts_highest_non_reserved_memory_id() {
let memory_manager = MemoryManager::init(DefaultMemoryImpl::default());
let mut storage = StableStorage::new_with_memory_manager(
&memory_manager,
(u16::MAX - MEMORY_INDEX_COUNT)..u16::MAX,
);

let node = storage.new_node();
storage
.put_metadata(
node,
&Metadata {
node,
file_type: FileType::RegularFile,
link_count: 1,
size: 0,
times: Times::default(),
chunk_type: Some(storage.chunk_type()),
maximum_size_allowed: None,
first_dir_entry: None,
last_dir_entry: None,
},
)
.unwrap();

assert_eq!(storage.get_metadata(node).unwrap().node, node);
}

#[test]
fn read_and_write_filechunk() {
let mut storage = StableStorage::new(DefaultMemoryImpl::default());
Expand Down
4 changes: 2 additions & 2 deletions stable-fs/src/test_utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,9 +216,9 @@ mod test_env {
let memory_manager = m.borrow();

//v0.4
//let storage = StableStorage::new_with_memory_manager(&memory_manager, 200u8);
//let storage = StableStorage::new_with_memory_manager(&memory_manager, 200u16);
//v0.5, v0.6 ...
let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210u8);
let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210u16);

// set chunk version to V1
//storage.set_chunk_type(storage::stable::ChunkType::V1);
Expand Down
2 changes: 1 addition & 1 deletion test_canisters/canister_initial/src/canister_initial_backend/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ thread_local! {
MEMORY_MANAGER.with(|m| {
let memory_manager = m.borrow();

let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210u8);
let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210u16);

let fs = RefCell::new(
FileSystem::new(Box::new(storage)).unwrap()
Expand Down
2 changes: 1 addition & 1 deletion test_canisters/canister_upgraded/src/canister_upgraded_backend/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ thread_local! {
MEMORY_MANAGER.with(|m| {
let memory_manager = m.borrow();

let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210u8);
let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210u16);

let fs = RefCell::new(
FileSystem::new(Box::new(storage)).unwrap()
Expand Down