From c53ed7cb03f47cfd3184bdb70708af571580c4a9 Mon Sep 17 00:00:00 2001 From: rdmrocha Date: Sun, 17 May 2026 21:48:12 +1000 Subject: [PATCH] Adds read-extraction for Xbox 360 STFS container packages --- CHANGELOG.md | 22 + Cargo.lock | 4 +- Cargo.toml | 2 +- README.md | 26 +- fatxlib/Cargo.toml | 2 +- fatxlib/src/stfs/block_translator.rs | 110 +++++ fatxlib/src/stfs/extract.rs | 630 ++++++++++++++++++++++++ fatxlib/src/stfs/file_entry.rs | 175 +++++++ fatxlib/src/{stfs.rs => stfs/header.rs} | 0 fatxlib/src/stfs/mod.rs | 14 + fatxlib/src/stfs/volume_descriptor.rs | 124 +++++ install.sh | 2 +- src/extract_stfs.rs | 253 ++++++++++ src/main.rs | 21 + 14 files changed, 1378 insertions(+), 7 deletions(-) create mode 100644 fatxlib/src/stfs/block_translator.rs create mode 100644 fatxlib/src/stfs/extract.rs create mode 100644 fatxlib/src/stfs/file_entry.rs rename fatxlib/src/{stfs.rs => stfs/header.rs} (100%) create mode 100644 fatxlib/src/stfs/mod.rs create mode 100644 fatxlib/src/stfs/volume_descriptor.rs create mode 100644 src/extract_stfs.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ff77122..6a9b052 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,28 @@ All notable changes to `xtafkit` will be documented in this file. +## [1.3.0] - 2026-05-17 + +Adds read-extraction for Xbox 360 STFS container packages — Arcade (XBLA), XBLIG, Title Updates, Marketplace DLC. New library surface under `fatxlib::stfs::extract` and a new `xtafkit extract-stfs` CLI subcommand. TUI integration for STFS is intentionally deferred to a later release. + +### STFS extraction +- Added `xtafkit extract-stfs ` for streaming Xbox 360 STFS containers (CON / LIVE / PIRS) to a local directory. `--to ` overrides the destination; `--dry-run` lists the entries with sizes and totals; `--json` emits machine-readable output for both dry-run and post-extract modes. +- Default destination (no `--to`) is `.//` taken straight from the STFS header — no catalog lookup, no `[TitleID]` suffix. The per-package `display_name` is consistently more specific than the title-id catalog mapping (notably for XBLIG, where every indie game shares the same system title-id and the catalog can only return a generic bucket name). +- Read-only / type-1 STFS only. Type-0 (read-write, used by save games / on-drive system files / CON packages) surfaces an explicit `"STFS type 0 (read-write) not supported yet"` error rather than producing wrong output. +- Block-index → byte-offset translator handles all interleaved hash levels: L0 every `0xAA` blocks, L1 every `0x70E4`, L2 every `0x4AF768`. Inline boundary tests pin offsets at `0xA9`, `0xAA`, `0x70E3`, `0x70E4` against literal hex values and assert strict monotonicity across boundaries. +- File chain follower covers both the consecutive fast path (no hash-block reads) and the fragmented case (next-block pointer threaded through the L0 hash block at offset `(N % 0xAA) * 24 + 0x15`). Walk is capped at `used_blocks` iterations to reject malformed cyclic chains, mirroring the existing FAT cycle rejection in `volume.rs`. +- Defensive parent-chain resolution in `extract_to_host`: rejects cyclic `parent_index` references and out-of-range parent pointers; refuses to overwrite existing output files. + +### Library API additions +- New `fatxlib::stfs` submodules: `volume_descriptor`, `block_translator`, `file_entry`, `extract`. Existing header parsing (`StfsHeader`, `parse_header`, `MIN_HEADER_BYTES`) moved into `fatxlib::stfs::header` and re-exported at the namespace root — no breaking changes for existing callers. +- `fatxlib::stfs::StfsPackage::{open, header, volume, entries, read_block_chain, read_file}` — read API for STFS packages. `read_file` streams through a writer; no full-file buffering even for multi-hundred-MiB packages. +- `fatxlib::stfs::extract::extract_to_host(&mut StfsPackage, &Path, Option) -> Result` — top-level walk + write, with progress callback shape matching the existing XISO extract (`(rel_path, file_size, total_bytes_so_far)`). +- `fatxlib::stfs::extract::ExtractReport` — `{ files, directories, bytes }` returned on success. + +### Testing / maintenance +- 27 inline synthetic tests across the four new STFS submodules: volume-descriptor parsing (type-0/1 detection, wrong-size rejection, truncation), type-1 block translator (boundary indices at every hash level plus monotonicity), file entry parsing (consecutive flag, directory flag, parent-index, non-ASCII tolerance), and end-to-end synthetic package extraction with a nested directory tree. +- v2 TUI extract-gating rule documented in the design spec: the future TUI sniff prompt will offer `(X)tract` only when the package contains a `default.xex` file (the only reliable signal that loose extraction produces something useful for alt-dashboards). Fallback: gate on `content_type == 0x000D0000`. The CLI `extract-stfs` is unrestricted. + ## [1.2.1] - 2026-05-17 Maintenance and hardening release on top of 1.2.0. No new features; all changes are bug fixes, internal refactors that reduce drift, and test coverage for areas that previously had none. diff --git a/Cargo.lock b/Cargo.lock index a748ec9..fed54b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -667,7 +667,7 @@ checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "fatxlib" -version = "1.2.1" +version = "1.3.0" dependencies = [ "bitflags 2.11.1", "byteorder", @@ -2685,7 +2685,7 @@ dependencies = [ [[package]] name = "xtafkit" -version = "1.2.1" +version = "1.3.0" dependencies = [ "base64", "chrono", diff --git a/Cargo.toml b/Cargo.toml index bb312d4..e59ef82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "xtafkit" -version = "1.2.1" +version = "1.3.0" edition = "2024" description = "Mac-native TUI workbench for Xbox 360 FATX/XTAF drives — browse, transfer, resolve titles, decode profiles" license = "Apache-2.0" diff --git a/README.md b/README.md index 8673f55..092b289 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ [![Release](https://img.shields.io/github/v/release/rdmrocha/xtafkit?include_prereleases&sort=semver)](https://github.com/rdmrocha/xtafkit/releases) [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE) +[![Buy me a coffee](https://img.shields.io/badge/Buy%20Me%20a%20coffee-ffdd00?logo=buy-me-a-coffee&logoColor=black)](http://paypal.me/rdmrocha) + Mac-native TUI workbench for Xbox 360 (XTAF) and Original Xbox (FATX) drives. Plug a console drive into your Mac over USB, then browse, transfer files, resolve game titles, decode profile gamertags, and work with XISO disc images from a polished terminal UI plus a small CLI surface. ## Highlights @@ -12,6 +14,7 @@ Mac-native TUI workbench for Xbox 360 (XTAF) and Original Xbox (FATX) drives. Pl - **Profile gamertag extraction** — decrypts the embedded Account file (ARC4 + HMAC-SHA1) to label profile XUIDs - **Per-file resolution** for Arcade / XNA / Marketplace / Installer folders, with one-keystroke bulk scan - **XISO extraction and GoD conversion** — stream XISO contents to a local directory or build Games-on-Demand packages for Xbox 360 backward-compatible titles +- **STFS extraction** — pull files out of Xbox 360 Arcade (XBLA), XBLIG, Title Update, and Marketplace DLC packages (CON / LIVE / PIRS) - **Sort toggle** — by resolved name or by raw ID (display format flips to match) - **Persistent caches** under `~/.config/xtafkit/` — plain text, human-editable - macOS-native I/O (`F_NOCACHE`, `F_RDAHEAD`, device-optimal alignment) @@ -62,15 +65,34 @@ xtafkit scan [--deep] detect FATX/XTAF partitions xtafkit mkimage [--size 1G] [--populate] [--format fatx|xtaf] xtafkit resolve STFS-based title / file resolution xtafkit extract [--keep-systemupdate] [--dry-run] +xtafkit extract-stfs [--to DIR] [--dry-run] xtafkit god [--trim compact|preserve-layout|none] [--dry-run] [--game-title TITLE] ``` -Seven subcommands total — file operations (download/upload/mkdir/rm/rename/copy/info/cleanup) live inside the TUI. +Eight subcommands total — file operations (download/upload/mkdir/rm/rename/copy/info/cleanup) live inside the TUI. ## XISO Tools `xtafkit extract` streams every file from an XISO to a local directory and skips `$SystemUpdate` by default. `xtafkit god` converts an XISO into a Games-on-Demand package; the default trim mode is `compact`, which repacks XDVDFS densely before GoD packaging. Pass `--trim preserve-layout` to keep mastered holes, or `--trim none` to use the full data partition. +## STFS extraction + +`xtafkit extract-stfs ` reads any Xbox 360 STFS container (CON / LIVE / PIRS) and writes its inner files to a local directory. Works on Arcade (XBLA), XBLIG, Title Updates, and Marketplace DLC. + +- Default destination is `.//` taken from the package's STFS header — no catalog lookup, no `[TitleID]` suffix. +- `--to ` overrides the destination explicitly. +- `--dry-run` lists every entry with its size and totals, exits without writing. +- `--json` emits machine-readable output for both `--dry-run` and post-extract modes. +- Read-only / type-1 STFS only. Type-0 packages (read-write — save games, on-drive system files) surface a clean error rather than producing wrong output. + +```bash +# Inspect what's inside without writing anything +xtafkit extract-stfs ./XBLA_pkg --dry-run + +# Extract to a chosen directory +xtafkit extract-stfs ./TU_patch --to ./out/title-update +``` + ## TUI ```bash @@ -168,7 +190,7 @@ Tests use file-backed FATX/XTAF images generated by `xtafkit mkimage`. No hardwa ## Origin -`xtafkit` started as a fork of [joshuareisbord/fatx-rs](https://github.com/joshuareisbord/fatx-rs), which provided the FATX/XTAF filesystem core. The project has since diverged — title catalog with merged Xbox 360 + Original Xbox sources, on-demand STFS resolution, profile gamertag decryption, slot-aware folder display, TUI-first workflow, XISO extraction, and Games-on-Demand conversion. Credit to the original author for the filesystem foundation. +`xtafkit` started as a fork of [joshuareisbord/fatx-rs](https://github.com/joshuareisbord/fatx-rs), which provided the FATX/XTAF filesystem core. The project has since diverged — title catalog with merged Xbox 360 + Original Xbox sources, on-demand STFS resolution, profile gamertag decryption, slot-aware folder display, TUI-first workflow, XISO extraction, Games-on-Demand conversion, and STFS container extraction. Credit to the original author for the filesystem foundation. ## License diff --git a/fatxlib/Cargo.toml b/fatxlib/Cargo.toml index bbaef3b..8612278 100644 --- a/fatxlib/Cargo.toml +++ b/fatxlib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fatxlib" -version = "1.2.1" +version = "1.3.0" edition = "2024" description = "A Rust library for reading and writing Xbox FATX file systems" license = "Apache-2.0" diff --git a/fatxlib/src/stfs/block_translator.rs b/fatxlib/src/stfs/block_translator.rs new file mode 100644 index 0000000..b57f701 --- /dev/null +++ b/fatxlib/src/stfs/block_translator.rs @@ -0,0 +1,110 @@ +//! STFS block-index → byte-offset translator (type 1, read-only format). +//! +//! Hash blocks are interleaved between data block groups: +//! - Every 0xAA (170) data blocks → one L0 hash block after the group +//! - Every 0x70E4 (28900) data blocks → one L1 hash block after the group of L0 groups +//! - Every 0x4AF768 data blocks → one L2 hash block (rarely reached) +//! +//! References: +//! - Free60: https://free60.org/System-Software/Formats/STFS/ +//! - py360 STFSPackage._getRealBlockNum +//! - Velocity StfsPackage::GetRealAddress + +/// First data block byte offset inside an STFS package (header is 0xB000 + +/// the first 0x1000 reserved). +pub const FIRST_DATA_BLOCK_OFFSET: u64 = 0xC000; + +/// Block size in bytes. +pub const BLOCK_SIZE: u64 = 0x1000; + +/// Data blocks per L0 hash group. +pub const BLOCKS_PER_L0: u32 = 0xAA; + +/// Data blocks per L1 hash group (`BLOCKS_PER_L0 * BLOCKS_PER_L0`). +pub const BLOCKS_PER_L1: u32 = 0x70E4; + +/// Data blocks per L2 hash group. +pub const BLOCKS_PER_L2: u32 = 0x4AF768; + +/// Translate a logical block index into a byte offset. +/// +/// Accounts for L0, L1, and L2 hash blocks interleaved between data block +/// groups in the type-1 (read-only / "male pack") layout. +pub fn block_to_byte_offset(block_index: u32) -> u64 { + let mut adjusted = block_index as u64; + if block_index >= BLOCKS_PER_L0 { + adjusted += (block_index / BLOCKS_PER_L0) as u64; + } + if block_index >= BLOCKS_PER_L1 { + adjusted += (block_index / BLOCKS_PER_L1) as u64; + } + if block_index >= BLOCKS_PER_L2 { + adjusted += (block_index / BLOCKS_PER_L2) as u64; + } + FIRST_DATA_BLOCK_OFFSET + adjusted * BLOCK_SIZE +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn block_zero_lands_at_0xc000() { + assert_eq!(block_to_byte_offset(0), 0xC000); + } + + #[test] + fn last_block_of_first_l0_group() { + // Block 0xA9 is the 170th block — no hash blocks before it. + // Offset = 0xC000 + 0xA9 * 0x1000 = 0xB5000 + assert_eq!(block_to_byte_offset(0xA9), 0xB5000); + } + + #[test] + fn first_block_of_second_l0_group_skips_one_l0_hash() { + // Block 0xAA: one L0 hash block sits between it and block 0xA9. + // Offset = 0xC000 + (0xAA + 1) * 0x1000 = 0xB7000 + assert_eq!(block_to_byte_offset(0xAA), 0xB7000); + } + + #[test] + fn second_block_of_second_l0_group() { + // Block 0xAB: same L0 hash skipped as block 0xAA. + // Offset = 0xC000 + (0xAB + 1) * 0x1000 = 0xB8000 + assert_eq!(block_to_byte_offset(0xAB), 0xB8000); + } + + #[test] + fn last_block_before_first_l1_hash() { + // Block 0x70E3: 169 complete L0 groups passed (0x70E3 / 0xAA = 0xA9). + // Offset = 0xC000 + (0x70E3 + 0xA9) * 0x1000 = 0xC000 + 0x718C000 = 0x7198000 + assert_eq!(block_to_byte_offset(0x70E3), 0x7198000); + } + + #[test] + fn first_block_after_l1_hash_skips_l0_and_l1() { + // Block 0x70E4: one L1 hash + one L0 hash inserted since 0x70E3. + // Offset = 0xC000 + (0x70E4 + 0xAA + 1) * 0x1000 + // = 0xC000 + 0x718F * 0x1000 + // = 0x719B000 + assert_eq!(block_to_byte_offset(0x70E4), 0x719B000); + } + + #[test] + fn block_translator_is_strictly_monotonic_across_boundaries() { + // Sanity: offsets must strictly increase as block index increases. + let probes: [u32; 9] = [0, 1, 0xA9, 0xAA, 0xAB, 0x70E3, 0x70E4, 0x70E5, 100_000]; + let mut last = 0u64; + for n in probes { + let off = block_to_byte_offset(n); + assert!( + off > last, + "non-monotonic at block 0x{:X}: 0x{:X} <= 0x{:X}", + n, + off, + last + ); + last = off; + } + } +} diff --git a/fatxlib/src/stfs/extract.rs b/fatxlib/src/stfs/extract.rs new file mode 100644 index 0000000..ad9d101 --- /dev/null +++ b/fatxlib/src/stfs/extract.rs @@ -0,0 +1,630 @@ +//! High-level STFS read + extract API. + +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; + +use crate::error::{FatxError, Result}; +use crate::stfs::block_translator::{BLOCK_SIZE, BLOCKS_PER_L0, block_to_byte_offset}; +use crate::stfs::file_entry::{self, ENTRIES_PER_BLOCK, FILE_ENTRY_SIZE, StfsEntry}; +use crate::stfs::header::{MIN_HEADER_BYTES, StfsHeader, parse_header}; +use crate::stfs::volume_descriptor::{self, VolumeDescriptor}; + +/// Summary of a completed extraction. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExtractReport { + pub files: usize, + pub directories: usize, + pub bytes: u64, +} + +/// Progress callback: `(relative_path, file_size, total_bytes_so_far)`. +pub type ProgressFn<'a> = &'a dyn Fn(&str, u64, u64); + +/// Walk `package` and extract every file under `dest_root`. Creates +/// directories as needed. Returns counts on success. +/// +/// `progress` is invoked once per file just before its write begins. +pub fn extract_to_host( + package: &mut StfsPackage, + dest_root: &Path, + progress: Option>, +) -> Result { + let entries = package.entries()?; + let paths = build_relative_paths(&entries)?; + + let mut files = 0usize; + let mut directories = 0usize; + let mut bytes = 0u64; + + for (idx, entry) in entries.iter().enumerate() { + let rel = &paths[idx]; + let target = dest_root.join(rel); + if entry.is_directory { + std::fs::create_dir_all(&target).map_err(FatxError::Io)?; + directories += 1; + continue; + } + if let Some(parent) = target.parent() { + std::fs::create_dir_all(parent).map_err(FatxError::Io)?; + } + if target.exists() { + return Err(FatxError::Other(format!( + "refusing to overwrite existing file: {}", + target.display(), + ))); + } + if let Some(cb) = progress { + cb(&rel.to_string_lossy(), entry.size, bytes); + } + let file = std::fs::File::create(&target).map_err(FatxError::Io)?; + let mut writer = std::io::BufWriter::new(file); + package.read_file(entry, &mut writer)?; + writer.flush().map_err(FatxError::Io)?; + files += 1; + bytes += entry.size; + } + + Ok(ExtractReport { + files, + directories, + bytes, + }) +} + +/// Build a vec of relative paths parallel to `entries`. Walks +/// `parent_index` chains; tolerates orphan entries by attaching them to +/// the root with a `` prefix. +fn build_relative_paths(entries: &[StfsEntry]) -> Result> { + let mut out: Vec> = vec![None; entries.len()]; + + fn resolve( + idx: usize, + entries: &[StfsEntry], + cache: &mut Vec>, + guard: &mut Vec, + ) -> Result { + if let Some(p) = &cache[idx] { + return Ok(p.clone()); + } + if guard[idx] { + return Err(FatxError::Other(format!( + "STFS entry parent chain cycles at index {}", + idx, + ))); + } + guard[idx] = true; + let entry = &entries[idx]; + let path = if entry.parent_index == -1 { + PathBuf::from(&entry.name) + } else { + let parent_idx = entry.parent_index as usize; + if parent_idx >= entries.len() { + return Err(FatxError::Other(format!( + "STFS entry {} references out-of-range parent {}", + idx, entry.parent_index, + ))); + } + let parent_path = resolve(parent_idx, entries, cache, guard)?; + parent_path.join(&entry.name) + }; + cache[idx] = Some(path.clone()); + Ok(path) + } + + let mut guard = vec![false; entries.len()]; + for i in 0..entries.len() { + resolve(i, entries, &mut out, &mut guard)?; + // reset guard for next traversal + for slot in guard.iter_mut() { + *slot = false; + } + } + Ok(out.into_iter().map(|p| p.unwrap()).collect()) +} + +pub struct StfsPackage { + reader: R, + header: StfsHeader, + volume: VolumeDescriptor, +} + +impl std::fmt::Debug for StfsPackage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StfsPackage") + .field("header", &self.header) + .field("volume", &self.volume) + .finish_non_exhaustive() + } +} + +impl StfsPackage { + /// Parse the STFS header + volume descriptor. Cheap; does not walk + /// the file table. + pub fn open(mut reader: R) -> Result { + let mut prefix = vec![0u8; MIN_HEADER_BYTES.max(0x379 + 0x24)]; + reader.seek(SeekFrom::Start(0)).map_err(FatxError::Io)?; + reader.read_exact(&mut prefix).map_err(FatxError::Io)?; + + let header = parse_header(&prefix).ok_or_else(|| { + FatxError::Other("Not an STFS package (bad magic or truncated header)".to_string()) + })?; + + let volume = volume_descriptor::parse(&prefix[0x379..0x379 + 0x24])?; + if !volume.read_only_format { + return Err(FatxError::Other( + "STFS type 0 (read-write) not supported yet — v1 supports type 1 only".to_string(), + )); + } + + Ok(Self { + reader, + header, + volume, + }) + } + + pub fn header(&self) -> &StfsHeader { + &self.header + } + + pub fn volume(&self) -> &VolumeDescriptor { + &self.volume + } + + /// Read one data block (4 KiB) into a fresh buffer. + fn read_data_block(&mut self, block_index: u32) -> Result> { + let total = self.volume.total_alloc_blocks; + if block_index >= total { + return Err(FatxError::Other(format!( + "STFS block index {} out of range (total_alloc_blocks = {})", + block_index, total, + ))); + } + let offset = block_to_byte_offset(block_index); + let mut buf = vec![0u8; BLOCK_SIZE as usize]; + self.reader + .seek(SeekFrom::Start(offset)) + .map_err(FatxError::Io)?; + self.reader.read_exact(&mut buf).map_err(FatxError::Io)?; + Ok(buf) + } + + /// L0 hash block byte offset for the group containing `data_block`. + fn l0_hash_block_offset(&self, data_block: u32) -> u64 { + let group_start = (data_block / BLOCKS_PER_L0) * BLOCKS_PER_L0; + // The L0 hash block sits AFTER the group's data blocks. Equivalently + // it occupies the position one block before block_to_byte_offset + // would place data block (group_start + BLOCKS_PER_L0). + block_to_byte_offset(group_start + BLOCKS_PER_L0) - BLOCK_SIZE + } + + /// Read the next-block pointer for `data_block` from its L0 hash entry. + /// Returns `0xFFFFFF` (sentinel end-of-chain) if the data block is the + /// last in its chain. + fn read_next_block(&mut self, data_block: u32) -> Result { + let hash_offset = self.l0_hash_block_offset(data_block); + let entry_offset = hash_offset + (data_block as u64 % BLOCKS_PER_L0 as u64) * 24 + 0x15; + let mut buf = [0u8; 3]; + self.reader + .seek(SeekFrom::Start(entry_offset)) + .map_err(FatxError::Io)?; + self.reader.read_exact(&mut buf).map_err(FatxError::Io)?; + Ok((buf[0] as u32) | ((buf[1] as u32) << 8) | ((buf[2] as u32) << 16)) + } + + /// Walk a file's block chain, returning the ordered list of data block + /// indices. Caps the walk at `entry.used_blocks` to reject malformed + /// cyclic chains. + pub fn read_block_chain(&mut self, entry: &StfsEntry) -> Result> { + if entry.used_blocks == 0 { + return Ok(Vec::new()); + } + if entry.consecutive { + return Ok((entry.start_block..entry.start_block + entry.used_blocks).collect()); + } + let mut chain = Vec::with_capacity(entry.used_blocks as usize); + let mut current = entry.start_block; + for _ in 0..entry.used_blocks { + chain.push(current); + let next = self.read_next_block(current)?; + if next == 0xFFFFFF { + break; + } + current = next; + } + Ok(chain) + } + + /// Stream the contents of one file entry through the writer. Truncates + /// the final block to honor `entry.size`. + pub fn read_file(&mut self, entry: &StfsEntry, writer: &mut W) -> Result { + let chain = self.read_block_chain(entry)?; + let mut remaining = entry.size; + for block_idx in chain { + let block = self.read_data_block(block_idx)?; + let take = remaining.min(BLOCK_SIZE) as usize; + writer.write_all(&block[..take]).map_err(FatxError::Io)?; + remaining -= take as u64; + if remaining == 0 { + break; + } + } + Ok(entry.size) + } + + /// Walk the file table block chain (v1: consecutive only). + pub fn entries(&mut self) -> Result> { + let count = self.volume.file_table_block_count as u32; + let first = self.volume.file_table_block_number; + let mut out = Vec::new(); + for i in 0..count { + let block = self.read_data_block(first + i)?; + for slot in 0..ENTRIES_PER_BLOCK { + let start = slot * FILE_ENTRY_SIZE; + let bytes = &block[start..start + FILE_ENTRY_SIZE]; + match file_entry::parse(bytes)? { + Some(entry) => out.push(entry), + None => return Ok(out), // empty slot terminates the table + } + } + } + Ok(out) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + /// Build the minimum bytes needed for `StfsPackage::open` to succeed. + /// Magic + title_id at 0x360 + display_name + title_name (already + /// covered by parse_header's MIN_HEADER_BYTES) + volume descriptor at + /// 0x379 (which overlaps the header region — we just patch those bytes). + fn synthetic_package(read_only_format: bool) -> Vec { + let mut buf = vec![0u8; MIN_HEADER_BYTES]; + // LIVE magic + buf[0..4].copy_from_slice(b"LIVE"); + // title_id at 0x360 + buf[0x360..0x364].copy_from_slice(&0x4D5307E6u32.to_be_bytes()); + // Volume descriptor at 0x379 + buf[0x379] = 0x24; // descriptor_size + buf[0x37B] = if read_only_format { 0x01 } else { 0x00 }; + buf[0x37C..0x37E].copy_from_slice(&1u16.to_be_bytes()); // file_table_block_count = 1 + // file_table_block_number = 0 (already zero) + buf[0x395..0x399].copy_from_slice(&1u32.to_be_bytes()); // total_alloc = 1 + buf + } + + #[test] + fn opens_synthetic_type_1_package() { + let bytes = synthetic_package(true); + let pkg = StfsPackage::open(Cursor::new(bytes)).expect("open"); + assert_eq!(&pkg.header().magic, b"LIVE"); + assert_eq!(pkg.header().title_id, 0x4D5307E6); + assert!(pkg.volume().read_only_format); + } + + #[test] + fn rejects_type_0_package() { + let bytes = synthetic_package(false); + let err = StfsPackage::open(Cursor::new(bytes)).expect_err("should reject"); + assert!(format!("{}", err).contains("type 0")); + } + + #[test] + fn rejects_bad_magic() { + let mut bytes = synthetic_package(true); + bytes[0..4].copy_from_slice(b"XXXX"); + let err = StfsPackage::open(Cursor::new(bytes)).expect_err("should reject"); + assert!(format!("{}", err).contains("Not an STFS package")); + } + + /// Build a more complete synthetic package: header + one file-table + /// block at block 0 containing two entries (one dir, one file). + fn synthetic_package_with_one_file_table_block(entries: &[Vec]) -> Vec { + use crate::stfs::block_translator::{BLOCK_SIZE, FIRST_DATA_BLOCK_OFFSET}; + + // Header region: MIN_HEADER_BYTES. + let mut buf = vec![0u8; MIN_HEADER_BYTES]; + buf[0..4].copy_from_slice(b"LIVE"); + buf[0x360..0x364].copy_from_slice(&0x4D5307E6u32.to_be_bytes()); + buf[0x379] = 0x24; + buf[0x37B] = 0x01; + buf[0x37C..0x37E].copy_from_slice(&1u16.to_be_bytes()); + // file_table_block_number stays 0 + buf[0x395..0x399].copy_from_slice(&2u32.to_be_bytes()); // total_alloc = 2 + + // Grow to the start of block 0. + let block_zero_offset = FIRST_DATA_BLOCK_OFFSET as usize; + if buf.len() < block_zero_offset { + buf.resize(block_zero_offset, 0); + } + // File table block 0: pack entries. + let mut ft_block = vec![0u8; BLOCK_SIZE as usize]; + for (i, e) in entries.iter().enumerate() { + ft_block[i * 0x40..(i + 1) * 0x40].copy_from_slice(e); + } + buf.extend_from_slice(&ft_block); + buf + } + + fn fe(name: &str, is_dir: bool, parent: i16, size: u32, start_block: u32) -> Vec { + let mut buf = vec![0u8; 0x40]; + let nb = name.as_bytes(); + buf[..nb.len()].copy_from_slice(nb); + let mut flags = nb.len() as u8 & 0x3F; + flags |= 0x40; // consecutive + if is_dir { + flags |= 0x80; + } + buf[0x28] = flags; + // used_blocks = 1 for files, 0 for dirs + if !is_dir { + buf[0x2C] = 1; + } + buf[0x2F] = (start_block & 0xFF) as u8; + buf[0x30] = ((start_block >> 8) & 0xFF) as u8; + buf[0x31] = ((start_block >> 16) & 0xFF) as u8; + buf[0x32..0x34].copy_from_slice(&parent.to_be_bytes()); + buf[0x34..0x38].copy_from_slice(&size.to_be_bytes()); + buf + } + + #[test] + fn entries_walks_single_file_table_block() { + let entries = vec![ + fe("Media", true, -1, 0, 0), + fe("default.xex", false, 0, 0x1000, 1), + ]; + let bytes = synthetic_package_with_one_file_table_block(&entries); + let mut pkg = StfsPackage::open(Cursor::new(bytes)).expect("open"); + let listed = pkg.entries().expect("entries"); + assert_eq!(listed.len(), 2); + assert_eq!(listed[0].name, "Media"); + assert!(listed[0].is_directory); + assert_eq!(listed[1].name, "default.xex"); + assert_eq!(listed[1].size, 0x1000); + assert_eq!(listed[1].parent_index, 0); + } + + #[test] + fn entries_stops_at_empty_slot() { + let entries = vec![fe("only.bin", false, -1, 0x100, 1)]; + let bytes = synthetic_package_with_one_file_table_block(&entries); + let mut pkg = StfsPackage::open(Cursor::new(bytes)).expect("open"); + let listed = pkg.entries().expect("entries"); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].name, "only.bin"); + } + + /// Build a synthetic package with one fragmented file whose blocks are + /// 5 → 3 → 1 (non-consecutive). Requires planting next-block pointers + /// inside the L0 hash block. + fn synthetic_package_with_fragmented_file() -> Vec { + use crate::stfs::block_translator::{BLOCK_SIZE, BLOCKS_PER_L0, FIRST_DATA_BLOCK_OFFSET}; + + // File table at block 0; data blocks 1..=5. + // total_alloc_blocks = 6 (covers blocks 0..=5). + let mut buf = vec![0u8; MIN_HEADER_BYTES]; + buf[0..4].copy_from_slice(b"LIVE"); + buf[0x360..0x364].copy_from_slice(&0x4D5307E6u32.to_be_bytes()); + buf[0x379] = 0x24; + buf[0x37B] = 0x01; + buf[0x37C..0x37E].copy_from_slice(&1u16.to_be_bytes()); + buf[0x395..0x399].copy_from_slice(&6u32.to_be_bytes()); // total_alloc = 6 + + let block_zero_offset = FIRST_DATA_BLOCK_OFFSET as usize; + buf.resize(block_zero_offset, 0); + + // Block 0: file table with one fragmented file + // "frag.bin", consecutive=false, used_blocks=3, start_block=5 + let mut ft_block = vec![0u8; BLOCK_SIZE as usize]; + let mut entry = vec![0u8; 0x40]; + let name = b"frag.bin"; + entry[..name.len()].copy_from_slice(name); + // flags: name length 8, consecutive OFF, dir OFF + entry[0x28] = name.len() as u8 & 0x3F; + entry[0x2C] = 3; // used_blocks + entry[0x2F] = 5; // start_block = 5 + entry[0x32..0x34].copy_from_slice(&(-1i16).to_be_bytes()); + // file_size = 3 * BLOCK_SIZE = 0x3000 + entry[0x34..0x38].copy_from_slice(&0x3000u32.to_be_bytes()); + ft_block[..0x40].copy_from_slice(&entry); + buf.extend_from_slice(&ft_block); + + // Blocks 1..=5: each filled with a single byte equal to the block index. + for b in 1u8..=5 { + let block_data = vec![b; BLOCK_SIZE as usize]; + buf.extend_from_slice(&block_data); + } + + // L0 hash block sits AT block_index = 0xAA per the type-1 layout — + // but our package has only 6 data blocks, so the hash block lives + // beyond the file proper. We need to plant next-block pointers + // somewhere the reader will look for them. + // + // For data block N, the next-block pointer lives in the L0 hash + // block at index (N / BLOCKS_PER_L0) * BLOCKS_PER_L0 + (BLOCKS_PER_L0 - 1) + 1 + // = the position after the group. With BLOCKS_PER_L0 = 0xAA and all + // our blocks in group 0, that's the hash block at block_to_byte_offset(0xAA) - BLOCK_SIZE. + // + // Compute that offset and pad the buffer to it, then plant the + // hash entries for blocks 5 → 3 → 1. + let hash_block_offset = + (FIRST_DATA_BLOCK_OFFSET + (BLOCKS_PER_L0 as u64) * BLOCK_SIZE) as usize; + if buf.len() < hash_block_offset + BLOCK_SIZE as usize { + buf.resize(hash_block_offset + BLOCK_SIZE as usize, 0); + } + let plant_next = |buf: &mut [u8], block: u32, next: u32| { + let entry_off = hash_block_offset + (block as usize % 0xAA) * 24 + 0x15; + buf[entry_off] = (next & 0xFF) as u8; + buf[entry_off + 1] = ((next >> 8) & 0xFF) as u8; + buf[entry_off + 2] = ((next >> 16) & 0xFF) as u8; + }; + plant_next(&mut buf, 5, 3); + plant_next(&mut buf, 3, 1); + // block 1 has no successor; plant 0xFFFFFF (end-of-chain) + plant_next(&mut buf, 1, 0xFFFFFF); + + buf + } + + #[test] + fn read_chain_follows_non_consecutive_pointers() { + let bytes = synthetic_package_with_fragmented_file(); + let mut pkg = StfsPackage::open(Cursor::new(bytes)).expect("open"); + let entries = pkg.entries().expect("entries"); + let entry = &entries[0]; + assert!(!entry.consecutive); + let chain = pkg.read_block_chain(entry).expect("chain"); + assert_eq!(chain, vec![5, 3, 1]); + } + + #[test] + fn read_chain_uses_fast_path_for_consecutive_files() { + let entries = vec![fe("file.bin", false, -1, 3 * 0x1000, 1)]; + // Mark consecutive (fe sets it true) + let bytes = synthetic_package_with_one_file_table_block(&entries); + let mut pkg = StfsPackage::open(Cursor::new(bytes)).expect("open"); + let listed = pkg.entries().expect("entries"); + // fe() sets used_blocks=1, override for this test + let mut entry = listed[0].clone(); + entry.used_blocks = 3; + entry.consecutive = true; + let chain = pkg.read_block_chain(&entry).expect("chain"); + assert_eq!(chain, vec![1, 2, 3]); + } + + #[test] + fn read_chain_caps_at_used_blocks_to_reject_cycles() { + use crate::stfs::block_translator::{BLOCK_SIZE, BLOCKS_PER_L0, FIRST_DATA_BLOCK_OFFSET}; + + // Plant a cycle: 5 → 3 → 5 → 3 → ... + // Start from the unpatched bytes and rewrite the next-pointer for block 3. + let mut raw = synthetic_package_with_fragmented_file(); + let hash_block_offset = + (FIRST_DATA_BLOCK_OFFSET + (BLOCKS_PER_L0 as u64) * BLOCK_SIZE) as usize; + // Rewrite next-pointer for block 3 to point back at block 5 + let entry_off = hash_block_offset + 3 * 24 + 0x15; + raw[entry_off] = 5; + raw[entry_off + 1] = 0; + raw[entry_off + 2] = 0; + let mut pkg = StfsPackage::open(Cursor::new(raw)).expect("open"); + let entry = pkg.entries().expect("entries")[0].clone(); + // used_blocks=3; walk should return exactly 3 blocks even though + // the chain loops 5 → 3 → 5 → 3 → ... + let chain = pkg.read_block_chain(&entry).expect("chain"); + assert_eq!(chain.len(), 3); + } + + #[test] + fn read_file_streams_consecutive_file_contents() { + // Build a package with one consecutive 2-block file containing + // [0xAA; BLOCK_SIZE] then [0xBB; BLOCK_SIZE - 1] (last block 1 byte short). + use crate::stfs::block_translator::{BLOCK_SIZE, FIRST_DATA_BLOCK_OFFSET}; + + let mut buf = vec![0u8; MIN_HEADER_BYTES]; + buf[0..4].copy_from_slice(b"LIVE"); + buf[0x379] = 0x24; + buf[0x37B] = 0x01; + buf[0x37C..0x37E].copy_from_slice(&1u16.to_be_bytes()); + buf[0x395..0x399].copy_from_slice(&3u32.to_be_bytes()); // total_alloc = 3 + + let block_zero = FIRST_DATA_BLOCK_OFFSET as usize; + buf.resize(block_zero, 0); + + let mut ft = vec![0u8; BLOCK_SIZE as usize]; + let name = b"data.bin"; + ft[..name.len()].copy_from_slice(name); + ft[0x28] = (name.len() as u8 & 0x3F) | 0x40; // consecutive + ft[0x2C] = 2; // used_blocks + ft[0x2F] = 1; // start_block + ft[0x32..0x34].copy_from_slice(&(-1i16).to_be_bytes()); + let file_size: u32 = (BLOCK_SIZE as u32) + (BLOCK_SIZE as u32) - 1; + ft[0x34..0x38].copy_from_slice(&file_size.to_be_bytes()); + buf.extend_from_slice(&ft); + + buf.extend_from_slice(&[0xAA; 0x1000]); // block 1 + let mut last_block = vec![0xBBu8; 0x1000]; + last_block[BLOCK_SIZE as usize - 1] = 0; // last byte unused + buf.extend_from_slice(&last_block); + + let mut pkg = StfsPackage::open(Cursor::new(buf)).expect("open"); + let entries = pkg.entries().expect("entries"); + let entry = &entries[0]; + let mut sink = Vec::new(); + pkg.read_file(entry, &mut sink).expect("read_file"); + + assert_eq!(sink.len(), file_size as usize); + assert!(sink[..0x1000].iter().all(|&b| b == 0xAA)); + assert!(sink[0x1000..].iter().all(|&b| b == 0xBB)); + } + + use tempfile::TempDir; + + #[test] + fn extract_to_host_writes_nested_tree() { + // Build: /Media/ -> /Media/cover.png (4 bytes "ABCD") + // /default.xex (4 bytes "MZRX") + use crate::stfs::block_translator::{BLOCK_SIZE, FIRST_DATA_BLOCK_OFFSET}; + + let mut buf = vec![0u8; MIN_HEADER_BYTES]; + buf[0..4].copy_from_slice(b"LIVE"); + buf[0x379] = 0x24; + buf[0x37B] = 0x01; + buf[0x37C..0x37E].copy_from_slice(&1u16.to_be_bytes()); + buf[0x395..0x399].copy_from_slice(&3u32.to_be_bytes()); // total_alloc = 3 + + let block_zero = FIRST_DATA_BLOCK_OFFSET as usize; + buf.resize(block_zero, 0); + + let mut ft = vec![0u8; BLOCK_SIZE as usize]; + // Entry 0: dir "Media", parent -1 (root) + let mut e0 = vec![0u8; 0x40]; + e0[..5].copy_from_slice(b"Media"); + e0[0x28] = 0x05 | 0x40 | 0x80; // dir + consecutive (irrelevant for dir) + e0[0x32..0x34].copy_from_slice(&(-1i16).to_be_bytes()); + ft[..0x40].copy_from_slice(&e0); + // Entry 1: file "cover.png", parent 0 (Media), start_block 1 + let mut e1 = vec![0u8; 0x40]; + e1[..9].copy_from_slice(b"cover.png"); + e1[0x28] = 0x09 | 0x40; + e1[0x2C] = 1; // used_blocks + e1[0x2F] = 1; // start_block + e1[0x32..0x34].copy_from_slice(&0i16.to_be_bytes()); + e1[0x34..0x38].copy_from_slice(&4u32.to_be_bytes()); // size + ft[0x40..0x80].copy_from_slice(&e1); + // Entry 2: file "default.xex", parent -1 (root), start_block 2 + let mut e2 = vec![0u8; 0x40]; + e2[..11].copy_from_slice(b"default.xex"); + e2[0x28] = 0x0B | 0x40; + e2[0x2C] = 1; + e2[0x2F] = 2; + e2[0x32..0x34].copy_from_slice(&(-1i16).to_be_bytes()); + e2[0x34..0x38].copy_from_slice(&4u32.to_be_bytes()); + ft[0x80..0xC0].copy_from_slice(&e2); + buf.extend_from_slice(&ft); + + // Block 1: cover.png contents + let mut b1 = vec![0u8; BLOCK_SIZE as usize]; + b1[..4].copy_from_slice(b"ABCD"); + buf.extend_from_slice(&b1); + // Block 2: default.xex contents + let mut b2 = vec![0u8; BLOCK_SIZE as usize]; + b2[..4].copy_from_slice(b"MZRX"); + buf.extend_from_slice(&b2); + + let mut pkg = StfsPackage::open(Cursor::new(buf)).expect("open"); + let tmp = TempDir::new().expect("tmp"); + let report = extract_to_host(&mut pkg, tmp.path(), None).expect("extract"); + assert_eq!(report.files, 2); + assert_eq!(report.directories, 1); + assert_eq!(report.bytes, 8); + + let cover = std::fs::read(tmp.path().join("Media/cover.png")).expect("read cover"); + assert_eq!(cover, b"ABCD"); + let xex = std::fs::read(tmp.path().join("default.xex")).expect("read xex"); + assert_eq!(xex, b"MZRX"); + } +} diff --git a/fatxlib/src/stfs/file_entry.rs b/fatxlib/src/stfs/file_entry.rs new file mode 100644 index 0000000..97c1097 --- /dev/null +++ b/fatxlib/src/stfs/file_entry.rs @@ -0,0 +1,175 @@ +//! STFS file table entry parser. +//! +//! Each file table block (4 KiB) holds up to 64 entries × 64 bytes. +//! An entry with `name_length == 0` marks end-of-table within a block. + +use crate::error::{FatxError, Result}; + +pub const FILE_ENTRY_SIZE: usize = 0x40; +pub const ENTRIES_PER_BLOCK: usize = 0x1000 / FILE_ENTRY_SIZE; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StfsEntry { + pub name: String, + pub is_directory: bool, + pub consecutive: bool, + pub allocated_blocks: u32, + pub used_blocks: u32, + pub start_block: u32, + /// Signed index into the entry list. `-1` (`0xFFFF`) = root directory parent. + pub parent_index: i16, + pub size: u64, + pub update_timestamp: u32, + pub access_timestamp: u32, +} + +impl StfsEntry { + /// True when this slot is unused (zero name length, no flags). + pub fn is_empty_slot(name_length_and_flags: u8) -> bool { + (name_length_and_flags & 0x3F) == 0 + } +} + +/// Parse one 64-byte entry. Returns `Ok(None)` for empty-slot entries. +pub fn parse(bytes: &[u8]) -> Result> { + if bytes.len() < FILE_ENTRY_SIZE { + return Err(FatxError::Other(format!( + "STFS file entry truncated: got {} bytes, need {}", + bytes.len(), + FILE_ENTRY_SIZE, + ))); + } + let flags = bytes[0x28]; + if StfsEntry::is_empty_slot(flags) { + return Ok(None); + } + let name_len = (flags & 0x3F) as usize; + let consecutive = (flags & 0x40) != 0; + let is_directory = (flags & 0x80) != 0; + + let name_bytes = &bytes[0..name_len]; + // STFS spec says ASCII; tolerate occasional non-printable bytes by + // replacing with '_' rather than rejecting the whole entry. + let name: String = name_bytes + .iter() + .map(|&b| { + if (0x20..=0x7E).contains(&b) { + b as char + } else { + '_' + } + }) + .collect(); + + let allocated_blocks = + (bytes[0x29] as u32) | ((bytes[0x2A] as u32) << 8) | ((bytes[0x2B] as u32) << 16); + let used_blocks = + (bytes[0x2C] as u32) | ((bytes[0x2D] as u32) << 8) | ((bytes[0x2E] as u32) << 16); + let start_block = + (bytes[0x2F] as u32) | ((bytes[0x30] as u32) << 8) | ((bytes[0x31] as u32) << 16); + let parent_index = i16::from_be_bytes([bytes[0x32], bytes[0x33]]); + let size = u32::from_be_bytes([bytes[0x34], bytes[0x35], bytes[0x36], bytes[0x37]]) as u64; + let update_timestamp = u32::from_be_bytes([bytes[0x38], bytes[0x39], bytes[0x3A], bytes[0x3B]]); + let access_timestamp = u32::from_be_bytes([bytes[0x3C], bytes[0x3D], bytes[0x3E], bytes[0x3F]]); + + Ok(Some(StfsEntry { + name, + is_directory, + consecutive, + allocated_blocks, + used_blocks, + start_block, + parent_index, + size, + update_timestamp, + access_timestamp, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn synthetic_entry( + name: &str, + is_dir: bool, + consecutive: bool, + used_blocks: u32, + start_block: u32, + parent_index: i16, + size: u32, + ) -> Vec { + let mut buf = vec![0u8; FILE_ENTRY_SIZE]; + let bytes = name.as_bytes(); + buf[..bytes.len()].copy_from_slice(bytes); + let mut flags = bytes.len() as u8 & 0x3F; + if consecutive { + flags |= 0x40; + } + if is_dir { + flags |= 0x80; + } + buf[0x28] = flags; + buf[0x29] = (used_blocks & 0xFF) as u8; + buf[0x2A] = ((used_blocks >> 8) & 0xFF) as u8; + buf[0x2B] = ((used_blocks >> 16) & 0xFF) as u8; + buf[0x2C] = (used_blocks & 0xFF) as u8; + buf[0x2D] = ((used_blocks >> 8) & 0xFF) as u8; + buf[0x2E] = ((used_blocks >> 16) & 0xFF) as u8; + buf[0x2F] = (start_block & 0xFF) as u8; + buf[0x30] = ((start_block >> 8) & 0xFF) as u8; + buf[0x31] = ((start_block >> 16) & 0xFF) as u8; + buf[0x32..0x34].copy_from_slice(&parent_index.to_be_bytes()); + buf[0x34..0x38].copy_from_slice(&size.to_be_bytes()); + buf + } + + #[test] + fn parses_consecutive_file_entry() { + let raw = synthetic_entry("default.xex", false, true, 8, 1, -1, 0x12345); + let entry = parse(&raw).expect("parse").expect("non-empty"); + assert_eq!(entry.name, "default.xex"); + assert!(!entry.is_directory); + assert!(entry.consecutive); + assert_eq!(entry.used_blocks, 8); + assert_eq!(entry.start_block, 1); + assert_eq!(entry.parent_index, -1); + assert_eq!(entry.size, 0x12345); + } + + #[test] + fn parses_directory_entry() { + let raw = synthetic_entry("Media", true, false, 0, 0, -1, 0); + let entry = parse(&raw).expect("parse").expect("non-empty"); + assert!(entry.is_directory); + assert!(!entry.consecutive); + assert_eq!(entry.name, "Media"); + } + + #[test] + fn empty_slot_returns_none() { + let raw = vec![0u8; FILE_ENTRY_SIZE]; + let entry = parse(&raw).expect("parse"); + assert!(entry.is_none()); + } + + #[test] + fn non_printable_bytes_in_name_become_underscore() { + let mut raw = synthetic_entry("abc", false, true, 1, 0, -1, 0); + raw[1] = 0x00; // corrupt middle byte + let entry = parse(&raw).expect("parse").expect("non-empty"); + assert_eq!(entry.name, "a_c"); + } + + #[test] + fn rejects_truncated_input() { + let raw = vec![0u8; FILE_ENTRY_SIZE - 1]; + let err = parse(&raw).expect_err("should reject"); + assert!(format!("{}", err).contains("truncated")); + } + + #[test] + fn entries_per_block_is_64() { + assert_eq!(ENTRIES_PER_BLOCK, 64); + } +} diff --git a/fatxlib/src/stfs.rs b/fatxlib/src/stfs/header.rs similarity index 100% rename from fatxlib/src/stfs.rs rename to fatxlib/src/stfs/header.rs diff --git a/fatxlib/src/stfs/mod.rs b/fatxlib/src/stfs/mod.rs new file mode 100644 index 0000000..bf4931c --- /dev/null +++ b/fatxlib/src/stfs/mod.rs @@ -0,0 +1,14 @@ +//! STFS (Secure Transacted File System) container format. +//! +//! Xbox 360 packages — CON (signed by console), LIVE (signed by Microsoft Live), +//! and PIRS (signed Microsoft installer) — share the same on-disk layout. +//! This module groups all STFS parsing and extraction logic. + +pub mod block_translator; +pub mod extract; +pub mod file_entry; +pub mod header; +pub mod volume_descriptor; + +pub use extract::StfsPackage; +pub use header::{MIN_HEADER_BYTES, StfsHeader, parse_header}; diff --git a/fatxlib/src/stfs/volume_descriptor.rs b/fatxlib/src/stfs/volume_descriptor.rs new file mode 100644 index 0000000..c095bc3 --- /dev/null +++ b/fatxlib/src/stfs/volume_descriptor.rs @@ -0,0 +1,124 @@ +//! STFS volume descriptor parser. +//! +//! Lives at offset 0x379 inside the STFS header. 36 bytes (size = 0x24). +//! Tells us where the file table lives and whether the package is type 0 +//! (read-write) or type 1 (read-only). v1 only supports type 1. + +use crate::error::{FatxError, Result}; + +/// Volume descriptor offset relative to the start of the STFS file. +pub const VOLUME_DESCRIPTOR_OFFSET: usize = 0x379; + +/// Expected `descriptor_size` value. Anything else is rejected. +pub const VOLUME_DESCRIPTOR_SIZE: u8 = 0x24; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VolumeDescriptor { + /// True for read-only ("male") packs. v1 supports only true. + pub read_only_format: bool, + pub file_table_block_count: u16, + pub file_table_block_number: u32, + pub total_alloc_blocks: u32, + pub total_unalloc_blocks: u32, +} + +/// Parse a volume descriptor from a slice positioned at offset 0x379 inside +/// the STFS header (i.e. pass `&header_bytes[0x379..0x379 + 0x24]`). +pub fn parse(bytes: &[u8]) -> Result { + if bytes.len() < VOLUME_DESCRIPTOR_SIZE as usize { + return Err(FatxError::Other(format!( + "STFS volume descriptor truncated: got {} bytes, need {}", + bytes.len(), + VOLUME_DESCRIPTOR_SIZE, + ))); + } + let descriptor_size = bytes[0]; + if descriptor_size != VOLUME_DESCRIPTOR_SIZE { + return Err(FatxError::Other(format!( + "Unsupported STFS volume descriptor size: 0x{:02X} (expected 0x{:02X})", + descriptor_size, VOLUME_DESCRIPTOR_SIZE, + ))); + } + let block_separation = bytes[2]; + let read_only_format = (block_separation & 0x01) != 0; + let file_table_block_count = u16::from_be_bytes([bytes[3], bytes[4]]); + // file_table_block_number is stored LITTLE-endian in a BE-dominated header + let file_table_block_number = + (bytes[5] as u32) | ((bytes[6] as u32) << 8) | ((bytes[7] as u32) << 16); + let total_alloc_blocks = + u32::from_be_bytes([bytes[0x1C], bytes[0x1D], bytes[0x1E], bytes[0x1F]]); + let total_unalloc_blocks = + u32::from_be_bytes([bytes[0x20], bytes[0x21], bytes[0x22], bytes[0x23]]); + Ok(VolumeDescriptor { + read_only_format, + file_table_block_count, + file_table_block_number, + total_alloc_blocks, + total_unalloc_blocks, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn synthetic_descriptor( + size: u8, + block_separation: u8, + file_table_block_count: u16, + file_table_block_number: u32, + total_alloc: u32, + total_unalloc: u32, + ) -> Vec { + let mut buf = vec![0u8; 0x24]; + buf[0] = size; + buf[2] = block_separation; + buf[3..5].copy_from_slice(&file_table_block_count.to_be_bytes()); + buf[5] = (file_table_block_number & 0xFF) as u8; + buf[6] = ((file_table_block_number >> 8) & 0xFF) as u8; + buf[7] = ((file_table_block_number >> 16) & 0xFF) as u8; + buf[0x1C..0x20].copy_from_slice(&total_alloc.to_be_bytes()); + buf[0x20..0x24].copy_from_slice(&total_unalloc.to_be_bytes()); + buf + } + + #[test] + fn parses_type_1_descriptor() { + let bytes = synthetic_descriptor(0x24, 0x01, 256, 0, 12_548, 0); + let vd = parse(&bytes).expect("parse"); + assert!(vd.read_only_format); + assert_eq!(vd.file_table_block_count, 256); + assert_eq!(vd.file_table_block_number, 0); + assert_eq!(vd.total_alloc_blocks, 12_548); + assert_eq!(vd.total_unalloc_blocks, 0); + } + + #[test] + fn detects_type_0_via_block_separation_bit_0_clear() { + let bytes = synthetic_descriptor(0x24, 0x00, 1, 0, 1, 0); + let vd = parse(&bytes).expect("parse"); + assert!(!vd.read_only_format); + } + + #[test] + fn file_table_block_number_uses_little_endian_within_be_stream() { + // Stored bytes 0x010203 (LE) → numeric value 0x00030201 + let bytes = synthetic_descriptor(0x24, 0x01, 1, 0x00030201, 1, 0); + let vd = parse(&bytes).expect("parse"); + assert_eq!(vd.file_table_block_number, 0x00030201); + } + + #[test] + fn rejects_wrong_descriptor_size() { + let bytes = synthetic_descriptor(0x23, 0x01, 1, 0, 1, 0); + let err = parse(&bytes).expect_err("should reject"); + assert!(format!("{}", err).contains("Unsupported STFS volume descriptor")); + } + + #[test] + fn rejects_truncated_input() { + let bytes = vec![0u8; 0x20]; + let err = parse(&bytes).expect_err("should reject"); + assert!(format!("{}", err).contains("truncated")); + } +} diff --git a/install.sh b/install.sh index 9759fef..28d3c25 100755 --- a/install.sh +++ b/install.sh @@ -10,7 +10,7 @@ # bash install.sh # # Options: -# XTAFKIT_VERSION=v1.2.1 bash install.sh # install a specific version +# XTAFKIT_VERSION=v1.3.0 bash install.sh # install a specific version # XTAFKIT_INSTALL_DIR=~/.local/bin bash install.sh # custom install directory set -e diff --git a/src/extract_stfs.rs b/src/extract_stfs.rs new file mode 100644 index 0000000..9623eeb --- /dev/null +++ b/src/extract_stfs.rs @@ -0,0 +1,253 @@ +//! `xtafkit extract-stfs` — extract an STFS package to a local directory. +//! +//! Mirrors the shape of `run_extract` (XISO) in main.rs. + +use std::fs::File; +use std::io; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use fatxlib::stfs::{StfsHeader, StfsPackage, extract::extract_to_host}; + +use crate::{cli_error, format_size, short_name}; + +pub fn run_extract_stfs(package: &Path, to: Option<&Path>, dry_run: bool, json: bool) { + let file = match File::open(package) { + Ok(f) => f, + Err(e) => { + cli_error(json, &format!("open {}: {}", package.display(), e)); + return; + } + }; + + let mut pkg = match StfsPackage::open(file) { + Ok(p) => p, + Err(e) => { + cli_error(json, &format!("parse {}: {}", package.display(), e)); + return; + } + }; + + let dest = match to { + Some(p) => p.to_path_buf(), + None => default_destination(package, pkg.header()), + }; + + let entries = match pkg.entries() { + Ok(e) => e, + Err(e) => { + cli_error(json, &format!("walk {}: {}", package.display(), e)); + return; + } + }; + let total_files = entries.iter().filter(|e| !e.is_directory).count(); + let total_bytes: u64 = entries + .iter() + .filter(|e| !e.is_directory) + .map(|e| e.size) + .sum(); + + if dry_run { + if json { + let entries_json: Vec<_> = entries + .iter() + .map(|e| { + serde_json::json!({ + "name": e.name, + "parent_index": e.parent_index, + "is_directory": e.is_directory, + "size": e.size, + "blocks_used": e.used_blocks, + "consecutive": e.consecutive, + }) + }) + .collect(); + println!( + "{}", + serde_json::json!({ + "package": package.display().to_string(), + "magic": String::from_utf8_lossy(&pkg.header().magic), + "title_id": format!("{:08X}", pkg.header().title_id), + "title_name": pkg.header().title_name, + "display_name": pkg.header().display_name, + "files": total_files, + "directories": entries.iter().filter(|e| e.is_directory).count(), + "bytes": total_bytes, + "entries": entries_json, + }) + ); + } else { + println!("Package: {}", package.display()); + println!("Magic: {}", String::from_utf8_lossy(&pkg.header().magic),); + println!( + "Title: {} (0x{:08X})", + pkg.header().best_name(), + pkg.header().title_id, + ); + println!("Files: {} ({})", total_files, format_size(total_bytes),); + println!(); + for entry in &entries { + let kind = if entry.is_directory { "dir " } else { "file" }; + println!( + " {} {:48} {:>10}", + kind, + entry.name, + if entry.is_directory { + String::from("-") + } else { + format_size(entry.size) + }, + ); + } + println!(); + println!("(dry-run; nothing written)"); + } + return; + } + + if let Err(e) = std::fs::create_dir_all(&dest) { + cli_error(json, &format!("create_dir_all {}: {}", dest.display(), e)); + return; + } + + let started = Instant::now(); + let last_progress = std::cell::Cell::new(Instant::now()); + let total_bytes_ref = total_bytes; + let cb = |path: &str, _size: u64, bytes_done: u64| { + if !json && last_progress.get().elapsed().as_millis() > 250 { + eprint!( + "\r {} ({}/{}) ", + short_name(path), + format_size(bytes_done), + format_size(total_bytes_ref), + ); + let _ = io::stderr().flush(); + last_progress.set(Instant::now()); + } + }; + + let report = match extract_to_host(&mut pkg, &dest, Some(&cb)) { + Ok(r) => r, + Err(e) => { + if !json { + eprintln!(); + } + cli_error(json, &format!("extract: {}", e)); + return; + } + }; + + let elapsed = started.elapsed(); + if !json { + eprint!("\r{:80}\r", ""); + } + if json { + println!( + "{}", + serde_json::json!({ + "package": package.display().to_string(), + "dest": dest.display().to_string(), + "title_id": format!("{:08X}", pkg.header().title_id), + "files": report.files, + "directories": report.directories, + "bytes": report.bytes, + "elapsed_secs": elapsed.as_secs_f64(), + }) + ); + } else { + println!( + "Extracted {} files, {} dirs ({}) → {} in {:?}", + report.files, + report.directories, + format_size(report.bytes), + dest.display(), + elapsed, + ); + } +} + +/// Default destination when `--to` is omitted. Prefers the STFS header's +/// own naming over the bundled catalog because the catalog can only resolve +/// `title_id` to a *system* name (e.g. XBLIG → "Community Package"), while +/// the per-package `display_name` carries the specific title (e.g. "DLC +/// Quest"). Resolution order: +/// 1. `header.display_name` — specific package label. +/// 2. `header.title_name` — often the parent/system label. +/// 3. source file stem — final fallback. +fn default_destination(package: &Path, header: &StfsHeader) -> PathBuf { + let candidate = if !header.display_name.trim().is_empty() { + header.display_name.trim().to_string() + } else if !header.title_name.trim().is_empty() { + header.title_name.trim().to_string() + } else { + return PathBuf::from( + package + .file_stem() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|| "stfs-extract".to_string()), + ); + }; + PathBuf::from(sanitize_fatx_name(&candidate)) +} + +/// Replace characters illegal as macOS / Windows path components. +fn sanitize_fatx_name(name: &str) -> String { + name.chars() + .map(|c| match c { + '/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' => '-', + c if (c as u32) < 0x20 => '-', + c => c, + }) + .collect::() + .trim() + .to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn header(display_name: &str, title_name: &str) -> StfsHeader { + StfsHeader { + magic: *b"LIVE", + title_id: 0, + title_name: title_name.to_string(), + display_name: display_name.to_string(), + } + } + + #[test] + fn default_destination_uses_display_name_when_present() { + // display_name is the specific package label; title_name is often + // a system bucket. We want the specific name. + let dest = default_destination( + Path::new("/tmp/pkg"), + &header("Specific Title", "System Bucket"), + ); + assert_eq!(dest, PathBuf::from("Specific Title")); + } + + #[test] + fn default_destination_omits_title_id_suffix() { + // Regression guard: no "[XXXXXXXX]" appended to the folder name. + let dest = + default_destination(Path::new("/tmp/pkg"), &header("Sample Game", "Sample Game")); + let s = dest.to_string_lossy(); + assert!(!s.contains('['), "unexpected title-id suffix: {s}"); + assert_eq!(dest, PathBuf::from("Sample Game")); + } + + #[test] + fn default_destination_falls_back_to_title_name_when_display_empty() { + let dest = default_destination(Path::new("/tmp/pkg"), &header("", "Bucket: Subtitle")); + // The colon is illegal on Windows/macOS and gets sanitized to '-'. + assert_eq!(dest, PathBuf::from("Bucket- Subtitle")); + } + + #[test] + fn default_destination_falls_back_to_file_stem_when_no_names() { + let dest = default_destination(Path::new("/tmp/some_pkg.stfs"), &header("", "")); + assert_eq!(dest, PathBuf::from("some_pkg")); + } +} diff --git a/src/main.rs b/src/main.rs index 0942cf4..a785f59 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,6 +6,7 @@ //! xtafkit scan /dev/rdisk4 //! xtafkit ls /dev/rdisk4 --partition "Data (E)" / +mod extract_stfs; mod mkimage; mod tui; @@ -189,6 +190,20 @@ enum Commands { #[arg(long)] dry_run: bool, }, + /// Extract every file from an Xbox 360 STFS package (CON / LIVE / PIRS) + /// to a local directory. Works on Arcade (XBLA), XBLIG, Title Updates, + /// Marketplace DLC, and other type-1 packages. + ExtractStfs { + /// Source STFS package + package: PathBuf, + /// Destination directory (created if missing). Defaults to + /// `./ []/` (or `.//` on catalog miss). + #[arg(long)] + to: Option, + /// Print the file list and totals without writing anything. + #[arg(long)] + dry_run: bool, + }, /// Convert an Xbox 360 XISO into a Games-on-Demand package in a local /// directory. Writes `///{,.data/}`. God { @@ -1146,6 +1161,12 @@ fn main() { dry_run, game_title, }) => run_god(&iso, &dest, &trim, dry_run, game_title.as_deref(), json), + + Some(Commands::ExtractStfs { + package, + to, + dry_run, + }) => extract_stfs::run_extract_stfs(&package, to.as_deref(), dry_run, json), } }