Further log stuff

Cargo.toml

@@ -5,7 +5,7 @@ resolver = "2"
 members = ["crates/*"]
 
 [workspace.package]
-version = "0.1.26"
+version = "0.1.27"
 repository = "https://github.com/human-solutions/builder"
 license = "MIT"
 edition = "2024"
@@ -41,6 +41,7 @@ brotli = "8.0"
 camino-fs = "0.1"
 cargo_metadata = "0.22"
 flate2 = "1.1"
+fs-err = "3.1"
 grass = "0.13"
 icu_locid = "1.5"
 lightningcss = { version = "1.0.0-alpha.67", features = ["browserslist"] }

crates/builder/src/main.rs

@@ -3,6 +3,7 @@ use std::env;
 
 use builder_command::{BuilderCmd, Cmd};
 use camino_fs::*;
+use common::site_fs;
 use common::{LOG_LEVEL, RELEASE, setup_logging};
 
 fn main() {
@@ -56,4 +57,9 @@ pub fn run(builder: BuilderCmd) {
             Cmd::SwiftPackage(cmd) => builder_swift_package::run(cmd),
         }
     }
+
+    // Finalize hash output files after all commands have completed
+    if let Err(e) = site_fs::finalize_hash_outputs() {
+        eprintln!("Failed to write hash output files: {}", e);
+    }
 }

crates/command/Cargo.toml

@@ -10,4 +10,5 @@ version.workspace = true
 
 [dependencies]
 camino-fs.workspace = true
+fs-err.workspace = true
 log.workspace = true

crates/command/src/lib.rs

@@ -14,11 +14,11 @@ pub use assemble::AssembleCmd;
 use camino_fs::Utf8PathBuf;
 pub use copy::CopyCmd;
 pub use fontforge::FontForgeCmd;
+use fs_err as fs;
 pub use localized::LocalizedCmd;
 use log::LevelFilter;
 pub use out::{Encoding, Output};
 pub use sass::SassCmd;
-use std::fs;
 pub use swift_package::SwiftPackageCmd;
 pub use uniffi::UniffiCmd;
 pub use wasm::{DebugSymbolsMode, Profile, WasmProcessingCmd};
@@ -32,10 +32,10 @@ pub enum LogLevel {
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum LogDestination {
-    Cargo,                    // via cargo::warning
-    File(Utf8PathBuf),       // given a path
-    Terminal,                // standard output
-    TerminalPlain,          // standard output, designed for when run in a Command that adds it's own prefixes to the logs
+    Cargo,             // via cargo::warning
+    File(Utf8PathBuf), // given a path
+    Terminal,          // standard output
+    TerminalPlain, // standard output, designed for when run in a Command that adds it's own prefixes to the logs
 }
 
 impl LogLevel {
@@ -196,15 +196,15 @@ impl Display for BuilderCmd {
             LogLevel::Trace => "trace",
         };
         writeln!(f, "log_level={}", log_level_str)?;
-        
+
         let log_destination_str = match &self.log_destination {
             LogDestination::Cargo => "cargo".to_string(),
             LogDestination::File(path) => format!("file:{}", path),
             LogDestination::Terminal => "terminal".to_string(),
             LogDestination::TerminalPlain => "terminal_plain".to_string(),
         };
         writeln!(f, "log_destination={}", log_destination_str)?;
-        
+
         writeln!(f, "release={}", self.release)?;
         writeln!(f, "builder_toml={}", self.builder_toml)?;
         for cmd in &self.cmds {
@@ -328,7 +328,9 @@ fn roundtrip() {
         .add_copy(CopyCmd::default())
         .add_swift_package(SwiftPackageCmd::default())
         .log_level(LogLevel::Verbose)
-        .log_destination(LogDestination::File(camino_fs::Utf8PathBuf::from("/tmp/builder.log")))
+        .log_destination(LogDestination::File(camino_fs::Utf8PathBuf::from(
+            "/tmp/builder.log",
+        )))
         .release(true)
         .builder_toml("builder.toml");
 

crates/command/src/out.rs

@@ -67,6 +67,9 @@ pub struct Output {
     all_encodings: bool,
 
     pub checksum: bool,
+
+    /// Optional path to write file hashes as a Rust file
+    pub hash_output_path: Option<Utf8PathBuf>,
 }
 
 impl Output {
@@ -79,6 +82,7 @@ impl Output {
             uncompressed: false,
             all_encodings: false,
             checksum: false,
+            hash_output_path: None,
         }
     }
 
@@ -91,6 +95,7 @@ impl Output {
             uncompressed: true,
             all_encodings: true,
             checksum: true,
+            hash_output_path: None,
         }
     }
 
@@ -103,6 +108,7 @@ impl Output {
             uncompressed: true,
             all_encodings: true,
             checksum: false,
+            hash_output_path: None,
         }
     }
 
@@ -111,6 +117,11 @@ impl Output {
         self
     }
 
+    pub fn hash_output_path<P: Into<Utf8PathBuf>>(mut self, path: P) -> Self {
+        self.hash_output_path = Some(path.into());
+        self
+    }
+
     pub fn uncompressed(&self) -> bool {
         // if none are set, then default to uncompressed
         let default_uncompressed = !self.uncompressed && !self.brotli && !self.gzip;
@@ -150,7 +161,10 @@ impl Display for Output {
         write!(f, "gzip={}\t", self.gzip)?;
         write!(f, "uncompressed={}\t", self.uncompressed)?;
         write!(f, "all_encodings={}\t", self.all_encodings)?;
-        write!(f, "checksum={}", self.checksum)?;
+        write!(f, "checksum={}\t", self.checksum)?;
+        if let Some(hash_output_path) = &self.hash_output_path {
+            write!(f, "hash_output_path={}\t", hash_output_path)?;
+        }
         Ok(())
     }
 }
@@ -160,7 +174,7 @@ impl FromStr for Output {
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         let mut cmd = Output::default();
-        for item in s.split('\t') {
+        for item in s.split('\t').filter(|s| !s.is_empty()) {
             let (key, value) = item.split_once('=').unwrap();
 
             match key {
@@ -171,6 +185,7 @@ impl FromStr for Output {
                 "uncompressed" => cmd.uncompressed = value.parse().unwrap(),
                 "all_encodings" => cmd.all_encodings = value.parse().unwrap(),
                 "checksum" => cmd.checksum = value.parse().unwrap(),
+                "hash_output_path" => cmd.hash_output_path = Some(value.into()),
                 _ => panic!("unknown key: {}", key),
             }
         }

crates/common/Cargo.toml

@@ -12,11 +12,12 @@ builder-command = { path = "../command" }
 
 anyhow.workspace = true
 base64.workspace = true
+brotli.workspace = true
 camino-fs.workspace = true
+flate2.workspace = true
+fs-err.workspace = true
 icu_locid.workspace = true
 log.workspace = true
-brotli.workspace = true
-flate2.workspace = true
 seahash.workspace = true
 simplelog.workspace = true
 time.workspace = true

crates/common/src/hash_output.rs

@@ -0,0 +1,197 @@
+use anyhow::Result;
+use camino_fs::{Utf8Path, Utf8PathExt};
+use std::collections::BTreeMap;
+
+/// Converts a file path to a valid Rust constant name
+/// Removes hash part, converts to uppercase, and replaces invalid characters with underscores
+fn file_path_to_const_name(file_path: &str) -> String {
+    let mut const_name = String::new();
+
+    // Split the path and remove any hash parts (parts ending with '=')
+    for part in file_path.split('/') {
+        if !const_name.is_empty() {
+            const_name.push('_');
+        }
+
+        // Split by '.' to separate name, potential hash, and extension
+        let parts: Vec<&str> = part.split('.').collect();
+
+        if parts.len() >= 2 {
+            // Add the name part
+            const_name.push_str(&sanitize_for_const(parts[0]));
+
+            // Find the extension (last part that's not a hash)
+            let mut ext_parts = Vec::new();
+            for p in parts.iter().skip(1) {
+                if !p.ends_with('=') {
+                    ext_parts.push(*p);
+                }
+            }
+
+            // Add extension parts
+            for ext in ext_parts {
+                const_name.push('_');
+                const_name.push_str(&sanitize_for_const(ext));
+            }
+        } else {
+            const_name.push_str(&sanitize_for_const(part));
+        }
+    }
+
+    const_name.to_uppercase()
+}
+
+/// Sanitizes a string part for use in a Rust constant name
+fn sanitize_for_const(s: &str) -> String {
+    s.chars()
+        .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' })
+        .collect()
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct HashEntry {
+    pub file_path: String,
+    pub hash: String,
+}
+
+impl HashEntry {
+    pub fn new<P: Into<String>, H: Into<String>>(file_path: P, hash: H) -> Self {
+        Self {
+            file_path: file_path.into(),
+            hash: hash.into(),
+        }
+    }
+}
+
+/// Collects hash entries and writes them to a Rust file
+pub struct HashCollector {
+    entries: BTreeMap<String, String>,
+}
+
+impl HashCollector {
+    pub fn new() -> Self {
+        Self {
+            entries: BTreeMap::new(),
+        }
+    }
+
+    pub fn add_entry<P: Into<String>, H: Into<String>>(&mut self, file_path: P, hash: H) {
+        self.entries.insert(file_path.into(), hash.into());
+    }
+
+    pub fn write_to_rust_file(&self, output_path: &Utf8Path) -> Result<()> {
+        let rust_content = self.generate_rust_code();
+
+        // Ensure parent directory exists
+        if let Some(parent) = output_path.parent() {
+            parent.mkdirs()?;
+        }
+
+        output_path.write(&rust_content)?;
+        Ok(())
+    }
+
+    fn generate_rust_code(&self) -> String {
+        let mut content = String::new();
+
+        content.push_str("// This file is auto-generated by the builder tool.\n");
+        content.push_str("// Do not edit manually - it will be overwritten.\n\n");
+
+        if self.entries.is_empty() {
+            content.push_str("// No files with hashes were generated.\n");
+        } else {
+            for (file_path, hash) in &self.entries {
+                let const_name = file_path_to_const_name(file_path);
+                content.push_str(&format!("pub const {}: &str = \"{}\";\n", const_name, hash));
+            }
+        }
+
+        content
+    }
+}
+
+impl Default for HashCollector {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_file_path_to_const_name() {
+        // Basic file names
+        assert_eq!(file_path_to_const_name("style.css"), "STYLE_CSS");
+        assert_eq!(file_path_to_const_name("script.js"), "SCRIPT_JS");
+
+        // Files with paths
+        assert_eq!(
+            file_path_to_const_name("assets/style.css"),
+            "ASSETS_STYLE_CSS"
+        );
+        assert_eq!(
+            file_path_to_const_name("js/modules/app.js"),
+            "JS_MODULES_APP_JS"
+        );
+
+        // Files with hashes (should be removed)
+        assert_eq!(file_path_to_const_name("style.abc123=.css"), "STYLE_CSS");
+        assert_eq!(
+            file_path_to_const_name("assets/script.def456=.js"),
+            "ASSETS_SCRIPT_JS"
+        );
+
+        // Special characters (should be sanitized)
+        assert_eq!(file_path_to_const_name("my-file.css"), "MY_FILE_CSS");
+        assert_eq!(file_path_to_const_name("file@2x.png"), "FILE_2X_PNG");
+    }
+
+    #[test]
+    fn test_sanitize_for_const() {
+        assert_eq!(sanitize_for_const("hello"), "hello");
+        assert_eq!(sanitize_for_const("hello-world"), "hello_world");
+        assert_eq!(sanitize_for_const("file@2x"), "file_2x");
+        assert_eq!(sanitize_for_const("123abc"), "123abc");
+    }
+
+    #[test]
+    fn test_hash_collector_empty() {
+        let collector = HashCollector::new();
+        let rust_code = collector.generate_rust_code();
+        assert!(rust_code.contains("// No files with hashes were generated."));
+    }
+
+    #[test]
+    fn test_hash_collector_with_entries() {
+        let mut collector = HashCollector::new();
+        collector.add_entry("style.css", "abc123");
+        collector.add_entry("script.js", "def456");
+
+        let rust_code = collector.generate_rust_code();
+        assert!(rust_code.contains("pub const STYLE_CSS: &str = \"abc123\";"));
+        assert!(rust_code.contains("pub const SCRIPT_JS: &str = \"def456\";"));
+        assert!(!rust_code.contains("FILE_HASHES")); // Should not contain the old array format
+    }
+
+    #[test]
+    fn test_hash_collector_with_complex_paths() {
+        let mut collector = HashCollector::new();
+        collector.add_entry("assets/styles/main.css", "hash1");
+        collector.add_entry("js/app.min.js", "hash2");
+        collector.add_entry("images/logo@2x.png", "hash3");
+
+        let rust_code = collector.generate_rust_code();
+        assert!(rust_code.contains("pub const ASSETS_STYLES_MAIN_CSS: &str = \"hash1\";"));
+        assert!(rust_code.contains("pub const JS_APP_MIN_JS: &str = \"hash2\";"));
+        assert!(rust_code.contains("pub const IMAGES_LOGO_2X_PNG: &str = \"hash3\";"));
+    }
+
+    #[test]
+    fn test_hash_entry_creation() {
+        let entry = HashEntry::new("test.txt", "hash123");
+        assert_eq!(entry.file_path, "test.txt");
+        assert_eq!(entry.hash, "hash123");
+    }
+}