From ec7754b2c402ecfc87027805b2240a7b5b0d84b6 Mon Sep 17 00:00:00 2001 From: ovo-Tim Date: Wed, 13 Aug 2025 17:35:55 +0800 Subject: [PATCH 01/15] feat(kulfi-utils): prepare for crates.io --- kulfi-utils/Cargo.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kulfi-utils/Cargo.toml b/kulfi-utils/Cargo.toml index e42998b..03e4c7c 100644 --- a/kulfi-utils/Cargo.toml +++ b/kulfi-utils/Cargo.toml @@ -3,9 +3,11 @@ name = "kulfi-utils" version = "0.1.1" edition.workspace = true authors.workspace = true -description = "ftnet utilities" +description = "Kulfi utilities" homepage.workspace = true license.workspace = true +repository.workspace = true +readme.workspace = true [dependencies] bb8.workspace = true From 1998d73b46ba9497111b8a6dc37464efdc98957d Mon Sep 17 00:00:00 2001 From: ovo-Tim Date: Wed, 13 Aug 2025 18:19:40 +0800 Subject: [PATCH 02/15] feat(ci): publish kulfi-utils to crate.io --- .github/workflows/release-kulfi-utils.yml | 47 +++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/release-kulfi-utils.yml diff --git a/.github/workflows/release-kulfi-utils.yml b/.github/workflows/release-kulfi-utils.yml new file mode 100644 index 0000000..a869075 --- /dev/null +++ b/.github/workflows/release-kulfi-utils.yml @@ -0,0 +1,47 @@ +name: Release kulfi-utils + +on: + workflow_dispatch: + +jobs: + build-for-linux-windows: + name: Build for Linux(x86-64 and aarch64) and Windows + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install Linux and Windows Cross Compilers + run: sudo apt-get install --yes --no-install-recommends musl-tools gcc-mingw-w64-x86-64-win32 gcc-aarch64-linux-gnu + - name: Install rustup targets + run: rustup target add x86_64-unknown-linux-musl x86_64-pc-windows-gnu aarch64-unknown-linux-gnu + - uses: Swatinem/rust-cache@v2 + with: + key: " build-for-linux-windows" + - name: Build + run: cargo package --target x86_64-unknown-linux-musl --target x86_64-pc-windows-gnu --target aarch64-unknown-linux-gnu + - uses: actions/upload-artifact@v4 + with: + path: target/package/*.crate + - name: publish to crates.io + run: | + cargo login ${{ secrets.CARGO_REGISTRY_TOKEN }} + cargo publish + + + build-for-macos: + name: Build for macOS + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + key: " build-for-macos" + - name: Build + run: cargo package + - uses: actions/upload-artifact@v4 + with: + path: target/package/*.crate + - name: publish to crates.io + run: | + cargo login ${{ secrets.CARGO_REGISTRY_TOKEN }} + cargo publish From eeaa02c83045b5716854ae06d0233846c30be28c Mon Sep 17 00:00:00 2001 From: ovo-Tim Date: Wed, 13 Aug 2025 21:21:39 +0800 Subject: [PATCH 03/15] feat(ci): Release kulfi-utils to crate.io --- .github/workflows/release-kulfi-utils.yml | 50 ++++------------------- 1 file changed, 9 insertions(+), 41 deletions(-) diff --git a/.github/workflows/release-kulfi-utils.yml b/.github/workflows/release-kulfi-utils.yml index a869075..ee1170e 100644 --- a/.github/workflows/release-kulfi-utils.yml +++ b/.github/workflows/release-kulfi-utils.yml @@ -4,44 +4,12 @@ on: workflow_dispatch: jobs: - build-for-linux-windows: - name: Build for Linux(x86-64 and aarch64) and Windows - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Install Linux and Windows Cross Compilers - run: sudo apt-get install --yes --no-install-recommends musl-tools gcc-mingw-w64-x86-64-win32 gcc-aarch64-linux-gnu - - name: Install rustup targets - run: rustup target add x86_64-unknown-linux-musl x86_64-pc-windows-gnu aarch64-unknown-linux-gnu - - uses: Swatinem/rust-cache@v2 - with: - key: " build-for-linux-windows" - - name: Build - run: cargo package --target x86_64-unknown-linux-musl --target x86_64-pc-windows-gnu --target aarch64-unknown-linux-gnu - - uses: actions/upload-artifact@v4 - with: - path: target/package/*.crate - - name: publish to crates.io - run: | - cargo login ${{ secrets.CARGO_REGISTRY_TOKEN }} - cargo publish - - - build-for-macos: - name: Build for macOS - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - with: - key: " build-for-macos" - - name: Build - run: cargo package - - uses: actions/upload-artifact@v4 - with: - path: target/package/*.crate - - name: publish to crates.io - run: | - cargo login ${{ secrets.CARGO_REGISTRY_TOKEN }} - cargo publish + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Publish to crates.io + run: | + cd kulfi-utils + cargo login ${{ secrets.CARGO_REGISTRY_TOKEN }} + cargo publish \ No newline at end of file From e1cd51e225dee4a7f9316e12c040250315089bf2 Mon Sep 17 00:00:00 2001 From: ovo-Tim Date: Fri, 15 Aug 2025 13:31:55 +0800 Subject: [PATCH 04/15] chore: Bump the version of kulfi-utils --- kulfi-utils/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kulfi-utils/Cargo.toml b/kulfi-utils/Cargo.toml index 03e4c7c..df01885 100644 --- a/kulfi-utils/Cargo.toml +++ b/kulfi-utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "kulfi-utils" -version = "0.1.1" +version = "0.1.2" edition.workspace = true authors.workspace = true description = "Kulfi utilities" From 49a5667acfb9471bb5e65018166e40038b7d0429 Mon Sep 17 00:00:00 2001 From: ovo-Tim Date: Fri, 15 Aug 2025 14:30:32 +0800 Subject: [PATCH 05/15] feat(ci): Also publish kulfi-id52 --- .github/workflows/release-kulfi-utils.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-kulfi-utils.yml b/.github/workflows/release-kulfi-utils.yml index ee1170e..9ad91e0 100644 --- a/.github/workflows/release-kulfi-utils.yml +++ b/.github/workflows/release-kulfi-utils.yml @@ -10,6 +10,6 @@ jobs: - uses: actions/checkout@v4 - name: Publish to crates.io run: | - cd kulfi-utils cargo login ${{ secrets.CARGO_REGISTRY_TOKEN }} - cargo publish \ No newline at end of file + cargo publish -p kulfi-id52 + cargo publish -p kulfi-utils \ No newline at end of file From 25876dab89838af499c7aa364de8e0d28ce04f40 Mon Sep 17 00:00:00 2001 From: ovo-Tim Date: Tue, 19 Aug 2025 22:23:18 +0800 Subject: [PATCH 06/15] feat: update kulfi-utils version to 0.1.2 --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 8a58043..53229be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3034,7 +3034,7 @@ dependencies = [ [[package]] name = "kulfi-utils" -version = "0.1.1" +version = "0.1.2" dependencies = [ "bb8", "bytes", From 58c1f7acde09b2bb0e520c719a7fb4e462a4ad33 Mon Sep 17 00:00:00 2001 From: ovo-Tim Date: Sun, 24 Aug 2025 00:41:45 +0800 Subject: [PATCH 07/15] feat: add malai.toml support for the first time.(http services work now) --- Cargo.lock | 17 ++- Cargo.toml | 18 ++-- kulfi-utils/Cargo.toml | 2 +- kulfi-utils/src/lib.rs | 2 +- kulfi-utils/src/secret.rs | 69 ++++++------ malai/Cargo.toml | 5 +- malai/src/expose_http.rs | 24 +++-- malai/src/folder/mod.rs | 9 ++ malai/src/main.rs | 63 ++++++++--- malai/src/run.rs | 164 ++++++++++++++++++++++++++++- malai/tests/http_example_conf.toml | 14 +++ 11 files changed, 321 insertions(+), 66 deletions(-) create mode 100644 malai/tests/http_example_conf.toml diff --git a/Cargo.lock b/Cargo.lock index 53229be..6294ec0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3205,7 +3205,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "malai" -version = "0.2.9" +version = "0.3.0" dependencies = [ "clap", "clap-verbosity-flag", @@ -3216,6 +3216,7 @@ dependencies = [ "hyper", "hyper-util", "iroh", + "kulfi-id52", "kulfi-utils", "mime_guess", "percent-encoding", @@ -3226,7 +3227,9 @@ dependencies = [ "tauri-plugin-opener", "tokio", "tokio-util", + "toml 0.9.5", "tracing", + "tracing-appender", "tracing-subscriber", "webbrowser", ] @@ -6623,6 +6626,18 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf" +dependencies = [ + "crossbeam-channel", + "thiserror 1.0.69", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.30" diff --git a/Cargo.toml b/Cargo.toml index b988530..aff4377 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ license = "UPL-1.0" repository = "https://github.com/fastn-stack/kulfi" homepage = "https://kulfi.app" publish = true -rust-version = "1.89" # update this when you update rust-toolchain.toml +rust-version = "1.89" # update this when you update rust-toolchain.toml [profile.release] strip = true @@ -40,17 +40,23 @@ directories = "6.0.0" eyre = "0.6" file-guard = "0.2.0" futures-util = "0.3" -http = "1" http-body-util = "0.1" hyper = { version = "1", features = ["full"] } hyper-util = { version = "0.1.15", features = ["tokio", "server"] } iroh = { version = "0.91", features = ["discovery-local-network"] } -keyring = { version = "3", features = ["apple-native", "windows-native", "linux-native", "vendored"] } -kulfi-utils = { path = "kulfi-utils", version = "0.1.0" } -kulfi-id52 = { path = "kulfi-id52", version = "0.1.0" } +keyring = { version = "3", features = [ + "apple-native", + "windows-native", + "linux-native", + "vendored", +] } +kulfi-utils = { path = "kulfi-utils" } +kulfi-id52 = { path = "kulfi-id52" } mime_guess = "2" percent-encoding = "2" -reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } +reqwest = { version = "0.12", default-features = false, features = [ + "rustls-tls", +] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" tauri-build = { version = "2", features = ["config-json5"] } diff --git a/kulfi-utils/Cargo.toml b/kulfi-utils/Cargo.toml index d7219aa..e09bd17 100644 --- a/kulfi-utils/Cargo.toml +++ b/kulfi-utils/Cargo.toml @@ -10,7 +10,7 @@ repository.workspace = true readme.workspace = true [dependencies] -kulfi-id52 = { path = "../kulfi-id52", version = "0.1.0" } +kulfi-id52.workspace = true bb8.workspace = true bytes.workspace = true colored.workspace = true diff --git a/kulfi-utils/src/lib.rs b/kulfi-utils/src/lib.rs index 8771937..e0ac948 100644 --- a/kulfi-utils/src/lib.rs +++ b/kulfi-utils/src/lib.rs @@ -10,7 +10,7 @@ mod http_to_peer; mod peer_to_http; mod ping; pub mod protocol; -mod secret; +pub mod secret; mod tcp; mod utils; mod utils_iroh; diff --git a/kulfi-utils/src/secret.rs b/kulfi-utils/src/secret.rs index a2bad79..cb47c98 100644 --- a/kulfi-utils/src/secret.rs +++ b/kulfi-utils/src/secret.rs @@ -37,50 +37,51 @@ pub fn get_secret_key(_id52: &str, _path: &str) -> eyre::Result eyre::Result<(String, kulfi_id52::SecretKey)> { + let e = kulfi_utils::secret::keyring_entry(&id52)?; + match e.get_secret() { + Ok(secret) => { + if secret.len() != 32 { + return Err(eyre::anyhow!( + "keyring: secret for {id52} has invalid length: {}", + secret.len() + )); + } + + let bytes: [u8; 32] = secret.try_into().expect("already checked for length"); + let secret_key = kulfi_id52::SecretKey::from_bytes(&bytes); + let id52 = secret_key.id52(); + Ok((id52, secret_key)) + } + Err(e) => { + tracing::error!("failed to read secret for {id52} from keyring: {e}"); + Err(e.into()) + } + } +} + #[tracing::instrument] pub async fn read_or_create_key() -> eyre::Result<(String, kulfi_id52::SecretKey)> { if let Ok(secret) = std::env::var(SECRET_KEY_ENV_VAR) { tracing::info!("Using secret key from environment variable {SECRET_KEY_ENV_VAR}"); return handle_secret(&secret); - } else { - match tokio::fs::read_to_string(SECRET_KEY_FILE).await { - Ok(secret) => { - tracing::info!("Using secret key from file {SECRET_KEY_FILE}"); - let secret = secret.trim_end(); - return handle_secret(secret); - } - Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(e) => { - tracing::error!("failed to read {SECRET_KEY_FILE}: {e}"); - return Err(e.into()); - } + } + match tokio::fs::read_to_string(SECRET_KEY_FILE).await { + Ok(secret) => { + tracing::info!("Using secret key from file {SECRET_KEY_FILE}"); + let secret = secret.trim_end(); + return handle_secret(secret); + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => { + tracing::error!("failed to read {SECRET_KEY_FILE}: {e}"); + return Err(e.into()); } } tracing::info!("No secret key found in environment or file, trying {ID52_FILE}"); match tokio::fs::read_to_string(ID52_FILE).await { - Ok(id52) => { - let e = keyring_entry(&id52)?; - match e.get_secret() { - Ok(secret) => { - if secret.len() != 32 { - return Err(eyre::anyhow!( - "keyring: secret for {id52} has invalid length: {}", - secret.len() - )); - } - - let bytes: [u8; 32] = secret.try_into().expect("already checked for length"); - let secret_key = kulfi_id52::SecretKey::from_bytes(&bytes); - let id52 = secret_key.id52(); - Ok((id52, secret_key)) - } - Err(e) => { - tracing::error!("failed to read secret for {id52} from keyring: {e}"); - Err(e.into()) - } - } - } + Ok(id52) => handle_identity(id52), Err(e) if e.kind() == std::io::ErrorKind::NotFound => generate_and_save_key().await, Err(e) => { tracing::error!("failed to read {ID52_FILE}: {e}"); diff --git a/malai/Cargo.toml b/malai/Cargo.toml index 8ff6694..9748f61 100644 --- a/malai/Cargo.toml +++ b/malai/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "malai" -version = "0.2.9" +version = "0.3.0" authors.workspace = true edition.workspace = true description = "malai: Kulfi Network Toolkit" @@ -24,6 +24,7 @@ hyper-util.workspace = true hyper.workspace = true iroh.workspace = true kulfi-utils.workspace = true +kulfi-id52.workspace = true mime_guess.workspace = true percent-encoding.workspace = true serde.workspace = true @@ -35,6 +36,8 @@ tokio-util.workspace = true tracing-subscriber.workspace = true tracing.workspace = true webbrowser.workspace = true +toml = "0.9.5" +tracing-appender = "0.2.3" [build-dependencies] tauri-build = { workspace = true, optional = true } diff --git a/malai/src/expose_http.rs b/malai/src/expose_http.rs index ebef3fa..b85287d 100644 --- a/malai/src/expose_http.rs +++ b/malai/src/expose_http.rs @@ -1,11 +1,19 @@ -pub async fn expose_http(host: String, port: u16, bridge: String, graceful: kulfi_utils::Graceful) { - let (id52, secret_key) = match kulfi_utils::read_or_create_key().await { - Ok(v) => v, - Err(e) => { - malai::identity_read_err_msg(e); - std::process::exit(1); - } - }; +pub async fn expose_http( + host: String, + port: u16, + bridge: String, + id52: String, + secret_key: kulfi_id52::SecretKey, + graceful: kulfi_utils::Graceful, +) { + // TODO(malai0.3): Clean here + // let (id52, secret_key) = match kulfi_utils::read_or_create_key().await { + // Ok(v) => v, + // Err(e) => { + // malai::identity_read_err_msg(e); + // std::process::exit(1); + // } + // }; let ep = match kulfi_utils::get_endpoint(secret_key).await { Ok(v) => v, diff --git a/malai/src/folder/mod.rs b/malai/src/folder/mod.rs index 8724ea2..d00e06e 100644 --- a/malai/src/folder/mod.rs +++ b/malai/src/folder/mod.rs @@ -58,10 +58,19 @@ pub async fn folder(path: String, bridge: String, graceful: kulfi_utils::Gracefu let graceful_for_expose_http = graceful.clone(); graceful.spawn(async move { + let (id52, secret_key) = match kulfi_utils::read_or_create_key().await { + Ok(v) => v, + Err(e) => { + malai::identity_read_err_msg(e); + std::process::exit(1); + } + }; malai::expose_http( "127.0.0.1".to_string(), port, bridge, + id52, + secret_key, graceful_for_expose_http, ) .await diff --git a/malai/src/main.rs b/malai/src/main.rs index 7eafe5d..c3fdce6 100644 --- a/malai/src/main.rs +++ b/malai/src/main.rs @@ -4,17 +4,39 @@ windows_subsystem = "windows" )] +use std::path::Path; + +use kulfi_utils::Graceful; + #[tokio::main] async fn main() -> eyre::Result<()> { use clap::Parser; - // run with RUST_LOG="malai=trace,kulfi_utils=trace" to see logs - tracing_subscriber::fmt::init(); - let cli = Cli::parse(); - let graceful = kulfi_utils::Graceful::default(); + if let Some(Command::Run { home }) = cli.command { + let home = match &home { + Some(home) => Path::new(home), + None => &std::env::current_dir()?, + }; + let conf_file = if home.is_file() { + home + } else { + &home.join("malai.toml") + }; + if !conf_file.exists() { + eprintln!("Unable to find malai.toml in {}", conf_file.display()); + } + malai::run(conf_file, graceful.clone()).await; + } else { + // run with RUST_LOG="malai=trace,kulfi_utils=trace" to see logs + tracing_subscriber::fmt::init(); + let _ = match_cli(cli, graceful.clone()); + } + graceful.shutdown().await +} +fn match_cli(cli: Cli, graceful: Graceful) -> eyre::Result<()> { match cli.command { Some(Command::Http { port, @@ -35,7 +57,22 @@ async fn main() -> eyre::Result<()> { tracing::info!(port, host, verbose = ?cli.verbose, "Exposing HTTP service on kulfi."); let graceful_for_export_http = graceful.clone(); graceful.spawn(async move { - malai::expose_http(host, port, bridge, graceful_for_export_http).await + let (id52, secret_key) = match kulfi_utils::read_or_create_key().await { + Ok(v) => v, + Err(e) => { + malai::identity_read_err_msg(e); + std::process::exit(1); + } + }; + malai::expose_http( + host, + port, + bridge, + id52, + secret_key, + graceful_for_export_http, + ) + .await }); } Some(Command::HttpBridge { proxy_target, port }) => { @@ -84,10 +121,9 @@ async fn main() -> eyre::Result<()> { let graceful_for_folder = graceful.clone(); graceful.spawn(async move { malai::folder(path, bridge, graceful_for_folder).await }); } - Some(Command::Run { home }) => { - tracing::info!(verbose = ?cli.verbose, "Running all services."); - let graceful_for_run = graceful.clone(); - graceful.spawn(async move { malai::run(home, graceful_for_run).await }); + Some(Command::Run { home: _ }) => { + // Handled brfore + return Ok(()); } Some(Command::HttpProxyRemote { public }) => { if !malai::public_check( @@ -126,8 +162,7 @@ async fn main() -> eyre::Result<()> { return Ok(()); } }; - - graceful.shutdown().await + return Ok(()); } #[derive(clap::Parser, Debug)] @@ -250,7 +285,11 @@ pub enum Command { }, #[clap(about = "Run all the services")] Run { - #[arg(long, help = "Malai Home", env = "MALAI_HOME")] + #[arg( + long, + help = "Malai Home directory or the config file", + env = "MALAI_HOME" + )] home: Option, }, #[clap(about = "Run an iroh remote server that handles requests from http-proxy.")] diff --git a/malai/src/run.rs b/malai/src/run.rs index f9029a0..5c73a46 100644 --- a/malai/src/run.rs +++ b/malai/src/run.rs @@ -1,3 +1,163 @@ -pub async fn run(_home: Option, _graceful: kulfi_utils::Graceful) { - todo!() +use eyre::{Context, ContextCompat}; +use serde::Deserialize; +use std::collections::HashMap; +use std::env; +use std::fs; +use std::path::Path; +use tracing_appender::rolling; + +#[allow(dead_code)] +#[derive(Deserialize, Debug)] +pub struct Config { + #[serde(default = "default_malai_conf")] + malai: MalaiConf, + http: Option, +} + +#[derive(Deserialize, Debug)] +struct MalaiConf { + log: Option, +} + +#[derive(Deserialize, Debug)] +struct HttpServices { + #[allow(dead_code)] + #[serde(flatten)] + services: HashMap, +} + +#[allow(dead_code)] +#[derive(Deserialize, Debug)] +struct HttpServiceConf { + identity: Option, // Leave None to read from env, .malai.secret-key file or .malai.id52 file and system keyring + port: u16, + public: bool, + active: bool, + #[serde(default = "default_host")] + host: String, + #[serde(default = "default_bridge")] + bridge: String, + // #[serde(rename = "access-log")] + // access_log: String, + // #[serde(rename = "debug-log")] + // debug_log: String, + // #[serde(rename = "error-log")] + // error_log: String, +} + +fn default_host() -> String { + "127.0.0.1".to_string() +} + +fn default_bridge() -> String { + match env::var("MALAI_HTTP_BRIDGE") { + Ok(value) => value, + Err(_) => "kulfi.site".to_string(), + } +} + +fn default_malai_conf() -> MalaiConf { + MalaiConf { log: None } +} + +fn parse_config(path: &Path) -> eyre::Result { + let conf_str = fs::read_to_string(&path) + .with_context(|| format!("Failed to read config file at {}", path.display()))?; + + let conf = toml::from_str(&conf_str).context("Failed to parse config file")?; + Ok(conf) +} + +fn set_up_logging(conf: &Config) -> eyre::Result<()> { + match &conf.malai.log { + Some(log_dir) => { + let log_dir = Path::new(&log_dir); + let file_appender = rolling::daily( + log_dir.parent().unwrap_or(Path::new("./")), + log_dir.file_name().context("Invalid log path.")?, + ); + let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender); + tracing_subscriber::fmt() + .with_writer(non_blocking) + .with_ansi(false) + .init(); + } + None => { + tracing_subscriber::fmt::init(); + } + } + Ok(()) +} + +async fn set_up_http_services(conf: &Config, graceful: kulfi_utils::Graceful) { + if let Some(http_conf) = &conf.http { + for (name, service_conf) in &http_conf.services { + println!("Starting HTTP services: {}", name); + // Check + if !service_conf.active { + continue; + } + if !service_conf.public { + tracing::warn!( + "You have to set public to true for service {}. Skipping.", + name + ); + continue; + } + let host = service_conf.host.clone(); + let port = service_conf.port; + let bridge = service_conf.bridge.clone(); + let graceful_clone = graceful.clone(); + + let (id52, secret_key) = match &service_conf.identity { + Some(id52) => match kulfi_utils::secret::handle_identity(id52.to_string()) { + Ok(v) => v, + Err(_e) => { + // The error message has been printed by tracing::error! + eprintln!("Failed to load identity for service: {}. Skipping.", name); + continue; + } + }, + None => match kulfi_utils::read_or_create_key().await { + Ok(v) => v, + Err(e) => { + eprintln!( + "Failed to load or create identity for service: {}. Skipping.", + name + ); + malai::identity_read_err_msg(e); + continue; + } + }, + }; + + graceful.spawn(async move { + malai::expose_http(host, port, bridge, id52, secret_key, graceful_clone).await + }); + } + } +} + +pub async fn run(conf_path: &Path, graceful: kulfi_utils::Graceful) { + let conf = match parse_config(conf_path) { + Ok(conf) => conf, + Err(e) => { + eprintln!("Failed to parse config: {}", e); + return; + } + }; + if let Err(e) = set_up_logging(&conf) { + eprintln!("Failed to set up logging: {}. Skipping.", e); + } + set_up_http_services(&conf, graceful.clone()).await; +} + +#[test] +fn parse_config_test() { + let conf = parse_config(Path::new("tests/http_example_conf.toml")).unwrap(); + println!("{:?}", conf); + assert!(conf.http.is_some()); + let http = conf.http.as_ref().expect("HTTP services should be present"); + assert!(http.services.get("service1").is_some()); + assert!(http.services.get("service2").is_some()); } diff --git a/malai/tests/http_example_conf.toml b/malai/tests/http_example_conf.toml new file mode 100644 index 0000000..94e8d5c --- /dev/null +++ b/malai/tests/http_example_conf.toml @@ -0,0 +1,14 @@ +[malai] +log = "/var/log/malai.log" + +[http.service1] +identity = "" +port = 3000 +public = true +active = true + +[http.service2] +identity = "" +port = 3001 +public = true +active = true From 911e4d80705158239a09cf25b1d5e53d107fac45 Mon Sep 17 00:00:00 2001 From: ovo-Tim Date: Sun, 24 Aug 2025 17:53:13 +0800 Subject: [PATCH 08/15] feat:malai identity --- kulfi-utils/src/lib.rs | 3 +- kulfi-utils/src/secret.rs | 22 +++++++++++-- kulfi/src/identity/create.rs | 6 +++- malai/src/expose_http.rs | 9 ------ malai/src/identity.rs | 56 +++++++++++++++++++++++++++++++++ malai/src/lib.rs | 2 ++ malai/src/main.rs | 61 +++++++++++++++++++++++++++++++++--- 7 files changed, 141 insertions(+), 18 deletions(-) create mode 100644 malai/src/identity.rs diff --git a/kulfi-utils/src/lib.rs b/kulfi-utils/src/lib.rs index e0ac948..328b11a 100644 --- a/kulfi-utils/src/lib.rs +++ b/kulfi-utils/src/lib.rs @@ -25,7 +25,8 @@ pub use peer_to_http::peer_to_http; pub use ping::{PONG, ping}; pub use protocol::{APNS_IDENTITY, Protocol, ProtocolHeader}; pub use secret::{ - SECRET_KEY_FILE, generate_and_save_key, generate_secret_key, get_secret_key, read_or_create_key, + ID52_FILE, SECRET_KEY_FILE, generate_and_save_key, generate_secret_key, get_secret_key, + read_or_create_key, }; pub use tcp::{peer_to_tcp, pipe_tcp_stream_over_iroh, tcp_to_peer}; pub use utils::mkdir; diff --git a/kulfi-utils/src/secret.rs b/kulfi-utils/src/secret.rs index cb47c98..e13b61a 100644 --- a/kulfi-utils/src/secret.rs +++ b/kulfi-utils/src/secret.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use eyre::WrapErr; pub const SECRET_KEY_ENV_VAR: &str = "KULFI_SECRET_KEY"; @@ -10,15 +12,27 @@ pub fn generate_secret_key() -> eyre::Result<(String, kulfi_id52::SecretKey)> { Ok((id52, secret_key)) } -pub async fn generate_and_save_key() -> eyre::Result<(String, kulfi_id52::SecretKey)> { +pub fn generate_and_save_key( + file: Option, +) -> eyre::Result<(String, kulfi_id52::SecretKey)> { let (id52, secret_key) = generate_secret_key()?; let e = keyring_entry(&id52)?; e.set_secret(&secret_key.to_bytes()) .wrap_err_with(|| format!("failed to save secret key for {id52}"))?; - tokio::fs::write(ID52_FILE, &id52).await?; + if let Some(file) = &file { + std::fs::write(file, &id52) + .wrap_err_with(|| format!("failed to save secret key to {}", &file.display()))?; + println!("Secret key saved to {}", file.display()); + } Ok((id52, secret_key)) } +pub fn delete_identity(id52: &str) -> eyre::Result<()> { + let e = keyring_entry(id52)?; + e.delete_credential()?; + Ok(()) +} + fn keyring_entry(id52: &str) -> eyre::Result { keyring::Entry::new("kulfi", id52) .wrap_err_with(|| format!("failed to create keyring Entry for {id52}")) @@ -82,7 +96,9 @@ pub async fn read_or_create_key() -> eyre::Result<(String, kulfi_id52::SecretKey tracing::info!("No secret key found in environment or file, trying {ID52_FILE}"); match tokio::fs::read_to_string(ID52_FILE).await { Ok(id52) => handle_identity(id52), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => generate_and_save_key().await, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + generate_and_save_key(Some(PathBuf::from(ID52_FILE))) + } Err(e) => { tracing::error!("failed to read {ID52_FILE}: {e}"); Err(e.into()) diff --git a/kulfi/src/identity/create.rs b/kulfi/src/identity/create.rs index b62f923..feaa41f 100644 --- a/kulfi/src/identity/create.rs +++ b/kulfi/src/identity/create.rs @@ -23,6 +23,8 @@ //! //! `logs` is the folder that contains the logs for this identity. This contains fastn access logs //! and other device access logs etc. + +use std::path::PathBuf; impl kulfi::Identity { #[tracing::instrument(skip(client_pools))] pub async fn create( @@ -31,7 +33,9 @@ impl kulfi::Identity { ) -> eyre::Result { use eyre::WrapErr; - let (id52, secret_key) = kulfi_utils::generate_and_save_key().await?; + let (id52, secret_key) = kulfi_utils::generate_and_save_key(Some(PathBuf::from( + kulfi_utils::secret::ID52_FILE, + )))?; let now = std::time::SystemTime::now(); let unixtime = now diff --git a/malai/src/expose_http.rs b/malai/src/expose_http.rs index b85287d..3d11d71 100644 --- a/malai/src/expose_http.rs +++ b/malai/src/expose_http.rs @@ -6,15 +6,6 @@ pub async fn expose_http( secret_key: kulfi_id52::SecretKey, graceful: kulfi_utils::Graceful, ) { - // TODO(malai0.3): Clean here - // let (id52, secret_key) = match kulfi_utils::read_or_create_key().await { - // Ok(v) => v, - // Err(e) => { - // malai::identity_read_err_msg(e); - // std::process::exit(1); - // } - // }; - let ep = match kulfi_utils::get_endpoint(secret_key).await { Ok(v) => v, Err(e) => { diff --git a/malai/src/identity.rs b/malai/src/identity.rs new file mode 100644 index 0000000..b553fc8 --- /dev/null +++ b/malai/src/identity.rs @@ -0,0 +1,56 @@ +use std::path::{Path, PathBuf}; + +fn get_identity_path(path: Option) -> Option { + let path = match path { + Some(path) => path, + None => return None, + }; + let path = Path::new(&path).to_path_buf(); + if path.is_dir() { + Some(path.join(kulfi_utils::secret::ID52_FILE)) + } else { + Some(path) + } +} + +pub fn create_identity(path: Option) -> eyre::Result<()> { + let path = get_identity_path(path); + let (id52, _) = kulfi_utils::secret::generate_and_save_key(path)?; + println!( + "Identity(ID52) created: {}. And the secret key has been saved to system keyring.", + id52 + ); + Ok(()) +} + +pub fn delete_identity(id52: Option, path: Option) -> eyre::Result<()> { + if let Some(id52) = id52 { + kulfi_utils::secret::delete_identity(&id52)?; + println!("Identity(ID52) deleted: {}", id52); + } + + let path = get_identity_path(path); + let path = match path { + Some(path) => path, + None => PathBuf::from(kulfi_utils::secret::ID52_FILE), + }; + if path.exists() { + let id52 = std::fs::read_to_string(&path)?; + if let Err(e) = kulfi_utils::secret::delete_identity(&id52) { + eprint!( + "Unable to delete identity(ID52): {} in file {}. Maybe you want to clean this file. Error: {}", + id52, + path.display(), + e + ); + } + println!( + "Identity(ID52) {} at file {} deleted.", + id52, + path.display() + ); + } else { + println!("Identity(ID52) file {} not found.", path.display()); + } + Ok(()) +} diff --git a/malai/src/lib.rs b/malai/src/lib.rs index 8844a50..f5d1e57 100644 --- a/malai/src/lib.rs +++ b/malai/src/lib.rs @@ -15,6 +15,7 @@ mod folder; mod http_bridge; mod http_proxy; mod http_proxy_remote; +mod identity; mod keygen; mod run; mod tcp_bridge; @@ -26,6 +27,7 @@ pub use folder::folder; pub use http_bridge::http_bridge; pub use http_proxy::{ProxyData, http_proxy}; pub use http_proxy_remote::http_proxy_remote; +pub use identity::{create_identity, delete_identity}; pub use keygen::keygen; pub use run::run; pub use tcp_bridge::tcp_bridge; diff --git a/malai/src/main.rs b/malai/src/main.rs index c3fdce6..793041d 100644 --- a/malai/src/main.rs +++ b/malai/src/main.rs @@ -28,15 +28,15 @@ async fn main() -> eyre::Result<()> { eprintln!("Unable to find malai.toml in {}", conf_file.display()); } malai::run(conf_file, graceful.clone()).await; + Ok(()) } else { // run with RUST_LOG="malai=trace,kulfi_utils=trace" to see logs tracing_subscriber::fmt::init(); - let _ = match_cli(cli, graceful.clone()); + match_cli(cli, graceful.clone()).await } - graceful.shutdown().await } -fn match_cli(cli: Cli, graceful: Graceful) -> eyre::Result<()> { +async fn match_cli(cli: Cli, graceful: Graceful) -> eyre::Result<()> { match cli.command { Some(Command::Http { port, @@ -149,6 +149,21 @@ fn match_cli(cli: Cli, graceful: Graceful) -> eyre::Result<()> { malai::keygen(file); return Ok(()); } + Some(Command::Identity { cmd }) => { + match cmd { + IdentityCmd::Create { file } => { + if let Err(e) = malai::create_identity(file) { + tracing::error!(error = ?e, "Error creating identity."); + } + } + IdentityCmd::Delete { id52, file } => { + if let Err(e) = malai::delete_identity(id52, file) { + tracing::error!(error = ?e, "Error deleting identity."); + } + } + } + return Ok(()); + } #[cfg(feature = "ui")] None => { tracing::info!(verbose = ?cli.verbose, "Starting UI."); @@ -162,7 +177,7 @@ fn match_cli(cli: Cli, graceful: Graceful) -> eyre::Result<()> { return Ok(()); } }; - return Ok(()); + graceful.shutdown().await } #[derive(clap::Parser, Debug)] @@ -318,4 +333,42 @@ pub enum Command { )] file: Option, }, + #[clap(about = "Create or delete ID52s in the system keyring")] + Identity { + #[clap(subcommand)] + cmd: IdentityCmd, + }, +} + +#[derive(clap::Subcommand, Debug)] +pub enum IdentityCmd { + #[clap(about = "Create a new identity and store the private key to system keyring.")] + Create { + #[arg( + long, + short, + num_args=0..=1, + default_missing_value=kulfi_utils::ID52_FILE, + help = "The file or the folder to store the private key." + )] + file: Option, + }, + #[clap(about = "Delete the identity from system keyring.")] + Delete { + #[arg( + long, + short, + num_args = 1, + help = "Delete the ID52 from system keyring." + )] + id52: Option, + #[arg( + long, + short, + num_args=0..=1, + default_missing_value=kulfi_utils::ID52_FILE, + help = "Delete the ID52 in the file from system keyring." + )] + file: Option, + }, } From 9a29e9e3d15ae2dbba5cc31254c17a149eba525d Mon Sep 17 00:00:00 2001 From: ovo-Tim Date: Mon, 25 Aug 2025 11:38:23 +0800 Subject: [PATCH 09/15] feat: tcp service support in malai.toml --- malai/src/expose_tcp.rs | 16 ++-- malai/src/main.rs | 15 +++- malai/src/run.rs | 118 ++++++++++++++++++++++------- malai/tests/http_example_conf.toml | 6 ++ 4 files changed, 117 insertions(+), 38 deletions(-) diff --git a/malai/src/expose_tcp.rs b/malai/src/expose_tcp.rs index 3affcef..779c67d 100644 --- a/malai/src/expose_tcp.rs +++ b/malai/src/expose_tcp.rs @@ -1,12 +1,10 @@ -pub async fn expose_tcp(host: String, port: u16, graceful: kulfi_utils::Graceful) { - let (id52, secret_key) = match kulfi_utils::read_or_create_key().await { - Ok(v) => v, - Err(e) => { - malai::identity_read_err_msg(e); - std::process::exit(1); - } - }; - +pub async fn expose_tcp( + host: String, + port: u16, + id52: String, + secret_key: kulfi_id52::SecretKey, + graceful: kulfi_utils::Graceful, +) { let ep = match kulfi_utils::get_endpoint(secret_key).await { Ok(v) => v, Err(e) => { diff --git a/malai/src/main.rs b/malai/src/main.rs index 793041d..e8f10f6 100644 --- a/malai/src/main.rs +++ b/malai/src/main.rs @@ -26,9 +26,10 @@ async fn main() -> eyre::Result<()> { }; if !conf_file.exists() { eprintln!("Unable to find malai.toml in {}", conf_file.display()); + return Ok(()); } malai::run(conf_file, graceful.clone()).await; - Ok(()) + graceful.shutdown().await } else { // run with RUST_LOG="malai=trace,kulfi_utils=trace" to see logs tracing_subscriber::fmt::init(); @@ -93,8 +94,16 @@ async fn match_cli(cli: Cli, graceful: Graceful) -> eyre::Result<()> { tracing::info!(port, host, verbose = ?cli.verbose, "Exposing TCP service on kulfi."); let graceful_for_expose_tcp = graceful.clone(); - graceful - .spawn(async move { malai::expose_tcp(host, port, graceful_for_expose_tcp).await }); + graceful.spawn(async move { + let (id52, secret_key) = match kulfi_utils::read_or_create_key().await { + Ok(v) => v, + Err(e) => { + malai::identity_read_err_msg(e); + std::process::exit(1); + } + }; + malai::expose_tcp(host, port, id52, secret_key, graceful_for_expose_tcp).await; + }); } Some(Command::TcpBridge { proxy_target, port }) => { tracing::info!(port, proxy_target, verbose = ?cli.verbose, "Starting TCP bridge."); diff --git a/malai/src/run.rs b/malai/src/run.rs index 5c73a46..93d6f0e 100644 --- a/malai/src/run.rs +++ b/malai/src/run.rs @@ -1,3 +1,4 @@ +use eyre::eyre; use eyre::{Context, ContextCompat}; use serde::Deserialize; use std::collections::HashMap; @@ -12,6 +13,7 @@ pub struct Config { #[serde(default = "default_malai_conf")] malai: MalaiConf, http: Option, + tcp: Option, } #[derive(Deserialize, Debug)] @@ -37,12 +39,24 @@ struct HttpServiceConf { host: String, #[serde(default = "default_bridge")] bridge: String, - // #[serde(rename = "access-log")] - // access_log: String, - // #[serde(rename = "debug-log")] - // debug_log: String, - // #[serde(rename = "error-log")] - // error_log: String, +} + +#[derive(Deserialize, Debug)] +struct TcpServices { + #[allow(dead_code)] + #[serde(flatten)] + services: HashMap, +} + +#[allow(dead_code)] +#[derive(Deserialize, Debug)] +struct TcpServiceConf { + identity: Option, // Leave None to read from env, .malai.secret-key file or .malai.id52 file and system keyring + port: u16, + public: bool, + active: bool, + #[serde(default = "default_host")] + host: String, } fn default_host() -> String { @@ -89,6 +103,25 @@ fn set_up_logging(conf: &Config) -> eyre::Result<()> { Ok(()) } +async fn load_identity(identity: &Option) -> eyre::Result<(String, kulfi_id52::SecretKey)> { + match identity { + Some(id52) => match kulfi_utils::secret::handle_identity(id52.to_string()) { + Ok(v) => Ok(v), + Err(_e) => Err(eyre!( + "Failed to load identity {} from system keyring.", + id52 + )), + }, + None => match kulfi_utils::read_or_create_key().await { + Ok(v) => Ok(v), + Err(e) => { + malai::identity_read_err_msg(e); + Err(eyre!("Failed to load/create identity from/to file.")) + } + }, + } +} + async fn set_up_http_services(conf: &Config, graceful: kulfi_utils::Graceful) { if let Some(http_conf) = &conf.http { for (name, service_conf) in &http_conf.services { @@ -109,26 +142,16 @@ async fn set_up_http_services(conf: &Config, graceful: kulfi_utils::Graceful) { let bridge = service_conf.bridge.clone(); let graceful_clone = graceful.clone(); - let (id52, secret_key) = match &service_conf.identity { - Some(id52) => match kulfi_utils::secret::handle_identity(id52.to_string()) { - Ok(v) => v, - Err(_e) => { - // The error message has been printed by tracing::error! - eprintln!("Failed to load identity for service: {}. Skipping.", name); - continue; - } - }, - None => match kulfi_utils::read_or_create_key().await { - Ok(v) => v, - Err(e) => { - eprintln!( - "Failed to load or create identity for service: {}. Skipping.", - name - ); - malai::identity_read_err_msg(e); - continue; - } - }, + let (id52, secret_key) = match load_identity(&service_conf.identity).await { + Ok(v) => v, + Err(e) => { + // The error message has been printed by tracing::error! + eprintln!( + "Failed to load identity for service {}: {} Skipping.", + name, e + ); + continue; + } }; graceful.spawn(async move { @@ -138,6 +161,44 @@ async fn set_up_http_services(conf: &Config, graceful: kulfi_utils::Graceful) { } } +async fn set_up_tcp_services(conf: &Config, graceful: kulfi_utils::Graceful) { + if let Some(tcp_conf) = &conf.tcp { + for (name, service_conf) in &tcp_conf.services { + println!("Starting TCP services: {}", name); + // Check + if !service_conf.active { + continue; + } + if !service_conf.public { + tracing::warn!( + "You have to set public to true for service {}. Skipping.", + name + ); + continue; + } + let host = service_conf.host.clone(); + let port = service_conf.port; + let graceful_clone = graceful.clone(); + + let (id52, secret_key) = match load_identity(&service_conf.identity).await { + Ok(v) => v, + Err(e) => { + // The error message has been printed by tracing::error! + eprintln!( + "Failed to load identity for service {}: {} Skipping.", + name, e + ); + continue; + } + }; + + graceful.spawn(async move { + malai::expose_tcp(host, port, id52, secret_key, graceful_clone).await + }); + } + } +} + pub async fn run(conf_path: &Path, graceful: kulfi_utils::Graceful) { let conf = match parse_config(conf_path) { Ok(conf) => conf, @@ -150,6 +211,7 @@ pub async fn run(conf_path: &Path, graceful: kulfi_utils::Graceful) { eprintln!("Failed to set up logging: {}. Skipping.", e); } set_up_http_services(&conf, graceful.clone()).await; + set_up_tcp_services(&conf, graceful.clone()).await; } #[test] @@ -160,4 +222,8 @@ fn parse_config_test() { let http = conf.http.as_ref().expect("HTTP services should be present"); assert!(http.services.get("service1").is_some()); assert!(http.services.get("service2").is_some()); + + assert!(conf.tcp.is_some()); + let tcp = conf.tcp.as_ref().expect("TCP services should be present"); + assert!(tcp.services.get("service3").is_some()); } diff --git a/malai/tests/http_example_conf.toml b/malai/tests/http_example_conf.toml index 94e8d5c..1252819 100644 --- a/malai/tests/http_example_conf.toml +++ b/malai/tests/http_example_conf.toml @@ -12,3 +12,9 @@ identity = "" port = 3001 public = true active = true + +[tcp.service3] +identity = "" +port = 3002 +public = true +active = true From c558e92f21fc4e2fb85c925de70f17f4149a8d57 Mon Sep 17 00:00:00 2001 From: ovo-Tim Date: Mon, 25 Aug 2025 16:35:27 +0800 Subject: [PATCH 10/15] fix: Make sure logs can be witten to log file. --- malai/src/main.rs | 3 ++- malai/src/run.rs | 31 +++++++++++++++++++++---------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/malai/src/main.rs b/malai/src/main.rs index e8f10f6..7d37639 100644 --- a/malai/src/main.rs +++ b/malai/src/main.rs @@ -28,7 +28,8 @@ async fn main() -> eyre::Result<()> { eprintln!("Unable to find malai.toml in {}", conf_file.display()); return Ok(()); } - malai::run(conf_file, graceful.clone()).await; + // Dropping this when main is ended makes sure the log will be written. + let _logging_guard = malai::run(conf_file, graceful.clone()).await; graceful.shutdown().await } else { // run with RUST_LOG="malai=trace,kulfi_utils=trace" to see logs diff --git a/malai/src/run.rs b/malai/src/run.rs index 93d6f0e..44e614f 100644 --- a/malai/src/run.rs +++ b/malai/src/run.rs @@ -1,10 +1,11 @@ +use eyre::Context; use eyre::eyre; -use eyre::{Context, ContextCompat}; use serde::Deserialize; use std::collections::HashMap; use std::env; use std::fs; use std::path::Path; +use tracing_appender::non_blocking::WorkerGuard; use tracing_appender::rolling; #[allow(dead_code)] @@ -82,25 +83,28 @@ fn parse_config(path: &Path) -> eyre::Result { Ok(conf) } -fn set_up_logging(conf: &Config) -> eyre::Result<()> { +fn set_up_logging(conf: &Config) -> eyre::Result> { match &conf.malai.log { Some(log_dir) => { let log_dir = Path::new(&log_dir); let file_appender = rolling::daily( log_dir.parent().unwrap_or(Path::new("./")), - log_dir.file_name().context("Invalid log path.")?, + log_dir + .file_name() + .unwrap_or_else(|| std::ffi::OsStr::new("malai.log")), ); - let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender); + let (non_blocking, guard) = tracing_appender::non_blocking(file_appender); tracing_subscriber::fmt() .with_writer(non_blocking) .with_ansi(false) .init(); + return Ok(Some(guard)); } None => { tracing_subscriber::fmt::init(); } } - Ok(()) + Ok(None) } async fn load_identity(identity: &Option) -> eyre::Result<(String, kulfi_id52::SecretKey)> { @@ -199,19 +203,26 @@ async fn set_up_tcp_services(conf: &Config, graceful: kulfi_utils::Graceful) { } } -pub async fn run(conf_path: &Path, graceful: kulfi_utils::Graceful) { +pub async fn run(conf_path: &Path, graceful: kulfi_utils::Graceful) -> Option { let conf = match parse_config(conf_path) { Ok(conf) => conf, Err(e) => { eprintln!("Failed to parse config: {}", e); - return; + return None; } }; - if let Err(e) = set_up_logging(&conf) { - eprintln!("Failed to set up logging: {}. Skipping.", e); - } + + let guard = match set_up_logging(&conf) { + Ok(guard) => guard, + Err(e) => { + eprintln!("Failed to set up logging: {}. Skipping.", e); + None + } + }; + set_up_http_services(&conf, graceful.clone()).await; set_up_tcp_services(&conf, graceful.clone()).await; + guard } #[test] From 55ddd6e3548edc92a4ca7c4cc12ed7ff70bea84f Mon Sep 17 00:00:00 2001 From: ovo-Tim Date: Mon, 25 Aug 2025 16:53:01 +0800 Subject: [PATCH 11/15] feat: use tracing for `malai run` log --- kulfi-utils/src/graceful.rs | 6 +++--- kulfi-utils/src/peer_to_http.rs | 2 +- kulfi/src/control_server/server.rs | 2 +- malai/src/http_bridge.rs | 2 +- malai/src/http_proxy.rs | 2 +- malai/src/run.rs | 33 +++++++++++++++++++++--------- 6 files changed, 30 insertions(+), 17 deletions(-) diff --git a/kulfi-utils/src/graceful.rs b/kulfi-utils/src/graceful.rs index 23a1f06..7d368dd 100644 --- a/kulfi-utils/src/graceful.rs +++ b/kulfi-utils/src/graceful.rs @@ -50,8 +50,8 @@ impl Graceful { .await .wrap_err_with(|| "failed to get ctrl-c signal handler")?; - tracing::info!("Received ctrl-c signal, showing info."); - tracing::info!("Pending tasks: {}", self.tracker.len()); + tracing::debug!("Received ctrl-c signal, showing info."); + tracing::debug!("Pending tasks: {}", self.tracker.len()); self.show_info_tx .send(true) @@ -85,7 +85,7 @@ impl Graceful { break; } _ = tokio::time::sleep(std::time::Duration::from_secs(3)) => { - tracing::info!("Timeout expired. Continuing..."); + tracing::debug!("Timeout expired. Continuing..."); println!("Did not receive ctrl+c within 3 secs. Press ctrl+c in quick succession to exit."); } } diff --git a/kulfi-utils/src/peer_to_http.rs b/kulfi-utils/src/peer_to_http.rs index d073f92..ae3c9cf 100644 --- a/kulfi-utils/src/peer_to_http.rs +++ b/kulfi-utils/src/peer_to_http.rs @@ -12,7 +12,7 @@ pub async fn peer_to_http( let req: crate::http::Request = crate::next_json(&mut recv).await?; - tracing::info!("got request: {req:?}"); + tracing::debug!("got request: {req:?}"); let mut r = hyper::Request::builder() .method(req.method.as_str()) diff --git a/kulfi/src/control_server/server.rs b/kulfi/src/control_server/server.rs index 22ddb88..ee637f4 100644 --- a/kulfi/src/control_server/server.rs +++ b/kulfi/src/control_server/server.rs @@ -84,7 +84,7 @@ async fn handle_request_( } }; - tracing::info!("got request for {id}"); + tracing::debug!("got request for {id}"); // if this is an identity, if so forward the request to fastn corresponding to that identity if let Some(fastn_port) = find_identity(id, id_map.clone()).await? { diff --git a/malai/src/http_bridge.rs b/malai/src/http_bridge.rs index 5bd47ce..0681eb1 100644 --- a/malai/src/http_bridge.rs +++ b/malai/src/http_bridge.rs @@ -135,7 +135,7 @@ async fn handle_request( } }; - tracing::info!("got request for {peer_id}"); + tracing::debug!("got request for {peer_id}"); kulfi_utils::http_to_peer( kulfi_utils::Protocol::Http.into(), diff --git a/malai/src/http_proxy.rs b/malai/src/http_proxy.rs index 6fd8171..723eff7 100644 --- a/malai/src/http_proxy.rs +++ b/malai/src/http_proxy.rs @@ -130,7 +130,7 @@ async fn handle_request( remote: String, graceful: kulfi_utils::Graceful, ) -> kulfi_utils::http::ProxyResult { - tracing::info!("got request for {remote}"); + tracing::debug!("got request for {remote}"); let graceful_for_upgrade = graceful.clone(); let host = match r diff --git a/malai/src/run.rs b/malai/src/run.rs index 44e614f..fffefa7 100644 --- a/malai/src/run.rs +++ b/malai/src/run.rs @@ -5,8 +5,10 @@ use std::collections::HashMap; use std::env; use std::fs; use std::path::Path; +use tracing::{error, info}; use tracing_appender::non_blocking::WorkerGuard; use tracing_appender::rolling; +use tracing_subscriber::{fmt, prelude::*}; #[allow(dead_code)] #[derive(Deserialize, Debug)] @@ -94,10 +96,21 @@ fn set_up_logging(conf: &Config) -> eyre::Result> { .unwrap_or_else(|| std::ffi::OsStr::new("malai.log")), ); let (non_blocking, guard) = tracing_appender::non_blocking(file_appender); - tracing_subscriber::fmt() - .with_writer(non_blocking) - .with_ansi(false) - .init(); + // tracing_subscriber::fmt() + // .with_writer(non_blocking) + // .with_ansi(false) + // .init(); + let subscriber = fmt::Subscriber::builder().finish().with( + fmt::Layer::new() + .with_writer(non_blocking) + .with_ansi(false) + .with_target(true) + .with_file(true) + .with_line_number(true), + ); + + tracing::subscriber::set_global_default(subscriber) + .expect("Failed to set tracing subscriber"); return Ok(Some(guard)); } None => { @@ -129,7 +142,7 @@ async fn load_identity(identity: &Option) -> eyre::Result<(String, kulfi async fn set_up_http_services(conf: &Config, graceful: kulfi_utils::Graceful) { if let Some(http_conf) = &conf.http { for (name, service_conf) in &http_conf.services { - println!("Starting HTTP services: {}", name); + info!("Starting HTTP services: {}", name); // Check if !service_conf.active { continue; @@ -150,7 +163,7 @@ async fn set_up_http_services(conf: &Config, graceful: kulfi_utils::Graceful) { Ok(v) => v, Err(e) => { // The error message has been printed by tracing::error! - eprintln!( + error!( "Failed to load identity for service {}: {} Skipping.", name, e ); @@ -168,7 +181,7 @@ async fn set_up_http_services(conf: &Config, graceful: kulfi_utils::Graceful) { async fn set_up_tcp_services(conf: &Config, graceful: kulfi_utils::Graceful) { if let Some(tcp_conf) = &conf.tcp { for (name, service_conf) in &tcp_conf.services { - println!("Starting TCP services: {}", name); + info!("Starting TCP services: {}", name); // Check if !service_conf.active { continue; @@ -188,7 +201,7 @@ async fn set_up_tcp_services(conf: &Config, graceful: kulfi_utils::Graceful) { Ok(v) => v, Err(e) => { // The error message has been printed by tracing::error! - eprintln!( + error!( "Failed to load identity for service {}: {} Skipping.", name, e ); @@ -207,7 +220,7 @@ pub async fn run(conf_path: &Path, graceful: kulfi_utils::Graceful) -> Option conf, Err(e) => { - eprintln!("Failed to parse config: {}", e); + error!("Failed to parse config: {}", e); return None; } }; @@ -215,7 +228,7 @@ pub async fn run(conf_path: &Path, graceful: kulfi_utils::Graceful) -> Option guard, Err(e) => { - eprintln!("Failed to set up logging: {}. Skipping.", e); + error!("Failed to set up logging: {}. Skipping.", e); None } }; From 6978e807832841f6163b3184a5f33b1b2e172027 Mon Sep 17 00:00:00 2001 From: ovo-Tim Date: Tue, 26 Aug 2025 20:39:18 +0800 Subject: [PATCH 12/15] feat: Detact used identities --- malai/src/run.rs | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/malai/src/run.rs b/malai/src/run.rs index fffefa7..0f722f5 100644 --- a/malai/src/run.rs +++ b/malai/src/run.rs @@ -2,6 +2,7 @@ use eyre::Context; use eyre::eyre; use serde::Deserialize; use std::collections::HashMap; +use std::collections::HashSet; use std::env; use std::fs; use std::path::Path; @@ -120,8 +121,11 @@ fn set_up_logging(conf: &Config) -> eyre::Result> { Ok(None) } -async fn load_identity(identity: &Option) -> eyre::Result<(String, kulfi_id52::SecretKey)> { - match identity { +async fn load_identity( + identity: &Option, + used_id52: &mut HashSet, +) -> eyre::Result<(String, kulfi_id52::SecretKey)> { + let (id52, secret_key) = match identity { Some(id52) => match kulfi_utils::secret::handle_identity(id52.to_string()) { Ok(v) => Ok(v), Err(_e) => Err(eyre!( @@ -136,10 +140,20 @@ async fn load_identity(identity: &Option) -> eyre::Result<(String, kulfi Err(eyre!("Failed to load/create identity from/to file.")) } }, + }?; + if used_id52.contains(&id52) { + Err(eyre!("Identity already used.")) + } else { + used_id52.insert(id52.clone()); + Ok((id52, secret_key)) } } -async fn set_up_http_services(conf: &Config, graceful: kulfi_utils::Graceful) { +async fn set_up_http_services( + conf: &Config, + used_id52: &mut HashSet, + graceful: kulfi_utils::Graceful, +) { if let Some(http_conf) = &conf.http { for (name, service_conf) in &http_conf.services { info!("Starting HTTP services: {}", name); @@ -159,7 +173,7 @@ async fn set_up_http_services(conf: &Config, graceful: kulfi_utils::Graceful) { let bridge = service_conf.bridge.clone(); let graceful_clone = graceful.clone(); - let (id52, secret_key) = match load_identity(&service_conf.identity).await { + let (id52, secret_key) = match load_identity(&service_conf.identity, used_id52).await { Ok(v) => v, Err(e) => { // The error message has been printed by tracing::error! @@ -178,7 +192,11 @@ async fn set_up_http_services(conf: &Config, graceful: kulfi_utils::Graceful) { } } -async fn set_up_tcp_services(conf: &Config, graceful: kulfi_utils::Graceful) { +async fn set_up_tcp_services( + conf: &Config, + used_id52: &mut HashSet, + graceful: kulfi_utils::Graceful, +) { if let Some(tcp_conf) = &conf.tcp { for (name, service_conf) in &tcp_conf.services { info!("Starting TCP services: {}", name); @@ -197,7 +215,7 @@ async fn set_up_tcp_services(conf: &Config, graceful: kulfi_utils::Graceful) { let port = service_conf.port; let graceful_clone = graceful.clone(); - let (id52, secret_key) = match load_identity(&service_conf.identity).await { + let (id52, secret_key) = match load_identity(&service_conf.identity, used_id52).await { Ok(v) => v, Err(e) => { // The error message has been printed by tracing::error! @@ -233,8 +251,10 @@ pub async fn run(conf_path: &Path, graceful: kulfi_utils::Graceful) -> Option = HashSet::new(); + + set_up_http_services(&conf, &mut used_id52, graceful.clone()).await; + set_up_tcp_services(&conf, &mut used_id52, graceful.clone()).await; guard } From bb03ceab1259ffec5d2d913107c75c2eee6c5541 Mon Sep 17 00:00:00 2001 From: ovo-Tim Date: Tue, 26 Aug 2025 21:13:45 +0800 Subject: [PATCH 13/15] perf: optimize release profile for size and speed --- Cargo.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index aff4377..cc39100 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,11 @@ publish = true rust-version = "1.89" # update this when you update rust-toolchain.toml [profile.release] -strip = true +codegen-units = 1 # Allows LLVM to perform better optimization. +lto = true # Enables link-time-optimizations. +opt-level = 3 +panic = "abort" # Higher performance by disabling panic handlers. +strip = true # Ensures debug symbols are removed. [workspace.dependencies] # Please do not specify a dependency more precisely than needed. If version "1" works, do From 140e9d0733dbd6aafbb245f5c6da2a36bc6c858b Mon Sep 17 00:00:00 2001 From: ovo-Tim Date: Wed, 27 Aug 2025 17:03:23 +0800 Subject: [PATCH 14/15] feat: support secret_file in malai.toml --- kulfi-utils/src/secret.rs | 4 +- malai/src/run.rs | 118 ++++++++++++++++++----------- malai/tests/http_example_conf.toml | 2 +- 3 files changed, 75 insertions(+), 49 deletions(-) diff --git a/kulfi-utils/src/secret.rs b/kulfi-utils/src/secret.rs index e13b61a..c2df22b 100644 --- a/kulfi-utils/src/secret.rs +++ b/kulfi-utils/src/secret.rs @@ -22,7 +22,7 @@ pub fn generate_and_save_key( if let Some(file) = &file { std::fs::write(file, &id52) .wrap_err_with(|| format!("failed to save secret key to {}", &file.display()))?; - println!("Secret key saved to {}", file.display()); + println!("ID52 saved to {}", file.display()); } Ok((id52, secret_key)) } @@ -38,7 +38,7 @@ fn keyring_entry(id52: &str) -> eyre::Result { .wrap_err_with(|| format!("failed to create keyring Entry for {id52}")) } -fn handle_secret(secret: &str) -> eyre::Result<(String, kulfi_id52::SecretKey)> { +pub fn handle_secret(secret: &str) -> eyre::Result<(String, kulfi_id52::SecretKey)> { use std::str::FromStr; let secret_key = kulfi_id52::SecretKey::from_str(secret).map_err(|e| eyre::anyhow!("{}", e))?; let id52 = secret_key.id52(); diff --git a/malai/src/run.rs b/malai/src/run.rs index 0f722f5..24e754e 100644 --- a/malai/src/run.rs +++ b/malai/src/run.rs @@ -1,4 +1,5 @@ use eyre::Context; +use eyre::ContextCompat; use eyre::eyre; use serde::Deserialize; use std::collections::HashMap; @@ -25,6 +26,13 @@ struct MalaiConf { log: Option, } +#[allow(dead_code)] +#[derive(Deserialize, Debug)] +struct IdentityConf { + identity: Option, + secret_file: Option, +} + #[derive(Deserialize, Debug)] struct HttpServices { #[allow(dead_code)] @@ -35,7 +43,8 @@ struct HttpServices { #[allow(dead_code)] #[derive(Deserialize, Debug)] struct HttpServiceConf { - identity: Option, // Leave None to read from env, .malai.secret-key file or .malai.id52 file and system keyring + #[serde(flatten)] + identity_conf: IdentityConf, // Leave None to read from env, .malai.secret-key file or .malai.id52 file and system keyring port: u16, public: bool, active: bool, @@ -55,7 +64,8 @@ struct TcpServices { #[allow(dead_code)] #[derive(Deserialize, Debug)] struct TcpServiceConf { - identity: Option, // Leave None to read from env, .malai.secret-key file or .malai.id52 file and system keyring + #[serde(flatten)] + identity_conf: IdentityConf, // Leave None to read from env, .malai.secret-key file or .malai.id52 file and system keyring port: u16, public: bool, active: bool, @@ -121,32 +131,38 @@ fn set_up_logging(conf: &Config) -> eyre::Result> { Ok(None) } +fn load_secret_from_file(path: &Path) -> eyre::Result<(String, kulfi_id52::SecretKey)> { + let secret_key = fs::read_to_string(path)?.trim().to_string(); + kulfi_utils::secret::handle_secret(&secret_key) +} + +fn check_used(used_id52: &mut HashSet, id52: &str) -> eyre::Result<()> { + if used_id52.contains(id52) { + Err(eyre!("Identity already used.")) + } else { + used_id52.insert(id52.to_string()); + Ok(()) + } +} + async fn load_identity( - identity: &Option, + identity_conf: &IdentityConf, used_id52: &mut HashSet, ) -> eyre::Result<(String, kulfi_id52::SecretKey)> { - let (id52, secret_key) = match identity { - Some(id52) => match kulfi_utils::secret::handle_identity(id52.to_string()) { - Ok(v) => Ok(v), - Err(_e) => Err(eyre!( - "Failed to load identity {} from system keyring.", - id52 - )), - }, - None => match kulfi_utils::read_or_create_key().await { - Ok(v) => Ok(v), - Err(e) => { - malai::identity_read_err_msg(e); - Err(eyre!("Failed to load/create identity from/to file.")) - } - }, - }?; - if used_id52.contains(&id52) { - Err(eyre!("Identity already used.")) + let (id52, secret_key) = if let Some(secret_path) = identity_conf.secret_file.as_ref() { + load_secret_from_file(Path::new(&secret_path))? } else { - used_id52.insert(id52.clone()); - Ok((id52, secret_key)) - } + let id52 = identity_conf + .identity + .as_ref() + .context("No identity specified. Please specify an identity or a secret key file.")?; + kulfi_utils::secret::handle_identity(id52.to_string()).context(format!( + "Failed to load identity {} from system keyring.", + id52 + ))? + }; + check_used(used_id52, &id52)?; + Ok((id52, secret_key)) } async fn set_up_http_services( @@ -173,17 +189,18 @@ async fn set_up_http_services( let bridge = service_conf.bridge.clone(); let graceful_clone = graceful.clone(); - let (id52, secret_key) = match load_identity(&service_conf.identity, used_id52).await { - Ok(v) => v, - Err(e) => { - // The error message has been printed by tracing::error! - error!( - "Failed to load identity for service {}: {} Skipping.", - name, e - ); - continue; - } - }; + let (id52, secret_key) = + match load_identity(&service_conf.identity_conf, used_id52).await { + Ok(v) => v, + Err(e) => { + // The error message has been printed by tracing::error! + error!( + "Failed to load identity for service {}: {} Skipping.", + name, e + ); + continue; + } + }; graceful.spawn(async move { malai::expose_http(host, port, bridge, id52, secret_key, graceful_clone).await @@ -215,17 +232,18 @@ async fn set_up_tcp_services( let port = service_conf.port; let graceful_clone = graceful.clone(); - let (id52, secret_key) = match load_identity(&service_conf.identity, used_id52).await { - Ok(v) => v, - Err(e) => { - // The error message has been printed by tracing::error! - error!( - "Failed to load identity for service {}: {} Skipping.", - name, e - ); - continue; - } - }; + let (id52, secret_key) = + match load_identity(&service_conf.identity_conf, used_id52).await { + Ok(v) => v, + Err(e) => { + // The error message has been printed by tracing::error! + error!( + "Failed to load identity for service {}: {} Skipping.", + name, e + ); + continue; + } + }; graceful.spawn(async move { malai::expose_tcp(host, port, id52, secret_key, graceful_clone).await @@ -266,6 +284,14 @@ fn parse_config_test() { let http = conf.http.as_ref().expect("HTTP services should be present"); assert!(http.services.get("service1").is_some()); assert!(http.services.get("service2").is_some()); + assert!( + http.services + .get("service2") + .unwrap() + .identity_conf + .identity + .is_some() + ); assert!(conf.tcp.is_some()); let tcp = conf.tcp.as_ref().expect("TCP services should be present"); diff --git a/malai/tests/http_example_conf.toml b/malai/tests/http_example_conf.toml index 1252819..dee4aba 100644 --- a/malai/tests/http_example_conf.toml +++ b/malai/tests/http_example_conf.toml @@ -8,7 +8,7 @@ public = true active = true [http.service2] -identity = "" +secret_file = "Path" port = 3001 public = true active = true From dd674f3683e626e405334f058a79e5bd6d4f7083 Mon Sep 17 00:00:00 2001 From: ovo-Tim Date: Wed, 27 Aug 2025 17:20:33 +0800 Subject: [PATCH 15/15] feat: better logging guard --- malai/src/main.rs | 3 +-- malai/src/run.rs | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/malai/src/main.rs b/malai/src/main.rs index 7d37639..e8f10f6 100644 --- a/malai/src/main.rs +++ b/malai/src/main.rs @@ -28,8 +28,7 @@ async fn main() -> eyre::Result<()> { eprintln!("Unable to find malai.toml in {}", conf_file.display()); return Ok(()); } - // Dropping this when main is ended makes sure the log will be written. - let _logging_guard = malai::run(conf_file, graceful.clone()).await; + malai::run(conf_file, graceful.clone()).await; graceful.shutdown().await } else { // run with RUST_LOG="malai=trace,kulfi_utils=trace" to see logs diff --git a/malai/src/run.rs b/malai/src/run.rs index 24e754e..371071f 100644 --- a/malai/src/run.rs +++ b/malai/src/run.rs @@ -7,11 +7,14 @@ use std::collections::HashSet; use std::env; use std::fs; use std::path::Path; +use std::sync::OnceLock; use tracing::{error, info}; use tracing_appender::non_blocking::WorkerGuard; use tracing_appender::rolling; use tracing_subscriber::{fmt, prelude::*}; +static LOG_GUARD: OnceLock = OnceLock::new(); + #[allow(dead_code)] #[derive(Deserialize, Debug)] pub struct Config { @@ -96,7 +99,7 @@ fn parse_config(path: &Path) -> eyre::Result { Ok(conf) } -fn set_up_logging(conf: &Config) -> eyre::Result> { +fn set_up_logging(conf: &Config) -> eyre::Result<()> { match &conf.malai.log { Some(log_dir) => { let log_dir = Path::new(&log_dir); @@ -107,6 +110,7 @@ fn set_up_logging(conf: &Config) -> eyre::Result> { .unwrap_or_else(|| std::ffi::OsStr::new("malai.log")), ); let (non_blocking, guard) = tracing_appender::non_blocking(file_appender); + LOG_GUARD.get_or_init(|| guard); // tracing_subscriber::fmt() // .with_writer(non_blocking) // .with_ansi(false) @@ -120,15 +124,13 @@ fn set_up_logging(conf: &Config) -> eyre::Result> { .with_line_number(true), ); - tracing::subscriber::set_global_default(subscriber) - .expect("Failed to set tracing subscriber"); - return Ok(Some(guard)); + tracing::subscriber::set_global_default(subscriber)?; } None => { tracing_subscriber::fmt::init(); } } - Ok(None) + Ok(()) } fn load_secret_from_file(path: &Path) -> eyre::Result<(String, kulfi_id52::SecretKey)> { @@ -252,20 +254,19 @@ async fn set_up_tcp_services( } } -pub async fn run(conf_path: &Path, graceful: kulfi_utils::Graceful) -> Option { +pub async fn run(conf_path: &Path, graceful: kulfi_utils::Graceful) { let conf = match parse_config(conf_path) { Ok(conf) => conf, Err(e) => { error!("Failed to parse config: {}", e); - return None; + return; } }; - let guard = match set_up_logging(&conf) { + match set_up_logging(&conf) { Ok(guard) => guard, Err(e) => { error!("Failed to set up logging: {}. Skipping.", e); - None } }; @@ -273,7 +274,6 @@ pub async fn run(conf_path: &Path, graceful: kulfi_utils::Graceful) -> Option