merge: pull request #1 from bitswired/feat/cb_channels

feat: use cross beam channels avoid lock and arc

Author: jimzer

Cargo.lock

@@ -1,7 +1,7 @@
 # kiru/Cargo.toml
 [package]
 name = "kiru"
-version = "0.1.9"
+version = "0.1.10"
 edition = "2021"
 description = "Fast text chunking for Rust"
 license = "MIT"
@@ -25,6 +25,7 @@ glob = { workspace = true }
 rayon = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
+crossbeam-channel = "0.5.15"
 
 [dev-dependencies]
 tempfile = { workspace = true }

kiru-core/Cargo.toml

@@ -1,8 +1,8 @@
+use crossbeam_channel::bounded;
 use glob::glob;
 use rayon::prelude::*;
-use std::sync::{mpsc, Mutex};
+use std::io;
 use std::thread;
-use std::{io, sync::Arc};
 use thiserror::Error;
 
 use crate::{BytesChunker, CharactersChunker, StreamType};
@@ -249,9 +249,8 @@ impl ChunkerWithStrategy {
             StreamType::from_source(source)?; // This validates the source
         }
 
-        let (sender, receiver) = mpsc::sync_channel(channel_size);
+        let (sender, receiver) = bounded(channel_size);
         let chunker_params = self.chunker_params.clone();
-        let receiver: Arc<Mutex<mpsc::Receiver<String>>> = Arc::new(Mutex::new(receiver));
 
         thread::spawn({
             move || {
@@ -273,13 +272,7 @@ impl ChunkerWithStrategy {
             }
         });
 
-        let iterator = std::iter::from_fn({
-            let receiver: Arc<Mutex<mpsc::Receiver<String>>> = Arc::clone(&receiver); // Clone Arc for iterator
-            move || {
-                let receiver = receiver.lock().unwrap();
-                receiver.recv().ok()
-            }
-        });
+        let iterator = std::iter::from_fn(move || receiver.recv().ok());
 
         Ok(Box::new(iterator))
     }

kiru-core/src/chunker.rs

@@ -1,7 +1,7 @@
 # kiru-py/Cargo.toml
 [package]
 name = "kiru-py"
-version = "0.1.9"
+version = "0.1.10"
 edition = "2021"
 description = "Python bindings for kiru text chunking library"
 repository = "https://github.com/bitswired/kiru"
@@ -17,6 +17,6 @@ pyo3 = { workspace = true }
 
 [dependencies.kiru]
 path = "../kiru-core"
-version = "0.1.9"
+version = "0.1.10"
 [build-dependencies]
 # None needed for simple bindings