From c28672bd22f071db2cbf379b5bb3b9830e015d01 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 31 Mar 2025 22:03:35 +0000 Subject: [PATCH 1/2] feat(archive_dirs): add archive_dirs package for organizing product images - Introduced a new package `archive_dirs` to manage archiving of product images. - Added logging functionality to track the archiving process. - Implemented directory traversal to find product image directories based on a product ID. - Created a progress bar for user feedback during file movement. - Enabled the removal of empty directories post-archiving. --- Cargo.toml | 1 + archive_dirs/Cargo.toml | 15 ++ archive_dirs/src/main.rs | 361 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 377 insertions(+) create mode 100644 archive_dirs/Cargo.toml create mode 100644 archive_dirs/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index ec185ca..ea5478f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "find_files_in_list", "random_pairs_of_s3file", "find_log_processtime", + "archive_dirs", # Add other tools here ] resolver = "2" # Add this line to specify resolver version 2 diff --git a/archive_dirs/Cargo.toml b/archive_dirs/Cargo.toml new file mode 100644 index 0000000..258de11 --- /dev/null +++ b/archive_dirs/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "archive_dirs" +version = "0.1.0" +edition = "2021" + + +[dependencies] +glob = "0.3.1" # For better pattern matching +walkdir = "2.4.0" # For directory traversal +humansize = "2.1.3" # For human-readable file sizes +indicatif = "0.17.8" # For progress bars +log = "0.4.20" # For logging +env_logger = "0.11.2" # For logging configuration +chrono = "0.4.34" # For timestamp in log filename +ring = "0.17.12" # Explicitly specify the fixed version of ring diff --git a/archive_dirs/src/main.rs b/archive_dirs/src/main.rs new file mode 100644 index 0000000..b319596 --- /dev/null +++ b/archive_dirs/src/main.rs @@ -0,0 +1,361 @@ +use chrono::Local; +use humansize::{format_size, BINARY}; +use indicatif::{ProgressBar, ProgressStyle}; +use log::{error, info, warn}; +use std::env; +use std::fs; +use std::fs::File; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::process; + +fn setup_logging(product_id: &str) -> io::Result<()> { + // Create logs directory if it doesn't exist + let logs_dir = PathBuf::from("logs"); + if !logs_dir.exists() { + fs::create_dir_all(&logs_dir)?; + } + + // Create log file with timestamp + let timestamp = Local::now().format("%Y%m%d_%H%M%S"); + let log_file = logs_dir.join(format!("archive_{}_{}.log", product_id, timestamp)); + + // Initialize file logger + let file = File::create(&log_file)?; + env_logger::Builder::new() + .target(env_logger::Target::Pipe(Box::new(file))) + .format(|buf, record| { + writeln!( + buf, + "{} [{}] {}", + Local::now().format("%Y-%m-%d %H:%M:%S"), + record.level(), + record.args() + ) + }) + .init(); + + info!("Logging initialized. Log file: {}", log_file.display()); + Ok(()) +} + +fn print_usage(program: &str) { + let usage = format!( + "Usage: {} [product_id] [custom_archive_dir]\n\ + {} --help\n\n\ + Arguments:\n\ + product_id The product ID to archive (required)\n\ + custom_archive_dir Optional custom archive directory name\n\n\ + Example:\n\ + {} wish\n\ + {} wish custom-archive", + program, program, program, program + ); + eprintln!("{}", usage); + error!("{}", usage); +} + +fn main() -> io::Result<()> { + let args: Vec = env::args().collect(); + + // Check for --help flag + if args.len() > 1 && (args[1] == "--help" || args[1] == "-h") { + print_usage(&args[0]); + process::exit(0); + } + + // Check if product_id parameter is provided + if args.len() < 2 { + eprintln!("Error: Product ID parameter is required."); + print_usage(&args[0]); + process::exit(1); + } + + let product_id = &args[1]; + + // Setup logging + setup_logging(product_id)?; + info!("Starting archive operation for product: {}", product_id); + + let pattern = format!("product_images-{}-202*", product_id); + let default_archive = format!("product_images-{}-archive", product_id); + + // Set archive directory (use custom if provided, otherwise use default) + let archive_dir = if args.len() >= 3 { + PathBuf::from(&args[2]) + } else { + PathBuf::from(&default_archive) + }; + + // Create archive directory if it doesn't exist + if !archive_dir.exists() { + println!("Creating archive directory: {}", archive_dir.display()); + info!("Creating archive directory: {}", archive_dir.display()); + fs::create_dir_all(&archive_dir)?; + } + + // Find all matching product image directories + println!("Finding product image directories for '{}'...", product_id); + info!("Finding product image directories for '{}'...", product_id); + let current_dir = env::current_dir()?; + let mut dirs: Vec = vec![]; + + for entry in fs::read_dir(¤t_dir)? { + let entry = entry?; + let path = entry.path(); + + if path.is_dir() { + if let Some(name) = path.file_name() { + let name_str = name.to_string_lossy(); + if glob_matches(&name_str, &pattern) { + dirs.push(path); + } + } + } + } + + // Sort directories by name (which contains the date) + dirs.sort(); + + // Check if any directories were found + if dirs.is_empty() { + let error_msg = format!( + "No product image directories found matching pattern '{}'!", + pattern + ); + eprintln!("{}", error_msg); + error!("{}", error_msg); + process::exit(1); + } + + // Print summary before moving + println!( + "Files will be moved from the following directories in this order (oldest to newest):" + ); + info!("Files will be moved from the following directories in this order (oldest to newest):"); + for dir in &dirs { + let dir_name = dir.file_name().unwrap().to_string_lossy(); + println!(" {}", dir_name); + info!(" {}", dir_name); + } + + println!("\nDisk usage of directories:"); + info!("Disk usage of directories:"); + let mut total_size = 0u64; + let mut total_files = 0; + for dir in &dirs { + let dir_size = calculate_dir_size(dir)?; + let file_count = count_files(dir)?; + total_size += dir_size; + total_files += file_count; + let msg = format!( + " {}: {} ({} files)", + dir.file_name().unwrap().to_string_lossy(), + format_size(dir_size, BINARY), + file_count + ); + println!("{}", msg); + info!("{}", msg); + } + + let total_msg = format!("Total disk usage: {}", format_size(total_size, BINARY)); + println!("{}", total_msg); + info!("{}", total_msg); + + let files_msg = format!("Total files to be moved: {}", total_files); + println!("{}", files_msg); + info!("{}", files_msg); + + let note_msg = "Note: Files with duplicate names will be overwritten by newer versions."; + println!("{}", note_msg); + info!("{}", note_msg); + + // Ask for confirmation + print!("Do you want to proceed? (y/n): "); + io::stdout().flush()?; + let mut response = String::new(); + io::stdin().read_line(&mut response)?; + if !response.trim().eq_ignore_ascii_case("y") { + println!("Operation cancelled."); + info!("Operation cancelled by user."); + process::exit(0); + } + + // Move files from each directory into the flat archive directory + println!("Moving files to {}...", archive_dir.display()); + info!("Moving files to {}...", archive_dir.display()); + let mut moved_count = 0; + let mut overwrite_count = 0; + + for dir in &dirs { + let dir_name = dir.file_name().unwrap().to_string_lossy(); + println!("Moving files from {}...", dir_name); + info!("Moving files from {}...", dir_name); + + let mut dir_moved = 0; + let mut dir_overwrite = 0; + + // Count total files in directory first + let total_files = count_files(dir)?; + + // Create progress bar + let pb = ProgressBar::new(total_files as u64); + pb.set_style(ProgressStyle::default_bar() + .template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({percent}%)") + .unwrap() + .progress_chars("#>-")); + + // Move only files to the flat archive directory + for entry in fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + + if path.is_file() { + let filename = path.file_name().unwrap(); + let dest_path = archive_dir.join(filename); + + // Check if file already exists in destination + if dest_path.exists() { + dir_overwrite += 1; + } + + // Move the file + fs::rename(&path, &dest_path)?; + dir_moved += 1; + pb.inc(1); + } + } + + pb.finish_with_message("completed"); + moved_count += dir_moved; + overwrite_count += dir_overwrite; + let move_msg = format!( + " Moved {} files from {} (overwrote {} files)", + dir_moved, dir_name, dir_overwrite + ); + println!("{}", move_msg); + info!("{}", move_msg); + } + + // Remove empty directories + println!("\nRemoving empty directories..."); + info!("Removing empty directories..."); + for dir in &dirs { + if let Err(e) = fs::remove_dir(dir) { + let warn_msg = format!( + "Warning: Failed to remove directory {}: {}", + dir.display(), + e + ); + eprintln!("{}", warn_msg); + warn!("{}", warn_msg); + } else { + let remove_msg = format!(" Removed: {}", dir.file_name().unwrap().to_string_lossy()); + println!("{}", remove_msg); + info!("{}", remove_msg); + } + } + + println!("\nOperation completed."); + info!("Operation completed."); + + let success_msg = format!( + "Successfully moved {} files to {}", + moved_count, + archive_dir.display() + ); + println!("{}", success_msg); + info!("{}", success_msg); + + // Print summary of overwritten files + println!("\nSummary:"); + info!("Summary:"); + let summary_msg = format!(" - Total files moved: {}", moved_count); + println!("{}", summary_msg); + info!("{}", summary_msg); + + let overwrite_msg = format!(" - Files overwritten: {}", overwrite_count); + println!("{}", overwrite_msg); + info!("{}", overwrite_msg); + + let unique_msg = format!( + " - Unique files in archive: {}", + moved_count - overwrite_count + ); + println!("{}", unique_msg); + info!("{}", unique_msg); + + // Print disk usage comparison + println!("\nDisk Usage:"); + info!("Disk Usage:"); + let initial_msg = format!( + " - Initial total size: {}", + format_size(total_size, BINARY) + ); + println!("{}", initial_msg); + info!("{}", initial_msg); + + // Print final archive directory size + let final_size = calculate_dir_size(&archive_dir)?; + let final_size_msg = format!( + " - Final archive size: {}", + format_size(final_size, BINARY) + ); + println!("{}", final_size_msg); + info!("{}", final_size_msg); + + Ok(()) +} + +// Count files in a directory (non-recursive) +fn count_files(dir: &Path) -> io::Result { + let mut count = 0; + for entry in fs::read_dir(dir)? { + if entry?.path().is_file() { + count += 1; + } + } + Ok(count) +} + +// Simple glob pattern matching for our specific case +fn glob_matches(value: &str, pattern: &str) -> bool { + let pattern_parts: Vec<&str> = pattern.split('*').collect(); + + if pattern_parts.is_empty() { + return false; + } + + // Check if string starts with the first part of the pattern + if !value.starts_with(pattern_parts[0]) { + return false; + } + + // If there's only a prefix pattern with *, we're done + if pattern_parts.len() == 1 { + return true; + } + + // Check ending (for patterns like "prefix*suffix") + if pattern_parts.len() == 2 && !pattern_parts[1].is_empty() { + return value.ends_with(pattern_parts[1]); + } + + // For more complex patterns, this is a simplification + // In a real implementation, we would use a proper glob crate + true +} + +// Calculate total size of a directory +fn calculate_dir_size(dir: &Path) -> io::Result { + let mut total_size = 0u64; + for entry in fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + + if path.is_file() { + total_size += path.metadata()?.len(); + } + } + Ok(total_size) +} From 4556663de9674c28c1d21c187ef93f87d2cae6ea Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 31 Mar 2025 22:41:19 +0000 Subject: [PATCH 2/2] fix(archive): add script to move product image files - Introduced a new script to archive product images from specific directories. - The script processes directories chronologically to ensure newer files overwrite older ones. - Added error handling for missing parameters and directory checks. - Enhanced file moving functionality with verification and logging of operations. --- archive_dirs/product_image-archive.sh | 113 ++++++++++++++++++++++++++ archive_dirs/src/main.rs | 62 ++++++++++---- 2 files changed, 159 insertions(+), 16 deletions(-) create mode 100755 archive_dirs/product_image-archive.sh diff --git a/archive_dirs/product_image-archive.sh b/archive_dirs/product_image-archive.sh new file mode 100755 index 0000000..7540441 --- /dev/null +++ b/archive_dirs/product_image-archive.sh @@ -0,0 +1,113 @@ +#!/bin/bash + +# Script to move all files from product image directories into a flat archive directory +# Processes directories in chronological order so newer files overwrite older ones +# Usage: ./archive_products.sh [product_id] [custom_archive_dir] +# +# Example: ./archive_products.sh wish +# - Will process all product_images-wish-* directories +# - Will use product_images-wish-archive as the default archive directory + +# Check if product_id parameter is provided +if [ -z "$1" ]; then + echo "Error: Product ID parameter is required." + echo "Usage: $0 [product_id] [custom_archive_dir]" + echo "Example: $0 wish" + exit 1 +fi + +PRODUCT_ID="$1" +PATTERN="product_images-${PRODUCT_ID}-202*" +DEFAULT_ARCHIVE="product_images-${PRODUCT_ID}-archive" + +# Set archive directory (use custom if provided, otherwise use default) +ARCHIVE_DIR="${2:-$DEFAULT_ARCHIVE}" + +# Create archive directory if it doesn't exist +if [ ! -d "$ARCHIVE_DIR" ]; then + echo "Creating archive directory: $ARCHIVE_DIR" + mkdir -p "$ARCHIVE_DIR" +fi + +# Find all matching product image directories and sort them by date in the directory name +echo "Finding product image directories for '${PRODUCT_ID}'..." +DIRS=$(find . -maxdepth 1 -type d -name "$PATTERN" | sort) + +# Check if any directories were found +if [ -z "$DIRS" ]; then + echo "No product image directories found matching pattern '$PATTERN'!" + exit 1 +fi + +# Print summary before moving +echo "Files will be moved from the following directories in this order (oldest to newest):" +echo "$DIRS" | sed 's/^\.\///' +echo "" +echo "Total space used by these directories:" +du -sh $(echo "$DIRS" | tr '\n' ' ') + +# Count total files to be moved +total_files=0 +for dir in $DIRS; do + dir_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + total_files=$((total_files + dir_files)) +done +echo "Total files to be moved: $total_files" + +echo "Note: Files with duplicate names will be overwritten by newer versions." + +# Ask for confirmation +read -p "Do you want to proceed? (y/n): " confirm +if [[ "$confirm" != "y" && "$confirm" != "Y" ]]; then + echo "Operation cancelled." + exit 0 +fi + +# Move files from each directory into the flat archive directory +echo "Moving files to $ARCHIVE_DIR..." +moved_count=0 +overwrite_count=0 + +for dir in $DIRS; do + dir_name=$(basename "$dir") + echo "Moving files from $dir_name..." + + # Count files in directory for reporting + file_count=$(find "$dir" -maxdepth 1 -type f | wc -l) + dir_moved=0 + dir_overwrite=0 + + # Move only files to the flat archive directory + for file in "$dir"/*; do + if [ -f "$file" ]; then + filename=$(basename "$file") + + # Check if file already exists in destination + if [ -f "$ARCHIVE_DIR/$filename" ]; then + dir_overwrite=$((dir_overwrite + 1)) + fi + + mv "$file" "$ARCHIVE_DIR/" + if [ $? -eq 0 ]; then + dir_moved=$((dir_moved + 1)) + fi + fi + done + + moved_count=$((moved_count + dir_moved)) + overwrite_count=$((overwrite_count + dir_overwrite)) + echo " Moved $dir_moved files from $dir_name (overwrote $dir_overwrite files)" +done + +echo "" +echo "Operation completed." +echo "Successfully moved $moved_count files to $ARCHIVE_DIR" +echo "Total files overwritten: $overwrite_count" + +# Print disk space comparison +echo "" +echo "Disk space in archive directory:" +du -sh "$ARCHIVE_DIR" +echo "" +echo "Remaining disk space in original directories:" +du -sh $(echo "$DIRS" | tr '\n' ' ') diff --git a/archive_dirs/src/main.rs b/archive_dirs/src/main.rs index b319596..bffaf5a 100644 --- a/archive_dirs/src/main.rs +++ b/archive_dirs/src/main.rs @@ -219,10 +219,27 @@ fn main() -> io::Result<()> { dir_overwrite += 1; } - // Move the file - fs::rename(&path, &dest_path)?; - dir_moved += 1; - pb.inc(1); + // Move the file and verify it was moved successfully + match fs::rename(&path, &dest_path) { + Ok(_) => { + // Verify the file exists in destination and not in source + if dest_path.exists() && !path.exists() { + dir_moved += 1; + pb.inc(1); + } else { + warn!("File {} was not properly moved", path.display()); + // Try to remove the source file if it still exists + if path.exists() { + if let Err(e) = fs::remove_file(&path) { + warn!("Failed to remove source file {}: {}", path.display(), e); + } + } + } + } + Err(e) => { + warn!("Failed to move file {}: {}", path.display(), e); + } + } } } @@ -241,18 +258,31 @@ fn main() -> io::Result<()> { println!("\nRemoving empty directories..."); info!("Removing empty directories..."); for dir in &dirs { - if let Err(e) = fs::remove_dir(dir) { - let warn_msg = format!( - "Warning: Failed to remove directory {}: {}", - dir.display(), - e - ); - eprintln!("{}", warn_msg); - warn!("{}", warn_msg); - } else { - let remove_msg = format!(" Removed: {}", dir.file_name().unwrap().to_string_lossy()); - println!("{}", remove_msg); - info!("{}", remove_msg); + // Double check if directory is empty before removing + if let Ok(mut entries) = fs::read_dir(dir) { + if entries.next().is_none() { + if let Err(e) = fs::remove_dir(dir) { + let warn_msg = format!( + "Warning: Failed to remove directory {}: {}", + dir.display(), + e + ); + eprintln!("{}", warn_msg); + warn!("{}", warn_msg); + } else { + let remove_msg = + format!(" Removed: {}", dir.file_name().unwrap().to_string_lossy()); + println!("{}", remove_msg); + info!("{}", remove_msg); + } + } else { + let warn_msg = format!( + "Warning: Directory {} is not empty, skipping removal", + dir.display() + ); + eprintln!("{}", warn_msg); + warn!("{}", warn_msg); + } } }