Update to Progress Bar for Intraday, and DB scripts

crates/av-cli/src/commands/load/intraday.rs

@@ -31,6 +31,7 @@ use anyhow::{Result, anyhow};
 use chrono::{DateTime, Utc};
 use clap::Parser;
 use diesel::prelude::*;
+use indicatif::{ProgressBar, ProgressStyle};
 use std::collections::HashMap;
 use std::sync::Arc;
 use tracing::{error, info, warn};
@@ -223,7 +224,15 @@ async fn save_intraday_prices_optimized(
 
       let mut conn = establish_connection(&database_url)?;
 
-      info!("💾 Processing {} intraday price records", prices.len());
+      // Set up progress bar for database insert/update operations
+      let progress = ProgressBar::new(prices.len() as u64);
+      progress.set_style(
+        ProgressStyle::default_bar()
+          .template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")
+          .unwrap()
+          .progress_chars("##-"),
+      );
+      progress.set_message("Saving to database");
 
       let mut saved_count = 0;
       let mut skipped_count = 0;
@@ -299,6 +308,7 @@ async fn save_intraday_prices_optimized(
               .on_conflict_do_nothing() // Safety net
               .execute(&mut conn)?;
 
+            progress.inc(chunk.len() as u64);
             saved_count += inserted;
           }
 
@@ -328,6 +338,11 @@ async fn save_intraday_prices_optimized(
         info!("Updated symbols table for {} symbols", sids.len());
       }
 
+      progress.finish_with_message(format!(
+        "Saved {} new, , skipped {} already existing records",
+        saved_count, skipped_count
+      ));
+
       info!(
         "✅ Database operation complete: {} new records saved, {} skipped (already existed)",
         saved_count, skipped_count

crates/av-database/postgres/scripts/backup_postgres.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+############################################################
+# Help                                                     #
+############################################################
+Help()
+{
+   # Display Help
+   echo "Backs up the PostgreSQL database to a timestamped directory using 'pg_basebackup'."
+   echo
+   echo "Syntax: backup_postgres.sh [-c|-d|-h|-i|-p|-u]"
+   echo "options:"
+   echo "c     The database password."
+   echo "d     The backup directory."
+   echo "h     These help details."
+   echo "i     The database hostname (default 'localhost')."
+   echo "p     The database post (default '5432')."
+   echo "u     The database username (default 'postgres')."
+   echo
+}
+
+############################################################
+# Echoes the error and help info, then exits.              #
+############################################################
+ErrorEcho() {
+  echo
+  echo "${1}"
+  echo
+  Help
+  exit;
+}
+
+############################################################
+############################################################
+# Main program                                             #
+############################################################
+############################################################
+
+# Set variables
+TimestampNow=$( date '+%F_%H-%M-%S' )
+BackupDirectory=""
+DBPassword=""
+DBHost="localhost"
+DBPort="5432"
+DBUSer="postgres"
+
+############################################################
+# Process the input options. Add options as needed.        #
+############################################################
+# Get the options
+while getopts ":c:d:hi:p:u:" option; do
+   case $option in
+      c) # Database Password/Credential
+        DBPassword=$OPTARG;;
+      d) # Backup directory
+        BackupDirectory=$OPTARG;;
+      h) # Display Help
+         Help
+         exit;;
+      i) # Database Host (IP Address or Hostname)
+        DBHost=$OPTARG;;
+      p) # Database Port
+        DBPort=$OPTARG;;
+      u) # Database User
+        DBUSer=$OPTARG;;
+     \?) # Invalid option
+         ErrorEcho "Error: Invalid option";;
+   esac
+done
+
+if [[ -z "${DBPassword}" ]]; then
+  ErrorEcho "Missing options -c, the database password. Please pass to this script when calling it."
+fi
+
+if [[ -z "${BackupDirectory}" ]]; then
+  ErrorEcho "Missing options -d, the backup directory. Please pass to this script when calling it."
+fi
+
+PGPASSWORD="${DBPassword}" \
+  pg_basebackup -h "${DBHost}" -p "${DBPort}" -U "${DBUSer}" -D "${BackupDirectory}"/"${TimestampNow}" -Ft -z -Xs -P

crates/av-database/postgres/scripts/populate_for_symbol.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+############################################################
+# Help                                                     #
+############################################################
+Help()
+{
+   # Display Help
+   echo "Populates Daily Prices and Intraday Prices for Symbols that were 'missing' and are no longer."
+   echo
+   echo "Syntax: populate_for_symbol.sh [-c|-d|-h|-i|-p|-s|-u]"
+   echo "options:"
+   echo "c     The database password (default 'dev_pw')."
+   echo "d     The database (default 'sec_master')."
+   echo "h     These help details."
+   echo "i     The database hostname (default 'localhost')."
+   echo "p     The database post (default '5432')."
+   echo "s     The Symbol to populate."
+   echo "u     The database username (default 'ts_user')."
+   echo
+}
+
+############################################################
+# Echoes the error and help info, then exits.              #
+############################################################
+ErrorEcho() {
+  echo
+  echo "${1}"
+  echo
+  Help
+  exit;
+}
+
+############################################################
+############################################################
+# Main program                                             #
+############################################################
+############################################################
+
+# Set variables
+Database="sec_master"
+DBPassword="dev_pw"
+DBHost="localhost"
+DBPort="5432"
+DBUSer="ts_user"
+Symbol=""
+Year=""
+Month=""
+FirstYear=true
+
+############################################################
+# Process the input options. Add options as needed.        #
+############################################################
+# Get the options
+while getopts ":c:d:hi:p:s:u:" option; do
+   case $option in
+      c) # Database Password/Credential
+        DBPassword=$OPTARG;;
+      d) # Database
+        Database=$OPTARG;;
+      h) # Display Help
+         Help
+         exit;;
+      i) # Database Host (IP Address or Hostname)
+        DBHost=$OPTARG;;
+      p) # Database Port
+        DBPort=$OPTARG;;
+      s) # Symbol
+        Symbol=$OPTARG;;
+      u) # Database User
+        DBUSer=$OPTARG;;
+     \?) # Invalid option
+         ErrorEcho "Error: Invalid option";;
+   esac
+done
+
+if [[ -z "$ALPHA_VANTAGE_API_KEY" ]]; then
+  ErrorEcho "Missing 'ALPHA_VANTAGE_API_KEY' environment variable, please set prior to running this script."
+fi
+
+if [[ -z "$DATABASE_URL" ]]; then
+  ErrorEcho "Missing 'DATABASE_URL' environment variable, please set prior to running this script."
+fi
+
+export RUSTFLAGS="-Awarnings"
+
+cargo run load daily --outputsize=full --concurrent=10 --api-delay=100 --symbol="${Symbol}"
+
+# Query database to find the first year and month this Symbol started getting Summary Prices, and use that as a starting date for Intraday Prices
+Year=$(psql "host=${DBHost} port=${DBPort} dbname=${Database} user=${DBUSer} password=${DBPassword}" -c "SELECT MIN(EXTRACT(YEAR FROM date)) FROM summaryprices WHERE symbol = '${Symbol}';" -t)
+Month=$(psql "host=${DBHost} port=${DBPort} dbname=${Database} user=${DBUSer} password=${DBPassword}" -c "SELECT MIN(EXTRACT(MONTH FROM date)) FROM summaryprices WHERE symbol = '${Symbol}' AND date >= '$((Year))-01-01' AND date <= '$((Year))-12-31';" -t)
+
+echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
+
+if ((Year < 2005)); then
+  echo "Initial date is out-of-bounds: $((Month))/$((Year)), reverting to 1/2005."
+  Year=2005
+  Month=1
+fi
+
+echo "Extracting Intraday Prices from Symbol '${Symbol}' from $((Month))/$((Year))."
+echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
+
+for year in $(seq $((Year)) 2025);
+do
+  if [[ "$FirstYear" = true ]]; then
+    for month in $(seq -f "%02g" $((Month)) 12);
+    do
+      cargo run load intraday --update --extended-hours --update-symbols --concurrent=5 --api-delay=250 --force-refresh --month="${year}"-"${month}" --symbol="${Symbol}"
+    done
+    FirstYear=false
+  else
+    for month in $(seq -f "%02g" 1 12);
+    do
+      cargo run load intraday --update --extended-hours --update-symbols --concurrent=5 --api-delay=250 --force-refresh --month="${year}"-"${month}" --symbol="${Symbol}"
+    done
+  fi
+done
+
+cargo run load intraday --force-refresh --concurrent=5 --api-delay=250 --symbol="${Symbol}"

crates/av-database/postgres/scripts/populate_intraday_from_year.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+############################################################
+# Help                                                     #
+############################################################
+Help()
+{
+   # Display Help
+   echo "Populates Intraday Prices for all Symbols from the year passed to 2026."
+   echo
+   echo "Syntax: populate_intraday_from_year.sh [-h|-y]"
+   echo "options:"
+   echo "h     These help details."
+   echo "y     The year to start from (default '2005')."
+   echo
+}
+
+############################################################
+# Echoes the error and help info, then exits.              #
+############################################################
+ErrorEcho() {
+  echo
+  echo "${1}"
+  echo
+  Help
+  exit;
+}
+
+############################################################
+############################################################
+# Main program                                             #
+############################################################
+############################################################
+
+# Set variables
+Year="2005"
+
+############################################################
+# Process the input options. Add options as needed.        #
+############################################################
+# Get the options
+while getopts ":hy:" option; do
+   case $option in
+      h) # Display Help
+         Help
+         exit;;
+      y) # Database User
+        Year=$OPTARG;;
+     \?) # Invalid option
+         ErrorEcho "Error: Invalid option";;
+   esac
+done
+
+if [[ -z "$ALPHA_VANTAGE_API_KEY" ]]; then
+  ErrorEcho "Missing 'ALPHA_VANTAGE_API_KEY' environment variable, please set prior to running this script."
+fi
+
+if [[ -z "$DATABASE_URL" ]]; then
+  ErrorEcho "Missing 'DATABASE_URL' environment variable, please set prior to running this script."
+fi
+
+export RUSTFLAGS="-Awarnings"
+
+if ((Year < 2005 || Year > 2026)); then
+  echo "Initial year is out-of-bounds: $((Year)), reverting to 2005."
+  Year=2005
+fi
+
+for year in $(seq "${Year}" 2026);
+do
+  for month in $(seq -f "%02g" 1 12);
+  do
+    cargo run load intraday --update --extended-hours --update-symbols --concurrent=5 --api-delay=250 --force-refresh --month="${year}"-"${month}"
+  done
+done