Upgrade Rails to 8.1.3 and merge DevOps work back onto develop

.dockerignore

@@ -6,9 +6,12 @@
 # Ignore bundler config.
 /.bundle
 
-# Ignore all environment files (except templates).
+# Ignore all environment files
 /.env*
-!/.env*.erb
+
+# Ignore Kamal files.
+/config/deploy*.yml
+/.kamal
 
 # Ignore all default key files.
 /config/master.key
@@ -45,3 +48,50 @@
 # Ignore Docker-related files
 /.dockerignore
 /Dockerfile*
+
+# SAPI
+/coverage
+/rdoc
+/private
+/public/system
+/public/downloads/*.pdf
+#LaTeX
+/public/latex/*.aux
+/public/latex/*.out
+/public/latex/*.log
+/public/latex/*.gz
+/public/latex/index.pdf
+/public/latex/history.pdf
+/public/sitemap*
+
+#checklist downloads
+/public/downloads/checklist/*.*
+
+#exports csvs
+/public/downloads/documents/*.csv
+/public/downloads/quotas/*.csv
+/public/downloads/cites_listings/*.csv
+/public/downloads/cites_suspensions/*.csv
+/public/downloads/eu_listings/*.csv
+/public/downloads/eu_decisions/*.csv
+/public/downloads/cms_listings/*.csv
+/public/downloads/checklist/*.pdf
+/public/downloads/checklist/*.csv
+/public/downloads/checklist/*.json
+/public/downloads/taxon_concepts_names/*.csv
+/public/downloads/synonyms_and_trade_names/*.csv
+/public/downloads/taxon_concepts_distributions/*.csv
+/public/downloads/shipments/*.csv
+/public/downloads/comptab/*.csv
+/public/downloads/gross_exports/*.csv
+/public/downloads/gross_imports/*.csv
+/public/downloads/net_exports/*.csv
+/public/downloads/net_imports/*.csv
+/public/downloads/trade_download_stats/*.csv
+/public/downloads/species_reference_output/*.csv
+/public/downloads/standard_reference_output/*.csv
+/public/downloads/common_names/*.csv
+/public/downloads/iucn_mappings/*.csv
+/public/downloads/cms_mappings/*.csv
+/public/downloads/orphaned_taxon_concepts/*.csv
+/public/uploads/*

.github/workflows/deploy.yml

@@ -0,0 +1,158 @@
+name: Deploy SAPI Rails API
+on:
+  push:
+    branches:
+      - staging
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    runs-on: self-hosted
+    environment: staging
+    steps:
+      - name: Set workflow start time
+        run: |
+          echo "START_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV
+
+      - uses: actions/checkout@v4
+
+      - name: Notify deployment start
+        id: notify-start
+        uses: unepwcmc/devops-actions/.github/actions/slack-notify@v1
+        with:
+          slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          slack-channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
+          notification-type: started
+          action-type: Deploy
+          environment: staging
+          repository: ${{ github.repository }}
+          repository-url: ${{ github.server_url }}/${{ github.repository }}
+          action-run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          actor: ${{ github.actor }}
+          actor-url: ${{ github.server_url }}/${{ github.actor }}
+          workflow-name: ${{ github.workflow }}
+          run-id: ${{ github.run_id }}
+          commit-message: ${{ github.event.head_commit.message || 'Manual workflow trigger' }}
+          runner-name: ${{ runner.name }}
+          start-time: ${{ env.START_TIME }}
+
+      - name: Export and validate kamal secrets
+        shell: bash
+        env:
+          SECRETS_JSON: ${{ toJSON(secrets) }}
+        run: |
+          if [[ ! -f ".kamal/secrets-common" ]]; then
+            echo "Error: .kamal/secrets-common file not found."
+            exit 1
+          fi
+
+          temp_secrets="$(mktemp)"
+          trap 'rm -f "$temp_secrets"' EXIT
+          printf '%s' "$SECRETS_JSON" > "$temp_secrets"
+
+          needed_vars=()
+          while IFS= read -r line; do
+            trimmed="$(echo "$line" | tr -d '\r' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
+            [[ -z "$trimmed" || "$trimmed" == \#* ]] && continue
+            [[ "$trimmed" != *=* ]] && continue
+
+            value="${trimmed#*=}"
+            value="${value#"${value%%[![:space:]]*}"}"
+            value="${value%"${value##*[![:space:]]}"}"
+
+            [[ "$value" != \$* ]] && continue
+            [[ "$value" == '$('* ]] && continue
+
+            if [[ "$value" == \$\{* ]]; then
+              if [[ "$value" =~ ^\$\{([A-Z0-9_]+) ]]; then
+                needed_vars+=("${BASH_REMATCH[1]}")
+              fi
+            elif [[ "$value" =~ ^\$([A-Z0-9_]+) ]]; then
+              needed_vars+=("${BASH_REMATCH[1]}")
+            fi
+          done < ".kamal/secrets-common"
+
+          IFS=$'\n' read -r -d '' -a unique_vars < <(printf '%s\n' "${needed_vars[@]}" | sort -u && printf '\0')
+
+          for var in "${unique_vars[@]}"; do
+            value=$(jq -r --arg key "$var" '.[$key] // empty' "$temp_secrets")
+            if [[ -z "$value" ]]; then
+              echo "Missing environment variable required in .kamal/secrets-common: ${var}" >&2
+              exit 1
+            fi
+
+            {
+              echo "${var}<<EOF"
+              echo "$value"
+              echo "EOF"
+            } >> "$GITHUB_ENV"
+          done
+
+          echo "✅ All required kamal secrets validated"
+
+      - name: Deploy with Kamal v2
+        uses: unepwcmc/devops-actions/.github/actions/kamal-v2.x-deploy@v1
+        with:
+          environment: staging
+          working-directory: .
+        env:
+          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+
+      - name: Set workflow end time and calculate duration
+        if: always()
+        run: |
+          echo "END_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV
+          if [[ -n "$START_TIME" ]]; then
+            start_timestamp=$(date -d "$START_TIME" +%s)
+            end_timestamp=$(date -d "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" +%s)
+            duration=$((end_timestamp - start_timestamp))
+            echo "DEPLOYMENT_DURATION=${duration}" >> $GITHUB_ENV
+          else
+            echo "DEPLOYMENT_DURATION=0" >> $GITHUB_ENV
+          fi
+
+      - name: Notify deployment success
+        if: success()
+        uses: unepwcmc/devops-actions/.github/actions/slack-notify@v1
+        with:
+          slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          slack-channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
+          notification-type: success
+          action-type: Deploy
+          environment: staging
+          repository: ${{ github.repository }}
+          repository-url: ${{ github.server_url }}/${{ github.repository }}
+          action-run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          actor: ${{ github.actor }}
+          actor-url: ${{ github.server_url }}/${{ github.actor }}
+          workflow-name: ${{ github.workflow }}
+          run-id: ${{ github.run_id }}
+          commit-message: ${{ github.event.head_commit.message || 'Manual workflow trigger' }}
+          runner-name: ${{ runner.name }}
+          start-time: ${{ env.START_TIME }}
+          end-time: ${{ env.END_TIME }}
+          deployment-duration: ${{ env.DEPLOYMENT_DURATION }}
+          update-message-ts: ${{ steps.notify-start.outputs.message-ts }}
+
+      - name: Notify deployment failure
+        if: failure()
+        uses: unepwcmc/devops-actions/.github/actions/slack-notify@v1
+        with:
+          slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          slack-channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
+          notification-type: failure
+          action-type: Deploy
+          environment: staging
+          repository: ${{ github.repository }}
+          repository-url: ${{ github.server_url }}/${{ github.repository }}
+          action-run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          actor: ${{ github.actor }}
+          actor-url: ${{ github.server_url }}/${{ github.actor }}
+          workflow-name: ${{ github.workflow }}
+          run-id: ${{ github.run_id }}
+          commit-message: ${{ github.event.head_commit.message || 'Manual workflow trigger' }}
+          runner-name: ${{ runner.name }}
+          start-time: ${{ env.START_TIME }}
+          end-time: ${{ env.END_TIME }}
+          deployment-duration: ${{ env.DEPLOYMENT_DURATION }}
+          update-message-ts: ${{ steps.notify-start.outputs.message-ts }}

.github/workflows/kamal-setup.yml

@@ -0,0 +1,162 @@
+name: Kamal v2 SAPI Setup (One-time per Environment)
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Environment to setup'
+        required: true
+        type: choice
+        options:
+          - staging
+      confirm:
+        description: 'Type "CONFIRM" to proceed with setup'
+        required: true
+        type: string
+
+jobs:
+  validate-input:
+    runs-on: self-hosted
+    steps:
+      - name: Validate confirmation
+        run: |
+          if [[ "${{ github.event.inputs.confirm }}" != "CONFIRM" ]]; then
+            echo "❌ Setup cancelled. You must type 'CONFIRM' to proceed."
+            echo "⚠️  This is a one-time setup operation that will:"
+            echo "   • Initialize Kamal v2 configuration for ${{ github.event.inputs.environment }}"
+            echo "   • Set up Docker containers and services"
+            echo "   • Configure the deployment infrastructure"
+            exit 1
+          fi
+          echo "✅ Setup confirmed for ${{ github.event.inputs.environment }} environment"
+
+  setup:
+    needs: validate-input
+    runs-on: self-hosted
+    environment: ${{ github.event.inputs.environment }}
+    env:
+      RAILS_MASTER_KEY: ${{ secrets.RAILS_MASTER_KEY }}
+      SAPI_DATABASE_HOST: ${{ secrets.SAPI_DATABASE_HOST }}
+      SAPI_DATABASE_NAME: ${{ secrets.SAPI_DATABASE_NAME }}
+      SAPI_DATABASE_USERNAME: ${{ secrets.SAPI_DATABASE_USERNAME }}
+      SAPI_DATABASE_PASSWORD: ${{ secrets.SAPI_DATABASE_PASSWORD }}
+      SAPI_DATABASE_PORT: ${{ secrets.SAPI_DATABASE_PORT }}
+      CAPTIVE_BREEDING_DATABASE_HOST: ${{ secrets.CAPTIVE_BREEDING_DATABASE_HOST }}
+      CAPTIVE_BREEDING_DATABASE_NAME: ${{ secrets.CAPTIVE_BREEDING_DATABASE_NAME }}
+      CAPTIVE_BREEDING_DATABASE_USERNAME: ${{ secrets.CAPTIVE_BREEDING_DATABASE_USERNAME }}
+      CAPTIVE_BREEDING_DATABASE_PASSWORD: ${{ secrets.CAPTIVE_BREEDING_DATABASE_PASSWORD }}
+      CAPTIVE_BREEDING_DATABASE_PORT: ${{ secrets.CAPTIVE_BREEDING_DATABASE_PORT }}
+      MAIL_USERNAME: ${{ secrets.MAIL_USERNAME }}
+      MAIL_PASSWORD: ${{ secrets.MAIL_PASSWORD }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      SAPI_SIDEKIQ_REDIS_URL: ${{ secrets.SAPI_SIDEKIQ_REDIS_URL }}
+      SAPI_SIDEKIQ_REDIS_CACHE_URL: ${{ secrets.SAPI_SIDEKIQ_REDIS_CACHE_URL }}
+      CERTIFICATE_PEM: ${{ secrets.CERTIFICATE_PEM }}
+      PRIVATE_KEY_PEM: ${{ secrets.PRIVATE_KEY_PEM }}
+    steps:
+      - name: Set workflow start time
+        run: |
+          echo "START_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV
+
+      - uses: actions/checkout@v4
+
+      - name: Notify setup start
+        id: notify-start
+        uses: unepwcmc/devops-actions/.github/actions/slack-notify@v1
+        with:
+          slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          slack-channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
+          notification-type: started
+          action-type: Setup
+          environment: ${{ github.event.inputs.environment }}
+          repository: ${{ github.repository }}
+          repository-url: ${{ github.server_url }}/${{ github.repository }}
+          action-run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          actor: ${{ github.actor }}
+          actor-url: ${{ github.server_url }}/${{ github.actor }}
+          workflow-name: ${{ github.workflow }}
+          run-id: ${{ github.run_id }}
+          commit-message: ${{ github.event.head_commit.message || 'Manual workflow trigger' }}
+          runner-name: ${{ runner.name }}
+          start-time: ${{ env.START_TIME }}
+
+      - name: Set up SSH agent
+        uses: webfactory/ssh-agent@v0.9.0
+        with:
+          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          working-directory: deploy
+          bundler-cache: false
+
+      - name: Install Kamal
+        working-directory: deploy
+        run: bundle install --quiet
+
+      - name: Run Kamal setup
+        run: BUNDLE_GEMFILE=deploy/Gemfile bundle exec kamal setup -d ${{ github.event.inputs.environment }}
+
+      - name: Set workflow end time and calculate duration
+        if: always()
+        run: |
+          echo "END_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV
+          if [[ -n "$START_TIME" ]]; then
+            start_timestamp=$(date -d "$START_TIME" +%s)
+            end_timestamp=$(date -d "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" +%s)
+            duration=$((end_timestamp - start_timestamp))
+            echo "DEPLOYMENT_DURATION=${duration}" >> $GITHUB_ENV
+          else
+            echo "DEPLOYMENT_DURATION=0" >> $GITHUB_ENV
+          fi
+
+      - name: Notify setup success
+        if: success()
+        uses: unepwcmc/devops-actions/.github/actions/slack-notify@v1
+        with:
+          slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          slack-channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
+          notification-type: success
+          action-type: Setup
+          environment: ${{ github.event.inputs.environment }}
+          repository: ${{ github.repository }}
+          repository-url: ${{ github.server_url }}/${{ github.repository }}
+          action-run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          actor: ${{ github.actor }}
+          actor-url: ${{ github.server_url }}/${{ github.actor }}
+          workflow-name: ${{ github.workflow }}
+          run-id: ${{ github.run_id }}
+          commit-message: ${{ github.event.head_commit.message || 'Manual workflow trigger' }}
+          runner-name: ${{ runner.name }}
+          start-time: ${{ env.START_TIME }}
+          end-time: ${{ env.END_TIME }}
+          deployment-duration: ${{ env.DEPLOYMENT_DURATION }}
+          update-message-ts: ${{ steps.notify-start.outputs.message-ts }}
+
+      - name: Notify setup failure
+        if: failure()
+        uses: unepwcmc/devops-actions/.github/actions/slack-notify@v1
+        with:
+          slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          slack-channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
+          notification-type: failure
+          action-type: Setup
+          environment: ${{ github.event.inputs.environment }}
+          repository: ${{ github.repository }}
+          repository-url: ${{ github.server_url }}/${{ github.repository }}
+          action-run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          actor: ${{ github.actor }}
+          actor-url: ${{ github.server_url }}/${{ github.actor }}
+          workflow-name: ${{ github.workflow }}
+          run-id: ${{ github.run_id }}
+          commit-message: ${{ github.event.head_commit.message || 'Manual workflow trigger' }}
+          runner-name: ${{ runner.name }}
+          start-time: ${{ env.START_TIME }}
+          end-time: ${{ env.END_TIME }}
+          deployment-duration: ${{ env.DEPLOYMENT_DURATION }}
+          update-message-ts: ${{ steps.notify-start.outputs.message-ts }}