Feature reverseproxy caching

.github/workflows/api.yml

@@ -0,0 +1,50 @@
+# API testing workflow config for CI/CD on Github
+
+name: API continuous integration
+
+# Controls when the action will run. 
+on:
+  # Triggers the workflow for main branch on git push
+  push:
+    branches: [main]
+  # Triggers workflow for or any branch on pull requests
+  pull_request:
+    branches:
+      - '**'
+  # Allows running this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow is defined of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "api_test"
+  api_test:
+    strategy:
+      # don't cancel other jobs if one fails
+      fail-fast: false
+      matrix:
+        os: [ubuntu, macos]
+    runs-on: ${{ matrix.os }}-latest  # Runs on latest builds of matrix OSes
+    env:
+      BUNDLE_WITHOUT: production # skip installing production gem (pg)
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v4
+
+      # Builds on a predefined action that has Ruby installed 
+      - uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+
+      - name: Setup test database
+        env: # define secrets in Github project -> settings -> secrets
+            DB_FILENAME: ${{ secrets.DB_FILENAME }}
+        run: RACK_ENV=test bundle exec rake db:migrate
+
+      - name: Run all specs
+        env: # define secrets in Github project -> settings -> secrets
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+          DB_FILENAME: ${{ secrets.DB_FILENAME }}
+          REPOSTORE_PATH: ${{ secrets.REPOSTORE_PATH }}
+          API_HOST: ${{ secrets.API_HOST }}
+        run: bundle exec rake spec

.gitignore

@@ -7,3 +7,4 @@ coverage/*
 !coverage/.resultset.json
 *.db
 repostore/**/
+_cache/

Gemfile

@@ -11,6 +11,7 @@ gem 'rake', '~> 13.0'
 
 # PRESENTATION LAYER
 gem 'multi_json', '~> 1.15'
+gem 'ostruct', '~> 0.0'
 gem 'roar', '~> 1.1'
 
 # APPLICATION LAYER
@@ -26,6 +27,11 @@ gem 'dry-monads', '~> 1.0'
 gem 'dry-transaction', '~> 0'
 gem 'dry-validation', '~> 1.0'
 
+# Caching
+gem 'rack-cache', '~> 1.13'
+gem 'redis', '~> 4.8'
+gem 'redis-rack-cache', '~> 2.2'
+
 # DOMAIN LAYER
 # Validation
 gem 'dry-struct', '~> 1.0'

Gemfile.lock

@@ -68,8 +68,6 @@ GEM
       dry-initializer (~> 3.2)
       dry-schema (~> 1.14)
       zeitwerk (~> 2.6)
-    faker (3.5.2)
-      i18n (>= 1.8.11, < 2)
     ffi (1.17.2)
     ffi (1.17.2-arm64-darwin)
     ffi (1.17.2-x86_64-linux-gnu)
@@ -83,7 +81,6 @@ GEM
       ruby_parser (~> 3.1, > 3.1.0)
       sexp_processor (~> 4.8)
     hashdiff (1.2.1)
-    headless (2.3.1)
     hirb (0.7.3)
     http (5.3.1)
       addressable (~> 2.8)
@@ -93,8 +90,6 @@ GEM
     http-cookie (1.1.0)
       domain_name (~> 0.5)
     http-form_data (2.3.0)
-    i18n (1.14.7)
-      concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
     json (2.16.0)
     language_server-protocol (3.17.0.5)
@@ -108,15 +103,12 @@ GEM
     logger (1.7.0)
     method_source (1.1.0)
     mini_portile2 (2.8.9)
-    minitest (5.26.1)
+    minitest (5.26.2)
     minitest-rg (5.3.0)
       minitest (~> 5.0)
+    multi_json (1.17.0)
     nio4r (2.7.5)
-    page-object (2.3.1)
-      page_navigation (>= 0.10)
-      watir (>= 6.10.3)
-    page_navigation (0.10)
-      data_magic (>= 0.22)
+    ostruct (0.6.3)
     parallel (1.27.0)
     parser (3.3.10.0)
       ast (~> 2.4.1)
@@ -134,26 +126,40 @@ GEM
       nio4r (~> 2.0)
     racc (1.8.1)
     rack (3.2.4)
+    rack-cache (1.17.0)
+      rack (>= 0.4)
     rack-session (0.3.0)
       rack (>= 3.0.0.beta1)
-    rack-test (2.1.0)
+    rack-test (2.2.0)
       rack (>= 1.3)
     rainbow (3.1.1)
     rake (13.3.1)
     rb-fsevent (0.11.2)
     rb-inotify (0.11.1)
       ffi (~> 1.0)
+    redis (4.8.1)
+    redis-rack-cache (2.2.1)
+      rack-cache (>= 1.10, < 2)
+      redis-store (>= 1.6, < 2)
+    redis-store (1.11.0)
+      redis (>= 4, < 6)
     reek (6.5.0)
       dry-schema (~> 1.13)
       logger (~> 1.6)
       parser (~> 3.3.0)
       rainbow (>= 2.0, < 4.0)
       rexml (~> 3.1)
     regexp_parser (2.11.3)
+    representable (3.2.0)
+      declarative (< 0.1.0)
+      trailblazer-option (>= 0.1.1, < 0.2.0)
+      uber (< 0.2.0)
     rerun (0.14.0)
       listen (~> 3.0)
     rexml (3.4.4)
-    roda (3.97.0)
+    roar (1.2.0)
+      representable (~> 3.1)
+    roda (3.98.0)
       rack
     rubocop (1.81.7)
       json (~> 2.3)
@@ -183,13 +189,6 @@ GEM
     ruby_parser (3.21.1)
       racc (~> 1.5)
       sexp_processor (~> 4.16)
-    rubyzip (3.2.2)
-    selenium-webdriver (4.38.0)
-      base64 (~> 0.2)
-      logger (~> 1.4)
-      rexml (~> 3.2, >= 3.2.5)
-      rubyzip (>= 1.2.2, < 4.0)
-      websocket (~> 1.0)
     sequel (5.98.0)
       bigdecimal
     sexp_processor (4.17.4)
@@ -201,23 +200,18 @@ GEM
     simplecov_json_formatter (0.1.4)
     sqlite3 (1.7.3)
       mini_portile2 (~> 2.8.0)
-    temple (0.10.4)
     thor (1.4.0)
-    tilt (2.6.1)
+    trailblazer-option (0.1.2)
+    uber (0.1.0)
     unicode-display_width (3.2.0)
       unicode-emoji (~> 4.1)
     unicode-emoji (4.1.0)
     vcr (6.3.1)
       base64
-    watir (7.3.0)
-      regexp_parser (>= 1.2, < 3)
-      selenium-webdriver (~> 4.2)
     webmock (3.26.1)
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    websocket (1.2.11)
-    yml_reader (0.7)
     zeitwerk (2.7.3)
 
 PLATFORMS
@@ -237,14 +231,19 @@ DEPENDENCIES
   hirb
   http (~> 5.0)
   logger (~> 1.0)
-  minitest (~> 5.20)
-  minitest-rg (~> 5.2)
-  page-object (~> 2.0)
+  minitest (~> 5.0)
+  minitest-rg (~> 5.0)
+  multi_json (~> 1.15)
+  ostruct (~> 0.0)
   pg (~> 1.0)
   pry
   puma (~> 6.0)
+  rack-cache (~> 1.13)
   rack-session (~> 0)
+  rack-test
   rake (~> 13.0)
+  redis (~> 4.8)
+  redis-rack-cache (~> 2.2)
   reek
   rerun
   roar (~> 1.1)

Rakefile

@@ -20,16 +20,16 @@ end
 
 desc 'Run the webserver and application and restart if code changes'
 task :rerun do
-  sh "rerun -c --ignore 'coverage/*' --ignore 'repostore/*' -- bundle exec puma"
+  sh "rerun -c --ignore 'coverage/*' --ignore 'repostore/*' -- rake run"
 end
 
 desc 'Run web app in default (dev) mode'
-task run: ['run:default']
+task run: ['run:dev']
 
 namespace :run do
   desc 'Run API in dev mode'
-  task :default do
-    sh 'rerun -c "bundle exec puma -p 9090"'
+  task :dev do
+    sh 'bundle exec puma -p 9090'
   end
 
   desc 'Run API in test mode'
@@ -108,6 +108,51 @@ namespace :repos do
   end
 end
 
+namespace :cache do
+  task :config do # rubocop:disable Rake/Desc
+    require_relative 'config/environment' # load config info
+    require_relative 'app/infrastructure/cache/redis_cache'
+    @api = CodePraise::App
+  end
+
+  desc 'Directory listing of local dev cache'
+  namespace :list do
+    desc 'Lists development cache'
+    task :dev do
+      puts 'Lists development cache'
+      list = `ls _cache/rack/meta`
+      puts 'No local cache found' if list.empty?
+      puts list
+    end
+
+    desc 'Lists production cache'
+    task :production => :config do
+      puts 'Finding production cache'
+      keys = CodePraise::Cache::Client.new(@api.config).keys
+      puts 'No keys found' if keys.none?
+      keys.each { |key| puts "Key: #{key}" }
+    end
+  end
+
+  namespace :wipe do
+    desc 'Delete development cache'
+    task :dev do
+      puts 'Deleting development cache'
+      sh 'rm -rf _cache/*'
+    end
+
+    desc 'Delete production cache'
+    task :production => :config do
+      print 'Are you sure you wish to wipe the production cache? (y/n) '
+      if $stdin.gets.chomp.downcase == 'y'
+        puts 'Deleting production cache'
+        wiped = CodePraise::Cache::Client.new(@api.config).wipe
+        wiped.each { |key| puts "Wiped: #{key}" }
+      end
+    end
+  end
+end
+
 desc 'Run application console'
 task :console do
   sh 'pry -r ./load_all'

app/application/controllers/app.rb

@@ -7,10 +7,11 @@ module CodePraise
   # Web App
   class App < Roda
     plugin :halt
-    plugin :flash
-    plugin :all_verbs # allows DELETE and other HTTP verbs beyond GET/POST
+    plugin :caching
+    # plugin :all_verbs # allows DELETE and other HTTP verbs beyond GET/POST
+
+    # use Rack::MethodOverride # for other HTTP verbs (with plugin all_verbs)
 
-    # rubocop:disable Metrics/BlockLength
     route do |routing|
       response['Content-Type'] = 'application/json'
 
@@ -31,6 +32,8 @@ class App < Roda
           routing.on String, String do |owner_name, project_name|
             # GET /projects/{owner_name}/{project_name}[/folder_namepath/]
             routing.get do
+              response.cache_control public: true, max_age: 120
+
               path_request = Request::ProjectPath.new(
                 owner_name, project_name, request
               )
@@ -86,6 +89,5 @@ class App < Roda
         end
       end
     end
-    # rubocop:enable Metrics/BlockLength
   end
 end

app/application/services/appraise_project.rb

@@ -17,6 +17,7 @@ class AppraiseProject
       NO_PROJ_ERR = 'Project not found'
       DB_ERR = 'Having trouble accessing the database'
       CLONE_ERR = 'Could not clone this project'
+      TOO_LARGE_ERR = 'Project is too large to clone'
       NO_FOLDER_ERR = 'Could not find that folder'
 
       def retrieve_remote_project(input)
@@ -38,6 +39,9 @@ def clone_remote(input)
         gitrepo.clone unless gitrepo.exists_locally?
 
         Success(input.merge(gitrepo:))
+      rescue GitRepo::Errors::TooLargeToClone
+        App.logger.warn "Project too large: #{input[:project].fullname} (#{input[:project].size} KB)"
+        Failure(Response::ApiResult.new(status: :forbidden, message: TOO_LARGE_ERR))
       rescue StandardError => error
         App.logger.error error.backtrace.join("\n")
         Failure(Response::ApiResult.new(status: :internal_error, message: CLONE_ERR))

app/domain/projects/entities/project.rb

@@ -11,7 +11,7 @@ module Entity
     class Project < Dry::Struct
       include Dry.Types
 
-      MAX_SIZE_KB = 10_000
+      MAX_SIZE_KB = 50_000
 
       attribute :id,            Integer.optional
       attribute :origin_id,     Strict::Integer

app/infrastructure/cache/redis_cache.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'redis'
+
+module CodePraise
+  module Cache
+    # Redis client utility
+    class Client
+      def initialize(config)
+        @redis = Redis.new(url: config.REDISCLOUD_URL)
+      end
+
+      def keys
+        @redis.keys
+      end
+
+      def wipe
+        keys.each { |key| @redis.del(key) }
+      end
+    end
+  end
+end

app/infrastructure/git/repositories/blame_reporter.rb

@@ -29,7 +29,7 @@ def files
           if @folder_name.empty?
             @local.files
           else
-            @local.files.select { _1.start_with?("#{@folder_name}/") }
+            @local.files.select { |file| file.start_with? "#{@folder_name}/" }
           end
       end
     end

app/infrastructure/git/repositories/local_repo.rb

@@ -13,7 +13,7 @@ module Errors
     class LocalGitRepo
       ONLY_FOLDERS = '**/'
       FILES_AND_FOLDERS = '**/*'
-      TEXT_FILES = %w[rb py c js java css html slim md yml json txt].join('|')
+      TEXT_FILES = %w[rb py js css html slim md erb].join('|')
       CODE_FILENAME_MATCH = /\.(#{TEXT_FILES})$/
 
       attr_reader :git_repo_path