Feature async

.github/workflows/api.yml

@@ -21,6 +21,8 @@ jobs:
     strategy:
       # don't cancel other jobs if one fails
       fail-fast: false
+      # maximum number of jobs that can run simultaneously
+      max-parallel: 1
       matrix:
         os: [ubuntu, macos]
     runs-on: ${{ matrix.os }}-latest  # Runs on latest builds of matrix OSes
@@ -47,4 +49,10 @@ jobs:
           DB_FILENAME: ${{ secrets.DB_FILENAME }}
           REPOSTORE_PATH: ${{ secrets.REPOSTORE_PATH }}
           API_HOST: ${{ secrets.API_HOST }}
-        run: bundle exec rake spec
+          CLONE_QUEUE_URL: ${{ secrets.CLONE_QUEUE_URL }}
+          AWS_REGION: ${{ secrets.AWS_REGION }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          rake worker:run:test &
+          bundle exec rake spec

Gemfile

@@ -40,12 +40,18 @@ gem 'dry-types', '~> 1.0'
 # INFRASTRUCTURE LAYER
 # Networking
 gem 'http', '~> 5.0'
+gem 'openssl', '~> 3.3.1' # resolves MacoOS CRL error: https://github.com/ruby/openssl/issues/949
 
 # Database
 gem 'hirb'
 # gem 'hirb-unicode' # incompatible with new rubocop
 gem 'sequel', '~> 5.0'
 
+# Asynchronicity
+gem 'aws-sdk-sqs', '~> 1.0'
+gem 'concurrent-ruby', '~> 1.0'
+gem 'shoryuken', '~> 6.0'
+
 group :development, :test do
   gem 'sqlite3', '~> 1.0'
 end

Gemfile.lock

@@ -1,9 +1,24 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.8.7)
-      public_suffix (>= 2.0.2, < 7.0)
+    addressable (2.8.8)
+      public_suffix (>= 2.0.2, < 8.0)
     ast (2.4.3)
+    aws-eventstream (1.4.0)
+    aws-partitions (1.1190.0)
+    aws-sdk-core (3.239.2)
+      aws-eventstream (~> 1, >= 1.3.0)
+      aws-partitions (~> 1, >= 1.992.0)
+      aws-sigv4 (~> 1.9)
+      base64
+      bigdecimal
+      jmespath (~> 1, >= 1.6.1)
+      logger
+    aws-sdk-sqs (1.107.0)
+      aws-sdk-core (~> 3, >= 3.239.1)
+      aws-sigv4 (~> 1.5)
+    aws-sigv4 (1.12.1)
+      aws-eventstream (~> 1, >= 1.0.2)
     base64 (0.3.0)
     bigdecimal (3.3.1)
     coderay (1.1.3)
@@ -91,7 +106,8 @@ GEM
       domain_name (~> 0.5)
     http-form_data (2.3.0)
     ice_nine (0.11.2)
-    json (2.16.0)
+    jmespath (1.6.2)
+    json (2.17.1)
     language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
     listen (3.9.0)
@@ -106,8 +122,9 @@ GEM
     minitest (5.26.2)
     minitest-rg (5.3.0)
       minitest (~> 5.0)
-    multi_json (1.17.0)
+    multi_json (1.18.0)
     nio4r (2.7.5)
+    openssl (3.3.2)
     ostruct (0.6.3)
     parallel (1.27.0)
     parser (3.3.10.0)
@@ -121,7 +138,7 @@ GEM
     pry (0.15.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (6.0.2)
+    public_suffix (7.0.0)
     puma (6.6.1)
       nio4r (~> 2.0)
     racc (1.8.1)
@@ -189,9 +206,13 @@ GEM
     ruby_parser (3.21.1)
       racc (~> 1.5)
       sexp_processor (~> 4.16)
-    sequel (5.98.0)
+    sequel (5.99.0)
       bigdecimal
     sexp_processor (4.17.4)
+    shoryuken (6.2.1)
+      aws-sdk-core (>= 2)
+      concurrent-ruby
+      thor
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -220,7 +241,9 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  aws-sdk-sqs (~> 1.0)
   base64
+  concurrent-ruby (~> 1.0)
   dry-monads (~> 1.0)
   dry-struct (~> 1.0)
   dry-transaction (~> 0)
@@ -234,6 +257,7 @@ DEPENDENCIES
   minitest (~> 5.0)
   minitest-rg (~> 5.0)
   multi_json (~> 1.15)
+  openssl (~> 3.3.1)
   ostruct (~> 0.0)
   pg (~> 1.0)
   pry
@@ -253,6 +277,7 @@ DEPENDENCIES
   rubocop-rake
   rubocop-sequel
   sequel (~> 5.0)
+  shoryuken (~> 6.0)
   simplecov (~> 0.0)
   sqlite3 (~> 1.0)
   vcr (~> 6.0)

Procfile

@@ -1 +1,3 @@
+release: rake db:migrate; rake queues:create
 web: bundle exec puma -t 5:5 -p ${PORT:-3000} -e ${RACK_ENV:-development}
+worker: bundle exec shoryuken -r ./workers/git_clone_worker.rb -C ./workers/shoryuken.yml

Rakefile

@@ -9,6 +9,7 @@ end
 
 desc 'Run unit and integration tests'
 Rake::TestTask.new(:spec) do |t|
+  puts 'Make sure worker is running in separate process'
   t.pattern = 'spec/tests/**/*_spec.rb'
   t.warning = false
 end
@@ -18,11 +19,6 @@ task :respec do
   sh "rerun -c 'rake spec' --ignore 'coverage/*' --ignore 'repostore/*'"
 end
 
-desc 'Run the webserver and application and restart if code changes'
-task :rerun do
-  sh "rerun -c --ignore 'coverage/*' --ignore 'repostore/*' -- rake run"
-end
-
 desc 'Run web app in default (dev) mode'
 task run: ['run:dev']
 
@@ -38,6 +34,11 @@ namespace :run do
   end
 end
 
+desc 'Keep restarting web app in dev mode upon changes'
+task :rerun do
+  sh "rerun -c --ignore 'coverage/*' --ignore 'repostore/*' --ignore '_cache/*' -- bundle exec puma -p 9090"
+end
+
 namespace :db do
   task :config do # rubocop:disable Rake/Desc
     require 'sequel'
@@ -155,6 +156,72 @@ namespace :cache do
   end
 end
 
+namespace :queues do
+  task :config do
+    require 'aws-sdk-sqs'
+    require_relative 'config/environment' # load config info
+    @api = CodePraise::App
+    @sqs = Aws::SQS::Client.new(
+      access_key_id: @api.config.AWS_ACCESS_KEY_ID,
+      secret_access_key: @api.config.AWS_SECRET_ACCESS_KEY,
+      region: @api.config.AWS_REGION
+    )
+    @q_name = @api.config.CLONE_QUEUE
+    @q_url = @sqs.get_queue_url(queue_name: @q_name).queue_url
+
+    puts "Environment: #{@api.environment}"
+  end
+
+  desc 'Create SQS queue for worker'
+  task :create => :config do
+    @sqs.create_queue(queue_name: @q_name)
+
+    puts 'Queue created:'
+    puts "  Name: #{@q_name}"
+    puts "  Region: #{@api.config.AWS_REGION}"
+    puts "  URL: #{@q_url}"
+  rescue StandardError => e
+    puts "Error creating queue: #{e}"
+  end
+
+  desc 'Report status of queue for worker'
+  task :status => :config do
+    puts 'Queue info:'
+    puts "  Name: #{@q_name}"
+    puts "  Region: #{@api.config.AWS_REGION}"
+    puts "  URL: #{@q_url}"
+  rescue StandardError => e
+    puts "Error finding queue: #{e}"
+  end
+
+  desc 'Purge messages in SQS queue for worker'
+  task :purge => :config do
+    @sqs.purge_queue(queue_url: @q_url)
+    puts "Queue #{@q_name} purged"
+  rescue StandardError => e
+    puts "Error purging queue: #{e}"
+  end
+end
+
+namespace :worker do
+  namespace :run do
+    desc 'Run the background cloning worker in development mode'
+    task :dev => :config do
+      sh 'RACK_ENV=development bundle exec shoryuken -r ./workers/git_clone_worker.rb -C ./workers/shoryuken_dev.yml'
+    end
+
+    desc 'Run the background cloning worker in testing mode'
+    task :test => :config do
+      sh 'RACK_ENV=test bundle exec shoryuken -r ./workers/git_clone_worker.rb -C ./workers/shoryuken_test.yml'
+    end
+
+    desc 'Run the background cloning worker in production mode'
+    task :production => :config do
+      sh 'RACK_ENV=production bundle exec shoryuken -r ./workers/git_clone_worker.rb -C ./workers/shoryuken.yml'
+    end
+  end
+end
+
 desc 'Run application console'
 task :console do
   sh 'pry -r ./load_all'

app/application/controllers/app.rb

@@ -32,7 +32,9 @@ class App < Roda
           routing.on String, String do |owner_name, project_name|
             # GET /projects/{owner_name}/{project_name}[/folder_namepath/]
             routing.get do
-              response.cache_control public: true, max_age: 120
+              App.configure :production do
+                response.cache_control public: true, max_age: 300
+              end
 
               path_request = Request::ProjectPath.new(
                 owner_name, project_name, request

app/application/services/appraise_project.rb

@@ -8,19 +8,23 @@ module Service
     class AppraiseProject
       include Dry::Transaction
 
-      step :retrieve_remote_project
-      step :clone_remote
+      step :find_project_details
+      step :check_project_eligibility
+      step :request_cloning_worker
       step :appraise_contributions
 
       private
 
+      # rubocop:disable Lint/UselessConstantScoping
       NO_PROJ_ERR = 'Project not found'
       DB_ERR = 'Having trouble accessing the database'
       CLONE_ERR = 'Could not clone this project'
-      TOO_LARGE_ERR = 'Project is too large to clone'
+      TOO_LARGE_ERR = 'Project is too large to analyze'
       NO_FOLDER_ERR = 'Could not find that folder'
+      PROCESSING_MSG = 'Processing the appraisal request; please check back later'
+      # rubocop:enable Lint/UselessConstantScoping
 
-      def retrieve_remote_project(input)
+      def find_project_details(input)
         input[:project] = Repository::For.klass(Entity::Project).find_full_name(
           input[:requested].owner_name, input[:requested].project_name
         )
@@ -34,16 +38,25 @@ def retrieve_remote_project(input)
         Failure(Response::ApiResult.new(status: :internal_error, message: DB_ERR))
       end
 
-      def clone_remote(input)
-        gitrepo = GitRepo.new(input[:project])
-        gitrepo.clone unless gitrepo.exists_locally?
+      def check_project_eligibility(input)
+        if input[:project].too_large?
+          Failure(Response::ApiResult.new(status: :forbidden, message: TOO_LARGE_ERR))
+        else
+          input[:gitrepo] = GitRepo.new(input[:project])
+          Success(input)
+        end
+      end
 
-        Success(input.merge(gitrepo:))
-      rescue GitRepo::Errors::TooLargeToClone
-        App.logger.warn "Project too large: #{input[:project].fullname} (#{input[:project].size} KB)"
-        Failure(Response::ApiResult.new(status: :forbidden, message: TOO_LARGE_ERR))
-      rescue StandardError => error
-        App.logger.error error.backtrace.join("\n")
+      def request_cloning_worker(input)
+        return Success(input) if input[:gitrepo].exists_locally?
+
+        Messaging::Queue
+          .new(App.config.CLONE_QUEUE_URL, App.config)
+          .send(Representer::Project.new(input[:project]).to_json)
+
+        Failure(Response::ApiResult.new(status: :processing, message: PROCESSING_MSG))
+      rescue StandardError => e
+        log_error(e)
         Failure(Response::ApiResult.new(status: :internal_error, message: CLONE_ERR))
       end
 
@@ -54,17 +67,21 @@ def appraise_contributions(input)
         appraisal = Response::ProjectFolderContributions.new(input[:project], input[:folder])
         Success(Response::ApiResult.new(status: :ok, message: appraisal))
       rescue StandardError
-        App.logger.error "Could not find: #{full_request_path(input)}"
+        # App.logger.error "Could not find: #{full_request_path(input)}"
         Failure(Response::ApiResult.new(status: :not_found, message: NO_FOLDER_ERR))
       end
 
-      # Helper methods
+      # Helper methods for steps
 
       def full_request_path(input)
         [input[:requested].owner_name,
          input[:requested].project_name,
          input[:requested].folder_name].join('/')
       end
+
+      def log_error(error)
+        App.logger.error [error.inspect, error.backtrace].flatten.join("\n")
+      end
     end
   end
 end