diff --git a/.github/workflows/api.yml b/.github/workflows/api.yml index 7738f60..791a00d 100644 --- a/.github/workflows/api.yml +++ b/.github/workflows/api.yml @@ -21,6 +21,8 @@ jobs: strategy: # don't cancel other jobs if one fails fail-fast: false + # maximum number of jobs that can run simultaneously + max-parallel: 1 matrix: os: [ubuntu, macos] runs-on: ${{ matrix.os }}-latest # Runs on latest builds of matrix OSes @@ -47,4 +49,10 @@ jobs: DB_FILENAME: ${{ secrets.DB_FILENAME }} REPOSTORE_PATH: ${{ secrets.REPOSTORE_PATH }} API_HOST: ${{ secrets.API_HOST }} - run: bundle exec rake spec + CLONE_QUEUE_URL: ${{ secrets.CLONE_QUEUE_URL }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + rake worker:run:test & + bundle exec rake spec \ No newline at end of file diff --git a/Gemfile b/Gemfile index 705452e..824bd11 100644 --- a/Gemfile +++ b/Gemfile @@ -40,12 +40,18 @@ gem 'dry-types', '~> 1.0' # INFRASTRUCTURE LAYER # Networking gem 'http', '~> 5.0' +gem 'openssl', '~> 3.3.1' # resolves MacoOS CRL error: https://github.com/ruby/openssl/issues/949 # Database gem 'hirb' # gem 'hirb-unicode' # incompatible with new rubocop gem 'sequel', '~> 5.0' +# Asynchronicity +gem 'aws-sdk-sqs', '~> 1.0' +gem 'concurrent-ruby', '~> 1.0' +gem 'shoryuken', '~> 6.0' + group :development, :test do gem 'sqlite3', '~> 1.0' end diff --git a/Gemfile.lock b/Gemfile.lock index dff5a03..aebf117 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,9 +1,24 @@ GEM remote: https://rubygems.org/ specs: - addressable (2.8.7) - public_suffix (>= 2.0.2, < 7.0) + addressable (2.8.8) + public_suffix (>= 2.0.2, < 8.0) ast (2.4.3) + aws-eventstream (1.4.0) + aws-partitions (1.1190.0) + aws-sdk-core (3.239.2) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) + base64 + bigdecimal + jmespath (~> 1, >= 1.6.1) + logger + aws-sdk-sqs (1.107.0) + aws-sdk-core (~> 3, >= 3.239.1) + aws-sigv4 (~> 1.5) + aws-sigv4 (1.12.1) + aws-eventstream (~> 1, >= 1.0.2) base64 (0.3.0) bigdecimal (3.3.1) coderay (1.1.3) @@ -91,7 +106,8 @@ GEM domain_name (~> 0.5) http-form_data (2.3.0) ice_nine (0.11.2) - json (2.16.0) + jmespath (1.6.2) + json (2.17.1) language_server-protocol (3.17.0.5) lint_roller (1.1.0) listen (3.9.0) @@ -106,8 +122,9 @@ GEM minitest (5.26.2) minitest-rg (5.3.0) minitest (~> 5.0) - multi_json (1.17.0) + multi_json (1.18.0) nio4r (2.7.5) + openssl (3.3.2) ostruct (0.6.3) parallel (1.27.0) parser (3.3.10.0) @@ -121,7 +138,7 @@ GEM pry (0.15.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (6.0.2) + public_suffix (7.0.0) puma (6.6.1) nio4r (~> 2.0) racc (1.8.1) @@ -189,9 +206,13 @@ GEM ruby_parser (3.21.1) racc (~> 1.5) sexp_processor (~> 4.16) - sequel (5.98.0) + sequel (5.99.0) bigdecimal sexp_processor (4.17.4) + shoryuken (6.2.1) + aws-sdk-core (>= 2) + concurrent-ruby + thor simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) @@ -220,7 +241,9 @@ PLATFORMS x86_64-linux DEPENDENCIES + aws-sdk-sqs (~> 1.0) base64 + concurrent-ruby (~> 1.0) dry-monads (~> 1.0) dry-struct (~> 1.0) dry-transaction (~> 0) @@ -234,6 +257,7 @@ DEPENDENCIES minitest (~> 5.0) minitest-rg (~> 5.0) multi_json (~> 1.15) + openssl (~> 3.3.1) ostruct (~> 0.0) pg (~> 1.0) pry @@ -253,6 +277,7 @@ DEPENDENCIES rubocop-rake rubocop-sequel sequel (~> 5.0) + shoryuken (~> 6.0) simplecov (~> 0.0) sqlite3 (~> 1.0) vcr (~> 6.0) diff --git a/Procfile b/Procfile index 69286bd..7b12f85 100644 --- a/Procfile +++ b/Procfile @@ -1 +1,3 @@ +release: rake db:migrate; rake queues:create web: bundle exec puma -t 5:5 -p ${PORT:-3000} -e ${RACK_ENV:-development} +worker: bundle exec shoryuken -r ./workers/git_clone_worker.rb -C ./workers/shoryuken.yml \ No newline at end of file diff --git a/Rakefile b/Rakefile index e3109bd..6f4f61c 100644 --- a/Rakefile +++ b/Rakefile @@ -9,6 +9,7 @@ end desc 'Run unit and integration tests' Rake::TestTask.new(:spec) do |t| + puts 'Make sure worker is running in separate process' t.pattern = 'spec/tests/**/*_spec.rb' t.warning = false end @@ -18,11 +19,6 @@ task :respec do sh "rerun -c 'rake spec' --ignore 'coverage/*' --ignore 'repostore/*'" end -desc 'Run the webserver and application and restart if code changes' -task :rerun do - sh "rerun -c --ignore 'coverage/*' --ignore 'repostore/*' -- rake run" -end - desc 'Run web app in default (dev) mode' task run: ['run:dev'] @@ -38,6 +34,11 @@ namespace :run do end end +desc 'Keep restarting web app in dev mode upon changes' +task :rerun do + sh "rerun -c --ignore 'coverage/*' --ignore 'repostore/*' --ignore '_cache/*' -- bundle exec puma -p 9090" +end + namespace :db do task :config do # rubocop:disable Rake/Desc require 'sequel' @@ -155,6 +156,72 @@ namespace :cache do end end +namespace :queues do + task :config do + require 'aws-sdk-sqs' + require_relative 'config/environment' # load config info + @api = CodePraise::App + @sqs = Aws::SQS::Client.new( + access_key_id: @api.config.AWS_ACCESS_KEY_ID, + secret_access_key: @api.config.AWS_SECRET_ACCESS_KEY, + region: @api.config.AWS_REGION + ) + @q_name = @api.config.CLONE_QUEUE + @q_url = @sqs.get_queue_url(queue_name: @q_name).queue_url + + puts "Environment: #{@api.environment}" + end + + desc 'Create SQS queue for worker' + task :create => :config do + @sqs.create_queue(queue_name: @q_name) + + puts 'Queue created:' + puts " Name: #{@q_name}" + puts " Region: #{@api.config.AWS_REGION}" + puts " URL: #{@q_url}" + rescue StandardError => e + puts "Error creating queue: #{e}" + end + + desc 'Report status of queue for worker' + task :status => :config do + puts 'Queue info:' + puts " Name: #{@q_name}" + puts " Region: #{@api.config.AWS_REGION}" + puts " URL: #{@q_url}" + rescue StandardError => e + puts "Error finding queue: #{e}" + end + + desc 'Purge messages in SQS queue for worker' + task :purge => :config do + @sqs.purge_queue(queue_url: @q_url) + puts "Queue #{@q_name} purged" + rescue StandardError => e + puts "Error purging queue: #{e}" + end +end + +namespace :worker do + namespace :run do + desc 'Run the background cloning worker in development mode' + task :dev => :config do + sh 'RACK_ENV=development bundle exec shoryuken -r ./workers/git_clone_worker.rb -C ./workers/shoryuken_dev.yml' + end + + desc 'Run the background cloning worker in testing mode' + task :test => :config do + sh 'RACK_ENV=test bundle exec shoryuken -r ./workers/git_clone_worker.rb -C ./workers/shoryuken_test.yml' + end + + desc 'Run the background cloning worker in production mode' + task :production => :config do + sh 'RACK_ENV=production bundle exec shoryuken -r ./workers/git_clone_worker.rb -C ./workers/shoryuken.yml' + end + end +end + desc 'Run application console' task :console do sh 'pry -r ./load_all' diff --git a/app/application/controllers/app.rb b/app/application/controllers/app.rb index e269360..fe2e01f 100644 --- a/app/application/controllers/app.rb +++ b/app/application/controllers/app.rb @@ -32,7 +32,9 @@ class App < Roda routing.on String, String do |owner_name, project_name| # GET /projects/{owner_name}/{project_name}[/folder_namepath/] routing.get do - response.cache_control public: true, max_age: 120 + App.configure :production do + response.cache_control public: true, max_age: 300 + end path_request = Request::ProjectPath.new( owner_name, project_name, request diff --git a/app/application/services/appraise_project.rb b/app/application/services/appraise_project.rb index f47bbd3..9dfc583 100644 --- a/app/application/services/appraise_project.rb +++ b/app/application/services/appraise_project.rb @@ -8,19 +8,23 @@ module Service class AppraiseProject include Dry::Transaction - step :retrieve_remote_project - step :clone_remote + step :find_project_details + step :check_project_eligibility + step :request_cloning_worker step :appraise_contributions private + # rubocop:disable Lint/UselessConstantScoping NO_PROJ_ERR = 'Project not found' DB_ERR = 'Having trouble accessing the database' CLONE_ERR = 'Could not clone this project' - TOO_LARGE_ERR = 'Project is too large to clone' + TOO_LARGE_ERR = 'Project is too large to analyze' NO_FOLDER_ERR = 'Could not find that folder' + PROCESSING_MSG = 'Processing the appraisal request; please check back later' + # rubocop:enable Lint/UselessConstantScoping - def retrieve_remote_project(input) + def find_project_details(input) input[:project] = Repository::For.klass(Entity::Project).find_full_name( input[:requested].owner_name, input[:requested].project_name ) @@ -34,16 +38,25 @@ def retrieve_remote_project(input) Failure(Response::ApiResult.new(status: :internal_error, message: DB_ERR)) end - def clone_remote(input) - gitrepo = GitRepo.new(input[:project]) - gitrepo.clone unless gitrepo.exists_locally? + def check_project_eligibility(input) + if input[:project].too_large? + Failure(Response::ApiResult.new(status: :forbidden, message: TOO_LARGE_ERR)) + else + input[:gitrepo] = GitRepo.new(input[:project]) + Success(input) + end + end - Success(input.merge(gitrepo:)) - rescue GitRepo::Errors::TooLargeToClone - App.logger.warn "Project too large: #{input[:project].fullname} (#{input[:project].size} KB)" - Failure(Response::ApiResult.new(status: :forbidden, message: TOO_LARGE_ERR)) - rescue StandardError => error - App.logger.error error.backtrace.join("\n") + def request_cloning_worker(input) + return Success(input) if input[:gitrepo].exists_locally? + + Messaging::Queue + .new(App.config.CLONE_QUEUE_URL, App.config) + .send(Representer::Project.new(input[:project]).to_json) + + Failure(Response::ApiResult.new(status: :processing, message: PROCESSING_MSG)) + rescue StandardError => e + log_error(e) Failure(Response::ApiResult.new(status: :internal_error, message: CLONE_ERR)) end @@ -54,17 +67,21 @@ def appraise_contributions(input) appraisal = Response::ProjectFolderContributions.new(input[:project], input[:folder]) Success(Response::ApiResult.new(status: :ok, message: appraisal)) rescue StandardError - App.logger.error "Could not find: #{full_request_path(input)}" + # App.logger.error "Could not find: #{full_request_path(input)}" Failure(Response::ApiResult.new(status: :not_found, message: NO_FOLDER_ERR)) end - # Helper methods + # Helper methods for steps def full_request_path(input) [input[:requested].owner_name, input[:requested].project_name, input[:requested].folder_name].join('/') end + + def log_error(error) + App.logger.error [error.inspect, error.backtrace].flatten.join("\n") + end end end end diff --git a/app/infrastructure/git/repositories/blame_reporter.rb b/app/infrastructure/git/repositories/blame_reporter.rb index 4c6fa67..0ad8601 100644 --- a/app/infrastructure/git/repositories/blame_reporter.rb +++ b/app/infrastructure/git/repositories/blame_reporter.rb @@ -8,6 +8,8 @@ module CodePraise module Git # Git contributions report parsing and reporting services class BlameReporter + NOT_FOUND_ERROR_MSG = 'Folder not found' + def initialize(gitrepo, folder_name) @local = gitrepo.local @folder_name = folder_name @@ -15,13 +17,11 @@ def initialize(gitrepo, folder_name) end def folder_report - raise('no files found in folder') if files.empty? + raise not_found_error unless folder_exists? - @local.in_repo do - files.map do |filename| - [filename, RepoFile.new(filename).blame] - end - end + filenames = @local.files.select { _1.start_with? @folder_name } + + @local.in_repo { analyze_files_concurrently(filenames) } end def files @@ -32,6 +32,40 @@ def files @local.files.select { |file| file.start_with? "#{@folder_name}/" } end end + + def folder_structure + @local.folder_structure + end + + def file_report(filename) + Git::RepoFile.new(filename).blame + end + + private + + def folder_exists? + return true if @folder_name.empty? + + @local.in_repo { Dir.exist? @folder_name } + end + + def not_found_error + "#{NOT_FOUND_ERROR_MSG} (#{@folder_name})" + end + + # synchronous reporting of a list of files + def analyze_files(filenames) + filenames.map { |fname| [fname, file_report(fname)] } + end + + # asynchronous reporting of a list of files + def analyze_files_concurrently(filenames) + filenames.map do |fname| + Concurrent::Promise + .execute { file_report(fname) } + .then { |freport| [fname, freport] } + end.map(&:value) + end end end end diff --git a/app/infrastructure/git/repositories/git_repo.rb b/app/infrastructure/git/repositories/git_repo.rb index a95ec39..6406d3e 100644 --- a/app/infrastructure/git/repositories/git_repo.rb +++ b/app/infrastructure/git/repositories/git_repo.rb @@ -9,17 +9,16 @@ class Errors CannotOverwriteLocalGitRepo = Class.new(StandardError) end - def initialize(repo, config = CodePraise::App.config) - @repo = repo - remote = Git::RemoteGitRepo.new(@repo.http_url) - @local = Git::LocalGitRepo.new(remote, config.REPOSTORE_PATH) + def initialize(project, config = CodePraise::App.config) + @project = project + remote = Git::RemoteGitRepo.new(@project.http_url) + @local = Git::LocalGitRepo.new(remote) end def local exists_locally? ? @local : raise(Errors::NoGitRepoFound) end - # Deliberately :reek:MissingSafeMethod for file system changes def delete @local.delete end @@ -28,9 +27,8 @@ def exists_locally? @local.exists? end - # Deliberately :reek:MissingSafeMethod for file system changes def clone - raise Errors::TooLargeToClone if @repo.too_large? + raise Errors::TooLargeToClone if @project.too_large? raise Errors::CannotOverwriteLocalGitRepo if exists_locally? @local.clone_remote { |line| yield line if block_given? } diff --git a/app/infrastructure/git/repositories/local_repo.rb b/app/infrastructure/git/repositories/local_repo.rb index ab1e94d..48af88a 100644 --- a/app/infrastructure/git/repositories/local_repo.rb +++ b/app/infrastructure/git/repositories/local_repo.rb @@ -16,9 +16,9 @@ class LocalGitRepo attr_reader :git_repo_path - def initialize(remote, repostore_path) + def initialize(remote) @remote = remote - @git_repo_path = [repostore_path, @remote.unique_id].join('/') + @git_repo_path = [ENV['REPOSTORE_PATH'], @remote.unique_id].join('/') end def clone_remote @@ -45,20 +45,19 @@ def exists? Dir.exist? @git_repo_path end - # Deliberately :reek:MissingSafeMethod delete def delete FileUtils.rm_rf(@git_repo_path) end + def wipe + FileUtils.rm_rf @git_repo_path + end + private def raise_unless_setup raise Errors::InvalidLocalGitRepo unless exists? end - - def wipe - FileUtils.rm_rf @git_repo_path - end end end end diff --git a/app/infrastructure/git/repositories/repo_store.rb b/app/infrastructure/git/repositories/repo_store.rb new file mode 100644 index 0000000..6ec2f4c --- /dev/null +++ b/app/infrastructure/git/repositories/repo_store.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module CodePraise + module Repository + # Collection of all local git repo clones + class RepoStore + def self.all_repos + Dir.glob(App.config.REPOSTORE_PATH + '/*') + .select { File.directory?(_1) } + end + + def self.wipe + all_repos.each { |dir| FileUtils.rm_r(dir) } + end + end + end +end diff --git a/app/infrastructure/messaging/queue.rb b/app/infrastructure/messaging/queue.rb new file mode 100644 index 0000000..8980d1b --- /dev/null +++ b/app/infrastructure/messaging/queue.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'aws-sdk-sqs' + +module CodePraise + module Messaging + ## Queue wrapper for AWS SQS + # Requires: AWS credentials loaded in ENV or through config file + class Queue + IDLE_TIMEOUT = 5 # seconds + + def initialize(queue_url, config) + @queue_url = queue_url + sqs = Aws::SQS::Client.new( + access_key_id: config.AWS_ACCESS_KEY_ID, + secret_access_key: config.AWS_SECRET_ACCESS_KEY, + region: config.AWS_REGION + ) + @queue = Aws::SQS::Queue.new(url: queue_url, client: sqs) + end + + ## Sends message to queue + # Usage: + # q = Messaging::Queue.new(App.config.CLONE_QUEUE_URL) + # q.send({data: "hello"}.to_json) + def send(message) + @queue.send_message(message_body: message) + end + + ## Polls queue, yielding each messge + # Usage: + # q = Messaging::Queue.new(App.config.CLONE_QUEUE_URL) + # q.poll { |msg| print msg.body.to_s } + def poll + poller = Aws::SQS::QueuePoller.new(@queue_url) + poller.poll(idle_timeout: IDLE_TIMEOUT) do |msg| + yield msg.body if block_given? + end + end + end + end +end diff --git a/app/presentation/representers/http_response_representer.rb b/app/presentation/representers/http_response_representer.rb index 08064dd..3af1990 100644 --- a/app/presentation/representers/http_response_representer.rb +++ b/app/presentation/representers/http_response_representer.rb @@ -20,7 +20,6 @@ class HttpResponse < Roar::Decorator ok: 200, created: 201, processing: 202, - no_content: 204, forbidden: 403, not_found: 404, diff --git a/app/presentation/responses/api_result.rb b/app/presentation/responses/api_result.rb index 6a273e4..7d4b7f9 100644 --- a/app/presentation/responses/api_result.rb +++ b/app/presentation/responses/api_result.rb @@ -3,7 +3,7 @@ module CodePraise module Response SUCCESS = Set.new( - %i[ok created processing no_content] + %i[ok created processing] ).freeze FAILURE = Set.new( diff --git a/config/secrets_example.yml b/config/secrets_example.yml index 04a7646..2b3725a 100644 --- a/config/secrets_example.yml +++ b/config/secrets_example.yml @@ -8,6 +8,11 @@ development: LOCAL_CACHE: _cache/rack API_HOST: http://localhost:9090 REDISCLOUD_URL: url-assigned-by-Redis-provider-on-Heroku + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: + AWS_REGION: + CLONE_QUEUE: codepraise-clone-development + CLONE_QUEUE_URL: app_test: DB_FILENAME: db/local/test.db @@ -15,6 +20,11 @@ app_test: LOCAL_CACHE: _cache/rack API_HOST: http://localhost:9090 REDISCLOUD_URL: url-assigned-by-Redis-provider-on-Heroku + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: + AWS_REGION: + CLONE_QUEUE: codepraise-clone-development + CLONE_QUEUE_URL: test: DB_FILENAME: db/local/test.db @@ -23,6 +33,11 @@ test: LOCAL_CACHE: _cache/rack API_HOST: http://localhost:9090 REDISCLOUD_URL: url-assigned-by-Redis-provider-on-Heroku + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: + AWS_REGION: + CLONE_QUEUE: codepraise-clone-development + CLONE_QUEUE_URL: production: # - assign DATABASE_URL in production @@ -30,3 +45,8 @@ production: REPOSTORE_PATH: repostore API_HOST: https://codepraise2022-api REDISCLOUD_URL: url-assigned-by-Redis-provider-on-Heroku + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: + AWS_REGION: + CLONE_QUEUE: codepraise-clone-production + CLONE_QUEUE_URL: diff --git a/coverage/.resultset.json b/coverage/.resultset.json index 166a4d1..6c23b72 100644 --- a/coverage/.resultset.json +++ b/coverage/.resultset.json @@ -16,7 +16,7 @@ 1, null, 1, - 56, + 58, null, null ] @@ -243,7 +243,7 @@ null, null, 1, - 6581, + 6496, null, null, 1, @@ -690,22 +690,22 @@ 1, null, 1, - 836, - 836, + 760, + 760, null, null, 1, - 13036, + 12887, null, null, 1, - 6527, + 6451, null, - 6509, + 6436, null, null, 1, - 456, + 380, null, null, 1, @@ -719,9 +719,9 @@ 1, null, 1, - 836, - 836, - 836, + 760, + 760, + 760, null, null, null, @@ -782,11 +782,11 @@ 1, null, 1, - 5, + 7, null, null, 1, - 3, + 9, null, null, 1, @@ -927,7 +927,7 @@ null, null, 1, - 25, + 28, null, null, 1, @@ -955,9 +955,9 @@ null, null, 1, - 92, + 104, null, - 92, + 104, null, null, null, @@ -966,8 +966,8 @@ null, null, 1, - 23, - 69, + 26, + 78, null, null, null, @@ -998,12 +998,12 @@ null, null, null, - 24, + 27, null, null, null, null, - 24, + 27, null, null, 1, @@ -1035,9 +1035,9 @@ null, null, 1, - 52, + 55, null, - 23, + 26, null, null, null, @@ -1083,16 +1083,16 @@ 1, null, 1, - 194, - 194, - 194, - 194, + 193, + 193, + 193, + 193, null, null, 1, - 3, - 3, - 3, + 2, + 2, + 2, null, null, 1, @@ -1107,21 +1107,21 @@ null, null, 1, - 3, - 3, + 2, + 2, null, null, 1, - 3, - 3, + 2, + 2, null, null, 1, - 388, + 386, null, null, 1, - 194, + 193, null, null, null, @@ -1132,8 +1132,8 @@ null, null, 1, - 2, - 10, + 1, + 5, null, null, null, @@ -1386,6 +1386,8 @@ null, 1, 1, + null, + 1, 6, 6, 6, @@ -1394,19 +1396,51 @@ 1, 6, null, - 5, - 5, - 190, + 310, null, + 10, + null, + null, + 1, + 0, + 0, + 0, + null, + 0, null, null, null, 1, - 11, + 0, + null, + null, + 1, + 190, + null, + null, + 1, + null, + 1, 6, - 3, null, - 186, + 6, + null, + null, + 1, + 1, + null, + null, + null, + 1, + 0, + null, + null, + null, + 1, + 5, + 190, + 190, + 190, null, null, null, @@ -1453,30 +1487,28 @@ null, null, 1, - 8, - 8, - 8, + 11, + 11, + 11, null, null, 1, 6, null, null, - null, - 1, 1, + 0, null, null, 1, - 16, - null, + 18, null, null, 1, - 3, - 2, + 1, + 1, null, - 12, + 6, null, null, null @@ -1503,47 +1535,46 @@ 1, null, 1, - 8, - 8, + 11, + 11, null, null, 1, - 12, - 2, + 6, + 1, null, null, 1, - 6, + 5, null, - 6, - 6, - 552, + 5, + 5, + 460, null, null, null, null, 1, - 11, - 22, + 13, + 26, null, null, 1, - 33, + 36, null, null, - null, - 1, 1, + 0, null, null, 1, + 0, null, - 1, - 17, null, + 1, null, 1, - 0, + 18, null, null, null, @@ -1568,15 +1599,15 @@ 1, null, 1, - 8, + 11, null, null, 1, - 8, + 11, null, null, 1, - 2, + 1, null, null, null, @@ -1587,6 +1618,27 @@ null ] }, + "/Users/soumyaray/Sync/Dropbox/ossdev/classes/SOA-class/projects/soa2025/api-codepraise-2025/app/infrastructure/git/repositories/repo_store.rb": { + "lines": [ + null, + null, + 1, + 1, + null, + 1, + 1, + 11, + 16, + null, + null, + 1, + 16, + null, + null, + null, + null + ] + }, "/Users/soumyaray/Sync/Dropbox/ossdev/classes/SOA-class/projects/soa2025/api-codepraise-2025/app/infrastructure/github/gateways/github_api.rb": { "lines": [ null, @@ -1805,6 +1857,52 @@ null ] }, + "/Users/soumyaray/Sync/Dropbox/ossdev/classes/SOA-class/projects/soa2025/api-codepraise-2025/app/infrastructure/messaging/queue.rb": { + "lines": [ + null, + null, + 1, + null, + 1, + 1, + null, + null, + 1, + 1, + null, + 1, + 3, + 3, + null, + null, + null, + null, + 3, + null, + null, + null, + null, + null, + null, + 1, + 3, + null, + null, + null, + null, + null, + null, + 1, + 0, + 0, + 0, + null, + null, + null, + null, + null + ] + }, "/Users/soumyaray/Sync/Dropbox/ossdev/classes/SOA-class/projects/soa2025/api-codepraise-2025/app/presentation/representers/contributor_representer.rb": { "lines": [ null, @@ -1994,9 +2092,8 @@ null, null, null, - null, 1, - 11, + 14, null, null, null, @@ -2096,17 +2193,17 @@ 1, null, 1, - 4, + 7, null, null, 1, null, 1, - 4, + 7, null, null, 1, - 4, + 7, null, null, null, @@ -2156,9 +2253,9 @@ null, 1, 1, - 29, + 32, null, - 28, + 31, null, null, null, @@ -2206,10 +2303,10 @@ null, null, 1, - 11, + 14, null, null, - 11, + 14, 1, null, 1, @@ -2220,22 +2317,24 @@ 1, null, null, + 13, + 13, + 13, + null, 10, - 10, - 10, + 8, + 0, null, - 7, - 5, null, - 5, + 8, null, null, null, - 5, + 8, null, - 5, - 3, - 3, + 8, + 6, + 6, null, null, 2, @@ -2376,16 +2475,16 @@ null, 1, 1, - 5, - 5, - 5, - 5, + 8, + 8, + 8, + 8, null, null, 1, null, 1, - 4, + 3, null, null, 1, @@ -2475,22 +2574,26 @@ 1, 1, 1, + 1, null, 1, null, + null, 1, 1, 1, 1, 1, + 1, + null, null, 1, - 7, + 10, null, null, null, - 7, - 5, + 10, + 8, null, 2, null, @@ -2499,13 +2602,22 @@ null, null, 1, - 5, - 5, + 8, + 1, + null, + 7, + 7, + null, null, - 4, null, 1, - 1, + 7, + null, + 3, + null, + null, + null, + 3, null, 0, 0, @@ -2518,18 +2630,22 @@ 3, 3, null, - 1, + null, 1, null, null, null, null, 1, - 1, + 0, null, null, null, null, + 1, + 0, + null, + null, null, null ] @@ -2595,14 +2711,16 @@ 7, 7, 7, + 7, + 7, null, null, null, null, 1, 30, - 2015, - 2015, + 2080, + 2080, null, null, 30, @@ -2978,6 +3096,8 @@ null, null, 1, + 1, + 1, null, null, 1, @@ -3273,6 +3393,6 @@ ] } }, - "timestamp": 1764473876 + "timestamp": 1764939916 } } diff --git a/spec/acceptance_tests b/spec/acceptance_tests index 1817c86..5c77fc1 100755 --- a/spec/acceptance_tests +++ b/spec/acceptance_tests @@ -4,7 +4,7 @@ trap "kill 0" EXIT # Run application server as backround process (using '&') # see: https://kb.iu.edu/d/afnz -RACK_ENV=test bundle exec puma -p 9000 & +rake worker:run:test & # Run acceptance tests on browser -rake spec:acceptance +bundle exec rake spec diff --git a/spec/helpers/vcr_helper.rb b/spec/helpers/vcr_helper.rb index b685e1c..aac161d 100644 --- a/spec/helpers/vcr_helper.rb +++ b/spec/helpers/vcr_helper.rb @@ -13,6 +13,8 @@ def self.setup_vcr vcr_config.cassette_library_dir = CASSETTES_FOLDER vcr_config.hook_into :webmock vcr_config.ignore_localhost = true # for acceptance tests + vcr_config.ignore_hosts 'sqs.us-east-1.amazonaws.com' + vcr_config.ignore_hosts 'sqs.ap-northeast-1.amazonaws.com' end end diff --git a/spec/tests/acceptance/api_spec.rb b/spec/tests/acceptance/api_spec.rb index edf2116..94d3f5d 100644 --- a/spec/tests/acceptance/api_spec.rb +++ b/spec/tests/acceptance/api_spec.rb @@ -17,6 +17,7 @@ def app before do VcrHelper.configure_vcr_for_github DatabaseHelper.wipe_database + CodePraise::Repository::RepoStore.wipe end after do @@ -35,11 +36,16 @@ def app end describe 'Appraise project folder route' do - it 'should be able to appraise a project folder' do + it 'should be able to appraise the root folder' do CodePraise::Service::AddProject.new.call( owner_name: USERNAME, project_name: PROJECT_NAME ) + get "/api/v1/projects/#{USERNAME}/#{PROJECT_NAME}" + _(last_response.status).must_equal 202 + + 5.times { sleep(1); print('_') } + get "/api/v1/projects/#{USERNAME}/#{PROJECT_NAME}" _(last_response.status).must_equal 200 appraisal = JSON.parse last_response.body @@ -58,6 +64,11 @@ def app owner_name: USERNAME, project_name: PROJECT_NAME ) + get "/api/v1/projects/#{USERNAME}/#{PROJECT_NAME}/spec" + _(last_response.status).must_equal 202 + + 5.times { sleep(1); print('_') } + get "/api/v1/projects/#{USERNAME}/#{PROJECT_NAME}/spec" _(last_response.status).must_equal 200 appraisal = JSON.parse last_response.body @@ -76,6 +87,11 @@ def app owner_name: USERNAME, project_name: PROJECT_NAME ) + get "/api/v1/projects/#{USERNAME}/#{PROJECT_NAME}/foobar" + _(last_response.status).must_equal 202 + + 5.times { sleep(1); print('_') } + get "/api/v1/projects/#{USERNAME}/#{PROJECT_NAME}/foobar" _(last_response.status).must_equal 404 _(JSON.parse(last_response.body)['status']).must_include 'not' diff --git a/spec/tests/integration/services/appraise_project_spec.rb b/spec/tests/integration/services/appraise_project_spec.rb index d4bac04..b4fc299 100644 --- a/spec/tests/integration/services/appraise_project_spec.rb +++ b/spec/tests/integration/services/appraise_project_spec.rb @@ -28,6 +28,8 @@ .new(GITHUB_TOKEN) .find(USERNAME, PROJECT_NAME) CodePraise::Repository::For.entity(gh_project).create(gh_project) + gitrepo = CodePraise::GitRepo.new(gh_project) + gitrepo.clone unless gitrepo.exists_locally? # WHEN: we request to appraise the project request = OpenStruct.new( diff --git a/workers/git_clone_worker.rb b/workers/git_clone_worker.rb new file mode 100644 index 0000000..bd895e5 --- /dev/null +++ b/workers/git_clone_worker.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require_relative '../require_app' +require_app + +require 'figaro' +require 'shoryuken' + +# Shoryuken worker class to clone repos in parallel +class GitCloneWorker + # Environment variables setup + Figaro.application = Figaro::Application.new( + environment: ENV['RACK_ENV'] || 'development', + path: File.expand_path('config/secrets.yml') + ) + Figaro.load + def self.config = Figaro.env + + Shoryuken.sqs_client = Aws::SQS::Client.new( + access_key_id: config.AWS_ACCESS_KEY_ID, + secret_access_key: config.AWS_SECRET_ACCESS_KEY, + region: config.AWS_REGION + ) + + include Shoryuken::Worker + + shoryuken_options queue: config.CLONE_QUEUE_URL, auto_delete: true + + def perform(_sqs_msg, request) + project = CodePraise::Representer::Project + .new(OpenStruct.new).from_json(request) + CodePraise::GitRepo.new(project).clone + rescue CodePraise::GitRepo::Errors::CannotOverwriteLocalGitRepo + puts 'CLONE EXISTS -- ignoring request' + end +end diff --git a/workers/shoryuken.yml b/workers/shoryuken.yml new file mode 100644 index 0000000..2a460f0 --- /dev/null +++ b/workers/shoryuken.yml @@ -0,0 +1,2 @@ +queues: + - https://sqs.us-east-1.amazonaws.com/503315808870/soa-codepraise-clone-production diff --git a/workers/shoryuken_dev.yml b/workers/shoryuken_dev.yml new file mode 100644 index 0000000..35848dd --- /dev/null +++ b/workers/shoryuken_dev.yml @@ -0,0 +1,2 @@ +queues: + - https://sqs.ap-northeast-1.amazonaws.com/503315808870/soa-codepraise-clone-dev diff --git a/workers/shoryuken_test.yml b/workers/shoryuken_test.yml new file mode 100644 index 0000000..be9aa3c --- /dev/null +++ b/workers/shoryuken_test.yml @@ -0,0 +1,2 @@ +queues: + - https://sqs.ap-northeast-1.amazonaws.com/503315808870/soa-codepraise-clone-test