From 8ff7a09ac3215bc5a6592160d21e37289cc8a480 Mon Sep 17 00:00:00 2001 From: Soumya Ray Date: Mon, 29 Nov 2021 14:04:34 +0800 Subject: [PATCH 1/4] Github CI workflow for API - New workflow in .github/workflows/ --- .github/workflows/api.yml | 50 +++++++++++++++++++ Gemfile | 1 + Gemfile.lock | 48 +++++++----------- Rakefile | 8 +-- .../git/repositories/blame_reporter.rb | 2 +- .../representers/project_representer.rb | 2 +- config/secrets_example.yml | 8 +-- 7 files changed, 77 insertions(+), 42 deletions(-) create mode 100644 .github/workflows/api.yml diff --git a/.github/workflows/api.yml b/.github/workflows/api.yml new file mode 100644 index 0000000..7738f60 --- /dev/null +++ b/.github/workflows/api.yml @@ -0,0 +1,50 @@ +# API testing workflow config for CI/CD on Github + +name: API continuous integration + +# Controls when the action will run. +on: + # Triggers the workflow for main branch on git push + push: + branches: [main] + # Triggers workflow for or any branch on pull requests + pull_request: + branches: + - '**' + # Allows running this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow is defined of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "api_test" + api_test: + strategy: + # don't cancel other jobs if one fails + fail-fast: false + matrix: + os: [ubuntu, macos] + runs-on: ${{ matrix.os }}-latest # Runs on latest builds of matrix OSes + env: + BUNDLE_WITHOUT: production # skip installing production gem (pg) + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v4 + + # Builds on a predefined action that has Ruby installed + - uses: ruby/setup-ruby@v1 + with: + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + + - name: Setup test database + env: # define secrets in Github project -> settings -> secrets + DB_FILENAME: ${{ secrets.DB_FILENAME }} + run: RACK_ENV=test bundle exec rake db:migrate + + - name: Run all specs + env: # define secrets in Github project -> settings -> secrets + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + DB_FILENAME: ${{ secrets.DB_FILENAME }} + REPOSTORE_PATH: ${{ secrets.REPOSTORE_PATH }} + API_HOST: ${{ secrets.API_HOST }} + run: bundle exec rake spec diff --git a/Gemfile b/Gemfile index 4717af9..dea31b5 100644 --- a/Gemfile +++ b/Gemfile @@ -11,6 +11,7 @@ gem 'rake', '~> 13.0' # PRESENTATION LAYER gem 'multi_json', '~> 1.15' +gem 'ostruct', '~> 0.0' gem 'roar', '~> 1.1' # APPLICATION LAYER diff --git a/Gemfile.lock b/Gemfile.lock index b67e962..f49f166 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -68,8 +68,6 @@ GEM dry-initializer (~> 3.2) dry-schema (~> 1.14) zeitwerk (~> 2.6) - faker (3.5.2) - i18n (>= 1.8.11, < 2) ffi (1.17.2) ffi (1.17.2-arm64-darwin) ffi (1.17.2-x86_64-linux-gnu) @@ -83,7 +81,6 @@ GEM ruby_parser (~> 3.1, > 3.1.0) sexp_processor (~> 4.8) hashdiff (1.2.1) - headless (2.3.1) hirb (0.7.3) http (5.3.1) addressable (~> 2.8) @@ -93,8 +90,6 @@ GEM http-cookie (1.1.0) domain_name (~> 0.5) http-form_data (2.3.0) - i18n (1.14.7) - concurrent-ruby (~> 1.0) ice_nine (0.11.2) json (2.16.0) language_server-protocol (3.17.0.5) @@ -108,15 +103,12 @@ GEM logger (1.7.0) method_source (1.1.0) mini_portile2 (2.8.9) - minitest (5.26.1) + minitest (5.26.2) minitest-rg (5.3.0) minitest (~> 5.0) + multi_json (1.17.0) nio4r (2.7.5) - page-object (2.3.1) - page_navigation (>= 0.10) - watir (>= 6.10.3) - page_navigation (0.10) - data_magic (>= 0.22) + ostruct (0.6.3) parallel (1.27.0) parser (3.3.10.0) ast (~> 2.4.1) @@ -136,7 +128,7 @@ GEM rack (3.2.4) rack-session (0.3.0) rack (>= 3.0.0.beta1) - rack-test (2.1.0) + rack-test (2.2.0) rack (>= 1.3) rainbow (3.1.1) rake (13.3.1) @@ -150,10 +142,16 @@ GEM rainbow (>= 2.0, < 4.0) rexml (~> 3.1) regexp_parser (2.11.3) + representable (3.2.0) + declarative (< 0.1.0) + trailblazer-option (>= 0.1.1, < 0.2.0) + uber (< 0.2.0) rerun (0.14.0) listen (~> 3.0) rexml (3.4.4) - roda (3.97.0) + roar (1.2.0) + representable (~> 3.1) + roda (3.98.0) rack rubocop (1.81.7) json (~> 2.3) @@ -183,13 +181,6 @@ GEM ruby_parser (3.21.1) racc (~> 1.5) sexp_processor (~> 4.16) - rubyzip (3.2.2) - selenium-webdriver (4.38.0) - base64 (~> 0.2) - logger (~> 1.4) - rexml (~> 3.2, >= 3.2.5) - rubyzip (>= 1.2.2, < 4.0) - websocket (~> 1.0) sequel (5.98.0) bigdecimal sexp_processor (4.17.4) @@ -201,23 +192,18 @@ GEM simplecov_json_formatter (0.1.4) sqlite3 (1.7.3) mini_portile2 (~> 2.8.0) - temple (0.10.4) thor (1.4.0) - tilt (2.6.1) + trailblazer-option (0.1.2) + uber (0.1.0) unicode-display_width (3.2.0) unicode-emoji (~> 4.1) unicode-emoji (4.1.0) vcr (6.3.1) base64 - watir (7.3.0) - regexp_parser (>= 1.2, < 3) - selenium-webdriver (~> 4.2) webmock (3.26.1) addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) - websocket (1.2.11) - yml_reader (0.7) zeitwerk (2.7.3) PLATFORMS @@ -237,13 +223,15 @@ DEPENDENCIES hirb http (~> 5.0) logger (~> 1.0) - minitest (~> 5.20) - minitest-rg (~> 5.2) - page-object (~> 2.0) + minitest (~> 5.0) + minitest-rg (~> 5.0) + multi_json (~> 1.15) + ostruct (~> 0.0) pg (~> 1.0) pry puma (~> 6.0) rack-session (~> 0) + rack-test rake (~> 13.0) reek rerun diff --git a/Rakefile b/Rakefile index 2b00894..dbd04ba 100644 --- a/Rakefile +++ b/Rakefile @@ -20,16 +20,16 @@ end desc 'Run the webserver and application and restart if code changes' task :rerun do - sh "rerun -c --ignore 'coverage/*' --ignore 'repostore/*' -- bundle exec puma" + sh "rerun -c --ignore 'coverage/*' --ignore 'repostore/*' -- rake run" end desc 'Run web app in default (dev) mode' -task run: ['run:default'] +task run: ['run:dev'] namespace :run do desc 'Run API in dev mode' - task :default do - sh 'rerun -c "bundle exec puma -p 9090"' + task :dev do + sh 'bundle exec puma -p 9090' end desc 'Run API in test mode' diff --git a/app/infrastructure/git/repositories/blame_reporter.rb b/app/infrastructure/git/repositories/blame_reporter.rb index 65448eb..4c6fa67 100644 --- a/app/infrastructure/git/repositories/blame_reporter.rb +++ b/app/infrastructure/git/repositories/blame_reporter.rb @@ -29,7 +29,7 @@ def files if @folder_name.empty? @local.files else - @local.files.select { _1.start_with?("#{@folder_name}/") } + @local.files.select { |file| file.start_with? "#{@folder_name}/" } end end end diff --git a/app/presentation/representers/project_representer.rb b/app/presentation/representers/project_representer.rb index 054f7d4..97d00d0 100644 --- a/app/presentation/representers/project_representer.rb +++ b/app/presentation/representers/project_representer.rb @@ -25,7 +25,7 @@ class Project < Roar::Decorator collection :contributors, extend: Representer::Member, class: OpenStruct link :self do - "#{App.config.API_HOST}/api/v1/projects/#{project_name}/#{owner_name}" + "#{App.config.API_HOST}/projects/#{project_name}/#{owner_name}" end private diff --git a/config/secrets_example.yml b/config/secrets_example.yml index 749fc16..81e1fc2 100644 --- a/config/secrets_example.yml +++ b/config/secrets_example.yml @@ -2,27 +2,23 @@ # Name your file secrets.yml development: - # Infrastructure DB_FILENAME: db/local/dev.db GITHUB_TOKEN: create_github_token REPOSTORE_PATH: repostore API_HOST: http://localhost:9090 app_test: - # Infrastructure - DB_FILENAME: app/infrastructure/database/local/test.db - REPOSTORE_PATH: app/infrastructure/git/repostore + DB_FILENAME: db/local/test.db + REPOSTORE_PATH: repostore API_HOST: http://localhost:9090 test: - # Infrastructure DB_FILENAME: db/local/test.db GITHUB_TOKEN: create_github_token REPOSTORE_PATH: repostore API_HOST: http://localhost:9090 production: - # Infrastructure # - assign DATABASE_URL in production GITHUB_TOKEN: create_github_token REPOSTORE_PATH: repostore From 300ebcf03c670f5217ba91d47b1deca7a96b8f09 Mon Sep 17 00:00:00 2001 From: Soumya Ray Date: Mon, 6 Dec 2021 08:03:57 +0800 Subject: [PATCH 2/4] reverse-proxy caching of project appraisal --- .gitignore | 1 + Gemfile | 5 ++ Gemfile.lock | 11 +++++ Rakefile | 45 ++++++++++++++++++ app/application/controllers/app.rb | 10 ++-- app/domain/projects/entities/project.rb | 2 +- app/infrastructure/cache/redis_cache.rb | 22 +++++++++ config/environment.rb | 25 +++++++++- config/secrets_example.yml | 4 ++ coverage/.resultset.json | 63 ++++++++++++++++++++++--- 10 files changed, 174 insertions(+), 14 deletions(-) create mode 100644 app/infrastructure/cache/redis_cache.rb diff --git a/.gitignore b/.gitignore index e40d924..8ecbd72 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ coverage/* !coverage/.resultset.json *.db repostore/**/ +_cache/ \ No newline at end of file diff --git a/Gemfile b/Gemfile index dea31b5..705452e 100644 --- a/Gemfile +++ b/Gemfile @@ -27,6 +27,11 @@ gem 'dry-monads', '~> 1.0' gem 'dry-transaction', '~> 0' gem 'dry-validation', '~> 1.0' +# Caching +gem 'rack-cache', '~> 1.13' +gem 'redis', '~> 4.8' +gem 'redis-rack-cache', '~> 2.2' + # DOMAIN LAYER # Validation gem 'dry-struct', '~> 1.0' diff --git a/Gemfile.lock b/Gemfile.lock index f49f166..dff5a03 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -126,6 +126,8 @@ GEM nio4r (~> 2.0) racc (1.8.1) rack (3.2.4) + rack-cache (1.17.0) + rack (>= 0.4) rack-session (0.3.0) rack (>= 3.0.0.beta1) rack-test (2.2.0) @@ -135,6 +137,12 @@ GEM rb-fsevent (0.11.2) rb-inotify (0.11.1) ffi (~> 1.0) + redis (4.8.1) + redis-rack-cache (2.2.1) + rack-cache (>= 1.10, < 2) + redis-store (>= 1.6, < 2) + redis-store (1.11.0) + redis (>= 4, < 6) reek (6.5.0) dry-schema (~> 1.13) logger (~> 1.6) @@ -230,9 +238,12 @@ DEPENDENCIES pg (~> 1.0) pry puma (~> 6.0) + rack-cache (~> 1.13) rack-session (~> 0) rack-test rake (~> 13.0) + redis (~> 4.8) + redis-rack-cache (~> 2.2) reek rerun roar (~> 1.1) diff --git a/Rakefile b/Rakefile index dbd04ba..385ae6c 100644 --- a/Rakefile +++ b/Rakefile @@ -108,6 +108,51 @@ namespace :repos do end end +namespace :cache do + task :config do # rubocop:disable Rake/Desc + require_relative 'config/environment' # load config info + require_relative 'app/infrastructure/cache/redis_cache' + @api = CodePraise::App + end + + desc 'Directory listing of local dev cache' + namespace :list do + desc 'Lists development cache' + task :dev do + puts 'Lists development cache' + list = `ls _cache/rack/meta` + puts 'No local cache found' if list.empty? + puts list + end + + desc 'Lists production cache' + task :production => :config do + puts 'Finding production cache' + keys = CodePraise::Cache::Client.new(@api.config).keys + puts 'No keys found' if keys.none? + keys.each { |key| puts "Key: #{key}" } + end + end + + namespace :wipe do + desc 'Delete development cache' + task :dev do + puts 'Deleting development cache' + sh 'rm -rf _cache/*' + end + + desc 'Delete production cache' + task :production => :config do + print 'Are you sure you wish to wipe the production cache? (y/n) ' + if $stdin.gets.chomp.downcase == 'y' + puts 'Deleting production cache' + wiped = CodePraise::Cache::Client.new(@api.config).wipe + wiped.each { |key| puts "Wiped: #{key}" } + end + end + end +end + desc 'Run application console' task :console do sh 'pry -r ./load_all' diff --git a/app/application/controllers/app.rb b/app/application/controllers/app.rb index e73139e..e269360 100644 --- a/app/application/controllers/app.rb +++ b/app/application/controllers/app.rb @@ -7,10 +7,11 @@ module CodePraise # Web App class App < Roda plugin :halt - plugin :flash - plugin :all_verbs # allows DELETE and other HTTP verbs beyond GET/POST + plugin :caching + # plugin :all_verbs # allows DELETE and other HTTP verbs beyond GET/POST + + # use Rack::MethodOverride # for other HTTP verbs (with plugin all_verbs) - # rubocop:disable Metrics/BlockLength route do |routing| response['Content-Type'] = 'application/json' @@ -31,6 +32,8 @@ class App < Roda routing.on String, String do |owner_name, project_name| # GET /projects/{owner_name}/{project_name}[/folder_namepath/] routing.get do + response.cache_control public: true, max_age: 120 + path_request = Request::ProjectPath.new( owner_name, project_name, request ) @@ -86,6 +89,5 @@ class App < Roda end end end - # rubocop:enable Metrics/BlockLength end end diff --git a/app/domain/projects/entities/project.rb b/app/domain/projects/entities/project.rb index 48458f6..6a32845 100644 --- a/app/domain/projects/entities/project.rb +++ b/app/domain/projects/entities/project.rb @@ -11,7 +11,7 @@ module Entity class Project < Dry::Struct include Dry.Types - MAX_SIZE_KB = 10_000 + MAX_SIZE_KB = 50_000 attribute :id, Integer.optional attribute :origin_id, Strict::Integer diff --git a/app/infrastructure/cache/redis_cache.rb b/app/infrastructure/cache/redis_cache.rb new file mode 100644 index 0000000..6544046 --- /dev/null +++ b/app/infrastructure/cache/redis_cache.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require 'redis' + +module CodePraise + module Cache + # Redis client utility + class Client + def initialize(config) + @redis = Redis.new(url: config.REDISCLOUD_URL) + end + + def keys + @redis.keys + end + + def wipe + keys.each { |key| @redis.del(key) } + end + end + end +end diff --git a/config/environment.rb b/config/environment.rb index cc1c7a5..c2249b4 100644 --- a/config/environment.rb +++ b/config/environment.rb @@ -5,7 +5,8 @@ require 'rack/session' require 'roda' require 'sequel' -# require 'delegate' # needed until Rack 2.3 fixes delegateclass bug +require 'rack/cache' +require 'redis-rack-cache' module CodePraise # Environment-specific configuration @@ -20,13 +21,33 @@ class App < Roda Figaro.load def self.config = Figaro.env + configure :development, :production do + plugin :common_logger, $stderr + end + + # Setup Cacheing mechanism + configure :development do + use Rack::Cache, + verbose: true, + metastore: 'file:_cache/rack/meta', + entitystore: 'file:_cache/rack/body' + end + + configure :production do + use Rack::Cache, + verbose: true, + metastore: "#{config.REDISCLOUD_URL}/0/metastore", + entitystore: "#{config.REDISCLOUD_URL}/0/entitystore" + end + + # Automated HTTP stubbing for testing only configure :app_test do require_relative '../spec/helpers/vcr_helper' VcrHelper.setup_vcr VcrHelper.configure_vcr_for_github(recording: :none) end - # Database Setup + # Database Setup (ensure DATABASE_URL already configured on production) configure :development, :test, :app_test do require 'pry'; # for breakpoints ENV['DATABASE_URL'] = "sqlite://#{config.DB_FILENAME}" diff --git a/config/secrets_example.yml b/config/secrets_example.yml index 81e1fc2..942b655 100644 --- a/config/secrets_example.yml +++ b/config/secrets_example.yml @@ -6,20 +6,24 @@ development: GITHUB_TOKEN: create_github_token REPOSTORE_PATH: repostore API_HOST: http://localhost:9090 + REDISCLOUD_URL: url-assigned-by-Redis-provider-on-Heroku app_test: DB_FILENAME: db/local/test.db REPOSTORE_PATH: repostore API_HOST: http://localhost:9090 + REDISCLOUD_URL: url-assigned-by-Redis-provider-on-Heroku test: DB_FILENAME: db/local/test.db GITHUB_TOKEN: create_github_token REPOSTORE_PATH: repostore API_HOST: http://localhost:9090 + REDISCLOUD_URL: url-assigned-by-Redis-provider-on-Heroku production: # - assign DATABASE_URL in production GITHUB_TOKEN: create_github_token REPOSTORE_PATH: repostore API_HOST: https://codepraise2022-api + REDISCLOUD_URL: url-assigned-by-Redis-provider-on-Heroku diff --git a/coverage/.resultset.json b/coverage/.resultset.json index 1e61c2c..7813e83 100644 --- a/coverage/.resultset.json +++ b/coverage/.resultset.json @@ -16,7 +16,7 @@ 1, null, 1, - 54, + 55, null, null ] @@ -30,7 +30,8 @@ 1, 1, 1, - null, + 1, + 1, null, 1, null, @@ -47,6 +48,26 @@ null, 1, 0, + null, + null, + null, + 1, + 0, + null, + null, + null, + null, + null, + 1, + 0, + null, + null, + null, + null, + null, + null, + 1, + 0, 0, 0, null, @@ -737,6 +758,32 @@ null ] }, + "/Users/soumyaray/Sync/Dropbox/ossdev/classes/SOA-class/projects/soa2025/api-codepraise-2025/app/infrastructure/cache/redis_cache.rb": { + "lines": [ + null, + null, + 1, + null, + 1, + 1, + null, + 1, + 1, + 0, + null, + null, + 1, + 0, + null, + null, + 1, + 0, + null, + null, + null, + null + ] + }, "/Users/soumyaray/Sync/Dropbox/ossdev/classes/SOA-class/projects/soa2025/api-codepraise-2025/app/infrastructure/database/orm/member_orm.rb": { "lines": [ null, @@ -2083,7 +2130,8 @@ 1, 1, 1, - 1, + null, + null, null, null, 1, @@ -2108,6 +2156,8 @@ 6, 4, null, + 4, + null, null, null, 4, @@ -2162,7 +2212,6 @@ null, null, null, - null, null ] }, @@ -2477,8 +2526,8 @@ null, 1, 29, - 1874, - 1874, + 1749, + 1749, null, null, 29, @@ -3059,6 +3108,6 @@ ] } }, - "timestamp": 1763267287 + "timestamp": 1763728123 } } From 8259287e9f946ad08c0e1e464d619d084f1700af Mon Sep 17 00:00:00 2001 From: Soumya Ray Date: Sat, 22 Nov 2025 17:53:23 +0800 Subject: [PATCH 3/4] fix: propagate err msg for too large repos --- app/application/services/appraise_project.rb | 4 + app/domain/projects/entities/project.rb | 2 +- coverage/.resultset.json | 244 ++++++++++--------- spec/tests/acceptance/api_spec.rb | 28 +++ 4 files changed, 157 insertions(+), 121 deletions(-) diff --git a/app/application/services/appraise_project.rb b/app/application/services/appraise_project.rb index 6464724..f47bbd3 100644 --- a/app/application/services/appraise_project.rb +++ b/app/application/services/appraise_project.rb @@ -17,6 +17,7 @@ class AppraiseProject NO_PROJ_ERR = 'Project not found' DB_ERR = 'Having trouble accessing the database' CLONE_ERR = 'Could not clone this project' + TOO_LARGE_ERR = 'Project is too large to clone' NO_FOLDER_ERR = 'Could not find that folder' def retrieve_remote_project(input) @@ -38,6 +39,9 @@ def clone_remote(input) gitrepo.clone unless gitrepo.exists_locally? Success(input.merge(gitrepo:)) + rescue GitRepo::Errors::TooLargeToClone + App.logger.warn "Project too large: #{input[:project].fullname} (#{input[:project].size} KB)" + Failure(Response::ApiResult.new(status: :forbidden, message: TOO_LARGE_ERR)) rescue StandardError => error App.logger.error error.backtrace.join("\n") Failure(Response::ApiResult.new(status: :internal_error, message: CLONE_ERR)) diff --git a/app/domain/projects/entities/project.rb b/app/domain/projects/entities/project.rb index 6a32845..e844015 100644 --- a/app/domain/projects/entities/project.rb +++ b/app/domain/projects/entities/project.rb @@ -11,7 +11,7 @@ module Entity class Project < Dry::Struct include Dry.Types - MAX_SIZE_KB = 50_000 + MAX_SIZE_KB = 30_000 attribute :id, Integer.optional attribute :origin_id, Strict::Integer diff --git a/coverage/.resultset.json b/coverage/.resultset.json index 7813e83..0757ba2 100644 --- a/coverage/.resultset.json +++ b/coverage/.resultset.json @@ -708,7 +708,7 @@ 1, null, 1, - 53, + 57, null, null, null, @@ -742,16 +742,16 @@ 1, null, 1, - 4, + 5, null, null, 1, - 0, + 1, null, null, 1, null, - 14, + 15, null, null, null, @@ -806,7 +806,7 @@ 1, null, 1, - 52, + 56, null, null, null, @@ -854,11 +854,11 @@ null, null, 1, - 22, + 25, null, null, 1, - 13, + 14, null, null, null, @@ -882,9 +882,9 @@ null, null, 1, - 80, + 92, null, - 80, + 92, null, null, null, @@ -893,13 +893,13 @@ null, null, 1, - 20, - 60, + 23, + 69, null, null, null, 1, - 52, + 56, null, null, null, @@ -925,12 +925,12 @@ null, null, null, - 21, + 24, null, null, null, null, - 21, + 24, null, null, 1, @@ -941,7 +941,7 @@ null, null, 1, - 13, + 14, null, null, 1, @@ -950,21 +950,21 @@ null, null, 1, - 13, - 13, + 14, + 14, null, null, 1, - 13, + 14, null, - 13, - 13, + 14, + 14, null, null, 1, - 47, + 52, null, - 20, + 23, null, null, null, @@ -975,21 +975,21 @@ null, 1, 1, - 13, + 14, null, null, 1, - 13, + 14, null, null, 1, - 13, + 14, null, - 13, - 13, + 14, + 14, null, - 13, - 39, + 14, + 42, null, null, null, @@ -1380,9 +1380,9 @@ null, null, 1, - 6, - 6, - 6, + 8, + 8, + 8, null, null, 1, @@ -1391,16 +1391,16 @@ null, null, 1, - 0, + 1, null, null, 1, - 12, + 14, null, null, null, 1, - 0, + 1, 0, null, 0, @@ -1432,8 +1432,8 @@ 1, null, 1, - 6, - 6, + 8, + 8, null, null, 1, @@ -1457,12 +1457,12 @@ null, null, 1, - 29, + 31, null, null, null, 1, - 0, + 1, null, null, 1, @@ -1497,11 +1497,11 @@ 1, null, 1, - 6, + 8, null, null, 1, - 6, + 8, null, null, 1, @@ -1529,45 +1529,45 @@ 1, null, 1, - 49, + 51, null, null, 1, - 27, + 28, null, null, null, null, 1, - 22, + 23, null, null, null, 1, 1, - 27, - 27, + 28, + 28, null, null, 1, - 27, + 28, null, null, null, null, 1, 1, - 49, + 51, null, null, 1, - 49, + 51, null, null, null, null, - 48, - 48, + 50, + 50, null, null, null, @@ -1583,7 +1583,7 @@ null, null, 1, - 48, + 50, null, null, 1, @@ -1605,29 +1605,29 @@ null, 1, 1, - 22, - 22, - 22, + 23, + 23, + 23, null, null, 1, - 22, - 66, + 23, + 69, null, null, null, 1, - 88, + 92, null, null, null, 1, 1, - 88, + 92, null, null, 1, - 88, + 92, null, null, null, @@ -1638,15 +1638,15 @@ 1, null, 1, - 88, + 92, null, null, 1, - 88, + 92, null, null, 1, - 88, + 92, null, null, null, @@ -1665,31 +1665,31 @@ null, 1, 1, - 27, - 27, - 27, + 28, + 28, + 28, null, null, 1, - 27, - 22, + 28, + 23, null, null, 1, - 22, + 23, null, null, null, 1, 1, - 22, - 22, + 23, + 23, null, null, null, null, 1, - 22, + 23, null, null, null, @@ -1702,31 +1702,31 @@ null, null, 1, - 22, + 23, null, null, 1, - 22, + 23, null, null, 1, - 22, + 23, null, null, 1, - 22, + 23, null, null, 1, - 22, + 23, null, null, 1, - 22, + 23, null, null, 1, - 22, + 23, null, null, null, @@ -1925,7 +1925,7 @@ null, null, 1, - 10, + 11, null, null, null, @@ -2085,9 +2085,9 @@ null, 1, 1, - 27, + 29, null, - 26, + 28, null, null, null, @@ -2135,10 +2135,10 @@ null, null, 1, - 10, + 11, null, null, - 10, + 11, 1, null, 1, @@ -2149,22 +2149,22 @@ 1, null, null, - 9, - 9, - 9, + 10, + 10, + 10, null, - 6, - 4, + 7, + 5, null, - 4, + 5, null, null, null, - 4, + 5, null, - 4, - 2, - 2, + 5, + 3, + 3, null, null, 2, @@ -2305,10 +2305,10 @@ null, 1, 1, - 4, - 4, - 4, - 4, + 5, + 5, + 5, + 5, null, null, 1, @@ -2347,24 +2347,24 @@ null, null, 1, - 11, + 12, 1, null, - 10, + 11, null, - 8, + 9, null, 3, null, null, 1, null, + 9, 8, - 7, null, 1, null, - 8, + 9, null, 0, 0, @@ -2373,7 +2373,7 @@ null, null, 1, - 10, + 11, null, null, null, @@ -2381,7 +2381,7 @@ null, null, 1, - 11, + 12, null, null, null, @@ -2411,14 +2411,15 @@ 1, 1, 1, + 1, null, 1, - 6, + 7, null, null, null, - 6, - 4, + 7, + 5, null, 2, null, @@ -2427,11 +2428,14 @@ null, null, 1, - 4, - 4, + 5, + 5, null, 4, null, + 1, + 1, + null, 0, 0, null, @@ -2525,12 +2529,12 @@ null, null, 1, - 29, - 1749, - 1749, + 30, + 1870, + 1870, null, null, - 29, + 30, null, null, null, @@ -2539,7 +2543,7 @@ null, null, 1, - 29, + 30, null, null ] @@ -2553,10 +2557,10 @@ null, 1, null, - 21, - 21, - 21, - 21, + 22, + 22, + 22, + 22, null, null ] @@ -3108,6 +3112,6 @@ ] } }, - "timestamp": 1763728123 + "timestamp": 1763793282 } } diff --git a/spec/tests/acceptance/api_spec.rb b/spec/tests/acceptance/api_spec.rb index 652578e..edf2116 100644 --- a/spec/tests/acceptance/api_spec.rb +++ b/spec/tests/acceptance/api_spec.rb @@ -90,6 +90,34 @@ def app _(last_response.status).must_equal 404 _(JSON.parse(last_response.body)['status']).must_include 'not' end + + it 'should report error for a project that is too large to clone' do + # First, add a project to the database + CodePraise::Service::AddProject.new.call( + owner_name: USERNAME, project_name: PROJECT_NAME + ) + + # Update the project's size directly in the database to exceed MAX_SIZE_KB + db_project = CodePraise::Database::ProjectOrm + .where(name: PROJECT_NAME).first + db_project.update(size: CodePraise::Entity::Project::MAX_SIZE_KB + 1000) + + # Reload the project to get updated size + project = CodePraise::Repository::For.klass(CodePraise::Entity::Project) + .find_full_name(USERNAME, PROJECT_NAME) + + # Delete local clone if it exists to force re-cloning + gitrepo = CodePraise::GitRepo.new(project) + gitrepo.delete if gitrepo.exists_locally? + + # Try to appraise - should fail with forbidden status + get "/api/v1/projects/#{USERNAME}/#{PROJECT_NAME}" + _(last_response.status).must_equal 403 + + response_body = JSON.parse(last_response.body) + _(response_body['status']).must_equal 'forbidden' + _(response_body['message']).must_include 'too large' + end end describe 'Add projects route' do From 1ec92527608f319d30d7a85ee8d7fc39fe5a3c3e Mon Sep 17 00:00:00 2001 From: Soumya Ray Date: Sat, 22 Nov 2025 20:24:35 +0800 Subject: [PATCH 4/4] fix: remove unwanted files from infrastructure reads --- app/domain/projects/entities/project.rb | 2 +- .../git/repositories/local_repo.rb | 2 +- coverage/.resultset.json | 299 +++++++++--------- .../layers/domain_contributions_spec.rb | 3 +- 4 files changed, 152 insertions(+), 154 deletions(-) diff --git a/app/domain/projects/entities/project.rb b/app/domain/projects/entities/project.rb index e844015..6a32845 100644 --- a/app/domain/projects/entities/project.rb +++ b/app/domain/projects/entities/project.rb @@ -11,7 +11,7 @@ module Entity class Project < Dry::Struct include Dry.Types - MAX_SIZE_KB = 30_000 + MAX_SIZE_KB = 50_000 attribute :id, Integer.optional attribute :origin_id, Strict::Integer diff --git a/app/infrastructure/git/repositories/local_repo.rb b/app/infrastructure/git/repositories/local_repo.rb index a37bfcc..91d35b4 100644 --- a/app/infrastructure/git/repositories/local_repo.rb +++ b/app/infrastructure/git/repositories/local_repo.rb @@ -13,7 +13,7 @@ module Errors class LocalGitRepo ONLY_FOLDERS = '**/' FILES_AND_FOLDERS = '**/*' - TEXT_FILES = %w[rb py c js java css html slim md yml json txt].join('|') + TEXT_FILES = %w[rb py js css html slim md erb].join('|') CODE_FILENAME_MATCH = /\.(#{TEXT_FILES})$/ attr_reader :git_repo_path diff --git a/coverage/.resultset.json b/coverage/.resultset.json index 0757ba2..6a7fc3e 100644 --- a/coverage/.resultset.json +++ b/coverage/.resultset.json @@ -125,7 +125,7 @@ null, null, 1, - 7564, + 7555, null, null, null, @@ -133,7 +133,7 @@ null, null, 1, - 6955, + 6940, null, null, null, @@ -167,20 +167,20 @@ 1, null, 1, - 194, - 194, + 190, + 190, null, null, 1, - 724, + 711, null, null, 1, - 203, + 199, null, null, 1, - 724, + 711, null, 711, null, @@ -189,17 +189,17 @@ null, null, 1, - 67, + 66, null, null, 1, null, 1, - 724, + 711, null, null, 1, - 724, + 711, null, null, null, @@ -215,7 +215,7 @@ null, 1, 1, - 203, + 199, null, null, 1, @@ -255,7 +255,7 @@ 1, null, 1, - 5372, + 5360, null, null, 1, @@ -339,7 +339,7 @@ 1, null, 1, - 6096, + 6071, null, null, null, @@ -360,20 +360,20 @@ 1, null, 1, - 52, + 48, null, - 52, - 52, - 52, - 52, + 48, + 48, + 48, + 48, null, null, 1, - 18, + 17, null, null, 1, - 18, + 17, null, null, 1, @@ -381,51 +381,51 @@ null, null, 1, - 18, + 17, null, null, 1, - 18, + 17, null, null, 1, - 60, + 58, null, null, 1, - 18, + 17, null, null, 1, null, 1, - 602, + 584, null, null, 1, - 52, + 48, null, null, 1, - 52, - 398, - 194, + 48, + 387, + 190, null, null, 1, - 52, + 48, null, - 204, - 204, + 197, + 197, null, null, null, 1, - 52, - 47, + 48, + 43, null, null, - 99, + 91, null, null, null, @@ -451,14 +451,14 @@ 1, null, 1, - 5372, - 5372, - 5372, - 5372, + 5360, + 5360, + 5360, + 5360, null, null, 1, - 11882, + 11870, null, null, null, @@ -474,14 +474,14 @@ null, 1, 1, - 151, + 139, null, null, null, null, 1, 1, - 1876, + 1845, null, null, null, @@ -497,7 +497,7 @@ 1, null, 1, - 362, + 356, null, null, null, @@ -505,15 +505,15 @@ null, 1, null, - 362, - 362, + 356, + 356, null, - 362, + 356, null, - 1103, - 1103, + 1091, + 1091, null, - 1103, + 1091, null, 0, 0, @@ -522,20 +522,20 @@ 0, 0, null, - 1103, + 1091, null, 86, 86, 86, null, null, - 1017, - 1017, - 1017, + 1005, + 1005, + 1005, null, null, null, - 362, + 356, null, null, null, @@ -556,8 +556,8 @@ 1, null, 1, - 565, - 565, + 546, + 546, null, null, null, @@ -584,29 +584,29 @@ null, null, 1, - 203, + 199, null, null, 1, - 6377, - 6377, - 6377, - 6377, + 6365, + 6365, + 6365, + 6365, null, null, 1, - 362, + 356, null, null, null, - 362, + 356, 446, null, null, - 362, - 1017, - 2120, - 1017, + 356, + 1005, + 2096, + 1005, null, null, null, @@ -622,7 +622,7 @@ 1, null, 1, - 13326, + 13305, null, null, null, @@ -656,32 +656,32 @@ 1, null, 1, - 388, - 388, + 380, + 380, null, null, 1, - 6096, + 6071, null, null, 1, - 6096, + 6071, null, null, 1, null, - 204, + 197, null, - 204, - 204, + 197, + 197, null, null, 1, null, 1, - 388, - 388, - 388, + 380, + 380, + 380, null, null, null, @@ -746,7 +746,7 @@ null, null, 1, - 1, + 2, null, null, 1, @@ -1010,57 +1010,57 @@ 1, null, 1, - 196, - 196, - 196, - 196, + 193, + 193, + 193, + 193, null, null, 1, - 1, - 1, - 1, + 2, + 2, + 2, null, null, 1, - 195, - 195, - 195, + 191, + 191, + 191, null, null, 1, - 195, - 195, + 191, + 191, null, null, 1, - 1, - 1, + 2, + 2, null, null, 1, - 1, - 1, + 2, + 2, null, null, 1, - 392, + 386, null, null, 1, - 196, + 193, null, null, null, null, null, 1, - 194, + 190, null, null, 1, - 0, - 0, + 1, + 5, null, null, null, @@ -1077,12 +1077,12 @@ null, 1, 1, - 5372, - 5372, + 5360, + 5360, null, null, 1, - 5372, + 5360, null, null, null, @@ -1092,7 +1092,7 @@ 1, null, 1, - 5372, + 5360, null, null, null, @@ -1123,16 +1123,16 @@ 1, 1, 5, - 194, + 190, null, null, null, null, 1, 1, - 194, - 194, - 194, + 190, + 190, + 190, null, null, null, @@ -1151,11 +1151,11 @@ null, 1, 1, - 194, + 190, null, null, 1, - 194, + 190, null, null, null, @@ -1164,20 +1164,20 @@ 1, null, 1, - 388, + 380, null, null, 1, - 5372, + 5360, null, null, 1, - 194, + 190, null, null, 1, - 194, - 5372, + 190, + 5360, null, null, null, @@ -1185,9 +1185,9 @@ null, 1, 1, - 5372, - 5372, - 5372, + 5360, + 5360, + 5360, null, null, 1, @@ -1197,14 +1197,14 @@ 1, null, 1, - 5372, + 5360, null, null, null, 1, null, 1, - 5372, + 5360, null, null, null, @@ -1238,7 +1238,7 @@ null, 1, 5, - 194, + 190, null, null, null, @@ -1258,13 +1258,13 @@ 1, null, 1, - 194, - 5372, + 190, + 5360, null, null, 1, - 194, - 194, + 190, + 190, null, 0, 0, @@ -1272,28 +1272,28 @@ null, null, 1, - 5372, + 5360, null, - 5372, + 5360, null, null, null, - 5372, - 57020, - 57020, + 5360, + 56896, + 56896, null, null, - 5372, + 5360, null, null, 1, - 5372, - 5372, - 5372, + 5360, + 5360, + 5360, null, null, 1, - 57020, + 56896, null, null, null, @@ -1323,7 +1323,7 @@ null, 5, 5, - 194, + 190, null, null, null, @@ -1333,7 +1333,7 @@ 6, 3, null, - 189, + 186, null, null, null, @@ -1352,11 +1352,11 @@ 1, null, 1, - 194, + 190, null, null, 1, - 194, + 190, null, null, null, @@ -1395,15 +1395,15 @@ null, null, 1, - 14, + 15, null, null, null, 1, + 2, 1, - 0, null, - 0, + 6, null, null, null @@ -1437,8 +1437,8 @@ null, null, 1, - 0, - 0, + 6, + 1, null, null, 1, @@ -1457,7 +1457,7 @@ null, null, 1, - 31, + 32, null, null, null, @@ -1505,7 +1505,7 @@ null, null, 1, - 0, + 1, null, null, null, @@ -2530,8 +2530,8 @@ null, 1, 30, - 1870, - 1870, + 1930, + 1930, null, null, 30, @@ -2619,7 +2619,6 @@ 1, null, 1, - 1, null, 1, null, @@ -3112,6 +3111,6 @@ ] } }, - "timestamp": 1763793282 + "timestamp": 1763966828 } } diff --git a/spec/tests/integration/layers/domain_contributions_spec.rb b/spec/tests/integration/layers/domain_contributions_spec.rb index 7e193c9..8a32fe9 100644 --- a/spec/tests/integration/layers/domain_contributions_spec.rb +++ b/spec/tests/integration/layers/domain_contributions_spec.rb @@ -49,8 +49,7 @@ it 'HAPPY: should get accurate contributions summary for specific folder' do forms = CodePraise::Mapper::Contributions.new(@gitrepo).for_folder('forms') - _(forms.subfolders.count).must_equal 1 - _(forms.subfolders.count).must_equal 1 + _(forms.subfolders.count).must_equal 0 _(forms.base_files.count).must_equal 2