Address Rails engine review feedback

Author: Qard

README.md

@@ -348,7 +348,11 @@ See examples: [eval.rb](./examples/eval.rb), [dataset.rb](./examples/eval/datase
 
 ### Dev Server
 
-Run evaluations from the Braintrust web UI against code in your own application. Define evaluators, pass them to the dev server, and start serving:
+Run evaluations from the Braintrust web UI against code in your own application.
+
+**Run as a Rack app**
+
+Define evaluators, pass them to the dev server, and start serving:
 
 ```ruby
 # eval_server.ru
@@ -378,6 +382,44 @@ run Braintrust::Server::Rack.app(
 bundle exec rackup eval_server.ru -p 8300 -o 0.0.0.0
 ```
 
+**Run as a Rails engine**
+
+Use the Rails engine when your evaluators live inside an existing Rails app and you want to mount the Braintrust endpoints into that application.
+
+```ruby
+# config/initializers/braintrust_server.rb
+require "braintrust/server/rails"
+
+Braintrust::Contrib::Rails::Engine.configure do |config|
+  config.evaluators = {
+    "food-classifier" => Braintrust::Eval::Evaluator.new(
+      task: ->(input:) { FoodClassifier.classify(input) },
+      scorers: [
+        Braintrust::Scorer.new("exact_match") { |expected:, output:| output == expected ? 1.0 : 0.0 }
+      ]
+    )
+  }
+
+  # Default is :clerk_token. Use :none for local development.
+  config.auth = :none
+end
+```
+
+```ruby
+# config/routes.rb
+Rails.application.routes.draw do
+  mount Braintrust::Contrib::Rails::Engine, at: "/braintrust"
+end
+```
+
+Mounted at `/braintrust`, the engine exposes:
+
+- `GET /braintrust/` for the health check
+- `GET /braintrust/list` and `POST /braintrust/list` to enumerate evaluators
+- `POST /braintrust/eval` to run an evaluation and stream SSE results
+
+See example: [contrib/rails/eval.rb](./examples/contrib/rails/eval.rb)
+
 **Custom evaluators**
 
 Evaluators can also be defined as subclasses:
@@ -412,7 +454,7 @@ gem "rack"
 gem "puma" # recommended
 ```
 
-See example: [server/eval.ru](./examples/server/eval.ru)
+See examples: [server/eval.ru](./examples/server/eval.ru), [contrib/rails/eval.rb](./examples/contrib/rails/eval.rb)
 
 ## Documentation
 

examples/README.md

@@ -33,6 +33,7 @@ BRAINTRUST_DEBUG=true ruby examples/login/login_basic.rb
 ### Dev Server Examples
 
 - **`server/eval.ru`**: Set up a dev server for remote evals — define evaluators (subclass or inline) and serve them via a Rack app. Start with: `bundle exec appraisal server rackup examples/server/eval.ru -p 8300 -o 0.0.0.0`
+- **`contrib/rails/eval.rb`**: Mount the dev server as a Rails engine and configure evaluators via `Braintrust::Contrib::Rails::Engine.configure`
 
 ## Coming Soon
 

lib/braintrust/contrib/rails/application_controller.rb

@@ -18,6 +18,14 @@ def authenticate!
           request.env["braintrust.auth"] = auth_result
           @braintrust_auth = auth_result
         end
+
+        def parse_json_body
+          body = request.body.read
+          return nil if body.nil? || body.empty?
+          JSON.parse(body)
+        rescue JSON::ParserError
+          nil
+        end
       end
     end
   end

lib/braintrust/contrib/rails/engine.rb

@@ -23,28 +23,23 @@ def self.evaluators
         end
 
         def self.auth_strategy
-          @auth_strategy ||= resolve_auth(config.auth)
+          resolve_auth(config.auth)
         end
 
         def self.list_service
-          @list_service ||= Server::Services::List.new(config.evaluators)
+          Server::Services::List.new(-> { config.evaluators })
         end
 
         # Long-lived so the state cache persists across requests.
         def self.eval_service
-          @eval_service ||= Server::Services::Eval.new(config.evaluators)
+          @eval_service ||= Server::Services::Eval.new(-> { config.evaluators })
         end
 
-        # Reset memoized services (useful in tests when config changes).
-        def self.reset_services!
-          @auth_strategy = nil
-          @list_service = nil
-          @eval_service = nil
-        end
-
-        def self.configure
-          yield config
-          reset_services!
+        # Support the explicit `|config|` style used by this integration while
+        # still delegating zero-arity DSL blocks to Rails' native implementation.
+        def self.configure(&block)
+          return super(&block) if block&.arity == 0
+          yield config if block
         end
 
         def self.resolve_auth(auth)

lib/braintrust/contrib/rails/eval_controller.rb

@@ -7,7 +7,7 @@ class EvalController < ApplicationController
         include ActionController::Live
 
         def create
-          body = parse_body
+          body = parse_json_body
           unless body
             render json: {"error" => "Invalid JSON body"}, status: :bad_request
             return
@@ -28,16 +28,6 @@ def create
         ensure
           response.stream.close
         end
-
-        private
-
-        def parse_body
-          body = request.body.read
-          return nil if body.nil? || body.empty?
-          JSON.parse(body)
-        rescue JSON::ParserError
-          nil
-        end
       end
     end
   end

lib/braintrust/server/services/eval_service.rb

@@ -21,7 +21,7 @@ def validate(body)
           name = body["name"]
           return {error: "Missing required field: name", status: 400} unless name
 
-          evaluator = @evaluators[name]
+          evaluator = current_evaluators[name]
           return {error: "Evaluator '#{name}' not found", status: 404} unless evaluator
 
           data = body["data"]
@@ -156,6 +156,11 @@ def build_state(auth)
 
         private
 
+        def current_evaluators
+          return @evaluators.call if @evaluators.respond_to?(:call)
+          @evaluators
+        end
+
         # Resolve data source from the data field.
         # Returns [cases, dataset] where exactly one is non-nil.
         def resolve_data_source(data)

lib/braintrust/server/services/list_service.rb

@@ -14,7 +14,7 @@ def initialize(evaluators)
 
         def call
           result = {}
-          @evaluators.each do |name, evaluator|
+          current_evaluators.each do |name, evaluator|
             scores = (evaluator.scorers || []).each_with_index.map do |scorer, i|
               scorer_name = scorer.respond_to?(:name) ? scorer.name : "score_#{i}"
               {"name" => scorer_name}
@@ -29,6 +29,11 @@ def call
 
         private
 
+        def current_evaluators
+          return @evaluators.call if @evaluators.respond_to?(:call)
+          @evaluators
+        end
+
         # Convert user-defined parameters to the dev server protocol format.
         # Wraps in a staticParameters container with "data" typed entries.
         def serialize_parameters(parameters)

test/braintrust/contrib/rails/engine_test.rb

@@ -19,29 +19,62 @@ def test_evaluators_returns_config_value
 
         def test_auth_strategy_returns_no_auth_for_none
           Engine.config.auth = :none
-          Engine.reset_services!
           assert_instance_of Braintrust::Server::Auth::NoAuth, Engine.auth_strategy
         end
 
         def test_auth_strategy_returns_clerk_token_by_default
           Engine.config.auth = :clerk_token
-          Engine.reset_services!
           assert_instance_of Braintrust::Server::Auth::ClerkToken, Engine.auth_strategy
         end
 
         def test_auth_strategy_accepts_custom_object
           custom = Braintrust::Server::Auth::NoAuth.new
           Engine.config.auth = custom
-          Engine.reset_services!
           assert_same custom, Engine.auth_strategy
         end
 
         def test_auth_strategy_raises_for_unknown_symbol
           Engine.config.auth = :jwt
-          Engine.reset_services!
           assert_raises(ArgumentError) { Engine.auth_strategy }
         end
 
+        def test_auth_strategy_raises_for_unknown_string
+          Engine.config.auth = "jwt"
+          assert_raises(ArgumentError) { Engine.auth_strategy }
+        end
+
+        def test_auth_strategy_reflects_config_changes_without_manual_reset
+          Engine.config.auth = :none
+          assert_instance_of Braintrust::Server::Auth::NoAuth, Engine.auth_strategy
+
+          Engine.config.auth = :clerk_token
+          assert_instance_of Braintrust::Server::Auth::ClerkToken, Engine.auth_strategy
+        end
+
+        def test_list_service_uses_latest_evaluators_without_manual_reset
+          first = Braintrust::Eval::Evaluator.new(task: ->(input) { input })
+          second = Braintrust::Eval::Evaluator.new(task: ->(input) { input })
+
+          Engine.config.evaluators = {"first" => first}
+          assert_equal ["first"], Engine.list_service.call.keys
+
+          Engine.config.evaluators = {"second" => second}
+          assert_equal ["second"], Engine.list_service.call.keys
+        end
+
+        def test_eval_service_uses_latest_evaluators_without_manual_reset
+          first = Braintrust::Eval::Evaluator.new(task: ->(input) { input })
+          second = Braintrust::Eval::Evaluator.new(task: ->(input) { input })
+          payload = {"data" => {"data" => [{"input" => "hello"}]}}
+
+          Engine.config.evaluators = {"first" => first}
+          service = Engine.eval_service
+          assert_same first, service.validate(payload.merge("name" => "first"))[:evaluator]
+
+          Engine.config.evaluators = {"second" => second}
+          assert_same second, service.validate(payload.merge("name" => "second"))[:evaluator]
+        end
+
         def test_eval_service_returns_eval_instance
           assert_instance_of Braintrust::Server::Services::Eval, Engine.eval_service
         end
@@ -56,16 +89,10 @@ def test_eval_service_is_memoized
           assert_same svc1, svc2
         end
 
-        def test_reset_services_clears_memoized_instances
-          svc1 = Engine.eval_service
-          Engine.reset_services!
-          svc2 = Engine.eval_service
-          refute_same svc1, svc2
-        end
-
-        def test_configure_yields_config_and_resets_services
+        def test_configure_yields_config_without_resetting_eval_service
           svc_before = Engine.eval_service
           evaluator = Braintrust::Eval::Evaluator.new(task: ->(input) { input })
+          payload = {"name" => "configured-eval", "data" => {"data" => [{"input" => "hello"}]}}
 
           Engine.configure do |config|
             config.evaluators = {"configured-eval" => evaluator}
@@ -74,7 +101,8 @@ def test_configure_yields_config_and_resets_services
 
           assert_same evaluator, Engine.evaluators["configured-eval"]
           assert_instance_of Braintrust::Server::Auth::NoAuth, Engine.auth_strategy
-          refute_same svc_before, Engine.eval_service
+          assert_same svc_before, Engine.eval_service
+          assert_same evaluator, Engine.eval_service.validate(payload)[:evaluator]
         end
 
         def test_cors_middleware_is_in_middleware_stack

test/support/rails_server_helper.rb

@@ -52,9 +52,11 @@ def rails_app
       end
 
       def reset_engine!(evaluators: {}, auth: :none)
-        Braintrust::Contrib::Rails::Engine.config.evaluators = evaluators
-        Braintrust::Contrib::Rails::Engine.config.auth = auth
-        Braintrust::Contrib::Rails::Engine.reset_services!
+        engine = Braintrust::Contrib::Rails::Engine
+        engine.config.evaluators = evaluators
+        engine.config.auth = auth
+        # Clear the long-lived eval service so cached state does not leak across tests.
+        engine.instance_variable_set(:@eval_service, nil)
       end
     end
   end