From 92dbbcb680244f94378c75182593d47feadf4d78 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Mon, 16 Mar 2026 20:56:01 +0000 Subject: [PATCH 1/4] Working Brotli and Gzip compression, configurable --- Gemfile | 2 + Gemfile.lock | 2 + README.md | 93 ++++++++++++++++++- lib/datastar.rb | 2 + lib/datastar/compressed_socket.rb | 71 +++++++++++++++ lib/datastar/configuration.rb | 6 +- lib/datastar/dispatcher.rb | 35 +++++++- lib/datastar/encoding_negotiation.rb | 92 +++++++++++++++++++ spec/compressed_socket_spec.rb | 128 +++++++++++++++++++++++++++ spec/dispatcher_spec.rb | 103 +++++++++++++++++++++ spec/encoding_negotiation_spec.rb | 94 ++++++++++++++++++++ 11 files changed, 624 insertions(+), 4 deletions(-) create mode 100644 lib/datastar/compressed_socket.rb create mode 100644 lib/datastar/encoding_negotiation.rb create mode 100644 spec/compressed_socket_spec.rb create mode 100644 spec/encoding_negotiation_spec.rb diff --git a/Gemfile b/Gemfile index 90c02e2..16736fe 100644 --- a/Gemfile +++ b/Gemfile @@ -17,4 +17,6 @@ group :test do gem 'async' # Puma to host test server gem 'puma' + # Brotli for compression tests + gem 'brotli' end diff --git a/Gemfile.lock b/Gemfile.lock index b3d2941..2503ed1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -15,6 +15,7 @@ GEM io-event (~> 1.11) metrics (~> 0.12) traces (~> 0.18) + brotli (0.8.0) console (1.34.2) fiber-annotation fiber-local (~> 1.1) @@ -77,6 +78,7 @@ PLATFORMS DEPENDENCIES async + brotli datastar! debug logger diff --git a/README.md b/README.md index 23a5b5e..5640de8 100644 --- a/README.md +++ b/README.md @@ -261,6 +261,91 @@ datastar.stream do |sse| end ``` +### Compression + +SSE data (JSON + HTML) is highly compressible, and long-lived connections benefit significantly from compression. This SDK supports opt-in Brotli and gzip compression for SSE streams. + +#### Enabling compression + +Per-instance: + +```ruby +datastar = Datastar.new(request:, response:, view_context:, compression: true) +``` + +Or globally: + +```ruby +Datastar.configure do |config| + config.compression = true +end +``` + +When enabled, the SDK negotiates compression with the client via the `Accept-Encoding` header and sets the appropriate `Content-Encoding` response header. If the client does not support compression, responses are sent uncompressed. + +#### Brotli vs gzip + +Brotli (`:br`) is preferred by default as it offers better compression ratios. If the optional [`brotli`](https://github.com/miyucy/brotli) gem is not installed, gzip is used as the fallback. Gzip uses Ruby built-in `zlib` and requires no extra dependencies. + +To use Brotli, add the gem to your `Gemfile`: + +```ruby +gem 'brotli' +``` + +#### Configuration options + +```ruby +Datastar.configure do |config| + # Enable compression (default: false) + # true enables both :br and :gzip + # Or pass an array to limit: [:gzip] + config.compression = true + + # Preferred encoding when client supports multiple (default: :br) + config.compression_preferred = :br + + # Encoder-specific options passed to the compressor (default: {}) + config.compression_options = { quality: 5 } +end +``` + +You can also set these per-instance: + +```ruby +datastar = Datastar.new( + request:, response:, view_context:, + compression: [:gzip], + compression_preferred: :gzip, + compression_options: { level: 1 } +) +``` + +#### Compressor options + +`compression_options` are passed directly to the underlying compressor. Available options depend on the negotiated encoding. + +**Gzip** (via `Zlib::Deflate`): + +| Option | Default | Description | +|--------|---------|-------------| +| `:level` | `Zlib::DEFAULT_COMPRESSION` | Compression level (0-9). 0 = none, 1 = fastest, 9 = smallest. `Zlib::BEST_SPEED` and `Zlib::BEST_COMPRESSION` also work. | +| `:mem_level` | `8` | Memory usage (1-9). Higher uses more memory for better compression. | +| `:strategy` | `Zlib::DEFAULT_STRATEGY` | Algorithm strategy. Alternatives: `Zlib::FILTERED`, `Zlib::HUFFMAN_ONLY`, `Zlib::RLE`, `Zlib::FIXED`. | + +**Brotli** (via `Brotli::Compressor`, requires the `brotli` gem): + +| Option | Default | Description | +|--------|---------|-------------| +| `:quality` | `11` | Compression quality (0-11). Lower is faster, higher compresses better. | +| `:lgwin` | `22` | Base-2 log of sliding window size (10-24). | +| `:lgblock` | `0` (auto) | Base-2 log of max input block size (16-24, or 0 for auto). | +| `:mode` | `:generic` | Compression mode: `:generic`, `:text`, or `:font`. `:text` is a good choice for SSE (UTF-8 HTML/JSON). | + +#### Proxy considerations + +Even with `X-Accel-Buffering: no` (set by default), some proxies like Nginx may buffer compressed responses. You may need to add `proxy_buffering off` to your Nginx configuration when using compression with SSE. + ### Global configuration ```ruby @@ -270,10 +355,14 @@ Datastar.configure do |config| config.on_error do |exception| Sentry.notify(exception) end - + # Global heartbeat interval (or false, to disable) - # Can be overriden on specific instances + # Can be overriden on specific instances config.heartbeat = 0.3 + + # Enable compression for SSE streams (default: false) + # See the Compression section above for details + config.compression = true end ``` diff --git a/lib/datastar.rb b/lib/datastar.rb index 9c1d293..e9d9dae 100644 --- a/lib/datastar.rb +++ b/lib/datastar.rb @@ -25,6 +25,8 @@ def self.from_rack_env(env, view_context: nil) end require_relative 'datastar/configuration' +require_relative 'datastar/compressed_socket' +require_relative 'datastar/encoding_negotiation' require_relative 'datastar/dispatcher' require_relative 'datastar/server_sent_event_generator' require_relative 'datastar/railtie' if defined?(Rails::Railtie) diff --git a/lib/datastar/compressed_socket.rb b/lib/datastar/compressed_socket.rb new file mode 100644 index 0000000..442c3ff --- /dev/null +++ b/lib/datastar/compressed_socket.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Datastar + # Decorator that wraps a socket and compresses data before writing. + # Supports Brotli and gzip compression. + # Used internally by Dispatcher when compression is negotiated. + module CompressedSocket + # Brotli compression using the `brotli` gem. + # Options are passed directly to Brotli::Compressor.new: + # :quality - Compression quality (0-11, default: 11). Lower is faster, higher compresses better. + # :lgwin - Base-2 log of the sliding window size (10-24, default: 22). + # :lgblock - Base-2 log of the maximum input block size (16-24, 0 = auto, default: 0). + # :mode - Compression mode (:generic, :text, or :font, default: :generic). + # Use :text for UTF-8 formatted text (HTML, JSON — good for SSE). + class Brotli + def initialize(socket, options = {}) + require 'brotli' + @socket = socket + @compressor = ::Brotli::Compressor.new(options) + end + + def <<(data) + compressed = @compressor.process(data) + @socket << compressed if compressed && !compressed.empty? + flushed = @compressor.flush + @socket << flushed if flushed && !flushed.empty? + self + end + + def close + final = @compressor.finish + @socket << final if final && !final.empty? + @socket.close + end + end + + # Gzip compression using Ruby's built-in zlib. + # Options: + # :level - Compression level (0-9, default: Zlib::DEFAULT_COMPRESSION). + # 0 = no compression, 1 = best speed, 9 = best compression. + # Zlib::BEST_SPEED (1) and Zlib::BEST_COMPRESSION (9) are also available. + # :mem_level - Memory usage level (1-9, default: 8). Higher uses more memory for better compression. + # :strategy - Compression strategy (default: Zlib::DEFAULT_STRATEGY). + # Zlib::FILTERED, Zlib::HUFFMAN_ONLY, Zlib::RLE, Zlib::FIXED are also available. + class Gzip + def initialize(socket, options = {}) + require 'zlib' + level = options.fetch(:level, Zlib::DEFAULT_COMPRESSION) + mem_level = options.fetch(:mem_level, Zlib::DEF_MEM_LEVEL) + strategy = options.fetch(:strategy, Zlib::DEFAULT_STRATEGY) + # Use raw deflate with gzip wrapping (window_bits 31 = 15 + 16) + @socket = socket + @deflate = Zlib::Deflate.new(level, 31, mem_level, strategy) + end + + def <<(data) + compressed = @deflate.deflate(data, Zlib::SYNC_FLUSH) + @socket << compressed if compressed && !compressed.empty? + self + end + + def close + final = @deflate.finish + @socket << final if final && !final.empty? + @socket.close + ensure + @deflate.close + end + end + end +end diff --git a/lib/datastar/configuration.rb b/lib/datastar/configuration.rb index 3df339b..bb75b5f 100644 --- a/lib/datastar/configuration.rb +++ b/lib/datastar/configuration.rb @@ -34,7 +34,8 @@ class Configuration RACK_FINALIZE = ->(_view_context, response) { response.finish } DEFAULT_HEARTBEAT = 3 - attr_accessor :executor, :error_callback, :finalize, :heartbeat, :logger + attr_accessor :executor, :error_callback, :finalize, :heartbeat, :logger, + :compression, :compression_preferred, :compression_options def initialize @executor = ThreadExecutor.new @@ -44,6 +45,9 @@ def initialize @error_callback = proc do |e| @logger.error("#{e.class} (#{e.message}):\n#{e.backtrace.join("\n")}") end + @compression = false + @compression_preferred = :br + @compression_options = {} end def on_error(callable = nil, &block) diff --git a/lib/datastar/dispatcher.rb b/lib/datastar/dispatcher.rb index 98e4600..fb2960b 100644 --- a/lib/datastar/dispatcher.rb +++ b/lib/datastar/dispatcher.rb @@ -44,7 +44,10 @@ def initialize( executor: Datastar.config.executor, error_callback: Datastar.config.error_callback, finalize: Datastar.config.finalize, - heartbeat: Datastar.config.heartbeat + heartbeat: Datastar.config.heartbeat, + compression: Datastar.config.compression, + compression_preferred: Datastar.config.compression_preferred, + compression_options: Datastar.config.compression_options ) @on_connect = [] @on_client_disconnect = [] @@ -68,6 +71,20 @@ def initialize( @heartbeat = heartbeat @heartbeat_on = false + + # Negotiate compression + @encoding = EncodingNegotiation.negotiate( + request, + preferred: compression_preferred, + enabled: compression + ) + @compression_options = compression_options + + if @encoding + encoding_value = @encoding == :br ? 'br' : 'gzip' + @response.headers['Content-Encoding'] = encoding_value + @response.headers['Vary'] = 'Accept-Encoding' + end end # Check if the request accepts SSE responses @@ -283,6 +300,7 @@ def stream_no_heartbeat(&block) # @api private def stream_one(streamer) proc do |socket| + socket = wrap_socket(socket) generator = ServerSentEventGenerator.new(socket, signals:, view_context: @view_context) @on_connect.each { |callable| callable.call(generator) } handling_sync_errors(generator, socket) do @@ -308,6 +326,7 @@ def stream_many(streamer) @queue ||= @executor.new_queue proc do |socket| + socket = wrap_socket(socket) signs = signals conn_generator = ServerSentEventGenerator.new(socket, signals: signs, view_context: @view_context) @on_connect.each { |callable| callable.call(conn_generator) } @@ -360,6 +379,20 @@ def stream_many(streamer) end end + # Wrap socket in a CompressedSocket if compression is negotiated + # @param socket [IO] + # @return [CompressedSocket, IO] + def wrap_socket(socket) + case @encoding + when :br + CompressedSocket::Brotli.new(socket, @compression_options) + when :gzip + CompressedSocket::Gzip.new(socket, @compression_options) + else + socket + end + end + # Handle errors caught during streaming # @param error [Exception] the error that occurred # @param socket [IO] the socket to pass to error handlers diff --git a/lib/datastar/encoding_negotiation.rb b/lib/datastar/encoding_negotiation.rb new file mode 100644 index 0000000..0d35d37 --- /dev/null +++ b/lib/datastar/encoding_negotiation.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +module Datastar + module EncodingNegotiation + ACCEPT_ENCODING = 'HTTP_ACCEPT_ENCODING' + + # Negotiate compression encoding based on request headers and configuration. + # + # @param request [Rack::Request] + # @param preferred [Symbol] preferred encoding (:br or :gzip) + # @param enabled [Boolean, Array] compression config + # @return [Symbol, nil] :br, :gzip, or nil + def self.negotiate(request, preferred:, enabled:) + return nil unless enabled + + accepted = parse_accept_encoding(request.get_header(ACCEPT_ENCODING).to_s) + return nil if accepted.empty? + + available = if enabled == true + %i[br gzip] + else + Array(enabled) + end + + # Try preferred encoding first + if available.include?(preferred) && accepted.include?(preferred) + return preferred if encoding_available?(preferred) + end + + # Fall back to other available encodings + (available - [preferred]).each do |enc| + return enc if accepted.include?(enc) && encoding_available?(enc) + end + + nil + end + + # Check if the encoding implementation is available + # @param encoding [Symbol] + # @return [Boolean] + def self.encoding_available?(encoding) + case encoding + when :br + brotli_available? + when :gzip + true # zlib is part of Ruby stdlib + else + false + end + end + + # Check if the brotli gem is installed (memoized) + # @return [Boolean] + def self.brotli_available? + return @brotli_available if defined?(@brotli_available) + + @brotli_available = begin + require 'brotli' + true + rescue LoadError + false + end + end + + # Reset memoized brotli availability (for testing) + def self.reset_brotli_cache! + remove_instance_variable(:@brotli_available) if defined?(@brotli_available) + end + + # Parse Accept-Encoding header into a set of encoding symbols + # @param header [String] + # @return [Set] + def self.parse_accept_encoding(header) + return Set.new if header.empty? + + encodings = Set.new + header.split(',').each do |part| + encoding, quality = part.strip.split(';', 2) + encoding = encoding.strip.downcase + # Skip if quality is explicitly 0 + if quality + q_val = quality.strip.match(/q=(\d+\.?\d*)/) + next if q_val && q_val[1].to_f == 0 + end + encodings << encoding.to_sym + end + encodings + end + + private_class_method :parse_accept_encoding + end +end diff --git a/spec/compressed_socket_spec.rb b/spec/compressed_socket_spec.rb new file mode 100644 index 0000000..9ce9a55 --- /dev/null +++ b/spec/compressed_socket_spec.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +require 'datastar' +require 'zlib' + +RSpec.describe Datastar::CompressedSocket do + let(:raw_socket) { StringSocket.new } + let(:sse_data) { "event: datastar-patch-signals\ndata: signals {\"foo\":\"bar\"}\n\n" } + + # A simple socket that collects binary data + class StringSocket + attr_reader :closed + + def initialize + @buffer = String.new(encoding: Encoding::BINARY) + @closed = false + end + + def <<(data) + @buffer << data.b + self + end + + def close + @closed = true + end + + def bytes + @buffer + end + end + + describe Datastar::CompressedSocket::Gzip do + subject(:socket) { described_class.new(raw_socket) } + + it 'compresses data and decompresses to original' do + socket << sse_data + socket.close + + decompressed = Zlib::Inflate.new(31).inflate(raw_socket.bytes) + expect(decompressed).to eq(sse_data) + end + + it 'flushes data after each write (data available before close)' do + socket << sse_data + # Data should be in raw_socket before close + expect(raw_socket.bytes).not_to be_empty + + partial = Zlib::Inflate.new(31).inflate(raw_socket.bytes) + expect(partial).to eq(sse_data) + end + + it 'handles multiple writes' do + data1 = "event: datastar-patch-signals\ndata: signals {\"a\":1}\n\n" + data2 = "event: datastar-patch-signals\ndata: signals {\"b\":2}\n\n" + + socket << data1 + socket << data2 + socket.close + + decompressed = Zlib::Inflate.new(31).inflate(raw_socket.bytes) + expect(decompressed).to eq(data1 + data2) + end + + it 'closes the underlying socket' do + socket << sse_data + socket.close + expect(raw_socket.closed).to be(true) + end + + it 'accepts compression level option' do + socket = described_class.new(raw_socket, level: Zlib::BEST_SPEED) + socket << sse_data + socket.close + + decompressed = Zlib::Inflate.new(31).inflate(raw_socket.bytes) + expect(decompressed).to eq(sse_data) + end + end + + describe Datastar::CompressedSocket::Brotli do + before do + skip 'brotli gem not available' unless brotli_available? + end + + subject(:socket) { described_class.new(raw_socket) } + + it 'compresses data and decompresses to original' do + socket << sse_data + socket.close + + decompressed = ::Brotli.inflate(raw_socket.bytes) + expect(decompressed).to eq(sse_data) + end + + it 'flushes data after each write (data available before close)' do + socket << sse_data + expect(raw_socket.bytes).not_to be_empty + end + + it 'handles multiple writes' do + data1 = "event: datastar-patch-signals\ndata: signals {\"a\":1}\n\n" + data2 = "event: datastar-patch-signals\ndata: signals {\"b\":2}\n\n" + + socket << data1 + socket << data2 + socket.close + + decompressed = ::Brotli.inflate(raw_socket.bytes) + expect(decompressed).to eq(data1 + data2) + end + + it 'closes the underlying socket' do + socket << sse_data + socket.close + expect(raw_socket.closed).to be(true) + end + end + + private + + def brotli_available? + require 'brotli' + true + rescue LoadError + false + end +end diff --git a/spec/dispatcher_spec.rb b/spec/dispatcher_spec.rb index 8c3b36b..7a37dab 100644 --- a/spec/dispatcher_spec.rb +++ b/spec/dispatcher_spec.rb @@ -610,8 +610,111 @@ def self.render_in(view_context) = %(
\n#{view_context} 'br, gzip' }) + dispatcher = Datastar.new(request:, response:, view_context:, compression: true) + + expect(dispatcher.response['Content-Encoding']).to eq('br') + expect(dispatcher.response['Vary']).to eq('Accept-Encoding') + end + + it 'sets Content-Encoding: gzip when compression enabled and client accepts gzip only' do + request = build_request('/events', headers: { 'HTTP_ACCEPT_ENCODING' => 'gzip' }) + dispatcher = Datastar.new(request:, response:, view_context:, compression: true) + + expect(dispatcher.response['Content-Encoding']).to eq('gzip') + expect(dispatcher.response['Vary']).to eq('Accept-Encoding') + end + + it 'does not set Content-Encoding when compression enabled but no Accept-Encoding' do + request = build_request('/events') + dispatcher = Datastar.new(request:, response:, view_context:, compression: true) + + expect(dispatcher.response['Content-Encoding']).to be_nil + expect(dispatcher.response['Vary']).to be_nil + end + + it 'does not set Content-Encoding when compression disabled' do + request = build_request('/events', headers: { 'HTTP_ACCEPT_ENCODING' => 'br, gzip' }) + dispatcher = Datastar.new(request:, response:, view_context:, compression: false) + + expect(dispatcher.response['Content-Encoding']).to be_nil + end + + it 'streams gzip-compressed data that decompresses correctly' do + request = build_request('/events', headers: { 'HTTP_ACCEPT_ENCODING' => 'gzip' }) + dispatcher = Datastar.new(request:, response:, view_context:, compression: true, heartbeat: false) + + dispatcher.patch_signals(foo: 'bar') + + raw_socket = BinarySocket.new + dispatcher.response.body.call(raw_socket) + + decompressed = Zlib::Inflate.new(31).inflate(raw_socket.bytes) + expect(decompressed).to include('datastar-patch-signals') + expect(decompressed).to include('"foo":"bar"') + end + + it 'streams brotli-compressed data that decompresses correctly' do + skip 'brotli gem not available' unless Datastar::EncodingNegotiation.brotli_available? + require 'brotli' + + request = build_request('/events', headers: { 'HTTP_ACCEPT_ENCODING' => 'br' }) + dispatcher = Datastar.new(request:, response:, view_context:, compression: true, heartbeat: false) + + dispatcher.patch_signals(foo: 'bar') + + raw_socket = BinarySocket.new + dispatcher.response.body.call(raw_socket) + + decompressed = Brotli.inflate(raw_socket.bytes) + expect(decompressed).to include('datastar-patch-signals') + expect(decompressed).to include('"foo":"bar"') + end + + it 'respects compression_preferred option' do + request = build_request('/events', headers: { 'HTTP_ACCEPT_ENCODING' => 'br, gzip' }) + dispatcher = Datastar.new(request:, response:, view_context:, compression: true, compression_preferred: :gzip) + + expect(dispatcher.response['Content-Encoding']).to eq('gzip') + end + + it 'respects compression as array of enabled encodings' do + request = build_request('/events', headers: { 'HTTP_ACCEPT_ENCODING' => 'br, gzip' }) + dispatcher = Datastar.new(request:, response:, view_context:, compression: [:gzip]) + + expect(dispatcher.response['Content-Encoding']).to eq('gzip') + end + end + private + # Binary socket for compression tests + class BinarySocket + attr_reader :closed + + def initialize + @buffer = String.new(encoding: Encoding::BINARY) + @closed = false + end + + def <<(data) + @buffer << data.b + self + end + + def close + @closed = true + end + + def bytes + @buffer + end + end + def build_request(path, method: 'GET', body: nil, content_type: 'application/json', accept: 'text/event-stream', headers: {}) headers = { 'HTTP_ACCEPT' => accept, diff --git a/spec/encoding_negotiation_spec.rb b/spec/encoding_negotiation_spec.rb new file mode 100644 index 0000000..1b44c04 --- /dev/null +++ b/spec/encoding_negotiation_spec.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'datastar' +require 'rack' + +RSpec.describe Datastar::EncodingNegotiation do + after { described_class.reset_brotli_cache! } + + describe '.negotiate' do + it 'returns nil when compression is disabled' do + request = build_request('Accept-Encoding' => 'br, gzip') + result = described_class.negotiate(request, preferred: :br, enabled: false) + expect(result).to be_nil + end + + it 'returns nil when no Accept-Encoding header' do + request = build_request + result = described_class.negotiate(request, preferred: :br, enabled: true) + expect(result).to be_nil + end + + it 'returns :gzip when client accepts gzip' do + request = build_request('Accept-Encoding' => 'gzip') + result = described_class.negotiate(request, preferred: :gzip, enabled: true) + expect(result).to eq(:gzip) + end + + it 'returns preferred encoding when both are accepted' do + request = build_request('Accept-Encoding' => 'br, gzip') + if described_class.brotli_available? + result = described_class.negotiate(request, preferred: :br, enabled: true) + expect(result).to eq(:br) + else + result = described_class.negotiate(request, preferred: :br, enabled: true) + expect(result).to eq(:gzip) + end + end + + it 'falls back to gzip when brotli preferred but unavailable' do + # Simulate brotli being unavailable + allow(described_class).to receive(:brotli_available?).and_return(false) + described_class.reset_brotli_cache! + + request = build_request('Accept-Encoding' => 'br, gzip') + result = described_class.negotiate(request, preferred: :br, enabled: true) + expect(result).to eq(:gzip) + end + + it 'returns nil when client encoding not in enabled list' do + request = build_request('Accept-Encoding' => 'br') + result = described_class.negotiate(request, preferred: :br, enabled: [:gzip]) + expect(result).to be_nil + end + + it 'respects enabled array' do + request = build_request('Accept-Encoding' => 'br, gzip') + result = described_class.negotiate(request, preferred: :br, enabled: [:gzip]) + expect(result).to eq(:gzip) + end + + it 'returns nil when Accept-Encoding has q=0 for all' do + request = build_request('Accept-Encoding' => 'gzip;q=0, br;q=0') + result = described_class.negotiate(request, preferred: :br, enabled: true) + expect(result).to be_nil + end + + it 'handles Accept-Encoding with quality values' do + request = build_request('Accept-Encoding' => 'gzip;q=1.0, br;q=0.5') + result = described_class.negotiate(request, preferred: :gzip, enabled: true) + expect(result).to eq(:gzip) + end + end + + describe '.brotli_available?' do + it 'returns a boolean' do + described_class.reset_brotli_cache! + expect(described_class.brotli_available?).to be(true).or be(false) + end + + it 'memoizes the result' do + described_class.reset_brotli_cache! + result1 = described_class.brotli_available? + result2 = described_class.brotli_available? + expect(result1).to eq(result2) + end + end + + private + + def build_request(headers = {}) + env = Rack::MockRequest.env_for('/', headers.transform_keys { |k| "HTTP_#{k.upcase.tr('-', '_')}" }) + Rack::Request.new(env) + end +end From b41b302bdbeedc27a73b0f9e3acb330245b7a86c Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Mon, 16 Mar 2026 22:17:59 +0000 Subject: [PATCH 2/4] Refactor into self-contained Compressor interface --- README.md | 89 +++++------ lib/datastar.rb | 3 +- lib/datastar/compressed_socket.rb | 71 --------- lib/datastar/compression_config.rb | 167 ++++++++++++++++++++ lib/datastar/compressor/brotli.rb | 56 +++++++ lib/datastar/compressor/gzip.rb | 60 ++++++++ lib/datastar/configuration.rb | 12 +- lib/datastar/dispatcher.rb | 28 +--- lib/datastar/encoding_negotiation.rb | 92 ----------- spec/compressed_socket_spec.rb | 10 +- spec/compression_config_spec.rb | 219 +++++++++++++++++++++++++++ spec/dispatcher_spec.rb | 9 +- spec/encoding_negotiation_spec.rb | 94 ------------ 13 files changed, 571 insertions(+), 339 deletions(-) delete mode 100644 lib/datastar/compressed_socket.rb create mode 100644 lib/datastar/compression_config.rb create mode 100644 lib/datastar/compressor/brotli.rb create mode 100644 lib/datastar/compressor/gzip.rb delete mode 100644 lib/datastar/encoding_negotiation.rb create mode 100644 spec/compression_config_spec.rb delete mode 100644 spec/encoding_negotiation_spec.rb diff --git a/README.md b/README.md index 5640de8..73cee12 100644 --- a/README.md +++ b/README.md @@ -261,6 +261,26 @@ datastar.stream do |sse| end ``` +### Global configuration + +```ruby +Datastar.configure do |config| + # Global on_error callback + # Can be overriden on specific instances + config.on_error do |exception| + Sentry.notify(exception) + end + + # Global heartbeat interval (or false, to disable) + # Can be overriden on specific instances + config.heartbeat = 0.3 + + # Enable compression for SSE streams (default: false) + # See the Compression section below for details + config.compression = true +end +``` + ### Compression SSE data (JSON + HTML) is highly compressible, and long-lived connections benefit significantly from compression. This SDK supports opt-in Brotli and gzip compression for SSE streams. @@ -285,7 +305,7 @@ When enabled, the SDK negotiates compression with the client via the `Accept-Enc #### Brotli vs gzip -Brotli (`:br`) is preferred by default as it offers better compression ratios. If the optional [`brotli`](https://github.com/miyucy/brotli) gem is not installed, gzip is used as the fallback. Gzip uses Ruby built-in `zlib` and requires no extra dependencies. +Brotli (`:br`) is preferred by default as it offers better compression ratios. It requires the host app to require the [`brotli`](https://github.com/miyucy/brotli) gem. Gzip uses Ruby built-in `zlib` and requires no extra dependencies. To use Brotli, add the gem to your `Gemfile`: @@ -298,15 +318,14 @@ gem 'brotli' ```ruby Datastar.configure do |config| # Enable compression (default: false) - # true enables both :br and :gzip - # Or pass an array to limit: [:gzip] + # true enables both :br and :gzip (br preferred) config.compression = true - # Preferred encoding when client supports multiple (default: :br) - config.compression_preferred = :br + # Or pass an array of encodings (first = preferred) + config.compression = [:br, :gzip] - # Encoder-specific options passed to the compressor (default: {}) - config.compression_options = { quality: 5 } + # Per-encoder options via [symbol, options] pairs + config.compression = [[:br, { quality: 5 }], :gzip] end ``` @@ -315,57 +334,41 @@ You can also set these per-instance: ```ruby datastar = Datastar.new( request:, response:, view_context:, - compression: [:gzip], - compression_preferred: :gzip, - compression_options: { level: 1 } + compression: [:gzip] # only gzip, no brotli +) + +# Or with per-encoder options +datastar = Datastar.new( + request:, response:, view_context:, + compression: [[:gzip, { level: 1 }]] ) ``` -#### Compressor options +#### Per-encoder options -`compression_options` are passed directly to the underlying compressor. Available options depend on the negotiated encoding. +Options are passed directly to the underlying compressor via the array form. Available options depend on the encoder. **Gzip** (via `Zlib::Deflate`): -| Option | Default | Description | -|--------|---------|-------------| -| `:level` | `Zlib::DEFAULT_COMPRESSION` | Compression level (0-9). 0 = none, 1 = fastest, 9 = smallest. `Zlib::BEST_SPEED` and `Zlib::BEST_COMPRESSION` also work. | -| `:mem_level` | `8` | Memory usage (1-9). Higher uses more memory for better compression. | -| `:strategy` | `Zlib::DEFAULT_STRATEGY` | Algorithm strategy. Alternatives: `Zlib::FILTERED`, `Zlib::HUFFMAN_ONLY`, `Zlib::RLE`, `Zlib::FIXED`. | +| Option | Default | Description | +| ------------ | --------------------------- | ------------------------------------------------------------ | +| `:level` | `Zlib::DEFAULT_COMPRESSION` | Compression level (0-9). 0 = none, 1 = fastest, 9 = smallest. `Zlib::BEST_SPEED` and `Zlib::BEST_COMPRESSION` also work. | +| `:mem_level` | `8` | Memory usage (1-9). Higher uses more memory for better compression. | +| `:strategy` | `Zlib::DEFAULT_STRATEGY` | Algorithm strategy. Alternatives: `Zlib::FILTERED`, `Zlib::HUFFMAN_ONLY`, `Zlib::RLE`, `Zlib::FIXED`. | **Brotli** (via `Brotli::Compressor`, requires the `brotli` gem): -| Option | Default | Description | -|--------|---------|-------------| -| `:quality` | `11` | Compression quality (0-11). Lower is faster, higher compresses better. | -| `:lgwin` | `22` | Base-2 log of sliding window size (10-24). | -| `:lgblock` | `0` (auto) | Base-2 log of max input block size (16-24, or 0 for auto). | -| `:mode` | `:generic` | Compression mode: `:generic`, `:text`, or `:font`. `:text` is a good choice for SSE (UTF-8 HTML/JSON). | +| Option | Default | Description | +| ---------- | ---------- | ------------------------------------------------------------ | +| `:quality` | `11` | Compression quality (0-11). Lower is faster, higher compresses better. | +| `:lgwin` | `22` | Base-2 log of sliding window size (10-24). | +| `:lgblock` | `0` (auto) | Base-2 log of max input block size (16-24, or 0 for auto). | +| `:mode` | `:generic` | Compression mode: `:generic`, `:text`, or `:font`. `:text` is a good choice for SSE (UTF-8 HTML/JSON). | #### Proxy considerations Even with `X-Accel-Buffering: no` (set by default), some proxies like Nginx may buffer compressed responses. You may need to add `proxy_buffering off` to your Nginx configuration when using compression with SSE. -### Global configuration - -```ruby -Datastar.configure do |config| - # Global on_error callback - # Can be overriden on specific instances - config.on_error do |exception| - Sentry.notify(exception) - end - - # Global heartbeat interval (or false, to disable) - # Can be overriden on specific instances - config.heartbeat = 0.3 - - # Enable compression for SSE streams (default: false) - # See the Compression section above for details - config.compression = true -end -``` - ### Rendering Rails templates In Rails, make sure to initialize Datastar with the `view_context` in a controller. diff --git a/lib/datastar.rb b/lib/datastar.rb index e9d9dae..e51db29 100644 --- a/lib/datastar.rb +++ b/lib/datastar.rb @@ -25,8 +25,7 @@ def self.from_rack_env(env, view_context: nil) end require_relative 'datastar/configuration' -require_relative 'datastar/compressed_socket' -require_relative 'datastar/encoding_negotiation' +require_relative 'datastar/compression_config' require_relative 'datastar/dispatcher' require_relative 'datastar/server_sent_event_generator' require_relative 'datastar/railtie' if defined?(Rails::Railtie) diff --git a/lib/datastar/compressed_socket.rb b/lib/datastar/compressed_socket.rb deleted file mode 100644 index 442c3ff..0000000 --- a/lib/datastar/compressed_socket.rb +++ /dev/null @@ -1,71 +0,0 @@ -# frozen_string_literal: true - -module Datastar - # Decorator that wraps a socket and compresses data before writing. - # Supports Brotli and gzip compression. - # Used internally by Dispatcher when compression is negotiated. - module CompressedSocket - # Brotli compression using the `brotli` gem. - # Options are passed directly to Brotli::Compressor.new: - # :quality - Compression quality (0-11, default: 11). Lower is faster, higher compresses better. - # :lgwin - Base-2 log of the sliding window size (10-24, default: 22). - # :lgblock - Base-2 log of the maximum input block size (16-24, 0 = auto, default: 0). - # :mode - Compression mode (:generic, :text, or :font, default: :generic). - # Use :text for UTF-8 formatted text (HTML, JSON — good for SSE). - class Brotli - def initialize(socket, options = {}) - require 'brotli' - @socket = socket - @compressor = ::Brotli::Compressor.new(options) - end - - def <<(data) - compressed = @compressor.process(data) - @socket << compressed if compressed && !compressed.empty? - flushed = @compressor.flush - @socket << flushed if flushed && !flushed.empty? - self - end - - def close - final = @compressor.finish - @socket << final if final && !final.empty? - @socket.close - end - end - - # Gzip compression using Ruby's built-in zlib. - # Options: - # :level - Compression level (0-9, default: Zlib::DEFAULT_COMPRESSION). - # 0 = no compression, 1 = best speed, 9 = best compression. - # Zlib::BEST_SPEED (1) and Zlib::BEST_COMPRESSION (9) are also available. - # :mem_level - Memory usage level (1-9, default: 8). Higher uses more memory for better compression. - # :strategy - Compression strategy (default: Zlib::DEFAULT_STRATEGY). - # Zlib::FILTERED, Zlib::HUFFMAN_ONLY, Zlib::RLE, Zlib::FIXED are also available. - class Gzip - def initialize(socket, options = {}) - require 'zlib' - level = options.fetch(:level, Zlib::DEFAULT_COMPRESSION) - mem_level = options.fetch(:mem_level, Zlib::DEF_MEM_LEVEL) - strategy = options.fetch(:strategy, Zlib::DEFAULT_STRATEGY) - # Use raw deflate with gzip wrapping (window_bits 31 = 15 + 16) - @socket = socket - @deflate = Zlib::Deflate.new(level, 31, mem_level, strategy) - end - - def <<(data) - compressed = @deflate.deflate(data, Zlib::SYNC_FLUSH) - @socket << compressed if compressed && !compressed.empty? - self - end - - def close - final = @deflate.finish - @socket << final if final && !final.empty? - @socket.close - ensure - @deflate.close - end - end - end -end diff --git a/lib/datastar/compression_config.rb b/lib/datastar/compression_config.rb new file mode 100644 index 0000000..6eec7aa --- /dev/null +++ b/lib/datastar/compression_config.rb @@ -0,0 +1,167 @@ +# frozen_string_literal: true + +require 'set' + +module Datastar + module Compressor + # Null compressor — no-op, used when compression is disabled or no match. + class Null + def encoding = nil + def wrap_socket(socket) = socket + def prepare_response(_response) = nil + end + + NONE = Null.new.freeze + end + + # Immutable value object that holds an ordered list of pre-built compressors + # and negotiates the best one for a given request. + # + # Use {.build} to create instances from user-facing configuration values. + # The first compressor in the list is preferred when the client supports multiple. + # + # @example Via global configuration + # Datastar.configure do |config| + # config.compression = true # [:br, :gzip] with default options + # config.compression = [:br, :gzip] # preferred = first in list + # config.compression = [[:br, { quality: 5 }], :gzip] # per-encoder options + # end + # + # @example Per-request negotiation (used internally by Dispatcher) + # compressor = Datastar.config.compression.negotiate(request) + # compressor.prepare_response(response) + # socket = compressor.wrap_socket(raw_socket) + class CompressionConfig + ACCEPT_ENCODING = 'HTTP_ACCEPT_ENCODING' + BLANK_HASH = {}.freeze + + # Build a {CompressionConfig} from various user-facing input forms. + # + # @param input [Boolean, Array, CompressionConfig] + # - +false+ / +nil+ — compression disabled (empty compressor list) + # - +true+ — enable +:br+ and +:gzip+ with default options + # - +Array+ — enable listed encodings with default options, e.g. +[:gzip]+ + # - +Array+ — enable with per-encoder options, + # e.g. +[[:br, { quality: 5 }], :gzip]+ + # - +CompressionConfig+ — returned as-is + # @return [CompressionConfig] + # @raise [ArgumentError] if +input+ is not a recognised form + # @raise [LoadError] if a requested encoder's gem is not available (e.g. +brotli+) + # + # @example Disable compression + # CompressionConfig.build(false) + # + # @example Enable all supported encodings + # CompressionConfig.build(true) + # + # @example Gzip only, with custom level + # CompressionConfig.build([[:gzip, { level: 1 }]]) + def self.build(input) + case input + when CompressionConfig + input + when false, nil + new([]) + when true + new([build_compressor(:br), build_compressor(:gzip)]) + when Array + compressors = input.map do |entry| + case entry + when Symbol + build_compressor(entry) + when Array + name, options = entry + build_compressor(name, options || BLANK_HASH) + else + raise ArgumentError, "Invalid compression entry: #{entry.inspect}. Expected Symbol or [Symbol, Hash]." + end + end + new(compressors) + else + raise ArgumentError, "Invalid compression value: #{input.inspect}. Expected true, false, or Array." + end + end + + def self.build_compressor(name, options = BLANK_HASH) + case name + when :br + require_relative 'compressor/brotli' + Compressor::Brotli.new(options) + when :gzip + require_relative 'compressor/gzip' + Compressor::Gzip.new(options) + else + raise ArgumentError, "Unknown compressor: #{name.inspect}. Expected :br or :gzip." + end + end + private_class_method :build_compressor + + # @param compressors [Array] + # ordered list of pre-built compressor instances. First = preferred. + def initialize(compressors) + @compressors = compressors.freeze + freeze + end + + # Whether any compressors are configured. + # + # @return [Boolean] + # + # @example + # CompressionConfig.build(false).enabled? # => false + # CompressionConfig.build(true).enabled? # => true + def enabled? + @compressors.any? + end + + # Negotiate compression with the client based on the +Accept-Encoding+ header. + # + # Iterates the configured compressors in order (first = preferred) and returns + # the first one whose encoding the client accepts. Returns {Compressor::NONE} + # when compression is disabled, the header is absent, or no match is found. + # + # No objects are created per-request — compressors are pre-built and reused. + # + # @param request [Rack::Request] + # @return [Compressor::Gzip, Compressor::Brotli, Compressor::Null] + # + # @example + # config = CompressionConfig.build([:gzip, :br]) + # compressor = config.negotiate(request) + # compressor.prepare_response(response) + # socket = compressor.wrap_socket(raw_socket) + def negotiate(request) + return Compressor::NONE unless enabled? + + accepted = parse_accept_encoding(request.get_header(ACCEPT_ENCODING).to_s) + return Compressor::NONE if accepted.empty? + + @compressors.each do |compressor| + return compressor if accepted.include?(compressor.encoding) + end + + Compressor::NONE + end + + private + + # Parse Accept-Encoding header into a set of encoding symbols + # @param header [String] + # @return [Set] + def parse_accept_encoding(header) + return Set.new if header.empty? + + encodings = Set.new + header.split(',').each do |part| + encoding, quality = part.strip.split(';', 2) + encoding = encoding.strip.downcase + if quality + q_val = quality.strip.match(/q=(\d+\.?\d*)/) + next if q_val && q_val[1].to_f == 0 + end + encodings << encoding.to_sym + end + encodings + end + end +end diff --git a/lib/datastar/compressor/brotli.rb b/lib/datastar/compressor/brotli.rb new file mode 100644 index 0000000..698bba3 --- /dev/null +++ b/lib/datastar/compressor/brotli.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'brotli' + +module Datastar + module Compressor + # Brotli compressor — built once at config time, reused across requests. + # Eagerly requires the brotli gem; raises LoadError at boot if missing. + class Brotli + attr_reader :encoding + + def initialize(options) + @options = options.freeze + @encoding = :br + freeze + end + + def prepare_response(response) + response.headers['Content-Encoding'] = 'br' + response.headers['Vary'] = 'Accept-Encoding' + end + + def wrap_socket(socket) + CompressedSocket.new(socket, @options) + end + + # Brotli compressed socket using the `brotli` gem. + # Options are passed directly to Brotli::Compressor.new: + # :quality - Compression quality (0-11, default: 11). Lower is faster, higher compresses better. + # :lgwin - Base-2 log of the sliding window size (10-24, default: 22). + # :lgblock - Base-2 log of the maximum input block size (16-24, 0 = auto, default: 0). + # :mode - Compression mode (:generic, :text, or :font, default: :generic). + # Use :text for UTF-8 formatted text (HTML, JSON — good for SSE). + class CompressedSocket + def initialize(socket, options = {}) + @socket = socket + @compressor = ::Brotli::Compressor.new(options) + end + + def <<(data) + compressed = @compressor.process(data) + @socket << compressed if compressed && !compressed.empty? + flushed = @compressor.flush + @socket << flushed if flushed && !flushed.empty? + self + end + + def close + final = @compressor.finish + @socket << final if final && !final.empty? + @socket.close + end + end + end + end +end diff --git a/lib/datastar/compressor/gzip.rb b/lib/datastar/compressor/gzip.rb new file mode 100644 index 0000000..21b3b6d --- /dev/null +++ b/lib/datastar/compressor/gzip.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'zlib' + +module Datastar + module Compressor + # Gzip compressor — built once at config time, reused across requests. + class Gzip + attr_reader :encoding + + def initialize(options) + @options = options.freeze + @encoding = :gzip + freeze + end + + def prepare_response(response) + response.headers['Content-Encoding'] = 'gzip' + response.headers['Vary'] = 'Accept-Encoding' + end + + def wrap_socket(socket) + CompressedSocket.new(socket, @options) + end + + # Gzip compressed socket using Ruby's built-in zlib. + # Options: + # :level - Compression level (0-9, default: Zlib::DEFAULT_COMPRESSION). + # 0 = no compression, 1 = best speed, 9 = best compression. + # Zlib::BEST_SPEED (1) and Zlib::BEST_COMPRESSION (9) also work. + # :mem_level - Memory usage level (1-9, default: 8). Higher uses more memory for better compression. + # :strategy - Compression strategy (default: Zlib::DEFAULT_STRATEGY). + # Zlib::FILTERED, Zlib::HUFFMAN_ONLY, Zlib::RLE, Zlib::FIXED are also available. + class CompressedSocket + def initialize(socket, options = {}) + level = options.fetch(:level, Zlib::DEFAULT_COMPRESSION) + mem_level = options.fetch(:mem_level, Zlib::DEF_MEM_LEVEL) + strategy = options.fetch(:strategy, Zlib::DEFAULT_STRATEGY) + # Use raw deflate with gzip wrapping (window_bits 31 = 15 + 16) + @socket = socket + @deflate = Zlib::Deflate.new(level, 31, mem_level, strategy) + end + + def <<(data) + compressed = @deflate.deflate(data, Zlib::SYNC_FLUSH) + @socket << compressed if compressed && !compressed.empty? + self + end + + def close + final = @deflate.finish + @socket << final if final && !final.empty? + @socket.close + ensure + @deflate.close + end + end + end + end +end diff --git a/lib/datastar/configuration.rb b/lib/datastar/configuration.rb index bb75b5f..12cb0d4 100644 --- a/lib/datastar/configuration.rb +++ b/lib/datastar/configuration.rb @@ -34,8 +34,8 @@ class Configuration RACK_FINALIZE = ->(_view_context, response) { response.finish } DEFAULT_HEARTBEAT = 3 - attr_accessor :executor, :error_callback, :finalize, :heartbeat, :logger, - :compression, :compression_preferred, :compression_options + attr_accessor :executor, :error_callback, :finalize, :heartbeat, :logger + attr_reader :compression def initialize @executor = ThreadExecutor.new @@ -45,9 +45,11 @@ def initialize @error_callback = proc do |e| @logger.error("#{e.class} (#{e.message}):\n#{e.backtrace.join("\n")}") end - @compression = false - @compression_preferred = :br - @compression_options = {} + @compression = CompressionConfig.build(false) + end + + def compression=(value) + @compression = value.is_a?(CompressionConfig) ? value : CompressionConfig.build(value) end def on_error(callable = nil, &block) diff --git a/lib/datastar/dispatcher.rb b/lib/datastar/dispatcher.rb index fb2960b..053ee96 100644 --- a/lib/datastar/dispatcher.rb +++ b/lib/datastar/dispatcher.rb @@ -45,9 +45,7 @@ def initialize( error_callback: Datastar.config.error_callback, finalize: Datastar.config.finalize, heartbeat: Datastar.config.heartbeat, - compression: Datastar.config.compression, - compression_preferred: Datastar.config.compression_preferred, - compression_options: Datastar.config.compression_options + compression: Datastar.config.compression ) @on_connect = [] @on_client_disconnect = [] @@ -73,18 +71,9 @@ def initialize( @heartbeat_on = false # Negotiate compression - @encoding = EncodingNegotiation.negotiate( - request, - preferred: compression_preferred, - enabled: compression - ) - @compression_options = compression_options - - if @encoding - encoding_value = @encoding == :br ? 'br' : 'gzip' - @response.headers['Content-Encoding'] = encoding_value - @response.headers['Vary'] = 'Accept-Encoding' - end + compression = CompressionConfig.build(compression) unless compression.is_a?(CompressionConfig) + @compressor = compression.negotiate(request) + @compressor.prepare_response(@response) end # Check if the request accepts SSE responses @@ -383,14 +372,7 @@ def stream_many(streamer) # @param socket [IO] # @return [CompressedSocket, IO] def wrap_socket(socket) - case @encoding - when :br - CompressedSocket::Brotli.new(socket, @compression_options) - when :gzip - CompressedSocket::Gzip.new(socket, @compression_options) - else - socket - end + @compressor.wrap_socket(socket) end # Handle errors caught during streaming diff --git a/lib/datastar/encoding_negotiation.rb b/lib/datastar/encoding_negotiation.rb deleted file mode 100644 index 0d35d37..0000000 --- a/lib/datastar/encoding_negotiation.rb +++ /dev/null @@ -1,92 +0,0 @@ -# frozen_string_literal: true - -module Datastar - module EncodingNegotiation - ACCEPT_ENCODING = 'HTTP_ACCEPT_ENCODING' - - # Negotiate compression encoding based on request headers and configuration. - # - # @param request [Rack::Request] - # @param preferred [Symbol] preferred encoding (:br or :gzip) - # @param enabled [Boolean, Array] compression config - # @return [Symbol, nil] :br, :gzip, or nil - def self.negotiate(request, preferred:, enabled:) - return nil unless enabled - - accepted = parse_accept_encoding(request.get_header(ACCEPT_ENCODING).to_s) - return nil if accepted.empty? - - available = if enabled == true - %i[br gzip] - else - Array(enabled) - end - - # Try preferred encoding first - if available.include?(preferred) && accepted.include?(preferred) - return preferred if encoding_available?(preferred) - end - - # Fall back to other available encodings - (available - [preferred]).each do |enc| - return enc if accepted.include?(enc) && encoding_available?(enc) - end - - nil - end - - # Check if the encoding implementation is available - # @param encoding [Symbol] - # @return [Boolean] - def self.encoding_available?(encoding) - case encoding - when :br - brotli_available? - when :gzip - true # zlib is part of Ruby stdlib - else - false - end - end - - # Check if the brotli gem is installed (memoized) - # @return [Boolean] - def self.brotli_available? - return @brotli_available if defined?(@brotli_available) - - @brotli_available = begin - require 'brotli' - true - rescue LoadError - false - end - end - - # Reset memoized brotli availability (for testing) - def self.reset_brotli_cache! - remove_instance_variable(:@brotli_available) if defined?(@brotli_available) - end - - # Parse Accept-Encoding header into a set of encoding symbols - # @param header [String] - # @return [Set] - def self.parse_accept_encoding(header) - return Set.new if header.empty? - - encodings = Set.new - header.split(',').each do |part| - encoding, quality = part.strip.split(';', 2) - encoding = encoding.strip.downcase - # Skip if quality is explicitly 0 - if quality - q_val = quality.strip.match(/q=(\d+\.?\d*)/) - next if q_val && q_val[1].to_f == 0 - end - encodings << encoding.to_sym - end - encodings - end - - private_class_method :parse_accept_encoding - end -end diff --git a/spec/compressed_socket_spec.rb b/spec/compressed_socket_spec.rb index 9ce9a55..2f03805 100644 --- a/spec/compressed_socket_spec.rb +++ b/spec/compressed_socket_spec.rb @@ -1,9 +1,11 @@ # frozen_string_literal: true require 'datastar' +require 'datastar/compressor/gzip' +require 'datastar/compressor/brotli' rescue nil require 'zlib' -RSpec.describe Datastar::CompressedSocket do +RSpec.describe 'Compressor compressed sockets' do let(:raw_socket) { StringSocket.new } let(:sse_data) { "event: datastar-patch-signals\ndata: signals {\"foo\":\"bar\"}\n\n" } @@ -30,7 +32,7 @@ def bytes end end - describe Datastar::CompressedSocket::Gzip do + describe Datastar::Compressor::Gzip::CompressedSocket do subject(:socket) { described_class.new(raw_socket) } it 'compresses data and decompresses to original' do @@ -78,12 +80,12 @@ def bytes end end - describe Datastar::CompressedSocket::Brotli do + describe 'Datastar::Compressor::Brotli::CompressedSocket' do before do skip 'brotli gem not available' unless brotli_available? end - subject(:socket) { described_class.new(raw_socket) } + subject(:socket) { Datastar::Compressor::Brotli::CompressedSocket.new(raw_socket) } it 'compresses data and decompresses to original' do socket << sse_data diff --git a/spec/compression_config_spec.rb b/spec/compression_config_spec.rb new file mode 100644 index 0000000..f873163 --- /dev/null +++ b/spec/compression_config_spec.rb @@ -0,0 +1,219 @@ +# frozen_string_literal: true + +require 'datastar' +require 'rack' + +RSpec.describe Datastar::CompressionConfig do + describe '.build' do + it 'returns a disabled config for false' do + config = described_class.build(false) + expect(config.enabled?).to be(false) + end + + it 'returns a disabled config for nil' do + config = described_class.build(nil) + expect(config.enabled?).to be(false) + end + + it 'returns an enabled config with br and gzip for true' do + config = described_class.build(true) + expect(config.enabled?).to be(true) + end + + it 'builds compressors from an array of symbols' do + config = described_class.build([:gzip]) + expect(config.enabled?).to be(true) + + request = build_request('Accept-Encoding' => 'gzip') + compressor = config.negotiate(request) + expect(compressor.encoding).to eq(:gzip) + end + + it 'builds compressors from nested array with options' do + config = described_class.build([[:gzip, { level: 1 }]]) + expect(config.enabled?).to be(true) + + request = build_request('Accept-Encoding' => 'gzip') + compressor = config.negotiate(request) + expect(compressor.encoding).to eq(:gzip) + end + + it 'builds mixed array of symbols and [symbol, hash] pairs' do + config = described_class.build([[:br, { quality: 5 }], :gzip]) + expect(config.enabled?).to be(true) + end + + it 'returns the input if already a CompressionConfig' do + original = described_class.build(true) + expect(described_class.build(original)).to equal(original) + end + + it 'raises ArgumentError for invalid input' do + expect { described_class.build('invalid') }.to raise_error(ArgumentError) + end + + it 'raises ArgumentError for unknown compressor symbol' do + expect { described_class.build([:deflate]) }.to raise_error(ArgumentError, /Unknown compressor/) + end + + it 'raises LoadError at build time if brotli compressor file cannot be loaded' do + allow(described_class).to receive(:build_compressor).and_call_original + allow(described_class).to receive(:build_compressor).with(:br).and_raise(LoadError) + expect { described_class.build([:br]) }.to raise_error(LoadError) + end + end + + describe '#negotiate' do + it 'returns Null compressor when disabled' do + config = described_class.build(false) + request = build_request('Accept-Encoding' => 'br, gzip') + compressor = config.negotiate(request) + expect(compressor).to be_a(Datastar::Compressor::Null) + expect(compressor.encoding).to be_nil + end + + it 'returns Null compressor when no Accept-Encoding header' do + config = described_class.build(true) + request = build_request + compressor = config.negotiate(request) + expect(compressor).to be_a(Datastar::Compressor::Null) + end + + it 'returns gzip compressor when client accepts gzip' do + config = described_class.build([:gzip]) + request = build_request('Accept-Encoding' => 'gzip') + compressor = config.negotiate(request) + expect(compressor.encoding).to eq(:gzip) + end + + it 'returns first compressor (preferred) when client supports both' do + begin + require 'brotli' + rescue LoadError + skip 'brotli gem not available' + end + + config = described_class.build([:br, :gzip]) + request = build_request('Accept-Encoding' => 'br, gzip') + compressor = config.negotiate(request) + expect(compressor.encoding).to eq(:br) + end + + it 'respects list order for preference' do + config = described_class.build([:gzip, :br]) + request = build_request('Accept-Encoding' => 'br, gzip') + compressor = config.negotiate(request) + expect(compressor.encoding).to eq(:gzip) + end + + it 'falls back to second compressor if client does not accept first' do + config = described_class.build([:gzip]) + request = build_request('Accept-Encoding' => 'gzip') + compressor = config.negotiate(request) + expect(compressor.encoding).to eq(:gzip) + end + + it 'returns Null when client encoding not in configured list' do + config = described_class.build([:gzip]) + request = build_request('Accept-Encoding' => 'br') + compressor = config.negotiate(request) + expect(compressor).to be_a(Datastar::Compressor::Null) + end + + it 'returns Null when Accept-Encoding has q=0 for all' do + config = described_class.build(true) + request = build_request('Accept-Encoding' => 'gzip;q=0, br;q=0') + compressor = config.negotiate(request) + expect(compressor).to be_a(Datastar::Compressor::Null) + end + + it 'handles Accept-Encoding with quality values' do + config = described_class.build([:gzip]) + request = build_request('Accept-Encoding' => 'gzip;q=1.0, br;q=0.5') + compressor = config.negotiate(request) + expect(compressor.encoding).to eq(:gzip) + end + end + + describe 'Compressor::NONE' do + subject(:null) { Datastar::Compressor::NONE } + + it 'is a frozen constant' do + expect(null).to be_frozen + expect(null).to equal(Datastar::Compressor::NONE) + end + + it 'returns nil encoding' do + expect(null.encoding).to be_nil + end + + it 'returns the socket unchanged from wrap_socket' do + socket = Object.new + expect(null.wrap_socket(socket)).to equal(socket) + end + + it 'prepare_response is a no-op' do + response = double('response') + expect(null.prepare_response(response)).to be_nil + end + end + + describe 'Compressor::Gzip' do + subject(:compressor) { Datastar::Compressor::Gzip.new({}) } + + it 'has :gzip encoding' do + expect(compressor.encoding).to eq(:gzip) + end + + it 'sets response headers' do + headers = {} + response = double('response', headers: headers) + compressor.prepare_response(response) + expect(headers['Content-Encoding']).to eq('gzip') + expect(headers['Vary']).to eq('Accept-Encoding') + end + + it 'wraps socket in Gzip::CompressedSocket' do + socket = Object.new + wrapped = compressor.wrap_socket(socket) + expect(wrapped).to be_a(Datastar::Compressor::Gzip::CompressedSocket) + end + end + + describe 'Compressor::Brotli' do + before do + begin + require 'brotli' + rescue LoadError + skip 'brotli gem not available' + end + end + + subject(:compressor) { Datastar::Compressor::Brotli.new({}) } + + it 'has :br encoding' do + expect(compressor.encoding).to eq(:br) + end + + it 'sets response headers' do + headers = {} + response = double('response', headers: headers) + compressor.prepare_response(response) + expect(headers['Content-Encoding']).to eq('br') + expect(headers['Vary']).to eq('Accept-Encoding') + end + + it 'wraps socket in Brotli::CompressedSocket' do + socket = Object.new + wrapped = compressor.wrap_socket(socket) + expect(wrapped).to be_a(Datastar::Compressor::Brotli::CompressedSocket) + end + end + + private + + def build_request(headers = {}) + env = Rack::MockRequest.env_for('/', headers.transform_keys { |k| "HTTP_#{k.upcase.tr('-', '_')}" }) + Rack::Request.new(env) + end +end diff --git a/spec/dispatcher_spec.rb b/spec/dispatcher_spec.rb index 7a37dab..1fd8303 100644 --- a/spec/dispatcher_spec.rb +++ b/spec/dispatcher_spec.rb @@ -612,7 +612,7 @@ def self.render_in(view_context) = %(
\n#{view_context} 'br, gzip' }) dispatcher = Datastar.new(request:, response:, view_context:, compression: true) @@ -659,8 +659,7 @@ def self.render_in(view_context) = %(
\n#{view_context} 'br' }) dispatcher = Datastar.new(request:, response:, view_context:, compression: true, heartbeat: false) @@ -675,9 +674,9 @@ def self.render_in(view_context) = %(
\n#{view_context} 'br, gzip' }) - dispatcher = Datastar.new(request:, response:, view_context:, compression: true, compression_preferred: :gzip) + dispatcher = Datastar.new(request:, response:, view_context:, compression: [:gzip, :br]) expect(dispatcher.response['Content-Encoding']).to eq('gzip') end diff --git a/spec/encoding_negotiation_spec.rb b/spec/encoding_negotiation_spec.rb deleted file mode 100644 index 1b44c04..0000000 --- a/spec/encoding_negotiation_spec.rb +++ /dev/null @@ -1,94 +0,0 @@ -# frozen_string_literal: true - -require 'datastar' -require 'rack' - -RSpec.describe Datastar::EncodingNegotiation do - after { described_class.reset_brotli_cache! } - - describe '.negotiate' do - it 'returns nil when compression is disabled' do - request = build_request('Accept-Encoding' => 'br, gzip') - result = described_class.negotiate(request, preferred: :br, enabled: false) - expect(result).to be_nil - end - - it 'returns nil when no Accept-Encoding header' do - request = build_request - result = described_class.negotiate(request, preferred: :br, enabled: true) - expect(result).to be_nil - end - - it 'returns :gzip when client accepts gzip' do - request = build_request('Accept-Encoding' => 'gzip') - result = described_class.negotiate(request, preferred: :gzip, enabled: true) - expect(result).to eq(:gzip) - end - - it 'returns preferred encoding when both are accepted' do - request = build_request('Accept-Encoding' => 'br, gzip') - if described_class.brotli_available? - result = described_class.negotiate(request, preferred: :br, enabled: true) - expect(result).to eq(:br) - else - result = described_class.negotiate(request, preferred: :br, enabled: true) - expect(result).to eq(:gzip) - end - end - - it 'falls back to gzip when brotli preferred but unavailable' do - # Simulate brotli being unavailable - allow(described_class).to receive(:brotli_available?).and_return(false) - described_class.reset_brotli_cache! - - request = build_request('Accept-Encoding' => 'br, gzip') - result = described_class.negotiate(request, preferred: :br, enabled: true) - expect(result).to eq(:gzip) - end - - it 'returns nil when client encoding not in enabled list' do - request = build_request('Accept-Encoding' => 'br') - result = described_class.negotiate(request, preferred: :br, enabled: [:gzip]) - expect(result).to be_nil - end - - it 'respects enabled array' do - request = build_request('Accept-Encoding' => 'br, gzip') - result = described_class.negotiate(request, preferred: :br, enabled: [:gzip]) - expect(result).to eq(:gzip) - end - - it 'returns nil when Accept-Encoding has q=0 for all' do - request = build_request('Accept-Encoding' => 'gzip;q=0, br;q=0') - result = described_class.negotiate(request, preferred: :br, enabled: true) - expect(result).to be_nil - end - - it 'handles Accept-Encoding with quality values' do - request = build_request('Accept-Encoding' => 'gzip;q=1.0, br;q=0.5') - result = described_class.negotiate(request, preferred: :gzip, enabled: true) - expect(result).to eq(:gzip) - end - end - - describe '.brotli_available?' do - it 'returns a boolean' do - described_class.reset_brotli_cache! - expect(described_class.brotli_available?).to be(true).or be(false) - end - - it 'memoizes the result' do - described_class.reset_brotli_cache! - result1 = described_class.brotli_available? - result2 = described_class.brotli_available? - expect(result1).to eq(result2) - end - end - - private - - def build_request(headers = {}) - env = Rack::MockRequest.env_for('/', headers.transform_keys { |k| "HTTP_#{k.upcase.tr('-', '_')}" }) - Rack::Request.new(env) - end -end From 519b7adfd67018276a9d37bc4230ef99f2c54a9b Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Mon, 16 Mar 2026 22:21:18 +0000 Subject: [PATCH 3/4] Compression benchmark bundle exec benchmark/compression.rb --- benchmarks/compression.rb | 251 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 benchmarks/compression.rb diff --git a/benchmarks/compression.rb b/benchmarks/compression.rb new file mode 100644 index 0000000..3966b94 --- /dev/null +++ b/benchmarks/compression.rb @@ -0,0 +1,251 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Benchmark: SSE compression payload sizes +# +# Compares bytes-over-the-wire for no compression, gzip, and brotli +# when streaming large HTML elements via Datastar's SSE protocol. +# +# Usage: +# bundle exec ruby benchmarks/compression.rb +# +# The benchmark patches realistic HTML payloads of increasing size +# through the full Datastar SSE pipeline (ServerSentEventGenerator → +# CompressedSocket → raw socket) and reports the resulting byte sizes +# and compression ratios. + +require 'bundler/setup' +require 'datastar' +require 'datastar/compressor/gzip' +require 'datastar/compressor/brotli' + +# --- Payload generators --------------------------------------------------- + +# A user-row partial, repeated N times inside a . +# Realistic: IDs, data attributes, mixed text, Tailwind-style classes. +def html_table(row_count) + rows = row_count.times.map do |i| + <<~HTML + + #{i + 1} + user-#{i}@example.com + #{%w[Admin Editor Viewer].sample} + 2025-01-#{(i % 28 + 1).to_s.rjust(2, '0')} + + + + + + HTML + end + + <<~HTML + + #{rows.join} + + HTML +end + +# A dashboard card with nested elements — charts placeholder, stats, lists. +def html_dashboard(card_count) + cards = card_count.times.map do |i| + <<~HTML +
+
+

Metric #{i + 1}

+ +#{rand(1..99)}% +
+
#{rand(1_000..99_999).to_s.chars.each_slice(3).map(&:join).join(',')}
+
+ #{8.times.map { |j| "
" }.join("\n ")} +
+
    + #{5.times.map { |j| "
  • Region #{j + 1}#{rand(100..9_999)}
    • " }.join("\n ")} +
+
+ HTML + end + + <<~HTML +
+ #{cards.join} +
+ HTML +end + +# --- Socket that counts bytes -------------------------------------------- + +class ByteCountingSocket + attr_reader :total_bytes + + def initialize + @total_bytes = 0 + end + + def <<(data) + @total_bytes += data.bytesize + self + end + + def close; end +end + +# --- Helpers -------------------------------------------------------------- + +# Pipe an HTML payload through the full SSE + compression stack +# and return the byte count that would go over the wire. +def measure_bytes(html, encoding) + socket = ByteCountingSocket.new + + wrapped = case encoding + when :none then socket + when :gzip then Datastar::Compressor::Gzip::CompressedSocket.new(socket) + when :br then Datastar::Compressor::Brotli::CompressedSocket.new(socket, mode: :text) + end + + generator = Datastar::ServerSentEventGenerator.new( + wrapped, + signals: {}, + view_context: nil + ) + + generator.patch_elements(html) + wrapped.close unless encoding == :none + + socket.total_bytes +end + +def format_bytes(bytes) + if bytes >= 1024 * 1024 + "%.1f MB" % (bytes / (1024.0 * 1024)) + elsif bytes >= 1024 + "%.1f KB" % (bytes / 1024.0) + else + "#{bytes} B" + end +end + +def ratio(original, compressed) + "%.1f%%" % ((1.0 - compressed.to_f / original) * 100) +end + +# --- Run benchmarks ------------------------------------------------------- + +SCENARIOS = [ + ["Table 10 rows", -> { html_table(10) }], + ["Table 50 rows", -> { html_table(50) }], + ["Table 200 rows", -> { html_table(200) }], + ["Table 1000 rows", -> { html_table(1000) }], + ["Dashboard 5 cards", -> { html_dashboard(5) }], + ["Dashboard 20 cards",-> { html_dashboard(20) }], + ["Dashboard 50 cards",-> { html_dashboard(50) }], +] + +ENCODINGS = %i[none gzip br] + +# Header +puts "Datastar SSE Compression Benchmark" +puts "=" * 90 +puts +puts format( + "%-22s %12s %12s %8s %12s %8s", + "Scenario", "No Compress", "Gzip", "Saved", "Brotli", "Saved" +) +puts "-" * 90 + +SCENARIOS.each do |name, generator| + html = generator.call + results = ENCODINGS.map { |enc| [enc, measure_bytes(html, enc)] }.to_h + none = results[:none] + + puts format( + "%-22s %12s %12s %8s %12s %8s", + name, + format_bytes(none), + format_bytes(results[:gzip]), + ratio(none, results[:gzip]), + format_bytes(results[:br]), + ratio(none, results[:br]) + ) +end + +# --- Streaming: multiple SSE events over one connection ------------------- + +# Simulates a long-lived SSE connection where rows are patched individually +# (e.g. a live-updating table). The compressor stays open across events, +# so repeated structure (CSS classes, attribute patterns) compresses +# increasingly well as the dictionary builds up. + +def measure_streaming_bytes(payloads, encoding) + socket = ByteCountingSocket.new + + wrapped = case encoding + when :none then socket + when :gzip then Datastar::Compressor::Gzip::CompressedSocket.new(socket) + when :br then Datastar::Compressor::Brotli::CompressedSocket.new(socket, mode: :text) + end + + generator = Datastar::ServerSentEventGenerator.new( + wrapped, + signals: {}, + view_context: nil + ) + + payloads.each { |html| generator.patch_elements(html) } + wrapped.close unless encoding == :none + + socket.total_bytes +end + +def table_rows(count) + count.times.map do |i| + <<~HTML + + #{i + 1} + user-#{i}@example.com + #{%w[Admin Editor Viewer].sample} + 2025-01-#{(i % 28 + 1).to_s.rjust(2, '0')} + + + + + + HTML + end +end + +puts +puts +puts "Streaming: individual row patches over one SSE connection" +puts "=" * 90 +puts +puts format( + "%-22s %12s %12s %8s %12s %8s", + "Scenario", "No Compress", "Gzip", "Saved", "Brotli", "Saved" +) +puts "-" * 90 + +[10, 50, 200, 1000].each do |count| + payloads = table_rows(count) + results = ENCODINGS.map { |enc| [enc, measure_streaming_bytes(payloads, enc)] }.to_h + none = results[:none] + + puts format( + "%-22s %12s %12s %8s %12s %8s", + "#{count} row patches", + format_bytes(none), + format_bytes(results[:gzip]), + ratio(none, results[:gzip]), + format_bytes(results[:br]), + ratio(none, results[:br]) + ) +end + +puts +puts "Notes:" +puts " - Single-event sizes include full SSE framing (event: / data: prefixes)" +puts " - Gzip: default compression level, gzip framing (window_bits=31)" +puts " - Brotli: default quality (11) with mode: :text" +puts " - Streaming rows: each row is a separate patch_elements SSE event" +puts " over one persistent compressed connection. The compressor dictionary" +puts " builds up across events, improving ratios for repetitive markup." From be4cfee74522d5bf2db82b3e0ff78ef72c7dce97 Mon Sep 17 00:00:00 2001 From: Ismael Celis Date: Mon, 16 Mar 2026 22:52:11 +0000 Subject: [PATCH 4/4] Let missing brotli raise --- spec/compressed_socket_spec.rb | 19 +++---------------- spec/compression_config_spec.rb | 14 -------------- spec/dispatcher_spec.rb | 4 ---- 3 files changed, 3 insertions(+), 34 deletions(-) diff --git a/spec/compressed_socket_spec.rb b/spec/compressed_socket_spec.rb index 2f03805..f208ab6 100644 --- a/spec/compressed_socket_spec.rb +++ b/spec/compressed_socket_spec.rb @@ -2,8 +2,7 @@ require 'datastar' require 'datastar/compressor/gzip' -require 'datastar/compressor/brotli' rescue nil -require 'zlib' +require 'datastar/compressor/brotli' RSpec.describe 'Compressor compressed sockets' do let(:raw_socket) { StringSocket.new } @@ -80,12 +79,8 @@ def bytes end end - describe 'Datastar::Compressor::Brotli::CompressedSocket' do - before do - skip 'brotli gem not available' unless brotli_available? - end - - subject(:socket) { Datastar::Compressor::Brotli::CompressedSocket.new(raw_socket) } + describe Datastar::Compressor::Brotli::CompressedSocket do + subject(:socket) { described_class.new(raw_socket) } it 'compresses data and decompresses to original' do socket << sse_data @@ -119,12 +114,4 @@ def bytes end end - private - - def brotli_available? - require 'brotli' - true - rescue LoadError - false - end end diff --git a/spec/compression_config_spec.rb b/spec/compression_config_spec.rb index f873163..a0e715d 100644 --- a/spec/compression_config_spec.rb +++ b/spec/compression_config_spec.rb @@ -87,12 +87,6 @@ end it 'returns first compressor (preferred) when client supports both' do - begin - require 'brotli' - rescue LoadError - skip 'brotli gem not available' - end - config = described_class.build([:br, :gzip]) request = build_request('Accept-Encoding' => 'br, gzip') compressor = config.negotiate(request) @@ -181,14 +175,6 @@ end describe 'Compressor::Brotli' do - before do - begin - require 'brotli' - rescue LoadError - skip 'brotli gem not available' - end - end - subject(:compressor) { Datastar::Compressor::Brotli.new({}) } it 'has :br encoding' do diff --git a/spec/dispatcher_spec.rb b/spec/dispatcher_spec.rb index 1fd8303..09b6157 100644 --- a/spec/dispatcher_spec.rb +++ b/spec/dispatcher_spec.rb @@ -612,8 +612,6 @@ def self.render_in(view_context) = %(
\n#{view_context} 'br, gzip' }) dispatcher = Datastar.new(request:, response:, view_context:, compression: true) @@ -659,8 +657,6 @@ def self.render_in(view_context) = %(
\n#{view_context} 'br' }) dispatcher = Datastar.new(request:, response:, view_context:, compression: true, heartbeat: false)