From 978f6eaf27e27f0e2484a7aa25ca70750d70f97f Mon Sep 17 00:00:00 2001 From: "Matthew M. Keeler" Date: Thu, 25 Jul 2024 11:45:53 -0400 Subject: [PATCH] feat: Add option to enable compression of event payloads (#291) --- contract-tests/client_entity.rb | 1 + contract-tests/service.rb | 2 + launchdarkly-server-sdk.gemspec | 1 + lib/ldclient-rb/config.rb | 20 ++++++++++ lib/ldclient-rb/impl/event_sender.rb | 14 ++++++- spec/http_util.rb | 23 ++++++----- spec/impl/event_sender_spec.rb | 57 ++++++++++++++++++++-------- 7 files changed, 92 insertions(+), 26 deletions(-) diff --git a/contract-tests/client_entity.rb b/contract-tests/client_entity.rb index 9c840912..203ac923 100644 --- a/contract-tests/client_entity.rb +++ b/contract-tests/client_entity.rb @@ -36,6 +36,7 @@ def initialize(log, config) opts[:private_attributes] = events[:globalPrivateAttributes] opts[:flush_interval] = (events[:flushIntervalMs] / 1_000) unless events[:flushIntervalMs].nil? opts[:omit_anonymous_contexts] = !!events[:omitAnonymousContexts] + opts[:compress_events] = !!events[:enableGzip] else opts[:send_events] = false end diff --git a/contract-tests/service.rb b/contract-tests/service.rb index 954715ee..7db6e0f9 100644 --- a/contract-tests/service.rb +++ b/contract-tests/service.rb @@ -34,6 +34,8 @@ 'secure-mode-hash', 'tags', 'migrations', + 'event-gzip', + 'optional-event-gzip', 'event-sampling', 'context-comparison', 'polling-gzip', diff --git a/launchdarkly-server-sdk.gemspec b/launchdarkly-server-sdk.gemspec index be94a681..26c73c8d 100644 --- a/launchdarkly-server-sdk.gemspec +++ b/launchdarkly-server-sdk.gemspec @@ -39,6 +39,7 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency "concurrent-ruby", "~> 1.1" spec.add_runtime_dependency "ld-eventsource", "2.2.2" spec.add_runtime_dependency "observer", "~> 0.1.2" + spec.add_runtime_dependency "zlib", "~> 3.1" unless RUBY_PLATFORM == "java" # Please keep ld-eventsource dependency as an exact version so that bugfixes to # that LD library are always associated with a new SDK version. diff --git a/lib/ldclient-rb/config.rb b/lib/ldclient-rb/config.rb index 9de83df0..9fb0d8a4 100644 --- a/lib/ldclient-rb/config.rb +++ b/lib/ldclient-rb/config.rb @@ -65,6 +65,7 @@ def initialize(opts = {}) @all_attributes_private = opts[:all_attributes_private] || false @private_attributes = opts[:private_attributes] || [] @send_events = opts.has_key?(:send_events) ? opts[:send_events] : Config.default_send_events + @compress_events = opts.has_key?(:compress_events) ? opts[:compress_events] : Config.default_compress_events @context_keys_capacity = opts[:context_keys_capacity] || Config.default_context_keys_capacity @context_keys_flush_interval = opts[:context_keys_flush_interval] || Config.default_context_keys_flush_interval @data_source = opts[:data_source] @@ -254,6 +255,17 @@ def offline? # attr_reader :send_events + # + # Should the event payload sent to LaunchDarkly use gzip compression. By default this is false to prevent backward + # breaking compatibility issues with older versions of the relay proxy. + # + # Customers not using the relay proxy are strongly encouraged to enable this feature to reduce egress bandwidth + # cost. + # + # @return [Boolean] + # + attr_reader :compress_events + # # The number of context keys that the event processor can remember at any one time. This reduces the # amount of duplicate context details sent in analytics events. @@ -539,6 +551,14 @@ def self.default_send_events true end + # + # The default value for {#compress_events}. + # @return [Boolean] false + # + def self.default_compress_events + false + end + # # The default value for {#context_keys_capacity}. # @return [Integer] 1000 diff --git a/lib/ldclient-rb/impl/event_sender.rb b/lib/ldclient-rb/impl/event_sender.rb index 4f4561d6..c730b04a 100644 --- a/lib/ldclient-rb/impl/event_sender.rb +++ b/lib/ldclient-rb/impl/event_sender.rb @@ -2,6 +2,8 @@ require "securerandom" require "http" +require "stringio" +require "zlib" module LaunchDarkly module Impl @@ -42,14 +44,24 @@ def send_event_data(event_data, description, is_diagnostic) @logger.debug { "[LDClient] sending #{description}: #{event_data}" } headers = {} headers["content-type"] = "application/json" + headers["content-encoding"] = "gzip" if @config.compress_events Impl::Util.default_http_headers(@sdk_key, @config).each { |k, v| headers[k] = v } unless is_diagnostic headers["X-LaunchDarkly-Event-Schema"] = CURRENT_SCHEMA_VERSION.to_s headers["X-LaunchDarkly-Payload-ID"] = payload_id end + + body = event_data + if @config.compress_events + gzip = Zlib::GzipWriter.new(StringIO.new) + gzip << event_data + + body = gzip.close.string + end + response = http_client.request("POST", uri, { headers: headers, - body: event_data, + body: body, }) rescue StandardError => exn @logger.warn { "[LDClient] Error sending events: #{exn.inspect}." } diff --git a/spec/http_util.rb b/spec/http_util.rb index 447c775a..a52e4bb6 100644 --- a/spec/http_util.rb +++ b/spec/http_util.rb @@ -1,14 +1,17 @@ require "webrick" require "webrick/httpproxy" require "webrick/https" +require "stringio" +require "zlib" class StubHTTPServer attr_reader :requests, :port @@next_port = 50000 - def initialize + def initialize(enable_compression: false) @port = StubHTTPServer.next_port + @enable_compression = enable_compression begin base_opts = { BindAddress: '127.0.0.1', @@ -73,14 +76,16 @@ def record_request(req, res) @requests_queue << [req, req.body] end - def await_request - r = @requests_queue.pop - r[0] - end - def await_request_with_body r = @requests_queue.pop - [r[0], r[1]] + request = r[0] + body = r[1] + + return [request, body.to_s] unless @enable_compression + + gz = Zlib::GzipReader.new(StringIO.new(body.to_s)) + + [request, gz.read] end end @@ -90,8 +95,8 @@ def method_missing(*) end end -def with_server(server = nil) - server = StubHTTPServer.new if server.nil? +def with_server(enable_compression: false) + server = StubHTTPServer.new(enable_compression: enable_compression) begin server.start yield server diff --git a/spec/impl/event_sender_spec.rb b/spec/impl/event_sender_spec.rb index c79855bc..e6a971bd 100644 --- a/spec/impl/event_sender_spec.rb +++ b/spec/impl/event_sender_spec.rb @@ -11,24 +11,48 @@ module Impl subject { EventSender } let(:sdk_key) { "sdk_key" } - let(:fake_data) { '{"things":[]}' } + let(:fake_data) { '{"things":[],"stuff":false,"other examples":["you", "me", "us", "we"]}' } - def make_sender(server) - make_sender_with_events_uri(server.base_uri.to_s) + def make_sender(config_options = {}) + config_options = {logger: $null_log}.merge(config_options) + subject.new(sdk_key, Config.new(config_options), nil, 0.1) end - def make_sender_with_events_uri(events_uri) - subject.new(sdk_key, Config.new(events_uri: events_uri, logger: $null_log, application: {id: "id", version: "version"}), nil, 0.1) + def with_sender_and_server(config_options = {}) + enable_compression = config_options[:compress_events] || false + with_server(enable_compression: enable_compression) do |server| + config_options[:events_uri] = server.base_uri.to_s + yield make_sender(config_options), server + end end - def with_sender_and_server - with_server do |server| - yield make_sender(server), server + it "sends analytics event data without compression enabled" do + with_sender_and_server(compress_events: false) do |es, server| + server.setup_ok_response("/bulk", "") + + result = es.send_event_data(fake_data, "", false) + + expect(result.success).to be true + expect(result.must_shutdown).to be false + expect(result.time_from_server).not_to be_nil + + req, body = server.await_request_with_body + expect(body).to eq fake_data + expect(req.header).to include({ + "authorization" => [ sdk_key ], + "content-type" => [ "application/json" ], + "user-agent" => [ "RubyClient/" + LaunchDarkly::VERSION ], + "x-launchdarkly-event-schema" => [ "4" ], + "connection" => [ "Keep-Alive" ], + }) + expect(req.header['x-launchdarkly-payload-id']).not_to eq [] + expect(req.header['content-encoding']).to eq [] + expect(req.header['content-length'][0].to_i).to eq fake_data.length end end - it "sends analytics event data" do - with_sender_and_server do |es, server| + it "sends analytics event data with compression enabled" do + with_sender_and_server(compress_events: true) do |es, server| server.setup_ok_response("/bulk", "") result = es.send_event_data(fake_data, "", false) @@ -37,17 +61,18 @@ def with_sender_and_server expect(result.must_shutdown).to be false expect(result.time_from_server).not_to be_nil - req = server.await_request - expect(req.body).to eq fake_data + req, body = server.await_request_with_body + expect(body).to eq fake_data expect(req.header).to include({ "authorization" => [ sdk_key ], + "content-encoding" => [ "gzip" ], "content-type" => [ "application/json" ], "user-agent" => [ "RubyClient/" + LaunchDarkly::VERSION ], "x-launchdarkly-event-schema" => [ "4" ], - "x-launchdarkly-tags" => [ "application-id/id application-version/version" ], "connection" => [ "Keep-Alive" ], }) expect(req.header['x-launchdarkly-payload-id']).not_to eq [] + expect(req.header['content-length'][0].to_i).to be > fake_data.length end end @@ -63,8 +88,8 @@ def with_sender_and_server result = es.send_event_data(fake_data, "", false) expect(result.success).to be true - req = server.await_request - expect(req.body).to eq fake_data + req, body = server.await_request_with_body + expect(body).to eq fake_data expect(req.host).to eq "fake-event-server" end end @@ -123,7 +148,7 @@ def with_sender_and_server begin ENV["http_proxy"] = proxy.base_uri.to_s - es = make_sender_with_events_uri(fake_target_uri) + es = make_sender(events_uri: fake_target_uri) result = es.send_event_data(fake_data, "", false)