From 63e43f389d418f24a38c79097e5090dc1374cd48 Mon Sep 17 00:00:00 2001 From: Jean Luis Urena Date: Tue, 15 Oct 2024 11:09:20 -0400 Subject: [PATCH 1/3] [ISSUE-71] Bump up activeresource bc supper is rails > 6.1 --- cached_resource.gemspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cached_resource.gemspec b/cached_resource.gemspec index b921563..49f2720 100644 --- a/cached_resource.gemspec +++ b/cached_resource.gemspec @@ -17,8 +17,8 @@ Gem::Specification.new do |s| s.required_ruby_version = ">= 3.0" - s.add_runtime_dependency "activeresource", ">= 4.0" - s.add_runtime_dependency "activesupport", ">= 4.0" + s.add_runtime_dependency "activeresource", ">= 6.1" + s.add_runtime_dependency "msgpack", "~> 1.7", ">= 1.7.3" s.add_runtime_dependency "nilio", ">= 1.0" s.add_development_dependency "concurrent-ruby" From 1e8a27de1e5b6bf8dfed6b88c457f4fcd1808ab2 Mon Sep 17 00:00:00 2001 From: Jean Luis Urena Date: Tue, 15 Oct 2024 12:00:13 -0400 Subject: [PATCH 2/3] [ISSUE-71] Serialize payload responses for cache value, optionally hash keys --- README.md | 1 + lib/cached_resource.rb | 3 ++- lib/cached_resource/caching.rb | 11 ++++++++--- lib/cached_resource/configuration.rb | 8 ++++++-- lib/cached_resource/version.rb | 2 +- spec/cached_resource/caching_spec.rb | 22 ++++++++++++++++++++++ spec/cached_resource/configuration_spec.rb | 4 ++++ 7 files changed, 44 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index f991bf6..04f8bb6 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ CachedResource accepts the following options as a hash: | `:collection_synchronize` | Use collections to generate cache entries for individuals. Update the existing cached principal collection when retrieving subsets of the principal collection or individuals. | `false` | | `:logger` | The logger to which CachedResource messages should be written. | The `Rails.logger` if available, or an `ActiveSupport::Logger` | | `:race_condition_ttl` | The race condition ttl, to prevent [dog pile effect](https://en.wikipedia.org/wiki/Cache_stampede) or [cache stampede](https://en.wikipedia.org/wiki/Cache_stampede). | `86400` | +| `:max_key_length` | Set a max key length. Anything over this key length will be hashed into a 256 hex key | `nil`, meaning it will never be hashed | `:ttl_randomization_scale` | A Range from which a random value will be selected to scale the ttl. | `1..2` | | `:ttl_randomization` | Enable ttl randomization. | `false` | | `:ttl` | The time in seconds until the cache should expire. | `604800` | diff --git a/lib/cached_resource.rb b/lib/cached_resource.rb index b3567a2..1a90a74 100644 --- a/lib/cached_resource.rb +++ b/lib/cached_resource.rb @@ -1,6 +1,7 @@ +require "msgpack" +require "nilio" require "ostruct" -require "nilio" require "active_support/cache" require "active_support/concern" require "active_support/logger" diff --git a/lib/cached_resource/caching.rb b/lib/cached_resource/caching.rb index 0878c12..f129b09 100644 --- a/lib/cached_resource/caching.rb +++ b/lib/cached_resource/caching.rb @@ -104,7 +104,7 @@ def is_any_collection?(*arguments) # Read a entry from the cache for the given key. def cache_read(key) object = cached_resource.cache.read(key).try do |json_cache| - json = ActiveSupport::JSON.decode(json_cache) + json = ActiveSupport::JSON.decode(MessagePack.unpack(json_cache)) unless json.nil? cache = json_to_object(json) @@ -130,7 +130,8 @@ def cache_write(key, object, *arguments) params = options[:params] prefix_options, query_options = split_options(params) - result = cached_resource.cache.write(key, object_to_json(object, prefix_options, query_options), race_condition_ttl: cached_resource.race_condition_ttl, expires_in: cached_resource.generate_ttl) + serialized_json = MessagePack.pack(object_to_json(object, prefix_options, query_options)) + result = cached_resource.cache.write(key, serialized_json, race_condition_ttl: cached_resource.race_condition_ttl, expires_in: cached_resource.generate_ttl) result && cached_resource.logger.info("#{CachedResource::Configuration::LOGGER_PREFIX} WRITE #{key}") result end @@ -158,7 +159,11 @@ def cache_key_delete_pattern # Generate the request cache key. def cache_key(*arguments) - "#{name_key}/#{arguments.join("/")}".downcase.delete(" ") + key = "#{name_key}/#{arguments.join("/")}".downcase.delete(" ") + if cached_resource.max_key_length && key.length > cached_resource.max_key_length + key = Digest::SHA256.hexdigest(key) + end + key end def name_key diff --git a/lib/cached_resource/configuration.rb b/lib/cached_resource/configuration.rb index ae381da..2ef096e 100644 --- a/lib/cached_resource/configuration.rb +++ b/lib/cached_resource/configuration.rb @@ -11,6 +11,9 @@ class Configuration < OpenStruct # prefix for log messages LOGGER_PREFIX = "[cached_resource]" + # Max key length + MAX_KEY_LENGTH = nil + # Initialize a Configuration with the given options, overriding any # defaults. The following options exist for cached resource: # :enabled, default: true @@ -32,10 +35,11 @@ def initialize(options = {}) collection_synchronize: false, enabled: true, logger: defined?(Rails.logger) && Rails.logger || LOGGER, + max_key_length: options.fetch(:max_key_length, MAX_KEY_LENGTH), race_condition_ttl: 86400, - ttl: 604800, + ttl_randomization_scale: 1..2, ttl_randomization: false, - ttl_randomization_scale: 1..2 + ttl: 604800 }.merge(options)) end diff --git a/lib/cached_resource/version.rb b/lib/cached_resource/version.rb index c4d22b3..1687cf3 100644 --- a/lib/cached_resource/version.rb +++ b/lib/cached_resource/version.rb @@ -1,3 +1,3 @@ module CachedResource - VERSION = "9.0.0" + VERSION = "9.1.0" end diff --git a/spec/cached_resource/caching_spec.rb b/spec/cached_resource/caching_spec.rb index c254c9f..46b8fbf 100644 --- a/spec/cached_resource/caching_spec.rb +++ b/spec/cached_resource/caching_spec.rb @@ -42,6 +42,7 @@ def read_from_cache(key, model = Thing) enabled: true, generate_ttl: 604800, logger: double(:thing_logger, info: nil, error: nil), + max_key_length: nil, race_condition_ttl: 86400, ttl_randomization_scale: 1..2, ttl_randomization: false, @@ -58,6 +59,7 @@ def read_from_cache(key, model = Thing) enabled: true, generate_ttl: 604800, logger: double(:not_the_thing_logger, info: nil, error: nil), + max_key_length: nil, race_condition_ttl: 86400, ttl_randomization_scale: 1..2, ttl_randomization: false, @@ -88,6 +90,26 @@ def read_from_cache(key, model = Thing) allow(not_the_thing_cached_resource).to receive(:enabled).and_return(true) end + context "When a `max_key_length` is set" do + before do + allow(thing_cached_resource).to receive(:max_key_length).and_return(10) + end + + context "When key length is greater than `max_key_length`" do + it "caches a response with hashed key" do + result = Thing.find(1, from: "path", params: {foo: "bar"}) + expect(read_from_cache(Digest::SHA256.hexdigest('thing/1/{:from=>"path",:params=>{:foo=>"bar"}}'))).to eq(result) + end + end + + context "When key length is less than `max_key_length`" do + it "caches a response with unhashed key" do + result = Thing.find(1) + expect(read_from_cache("thing/1")).to eq(result) + end + end + end + context "Caching single resource" do it "caches a response" do result = Thing.find(1) diff --git a/spec/cached_resource/configuration_spec.rb b/spec/cached_resource/configuration_spec.rb index a0826be..93c01fa 100644 --- a/spec/cached_resource/configuration_spec.rb +++ b/spec/cached_resource/configuration_spec.rb @@ -56,6 +56,10 @@ class Bar3 < Bar expect(configuration.ttl).to eq(604800) end + it "should have key length of Configuration::MAX_KEY_LENGTH" do + expect(configuration.max_key_length).to eq(CachedResource::Configuration::MAX_KEY_LENGTH) + end + it "should disable collection synchronization" do expect(configuration.collection_synchronize).to eq(false) end From ccf2d44b34d4e626ceab8c9e6095458dde8c76ae Mon Sep 17 00:00:00 2001 From: Jean Luis Urena Date: Thu, 28 Nov 2024 09:16:22 -0500 Subject: [PATCH 3/3] Make 'compression' configurable and optional. false by default --- README.md | 30 ++++++++++---------- cached_resource.gemspec | 2 +- lib/cached_resource/cached_resource.rb | 1 + lib/cached_resource/caching.rb | 6 ++-- lib/cached_resource/configuration.rb | 3 +- spec/cached_resource/caching_spec.rb | 39 +++++++++++++++++++------- 6 files changed, 52 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 04f8bb6..0bdd4dc 100644 --- a/README.md +++ b/README.md @@ -39,20 +39,21 @@ end ### Options CachedResource accepts the following options as a hash: -| Option | Description | Default | -|----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------| -| `:enabled` | Enables or disables caching. | `true` | -| `:cache_collections` | Set to false to always remake a request for collections. | `true` | -| `:cache` | The cache store that CacheResource should use. | The `Rails.cache` if available, or an `ActiveSupport::Cache::MemoryStore` | -| `:cache_key_prefix` | A prefix to be added to the cache keys. | `nil` | -| `:collection_arguments` | The arguments that identify the principal collection request. | `[:all]` | -| `:collection_synchronize` | Use collections to generate cache entries for individuals. Update the existing cached principal collection when retrieving subsets of the principal collection or individuals. | `false` | -| `:logger` | The logger to which CachedResource messages should be written. | The `Rails.logger` if available, or an `ActiveSupport::Logger` | -| `:race_condition_ttl` | The race condition ttl, to prevent [dog pile effect](https://en.wikipedia.org/wiki/Cache_stampede) or [cache stampede](https://en.wikipedia.org/wiki/Cache_stampede). | `86400` | -| `:max_key_length` | Set a max key length. Anything over this key length will be hashed into a 256 hex key | `nil`, meaning it will never be hashed -| `:ttl_randomization_scale` | A Range from which a random value will be selected to scale the ttl. | `1..2` | -| `:ttl_randomization` | Enable ttl randomization. | `false` | -| `:ttl` | The time in seconds until the cache should expire. | `604800` | +| Option | Description | Default | +|----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------| +| `:enabled` | Enables or disables caching. | `true` | +| `:cache_collections` | Set to `false` to always remake a request for collections. | `true` | +| `:cache` | The cache store that CachedResource should use. | `Rails.cache` (if available), or an `ActiveSupport::Cache::MemoryStore`. | +| `:cache_key_prefix` | A prefix to be added to the cache keys. | `nil` | +| `:collection_arguments` | The arguments that identify the principal collection request. | `[:all]` | +| `:collection_synchronize` | Use collections to generate cache entries for individuals. Updates the cached principal collection when retrieving subsets or individuals. | `false` | +| `:compress` | Whether to compress cache values using the [`msgpack`](https://rubygems.org/gems/msgpack) gem. The `msgpack` gem must be part of your dependencies. | `false` | +| `:logger` | The logger to which CachedResource messages should be written. | `Rails.logger` (if available), or an `ActiveSupport::Logger`. | +| `:race_condition_ttl` | The race condition TTL, to prevent the [dog-pile effect](https://en.wikipedia.org/wiki/Cache_stampede) or [cache stampede](https://en.wikipedia.org/wiki/Cache_stampede). | `86400` | +| `:max_key_length` | Sets a max key length. Keys exceeding this length will be hashed into a 256-character hex key. | `nil` (keys are never hashed). | +| `:ttl_randomization_scale` | A range from which a random value is selected to scale the TTL. | `1..2` | +| `:ttl_randomization` | Enables TTL randomization. | `false` | +| `:ttl` | The time in seconds until the cache should expire. | `604800` | For example: ```ruby @@ -145,7 +146,6 @@ To automatically apply linter fixes: `bundle exec rake standard:fix` ## Credit/Inspiration * quamen and [this gist](http://gist.github.com/947734) -* latimes and [this plugin](http://github.com/latimes/cached_resource) ## Feedback/Problems Feedback is greatly appreciated! Check out this project's [issue tracker](https://github.com/Ahsizara/cached_resource/issues) if you've got anything to say. diff --git a/cached_resource.gemspec b/cached_resource.gemspec index 49f2720..fe4b5ef 100644 --- a/cached_resource.gemspec +++ b/cached_resource.gemspec @@ -18,10 +18,10 @@ Gem::Specification.new do |s| s.required_ruby_version = ">= 3.0" s.add_runtime_dependency "activeresource", ">= 6.1" - s.add_runtime_dependency "msgpack", "~> 1.7", ">= 1.7.3" s.add_runtime_dependency "nilio", ">= 1.0" s.add_development_dependency "concurrent-ruby" + s.add_development_dependency "msgpack", "~> 1.7", ">= 1.7.3" s.add_development_dependency "pry-byebug" s.add_development_dependency "rake" s.add_development_dependency "rspec" diff --git a/lib/cached_resource/cached_resource.rb b/lib/cached_resource/cached_resource.rb index 3e89b1a..6f8a8c1 100644 --- a/lib/cached_resource/cached_resource.rb +++ b/lib/cached_resource/cached_resource.rb @@ -17,6 +17,7 @@ def cached_resource(options = {}) # Set up cached resource for this class by creating a new configuration # and establishing the necessary methods. def setup_cached_resource!(options) + require "msgpack" if options[:compress] @cached_resource = CachedResource::Configuration.new(options) send :include, CachedResource::Caching @cached_resource diff --git a/lib/cached_resource/caching.rb b/lib/cached_resource/caching.rb index f129b09..e84275f 100644 --- a/lib/cached_resource/caching.rb +++ b/lib/cached_resource/caching.rb @@ -104,7 +104,8 @@ def is_any_collection?(*arguments) # Read a entry from the cache for the given key. def cache_read(key) object = cached_resource.cache.read(key).try do |json_cache| - json = ActiveSupport::JSON.decode(MessagePack.unpack(json_cache)) + json = cached_resource.compress ? MessagePack.unpack(json_cache) : json_cache + json = ActiveSupport::JSON.decode(json) unless json.nil? cache = json_to_object(json) @@ -130,7 +131,8 @@ def cache_write(key, object, *arguments) params = options[:params] prefix_options, query_options = split_options(params) - serialized_json = MessagePack.pack(object_to_json(object, prefix_options, query_options)) + serialized_json = object_to_json(object, prefix_options, query_options) + serialized_json = MessagePack.pack(serialized_json) if cached_resource.compress result = cached_resource.cache.write(key, serialized_json, race_condition_ttl: cached_resource.race_condition_ttl, expires_in: cached_resource.generate_ttl) result && cached_resource.logger.info("#{CachedResource::Configuration::LOGGER_PREFIX} WRITE #{key}") result diff --git a/lib/cached_resource/configuration.rb b/lib/cached_resource/configuration.rb index 2ef096e..a8f4947 100644 --- a/lib/cached_resource/configuration.rb +++ b/lib/cached_resource/configuration.rb @@ -28,11 +28,12 @@ class Configuration < OpenStruct # :cache_collections, default: true def initialize(options = {}) super({ - cache: defined?(Rails.cache) && Rails.cache || CACHE, cache_collections: true, cache_key_prefix: nil, + cache: defined?(Rails.cache) && Rails.cache || CACHE, collection_arguments: [:all], collection_synchronize: false, + compress: false, enabled: true, logger: defined?(Rails.logger) && Rails.logger || LOGGER, max_key_length: options.fetch(:max_key_length, MAX_KEY_LENGTH), diff --git a/spec/cached_resource/caching_spec.rb b/spec/cached_resource/caching_spec.rb index 46b8fbf..8885360 100644 --- a/spec/cached_resource/caching_spec.rb +++ b/spec/cached_resource/caching_spec.rb @@ -39,14 +39,15 @@ def read_from_cache(key, model = Thing) cache: CACHE, collection_arguments: [:all], collection_synchronize: false, + compress: false, enabled: true, - generate_ttl: 604800, + generate_ttl: 604_800, logger: double(:thing_logger, info: nil, error: nil), max_key_length: nil, - race_condition_ttl: 86400, + race_condition_ttl: 86_400, ttl_randomization_scale: 1..2, ttl_randomization: false, - ttl: 604800) + ttl: 604_800) end let(:not_the_thing_cached_resource) do @@ -56,14 +57,15 @@ def read_from_cache(key, model = Thing) cache: CACHE, collection_arguments: [:all], collection_synchronize: false, + compress: false, enabled: true, - generate_ttl: 604800, + generate_ttl: 604_800, logger: double(:not_the_thing_logger, info: nil, error: nil), max_key_length: nil, - race_condition_ttl: 86400, + race_condition_ttl: 86_400, ttl_randomization_scale: 1..2, ttl_randomization: false, - ttl: 604800) + ttl: 604_800) end before do @@ -217,12 +219,14 @@ def read_from_cache(key, model = Thing) context "custom collection arguments" do before do - allow(thing_cached_resource).to receive(:collection_arguments).and_return([:all, params: {name: 42}]) + allow(thing_cached_resource).to receive(:collection_arguments).and_return([:all, {params: {name: 42}}]) end it "checks for custom collection arguments" do Thing.all - expect { Thing.find(:all, params: {name: 42}) }.to change(ActiveResource::HttpMock.requests, :length).from(1).to(2) + expect do + Thing.find(:all, params: {name: 42}) + end.to change(ActiveResource::HttpMock.requests, :length).from(1).to(2) end end end @@ -247,6 +251,21 @@ def read_from_cache(key, model = Thing) end end + context "compress" do + before do + allow(thing_cached_resource).to receive(:compress).and_return(true) + allow(MessagePack).to receive(:pack).and_call_original + allow(MessagePack).to receive(:unpack).and_call_original + end + + it "compresses the cache entry" do + result = Thing.find(1) + expect(read_from_cache("thing/1")).to eq(result) + expect(MessagePack).to have_received(:pack) + expect(MessagePack).to have_received(:unpack) + end + end + context "when cache prefix is set" do context "cache_key_prefix is a string" do before { allow(thing_cached_resource).to receive(:cache_key_prefix).and_return("prefix123") } @@ -351,14 +370,14 @@ def read_from_cache(key, model = Thing) context "with cache ActiveSupport::Cache::MemoryStore" do let(:cache_class) { ActiveSupport::Cache::MemoryStore.new } it do - expect(Thing.send(:cache_key_delete_pattern)).to eq(/^thing\//) + expect(Thing.send(:cache_key_delete_pattern)).to eq(%r{^thing/}) end end context "with cache ActiveSupport::Cache::FileStore" do let(:cache_class) { ActiveSupport::Cache::FileStore.new("tmp/") } it do - expect(Thing.send(:cache_key_delete_pattern)).to eq(/^thing\//) + expect(Thing.send(:cache_key_delete_pattern)).to eq(%r{^thing/}) end end