From f90ef57ca996481fd590d0c2c8492980f00929c4 Mon Sep 17 00:00:00 2001 From: Wu Date: Wed, 23 May 2018 20:43:15 -0700 Subject: [PATCH 1/4] Add Privacy Notice to Docs --- .yardopts | 1 + doc-src/templates/default/layout/html/footer.erb | 6 ++++++ doc-src/templates/default/setup.rb | 3 +++ 3 files changed, 10 insertions(+) create mode 100644 doc-src/templates/default/layout/html/footer.erb create mode 100644 doc-src/templates/default/setup.rb diff --git a/.yardopts b/.yardopts index 498d493..22b0e05 100644 --- a/.yardopts +++ b/.yardopts @@ -1,4 +1,5 @@ --title 'AWS X-Ray SDK for Ruby' --markup markdown +--template-path doc-src/templates --no-progress - LICENSE diff --git a/doc-src/templates/default/layout/html/footer.erb b/doc-src/templates/default/layout/html/footer.erb new file mode 100644 index 0000000..7bff86f --- /dev/null +++ b/doc-src/templates/default/layout/html/footer.erb @@ -0,0 +1,6 @@ + \ No newline at end of file diff --git a/doc-src/templates/default/setup.rb b/doc-src/templates/default/setup.rb new file mode 100644 index 0000000..81da9dc --- /dev/null +++ b/doc-src/templates/default/setup.rb @@ -0,0 +1,3 @@ +def javascripts + (super + %w(js/tabs.js)).uniq +end From 94aa0ebd93179f72031925f8c4dff576abef2cc8 Mon Sep 17 00:00:00 2001 From: Wu Date: Wed, 12 Sep 2018 18:50:32 -0700 Subject: [PATCH 2/4] service sampling support tons of fixes update change log --- CHANGELOG.md | 9 ++ aws-xray-sdk.gemspec | 1 + lib/aws-xray-sdk/configuration.rb | 17 +- lib/aws-xray-sdk/daemon_config.rb | 59 +++++++ lib/aws-xray-sdk/emitter/default_emitter.rb | 31 ++-- lib/aws-xray-sdk/emitter/emitter.rb | 4 +- lib/aws-xray-sdk/facets/aws_sdk.rb | 18 ++- lib/aws-xray-sdk/facets/helper.rb | 15 +- lib/aws-xray-sdk/facets/net_http.rb | 4 + lib/aws-xray-sdk/facets/rack.rb | 9 +- lib/aws-xray-sdk/model/dummy_entities.rb | 4 + lib/aws-xray-sdk/model/segment.rb | 9 +- lib/aws-xray-sdk/recorder.rb | 11 +- lib/aws-xray-sdk/sampling/connector.rb | 71 +++++++++ lib/aws-xray-sdk/sampling/default_sampler.rb | 150 +++++++++--------- lib/aws-xray-sdk/sampling/lead_poller.rb | 72 +++++++++ lib/aws-xray-sdk/sampling/local/reservoir.rb | 35 ++++ lib/aws-xray-sdk/sampling/local/sampler.rb | 110 +++++++++++++ .../sampling/local/sampling_rule.rb | 63 ++++++++ lib/aws-xray-sdk/sampling/reservoir.rb | 92 ++++++++--- lib/aws-xray-sdk/sampling/rule_cache.rb | 86 ++++++++++ lib/aws-xray-sdk/sampling/rule_poller.rb | 39 +++++ lib/aws-xray-sdk/sampling/sampler.rb | 6 +- .../sampling/sampling_decision.rb | 8 + lib/aws-xray-sdk/sampling/sampling_rule.rb | 137 ++++++++++++---- lib/aws-xray-sdk/version.rb | 2 +- test/aws-xray-sdk/tc_aws_sdk.rb | 18 ++- test/aws-xray-sdk/tc_daemon_config.rb | 58 +++++++ .../{tc_sampling.rb => tc_local_sampling.rb} | 53 ++++--- test/aws-xray-sdk/tc_recorder.rb | 14 ++ test/aws-xray-sdk/tc_sampling_rule_cache.rb | 117 ++++++++++++++ test/aws-xray-sdk/tc_segment.rb | 22 +++ test/test_helper.rb | 8 + 33 files changed, 1128 insertions(+), 224 deletions(-) create mode 100644 lib/aws-xray-sdk/daemon_config.rb create mode 100644 lib/aws-xray-sdk/sampling/connector.rb create mode 100644 lib/aws-xray-sdk/sampling/lead_poller.rb create mode 100644 lib/aws-xray-sdk/sampling/local/reservoir.rb create mode 100644 lib/aws-xray-sdk/sampling/local/sampler.rb create mode 100644 lib/aws-xray-sdk/sampling/local/sampling_rule.rb create mode 100644 lib/aws-xray-sdk/sampling/rule_cache.rb create mode 100644 lib/aws-xray-sdk/sampling/rule_poller.rb create mode 100644 lib/aws-xray-sdk/sampling/sampling_decision.rb create mode 100644 test/aws-xray-sdk/tc_daemon_config.rb rename test/aws-xray-sdk/{tc_sampling.rb => tc_local_sampling.rb} (62%) create mode 100644 test/aws-xray-sdk/tc_sampling_rule_cache.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 09c8649..1f16efd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +0.11.0 (2018-09-25) +------------------- +* **Breaking**: The default sampler now launches background tasks to poll sampling rules from X-Ray service. See more details on how to create sampling rules: https://docs.aws.amazon.com/xray/latest/devguide/xray-console-sampling.html. +* **Breaking**: The sampling modules related to local sampling rules have been renamed and moved to `sampling/local` namespace. +* **Breaking**: The default json serializer is switched to `multi_json` for experimental JRuby support. Now you need to specify `oj` or `jrjackson` in Gemfile. [PR#5](https://github.com/aws/aws-xray-sdk-ruby/pull/5) +* **Breaking**: The SDK now requires `aws-sdk-xray` >= `1.4.0`. +* Feature: Environment variable `AWS_XRAY_DAEMON_ADDRESS` now takes an additional notation in `tcp:127.0.0.1:2000 udp:127.0.0.2:2001` to set TCP and UDP destination separately. By default it assumes a X-Ray daemon listening to both UDP and TCP traffic on `127.0.0.1:2000`. +* Bugfix - Call only once if `current_entity` is nil. [PR#9](https://github.com/aws/aws-xray-sdk-ruby/pull/9) + 0.10.2 (2018-03-30) ------------------- * Feature - Added SDK and Ruby runtime information to sampled segments. diff --git a/aws-xray-sdk.gemspec b/aws-xray-sdk.gemspec index 788a7e6..df86ee5 100644 --- a/aws-xray-sdk.gemspec +++ b/aws-xray-sdk.gemspec @@ -21,6 +21,7 @@ Gem::Specification.new do |spec| spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) spec.require_paths = ['lib'] + spec.add_dependency 'aws-sdk-xray', '~> 1.4.0' spec.add_dependency 'multi_json', '~> 1' spec.add_development_dependency 'aws-sdk-dynamodb', '~> 1' diff --git a/lib/aws-xray-sdk/configuration.rb b/lib/aws-xray-sdk/configuration.rb index da1355b..8c54a09 100644 --- a/lib/aws-xray-sdk/configuration.rb +++ b/lib/aws-xray-sdk/configuration.rb @@ -2,6 +2,7 @@ require 'aws-xray-sdk/patcher' require 'aws-xray-sdk/emitter/default_emitter' require 'aws-xray-sdk/context/default_context' +require 'aws-xray-sdk/daemon_config' require 'aws-xray-sdk/sampling/default_sampler' require 'aws-xray-sdk/streaming/default_streamer' require 'aws-xray-sdk/segment_naming/dynamic_naming' @@ -17,9 +18,9 @@ class Configuration include Patcher SEGMENT_NAME_KEY = 'AWS_XRAY_TRACING_NAME'.freeze - CONFIG_KEY = %I[logger name sampling plugins daemon_address segment_naming - naming_pattern emitter streamer context context_missing - sampling_rules stream_threshold patch].freeze + CONFIG_KEY = %I[logger name sampling plugins daemon_address + segment_naming naming_pattern emitter streamer context + context_missing sampling_rules stream_threshold patch].freeze def initialize @name = ENV[SEGMENT_NAME_KEY] @@ -38,9 +39,12 @@ def name=(v) @name = ENV[SEGMENT_NAME_KEY] || v end - # proxy method to the emitter's daemon_address config. + # setting daemon address for components communicate with X-Ray daemon. def daemon_address=(v) - emitter.daemon_address = v + v = ENV[DaemonConfig::DAEMON_ADDRESS_KEY] || v + config = DaemonConfig.new(v) + emitter.daemon_config = config + sampler.daemon_config = config if sampler.respond_to?(:daemon_config=) end # proxy method to the context's context_missing config. @@ -63,8 +67,7 @@ def naming_pattern=(v) segment_naming.pattern = v end - # makes a sampling decision based on internal configure, e.g. - # if sampling enabled and the default sampling rule. + # makes a sampling decision without incoming filters. def sample? return true unless sampling sampler.sample? diff --git a/lib/aws-xray-sdk/daemon_config.rb b/lib/aws-xray-sdk/daemon_config.rb new file mode 100644 index 0000000..ba6bdc4 --- /dev/null +++ b/lib/aws-xray-sdk/daemon_config.rb @@ -0,0 +1,59 @@ +require 'aws-xray-sdk/exceptions' + +module XRay + # The class that stores X-Ray daemon configuration about + # the ip address and port for UDP and TCP port. It gets the address + # string from `AWS_XRAY_DAEMON_ADDRESS` and then from recorder's + # configuration for `daemon_address`. + # A notation of `127.0.0.1:2000` or `tcp:127.0.0.1:2000 udp:127.0.0.2:2001` + # are both acceptable. The former one means UDP and TCP are running at + # the same address. By default it assumes a X-Ray daemon + # running at `127.0.0.1:2000` listening to both UDP and TCP traffic. + class DaemonConfig + DAEMON_ADDRESS_KEY = 'AWS_XRAY_DAEMON_ADDRESS'.freeze + attr_reader :tcp_ip, :tcp_port, :udp_ip, :udp_port + @@dafault_addr = '127.0.0.1:2000' + + def initialize(addr: @@dafault_addr) + update_address(addr) + end + + def update_address(v) + v = ENV[DAEMON_ADDRESS_KEY] || v + update_addr(v) + rescue StandardError + raise InvalidDaemonAddressError, %(Invalid X-Ray daemon address specified: #{v}.) + end + + private + + def update_addr(v) + parts = v.split(' ') + if parts.length == 1 # format of '127.0.0.1:2000' + addr = parts[0].split(':') + raise InvalidDaemonAddressError unless addr.length == 2 + @tcp_ip = addr[0] + @tcp_port = addr[1].to_i + @udp_ip = addr[0] + @udp_port = addr[1].to_i + else + set_tcp_udp(parts) # format of 'tcp:127.0.0.1:2000 udp:127.0.0.2:2001' + end + end + + def set_tcp_udp(parts) + part1 = parts[0] + part2 = parts[1] + key1 = part1.split(':')[0] + key2 = part2.split(':')[0] + addr_h = {} + addr_h[key1] = part1.split(':') + addr_h[key2] = part2.split(':') + + @tcp_ip = addr_h['tcp'][1] + @tcp_port = addr_h['tcp'][2].to_i + @udp_ip = addr_h['udp'][1] + @udp_port = addr_h['udp'][2].to_i + end + end +end diff --git a/lib/aws-xray-sdk/emitter/default_emitter.rb b/lib/aws-xray-sdk/emitter/default_emitter.rb index 13c332b..b8d759d 100644 --- a/lib/aws-xray-sdk/emitter/default_emitter.rb +++ b/lib/aws-xray-sdk/emitter/default_emitter.rb @@ -2,6 +2,7 @@ require 'aws-xray-sdk/logger' require 'aws-xray-sdk/emitter/emitter' require 'aws-xray-sdk/exceptions' +require 'aws-xray-sdk/daemon_config' module XRay # The default emitter the X-Ray recorder uses to send segments/subsegments @@ -10,12 +11,11 @@ class DefaultEmitter include Emitter include Logging - attr_reader :address + attr_reader :daemon_config - def initialize + def initialize(daemon_config: DaemonConfig.new) @socket = UDPSocket.new - @address = ENV[DAEMON_ADDRESS_KEY] || '127.0.0.1:2000' - configure_socket(@address) + self.daemon_config = daemon_config end # Serializes a segment/subsegment and sends it to the X-Ray daemon @@ -25,29 +25,18 @@ def send_entity(entity:) return nil unless entity.sampled begin payload = %(#{@@protocol_header}#{@@protocol_delimiter}#{entity.to_json}) - logger.debug %(sending payload #{payload} to daemon at #{address}.) + logger.debug %(sending payload #{payload} to daemon at #{@address}.) @socket.send payload, 0 rescue StandardError => e logger.warn %(failed to send payload due to #{e.message}) end end - def daemon_address=(v) - v = ENV[DAEMON_ADDRESS_KEY] || v - @address = v - configure_socket(v) - end - - private - - def configure_socket(v) - begin - addr = v.split(':') - host, ip = addr[0], addr[1].to_i - @socket.connect(host, ip) - rescue StandardError - raise InvalidDaemonAddressError, %(Invalid X-Ray daemon address specified: #{v}.) - end + def daemon_config=(v) + @address = %(#{v.udp_ip}:#{v.udp_port}) + @socket.connect(v.udp_ip, v.udp_port) + rescue StandardError + raise InvalidDaemonAddressError, %(Invalid X-Ray daemon address specified: #{v}.) end end end diff --git a/lib/aws-xray-sdk/emitter/emitter.rb b/lib/aws-xray-sdk/emitter/emitter.rb index 14aeb98..3af1ee2 100644 --- a/lib/aws-xray-sdk/emitter/emitter.rb +++ b/lib/aws-xray-sdk/emitter/emitter.rb @@ -4,8 +4,6 @@ module XRay # The emitter interface the X-Ray recorder uses to send segments/subsegments # to the X-Ray daemon over UDP. module Emitter - DAEMON_ADDRESS_KEY = 'AWS_XRAY_DAEMON_ADDRESS'.freeze - @@protocol_header = { format: 'json', version: 1 @@ -17,7 +15,7 @@ def send_entity(entity:) raise 'Not implemented' end - def daemon_address=(v) + def daemon_config=(v) raise 'Not implemented' end end diff --git a/lib/aws-xray-sdk/facets/aws_sdk.rb b/lib/aws-xray-sdk/facets/aws_sdk.rb index 7ce2040..7779a72 100644 --- a/lib/aws-xray-sdk/facets/aws_sdk.rb +++ b/lib/aws-xray-sdk/facets/aws_sdk.rb @@ -17,14 +17,18 @@ class Handler < Seahorse::Client::Handler include XRay::Facets::Helper def call(context) - recorder = Aws.config[:xray_recorder] - if recorder.current_entity.nil? - super - end - operation = context.operation_name service_name = context.client.class.api.metadata['serviceAbbreviation'] || context.client.class.to_s.split('::')[1] + if skip?(service_name, operation) + return super + end + + recorder = Aws.config[:xray_recorder] + if recorder.nil? || recorder.current_entity.nil? + return super + end + recorder.capture(service_name, namespace: 'aws') do |subsegment| # inject header string before calling downstream AWS services context.http_request.headers[TRACE_HEADER] = prep_header_str entity: subsegment @@ -109,6 +113,10 @@ def process_descriptor(target:, descriptor:, meta:) v = target.keys if descriptor[:get_keys] meta[descriptor[:rename_to]] = v end + + def skip?(service, op) + return service == 'XRay' && (op == :get_sampling_rules || op == :get_sampling_targets) + end end end diff --git a/lib/aws-xray-sdk/facets/helper.rb b/lib/aws-xray-sdk/facets/helper.rb index 681f682..efdbdab 100644 --- a/lib/aws-xray-sdk/facets/helper.rb +++ b/lib/aws-xray-sdk/facets/helper.rb @@ -26,21 +26,14 @@ def construct_header(headers:) # the highest precedence. If the `trace_header` doesn't contain # sampling decision then it checks if sampling is enabled or not # in the recorder. If not enbaled it returns 'true'. Otherwise it uses - # sampling rule to decide. - def should_sample?(header_obj:, recorder:, - host: nil, method: nil, path: nil, - **args) + # sampling rules to decide. + def should_sample?(header_obj:, recorder:, sampling_req:, **args) # check outside decision if i = header_obj.sampled - if i.zero? - false - else - true - end + !i.zero? # check sampling rules elsif recorder.sampling_enabled? - recorder.sampler.sample_request?(service_name: host, url_path: path, - http_method: method) + recorder.sampler.sample_request?(sampling_req) # sample if no reason not to else true diff --git a/lib/aws-xray-sdk/facets/net_http.rb b/lib/aws-xray-sdk/facets/net_http.rb index 08e008e..552b635 100644 --- a/lib/aws-xray-sdk/facets/net_http.rb +++ b/lib/aws-xray-sdk/facets/net_http.rb @@ -21,6 +21,10 @@ def initialize(*options) end def request(req, body = nil, &block) + if req.path && (req.path == ('/GetSamplingRules') || req.path == ('/SamplingTargets')) + return super + end + entity = XRay.recorder.current_entity capture = !(entity && entity.namespace && entity.namespace == 'aws'.freeze) if started? && capture && entity diff --git a/lib/aws-xray-sdk/facets/rack.rb b/lib/aws-xray-sdk/facets/rack.rb index c3f8307..49df33a 100644 --- a/lib/aws-xray-sdk/facets/rack.rb +++ b/lib/aws-xray-sdk/facets/rack.rb @@ -22,16 +22,15 @@ def call(env) host = req.host url_path = req.path method = req.request_method + # get segment name from host header if applicable + seg_name = @recorder.segment_naming.provide_name(host: req.host) # get sampling decision sampled = should_sample?( - header_obj: header, recorder: @recorder, - host: host, method: method, path: url_path + header_obj: header, recorder: @recorder, sampling_req: + { host: host, http_method: method, url_path: url_path, service: seg_name } ) - # get segment name from host header if applicable - seg_name = @recorder.segment_naming.provide_name(host: req.host) - # begin the segment segment = @recorder.begin_segment seg_name, trace_id: header.root, parent_id: header.parent_id, sampled: sampled diff --git a/lib/aws-xray-sdk/model/dummy_entities.rb b/lib/aws-xray-sdk/model/dummy_entities.rb index c6d20d2..3be3e9a 100644 --- a/lib/aws-xray-sdk/model/dummy_entities.rb +++ b/lib/aws-xray-sdk/model/dummy_entities.rb @@ -38,6 +38,10 @@ def aws=(v) # no-op end + def sampling_rule_name=(v) + # no-op + end + def to_h # no-op end diff --git a/lib/aws-xray-sdk/model/segment.rb b/lib/aws-xray-sdk/model/segment.rb index 58440db..2f9c506 100644 --- a/lib/aws-xray-sdk/model/segment.rb +++ b/lib/aws-xray-sdk/model/segment.rb @@ -6,7 +6,8 @@ module XRay # details about the request, and details about the work done. class Segment include Entity - attr_accessor :ref_counter, :subsegment_size, :origin, :user, :service + attr_accessor :ref_counter, :subsegment_size, :origin, + :user, :service # @param [String] trace_id Manually crafted trace id. # @param [String] name Must be specified either on object creation or @@ -39,6 +40,12 @@ def remove_subsegment(subsegment:) @subsegment_size = subsegment_size - subsegment.all_children_count - 1 end + def sampling_rule_name=(v) + @aws ||= {} + @aws[:xray] ||= {} + @aws[:xray][:sampling_rule_name] = v + end + def decrement_ref_counter @ref_counter -= 1 end diff --git a/lib/aws-xray-sdk/recorder.rb b/lib/aws-xray-sdk/recorder.rb index 13d3d5f..b4a8e33 100644 --- a/lib/aws-xray-sdk/recorder.rb +++ b/lib/aws-xray-sdk/recorder.rb @@ -12,11 +12,12 @@ module XRay # and send them to the X-Ray daemon. It is also responsible for managing # context. class Recorder - attr_reader :config + attr_reader :config, :origin def initialize(user_config: nil) @config = Configuration.new @config.configure(user_config) unless user_config.nil? + @origin = nil end # Begin a segment for the current context. The recorder @@ -31,7 +32,7 @@ def begin_segment(name, trace_id: nil, parent_id: nil, sampled: nil) sample = sampled.nil? ? config.sample? : sampled if sample segment = Segment.new name: seg_name, trace_id: trace_id, parent_id: parent_id - populate_runtime_context(segment) + populate_runtime_context(segment, sample) else segment = DummySegment.new name: seg_name, trace_id: trace_id, parent_id: parent_id end @@ -204,14 +205,14 @@ def sampling_enabled? private_class_method - def populate_runtime_context(segment) + def populate_runtime_context(segment, sample) @aws ||= begin aws = {} config.plugins.each do |p| meta = p.aws if meta.is_a?(Hash) && !meta.empty? aws.merge! meta - segment.origin = p::ORIGIN + @origin = p::ORIGIN end end xray_meta = { xray: @@ -230,6 +231,8 @@ def populate_runtime_context(segment) segment.aws = @aws segment.service = @service + segment.origin = @origin + segment.sampling_rule_name = sample if sample.is_a?(String) end end end diff --git a/lib/aws-xray-sdk/sampling/connector.rb b/lib/aws-xray-sdk/sampling/connector.rb new file mode 100644 index 0000000..8f7d643 --- /dev/null +++ b/lib/aws-xray-sdk/sampling/connector.rb @@ -0,0 +1,71 @@ +require 'securerandom' +require 'aws-xray-sdk/sampling/sampling_rule' +require 'aws-xray-sdk/logger' + +module XRay + # Connector class that translates Sampling poller functions to + # actual X-Ray back-end APIs and communicates with X-Ray daemon + # as the signing proxy. + class ServiceConnector + include Logging + attr_accessor :xray_client + @@client_id = SecureRandom.hex(12) + + def initialize + update_xray_client + end + + def fetch_sampling_rules + rules = [] + records = @xray_client.get_sampling_rules.sampling_rule_records + records.each { |record| rules << SamplingRule.new(record.sampling_rule) if rule_valid?(record.sampling_rule) } + rules + end + + def fetch_sampling_targets(rules) + now = Time.now.to_i + reports = generate_reports(rules, now) + resp = @xray_client.get_sampling_targets({ sampling_statistics_documents: reports }) + { + last_modified: resp.last_rule_modification, + documents: resp.sampling_target_documents + } + end + + def update_xray_client(ip: '127.0.0.1', port: 2000) + require 'aws-sdk-xray' + @xray_client = Aws::XRay::Client.new( + endpoint: %(http://#{ip}:#{port}), + access_key_id: 'dummy', # AWS Ruby SDK doesn't support unsigned request + secret_access_key: 'dummy' + ) + end + + def daemon_config=(v) + update_xray_client ip: v.tcp_ip, port: v.tcp_port + end + + private + + def generate_reports(rules, now) + reports = [] + rules.each do |rule| + report = rule.snapshot_statistics + report[:rule_name] = rule.name + report[:timestamp] = now + report[:client_id] = @@client_id + reports << report + end + reports + end + + def rule_valid?(rule) + return false if rule.version != 1 + # rules has resource ARN and attributes configured + # doesn't apply to this SDK + return false unless rule.resource_arn == '*' + return false unless rule.attributes.nil? || rule.attributes.empty? + true + end + end +end diff --git a/lib/aws-xray-sdk/sampling/default_sampler.rb b/lib/aws-xray-sdk/sampling/default_sampler.rb index 7ef227f..c98e233 100644 --- a/lib/aws-xray-sdk/sampling/default_sampler.rb +++ b/lib/aws-xray-sdk/sampling/default_sampler.rb @@ -1,105 +1,99 @@ +require 'aws-xray-sdk/logger' +require 'aws-xray-sdk/sampling/local/sampler' +require 'aws-xray-sdk/sampling/lead_poller' +require 'aws-xray-sdk/sampling/rule_cache' require 'aws-xray-sdk/sampling/sampler' require 'aws-xray-sdk/sampling/sampling_rule' -require 'aws-xray-sdk/exceptions' +require 'aws-xray-sdk/sampling/sampling_decision' module XRay - # The default sampler that uses internally defined - # sampling rule and reservoir models to decide sampling decision. - # It also uses the default sampling rule. - # An example definition: - # { - # version: 1, - # rules: [ - # { - # description: 'Player moves.', - # service_name: '*', - # http_method: '*', - # url_path: '/api/move/*', - # fixed_target: 0, - # rate: 0.05 - # } - # ], - # default: { - # fixed_target: 1, - # rate: 0.1 - # } - # } - # This example defines one custom rule and a default rule. - # The custom rule applies a five-percent sampling rate with no minimum - # number of requests to trace for paths under /api/move/. The default - # rule traces the first request each second and 10 percent of additional requests. - # The SDK applies custom rules in the order in which they are defined. - # If a request matches multiple custom rules, the SDK applies only the first rule. + # Making sampling decisions based on service sampling rules defined + # by X-Ray control plane APIs. It will fall back to local sampling rules + # if service sampling rules are not available or expired. class DefaultSampler include Sampler - DEFAULT_RULES = { - version: 1, - default: { - fixed_target: 1, - rate: 0.05 - }, - rules: [] - }.freeze + include Logging + attr_reader :cache, :local_sampler, :poller + attr_accessor :origin def initialize - load_sampling_rules(DEFAULT_RULES) + @local_sampler = LocalSampler.new + @cache = RuleCache.new + @poller = LeadPoller.new(@cache) + + @started = false + @origin = nil + @lock = Mutex.new end - # Return True if the sampler decide to sample based on input - # information and sampling rules. It will first check if any - # custom rule should be applied, if not it falls back to the - # default sampling rule. - # All arugments are extracted from incoming requests by - # X-Ray middleware to perform path based sampling. - def sample_request?(service_name:, url_path:, http_method:) - # directly fallback to non-path-based if all arguments are nil - return sample? unless service_name || url_path || http_method - @custom_rules ||= [] - @custom_rules.each do |c| - return should_sample?(c) if c.applies?(target_name: service_name, target_path: url_path, target_method: http_method) + # Start background threads to poll sampling rules + def start + @lock.synchronize do + unless @started + @poller.start + @started = true + end + end + end + + # Return the rule name if it decides to sample based on + # a service sampling rule matching. If there is no match + # it will fallback to local defined sampling rules. + def sample_request?(sampling_req) + start unless @started + now = Time.now.to_i + if sampling_req.nil? + sampling_req = { service_type: @origin } if @origin + elsif !sampling_req.key?(:service_type) + sampling_req[:service_type] = @origin if @origin + end + + matched_rule = @cache.get_matched_rule(sampling_req, now: now) + if !matched_rule.nil? + logger.debug %(Rule #{matched_rule.name} is selected to make a sampling decision.') + process_matched_rule(matched_rule, now) + else + logger.warn %(No effective centralized sampling rule match. Fallback to local rules.) + @local_sampler.sample_request?(sampling_req) end - sample? end - # Decides if should sample based on non-path-based rule. - # Currently only the default rule is not path-based. def sample? - should_sample?(@default_rule) + sample_request? nil end - # @param [Hash] v The sampling rules definition. + # @param [Hash] v Local sampling rules definition. + # This configuration has lower priority than service + # sampling rules and only has effect when those rules + # are not available or expired. def sampling_rules=(v) - load_sampling_rules(v) + @local_sampler.sampling_rules = v end - # @return [Array] An array of [SamplingRule] - def sampling_rules - all_rules = [] - all_rules << @default_rule - all_rules << @custom_rules unless @custom_rules.empty? - all_rules + def daemon_config=(v) + @poller.connector.daemon_config = v end private - def should_sample?(rule) - return true if rule.reservoir.take - Random.rand <= rule.rate - end - - def load_sampling_rules(v) - version = v[:version] - if version != 1 - raise InvalidSamplingConfigError, %('Sampling rule version #{version} is not supported.') - end - unless v[:default] - raise InvalidSamplingConfigError, 'A default rule must be provided.' - end - @default_rule = SamplingRule.new rule_definition: v[:default], default: true - @custom_rules = [] - v[:rules].each do |d| - @custom_rules << SamplingRule.new(rule_definition: d) + def process_matched_rule(rule, now) + # As long as a rule is matched we increment request counter. + rule.increment_request_count + reservoir = rule.reservoir + sample = true + # We check if we can borrow or take from reservoir first. + decision = reservoir.borrow_or_take(now, rule.borrowable?) + if decision == SamplingDecision::BORROW + rule.increment_borrow_count + elsif decision == SamplingDecision::TAKE + rule.increment_sampled_count + # Otherwise we compute based on fixed rate of this sampling rule. + elsif rand <= rule.rate + rule.increment_sampled_count + else + sample = false end + sample ? rule.name : false end end end diff --git a/lib/aws-xray-sdk/sampling/lead_poller.rb b/lib/aws-xray-sdk/sampling/lead_poller.rb new file mode 100644 index 0000000..e39440e --- /dev/null +++ b/lib/aws-xray-sdk/sampling/lead_poller.rb @@ -0,0 +1,72 @@ +require 'aws-xray-sdk/logger' +require 'aws-xray-sdk/sampling/connector' +require 'aws-xray-sdk/sampling/rule_poller' + +module XRay + # The poller to report the current statistics of all + # sampling rules and retrieve the new allocated + # sampling quota and TTL from X-Ray service. It also + # controls the rule poller. + class LeadPoller + include Logging + attr_reader :connector + @@interval = 10 # working frequency of the lead poller + @@rule_interval = 5 * 60 # 5 minutes on polling rules + + def initialize(cache) + @cache = cache + @connector = ServiceConnector.new + @rule_poller = RulePoller.new cache: @cache, connector: @connector + @rule_poller_elapsed = 0 + end + + def start + @rule_poller.run + Thread.new { worker } + end + + def worker + loop do + sleep_time = @@interval + rand + sleep sleep_time + @rule_poller_elapsed += sleep_time + refresh_cache + if @rule_poller_elapsed >= @@rule_interval + @rule_poller.run + @rule_poller_elapsed = 0 + end + end + end + + private + + def refresh_cache + candidates = get_candidates(@cache.rules) + if candidates.empty? + logger.debug %(No X-Ray sampling rules to report statistics. Skipping.) + return + end + + result = @connector.fetch_sampling_targets(candidates) + targets = {} + result[:documents].each { |doc| targets[doc.rule_name] = doc } + @cache.load_targets(targets) + + return unless @cache.last_updated && result[:last_modified].to_i > @cache.last_updated + logger.info 'Performing out-of-band sampling rule polling to fetch updated rules.' + @rule_poller.run + @rule_poller_elapsed = 0 + rescue StandardError => e + logger.warn %(failed to fetch X-Ray sampling targets due to #{e.message}) + end + + # Don't report a rule statistics if any of the conditions is met: + # 1. The report time hasn't come(some rules might have larger report intervals). + # 2. The rule is never matched. + def get_candidates(rules) + candidates = [] + rules.each { |rule| candidates << rule if rule.ever_matched? && rule.time_to_report? } + candidates + end + end +end diff --git a/lib/aws-xray-sdk/sampling/local/reservoir.rb b/lib/aws-xray-sdk/sampling/local/reservoir.rb new file mode 100644 index 0000000..9a081f5 --- /dev/null +++ b/lib/aws-xray-sdk/sampling/local/reservoir.rb @@ -0,0 +1,35 @@ +module XRay + # Keeps track of the number of sampled segments within + # a single second in the local process. This class is + # implemented to be thread-safe to achieve accurate sampling. + class LocalReservoir + # @param [Integer] traces_per_sec The number of guranteed sampled + # segments per second. + def initialize(traces_per_sec: 0) + @traces_per_sec = traces_per_sec + @used_this_sec = 0 + @this_sec = Time.now.to_i + @lock = Mutex.new + end + + # Returns `true` if there are quota left within the + # current second, otherwise returns `false`. + def take + # nothing to provide if reserved is set to 0 + return false if @traces_per_sec.zero? + @lock.synchronize do + now = Time.now.to_i + # refresh time frame + if now != @this_sec + @used_this_sec = 0 + @this_sec = now + end + # return false if reserved item ran out + return false unless @used_this_sec < @traces_per_sec + # otherwise increment used counter and return true + @used_this_sec += 1 + return true + end + end + end +end diff --git a/lib/aws-xray-sdk/sampling/local/sampler.rb b/lib/aws-xray-sdk/sampling/local/sampler.rb new file mode 100644 index 0000000..c731801 --- /dev/null +++ b/lib/aws-xray-sdk/sampling/local/sampler.rb @@ -0,0 +1,110 @@ +require 'aws-xray-sdk/sampling/sampler' +require 'aws-xray-sdk/sampling/local/sampling_rule' +require 'aws-xray-sdk/exceptions' + +module XRay + # The local sampler that uses locally defined + # sampling rule and reservoir models to decide sampling decision. + # It also uses the default sampling rule. + # An example definition: + # { + # version: 2, + # rules: [ + # { + # description: 'Player moves.', + # host: '*', + # http_method: '*', + # url_path: '/api/move/*', + # fixed_target: 0, + # rate: 0.05 + # } + # ], + # default: { + # fixed_target: 1, + # rate: 0.1 + # } + # } + # This example defines one custom rule and a default rule. + # The custom rule applies a five-percent sampling rate with no minimum + # number of requests to trace for paths under /api/move/. The default + # rule traces the first request each second and 10 percent of additional requests. + # The SDK applies custom rules in the order in which they are defined. + # If a request matches multiple custom rules, the SDK applies only the first rule. + class LocalSampler + include Sampler + DEFAULT_RULES = { + version: 2, + default: { + fixed_target: 1, + rate: 0.05 + }, + rules: [] + }.freeze + + SUPPORTED_VERSION = [1, 2].freeze + + def initialize + load_sampling_rules(DEFAULT_RULES) + end + + # Return True if the sampler decide to sample based on input + # information and sampling rules. It will first check if any + # custom rule should be applied, if not it falls back to the + # default sampling rule. + # All arugments are extracted from incoming requests by + # X-Ray middleware to perform path based sampling. + def sample_request?(sampling_req) + sample = sample? + return sample if sampling_req.nil? || sampling_req.empty? + @custom_rules ||= [] + @custom_rules.each do |c| + return should_sample?(c) if c.applies?(sampling_req: sampling_req) + end + # use previously made decision based on default rule + # if no path-based rule has been matched + sample + end + + # Decides if should sample based on non-path-based rule. + # Currently only the default rule is non-path-based. + def sample? + should_sample?(@default_rule) + end + + # @param [Hash] v The sampling rules definition. + def sampling_rules=(v) + load_sampling_rules(v) + end + + # @return [Array] An array of [SamplingRule] + def sampling_rules + all_rules = [] + all_rules << @default_rule + all_rules << @custom_rules unless @custom_rules.empty? + all_rules + end + + private + + def should_sample?(rule) + return true if rule.reservoir.take + Random.rand <= rule.rate + end + + def load_sampling_rules(v) + version = v[:version] + unless SUPPORTED_VERSION.include?(version) + raise InvalidSamplingConfigError, %('Sampling rule version #{version} is not supported.') + end + unless v[:default] + raise InvalidSamplingConfigError, 'A default rule must be provided.' + end + @default_rule = LocalSamplingRule.new rule_definition: v[:default], default: true + @custom_rules = [] + v[:rules].each do |d| + d[:host] = d[:service_name] if version == 1 + @custom_rules << LocalSamplingRule.new(rule_definition: d) + end + end + end +end diff --git a/lib/aws-xray-sdk/sampling/local/sampling_rule.rb b/lib/aws-xray-sdk/sampling/local/sampling_rule.rb new file mode 100644 index 0000000..8b8fb1c --- /dev/null +++ b/lib/aws-xray-sdk/sampling/local/sampling_rule.rb @@ -0,0 +1,63 @@ +require 'aws-xray-sdk/exceptions' +require 'aws-xray-sdk/sampling/local/reservoir' +require 'aws-xray-sdk/search_pattern' + +module XRay + # One SamplingRule object represents one rule defined from the rules hash definition. + # It can be either a custom rule or the default rule. + class LocalSamplingRule + attr_reader :fixed_target, :rate, :host, + :method, :path, :reservoir, :default + + # @param [Hash] rule_definition Hash that defines a single rule. + # @param default A boolean flag indicates if this rule is the default rule. + def initialize(rule_definition:, default: false) + @fixed_target = rule_definition[:fixed_target] + @rate = rule_definition[:rate] + + @host = rule_definition[:host] + @method = rule_definition[:http_method] + @path = rule_definition[:url_path] + + @default = default + validate + @reservoir = LocalReservoir.new traces_per_sec: @fixed_target + end + + # Determines whether or not this sampling rule applies to + # the incoming request based on some of the request's parameters. + # Any None parameters provided will be considered an implicit match. + def applies?(sampling_req) + return false if sampling_req.nil? || sampling_req.empty? + + host = sampling_req[:host] + url_path = sampling_req[:url_path] + http_method = sampling_req[:http_method] + + host_match = !host || SearchPattern.wildcard_match?(pattern: @host, text: host) + path_match = !url_path || SearchPattern.wildcard_match?(pattern: @path, text: url_path) + method_match = !http_method || SearchPattern.wildcard_match?(pattern: @method, text: http_method) + host_match && path_match && method_match + end + + private + + def validate + if @fixed_target < 0 || @rate < 0 + raise InvalidSamplingConfigError, 'All rules must have non-negative values for fixed_target and rate.' + end + + if @default + # validate default rule + if @host || @method || @path + raise InvalidSamplingConfigError, 'The default rule must not specify values for url_path, service_name, or http_method.' + end + else + # validate custom rule + unless @host && @method && @path + raise InvalidSamplingConfigError, 'All non-default rules must have values for url_path, service_name, and http_method.' + end + end + end + end +end diff --git a/lib/aws-xray-sdk/sampling/reservoir.rb b/lib/aws-xray-sdk/sampling/reservoir.rb index ee3ceb3..f5292eb 100644 --- a/lib/aws-xray-sdk/sampling/reservoir.rb +++ b/lib/aws-xray-sdk/sampling/reservoir.rb @@ -1,35 +1,79 @@ +require 'date' +require 'aws-xray-sdk/sampling/sampling_decision' + module XRay - # Keeps track of the number of sampled segments within - # a single second. This class is implemented to be - # thread-safe to achieve accurate sampling. + # Centralized thread-safe reservoir which holds fixed sampling + # quota for the current instance, borrowed count and TTL. class Reservoir - # @param [Integer] traces_per_sec The number of guranteed sampled - # segments per second. - def initialize(traces_per_sec: 0) - @traces_per_sec = traces_per_sec - @used_this_sec = 0 - @this_sec = Time.now.to_i + attr_reader :quota, :ttl + + def initialize + @quota = nil + @ttl = nil + + @this_sec = 0 + @taken_this_sec = 0 + @borrowed_this_sec = 0 + + @report_interval = 1 + @report_elapsed = 0 + @lock = Mutex.new end - # Returns `true` if there are segments left within the - # current second, otherwise returns `false`. - def take - # nothing to provide if reserved is set to 0 - return false if @traces_per_sec.zero? + # Decide whether to borrow or take one quota from + # the reservoir. Return `false` if it can neither + # borrow nor take. This method is thread-safe. + def borrow_or_take(now, borrowable) @lock.synchronize do - now = Time.now.to_i - # refresh time frame - if now != @this_sec - @used_this_sec = 0 - @this_sec = now + reset_new_sec(now) + # Don't borrow if the quota is available and fresh. + if quota_fresh?(now) + return SamplingDecision::NOT_SAMPLE if @taken_this_sec >= @quota + @taken_this_sec += 1 + return SamplingDecision::TAKE end - # return false if reserved item ran out - return false unless @used_this_sec < @traces_per_sec - # otherwise increment used counter and return true - @used_this_sec += 1 - return true + + # Otherwise try to borrow if the quota is not present or expired. + if borrowable + return SamplingDecision::NOT_SAMPLE if @borrowed_this_sec >= 1 + @borrowed_this_sec += 1 + return SamplingDecision::BORROW + end + + # Cannot sample if quota expires and cannot borrow + SamplingDecision::NOT_SAMPLE + end + end + + def load_target_info(quota:, ttl:, interval:) + @quota = quota unless quota.nil? + @ttl = ttl.to_i unless ttl.nil? + @interval = interval / 10 unless interval.nil? + end + + def time_to_report? + if @report_elapsed + 1 >= @report_interval + @report_elapsed = 0 + true + else + @report_elapsed += 1 + false end end + + private + + # Reset the counter if now enters a new one-second window + def reset_new_sec(now) + return if now == @this_sec + @taken_this_sec = 0 + @borrowed_this_sec = 0 + @this_sec = now + end + + def quota_fresh?(now) + @quota && @quota >= 0 && @ttl && @ttl >= now + end end end diff --git a/lib/aws-xray-sdk/sampling/rule_cache.rb b/lib/aws-xray-sdk/sampling/rule_cache.rb new file mode 100644 index 0000000..2b5f1c1 --- /dev/null +++ b/lib/aws-xray-sdk/sampling/rule_cache.rb @@ -0,0 +1,86 @@ +require 'aws-xray-sdk/logger' + +module XRay + # Cache sampling rules and quota retrieved by `TargetPoller` + # and `RulePoller`. It will not return anything if it expires. + class RuleCache + include Logging + attr_accessor :last_updated + @@TTL = 60 * 60 # 1 hour + + def initialize + @rules = [] + @last_updated = nil + @lock = Mutex.new + end + + def get_matched_rule(sampling_req, now: Time.now.to_i) + return nil if expired?(now) + matched = nil + rules.each do |rule| + matched = rule if matched.nil? && rule.applies?(sampling_req) + matched = rule if matched.nil? && rule.default? + end + matched + end + + def load_rules(new_rules) + @lock.synchronize do + # Simply assign rules and sort if cache is empty + if @rules.empty? + @rules = new_rules + return sort_rules + end + + # otherwise we need to merge new rules and current rules + curr_rules = {} + @rules.each do |rule| + curr_rules[rule.name] = rule + end + # Update the rules in the cache + @rules = new_rules + # Transfer state information + @rules.each do |rule| + curr_rule = curr_rules[rule.name] + rule.merge(curr_rule) unless curr_rule.nil? + end + sort_rules + end + end + + def load_targets(targets_h) + @lock.synchronize do + @rules.each do |rule| + target = targets_h[rule.name] + next if target.nil? + rule.rate = target.fixed_rate + rule.reservoir.load_target_info( + quota: target.reservoir_quota, + ttl: target.reservoir_quota_ttl, + interval: target.interval + ) + end + end + end + + def rules + @lock.synchronize do + @rules + end + end + + private + + # The cache should maintain the order of the rules based on + # priority. If priority is the same we sort name by alphabet + # as rule name is unique. + def sort_rules + @rules.sort_by! { |rule| [rule.priority, rule.name] } + end + + def expired?(now) + # The cache is treated as expired if it is never loaded. + @last_updated.nil? || now > @last_updated + @@TTL + end + end +end diff --git a/lib/aws-xray-sdk/sampling/rule_poller.rb b/lib/aws-xray-sdk/sampling/rule_poller.rb new file mode 100644 index 0000000..06c6c56 --- /dev/null +++ b/lib/aws-xray-sdk/sampling/rule_poller.rb @@ -0,0 +1,39 @@ +require 'aws-xray-sdk/logger' + +module XRay + # Polls sampling rules from X-Ray service + class RulePoller + include Logging + attr_reader :cache, :connector + + def initialize(cache:, connector:) + @cache = cache + @connector = connector + @worker = Thread.new { poll } + end + + def run + @worker.run + end + + private + + def poll + loop do + Thread.stop + refresh_cache + end + end + + def refresh_cache + now = Time.now.to_i + rules = @connector.fetch_sampling_rules + unless rules.nil? || rules.empty? + @cache.load_rules(rules) + @cache.last_updated = now + end + rescue StandardError => e + logger.warn %(failed to fetch X-Ray sampling rules due to #{e.backtrace}) + end + end +end diff --git a/lib/aws-xray-sdk/sampling/sampler.rb b/lib/aws-xray-sdk/sampling/sampler.rb index a2d0cca..38bbef6 100644 --- a/lib/aws-xray-sdk/sampling/sampler.rb +++ b/lib/aws-xray-sdk/sampling/sampler.rb @@ -6,12 +6,12 @@ module XRay module Sampler # Decides if a segment should be sampled for an incoming request. # Used in case of middleware. - def sample_request?(service_name:, url_path:, http_method:) + def sample_request?(sampling_req:) raise 'Not implemented' end - # Decides if a segment should be sampled merely based on internal - # sampling rules. + # Sample purely based on cached sampling rules without + # any incoming rules matching information. def sample? raise 'Not implemented' end diff --git a/lib/aws-xray-sdk/sampling/sampling_decision.rb b/lib/aws-xray-sdk/sampling/sampling_decision.rb new file mode 100644 index 0000000..b0070bc --- /dev/null +++ b/lib/aws-xray-sdk/sampling/sampling_decision.rb @@ -0,0 +1,8 @@ +module XRay + # Stores the enum style sampling decisions for default sampler + module SamplingDecision + TAKE = 'take'.freeze + BORROW = 'borrow'.freeze + NOT_SAMPLE = 'no'.freeze + end +end diff --git a/lib/aws-xray-sdk/sampling/sampling_rule.rb b/lib/aws-xray-sdk/sampling/sampling_rule.rb index 1947ea0..82c253c 100644 --- a/lib/aws-xray-sdk/sampling/sampling_rule.rb +++ b/lib/aws-xray-sdk/sampling/sampling_rule.rb @@ -3,55 +3,122 @@ require 'aws-xray-sdk/search_pattern' module XRay - # One SamplingRule object represents one rule defined from the rules hash definition. - # It can be either a custom rule or the default rule. + # Service sampling rule data model class SamplingRule - attr_reader :fixed_target, :rate, :service_name, - :method, :path, :reservoir, :default + attr_reader :name, :priority, + :request_count, :borrow_count, :sampled_count + attr_accessor :reservoir, :rate - # @param [Hash] rule_definition Hash that defines a single rule. - # @param default A boolean flag indicates if this rule is the default rule. - def initialize(rule_definition:, default: false) - @fixed_target = rule_definition[:fixed_target] - @rate = rule_definition[:rate] + # @param Struct defined here https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/XRay/Types/SamplingRule.html. + def initialize(record) + @name = record.rule_name + @priority = record.priority + @rate = record.fixed_rate - @service_name = rule_definition[:service_name] - @method = rule_definition[:http_method] - @path = rule_definition[:url_path] + @host = record.host + @method = record.http_method + @path = record.url_path + @service = record.service_name + @service_type = record.service_type - @default = default - validate - @reservoir = Reservoir.new traces_per_sec: @fixed_target + @reservoir_size = record.reservoir_size + @reservoir = Reservoir.new + reset_statistics + + @lock = Mutex.new end # Determines whether or not this sampling rule applies to # the incoming request based on some of the request's parameters. - # Any None parameters provided will be considered an implicit match. - def applies?(target_name:, target_path:, target_method:) - name_match = !target_name || SearchPattern.wildcard_match?(pattern: @service_name, text: target_name) - path_match = !target_path || SearchPattern.wildcard_match?(pattern: @path, text: target_path) - method_match = !target_method || SearchPattern.wildcard_match?(pattern: @method, text: target_method) - name_match && path_match && method_match + # Any Nil parameters provided will be considered as implicit matches + # as the rule matching is a best effort. + def applies?(sampling_req) + return false if sampling_req.nil? || sampling_req.empty? + + host = sampling_req[:host] + http_method = sampling_req[:http_method] + url_path = sampling_req[:url_path] + service = sampling_req[:service] + + host_match = !host || SearchPattern.wildcard_match?(pattern: @host, text: host) + path_match = !url_path || SearchPattern.wildcard_match?(pattern: @path, text: url_path) + method_match = !http_method || SearchPattern.wildcard_match?(pattern: @method, text: http_method) + service_match = !service || SearchPattern.wildcard_match?(pattern: @service, text: service) + + # if sampling request contains service type we assmue + # the origin (a.k.a AWS plugins are set and effective) + if sampling_req.key?(:service_type) + service_type = sampling_req[:service_type] + service_type_match = SearchPattern.wildcard_match?(pattern: @service_type, text: service_type) + else + service_type_match = @service_type == '*' + end + host_match && path_match && method_match && service_match && service_type_match end - private + def snapshot_statistics + @lock.synchronize do + report = { + request_count: @request_count, + borrow_count: @borrow_count, + sampled_count: @sampled_count + } + reset_statistics + report + end + end - def validate - if @fixed_target < 0 || @rate < 0 - raise InvalidSamplingConfigError, 'All rules must have non-negative values for fixed_target and rate.' + def merge(rule) + @lock.synchronize do + @request_count = rule.request_count + @borrow_count = rule.borrow_count + @sampled_count = rule.sampled_count + @reservoir = rule.reservoir + rule.reservoir = nil end + end - if @default - # validate default rule - if @service_name || @method || @path - raise InvalidSamplingConfigError, 'The default rule must not specify values for url_path, service_name, or http_method.' - end - else - # validate custom rule - unless @service_name && @method && @path - raise InvalidSamplingConfigError, 'All non-default rules must have values for url_path, service_name, and http_method.' - end + def borrowable? + @reservoir_size != 0 + end + + # Return `true` if this rule is the default rule. + def default? + @name == 'Default' + end + + def ever_matched? + @request_count > 0 + end + + def time_to_report? + @reservoir.time_to_report? + end + + def increment_request_count + @lock.synchronize do + @request_count += 1 + end + end + + def increment_borrow_count + @lock.synchronize do + @borrow_count += 1 end end + + def increment_sampled_count + @lock.synchronize do + @sampled_count += 1 + end + end + + private + + def reset_statistics + @request_count = 0 + @borrow_count = 0 + @sampled_count = 0 + end end end diff --git a/lib/aws-xray-sdk/version.rb b/lib/aws-xray-sdk/version.rb index fab0a49..6bd4ef3 100644 --- a/lib/aws-xray-sdk/version.rb +++ b/lib/aws-xray-sdk/version.rb @@ -1,3 +1,3 @@ module XRay - VERSION = '0.10.2' + VERSION = '0.11.0' end diff --git a/test/aws-xray-sdk/tc_aws_sdk.rb b/test/aws-xray-sdk/tc_aws_sdk.rb index 801659b..65c07dc 100644 --- a/test/aws-xray-sdk/tc_aws_sdk.rb +++ b/test/aws-xray-sdk/tc_aws_sdk.rb @@ -1,7 +1,8 @@ require_relative '../test_helper' require 'aws-xray-sdk' -require 'aws-sdk-s3' require 'aws-sdk-dynamodb' +require 'aws-sdk-s3' +require 'aws-sdk-xray' # Test subsegments recording on AWS Ruby SDK class TestAwsSdk < Minitest::Test @@ -96,7 +97,19 @@ def test_capture_map_keys assert_equal mocked_resp[:consumed_capacity], aws_meta[:consumed_capacity] end - def test_capiture_client_error + def test_not_capture_sampling_calls + xray = Aws::XRay::Client.new(stub_responses: true) + + rules_resp = xray.stub_data(:get_sampling_rules, {}) + xray.stub_responses(:get_sampling_rules, rules_resp) + xray.get_sampling_rules + + targets_resp = xray.stub_data(:get_sampling_targets, {}) + xray.stub_responses(:get_sampling_targets, targets_resp) + xray.get_sampling_targets sampling_statistics_documents: {} + end + + def test_capture_client_error @@recorder.begin_segment name s3 = Aws::S3::Client.new(stub_responses: true) s3.stub_responses(:head_bucket, Timeout::Error) @@ -131,5 +144,6 @@ def test_log_error_pass_through assert_nil recorder.emitter.entities # s3 call is executed assert_equal '1', resp.buckets[0].name + recorder.context.clear! end end diff --git a/test/aws-xray-sdk/tc_daemon_config.rb b/test/aws-xray-sdk/tc_daemon_config.rb new file mode 100644 index 0000000..d292c83 --- /dev/null +++ b/test/aws-xray-sdk/tc_daemon_config.rb @@ -0,0 +1,58 @@ +require_relative '../test_helper' +require 'aws-xray-sdk/daemon_config' +require 'aws-xray-sdk/exceptions' + +# Test daemon configuration +class TestDaemonConfig < Minitest::Test + def test_single_address + ip = '192.168.0.1' + port = 8000 + config = XRay::DaemonConfig.new addr: %(#{ip}:#{port}) + + assert_equal config.udp_ip, ip + assert_equal config.udp_port, port + assert_equal config.tcp_ip, ip + assert_equal config.tcp_port, port + end + + def test_tcp_and_udp + tcp_ip = '192.168.0.1' + tcp_port = 8000 + udp_ip = '127.0.0.1' + udp_port = 3000 + tcp = %(tcp:#{tcp_ip}:#{tcp_port}) + udp = %(udp:#{udp_ip}:#{udp_port}) + + config = XRay::DaemonConfig.new addr: %(#{tcp} #{udp}) + + assert_equal udp_ip, config.udp_ip + assert_equal udp_port, config.udp_port + assert_equal tcp_ip, config.tcp_ip + assert_equal tcp_port, config.tcp_port + + config.update_address %(#{udp} #{tcp}) + + assert_equal udp_ip, config.udp_ip + assert_equal udp_port, config.udp_port + assert_equal tcp_ip, config.tcp_ip + assert_equal tcp_port, config.tcp_port + end + + def test_invalid_config + assert_raises XRay::InvalidDaemonAddressError do + XRay::DaemonConfig.new addr: 'tcp:127.0.0.1:2000' + end + + assert_raises XRay::InvalidDaemonAddressError do + XRay::DaemonConfig.new addr: '127.0.0.1' + end + + assert_raises XRay::InvalidDaemonAddressError do + XRay::DaemonConfig.new addr: 'tcp:127.0.0.1:2000 tcp:127.0.0.1:3000' + end + + assert_raises XRay::InvalidDaemonAddressError do + XRay::DaemonConfig.new addr: 'tcp:127.0.0.1:2000udp:127.0.0.1:3000' + end + end +end diff --git a/test/aws-xray-sdk/tc_sampling.rb b/test/aws-xray-sdk/tc_local_sampling.rb similarity index 62% rename from test/aws-xray-sdk/tc_sampling.rb rename to test/aws-xray-sdk/tc_local_sampling.rb index b874316..e35ac7b 100644 --- a/test/aws-xray-sdk/tc_sampling.rb +++ b/test/aws-xray-sdk/tc_local_sampling.rb @@ -1,6 +1,6 @@ -require 'aws-xray-sdk/sampling/sampling_rule' -require 'aws-xray-sdk/sampling/reservoir' -require 'aws-xray-sdk/sampling/default_sampler' +require 'aws-xray-sdk/sampling/local/reservoir' +require 'aws-xray-sdk/sampling/local/sampler' +require 'aws-xray-sdk/sampling/local/sampling_rule' require 'aws-xray-sdk/exceptions' # Test sampling models and the default sampler @@ -8,37 +8,42 @@ class TestSampling < Minitest::Test VALID_RULE_DEF = { fixed_target: 1, rate: 0.5, - service_name: '*', + host: '*', url_path: '*/ping', http_method: 'PUT' }.freeze def test_reservoir_pass_through - reservoir = XRay::Reservoir.new traces_per_sec: 1 + reservoir = XRay::LocalReservoir.new traces_per_sec: 1 assert reservoir.take - reservoir2 = XRay::Reservoir.new + reservoir2 = XRay::LocalReservoir.new refute reservoir2.take end def test_simple_single_rule - rule = XRay::SamplingRule.new rule_definition: VALID_RULE_DEF + rule = XRay::LocalSamplingRule.new rule_definition: VALID_RULE_DEF assert_equal 1, rule.fixed_target assert_equal 0.5, rule.rate - assert_equal '*', rule.service_name + assert_equal '*', rule.host assert_equal '*/ping', rule.path assert_equal 'PUT', rule.method assert rule.reservoir.take end def test_rule_request_matching - rule = XRay::SamplingRule.new rule_definition: VALID_RULE_DEF + rule = XRay::LocalSamplingRule.new rule_definition: VALID_RULE_DEF - assert rule.applies? target_name: nil, target_path: '/ping', target_method: 'put' - assert rule.applies? target_name: 'a', target_path: nil, target_method: 'put' - assert rule.applies? target_name: 'a', target_path: '/ping', target_method: nil - assert rule.applies? target_name: 'a', target_path: '/ping', target_method: 'PUT' - refute rule.applies? target_name: 'a', target_path: '/sping', target_method: 'PUT' + req1 = { host: nil, url_path: '/ping', http_method: 'put' } + req2 = { host: 'a', url_path: nil, http_method: 'put' } + req3 = { host: 'a', url_path: '/ping', http_method: nil } + req4 = { host: 'a', url_path: '/ping', http_method: 'PUT' } + req5 = { host: 'a', url_path: '/sping', http_method: 'PUT' } + assert rule.applies?(req1) + assert rule.applies?(req2) + assert rule.applies?(req3) + assert rule.applies?(req4) + refute rule.applies?(req5) end def test_invalid_single_rule @@ -46,20 +51,20 @@ def test_invalid_single_rule rule_def1 = { fixed_target: 1, rate: 0.5, - service_name: '*', + host: '*', http_method: 'GET' } assert_raises XRay::InvalidSamplingConfigError do - XRay::SamplingRule.new rule_definition: rule_def1 + XRay::LocalSamplingRule.new rule_definition: rule_def1 end # extra field for default rule rule_def2 = { fixed_target: 1, rate: 0.5, - service_name: '*' + host: '*' } assert_raises XRay::InvalidSamplingConfigError do - XRay::SamplingRule.new rule_definition: rule_def2, default: true + XRay::LocalSamplingRule.new rule_definition: rule_def2, default: true end # invalid value rule_def3 = { @@ -67,16 +72,16 @@ def test_invalid_single_rule rate: -0.5 } assert_raises XRay::InvalidSamplingConfigError do - XRay::SamplingRule.new rule_definition: rule_def3, default: true + XRay::LocalSamplingRule.new rule_definition: rule_def3, default: true end end EXAMPLE_CONFIG = { - version: 1, + version: 2, rules: [ { description: 'Player moves.', - service_name: '*', + host: '*', http_method: '*', url_path: '*/ping', fixed_target: 0, @@ -89,8 +94,8 @@ def test_invalid_single_rule } }.freeze - def test_default_sampler - sampler = XRay::DefaultSampler.new + def test_local_sampler + sampler = XRay::LocalSampler.new assert sampler.sample? # should only has default rule assert_equal 1, sampler.sampling_rules.count @@ -102,7 +107,7 @@ def test_default_sampler end def test_invalid_rules_config - sampler = XRay::DefaultSampler.new + sampler = XRay::LocalSampler.new config1 = EXAMPLE_CONFIG.merge(version: nil) assert_raises XRay::InvalidSamplingConfigError do sampler.sampling_rules = config1 diff --git a/test/aws-xray-sdk/tc_recorder.rb b/test/aws-xray-sdk/tc_recorder.rb index 96f1616..3ece24f 100644 --- a/test/aws-xray-sdk/tc_recorder.rb +++ b/test/aws-xray-sdk/tc_recorder.rb @@ -93,6 +93,20 @@ def test_sampled_block assert @@recorder.sampled? end + def test_sampling_segments + recorder = XRay::Recorder.new + config = { + sampling: true, + emitter: XRay::TestHelper::StubbedEmitter.new, + sampler: XRay::TestHelper::StubbedDefaultSampler.new + } + recorder.configure(config) + + segment = recorder.begin_segment('name') + assert segment.sampled + recorder.context.clear! + end + def test_add_annotation segment = @@recorder.begin_segment name @@recorder.annotations[:k] = 1 diff --git a/test/aws-xray-sdk/tc_sampling_rule_cache.rb b/test/aws-xray-sdk/tc_sampling_rule_cache.rb new file mode 100644 index 0000000..4b84d1b --- /dev/null +++ b/test/aws-xray-sdk/tc_sampling_rule_cache.rb @@ -0,0 +1,117 @@ +require_relative '../test_helper' +require 'aws-xray-sdk/sampling/rule_cache' +require 'aws-xray-sdk/sampling/sampling_rule' + +# Test sampling rule cache +class TestSamplingRuleCache < Minitest::Test + RuleDef = Struct.new(:rule_name, :priority, :fixed_rate, + :host, :http_method, :url_path, + :service_name, :service_type, :reservoir_size) + SamplingTarget = Struct.new(:fixed_rate, :reservoir_quota, + :reservoir_quota_ttl, :interval) + + def setup + @@rule0 = XRay::SamplingRule.new RuleDef.new('a', 1, 0.1, '*mydomain*', 'GET', + 'myop', 'random_name', '*', 1) + @@rule1 = XRay::SamplingRule.new RuleDef.new('aa', 2, 0.1, '*random*', 'POST', + 'random', 'proxy', '*', 1) + @@rule2 = XRay::SamplingRule.new RuleDef.new('b', 2, 0.1, '*', 'GET', 'ping', + 'myapp', 'AWS::EC2::Instance', 1) + @@rule_default = XRay::SamplingRule.new RuleDef.new('Default', 10000, 0.1, '*', + '*', '*', '*', '*', 1) + end + + def test_rules_sorting + cache = XRay::RuleCache.new + cache.load_rules [@@rule_default, @@rule2, @@rule0, @@rule1] + sorted_rules = cache.rules + + assert_equal 'a', sorted_rules[0].name + assert_equal 'aa', sorted_rules[1].name + assert_equal 'b', sorted_rules[2].name + assert_equal 'Default', sorted_rules[3].name + end + + def test_evict_deleted_rules + cache = XRay::RuleCache.new + cache.load_rules [@@rule_default, @@rule0, @@rule1] + cache.load_rules [@@rule_default, @@rule2] + + rules = cache.rules + assert_equal 2, rules.length + assert rules.include?(@@rule_default) + assert rules.include?(@@rule2) + end + + def test_preserving_sampling_statistics + now = Time.now.to_i + cache = XRay::RuleCache.new + cache.load_rules [@@rule_default, @@rule0] + @@rule0.increment_request_count + @@rule0.increment_sampled_count + @@rule0.reservoir.load_target_info quota: 3, ttl: now, interval: nil + updated_rule0 = XRay::SamplingRule.new RuleDef.new('a', 1, 0.1, '*', 'GET', + 'myop', '*', '*', 1) + cache.load_rules [@@rule_default, updated_rule0] + new_rule0 = cache.rules[0] + + assert_equal 1, new_rule0.request_count + assert_equal 1, new_rule0.sampled_count + assert_equal 3, new_rule0.reservoir.quota + assert_equal now, new_rule0.reservoir.ttl + end + + def test_target_mapping + cache = XRay::RuleCache.new + cache.load_rules [@@rule_default, @@rule0] + targets = { + 'a' => SamplingTarget.new(0.1, 5, nil, nil), + 'b'=> SamplingTarget.new(0.1, 6, nil, nil), + 'Default' => SamplingTarget.new(0.1, 7, nil, nil), + } + cache.load_targets(targets) + + assert_equal 5, cache.rules[0].reservoir.quota + assert_equal 7, cache.rules[1].reservoir.quota + end + + def test_expired_cache + now = Time.now.to_i + cache = XRay::RuleCache.new + cache.load_rules [@@rule_default, @@rule2, @@rule0, @@rule1] + cache.last_updated = now - 60 * 60 * 3 # 2 hours passed cache TTL + + assert_nil cache.get_matched_rule({host: 'nomatch'}, now: now) + + cache.last_updated = now + assert cache.get_matched_rule({http_method: 'nil', host: 'nil'}, now: now).default? + end + + def test_rule_matching + now = Time.now.to_i + cache = XRay::RuleCache.new + cache.load_rules [@@rule_default, @@rule2, @@rule0, @@rule1] + cache.last_updated = now + + sampling_req = {host: 'mydomain.com'} + rule = cache.get_matched_rule(sampling_req, now: now) + assert_equal 'a', rule.name + + sampling_req = {http_method: 'POST'} + rule = cache.get_matched_rule(sampling_req, now: now) + assert_equal 'aa', rule.name + + sampling_req = {service: 'proxy'} + rule = cache.get_matched_rule(sampling_req, now: now) + assert_equal 'aa', rule.name + + sampling_req = {host: 'unkown', service_type: 'AWS::EC2::Instance'} + rule = cache.get_matched_rule(sampling_req, now: now) + assert_equal 'b', rule.name + + # Default should be always returned when there is no match + sampling_req = {host: 'unknown', url_path: 'unknown'} + rule = cache.get_matched_rule(sampling_req, now: now) + assert rule.default? + end +end diff --git a/test/aws-xray-sdk/tc_segment.rb b/test/aws-xray-sdk/tc_segment.rb index ffbb284..4291778 100644 --- a/test/aws-xray-sdk/tc_segment.rb +++ b/test/aws-xray-sdk/tc_segment.rb @@ -98,6 +98,28 @@ def test_numeric_annotation_value assert_equal 'NaN', at_h[:k4] end + def test_sampling_rule_name + sdk = 'ruby' + aws = { + xray: { + sdk: sdk + } + } + segment1 = XRay::Segment.new name: 'seg1' + segment2 = XRay::Segment.new name: 'seg2' + + segment1.aws = aws + segment2.aws = aws + + segment1.sampling_rule_name = 'rule1' + segment2.sampling_rule_name = 'rule2' + + assert segment1.aws[:xray][:sdk] = sdk + assert segment1.aws[:xray][:sampling_rule_name] = 'rule1' + assert segment2.aws[:xray][:sdk] = sdk + assert segment2.aws[:xray][:sampling_rule_name] = 'rule2' + end + def test_add_subsegment segment = XRay::Segment.new name: name subsegment = XRay::Subsegment.new name: name, segment: segment diff --git a/test/test_helper.rb b/test/test_helper.rb index 8fa3723..01d9d01 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -3,6 +3,7 @@ require 'minitest/autorun' require 'aws-xray-sdk/emitter/emitter' +require 'aws-xray-sdk/sampling/default_sampler' if RUBY_PLATFORM == 'java' require 'jrjackson' @@ -28,5 +29,12 @@ def clear @entities = [] end end + + # The stubbed sampler doesn't spawn threads to call X-Ray service. + class StubbedDefaultSampler < DefaultSampler + def start + # no-op + end + end end end From fa3a55e856ae2fb3d32348c9d0d93965cf5875e7 Mon Sep 17 00:00:00 2001 From: Wu Date: Tue, 25 Sep 2018 13:31:12 -0700 Subject: [PATCH 3/4] minor merging fix --- lib/aws-xray-sdk/sampling/connector.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/aws-xray-sdk/sampling/connector.rb b/lib/aws-xray-sdk/sampling/connector.rb index 8f7d643..b31d091 100644 --- a/lib/aws-xray-sdk/sampling/connector.rb +++ b/lib/aws-xray-sdk/sampling/connector.rb @@ -37,7 +37,8 @@ def update_xray_client(ip: '127.0.0.1', port: 2000) @xray_client = Aws::XRay::Client.new( endpoint: %(http://#{ip}:#{port}), access_key_id: 'dummy', # AWS Ruby SDK doesn't support unsigned request - secret_access_key: 'dummy' + secret_access_key: 'dummy', + region: 'us-west-2' # not used ) end From 3e6910bce7d1cfd7b5918a8d3ff1befb93f6b680 Mon Sep 17 00:00:00 2001 From: Wu Date: Tue, 25 Sep 2018 14:46:26 -0700 Subject: [PATCH 4/4] update readme --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3f5f088..2c20688 100644 --- a/README.md +++ b/README.md @@ -51,13 +51,15 @@ The [API Reference](http://docs.aws.amazon.com/xray-sdk-for-ruby/latest/referenc ```ruby require 'aws-xray-sdk' -# configure path based sampling rules in case of web app +# For configuring sampling rules through X-Ray service +# please see https://docs.aws.amazon.com/xray/latest/devguide/xray-console-sampling.html. +# The doc below defines local fallback sampling rules which has lower priority. my_sampling_rules = { - version: 1, + version: 2, rules: [ { description: 'healthcheck', - service_name: '*', + host: '*', http_method: 'GET', url_path: '/ping', fixed_target: 0,