diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b22aedb --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +*.gem +*.rbc + +.bundle +.config +.coverage +.yardoc +Gemfile.lock + +doc/ +pkg/ + +tmp +test/tmp +coverage +vendor/ diff --git a/.yardopts b/.yardopts new file mode 100644 index 0000000..971c6ac --- /dev/null +++ b/.yardopts @@ -0,0 +1,5 @@ +--title 'AWS X-Ray SDK for Ruby' +--markup markdown +--markup-provider rdiscount +--no-progress +- LICENSE diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..fa75df1 --- /dev/null +++ b/Gemfile @@ -0,0 +1,3 @@ +source 'https://rubygems.org' + +gemspec diff --git a/LICENSE b/LICENSE index d645695..261eeb9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,3 @@ - Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ diff --git a/NOTICE b/NOTICE index f883730..2971941 100644 --- a/NOTICE +++ b/NOTICE @@ -1,2 +1,3 @@ -AWS Xray SDK Ruby -Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +AWS X-Ray SDK for Ruby +Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + diff --git a/README.md b/README.md index 794dcb8..d80ba7a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,189 @@ -AWS Xray SDK Ruby +# AWS X-Ray SDK for Ruby (beta) + +![Screenshot of the AWS X-Ray console](/images/example_servicemap.png?raw=true) + +## Installing + +The AWS X-Ray SDK for Ruby is compatible with Ruby 2.3.6 and newer Ruby versions. + +To install the Ruby gem for your project, add it to your project Gemfile. + +``` +# Gemfile +gem 'aws-xray-sdk' +``` +Then run `bundle install`. + +## Getting Help + +Use the following community resources for getting help with the SDK. We use the GitHub +issues for tracking bugs and feature requests. + +* Ask a question in the [AWS X-Ray Forum](https://forums.aws.amazon.com/forum.jspa?forumID=241&start=0). +* Open a support ticket with [AWS Support](http://docs.aws.amazon.com/awssupport/latest/user/getting-started.html). +* If you think you may have found a bug, open an [issue](https://github.com/aws/aws-xray-sdk-ruby/issues/new). + +## Opening Issues + +If you encounter a bug with the AWS X-Ray SDK for Ruby, we want to hear about +it. Before opening a new issue, search the [existing issues](https://github.com/aws/aws-xray-sdk-ruby/issues) +to see if others are also experiencing the issue. Include the version of the AWS X-Ray +SDK for Ruby, Ruby language, and other gems if applicable. In addition, +include the repro case when appropriate. + +The GitHub issues are intended for bug reports and feature requests. For help and +questions about using the AWS SDK for Ruby, use the resources listed +in the [Getting Help](https://github.com/aws/aws-xray-sdk-ruby#getting-help) section. Keeping the list of open issues lean helps us respond in a timely manner. + +## Documentation + +The [developer guide](https://docs.aws.amazon.com/xray/latest/devguide) provides in-depth guidance about using the AWS X-Ray service. +The [API Reference](http://docs.aws.amazon.com/xray-sdk-for-ruby/latest/reference/) provides documentation for public APIs of all classes in the SDK. + +## Quick Start + +**Configuration** + +```ruby +require 'aws-xray-sdk' + +# configure path based sampling rules in case of web app +my_sampling_rules = { + version: 1, + rules: [ + { + description: 'healthcheck', + service_name: '*', + http_method: 'GET', + url_path: '/ping', + fixed_target: 0, + rate: 0 + } + ], + default: { + fixed_target: 1, + rate: 0.2 + } +} + +user_config = { + sampling: true, + sampling_rules: my_sampling_rules, + name: 'default_segment_name', + daemon_address: '127.0.0.1:3000', + context_missing: 'LOG_ERROR', + patch: %I[net_http aws_sdk] +} + +XRay.recorder.configure(user_config) +``` + +**Working with Rails** + +```ruby +# Edit Gemfile to add XRay middleware +gem 'aws-xray-sdk', require: ['aws-xray-sdk/facets/rails/railtie'] + +# Configure the recorder in app_root/config/initializers/aws_xray.rb +Rails.application.config.xray = { + # default segment name generated by XRay middleware + name: 'myrails', + patch: %I[net_http aws_sdk], + # record db transactions as subsegments + active_record: true +} +``` + +**Adding metadata/annotations using recorder** + +```ruby +require 'aws-xray-sdk' + +# Add annotations to the current active entity +XRay.recorder.annotations[:k1] = 'v1' +XRay.recorder.annotations.update k2: 'v2', k3: 3 + +# Add metadata to the current active entity +XRay.recorder.metadata[:k1] = 'v1' # add to default namespace +XRay.recorder.metadata(namespace: :my_ns).update k2: 'v2' + +XRay.recorder.sampled? do + XRay.recorder.metadata.update my_meta # expensive metadata generation here +end +``` + +**Capture** + +```ruby +require 'aws-xray-sdk' + +XRay.recorder.capture('name_for_subsegment') do |subsegment| + resp = myfunc() + subsegment.annotations.update k1: 'v1' + resp +end + +# Manually apply the parent segment for the captured subsegment +XRay.recorder.capture('name_for_subsegment', segment: my_segment) do |subsegment| + myfunc() +end +``` + +**Thread Injection** + +```ruby +require 'aws-xray-sdk' + +XRay.recorder.configure({patch: %I[net_http]}) + +uri = URI.parse('http://aws.amazon.com/') +# Get the active entity from current call context +entity = XRay.recorder.current_entity + +workers = (0...3).map do + Thread.new do + begin + # set X-Ray context for this thread + XRay.recorder.inject_context entity do + http = Net::HTTP.new(uri.host, uri.port) + http.request(Net::HTTP::Get.new(uri.request_uri)) + end + rescue ThreadError + end + end +end + +workers.map(&:join) +``` + +**Start a custom segment/subsegment** + +```ruby +require 'aws-xray-sdk' + +# Start a segment +segment = XRay.recorder.begin_segment 'my_service' +# Start a subsegment +subsegment = XRay.recorder.begin_subsegment 'outbound_call', namespace: 'remote' + +# Add metadata or annotation here if necessary +my_annotations = { + k1: 'v1', + k2: 1024 +} +segment.annotations.update my_annotations + +# Add metadata to default namespace +subsegment.metadata[:k1] = 'v1' + +# Set user for the segment (subsegment is not supported) +segment.user = 'my_name' + +# End segment/subsegment +XRay.recorder.end_subsegment +XRay.recorder.end_segment +``` ## License -This library is licensed under the Apache 2.0 License. +The AWS X-Ray SDK for Ruby is licensed under the Apache 2.0 License. See LICENSE and NOTICE for more information. diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..706907a --- /dev/null +++ b/Rakefile @@ -0,0 +1,23 @@ +require 'bundler/gem_tasks' + +Rake::Task['release'].clear + +# default task +task default: %i[yard test] + +require 'rake' +require 'rake/testtask' +require 'yard' + +# tasks +desc 'execute all tests' +Rake::TestTask.new :test do |t| + t.test_files = FileList['test/**/tc_*.rb'] + t.verbose = false + t.warning = false +end + +desc 'generate API reference documentation' +YARD::Rake::YardocTask.new :yard do |t| + t.files = ['lib/**/*.rb'] +end diff --git a/aws-xray-sdk.gemspec b/aws-xray-sdk.gemspec new file mode 100644 index 0000000..885c36e --- /dev/null +++ b/aws-xray-sdk.gemspec @@ -0,0 +1,31 @@ +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'aws-xray-sdk/version' + +Gem::Specification.new do |spec| + spec.name = 'aws-xray-sdk' + spec.version = XRay::VERSION + spec.author = 'Amazon Web Services' + spec.email = 'aws-xray-ruby@amazon.com' + spec.summary = 'AWS X-Ray SDK for Ruby' + spec.description = 'The AWS X-Ray SDK for Ruby enables Ruby developers to record and emit information from within their applications to the AWS X-Ray service.' + spec.homepage = 'https://github.com/aws/aws-xray-sdk-ruby' + spec.license = 'Apache-2.0' + + spec.files = Dir.glob('lib/**/*') + spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } + spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) + spec.require_paths = ['lib'] + + spec.add_dependency 'oj', '~> 3.0' + + spec.add_development_dependency 'aws-sdk-dynamodb', '~> 1' + spec.add_development_dependency 'aws-sdk-s3', '~> 1' + spec.add_development_dependency 'bundler', '~> 1.0' + spec.add_development_dependency 'minitest', '~> 5.0' + spec.add_development_dependency 'rake', '~> 12.0' + spec.add_development_dependency 'rdiscount', '~> 2.2' + spec.add_development_dependency 'simplecov', '~> 0.15' + spec.add_development_dependency 'webmock', '~> 3.0' + spec.add_development_dependency 'yard', '~> 0.9' +end diff --git a/images/example_servicemap.png b/images/example_servicemap.png new file mode 100644 index 0000000..3f00ba4 Binary files /dev/null and b/images/example_servicemap.png differ diff --git a/lib/aws-xray-sdk.rb b/lib/aws-xray-sdk.rb new file mode 100644 index 0000000..b335b0a --- /dev/null +++ b/lib/aws-xray-sdk.rb @@ -0,0 +1,10 @@ +require 'aws-xray-sdk/recorder' + +module XRay + @recorder = Recorder.new + + # providing the default global recorder + def self.recorder + @recorder + end +end diff --git a/lib/aws-xray-sdk/configuration.rb b/lib/aws-xray-sdk/configuration.rb new file mode 100644 index 0000000..da1355b --- /dev/null +++ b/lib/aws-xray-sdk/configuration.rb @@ -0,0 +1,158 @@ +require 'aws-xray-sdk/exceptions' +require 'aws-xray-sdk/patcher' +require 'aws-xray-sdk/emitter/default_emitter' +require 'aws-xray-sdk/context/default_context' +require 'aws-xray-sdk/sampling/default_sampler' +require 'aws-xray-sdk/streaming/default_streamer' +require 'aws-xray-sdk/segment_naming/dynamic_naming' +require 'aws-xray-sdk/plugins/ec2' +require 'aws-xray-sdk/plugins/ecs' +require 'aws-xray-sdk/plugins/elastic_beanstalk' +require 'aws-xray-sdk/logger' + +module XRay + # This class stores all configurations for X-Ray recorder + # and should be initialized only once. + class Configuration + include Patcher + + SEGMENT_NAME_KEY = 'AWS_XRAY_TRACING_NAME'.freeze + CONFIG_KEY = %I[logger name sampling plugins daemon_address segment_naming + naming_pattern emitter streamer context context_missing + sampling_rules stream_threshold patch].freeze + + def initialize + @name = ENV[SEGMENT_NAME_KEY] + @sampling = true + @emitter = DefaultEmitter.new + @context = DefaultContext.new + @sampler = DefaultSampler.new + @streamer = DefaultStreamer.new + @segment_naming = DynamicNaming.new fallback: @name + @plugins = [] + end + + # @param [String] v The default segment name. + # Environment vairable takes higher precedence. + def name=(v) + @name = ENV[SEGMENT_NAME_KEY] || v + end + + # proxy method to the emitter's daemon_address config. + def daemon_address=(v) + emitter.daemon_address = v + end + + # proxy method to the context's context_missing config. + def context_missing=(v) + context.context_missing = v + end + + # proxy method to the sampler's sampling rule config. + def sampling_rules=(v) + sampler.sampling_rules = v + end + + # proxy method to the streamer's stream threshold config. + def stream_threshold=(v) + streamer.stream_threshold = v + end + + # proxy method to the dynamic naming's pattern config. + def naming_pattern=(v) + segment_naming.pattern = v + end + + # makes a sampling decision based on internal configure, e.g. + # if sampling enabled and the default sampling rule. + def sample? + return true unless sampling + sampler.sample? + end + + # @param [Hash] user_config The user configuration overrides. + def configure(user_config) + raise InvalidConfigurationError.new('User config must be a Hash.') unless user_config.is_a?(Hash) + return if user_config.empty? + + user_config.each_key do |key| + case key + when :logger + XRay::Logging.logger = user_config[key] + when :name + self.name = user_config[key] + when :context + self.context = user_config[key] + when :context_missing + self.context_missing = user_config[key] + when :sampler + self.sampler = user_config[key] + when :sampling_rules + self.sampling_rules = user_config[key] + when :sampling + self.sampling = user_config[key] + when :emitter + self.emitter = user_config[key] + when :daemon_address + self.daemon_address = user_config[key] + when :segment_naming + self.segment_naming = user_config[key] + when :naming_pattern + self.naming_pattern = user_config[key] + when :streamer + self.streamer = user_config[key] + when :stream_threshold + self.stream_threshold = user_config[key] + when :plugins + self.plugins = load_plugins(user_config[key]) + when :patch + patch(user_config[key]) + else + raise InvalidConfigurationError.new(%(Invalid config key #{key}.)) + end + end + end + + attr_accessor :emitter + + attr_accessor :context + + attr_accessor :sampler + + attr_accessor :streamer + + attr_accessor :segment_naming + + attr_accessor :plugins + + attr_accessor :sampling + + # @return [String] The default segment name. + attr_reader :name + + # The global logger used across the X-Ray SDK. + # @return [Logger] + attr_reader :logger + + private + + def load_plugins(symbols) + plugins = [] + symbols.each do |symbol| + case symbol + when :ec2 + plugins << XRay::Plugins::EC2 + when :ecs + plugins << XRay::Plugins::ECS + when :elastic_beanstalk + plugins << XRay::Plugins::ElasticBeanstalk + else + raise InvalidConfigurationError.new(%(Unsupported plugin #{symbol}.)) + end + end + # eager loads aws metadata to eliminate impact on first incoming request + plugins.each(&:aws) + plugins + end + end +end diff --git a/lib/aws-xray-sdk/context/context.rb b/lib/aws-xray-sdk/context/context.rb new file mode 100644 index 0000000..630f740 --- /dev/null +++ b/lib/aws-xray-sdk/context/context.rb @@ -0,0 +1,26 @@ +module XRay + # The interface of context management for the X-Ray recorder. + module Context + # @param [Entity] entity The entity to be stored in the context. + def store_entity(entity:) + raise 'Not implemented' + end + + def current_entity + raise 'Not implemented' + end + + def clear! + raise 'Not implemented' + end + + # Put current active entity to the new context storage. + def inject_context(entity, target_ctx: nil) + raise 'Not implemented' + end + + def handle_context_missing + raise 'Not implemented' + end + end +end diff --git a/lib/aws-xray-sdk/context/default_context.rb b/lib/aws-xray-sdk/context/default_context.rb new file mode 100644 index 0000000..72643ba --- /dev/null +++ b/lib/aws-xray-sdk/context/default_context.rb @@ -0,0 +1,81 @@ +require 'aws-xray-sdk/logger' +require 'aws-xray-sdk/context/context' +require 'aws-xray-sdk/exceptions' + +module XRay + # The default context storage management used by + # the X-Ray recorder. It uses thread local to store + # segments and subsegments. + class DefaultContext + include Context + include Logging + + LOCAL_KEY = '_aws_xray_entity'.freeze + CONTEXT_MISSING_KEY = 'AWS_XRAY_CONTEXT_MISSING'.freeze + SUPPORTED_STRATEGY = %w[RUNTIME_ERROR LOG_ERROR].freeze + DEFAULT_STRATEGY = SUPPORTED_STRATEGY[0] + + attr_reader :context_missing + + def initialize + strategy = ENV[CONTEXT_MISSING_KEY] || DEFAULT_STRATEGY + @context_missing = sanitize_strategy(strategy) + end + + # @param [Entity] entity The entity to be stored in the context. + def store_entity(entity:) + Thread.current[LOCAL_KEY] = entity + end + + # @return [Entity] The current active entity(could be segment or subsegment). + def current_entity + if entity = Thread.current[LOCAL_KEY] + entity + else + handle_context_missing + end + end + + # Clear the current thread local storage on X-Ray related entities. + def clear! + Thread.current[LOCAL_KEY] = nil + end + + # @param [Entity] entity The entity to inject. + # @param [Thread] target_ctx Put the provided entity to the new thread. + def inject_context(entity, target_ctx: nil) + target_ctx ||= Thread.current + target_ctx[LOCAL_KEY] = entity if entity + end + + # When the current entity needs to be accessed but there is none, + # it handles the missing context based on the configuration. + # On `RUNTIME_ERROR` it raises `ContextMissingError`. + # On 'LOG_ERROR' it logs an error message and return `nil`. + def handle_context_missing + case context_missing + when 'RUNTIME_ERROR' + raise ContextMissingError + when 'LOG_ERROR' + logger.error %(can not find the current context.) + end + nil + end + + def context_missing=(v) + strategy = ENV[CONTEXT_MISSING_KEY] || v + @context_missing = sanitize_strategy(strategy) + end + + private + + def sanitize_strategy(v) + if SUPPORTED_STRATEGY.include?(v) + v + else + logger.warn %(context missing #{v} is not supported, switch to default #{DEFAULT_STRATEGY}.) + DEFAULT_STRATEGY + end + end + end +end diff --git a/lib/aws-xray-sdk/emitter/default_emitter.rb b/lib/aws-xray-sdk/emitter/default_emitter.rb new file mode 100644 index 0000000..13c332b --- /dev/null +++ b/lib/aws-xray-sdk/emitter/default_emitter.rb @@ -0,0 +1,53 @@ +require 'socket' +require 'aws-xray-sdk/logger' +require 'aws-xray-sdk/emitter/emitter' +require 'aws-xray-sdk/exceptions' + +module XRay + # The default emitter the X-Ray recorder uses to send segments/subsegments + # to the X-Ray daemon over UDP using a non-blocking socket. + class DefaultEmitter + include Emitter + include Logging + + attr_reader :address + + def initialize + @socket = UDPSocket.new + @address = ENV[DAEMON_ADDRESS_KEY] || '127.0.0.1:2000' + configure_socket(@address) + end + + # Serializes a segment/subsegment and sends it to the X-Ray daemon + # over UDP. It is no-op for non-sampled entity. + # @param [Entity] entity The entity to send + def send_entity(entity:) + return nil unless entity.sampled + begin + payload = %(#{@@protocol_header}#{@@protocol_delimiter}#{entity.to_json}) + logger.debug %(sending payload #{payload} to daemon at #{address}.) + @socket.send payload, 0 + rescue StandardError => e + logger.warn %(failed to send payload due to #{e.message}) + end + end + + def daemon_address=(v) + v = ENV[DAEMON_ADDRESS_KEY] || v + @address = v + configure_socket(v) + end + + private + + def configure_socket(v) + begin + addr = v.split(':') + host, ip = addr[0], addr[1].to_i + @socket.connect(host, ip) + rescue StandardError + raise InvalidDaemonAddressError, %(Invalid X-Ray daemon address specified: #{v}.) + end + end + end +end diff --git a/lib/aws-xray-sdk/emitter/emitter.rb b/lib/aws-xray-sdk/emitter/emitter.rb new file mode 100644 index 0000000..14aeb98 --- /dev/null +++ b/lib/aws-xray-sdk/emitter/emitter.rb @@ -0,0 +1,24 @@ +require 'json' + +module XRay + # The emitter interface the X-Ray recorder uses to send segments/subsegments + # to the X-Ray daemon over UDP. + module Emitter + DAEMON_ADDRESS_KEY = 'AWS_XRAY_DAEMON_ADDRESS'.freeze + + @@protocol_header = { + format: 'json', + version: 1 + }.to_json + @@protocol_delimiter = "\n" + + # @param [Entity] entity Entity to send. + def send_entity(entity:) + raise 'Not implemented' + end + + def daemon_address=(v) + raise 'Not implemented' + end + end +end diff --git a/lib/aws-xray-sdk/exceptions.rb b/lib/aws-xray-sdk/exceptions.rb new file mode 100644 index 0000000..abbf9fe --- /dev/null +++ b/lib/aws-xray-sdk/exceptions.rb @@ -0,0 +1,31 @@ +module XRay + # All custom exception thrown by the SDK should subclass AwsXRayError. + class AwsXRaySdkError < ::StandardError; end + + class EntityClosedError < AwsXRaySdkError + def initialize + super('Segment or subsegment already ended.') + end + end + + class ContextMissingError < AwsXRaySdkError + def initialize + super('Can not find any active segment or subsegment.') + end + end + + class SegmentNameMissingError < AwsXRaySdkError + end + + class InvalidDaemonAddressError < AwsXRaySdkError + end + + class InvalidSamplingConfigError < AwsXRaySdkError + end + + class InvalidConfigurationError < AwsXRaySdkError + end + + class UnsupportedPatchingTargetError < AwsXRaySdkError + end +end diff --git a/lib/aws-xray-sdk/facets/aws_sdk.rb b/lib/aws-xray-sdk/facets/aws_sdk.rb new file mode 100644 index 0000000..0a7237a --- /dev/null +++ b/lib/aws-xray-sdk/facets/aws_sdk.rb @@ -0,0 +1,127 @@ +require 'aws-sdk-core' +require 'aws-xray-sdk/facets/helper' +require 'aws-xray-sdk/facets/resources/aws_params_whitelist' +require 'aws-xray-sdk/facets/resources/aws_services_whitelist' + +module XRay + class AwsSDKPlugin < Seahorse::Client::Plugin + option :xray_recorder, default: XRay.recorder + + def add_handlers(handlers, config) + # run before Seahorse::Client::Plugin::ParamValidator (priority 50) + handlers.add Handler, step: :validate, priority: 49 + end + + # Handler to capture AWS API calls as subsegments + class Handler < Seahorse::Client::Handler + include XRay::Facets::Helper + + def call(context) + recorder = Aws.config[:xray_recorder] + operation = context.operation_name + service_name = context.client.class.api.metadata['serviceAbbreviation'] || + context.client.class.to_s.split('::')[1] + recorder.capture service_name, namespace: 'aws' do |subsegment| + # inject header string before calling downstream AWS services + context.http_request.headers[TRACE_HEADER] = prep_header_str entity: subsegment + response = @handler.call(context) + http_response = context.http_response + resp_meta = { + status: http_response.status_code, + content_length: http_response.headers['content-length'].to_i + } + aws = { + # XRay back-end right now has strict operation name matching + operation: sanitize_op_name(operation), + region: context.client.config.region, + retries: context.retries, + request_id: http_response.headers['x-amzn-requestid'] + } + # S3 returns special request id in response headers + if service_name == 'S3' + aws[:id_2] = http_response.headers['x-amz-id-2'] + end + + operation_h = AwsParams.whitelist[:services] + .fetch(service_name.to_sym, {}) + .fetch(:operations, {})[operation] + unless operation_h.nil? + params_capture req_params: context.params, resp_params: response.to_h, + capture: operation_h, meta: aws + end + subsegment.aws = aws + if err = response.error + subsegment.add_exception exception: err, remote: true + end + subsegment.merge_http_response response: resp_meta + response + end + end + + private + + def inject_headers(request:, entity:) + request.headers[TRACE_HEADER] = prep_header_str entity: entity + end + + def sanitize_op_name(opname) + opname.to_s.split('_').collect(&:capitalize).join if opname + end + + def params_capture(req_params:, resp_params:, capture:, meta:) + if norm = capture[:request_parameters] + capture_normal params: req_params, capture: norm, meta: meta + end + + if norm = capture[:response_parameters] + capture_normal params: resp_params, capture: norm, meta: meta + end + + if spec = capture[:request_descriptors] + capture_special params: req_params, capture: spec, meta: meta + end + + if spec = capture[:response_descriptors] + capture_special params: resp_params, capture: spec, meta: meta + end + end + + def capture_normal(params:, capture:, meta:) + params.each_key do |key| + meta[key] = params[key] if capture.include?(key) + end + end + + def capture_special(params:, capture:, meta:) + params.each_key do |key| + process_descriptor(target: params[key], descriptor: capture[key], meta: meta) if capture.include?(key) + end + end + + def process_descriptor(target:, descriptor:, meta:) + # "get_count" = true + v = target.length if descriptor[:get_count] + # "get_keys" = true + v = target.keys if descriptor[:get_keys] + meta[descriptor[:rename_to]] = v + end + end + end + + # Add X-Ray plugin to AWS SDK clients + module AwsSDKPatcher + def self.patch(services: nil, recorder: XRay.recorder) + force = services.nil? + services ||= AwsServices.whitelist + services.each do |s| + begin + Aws.const_get(%(#{s}::Client)).add_plugin XRay::AwsSDKPlugin + Aws.config.update xray_recorder: recorder + rescue NameError + # swallow the error if no explicit user config + raise unless force + end + end + end + end +end diff --git a/lib/aws-xray-sdk/facets/helper.rb b/lib/aws-xray-sdk/facets/helper.rb new file mode 100644 index 0000000..681f682 --- /dev/null +++ b/lib/aws-xray-sdk/facets/helper.rb @@ -0,0 +1,61 @@ +require 'aws-xray-sdk/model/trace_header' + +module XRay + module Facets + # Hepler functions shared for all external frameworks/libraries + # like make sampling decisions from incoming http requests etc. + module Helper + TRACE_HEADER = 'X-Amzn-Trace-Id'.freeze + TRACE_HEADER_PROXY = 'HTTP_X_AMZN_TRACE_ID'.freeze + + # Construct a `TraceHeader` object from headers + # of the incoming request. This method should always return + # a `TraceHeader` object regardless of tracing header's presence + # in the incoming request. + # @param [Hash] headers Hash that contains X-Ray trace header key. + # @return [TraceHeader] The new constructed trace header object. + def construct_header(headers:) + if v = headers[TRACE_HEADER_PROXY] || headers[TRACE_HEADER] + TraceHeader.from_header_string header_str: v + else + TraceHeader.empty_header + end + end + + # The sampling decision coming from `trace_header` always has + # the highest precedence. If the `trace_header` doesn't contain + # sampling decision then it checks if sampling is enabled or not + # in the recorder. If not enbaled it returns 'true'. Otherwise it uses + # sampling rule to decide. + def should_sample?(header_obj:, recorder:, + host: nil, method: nil, path: nil, + **args) + # check outside decision + if i = header_obj.sampled + if i.zero? + false + else + true + end + # check sampling rules + elsif recorder.sampling_enabled? + recorder.sampler.sample_request?(service_name: host, url_path: path, + http_method: method) + # sample if no reason not to + else + true + end + end + + # Prepares a X-Ray header string based on the provided Segment/Subsegment. + def prep_header_str(entity:) + return '' if entity.nil? + root = entity.segment.trace_id + parent_id = entity.id + sampled = entity.sampled ? 1 : 0 + header = TraceHeader.new root: root, parent_id: parent_id, sampled: sampled + header.header_string + end + end + end +end diff --git a/lib/aws-xray-sdk/facets/net_http.rb b/lib/aws-xray-sdk/facets/net_http.rb new file mode 100644 index 0000000..2511b59 --- /dev/null +++ b/lib/aws-xray-sdk/facets/net_http.rb @@ -0,0 +1,61 @@ +require 'net/http' +require 'aws-xray-sdk/facets/helper' + +module XRay + # Patch net/http to be traced by X-Ray + module NetHttp + # Class level interceptor to capture http requests as subsegments + module HTTPClassInterceptor + def new(*options) + o = super(*options) + o + end + end + + # Instance level interceptor to capture http requests as subsegments + module HTTPInstanceInterceptor + include XRay::Facets::Helper + + def initialize(*options) + super(*options) + end + + def request(req, body = nil, &block) + entity = XRay.recorder.current_entity + capture = !(entity && entity.namespace && entity.namespace == 'aws'.freeze) + if started? && capture && entity + XRay.recorder.capture(address, namespace: 'remote') do |subsegment| + protocol = use_ssl? ? 'https'.freeze : 'http'.freeze + # avoid modifying original variable + iport = port.nil? ? nil : %(:#{port}) + # do not capture query string + path = req.path.split('?')[0] if req.path + uri = %(#{protocol}://#{address}#{iport}#{path}) + req_meta = { + url: uri, + method: req.method + } + subsegment.merge_http_request request: req_meta + req[TRACE_HEADER] = prep_header_str entity: subsegment + begin + res = super + res_meta = { + status: res.code.to_i, + content_length: res.content_length + } + subsegment.merge_http_response response: res_meta + rescue Exception => e + subsegment.add_exception exception: e + raise e + end + end + else + super + end + end + end + + ::Net::HTTP.singleton_class.prepend HTTPClassInterceptor + ::Net::HTTP.prepend HTTPInstanceInterceptor + end +end diff --git a/lib/aws-xray-sdk/facets/rack.rb b/lib/aws-xray-sdk/facets/rack.rb new file mode 100644 index 0000000..f29f63e --- /dev/null +++ b/lib/aws-xray-sdk/facets/rack.rb @@ -0,0 +1,87 @@ +require 'rack/request' +require 'aws-xray-sdk' +require 'aws-xray-sdk/facets/helper' + +module XRay + module Rack + # Rack middleware that generates a segment for each request/response cycle. + class Middleware + include XRay::Facets::Helper + X_FORWARD = 'HTTP_X_FORWARDED_FOR'.freeze + + def initialize(app, recorder: nil) + @app = app + @recorder = recorder || XRay.recorder + end + + def call(env) + header = construct_header(headers: env) + req = ::Rack::Request.new(env) + + # params required for path based sampling + host = req.host + url_path = req.path + method = req.request_method + + # get sampling decision + sampled = should_sample?( + header_obj: header, recorder: @recorder, + host: host, method: method, path: url_path + ) + + # get segment name from host header if applicable + seg_name = @recorder.segment_naming.provide_name(host: req.host) + + # begin the segment + segment = @recorder.begin_segment seg_name, trace_id: header.root, parent_id: header.parent_id, + sampled: sampled + + # add neccessary http request metadata to the segment + req_meta = extract_request_meta(req) + segment.merge_http_request request: req_meta unless req_meta.empty? + begin + status, headers, body = @app.call env + resp_meta = {} + resp_meta[:status] = status + # Don't set content_length if it is not available on headers. + resp_obj = ::Rack::Response.new body: body, status: status, headers: headers + if len = resp_obj.content_length + resp_meta[:content_length] = len + end + segment.merge_http_response response: resp_meta + [status, headers, body] + rescue Exception => e + segment.apply_status_code status: 500 + segment.add_exception exception: e + raise e + ensure + @recorder.end_segment + end + end + + private + + def extract_request_meta(req) + req_meta = {} + req_meta[:url] = req.url if req.url + req_meta[:user_agent] = req.user_agent if req.user_agent + req_meta[:method] = req.request_method if req.request_method + if req.has_header?(X_FORWARD) + req_meta[:client_ip] = get_ip(req.get_header(X_FORWARD)) + req_meta[:x_forwarded_for] = true + elsif v = req.ip + req_meta[:client_ip] = v + end + req_meta + end + + def get_ip(ips) + if ips.respond_to?(:length) + ips[ips.length - 1] + else + ips + end + end + end + end +end diff --git a/lib/aws-xray-sdk/facets/rails/active_record.rb b/lib/aws-xray-sdk/facets/rails/active_record.rb new file mode 100644 index 0000000..e9b4fd6 --- /dev/null +++ b/lib/aws-xray-sdk/facets/rails/active_record.rb @@ -0,0 +1,66 @@ +require 'active_record' + +module XRay + module Rails + # Recording Rails database transactions as subsegments. + module ActiveRecord + class << self + IGNORE_OPS = ['SCHEMA', 'ActiveRecord::SchemaMigration Load', + 'ActiveRecord::InternalMetadata Load'].freeze + DB_TYPE_MAPPING = { + mysql2: 'MySQL', + postgresql: 'PostgreSQL' + }.freeze + + def record(transaction) + payload = transaction.payload + pool, conn = get_pool_n_conn(payload[:connection_id]) + + return if IGNORE_OPS.include?(payload[:name]) || pool.nil? || conn.nil? + db_config = pool.spec.config + name, sql = build_name_sql_meta config: db_config, conn: conn + subsegment = XRay.recorder.begin_subsegment name, namespace: 'remote' + subsegment.start_time = transaction.time.to_f + subsegment.sql = sql + XRay.recorder.end_subsegment end_time: transaction.end.to_f + end + + private + + def build_name_sql_meta(config:, conn:) + # extract all available info + adapter = config[:adapter] + database = config[:database] + host = config[:host].nil? ? nil : %(@#{config[:host]}) + port = config[:port].nil? ? nil : %(:#{config[:port]}) + username = config[:username] + + # assemble subsegment name + name = %(#{database}#{host}) + # assemble sql meta + sql = {} + sql[:user] = username + sql[:url] = %(#{adapter}://#{username}#{host}#{port}/#{database}) + sql[:database_type] = DB_TYPE_MAPPING[adapter.to_sym] + [name, sql] + end + + def get_pool_n_conn(conn_id) + pool, conn = nil, nil + ::ActiveRecord::Base.connection_handler.connection_pool_list.each do |p| + conn = p.connections.select { |c| c.object_id == conn_id } + pool = p unless conn.nil? + end + [pool, conn] + end + end + end + end +end + +# Add a hook on database transactions using Rails instrumentation API +ActiveSupport::Notifications.subscribe('sql.active_record') do |*args| + # We need the full event which has all the timing info + transaction = ActiveSupport::Notifications::Event.new(*args) + XRay::Rails::ActiveRecord.record(transaction) +end diff --git a/lib/aws-xray-sdk/facets/rails/ex_middleware.rb b/lib/aws-xray-sdk/facets/rails/ex_middleware.rb new file mode 100644 index 0000000..101b618 --- /dev/null +++ b/lib/aws-xray-sdk/facets/rails/ex_middleware.rb @@ -0,0 +1,24 @@ +require 'aws-xray-sdk' + +module XRay + module Rails + # Middleware for capturing unhandled exceptions from views/controller. + # To properly capture exceptions this middleware needs to be placed + # after the default exception handling middleware. Otherwise they will + # be swallowed. + class ExceptionMiddleware + def initialize(app, recorder: nil) + @app = app + @recorder = recorder || XRay.recorder + end + + def call(env) + @app.call(env) + rescue Exception => e + segment = @recorder.current_segment + segment.add_exception exception: e if segment + raise e + end + end + end +end diff --git a/lib/aws-xray-sdk/facets/rails/railtie.rb b/lib/aws-xray-sdk/facets/rails/railtie.rb new file mode 100644 index 0000000..b3b364f --- /dev/null +++ b/lib/aws-xray-sdk/facets/rails/railtie.rb @@ -0,0 +1,23 @@ +require 'aws-xray-sdk/facets/rack' +require 'aws-xray-sdk/facets/rails/ex_middleware' + +module XRay + # configure X-Ray instrumentation for rails framework + class Railtie < ::Rails::Railtie + RAILS_OPTIONS = %I[active_record].freeze + + initializer "aws-xray-sdk.rack_middleware" do |app| + app.middleware.insert 0, Rack::Middleware + app.middleware.use XRay::Rails::ExceptionMiddleware + end + + config.after_initialize do |app| + if app.config.respond_to?('xray') + options = app.config.xray + require 'aws-xray-sdk/facets/rails/active_record' if options[:active_record] + general_options = options.reject { |k, v| RAILS_OPTIONS.include?(k) } + XRay.recorder.configure(general_options) + end + end + end +end diff --git a/lib/aws-xray-sdk/facets/resources/aws_params_whitelist.rb b/lib/aws-xray-sdk/facets/resources/aws_params_whitelist.rb new file mode 100644 index 0000000..cb4949c --- /dev/null +++ b/lib/aws-xray-sdk/facets/resources/aws_params_whitelist.rb @@ -0,0 +1,340 @@ +module XRay + # AWS SDK parameters whitelisted will be recorded + # as metadata on AWS subsegments + module AwsParams + @whitelist = { + services: { + DynamoDB: { + operations: { + batch_get_item: { + request_descriptors: { + request_items: { + map: true, + get_keys: true, + rename_to: :table_names + } + }, + response_parameters: %I[ + consumed_capacity + ] + }, + batch_write_item: { + request_descriptors: { + request_items: { + map: true, + get_keys: true, + rename_to: :table_names + } + }, + response_parameters: %I[ + consumed_capacity + item_collection_metrics + ] + }, + create_table: { + request_parameters: %I[ + global_secondary_indexes + local_secondary_indexes + provisioned_throughput + table_name + ] + }, + delete_item: { + request_parameters: %I[ + table_name + ], + response_parameters: %I[ + consumed_capacity + item_collection_metrics + ] + }, + delete_table: { + request_parameters: %I[ + table_name + ] + }, + describe_table: { + request_parameters: %I[ + table_name + ] + }, + get_item: { + request_parameters: %I[ + consistent_read + projection_expression + table_name + ], + response_parameters: %I[ + consumed_capacity + ] + }, + list_tables: { + request_parameters: %I[ + exclusive_start_table_name + limit + ], + response_descriptors: { + table_names: { + list: true, + get_count: true, + rename_to: :table_count + } + } + }, + put_item: { + request_parameters: %I[ + table_name + ], + response_parameters: %I[ + consumed_capacity + item_collection_metrics + ] + }, + query: { + request_parameters: %I[ + attributes_to_get + consistent_read + index_name + limit + projection_expression + scan_index_forward + select + table_name + ], + response_parameters: %I[ + consumed_capacity + ] + }, + scan: { + request_parameters: %I[ + attributes_to_get + consistent_read + index_name + limit + projection_expression + segment + select + table_name + total_segments + ], + response_parameters: %I[ + consumed_capacity + count + scanned_count + ] + }, + update_item: { + request_parameters: %I[ + table_name + ], + response_parameters: %I[ + consumed_capacity + item_collection_metrics + ] + }, + update_table: { + request_parameters: %I[ + attribute_definitions + global_secondary_index_updates + provisioned_throughput + table_name + ] + } + } + }, + SQS: { + operations: { + add_permission: { + request_parameters: %I[ + label + queue_url + ] + }, + change_message_visibility: { + request_parameters: %I[ + queue_url + visibility_timeout + ] + }, + change_message_visibility_batch: { + request_parameters: %I[ + queue_url + ], + response_parameters: %I[ + failed + ] + }, + create_queue: { + request_parameters: %I[ + attributes + queue_name + ] + }, + delete_message: { + request_parameters: %I[ + queue_urls + ] + }, + delete_message_batch: { + request_parameters: %I[ + queue_url + ], + response_parameters: %I[ + failed + ] + }, + delete_queue: { + request_parameters: %I[ + queue_url + ] + }, + get_queue_attributes: { + request_parameters: %I[ + queue_url + ], + response_parameters: %I[ + attributes + ] + }, + get_queue_url: { + request_parameters: %I[ + queue_name + queue_owner_aws_account_id + ], + response_parameters: %I[ + queue_url + ] + }, + list_dead_letter_source_queues: { + request_parameters: %I[ + queue_url + ], + response_parameters: %I[ + queue_urls + ] + }, + list_queues: { + request_parameters: %I[ + queue_name_prefix + ], + response_descriptors: { + queue_urls: { + list: true, + get_count: true, + rename_to: :queue_count + } + } + }, + purge_queue: { + request_parameters: %I[ + queue_url + ] + }, + receive_message: { + request_parameters: %I[ + attribute_names + max_number_of_messages + message_attribute_names + queue_url + visibility_timeout + wait_time_seconds + ], + response_descriptors: { + messages: { + list: true, + get_count: true, + rename_to: :message_count + } + } + }, + remove_permission: { + request_parameters: %I[ + queue_url + ] + }, + send_message: { + request_parameters: %I[ + delay_seconds + queue_url + ], + request_descriptors: { + message_attributes: { + map: true, + get_keys: true, + rename_to: :message_attribute_names + } + }, + response_parameters: %I[ + message_id + ] + }, + send_message_batch: { + request_parameters: %I[ + queue_url + ], + request_descriptors: { + entries: { + list: true, + get_count: true, + rename_to: :message_count + } + }, + response_descriptors: { + failed: { + list: true, + get_count: true, + rename_to: :failed_count + }, + successful: { + list: true, + get_count: true, + rename_to: :successful_count + } + } + }, + set_queue_attributes: { + request_parameters: %I[ + queue_url + ], + request_descriptors: { + attributes: { + map: true, + get_keys: true, + rename_to: :attribute_names + } + } + } + } + }, + Lambda: { + operations: { + invoke: { + request_parameters: %I[ + function_name + invocation_type + log_type + qualifier + ], + response_parameters: %I[ + function_error + status_code + ] + }, + invoke_async: { + request_parameters: %I[ + function_name + ], + response_parameters: %I[ + status + ] + } + } + } + } + } + + def self.whitelist + @whitelist + end + end +end diff --git a/lib/aws-xray-sdk/facets/resources/aws_services_whitelist.rb b/lib/aws-xray-sdk/facets/resources/aws_services_whitelist.rb new file mode 100644 index 0000000..37e275e --- /dev/null +++ b/lib/aws-xray-sdk/facets/resources/aws_services_whitelist.rb @@ -0,0 +1,147 @@ +module XRay + # AWS Services listed below will be recorded as subsegments + module AwsServices + # exausted list can be tracked at http://docs.aws.amazon.com/sdk-for-ruby/v3/api/Seahorse/Client/Base.html + @whitelist = %I[ + ACM + APIGateway + AlexaForBusiness + AppStream + AppSync + ApplicationAutoScaling + ApplicationDiscoveryService + Athena + AutoScaling + Batch + Budgets + Cloud9 + CloudDirectory + CloudFormatioin + CloudFront + CloudHSM + CloudHSMV2 + CloudSearch + CloudSearchDomain + CloudTrail + CloudWatch + CloudWatchEvents + CloudWatchLogs + CodeBuild + CodeCommit + CodeDeploy + CodePipeline + CodeStar + CognitoIdentity + CognitoIdentityProvider + CognitoSync + Comprehend + ConfigService + CostExplore + CostandUsageReportService + DAX + DataPipeline + DatabaseMigrationService + DeviceFarm + DirectConnect + DirectoryService + DynamoDB + DynamoDBStreams + EC2 + ECR + ECS + EFS + EMR + ElastiCache + ElasticBeanstalk + ElasticLoadBalancing + ElasticLoadBalancingV2 + ElasticTranscoder + ElasticsearchService + Firehost + GameLift + Glacier + Glue + Greengrass + GuardDuty + Health + IAM + ImportExport + Inspector + IoT + IoTDataPlane + IoTJobsDataPlane + KMS + Kinesis + KinesisAnalytics + KinesisVideo + KinesisVideoArchiveMedia + KinesisVideoMedia + Lambda + LambdaPreview + Lex + LexModelBuildingService + LexRuntimeService + Lightsail + MQ + MTurk + MachineLearning + MarketplaceCommerceAnalytics + MarketplaceEntitlementService + MarketplaceMetering + MediaConvert + MediaLive + MediaPackage + MediaStore + MediaStoreData + MigrationHub + Mobile + OpsWorks + OpsWorksCM + Organizations + Pinpoint + Polly + Pricing + RDS + Redshift + Rekognition + ResourceGroups + ResourceGroupsTaggingAPI + Route53 + Route53Domains + S3 + SES + SFN + SMS + SNS + SQS + SSM + STS + SWF + SageMaker + SageMakerRuntime + ServerlessApplicationRepository + ServiceCatalog + ServiceDiscovery + Shield + SimpleDB + Snowball + States + StorageGateway + Support + Translate + WAF + WAFRegional + WorkDocs + WorkSpaces + XRay + ] + + def self.whitelist + @whitelist + end + + def self.whitelist=(v) + @whitelist = v + end + end +end diff --git a/lib/aws-xray-sdk/logger.rb b/lib/aws-xray-sdk/logger.rb new file mode 100644 index 0000000..4775028 --- /dev/null +++ b/lib/aws-xray-sdk/logger.rb @@ -0,0 +1,19 @@ +require 'logger' + +module XRay + # Provide global logger for classes that include this module. + # It serves as a proxy to global recorder's logger. + module Logging + def logger + Logging.logger + end + + def self.logger + @logger ||= Logger.new($stdout).tap { |l| l.level = Logger::INFO } + end + + def self.logger=(v) + @logger = v + end + end +end diff --git a/lib/aws-xray-sdk/model/annotations.rb b/lib/aws-xray-sdk/model/annotations.rb new file mode 100644 index 0000000..1c27d3c --- /dev/null +++ b/lib/aws-xray-sdk/model/annotations.rb @@ -0,0 +1,97 @@ +require 'aws-xray-sdk/logger' +require 'aws-xray-sdk/exceptions' + +module XRay + # Annotations are simple key-value pairs that are indexed for use with filter expressions. + # Use annotations to record data that you want to use to group traces in the console, + # or when calling the GetTraceSummaries API. + class Annotations + include Logging + + def initialize(entity) + @entity = entity + @data = {} + end + + def [](key) + @data[key] + end + + # @param [Symbol] k Only characters in `A-Za-z0-9_` are supported. + # @param v Only `Numeric`, `String` true/false is supported. + def []=(k, v) + raise EntityClosedError if @entity.closed? + if key_supported?(k) && value_supported?(v) + @data[k] = v + else + logger.warn %(Dropping annotation with key #{k} due to invalid characters.) + end + end + + # @param [Hash] h Update annotations with a single input hash. + def update(h) + raise EntityClosedError if @entity.closed? + filtered = filter_annotations(h) + @data.merge!(filtered) + end + + def to_h + sanitize_values(@data) + end + + private + + def filter_annotations(h) + h.delete_if do |k, v| + drop = !key_supported?(k) || !value_supported?(v) + logger.warn %(Dropping annotation with key #{k} due to invalid characters.) if drop + drop + end + end + + def sanitize_values(h) + h.each_pair do |k, v| + if v.is_a?(Float) + h[k] = v.to_s if v.nan? || v.infinite? == 1 || v.infinite? == -1 + end + end + end + + def key_supported?(k) + k.match(/[A-Za-z0-9_]+/) + end + + def value_supported?(v) + case v + when Numeric + true + when true, false + true + else + v.is_a?(String) + end + end + end + + # Singleton facade annotations class doing no-op for performance + # in case of not sampled X-Ray entities. + module FacadeAnnotations + class << self + def [](key) + # no-op + end + + def []=(k, v) + # no-op + end + + def update(h) + # no-op + end + + def to_h + # no-op + end + end + end +end diff --git a/lib/aws-xray-sdk/model/cause.rb b/lib/aws-xray-sdk/model/cause.rb new file mode 100644 index 0000000..3fe0849 --- /dev/null +++ b/lib/aws-xray-sdk/model/cause.rb @@ -0,0 +1,70 @@ +require 'oj' + +module XRay + # Represents cause section in segment and subsegment document. + # It records information about application runtime exceptions. + class Cause + attr_reader :id + @@depth = 15 + + def initialize(exception: nil, id: nil, remote: false) + if exception + @exception_h = normalize e: exception, remote: remote + end + @id = id + end + + def to_h + return id if id + h = { + working_directory: Dir.pwd, + paths: Gem.paths.path, + exceptions: @exception_h + } + h + end + + def to_json + @to_json ||= begin + Oj.dump to_h, mode: :compat, use_as_json: true + end + end + + private + + def normalize(e:, remote: false) + exceptions = [] + exceptions << normalize_exception(e: e, remote: remote) + + # don't propagate remote flag + while e.cause + exceptions << normalize_exception(e: e.cause) + e = e.cause + end + + exceptions + end + + def normalize_exception(e:, remote: false) + h = { + message: e.to_s, + type: e.class.to_s + } + h[:remote] = true if remote + + backtrace = e.backtrace_locations + return h unless backtrace + h[:stack] = backtrace.first(@@depth).collect do |t| + { + path: t.path, + line: t.lineno, + label: t.label + } + end + + truncated = backtrace.size - @@depth + h[:truncated] = truncated if truncated > 0 + h + end + end +end diff --git a/lib/aws-xray-sdk/model/dummy_entities.rb b/lib/aws-xray-sdk/model/dummy_entities.rb new file mode 100644 index 0000000..c6d20d2 --- /dev/null +++ b/lib/aws-xray-sdk/model/dummy_entities.rb @@ -0,0 +1,72 @@ +require 'aws-xray-sdk/model/segment' +require 'aws-xray-sdk/model/subsegment' +require 'aws-xray-sdk/model/annotations' +require 'aws-xray-sdk/model/metadata' + +module XRay + # defines common no-op methods for dummy segments/subsegments + module DummyEntity + def sampled + false + end + + def annotations + FacadeAnnotations + end + + def metadata(namespace: :default) + FacadeMetadata + end + + def apply_status_code(status:) + # no-op + end + + def merge_http_request(request:) + # no-op + end + + def merge_http_response(response:) + # no-op + end + + def add_exception(exception:, remote: false) + # no-op + end + + def aws=(v) + # no-op + end + + def to_h + # no-op + end + + def to_json + # no-op + end + end + + # A dummy segment is created when ``xray_recorder`` decides to not sample + # the segment based on sampling decisions. + # Adding data to a dummy segment becomes a no-op except for + # subsegments. This is to reduce the memory footprint of the SDK. + # A dummy segment will not be sent to the X-Ray daemon by the default emitter. + # Manually create dummy segments is not recommended. + class DummySegment < Segment + include DummyEntity + end + + # A dummy subsegment will be created when ``xray_recorder`` tries + # to create a subsegment under a not sampled segment. Adding data + # to a dummy subsegment becomes no-op except for child subsegments. + # Dummy subsegment will not be sent to the X-Ray daemon by the default emitter. + # Manually create dummy subsegments is not recommended. + class DummySubsegment < Subsegment + include DummyEntity + + def sql=(v) + # no-op + end + end +end diff --git a/lib/aws-xray-sdk/model/entity.rb b/lib/aws-xray-sdk/model/entity.rb new file mode 100644 index 0000000..708093d --- /dev/null +++ b/lib/aws-xray-sdk/model/entity.rb @@ -0,0 +1,187 @@ +require 'securerandom' +require 'bigdecimal' +require 'oj' +require 'aws-xray-sdk/exceptions' +require 'aws-xray-sdk/model/cause' +require 'aws-xray-sdk/model/annotations' +require 'aws-xray-sdk/model/metadata' + +module XRay + # This module contains common properties and methods + # used by segment and subsegment class. + module Entity + attr_reader :name, :exception, :cause, :namespace, + :http_request, :http_response + attr_accessor :parent, :throttle, :error, :fault, :sampled, :aws, + :start_time, :end_time + + HTTP_REQUEST_KEY = %I[url method user_agent client_ip x_forwarded_for].freeze + HTTP_RESPONSE_KEY = %I[status content_length].freeze + + # Generates a random 8-digit hex number as the entity id and returns it. + def id + @id ||= begin + SecureRandom.hex(8) + end + end + + def closed? + @closed ||= false + end + + # @param [Float] end_time End time on epoch. + def close(end_time: nil) + raise EntityClosedError if closed? + @end_time = end_time || Time.now.to_f + @closed = true + end + + # @return [Array] The children subsegments of this entity. + def subsegments + @subsegments ||= [] + end + + # @param [Subsegment] subsegment Append the provided subsegment to children subsegments. + def add_subsegment(subsegment:) + raise EntityClosedError if closed? + subsegment.sampled = sampled + subsegment.parent = self + subsegments << subsegment + nil + end + + # @param [Subsegment] subsegment Remove the provided subsegment from children subsegments. + # @return [Subsegment] The deleted subsegment if the deletion is successful. + def remove_subsegment(subsegment:) + subsegments.delete(subsegment) + subsegment + end + + def annotations + @annotations ||= Annotations.new(self) + end + + def metadata(namespace: :default) + @metadata ||= Metadata.new(self) + @metadata.sub_meta(namespace) + end + + # Set error/fault/throttle flags based on http status code. + # This method is idempotent. + # @param [Integer] status + def apply_status_code(status:) + raise EntityClosedError if closed? + case status.to_i + when 429 + @throttle = true + @error = true + @fault = false + when 400..499 + @error = true + @throttle = false + @fault = false + when 500..599 + @fault = true + @error = false + @throttle = false + end + + @http_response ||= {} + @http_response[:status] = status.to_i + end + + # @param [Hash] request Supported keys are `:url`, `:user_agent`, `:client_ip`, + # `:x_forwarded_for`, `:method`. Value can be one of + # String or Integer or Boolean types depend on the key. + def merge_http_request(request:) + raise EntityClosedError if closed? + request.delete_if { |k| !HTTP_REQUEST_KEY.include?(k) } + @http_request ||= {} + @http_request.merge!(request) + end + + # @param [Hash] response Supported keys are `:status`, `:content_length`. + # Value can be one of String or Integer types depend on the key. + def merge_http_response(response:) + raise EntityClosedError if closed? + response.delete_if { |k| !HTTP_RESPONSE_KEY.include?(k) } + @http_response ||= {} + @http_response.merge!(response) + apply_status_code status: response[:status] if response.key?(:status) + end + + # @param [Exception] exception The exception object to capture. + # @param remote A boolean flag indicates whether the exception is + # returned from the downstream service. + def add_exception(exception:, remote: false) + raise EntityClosedError if closed? + @fault = true + @exception = exception + if cause_id = find_root_cause(exception) + @cause = Cause.new id: cause_id + else + @cause = Cause.new exception: exception, remote: remote + end + end + + # @return [String] Cause id is the id of the subsegment where + # the exception originally comes from. + def cause_id + return @cause.id if @cause + end + + # @return [Hash] The hash that contains all attributes that will + # be later serialized and sent out. + def to_h + h = { + name: name, + id: id, + start_time: start_time + } + if closed? + h[:end_time] = end_time + else + h[:in_progress] = true + end + h[:subsegments] = subsegments unless subsegments.empty? + h[:aws] = aws if aws + if http_request || http_response + h[:http] = {} + h[:http][:request] = http_request if http_request + h[:http][:response] = http_response if http_response + end + if (a = annotations.to_h) && !a.empty? + h[:annotations] = a + end + if (m = @metadata) && !m.to_h.empty? + h[:metadata] = m.to_h + end + + h[:parent_id] = parent.id if parent + # make sure the value in hash can only be boolean true + h[:fault] = !!fault if fault + h[:error] = !!error if error + h[:throttle] = !!throttle if throttle + h[:cause] = cause.to_h if cause + h + end + + def to_json + @to_json ||= begin + Oj.dump to_h, mode: :compat, use_as_json: true + end + end + + private + + def find_root_cause(e) + subsegment = subsegments.find { |i| i.exception.hash == e.hash } + return nil unless subsegment + if cause_id = subsegment.cause_id + cause_id + else + subsegment.id + end + end + end +end diff --git a/lib/aws-xray-sdk/model/metadata.rb b/lib/aws-xray-sdk/model/metadata.rb new file mode 100644 index 0000000..ec6e3ce --- /dev/null +++ b/lib/aws-xray-sdk/model/metadata.rb @@ -0,0 +1,77 @@ +require 'oj' +require 'aws-xray-sdk/exceptions' + +module XRay + # Metadata are key-value pairs with values of any type, including objects + # and lists, but that are not indexed. Use metadata to record data + # you want to store in the trace but don't need to use for searching traces. + class Metadata + def initialize(entity) + @data = {} + @entity = entity + end + + def sub_meta(namespace) + @data[namespace] = SubMeta.new(@entity) unless @data[namespace] + @data[namespace] + end + + def to_h + @data + end + end + + # The actual class that stores all data under a certain namespace. + class SubMeta + def initialize(entity) + @data = {} + @entity = entity + end + + def [](key) + @data[key] + end + + def []=(k, v) + raise EntityClosedError if @entity.closed? + @data[k] = v + end + + def update(h) + raise EntityClosedError if @entity.closed? + @data.merge!(h) + end + + def to_h + @data + end + + def to_json + @to_json ||= begin + Oj.dump to_h, mode: :compat, use_as_json: true + end + end + end + + # Singleton facade metadata class doing no-op for performance + # in case of not sampled X-Ray entities. + module FacadeMetadata + class << self + def [](key) + # no-op + end + + def []=(k, v) + # no-op + end + + def update(h) + # no-op + end + + def to_h + # no-op + end + end + end +end diff --git a/lib/aws-xray-sdk/model/segment.rb b/lib/aws-xray-sdk/model/segment.rb new file mode 100644 index 0000000..9306116 --- /dev/null +++ b/lib/aws-xray-sdk/model/segment.rb @@ -0,0 +1,63 @@ +require 'aws-xray-sdk/model/entity' + +module XRay + # The compute resources running your application logic send data + # about their work as segments. A segment provides the resource's name, + # details about the request, and details about the work done. + class Segment + include Entity + attr_accessor :ref_counter, :subsegment_size, :origin, :user + + # @param [String] trace_id Manually crafted trace id. + # @param [String] name Must be specified either on object creation or + # on environment variable `AWS_TRACING_NAME`. The latter has higher precedence. + # @param [String] parent_id ID of the segment/subsegment representing the upstream caller. + def initialize(trace_id: nil, name: nil, parent_id: nil) + @trace_id = trace_id + @name = ENV['AWS_TRACING_NAME'] || name + @parent_id = parent_id + @start_time = Time.now.to_f + @ref_counter = 0 + @subsegment_size = 0 + @sampled = true + end + + def trace_id + @trace_id ||= begin + %[1-#{Time.now.to_i.to_s(16)}-#{SecureRandom.hex(12)}] + end + end + + def add_subsegment(subsegment:) + super subsegment: subsegment + @ref_counter += 1 + @subsegment_size += 1 + end + + def remove_subsegment(subsegment:) + super subsegment: subsegment + @subsegment_size = subsegment_size - subsegment.all_children_count - 1 + end + + def decrement_ref_counter + @ref_counter -= 1 + end + + def ready_to_send? + closed? && ref_counter.zero? + end + + def to_h + h = super + h[:trace_id] = trace_id + h[:origin] = origin if origin + h[:parent_id] = @parent_id if @parent_id + h[:user] = user if user + h + end + + def segment + self + end + end +end diff --git a/lib/aws-xray-sdk/model/subsegment.rb b/lib/aws-xray-sdk/model/subsegment.rb new file mode 100644 index 0000000..7abe59a --- /dev/null +++ b/lib/aws-xray-sdk/model/subsegment.rb @@ -0,0 +1,67 @@ +require 'aws-xray-sdk/model/entity' + +module XRay + # The work done in a single segment can be broke down into subsegments. + # Subsegments provide more granular timing information and details about + # downstream calls that your application made to fulfill the original request. + # A subsegment can contain additional details about a call to an AWS service, + # an external HTTP API, or an SQL database. + class Subsegment + include Entity + + attr_reader :segment + attr_accessor :sql + + # @param [String] name The subsegment name. + # @param [Segment] segment The root parent segment. This segment + # may not be its direct parent. + # @param [String] namespace Currently supported namespaces are + # 'remote', 'aws', 'local'. + def initialize(name:, segment:, namespace: 'local') + @name = name + @segment = segment + @namespace = namespace + @start_time = Time.now.to_f + @sampled = true + end + + def add_subsegment(subsegment:) + super subsegment: subsegment + segment.ref_counter += 1 + segment.subsegment_size += 1 + end + + def remove_subsegment(subsegment:) + super subsegment: subsegment + cur = segment.subsegment_size + segment.subsegment_size = cur - subsegment.all_children_count - 1 + end + + def close(end_time: nil) + super end_time: end_time + segment.decrement_ref_counter + end + + def sql + @sql ||= {} + end + + # Returns the number of its direct and indirect children. + # This is useful when we remove the reference to a subsegment + # and need to keep remaining subsegment size accurate. + def all_children_count + size = subsegments.count + subsegments.each { |v| size += v.all_children_count } + size + end + + def to_h + h = super + h[:trace_id] = segment.trace_id + h[:sql] = sql unless sql.empty? + h[:type] = 'subsegment' + h[:namespace] = namespace if namespace + h + end + end +end diff --git a/lib/aws-xray-sdk/model/trace_header.rb b/lib/aws-xray-sdk/model/trace_header.rb new file mode 100644 index 0000000..4d0f390 --- /dev/null +++ b/lib/aws-xray-sdk/model/trace_header.rb @@ -0,0 +1,54 @@ +require 'aws-xray-sdk/logger' + +module XRay + # The sampling decision and trace ID are added to HTTP requests in + # tracing headers named ``X-Amzn-Trace-Id``. The first X-Ray-integrated + # service that the request hits adds a tracing header, which is read + # by the X-Ray SDK and included in the response. + class TraceHeader + include Logging + attr_accessor :root, :parent_id, :sampled + + # @param [String] root Trace id. + # @param [String] parent_id The id of the parent segment or subsegment. + # @param [Integer] sampled 0 means not sampled. + def initialize(root:, parent_id:, sampled:) + @root = root + @parent_id = parent_id + @sampled = sampled.to_i if sampled + end + + def self.from_header_string(header_str:) + empty_header if header_str.to_s.empty? + header = header_str.delete(' ').downcase + tmp = {} + begin + fields = header.split(';') + fields.each do |f| + pair = f.split('=') + tmp[pair[0].to_sym] = pair[1] + end + new root: tmp[:root], parent_id: tmp[:parent], sampled: tmp[:sampled] + rescue StandardError + logger.warn %(Invalid trace header #{header}. Ignored.) + empty_header + end + end + + # @return [String] The heading string constructed based on this header object. + def header_string + return '' unless root + if !parent_id + %(Root=#{root};Sampled=#{sampled}) + elsif !sampled + %(Root=#{root};Parent=#{parent_id}) + else + %(Root=#{root};Parent=#{parent_id};Sampled=#{sampled}) + end + end + + def self.empty_header + new root: nil, parent_id: nil, sampled: nil + end + end +end diff --git a/lib/aws-xray-sdk/patcher.rb b/lib/aws-xray-sdk/patcher.rb new file mode 100644 index 0000000..1a25fe5 --- /dev/null +++ b/lib/aws-xray-sdk/patcher.rb @@ -0,0 +1,21 @@ +require 'aws-xray-sdk/exceptions' + +module XRay + # Patching external libraries/frameworks to be traced by X-Ray recorder. + module Patcher + # @param [Array] targets A list of libraries/frameworks to patch. + def patch(targets) + targets.each do |l| + case l + when :net_http + require 'aws-xray-sdk/facets/net_http' + when :aws_sdk + require 'aws-xray-sdk/facets/aws_sdk' + XRay::AwsSDKPatcher.patch + else + raise UnsupportedPatchingTargetError.new(%(#{l} is not supported by X-Ray SDK.)) + end + end + end + end +end diff --git a/lib/aws-xray-sdk/plugins/ec2.rb b/lib/aws-xray-sdk/plugins/ec2.rb new file mode 100644 index 0000000..bfb6cb2 --- /dev/null +++ b/lib/aws-xray-sdk/plugins/ec2.rb @@ -0,0 +1,39 @@ +require 'open-uri' +require 'aws-xray-sdk/logger' + +module XRay + module Plugins + # A plugin that gets the EC2 instance-id and AZ if running on an EC2 instance. + module EC2 + include Logging + + ORIGIN = 'AWS::EC2::Instance'.freeze + # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html#instancedata-data-retrieval + ID_ADDR = 'http://169.254.169.254/latest/meta-data/instance-id'.freeze + AZ_ADDR = 'http://169.254.169.254/latest/meta-data/placement/availability-zone'.freeze + + def self.aws + @@aws ||= begin + instance_id = open(ID_ADDR, open_timeout: 1).read + az = open(AZ_ADDR, open_timeout: 1).read + { + ec2: { + instance_id: instance_id, + availability_zone: az + } + } + rescue StandardError => e + # Two attempts in total to get EC2 metadata + @retries ||= 0 + if @retries < 1 + @retries += 1 + retry + else + @@aws = {} + Logging.logger.warn %(can not get the ec2 instance metadata due to: #{e.message}.) + end + end + end + end + end +end diff --git a/lib/aws-xray-sdk/plugins/ecs.rb b/lib/aws-xray-sdk/plugins/ecs.rb new file mode 100644 index 0000000..c2eb47f --- /dev/null +++ b/lib/aws-xray-sdk/plugins/ecs.rb @@ -0,0 +1,23 @@ +require 'socket' +require 'aws-xray-sdk/logger' + +module XRay + module Plugins + # Due to lack of ECS container metadata service, the only host information + # available is the host name. + module ECS + include Logging + + ORIGIN = 'AWS::ECS::Container'.freeze + + def self.aws + @@aws ||= begin + { ecs: { container: Socket.gethostname } } + rescue StandardError => e + @@aws = {} + Logging.logger.warn %(can not get the ecs container hostname due to: #{e.message}.) + end + end + end + end +end diff --git a/lib/aws-xray-sdk/plugins/elastic_beanstalk.rb b/lib/aws-xray-sdk/plugins/elastic_beanstalk.rb new file mode 100644 index 0000000..b23b7d7 --- /dev/null +++ b/lib/aws-xray-sdk/plugins/elastic_beanstalk.rb @@ -0,0 +1,25 @@ +require 'oj' +require 'aws-xray-sdk/logger' + +module XRay + module Plugins + # A plugin that records information about the elastic beanstalk environment + # hosting your application. + module ElasticBeanstalk + include Logging + + CONF_PATH = '/var/elasticbeanstalk/xray/environment.conf'.freeze + ORIGIN = 'AWS::ElasticBeanstalk::Environment'.freeze + + def self.aws + @@aws ||= begin + file = File.open(CONF_PATH) + { elastic_beanstalk: Oj.load(file) } + rescue StandardError => e + @@aws = {} + Logging.logger.warn %(can not get the environment config due to: #{e.message}.) + end + end + end + end +end diff --git a/lib/aws-xray-sdk/recorder.rb b/lib/aws-xray-sdk/recorder.rb new file mode 100644 index 0000000..de5e24d --- /dev/null +++ b/lib/aws-xray-sdk/recorder.rb @@ -0,0 +1,209 @@ +require 'aws-xray-sdk/configuration' +require 'aws-xray-sdk/exceptions' +require 'aws-xray-sdk/model/segment' +require 'aws-xray-sdk/model/subsegment' +require 'aws-xray-sdk/model/dummy_entities' +require 'aws-xray-sdk/model/annotations' +require 'aws-xray-sdk/model/metadata' + +module XRay + # A global AWS X-Ray recorder that will begin/end segments/subsegments + # and send them to the X-Ray daemon. It is also responsible for managing + # context. + class Recorder + attr_reader :config + + def initialize(user_config: nil) + @config = Configuration.new + @config.configure(user_config) unless user_config.nil? + end + + # Begin a segment for the current context. The recorder + # only keeps one segment at a time. Create a second one without + # closing existing one will overwrite the existing one. + # @return [Segment] thew newly created segment. + def begin_segment(name, trace_id: nil, parent_id: nil, sampled: nil) + seg_name = name || config.name + raise SegmentNameMissingError if seg_name.to_s.empty? + + # sampling decision comes from outside has higher precedence. + sample = sampled.nil? ? config.sample? : sampled + if sample + segment = Segment.new name: seg_name, trace_id: trace_id, parent_id: parent_id + populate_runtime_context(segment) + else + segment = DummySegment.new name: seg_name, trace_id: trace_id, parent_id: parent_id + end + context.store_entity entity: segment + segment + end + + # @return [Segment] the active segment tied to the current context. + # If the current context is under a subsegment, it returns its parent segment. + def current_segment + entity = current_entity + entity.segment if entity + end + + # End the current segment and send it to X-Ray daemon if it is ready. + def end_segment(end_time: nil) + segment = current_segment + return unless segment + segment.close end_time: end_time + context.clear! + emitter.send_entity entity: segment if segment.ready_to_send? + end + + # Begin a new subsegment and add it to be the child of the current active + # subsegment or segment. Also tie the new created subsegment to the current context. + # Its sampling decision will follow its parent. + # @return [Subsegment] the newly created subsegment. + def begin_subsegment(name, namespace: nil, segment: nil) + entity = segment || current_entity + return unless entity + if entity.sampled + subsegment = Subsegment.new name: name, segment: entity.segment, namespace: namespace + else + subsegment = DummySubsegment.new name: name, segment: entity.segment + end + # attach the new created subsegment under the current active entity + entity.add_subsegment subsegment: subsegment + # associate the new subsegment to the current context + context.store_entity entity: subsegment + subsegment + end + + # @return [Subsegment] the active subsegment tied to the current context. + # Returns nil if the current context has no associated subsegment. + def current_subsegment + entity = context.current_entity + entity.is_a?(Subsegment) ? entity : nil + end + + # End the current active subsegment. It also send the entire segment if + # this subsegment is the last one open or stream out subsegments of its + # parent segment if the stream threshold is breached. + def end_subsegment(end_time: nil) + entity = current_entity + return unless entity.is_a?(Subsegment) + entity.close end_time: end_time + # update current context + if entity.parent.closed? + context.clear! + else + context.store_entity entity: entity.parent + end + # check if the entire segment can be send. + # If not, stream subsegments when threshold is reached. + segment = entity.segment + if segment.ready_to_send? + emitter.send_entity entity: segment + elsif streamer.eligible? segment: segment + streamer.stream_subsegments root: segment, emitter: emitter + end + end + + # Record the passed block as a subsegment. + def capture(name, namespace: nil, segment: nil) + subsegment = begin_subsegment name, namespace: namespace, segment: segment + begin + yield subsegment + rescue Exception => e + subsegment.add_exception exception: e + raise e + ensure + end_subsegment + end + end + + # Returns current segment or subsegment that associated to the current context. + # This is a proxy method to Context class current_entity. + def current_entity + context.current_entity + end + + def inject_context(entity, target_ctx: nil) + context.inject_context entity, target_ctx: target_ctx + return unless block_given? + yield + context.clear! + end + + def clear_context + context.clear! + end + + def sampled? + entity = current_entity + if block_given? + yield if entity && entity.sampled + else + entity && entity.sampled + end + end + + # A proxy method to get the annotations from the current active entity. + def annotations + entity = current_entity + if entity + entity.annotations + else + FacadeAnnotations + end + end + + # A proxy method to get the metadata under provided namespace + # from the current active entity. + def metadata(namespace: :default) + entity = current_entity + if entity + entity.metadata(namespace: namespace) + else + FacadeMetadata + end + end + + # A proxy method to XRay::Configuration.configure + def configure(user_config) + config.configure(user_config) + end + + def context + config.context + end + + def sampler + config.sampler + end + + def emitter + config.emitter + end + + def streamer + config.streamer + end + + def segment_naming + config.segment_naming + end + + def sampling_enabled? + config.sampling + end + + private_class_method + + def populate_runtime_context(segment) + aws = {} + config.plugins.each do |p| + meta = p.aws + if meta.is_a?(Hash) && !meta.empty? + aws.merge! meta + segment.origin = p::ORIGIN + end + end + segment.aws = aws unless aws.empty? + end + end +end diff --git a/lib/aws-xray-sdk/sampling/default_sampler.rb b/lib/aws-xray-sdk/sampling/default_sampler.rb new file mode 100644 index 0000000..7ef227f --- /dev/null +++ b/lib/aws-xray-sdk/sampling/default_sampler.rb @@ -0,0 +1,105 @@ +require 'aws-xray-sdk/sampling/sampler' +require 'aws-xray-sdk/sampling/sampling_rule' +require 'aws-xray-sdk/exceptions' + +module XRay + # The default sampler that uses internally defined + # sampling rule and reservoir models to decide sampling decision. + # It also uses the default sampling rule. + # An example definition: + # { + # version: 1, + # rules: [ + # { + # description: 'Player moves.', + # service_name: '*', + # http_method: '*', + # url_path: '/api/move/*', + # fixed_target: 0, + # rate: 0.05 + # } + # ], + # default: { + # fixed_target: 1, + # rate: 0.1 + # } + # } + # This example defines one custom rule and a default rule. + # The custom rule applies a five-percent sampling rate with no minimum + # number of requests to trace for paths under /api/move/. The default + # rule traces the first request each second and 10 percent of additional requests. + # The SDK applies custom rules in the order in which they are defined. + # If a request matches multiple custom rules, the SDK applies only the first rule. + class DefaultSampler + include Sampler + DEFAULT_RULES = { + version: 1, + default: { + fixed_target: 1, + rate: 0.05 + }, + rules: [] + }.freeze + + def initialize + load_sampling_rules(DEFAULT_RULES) + end + + # Return True if the sampler decide to sample based on input + # information and sampling rules. It will first check if any + # custom rule should be applied, if not it falls back to the + # default sampling rule. + # All arugments are extracted from incoming requests by + # X-Ray middleware to perform path based sampling. + def sample_request?(service_name:, url_path:, http_method:) + # directly fallback to non-path-based if all arguments are nil + return sample? unless service_name || url_path || http_method + @custom_rules ||= [] + @custom_rules.each do |c| + return should_sample?(c) if c.applies?(target_name: service_name, target_path: url_path, target_method: http_method) + end + sample? + end + + # Decides if should sample based on non-path-based rule. + # Currently only the default rule is not path-based. + def sample? + should_sample?(@default_rule) + end + + # @param [Hash] v The sampling rules definition. + def sampling_rules=(v) + load_sampling_rules(v) + end + + # @return [Array] An array of [SamplingRule] + def sampling_rules + all_rules = [] + all_rules << @default_rule + all_rules << @custom_rules unless @custom_rules.empty? + all_rules + end + + private + + def should_sample?(rule) + return true if rule.reservoir.take + Random.rand <= rule.rate + end + + def load_sampling_rules(v) + version = v[:version] + if version != 1 + raise InvalidSamplingConfigError, %('Sampling rule version #{version} is not supported.') + end + unless v[:default] + raise InvalidSamplingConfigError, 'A default rule must be provided.' + end + @default_rule = SamplingRule.new rule_definition: v[:default], default: true + @custom_rules = [] + v[:rules].each do |d| + @custom_rules << SamplingRule.new(rule_definition: d) + end + end + end +end diff --git a/lib/aws-xray-sdk/sampling/reservoir.rb b/lib/aws-xray-sdk/sampling/reservoir.rb new file mode 100644 index 0000000..ee3ceb3 --- /dev/null +++ b/lib/aws-xray-sdk/sampling/reservoir.rb @@ -0,0 +1,35 @@ +module XRay + # Keeps track of the number of sampled segments within + # a single second. This class is implemented to be + # thread-safe to achieve accurate sampling. + class Reservoir + # @param [Integer] traces_per_sec The number of guranteed sampled + # segments per second. + def initialize(traces_per_sec: 0) + @traces_per_sec = traces_per_sec + @used_this_sec = 0 + @this_sec = Time.now.to_i + @lock = Mutex.new + end + + # Returns `true` if there are segments left within the + # current second, otherwise returns `false`. + def take + # nothing to provide if reserved is set to 0 + return false if @traces_per_sec.zero? + @lock.synchronize do + now = Time.now.to_i + # refresh time frame + if now != @this_sec + @used_this_sec = 0 + @this_sec = now + end + # return false if reserved item ran out + return false unless @used_this_sec < @traces_per_sec + # otherwise increment used counter and return true + @used_this_sec += 1 + return true + end + end + end +end diff --git a/lib/aws-xray-sdk/sampling/sampler.rb b/lib/aws-xray-sdk/sampling/sampler.rb new file mode 100644 index 0000000..a2d0cca --- /dev/null +++ b/lib/aws-xray-sdk/sampling/sampler.rb @@ -0,0 +1,27 @@ +module XRay + # The sampler interface that calculates if a segment + # should be sampled or not upon creation based on the + # sampling rules it holds. It doesn't respect sampling decision + # from upstream. + module Sampler + # Decides if a segment should be sampled for an incoming request. + # Used in case of middleware. + def sample_request?(service_name:, url_path:, http_method:) + raise 'Not implemented' + end + + # Decides if a segment should be sampled merely based on internal + # sampling rules. + def sample? + raise 'Not implemented' + end + + def sampling_rules=(v) + raise 'Not implemented' + end + + def sampling_rules + raise 'Not implemented' + end + end +end diff --git a/lib/aws-xray-sdk/sampling/sampling_rule.rb b/lib/aws-xray-sdk/sampling/sampling_rule.rb new file mode 100644 index 0000000..1947ea0 --- /dev/null +++ b/lib/aws-xray-sdk/sampling/sampling_rule.rb @@ -0,0 +1,57 @@ +require 'aws-xray-sdk/exceptions' +require 'aws-xray-sdk/sampling/reservoir' +require 'aws-xray-sdk/search_pattern' + +module XRay + # One SamplingRule object represents one rule defined from the rules hash definition. + # It can be either a custom rule or the default rule. + class SamplingRule + attr_reader :fixed_target, :rate, :service_name, + :method, :path, :reservoir, :default + + # @param [Hash] rule_definition Hash that defines a single rule. + # @param default A boolean flag indicates if this rule is the default rule. + def initialize(rule_definition:, default: false) + @fixed_target = rule_definition[:fixed_target] + @rate = rule_definition[:rate] + + @service_name = rule_definition[:service_name] + @method = rule_definition[:http_method] + @path = rule_definition[:url_path] + + @default = default + validate + @reservoir = Reservoir.new traces_per_sec: @fixed_target + end + + # Determines whether or not this sampling rule applies to + # the incoming request based on some of the request's parameters. + # Any None parameters provided will be considered an implicit match. + def applies?(target_name:, target_path:, target_method:) + name_match = !target_name || SearchPattern.wildcard_match?(pattern: @service_name, text: target_name) + path_match = !target_path || SearchPattern.wildcard_match?(pattern: @path, text: target_path) + method_match = !target_method || SearchPattern.wildcard_match?(pattern: @method, text: target_method) + name_match && path_match && method_match + end + + private + + def validate + if @fixed_target < 0 || @rate < 0 + raise InvalidSamplingConfigError, 'All rules must have non-negative values for fixed_target and rate.' + end + + if @default + # validate default rule + if @service_name || @method || @path + raise InvalidSamplingConfigError, 'The default rule must not specify values for url_path, service_name, or http_method.' + end + else + # validate custom rule + unless @service_name && @method && @path + raise InvalidSamplingConfigError, 'All non-default rules must have values for url_path, service_name, and http_method.' + end + end + end + end +end diff --git a/lib/aws-xray-sdk/search_pattern.rb b/lib/aws-xray-sdk/search_pattern.rb new file mode 100644 index 0000000..a17a3ce --- /dev/null +++ b/lib/aws-xray-sdk/search_pattern.rb @@ -0,0 +1,82 @@ +module XRay + # custom pattern matching for performance and the SDK use cases. + module SearchPattern + # Performs a case-insensitive wildcard match against two strings. + # This method works with pseduo-regex chars; specifically ? and * are supported. + # An asterisk (*) represents any combination of characters. + # A question mark (?) represents any single character. + # @param [String] pattern The regex-like pattern to be compared against. + # @param [String] text The string to compare against the pattern. + # @param case_insensitive A boolean flag. Default is true. + def self.wildcard_match?(pattern:, text:, case_insensitive: true) + return false unless pattern && text + pattern_len = pattern.length + text_len = text.length + return text_len.zero? if pattern_len.zero? + # Check the special case of a single * pattern, as it's common + return true if pattern == '*' + + if case_insensitive + # do not mutate original input + pattern = pattern.downcase + text = text.downcase + end + # Infix globs are relatively rare, and the below search is expensive. + # Check for infix globs and, in their absence, do the simple thing. + if !pattern.include?('*') || pattern.index('*') == pattern_len - 1 + return simple_wildcard_match? pattern: pattern, text: text + end + + # The res[i] is used to record if there is a match between + # the first i chars in text and the first j chars in pattern. + # So will return res[textLength+1] in the end + # Loop from the beginning of the pattern + # case not '*': if text[i]==pattern[j] or pattern[j] is '?', + # and res[i] is true, set res[i+1] to true, otherwise false. + # case '*': since '*' can match any globing, as long as there is a true + # in res before i, all the res[i+1], res[i+2],...,res[textLength] + # could be true + res = Array.new(text_len + 1) + res[0] = true + (0...pattern_len).each do |j| + p = pattern[j] + if p != '*' + (text_len - 1).downto(0) do |i| + res[i + 1] = res[i] && (p == '?' || (p == text[i])) + end + else + i = 0 + i += 1 while i <= text_len && !res[i] + (i..text_len + 1).each do |m| + res[m] = true + end + end + res[0] = res[0] && (p == '*') + end + res[text_len] + end + + private_class_method + + def self.simple_wildcard_match?(pattern:, text:) + j = 0 + pattern_len = pattern.length + text_len = text.length + (0...pattern_len).each do |i| + p = pattern[i] + # Presumption for this method is that globs only occur at end + return true if p == '*' + if p == '?' + # No character to match + return false if j == text_len + else + return false if j >= text_len || p != text[j] + end + j += 1 + end + # Ate up all the pattern and didn't end at a glob, so a match + # will have consumed all the text + j == text_len + end + end +end diff --git a/lib/aws-xray-sdk/segment_naming/dynamic_naming.rb b/lib/aws-xray-sdk/segment_naming/dynamic_naming.rb new file mode 100644 index 0000000..a896e2a --- /dev/null +++ b/lib/aws-xray-sdk/segment_naming/dynamic_naming.rb @@ -0,0 +1,26 @@ +require 'aws-xray-sdk/segment_naming/segment_naming' +require 'aws-xray-sdk/search_pattern' + +module XRay + # Decides what name to use on a segment generated from an incoming request. + # This default naming takes the host name and compares it to a pre-defined pattern. + # If the host name matches that pattern, it returns the host name, otherwise + # it returns the fallback name. The host name usually comes from the incoming + # request's headers. + class DynamicNaming + include SegmentNaming + + # @param [String] fallback The fallback name used when there is no match + # between host name and specified pattern. + def initialize(fallback:) + @fallback = fallback + end + + # @param [String] host The host name fetched from the incoming request's header. + def provide_name(host:) + # use fallback name when either the pattern or host name is unavailable. + return fallback unless pattern && !pattern.empty? && host && !host.empty? + SearchPattern.wildcard_match?(pattern: pattern, text: host) ? host : fallback + end + end +end diff --git a/lib/aws-xray-sdk/segment_naming/segment_naming.rb b/lib/aws-xray-sdk/segment_naming/segment_naming.rb new file mode 100644 index 0000000..7d6a176 --- /dev/null +++ b/lib/aws-xray-sdk/segment_naming/segment_naming.rb @@ -0,0 +1,10 @@ +module XRay + # The interface that provides the segment name + # based on host name, pattern and the fallback name. + module SegmentNaming + attr_accessor :fallback, :pattern + def provide_name(host:) + raise 'Not implemented' + end + end +end diff --git a/lib/aws-xray-sdk/streaming/default_streamer.rb b/lib/aws-xray-sdk/streaming/default_streamer.rb new file mode 100644 index 0000000..ebebe6b --- /dev/null +++ b/lib/aws-xray-sdk/streaming/default_streamer.rb @@ -0,0 +1,53 @@ +require 'aws-xray-sdk/streaming/streamer' +require 'aws-xray-sdk/logger' + +module XRay + # The default streamer use subsegment count as the threshold + # for performance reasons and it streams out subtrees + # where all the nodes in it are completed. + class DefaultStreamer + include Streamer + include Logging + + def initialize + @stream_threshold = 50 + end + + # @param [Segment] segment Check if the provided segment exceeds + # the threshold to stream. + def eligible?(segment:) + # only get subsegments to stream from sampled segments. + segment && segment.sampled && segment.subsegment_size >= stream_threshold + end + + # @param [Segment] root The target segment to stream subsegments from. + # @param [Emitter] emitter The emitter employed to send data to the daemon. + def stream_subsegments(root:, emitter:) + children = root.subsegments + children_ready = [] + + unless children.empty? + # Collect ready subtrees from root. + children.each do |child| + children_ready << child if stream_subsegments root: child, emitter: emitter + end + end + + # If this subtree is ready, go back to the root's parent + # to try to find a bigger subtree + return true if children_ready.length == children.length && root.closed? + # Otherwise this subtree has at least one non-ready node. + # Only stream its ready child subtrees. + children_ready.each do |child| + root.remove_subsegment subsegment: child + emitter.send_entity entity: child + end + # Return false so this node won't be added to its parent's ready children. + false + end + + # @return [Integer] The maximum number of subsegments a segment + # can hold before streaming. + attr_accessor :stream_threshold + end +end diff --git a/lib/aws-xray-sdk/streaming/streamer.rb b/lib/aws-xray-sdk/streaming/streamer.rb new file mode 100644 index 0000000..1c52140 --- /dev/null +++ b/lib/aws-xray-sdk/streaming/streamer.rb @@ -0,0 +1,17 @@ +module XRay + # The interface used by the X-Ray recoder to get eligible subsegments + # to be streamed out from a given segment. + module Streamer + def eligible?(segment:) + raise 'Not implemented' + end + + def subsegments_to_stream(segment:, emitter:, force: false) + raise 'Not implemented' + end + + def stream_threshold=(v) + raise 'Not implemented' + end + end +end diff --git a/lib/aws-xray-sdk/version.rb b/lib/aws-xray-sdk/version.rb new file mode 100644 index 0000000..cab21bb --- /dev/null +++ b/lib/aws-xray-sdk/version.rb @@ -0,0 +1,3 @@ +module XRay + VERSION = '0.9.0' +end diff --git a/test/aws-xray-sdk/tc_aws_sdk.rb b/test/aws-xray-sdk/tc_aws_sdk.rb new file mode 100644 index 0000000..7fcbc67 --- /dev/null +++ b/test/aws-xray-sdk/tc_aws_sdk.rb @@ -0,0 +1,113 @@ +require_relative '../test_helper' +require 'aws-xray-sdk' +require 'aws-sdk-s3' +require 'aws-sdk-dynamodb' + +# Test subsegments recording on AWS Ruby SDK +class TestAwsSdk < Minitest::Test + @@recorder = XRay::Recorder.new + config = { + sampling: false, + emitter: XRay::TestHelper::StubbedEmitter.new, + patch: %I[aws_sdk] + } + @@recorder.configure(config) + Aws.config.update xray_recorder: @@recorder + + def setup + @@recorder.context.clear! + @@recorder.emitter.clear + end + + def teardown + @@recorder.context.clear! + @@recorder.emitter.clear + end + + # By default the X-Ray SDK doesn't have parameter whitelisting for S3 APIs. + def test_simple_s3_call + @@recorder.begin_segment name + s3 = Aws::S3::Client.new(stub_responses: true) + bucket_data = s3.stub_data(:list_buckets, buckets: [{ name: '1' }, { name: '2' }]) + s3.stub_responses(:list_buckets, bucket_data) + s3.list_buckets + @@recorder.end_segment + subsegment = @@recorder.emitter.entities[0].subsegments[0] + + assert_equal 'aws', subsegment.namespace + aws_meta = subsegment.aws + assert_equal 'ListBuckets', aws_meta[:operation] + end + + def test_return_list_count + @@recorder.begin_segment name + dynamodb = Aws::DynamoDB::Client.new(stub_responses: true) + table_list = dynamodb.stub_data(:list_tables, table_names: %w[t1 t2]) + dynamodb.stub_responses(:list_tables, table_list) + dynamodb.list_tables + @@recorder.end_segment + subsegment = @@recorder.emitter.entities[0].subsegments[0] + + assert_equal 'aws', subsegment.namespace + aws_meta = subsegment.aws + assert_equal 'ListTables', aws_meta[:operation] + assert_equal 2, aws_meta[:table_count] + refute aws_meta.key?(:table_names) + end + + def test_capture_map_keys + @@recorder.begin_segment name + dynamodb = Aws::DynamoDB::Client.new(stub_responses: true) + + mocked_req = { + request_items: { + table1: { + keys: [{ id: 1 }] + }, + table2: { + keys: [{ id: 2 }] + } + } + } + + mocked_resp = { + consumed_capacity: [ + { + table_name: 'table1', + capacity_units: 1.0 + }, + { + table_name: 'table2', + capacity_units: 2.0 + } + ] + } + + resp = dynamodb.stub_data(:batch_get_item, mocked_resp) + dynamodb.stub_responses(:batch_get_item, resp) + dynamodb.batch_get_item mocked_req + @@recorder.end_segment + subsegment = @@recorder.emitter.entities[0].subsegments[0] + + assert_equal 'aws', subsegment.namespace + aws_meta = subsegment.aws + assert_equal 'BatchGetItem', aws_meta[:operation] + assert_equal %w[table1 table2], aws_meta[:table_names].sort + assert_equal mocked_resp[:consumed_capacity], aws_meta[:consumed_capacity] + end + + def test_capiture_client_error + @@recorder.begin_segment name + s3 = Aws::S3::Client.new(stub_responses: true) + s3.stub_responses(:head_bucket, Timeout::Error) + # makes sure the capture code doesn't swallow the error + assert_raises Timeout::Error do + s3.head_bucket bucket: 'my_bucket' + end + @@recorder.end_segment + subsegment = @@recorder.emitter.entities[0].subsegments[0] + ex_h = subsegment.to_h[:cause][:exceptions][0] + assert_equal 'Timeout::Error', ex_h[:message] + assert_equal 'Timeout::Error', ex_h[:type] + end +end diff --git a/test/aws-xray-sdk/tc_cause.rb b/test/aws-xray-sdk/tc_cause.rb new file mode 100644 index 0000000..b7f9e41 --- /dev/null +++ b/test/aws-xray-sdk/tc_cause.rb @@ -0,0 +1,87 @@ +require_relative '../test_helper' +require 'aws-xray-sdk/model/segment' +require 'aws-xray-sdk/model/subsegment' + +# Test exception recording +class TestCauses < Minitest::Test + def test_simple_exception + segment = XRay::Segment.new name: name + begin + 1 / 0 + rescue ZeroDivisionError => e + segment.add_exception exception: e + end + + assert segment.fault + # check top level fields + h = segment.cause.to_h + assert_equal 1, h[:exceptions].count + assert_nil h[:remote] + refute_nil h[:working_directory] + refute_nil h[:paths] + + # check exception entry + eh = h[:exceptions][0] + assert_equal 'ZeroDivisionError', eh[:type] + refute_nil eh[:message] + + # check stack entry + stack = eh[:stack][0] + assert stack[:line].is_a?(Integer) + refute_nil stack[:label] + refute_nil stack[:path] + end + + def test_remote_flag + segment = XRay::Segment.new name: name + begin + raise StandardError + rescue StandardError => e + segment.add_exception exception: e, remote: true + end + exception = segment.cause.to_h[:exceptions][0] + assert_equal true, exception[:remote] + end + + def test_chained_exception + segment = XRay::Segment.new name: name + begin + fail_and_raise + rescue StandardError => e + segment.add_exception exception: e + end + + # check top level fields + h = segment.cause.to_h + assert_equal 2, h[:exceptions].count + + # check exceptions type + exceptions = h[:exceptions] + assert_equal 'StandardError', exceptions[0][:type] + assert_equal 'ZeroDivisionError', exceptions[1][:type] + end + + def test_duplicate_exception + segment = XRay::Segment.new name: name + subsegment = XRay::Subsegment.new name: name, segment: segment + segment.add_subsegment subsegment: subsegment + begin + 1 / 0 + rescue ZeroDivisionError => e + subsegment.add_exception exception: e + segment.add_exception exception: e + end + + assert_equal subsegment.id, segment.cause_id + cause_id = segment.to_h[:cause] + assert_equal subsegment.id, cause_id + end + + private + + def fail_and_raise + raise ZeroDivisionError + rescue + raise StandardError + end +end diff --git a/test/aws-xray-sdk/tc_context.rb b/test/aws-xray-sdk/tc_context.rb new file mode 100644 index 0000000..9374b28 --- /dev/null +++ b/test/aws-xray-sdk/tc_context.rb @@ -0,0 +1,42 @@ +require 'aws-xray-sdk/exceptions' +require 'aws-xray-sdk/model/segment' +require 'aws-xray-sdk/context/default_context' + +# Test context management +class TestContext < Minitest::Test + def test_segment_crud + context = XRay::DefaultContext.new + context.clear! + segment = XRay::Segment.new name: name + context.store_entity entity: segment + assert_equal segment, context.current_entity + segment2 = XRay::Segment.new name: name + context.store_entity entity: segment2 + assert_equal segment2, context.current_entity + context.clear! + end + + def test_change_context_missing + context = XRay::DefaultContext.new + context.clear! + context.context_missing = 'UNKWON' + assert_equal 'RUNTIME_ERROR', context.context_missing + context.context_missing = 'LOG_ERROR' + assert_equal 'LOG_ERROR', context.context_missing + end + + def test_runtime_error + context = XRay::DefaultContext.new + context.clear! + assert_raises XRay::ContextMissingError do + context.current_entity + end + end + + def test_log_error + context = XRay::DefaultContext.new + context.clear! + context.context_missing = 'LOG_ERROR' + refute context.current_entity + end +end diff --git a/test/aws-xray-sdk/tc_dummy_entities.rb b/test/aws-xray-sdk/tc_dummy_entities.rb new file mode 100644 index 0000000..58c6b09 --- /dev/null +++ b/test/aws-xray-sdk/tc_dummy_entities.rb @@ -0,0 +1,45 @@ +require 'aws-xray-sdk/model/dummy_entities' + +# Test dummy segments and dummy subsegments +class TestDummyEntities < Minitest::Test + def test_no_sample + segment = XRay::DummySegment.new name: name + refute segment.sampled + subegment = XRay::DummySubsegment.new name: segment, segment: segment + refute subegment.sampled + end + + def test_no_meta + segment = XRay::DummySegment.new name: name + subegment = XRay::DummySubsegment.new name: 'dummy', segment: segment + entities = [segment, subegment] + entities.each do |e| + e.metadata.update k: 'v' + e.annotations.update k: 'v' + e.merge_http_request request: { url: '/ping' } + e.merge_http_response response: { status: 200 } + e.aws = { sdk: 'ruby' } + end + + entities.each do |e| + refute e.aws + refute e.http_request + refute e.http_response + refute e.annotations.to_h + refute e.metadata.to_h + end + end + + def test_structure_intact + segment = XRay::DummySegment.new name: name + subsegment1 = XRay::DummySubsegment.new name: 'dummy', segment: segment + subsegment2 = XRay::DummySubsegment.new name: 'dummy', segment: segment + segment.add_subsegment subsegment: subsegment1 + subsegment1.add_subsegment subsegment: subsegment2 + + assert_equal 2, segment.subsegment_size + assert_equal 2, segment.ref_counter + assert_equal subsegment1, segment.subsegments[0] + assert_equal subsegment2, subsegment1.subsegments[0] + end +end diff --git a/test/aws-xray-sdk/tc_dynamic_naming.rb b/test/aws-xray-sdk/tc_dynamic_naming.rb new file mode 100644 index 0000000..dd9411b --- /dev/null +++ b/test/aws-xray-sdk/tc_dynamic_naming.rb @@ -0,0 +1,30 @@ +require 'aws-xray-sdk/segment_naming/dynamic_naming' + +# Test dynamic naming suite +class TestDynamicNaming < Minitest::Test + def test_no_pattern_specified + fallback = 'test' + naming = XRay::DynamicNaming.new fallback: fallback + assert_equal fallback, naming.provide_name(host: 'example.com') + assert_equal fallback, naming.provide_name(host: nil) + assert_equal fallback, naming.provide_name(host: '') + end + + def test_hostname_unavailable + fallback = 'test' + naming = XRay::DynamicNaming.new fallback: fallback + naming.pattern = '*' + assert_equal fallback, naming.provide_name(host: nil) + assert_equal fallback, naming.provide_name(host: '') + end + + def test_pattern_matching + fallback = 'test' + naming = XRay::DynamicNaming.new fallback: fallback + naming.pattern = '*mydomain*' + host = 'www.mydomain.com' + assert_equal host, naming.provide_name(host: host) + refute_equal host, naming.provide_name(host: '127.0.0.1') + assert_equal fallback, naming.provide_name(host: '127.0.0.1') + end +end diff --git a/test/aws-xray-sdk/tc_emitter.rb b/test/aws-xray-sdk/tc_emitter.rb new file mode 100644 index 0000000..9307abe --- /dev/null +++ b/test/aws-xray-sdk/tc_emitter.rb @@ -0,0 +1,25 @@ +require 'aws-xray-sdk/emitter/default_emitter' +require 'aws-xray-sdk/model/segment' +require 'aws-xray-sdk/exceptions' + +class TestEmitter < Minitest::Test + def test_pass_through + segment = XRay::Segment.new name: name + segment.close + emitter = XRay::DefaultEmitter.new + emitter.send_entity entity: segment + end + + def test_invalid_daemon_address + segment = XRay::Segment.new name: name + segment.close + assert_raises XRay::InvalidDaemonAddressError do + emitter = XRay::DefaultEmitter.new + emitter.daemon_address = 'blah' + end + assert_raises XRay::InvalidDaemonAddressError do + emitter = XRay::DefaultEmitter.new + emitter.daemon_address = '127.0.0.1' + end + end +end diff --git a/test/aws-xray-sdk/tc_plugin.rb b/test/aws-xray-sdk/tc_plugin.rb new file mode 100644 index 0000000..0e349a6 --- /dev/null +++ b/test/aws-xray-sdk/tc_plugin.rb @@ -0,0 +1,42 @@ +require 'webmock/minitest' +require 'aws-xray-sdk/plugins/elastic_beanstalk' +require 'aws-xray-sdk/plugins/ec2' +require 'aws-xray-sdk/plugins/ecs' +require 'aws-xray-sdk/recorder' + +# test AWS service plugins pass through +class TestPlugins < Minitest::Test + def test_origin_all_set + assert XRay::Plugins::ElasticBeanstalk::ORIGIN + assert XRay::Plugins::EC2::ORIGIN + assert XRay::Plugins::ECS::ORIGIN + end + + # all plugins should have method 'aws' and it should not break + # when running on any machine. + def test_get_runtime_context + XRay::Plugins::ElasticBeanstalk.aws + stub_request(:any, XRay::Plugins::EC2::ID_ADDR).to_raise(StandardError) + stub_request(:any, XRay::Plugins::EC2::AZ_ADDR).to_raise(StandardError) + XRay::Plugins::EC2.aws + XRay::Plugins::ECS.aws + WebMock.reset! + end + + def test_mocked_ec2_metadata + instance_id = "abc" + az = "us-east-1a" + stub_request(:any, XRay::Plugins::EC2::ID_ADDR) + .to_return(body: instance_id, status: 200) + stub_request(:any, XRay::Plugins::EC2::AZ_ADDR) + .to_return(body: az, status: 200) + expected = { + ec2: { + instance_id: instance_id, + avaliablity_zone: az + } + } + assert expected, XRay::Plugins::EC2.aws + WebMock.reset! + end +end diff --git a/test/aws-xray-sdk/tc_recorder.rb b/test/aws-xray-sdk/tc_recorder.rb new file mode 100644 index 0000000..e0b5a59 --- /dev/null +++ b/test/aws-xray-sdk/tc_recorder.rb @@ -0,0 +1,143 @@ +require_relative '../test_helper' +require 'aws-xray-sdk/recorder' +require 'aws-xray-sdk/exceptions' + +# Test global X-Ray recorder +class TestRecorder < Minitest::Test + @@recorder = XRay::Recorder.new + @@recorder.config.sampling = false + @@recorder.config.emitter = XRay::TestHelper::StubbedEmitter.new + + def setup + @@recorder.context.clear! + @@recorder.emitter.clear + end + + def teardown + @@recorder.context.clear! + @@recorder.emitter.clear + end + + def test_get_segment + segment = @@recorder.begin_segment name: name + assert_equal segment, @@recorder.current_segment + assert segment.sampled + end + + def test_get_subsegment + segment = @@recorder.begin_segment name: name + subsegment = @@recorder.begin_subsegment name: name + assert_equal segment, @@recorder.current_segment + assert_equal subsegment, @@recorder.current_subsegment + assert segment.sampled + end + + def test_send_segment + segment = @@recorder.begin_segment name: name + @@recorder.end_segment + assert_equal segment, @@recorder.emitter.entities[0] + assert_raises XRay::ContextMissingError do + @@recorder.current_segment + end + end + + def test_subsegment_capture + segment = @@recorder.begin_segment name + at = { k: 'v' } + @@recorder.capture('compute') do |subsegment| + subsegment.annotations.update at + end + subsegment = segment.subsegments[0] + assert_equal at, subsegment.annotations.to_h + assert subsegment.closed? + end + + def test_nested_subsegments + @@recorder.begin_segment name + subsegment1 = @@recorder.begin_subsegment name + @@recorder.begin_subsegment name + @@recorder.begin_subsegment name + @@recorder.end_subsegment + @@recorder.end_subsegment + assert_equal subsegment1, @@recorder.current_subsegment + end + + def test_subsegments_streaming + threshold = @@recorder.streamer.stream_threshold + segment = @@recorder.begin_segment name + (threshold + 1).times do |i| + @@recorder.begin_subsegment i.to_s + end + (threshold + 1).times do + @@recorder.end_subsegment + end + assert_equal 2, @@recorder.emitter.entities.count + # segment should not be prematurely sent in this case + refute @@recorder.emitter.entities.include?(segment) + @@recorder.end_segment + + assert_equal 3, @@recorder.emitter.entities.count + assert_equal segment, @@recorder.emitter.entities[2] + end + + def test_sampled_block + @@recorder.begin_segment name + x = 0 + @@recorder.sampled? do + x = 1 + end + assert_equal 1, x + assert @@recorder.sampled? + end + + def test_add_annotation + segment = @@recorder.begin_segment name + @@recorder.annotations[:k] = 1 + assert_equal 1, segment.annotations.to_h[:k] + + subsegment = @@recorder.begin_subsegment name, segment: segment + @@recorder.annotations[:k2] = 2 + assert_equal 2, subsegment.annotations.to_h[:k2] + refute segment.annotations.to_h.key?(:k2) + end + + def test_add_metadata + segment = @@recorder.begin_segment name + @@recorder.metadata[:k] = 1 + assert_equal 1, segment.to_h[:metadata][:default][:k] + subsegment = @@recorder.begin_subsegment name, segment: segment + @@recorder.metadata(namespace: :ns)[:k] = 1 + @@recorder.metadata(namespace: :ns).update k2: 2 + assert_equal 1, subsegment.to_h[:metadata][:ns][:k] + assert_equal 2, subsegment.to_h[:metadata][:ns][:k2] + end + + def test_thread_infection + segment = @@recorder.begin_segment name + thread = Thread.new { + @@recorder.inject_context segment do + @@recorder.begin_subsegment 'my_sub' + @@recorder.end_subsegment + end + } + thread.join + @@recorder.end_segment + + sent_entity = @@recorder.emitter.entities[0] + assert_equal segment, sent_entity + subsegment = sent_entity.subsegments[0] + assert_equal 'my_sub', subsegment.name + assert_equal segment.id, subsegment.parent.id + end + + def test_context_missing_passthrough + recorder = XRay::Recorder.new + recorder.config.context_missing = 'LOG_ERROR' + recorder.annotations[:k] = 1 + recorder.sampled? do + recorder.annotations.update k2: 2 + recorder.metadata.update k3: 3 + end + recorder.metadata[:foo] = 'bar' + end +end diff --git a/test/aws-xray-sdk/tc_sampling.rb b/test/aws-xray-sdk/tc_sampling.rb new file mode 100644 index 0000000..b874316 --- /dev/null +++ b/test/aws-xray-sdk/tc_sampling.rb @@ -0,0 +1,115 @@ +require 'aws-xray-sdk/sampling/sampling_rule' +require 'aws-xray-sdk/sampling/reservoir' +require 'aws-xray-sdk/sampling/default_sampler' +require 'aws-xray-sdk/exceptions' + +# Test sampling models and the default sampler +class TestSampling < Minitest::Test + VALID_RULE_DEF = { + fixed_target: 1, + rate: 0.5, + service_name: '*', + url_path: '*/ping', + http_method: 'PUT' + }.freeze + + def test_reservoir_pass_through + reservoir = XRay::Reservoir.new traces_per_sec: 1 + assert reservoir.take + reservoir2 = XRay::Reservoir.new + refute reservoir2.take + end + + def test_simple_single_rule + rule = XRay::SamplingRule.new rule_definition: VALID_RULE_DEF + + assert_equal 1, rule.fixed_target + assert_equal 0.5, rule.rate + assert_equal '*', rule.service_name + assert_equal '*/ping', rule.path + assert_equal 'PUT', rule.method + assert rule.reservoir.take + end + + def test_rule_request_matching + rule = XRay::SamplingRule.new rule_definition: VALID_RULE_DEF + + assert rule.applies? target_name: nil, target_path: '/ping', target_method: 'put' + assert rule.applies? target_name: 'a', target_path: nil, target_method: 'put' + assert rule.applies? target_name: 'a', target_path: '/ping', target_method: nil + assert rule.applies? target_name: 'a', target_path: '/ping', target_method: 'PUT' + refute rule.applies? target_name: 'a', target_path: '/sping', target_method: 'PUT' + end + + def test_invalid_single_rule + # missing path + rule_def1 = { + fixed_target: 1, + rate: 0.5, + service_name: '*', + http_method: 'GET' + } + assert_raises XRay::InvalidSamplingConfigError do + XRay::SamplingRule.new rule_definition: rule_def1 + end + # extra field for default rule + rule_def2 = { + fixed_target: 1, + rate: 0.5, + service_name: '*' + } + assert_raises XRay::InvalidSamplingConfigError do + XRay::SamplingRule.new rule_definition: rule_def2, default: true + end + # invalid value + rule_def3 = { + fixed_target: 1, + rate: -0.5 + } + assert_raises XRay::InvalidSamplingConfigError do + XRay::SamplingRule.new rule_definition: rule_def3, default: true + end + end + + EXAMPLE_CONFIG = { + version: 1, + rules: [ + { + description: 'Player moves.', + service_name: '*', + http_method: '*', + url_path: '*/ping', + fixed_target: 0, + rate: 0 + } + ], + default: { + fixed_target: 1, + rate: 0.1 + } + }.freeze + + def test_default_sampler + sampler = XRay::DefaultSampler.new + assert sampler.sample? + # should only has default rule + assert_equal 1, sampler.sampling_rules.count + sampler.sampling_rules = EXAMPLE_CONFIG + # now has one extra custom rule + assert_equal 2, sampler.sampling_rules.count + # don't sample health checks based on the custom rule + refute sampler.sample_request? service_name: '*', url_path: '/ping', http_method: 'GET' + end + + def test_invalid_rules_config + sampler = XRay::DefaultSampler.new + config1 = EXAMPLE_CONFIG.merge(version: nil) + assert_raises XRay::InvalidSamplingConfigError do + sampler.sampling_rules = config1 + end + config2 = EXAMPLE_CONFIG.merge(default: nil) + assert_raises XRay::InvalidSamplingConfigError do + sampler.sampling_rules = config2 + end + end +end diff --git a/test/aws-xray-sdk/tc_search_pattern.rb b/test/aws-xray-sdk/tc_search_pattern.rb new file mode 100644 index 0000000..a787fcb --- /dev/null +++ b/test/aws-xray-sdk/tc_search_pattern.rb @@ -0,0 +1,40 @@ +require 'aws-xray-sdk/search_pattern' + +# Test wildcard matching +class TestSearchPattern < Minitest::Test + def test_corner_case + assert XRay::SearchPattern.wildcard_match? pattern: '*', text: '' + assert XRay::SearchPattern.wildcard_match? pattern: '', text: '' + refute XRay::SearchPattern.wildcard_match? pattern: '', text: 'a' + refute XRay::SearchPattern.wildcard_match? pattern: '*', text: nil + refute XRay::SearchPattern.wildcard_match? pattern: nil, text: '' + refute XRay::SearchPattern.wildcard_match? pattern: nil, text: nil + end + + def test_no_special_character + pattern = 'test' + assert XRay::SearchPattern.wildcard_match? pattern: pattern, text: pattern + refute XRay::SearchPattern.wildcard_match? pattern: pattern, text: "a#{pattern}" + refute XRay::SearchPattern.wildcard_match? pattern: pattern, text: "#{pattern}a" + end + + def test_star + assert XRay::SearchPattern.wildcard_match? pattern: '*', text: 'test' + assert XRay::SearchPattern.wildcard_match? pattern: 'www.*', text: 'www.test.com' + assert XRay::SearchPattern.wildcard_match? pattern: '*.com', text: 'test.com' + assert XRay::SearchPattern.wildcard_match? pattern: 'www.*.com', text: 'www.test.com' + assert XRay::SearchPattern.wildcard_match? pattern: '*test.*', text: 'www.test.org' + end + + def test_question_mark + assert XRay::SearchPattern.wildcard_match? pattern: 'te?t', text: 'test' + assert XRay::SearchPattern.wildcard_match? pattern: '??st', text: 'test' + assert XRay::SearchPattern.wildcard_match? pattern: 'te??', text: 'test' + refute XRay::SearchPattern.wildcard_match? pattern: 'te?t', text: 'tet' + end + + def test_mixed_characters + assert XRay::SearchPattern.wildcard_match? pattern: '*test?.*', text: 'www.test3.com' + refute XRay::SearchPattern.wildcard_match? pattern: '*test?.*', text: 'www.test.com' + end +end diff --git a/test/aws-xray-sdk/tc_segment.rb b/test/aws-xray-sdk/tc_segment.rb new file mode 100644 index 0000000..ffbb284 --- /dev/null +++ b/test/aws-xray-sdk/tc_segment.rb @@ -0,0 +1,140 @@ +require 'bigdecimal' +require 'aws-xray-sdk/model/segment' +require 'aws-xray-sdk/model/subsegment' + +# Segment data model test suite +class TestSegment < Minitest::Test + def test_minimal_segment + segment = XRay::Segment.new name: name + assert_equal name, segment.name + assert segment.sampled + assert_nil segment.end_time + refute_nil segment.start_time + refute_nil segment.trace_id + refute_nil segment.id + end + + def test_minimal_json + segment = XRay::Segment.new name: name + segment.close + json = segment.to_json + h = eval(json) + refute_nil h[:trace_id] + refute_nil h[:id] + refute_nil h[:start_time] + refute_nil h[:end_time] + refute h.key?(:error) + refute h.key?(:throttle) + refute h.key?(:fault) + refute h.key?(:cause) + refute h.key?(:metadata) + refute h.key?(:annotations) + refute h.key?(:user) + refute h.key?(:parent_id) + refute h.key?(:in_progress) + refute h.key?(:subsegments) + refute h.key?(:http) + refute h.key?(:aws) + end + + def test_apply_status_code + segment1 = XRay::Segment.new name: name + segment1.apply_status_code status: 200 + refute segment1.fault + refute segment1.error + refute segment1.throttle + assert_equal 200, segment1.http_response[:status] + + segment2 = XRay::Segment.new name: name + segment2.apply_status_code status: 500 + assert segment2.fault + refute segment2.error + refute segment2.throttle + + segment3 = XRay::Segment.new name: name + segment3.apply_status_code status: 400 + assert segment3.error + refute segment3.fault + refute segment3.throttle + + segment4 = XRay::Segment.new name: name + segment4.apply_status_code status: 429 + assert segment4.error + assert segment4.throttle + refute segment4.fault + end + + def test_annotations + segment = XRay::Segment.new name: name + segment.annotations.update key1: 'value', key2: 2 + assert_equal segment.annotations[:key1], 'value' + assert_equal segment.annotations[:key2], 2 + + segment.annotations[:key2] = 3 + assert_equal segment.annotations.to_h, { key1: 'value' }.merge(key2: 3) + + # annotation key contains invalid character should be dropped. + at3 = { 福: true } + segment.annotations.update at3 + refute segment.annotations.to_h.key?(:福) + + # annotation value with unsupported type should be dropped. + segment.annotations[:key3] = {} + refute segment.annotations.to_h.key?(:key3) + end + + def test_numeric_annotation_value + segment = XRay::Segment.new name: name + annotations = { + k1: Rational(1 / 2), + k2: BigDecimal(1), + k3: 1 / 0.0, # Infinity + k4: 0 / 0.0, # NaN + } + segment.annotations.update annotations + h = eval(segment.to_json) + at_h = h[:annotations] + assert_equal 'Infinity', at_h[:k3] + assert_equal 'NaN', at_h[:k4] + end + + def test_add_subsegment + segment = XRay::Segment.new name: name + subsegment = XRay::Subsegment.new name: name, segment: segment + segment.add_subsegment subsegment: subsegment + assert_equal segment.ref_counter, 1 + assert_equal segment.subsegment_size, 1 + assert_equal segment.subsegments.count, 1 + assert_equal segment.subsegments[0], subsegment + + subsegment.close + assert_equal segment.ref_counter, 0 + refute segment.ready_to_send? + end + + def test_remove_subsegment + segment = XRay::Segment.new name: name + subsegment = XRay::Subsegment.new name: name, segment: segment + segment.add_subsegment subsegment: subsegment + segment.remove_subsegment subsegment: subsegment + assert segment.subsegments.empty? + assert_equal segment.subsegment_size, 0 + end + + def test_mutate_closed + segment = XRay::Segment.new name: name + segment.close + assert_raises XRay::EntityClosedError do + segment.close + end + + assert_raises XRay::EntityClosedError do + segment.annotations[:k] = 1 + end + + assert_raises XRay::EntityClosedError do + subsegment = XRay::Subsegment.new name: name, segment: segment + segment.add_subsegment subsegment: subsegment + end + end +end diff --git a/test/aws-xray-sdk/tc_streaming.rb b/test/aws-xray-sdk/tc_streaming.rb new file mode 100644 index 0000000..38a36c3 --- /dev/null +++ b/test/aws-xray-sdk/tc_streaming.rb @@ -0,0 +1,88 @@ +require_relative '../test_helper' +require 'aws-xray-sdk/model/segment' +require 'aws-xray-sdk/model/subsegment' +require 'aws-xray-sdk/streaming/default_streamer' + +# Subtree streaming test suite +class TestStreaming < Minitest::Test + + def test_segment_eligibility + segment = XRay::Segment.new name: name + subsegment = XRay::Subsegment.new name: name, segment: segment + segment.add_subsegment subsegment: subsegment + streamer = XRay::DefaultStreamer.new + streamer.stream_threshold = 1 + + refute streamer.eligible? segment: nil + assert streamer.eligible? segment: segment + end + + def test_single_subsegment + segment = XRay::Segment.new name: name + subsegment = XRay::Subsegment.new name: name, segment: segment + segment.add_subsegment subsegment: subsegment + subsegment.close + + streamer = XRay::DefaultStreamer.new + streamer.stream_threshold = 1 + emitter = XRay::TestHelper::StubbedEmitter.new + streamer.stream_subsegments root: segment, emitter: emitter + assert_equal 1, emitter.entities.count + assert_equal 0, segment.subsegment_size + end + + # all segment/subsegments has only one child subsegment. + def test_single_path + segment = XRay::Segment.new name: name + subsegment1 = XRay::Subsegment.new name: name, segment: segment + subsegment2 = XRay::Subsegment.new name: name, segment: segment + subsegment3 = XRay::Subsegment.new name: name, segment: segment + segment.add_subsegment subsegment: subsegment1 + subsegment1.add_subsegment subsegment: subsegment2 + subsegment2.add_subsegment subsegment: subsegment3 + subsegment3.close + subsegment2.close + + streamer = XRay::DefaultStreamer.new + streamer.stream_threshold = 1 + emitter = XRay::TestHelper::StubbedEmitter.new + streamer.stream_subsegments root: segment, emitter: emitter + + streamed_out = emitter.entities + # subtree with root node subsegment2 should be streamed out + assert_equal 1, streamed_out.count + assert_equal subsegment2, streamed_out[0] + assert_equal 1, segment.subsegment_size + assert_equal subsegment1, segment.subsegments[0] + # check reference removal + refute segment.subsegments.include?(subsegment2) + end + + # root segment has two subtrees eligible to stream + def test_multi_subtrees + segment = XRay::Segment.new name: name + subsegments = [] + 4.times do + subsegments << XRay::Subsegment.new(name: name, segment: segment) + end + subsegments[0].add_subsegment subsegment: subsegments[2] + subsegments[1].add_subsegment subsegment: subsegments[3] + segment.add_subsegment subsegment: subsegments[0] + segment.add_subsegment subsegment: subsegments[1] + subsegments.each &:close + + streamer = XRay::DefaultStreamer.new + streamer.stream_threshold = 1 + emitter = XRay::TestHelper::StubbedEmitter.new + streamer.stream_subsegments root: segment, emitter: emitter + + streamed_out = emitter.entities + # subtree with root node subsegment0 and subsegment1 should be streamed out + assert_equal 2, streamed_out.count + assert streamed_out.include?(subsegments[0]) + assert streamed_out.include?(subsegments[1]) + assert_equal 0, segment.subsegment_size + # check reference removal + assert segment.subsegments.empty? + end +end diff --git a/test/aws-xray-sdk/tc_subsegment.rb b/test/aws-xray-sdk/tc_subsegment.rb new file mode 100644 index 0000000..e29e465 --- /dev/null +++ b/test/aws-xray-sdk/tc_subsegment.rb @@ -0,0 +1,56 @@ +require 'aws-xray-sdk/model/segment' +require 'aws-xray-sdk/model/subsegment' + +# Subsegment data model test suite +class TestSubsegment < Minitest::Test + def test_minimal_subsegment + segment = XRay::Segment.new name: name + subsegment = XRay::Subsegment.new name: name, segment: segment + assert_equal segment, subsegment.segment + assert subsegment.sampled + assert_nil subsegment.end_time + refute_nil subsegment.start_time + refute_nil subsegment.id + end + + def test_minimal_json + segment = XRay::Segment.new name: name + subsegment = XRay::Subsegment.new name: name, segment: segment + segment.add_subsegment subsegment: subsegment + json = segment.to_json + h = eval(json) + refute_nil h[:subsegments] + refute h[:subsegments].empty? + + sub_h = h[:subsegments][0] + assert_equal 'subsegment', sub_h[:type] + assert_equal segment.id, sub_h[:parent_id] + assert_equal segment.trace_id, sub_h[:trace_id] + assert sub_h[:in_progress] + refute_nil sub_h[:start_time] + refute sub_h.key?(:sql) + refute sub_h.key?(:end_time) + end + + def test_nested_subsegments + segment = XRay::Segment.new name: name + subseg1 = XRay::Subsegment.new name: name, segment: segment + subseg2 = XRay::Subsegment.new name: name, segment: segment + subseg3 = XRay::Subsegment.new name: name, segment: segment + subseg4 = XRay::Subsegment.new name: name, segment: segment + segment.add_subsegment subsegment: subseg1 + subseg1.add_subsegment subsegment: subseg2 + subseg1.add_subsegment subsegment: subseg3 + subseg3.add_subsegment subsegment: subseg4 + + assert_equal 4, segment.subsegment_size + assert_equal 4, segment.ref_counter + + subseg4.close + subseg3.close + assert_equal 2, segment.ref_counter + + subseg1.remove_subsegment subsegment: subseg3 + assert_equal 2, segment.subsegment_size + end +end diff --git a/test/aws-xray-sdk/tc_trace_header.rb b/test/aws-xray-sdk/tc_trace_header.rb new file mode 100644 index 0000000..e263fee --- /dev/null +++ b/test/aws-xray-sdk/tc_trace_header.rb @@ -0,0 +1,63 @@ +require 'aws-xray-sdk/model/trace_header' + +# Test TraceHeader data model +class TestTraceHeader < Minitest::Test + TRACE_ID = '1-5759e988-bd862e3fe1be46a994272793'.freeze + PARENT_ID = '53995c3f42cd8ad8'.freeze + + def test_no_sample + header = XRay::TraceHeader.new root: TRACE_ID, parent_id: PARENT_ID, sampled: nil + refute header.sampled + assert_equal TRACE_ID, header.root + assert_equal PARENT_ID, header.parent_id + assert_equal %(Root=#{TRACE_ID};Parent=#{PARENT_ID}), header.header_string + end + + def test_no_parent + header = XRay::TraceHeader.new root: TRACE_ID, parent_id: nil, sampled: 1 + assert_equal 1, header.sampled + assert_equal TRACE_ID, header.root + assert_equal %(Root=#{TRACE_ID};Sampled=1), header.header_string + end + + def test_from_full_header_str + header_str = %(Root=#{TRACE_ID};Parent=#{PARENT_ID};Sampled=0) + header = XRay::TraceHeader.from_header_string header_str: header_str + assert_equal 0, header.sampled + assert_equal TRACE_ID, header.root + assert_equal PARENT_ID, header.parent_id + end + + def test_from_partial_header_str + # missing parent_id + header_str1 = %(Root=#{TRACE_ID};Sampled=0) + header = XRay::TraceHeader.from_header_string header_str: header_str1 + assert_equal 0, header.sampled + assert_equal TRACE_ID, header.root + refute header.parent_id + + # missing sampling + header_str2 = %(Root=#{TRACE_ID};Parent=#{PARENT_ID}) + header = XRay::TraceHeader.from_header_string header_str: header_str2 + assert_equal PARENT_ID, header.parent_id + assert_equal TRACE_ID, header.root + refute header.sampled + end + + def test_invalid_header_str + header_str = 'some random header string' + header = XRay::TraceHeader.from_header_string header_str: header_str + refute header.sampled + refute header.root + refute header.parent_id + end + + def test_header_str_variant + # casing and whitespaces + header_str = %(ROOT=#{TRACE_ID}; PARENT=#{PARENT_ID}; SAMPLED=1) + header = XRay::TraceHeader.from_header_string header_str: header_str + assert_equal 1, header.sampled + assert_equal TRACE_ID, header.root + assert_equal PARENT_ID, header.parent_id + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb new file mode 100644 index 0000000..cfdc8e6 --- /dev/null +++ b/test/test_helper.rb @@ -0,0 +1,26 @@ +require 'simplecov' +SimpleCov.start + +require 'minitest/autorun' +require 'aws-xray-sdk/emitter/emitter' + +module XRay + # holds all testing needed classes and methods + module TestHelper + # Emitter for testing that holds all entity it is about to send. + class StubbedEmitter + include Emitter + + attr_reader :entities + + def send_entity(entity:) + @entities ||= [] + @entities << entity + end + + def clear + @entities = [] + end + end + end +end