From 7c9ddf92c53285b60b6a417ab131e0b7c68aa4ad Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Thu, 11 Jan 2024 11:55:49 -0800 Subject: [PATCH 1/7] Add AwsAttributePropagatingSpanProcessor. --- ...ws_attribute_propagating_span_processor.py | 91 +++++++++++++++++++ ...bute_propagating_span_processor_builder.py | 53 +++++++++++ 2 files changed, 144 insertions(+) create mode 100644 aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_attribute_propagating_span_processor.py create mode 100644 aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_attribute_propagating_span_processor_builder.py diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_attribute_propagating_span_processor.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_attribute_propagating_span_processor.py new file mode 100644 index 000000000..6fd29ef14 --- /dev/null +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_attribute_propagating_span_processor.py @@ -0,0 +1,91 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from typing import Callable, List, Optional + +from typing_extensions import override + +from amazon.opentelemetry.distro.aws_attribute_keys import AwsAttributeKeys +from amazon.opentelemetry.distro.aws_span_processing_util import AwsSpanProcessingUtil +from opentelemetry.context import Context +from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor +from opentelemetry.trace import SpanKind +from opentelemetry.trace.propagation import get_current_span + + +class AwsAttributePropagatingSpanProcessor(SpanProcessor): + """AwsAttributePropagatingSpanProcessor is SpanProcessor that propagates attributes from parent to child spans + + AwsAttributePropagatingSpanProcessor handles the propagation of attributes from parent spans to child spans, specified in self._attribute_keys_to_propagate. + AwsAttributePropagatingSpanProcessor also propagates configurable data from parent spans to child spans, as a new attribute specified by self._propagation_data_key. + Propagated data can be configured via the self._propagation_data_extractor. + Span data propagation only starts from local root server/consumer spans, but from there will be propagated to any descendant spans. If the span is a CONSUMER + PROCESS with the parent also a CONSUMER, it will set attribute AWS_CONSUMER_PARENT_SPAN_KIND as CONSUMER to indicate that dependency metrics should not be generated for this span. + """ + + _propagation_data_extractor: Callable[[ReadableSpan], str] + _propagation_data_key: str + _attribute_keys_to_propagate: List[str] + + def __init__( + self, + propagation_data_extractor: Callable[[ReadableSpan], str], + propagation_data_key: str, + attribute_keys_to_propagate: List[str], + ): + self._propagation_data_extractor = propagation_data_extractor + self._propagation_data_key = propagation_data_key + self._attribute_keys_to_propagate = attribute_keys_to_propagate + + @override + def on_start(self, span: Span, parent_context: Optional[Context] = None) -> None: + parent_span: ReadableSpan = get_current_span(parent_context) + if isinstance(parent_span, ReadableSpan): + # Add the AWS_SDK_DESCENDANT attribute to the immediate child spans of AWS SDK span. + # This attribute helps the backend differentiate between SDK spans and their immediate children. + # It's assumed that the HTTP spans are immediate children of the AWS SDK span + # TODO: we should have a contract test to check the immediate children are HTTP span + if AwsSpanProcessingUtil.is_aws_sdk_span(parent_span): + span.set_attribute(AwsAttributeKeys.AWS_SDK_DESCENDANT, "true") + + if SpanKind.INTERNAL == parent_span.kind: + for key_to_propagate in self._attribute_keys_to_propagate: + value_to_propagate: str = parent_span.attributes.get(key_to_propagate) + if value_to_propagate is not None: + span.set_attribute(key_to_propagate, value_to_propagate) + + # We cannot guarantee that messaging.operation is set onStart, it could be set after the fact. + # To work around this, add the AWS_CONSUMER_PARENT_SPAN_KIND attribute if parent and child are both CONSUMER + # then check later if a metric should be generated. + if self._is_consumer_kind(span) and self._is_consumer_kind(parent_span): + span.set_attribute(AwsAttributeKeys.AWS_CONSUMER_PARENT_SPAN_KIND, parent_span.kind.name) + + propagation_data: str = None + if AwsSpanProcessingUtil.is_local_root(span): + if not self._is_server_kind(span): + propagation_data = self._propagation_data_extractor(span) + elif self._is_server_kind(parent_span): + propagation_data = self._propagation_data_extractor(parent_span) + else: + propagation_data = parent_span.attributes.get(self._propagation_data_key) + + if propagation_data is not None: + span.set_attribute(self._propagation_data_key, propagation_data) + + @override + def on_end(self, span: ReadableSpan) -> None: + pass + + @override + def shutdown(self) -> None: + self.force_flush() + + # pylint: disable=no-self-use + @override + def force_flush(self, timeout_millis: int = None) -> bool: + return True + + def _is_consumer_kind(self, span: ReadableSpan) -> bool: + return SpanKind.CONSUMER == span.kind + + def _is_server_kind(self, span: ReadableSpan) -> bool: + return SpanKind.SERVER == span.kind diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_attribute_propagating_span_processor_builder.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_attribute_propagating_span_processor_builder.py new file mode 100644 index 000000000..7baa2a876 --- /dev/null +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_attribute_propagating_span_processor_builder.py @@ -0,0 +1,53 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from typing import Callable, List + +from amazon.opentelemetry.distro.aws_attribute_keys import AwsAttributeKeys +from amazon.opentelemetry.distro.aws_attribute_propagating_span_processor import AwsAttributePropagatingSpanProcessor +from amazon.opentelemetry.distro.aws_span_processing_util import AwsSpanProcessingUtil +from opentelemetry.sdk.trace import ReadableSpan + + +class AwsAttributePropagatingSpanProcessorBuilder: + """ + AwsAttributePropagatingSpanProcessorBuilder is used to construct a {@link + AwsAttributePropagatingSpanProcessor}. If {@link #set_propagation_data_extractor}, {@link#set_propagation_data_key} + or {@link #set_attributes_keys_to_propagate} are not invoked, the builder defaults to using specific propagation targets. + """ + + _propagation_data_extractor: str = AwsSpanProcessingUtil.get_ingress_operation + _propagation_data_key: str = AwsAttributeKeys.AWS_LOCAL_OPERATION + _attributes_keys_to_propagate: List[str] = [ + AwsAttributeKeys.AWS_REMOTE_SERVICE, + AwsAttributeKeys.AWS_REMOTE_OPERATION, + ] + + def __init__(self): + pass + + def set_propagation_data_extractor( + self, propagation_data_extractor: Callable[[ReadableSpan], str] + ) -> "AwsAttributePropagatingSpanProcessorBuilder": + if propagation_data_extractor is None: + raise ValueError("propagation_data_extractor must not be None") + self._propagation_data_extractor = propagation_data_extractor + return self + + def set_propagation_data_key(self, propagation_data_key: str) -> "AwsAttributePropagatingSpanProcessorBuilder": + if propagation_data_key is None: + raise ValueError("propagation_data_key must not be None") + self._propagation_data_key = propagation_data_key + return self + + def set_attributes_keys_to_propagate( + self, attributes_keys_to_propagate: List[str] + ) -> "AwsAttributePropagatingSpanProcessorBuilder": + if attributes_keys_to_propagate is None: + raise ValueError("attributes_keys_to_propagate must not be None") + self._attributes_keys_to_propagate = attributes_keys_to_propagate + return self + + def build(self) -> AwsAttributePropagatingSpanProcessor: + return AwsAttributePropagatingSpanProcessor( + self._propagation_data_extractor, self._propagation_data_key, self._attributes_keys_to_propagate + ) From fce04990d134a64614deccce3285bf8cf1699d5a Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Thu, 11 Jan 2024 15:08:02 -0800 Subject: [PATCH 2/7] Address comments. --- .../distro/_aws_attribute_keys.py | 2 +- .../distro/_aws_span_processing_util.py | 158 ++++++++++++++++++ ...> attribute_propagating_span_processor.py} | 32 ++-- ...bute_propagating_span_processor_builder.py | 50 ++++++ ...bute_propagating_span_processor_builder.py | 53 ------ ...st_attribute_propagating_span_processor.py | 11 ++ ...bute_propagating_span_processor_builder.py | 13 ++ 7 files changed, 250 insertions(+), 69 deletions(-) create mode 100644 aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_span_processing_util.py rename aws-opentelemetry-distro/src/amazon/opentelemetry/distro/{aws_attribute_propagating_span_processor.py => attribute_propagating_span_processor.py} (80%) create mode 100644 aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor_builder.py delete mode 100644 aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_attribute_propagating_span_processor_builder.py create mode 100644 aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_attribute_propagating_span_processor.py create mode 100644 aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_attribute_propagating_span_processor_builder.py diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_attribute_keys.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_attribute_keys.py index b0ada931d..0d075f6d1 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_attribute_keys.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_attribute_keys.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 -class AwsAttributeKeys: +class _AwsAttributeKeys: """Utility class holding attribute keys with special meaning to AWS components""" AWS_SPAN_KIND: str = "aws.span.kind" diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_span_processing_util.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_span_processing_util.py new file mode 100644 index 000000000..998e1d537 --- /dev/null +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_span_processing_util.py @@ -0,0 +1,158 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Utility module designed to support shared logic across AWS Span Processors.""" +from amazon.opentelemetry.distro._aws_attribute_keys import _AwsAttributeKeys +from opentelemetry.sdk.trace import InstrumentationScope, ReadableSpan +from opentelemetry.semconv.trace import MessagingOperationValues, SpanAttributes +from opentelemetry.trace import SpanKind + +# Default attribute values if no valid span attribute value is identified +UNKNOWN_SERVICE: str = "UnknownService" +UNKNOWN_OPERATION: str = "UnknownOperation" +UNKNOWN_REMOTE_SERVICE: str = "UnknownRemoteService" +UNKNOWN_REMOTE_OPERATION: str = "UnknownRemoteOperation" +INTERNAL_OPERATION: str = "InternalOperation" +LOCAL_ROOT: str = "LOCAL_ROOT" + +# Useful constants +_SQS_RECEIVE_MESSAGE_SPAN_NAME: str = "Sqs.ReceiveMessage" +_AWS_SDK_INSTRUMENTATION_SCOPE_PREFIX: str = "io.opentelemetry.aws-sdk-" + + +def get_ingress_operation(span: ReadableSpan) -> str: + """ + Ingress operation (i.e. operation for Server and Consumer spans) will be generated from "http.method + http.target/ + with the first API path parameter" if the default span name is None, UnknownOperation or http.method value. + """ + operation: str = span.name + if should_use_internal_operation(span): + operation = INTERNAL_OPERATION + elif not _is_valid_operation(span, operation): + operation = _generate_ingress_operation(span) + return operation + + +def get_egress_operation(span: ReadableSpan) -> str: + if should_use_internal_operation(span): + return INTERNAL_OPERATION + return span.attributes.get(_AwsAttributeKeys.AWS_LOCAL_OPERATION) + + +def extract_api_path_value(http_target: str) -> str: + """Extract the first part from API http target if it exists + + Args + http_target - http request target string value. Eg, /payment/1234 + Returns + the first part from the http target. Eg, /payment + :return: + """ + if http_target is None or len(http_target) == 0: + return "/" + paths: [str] = http_target.split("/") + if len(paths) > 1: + return "/" + paths[1] + return "/" + + +def is_key_present(span: ReadableSpan, key: str) -> bool: + return span.attributes.get(key) is not None + + +def is_aws_sdk_span(span: ReadableSpan) -> bool: + # https://opentelemetry.io/docs/specs/otel/trace/semantic_conventions/instrumentation/aws-sdk/#common-attributes + return "aws-api" == span.attributes.get(SpanAttributes.RPC_SYSTEM) + + +def should_generate_service_metric_attributes(span: ReadableSpan) -> bool: + return (is_local_root(span) and not _is_sqs_receive_message_consumer_span(span)) or SpanKind.SERVER == span.kind + + +def should_generate_dependency_metric_attributes(span: ReadableSpan) -> bool: + return ( + SpanKind.CLIENT == span.kind + or SpanKind.PRODUCER == span.kind + or (_is_dependency_consumer_span(span) and not _is_sqs_receive_message_consumer_span(span)) + ) + + +def is_consumer_process_span(span: ReadableSpan) -> bool: + messaging_operation: str = span.attributes.get(SpanAttributes.MESSAGING_OPERATION) + return SpanKind.CONSUMER == span.kind and MessagingOperationValues.PROCESS == messaging_operation + + +def should_use_internal_operation(span: ReadableSpan) -> bool: + """ + Any spans that are Local Roots and also not SERVER should have aws.local.operation renamed toInternalOperation. + """ + return is_local_root(span) and not SpanKind.SERVER == span.kind + + +def is_local_root(span: ReadableSpan) -> bool: + """ + A span is a local root if it has no parent or if the parent is remote. This function checks the parent context + and returns true if it is a local root. + """ + return span.parent is None or not span.parent.is_valid or span.parent.is_remote + + +def _is_sqs_receive_message_consumer_span(span: ReadableSpan) -> bool: + """To identify the SQS consumer spans produced by AWS SDK instrumentation""" + messaging_operation: str = span.attributes.get(SpanAttributes.MESSAGING_OPERATION) + instrumentation_scope: InstrumentationScope = span.instrumentation_scope + + return ( + _SQS_RECEIVE_MESSAGE_SPAN_NAME.casefold() == span.name.casefold() + and SpanKind.CONSUMER == span.kind + and instrumentation_scope is not None + and instrumentation_scope.name.startswith(_AWS_SDK_INSTRUMENTATION_SCOPE_PREFIX) + and (messaging_operation is None or messaging_operation == MessagingOperationValues.PROCESS) + ) + + +def _is_dependency_consumer_span(span: ReadableSpan) -> bool: + if SpanKind.CONSUMER != span.kind: + return False + + if is_consumer_process_span(span): + if is_local_root(span): + return True + parent_span_kind: str = span.attributes.get(_AwsAttributeKeys.AWS_CONSUMER_PARENT_SPAN_KIND) + return SpanKind.CONSUMER != parent_span_kind + + return True + + +def _is_valid_operation(span: ReadableSpan, operation: str) -> bool: + """ + When Span name is null, UnknownOperation or HttpMethod value, it will be treated as invalid local operation value + that needs to be further processed + """ + if operation is None or operation == UNKNOWN_OPERATION: + return False + + if is_key_present(span, SpanAttributes.HTTP_METHOD): + http_method: str = span.attributes.get(SpanAttributes.HTTP_METHOD) + return operation != http_method + + return True + + +def _generate_ingress_operation(span: ReadableSpan) -> str: + """ + When span name is not meaningful(null, unknown or http_method value) as operation name for http use cases. Will try + to extract the operation name from http target string + """ + operation: str = UNKNOWN_OPERATION + if is_key_present(span, SpanAttributes.HTTP_TARGET): + http_target: str = span.attributes.get(SpanAttributes.HTTP_TARGET) + # get the first part from API path string as operation value + # the more levels/parts we get from API path the higher chance for getting high cardinality data + if http_target is not None: + operation = extract_api_path_value(http_target) + if is_key_present(span, SpanAttributes.HTTP_METHOD): + http_method: str = span.attributes.get(SpanAttributes.HTTP_METHOD) + if http_method is not None: + operation = http_method + " " + operation + + return operation diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_attribute_propagating_span_processor.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py similarity index 80% rename from aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_attribute_propagating_span_processor.py rename to aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py index 6fd29ef14..b08f37c58 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_attribute_propagating_span_processor.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py @@ -4,15 +4,15 @@ from typing_extensions import override -from amazon.opentelemetry.distro.aws_attribute_keys import AwsAttributeKeys -from amazon.opentelemetry.distro.aws_span_processing_util import AwsSpanProcessingUtil +from amazon.opentelemetry.distro._aws_attribute_keys import _AwsAttributeKeys +from amazon.opentelemetry.distro._aws_span_processing_util import is_aws_sdk_span, is_local_root from opentelemetry.context import Context from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor from opentelemetry.trace import SpanKind from opentelemetry.trace.propagation import get_current_span -class AwsAttributePropagatingSpanProcessor(SpanProcessor): +class AttributePropagatingSpanProcessor(SpanProcessor): """AwsAttributePropagatingSpanProcessor is SpanProcessor that propagates attributes from parent to child spans AwsAttributePropagatingSpanProcessor handles the propagation of attributes from parent spans to child spans, specified in self._attribute_keys_to_propagate. @@ -44,8 +44,8 @@ def on_start(self, span: Span, parent_context: Optional[Context] = None) -> None # This attribute helps the backend differentiate between SDK spans and their immediate children. # It's assumed that the HTTP spans are immediate children of the AWS SDK span # TODO: we should have a contract test to check the immediate children are HTTP span - if AwsSpanProcessingUtil.is_aws_sdk_span(parent_span): - span.set_attribute(AwsAttributeKeys.AWS_SDK_DESCENDANT, "true") + if is_aws_sdk_span(parent_span): + span.set_attribute(_AwsAttributeKeys.AWS_SDK_DESCENDANT, "true") if SpanKind.INTERNAL == parent_span.kind: for key_to_propagate in self._attribute_keys_to_propagate: @@ -56,14 +56,14 @@ def on_start(self, span: Span, parent_context: Optional[Context] = None) -> None # We cannot guarantee that messaging.operation is set onStart, it could be set after the fact. # To work around this, add the AWS_CONSUMER_PARENT_SPAN_KIND attribute if parent and child are both CONSUMER # then check later if a metric should be generated. - if self._is_consumer_kind(span) and self._is_consumer_kind(parent_span): - span.set_attribute(AwsAttributeKeys.AWS_CONSUMER_PARENT_SPAN_KIND, parent_span.kind.name) + if _is_consumer_kind(span) and _is_consumer_kind(parent_span): + span.set_attribute(_AwsAttributeKeys.AWS_CONSUMER_PARENT_SPAN_KIND, parent_span.kind.name) propagation_data: str = None - if AwsSpanProcessingUtil.is_local_root(span): - if not self._is_server_kind(span): + if is_local_root(span): + if not _is_server_kind(span): propagation_data = self._propagation_data_extractor(span) - elif self._is_server_kind(parent_span): + elif _is_server_kind(parent_span): propagation_data = self._propagation_data_extractor(parent_span) else: propagation_data = parent_span.attributes.get(self._propagation_data_key) @@ -73,7 +73,7 @@ def on_start(self, span: Span, parent_context: Optional[Context] = None) -> None @override def on_end(self, span: ReadableSpan) -> None: - pass + return @override def shutdown(self) -> None: @@ -84,8 +84,10 @@ def shutdown(self) -> None: def force_flush(self, timeout_millis: int = None) -> bool: return True - def _is_consumer_kind(self, span: ReadableSpan) -> bool: - return SpanKind.CONSUMER == span.kind - def _is_server_kind(self, span: ReadableSpan) -> bool: - return SpanKind.SERVER == span.kind +def _is_consumer_kind(span: ReadableSpan) -> bool: + return SpanKind.CONSUMER == span.kind + + +def _is_server_kind(span: ReadableSpan) -> bool: + return SpanKind.SERVER == span.kind diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor_builder.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor_builder.py new file mode 100644 index 000000000..4a7c4e405 --- /dev/null +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor_builder.py @@ -0,0 +1,50 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from typing import Callable, List + +from amazon.opentelemetry.distro._aws_attribute_keys import _AwsAttributeKeys +from amazon.opentelemetry.distro._aws_span_processing_util import get_ingress_operation +from amazon.opentelemetry.distro.attribute_propagating_span_processor import AttributePropagatingSpanProcessor +from opentelemetry.sdk.trace import ReadableSpan + + +class AttributePropagatingSpanProcessorBuilder: + """ + AttributePropagatingSpanProcessorBuilder is used to construct a AttributePropagatingSpanProcessor. + If set_propagation_data_extractor, set_propagation_data_key or set_attributes_keys_to_propagate are not invoked, + the builder defaults to using specific propagation targets. + """ + + _propagation_data_extractor: str = get_ingress_operation + _propagation_data_key: str = _AwsAttributeKeys.AWS_LOCAL_OPERATION + _attributes_keys_to_propagate: List[str] = [ + _AwsAttributeKeys.AWS_REMOTE_SERVICE, + _AwsAttributeKeys.AWS_REMOTE_OPERATION, + ] + + def set_propagation_data_extractor( + self, propagation_data_extractor: Callable[[ReadableSpan], str] + ) -> "AttributePropagatingSpanProcessorBuilder": + if propagation_data_extractor is None: + raise ValueError("propagation_data_extractor must not be None") + self._propagation_data_extractor = propagation_data_extractor + return self + + def set_propagation_data_key(self, propagation_data_key: str) -> "AttributePropagatingSpanProcessorBuilder": + if propagation_data_key is None: + raise ValueError("propagation_data_key must not be None") + self._propagation_data_key = propagation_data_key + return self + + def set_attributes_keys_to_propagate( + self, attributes_keys_to_propagate: List[str] + ) -> "AttributePropagatingSpanProcessorBuilder": + if attributes_keys_to_propagate is None: + raise ValueError("attributes_keys_to_propagate must not be None") + self._attributes_keys_to_propagate = tuple(attributes_keys_to_propagate) + return self + + def build(self) -> AttributePropagatingSpanProcessor: + return AttributePropagatingSpanProcessor( + self._propagation_data_extractor, self._propagation_data_key, self._attributes_keys_to_propagate + ) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_attribute_propagating_span_processor_builder.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_attribute_propagating_span_processor_builder.py deleted file mode 100644 index 7baa2a876..000000000 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_attribute_propagating_span_processor_builder.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -from typing import Callable, List - -from amazon.opentelemetry.distro.aws_attribute_keys import AwsAttributeKeys -from amazon.opentelemetry.distro.aws_attribute_propagating_span_processor import AwsAttributePropagatingSpanProcessor -from amazon.opentelemetry.distro.aws_span_processing_util import AwsSpanProcessingUtil -from opentelemetry.sdk.trace import ReadableSpan - - -class AwsAttributePropagatingSpanProcessorBuilder: - """ - AwsAttributePropagatingSpanProcessorBuilder is used to construct a {@link - AwsAttributePropagatingSpanProcessor}. If {@link #set_propagation_data_extractor}, {@link#set_propagation_data_key} - or {@link #set_attributes_keys_to_propagate} are not invoked, the builder defaults to using specific propagation targets. - """ - - _propagation_data_extractor: str = AwsSpanProcessingUtil.get_ingress_operation - _propagation_data_key: str = AwsAttributeKeys.AWS_LOCAL_OPERATION - _attributes_keys_to_propagate: List[str] = [ - AwsAttributeKeys.AWS_REMOTE_SERVICE, - AwsAttributeKeys.AWS_REMOTE_OPERATION, - ] - - def __init__(self): - pass - - def set_propagation_data_extractor( - self, propagation_data_extractor: Callable[[ReadableSpan], str] - ) -> "AwsAttributePropagatingSpanProcessorBuilder": - if propagation_data_extractor is None: - raise ValueError("propagation_data_extractor must not be None") - self._propagation_data_extractor = propagation_data_extractor - return self - - def set_propagation_data_key(self, propagation_data_key: str) -> "AwsAttributePropagatingSpanProcessorBuilder": - if propagation_data_key is None: - raise ValueError("propagation_data_key must not be None") - self._propagation_data_key = propagation_data_key - return self - - def set_attributes_keys_to_propagate( - self, attributes_keys_to_propagate: List[str] - ) -> "AwsAttributePropagatingSpanProcessorBuilder": - if attributes_keys_to_propagate is None: - raise ValueError("attributes_keys_to_propagate must not be None") - self._attributes_keys_to_propagate = attributes_keys_to_propagate - return self - - def build(self) -> AwsAttributePropagatingSpanProcessor: - return AwsAttributePropagatingSpanProcessor( - self._propagation_data_extractor, self._propagation_data_key, self._attributes_keys_to_propagate - ) diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_attribute_propagating_span_processor.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_attribute_propagating_span_processor.py new file mode 100644 index 000000000..a29cee5a4 --- /dev/null +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_attribute_propagating_span_processor.py @@ -0,0 +1,11 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from unittest import TestCase + +from amazon.opentelemetry.distro.attribute_propagating_span_processor import AttributePropagatingSpanProcessor + + +class TestAttributePropagatingSpanProcessor(TestCase): + def test_basic(self): + processor: AttributePropagatingSpanProcessor = AttributePropagatingSpanProcessor(None, None, None) + self.assertTrue(processor.force_flush) diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_attribute_propagating_span_processor_builder.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_attribute_propagating_span_processor_builder.py new file mode 100644 index 000000000..dbd30969b --- /dev/null +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_attribute_propagating_span_processor_builder.py @@ -0,0 +1,13 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from unittest import TestCase + +from amazon.opentelemetry.distro.attribute_propagating_span_processor_builder import ( + AttributePropagatingSpanProcessorBuilder, +) + + +class TestAttributePropagatingSpanProcessorBuilder(TestCase): + def test_basic(self): + builder: AttributePropagatingSpanProcessorBuilder = AttributePropagatingSpanProcessorBuilder() + self.assertIs(builder.set_propagation_data_key("test"), builder) From 36eda0e6e71921591d5542c13bc5d6abd9e80212 Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Thu, 11 Jan 2024 15:10:12 -0800 Subject: [PATCH 3/7] Address flake8 line restriction. --- .../distro/attribute_propagating_span_processor.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py index b08f37c58..3054475d7 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py @@ -15,11 +15,14 @@ class AttributePropagatingSpanProcessor(SpanProcessor): """AwsAttributePropagatingSpanProcessor is SpanProcessor that propagates attributes from parent to child spans - AwsAttributePropagatingSpanProcessor handles the propagation of attributes from parent spans to child spans, specified in self._attribute_keys_to_propagate. - AwsAttributePropagatingSpanProcessor also propagates configurable data from parent spans to child spans, as a new attribute specified by self._propagation_data_key. + AwsAttributePropagatingSpanProcessor handles the propagation of attributes from parent spans to child spans, + specified in self._attribute_keys_to_propagate. AwsAttributePropagatingSpanProcessor also propagates + configurable data from parent spans to child spans, as a new attribute specified by self._propagation_data_key. Propagated data can be configured via the self._propagation_data_extractor. - Span data propagation only starts from local root server/consumer spans, but from there will be propagated to any descendant spans. If the span is a CONSUMER - PROCESS with the parent also a CONSUMER, it will set attribute AWS_CONSUMER_PARENT_SPAN_KIND as CONSUMER to indicate that dependency metrics should not be generated for this span. + Span data propagation only starts from local root server/consumer spans, + but from there will be propagated to any descendant spans. If the span is a CONSUMER + PROCESS with the parent also a CONSUMER, it will set attribute AWS_CONSUMER_PARENT_SPAN_KIND as CONSUMER + to indicate that dependency metrics should not be generated for this span. """ _propagation_data_extractor: Callable[[ReadableSpan], str] From 251503ee7eef1d482d381689f13bb0ce849adf77 Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Thu, 11 Jan 2024 15:14:50 -0800 Subject: [PATCH 4/7] Add # pylint: disable=no-self-use. --- .../opentelemetry/distro/attribute_propagating_span_processor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py index 3054475d7..279daa72a 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py @@ -74,6 +74,7 @@ def on_start(self, span: Span, parent_context: Optional[Context] = None) -> None if propagation_data is not None: span.set_attribute(self._propagation_data_key, propagation_data) + # pylint: disable=no-self-use @override def on_end(self, span: ReadableSpan) -> None: return From 16d64f62eafb1685819e7be8700877f3acc79097 Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Fri, 12 Jan 2024 09:31:40 -0800 Subject: [PATCH 5/7] delete. --- .../distro/_aws_span_processing_util.py | 158 ------------------ 1 file changed, 158 deletions(-) delete mode 100644 aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_span_processing_util.py diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_span_processing_util.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_span_processing_util.py deleted file mode 100644 index 998e1d537..000000000 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_span_processing_util.py +++ /dev/null @@ -1,158 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Utility module designed to support shared logic across AWS Span Processors.""" -from amazon.opentelemetry.distro._aws_attribute_keys import _AwsAttributeKeys -from opentelemetry.sdk.trace import InstrumentationScope, ReadableSpan -from opentelemetry.semconv.trace import MessagingOperationValues, SpanAttributes -from opentelemetry.trace import SpanKind - -# Default attribute values if no valid span attribute value is identified -UNKNOWN_SERVICE: str = "UnknownService" -UNKNOWN_OPERATION: str = "UnknownOperation" -UNKNOWN_REMOTE_SERVICE: str = "UnknownRemoteService" -UNKNOWN_REMOTE_OPERATION: str = "UnknownRemoteOperation" -INTERNAL_OPERATION: str = "InternalOperation" -LOCAL_ROOT: str = "LOCAL_ROOT" - -# Useful constants -_SQS_RECEIVE_MESSAGE_SPAN_NAME: str = "Sqs.ReceiveMessage" -_AWS_SDK_INSTRUMENTATION_SCOPE_PREFIX: str = "io.opentelemetry.aws-sdk-" - - -def get_ingress_operation(span: ReadableSpan) -> str: - """ - Ingress operation (i.e. operation for Server and Consumer spans) will be generated from "http.method + http.target/ - with the first API path parameter" if the default span name is None, UnknownOperation or http.method value. - """ - operation: str = span.name - if should_use_internal_operation(span): - operation = INTERNAL_OPERATION - elif not _is_valid_operation(span, operation): - operation = _generate_ingress_operation(span) - return operation - - -def get_egress_operation(span: ReadableSpan) -> str: - if should_use_internal_operation(span): - return INTERNAL_OPERATION - return span.attributes.get(_AwsAttributeKeys.AWS_LOCAL_OPERATION) - - -def extract_api_path_value(http_target: str) -> str: - """Extract the first part from API http target if it exists - - Args - http_target - http request target string value. Eg, /payment/1234 - Returns - the first part from the http target. Eg, /payment - :return: - """ - if http_target is None or len(http_target) == 0: - return "/" - paths: [str] = http_target.split("/") - if len(paths) > 1: - return "/" + paths[1] - return "/" - - -def is_key_present(span: ReadableSpan, key: str) -> bool: - return span.attributes.get(key) is not None - - -def is_aws_sdk_span(span: ReadableSpan) -> bool: - # https://opentelemetry.io/docs/specs/otel/trace/semantic_conventions/instrumentation/aws-sdk/#common-attributes - return "aws-api" == span.attributes.get(SpanAttributes.RPC_SYSTEM) - - -def should_generate_service_metric_attributes(span: ReadableSpan) -> bool: - return (is_local_root(span) and not _is_sqs_receive_message_consumer_span(span)) or SpanKind.SERVER == span.kind - - -def should_generate_dependency_metric_attributes(span: ReadableSpan) -> bool: - return ( - SpanKind.CLIENT == span.kind - or SpanKind.PRODUCER == span.kind - or (_is_dependency_consumer_span(span) and not _is_sqs_receive_message_consumer_span(span)) - ) - - -def is_consumer_process_span(span: ReadableSpan) -> bool: - messaging_operation: str = span.attributes.get(SpanAttributes.MESSAGING_OPERATION) - return SpanKind.CONSUMER == span.kind and MessagingOperationValues.PROCESS == messaging_operation - - -def should_use_internal_operation(span: ReadableSpan) -> bool: - """ - Any spans that are Local Roots and also not SERVER should have aws.local.operation renamed toInternalOperation. - """ - return is_local_root(span) and not SpanKind.SERVER == span.kind - - -def is_local_root(span: ReadableSpan) -> bool: - """ - A span is a local root if it has no parent or if the parent is remote. This function checks the parent context - and returns true if it is a local root. - """ - return span.parent is None or not span.parent.is_valid or span.parent.is_remote - - -def _is_sqs_receive_message_consumer_span(span: ReadableSpan) -> bool: - """To identify the SQS consumer spans produced by AWS SDK instrumentation""" - messaging_operation: str = span.attributes.get(SpanAttributes.MESSAGING_OPERATION) - instrumentation_scope: InstrumentationScope = span.instrumentation_scope - - return ( - _SQS_RECEIVE_MESSAGE_SPAN_NAME.casefold() == span.name.casefold() - and SpanKind.CONSUMER == span.kind - and instrumentation_scope is not None - and instrumentation_scope.name.startswith(_AWS_SDK_INSTRUMENTATION_SCOPE_PREFIX) - and (messaging_operation is None or messaging_operation == MessagingOperationValues.PROCESS) - ) - - -def _is_dependency_consumer_span(span: ReadableSpan) -> bool: - if SpanKind.CONSUMER != span.kind: - return False - - if is_consumer_process_span(span): - if is_local_root(span): - return True - parent_span_kind: str = span.attributes.get(_AwsAttributeKeys.AWS_CONSUMER_PARENT_SPAN_KIND) - return SpanKind.CONSUMER != parent_span_kind - - return True - - -def _is_valid_operation(span: ReadableSpan, operation: str) -> bool: - """ - When Span name is null, UnknownOperation or HttpMethod value, it will be treated as invalid local operation value - that needs to be further processed - """ - if operation is None or operation == UNKNOWN_OPERATION: - return False - - if is_key_present(span, SpanAttributes.HTTP_METHOD): - http_method: str = span.attributes.get(SpanAttributes.HTTP_METHOD) - return operation != http_method - - return True - - -def _generate_ingress_operation(span: ReadableSpan) -> str: - """ - When span name is not meaningful(null, unknown or http_method value) as operation name for http use cases. Will try - to extract the operation name from http target string - """ - operation: str = UNKNOWN_OPERATION - if is_key_present(span, SpanAttributes.HTTP_TARGET): - http_target: str = span.attributes.get(SpanAttributes.HTTP_TARGET) - # get the first part from API path string as operation value - # the more levels/parts we get from API path the higher chance for getting high cardinality data - if http_target is not None: - operation = extract_api_path_value(http_target) - if is_key_present(span, SpanAttributes.HTTP_METHOD): - http_method: str = span.attributes.get(SpanAttributes.HTTP_METHOD) - if http_method is not None: - operation = http_method + " " + operation - - return operation From dc2f942396e0a47fe446d7e59e87f43a0d80fbe7 Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Fri, 12 Jan 2024 09:47:00 -0800 Subject: [PATCH 6/7] Use Tuple instead of List. --- .../opentelemetry/distro/_aws_span_processing_util.py | 6 +++--- .../distro/attribute_propagating_span_processor.py | 6 +++--- .../distro/attribute_propagating_span_processor_builder.py | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_span_processing_util.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_span_processing_util.py index 00be704c5..139890b96 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_span_processing_util.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_span_processing_util.py @@ -1,7 +1,7 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 """Utility module designed to support shared logic across AWS Span Processors.""" -from amazon.opentelemetry.distro._aws_attribute_keys import AwsAttributeKeys +from amazon.opentelemetry.distro._aws_attribute_keys import _AwsAttributeKeys from opentelemetry.sdk.trace import InstrumentationScope, ReadableSpan from opentelemetry.semconv.trace import MessagingOperationValues, SpanAttributes from opentelemetry.trace import SpanKind @@ -35,7 +35,7 @@ def get_ingress_operation(__, span: ReadableSpan) -> str: def get_egress_operation(span: ReadableSpan) -> str: if should_use_internal_operation(span): return INTERNAL_OPERATION - return span.attributes.get(AwsAttributeKeys.AWS_LOCAL_OPERATION) + return span.attributes.get(_AwsAttributeKeys.AWS_LOCAL_OPERATION) def extract_api_path_value(http_target: str) -> str: @@ -117,7 +117,7 @@ def _is_dependency_consumer_span(span: ReadableSpan) -> bool: if is_consumer_process_span(span): if is_local_root(span): return True - parent_span_kind: str = span.attributes.get(AwsAttributeKeys.AWS_CONSUMER_PARENT_SPAN_KIND) + parent_span_kind: str = span.attributes.get(_AwsAttributeKeys.AWS_CONSUMER_PARENT_SPAN_KIND) return SpanKind.CONSUMER != parent_span_kind return True diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py index 279daa72a..1ed6a3912 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py @@ -1,6 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from typing import Callable, List, Optional +from typing import Callable, List, Optional, Tuple from typing_extensions import override @@ -27,13 +27,13 @@ class AttributePropagatingSpanProcessor(SpanProcessor): _propagation_data_extractor: Callable[[ReadableSpan], str] _propagation_data_key: str - _attribute_keys_to_propagate: List[str] + _attribute_keys_to_propagate: Tuple[str, ...] def __init__( self, propagation_data_extractor: Callable[[ReadableSpan], str], propagation_data_key: str, - attribute_keys_to_propagate: List[str], + attribute_keys_to_propagate: Tuple[str, ...], ): self._propagation_data_extractor = propagation_data_extractor self._propagation_data_key = propagation_data_key diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor_builder.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor_builder.py index 4a7c4e405..64e775529 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor_builder.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor_builder.py @@ -1,6 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from typing import Callable, List +from typing import Callable, List, Tuple from amazon.opentelemetry.distro._aws_attribute_keys import _AwsAttributeKeys from amazon.opentelemetry.distro._aws_span_processing_util import get_ingress_operation @@ -17,10 +17,10 @@ class AttributePropagatingSpanProcessorBuilder: _propagation_data_extractor: str = get_ingress_operation _propagation_data_key: str = _AwsAttributeKeys.AWS_LOCAL_OPERATION - _attributes_keys_to_propagate: List[str] = [ + _attributes_keys_to_propagate: Tuple[str, ...] = ( _AwsAttributeKeys.AWS_REMOTE_SERVICE, _AwsAttributeKeys.AWS_REMOTE_OPERATION, - ] + ) def set_propagation_data_extractor( self, propagation_data_extractor: Callable[[ReadableSpan], str] From 4f271f50c036e670bf5670f8f7a1a2174f546d41 Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Fri, 12 Jan 2024 09:49:48 -0800 Subject: [PATCH 7/7] Delete unused import --- .../distro/attribute_propagating_span_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py index 1ed6a3912..21b5d43cf 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/attribute_propagating_span_processor.py @@ -1,6 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from typing import Callable, List, Optional, Tuple +from typing import Callable, Optional, Tuple from typing_extensions import override