feat: Improvements to the message decryption process (2.x) (#213)

See GHSA-89v2-g37m-g3ff

CHANGELOG.rst

@@ -2,6 +2,15 @@
 Changelog
 *********
 
+2.2.0 -- 2021-05-27
+===================
+
+Features
+--------
+* Improvements to the message decryption process
+
+  See https://github.com/aws/aws-encryption-sdk-cli/security/advisories/GHSA-89v2-g37m-g3ff.
+
 2.1.0 -- 2020-10-27
 ===================
 

README.rst

@@ -43,7 +43,7 @@ Required Prerequisites
 ======================
 
 * Python 2.7+ or 3.4+
-* aws-encryption-sdk >= 2.0.0
+* aws-encryption-sdk >= 2.2.0
 
 Installation
 ============
@@ -168,7 +168,7 @@ Metadata Contents
 `````````````````
 The metadata JSON contains the following fields:
 
-* ``"mode"`` : ``"encrypt"``/``"decrypt"``
+* ``"mode"`` : ``"encrypt"``/``"decrypt"``/``"decrypt-unsigned"``
 * ``"input"`` : Full path to input file (or ``"<stdin>"`` if stdin)
 * ``"output"`` : Full path to output file (or ``"<stdout>"`` if stdout)
 * ``"header"`` : JSON representation of `message header data`_
@@ -342,7 +342,6 @@ Allowed parameters:
 * **max_messages_encrypted** :  Determines how long each entry can remain in the cache, beginning when it was added.
 * **max_bytes_encrypted** : Specifies the maximum number of bytes that a cached data key can encrypt.
 
-
 Logging and Verbosity
 ---------------------
 The ``-v`` argument allows you to tune the verbosity of the built-in logging to your desired level.

requirements.txt

@@ -1,4 +1,4 @@
 base64io>=1.0.1
-aws-encryption-sdk~=2.0
+aws-encryption-sdk~=2.2
 setuptools
 attrs>=17.1.0

src/aws_encryption_sdk_cli/__init__.py

@@ -185,6 +185,8 @@ def process_cli_request(stream_args, parsed_args):  # noqa: C901
         required_encryption_context=parsed_args.encryption_context,
         required_encryption_context_keys=parsed_args.required_encryption_context_keys,
         commitment_policy=commitment_policy,
+        buffer_output=parsed_args.buffer,
+        max_encrypted_data_keys=parsed_args.max_encrypted_data_keys,
     )
 
     if parsed_args.input == "-":

src/aws_encryption_sdk_cli/internal/arg_parsing.py

@@ -192,6 +192,14 @@ def _build_parser():
         "-d", "--decrypt", dest="action", action="store_const", const="decrypt", help="Decrypt data"
     )
     parser.add_dummy_redirect_argument("--decrypt")
+    operating_action.add_argument(
+        "--decrypt-unsigned",
+        dest="action",
+        action="store_const",
+        const="decrypt-unsigned",
+        help="Decrypt data and enforce messages are unsigned during decryption.",
+    )
+    parser.add_dummy_redirect_argument("--decrypt-unsigned")
 
     # For each argument added to this group, a dummy redirect argument must
     # be added to the parent parser for each long form option string.
@@ -258,6 +266,10 @@ def _build_parser():
         ),
     )
 
+    parser.add_argument(
+        "-b", "--buffer", action="store_true", help="Buffer result in memory before releasing to output"
+    )
+
     parser.add_argument(
         "-i",
         "--input",
@@ -315,6 +327,13 @@ def _build_parser():
         ),
     )
 
+    parser.add_argument(
+        "--max-encrypted-data-keys",
+        type=int,
+        action=UniqueStoreAction,
+        help="Maximum number of encrypted data keys to wrap (during encryption) or to unwrap (during decryption)",
+    )
+
     parser.add_argument(
         "--suffix",
         nargs="?",

src/aws_encryption_sdk_cli/internal/identifiers.py

@@ -31,10 +31,14 @@
     "DEFAULT_MASTER_KEY_PROVIDER",
     "OperationResult",
 )
-__version__ = "2.1.0"  # type: str
+__version__ = "2.2.0"  # type: str
 
 #: Suffix added to output files if specific output filename is not specified.
-OUTPUT_SUFFIX = {"encrypt": ".encrypted", "decrypt": ".decrypted"}  # type: Dict[str, str]
+OUTPUT_SUFFIX = {
+    "encrypt": ".encrypted",
+    "decrypt": ".decrypted",
+    "decrypt-unsigned": ".decrypted",
+}  # type: Dict[str, str]
 
 ALGORITHM_NAMES = {
     alg for alg in dir(aws_encryption_sdk.Algorithm) if not alg.startswith("_")

src/aws_encryption_sdk_cli/internal/io_handling.py

@@ -24,6 +24,7 @@
 from aws_encryption_sdk.materials_managers import CommitmentPolicy  # noqa pylint: disable=unused-import
 from base64io import Base64IO
 
+from aws_encryption_sdk_cli.exceptions import BadUserArgumentError
 from aws_encryption_sdk_cli.internal.identifiers import OUTPUT_SUFFIX, OperationResult
 from aws_encryption_sdk_cli.internal.logging_utils import LOGGER_NAME
 from aws_encryption_sdk_cli.internal.metadata import MetadataWriter, json_ready_header, json_ready_header_auth
@@ -142,6 +143,21 @@ def _output_dir(source_root, destination_root, source_dir):
     return os.path.join(destination_root, suffix)
 
 
+def _is_decrypt_mode(mode):
+    # type: (str) -> bool
+    """
+    Determines whether the provided mode does decryption
+
+    :param str filepath: Full file path to file in question
+    :rtype: bool
+    """
+    if mode in ("decrypt", "decrypt-unsigned"):
+        return True
+    if mode == "encrypt":
+        return False
+    raise BadUserArgumentError("Mode {mode} has not been implemented".format(mode=mode))
+
+
 @attr.s(hash=False, init=False)
 class IOHandler(object):
     """Common handler for all IO operations. Holds common configuration values used for all
@@ -153,6 +169,7 @@ class IOHandler(object):
     :param bool no_overwrite: Should never overwrite existing files
     :param bool decode_input: Should input be base64 decoded before operation
     :param bool encode_output: Should output be base64 encoded after operation
+    :param bool buffer_output: Should buffer entire output before releasing to destination
     :param dict required_encryption_context: Encryption context key-value pairs to require
     :param list required_encryption_context_keys: Encryption context keys to require
     """
@@ -162,12 +179,13 @@ class IOHandler(object):
     no_overwrite = attr.ib(validator=attr.validators.instance_of(bool))
     decode_input = attr.ib(validator=attr.validators.instance_of(bool))
     encode_output = attr.ib(validator=attr.validators.instance_of(bool))
+    buffer_output = attr.ib(validator=attr.validators.instance_of(bool))
     required_encryption_context = attr.ib(validator=attr.validators.instance_of(dict))
     required_encryption_context_keys = attr.ib(
         validator=attr.validators.instance_of(list)
     )  # noqa pylint: disable=invalid-name
 
-    def __init__(
+    def __init__(  # noqa pylint: disable=too-many-arguments
         self,
         metadata_writer,  # type: MetadataWriter
         interactive,  # type: bool
@@ -177,6 +195,8 @@ def __init__(
         required_encryption_context,  # type: Dict[str, str]
         required_encryption_context_keys,  # type: List[str]
         commitment_policy,  # type: CommitmentPolicy
+        buffer_output,
+        max_encrypted_data_keys,  # type: Union[None, int]
     ):
         # type: (...) -> None
         """Workaround pending resolution of attrs/mypy interaction.
@@ -190,7 +210,11 @@ def __init__(
         self.encode_output = encode_output
         self.required_encryption_context = required_encryption_context
         self.required_encryption_context_keys = required_encryption_context_keys  # pylint: disable=invalid-name
-        self.client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=commitment_policy)
+        self.buffer_output = buffer_output
+        self.client = aws_encryption_sdk.EncryptionSDKClient(
+            commitment_policy=commitment_policy,
+            max_encrypted_data_keys=max_encrypted_data_keys,
+        )
         attr.validate(self)
 
     def _single_io_write(self, stream_args, source, destination_writer):
@@ -223,7 +247,7 @@ def _single_io_write(self, stream_args, source, destination_writer):
                 else:
                     metadata_kwargs["header_auth"] = json_ready_header_auth(header_auth)
 
-                if stream_args["mode"] == "decrypt":
+                if _is_decrypt_mode(str(stream_args["mode"])):
                     discovered_ec = handler.header.encryption_context
                     missing_keys = set(self.required_encryption_context_keys).difference(set(discovered_ec.keys()))
                     missing_pairs = set(self.required_encryption_context.items()).difference(set(discovered_ec.items()))
@@ -243,9 +267,12 @@ def _single_io_write(self, stream_args, source, destination_writer):
                         return OperationResult.FAILED_VALIDATION
 
                 metadata.write_metadata(**metadata_kwargs)
-                for chunk in handler:
-                    _destination.write(chunk)
-                    _destination.flush()
+                if self.buffer_output:
+                    _destination.write(handler.read())
+                else:
+                    for chunk in handler:
+                        _destination.write(chunk)
+                        _destination.flush()
         return OperationResult.SUCCESS
 
     def process_single_operation(self, stream_args, source, destination):

test/integration/integration_test_utils.py

@@ -72,7 +72,7 @@ def encrypt_args_template(metadata=False, caching=False, encode=False, decode=Fa
     return template
 
 
-def decrypt_args_template(metadata=False, encode=False, decode=False, discovery=True):
+def decrypt_args_template(metadata=False, encode=False, decode=False, discovery=True, buffer=False):
     template = "-d -i {source} -o {target} "
     if metadata:
         template += " {metadata}"
@@ -84,6 +84,17 @@ def decrypt_args_template(metadata=False, encode=False, decode=False, discovery=
         template += " --decode"
     if discovery:
         template += " --wrapping-keys discovery=true"
+    if buffer:
+        template += " --buffer"
+    return template
+
+
+def decrypt_unsigned_args_template(metadata=False):
+    template = "--decrypt-unsigned -i {source} -o {target} --wrapping-keys discovery=true"
+    if metadata:
+        template += " {metadata}"
+    else:
+        template += " -S"
     return template