feat!: Flexible checksum v2

AWSSDKSwiftCLI/Sources/AWSSDKSwiftCLI/Resources/Package.Base.txt

@@ -103,7 +103,8 @@ private var runtimeTargets: [Target] {
                 .smithyEventStreamsAuthAPI,
                 .awsSDKCommon,
                 .awsSDKHTTPAuth,
-                .awsSDKIdentity
+                .awsSDKIdentity,
+                .awsSDKChecksums,
             ],
             path: "Sources/Core/AWSClientRuntime/Sources/AWSClientRuntime",
             resources: [

Sources/Core/AWSClientRuntime/Sources/AWSClientRuntime/AWSClientConfigDefaultsProvider.swift

@@ -18,6 +18,7 @@ import enum ClientRuntime.ClientLogMode
 import struct SmithyRetries.DefaultRetryStrategy
 import struct SmithyRetries.ExponentialBackoffStrategy
 import struct SmithyRetriesAPI.RetryStrategyOptions
+import enum AWSSDKChecksums.AWSChecksumCalculationMode
 
 typealias RuntimeConfigType = DefaultSDKRuntimeConfiguration<DefaultRetryStrategy, DefaultRetryErrorInfoProvider>
 
@@ -84,6 +85,40 @@ public class AWSClientConfigDefaultsProvider {
         return resolvedAppID
     }
 
+    public static func requestChecksumCalculation(
+        _ requestChecksumCalculation: AWSChecksumCalculationMode? = nil
+    ) throws -> AWSChecksumCalculationMode {
+        let fileBasedConfig = try CRTFileBasedConfiguration.make()
+        let resolvedRequestChecksumCalculation: AWSChecksumCalculationMode
+        if let requestChecksumCalculation {
+            resolvedRequestChecksumCalculation = requestChecksumCalculation
+        } else {
+            resolvedRequestChecksumCalculation = AWSChecksumsConfig.requestChecksumCalculation(
+                configValue: nil,
+                profileName: nil,
+                fileBasedConfig: fileBasedConfig
+            )
+        }
+        return resolvedRequestChecksumCalculation
+    }
+
+    public static func responseChecksumValidation(
+        _ responseChecksumValidation: AWSChecksumCalculationMode? = nil
+    ) throws -> AWSChecksumCalculationMode {
+        let fileBasedConfig = try CRTFileBasedConfiguration.make()
+        let resolvedResponseChecksumValidation: AWSChecksumCalculationMode
+        if let responseChecksumValidation {
+            resolvedResponseChecksumValidation = responseChecksumValidation
+        } else {
+            resolvedResponseChecksumValidation = AWSChecksumsConfig.responseChecksumValidation(
+                configValue: nil,
+                profileName: nil,
+                fileBasedConfig: fileBasedConfig
+            )
+        }
+        return resolvedResponseChecksumValidation
+    }
+
     public static func retryMode(_ retryMode: AWSRetryMode? = nil) throws -> AWSRetryMode {
         let fileBasedConfig = try CRTFileBasedConfiguration.make()
         let resolvedRetryMode: AWSRetryMode?

Sources/Core/AWSClientRuntime/Sources/AWSClientRuntime/Config/AWSChecksumsConfig.swift

@@ -0,0 +1,41 @@
+//
+// Copyright Amazon.com Inc. or its affiliates.
+// All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+import enum AWSSDKChecksums.AWSChecksumCalculationMode
+@_spi(FileBasedConfig) import AWSSDKCommon
+
+public enum AWSChecksumsConfig {
+    static func requestChecksumCalculation(
+        configValue: AWSChecksumCalculationMode?,
+        profileName: String?,
+        fileBasedConfig: FileBasedConfiguration
+    ) -> AWSChecksumCalculationMode {
+        return FieldResolver(
+            configValue: configValue,
+            envVarName: "AWS_REQUEST_CHECKSUM_CALCULATION",
+            configFieldName: "request_checksum_calculation",
+            fileBasedConfig: fileBasedConfig,
+            profileName: profileName,
+            converter: { AWSChecksumCalculationMode(caseInsensitiveRawValue: $0) }
+        ).value ?? .whenSupported
+    }
+
+    static func responseChecksumValidation(
+        configValue: AWSChecksumCalculationMode?,
+        profileName: String?,
+        fileBasedConfig: FileBasedConfiguration
+    ) -> AWSChecksumCalculationMode {
+        return FieldResolver(
+            configValue: configValue,
+            envVarName: "AWS_RESPONSE_CHECKSUM_VALIDATION",
+            configFieldName: "response_checksum_validation",
+            fileBasedConfig: fileBasedConfig,
+            profileName: profileName,
+            converter: { AWSChecksumCalculationMode(caseInsensitiveRawValue: $0) }
+        ).value ?? .whenSupported
+    }
+}

Sources/Core/AWSClientRuntime/Sources/AWSClientRuntime/Config/AWSDefaultClientConfiguration.swift

@@ -7,6 +7,7 @@
 
 import SmithyIdentity
 import SmithyIdentityAPI
+import enum AWSSDKChecksums.AWSChecksumCalculationMode
 
 public protocol AWSDefaultClientConfiguration {
     /// The AWS credential identity resolver to be used for AWS credentials.
@@ -45,4 +46,22 @@ public protocol AWSDefaultClientConfiguration {
     ///
     /// If set, this value gets used when resolving max attempts value from the standard progression of potential sources. If no value could be resolved, the SDK uses max attempts value of 3 by default.
     var maxAttempts: Int? { get set }
+
+    /// The AWS request checksum calculation mode to use.
+    ///
+    /// If `.whenRequired`, the client calculates checksum for the request payload only if the operation requires it.
+    /// If `.whenSupported`, the client calculates checksum for the request payload if the operation supports it.
+    ///
+    /// Default mode is `.whenSupported`.
+    ///
+    /// If no algorithm was chosen and no checksum was provided, CRC32 checksum algorithm is used by default.
+    var requestChecksumCalculation: AWSChecksumCalculationMode { get set }
+
+    /// The AWS response checksum calculation mode to use.
+    ///
+    /// If `.whenRequired`, the client validates checksum of the response only if the top-level input field for `requestValidationModeMember` is set to `.enabled` and SDK supports the checksum algorithm.
+    /// If `.whenSupported`, the client validates checksum of the response if the operation supports it and SDK supports at least one of the checksum algorithms returend by service.
+    ///
+    /// Default mode is `.whenSupported`.
+    var responseChecksumValidation: AWSChecksumCalculationMode { get set }
 }

Sources/Core/AWSClientRuntime/Sources/AWSClientRuntime/Middlewares/FlexibleChecksumsRequestMiddleware.swift

@@ -8,7 +8,9 @@
 import enum SmithyChecksumsAPI.ChecksumAlgorithm
 import enum SmithyChecksums.ChecksumMismatchException
 import enum Smithy.ClientError
+import struct Smithy.URIQueryItem
 import class Smithy.Context
+import struct Foundation.Data
 import AwsCommonRuntimeKit
 import AWSSDKChecksums
 import ClientRuntime
@@ -18,10 +20,18 @@
 
     public let id: String = "FlexibleChecksumsRequestMiddleware"
 
+    let requestChecksumRequired: Bool
     let checksumAlgorithm: String?
-
-    public init(checksumAlgorithm: String?) {
+    let checksumAlgoHeaderName: String?
+
+    public init(
+        requestChecksumRequired: Bool,
+        checksumAlgorithm: String?,
+        checksumAlgoHeaderName: String?
+    ) {
+        self.requestChecksumRequired = requestChecksumRequired
         self.checksumAlgorithm = checksumAlgorithm
+        self.checksumAlgoHeaderName = checksumAlgoHeaderName
     }
 
     private func addHeaders(builder: HTTPRequestBuilder, attributes: Context) async throws {
@@ -32,62 +42,127 @@
             }
         }
 
-        // Initialize logger
-        guard let logger = attributes.getLogger() else {
-            throw ClientError.unknownError("No logger found!")
-        }
+        let checksumHeaderPrefix = "x-amz-checksum-"
 
-        guard let checksumString = checksumAlgorithm else {
-            logger.info("No checksum provided! Skipping flexible checksums workflow...")
+        // If it's a PRESIGN_URL flow, add checksum headers present in the request to the request's query items. If only the algo header was configured in input, it's up to user to add the checksum hash query item along with the matching body when they send request using the presigned URL. If the checksum hash header was configured in input, it's up to user to provide a body whose checksum matches the value of the checksum hash header when they send request using the presigned URL.
+        if attributes.getFlowType() == .PRESIGN_URL {
+            let checksumAlgoHeader = builder.headers.headers.first(where: {
+                $0.name.lowercased() == checksumAlgoHeaderName?.lowercased()
+            })
+            if let checksumAlgoHeader {
+                builder.withQueryItem(URIQueryItem(
+                    name: checksumAlgoHeader.name,
+                    value: checksumAlgoHeader.value.first
+                ))
+            }
+            let checksumHeader = builder.headers.headers.first(where: {
+                $0.name.lowercased().starts(with: checksumHeaderPrefix) &&
+                $0.name.lowercased() != checksumAlgoHeaderName?.lowercased()
+            })
+            if let checksumHeader {
+                builder.withQueryItem(URIQueryItem(
+                    name: checksumHeader.name,
+                    value: checksumHeader.value.first
+                ))
+            }
+            // Skip default request checksum calculation logic.
             return
         }
 
-        guard let checksumHashFunction = ChecksumAlgorithm.from(string: checksumString) else {
-            logger.info("Found no supported checksums! Skipping flexible checksums workflow...")
-            return
+        // Initialize logger
+        guard let logger = attributes.getLogger() else {
+            throw ClientError.unknownError("No logger found!")
         }
 
-        // Determine the header name
-        let headerName = "x-amz-checksum-\(checksumHashFunction)"
-        logger.debug("Resolved checksum header name: \(headerName)")
-
         // Check if any checksum header is already provided by the user
-        let checksumHeaderPrefix = "x-amz-checksum-"
         if builder.headers.headers.contains(where: {
             $0.name.lowercased().starts(with: checksumHeaderPrefix) &&
-            $0.name.lowercased() != "x-amz-checksum-algorithm"
+            $0.name.lowercased() != checksumAlgoHeaderName?.lowercased()
         }) {
             logger.debug("Checksum header already provided by the user. Skipping calculation.")
             return
         }
 
+        var checksumHashFunction: ChecksumAlgorithm
+        if let checksumAlgorithm {
+            if let hashFunction = ChecksumAlgorithm.from(string: checksumAlgorithm) {
+                // If user chose supported algorithm, continue
+                checksumHashFunction = hashFunction
+            } else {
+                // If user chose unsupported algorithm, throw error
+                throw ClientError.invalidValue("Error: Checksum algorithm \(checksumAlgorithm) is not supported!")
+            }
+        } else {
+            // If user didn't choose an algorithm, then:
+            if requestChecksumRequired || (attributes.requestChecksumCalculation == .whenSupported) {
+                // If requestChecksumRequired == true OR RequestChecksumCalculation == when_supported, use CRC32 as default algorithm.
+                checksumHashFunction = ChecksumAlgorithm.from(string: "crc32")!
+                logger.info("No algorithm chosen by user. Defaulting to CRC32 checksum algorithm.")
+                // If the member specified by `requestAlgorithmMember` has `httpHeader` property set in model,
+                //   set the header specified in `httpHeader` with SDK's default algorithm: crc32
+                // TODO: UNCOMMENT AFTER FEATURE RELEASE
+//                if let checksumAlgoHeaderName {
+//                    builder.updateHeader(name: checksumAlgoHeaderName, value: "crc32")
+//                }
+            } else {
+                // If requestChecksumRequired == false AND RequestChecksumCalculation == when_required, skip calculation.
+                logger.info("Checksum not required for the operation.")
+                logger.info("Client config `requestChecksumCalculation` set to `.whenRequired`")
+                logger.info("No checksum algorithm chosen by the user. Skipping checksum calculation...")
+                return
+            }
+        }
+
+        // Save resolved ChecksumAlgorithm to interceptor context.
+        attributes.checksum = checksumHashFunction
+
+        // Determine the checksum header name
+        let checksumHashHeaderName = "x-amz-checksum-\(checksumHashFunction)"
+        logger.debug("Resolved checksum header name: \(checksumHashHeaderName)")
+
         // Handle body vs handle stream
         switch builder.body {
         case .data(let data):
-            guard let data else {
-                throw ClientError.dataNotFound("Cannot calculate checksum of empty body!")
+            try await calculateAndAddChecksumHeader(data: data)
+        case .stream(let stream):
+            if stream.isEligibleForChunkedStreaming {
+                // Handle calculating and adding checksum header in ChunkedStream
+                builder.updateHeader(name: "x-amz-trailer", value: [checksumHashHeaderName])
+            } else {
+                // If not eligible for chunked streaming, calculate and add checksum to request header now
+                //   instead of in a trailing header.
+                let streamBytes: Data?
+                let currentPosition = stream.position
+                if stream.isSeekable {
+                    // Explicit seek to beginning for correct behavior of FileHandle
+                    try stream.seek(toOffset: 0)
+                    streamBytes = try await stream.readToEnd()
+                    try stream.seek(toOffset: currentPosition)
+                } else {
+                    streamBytes = try await stream.readToEndAsync()
+                    builder.withBody(.data(streamBytes))
+                }
+                try await calculateAndAddChecksumHeader(data: streamBytes)
             }
+        case .noStream:
+            logger.info("Request body is empty. Skipping request checksum calculation...")
+        }
 
-            if builder.headers.value(for: headerName) == nil {
+        func calculateAndAddChecksumHeader(data: Data?) async throws {
+            guard let data else {
+                logger.info("Request body is empty. Skipping request checksum calculation...")
+                return
+            }
+            if builder.headers.value(for: checksumHashHeaderName) == nil {
                 logger.debug("Calculating checksum")
             }
-
             // Create checksum instance
             let checksum = checksumHashFunction.createChecksum()
-
             // Pass data to hash
             try checksum.update(chunk: data)
-
             // Retrieve the hash
             let hash = try checksum.digest().toBase64String()
-
-            builder.updateHeader(name: headerName, value: [hash])
-        case .stream:
-            // Will handle calculating checksum and setting header later
-            attributes.checksum = checksumHashFunction
-            builder.updateHeader(name: "x-amz-trailer", value: [headerName])
-        case .noStream:
-            throw ClientError.dataNotFound("Cannot calculate the checksum of an empty body!")
+            builder.updateHeader(name: checksumHashHeaderName, value: [hash])
         }
     }
 }

Sources/Core/AWSClientRuntime/Sources/AWSClientRuntime/Middlewares/FlexibleChecksumsResponseMiddleware.swift

@@ -15,14 +15,15 @@ public struct FlexibleChecksumsResponseMiddleware<OperationStackInput, Operation
     let CHECKSUM_HEADER_VALIDATION_PRIORITY_LIST: [String] = [
         ChecksumAlgorithm.crc32c,
         .crc32,
+        .crc64nvme,
         .sha1,
         .sha256
     ].sorted().map { $0.toString() }
 
-    let validationMode: Bool
+    let validationMode: String
     let priorityList: [String]
 
-    public init(validationMode: Bool, priorityList: [String] = []) {
+    public init(validationMode: String, priorityList: [String] = []) {
         self.validationMode = validationMode
         self.priorityList = !priorityList.isEmpty
             ? withPriority(checksums: priorityList)
@@ -31,13 +32,15 @@ public struct FlexibleChecksumsResponseMiddleware<OperationStackInput, Operation
 
     private func validateChecksum(response: HTTPResponse, logger: any LogAgent, attributes: Context) async throws {
         // Exit if validation should not be performed
-        if !validationMode {
+        if validationMode != "ENABLED" && attributes.responseChecksumValidation == .whenRequired {
             logger.info("Checksum validation should not be performed! Skipping workflow...")
             return
         }
 
         let checksumHeaderIsPresent = priorityList.first {
-            response.headers.value(for: "x-amz-checksum-\($0)") != nil
+            response.headers.value(for: "x-amz-checksum-\($0)") != nil  &&
+            // Checksum of checksums has "-#" at the end and should be ignored.
+            !(response.headers.value(for: "x-amz-checksum-\($0)")!.hasSuffix("-#"))
         }
 
         guard let checksumHeader = checksumHeaderIsPresent else {
@@ -65,7 +68,8 @@ public struct FlexibleChecksumsResponseMiddleware<OperationStackInput, Operation
         switch response.body {
         case .data(let data):
             guard let data else {
-                throw ClientError.dataNotFound("Cannot calculate checksum of empty body!")
+                logger.info("Response body is empty. Skipping response checksum validation...")
+                return
             }
 
             let responseChecksumHasher = responseChecksum.createChecksum()
@@ -87,7 +91,8 @@ public struct FlexibleChecksumsResponseMiddleware<OperationStackInput, Operation
             attributes.httpResponse = response
             attributes.httpResponse?.body = validatingStream
         case .noStream:
-            throw ClientError.dataNotFound("Cannot calculate the checksum of an empty body!")
+            logger.info("Response body is empty. Skipping response checksum validation...")
+            return
         }
     }
 }