Respect request limits when cleaning EC2

Fix errors like: ``` 1) 22-Jan-2024 12:02:54 12:02:54,188 WARN {} Failed to terminate 1633 instances 22-Jan-2024 12:02:54 com.amazonaws.services.ec2.model.AmazonEC2Exception: You have exceeded the number of resources allowed in a single call of this type (Service: AmazonEC2; Status Code: 400; Error Code: ResourceCountExceeded; Request ID: b09c4c14-5f5f-4570-a4b6-a04762db0a5b; Proxy: null) 22-Jan-2024 12:02:54 at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1811) ~[aws-java-sdk-core-1.11.817.jar:?] 22-Jan-2024 12:02:54 at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1395) ~[aws-java-sdk-core-1.11.817.jar:?] 22-Jan-2024 12:02:54 at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1371) ~[aws-java-sdk-core-1.11.817.jar:?] 22-Jan-2024 12:02:54 at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.817.jar:?] 22-Jan-2024 12:02:54 at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.817.jar:?] 22-Jan-2024 12:02:54 at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.817.jar:?] 22-Jan-2024 12:02:54 at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.817.jar:?] 22-Jan-2024 12:02:54 at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.817.jar:?] 22-Jan-2024 12:02:54 at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.817.jar:?] 22-Jan-2024 12:02:54 at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.817.jar:?] 22-Jan-2024 12:02:54 at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.817.jar:?] 22-Jan-2024 12:02:54 at com.amazonaws.services.ec2.AmazonEC2Client.doInvoke(AmazonEC2Client.java:25451) ~[aws-java-sdk-ec2-1.11.817.jar:?] 22-Jan-2024 12:02:54 at com.amazonaws.services.ec2.AmazonEC2Client.invoke(AmazonEC2Client.java:25418) ~[aws-java-sdk-ec2-1.11.817.jar:?] 22-Jan-2024 12:02:54 at com.amazonaws.services.ec2.AmazonEC2Client.invoke(AmazonEC2Client.java:25407) ~[aws-java-sdk-ec2-1.11.817.jar:?] 22-Jan-2024 12:02:54 at com.amazonaws.services.ec2.AmazonEC2Client.executeTerminateInstances(AmazonEC2Client.java:24995) ~[aws-java-sdk-ec2-1.11.817.jar:?] 22-Jan-2024 12:02:54 at com.amazonaws.services.ec2.AmazonEC2Client.terminateInstances(AmazonEC2Client.java:24966) ~[aws-java-sdk-ec2-1.11.817.jar:?] 22-Jan-2024 12:02:54 at com.atlassian.performance.tools.aws.api.TerminationBatchingEc2.terminate(TerminationBatchingEc2.kt:60) ~[aws-resources-1.14.0.jar:?] 22-Jan-2024 12:02:54 at com.atlassian.performance.tools.aws.api.TerminationBatchingEc2.tryToTerminate(TerminationBatchingEc2.kt:46) ~[aws-resources-1.14.0.jar:?] 22-Jan-2024 12:02:54 at com.atlassian.performance.tools.aws.api.TerminationBatchingEc2.access$tryToTerminate(TerminationBatchingEc2.kt:18) ~[aws-resources-1.14.0.jar:?] 22-Jan-2024 12:02:54 at com.atlassian.performance.tools.aws.api.TerminationBatchingEc2$$special$$inlined$timer$1.run(Timer.kt:149) ~[aws-resources-1.14.0.jar:?] 22-Jan-2024 12:02:54 at java.util.TimerThread.mainLoop(Timer.java:556) ~[?:?] 22-Jan-2024 12:02:54 at java.util.TimerThread.run(Timer.java:506) ~[?:?] 2) 2024-01-22T12:54:37,605Z WARN ec2-polling {} [] [com.atlassian.performance.tools.aws.api.TerminationPollingEc2] Failed to poll instances com.amazonaws.services.ec2.model.AmazonEC2Exception: The maximum number of filter values specified on a single call is 200 (Service: AmazonEC2; Status Code: 400; Error Code: FilterLimitExceeded; Request ID: 39d3fdde-d440-4106-b28f-bf76c4a0dcf9; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1811) ~[aws-java-sdk-core-1.11.817.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1395) ~[aws-java-sdk-core-1.11.817.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1371) ~[aws-java-sdk-core-1.11.817.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.817.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.817.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.817.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.817.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.817.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.817.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.817.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.817.jar:?] at com.amazonaws.services.ec2.AmazonEC2Client.doInvoke(AmazonEC2Client.java:25451) ~[aws-java-sdk-ec2-1.11.817.jar:?] at com.amazonaws.services.ec2.AmazonEC2Client.invoke(AmazonEC2Client.java:25418) ~[aws-java-sdk-ec2-1.11.817.jar:?] at com.amazonaws.services.ec2.AmazonEC2Client.invoke(AmazonEC2Client.java:25407) ~[aws-java-sdk-ec2-1.11.817.jar:?] at com.amazonaws.services.ec2.AmazonEC2Client.executeDescribeInstances(AmazonEC2Client.java:12022) ~[aws-java-sdk-ec2-1.11.817.jar:?] at com.amazonaws.services.ec2.AmazonEC2Client.describeInstances(AmazonEC2Client.java:11993) ~[aws-java-sdk-ec2-1.11.817.jar:?] at com.atlassian.performance.tools.aws.TokenScrollingEc2.scrollThroughInstances(TokenScrollingEc2.kt:22) ~[main/:?] at com.atlassian.performance.tools.aws.api.TerminationPollingEc2.pollUntilTermination(TerminationPollingEc2.kt:64) ~[main/:?] at com.atlassian.performance.tools.aws.api.TerminationPollingEc2.tryPolling(TerminationPollingEc2.kt:49) [main/:?] at com.atlassian.performance.tools.aws.api.TerminationPollingEc2.access$tryPolling(TerminationPollingEc2.kt:19) [main/:?] at com.atlassian.performance.tools.aws.api.TerminationPollingEc2$$special$$inlined$timer$1.run(Timer.kt:149) [main/:?] at java.util.TimerThread.mainLoop(Timer.java:556) [?:?] at java.util.TimerThread.run(Timer.java:506) [?:?] ```
atlassian-labs · Jan 22, 2024 · 831e7bb · 831e7bb
1 parent a47234c
commit 831e7bb
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 6 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -25,6 +25,7 @@ Dropping a requirement of a major version of a dependency is a new contract.
 
 ### Fixed
 - Tag security groups atomically when created.
+- Respect AWS request size limits in `TerminationBatchingEc2` and `TerminationPollingEc2`.
 
 ## [1.14.0] - 2024-01-04
 [1.14.0]: https://github.com/atlassian-labs/aws-resources/compare/release-1.13.0...release-1.14.0

diff --git a/src/main/kotlin/com/atlassian/performance/tools/aws/api/TerminationBatchingEc2.kt b/src/main/kotlin/com/atlassian/performance/tools/aws/api/TerminationBatchingEc2.kt
@@ -55,7 +55,7 @@ class TerminationBatchingEc2(
             logger.trace("No instances to terminate")
             return
         }
-        val instanceIds = terminations.keys
+        val instanceIds = terminations.keys.take(1000)
         logger.debug("Starting batch termination of $instanceIds")
         ec2.terminateInstances(TerminateInstancesRequest().withInstanceIds(instanceIds))
         instanceIds.forEach { instanceId ->
@@ -64,4 +64,4 @@ class TerminationBatchingEc2(
                 .thenAccept { terminatedId -> terminations.remove(terminatedId)?.complete(terminatedId) }
         }
     }
-}
+}
diff --git a/src/main/kotlin/com/atlassian/performance/tools/aws/api/TerminationPollingEc2.kt b/src/main/kotlin/com/atlassian/performance/tools/aws/api/TerminationPollingEc2.kt
@@ -58,9 +58,9 @@ class TerminationPollingEc2(
             logger.debug("No instances to poll")
             return
         }
-        val instanceIds = polls.keys.toList()
+        val instanceIds = polls.keys.take(200)
         logger.debug("Polling $instanceIds")
-        val foundInstanceIds = mutableListOf<String>()
+        val foundInstanceIds = mutableSetOf<String>()
         scrollingEc2.scrollThroughInstances(
             Filter("instance-id", instanceIds)
         ) { instanceBatch ->
@@ -83,4 +83,4 @@ class TerminationPollingEc2(
             .remove(instanceId)
             ?.complete(instanceId)
     }
-}
+}
diff --git a/...ain/kotlin/com/atlassian/performance/tools/aws/api/housekeeping/ConcurrentHousekeeping.kt b/...ain/kotlin/com/atlassian/performance/tools/aws/api/housekeeping/ConcurrentHousekeeping.kt
@@ -33,7 +33,7 @@ class ConcurrentHousekeeping(
         val securityGroups = aws.ec2.describeSecurityGroups().securityGroups.map { securityGroup ->
             Ec2SecurityGroup(securityGroup, aws.ec2)
         }.filter { it.isExpired() }
-        waitUntilReleased(securityGroups)
+        waitUntilReleased(securityGroups, Duration.ofMinutes(3))
 
         Cloudformation(aws, aws.cloudformation).consumeExpiredStacks(Consumer { stacks ->
             waitUntilReleased(stacks, stackTimeout)