Cleanup flaky tests (#2166)

* make tests less flaky

* cassettes

datadog/tests/cassettes/TestAccDatadogMonitorDatasource.freeze

@@ -1 +1 @@
-2023-06-16T12:00:53.754-04:00
+2023-10-31T14:17:04.187542-04:00

datadog/tests/cassettes/TestAccDatadogMonitor_ComposeWithSyntheticsTest.freeze

@@ -1 +1 @@
-2023-06-16T11:56:44.238284-04:00
+2023-10-31T14:16:21.62046-04:00

datadog/tests/cassettes/TestAccDatadogMonitorsDatasource.freeze

@@ -1 +1 @@
-2023-06-16T12:01:14.985149-04:00
+2023-10-31T14:16:39.088182-04:00

datadog/tests/cassettes/TestAccDatadogSyntheticsTestMultistepApi_Basic.freeze

@@ -1 +1 @@
-2023-08-17T10:19:08.476225+02:00
+2023-10-31T14:16:01.698388-04:00

datadog/tests/data_source_datadog_synthetics_global_variable_test.go

@@ -19,7 +19,7 @@ func TestAccDatadogSyntheticsGlobalVariable(t *testing.T) {
 	resource.Test(t, resource.TestCase{
 		PreCheck:          func() { testAccPreCheck(t) },
 		ProviderFactories: accProviders,
-		CheckDestroy:      testSyntheticsResourceIsDestroyed(accProvider),
+		CheckDestroy:      testSyntheticsGlobalVariableResourceIsDestroyed(accProvider),
 		Steps: []resource.TestStep{
 			{
 				Config: testAccCheckDatadogSyntheticsGlobalVariableConfig(uniq),

datadog/tests/data_source_datadog_synthetics_test_test.go

@@ -19,7 +19,7 @@ func TestAccDatadogSyntheticsTest(t *testing.T) {
 	resource.Test(t, resource.TestCase{
 		PreCheck:          func() { testAccPreCheck(t) },
 		ProviderFactories: accProviders,
-		CheckDestroy:      testSyntheticsResourceIsDestroyed(accProvider),
+		CheckDestroy:      testSyntheticsTestIsDestroyed(accProvider),
 		Steps: []resource.TestStep{
 			{
 				Config: testAccCheckDatadogSyntheticsTestConfig(uniq),
@@ -38,7 +38,7 @@ func TestAccDatadogSyntheticsTestWithUrl(t *testing.T) {
 	resource.Test(t, resource.TestCase{
 		PreCheck:          func() { testAccPreCheck(t) },
 		ProviderFactories: accProviders,
-		CheckDestroy:      testSyntheticsResourceIsDestroyed(accProvider),
+		CheckDestroy:      testSyntheticsTestIsDestroyed(accProvider),
 		Steps: []resource.TestStep{
 			{
 				Config: testAccCheckDatadogSyntheticsTestConfigWithUrl(uniq),

datadog/tests/resource_datadog_dashboard_list_test.go

@@ -203,7 +203,7 @@ func testAccCheckDatadogDashListDestroyWithFw(accProvider *fwprovider.FrameworkP
 
 func datadogDashListDestroyHelper(ctx context.Context, s *terraform.State, apiInstances *utils.ApiInstances) error {
 	for _, r := range s.RootModule().Resources {
-		if !strings.Contains(r.Primary.Attributes["name"], "List") {
+		if r.Type != "datadog_dashboard_list" {
 			continue
 		}
 		id, _ := strconv.Atoi(r.Primary.ID)

datadog/tests/resource_datadog_monitor_test.go

@@ -837,7 +837,7 @@ func testAccCheckDatadogMonitorDestroy(accProvider func() (*schema.Provider, err
 		apiInstances := providerConf.DatadogApiInstances
 		auth := providerConf.Auth
 
-		if err := destroyHelper(auth, s, apiInstances); err != nil {
+		if err := destroyMonitorHelper(auth, s, apiInstances); err != nil {
 			return err
 		}
 		return nil
@@ -1568,9 +1568,13 @@ resource "datadog_monitor" "foo" {
 }`, uniq)
 }
 
-func destroyHelper(ctx context.Context, s *terraform.State, apiInstances *utils.ApiInstances) error {
-	err := utils.Retry(2, 10, func() error {
-		for _, r := range s.RootModule().Resources {
+func destroyMonitorHelper(ctx context.Context, s *terraform.State, apiInstances *utils.ApiInstances) error {
+	for _, r := range s.RootModule().Resources {
+		if r.Type != "datadog_monitor" {
+			continue
+		}
+
+		err := utils.Retry(2, 10, func() error {
 			i, _ := strconv.ParseInt(r.Primary.ID, 10, 64)
 			_, httpresp, err := apiInstances.GetMonitorsApiV1().GetMonitor(ctx, i)
 			if err != nil {
@@ -1580,10 +1584,14 @@ func destroyHelper(ctx context.Context, s *terraform.State, apiInstances *utils.
 				return &utils.RetryableError{Prob: fmt.Sprintf("received an error retrieving Monitor %s", err)}
 			}
 			return &utils.RetryableError{Prob: "Monitor still exists"}
+		})
+
+		if err != nil {
+			return err
 		}
-		return nil
-	})
-	return err
+	}
+
+	return nil
 }
 
 func existsHelper(ctx context.Context, s *terraform.State, apiInstances *utils.ApiInstances) error {

datadog/tests/resource_datadog_service_level_objective_test.go

@@ -262,21 +262,28 @@ func testAccCheckDatadogServiceLevelObjectiveDestroy(accProvider func() (*schema
 }
 
 func destroyServiceLevelObjectiveHelper(ctx context.Context, s *terraform.State, apiInstances *utils.ApiInstances) error {
-	err := utils.Retry(2, 5, func() error {
-		for _, r := range s.RootModule().Resources {
-			if r.Primary.ID != "" && r.Type == "datadog_service_level_objective" {
-				if _, httpResp, err := apiInstances.GetServiceLevelObjectivesApiV1().GetSLO(ctx, r.Primary.ID); err != nil {
-					if httpResp != nil && httpResp.StatusCode == 404 {
-						continue
-					}
-					return &utils.FatalError{Prob: fmt.Sprintf("received an error retrieving service level objective %s", err)}
+	for _, r := range s.RootModule().Resources {
+		if r.Type != "datadog_service_level_objective" {
+			continue
+		}
+
+		err := utils.Retry(2, 5, func() error {
+			if _, httpResp, err := apiInstances.GetServiceLevelObjectivesApiV1().GetSLO(ctx, r.Primary.ID); err != nil {
+				if httpResp != nil && httpResp.StatusCode == 404 {
+					return nil
 				}
-				return &utils.RetryableError{Prob: "service Level Objective still exists"}
+				return &utils.FatalError{Prob: fmt.Sprintf("received an error retrieving service level objective %s", err)}
 			}
+			return &utils.RetryableError{Prob: "service Level Objective still exists"}
+		})
+
+		if err != nil {
+			return nil
 		}
-		return nil
-	})
-	return err
+
+	}
+
+	return nil
 }
 
 func existsServiceLevelObjectiveHelper(ctx context.Context, s *terraform.State, apiInstances *utils.ApiInstances) error {

datadog/tests/resource_datadog_synthetics_global_variable_test.go

@@ -27,7 +27,7 @@ func TestAccDatadogSyntheticsGlobalVariable_importBasic(t *testing.T) {
 	resource.Test(t, resource.TestCase{
 		PreCheck:          func() { testAccPreCheck(t) },
 		ProviderFactories: accProviders,
-		CheckDestroy:      testSyntheticsResourceIsDestroyed(accProvider),
+		CheckDestroy:      testSyntheticsGlobalVariableResourceIsDestroyed(accProvider),
 		Steps: []resource.TestStep{
 			{
 				Config: createSyntheticsGlobalVariableConfig(variableName, roleName),
@@ -49,7 +49,7 @@ func TestAccDatadogSyntheticsGlobalVariable_Basic(t *testing.T) {
 	resource.Test(t, resource.TestCase{
 		PreCheck:          func() { testAccPreCheck(t) },
 		ProviderFactories: accProviders,
-		CheckDestroy:      testSyntheticsResourceIsDestroyed(accProvider),
+		CheckDestroy:      testSyntheticsGlobalVariableResourceIsDestroyed(accProvider),
 		Steps: []resource.TestStep{
 			createSyntheticsGlobalVariableStep(ctx, accProvider, t),
 		},
@@ -64,7 +64,7 @@ func TestAccDatadogSyntheticsGlobalVariableSecure_Basic(t *testing.T) {
 	resource.Test(t, resource.TestCase{
 		PreCheck:          func() { testAccPreCheck(t) },
 		ProviderFactories: accProviders,
-		CheckDestroy:      testSyntheticsResourceIsDestroyed(accProvider),
+		CheckDestroy:      testSyntheticsGlobalVariableResourceIsDestroyed(accProvider),
 		Steps: []resource.TestStep{
 			createSyntheticsGlobalVariableSecureStep(ctx, accProvider, t),
 		},
@@ -79,7 +79,7 @@ func TestAccDatadogSyntheticsGlobalVariable_Updated(t *testing.T) {
 	resource.Test(t, resource.TestCase{
 		PreCheck:          func() { testAccPreCheck(t) },
 		ProviderFactories: accProviders,
-		CheckDestroy:      testSyntheticsResourceIsDestroyed(accProvider),
+		CheckDestroy:      testSyntheticsGlobalVariableResourceIsDestroyed(accProvider),
 		Steps: []resource.TestStep{
 			createSyntheticsGlobalVariableStep(ctx, accProvider, t),
 			updateSyntheticsGlobalVariableStep(ctx, accProvider, t),
@@ -95,7 +95,7 @@ func TestAccDatadogSyntheticsGlobalVariableSecure_Updated(t *testing.T) {
 	resource.Test(t, resource.TestCase{
 		PreCheck:          func() { testAccPreCheck(t) },
 		ProviderFactories: accProviders,
-		CheckDestroy:      testSyntheticsResourceIsDestroyed(accProvider),
+		CheckDestroy:      testSyntheticsGlobalVariableResourceIsDestroyed(accProvider),
 		Steps: []resource.TestStep{
 			createSyntheticsGlobalVariableSecureStep(ctx, accProvider, t),
 			updateSyntheticsGlobalVariableSecureStep(ctx, accProvider, t),
@@ -111,7 +111,7 @@ func TestAccDatadogSyntheticsGlobalVariableFromTest_Basic(t *testing.T) {
 	resource.Test(t, resource.TestCase{
 		PreCheck:          func() { testAccPreCheck(t) },
 		ProviderFactories: accProviders,
-		CheckDestroy:      testSyntheticsResourceIsDestroyed(accProvider),
+		CheckDestroy:      testSyntheticsGlobalVariableResourceIsDestroyed(accProvider),
 		Steps: []resource.TestStep{
 			createSyntheticsGlobalVariableFromTestStep(ctx, accProvider, t),
 		},
@@ -126,7 +126,7 @@ func TestAccDatadogSyntheticsGlobalVariableFromTest_LocalVariable(t *testing.T)
 	resource.Test(t, resource.TestCase{
 		PreCheck:          func() { testAccPreCheck(t) },
 		ProviderFactories: accProviders,
-		CheckDestroy:      testSyntheticsResourceIsDestroyed(accProvider),
+		CheckDestroy:      testSyntheticsGlobalVariableResourceIsDestroyed(accProvider),
 		Steps: []resource.TestStep{
 			createSyntheticsGlobalVariableFromTestLocalVariableStep(ctx, accProvider, t),
 		},
@@ -494,12 +494,6 @@ func testSyntheticsResourceExists(accProvider func() (*schema.Provider, error))
 		auth := providerConf.Auth
 
 		for _, r := range s.RootModule().Resources {
-			if r.Type == "datadog_synthetics_test" {
-				if _, _, err := apiInstances.GetSyntheticsApiV1().GetTest(auth, r.Primary.ID); err != nil {
-					return fmt.Errorf("received an error retrieving synthetics test %s", err)
-				}
-			}
-
 			if r.Type == "datadog_synthetics_global_variable" {
 				if _, _, err := apiInstances.GetSyntheticsApiV1().GetGlobalVariable(auth, r.Primary.ID); err != nil {
 					return fmt.Errorf("received an error retrieving synthetics global variable %s", err)
@@ -510,28 +504,18 @@ func testSyntheticsResourceExists(accProvider func() (*schema.Provider, error))
 	}
 }
 
-func testSyntheticsResourceIsDestroyed(accProvider func() (*schema.Provider, error)) resource.TestCheckFunc {
+func testSyntheticsGlobalVariableResourceIsDestroyed(accProvider func() (*schema.Provider, error)) resource.TestCheckFunc {
 	return func(s *terraform.State) error {
 		provider, _ := accProvider()
 		providerConf := provider.Meta().(*datadog.ProviderConfiguration)
 		apiInstances := providerConf.DatadogApiInstances
 		auth := providerConf.Auth
 
 		for _, r := range s.RootModule().Resources {
-			if r.Type == "datadog_role" {
+			if r.Type != "datadog_synthetics_global_variable" {
 				continue
 			}
 
-			if r.Type == "datadog_synthetics_test" {
-				if _, _, err := apiInstances.GetSyntheticsApiV1().GetTest(auth, r.Primary.ID); err != nil {
-					if strings.Contains(err.Error(), "404 Not Found") {
-						continue
-					}
-					return fmt.Errorf("received an error retrieving synthetics test %s", err)
-				}
-				return fmt.Errorf("synthetics test still exists")
-			}
-
 			if _, _, err := apiInstances.GetSyntheticsApiV1().GetGlobalVariable(auth, r.Primary.ID); err != nil {
 				if strings.Contains(err.Error(), "404 Not Found") {
 					continue

datadog/tests/resource_datadog_synthetics_test_test.go

@@ -516,7 +516,7 @@ func TestAccDatadogSyntheticsTestMultistepApi_Basic(t *testing.T) {
 	resource.Test(t, resource.TestCase{
 		PreCheck:          func() { testAccPreCheck(t) },
 		ProviderFactories: accProviders,
-		CheckDestroy:      testSyntheticsResourceIsDestroyed(accProvider),
+		CheckDestroy:      testSyntheticsTestIsDestroyed(accProvider),
 		Steps: []resource.TestStep{
 			createSyntheticsMultistepAPITest(ctx, accProvider, t),
 		},
@@ -4457,6 +4457,10 @@ func testSyntheticsTestIsDestroyed(accProvider func() (*schema.Provider, error))
 		auth := providerConf.Auth
 
 		for _, r := range s.RootModule().Resources {
+			if r.Type != "datadog_synthetics_test" {
+				continue
+			}
+
 			if _, _, err := apiInstances.GetSyntheticsApiV1().GetTest(auth, r.Primary.ID); err != nil {
 				if strings.Contains(err.Error(), "404 Not Found") {
 					continue

datadog/tests/resource_datadog_team_link_test.go

@@ -105,22 +105,22 @@ func testAccCheckDatadogTeamLinkDestroy(accProvider *fwprovider.FrameworkProvide
 		apiInstances := accProvider.DatadogApiInstances
 		auth := accProvider.Auth
 
-		if err := TeamLinkDestroyHelper(auth, s, apiInstances); err != nil {
+		if err := teamLinkDestroyHelper(auth, s, apiInstances); err != nil {
 			return err
 		}
 		return nil
 	}
 }
 
-func TeamLinkDestroyHelper(auth context.Context, s *terraform.State, apiInstances *utils.ApiInstances) error {
-	err := utils.Retry(2, 10, func() error {
-		for _, r := range s.RootModule().Resources {
-			if r.Type != "resource_datadog_team_link" {
-				continue
-			}
-			teamId := r.Primary.Attributes["team_id"]
-			id := r.Primary.ID
+func teamLinkDestroyHelper(auth context.Context, s *terraform.State, apiInstances *utils.ApiInstances) error {
+	for _, r := range s.RootModule().Resources {
+		if r.Type != "resource_datadog_team_link" {
+			continue
+		}
+		teamId := r.Primary.Attributes["team_id"]
+		id := r.Primary.ID
 
+		err := utils.Retry(2, 10, func() error {
 			_, httpResp, err := apiInstances.GetTeamsApiV2().GetTeamLink(auth, teamId, id)
 			if err != nil {
 				if httpResp != nil && httpResp.StatusCode == 404 {
@@ -129,10 +129,12 @@ func TeamLinkDestroyHelper(auth context.Context, s *terraform.State, apiInstance
 				return &utils.RetryableError{Prob: fmt.Sprintf("received an error retrieving TeamLink %s", err)}
 			}
 			return &utils.RetryableError{Prob: "TeamLink still exists"}
+		})
+		if err != nil {
+			return nil
 		}
-		return nil
-	})
-	return err
+	}
+	return nil
 }
 
 func testAccCheckDatadogTeamLinkExists(accProvider *fwprovider.FrameworkProvider) resource.TestCheckFunc {

datadog/tests/resource_datadog_team_membership_test.go

@@ -62,20 +62,21 @@ func testAccCheckDatadogTeamMembershipDestroy(accProvider *fwprovider.FrameworkP
 		apiInstances := accProvider.DatadogApiInstances
 		auth := accProvider.Auth
 
-		if err := TeamMembershipDestroyHelper(auth, s, apiInstances); err != nil {
+		if err := teamMembershipDestroyHelper(auth, s, apiInstances); err != nil {
 			return err
 		}
 		return nil
 	}
 }
 
-func TeamMembershipDestroyHelper(auth context.Context, s *terraform.State, apiInstances *utils.ApiInstances) error {
-	err := utils.Retry(2, 10, func() error {
-		for _, r := range s.RootModule().Resources {
-			if r.Type != "resource_datadog_team_membership" {
-				continue
-			}
-			teamId := r.Primary.ID
+func teamMembershipDestroyHelper(auth context.Context, s *terraform.State, apiInstances *utils.ApiInstances) error {
+	for _, r := range s.RootModule().Resources {
+		if r.Type != "resource_datadog_team_membership" {
+			continue
+		}
+		teamId := r.Primary.ID
+
+		err := utils.Retry(2, 10, func() error {
 			r, httpResp, err := apiInstances.GetTeamsApiV2().GetTeamMemberships(auth, teamId)
 			if err != nil {
 				if httpResp != nil && httpResp.StatusCode == 404 {
@@ -88,10 +89,13 @@ func TeamMembershipDestroyHelper(auth context.Context, s *terraform.State, apiIn
 					return &utils.RetryableError{Prob: "TeamMembership still exists"}
 				}
 			}
+			return nil
+		})
+		if err != nil {
+			return err
 		}
-		return nil
-	})
-	return err
+	}
+	return nil
 }
 
 func testAccCheckDatadogTeamMembershipExists(accProvider *fwprovider.FrameworkProvider) resource.TestCheckFunc {

datadog/tests/resource_datadog_team_test.go

@@ -81,13 +81,13 @@ func testAccCheckDatadogTeamDestroy(accProvider *fwprovider.FrameworkProvider) f
 }
 
 func TeamDestroyHelper(auth context.Context, s *terraform.State, apiInstances *utils.ApiInstances) error {
-	err := utils.Retry(2, 10, func() error {
-		for _, r := range s.RootModule().Resources {
-			if r.Type != "resource_datadog_team" {
-				continue
-			}
-			id := r.Primary.ID
+	for _, r := range s.RootModule().Resources {
+		if r.Type != "resource_datadog_team" {
+			continue
+		}
+		id := r.Primary.ID
 
+		err := utils.Retry(2, 10, func() error {
 			_, httpResp, err := apiInstances.GetTeamsApiV2().GetTeam(auth, id)
 			if err != nil {
 				if httpResp != nil && httpResp.StatusCode == 404 {
@@ -96,10 +96,13 @@ func TeamDestroyHelper(auth context.Context, s *terraform.State, apiInstances *u
 				return &utils.RetryableError{Prob: fmt.Sprintf("received an error retrieving Team %s", err)}
 			}
 			return &utils.RetryableError{Prob: "Team still exists"}
+		})
+
+		if err != nil {
+			return err
 		}
-		return nil
-	})
-	return err
+	}
+	return nil
 }
 
 func testAccCheckDatadogTeamExists(accProvider *fwprovider.FrameworkProvider) resource.TestCheckFunc {