Integration tests now fail if there are log errors unless the test opts out.

cmd/state-installer/test/integration/installer_int_test.go

@@ -145,6 +145,7 @@ func (suite *InstallerIntegrationTestSuite) TestInstallIncompatible() {
 	// Assert output
 	cp.Expect("not compatible")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 }
 
 func (suite *InstallerIntegrationTestSuite) TestInstallNoErrorTips() {
@@ -164,6 +165,7 @@ func (suite *InstallerIntegrationTestSuite) TestInstallNoErrorTips() {
 
 	cp.ExpectExitCode(1)
 	suite.Assert().NotContains(cp.Output(), "Need More Help?", "error tips should not be displayed when invoking installer")
+	ts.IgnoreLogErrors()
 }
 
 func (suite *InstallerIntegrationTestSuite) TestInstallErrorTips() {

cmd/state-svc/test/integration/svc_int_test.go

@@ -43,6 +43,7 @@ func (suite *SvcIntegrationTestSuite) TestStartStop() {
 	cp = ts.SpawnCmdWithOpts(ts.SvcExe, e2e.OptArgs("status"))
 	cp.Expect("Service cannot be reached")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 
 	cp = ts.SpawnCmdWithOpts(ts.SvcExe, e2e.OptArgs("start"))
 	cp.Expect("Starting")
@@ -91,6 +92,7 @@ func (suite *SvcIntegrationTestSuite) TestSignals() {
 
 	suite.OnlyRunForTags(tagsuite.Service)
 	ts := e2e.New(suite.T(), false)
+	ts.IgnoreLogErrors()
 	defer ts.Close()
 
 	// SIGINT (^C)
@@ -145,6 +147,7 @@ func (suite *SvcIntegrationTestSuite) TestStartDuplicateErrorOutput() {
 	cp = ts.SpawnCmdWithOpts(ts.SvcExe, e2e.OptArgs("foreground"))
 	cp.Expect("An existing server instance appears to be in use")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 
 	cp = ts.SpawnCmdWithOpts(ts.SvcExe, e2e.OptArgs("stop"))
 	cp.ExpectExitCode(0)

internal/testhelpers/e2e/session.go

@@ -57,12 +57,13 @@ type Session struct {
 	retainDirs      bool
 	createdProjects []*project.Namespaced
 	// users created during session
-	users       []string
-	t           *testing.T
-	Exe         string
-	SvcExe      string
-	ExecutorExe string
-	spawned     []*SpawnedCmd
+	users           []string
+	t               *testing.T
+	Exe             string
+	SvcExe          string
+	ExecutorExe     string
+	spawned         []*SpawnedCmd
+	ignoreLogErrors bool
 }
 
 var (
@@ -638,6 +639,10 @@ func (s *Session) Close() error {
 		}
 	}
 
+	if !s.ignoreLogErrors {
+		s.detectLogErrors()
+	}
+
 	return nil
 }
 
@@ -742,12 +747,19 @@ func (s *Session) DebugLogsDump() string {
 	return result
 }
 
+// IgnoreLogErrors disables log error checking after the session closes.
+// Normally, logged errors automatically cause test failures, so calling this is needed for tests
+// with expected errors.
+func (s *Session) IgnoreLogErrors() {
+	s.ignoreLogErrors = true
+}
+
 var errorOrPanicRegex = regexp.MustCompile(`(?:\[ERR:|Panic:)`)
 
-func (s *Session) DetectLogErrors() {
+func (s *Session) detectLogErrors() {
 	for _, path := range s.LogFiles() {
 		if contents := string(fileutils.ReadFileUnsafe(path)); errorOrPanicRegex.MatchString(contents) {
-			s.t.Errorf("Found error and/or panic in log file %s, contents:\n%s", path, contents)
+			s.t.Errorf("Found error and/or panic in log file %s\nIf this was expected, call session.IgnoreLogErrors() to avoid this check\nLog contents:\n%s", path, contents)
 		}
 	}
 }

test/integration/activate_int_test.go

@@ -201,6 +201,7 @@ func (suite *ActivateIntegrationTestSuite) TestActivatePythonByHostOnly() {
 			suite.Fail("Expected exactly ONE error message, got: ", cp.Snapshot())
 		}
 	}
+	ts.IgnoreLogErrors()
 }
 
 func (suite *ActivateIntegrationTestSuite) assertCompletedStatusBarReport(snapshot string) {
@@ -553,6 +554,7 @@ func (suite *ActivateIntegrationTestSuite) TestActivateCommitURL() {
 	cp := ts.Spawn("activate")
 	cp.Expect("Cannot activate a headless project", e2e.RuntimeSourcingTimeoutOpt)
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 }
 
 func (suite *ActivateIntegrationTestSuite) TestActivate_AlreadyActive() {

test/integration/analytics_int_test.go

@@ -496,6 +496,7 @@ func (suite *AnalyticsIntegrationTestSuite) TestInputError() {
 
 	cp := ts.Spawn("clean", "uninstall", "badarg", "--mono")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 
 	events := parseAnalyticsEvents(suite, ts)
 	suite.assertSequentialEvents(events)

test/integration/auth_int_test.go

@@ -78,6 +78,7 @@ func (suite *AuthIntegrationTestSuite) loginFlags(ts *e2e.Session, username stri
 	cp := ts.Spawn(tagsuite.Auth, "--username", username, "--password", "bad-password")
 	cp.Expect("You are not authorized, did you provide valid login credentials?")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 }
 
 func (suite *AuthIntegrationTestSuite) ensureLogout(ts *e2e.Session) {

test/integration/bundle_int_test.go

@@ -83,6 +83,7 @@ func (suite *BundleIntegrationTestSuite) TestBundle_project_invalid() {
 	cp := ts.Spawn("bundles", "--namespace", "junk/junk")
 	cp.Expect("The requested project junk does not exist under junk")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 }
 
 func (suite *BundleIntegrationTestSuite) TestBundle_searchSimple() {
@@ -131,6 +132,7 @@ func (suite *BundleIntegrationTestSuite) TestBundle_searchWithExactTermWrongTerm
 	cp := ts.Spawn("bundles", "search", "xxxUtilitiesxxx", "--exact-term")
 	cp.Expect("No bundles in our catalog match")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 }
 
 func (suite *BundleIntegrationTestSuite) TestBundle_searchWithLang() {
@@ -153,6 +155,7 @@ func (suite *BundleIntegrationTestSuite) TestBundle_searchWithWrongLang() {
 	cp := ts.Spawn("bundles", "search", "Utilities", "--language=python")
 	cp.Expect("No bundles in our catalog match")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 }
 
 func (suite *BundleIntegrationTestSuite) TestBundle_searchWithBadLang() {
@@ -164,6 +167,7 @@ func (suite *BundleIntegrationTestSuite) TestBundle_searchWithBadLang() {
 	cp := ts.Spawn("bundles", "search", "Utilities", "--language=bad")
 	cp.Expect("Cannot obtain search")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 }
 
 func (suite *BundleIntegrationTestSuite) TestBundle_detached_operation() {
@@ -198,6 +202,7 @@ func (suite *BundleIntegrationTestSuite) TestBundle_detached_operation() {
 		cp := ts.Spawn("bundles", "install", "[email protected]")
 		cp.ExpectRe("(?:bundle updated|being built)")
 		cp.ExpectExitCode(1)
+		ts.IgnoreLogErrors()
 	})
 	*/
 

test/integration/checkout_int_test.go

@@ -96,6 +96,7 @@ func (suite *CheckoutIntegrationTestSuite) TestCheckoutNonEmptyDir() {
 	)
 	cp.Expect("already a project checked out at")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 
 	if strings.Count(cp.Snapshot(), " x ") != 1 {
 		suite.Fail("Expected exactly ONE error message, got: ", cp.Snapshot())
@@ -164,6 +165,7 @@ func (suite *CheckoutIntegrationTestSuite) TestCheckoutWithFlags() {
 	cp = ts.SpawnWithOpts(e2e.OptArgs("checkout", "ActiveState-CLI/Python-3.9", branchPath, "--branch", "doesNotExist"))
 	cp.Expect("This project has no branch with label matching doesNotExist")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 }
 
 func (suite *CheckoutIntegrationTestSuite) TestCheckoutCustomRTPath() {
@@ -217,6 +219,7 @@ func (suite *CheckoutIntegrationTestSuite) TestCheckoutNotFound() {
 	cp.Expect("does not exist under")         // error
 	cp.Expect("If this is a private project") // tip
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 
 	if strings.Count(cp.Snapshot(), " x ") != 1 {
 		suite.Fail("Expected exactly ONE error message, got: ", cp.Snapshot())
@@ -236,6 +239,7 @@ func (suite *CheckoutIntegrationTestSuite) TestCheckoutAlreadyCheckedOut() {
 	cp = ts.SpawnWithOpts(e2e.OptArgs("checkout", "ActiveState-CLI/small-python"))
 	cp.Expect("already a project checked out at")
 	cp.ExpectNotExitCode(0)
+	ts.IgnoreLogErrors()
 }
 
 func (suite *CheckoutIntegrationTestSuite) TestJSON() {
@@ -252,6 +256,7 @@ func (suite *CheckoutIntegrationTestSuite) TestJSON() {
 	cp.Expect(`"tips":["If this is a private project`) // tip
 	cp.ExpectNotExitCode(0)
 	AssertValidJSON(suite.T(), cp)
+	ts.IgnoreLogErrors()
 }
 
 func (suite *CheckoutIntegrationTestSuite) TestCheckoutCaseInsensitive() {

test/integration/condition_int_test.go

@@ -111,6 +111,7 @@ func (suite *ConditionIntegrationTestSuite) TestConditionSyntaxError() {
 	)
 	cp.Expect(`not defined`) // for now we aren't passing the error up the chain, so invalid syntax will lead to empty result
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 }
 
 func (suite *ConditionIntegrationTestSuite) PrepareActiveStateYAML(ts *e2e.Session) {

test/integration/config_int_test.go

@@ -21,6 +21,7 @@ func (suite *ConfigIntegrationTestSuite) TestConfig() {
 	cp := ts.Spawn("config", "set", "invalid++", "value")
 	cp.Expect("Invalid")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 
 	cp = ts.Spawn("config", "set", constants.UnstableConfig, "true")
 	cp.Expect("Successfully")

test/integration/cve_int_test.go

@@ -97,6 +97,7 @@ func (suite *CveIntegrationTestSuite) TestCveInvalidProject() {
 	cp.Expect("Found no project with specified organization and name")
 
 	cp.ExpectNotExitCode(0)
+	ts.IgnoreLogErrors()
 }
 
 func (suite *CveIntegrationTestSuite) TestJSON() {

test/integration/deploy_int_test.go

@@ -276,6 +276,7 @@ func (suite *DeployIntegrationTestSuite) TestDeployConfigure() {
 	)
 	cp.Expect("need to run the install step")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 	suite.InstallAndAssert(ts, targetPath)
 
 	if runtime.GOOS != "windows" {
@@ -352,6 +353,7 @@ func (suite *DeployIntegrationTestSuite) TestDeploySymlink() {
 	)
 	cp.Expect("need to run the install step")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 	suite.InstallAndAssert(ts, targetPath)
 
 	if runtime.GOOS != "darwin" {
@@ -393,6 +395,7 @@ func (suite *DeployIntegrationTestSuite) TestDeployReport() {
 	)
 	cp.Expect("need to run the install step")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 	suite.InstallAndAssert(ts, targetPath)
 
 	cp = ts.SpawnWithOpts(
@@ -482,6 +485,7 @@ func (suite *DeployIntegrationTestSuite) TestDeployUninstall() {
 	)
 	cp.Expect("no deployed runtime")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 	suite.True(fileutils.IsDir(ts.Dirs.Work), "Work dir was unexpectedly deleted")
 
 	// Trying to uninstall in a non-deployment directory should fail.

test/integration/errors_int_test.go

@@ -23,6 +23,7 @@ func (suite *ErrorsIntegrationTestSuite) TestTips() {
 	cp.Expect("Run →")
 	cp.Expect("Ask For Help →")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 }
 
 func (suite *ErrorsIntegrationTestSuite) TestMultiError() {
@@ -33,6 +34,7 @@ func (suite *ErrorsIntegrationTestSuite) TestMultiError() {
 	cp := ts.Spawn("__test", "multierror")
 	cp.ExpectRe(`\s+x error1\s+\s+x error2\s+x error3\s+x error4\s+█\s+Need More Help`)
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 }
 
 func TestErrorsIntegrationTestSuite(t *testing.T) {

test/integration/export_int_test.go

@@ -56,6 +56,7 @@ func (suite *ExportIntegrationTestSuite) TestExport_InvalidPlatform() {
 	ts.PrepareProject("cli-integration-tests/Export", "")
 	cp := ts.Spawn("export", "recipe", "--platform", "junk")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 }
 
 func (suite *ExportIntegrationTestSuite) TestExport_ConfigDir() {
@@ -66,6 +67,7 @@ func (suite *ExportIntegrationTestSuite) TestExport_ConfigDir() {
 	ts.PrepareProject("cli-integration-tests/Export", "")
 	cp := ts.Spawn("export", "config", "--filter", "junk")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 }
 
 func (suite *ExportIntegrationTestSuite) TestExport_Config() {

test/integration/fork_int_test.go

@@ -37,6 +37,7 @@ func (suite *ForkIntegrationTestSuite) TestFork() {
 	cp = ts.Spawn("fork", "ActiveState-CLI/Python3", "--name", "Test-Python3", "--org", username)
 	cp.Expect(`You already have a project with the name`)
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 }
 
 func (suite *ForkIntegrationTestSuite) TestFork_FailNameExists() {
@@ -48,6 +49,7 @@ func (suite *ForkIntegrationTestSuite) TestFork_FailNameExists() {
 	cp := ts.SpawnWithOpts(e2e.OptArgs("fork", "ActiveState-CLI/Python3", "--org", e2e.PersistentUsername))
 	cp.Expect("You already have a project with the name 'Python3'", termtest.OptExpectTimeout(30*time.Second))
 	cp.ExpectNotExitCode(0)
+	ts.IgnoreLogErrors()
 }
 
 func TestForkIntegrationTestSuite(t *testing.T) {

test/integration/hello_int_example_test.go

@@ -29,6 +29,7 @@ func (suite *HelloIntegrationTestSuite) TestHello() {
 	cp = ts.Spawn("_hello", "")
 	cp.Expect("Cannot say hello because no name was provided")
 	cp.ExpectNotExitCode(0)
+	ts.IgnoreLogErrors()
 
 	cp = ts.Spawn("_hello", "Person", "--extra")
 	cp.Expect("Project: ActiveState-CLI/small-python")

test/integration/import_int_test.go

@@ -86,6 +86,7 @@ func (suite *ImportIntegrationTestSuite) TestImport() {
 		cp.Expect("already exists")
 		cp.ExpectNotExitCode(0)
 	})
+	ts.IgnoreLogErrors()
 }
 
 func TestImportIntegrationTestSuite(t *testing.T) {

test/integration/info_int_test.go

@@ -44,6 +44,7 @@ func (suite *InfoIntegrationTestSuite) TestInfo_UnavailableVersion() {
 	cp := ts.Spawn("info", "[email protected]", "--language", "python")
 	cp.Expect("Could not find version 9.9.9 for package pylint")
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 }
 
 func (suite *InfoIntegrationTestSuite) TestJSON() {
@@ -62,6 +63,7 @@ func (suite *InfoIntegrationTestSuite) TestJSON() {
 	cp.Expect(`"error":`)
 	cp.ExpectExitCode(1)
 	AssertValidJSON(suite.T(), cp)
+	ts.IgnoreLogErrors()
 }
 
 func TestInfoIntegrationTestSuite(t *testing.T) {

test/integration/init_int_test.go

@@ -100,6 +100,7 @@ func (suite *InitIntegrationTestSuite) TestInit_NoLanguage() {
 
 	cp := ts.Spawn("init", "test-user/test-project")
 	cp.ExpectNotExitCode(0)
+	ts.IgnoreLogErrors()
 }
 
 func (suite *InitIntegrationTestSuite) TestInit_InferLanguageFromUse() {

test/integration/install_int_test.go

@@ -35,6 +35,7 @@ func (suite *InstallIntegrationTestSuite) TestInstall_InvalidCommit() {
 	//cp.Expect("Could not find or read the commit file") // re-enable in DX-2307
 	cp.Expect("Invalid commit ID") // remove in DX-2307
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 
 	// Re-enable in DX-2307.
 	//if strings.Count(cp.Snapshot(), " x ") != 1 {
@@ -52,6 +53,7 @@ func (suite *InstallIntegrationTestSuite) TestInstall_NoMatches_NoAlternatives()
 	cp.Expect("No results found for search term")
 	cp.Expect("find alternatives") // This verifies no alternatives were found
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 
 	if strings.Count(strings.ReplaceAll(cp.Snapshot(), " x Failed", ""), " x ") != 1 {
 		suite.Fail("Expected exactly ONE error message, got: ", cp.Snapshot())
@@ -68,6 +70,7 @@ func (suite *InstallIntegrationTestSuite) TestInstall_NoMatches_Alternatives() {
 	cp.Expect("No results found for search term")
 	cp.Expect("did you mean") // This verifies alternatives were found
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 
 	if strings.Count(strings.ReplaceAll(cp.Snapshot(), " x Failed", ""), " x ") != 1 {
 		suite.Fail("Expected exactly ONE error message, got: ", cp.Snapshot())
@@ -83,6 +86,7 @@ func (suite *InstallIntegrationTestSuite) TestInstall_BuildPlannerError() {
 	cp := ts.SpawnWithOpts(e2e.OptArgs("install", "[email protected]"), e2e.OptAppendEnv(constants.DisableRuntime+"=true"))
 	cp.Expect("Could not plan build, platform responded with", e2e.RuntimeSourcingTimeoutOpt)
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 
 	if strings.Count(strings.ReplaceAll(cp.Snapshot(), " x Failed", ""), " x ") != 1 {
 		suite.Fail("Expected exactly ONE error message, got: ", cp.Snapshot())

test/integration/install_scripts_int_test.go

@@ -185,6 +185,7 @@ func (suite *InstallScriptsIntegrationTestSuite) TestInstall_NonEmptyTarget() {
 	// because of powershell.
 	// Since we asserted the expected error above we don't need to go on a wild goose chase here.
 	cp.ExpectExit()
+	ts.IgnoreLogErrors()
 }
 
 func (suite *InstallScriptsIntegrationTestSuite) TestInstall_VersionDoesNotExist() {
@@ -206,6 +207,7 @@ func (suite *InstallScriptsIntegrationTestSuite) TestInstall_VersionDoesNotExist
 		cp.Expect("Could not download")
 	}
 	cp.ExpectExitCode(1)
+	ts.IgnoreLogErrors()
 }
 
 // scriptPath returns the path to an installation script copied to targetDir, if useTestUrl is true, the install script is modified to download from the local test server instead