From ff7d67448765ba5f102732d21b7cb7bf6b0c7643 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Wed, 21 Aug 2024 16:02:41 +0300 Subject: [PATCH 01/33] optimize test run time --- test/integration/scan_test.go | 77 +++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 35 deletions(-) diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index 67f056348..c983c9caa 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -8,7 +8,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/google/uuid" "io" "log" "os" @@ -19,6 +18,8 @@ import ( "testing" "time" + "github.com/google/uuid" + "github.com/checkmarx/ast-cli/internal/commands" realtime "github.com/checkmarx/ast-cli/internal/commands/scarealtime" "github.com/checkmarx/ast-cli/internal/commands/scarealtime/scaconfig" @@ -82,7 +83,7 @@ func TestCreateScan_WithOnlyValidApikeyFlag_Success(t *testing.T) { "scan", "create", flag(params.ProjectName), "project", flag(params.SourcesFlag), "data/insecure.zip", - flag(params.ScanTypes), "iac-security", + flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", flag(params.AstAPIKeyFlag), originals[params.AstAPIKeyEnv], } @@ -106,7 +107,7 @@ func TestCreateScan_WithOnlyValidApikeyEnvVar_Success(t *testing.T) { "scan", "create", flag(params.ProjectName), "project", flag(params.SourcesFlag), "data/insecure.zip", - flag(params.ScanTypes), "iac-security", + flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", } @@ -130,7 +131,7 @@ func TestCreateScan_WithOnlyInvalidApikeyEnvVar_Fail(t *testing.T) { "scan", "create", flag(params.ProjectName), "project", flag(params.SourcesFlag), "data/insecure.zip", - flag(params.ScanTypes), "iac-security", + flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", } @@ -154,7 +155,7 @@ func TestCreateScan_WithOnlyInvalidApikeyFlag_Fail(t *testing.T) { "scan", "create", flag(params.ProjectName), "project", flag(params.SourcesFlag), "data/insecure.zip", - flag(params.ScanTypes), "iac-security", + flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", flag(params.AstAPIKeyFlag), "invalid_apikey", } @@ -179,7 +180,7 @@ func TestCreateScan_WithValidClientCredentialsFlag_Success(t *testing.T) { "scan", "create", flag(params.ProjectName), "project", flag(params.SourcesFlag), "data/insecure.zip", - flag(params.ScanTypes), "iac-security", + flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", flag(params.AccessKeyIDFlag), originals[params.AccessKeyIDEnv], flag(params.AccessKeySecretFlag), originals[params.AccessKeySecretEnv], @@ -206,7 +207,7 @@ func TestCreateScan_WithInvalidClientCredentialsFlag_Fail(t *testing.T) { "scan", "create", flag(params.ProjectName), "project", flag(params.SourcesFlag), "data/insecure.zip", - flag(params.ScanTypes), "iac-security", + flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", flag(params.AccessKeyIDFlag), "invalid_client_ID", flag(params.AccessKeySecretFlag), "invalid_client_secret", @@ -229,7 +230,7 @@ func TestCreateScan_WithValidClientCredentialsEnvVars_Success(t *testing.T) { "scan", "create", flag(params.ProjectName), "project", flag(params.SourcesFlag), "data/insecure.zip", - flag(params.ScanTypes), "iac-security", + flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", } @@ -253,7 +254,7 @@ func TestCreateScan_WithInvalidClientCredentialsEnvVars_Fail(t *testing.T) { "scan", "create", flag(params.ProjectName), "project", flag(params.SourcesFlag), "data/insecure.zip", - flag(params.ScanTypes), "iac-security", + flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", } @@ -283,7 +284,7 @@ func TestScanCreateEmptyProjectName(t *testing.T) { "scan", "create", flag(params.ProjectName), "", flag(params.SourcesFlag), ".", - flag(params.ScanTypes), "sast", + flag(params.ScanTypes), params.SastType, flag(params.BranchFlag), "dummy_branch", } @@ -297,7 +298,7 @@ func TestScanCreate_ExistingApplicationAndExistingProject_CreateScanSuccessfully flag(params.ApplicationName), "my-application", flag(params.ProjectName), "my-project", flag(params.SourcesFlag), ".", - flag(params.ScanTypes), "sast", + flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", } @@ -310,7 +311,7 @@ func TestScanCreate_FolderWithSymbolicLinkWithAbsolutePath_CreateScanSuccessfull "scan", "create", flag(params.ProjectName), "my-project", flag(params.SourcesFlag), "data/project-with-directory-symlink", - flag(params.ScanTypes), "iac-security", + flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", } err, _ := executeCommand(t, args...) @@ -322,7 +323,7 @@ func TestScanCreate_FolderWithSymbolicLinkWithRelativePath_CreateScanSuccessfull "scan", "create", flag(params.ProjectName), "my-project", flag(params.SourcesFlag), "data/symlink-relative-path-folder", - flag(params.ScanTypes), "iac-security", + flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", } err, _ := executeCommand(t, args...) @@ -335,7 +336,7 @@ func TestScanCreate_ExistingApplicationAndNotExistingProject_CreatingNewProjectA flag(params.ApplicationName), "my-application", flag(params.ProjectName), projectNameRandom, flag(params.SourcesFlag), ".", - flag(params.ScanTypes), "sast", + flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", flag(params.ScanInfoFormatFlag), printer.FormatJSON, } @@ -351,7 +352,7 @@ func TestScanCreate_ApplicationDoesntExist_FailScanWithError(t *testing.T) { flag(params.ApplicationName), "application-that-doesnt-exist", flag(params.ProjectName), "my-project", flag(params.SourcesFlag), ".", - flag(params.ScanTypes), "sast", + flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", } @@ -365,7 +366,7 @@ func TestContainerEngineScansE2E_ContainerImagesFlagAndScanType(t *testing.T) { "scan", "create", flag(params.ProjectName), getProjectNameForScanTests(), flag(params.SourcesFlag), "data/Dockerfile-mysql571.zip", - flag(params.ScanTypes), "container-security", + flag(params.ScanTypes), params.ContainersTypeFlag, flag(params.ContainerImagesFlag), "nginx:alpine,debian:9", flag(params.BranchFlag), "dummy_branch", flag(params.ScanInfoFormatFlag), printer.FormatJSON, @@ -386,6 +387,7 @@ func TestContainerEngineScansE2E_ContainerImagesFlagOnly(t *testing.T) { flag(params.SourcesFlag), "data/insecure.zip", flag(params.ContainerImagesFlag), "nginx:alpine", flag(params.BranchFlag), "dummy_branch", + flag(params.ScanTypes), params.ContainersTypeFlag, flag(params.ScanInfoFormatFlag), printer.FormatJSON, } if isFFEnabled(t, wrappers.ContainerEngineCLIEnabled) { @@ -405,6 +407,7 @@ func TestContainerEngineScansE2E_ContainerImagesAndDebugFlags(t *testing.T) { flag(params.ContainerImagesFlag), "mysql:5.7", flag(params.BranchFlag), "dummy_branch", flag(params.DebugFlag), + flag(params.ScanTypeFlag), params.ContainersTypeFlag, flag(params.ScanInfoFormatFlag), printer.FormatJSON, } if isFFEnabled(t, wrappers.ContainerEngineCLIEnabled) { @@ -424,6 +427,7 @@ func TestContainerEngineScansE2E_ContainerImagesFlagAndEmptyFolderProject(t *tes flag(params.ContainerImagesFlag), "mysql:5.7", flag(params.BranchFlag), "dummy_branch", flag(params.ScanInfoFormatFlag), printer.FormatJSON, + flag(params.ScanTypeFlag), params.ContainersTypeFlag, } if isFFEnabled(t, wrappers.ContainerEngineCLIEnabled) { scanID, projectID := executeCreateScan(t, testArgs) @@ -477,9 +481,9 @@ func createScanWithFastScan(t *testing.T, source string, name string, tags map[s } func TestScansUpdateProjectGroups(t *testing.T) { - scanID, projectID := executeCreateScan(t, getCreateArgs(Zip, Tags, "sast")) + scanID, projectID := executeCreateScan(t, getCreateArgs(Zip, Tags, params.IacType)) response := listScanByID(t, scanID) - scanID, projectID = executeCreateScan(t, getCreateArgsWithNameAndGroups(Zip, Tags, Groups, response[0].ProjectName, "sast")) + scanID, projectID = executeCreateScan(t, getCreateArgsWithNameAndGroups(Zip, Tags, Groups, response[0].ProjectName, params.IacType)) defer deleteProject(t, projectID) executeScanAssertions(t, projectID, scanID, Tags) @@ -523,7 +527,7 @@ func TestScaResolverArg(t *testing.T) { t, Dir, map[string]string{}, - "sast,iac-security", + params.IacType, viper.GetString(resolverEnvVar), ) @@ -544,7 +548,7 @@ func TestScaResolverArgFailed(t *testing.T) { flag(params.ProjectName), "resolver", flag(params.SourcesFlag), ".", flag(params.ScaResolverFlag), "./nonexisting", - flag(params.ScanTypes), "sast,iac-security,sca", + flag(params.ScanTypes), "iac-security,sca", flag(params.BranchFlag), "dummy_branch", } @@ -556,7 +560,7 @@ func TestScaResolverArgFailed(t *testing.T) { flag(params.ProjectName), "resolver", flag(params.SourcesFlag), ".", flag(params.ScaResolverFlag), viper.GetString(resolverEnvVar), - flag(params.ScanTypes), "sast,iac-security,sca", + flag(params.ScanTypes), "iac-security,sca", flag(params.BranchFlag), "dummy_branch", flag(params.ScaResolverParamsFlag), "-q --invalid-param \"invalid\"", } @@ -581,7 +585,7 @@ func TestIncrementalScan(t *testing.T) { // Start a scan guaranteed to take considerable time, cancel it and assert the status func TestCancelScan(t *testing.T) { - scanID, projectID := createScanSastNoWait(t, SlowRepo, map[string]string{}) + scanID, projectID := createScanNoWait(t, SlowRepo, map[string]string{}, GenerateRandomProjectNameForScan()) defer deleteProject(t, projectID) defer deleteScan(t, scanID) @@ -605,7 +609,7 @@ func TestScanCreateIncludeFilter(t *testing.T) { "scan", "create", flag(params.ProjectName), projectName, flag(params.SourcesFlag), ".", - flag(params.ScanTypes), "sast", + flag(params.ScanTypes), params.IacType, flag(params.PresetName), "Checkmarx Default", flag(params.SourceDirFilterFlag), "!*go,!*Dockerfile,!*js,!*json,!*tf", flag(params.IacsFilterFlag), "!Dockerfile", @@ -625,7 +629,7 @@ func TestScanCreateWithThresholdShouldBlock(t *testing.T) { "scan", "create", flag(params.ProjectName), projectName, flag(params.SourcesFlag), Zip, - flag(params.ScanTypes), "sast", + flag(params.ScanTypes), params.SastType, flag(params.PresetName), "Checkmarx Default", flag(params.Threshold), "sast-high=1;sast-low=1;", flag(params.KicsFilterFlag), "!Dockerfile", @@ -643,9 +647,9 @@ func TestScanCreateWithThreshold(t *testing.T) { "scan", "create", flag(params.ProjectName), projectName, flag(params.SourcesFlag), Zip, - flag(params.ScanTypes), "sast", + flag(params.ScanTypes), params.IacType, flag(params.PresetName), "Checkmarx Default", - flag(params.Threshold), "sast-high=100;", + flag(params.Threshold), "iac-security-high=100;", flag(params.KicsFilterFlag), "!Dockerfile", flag(params.BranchFlag), "dummy_branch", } @@ -663,6 +667,7 @@ func TestScansAPISecThresholdShouldBlock(t *testing.T) { flag(params.BranchFlag), "dummy_branch", flag(params.ScanInfoFormatFlag), printer.FormatJSON, flag(params.ScanTypes), "sast, api-security", + flag(params.SastFastScanFlag), flag(params.Threshold), "api-security-high=1", } _, _ = executeCommand(t, testArgs...) @@ -677,9 +682,9 @@ func TestScanCreateWithThresholdParseError(t *testing.T) { "scan", "create", flag(params.ProjectName), projectName, flag(params.SourcesFlag), Zip, - flag(params.ScanTypes), "sast, sca", + flag(params.ScanTypes), "sca", flag(params.PresetName), "Checkmarx Default", - flag(params.Threshold), "sast-high=error; sca-high=error;", + flag(params.Threshold), "sca-high=error;", flag(params.BranchFlag), "dummy_branch", } @@ -707,10 +712,10 @@ func TestScanCreateWithThresholdAndReportGenerate(t *testing.T) { "scan", "create", flag(params.ProjectName), projectName, flag(params.SourcesFlag), Zip, - flag(params.ScanTypes), "sast, sca", + flag(params.ScanTypes), "sca", flag(params.SastRedundancyFlag), flag(params.PresetName), "Checkmarx Default", - flag(params.Threshold), "sast-high=1;sast-low=1; sca-high=1", + flag(params.Threshold), "sca-high=1", flag(params.BranchFlag), "dummy_branch", flag(params.TargetFormatFlag), "json", flag(params.TargetPathFlag), "/tmp/", @@ -736,7 +741,7 @@ func TestScanCreateIgnoreExclusionFolders(t *testing.T) { "scan", "create", flag(params.ProjectName), projectName, flag(params.SourcesFlag), "../..", - flag(params.ScanTypes), "sast,sca", + flag(params.ScanTypes), params.IacType, flag(params.PresetName), "Checkmarx Default", flag(params.SourceDirFilterFlag), ".git,*.js", // needed one code file or the scan will end with partial code flag(params.BranchFlag), "dummy_branch", @@ -1161,6 +1166,7 @@ func TestScanCreateWithSSHKey(t *testing.T) { flag(params.SourcesFlag), SSHRepo, flag(params.BranchFlag), "main", flag(params.SSHKeyFlag), SSHKeyFilePath, + flag(params.ScanTypeFlag), params.IacType, flag(params.IgnorePolicyFlag), } @@ -1177,6 +1183,7 @@ func TestCreateScanFilterZipFile(t *testing.T) { flag(params.SourcesFlag), Zip, flag(params.SourceDirFilterFlag), "!*.html", flag(params.IgnorePolicyFlag), + flag(params.ScanTypeFlag), params.IacType, } executeCmdWithTimeOutNilAssertion(t, "Scan must complete successfully", 10*time.Minute, args...) @@ -1640,7 +1647,7 @@ func TestScanWithPolicy(t *testing.T) { args := []string{scanCommand, "create", flag(params.ProjectName), "TiagoBaptista/testingCli/testingCli", flag(params.SourcesFlag), Zip, - flag(params.ScanTypes), "sast", + flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "main", flag(params.TargetFormatFlag), "markdown,summaryConsole,summaryHTML"} @@ -1713,7 +1720,7 @@ func TestCreateScan_WithTypeScsMissingRepoURL_Fail(t *testing.T) { "scan", "create", flag(params.ProjectName), projectName, flag(params.SourcesFlag), Zip, - flag(params.ScanTypes), "sast, scs", + flag(params.ScanTypes), "iac-security, scs", flag(params.BranchFlag), "main", flag(params.SCSRepoTokenFlag), scsRepoToken, } @@ -1729,7 +1736,7 @@ func TestCreateScan_WithTypeScsMissingRepoToken_Fail(t *testing.T) { "scan", "create", flag(params.ProjectName), projectName, flag(params.SourcesFlag), Zip, - flag(params.ScanTypes), "sast, scs", + flag(params.ScanTypes), "iac-security, scs", flag(params.BranchFlag), "main", flag(params.SCSRepoURLFlag), scsRepoURL, } @@ -1821,7 +1828,7 @@ func addSCSDefaultFlagsToArgs(args *[]string) { func TestCreateScanAndValidateCheckmarxDomains(t *testing.T) { wrappers.Domains = make(map[string]struct{}) - _, _ = executeCreateScan(t, getCreateArgsWithGroups(Zip, Tags, Groups, "sast,iac-security,sca")) + _, _ = executeCreateScan(t, getCreateArgsWithGroups(Zip, Tags, Groups, "iac-security")) usedDomainsInTests := []string{"deu.iam.checkmarx.net", "deu.ast.checkmarx.net"} validateCheckmarxDomains(t, usedDomainsInTests) } From cb72cfd87b5f4e9a3d9df98fd81567734b14f62a Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Thu, 22 Aug 2024 09:34:58 +0300 Subject: [PATCH 02/33] optimize test run time --- test/integration/result_test.go | 21 ++++++++++++++++++++- test/integration/scan_test.go | 14 +++++++------- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/test/integration/result_test.go b/test/integration/result_test.go index 475b53c6e..d6913e678 100644 --- a/test/integration/result_test.go +++ b/test/integration/result_test.go @@ -27,6 +27,7 @@ const ( ) func TestResultsExitCode_OnSendingFakeScanId_ShouldReturnNotFoundError(t *testing.T) { + t.Parallel() bindKeysToEnvAndDefault(t) scansPath := viper.GetString(params.ScansPathKey) scansWrapper := wrappers.NewHTTPScansWrapper(scansPath) @@ -36,6 +37,7 @@ func TestResultsExitCode_OnSendingFakeScanId_ShouldReturnNotFoundError(t *testin } func TestResultsExitCode_OnSuccessfulScan_ShouldReturnStatusCompleted(t *testing.T) { + t.Parallel() scanID, _ := getRootScan(t) scansPath := viper.GetString(params.ScansPathKey) @@ -50,6 +52,7 @@ func TestResultsExitCode_OnSuccessfulScan_ShouldReturnStatusCompleted(t *testing } func TestResultsExitCode_NoScanIdSent_FailCommandWithError(t *testing.T) { + t.Parallel() bindKeysToEnvAndDefault(t) args := []string{ "results", "exit-code", @@ -61,6 +64,7 @@ func TestResultsExitCode_NoScanIdSent_FailCommandWithError(t *testing.T) { } func TestResultsExitCode_FakeScanIdSent_FailCommandWithError(t *testing.T) { + t.Parallel() bindKeysToEnvAndDefault(t) args := []string{ "results", "exit-code", @@ -73,6 +77,7 @@ func TestResultsExitCode_FakeScanIdSent_FailCommandWithError(t *testing.T) { } func TestResultListJson(t *testing.T) { + t.Parallel() assertRequiredParameter(t, "Please provide a scan ID", "results", "show") scanID, _ := getRootScan(t) @@ -125,7 +130,7 @@ func assertResultFilesCreated(t *testing.T) { } func TestResultListForGlReports(t *testing.T) { - + t.Parallel() assertRequiredParameter(t, "Please provide a scan ID", "results", "show") scanID, _ := getRootScan(t) @@ -158,6 +163,7 @@ func TestResultListForGlReports(t *testing.T) { } func assertGlResultFilesCreated(t *testing.T) { + t.Parallel() extensions := []string{printer.FormatGLSast, printer.FormatGLSca} @@ -173,6 +179,7 @@ func assertGlResultFilesCreated(t *testing.T) { } func TestResultsShowParamFailed(t *testing.T) { + t.Parallel() args := []string{ "results", "show", @@ -183,6 +190,7 @@ func TestResultsShowParamFailed(t *testing.T) { } func TestCodeBashingParamFailed(t *testing.T) { + t.Parallel() args := []string{ "results", "codebashing", @@ -193,6 +201,7 @@ func TestCodeBashingParamFailed(t *testing.T) { } func TestCodeBashingList(t *testing.T) { + t.Parallel() outputBuffer := executeCmdNilAssertion( t, "Getting results should pass", @@ -210,6 +219,7 @@ func TestCodeBashingList(t *testing.T) { } func TestCodeBashingListJson(t *testing.T) { + t.Parallel() outputBuffer := executeCmdNilAssertion( t, "Getting results should pass", @@ -228,6 +238,7 @@ func TestCodeBashingListJson(t *testing.T) { } func TestCodeBashingListTable(t *testing.T) { + t.Parallel() outputBuffer := executeCmdNilAssertion( t, "Getting results should pass", @@ -242,6 +253,7 @@ func TestCodeBashingListTable(t *testing.T) { } func TestCodeBashingListEmpty(t *testing.T) { + t.Parallel() args := []string{ "results", "codebashing", @@ -255,6 +267,7 @@ func TestCodeBashingListEmpty(t *testing.T) { } func TestCodeBashingFailedListingAuth(t *testing.T) { + t.Parallel() args := []string{ "results", "codebashing", @@ -270,6 +283,7 @@ func TestCodeBashingFailedListingAuth(t *testing.T) { } func TestResultsGeneratingPdfReportWithInvalidPdfOptions(t *testing.T) { + t.Parallel() scanID, _ := getRootScan(t) args := []string{ @@ -284,6 +298,7 @@ func TestResultsGeneratingPdfReportWithInvalidPdfOptions(t *testing.T) { } func TestResultsGeneratingPdfReportWithInvalidEmail(t *testing.T) { + t.Parallel() scanID, _ := getRootScan(t) args := []string{ @@ -352,6 +367,7 @@ func TestResultsGeneratingPdfReportAndSendToEmail(t *testing.T) { } func TestResultsGeneratingSBOMWrongScanType(t *testing.T) { + t.Parallel() scanID, _ := getRootScan(t) args := []string{ @@ -394,6 +410,7 @@ func TestResultsGeneratingSBOM(t *testing.T) { } func TestResultsWrongScanID(t *testing.T) { + t.Parallel() args := []string{ "results", "show", flag(params.ScanIDFlag), "wrong", @@ -435,6 +452,7 @@ func TestResultsCounterJsonOutput(t *testing.T) { } func TestResultsCounterGlSastOutput(t *testing.T) { + t.Parallel() scanID, _ := getRootScan(t) _ = executeCmdNilAssertion( t, "Results show generating gl-sast report with options should pass", @@ -516,6 +534,7 @@ func TestResultsGeneratingJsonReportWithSeverityHighAndWithoutNotExploitable(t * } func TestResultExcludeNotExploitableFailScanId(t *testing.T) { + t.Parallel() bindKeysToEnvAndDefault(t) args := []string{ "results", "show", diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index c983c9caa..d56d61adc 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -407,7 +407,7 @@ func TestContainerEngineScansE2E_ContainerImagesAndDebugFlags(t *testing.T) { flag(params.ContainerImagesFlag), "mysql:5.7", flag(params.BranchFlag), "dummy_branch", flag(params.DebugFlag), - flag(params.ScanTypeFlag), params.ContainersTypeFlag, + flag(params.ScanTypes), params.ContainersTypeFlag, flag(params.ScanInfoFormatFlag), printer.FormatJSON, } if isFFEnabled(t, wrappers.ContainerEngineCLIEnabled) { @@ -427,7 +427,7 @@ func TestContainerEngineScansE2E_ContainerImagesFlagAndEmptyFolderProject(t *tes flag(params.ContainerImagesFlag), "mysql:5.7", flag(params.BranchFlag), "dummy_branch", flag(params.ScanInfoFormatFlag), printer.FormatJSON, - flag(params.ScanTypeFlag), params.ContainersTypeFlag, + flag(params.ScanTypes), params.ContainersTypeFlag, } if isFFEnabled(t, wrappers.ContainerEngineCLIEnabled) { scanID, projectID := executeCreateScan(t, testArgs) @@ -1038,7 +1038,7 @@ func TestScanLogsSAST(t *testing.T) { args := []string{ "scan", "logs", flag(params.ScanIDFlag), scanID, - flag(params.ScanTypeFlag), "sast", + flag(params.ScanTypes), "sast", } cmd := createASTIntegrationTestCommand(t) err := execute(cmd, args...) @@ -1050,7 +1050,7 @@ func TestScanLogsKICSDeprecated(t *testing.T) { args := []string{ "scan", "logs", flag(params.ScanIDFlag), scanID, - flag(params.ScanTypeFlag), "kics", + flag(params.ScanTypes), "kics", } cmd := createASTIntegrationTestCommand(t) err := execute(cmd, args...) @@ -1062,7 +1062,7 @@ func TestScanLogsKICS(t *testing.T) { args := []string{ "scan", "logs", flag(params.ScanIDFlag), scanID, - flag(params.ScanTypeFlag), "iac-security", + flag(params.ScanTypes), "iac-security", } cmd := createASTIntegrationTestCommand(t) err := execute(cmd, args...) @@ -1166,7 +1166,7 @@ func TestScanCreateWithSSHKey(t *testing.T) { flag(params.SourcesFlag), SSHRepo, flag(params.BranchFlag), "main", flag(params.SSHKeyFlag), SSHKeyFilePath, - flag(params.ScanTypeFlag), params.IacType, + flag(params.ScanTypes), params.IacType, flag(params.IgnorePolicyFlag), } @@ -1183,7 +1183,7 @@ func TestCreateScanFilterZipFile(t *testing.T) { flag(params.SourcesFlag), Zip, flag(params.SourceDirFilterFlag), "!*.html", flag(params.IgnorePolicyFlag), - flag(params.ScanTypeFlag), params.IacType, + flag(params.ScanTypes), params.IacType, } executeCmdWithTimeOutNilAssertion(t, "Scan must complete successfully", 10*time.Minute, args...) From 839497bba460d6bb4f583d36eb1e9837229885f8 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Thu, 22 Aug 2024 09:41:10 +0300 Subject: [PATCH 03/33] failed the job if some test failed --- internal/commands/.scripts/integration_up.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 2168c7ff3..2bd3937fd 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -14,6 +14,7 @@ go test \ -tags integration \ -v \ -timeout 210m \ + -failfast \ -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ -coverprofile cover.out \ github.com/checkmarx/ast-cli/test/integration From 972b05ed78ccac1be0336803bc47f1b7ae3281f5 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Thu, 22 Aug 2024 10:11:50 +0300 Subject: [PATCH 04/33] revert parallel --- test/integration/result_test.go | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/test/integration/result_test.go b/test/integration/result_test.go index d6913e678..fb2b827de 100644 --- a/test/integration/result_test.go +++ b/test/integration/result_test.go @@ -27,7 +27,6 @@ const ( ) func TestResultsExitCode_OnSendingFakeScanId_ShouldReturnNotFoundError(t *testing.T) { - t.Parallel() bindKeysToEnvAndDefault(t) scansPath := viper.GetString(params.ScansPathKey) scansWrapper := wrappers.NewHTTPScansWrapper(scansPath) @@ -37,7 +36,6 @@ func TestResultsExitCode_OnSendingFakeScanId_ShouldReturnNotFoundError(t *testin } func TestResultsExitCode_OnSuccessfulScan_ShouldReturnStatusCompleted(t *testing.T) { - t.Parallel() scanID, _ := getRootScan(t) scansPath := viper.GetString(params.ScansPathKey) @@ -52,7 +50,6 @@ func TestResultsExitCode_OnSuccessfulScan_ShouldReturnStatusCompleted(t *testing } func TestResultsExitCode_NoScanIdSent_FailCommandWithError(t *testing.T) { - t.Parallel() bindKeysToEnvAndDefault(t) args := []string{ "results", "exit-code", @@ -64,7 +61,6 @@ func TestResultsExitCode_NoScanIdSent_FailCommandWithError(t *testing.T) { } func TestResultsExitCode_FakeScanIdSent_FailCommandWithError(t *testing.T) { - t.Parallel() bindKeysToEnvAndDefault(t) args := []string{ "results", "exit-code", @@ -77,7 +73,6 @@ func TestResultsExitCode_FakeScanIdSent_FailCommandWithError(t *testing.T) { } func TestResultListJson(t *testing.T) { - t.Parallel() assertRequiredParameter(t, "Please provide a scan ID", "results", "show") scanID, _ := getRootScan(t) @@ -130,7 +125,6 @@ func assertResultFilesCreated(t *testing.T) { } func TestResultListForGlReports(t *testing.T) { - t.Parallel() assertRequiredParameter(t, "Please provide a scan ID", "results", "show") scanID, _ := getRootScan(t) @@ -163,7 +157,6 @@ func TestResultListForGlReports(t *testing.T) { } func assertGlResultFilesCreated(t *testing.T) { - t.Parallel() extensions := []string{printer.FormatGLSast, printer.FormatGLSca} @@ -179,7 +172,6 @@ func assertGlResultFilesCreated(t *testing.T) { } func TestResultsShowParamFailed(t *testing.T) { - t.Parallel() args := []string{ "results", "show", @@ -190,7 +182,6 @@ func TestResultsShowParamFailed(t *testing.T) { } func TestCodeBashingParamFailed(t *testing.T) { - t.Parallel() args := []string{ "results", "codebashing", @@ -201,7 +192,6 @@ func TestCodeBashingParamFailed(t *testing.T) { } func TestCodeBashingList(t *testing.T) { - t.Parallel() outputBuffer := executeCmdNilAssertion( t, "Getting results should pass", @@ -219,7 +209,6 @@ func TestCodeBashingList(t *testing.T) { } func TestCodeBashingListJson(t *testing.T) { - t.Parallel() outputBuffer := executeCmdNilAssertion( t, "Getting results should pass", @@ -238,7 +227,6 @@ func TestCodeBashingListJson(t *testing.T) { } func TestCodeBashingListTable(t *testing.T) { - t.Parallel() outputBuffer := executeCmdNilAssertion( t, "Getting results should pass", @@ -253,7 +241,6 @@ func TestCodeBashingListTable(t *testing.T) { } func TestCodeBashingListEmpty(t *testing.T) { - t.Parallel() args := []string{ "results", "codebashing", @@ -267,7 +254,6 @@ func TestCodeBashingListEmpty(t *testing.T) { } func TestCodeBashingFailedListingAuth(t *testing.T) { - t.Parallel() args := []string{ "results", "codebashing", @@ -283,7 +269,6 @@ func TestCodeBashingFailedListingAuth(t *testing.T) { } func TestResultsGeneratingPdfReportWithInvalidPdfOptions(t *testing.T) { - t.Parallel() scanID, _ := getRootScan(t) args := []string{ @@ -298,7 +283,6 @@ func TestResultsGeneratingPdfReportWithInvalidPdfOptions(t *testing.T) { } func TestResultsGeneratingPdfReportWithInvalidEmail(t *testing.T) { - t.Parallel() scanID, _ := getRootScan(t) args := []string{ @@ -367,7 +351,6 @@ func TestResultsGeneratingPdfReportAndSendToEmail(t *testing.T) { } func TestResultsGeneratingSBOMWrongScanType(t *testing.T) { - t.Parallel() scanID, _ := getRootScan(t) args := []string{ @@ -410,7 +393,6 @@ func TestResultsGeneratingSBOM(t *testing.T) { } func TestResultsWrongScanID(t *testing.T) { - t.Parallel() args := []string{ "results", "show", flag(params.ScanIDFlag), "wrong", @@ -452,7 +434,6 @@ func TestResultsCounterJsonOutput(t *testing.T) { } func TestResultsCounterGlSastOutput(t *testing.T) { - t.Parallel() scanID, _ := getRootScan(t) _ = executeCmdNilAssertion( t, "Results show generating gl-sast report with options should pass", @@ -534,7 +515,6 @@ func TestResultsGeneratingJsonReportWithSeverityHighAndWithoutNotExploitable(t * } func TestResultExcludeNotExploitableFailScanId(t *testing.T) { - t.Parallel() bindKeysToEnvAndDefault(t) args := []string{ "results", "show", From 5ebd2a7c0e9467a1b7e768e0789dc9fcb7b74854 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Thu, 22 Aug 2024 10:29:01 +0300 Subject: [PATCH 05/33] try go test sum --- .github/workflows/ci.yml | 2 ++ internal/commands/.scripts/integration_up.sh | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78117e256..b0f697e72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,8 @@ jobs: - run: go version - name: Go Build run: go build -o ./bin/cx ./cmd + - name: Install gotestsum + run: go install gotest.tools/gotestsum@latest - name: Go Integration test shell: bash env: diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 2bd3937fd..83c7c899d 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -10,11 +10,10 @@ wget https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.g tar -xzvf ScaResolver-linux64.tar.gz -C /tmp rm -rf ScaResolver-linux64.tar.gz -go test \ +gotestsum --rerun-fails=2 --go test \ -tags integration \ -v \ -timeout 210m \ - -failfast \ -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ -coverprofile cover.out \ github.com/checkmarx/ast-cli/test/integration From 7d912406783da16f8580876543aa2e4e9603cde8 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Thu, 22 Aug 2024 10:48:41 +0300 Subject: [PATCH 06/33] revert try go test sum --- .github/workflows/ci.yml | 2 -- internal/commands/.scripts/integration_up.sh | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0f697e72..78117e256 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,8 +43,6 @@ jobs: - run: go version - name: Go Build run: go build -o ./bin/cx ./cmd - - name: Install gotestsum - run: go install gotest.tools/gotestsum@latest - name: Go Integration test shell: bash env: diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 83c7c899d..2bd3937fd 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -10,10 +10,11 @@ wget https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.g tar -xzvf ScaResolver-linux64.tar.gz -C /tmp rm -rf ScaResolver-linux64.tar.gz -gotestsum --rerun-fails=2 --go test \ +go test \ -tags integration \ -v \ -timeout 210m \ + -failfast \ -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ -coverprofile cover.out \ github.com/checkmarx/ast-cli/test/integration From c79ba820bc16ad72c770165e49413bf8c05899c2 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Thu, 22 Aug 2024 11:12:29 +0300 Subject: [PATCH 07/33] try go test sum --- .github/workflows/ci.yml | 2 ++ internal/commands/.scripts/integration_up.sh | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78117e256..b0f697e72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,8 @@ jobs: - run: go version - name: Go Build run: go build -o ./bin/cx ./cmd + - name: Install gotestsum + run: go install gotest.tools/gotestsum@latest - name: Go Integration test shell: bash env: diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 2bd3937fd..203c8103a 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -10,15 +10,15 @@ wget https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.g tar -xzvf ScaResolver-linux64.tar.gz -C /tmp rm -rf ScaResolver-linux64.tar.gz -go test \ - -tags integration \ +gotestsum --rerun-fails=2 \ + -- -tags=integration \ -v \ -timeout 210m \ - -failfast \ -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ -coverprofile cover.out \ github.com/checkmarx/ast-cli/test/integration + status=$? echo "status value after tests $status" if [ $status -ne 0 ]; then From 4e0e5d4301866f8c37ae98418904d0dfc0c35392 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Thu, 22 Aug 2024 11:17:25 +0300 Subject: [PATCH 08/33] try go test sum --- internal/commands/.scripts/integration_up.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 203c8103a..00beec3d4 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -10,13 +10,12 @@ wget https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.g tar -xzvf ScaResolver-linux64.tar.gz -C /tmp rm -rf ScaResolver-linux64.tar.gz -gotestsum --rerun-fails=2 \ - -- -tags=integration \ - -v \ - -timeout 210m \ - -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ - -coverprofile cover.out \ - github.com/checkmarx/ast-cli/test/integration +gotestsum --rerun-fails=2 --format=short \ +--packages github.com/checkmarx/ast-cli/test/integration -- \ +-tags=integration -v -timeout 210m \ +-coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ +-coverprofile cover.out + status=$? From d7b4ebd7969ce46165663adcdf3b8db64e812f85 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Sun, 25 Aug 2024 09:07:14 +0300 Subject: [PATCH 09/33] remove scs score card test --- test/integration/scan_test.go | 62 +++++++++++++++++------------------ 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index d56d61adc..2f5a019b7 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -865,9 +865,9 @@ func executeScanAssertions(t *testing.T, projectID, scanID string, tags map[stri func createScan(t *testing.T, source string, tags map[string]string) (string, string) { if isFFEnabled(t, wrappers.ContainerEngineCLIEnabled) { - return executeCreateScan(t, getCreateArgs(source, tags, "sast , sca , iac-security , api-security, scs,container-security")) + return executeCreateScan(t, getCreateArgs(source, tags, "sast , sca , iac-security , api-security, container-security")) } else { - return executeCreateScan(t, getCreateArgs(source, tags, "sast , sca , iac-security , api-security, scs")) + return executeCreateScan(t, getCreateArgs(source, tags, "sast , sca , iac-security , api-security")) } } @@ -1667,21 +1667,21 @@ func TestScanWithPolicyTimeout(t *testing.T) { assert.Error(t, err, "--policy-timeout should be equal or higher than 0") } -func TestCreateScan_WithTypeScs_Success(t *testing.T) { - _, projectName := getRootProject(t) - - args := []string{ - "scan", "create", - flag(params.ProjectName), projectName, - flag(params.SourcesFlag), Zip, - flag(params.ScanTypes), "scs", - flag(params.BranchFlag), "main", - flag(params.SCSRepoURLFlag), scsRepoURL, - flag(params.SCSRepoTokenFlag), scsRepoToken, - } - - executeCmdWithTimeOutNilAssertion(t, "SCS scan must complete successfully", 4*time.Minute, args...) -} +//func TestCreateScan_WithTypeScs_Success(t *testing.T) { +// _, projectName := getRootProject(t) +// +// args := []string{ +// "scan", "create", +// flag(params.ProjectName), projectName, +// flag(params.SourcesFlag), Zip, +// flag(params.ScanTypes), "scs", +// flag(params.BranchFlag), "main", +// flag(params.SCSRepoURLFlag), scsRepoURL, +// flag(params.SCSRepoTokenFlag), scsRepoToken, +// } +// +// executeCmdWithTimeOutNilAssertion(t, "SCS scan must complete successfully", 4*time.Minute, args...) +//} func TestCreateScan_WithNoScanTypesFlag_SuccessAndScsNotScanned(t *testing.T) { _, projectName := getRootProject(t) @@ -1698,20 +1698,20 @@ func TestCreateScan_WithNoScanTypesFlag_SuccessAndScsNotScanned(t *testing.T) { assert.Assert(t, !strings.Contains(output.String(), params.ScsType), "Scs scan must not run if all required flags are not provided") } -func TestCreateScan_WithNoScanTypesFlagButScsFlagsPresent_SuccessAndScsScanned(t *testing.T) { - _, projectName := getRootProject(t) - args := []string{ - "scan", "create", - flag(params.ProjectName), projectName, - flag(params.SourcesFlag), Zip, - flag(params.BranchFlag), "main", - flag(params.SCSRepoURLFlag), scsRepoURL, - flag(params.SCSRepoTokenFlag), scsRepoToken, - } - - output := executeCmdWithTimeOutNilAssertion(t, "Scan must complete successfully if no scan-types specified and scs-repo flags are present", 4*time.Minute, args...) - assert.Assert(t, strings.Contains(output.String(), params.ScsType), "Scs scan should run if all required flags are provided") -} +//func TestCreateScan_WithNoScanTypesFlagButScsFlagsPresent_SuccessAndScsScanned(t *testing.T) { +// _, projectName := getRootProject(t) +// args := []string{ +// "scan", "create", +// flag(params.ProjectName), projectName, +// flag(params.SourcesFlag), Zip, +// flag(params.BranchFlag), "main", +// flag(params.SCSRepoURLFlag), scsRepoURL, +// flag(params.SCSRepoTokenFlag), scsRepoToken, +// } +// +// output := executeCmdWithTimeOutNilAssertion(t, "Scan must complete successfully if no scan-types specified and scs-repo flags are present", 4*time.Minute, args...) +// assert.Assert(t, strings.Contains(output.String(), params.ScsType), "Scs scan should run if all required flags are provided") +//} func TestCreateScan_WithTypeScsMissingRepoURL_Fail(t *testing.T) { _, projectName := getRootProject(t) From eb0bed0045cbb352a3f76447d8440d544581cfa1 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Sun, 25 Aug 2024 10:37:15 +0300 Subject: [PATCH 10/33] remove scs score card test --- test/integration/scan_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index 2f5a019b7..3572af92a 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -923,6 +923,8 @@ func getCreateArgsWithName(source string, tags map[string]string, projectName, s return getCreateArgsWithNameAndGroups(source, tags, nil, projectName, scanTypes) } func getCreateArgsWithNameAndGroups(source string, tags map[string]string, groups []string, projectName, scanTypes string) []string { + scanTypes = removeSCSScanType(scanTypes) + args := []string{ "scan", "create", flag(params.ProjectName), projectName, @@ -941,6 +943,20 @@ func getCreateArgsWithNameAndGroups(source string, tags map[string]string, group return args } +func removeSCSScanType(scanTypes string) string { + if strings.Contains(scanTypes, "scs") { + types := strings.Split(scanTypes, ",") + for i, t := range types { + if t == "scs" { + types = append(types[:i], types[i+1:]...) + break + } + } + return strings.Join(types, ",") + } + return scanTypes +} + func executeCreateScan(t *testing.T, args []string) (string, string) { buffer := executeScanGetBuffer(t, args) From d346bae6534fbf6a47c2f78da6c01a656207e229 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Sun, 25 Aug 2024 12:09:23 +0300 Subject: [PATCH 11/33] set global timeout --- test/integration/scan_test.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index 3572af92a..6bce58e25 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -52,6 +52,7 @@ const ( invalidClientSecret = "invalidClientSecret" invalidAPIKey = "invalidAPI" invalidTenant = "invalidTenant" + timeout = 10 * time.Minute ) var ( @@ -617,7 +618,7 @@ func TestScanCreateIncludeFilter(t *testing.T) { } args[11] = "*js" - executeCmdWithTimeOutNilAssertion(t, "Including zip should fix the scan", 5*time.Minute, args...) + executeCmdWithTimeOutNilAssertion(t, "Including zip should fix the scan", timeout, args...) } // Create a scan with the sources @@ -972,7 +973,7 @@ func executeCreateScan(t *testing.T, args []string) (string, string) { } func executeScanGetBuffer(t *testing.T, args []string) *bytes.Buffer { - return executeCmdWithTimeOutNilAssertion(t, "Creating a scan should pass", 10*time.Minute, args...) + return executeCmdWithTimeOutNilAssertion(t, "Creating a scan should pass", timeout, args...) } func deleteScan(t *testing.T, scanID string) { @@ -1186,7 +1187,7 @@ func TestScanCreateWithSSHKey(t *testing.T) { flag(params.IgnorePolicyFlag), } - executeCmdWithTimeOutNilAssertion(t, "Create a scan with ssh-key should pass", 4*time.Minute, args...) + executeCmdWithTimeOutNilAssertion(t, "Create a scan with ssh-key should pass", timeout, args...) } func TestCreateScanFilterZipFile(t *testing.T) { @@ -1202,7 +1203,7 @@ func TestCreateScanFilterZipFile(t *testing.T) { flag(params.ScanTypes), params.IacType, } - executeCmdWithTimeOutNilAssertion(t, "Scan must complete successfully", 10*time.Minute, args...) + executeCmdWithTimeOutNilAssertion(t, "Scan must complete successfully", timeout, args...) } func TestRunKicsScan(t *testing.T) { @@ -1710,7 +1711,7 @@ func TestCreateScan_WithNoScanTypesFlag_SuccessAndScsNotScanned(t *testing.T) { flag(params.SCSRepoTokenFlag), scsRepoToken, } - output := executeCmdWithTimeOutNilAssertion(t, "Scan must complete successfully if no scan-types specified, even if missing scs-repo flags", 4*time.Minute, args...) + output := executeCmdWithTimeOutNilAssertion(t, "Scan must complete successfully if no scan-types specified, even if missing scs-repo flags", timeout, args...) assert.Assert(t, !strings.Contains(output.String(), params.ScsType), "Scs scan must not run if all required flags are not provided") } @@ -1774,7 +1775,7 @@ func TestCreateScan_WithTypeScsOnlySecretDetection_Success(t *testing.T) { } executeCmdWithTimeOutNilAssertion(t, - "SCS with only secret-detection scan must complete successfully, even if missing scs-repo flags", 4*time.Minute, args...) + "SCS with only secret-detection scan must complete successfully, even if missing scs-repo flags", timeout, args...) } func TestCreateScan_WithNoScanTypesFlagScsAndOnlySecretDetection_Success(t *testing.T) { @@ -1789,7 +1790,7 @@ func TestCreateScan_WithNoScanTypesFlagScsAndOnlySecretDetection_Success(t *test } executeCmdWithTimeOutNilAssertion(t, - "SCS with only secret-detection scan must complete successfully, even if missing scs-repo flags", 4*time.Minute, args...) + "SCS with only secret-detection scan must complete successfully, even if missing scs-repo flags", timeout, args...) } func TestCreateScan_WithScanTypesScsAndOnlyScorecardMissingRepoFlags_Fail(t *testing.T) { From 27ed70b6c8ed5abe0685141bee10c5a34861f4e0 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Sun, 25 Aug 2024 14:26:09 +0300 Subject: [PATCH 12/33] set global timeout --- test/integration/scan_test.go | 77 ++++++++++++++--------------------- 1 file changed, 31 insertions(+), 46 deletions(-) diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index 6bce58e25..a2e4506bb 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -866,9 +866,9 @@ func executeScanAssertions(t *testing.T, projectID, scanID string, tags map[stri func createScan(t *testing.T, source string, tags map[string]string) (string, string) { if isFFEnabled(t, wrappers.ContainerEngineCLIEnabled) { - return executeCreateScan(t, getCreateArgs(source, tags, "sast , sca , iac-security , api-security, container-security")) + return executeCreateScan(t, getCreateArgs(source, tags, "sast , sca , iac-security , api-security, container-security, scs")) } else { - return executeCreateScan(t, getCreateArgs(source, tags, "sast , sca , iac-security , api-security")) + return executeCreateScan(t, getCreateArgs(source, tags, "sast , sca , iac-security , api-security, scs")) } } @@ -924,7 +924,6 @@ func getCreateArgsWithName(source string, tags map[string]string, projectName, s return getCreateArgsWithNameAndGroups(source, tags, nil, projectName, scanTypes) } func getCreateArgsWithNameAndGroups(source string, tags map[string]string, groups []string, projectName, scanTypes string) []string { - scanTypes = removeSCSScanType(scanTypes) args := []string{ "scan", "create", @@ -944,20 +943,6 @@ func getCreateArgsWithNameAndGroups(source string, tags map[string]string, group return args } -func removeSCSScanType(scanTypes string) string { - if strings.Contains(scanTypes, "scs") { - types := strings.Split(scanTypes, ",") - for i, t := range types { - if t == "scs" { - types = append(types[:i], types[i+1:]...) - break - } - } - return strings.Join(types, ",") - } - return scanTypes -} - func executeCreateScan(t *testing.T, args []string) (string, string) { buffer := executeScanGetBuffer(t, args) @@ -1684,21 +1669,21 @@ func TestScanWithPolicyTimeout(t *testing.T) { assert.Error(t, err, "--policy-timeout should be equal or higher than 0") } -//func TestCreateScan_WithTypeScs_Success(t *testing.T) { -// _, projectName := getRootProject(t) -// -// args := []string{ -// "scan", "create", -// flag(params.ProjectName), projectName, -// flag(params.SourcesFlag), Zip, -// flag(params.ScanTypes), "scs", -// flag(params.BranchFlag), "main", -// flag(params.SCSRepoURLFlag), scsRepoURL, -// flag(params.SCSRepoTokenFlag), scsRepoToken, -// } -// -// executeCmdWithTimeOutNilAssertion(t, "SCS scan must complete successfully", 4*time.Minute, args...) -//} +func TestCreateScan_WithTypeScs_Success(t *testing.T) { + _, projectName := getRootProject(t) + + args := []string{ + "scan", "create", + flag(params.ProjectName), projectName, + flag(params.SourcesFlag), Zip, + flag(params.ScanTypes), "scs", + flag(params.BranchFlag), "main", + flag(params.SCSRepoURLFlag), scsRepoURL, + flag(params.SCSRepoTokenFlag), scsRepoToken, + } + + executeCmdWithTimeOutNilAssertion(t, "SCS scan must complete successfully", 4*time.Minute, args...) +} func TestCreateScan_WithNoScanTypesFlag_SuccessAndScsNotScanned(t *testing.T) { _, projectName := getRootProject(t) @@ -1715,20 +1700,20 @@ func TestCreateScan_WithNoScanTypesFlag_SuccessAndScsNotScanned(t *testing.T) { assert.Assert(t, !strings.Contains(output.String(), params.ScsType), "Scs scan must not run if all required flags are not provided") } -//func TestCreateScan_WithNoScanTypesFlagButScsFlagsPresent_SuccessAndScsScanned(t *testing.T) { -// _, projectName := getRootProject(t) -// args := []string{ -// "scan", "create", -// flag(params.ProjectName), projectName, -// flag(params.SourcesFlag), Zip, -// flag(params.BranchFlag), "main", -// flag(params.SCSRepoURLFlag), scsRepoURL, -// flag(params.SCSRepoTokenFlag), scsRepoToken, -// } -// -// output := executeCmdWithTimeOutNilAssertion(t, "Scan must complete successfully if no scan-types specified and scs-repo flags are present", 4*time.Minute, args...) -// assert.Assert(t, strings.Contains(output.String(), params.ScsType), "Scs scan should run if all required flags are provided") -//} +func TestCreateScan_WithNoScanTypesFlagButScsFlagsPresent_SuccessAndScsScanned(t *testing.T) { + _, projectName := getRootProject(t) + args := []string{ + "scan", "create", + flag(params.ProjectName), projectName, + flag(params.SourcesFlag), Zip, + flag(params.BranchFlag), "main", + flag(params.SCSRepoURLFlag), scsRepoURL, + flag(params.SCSRepoTokenFlag), scsRepoToken, + } + + output := executeCmdWithTimeOutNilAssertion(t, "Scan must complete successfully if no scan-types specified and scs-repo flags are present", 4*time.Minute, args...) + assert.Assert(t, strings.Contains(output.String(), params.ScsType), "Scs scan should run if all required flags are provided") +} func TestCreateScan_WithTypeScsMissingRepoURL_Fail(t *testing.T) { _, projectName := getRootProject(t) From 2a0d02afa5fd8a18101f3f29eca6da8f863b5fdf Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Mon, 26 Aug 2024 10:10:22 +0300 Subject: [PATCH 13/33] Revert TestCancelscan test --- test/integration/scan_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index a2e4506bb..10a7e95ca 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -586,7 +586,7 @@ func TestIncrementalScan(t *testing.T) { // Start a scan guaranteed to take considerable time, cancel it and assert the status func TestCancelScan(t *testing.T) { - scanID, projectID := createScanNoWait(t, SlowRepo, map[string]string{}, GenerateRandomProjectNameForScan()) + scanID, projectID := createScanSastNoWait(t, SlowRepo, map[string]string{}) defer deleteProject(t, projectID) defer deleteScan(t, scanID) From 6f82c740b84a55bd558d8daae6b5975fd92230ba Mon Sep 17 00:00:00 2001 From: AlvoBen <144705560+AlvoBen@users.noreply.github.com> Date: Mon, 26 Aug 2024 11:47:08 +0300 Subject: [PATCH 14/33] Update integration_up.sh --- internal/commands/.scripts/integration_up.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 00beec3d4..eda4cf7da 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -10,11 +10,13 @@ wget https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.g tar -xzvf ScaResolver-linux64.tar.gz -C /tmp rm -rf ScaResolver-linux64.tar.gz -gotestsum --rerun-fails=2 --format=short \ ---packages github.com/checkmarx/ast-cli/test/integration -- \ --tags=integration -v -timeout 210m \ --coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ --coverprofile cover.out +go test \ + -tags integration \ + -v \ + -timeout 210m \ + -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ + -coverprofile cover.out \ + github.com/checkmarx/ast-cli/test/integration From 6de1a18ce5ea73228f2e5875e3315a604347c5b6 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Tue, 27 Aug 2024 10:20:24 +0300 Subject: [PATCH 15/33] try implement re run tests mechanism --- internal/commands/.scripts/integration_up.sh | 80 +++++++++++++++----- 1 file changed, 61 insertions(+), 19 deletions(-) diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index eda4cf7da..927a40d58 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -1,30 +1,72 @@ -docker run \ - --name squid \ - -d \ - -p $PROXY_PORT:3128 \ - -v $(pwd)/internal/commands/.scripts/squid/squid.conf:/etc/squid/squid.conf \ - -v $(pwd)/internal/commands/.scripts/squid/passwords:/etc/squid/passwords \ - ubuntu/squid:5.2-22.04_beta +#!/bin/bash -wget https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.gz -tar -xzvf ScaResolver-linux64.tar.gz -C /tmp -rm -rf ScaResolver-linux64.tar.gz +# Step 1: Check if the failedTests file exists +FAILED_TESTS_FILE="failedTests" -go test \ - -tags integration \ - -v \ - -timeout 210m \ - -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ - -coverprofile cover.out \ - github.com/checkmarx/ast-cli/test/integration +if [ -f "$FAILED_TESTS_FILE" ]; then + # Step 2.1: If it exists, run all the tests listed in this file + echo "Running tests from $FAILED_TESTS_FILE..." + while IFS= read -r testName; do + go test \ + -tags integration \ + -v \ + -timeout 210m \ + -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ + -coverprofile cover.out \ + -run "^$testName$" \ + github.com/checkmarx/ast-cli/test/integration + done < "$FAILED_TESTS_FILE" +else + # Step 2.2: If not, create the failedTests file + echo "Creating $FAILED_TESTS_FILE..." + touch "$FAILED_TESTS_FILE" +fi +# Step 3: Run all tests and write failed test names to failedTests file +echo "Running all tests..." +go test \ + -tags integration \ + -v \ + -timeout 210m \ + -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ + -coverprofile cover.out \ + github.com/checkmarx/ast-cli/test/integration 2>&1 | tee test_output.log +grep -E "^--- FAIL: " test_output.log | awk '{print $3}' > "$FAILED_TESTS_FILE" status=$? echo "status value after tests $status" if [ $status -ne 0 ]; then echo "Integration tests failed" - rm cover.out fi -go tool cover -html=cover.out -o coverage.html +# Step 4: Check if failedTests file is empty +if [ ! -s "$FAILED_TESTS_FILE" ]; then + # If empty, exit with success + echo "All tests passed." + rm -f "$FAILED_TESTS_FILE" test_output.log + exit 0 +else + # If not empty, rerun the failed tests + echo "Rerunning failed tests..." + while IFS= read -r testName; do + go test \ + -tags integration \ + -v \ + -timeout 210m \ + -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ + -coverprofile cover.out \ + -run "^$testName$" \ + github.com/checkmarx/ast-cli/test/integration + done < "$FAILED_TESTS_FILE" + + # Check if any tests failed again + if [ -s "$FAILED_TESTS_FILE" ]; then + echo "Some tests are still failing." + exit 1 + else + echo "All failed tests passed on rerun." + rm -f "$FAILED_TESTS_FILE" test_output.log + exit 0 + fi +fi From 3b3f86b736467d20158293d12e14b3d2b6db89c1 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Tue, 27 Aug 2024 10:33:20 +0300 Subject: [PATCH 16/33] try implement re run tests mechanism --- internal/commands/.scripts/integration_up.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 927a40d58..fbcefa9c9 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -1,3 +1,15 @@ +docker run \ + --name squid \ + -d \ + -p $PROXY_PORT:3128 \ + -v $(pwd)/internal/commands/.scripts/squid/squid.conf:/etc/squid/squid.conf \ + -v $(pwd)/internal/commands/.scripts/squid/passwords:/etc/squid/passwords \ + ubuntu/squid:5.2-22.04_beta + +wget https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.gz +tar -xzvf ScaResolver-linux64.tar.gz -C /tmp +rm -rf ScaResolver-linux64.tar.gz + #!/bin/bash # Step 1: Check if the failedTests file exists @@ -70,3 +82,4 @@ else exit 0 fi fi + From e54ff25a6000be0fc3577f9c535897ac5310b903 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Tue, 27 Aug 2024 12:10:45 +0300 Subject: [PATCH 17/33] try implement re run tests mechanism --- internal/commands/.scripts/integration_up.sh | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index fbcefa9c9..9383712b2 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -1,15 +1,3 @@ -docker run \ - --name squid \ - -d \ - -p $PROXY_PORT:3128 \ - -v $(pwd)/internal/commands/.scripts/squid/squid.conf:/etc/squid/squid.conf \ - -v $(pwd)/internal/commands/.scripts/squid/passwords:/etc/squid/passwords \ - ubuntu/squid:5.2-22.04_beta - -wget https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.gz -tar -xzvf ScaResolver-linux64.tar.gz -C /tmp -rm -rf ScaResolver-linux64.tar.gz - #!/bin/bash # Step 1: Check if the failedTests file exists @@ -61,6 +49,7 @@ if [ ! -s "$FAILED_TESTS_FILE" ]; then else # If not empty, rerun the failed tests echo "Rerunning failed tests..." + rerun_status=0 while IFS= read -r testName; do go test \ -tags integration \ @@ -69,12 +58,13 @@ else -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ -coverprofile cover.out \ -run "^$testName$" \ - github.com/checkmarx/ast-cli/test/integration + github.com/checkmarx/ast-cli/test/integration || rerun_status=1 done < "$FAILED_TESTS_FILE" # Check if any tests failed again - if [ -s "$FAILED_TESTS_FILE" ]; then + if [ $rerun_status -eq 1 ]; then echo "Some tests are still failing." + rm -f "$FAILED_TESTS_FILE" test_output.log exit 1 else echo "All failed tests passed on rerun." @@ -82,4 +72,3 @@ else exit 0 fi fi - From c698aa9251b7ae0512367cdb0788e548adfb33a6 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Tue, 27 Aug 2024 12:11:50 +0300 Subject: [PATCH 18/33] try implement re run tests mechanism --- internal/commands/.scripts/integration_up.sh | 21 +++----------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 9383712b2..3fdfd8f85 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -3,24 +3,9 @@ # Step 1: Check if the failedTests file exists FAILED_TESTS_FILE="failedTests" -if [ -f "$FAILED_TESTS_FILE" ]; then - # Step 2.1: If it exists, run all the tests listed in this file - echo "Running tests from $FAILED_TESTS_FILE..." - while IFS= read -r testName; do - go test \ - -tags integration \ - -v \ - -timeout 210m \ - -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ - -coverprofile cover.out \ - -run "^$testName$" \ - github.com/checkmarx/ast-cli/test/integration - done < "$FAILED_TESTS_FILE" -else - # Step 2.2: If not, create the failedTests file - echo "Creating $FAILED_TESTS_FILE..." - touch "$FAILED_TESTS_FILE" -fi +# Step 2: Create the failedTests file +echo "Creating $FAILED_TESTS_FILE..." +touch "$FAILED_TESTS_FILE" # Step 3: Run all tests and write failed test names to failedTests file echo "Running all tests..." From 76c8f16b461c47354d9fb698083edd3ec2ff57ad Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Tue, 27 Aug 2024 12:17:31 +0300 Subject: [PATCH 19/33] try implement re run tests mechanism --- internal/commands/.scripts/integration_up.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 3fdfd8f85..a9c4404e3 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -46,6 +46,8 @@ else github.com/checkmarx/ast-cli/test/integration || rerun_status=1 done < "$FAILED_TESTS_FILE" + go tool cover -html=cover.out -o coverage.html + # Check if any tests failed again if [ $rerun_status -eq 1 ]; then echo "Some tests are still failing." From 257c869c6dd143940d4e886bfab7e39c7296dd04 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Tue, 27 Aug 2024 12:42:32 +0300 Subject: [PATCH 20/33] update integration_up.sh --- internal/commands/.scripts/integration_up.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index a9c4404e3..4ba3fbe3d 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -1,5 +1,14 @@ -#!/bin/bash +docker run \ + --name squid \ + -d \ + -p $PROXY_PORT:3128 \ + -v $(pwd)/internal/commands/.scripts/squid/squid.conf:/etc/squid/squid.conf \ + -v $(pwd)/internal/commands/.scripts/squid/passwords:/etc/squid/passwords \ + ubuntu/squid:5.2-22.04_beta +wget https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.gz +tar -xzvf ScaResolver-linux64.tar.gz -C /tmp +rm -rf ScaResolver-linux64.tar.gz # Step 1: Check if the failedTests file exists FAILED_TESTS_FILE="failedTests" From bf89a6802a54ecbb72ce4ef695620ab19968f578 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Tue, 27 Aug 2024 16:19:18 +0300 Subject: [PATCH 21/33] fix coverage --- internal/commands/.scripts/integration_up.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 4ba3fbe3d..3ccbd0c27 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -26,6 +26,9 @@ go test \ -coverprofile cover.out \ github.com/checkmarx/ast-cli/test/integration 2>&1 | tee test_output.log + +go tool cover -html=cover.out -o coverage.html + grep -E "^--- FAIL: " test_output.log | awk '{print $3}' > "$FAILED_TESTS_FILE" status=$? @@ -49,14 +52,10 @@ else -tags integration \ -v \ -timeout 210m \ - -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ - -coverprofile cover.out \ -run "^$testName$" \ github.com/checkmarx/ast-cli/test/integration || rerun_status=1 done < "$FAILED_TESTS_FILE" - go tool cover -html=cover.out -o coverage.html - # Check if any tests failed again if [ $rerun_status -eq 1 ]; then echo "Some tests are still failing." From 7e1b2fff125642226c524d3e522649aa49515fd2 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Tue, 27 Aug 2024 17:22:41 +0300 Subject: [PATCH 22/33] merge coverages --- .github/workflows/ci.yml | 4 ++-- internal/commands/.scripts/integration_up.sh | 24 ++++++++++++++++---- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0f697e72..10c6172fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,8 +43,8 @@ jobs: - run: go version - name: Go Build run: go build -o ./bin/cx ./cmd - - name: Install gotestsum - run: go install gotest.tools/gotestsum@latest + - name: Install gocovmerge + run: go install github.com/wadey/gocovmerge@latest - name: Go Integration test shell: bash env: diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 3ccbd0c27..853c58310 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -1,3 +1,4 @@ +# Start the Squid proxy in a Docker container docker run \ --name squid \ -d \ @@ -6,9 +7,11 @@ docker run \ -v $(pwd)/internal/commands/.scripts/squid/passwords:/etc/squid/passwords \ ubuntu/squid:5.2-22.04_beta +# Download and extract the ScaResolver tool wget https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.gz tar -xzvf ScaResolver-linux64.tar.gz -C /tmp rm -rf ScaResolver-linux64.tar.gz + # Step 1: Check if the failedTests file exists FAILED_TESTS_FILE="failedTests" @@ -26,11 +29,13 @@ go test \ -coverprofile cover.out \ github.com/checkmarx/ast-cli/test/integration 2>&1 | tee test_output.log - +# Generate the initial HTML coverage report go tool cover -html=cover.out -o coverage.html +# Extract names of failed tests and save them in the failedTests file grep -E "^--- FAIL: " test_output.log | awk '{print $3}' > "$FAILED_TESTS_FILE" +# Capture the exit status of the tests status=$? echo "status value after tests $status" if [ $status -ne 0 ]; then @@ -39,12 +44,12 @@ fi # Step 4: Check if failedTests file is empty if [ ! -s "$FAILED_TESTS_FILE" ]; then - # If empty, exit with success + # If the file is empty, all tests passed echo "All tests passed." rm -f "$FAILED_TESTS_FILE" test_output.log exit 0 else - # If not empty, rerun the failed tests + # If the file is not empty, rerun the failed tests echo "Rerunning failed tests..." rerun_status=0 while IFS= read -r testName; do @@ -52,11 +57,22 @@ else -tags integration \ -v \ -timeout 210m \ + -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ + -coverprofile cover_rerun.out \ -run "^$testName$" \ github.com/checkmarx/ast-cli/test/integration || rerun_status=1 done < "$FAILED_TESTS_FILE" - # Check if any tests failed again + # Step 5: Merge the original and rerun coverage profiles + if [ -f cover_rerun.out ]; then + echo "Merging coverage profiles..." + gocovmerge cover.out cover_rerun.out > merged_coverage.out + mv merged_coverage.out cover.out + go tool cover -html=cover.out -o coverage.html + rm -f cover_rerun.out + fi + + # Step 6: Check if any tests failed again if [ $rerun_status -eq 1 ]; then echo "Some tests are still failing." rm -f "$FAILED_TESTS_FILE" test_output.log From 14a097c2aa25800ba948485314a3c158ff11d91f Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Wed, 28 Aug 2024 09:50:47 +0300 Subject: [PATCH 23/33] make test independent --- test/integration/pr_test.go | 9 +++-- test/integration/scan_test.go | 72 ++++++++++++++++++----------------- 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/test/integration/pr_test.go b/test/integration/pr_test.go index f8cb5e9fd..8a18aae21 100644 --- a/test/integration/pr_test.go +++ b/test/integration/pr_test.go @@ -3,12 +3,13 @@ package integration import ( - "github.com/checkmarx/ast-cli/internal/commands/util" - "github.com/checkmarx/ast-cli/internal/logger" "os" "strings" "testing" + "github.com/checkmarx/ast-cli/internal/commands/util" + "github.com/checkmarx/ast-cli/internal/logger" + "github.com/checkmarx/ast-cli/internal/params" "gotest.tools/assert" ) @@ -118,7 +119,7 @@ func TestPRGitlabDecorationFailure(t *testing.T) { } func TestPRGithubDecoration_WhenScanIsRunning_ShouldAvoidPRDecorationCommand(t *testing.T) { - scanID, _ := createScanNoWait(t, Zip, Tags, GenerateRandomProjectNameForScan()) + scanID, _ := createScanNoWait(t, Zip, Tags, getProjectNameForScanTests()) args := []string{ "utils", "pr", @@ -149,7 +150,7 @@ func TestPRGithubDecoration_WhenScanIsRunning_ShouldAvoidPRDecorationCommand(t * } func TestPRGitlabDecoration_WhenScanIsRunning_ShouldAvoidPRDecorationCommand(t *testing.T) { - scanID, _ := createScanNoWait(t, Zip, Tags, GenerateRandomProjectNameForScan()) + scanID, _ := createScanNoWait(t, Zip, Tags, getProjectNameForScanTests()) args := []string{ "utils", "pr", diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index e5292215b..5f4d5fb42 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -83,7 +83,7 @@ func TestCreateScan_WithOnlyValidApikeyFlag_Success(t *testing.T) { args := []string{ "scan", "create", - flag(params.ProjectName), "project", + flag(params.ProjectName), getProjectNameForScanTests(), flag(params.SourcesFlag), "data/insecure.zip", flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", @@ -107,7 +107,7 @@ func TestCreateScan_WithOnlyValidApikeyEnvVar_Success(t *testing.T) { args := []string{ "scan", "create", - flag(params.ProjectName), "project", + flag(params.ProjectName), getProjectNameForScanTests(), flag(params.SourcesFlag), "data/insecure.zip", flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", @@ -131,7 +131,7 @@ func TestCreateScan_WithOnlyInvalidApikeyEnvVar_Fail(t *testing.T) { args := []string{ "scan", "create", - flag(params.ProjectName), "project", + flag(params.ProjectName), getProjectNameForScanTests(), flag(params.SourcesFlag), "data/insecure.zip", flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", @@ -155,7 +155,7 @@ func TestCreateScan_WithOnlyInvalidApikeyFlag_Fail(t *testing.T) { args := []string{ "scan", "create", - flag(params.ProjectName), "project", + flag(params.ProjectName), getProjectNameForScanTests(), flag(params.SourcesFlag), "data/insecure.zip", flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", @@ -180,7 +180,7 @@ func TestCreateScan_WithValidClientCredentialsFlag_Success(t *testing.T) { args := []string{ "scan", "create", - flag(params.ProjectName), "project", + flag(params.ProjectName), getProjectNameForScanTests(), flag(params.SourcesFlag), "data/insecure.zip", flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", @@ -207,7 +207,7 @@ func TestCreateScan_WithInvalidClientCredentialsFlag_Fail(t *testing.T) { args := []string{ "scan", "create", - flag(params.ProjectName), "project", + flag(params.ProjectName), getProjectNameForScanTests(), flag(params.SourcesFlag), "data/insecure.zip", flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", @@ -230,7 +230,7 @@ func TestCreateScan_WithValidClientCredentialsEnvVars_Success(t *testing.T) { args := []string{ "scan", "create", - flag(params.ProjectName), "project", + flag(params.ProjectName), getProjectNameForScanTests(), flag(params.SourcesFlag), "data/insecure.zip", flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", @@ -254,7 +254,7 @@ func TestCreateScan_WithInvalidClientCredentialsEnvVars_Fail(t *testing.T) { args := []string{ "scan", "create", - flag(params.ProjectName), "project", + flag(params.ProjectName), getProjectNameForScanTests(), flag(params.SourcesFlag), "data/insecure.zip", flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", @@ -298,7 +298,7 @@ func TestScanCreate_ExistingApplicationAndExistingProject_CreateScanSuccessfully args := []string{ "scan", "create", flag(params.ApplicationName), "my-application", - flag(params.ProjectName), "my-project", + flag(params.ProjectName), getProjectNameForScanTests(), flag(params.SourcesFlag), ".", flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", @@ -311,7 +311,7 @@ func TestScanCreate_ExistingApplicationAndExistingProject_CreateScanSuccessfully func TestScanCreate_FolderWithSymbolicLinkWithAbsolutePath_CreateScanSuccessfully(t *testing.T) { args := []string{ "scan", "create", - flag(params.ProjectName), "my-project", + flag(params.ProjectName), getProjectNameForScanTests(), flag(params.SourcesFlag), "data/project-with-directory-symlink", flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", @@ -323,7 +323,7 @@ func TestScanCreate_FolderWithSymbolicLinkWithAbsolutePath_CreateScanSuccessfull func TestScanCreate_FolderWithSymbolicLinkWithRelativePath_CreateScanSuccessfully(t *testing.T) { args := []string{ "scan", "create", - flag(params.ProjectName), "my-project", + flag(params.ProjectName), getProjectNameForScanTests(), flag(params.SourcesFlag), "data/symlink-relative-path-folder", flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", @@ -336,14 +336,13 @@ func TestScanCreate_ExistingApplicationAndNotExistingProject_CreatingNewProjectA args := []string{ "scan", "create", flag(params.ApplicationName), "my-application", - flag(params.ProjectName), projectNameRandom, + flag(params.ProjectName), GenerateRandomProjectNameForScan(), flag(params.SourcesFlag), ".", flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", flag(params.ScanInfoFormatFlag), printer.FormatJSON, } scanID, projectID := executeCreateScan(t, args) - defer deleteProject(t, projectID) assert.Assert(t, scanID != "", "Scan ID should not be empty") assert.Assert(t, projectID != "", "Project ID should not be empty") } @@ -352,7 +351,7 @@ func TestScanCreate_ApplicationDoesntExist_FailScanWithError(t *testing.T) { args := []string{ "scan", "create", flag(params.ApplicationName), "application-that-doesnt-exist", - flag(params.ProjectName), "my-project", + flag(params.ProjectName), getProjectNameForScanTests(), flag(params.SourcesFlag), ".", flag(params.ScanTypes), params.IacType, flag(params.BranchFlag), "dummy_branch", @@ -375,7 +374,6 @@ func TestContainerEngineScansE2E_ContainerImagesFlagAndScanType(t *testing.T) { } if isFFEnabled(t, wrappers.ContainerEngineCLIEnabled) { scanID, projectID := executeCreateScan(t, testArgs) - defer deleteProject(t, projectID) assert.Assert(t, scanID != "", "Scan ID should not be empty") assert.Assert(t, projectID != "", "Project ID should not be empty") } @@ -394,7 +392,6 @@ func TestContainerEngineScansE2E_ContainerImagesFlagOnly(t *testing.T) { } if isFFEnabled(t, wrappers.ContainerEngineCLIEnabled) { scanID, projectID := executeCreateScan(t, testArgs) - defer deleteProject(t, projectID) assert.Assert(t, scanID != "", "Scan ID should not be empty") assert.Assert(t, projectID != "", "Project ID should not be empty") } @@ -414,7 +411,6 @@ func TestContainerEngineScansE2E_ContainerImagesAndDebugFlags(t *testing.T) { } if isFFEnabled(t, wrappers.ContainerEngineCLIEnabled) { scanID, projectID := executeCreateScan(t, testArgs) - defer deleteProject(t, projectID) assert.Assert(t, scanID != "", "Scan ID should not be empty") assert.Assert(t, projectID != "", "Project ID should not be empty") } @@ -433,7 +429,6 @@ func TestContainerEngineScansE2E_ContainerImagesFlagAndEmptyFolderProject(t *tes } if isFFEnabled(t, wrappers.ContainerEngineCLIEnabled) { scanID, projectID := executeCreateScan(t, testArgs) - defer deleteProject(t, projectID) assert.Assert(t, scanID != "", "Scan ID should not be empty") assert.Assert(t, projectID != "", "Project ID should not be empty") } @@ -457,8 +452,8 @@ func TestContainerEngineScansE2E_InvalidContainerImagesFlag(t *testing.T) { // Create scans from current dir, zip and url and perform assertions in executeScanAssertions func TestScansE2E(t *testing.T) { - scanID, projectID := executeCreateScan(t, getCreateArgsWithGroups(Zip, Tags, Groups, "sast,iac-security,sca")) - defer deleteProject(t, projectID) + cleanupCxZipFiles(t) + scanID, projectID := executeCreateScan(t, getCreateArgsWithGroups(Zip, Tags, Groups, params.IacType)) executeScanAssertions(t, projectID, scanID, Tags) glob, err := filepath.Glob(filepath.Join(os.TempDir(), "cx*.zip")) @@ -469,11 +464,23 @@ func TestScansE2E(t *testing.T) { assert.Equal(t, len(glob), 0, "Zip file not removed") } +func cleanupCxZipFiles(t *testing.T) { + glob, err := filepath.Glob(filepath.Join(os.TempDir(), "cx*.zip")) + if err != nil { + t.Logf("Failed to search for cx*.zip files: %v", err) + } + for _, file := range glob { + err = os.Remove(file) + if err != nil { + t.Logf("Failed to remove file %s: %v", file, err) + } + } +} + func TestFastScan(t *testing.T) { projectName := getProjectNameForScanTests() // Create a scan scanID, projectID := createScanWithFastScan(t, Dir, projectName, map[string]string{}) - defer deleteProject(t, projectID) executeScanAssertions(t, projectID, scanID, map[string]string{}) } @@ -483,10 +490,10 @@ func createScanWithFastScan(t *testing.T, source string, name string, tags map[s } func TestScansUpdateProjectGroups(t *testing.T) { + cleanupCxZipFiles(t) scanID, projectID := executeCreateScan(t, getCreateArgs(Zip, Tags, params.IacType)) response := listScanByID(t, scanID) scanID, projectID = executeCreateScan(t, getCreateArgsWithNameAndGroups(Zip, Tags, Groups, response[0].ProjectName, params.IacType)) - defer deleteProject(t, projectID) executeScanAssertions(t, projectID, scanID, Tags) glob, err := filepath.Glob(filepath.Join(os.TempDir(), "cx*.zip")) @@ -499,7 +506,7 @@ func TestScansUpdateProjectGroups(t *testing.T) { func TestInvalidSource(t *testing.T) { args := []string{scanCommand, "create", - flag(params.ProjectName), "TestProject", + flag(params.ProjectName), getProjectNameForScanTests(), flag(params.SourcesFlag), "invalidSource", flag(params.ScanTypes), "sast", flag(params.BranchFlag), "dummy_branch"} @@ -533,8 +540,6 @@ func TestScaResolverArg(t *testing.T) { viper.GetString(resolverEnvVar), ) - defer deleteProject(t, projectID) - assert.Assert( t, pollScanUntilStatus(t, scanID, wrappers.ScanCompleted, FullScanWait, ScanPollSleep), @@ -547,7 +552,7 @@ func TestScaResolverArg(t *testing.T) { func TestScaResolverArgFailed(t *testing.T) { args := []string{ "scan", "create", - flag(params.ProjectName), "resolver", + flag(params.ProjectName), getProjectNameForScanTests(), flag(params.SourcesFlag), ".", flag(params.ScaResolverFlag), "./nonexisting", flag(params.ScanTypes), "iac-security,sca", @@ -559,7 +564,7 @@ func TestScaResolverArgFailed(t *testing.T) { args = []string{ "scan", "create", - flag(params.ProjectName), "resolver", + flag(params.ProjectName), getProjectNameForScanTests(), flag(params.SourcesFlag), ".", flag(params.ScaResolverFlag), viper.GetString(resolverEnvVar), flag(params.ScanTypes), "iac-security,sca", @@ -576,7 +581,6 @@ func TestIncrementalScan(t *testing.T) { projectName := getProjectNameForScanTests() scanID, projectID := createScanIncremental(t, Dir, projectName, map[string]string{}) - defer deleteProject(t, projectID) scanIDInc, projectIDInc := createScanIncremental(t, Dir, projectName, map[string]string{}) assert.Assert(t, projectID == projectIDInc, "Project IDs should match") @@ -587,9 +591,8 @@ func TestIncrementalScan(t *testing.T) { // Start a scan guaranteed to take considerable time, cancel it and assert the status func TestCancelScan(t *testing.T) { - scanID, projectID := createScanSastNoWait(t, SlowRepo, map[string]string{}) + scanID, _ := createScanSastNoWait(t, SlowRepo, map[string]string{}) - defer deleteProject(t, projectID) defer deleteScan(t, scanID) // canceling too quickly after creating fails the scan... @@ -664,7 +667,7 @@ func TestScansAPISecThresholdShouldBlock(t *testing.T) { createASTIntegrationTestCommand(t) testArgs := []string{ "scan", "create", - flag(params.ProjectName), "my-project", + flag(params.ProjectName), getProjectNameForScanTests(), flag(params.SourcesFlag), "data/sources.zip", flag(params.BranchFlag), "dummy_branch", flag(params.ScanInfoFormatFlag), printer.FormatJSON, @@ -1177,7 +1180,7 @@ func TestScanCreateWithSSHKey(t *testing.T) { } func TestScanGLReportValidation(t *testing.T) { - projectName := GenerateRandomProjectNameForScan() + projectName := getProjectNameForScanTests() args := []string{ "scan", "create", flag(params.ProjectName), projectName, @@ -1190,7 +1193,6 @@ func TestScanGLReportValidation(t *testing.T) { err, _ := executeCommand(t, args...) assert.NilError(t, err, err) - deleteProjectByName(t, projectName) reportFilePath := "./cx_result.gl-sca-report.json" schemaURL := "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dependency-scanning-report-format.json" @@ -1880,7 +1882,7 @@ func validateCheckmarxDomains(t *testing.T, usedDomainsInTests []string) { } func TestCreateScan_TwoScansWithSameBranchNameWithWhiteSpace_Success(t *testing.T) { - projectName := GenerateRandomProjectNameForScan() + projectName := getProjectNameForScanTests() args := []string{ scanCommand, "create", flag(params.ProjectName), projectName, @@ -1924,7 +1926,7 @@ func TestCreateAsyncScan_CallExportServiceBeforeScanFinishWithRetry_Success(t *t configuration.LoadConfiguration() args := []string{ "scan", "create", - flag(params.ProjectName), GenerateRandomProjectNameForScan(), + flag(params.ProjectName), getProjectNameForScanTests(), flag(params.SourcesFlag), "data/empty-folder.zip", flag(params.ScanTypes), "sca", flag(params.BranchFlag), "main", From 1aff8dedccf2ae9171cecfb489db9f7c33a0b828 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Wed, 28 Aug 2024 11:45:43 +0300 Subject: [PATCH 24/33] make test independent --- test/integration/scan_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index 5f4d5fb42..bca8596d7 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -1882,7 +1882,7 @@ func validateCheckmarxDomains(t *testing.T, usedDomainsInTests []string) { } func TestCreateScan_TwoScansWithSameBranchNameWithWhiteSpace_Success(t *testing.T) { - projectName := getProjectNameForScanTests() + projectName := GenerateRandomProjectNameForScan() args := []string{ scanCommand, "create", flag(params.ProjectName), projectName, From 56f3cfc7a6dae4487a12da5795adb9fff323caaa Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Wed, 28 Aug 2024 11:50:05 +0300 Subject: [PATCH 25/33] make test independent --- test/integration/scan_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index bca8596d7..0ba22c6fa 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -1028,7 +1028,7 @@ func pollScanUntilStatus(t *testing.T, scanID string, requiredStatus wrappers.Sc // Get a scan workflow and assert it fails func TestScanWorkflow(t *testing.T) { - scanID, _ := getRootScan(t) + scanID := "fake-scan-id" args := []string{ "scan", "workflow", flag(params.ScanIDFlag), scanID, From 1a2279c7951ad9095f71bf9b05bd9687367273a8 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Wed, 28 Aug 2024 12:22:14 +0300 Subject: [PATCH 26/33] add ckean data flow --- internal/commands/.scripts/integration_up.sh | 18 +++++-- test/cleandata/clean-data_test.go | 49 ++++++++++++++++++++ test/integration/root_test.go | 1 + test/integration/scan_test.go | 18 ++++++- 4 files changed, 80 insertions(+), 6 deletions(-) create mode 100644 test/cleandata/clean-data_test.go diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 853c58310..2deac8285 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -47,7 +47,6 @@ if [ ! -s "$FAILED_TESTS_FILE" ]; then # If the file is empty, all tests passed echo "All tests passed." rm -f "$FAILED_TESTS_FILE" test_output.log - exit 0 else # If the file is not empty, rerun the failed tests echo "Rerunning failed tests..." @@ -75,11 +74,20 @@ else # Step 6: Check if any tests failed again if [ $rerun_status -eq 1 ]; then echo "Some tests are still failing." - rm -f "$FAILED_TESTS_FILE" test_output.log - exit 1 else echo "All failed tests passed on rerun." - rm -f "$FAILED_TESTS_FILE" test_output.log - exit 0 fi fi + +# Step 7: Run the cleandata package to delete projects +echo "Running cleandata to clean up projects..." +go test -v github.com/checkmarx/ast-cli/internal/cleandata + +# Step 8: Final cleanup and exit +rm -f "$FAILED_TESTS_FILE" test_output.log + +if [ $status -ne 0 ] || [ $rerun_status -eq 1 ]; then + exit 1 +else + exit 0 +fi diff --git a/test/cleandata/clean-data_test.go b/test/cleandata/clean-data_test.go new file mode 100644 index 000000000..8f0c356e7 --- /dev/null +++ b/test/cleandata/clean-data_test.go @@ -0,0 +1,49 @@ +package cleandata + +import ( + "bufio" + "fmt" + "log" + "os" + "testing" + + "github.com/checkmarx/ast-cli/internal/params" + "github.com/checkmarx/ast-cli/internal/wrappers" + "github.com/spf13/viper" +) + +const ProjectNameFile = "projectName.txt" + +func DeleteProjectByName(projectName string) { + projectsWrapper := wrappers.NewHTTPProjectsWrapper(viper.GetString(params.ProjectsPathKey)) + projectModel, _, err := projectsWrapper.GetByName(projectName) + if err == nil && projectModel != nil { + _, _ = projectsWrapper.Delete(projectModel.ID) + } +} + +func TestDeleteProjectsFromFile(t *testing.T) { + projectNameFile := fmt.Sprint("../integration/", ProjectNameFile) // Replace with your actual file path + + file, err := os.Open(projectNameFile) + if err != nil { + log.Printf("Failed to open project name file: %v", err) + } + defer func(file *os.File) { + if err := file.Close(); err != nil { + log.Printf("Failed to close file: %v", err) + } + }(file) + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + projectName := scanner.Text() + log.Printf("Attempting to delete project: %s", projectName) + DeleteProjectByName(projectName) + log.Printf("Project deleted: %s", projectName) + } + + if err := scanner.Err(); err != nil { + log.Printf("Error reading project name file: %v", err) + } +} diff --git a/test/integration/root_test.go b/test/integration/root_test.go index 097dae1ad..cf2c0754e 100644 --- a/test/integration/root_test.go +++ b/test/integration/root_test.go @@ -104,6 +104,7 @@ func getRootProject(t *testing.T) (string, string) { } rootProjectId, rootProjectName = createProject(t, Tags, Groups) + _ = WriteProjectNameToFile(rootProjectName) return rootProjectId, rootProjectName } diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index 0ba22c6fa..f6a998a16 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -54,6 +54,7 @@ const ( invalidAPIKey = "invalidAPI" invalidTenant = "invalidTenant" timeout = 10 * time.Minute + ProjectNameFile = "projectName.txt" ) var ( @@ -1940,5 +1941,20 @@ func TestCreateAsyncScan_CallExportServiceBeforeScanFinishWithRetry_Success(t *t } func GenerateRandomProjectNameForScan() string { - return fmt.Sprint("ast-cli-scan-", uuid.New().String()) + projectName := fmt.Sprintf("ast-cli-scan-%s", uuid.New().String()) + _ = WriteProjectNameToFile(projectName) + return projectName +} + +func WriteProjectNameToFile(projectName string) error { + f, err := os.OpenFile(ProjectNameFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer f.Close() + + if _, err := f.WriteString(projectName); err != nil { + return err + } + return nil } From 069d8b27c08c94d037deb5c6c6ae044b24c24f17 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Wed, 28 Aug 2024 12:32:31 +0300 Subject: [PATCH 27/33] add ckean data flow --- test/integration/result_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/integration/result_test.go b/test/integration/result_test.go index fb2b827de..d9d2ff340 100644 --- a/test/integration/result_test.go +++ b/test/integration/result_test.go @@ -105,8 +105,6 @@ func TestResultListJson(t *testing.T) { assert.Assert(t, uint(len(result.Results)) == result.TotalCount, "Should have results") assertResultFilesCreated(t) - - deleteScanAndProject() } // assert all files were created @@ -152,8 +150,6 @@ func TestResultListForGlReports(t *testing.T) { assert.Assert(t, uint(len(result.Results)) == result.TotalCount, "Should have results") assertGlResultFilesCreated(t) - - deleteScanAndProject() } func assertGlResultFilesCreated(t *testing.T) { From c2c0ef0985068ba234c37519cf7a302a1a47c3f8 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Wed, 28 Aug 2024 12:34:57 +0300 Subject: [PATCH 28/33] change rerun timeout to 60m --- internal/commands/.scripts/integration_up.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 2deac8285..69cafd976 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -55,7 +55,7 @@ else go test \ -tags integration \ -v \ - -timeout 210m \ + -timeout 30m \ -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ -coverprofile cover_rerun.out \ -run "^$testName$" \ From 025d7e34d91e87ba5e06812e54fba8911a37cbfa Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Wed, 28 Aug 2024 12:48:47 +0300 Subject: [PATCH 29/33] change rerun timeout to 60m --- internal/commands/.scripts/integration_up.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 69cafd976..62708467b 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -81,7 +81,7 @@ fi # Step 7: Run the cleandata package to delete projects echo "Running cleandata to clean up projects..." -go test -v github.com/checkmarx/ast-cli/internal/cleandata +go test -v github.com/checkmarx/ast-cli/test/cleandata # Step 8: Final cleanup and exit rm -f "$FAILED_TESTS_FILE" test_output.log From 3a1baab0e28cf339e11eada4cadb680a24985ea4 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Wed, 28 Aug 2024 13:08:01 +0300 Subject: [PATCH 30/33] change rerun timeout to 60m --- test/integration/scan_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index f6a998a16..719d8b973 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -1953,7 +1953,7 @@ func WriteProjectNameToFile(projectName string) error { } defer f.Close() - if _, err := f.WriteString(projectName); err != nil { + if _, err := f.WriteString(projectName + "\n"); err != nil { return err } return nil From d6766ae5a6ae0d2cfba48877ed2451bee3f46d42 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Wed, 28 Aug 2024 13:12:21 +0300 Subject: [PATCH 31/33] fix clean data --- test/cleandata/clean-data_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/cleandata/clean-data_test.go b/test/cleandata/clean-data_test.go index 8f0c356e7..3a6559af8 100644 --- a/test/cleandata/clean-data_test.go +++ b/test/cleandata/clean-data_test.go @@ -9,6 +9,7 @@ import ( "github.com/checkmarx/ast-cli/internal/params" "github.com/checkmarx/ast-cli/internal/wrappers" + "github.com/checkmarx/ast-cli/internal/wrappers/configuration" "github.com/spf13/viper" ) @@ -23,6 +24,7 @@ func DeleteProjectByName(projectName string) { } func TestDeleteProjectsFromFile(t *testing.T) { + configuration.LoadConfiguration() projectNameFile := fmt.Sprint("../integration/", ProjectNameFile) // Replace with your actual file path file, err := os.Open(projectNameFile) From 6abb12aa325a710519d81db3b4dabf019750e049 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Wed, 28 Aug 2024 14:35:31 +0300 Subject: [PATCH 32/33] fix clean data --- test/integration/root_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/integration/root_test.go b/test/integration/root_test.go index cf2c0754e..c2d63a0d0 100644 --- a/test/integration/root_test.go +++ b/test/integration/root_test.go @@ -104,7 +104,11 @@ func getRootProject(t *testing.T) (string, string) { } rootProjectId, rootProjectName = createProject(t, Tags, Groups) + + //--------------------Write project name to file to delete it later-------------------- + _ = WriteProjectNameToFile(getProjectNameForTest()) _ = WriteProjectNameToFile(rootProjectName) + //------------------------------------------------------------------------------------- return rootProjectId, rootProjectName } From 3808b5a2810c757a1a91771dacd062869db9a394 Mon Sep 17 00:00:00 2001 From: AlvoBen Date: Wed, 28 Aug 2024 16:04:58 +0300 Subject: [PATCH 33/33] fix clean data --- test/integration/root_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/root_test.go b/test/integration/root_test.go index c2d63a0d0..e5346ade7 100644 --- a/test/integration/root_test.go +++ b/test/integration/root_test.go @@ -106,7 +106,7 @@ func getRootProject(t *testing.T) (string, string) { rootProjectId, rootProjectName = createProject(t, Tags, Groups) //--------------------Write project name to file to delete it later-------------------- - _ = WriteProjectNameToFile(getProjectNameForTest()) + _ = WriteProjectNameToFile(fmt.Sprint(getProjectNameForTest(), "_for_scan")) _ = WriteProjectNameToFile(rootProjectName) //-------------------------------------------------------------------------------------