diff --git a/x-pack/plugins/osquery/cypress/e2e/all/packs_integration.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/packs_integration.cy.ts
index 30b8a1641fb94..9e10e1eb0d889 100644
--- a/x-pack/plugins/osquery/cypress/e2e/all/packs_integration.cy.ts
+++ b/x-pack/plugins/osquery/cypress/e2e/all/packs_integration.cy.ts
@@ -224,6 +224,7 @@ describe('ALL - Packs', { tags: ['@ess', '@serverless'] }, () => {
         cy.getBySel('createAgentPolicyNameField').type(agentPolicy);
         cy.getBySel('createAgentPolicyFlyoutBtn').click();
         cy.contains(`Agent policy '${agentPolicy}' created`).click();
+        closeToastIfVisible();
         cy.contains(agentPolicy).click();
         cy.contains('Add integration').click();
         cy.getBySel('epmList.searchBar').type('osquery');
diff --git a/x-pack/plugins/osquery/package.json b/x-pack/plugins/osquery/package.json
index 4e9070d780d28..a708e9a383031 100644
--- a/x-pack/plugins/osquery/package.json
+++ b/x-pack/plugins/osquery/package.json
@@ -12,7 +12,7 @@
     "cypress:run": "yarn cypress run",
     "cypress:serverless": "NODE_OPTIONS=--openssl-legacy-provider node ../security_solution/scripts/start_cypress_parallel --config-file ../osquery/cypress/serverless_cypress.config.ts --ftr-config-file ../../../x-pack/test/osquery_cypress/serverless_cli_config",
     "cypress:serverless:open": "yarn cypress:serverless open",
-    "cypress:serverless:run": "yarn cypress:serverless run",
+    "cypress:serverless:run": "yarn cypress:serverless run --spec ./cypress/e2e/all/packs_integration.cy.ts",
     "nyc": "../../../node_modules/.bin/nyc report --reporter=text-summary",
     "junit:merge": "../../../node_modules/.bin/mochawesome-merge ../../../target/kibana-osquery/cypress/results/mochawesome*.json > ../../../target/kibana-osquery/cypress/results/output.json && ../../../node_modules/.bin/marge ../../../target/kibana-osquery/cypress/results/output.json --reportDir ../../../target/kibana-osquery/cypress/results && yarn junit:transform && mkdir -p ../../../target/junit && cp ../../../target/kibana-osquery/cypress/results/*.xml ../../../target/junit/",
     "junit:transform": "node ../security_solution/scripts/junit_transformer --pathPattern '../../../target/kibana-osquery/cypress/results/*.xml' --rootDirectory ../../../ --reportName 'Osquery Cypress' --writeInPlace",