Improve Local Testing experience (#114)

* Added local_mode and cli args. Calculate local_identifier based on local_mode. * Adding Local Binary start and stop logic * Adding demand message for local-identifier and local-mode options * Adding log messages for sync and local mode * upgrading yargs and putting requiredArgs * passed already failing 12 cases * added new test cases for the modified code * added new test cases and passed already failing test cases * removed unwanted files * Checking local identifier running in always on mode * local inferred, local mode inferred, sync inferred. - Adding checks and params for local, local mode, and sync inferred. * passed all failing tests and added new specs * Adding local start and stop error * Updated validation messages for local cli * Send the value of local_mode not the true/false * Bug fixes for callback stop and start binary * Removing bshost. * Adding local_config_file for passing configuration for local binary * Updating inferred event logic - If the user passed anything explicitly, then it is not inferred. - This is to understand which of the "auto-fallback" flows does CLI fall in. - Inverted the logic of the *_inferred events. * updating user agent to 1.7.2 * written extra specs and passed failing spec * removed unwanted log files * covered new functions under test * Modified local unit tests * Fixing Unit test cases * removed local object creation * Indentation and minor null pointer fixes. * Adding placeholders for local_mode and local_config_file in default json * Local Start fail error. * Fix Local Inferred logic * Fixed local_mode not set to always-on when local_identifier is supplied in browserstack.json * Send local_identifier_error when local_identifier not running in stopBinary * Local mode and id validation * Added test cases for local testing bug fixes * Warning message in case of local_mode is invalid * Updated message for invalid local config file * Fixed cli args precendence on-demand bug * Updating version to 1.8.0 Co-authored-by: Surya Tripathi <[email protected]> Co-authored-by: roshan <[email protected]>
browserstack · Mar 12, 2021 · ff9be4d · ff9be4d
1 parent 69da94e
commit ff9be4d
Show file tree

Hide file tree

Showing 11 changed files with 970 additions and 69 deletions.
diff --git a/bin/commands/runs.js b/bin/commands/runs.js
@@ -40,10 +40,16 @@ module.exports = function run(args) {
     utils.setTestEnvs(bsConfig, args);
 
     //accept the local from env variable if provided
-    utils.setLocal(bsConfig);
+    utils.setLocal(bsConfig, args);
+
+    // set Local Mode (on-demand/ always-on)
+    utils.setLocalMode(bsConfig, args);
 
     //accept the local identifier from env variable if provided
-    utils.setLocalIdentifier(bsConfig);
+    utils.setLocalIdentifier(bsConfig, args);
+
+    // set Local Config File
+    utils.setLocalConfigFile(bsConfig, args);
 
     // run test in headed mode
     utils.setHeaded(bsConfig, args);
@@ -61,8 +67,12 @@ module.exports = function run(args) {
       return archiver.archive(bsConfig.run_settings, config.fileName, args.exclude).then(function (data) {
 
         // Uploaded zip file
-        return zipUploader.zipUpload(bsConfig, config.fileName).then(function (zip) {
+        return zipUploader.zipUpload(bsConfig, config.fileName).then(async function (zip) {
           // Create build
+
+          //setup Local Testing
+          let bs_local = await utils.setupLocalTesting(bsConfig, args);
+
           return build.createBuild(bsConfig, zip).then(function (data) {
             let message = `${data.message}! ${Constants.userMessages.BUILD_CREATED} with build id: ${data.build_id}`;
             let dashboardLink = `${Constants.userMessages.VISIT_DASHBOARD} ${data.dashboard_url}`;
@@ -82,7 +92,11 @@ module.exports = function run(args) {
             }
 
             if (args.sync) {
-              syncRunner.pollBuildStatus(bsConfig, data).then((exitCode) => {
+              syncRunner.pollBuildStatus(bsConfig, data).then(async (exitCode) => {
+
+                // stop the Local instance
+                await utils.stopLocalBinary(bsConfig, bs_local, args);
+
                 // Generate custom report!
                 reportGenerator(bsConfig, data.build_id, args, function(){
                   utils.sendUsageReport(bsConfig, args, `${message}\n${dashboardLink}`, Constants.messageTypes.SUCCESS, null);
@@ -96,17 +110,24 @@ module.exports = function run(args) {
             if(!args.sync) logger.info(Constants.userMessages.EXIT_SYNC_CLI_MESSAGE.replace("<build-id>",data.build_id));
             utils.sendUsageReport(bsConfig, args, `${message}\n${dashboardLink}`, Constants.messageTypes.SUCCESS, null);
             return;
-          }).catch(function (err) {
+          }).catch(async function (err) {
             // Build creation failed
             logger.error(err);
+            // stop the Local instance
+            await utils.stopLocalBinary(bsConfig, bs_local, args);
+
             utils.sendUsageReport(bsConfig, args, err, Constants.messageTypes.ERROR, 'build_failed');
           });
         }).catch(function (err) {
-          // Zip Upload failed
+          // Zip Upload failed | Local Start failed 
           logger.error(err);
-          logger.error(Constants.userMessages.ZIP_UPLOAD_FAILED);
-          fileHelpers.deleteZip();
-          utils.sendUsageReport(bsConfig, args, `${err}\n${Constants.userMessages.ZIP_UPLOAD_FAILED}`, Constants.messageTypes.ERROR, 'zip_upload_failed');
+          if(err === Constants.userMessages.LOCAL_START_FAILED){
+            utils.sendUsageReport(bsConfig, args, `${err}\n${Constants.userMessages.LOCAL_START_FAILED}`, Constants.messageTypes.ERROR, 'local_start_failed');
+          } else {
+            logger.error(Constants.userMessages.ZIP_UPLOAD_FAILED);
+            fileHelpers.deleteZip();
+            utils.sendUsageReport(bsConfig, args, `${err}\n${Constants.userMessages.ZIP_UPLOAD_FAILED}`, Constants.messageTypes.ERROR, 'zip_upload_failed');
+          }
         });
       }).catch(function (err) {
         // Zipping failed

diff --git a/bin/helpers/capabilityHelper.js b/bin/helpers/capabilityHelper.js
@@ -42,18 +42,47 @@ const caps = (bsConfig, zip) => {
       reject("Test suite is empty");
     }
 
+    // Inferred settings
+    if(bsConfig.connection_settings){
+      if (bsConfig.connection_settings.local_mode_inferred) {
+        obj.local_mode_inferred = bsConfig.connection_settings.local_mode_inferred;
+      }
+
+      if (bsConfig.connection_settings.local_inferred) {
+        obj.local_inferred = bsConfig.connection_settings.local_inferred;
+      }
+
+      if (bsConfig.connection_settings.sync_inferred) {
+        obj.sync_inferred = bsConfig.connection_settings.sync_inferred;
+        logger.info('Setting "sync" mode to enable Local testing.');
+      }
+    }
+
     // Local
     obj.local = false;
-    if (bsConfig.connection_settings && bsConfig.connection_settings.local === true) obj.local = true;
-    logger.info(`Local is set to: ${obj.local} (${obj.local ? Constants.userMessages.LOCAL_TRUE : Constants.userMessages.LOCAL_FALSE})`);
+    if (bsConfig.connection_settings && bsConfig.connection_settings.local === true) {
+      obj.local = true;
+    }
+
+    obj.localMode = null;
+    // Local Mode
+    if (obj.local === true && bsConfig.connection_settings.local_mode) {
+      obj.localMode = bsConfig.connection_settings.local_mode;
+      if (bsConfig.connection_settings.user_defined_local_mode_warning) {
+        logger.warn(Constants.userMessages.INVALID_LOCAL_MODE_WARNING);
+      }
+      logger.info(`Local testing set up in ${obj.localMode} mode.`);
+    }
 
     // Local Identifier
     obj.localIdentifier = null;
     if (obj.local === true && (bsConfig.connection_settings.localIdentifier || bsConfig.connection_settings.local_identifier)) {
       obj.localIdentifier = bsConfig.connection_settings.localIdentifier || bsConfig.connection_settings.local_identifier;
-      logger.log(`Local Identifier is set to: ${obj.localIdentifier}`);
+      logger.info(`Local testing identifier: ${obj.localIdentifier}`);
     }
 
+    logger.info(`Local is set to: ${obj.local} (${obj.local ? Constants.userMessages.LOCAL_TRUE : Constants.userMessages.LOCAL_FALSE})`);
+
     // Project name
     obj.project = "project-name";
     // Build name
@@ -94,9 +123,9 @@ const caps = (bsConfig, zip) => {
 
     if(obj.parallels === Constants.cliMessages.RUN.DEFAULT_PARALLEL_MESSAGE) obj.parallels = undefined
 
-    if (obj.project) logger.log(`Project name is: ${obj.project}`);
+    if (obj.project) logger.info(`Project name is: ${obj.project}`);
 
-    if (obj.customBuildName) logger.log(`Build name is: ${obj.customBuildName}`);
+    if (obj.customBuildName) logger.info(`Build name is: ${obj.customBuildName}`);
 
     if (obj.callbackURL) logger.info(`callback url is : ${obj.callbackURL}`);
 
@@ -132,6 +161,16 @@ const validate = (bsConfig, args) => {
     // if parallels specified via arguments validate only arguments
     if (!Utils.isUndefined(args) && !Utils.isUndefined(args.parallels) && !Utils.isParallelValid(args.parallels)) reject(Constants.validationMessages.INVALID_PARALLELS_CONFIGURATION);
 
+    // validate local args i.e --local-mode and --local-identifier
+
+    if( Utils.searchForOption('--local-identifier') && (Utils.isUndefined(args.localIdentifier) || (!Utils.isUndefined(args.localIdentifier) && !args.localIdentifier.trim()))) reject(Constants.validationMessages.INVALID_CLI_LOCAL_IDENTIFIER);
+
+    if( Utils.getLocalFlag(bsConfig.connection_settings) && (Utils.isUndefined(bsConfig["connection_settings"]["local_identifier"]) || ( !Utils.isUndefined(bsConfig["connection_settings"]["local_identifier"]) && !bsConfig["connection_settings"]["local_identifier"].trim()))) reject(Constants.validationMessages.INVALID_LOCAL_IDENTIFIER);
+
+    if( Utils.searchForOption('--local-mode') && ( Utils.isUndefined(args.localMode) || (!Utils.isUndefined(args.localMode) && !["always-on","on-demand"].includes(args.localMode)))) reject(Constants.validationMessages.INVALID_LOCAL_MODE);
+
+    if( Utils.searchForOption('--local-config-file') && ( Utils.isUndefined(args.localConfigFile) || (!Utils.isUndefined(args.localConfigFile) && !fs.existsSync(args.localConfigFile)))) reject(Constants.validationMessages.INVALID_LOCAL_CONFIG_FILE);
+
     // validate if config file provided exists or not when cypress_config_file provided
     // validate the cypressProjectDir key otherwise.
     let cypressConfigFilePath = bsConfig.run_settings.cypressConfigFilePath;

diff --git a/bin/helpers/config.json b/bin/helpers/config.json
@@ -2,5 +2,6 @@
   "uploadUrl": "https://api-cloud.browserstack.com/automate-frameworks/cypress/upload",
   "rails_host": "https://api.browserstack.com",
   "dashboardUrl": "https://automate.browserstack.com/dashboard/v2/builds/",
-  "usageReportingUrl": "https://eds.browserstack.com:443/send_event_cy_internal"
+  "usageReportingUrl": "https://eds.browserstack.com:443/send_event_cy_internal",
+  "localTestingListUrl": "https://www.browserstack.com/local/v1/list"
 }
diff --git a/bin/helpers/constants.js b/bin/helpers/constants.js
@@ -37,7 +37,10 @@ const userMessages = {
   FATAL_NETWORK_ERROR: `fatal: unable to access '${config.buildUrl}': Could not resolve host: ${config.rails_host}`,
   RETRY_LIMIT_EXCEEDED: `Max retries exceeded trying to connect to the host (retries: ${config.retries})`,
   CHECK_DASHBOARD_AT: "Please check the build status at: ",
-  CYPRESS_VERSION_CHANGED: "Your build will run using Cypress <actualVersion> instead of Cypress <preferredVersion>. Read more about supported versions here: http://browserstack.com/docs/automate/cypress/supported-versions"
+  CYPRESS_VERSION_CHANGED: "Your build will run using Cypress <actualVersion> instead of Cypress <preferredVersion>. Read more about supported versions here: http://browserstack.com/docs/automate/cypress/supported-versions",
+  LOCAL_START_FAILED: "Local Testing setup failed.",
+  LOCAL_STOP_FAILED: "Local Binary stop failed.",
+  INVALID_LOCAL_MODE_WARNING: "Invalid value specified for local_mode. local_mode: (\"always-on\" | \"on-demand\"). For more info, check out https://www.browserstack.com/docs/automate/cypress/cli-reference"
 };
 
 const validationMessages = {
@@ -57,7 +60,11 @@ const validationMessages = {
   INVALID_CYPRESS_JSON: "cypress.json is not a valid json",
   INVALID_DEFAULT_AUTH_PARAMS: "Your username and access key are required to run your tests on BrowserStack. Learn more at https://www.browserstack.com/docs/automate/cypress/authentication",
   LOCAL_NOT_SET: "To test <baseUrlValue> on BrowserStack, you will have to set up Local testing. Read more here: https://www.browserstack.com/docs/automate/cypress/local-testing",
-  INCORRECT_DIRECTORY_STRUCTURE: "No tests to run. Note that your Cypress tests should be in the same directory where the cypress.json exists."
+  INCORRECT_DIRECTORY_STRUCTURE: "No tests to run. Note that your Cypress tests should be in the same directory where the cypress.json exists.",
+  INVALID_CLI_LOCAL_IDENTIFIER: "When using --local-identifier, a value needs to be supplied. \n--local-identifier <String>.\nFor more info, check out https://www.browserstack.com/docs/automate/cypress/cli-reference",
+  INVALID_LOCAL_MODE: "When using --local-mode, a value needs to be supplied. \n--local-mode (\"always-on\" | \"on-demand\").\nFor more info, check out https://www.browserstack.com/docs/automate/cypress/cli-reference",
+  INVALID_LOCAL_CONFIG_FILE: "Using --local-config-file requires an input of the form /path/to/config-file.yml.\nFor more info, check out https://www.browserstack.com/docs/automate/cypress/cli-reference",
+  INVALID_LOCAL_IDENTIFIER: "Invalid value specified for local_identifier. For more info, check out https://www.browserstack.com/docs/automate/cypress/cli-reference"
 };
 
 const cliMessages = {
@@ -94,6 +101,10 @@ const cliMessages = {
     SYNC_DESCRIPTION: "Makes the run command in sync",
     BUILD_REPORT_MESSAGE: "See the entire build report here",
     HEADED: "Run your tests in a headed browser instead of a headless browser",
+    LOCAL: "Accepted values: (true | false) - create a local testing connection to let you test staging and localhost websites, or sites behind proxies; learn more at browserstack.com/local-testing",
+    LOCAL_MODE: 'Accepted values: ("always-on" | "on-demand") - if you choose to keep the binary "always-on", it will speed up your tests by keeping the Local connection warmed up in the background; otherwise, you can choose to have it spawn and killed for every build',
+    LOCAL_IDENTIFIER: "Accepted values: String - assign an identifier to your Local process instance",
+    LOCAL_CONFIG_FILE: "Accepted values: String - path to local config-file to your Local process instance. Learn more at https://www.browserstack.com/local-testing/binary-params"
   },
   COMMON: {
     DISABLE_USAGE_REPORTING: "Disable usage reporting",

diff --git a/bin/helpers/usageReporting.js b/bin/helpers/usageReporting.js
@@ -231,5 +231,6 @@ function send(args) {
 }
 
 module.exports = {
-  send
-}
+  send,
+  cli_version_and_path,
+};
-Original file line number
+Diff line change
@@ Expand Up / @@ -231,5 +231,6 @@ function send(args) { @@
     }
     module.exports = {
-      send
-    }
+      send,
+      cli_version_and_path,
+    };