Disabling experiments. New mode: off

.github/workflows/build.yml

@@ -37,7 +37,7 @@ jobs:
  - uses: actions/setup-node@v3
  with:
  node-version-file: '.nvmrc'
- - run: echo "EXPERIMENT_GROUP=autoassign" >> $GITHUB_ENV
+ - run: echo "EXPERIMENT_GROUP=off" >> $GITHUB_ENV
  - run: npm install
  - run: npm run build:prod
  - run: npm run pack:html

README.md

@@ -95,6 +95,7 @@ Los grupos experimentales pueden ser: `notAffected`, `control`, `treatment` y `a
 - `control`: al finalizar un desafío muestra las expectativas existentes para el mismo, sin informar si se cumplieron o no.
 - `treatment`: tiene un período de entrenamiento en el cual muestra las expectativas existentes sobre el desafío, junto con una barra de progreso para las que se cumplieron. Se agrega feedback sobre los bloques. Una vez terminado el período de entrenamiento pasa a tener un comportamiento similar a *control*.
 - `autoassign`: asigna aleatoriamente un grupo experimental en tiempo de ejecución (elige entre `control` y `treatment).
+- `off`: Siempre muestra las expectativas y la barra de progreso. Deja de mostrar la encuesta de preguntas al usuario.
 
 ### Empaquetando instaladores:
 

README_en.md

@@ -114,12 +114,13 @@ In Windows this is done with:
 SET "EXPERIMENT_GROUP=treatment" && npm run start
 ```
 
-The experiment groups are: `notAffected`, `control`, `treatment` and `autoassign`.
+The experiment groups are: `notAffected`, `control`, `treatment`, `autoassign` and `off`.
 
 - `notAffected`: (default) it does not show any feedback on current solution, just challenge completion.
 - `control`: after completing a challenge its expectations are shown, but with no information regarding whether they were fulfilled.
 - `treatment`: this one has a training period. During this time expectations are shown for each challenge, alongside a progress bar that awards points for each passed expectation. Blocks in the workspace have their own feedback too. Once the training period is over, the app behavior changes to *control*.
 - `autoassign`: randomly assigns an experiment group at run time (between `control` and `treatment`).
+- `off`: always shows expectations and points. Doesn't show survey questions to the user.
 
 ### Packing installers:
 

app/components/personal-survey.js

@@ -7,6 +7,7 @@ export default Component.extend({
  pilasBloquesApi: service(),
  pilasBloquesAnalytics: service(),
  paperToaster: service(),
+ experiments: service(),
 
  /** Dialog descriptions according to SurveyJS library.
  ** Additional field: askEachSession, which tells the app to ask the question each time*/
@@ -71,7 +72,7 @@ export default Component.extend({
  ],
 
  didInsertElement() {
- if (this.pilasBloquesApi.isConnected()) this.showNextQuestion()
+ if (!this.experiments.isOff() && this.pilasBloquesApi.isConnected()) this.showNextQuestion()
  },
 
  showNextQuestion() {

app/services/experiments.js

@@ -11,7 +11,7 @@ export default Service.extend({
  challengeExpectations: service(),
 
  //This order is important, do NOT change
- possibleGroups: ["treatment", "control", "notAffected"],
+ possibleGroups: ["treatment", "control", "notAffected", "off"],
  decompositionTreatmentLength: ENV.decompositionTreatmentLength,
 
  solvedChallenges: computed('storage', function () {
@@ -30,6 +30,10 @@ export default Service.extend({
  return !(this.isTreatmentGroup() || this.isControlGroup())
  },
 
+ isOff(){
+ return this.experimentGroup() === "off"
+ },
+
  isAutoAssignStrategy() {
  return this.groupSelectionStrategy === "autoassign"
  },
@@ -102,11 +106,11 @@ export default Service.extend({
  },
 
  shouldShowBlocksWarningExpectationFeedback() {
- return this.isTreatmentGroup() && !this.feedbackIsDisabled()
+ return this.isOff() || this.isTreatmentGroup() && !this.feedbackIsDisabled()
  },
 
  shouldShowScoredExpectations() {
- return !(this.isControlGroup() || this.feedbackIsDisabled())
+ return this.isOff() || !(this.isControlGroup() || this.feedbackIsDisabled())
  },
 
  feedbackIsDisabled() {

config/environment.js

@@ -2,7 +2,7 @@
 
 var experimentGroup = process.env.EXPERIMENT_GROUP
 if (!experimentGroup) {
- experimentGroup = 'notAffected'
+ experimentGroup = 'off'
  console.log(`\nInfo: EXPERIMENT_GROUP variable not set. Building Pilas Bloques in default mode: ${experimentGroup}. See README.md for valid EXPERIMENT_GROUP values`)
 }
 

tests/integration/components/personal-survey-test.js

@@ -6,14 +6,16 @@ import { setupLoggedUser, setUpTestLocale } from '../../helpers/utils'
 import fetchMock from 'fetch-mock'
 
 module('Integration | Component | survey-window', function (hooks) {
- var api
+ var api, experiments
 
  setupRenderingTest(hooks)
  setupLoggedUser(hooks)
  setUpTestLocale(hooks)
 
  hooks.beforeEach(function () {
  api = this.owner.lookup('service:pilas-bloques-api')
+ experiments = this.owner.lookup('service:experiments')
+ experiments.set('groupSelectionStrategy', 'notAffected')
  })
 
  hooks.afterEach(function () {
@@ -46,4 +48,10 @@ module('Integration | Component | survey-window', function (hooks) {
  await render(hbs`<SessionButton />`)
  assert.notOk(surveyExists())
  })
+
+ test('When experiments are off, survey doesnt show', async function (assert) {
+ experiments.set('groupSelectionStrategy', 'off')
+ await render(hbs`<SessionButton />`)
+ assert.notOk(surveyExists())
+ })
 })

tests/unit/services/experiments-test.js

@@ -53,16 +53,19 @@ module('Unit | Service | experiments', function (hooks) {
 
  //Show non scored expects
 
- testShouldShowScoredExpectations('control', 'enabled', false)
- testShouldShowScoredExpectations('treatment', 'enabled', true)
- testShouldShowScoredExpectations('treatment', 'disabled', false, solvedChallengesFeedbackDisabled)
+ testShouldShowScoredExpectations('control', false)
+ testShouldShowScoredExpectations('treatment', true)
+ testShouldShowScoredExpectations('treatment', false, solvedChallengesFeedbackDisabled)
+ testShouldShowScoredExpectations('off', true)
+ testShouldShowScoredExpectations('off', true, solvedChallengesFeedbackDisabled)
 
  //Show feedback expectations (bubbles)
 
- testShouldShowBlocksWarningExpectationsFeedback('control', 'enabled', false)
- testShouldShowBlocksWarningExpectationsFeedback('treatment', 'enabled', true)
- testShouldShowBlocksWarningExpectationsFeedback('treatment', 'disabled', false, solvedChallengesFeedbackDisabled)
-
+ testShouldShowBlocksWarningExpectationsFeedback('control', false)
+ testShouldShowBlocksWarningExpectationsFeedback('treatment', true)
+ testShouldShowBlocksWarningExpectationsFeedback('treatment', false, solvedChallengesFeedbackDisabled)
+ testShouldShowBlocksWarningExpectationsFeedback('off', true)
+ testShouldShowBlocksWarningExpectationsFeedback('off', true, solvedChallengesFeedbackDisabled)
 
  //Congratulations modal
 
@@ -167,15 +170,16 @@ module('Unit | Service | experiments', function (hooks) {
  assert.ok(pilasBloquesApiMock.saveExperimentGroup.called)
  })
 
- function testShouldShowScoredExpectations(group, feedback, shouldShow, solvedChallenges) {
- testShouldShow('scored expects', group, feedback, shouldShow, (() => experiments.shouldShowScoredExpectations()), solvedChallenges)
+ function testShouldShowScoredExpectations(group, shouldShow, solvedChallenges) {
+ testShouldShow('scored expects', group, shouldShow, (() => experiments.shouldShowScoredExpectations()), solvedChallenges)
  }
 
- function testShouldShowBlocksWarningExpectationsFeedback(group, feedback, shouldShow, solvedChallenges) {
- testShouldShow('blocks warning expectation feedback', group, feedback, shouldShow, (() => experiments.shouldShowBlocksWarningExpectationFeedback()), solvedChallenges)
+ function testShouldShowBlocksWarningExpectationsFeedback(group, shouldShow, solvedChallenges) {
+ testShouldShow('blocks warning expectation feedback', group, shouldShow, (() => experiments.shouldShowBlocksWarningExpectationFeedback()), solvedChallenges)
  }
 
- function testShouldShow(name, group, feedback, shouldShow, callback, solvedChallenges = []) {
+ function testShouldShow(name, group, shouldShow, callback, solvedChallenges = []) {
+ const feedback = solvedChallenges.length == 0 ? 'enabled' : 'disabled'
  test(`Should ${shouldShow ? "" : "NOT"} show ${name} - ${group} group and feedback ${feedback}`, function (assert) {
  storageMock.solvedChallenges = solvedChallenges
  experiments.set('groupSelectionStrategy', group)