Skip to content

Commit

Permalink
merge master
Browse files Browse the repository at this point in the history
  • Loading branch information
@aspicer
aspicer committed Oct 17, 2024
2 parents 7354b7d + a9b5a60 commit b3a361a
Show file tree
Hide file tree
Showing 160 changed files with 6,004 additions and 1,834 deletions.
1 change: 1 addition & 0 deletions bin/copy-posthog-js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ set -e

cp node_modules/posthog-js/dist/array.js* frontend/dist/
cp node_modules/posthog-js/dist/array.full.js* frontend/dist/
cp node_modules/posthog-js/dist/array.full.es5.js* frontend/dist/
cp node_modules/posthog-js/dist/recorder.js* frontend/dist/
cp node_modules/posthog-js/dist/recorder-v2.js* frontend/dist/
cp node_modules/posthog-js/dist/surveys.js* frontend/dist/
Expand Down
37 changes: 34 additions & 3 deletions cypress/e2e/alerts.cy.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,22 @@ describe('Alerts', () => {
const createAlert = (
name: string = 'Alert name',
lowerThreshold: string = '100',
upperThreshold: string = '200'
upperThreshold: string = '200',
condition?: string
): void => {
cy.get('[data-attr=more-button]').click()
cy.contains('Manage alerts').click()
cy.contains('New alert').click()

cy.get('[data-attr=alertForm-name]').clear().type(name)
cy.get('[data-attr=subscribed-users').click().type('{downarrow}{enter}')

if (condition) {
cy.get('[data-attr=alertForm-condition').click()
cy.contains(condition).click()
cy.contains('%').click()
}

cy.get('[data-attr=alertForm-lower-threshold').clear().type(lowerThreshold)
cy.get('[data-attr=alertForm-upper-threshold').clear().type(upperThreshold)
cy.contains('Create alert').click()
Expand All @@ -39,7 +47,6 @@ describe('Alerts', () => {
cy.get('[data-attr=insight-edit-button]').click()
cy.get('[data-attr=chart-filter]').click()
cy.contains(displayType).click()
cy.get('.insight-empty-state').should('not.exist')
cy.get('[data-attr=insight-save-button]').contains('Save').click()
cy.url().should('not.include', '/edit')
}
Expand Down Expand Up @@ -69,7 +76,7 @@ describe('Alerts', () => {
})

it('Should warn about an alert deletion', () => {
setInsightDisplayTypeAndSave('Number')
setInsightDisplayTypeAndSave('Area chart')

createAlert('Alert to be deleted because of a changed insight')

Expand All @@ -90,4 +97,28 @@ describe('Alerts', () => {
cy.contains('Manage alerts').click()
cy.contains('Alert to be deleted because of a changed insight').should('not.exist')
})

it('Should allow create and delete a relative alert', () => {
cy.get('[data-attr=more-button]').click()
// Alerts should be disabled for trends represented with graphs
cy.get('[data-attr=manage-alerts-button]').should('have.attr', 'aria-disabled', 'true')

setInsightDisplayTypeAndSave('Bar chart')

createAlert('Alert name', '10', '20', 'increases by')
cy.reload()

// Check the alert has the same values as when it was created
cy.get('[data-attr=more-button]').click()
cy.contains('Manage alerts').click()
cy.get('[data-attr=alert-list-item]').contains('Alert name').click()
cy.get('[data-attr=alertForm-name]').should('have.value', 'Alert name')
cy.get('[data-attr=alertForm-lower-threshold').should('have.value', '10')
cy.get('[data-attr=alertForm-upper-threshold').should('have.value', '20')
cy.contains('Delete alert').click()
cy.wait(2000)

cy.reload()
cy.contains('Alert name').should('not.exist')
})
})
18 changes: 18 additions & 0 deletions cypress/e2e/notebooks-insights.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import { insight, savedInsights } from '../productAnalytics'

describe('Notebooks', () => {
beforeEach(() => {
cy.clickNavMenu('notebooks')
cy.location('pathname').should('include', '/notebooks')
})
;['SQL', 'TRENDS', 'FUNNELS', 'RETENTION', 'PATHS', 'STICKINESS', 'LIFECYCLE'].forEach((insightType) => {
it(`Can add a ${insightType} insight`, () => {
savedInsights.createNewInsightOfType(insightType)
insight.editName(`${insightType} Insight`)
insight.save()
cy.get('[data-attr="notebooks-add-button"]').click()
cy.get('[data-attr="notebooks-select-button-create"]').click()
cy.get('.ErrorBoundary').should('not.exist')
})
})
})
2 changes: 1 addition & 1 deletion ee/clickhouse/queries/experiments/funnel_experiment_result.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

from posthog.constants import ExperimentNoResultsErrorKeys
from posthog.hogql_queries.experiments import CONTROL_VARIANT_KEY
from posthog.hogql_queries.experiments.funnel_statistics import (
from posthog.hogql_queries.experiments.funnels_statistics import (
are_results_significant,
calculate_credible_intervals,
calculate_probabilities,
Expand Down
88 changes: 44 additions & 44 deletions ee/clickhouse/queries/experiments/test_funnel_experiment_result.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@

from flaky import flaky

from posthog.hogql_queries.experiments.funnel_statistics import (
from posthog.hogql_queries.experiments.funnels_statistics import (
are_results_significant,
calculate_expected_loss,
calculate_probabilities,
calculate_credible_intervals as calculate_funnel_credible_intervals,
)
from posthog.schema import ExperimentSignificanceCode, ExperimentVariantFunnelResult
from posthog.schema import ExperimentSignificanceCode, ExperimentVariantFunnelsBaseStats

Probability = float

Expand All @@ -25,7 +25,7 @@ def logbeta(x: int, y: int) -> float:


def calculate_probability_of_winning_for_target(
target_variant: ExperimentVariantFunnelResult, other_variants: list[ExperimentVariantFunnelResult]
target_variant: ExperimentVariantFunnelsBaseStats, other_variants: list[ExperimentVariantFunnelsBaseStats]
) -> Probability:
"""
Calculates the probability of winning for target variant.
Expand Down Expand Up @@ -146,8 +146,8 @@ def probability_D_beats_A_B_and_C(
@flaky(max_runs=10, min_passes=1)
class TestFunnelExperimentCalculator(unittest.TestCase):
def test_calculate_results(self):
variant_test = ExperimentVariantFunnelResult(key="A", success_count=100, failure_count=10)
variant_control = ExperimentVariantFunnelResult(key="B", success_count=100, failure_count=18)
variant_test = ExperimentVariantFunnelsBaseStats(key="A", success_count=100, failure_count=10)
variant_control = ExperimentVariantFunnelsBaseStats(key="B", success_count=100, failure_count=18)

_, probability = calculate_probabilities(variant_control, [variant_test])
self.assertAlmostEqual(probability, 0.918, places=2)
Expand All @@ -164,8 +164,8 @@ def test_calculate_results(self):
self.assertAlmostEqual(credible_intervals[variant_test.key][1], 0.9494, places=3)

def test_simulation_result_is_close_to_closed_form_solution(self):
variant_test = ExperimentVariantFunnelResult(key="A", success_count=100, failure_count=10)
variant_control = ExperimentVariantFunnelResult(key="B", success_count=100, failure_count=18)
variant_test = ExperimentVariantFunnelsBaseStats(key="A", success_count=100, failure_count=10)
variant_control = ExperimentVariantFunnelsBaseStats(key="B", success_count=100, failure_count=18)

_, probability = calculate_probabilities(variant_control, [variant_test])
self.assertAlmostEqual(probability, 0.918, places=1)
Expand All @@ -174,9 +174,9 @@ def test_simulation_result_is_close_to_closed_form_solution(self):
self.assertAlmostEqual(probability, alternative_probability, places=1)

def test_calculate_results_for_two_test_variants(self):
variant_test_1 = ExperimentVariantFunnelResult(key="A", success_count=100, failure_count=10)
variant_test_2 = ExperimentVariantFunnelResult(key="B", success_count=100, failure_count=3)
variant_control = ExperimentVariantFunnelResult(key="C", success_count=100, failure_count=18)
variant_test_1 = ExperimentVariantFunnelsBaseStats(key="A", success_count=100, failure_count=10)
variant_test_2 = ExperimentVariantFunnelsBaseStats(key="B", success_count=100, failure_count=3)
variant_control = ExperimentVariantFunnelsBaseStats(key="C", success_count=100, failure_count=18)

probabilities = calculate_probabilities(variant_control, [variant_test_1, variant_test_2])
self.assertAlmostEqual(sum(probabilities), 1)
Expand Down Expand Up @@ -210,9 +210,9 @@ def test_calculate_results_for_two_test_variants(self):
self.assertAlmostEqual(credible_intervals[variant_test_2.key][1], 0.9894, places=3)

def test_calculate_results_for_two_test_variants_almost_equal(self):
variant_test_1 = ExperimentVariantFunnelResult(key="A", success_count=120, failure_count=60)
variant_test_2 = ExperimentVariantFunnelResult(key="B", success_count=110, failure_count=52)
variant_control = ExperimentVariantFunnelResult(key="C", success_count=130, failure_count=65)
variant_test_1 = ExperimentVariantFunnelsBaseStats(key="A", success_count=120, failure_count=60)
variant_test_2 = ExperimentVariantFunnelsBaseStats(key="B", success_count=110, failure_count=52)
variant_control = ExperimentVariantFunnelsBaseStats(key="C", success_count=130, failure_count=65)

probabilities = calculate_probabilities(variant_control, [variant_test_1, variant_test_2])
self.assertAlmostEqual(sum(probabilities), 1)
Expand Down Expand Up @@ -245,8 +245,8 @@ def test_calculate_results_for_two_test_variants_almost_equal(self):
self.assertAlmostEqual(credible_intervals[variant_test_2.key][1], 0.7460, places=3)

def test_absolute_loss_less_than_one_percent_but_not_significant(self):
variant_test_1 = ExperimentVariantFunnelResult(key="A", success_count=286, failure_count=2014)
variant_control = ExperimentVariantFunnelResult(key="B", success_count=267, failure_count=2031)
variant_test_1 = ExperimentVariantFunnelsBaseStats(key="A", success_count=286, failure_count=2014)
variant_control = ExperimentVariantFunnelsBaseStats(key="B", success_count=267, failure_count=2031)

probabilities = calculate_probabilities(variant_control, [variant_test_1])
self.assertAlmostEqual(sum(probabilities), 1)
Expand All @@ -267,10 +267,10 @@ def test_absolute_loss_less_than_one_percent_but_not_significant(self):
self.assertAlmostEqual(credible_intervals[variant_test_1.key][1], 0.1384, places=3)

def test_calculate_results_for_three_test_variants(self):
variant_test_1 = ExperimentVariantFunnelResult(key="A", success_count=100, failure_count=10)
variant_test_2 = ExperimentVariantFunnelResult(key="B", success_count=100, failure_count=3)
variant_test_3 = ExperimentVariantFunnelResult(key="C", success_count=100, failure_count=30)
variant_control = ExperimentVariantFunnelResult(key="D", success_count=100, failure_count=18)
variant_test_1 = ExperimentVariantFunnelsBaseStats(key="A", success_count=100, failure_count=10)
variant_test_2 = ExperimentVariantFunnelsBaseStats(key="B", success_count=100, failure_count=3)
variant_test_3 = ExperimentVariantFunnelsBaseStats(key="C", success_count=100, failure_count=30)
variant_control = ExperimentVariantFunnelsBaseStats(key="D", success_count=100, failure_count=18)

probabilities = calculate_probabilities(variant_control, [variant_test_1, variant_test_2, variant_test_3])
self.assertAlmostEqual(sum(probabilities), 1)
Expand Down Expand Up @@ -313,10 +313,10 @@ def test_calculate_results_for_three_test_variants(self):
self.assertAlmostEqual(credible_intervals[variant_test_3.key][1], 0.8332, places=3)

def test_calculate_results_for_three_test_variants_almost_equal(self):
variant_test_1 = ExperimentVariantFunnelResult(key="A", success_count=120, failure_count=60)
variant_test_2 = ExperimentVariantFunnelResult(key="B", success_count=110, failure_count=52)
variant_test_3 = ExperimentVariantFunnelResult(key="C", success_count=100, failure_count=46)
variant_control = ExperimentVariantFunnelResult(key="D", success_count=130, failure_count=65)
variant_test_1 = ExperimentVariantFunnelsBaseStats(key="A", success_count=120, failure_count=60)
variant_test_2 = ExperimentVariantFunnelsBaseStats(key="B", success_count=110, failure_count=52)
variant_test_3 = ExperimentVariantFunnelsBaseStats(key="C", success_count=100, failure_count=46)
variant_control = ExperimentVariantFunnelsBaseStats(key="D", success_count=130, failure_count=65)

probabilities = calculate_probabilities(variant_control, [variant_test_1, variant_test_2, variant_test_3])
self.assertAlmostEqual(sum(probabilities), 1)
Expand Down Expand Up @@ -357,10 +357,10 @@ def test_calculate_results_for_three_test_variants_almost_equal(self):
self.assertAlmostEqual(credible_intervals[variant_test_3.key][1], 0.7547, places=3)

def test_calculate_results_for_three_test_variants_much_better_than_control(self):
variant_test_1 = ExperimentVariantFunnelResult(key="A", success_count=130, failure_count=60)
variant_test_2 = ExperimentVariantFunnelResult(key="B", success_count=135, failure_count=62)
variant_test_3 = ExperimentVariantFunnelResult(key="C", success_count=132, failure_count=60)
variant_control = ExperimentVariantFunnelResult(key="D", success_count=80, failure_count=65)
variant_test_1 = ExperimentVariantFunnelsBaseStats(key="A", success_count=130, failure_count=60)
variant_test_2 = ExperimentVariantFunnelsBaseStats(key="B", success_count=135, failure_count=62)
variant_test_3 = ExperimentVariantFunnelsBaseStats(key="C", success_count=132, failure_count=60)
variant_control = ExperimentVariantFunnelsBaseStats(key="D", success_count=80, failure_count=65)

probabilities = calculate_probabilities(variant_control, [variant_test_1, variant_test_2, variant_test_3])
self.assertAlmostEqual(sum(probabilities), 1)
Expand Down Expand Up @@ -392,14 +392,14 @@ def test_calculate_results_for_three_test_variants_much_better_than_control(self
self.assertAlmostEqual(credible_intervals[variant_test_3.key][1], 0.7488, places=3)

def test_calculate_results_for_seven_test_variants(self):
variant_test_1 = ExperimentVariantFunnelResult(key="A", success_count=100, failure_count=17)
variant_test_2 = ExperimentVariantFunnelResult(key="B", success_count=100, failure_count=16)
variant_test_3 = ExperimentVariantFunnelResult(key="C", success_count=100, failure_count=30)
variant_test_4 = ExperimentVariantFunnelResult(key="D", success_count=100, failure_count=31)
variant_test_5 = ExperimentVariantFunnelResult(key="E", success_count=100, failure_count=29)
variant_test_6 = ExperimentVariantFunnelResult(key="F", success_count=100, failure_count=32)
variant_test_7 = ExperimentVariantFunnelResult(key="G", success_count=100, failure_count=33)
variant_control = ExperimentVariantFunnelResult(key="H", success_count=100, failure_count=18)
variant_test_1 = ExperimentVariantFunnelsBaseStats(key="A", success_count=100, failure_count=17)
variant_test_2 = ExperimentVariantFunnelsBaseStats(key="B", success_count=100, failure_count=16)
variant_test_3 = ExperimentVariantFunnelsBaseStats(key="C", success_count=100, failure_count=30)
variant_test_4 = ExperimentVariantFunnelsBaseStats(key="D", success_count=100, failure_count=31)
variant_test_5 = ExperimentVariantFunnelsBaseStats(key="E", success_count=100, failure_count=29)
variant_test_6 = ExperimentVariantFunnelsBaseStats(key="F", success_count=100, failure_count=32)
variant_test_7 = ExperimentVariantFunnelsBaseStats(key="G", success_count=100, failure_count=33)
variant_control = ExperimentVariantFunnelsBaseStats(key="H", success_count=100, failure_count=18)

probabilities = calculate_probabilities(
variant_control,
Expand Down Expand Up @@ -487,8 +487,8 @@ def test_calculate_results_for_seven_test_variants(self):
self.assertAlmostEqual(credible_intervals[variant_test_7.key][1], 0.8174, places=3)

def test_calculate_results_control_is_significant(self):
variant_test = ExperimentVariantFunnelResult(key="test", success_count=100, failure_count=18)
variant_control = ExperimentVariantFunnelResult(key="control", success_count=100, failure_count=10)
variant_test = ExperimentVariantFunnelsBaseStats(key="test", success_count=100, failure_count=18)
variant_control = ExperimentVariantFunnelsBaseStats(key="control", success_count=100, failure_count=10)

probabilities = calculate_probabilities(variant_control, [variant_test])

Expand All @@ -507,13 +507,13 @@ def test_calculate_results_control_is_significant(self):
self.assertAlmostEqual(credible_intervals[variant_test.key][1], 0.9010, places=3)

def test_calculate_results_many_variants_control_is_significant(self):
variant_test_1 = ExperimentVariantFunnelResult(key="test_1", success_count=100, failure_count=20)
variant_test_2 = ExperimentVariantFunnelResult(key="test_2", success_count=100, failure_count=21)
variant_test_3 = ExperimentVariantFunnelResult(key="test_3", success_count=100, failure_count=22)
variant_test_4 = ExperimentVariantFunnelResult(key="test_4", success_count=100, failure_count=23)
variant_test_5 = ExperimentVariantFunnelResult(key="test_5", success_count=100, failure_count=24)
variant_test_6 = ExperimentVariantFunnelResult(key="test_6", success_count=100, failure_count=25)
variant_control = ExperimentVariantFunnelResult(key="control", success_count=100, failure_count=10)
variant_test_1 = ExperimentVariantFunnelsBaseStats(key="test_1", success_count=100, failure_count=20)
variant_test_2 = ExperimentVariantFunnelsBaseStats(key="test_2", success_count=100, failure_count=21)
variant_test_3 = ExperimentVariantFunnelsBaseStats(key="test_3", success_count=100, failure_count=22)
variant_test_4 = ExperimentVariantFunnelsBaseStats(key="test_4", success_count=100, failure_count=23)
variant_test_5 = ExperimentVariantFunnelsBaseStats(key="test_5", success_count=100, failure_count=24)
variant_test_6 = ExperimentVariantFunnelsBaseStats(key="test_6", success_count=100, failure_count=25)
variant_control = ExperimentVariantFunnelsBaseStats(key="control", success_count=100, failure_count=10)

variants_test = [
variant_test_1,
Expand Down
Loading

0 comments on commit b3a361a

Please sign in to comment.