Merge branch 'staging' into 'master'

Release v2.2.0

See merge request bullet-train/bullet-train-api!132

.gitignore

@@ -12,3 +12,4 @@ checkstyle.txt
 .env
 .direnv
 .envrc
+.elasticbeanstalk/

Pipfile

@@ -16,6 +16,7 @@ pylint = "*"
 "autopep8" = "*"
 pytest = "*"
 pytest-django = "*"
+django-test-migrations = "*"
 
 [packages]
 appdirs = "*"
@@ -33,9 +34,7 @@ sendgrid-django = "*"
 psycopg2-binary = "*"
 coreapi = "*"
 Django = "<3.0"
-numpy = "*"
 django-simple-history = "*"
-twisted = {version = "*",extras = ["tls"]}
 django-debug-toolbar = "*"
 google-api-python-client = "*"
 "oauth2client" = "*"
@@ -46,8 +45,8 @@ chargebee = "*"
 python-http-client = "<3.2.0" # 3.2.0 is the latest but throws an error on installation saying that it's not found
 django-health-check = "*"
 django-storages = "*"
-boto3 = "*"
 django-environ = "*"
 django-trench = "*"
 djoser = "*"
 influxdb-client = "*"
+django-ordered-model = "*"

readme.md

@@ -120,7 +120,6 @@ The application relies on the following environment variables to run:
 * `INFLUXDB_URL`: The URL for your InfluxDB database
 * `INFLUXDB_ORG`: The organisation string for your InfluxDB API call.
 * `GA_TABLE_ID`: GA table ID (view) to query when looking for organisation usage
-* `USE_S3_STORAGE`: 'True' to store static files in s3
 * `AWS_STORAGE_BUCKET_NAME`: bucket name to store static files. Required if `USE_S3_STORAGE' is true.
 * `AWS_S3_REGION_NAME`: region name of the static files bucket. Defaults to eu-west-2.
 * `ALLOWED_ADMIN_IP_ADDRESSES`: restrict access to the django admin console to a comma separated list of IP addresses (e.g. `127.0.0.1,127.0.0.2`) 

src/app/settings/common.py

@@ -107,6 +107,9 @@
     # health check plugins
     'health_check',
     'health_check.db',
+
+    # Used for ordering models (e.g. FeatureSegment)
+    'ordered_model',
 ]
 
 if GOOGLE_ANALYTICS_KEY or INFLUXDB_TOKEN:
@@ -326,14 +329,6 @@
     }
 }
 
-if env.bool('USE_S3_STORAGE', default=False):
-    STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
-    AWS_STORAGE_BUCKET_NAME = os.environ['AWS_STORAGE_BUCKET_NAME']
-    AWS_S3_REGION_NAME = os.environ.get('AWS_S3_REGION_NAME', 'eu-west-2')
-    AWS_LOCATION = 'static'
-    AWS_DEFAULT_ACL = 'public-read'
-    AWS_S3_ADDRESSING_STYLE = 'virtual'
-
 LOG_LEVEL = env.str('LOG_LEVEL', 'WARNING')
 
 TRENCH_AUTH = {

src/app/settings/master.py

@@ -40,4 +40,4 @@
 REST_FRAMEWORK['PAGE_SIZE'] = 999
 
 SECURE_SSL_REDIRECT = True
-SECURE_REDIRECT_EXEMPT = [r'^/$', r'^$']  # root is exempt as it's used for EB health checks
+SECURE_REDIRECT_EXEMPT = [r'^health$']  # /health is exempt as it's used for EB health checks

src/app/settings/staging.py

@@ -40,4 +40,4 @@
 REST_FRAMEWORK['PAGE_SIZE'] = 999
 
 SECURE_SSL_REDIRECT = True
-SECURE_REDIRECT_EXEMPT = [r'^/$', r'^$']  # root is exempt as it's used for EB health checks
+SECURE_REDIRECT_EXEMPT = [r'^health$']  # /health is exempt as it's used for EB health checks

src/audit/models.py

@@ -9,7 +9,7 @@
 FEATURE_UPDATED_MESSAGE = "Flag / Remote Config updated: %s"
 SEGMENT_CREATED_MESSAGE = "New Segment created: %s"
 SEGMENT_UPDATED_MESSAGE = "Segment updated: %s"
-FEATURE_SEGMENT_UPDATED_MESSAGE = "Segment rules updated for flag: %s"
+FEATURE_SEGMENT_UPDATED_MESSAGE = "Segment rules updated for flag: %s in environment: %s"
 ENVIRONMENT_CREATED_MESSAGE = "New Environment created: %s"
 ENVIRONMENT_UPDATED_MESSAGE = "Environment updated: %s"
 FEATURE_STATE_UPDATED_MESSAGE = "Flag state / Remote Config value updated for feature: %s"
@@ -45,3 +45,15 @@ class Meta:
 
     def __str__(self):
         return "Audit Log %s" % self.id
+
+    @classmethod
+    def create_record(cls, obj, obj_type, log_message, author, project=None, environment=None):
+        cls.objects.create(
+            related_object_id=obj.id,
+            related_object_type=obj_type.name,
+            log=log_message,
+            author=author,
+            project=project,
+            environment=environment
+        )
+

src/audit/signals.py

@@ -5,10 +5,10 @@
 
 from audit.models import AuditLog
 from audit.serializers import AuditLogSerializer
+from util.logging import get_logger
 from webhooks.webhooks import call_organisation_webhooks, WebhookEventType
 
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
+logger = get_logger(__name__)
 
 
 @receiver(post_save, sender=AuditLog)

src/environments/models.py

@@ -123,7 +123,9 @@ def get_all_feature_states(self):
         # define sub queries
         belongs_to_environment_query = Q(environment=self.environment)
         overridden_for_identity_query = Q(identity=self)
-        overridden_for_segment_query = Q(feature_segment__segment__in=segments)
+        overridden_for_segment_query = Q(
+            feature_segment__segment__in=segments, feature_segment__environment=self.environment
+        )
         environment_default_query = Q(identity=None, feature_segment=None)
 
         # define the full query
@@ -135,6 +137,8 @@ def get_all_feature_states(self):
 
         all_flags = FeatureState.objects.select_related(*select_related_args).filter(full_query)
 
+        # iterate over all the flags and build a dictionary keyed on feature with the highest priority flag
+        # for the given identity as the value.
         identity_flags = {}
         for flag in all_flags:
             if flag.feature_id not in identity_flags:

src/environments/tests/test_models.py

@@ -6,7 +6,7 @@
 from features.utils import INTEGER, STRING, BOOLEAN
 from organisations.models import Organisation
 from projects.models import Project
-from segments.models import Segment, SegmentRule, Condition, EQUAL, GREATER_THAN_INCLUSIVE
+from segments.models import Segment, SegmentRule, Condition, EQUAL, GREATER_THAN_INCLUSIVE, GREATER_THAN
 from util.tests import Helper
 
 
@@ -190,9 +190,9 @@ def test_get_all_feature_states_for_identity_returns_correct_values_for_matching
         remote_config = Feature.objects.create(name='test-remote-config', project=self.project,
                                                initial_value='initial-value', type='CONFIG')
 
-        FeatureSegment.objects.create(feature=feature_flag, segment=segment, enabled=True)
+        FeatureSegment.objects.create(feature=feature_flag, segment=segment, environment=self.environment, enabled=True)
         overridden_value = 'overridden-value'
-        FeatureSegment.objects.create(feature=remote_config, segment=segment,
+        FeatureSegment.objects.create(feature=remote_config, segment=segment, environment=self.environment,
                                       value=overridden_value, value_type=STRING)
 
         # When
@@ -221,9 +221,9 @@ def test_get_all_feature_states_for_identity_returns_correct_values_for_identity
         remote_config = Feature.objects.create(name='test-remote-config', project=self.project,
                                                initial_value=initial_value, type='CONFIG')
 
-        FeatureSegment.objects.create(feature=feature_flag, segment=segment, enabled=True)
+        FeatureSegment.objects.create(feature=feature_flag, segment=segment, environment=self.environment, enabled=True)
         overridden_value = 'overridden-value'
-        FeatureSegment.objects.create(feature=remote_config, segment=segment,
+        FeatureSegment.objects.create(feature=remote_config, segment=segment, environment=self.environment,
                                       value=overridden_value, value_type=STRING)
 
         # When
@@ -252,7 +252,7 @@ def test_get_all_feature_states_for_identity_returns_correct_value_for_matching_
         # Feature segment value is converted to string in the serializer so we set as a string value here to test
         # bool value
         overridden_value = '12'
-        FeatureSegment.objects.create(feature=remote_config, segment=segment,
+        FeatureSegment.objects.create(feature=remote_config, segment=segment, environment=self.environment,
                                       value=overridden_value, value_type=INTEGER)
 
         # When
@@ -279,7 +279,7 @@ def test_get_all_feature_states_for_identity_returns_correct_value_for_matching_
         # Feature segment value is converted to string in the serializer so we set as a string value here to test
         # bool value
         overridden_value = 'false'
-        FeatureSegment.objects.create(feature=remote_config, segment=segment,
+        FeatureSegment.objects.create(feature=remote_config, segment=segment, environment=self.environment,
                                       value=overridden_value, value_type=BOOLEAN)
 
         # When
@@ -313,11 +313,11 @@ def test_get_all_feature_states_highest_value_of_highest_priority_segment(self):
 
         # which is overridden by both segments with different values
         overridden_value_1 = 'overridden-value-1'
-        FeatureSegment.objects.create(feature=remote_config, segment=segment_1,
+        FeatureSegment.objects.create(feature=remote_config, segment=segment_1, environment=self.environment,
                                       value=overridden_value_1, value_type=STRING, priority=1)
 
         overridden_value_2 = 'overridden-value-2'
-        FeatureSegment.objects.create(feature=remote_config, segment=segment_2,
+        FeatureSegment.objects.create(feature=remote_config, segment=segment_2, environment=self.environment,
                                       value=overridden_value_2, value_type=STRING, priority=2)
 
         # When - we get all feature states for an identity
@@ -327,3 +327,38 @@ def test_get_all_feature_states_highest_value_of_highest_priority_segment(self):
         assert len(feature_states) == 1
         remote_config_feature_state = next(filter(lambda fs: fs.feature == remote_config, feature_states))
         assert remote_config_feature_state.get_feature_state_value() == overridden_value_1
+
+    def test_remote_config_override(self):
+        """specific test for bug raised following work to make feature segments unique to an environment"""
+        # GIVEN - an identity with a trait that has a value of 10
+        identity = Identity.objects.create(identifier="test", environment=self.environment)
+        trait = Trait.objects.create(identity=identity, trait_key="my_trait", integer_value=10, value_type=INTEGER)
+
+        # and a segment that matches users that have a value for this trait greater than 5
+        segment = Segment.objects.create(name="Test segment", project=self.project)
+        segment_rule = SegmentRule.objects.create(segment=segment, type=SegmentRule.ALL_RULE)
+        condition = Condition.objects.create(
+            rule=segment_rule, operator=GREATER_THAN, value="5", property=trait.trait_key
+        )
+
+        # and a feature that has a segment override in the same environment as the identity
+        remote_config = Feature.objects.create(name="my_feature", initial_value="initial value", project=self.project)
+        feature_segment = FeatureSegment.objects.create(
+            feature=remote_config,
+            environment=self.environment,
+            segment=segment,
+            value="overridden value 1",
+            value_type=STRING
+        )
+
+        # WHEN - the value on the feature segment is updated and we get all the feature states for the identity
+        feature_segment.value = "overridden value 2"
+        feature_segment.save()
+        feature_states = identity.get_all_feature_states()
+
+        # THEN - the feature state value is correctly set to the newly updated feature segment value
+        assert len(feature_states) == 1
+
+        overridden_feature_state = feature_states[0]
+        assert overridden_feature_state.get_feature_state_value() == feature_segment.value
+

src/environments/tests/test_views.py

@@ -511,7 +511,9 @@ def test_identities_endpoint_returns_value_for_segment_if_identity_in_segment(se
         segment = Segment.objects.create(name='Test Segment', project=self.project)
         segment_rule = SegmentRule.objects.create(segment=segment, type=SegmentRule.ALL_RULE)
         Condition.objects.create(operator='EQUAL', property=trait_key, value=trait_value, rule=segment_rule)
-        FeatureSegment.objects.create(segment=segment, feature=self.feature_2, enabled=True, priority=1)
+        FeatureSegment.objects.create(
+            segment=segment, feature=self.feature_2, environment=self.environment, enabled=True, priority=1
+        )
 
         # When
         response = self.client.get(url)
@@ -534,7 +536,9 @@ def test_identities_endpoint_returns_value_for_segment_if_identity_in_segment_an
         segment = Segment.objects.create(name='Test Segment', project=self.project)
         segment_rule = SegmentRule.objects.create(segment=segment, type=SegmentRule.ALL_RULE)
         Condition.objects.create(operator='EQUAL', property=trait_key, value=trait_value, rule=segment_rule)
-        FeatureSegment.objects.create(segment=segment, feature=self.feature_1, enabled=True, priority=1)
+        FeatureSegment.objects.create(
+            segment=segment, feature=self.feature_1, environment=self.environment, enabled=True, priority=1
+        )
 
         # When
         response = self.client.get(url)
@@ -557,7 +561,9 @@ def test_identities_endpoint_returns_value_for_segment_if_rule_type_percentage_s
         Condition.objects.create(operator=models.PERCENTAGE_SPLIT,
                                  value=(identity_percentage_value + (1 - identity_percentage_value) / 2) * 100.0,
                                  rule=segment_rule)
-        FeatureSegment.objects.create(segment=segment, feature=self.feature_1, enabled=True, priority=1)
+        FeatureSegment.objects.create(
+            segment=segment, feature=self.feature_1, environment=self.environment, enabled=True, priority=1
+        )
 
         # When
         self.client.credentials(HTTP_X_ENVIRONMENT_KEY=self.environment.api_key)
@@ -580,7 +586,9 @@ def test_identities_endpoint_returns_default_value_if_rule_type_percentage_split
         Condition.objects.create(operator=models.PERCENTAGE_SPLIT,
                                  value=identity_percentage_value / 2,
                                  rule=segment_rule)
-        FeatureSegment.objects.create(segment=segment, feature=self.feature_1, enabled=True, priority=1)
+        FeatureSegment.objects.create(
+            segment=segment, feature=self.feature_1, environment=self.environment, enabled=True, priority=1
+        )
 
         # When
         self.client.credentials(HTTP_X_ENVIRONMENT_KEY=self.environment.api_key)

src/features/apps.py

@@ -8,4 +8,5 @@ class FeaturesConfig(AppConfig):
     name = 'features'
 
     def ready(self):
-        pass
+        # noinspection PyUnresolvedReferences
+        import features.signals

src/features/fields.py

@@ -0,0 +1,19 @@
+from rest_framework import serializers
+
+from features.utils import INTEGER, BOOLEAN, STRING
+
+
+class FeatureSegmentValueField(serializers.Field):
+    def to_internal_value(self, data):
+        if data is not None:
+            # grab the type of the value and set the context for use
+            # in the create / update methods on the serializer
+            value_type = type(data).__name__
+            value_types = [STRING, BOOLEAN, INTEGER]
+            value_type = value_type if value_type in value_types else STRING
+            self.context['value_type'] = value_type
+
+            return str(data)
+
+    def to_representation(self, value):
+        return self.root.instance.get_value()

src/features/helpers.py

@@ -0,0 +1,13 @@
+import typing
+
+from features.utils import INTEGER, BOOLEAN
+
+
+def get_correctly_typed_value(value_type: str, string_value: str) -> typing.Any:
+    if value_type == INTEGER:
+        return int(string_value)
+    elif value_type == BOOLEAN:
+        return string_value == 'True'
+
+    return string_value
+

src/features/migrations/0017_auto_20200607_1005.py

@@ -0,0 +1,25 @@
+# Generated by Django 2.2.12 on 2020-06-07 10:05
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('environments', '0012_auto_20200504_1322'),
+        ('segments', '0007_auto_20190906_1416'),
+        ('features', '0016_auto_20190916_1717'),
+    ]
+
+    operations = [
+        # first, add the field, allowing null values
+        migrations.AddField(
+            model_name='featuresegment',
+            name='environment',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='feature_segments', to='environments.Environment'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='featuresegment',
+            unique_together={('feature', 'environment', 'segment')},
+        ),
+    ]