From 7b314cc34c636d47db8414de38dab0f87d396e45 Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Fri, 10 Feb 2017 04:05:57 +0100 Subject: [PATCH] Add "multi-indexes" feature in contrib This allow to easily compose complexe indexes based on many ones, for example a DateTime index allowing filtering on date, time, year, month... --- limpyd/contrib/indexes.py | 210 ++++++++++++++++++++++++++++++++++++++ tests/contrib/indexes.py | 168 ++++++++++++++++++++++++++++++ 2 files changed, 378 insertions(+) create mode 100644 limpyd/contrib/indexes.py create mode 100644 tests/contrib/indexes.py diff --git a/limpyd/contrib/indexes.py b/limpyd/contrib/indexes.py new file mode 100644 index 0000000..1b63d9e --- /dev/null +++ b/limpyd/contrib/indexes.py @@ -0,0 +1,210 @@ +# -*- coding:utf-8 -*- +from __future__ import unicode_literals + +from limpyd.indexes import BaseIndex +from limpyd.utils import cached_property + + +class MultiIndexes(BaseIndex): + """An index that is a proxy to many ones + + This must not be used directly as a class, but a new index class must be + created by using the ``create`` class method + + Attributes + ---------- + index_classes: list + The index classes composing this multi-indexes class + key: str + A key to avoid collision with another index/multi-index + that will be passed to a field. + + Examples + -------- + + >>> multi_index = MultiIndexes.compose([MyIndex, MyOtherIndex]) + >>> class MyModel(RedisModel): + ... field = StringField(indexes=[multi_index]) + + """ + index_classes = [] + + @classmethod + def compose(cls, index_classes, key=None, transform=None, name=None): + """Create a new class with the given index classes + + Parameters + ----------- + index_classes: list + The list of index classes to be used in the multi-index class to create + name: str + The name of the new multi-index class. If not set, it will be the same + as the current class + key: str + A key to augment the default key of each index, to avoid collision. + transform: callable + None by default, can be set to a function that will transform the value to be indexed. + This callable can accept one (`value`) or two (`self`, `value`) arguments + + """ + + attrs = {} + if index_classes: + attrs['index_classes'] = index_classes + + klass = type(str(name or cls.__name__), (cls, ), attrs) + + # let the ``configure`` method manage some fields + configure_attrs = {} + if key is not None: + configure_attrs['key'] = key + if transform is not None: + configure_attrs['transform'] = transform + + if configure_attrs: + klass = klass.configure(**configure_attrs) + + return klass + + @cached_property + def _indexes(self): + """Instantiate the indexes only when asked + + Returns + ------- + list + A list of all indexes, tied to the field. + + """ + return [index_class(field=self.field) for index_class in self.index_classes] + + def can_handle_suffix(self, suffix): + """Tell if one of the managed indexes can be used for the given filter prefix + + For parameters, see BaseIndex.can_handle_suffix + + """ + for index in self._indexes: + if index.can_handle_suffix(suffix): + return True + + return False + + def _reset_cache(self): + """Reset attributes used to potentially rollback the indexes + + For the parameters, seen BaseIndex._reset_cache + + """ + for index in self._indexes: + index._reset_cache() + + def _rollback(self): + """Restore the index in its previous state + + For the parameters, seen BaseIndex._rollback + + """ + for index in self._indexes: + index._rollback() + + def get_unique_index(self): + """Returns the first index handling uniqueness + + Returns + ------- + BaseIndex + The first index capable of handling uniqueness + + Raises + ------ + IndexError + If not index is capable of handling uniqueness + + """ + return [index for index in self._indexes if index.handle_uniqueness][0] + + @property + def handle_uniqueness(self): + """Tell if at least one of the indexes can handle uniqueness + + Returns + ------- + bool + ``True`` if this multi-index can handle uniqueness. + + """ + try: + self.get_unique_index() + except IndexError: + return False + else: + return True + + def prepare_args(self, args, transform=True): + """Prepare args to be used by a sub-index + + Parameters + ---------- + args: list + The while list of arguments passed to add, check_uniqueness, get_filtered_keys... + transform: bool + If ``True``, the last entry in `args`, ie the value, will be transformed. + Else it will be kept as is. + + """ + updated_args = list(args) + if transform: + updated_args[-1] = self.transform_value(updated_args[-1]) + if self.key: + updated_args.insert(-1, self.key) + + return updated_args + + def check_uniqueness(self, *args): + """For a unique index, check if the given args are not used twice + + For the parameters, seen BaseIndex.check_uniqueness + + """ + self.get_unique_index().check_uniqueness(*self.prepare_args(args, transform=False)) + + def add(self, *args, **kwargs): + """Add the instance tied to the field to all the indexes + + For the parameters, seen BaseIndex.add + + """ + + check_uniqueness = kwargs.pop('check_uniqueness', False) + args = self.prepare_args(args) + + for index in self._indexes: + index.add(*args, check_uniqueness=check_uniqueness and index.handle_uniqueness, **kwargs) + if check_uniqueness and index.handle_uniqueness: + check_uniqueness = False + + def remove(self, *args): + """Remove the instance tied to the field from all the indexes + + For the parameters, seen BaseIndex.remove + + """ + + args = self.prepare_args(args) + + for index in self._indexes: + index.remove(*args) + + def get_filtered_keys(self, suffix, *args, **kwargs): + """Returns the index keys to be used by the collection for the given args + + For the parameters, see BaseIndex.get_filtered_keys + + """ + + args = self.prepare_args(args, transform=False) + + for index in self._indexes: + if index.can_handle_suffix(suffix): + return index.get_filtered_keys(suffix, *args, **kwargs) diff --git a/tests/contrib/indexes.py b/tests/contrib/indexes.py new file mode 100644 index 0000000..4f63116 --- /dev/null +++ b/tests/contrib/indexes.py @@ -0,0 +1,168 @@ +# -*- coding:utf-8 -*- +from __future__ import absolute_import +from __future__ import unicode_literals + +from limpyd import fields +from limpyd.contrib.indexes import MultiIndexes +from limpyd.exceptions import ImplementationError, UniquenessError +from limpyd.indexes import BaseIndex, NumberRangeIndex, TextRangeIndex, EqualIndex + +from ..base import LimpydBaseTest +from ..indexes import ReverseEqualIndex +from ..model import TestRedisModel + + +class MultiIndexesTestCase(LimpydBaseTest): + + def test_can_be_created_with_many_indexes(self): + index_class = MultiIndexes.compose([TextRangeIndex, ReverseEqualIndex]) + + self.assertTrue(issubclass(index_class, BaseIndex)) + self.assertTrue(issubclass(index_class, MultiIndexes)) + self.assertEqual(index_class.__name__, 'MultiIndexes') + self.assertEqual(index_class.index_classes, [TextRangeIndex, ReverseEqualIndex]) + + index_class = MultiIndexes.compose([TextRangeIndex, NumberRangeIndex], name='MyMultiIndex') + self.assertEqual(index_class.__name__, 'MyMultiIndex') + + def test_multi_index_with_only_one_should_behave_like_the_one(self): + index_class = MultiIndexes.compose([EqualIndex]) + + class MultiIndexOneIndexTestModel(TestRedisModel): + name = fields.StringField(indexable=True, indexes=[index_class], unique=True) + + obj1 = MultiIndexOneIndexTestModel(name="foo") + pk1 = obj1.pk.get() + obj2 = MultiIndexOneIndexTestModel(name="bar") + pk2 = obj2.pk.get() + + # test without suffix + self.assertSetEqual( + set(MultiIndexOneIndexTestModel.collection(name='foo')), + {pk1} + ) + + self.assertSetEqual( + set(MultiIndexOneIndexTestModel.collection(name='bar')), + {pk2} + ) + + self.assertSetEqual( + set(MultiIndexOneIndexTestModel.collection(name='foobar')), + set() + ) + + # test with suffix + self.assertSetEqual( + set(MultiIndexOneIndexTestModel.collection(name__eq='bar')), + {pk2} + ) + + # test invalid suffix + with self.assertRaises(ImplementationError): + MultiIndexOneIndexTestModel.collection(name__gte='bar') + + # test uniqueness + with self.assertRaises(UniquenessError): + MultiIndexOneIndexTestModel(name="foo") + + def test_chaining_should_work(self): + + index_class = MultiIndexes.compose([ + MultiIndexes.compose([ + MultiIndexes.compose([ + MultiIndexes.compose([ + EqualIndex + ]) + ]) + ]) + ]) + + class ChainingIndexTestModel(TestRedisModel): + name = fields.StringField(indexable=True, indexes=[index_class], unique=True) + + obj1 = ChainingIndexTestModel(name="foo") + pk1 = obj1.pk.get() + obj2 = ChainingIndexTestModel(name="bar") + pk2 = obj2.pk.get() + + with self.assertRaises(UniquenessError): + ChainingIndexTestModel(name="foo") + + self.assertEqual( + set(ChainingIndexTestModel.collection(name='foo')), + {pk1} + ) + + def test_filtering(self): + + index_class = MultiIndexes.compose([ + EqualIndex.configure( + prefix='first_letter', + transform=lambda v: v[0] if v else '', + handle_uniqueness=False + ), + EqualIndex + ]) + + class MultiIndexTestModel(TestRedisModel): + name = fields.StringField(indexable=True, indexes=[index_class], unique=True) + + obj1 = MultiIndexTestModel(name="foo") + pk1 = obj1.pk.get() + obj2 = MultiIndexTestModel(name="bar") + pk2 = obj2.pk.get() + + # we should not be able to add another with the same name + with self.assertRaises(UniquenessError): + MultiIndexTestModel(name="foo") + + # but we can with the first letter being the same + # because our special index does not handle uniqueness + obj3 = MultiIndexTestModel(name='baz') + pk3 = obj3.pk.get() + + # access without prefix: the simple should be used + self.assertSetEqual( + set(MultiIndexTestModel.collection(name='foo')), + {pk1} + ) + + # nothing with the first letter + self.assertSetEqual( + set(MultiIndexTestModel.collection(name='f')), + set() + ) + + # the same with `eq` suffix + self.assertSetEqual( + set(MultiIndexTestModel.collection(name__eq='foo')), + {pk1} + ) + self.assertSetEqual( + set(MultiIndexTestModel.collection(name__eq='f')), + set() + ) + + # access with the suffix: the special index should be used + self.assertSetEqual( + set(MultiIndexTestModel.collection(name__first_letter='b')), + {pk2, pk3} + ) + # also with the `eq` suffix + self.assertSetEqual( + set(MultiIndexTestModel.collection(name__first_letter__eq='b')), + {pk2, pk3} + ) + + # and nothing with the full name + self.assertSetEqual( + set(MultiIndexTestModel.collection(name__first_letter='bar')), + set() + ) + + # and it should work with both indexes + self.assertSetEqual( + set(MultiIndexTestModel.collection(name__first_letter='b', name='bar')), + {pk2} + )