Rebase with upstream

django_elasticsearch/contrib/restframework/base.py

@@ -12,8 +12,14 @@ class AutoCompletionMixin(ListModelMixin):
 
     @list_route()
     def autocomplete(self, request, **kwargs):
-        field_name = request.QUERY_PARAMS.get('f', None)
-        query = request.QUERY_PARAMS.get('q', '')
+        try:
+            qp = request.query_params
+        except AttributeError:
+            # restframework 2
+            qp = request.QUERY_PARAMS
+
+        field_name = qp.get('f', None)
+        query = qp.get('q', '')
 
         try:
             data = self.model.es.complete(field_name, query)

django_elasticsearch/contrib/restframework/restframework3.py

@@ -2,7 +2,7 @@
 from django.conf import settings
 
 from rest_framework.response import Response
-from rest_framework.compat import OrderedDict
+from rest_framework.serializers import OrderedDict
 from rest_framework.settings import api_settings
 from rest_framework.filters import OrderingFilter
 from rest_framework.filters import DjangoFilterBackend

django_elasticsearch/managers.py

@@ -1,8 +1,15 @@
 # -*- coding: utf-8 -*-
 import json
+try:
+    import importlib
+except ImportError:  # python < 2.7
+    from django.utils import importlib
 
 from django.conf import settings
-from django.utils import importlib
+try:
+    from django.utils import importlib
+except:
+    import importlib
 from django.db.models import FieldDoesNotExist
 
 from django_elasticsearch.query import EsQueryset

django_elasticsearch/models.py

@@ -1,8 +1,18 @@
 # -*- coding: utf-8 -*-
+from django import get_version
 from django.conf import settings
 from django.db.models import Model
-from django.db.models.signals import post_save, post_delete, post_syncdb
+from django.db.models.signals import post_save, post_delete
+try:
+    from django.db.models.signals import post_migrate
+except ImportError:  # django <= 1.6
+    from django.db.models.signals import post_syncdb as post_migrate
+
 from django.db.models.signals import class_prepared
+try:
+    from django.db.models.signals import post_migrate
+except ImportError:  # django <= 1.6
+    from django.db.models.signals import post_syncdb as post_migrate
 
 from django_elasticsearch.serializers import EsJsonSerializer
 from django_elasticsearch.managers import ElasticsearchManager
@@ -68,14 +78,20 @@ def es_delete_callback(sender, instance, **kwargs):
     instance.es.delete()
 
 
-def es_syncdb_callback(sender, app, created_models, **kwargs):
-    for model in created_models:
+def es_syncdb_callback(sender, app=None, created_models=[], **kwargs):
+    if int(get_version()[2]) > 6:
+        models = sender.get_models()
+    else:
+        models = created_models
+
+    for model in models:
         if issubclass(model, EsIndexable):
             model.es.create_index()
 
+
 if getattr(settings, 'ELASTICSEARCH_AUTO_INDEX', False):
     # Note: can't specify the sender class because EsIndexable is Abstract,
     # see: https://code.djangoproject.com/ticket/9318
     post_save.connect(es_save_callback)
     post_delete.connect(es_delete_callback)
-    post_syncdb.connect(es_syncdb_callback)
+    post_migrate.connect(es_syncdb_callback)

django_elasticsearch/query.py

@@ -116,19 +116,8 @@ def __nonzero__(self):
         return self._total != 0
 
     def __len__(self):
-        # if we pass a body without a query, elasticsearch complains
-        if self._total:
-            return self._total
-        if self.mode == self.MODE_MLT:
-            # Note: there is no count on the mlt api, need to fetch the results
-            self.do_search()
-        else:
-            r = es_client.count(
-                index=self.index,
-                doc_type=self.doc_type,
-                body=self.make_search_body() or None)
-            self._total = r['count']
-        return self._total
+        self.do_search()
+        return len(self._result_cache)
 
     def make_search_body(self):
         body = {}
@@ -185,12 +174,13 @@ def make_search_body(self):
                     filtr = {'query': {'match': {field_name: {'query': value}}}}
 
                 elif operator in ['gt', 'gte', 'lt', 'lte']:
-                    filtr = {'range': {field_name: {operator: value}}}
+                    filtr = {'bool': {'must': [{'range': {field_name: {
+                        operator: value}}}]}}
 
                 elif operator == 'range':
-                    filtr = {'range': {field_name: {
+                    filtr = {'bool': {'must': [{'range': {field_name: {
                         'gte': value[0],
-                        'lte': value[1]}}}
+                        'lte': value[1]}}}]}}
 
                 elif operator == 'isnull':
                     if value:
@@ -215,9 +205,12 @@ def response(self):
         self.do_search()
         return self._response
 
+    def _fetch_all(self):
+        self.do_search()
+
     def do_search(self):
         if self.is_evaluated:
-            return self
+            return
 
         body = self.make_search_body()
         if self.facets_fields:
@@ -273,6 +266,7 @@ def do_search(self):
         else:
             if 'from' in search_params:
                 search_params['from_'] = search_params.pop('from')
+
             r = es_client.search(**search_params)
 
         self._response = r
@@ -291,7 +285,8 @@ def do_search(self):
         self._max_score = r['hits']['max_score']
 
         self._total = r['hits']['total']
-        return self
+
+        return
 
     def query(self, query):
         clone = self._clone()
@@ -417,7 +412,19 @@ def suggestions(self):
         return self._suggestions
 
     def count(self):
-        return self.__len__()
+        # if we pass a body without a query, elasticsearch complains
+        if self._total:
+            return self._total
+        if self.mode == self.MODE_MLT:
+            # Note: there is no count on the mlt api, need to fetch the results
+            self.do_search()
+        else:
+            r = es_client.count(
+                index=self.index,
+                doc_type=self.doc_type,
+                body=self.make_search_body() or None)
+            self._total = r['count']
+        return self._total
 
     def deserialize(self):
         self._deserialize = True
@@ -429,3 +436,6 @@ def extra(self, body):
         clone = self._clone()
         clone.extra_body = body
         return clone
+
+    def prefetch_related(self):
+        raise NotImplementedError(".prefetch_related is not available for an EsQueryset.")

django_elasticsearch/tests/__init__.py

@@ -1,4 +1,5 @@
 from django_elasticsearch.tests.test_indexable import EsIndexableTestCase
+from django_elasticsearch.tests.test_indexable import EsAutoIndexTestCase
 from django_elasticsearch.tests.test_qs import EsQuerysetTestCase
 from django_elasticsearch.tests.test_views import EsViewTestCase
 from django_elasticsearch.tests.test_serializer import EsJsonSerializerTestCase
@@ -8,5 +9,6 @@
 __all__ = ['EsQuerysetTestCase',
            'EsViewTestCase',
            'EsIndexableTestCase',
+           'EsAutoIndexTestCase',
            'EsJsonSerializerTestCase',
            'EsRestFrameworkTestCase']

django_elasticsearch/tests/test_indexable.py

@@ -9,6 +9,8 @@
 
 from test_app.models import TestModel
 
+from django import get_version
+
 
 class EsIndexableTestCase(TestCase):
     def setUp(self):
@@ -51,40 +53,39 @@ def test_delete(self):
 
     def test_mlt(self):
         qs = self.instance.es.mlt(mlt_fields=['first_name',], min_term_freq=1, min_doc_freq=1)
-        self.assertEqual(len(qs), 0)
+        self.assertEqual(qs.count(), 0)
 
         a = TestModel.objects.create(username=u"2", first_name=u"woot", last_name=u"foo fooo")
         a.es.do_index()
         a.es.do_update()
 
         results = self.instance.es.mlt(mlt_fields=['first_name',], min_term_freq=1, min_doc_freq=1).deserialize()
-        self.assertEqual(len(results), 1)
+        self.assertEqual(results.count(), 1)
         self.assertEqual(results[0], a)
 
     def test_search(self):
         hits = TestModel.es.search('wee')
-        self.assertEqual(len(hits), 0)
+        self.assertEqual(hits.count(), 0)
 
         hits = TestModel.es.search('woot')
-        self.assertEqual(len(hits), 1)
+        self.assertEqual(hits.count(), 1)
 
     def test_search_with_facets(self):
         s = TestModel.es.search('whatever').facet(['first_name',])
         self.assertEqual(s.count(), 0)
-        expected = {u'doc_count': 1,
-                    u'first_name': {u'buckets': [{u'doc_count': 1,
-                                                  u'key': u'woot'}]}}
-        self.assertEqual(s.facets, expected)
+        expected = [{u'doc_count': 1, u'key': u'woot'}]
+        self.assertEqual(s.facets['doc_count'], 1)
+        self.assertEqual(s.facets['first_name']['buckets'], expected)
 
     def test_fuzziness(self):
         hits = TestModel.es.search('woo')  # instead of woot
-        self.assertEqual(len(hits), 1)
+        self.assertEqual(hits.count(), 1)
 
         hits = TestModel.es.search('woo', fuzziness=0)
-        self.assertEqual(len(hits), 0)
+        self.assertEqual(hits.count(), 0)
 
         hits = TestModel.es.search('waat', fuzziness=2)
-        self.assertEqual(len(hits), 1)
+        self.assertEqual(hits.count(), 1)
 
     @withattrs(TestModel.Elasticsearch, 'fields', ['username'])
     @withattrs(TestModel.Elasticsearch, 'mappings', {"username": {"boost": 20}})
@@ -181,3 +182,57 @@ def test_diff(self):
         # force diff to reload from db
         deserialized = TestModel.es.all().deserialize()[0]
         self.assertEqual(deserialized.es.diff(), {})
+
+
+class EsAutoIndexTestCase(TestCase):
+    """
+    integration test with django's db callbacks
+    """
+
+    def setUp(self):
+        from django.db.models.signals import post_save, post_delete
+        try:
+            from django.db.models.signals import post_migrate
+        except ImportError:  # django <= 1.6
+            from django.db.models.signals import post_syncdb as post_migrate
+
+        from django_elasticsearch.models import es_save_callback
+        from django_elasticsearch.models import es_delete_callback
+        from django_elasticsearch.models import es_syncdb_callback
+        try:
+            from django.apps import apps
+            app = apps.get_app_config('django_elasticsearch')
+        except ImportError: # django 1.4
+            from django.db.models import get_app
+            app = get_app('django_elasticsearch')
+
+        post_save.connect(es_save_callback)
+        post_delete.connect(es_delete_callback)
+        post_migrate.connect(es_syncdb_callback)
+
+        if int(get_version()[2]) >= 6:
+            sender = app
+        else:
+            sender = None
+        post_migrate.send(sender=sender,
+                          app_config=app,
+                          app=app,  # django 1.4
+                          created_models=[TestModel,],
+                          verbosity=2)
+
+        self.instance = TestModel.objects.create(username=u"1",
+                                                 first_name=u"woot",
+                                                 last_name=u"foo")
+        self.instance.es.do_index()
+
+    def test_auto_save(self):
+        self.instance.first_name = u'Test'
+        self.instance.save()
+        TestModel.es.do_update()
+        self.assertEqual(TestModel.es.filter(first_name=u'Test').count(), 1)
+
+    def test_auto_delete(self):
+        self.instance.es.delete()
+        TestModel.es.do_update()
+        self.assertEqual(TestModel.es.filter(first_name=u'Test').count(), 0)
+        self.assertEqual(TestModel.es.filter(first_name=u'Test').count(), 0)

django_elasticsearch/tests/test_qs.py

@@ -5,6 +5,7 @@
 from django.test import TestCase
 from django.test.utils import override_settings
 from django.contrib.auth.models import Group
+from django.template import Template, Context
 
 from django_elasticsearch.client import es_client
 from django_elasticsearch.managers import EsQueryset
@@ -80,7 +81,8 @@ def test_use_cache(self):
             list(qs)
             # use cache
             list(qs)
-        mocked.assert_called_once()
+
+        self.assertEqual(len(mocked.mock_calls), 1)
 
         # same for a sliced query
         with mock.patch.object(EsQueryset,
@@ -90,22 +92,20 @@ def test_use_cache(self):
             list(qs[0:5])
             # use cache
             list(qs[0:5])
-        mocked.assert_called_once()
+
+        self.assertEqual(len(mocked.mock_calls), 1)
 
     def test_facets(self):
         qs = TestModel.es.queryset.facet(['last_name'])
-        expected = {u'doc_count': 4,
-                    u'last_name': {u'buckets': [{u'doc_count': 3,
-                                                 u'key': u'smith'},
-                                                {u'doc_count': 1,
-                                                 u'key': u'bar'}]}}
-        self.assertEqual(expected, qs.facets)
+        expected = [{u'doc_count': 3, u'key': u'smith'},
+                    {u'doc_count': 1, u'key': u'bar'}]
+        self.assertEqual(qs.facets['doc_count'], 4)
+        self.assertEqual(qs.facets['last_name']['buckets'], expected)
 
     def test_non_global_facets(self):
         qs = TestModel.es.search("Foo").facet(['last_name'], use_globals=False)
-        expected = {u'last_name': {u'buckets': [{u'doc_count': 1,
-                                                 u'key': u'bar'}]}}
-        self.assertEqual(expected, qs.facets)
+        expected = [{u'doc_count': 1, u'key': u'bar'}]
+        self.assertEqual(qs.facets['last_name']['buckets'], expected)
 
     def test_suggestions(self):
         qs = TestModel.es.search('smath').suggest(['last_name',], limit=3)
@@ -244,6 +244,7 @@ def test_filter_date_range(self):
         time.sleep(2)
 
         contents = TestModel.es.filter(date_joined_exp__iso__gte=self.t2.date_joined.isoformat()).deserialize()
+
         self.assertTrue(self.t1 not in contents)
         self.assertTrue(self.t2 in contents)
         self.assertTrue(self.t3 in contents)
@@ -356,3 +357,20 @@ def test_extra(self):
 
         # make sure it didn't break the query otherwise
         self.assertTrue(q.deserialize())
+
+    # some attributes were missing on the queryset
+    # raising an AttributeError when passed to a template
+    def test_qs_attributes_from_template(self):
+        qs = self.t1.es.all().order_by('id')
+        t = Template("{% for e in qs %}{{e.username}}. {% endfor %}")
+        expected = u'woot woot. woot. BigMama. foo. '
+        result = t.render(Context({'qs': qs}))
+        self.assertEqual(result, expected)
+
+    def test_prefetch_related(self):
+        with self.assertRaises(NotImplementedError):
+            TestModel.es.all().prefetch_related()
+
+    def test_range_plus_must(self):
+        q = TestModel.es.filter(date_joined__gt='now-10d').filter(first_name="John")
+        self.assertEqual(q.count(), 1)