diff --git a/django_elasticsearch/contrib/restframework/base.py b/django_elasticsearch/contrib/restframework/base.py index ac314a5..6d29429 100644 --- a/django_elasticsearch/contrib/restframework/base.py +++ b/django_elasticsearch/contrib/restframework/base.py @@ -12,8 +12,14 @@ class AutoCompletionMixin(ListModelMixin): @list_route() def autocomplete(self, request, **kwargs): - field_name = request.QUERY_PARAMS.get('f', None) - query = request.QUERY_PARAMS.get('q', '') + try: + qp = request.query_params + except AttributeError: + # restframework 2 + qp = request.QUERY_PARAMS + + field_name = qp.get('f', None) + query = qp.get('q', '') try: data = self.model.es.complete(field_name, query) diff --git a/django_elasticsearch/contrib/restframework/restframework3.py b/django_elasticsearch/contrib/restframework/restframework3.py index 8768447..090cc1e 100644 --- a/django_elasticsearch/contrib/restframework/restframework3.py +++ b/django_elasticsearch/contrib/restframework/restframework3.py @@ -2,7 +2,7 @@ from django.conf import settings from rest_framework.response import Response -from rest_framework.compat import OrderedDict +from rest_framework.serializers import OrderedDict from rest_framework.settings import api_settings from rest_framework.filters import OrderingFilter from rest_framework.filters import DjangoFilterBackend diff --git a/django_elasticsearch/managers.py b/django_elasticsearch/managers.py index 1f5cc7f..edf2718 100644 --- a/django_elasticsearch/managers.py +++ b/django_elasticsearch/managers.py @@ -1,8 +1,15 @@ # -*- coding: utf-8 -*- import json +try: + import importlib +except ImportError: # python < 2.7 + from django.utils import importlib from django.conf import settings -from django.utils import importlib +try: + from django.utils import importlib +except: + import importlib from django.db.models import FieldDoesNotExist from django_elasticsearch.query import EsQueryset diff --git a/django_elasticsearch/models.py b/django_elasticsearch/models.py index df34dae..39ab19f 100644 --- a/django_elasticsearch/models.py +++ b/django_elasticsearch/models.py @@ -1,8 +1,18 @@ # -*- coding: utf-8 -*- +from django import get_version from django.conf import settings from django.db.models import Model -from django.db.models.signals import post_save, post_delete, post_syncdb +from django.db.models.signals import post_save, post_delete +try: + from django.db.models.signals import post_migrate +except ImportError: # django <= 1.6 + from django.db.models.signals import post_syncdb as post_migrate + from django.db.models.signals import class_prepared +try: + from django.db.models.signals import post_migrate +except ImportError: # django <= 1.6 + from django.db.models.signals import post_syncdb as post_migrate from django_elasticsearch.serializers import EsJsonSerializer from django_elasticsearch.managers import ElasticsearchManager @@ -68,14 +78,20 @@ def es_delete_callback(sender, instance, **kwargs): instance.es.delete() -def es_syncdb_callback(sender, app, created_models, **kwargs): - for model in created_models: +def es_syncdb_callback(sender, app=None, created_models=[], **kwargs): + if int(get_version()[2]) > 6: + models = sender.get_models() + else: + models = created_models + + for model in models: if issubclass(model, EsIndexable): model.es.create_index() + if getattr(settings, 'ELASTICSEARCH_AUTO_INDEX', False): # Note: can't specify the sender class because EsIndexable is Abstract, # see: https://code.djangoproject.com/ticket/9318 post_save.connect(es_save_callback) post_delete.connect(es_delete_callback) - post_syncdb.connect(es_syncdb_callback) + post_migrate.connect(es_syncdb_callback) diff --git a/django_elasticsearch/query.py b/django_elasticsearch/query.py index f7d689f..e4644bb 100644 --- a/django_elasticsearch/query.py +++ b/django_elasticsearch/query.py @@ -116,19 +116,8 @@ def __nonzero__(self): return self._total != 0 def __len__(self): - # if we pass a body without a query, elasticsearch complains - if self._total: - return self._total - if self.mode == self.MODE_MLT: - # Note: there is no count on the mlt api, need to fetch the results - self.do_search() - else: - r = es_client.count( - index=self.index, - doc_type=self.doc_type, - body=self.make_search_body() or None) - self._total = r['count'] - return self._total + self.do_search() + return len(self._result_cache) def make_search_body(self): body = {} @@ -185,12 +174,13 @@ def make_search_body(self): filtr = {'query': {'match': {field_name: {'query': value}}}} elif operator in ['gt', 'gte', 'lt', 'lte']: - filtr = {'range': {field_name: {operator: value}}} + filtr = {'bool': {'must': [{'range': {field_name: { + operator: value}}}]}} elif operator == 'range': - filtr = {'range': {field_name: { + filtr = {'bool': {'must': [{'range': {field_name: { 'gte': value[0], - 'lte': value[1]}}} + 'lte': value[1]}}}]}} elif operator == 'isnull': if value: @@ -215,9 +205,12 @@ def response(self): self.do_search() return self._response + def _fetch_all(self): + self.do_search() + def do_search(self): if self.is_evaluated: - return self + return body = self.make_search_body() if self.facets_fields: @@ -273,6 +266,7 @@ def do_search(self): else: if 'from' in search_params: search_params['from_'] = search_params.pop('from') + r = es_client.search(**search_params) self._response = r @@ -291,7 +285,8 @@ def do_search(self): self._max_score = r['hits']['max_score'] self._total = r['hits']['total'] - return self + + return def query(self, query): clone = self._clone() @@ -417,7 +412,19 @@ def suggestions(self): return self._suggestions def count(self): - return self.__len__() + # if we pass a body without a query, elasticsearch complains + if self._total: + return self._total + if self.mode == self.MODE_MLT: + # Note: there is no count on the mlt api, need to fetch the results + self.do_search() + else: + r = es_client.count( + index=self.index, + doc_type=self.doc_type, + body=self.make_search_body() or None) + self._total = r['count'] + return self._total def deserialize(self): self._deserialize = True @@ -429,3 +436,6 @@ def extra(self, body): clone = self._clone() clone.extra_body = body return clone + + def prefetch_related(self): + raise NotImplementedError(".prefetch_related is not available for an EsQueryset.") diff --git a/django_elasticsearch/tests/__init__.py b/django_elasticsearch/tests/__init__.py index cc2e8a1..3596cf4 100644 --- a/django_elasticsearch/tests/__init__.py +++ b/django_elasticsearch/tests/__init__.py @@ -1,4 +1,5 @@ from django_elasticsearch.tests.test_indexable import EsIndexableTestCase +from django_elasticsearch.tests.test_indexable import EsAutoIndexTestCase from django_elasticsearch.tests.test_qs import EsQuerysetTestCase from django_elasticsearch.tests.test_views import EsViewTestCase from django_elasticsearch.tests.test_serializer import EsJsonSerializerTestCase @@ -8,5 +9,6 @@ __all__ = ['EsQuerysetTestCase', 'EsViewTestCase', 'EsIndexableTestCase', + 'EsAutoIndexTestCase', 'EsJsonSerializerTestCase', 'EsRestFrameworkTestCase'] diff --git a/django_elasticsearch/tests/test_indexable.py b/django_elasticsearch/tests/test_indexable.py index a109fae..1d0d9b2 100644 --- a/django_elasticsearch/tests/test_indexable.py +++ b/django_elasticsearch/tests/test_indexable.py @@ -9,6 +9,8 @@ from test_app.models import TestModel +from django import get_version + class EsIndexableTestCase(TestCase): def setUp(self): @@ -51,40 +53,39 @@ def test_delete(self): def test_mlt(self): qs = self.instance.es.mlt(mlt_fields=['first_name',], min_term_freq=1, min_doc_freq=1) - self.assertEqual(len(qs), 0) + self.assertEqual(qs.count(), 0) a = TestModel.objects.create(username=u"2", first_name=u"woot", last_name=u"foo fooo") a.es.do_index() a.es.do_update() results = self.instance.es.mlt(mlt_fields=['first_name',], min_term_freq=1, min_doc_freq=1).deserialize() - self.assertEqual(len(results), 1) + self.assertEqual(results.count(), 1) self.assertEqual(results[0], a) def test_search(self): hits = TestModel.es.search('wee') - self.assertEqual(len(hits), 0) + self.assertEqual(hits.count(), 0) hits = TestModel.es.search('woot') - self.assertEqual(len(hits), 1) + self.assertEqual(hits.count(), 1) def test_search_with_facets(self): s = TestModel.es.search('whatever').facet(['first_name',]) self.assertEqual(s.count(), 0) - expected = {u'doc_count': 1, - u'first_name': {u'buckets': [{u'doc_count': 1, - u'key': u'woot'}]}} - self.assertEqual(s.facets, expected) + expected = [{u'doc_count': 1, u'key': u'woot'}] + self.assertEqual(s.facets['doc_count'], 1) + self.assertEqual(s.facets['first_name']['buckets'], expected) def test_fuzziness(self): hits = TestModel.es.search('woo') # instead of woot - self.assertEqual(len(hits), 1) + self.assertEqual(hits.count(), 1) hits = TestModel.es.search('woo', fuzziness=0) - self.assertEqual(len(hits), 0) + self.assertEqual(hits.count(), 0) hits = TestModel.es.search('waat', fuzziness=2) - self.assertEqual(len(hits), 1) + self.assertEqual(hits.count(), 1) @withattrs(TestModel.Elasticsearch, 'fields', ['username']) @withattrs(TestModel.Elasticsearch, 'mappings', {"username": {"boost": 20}}) @@ -181,3 +182,57 @@ def test_diff(self): # force diff to reload from db deserialized = TestModel.es.all().deserialize()[0] self.assertEqual(deserialized.es.diff(), {}) + + +class EsAutoIndexTestCase(TestCase): + """ + integration test with django's db callbacks + """ + + def setUp(self): + from django.db.models.signals import post_save, post_delete + try: + from django.db.models.signals import post_migrate + except ImportError: # django <= 1.6 + from django.db.models.signals import post_syncdb as post_migrate + + from django_elasticsearch.models import es_save_callback + from django_elasticsearch.models import es_delete_callback + from django_elasticsearch.models import es_syncdb_callback + try: + from django.apps import apps + app = apps.get_app_config('django_elasticsearch') + except ImportError: # django 1.4 + from django.db.models import get_app + app = get_app('django_elasticsearch') + + post_save.connect(es_save_callback) + post_delete.connect(es_delete_callback) + post_migrate.connect(es_syncdb_callback) + + if int(get_version()[2]) >= 6: + sender = app + else: + sender = None + post_migrate.send(sender=sender, + app_config=app, + app=app, # django 1.4 + created_models=[TestModel,], + verbosity=2) + + self.instance = TestModel.objects.create(username=u"1", + first_name=u"woot", + last_name=u"foo") + self.instance.es.do_index() + + def test_auto_save(self): + self.instance.first_name = u'Test' + self.instance.save() + TestModel.es.do_update() + self.assertEqual(TestModel.es.filter(first_name=u'Test').count(), 1) + + def test_auto_delete(self): + self.instance.es.delete() + TestModel.es.do_update() + self.assertEqual(TestModel.es.filter(first_name=u'Test').count(), 0) + self.assertEqual(TestModel.es.filter(first_name=u'Test').count(), 0) diff --git a/django_elasticsearch/tests/test_qs.py b/django_elasticsearch/tests/test_qs.py index 85deb7b..60f8c12 100644 --- a/django_elasticsearch/tests/test_qs.py +++ b/django_elasticsearch/tests/test_qs.py @@ -5,6 +5,7 @@ from django.test import TestCase from django.test.utils import override_settings from django.contrib.auth.models import Group +from django.template import Template, Context from django_elasticsearch.client import es_client from django_elasticsearch.managers import EsQueryset @@ -80,7 +81,8 @@ def test_use_cache(self): list(qs) # use cache list(qs) - mocked.assert_called_once() + + self.assertEqual(len(mocked.mock_calls), 1) # same for a sliced query with mock.patch.object(EsQueryset, @@ -90,22 +92,20 @@ def test_use_cache(self): list(qs[0:5]) # use cache list(qs[0:5]) - mocked.assert_called_once() + + self.assertEqual(len(mocked.mock_calls), 1) def test_facets(self): qs = TestModel.es.queryset.facet(['last_name']) - expected = {u'doc_count': 4, - u'last_name': {u'buckets': [{u'doc_count': 3, - u'key': u'smith'}, - {u'doc_count': 1, - u'key': u'bar'}]}} - self.assertEqual(expected, qs.facets) + expected = [{u'doc_count': 3, u'key': u'smith'}, + {u'doc_count': 1, u'key': u'bar'}] + self.assertEqual(qs.facets['doc_count'], 4) + self.assertEqual(qs.facets['last_name']['buckets'], expected) def test_non_global_facets(self): qs = TestModel.es.search("Foo").facet(['last_name'], use_globals=False) - expected = {u'last_name': {u'buckets': [{u'doc_count': 1, - u'key': u'bar'}]}} - self.assertEqual(expected, qs.facets) + expected = [{u'doc_count': 1, u'key': u'bar'}] + self.assertEqual(qs.facets['last_name']['buckets'], expected) def test_suggestions(self): qs = TestModel.es.search('smath').suggest(['last_name',], limit=3) @@ -244,6 +244,7 @@ def test_filter_date_range(self): time.sleep(2) contents = TestModel.es.filter(date_joined_exp__iso__gte=self.t2.date_joined.isoformat()).deserialize() + self.assertTrue(self.t1 not in contents) self.assertTrue(self.t2 in contents) self.assertTrue(self.t3 in contents) @@ -356,3 +357,20 @@ def test_extra(self): # make sure it didn't break the query otherwise self.assertTrue(q.deserialize()) + + # some attributes were missing on the queryset + # raising an AttributeError when passed to a template + def test_qs_attributes_from_template(self): + qs = self.t1.es.all().order_by('id') + t = Template("{% for e in qs %}{{e.username}}. {% endfor %}") + expected = u'woot woot. woot. BigMama. foo. ' + result = t.render(Context({'qs': qs})) + self.assertEqual(result, expected) + + def test_prefetch_related(self): + with self.assertRaises(NotImplementedError): + TestModel.es.all().prefetch_related() + + def test_range_plus_must(self): + q = TestModel.es.filter(date_joined__gt='now-10d').filter(first_name="John") + self.assertEqual(q.count(), 1) diff --git a/django_elasticsearch/tests/test_restframework.py b/django_elasticsearch/tests/test_restframework.py index 2fbd476..8396641 100644 --- a/django_elasticsearch/tests/test_restframework.py +++ b/django_elasticsearch/tests/test_restframework.py @@ -111,10 +111,9 @@ def test_facets(self): queryset = TestModel.es.all() filter_backend = ElasticsearchFilterBackend() s = filter_backend.filter_queryset(self.fake_request, queryset, self.fake_view) - expected = {u'doc_count': 3, - u'first_name': {u'buckets': [{u'doc_count': 1, - u'key': u'test'}]}} - self.assertEqual(s.facets, expected) + expected = [{u'doc_count': 1, u'key': u'test'}] + self.assertEqual(s.facets['doc_count'], 3) + self.assertEqual(s.facets['first_name']['buckets'], expected) @withattrs(TestModel.Elasticsearch, 'facets_fields', ['first_name',]) def test_faceted_viewset(self): diff --git a/readme.md b/readme.md index 6377f1d..55b19ba 100644 --- a/readme.md +++ b/readme.md @@ -59,6 +59,14 @@ Like a regular Queryset, an EsQueryset is lazy, and if evaluated, returns a list > django-elasticsearch **DOES NOT** index documents by itself unless told to, either set settings.ELASTICSEARCH_AUTO_INDEX to True to index your models when you save them, or call directly myinstance.es.do_index(). +To specify the size of output of documents, it is necessary to make a slice of data, for example: + +``` +len(list(MyModel.es.search('value'))) +>>> 10 +len(list(MyModel.es.search('value')[0:100])) +>>> 42 +``` CONFIGURATION ============= @@ -88,7 +96,7 @@ Project scope configuration (django settings): * **ELASTICSEARCH_CONNECTION_KWARGS** Defaults to {} - Additional kwargs to be passed to at the instanciation of the elasticsearch client. Useful to manage HTTPS connection for example ([Reference](http://elasticsearch-py.readthedocs.org/en/master/api.html#elasticsearch.Elasticsearch)). + Additional kwargs to be passed to at the instantiation of the elasticsearch client. Useful to manage HTTPS connection for example ([Reference](http://elasticsearch-py.readthedocs.org/en/master/api.html#elasticsearch.Elasticsearch)). Model scope configuration: -------------------------- @@ -107,9 +115,9 @@ Each EsIndexable model receive an Elasticsearch class that contains its options Defaults to None The fields to be indexed by elasticsearch, if left to None, all models fields will be indexed. -* **mapping** +* **mappings** Defaults to None - You can override some or all of the fields mapping with this dictionnary + You can override some or all of the fields mapping with this dictionary Example: ```python @@ -118,7 +126,7 @@ Each EsIndexable model receive an Elasticsearch class that contains its options title = models.CharField(max_length=64) class Elasticsearch(EsIndexable.Elasticsearch): - mappings = {'title': {'boost': 2.0} + mappings = {'title': {'boost': 2.0}} ``` In this example we only override the 'boost' attribute of the 'title' field, but there are plenty of possible configurations, see [the docs](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-put-mapping.html). @@ -163,7 +171,7 @@ The Elasticsearch manager is available from the 'es' attribute of EsIndexable Mo suggest_fields=None, suggest_limit=None, fuzziness=None) - Returns a configurated EsQueryset with the given options, or the defaults set in ```EsIndexable.Elasticsearch```. + Returns a configured EsQueryset with the given options, or the defaults set in ```EsIndexable.Elasticsearch```. * **es.all**() Proxy to an empty query ```.search("")```. @@ -218,10 +226,10 @@ The Elasticsearch manager is available from the 'es' attribute of EsIndexable Mo EsQueryset API: --------------- -This class is as close as possible to a standard relational db Queryset, however the db operations (update and delete) are disactivated (i'm open for discution on if and how to implement these). Note that just like regular Querysets, EsQuerysets are lazy, they can be ordered, filtered and faceted. +This class is as close as possible to a standard relational db Queryset, however the db operations (update and delete) are deactivated (i'm open for discussion on if and how to implement these). Note that just like regular Querysets, EsQuerysets are lazy, they can be ordered, filtered and faceted. Note that the return value of the queryset is higly dependent on your mapping, for example, if you want to be able to do an exact filtering with filter() you need a field with {"index" : "not_analyzed"}. -Also by defaut, filters are case insensitive, if you have a case sensitive tokenizer, you need to instanciate EsQueryset with ignore_case=False. +Also by default, filters are case insensitive, if you have a case sensitive tokenizer, you need to instantiate EsQueryset with ignore_case=False. An EsQueryset acts a lot like a regular Queryset: ``` @@ -379,14 +387,14 @@ Two loggers are available 'elasticsearch' and 'elasticsearch.trace'. FAILING GRACEFULLY ================== -You can catch ```elasticsearch.ConnectionError``` and ```elasticsearch.TransportError``` if you want to recover from an error on elasticsearch side. There is an exemple of it in ```django_elasticsearch.views.ElasticsearchListView```. +You can catch ```elasticsearch.ConnectionError``` and ```elasticsearch.TransportError``` if you want to recover from an error on elasticsearch side. There is an example of it in ```django_elasticsearch.views.ElasticsearchListView```. You can also use the ```MyModel.es.check_cluster()``` method which returns True if the cluster is available, in case you want to make sure of it before doing anything. TESTS ===== -Django-elasticsearch has a 95% test coverage, and tests pass for django 1.4 to 1.8. +Django-elasticsearch has a 95% test coverage, and tests pass for django 1.4 to 1.9. Using tox --------- @@ -438,4 +446,4 @@ coverage run --source=django_elasticsearch --omit='*tests*','*migrations*' manag NOTES ===== -Why not make a django database backend ? Because django *does not* support non relational databases, which means that the db backend API is very heavily designed around SQL. I'm usually in favor of hiding the complexity, but in this case for every bit that feels right - auto db and test db creation, client handling, .. - there is one that feels wrong and keeping up with the api changes makes it worse. There is an avorted prototype branch (feature/db-backend) going this way though. \ No newline at end of file +Why not make a django database backend ? Because django *does not* support non relational databases, which means that the db backend API is very heavily designed around SQL. I'm usually in favor of hiding the complexity, but in this case for every bit that feels right - auto db and test db creation, client handling, .. - there is one that feels wrong and keeping up with the api changes makes it worse. There is an avorted prototype branch (feature/db-backend) going this way though. diff --git a/requirements.txt b/requirements.txt index 174c3f8..6289dc0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -elasticsearch +# fixes pyelasticsearchversion to 1.* +elasticsearch>=1.3, <2.0 diff --git a/test_project/test_project/urls.py b/test_project/test_project/urls.py index 0a3a830..2e41e2f 100644 --- a/test_project/test_project/urls.py +++ b/test_project/test_project/urls.py @@ -1,4 +1,5 @@ from django.conf.urls import patterns, include, url +from django.http import HttpResponseNotFound # Uncomment the next two lines to enable the admin: # from django.contrib import admin @@ -7,3 +8,8 @@ urlpatterns = patterns('', url(r'^', include('test_app.urls')), ) + +def custom404(request): + return HttpResponseNotFound(status=404) + +handler404 = 'test_project.urls.custom404' diff --git a/test_project/tox.ini b/test_project/tox.ini index 7d68647..547e86b 100644 --- a/test_project/tox.ini +++ b/test_project/tox.ini @@ -5,7 +5,7 @@ [tox] # envlist = py{27,34}-django{14,16,17,18} -envlist = py{27}-django{14,16,17,18} +envlist = py{27}-django{14,16,17,18,19} skipsdist=True [testenv] @@ -15,7 +15,8 @@ deps = django16: django>=1.6, <1.7 django17: django>=1.7, <1.8 django18: django>=1.8, <1.9 + django19: django>=1.9, <2.0 django{14,16,17}: djangorestframework>=2.4, <3.0 - django18: djangorestframework>3.0 + django{18,19}: djangorestframework>3.0, <3.2 -r../requirements.txt -rrequirements.txt