From e4b258bba37c5cd4a895bb381568d6835c53e919 Mon Sep 17 00:00:00 2001 From: Phillip Ahereza Date: Mon, 13 Aug 2018 14:22:09 +0300 Subject: [PATCH 1/8] added backend settings --- takwimu/models/dashboard.py | 13 ++++++++---- takwimu/settings.py | 41 +++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/takwimu/models/dashboard.py b/takwimu/models/dashboard.py index c7ff73dd..e3f8408f 100644 --- a/takwimu/models/dashboard.py +++ b/takwimu/models/dashboard.py @@ -47,12 +47,12 @@ class TopicPageDataIndicators(Orderable, TopicPageDataIndicator): class TopicPage(Page): - ''' + """ Topic Editor ------------ All data indicators are made available to profiles + sections via topics. This therefore serves as an editorial interface to create topics and link indicators to it. - ''' + """ description = models.TextField(blank=True) icon = IconField() #adding icon property that uses the djano-fontawesome app structure @@ -64,7 +64,6 @@ class TopicPage(Page): # Search index configuration search_fields = Page.search_fields + [ - index.SearchField('title'), index.SearchField('description') ] @@ -361,13 +360,19 @@ class ExplainerSteps(Page): StreamFieldPanel('steps'), ] -class FAQ(models.Model): + +class FAQ(index.Indexed, models.Model): question = models.TextField() answer = RichTextField() cta_one_url = models.URLField("'Find Out More' button URL", default="https://takwimu.zendesk.com/") cta_two_name = models.TextField("Second button Name (optional)", blank=True) cta_two_url = models.URLField("Second button URL (optional)", blank=True) + search_fields = [ + index.SearchField('question'), + index.SearchField('answer'), + ] + def __str__(self): return self.question.encode('ascii', 'ignore') diff --git a/takwimu/settings.py b/takwimu/settings.py index 9d39a752..72b34393 100644 --- a/takwimu/settings.py +++ b/takwimu/settings.py @@ -1,6 +1,9 @@ # coding=utf-8 import os +from elasticsearch import RequestsHttpConnection +from requests_aws4auth import AWS4Auth + from hurumap.settings import * # noqa # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -107,3 +110,41 @@ ZENDESK_API = 'https://takwimu.zendesk.com/api/v2/requests.json' ZENDESK_API_TOKEN = os.environ.get('ZENDESK_API_TOKEN') + +# ------------------------------------------------------------------------------------- +# WAGTAIL Search +# ------------------------------------------------------------------------------------- +if DEBUG: + WAGTAILSEARCH_BACKENDS = { + 'default': { + 'BACKEND': 'wagtail.wagtailsearch.backends.elasticsearch5', + 'URLS': ['http://localhost:9200'], + 'INDEX': 'takwimu', + 'TIMEOUT': 5, + 'OPTIONS': {}, + 'INDEX_SETTINGS': {}, + } + } +else: + ES_ACCESS_KEY = os.environ.get('ES_ACCESS_KEY') + ES_SECRET_KEY = os.environ.get('ES_SECRET_KEY') + ES_REGION = os.environ.get('ES_REGION') + ES_CLUSTER_NAME = os.environ.get('ES_CLUSTERNAME') + + WAGTAILSEARCH_BACKENDS = { + 'default': { + 'BACKEND': 'wagtail.wagtailsearch.backends.elasticsearch5', + 'INDEX': 'wagtail', + 'TIMEOUT': 5, + 'HOSTS': [{ + 'host': '{}.{}.es.amazonaws.com'.format(ES_CLUSTER_NAME, ES_REGION), + 'port': 443, + 'use_ssl': True, + 'verify_certs': True, + 'http_auth': AWS4Auth(ES_ACCESS_KEY, ES_SECRET_KEY, ES_REGION, 'es'), + }], + 'OPTIONS': { + 'connection_class': RequestsHttpConnection, + }, + } + } \ No newline at end of file From b1bbdf43f5e2d2614d5efe4c457e8b1a2105b28c Mon Sep 17 00:00:00 2001 From: Phillip Ahereza Date: Mon, 13 Aug 2018 14:46:41 +0300 Subject: [PATCH 2/8] add testing view --- takwimu/templates/search_results.html | 35 +++++++++++++++++++++++++++ takwimu/urls.py | 4 ++- takwimu/views.py | 23 +++++++++++++++++- 3 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 takwimu/templates/search_results.html diff --git a/takwimu/templates/search_results.html b/takwimu/templates/search_results.html new file mode 100644 index 00000000..cc243c69 --- /dev/null +++ b/takwimu/templates/search_results.html @@ -0,0 +1,35 @@ +{% extends 'takwimu/_layouts/page.html' %} +{% load wagtailcore_tags %} +{% load staticfiles %} + +{% block content %} + + {% include 'takwimu/_includes/homepage/hero.html' %} + +
+
+
+ + +
+
+ + {% if search_results %} +
    + {% for result in search_results %} +
  • +

    {{ result }}

    + {% if result.search_description %} + {{ result.search_description|safe }} + {% endif %} +
    • + {% endfor %} +
+ {% elif search_query %} + No results found + {% else %} + Please type something into the search box + {% endif %} +
+ +{% endblock %} \ No newline at end of file diff --git a/takwimu/urls.py b/takwimu/urls.py index 5900b3f9..95b313ba 100644 --- a/takwimu/urls.py +++ b/takwimu/urls.py @@ -3,7 +3,7 @@ from django.views.generic import RedirectView from takwimu import settings -from takwimu.views import HomePageView, SupportServicesIndexView, AboutUsView, LegalView, TopicView +from takwimu.views import HomePageView, SupportServicesIndexView, AboutUsView, LegalView, TopicView, search_view from takwimu.views import handler404, handler500 from wazimap.views import HomepageView as ProfileView from takwimu.feed import CountryProfileFeed @@ -20,6 +20,8 @@ url(r'^profiles/$', ProfileView.as_view(), name='profiles'), url(r'^topics/$', TopicView.as_view(), name='topics'), url(r'^feed/$', CountryProfileFeed(), name='rss_feed'), + # testing search page + url(r'^search$', search_view, name='search'), ] + \ hurumap_urlpatterns diff --git a/takwimu/views.py b/takwimu/views.py index 4e2746e0..3cd3bd33 100644 --- a/takwimu/views.py +++ b/takwimu/views.py @@ -2,9 +2,11 @@ import requests from django.conf import settings -from django.shortcuts import render_to_response +from django.shortcuts import render_to_response, render from django.template import RequestContext from django.views.generic import TemplateView, FormView +from wagtail.wagtailcore.models import Page +from wagtail.wagtailsearch.models import Query from takwimu.models.dashboard import ExplainerSteps, FAQ, Testimonial from forms import SupportServicesContactForm @@ -133,3 +135,22 @@ def form_invalid(self, form): print('\n\n\n\n\n\n') print form.data return super(SupportServicesIndexView, self).form_invalid(form) + +# view for testing search functionality +def search_view(request): + # TODO: 13/08/2018 Remove view, url config and template + # Search + search_query = request.GET.get('query', None) + if search_query: + search_results = Page.objects.live().search(search_query) + + # Log the query so Wagtail can suggest promoted results + Query.get(search_query).add_hit() + else: + search_results = Page.objects.none() + + # Render template + return render(request, 'search_results.html', { + 'search_query': search_query, + 'search_results': search_results, + }) From f4b82cea0d69c3f98724bfa5e7af9a4ec00005fd Mon Sep 17 00:00:00 2001 From: Phillip Ahereza Date: Mon, 13 Aug 2018 15:01:52 +0300 Subject: [PATCH 3/8] added new dependencies to requirements.txt --- requirements.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/requirements.txt b/requirements.txt index e41df454..21a41d6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,8 @@ git+https://github.com/CodeForAfricaLabs/django-fontawesome.git@master#egg=djang Markdown==2.6.11 django-debug-toolbar + +elasticsearch==5.5.3 + +requests-aws4auth==0.9 + From 6c2035c6fe3797f45630231000ada839839719db Mon Sep 17 00:00:00 2001 From: Phillip Ahereza Date: Wed, 15 Aug 2018 17:45:02 +0300 Subject: [PATCH 4/8] tried adding elasticsearch to docker --- docker-compose.yml | 7 +++++++ docker-entrypoint.sh | 1 + takwimu/settings.py | 46 +++++++++++++------------------------------- takwimu/urls.py | 2 +- takwimu/views.py | 8 +++++--- 5 files changed, 27 insertions(+), 37 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 32125b64..1b408546 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,12 +10,18 @@ services: - POSTGRES_PASSWORD=takwimu - POSTGRES_DB=takwimu + es: + image: elasticsearch:5.6.10 + ports: + - "9200:9200" + web: build: . ports: - "8000:8000" depends_on: - db + - es volumes: - "./:/srv/takwimu" environment: @@ -27,4 +33,5 @@ services: - PGUSER=takwimu - PGPASSWORD=takwimu - PYTHONDONTWRITEBYTECODE="True" + - ES_HOST=http://es:9200 # - DJANGO_DEBUG=False # For testing deploys diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 86fc6984..85acb0f6 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -3,6 +3,7 @@ python manage.py migrate --noinput # Apply database migrations cat takwimu/sql/*.sql | psql # Upload tables / data python manage.py compilescss # Compile SCSS (offline) python manage.py collectstatic --noinput # Collect static files +python manage.py update_index # Update search index # Prepare log files and start outputting logs to stdout touch /srv/logs/gunicorn.log diff --git a/takwimu/settings.py b/takwimu/settings.py index 59c56a06..f22c8ca6 100644 --- a/takwimu/settings.py +++ b/takwimu/settings.py @@ -117,37 +117,17 @@ # ------------------------------------------------------------------------------------- # WAGTAIL Search # ------------------------------------------------------------------------------------- -if DEBUG: - WAGTAILSEARCH_BACKENDS = { - 'default': { - 'BACKEND': 'wagtail.wagtailsearch.backends.elasticsearch5', - 'URLS': ['http://localhost:9200'], - 'INDEX': 'takwimu', - 'TIMEOUT': 5, - 'OPTIONS': {}, - 'INDEX_SETTINGS': {}, - } + +ES_HOST = os.environ.get('ES_HOST', 'http://localhost:9200') + +WAGTAILSEARCH_BACKENDS = { + 'default': { + 'BACKEND': 'wagtail.wagtailsearch.backends.elasticsearch5', + 'URLS': [ES_HOST], + 'INDEX': 'takwimu', + 'TIMEOUT': 5, + 'OPTIONS': {}, + 'INDEX_SETTINGS': {}, } -else: - ES_ACCESS_KEY = os.environ.get('ES_ACCESS_KEY') - ES_SECRET_KEY = os.environ.get('ES_SECRET_KEY') - ES_REGION = os.environ.get('ES_REGION') - ES_CLUSTER_NAME = os.environ.get('ES_CLUSTERNAME') - - WAGTAILSEARCH_BACKENDS = { - 'default': { - 'BACKEND': 'wagtail.wagtailsearch.backends.elasticsearch5', - 'INDEX': 'wagtail', - 'TIMEOUT': 5, - 'HOSTS': [{ - 'host': '{}.{}.es.amazonaws.com'.format(ES_CLUSTER_NAME, ES_REGION), - 'port': 443, - 'use_ssl': True, - 'verify_certs': True, - 'http_auth': AWS4Auth(ES_ACCESS_KEY, ES_SECRET_KEY, ES_REGION, 'es'), - }], - 'OPTIONS': { - 'connection_class': RequestsHttpConnection, - }, - } - } \ No newline at end of file +} + diff --git a/takwimu/urls.py b/takwimu/urls.py index 95b313ba..30c0f63a 100644 --- a/takwimu/urls.py +++ b/takwimu/urls.py @@ -21,7 +21,7 @@ url(r'^topics/$', TopicView.as_view(), name='topics'), url(r'^feed/$', CountryProfileFeed(), name='rss_feed'), # testing search page - url(r'^search$', search_view, name='search'), + url(r'^search/$', search_view, name='search'), ] + \ hurumap_urlpatterns diff --git a/takwimu/views.py b/takwimu/views.py index 3cd3bd33..a723b815 100644 --- a/takwimu/views.py +++ b/takwimu/views.py @@ -4,7 +4,7 @@ from django.conf import settings from django.shortcuts import render_to_response, render from django.template import RequestContext -from django.views.generic import TemplateView, FormView +from django.views.generic import TemplateView, FormView, View from wagtail.wagtailcore.models import Page from wagtail.wagtailsearch.models import Query @@ -140,8 +140,8 @@ def form_invalid(self, form): def search_view(request): # TODO: 13/08/2018 Remove view, url config and template # Search - search_query = request.GET.get('query', None) - if search_query: + search_query = request.GET.get('query', '') + if search_query != '': search_results = Page.objects.live().search(search_query) # Log the query so Wagtail can suggest promoted results @@ -154,3 +154,5 @@ def search_view(request): 'search_query': search_query, 'search_results': search_results, }) + + From ef0ceb3b9cbf4ddd917ffca594623ca99b2b5100 Mon Sep 17 00:00:00 2001 From: Phillip Ahereza Date: Thu, 16 Aug 2018 11:53:42 +0300 Subject: [PATCH 5/8] changed to cbv --- takwimu/urls.py | 4 ++-- takwimu/views.py | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/takwimu/urls.py b/takwimu/urls.py index 30c0f63a..5d060620 100644 --- a/takwimu/urls.py +++ b/takwimu/urls.py @@ -3,7 +3,7 @@ from django.views.generic import RedirectView from takwimu import settings -from takwimu.views import HomePageView, SupportServicesIndexView, AboutUsView, LegalView, TopicView, search_view +from takwimu.views import HomePageView, SupportServicesIndexView, AboutUsView, LegalView, TopicView, SearchView from takwimu.views import handler404, handler500 from wazimap.views import HomepageView as ProfileView from takwimu.feed import CountryProfileFeed @@ -21,7 +21,7 @@ url(r'^topics/$', TopicView.as_view(), name='topics'), url(r'^feed/$', CountryProfileFeed(), name='rss_feed'), # testing search page - url(r'^search/$', search_view, name='search'), + url(r'^search/$', SearchView.as_view(), name='search'), ] + \ hurumap_urlpatterns diff --git a/takwimu/views.py b/takwimu/views.py index a723b815..70761c23 100644 --- a/takwimu/views.py +++ b/takwimu/views.py @@ -5,6 +5,7 @@ from django.shortcuts import render_to_response, render from django.template import RequestContext from django.views.generic import TemplateView, FormView, View +from django.views.generic.base import TemplateView from wagtail.wagtailcore.models import Page from wagtail.wagtailsearch.models import Query @@ -44,6 +45,7 @@ class LegalView(TemplateView): """ template_name = 'takwimu/about/legal.html' + class TopicView(TemplateView): """ Topic View: @@ -136,6 +138,7 @@ def form_invalid(self, form): print form.data return super(SupportServicesIndexView, self).form_invalid(form) + # view for testing search functionality def search_view(request): # TODO: 13/08/2018 Remove view, url config and template @@ -156,3 +159,20 @@ def search_view(request): }) +class SearchView(TemplateView): + template_name = 'search_results.html' + + def get(self, request, *args, **kwargs): + search_query = request.GET.get('query', '') + if search_query != '': + search_results = Page.objects.live().search(search_query) + + # Log the query so Wagtail can suggest promoted results + Query.get(search_query).add_hit() + else: + search_results = Page.objects.none() + + return render(request, self.template_name, { + 'search_query': search_query, + 'search_results': search_results, + }) From d9ec1ebc28c8b5fe6875e28434a3fdaf5e4e46dc Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Thu, 16 Aug 2018 12:11:19 +0300 Subject: [PATCH 6/8] ES Settings & docker-compose support --- docker-compose.yml | 4 +++- takwimu/settings.py | 51 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1b408546..9260b301 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,6 +32,8 @@ services: - PGDATABASE=takwimu - PGUSER=takwimu - PGPASSWORD=takwimu + - TAKWIMU_ES_INDEX=takwimu + - TAKWIMU_ES_TIMEOUT=10 + - TAKWIMU_ES_URL=http://es:9200 - PYTHONDONTWRITEBYTECODE="True" - - ES_HOST=http://es:9200 # - DJANGO_DEBUG=False # For testing deploys diff --git a/takwimu/settings.py b/takwimu/settings.py index f22c8ca6..3b064f02 100644 --- a/takwimu/settings.py +++ b/takwimu/settings.py @@ -118,16 +118,43 @@ # WAGTAIL Search # ------------------------------------------------------------------------------------- -ES_HOST = os.environ.get('ES_HOST', 'http://localhost:9200') - -WAGTAILSEARCH_BACKENDS = { - 'default': { - 'BACKEND': 'wagtail.wagtailsearch.backends.elasticsearch5', - 'URLS': [ES_HOST], - 'INDEX': 'takwimu', - 'TIMEOUT': 5, - 'OPTIONS': {}, - 'INDEX_SETTINGS': {}, - } -} +TAKWIMU_ES_INDEX = os.environ.get('TAKWIMU_ES_INDEX', 'takwimu') +TAKWIMU_ES_TIMEOUT = int(os.environ.get('TAKWIMU_ES_HOST', '5')) +TAKWIMU_ES_URL = os.environ.get('TAKWIMU_ES_URL', 'http://localhost:9200') + +# Support for AWS ElasticSearch service. If HOST_TYPE is anything other than +# 'AWS', the default configuration will be used. +TAKWIMU_ES_HOST_TYPE = os.environ.get('TAKWIMU_ES_HOST_TYPE', '') +if TAKWIMU_ES_HOST_TYPE.lower() == 'aws': + TAKWIMU_ES_AWS_ACCESS_KEY = os.environ.get('TAKWIMU_ES_AWS_ACCESS_KEY', '') + TAKWIMU_ES_AWS_SECRET_KEY = os.environ.get('TAKWIMU_ES_AWS_SECRET_KEY', '') + TAKWIMU_ES_AWS_REGION = os.environ.get('TAKWIMU_ES_AWS_REGION', '') + WAGTAILSEARCH_BACKENDS = { + 'default': { + 'BACKEND': 'wagtail.wagtailsearch.backends.elasticsearch5', + 'INDEX': TAKWIMU_ES_INDEX, + 'TIMEOUT': TAKWIMU_ES_TIMEOUT, + 'HOSTS': [{ + 'host': TAKWIMU_ES_URL, + 'port': 443, + 'use_ssl': True, + 'verify_certs': True, + 'http_auth': AWS4Auth(TAKWIMU_ES_AWS_ACCESS_KEY, TAKWIMU_ES_AWS_SECRET_KEY, TAKWIMU_ES_AWS_REGION, 'es'), + }], + 'OPTIONS': { + 'connection_class': RequestsHttpConnection, + }, + } + } +else: + WAGTAILSEARCH_BACKENDS = { + 'default': { + 'BACKEND': 'wagtail.wagtailsearch.backends.elasticsearch5', + 'INDEX': TAKWIMU_ES_INDEX, + 'TIMEOUT': TAKWIMU_ES_TIMEOUT, + 'URLS': [TAKWIMU_ES_URL], + 'OPTIONS': {}, + 'INDEX_SETTINGS': {}, + } + } From 5e8af8e840442f2fb96950038f7cb2a6d539aec3 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Fri, 17 Aug 2018 12:24:36 +0300 Subject: [PATCH 7/8] Fix ElasticSearch connection errors --- docker-compose.yml | 5 +++-- takwimu/settings.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9260b301..c4738280 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,13 +27,14 @@ services: environment: - DATABASE_URL=postgresql://takwimu:takwimu@db:5432/takwimu - DJANGO_SECRET_KEY=somethingsecret - - HURUMAP_URL='http://localhost:8000' + - HURUMAP_URL=http://localhost:8000 - PGHOST=db - PGDATABASE=takwimu - PGUSER=takwimu - PGPASSWORD=takwimu - TAKWIMU_ES_INDEX=takwimu - - TAKWIMU_ES_TIMEOUT=10 + # A lot of ConnectionReadTimeout errors, when value this is low + - TAKWIMU_ES_TIMEOUT=30 - TAKWIMU_ES_URL=http://es:9200 - PYTHONDONTWRITEBYTECODE="True" # - DJANGO_DEBUG=False # For testing deploys diff --git a/takwimu/settings.py b/takwimu/settings.py index 3b064f02..35904b89 100644 --- a/takwimu/settings.py +++ b/takwimu/settings.py @@ -120,7 +120,7 @@ TAKWIMU_ES_INDEX = os.environ.get('TAKWIMU_ES_INDEX', 'takwimu') -TAKWIMU_ES_TIMEOUT = int(os.environ.get('TAKWIMU_ES_HOST', '5')) +TAKWIMU_ES_TIMEOUT = int(os.environ.get('TAKWIMU_ES_TIMEOUT', '5')) TAKWIMU_ES_URL = os.environ.get('TAKWIMU_ES_URL', 'http://localhost:9200') # Support for AWS ElasticSearch service. If HOST_TYPE is anything other than From 7884b064f1258c2d429dea03dc8c77dddf15e528 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Mon, 20 Aug 2018 17:42:21 +0300 Subject: [PATCH 8/8] Remove unused function + code format --- takwimu/templates/search_results.html | 2 +- takwimu/urls.py | 1 - takwimu/views.py | 33 ++++++++------------------- 3 files changed, 10 insertions(+), 26 deletions(-) diff --git a/takwimu/templates/search_results.html b/takwimu/templates/search_results.html index cc243c69..2e881886 100644 --- a/takwimu/templates/search_results.html +++ b/takwimu/templates/search_results.html @@ -9,7 +9,7 @@
- +
diff --git a/takwimu/urls.py b/takwimu/urls.py index 5d060620..8ca262f1 100644 --- a/takwimu/urls.py +++ b/takwimu/urls.py @@ -20,7 +20,6 @@ url(r'^profiles/$', ProfileView.as_view(), name='profiles'), url(r'^topics/$', TopicView.as_view(), name='topics'), url(r'^feed/$', CountryProfileFeed(), name='rss_feed'), - # testing search page url(r'^search/$', SearchView.as_view(), name='search'), ] + \ hurumap_urlpatterns diff --git a/takwimu/views.py b/takwimu/views.py index 70761c23..23e6fef3 100644 --- a/takwimu/views.py +++ b/takwimu/views.py @@ -139,38 +139,23 @@ def form_invalid(self, form): return super(SupportServicesIndexView, self).form_invalid(form) -# view for testing search functionality -def search_view(request): - # TODO: 13/08/2018 Remove view, url config and template - # Search - search_query = request.GET.get('query', '') - if search_query != '': - search_results = Page.objects.live().search(search_query) - - # Log the query so Wagtail can suggest promoted results - Query.get(search_query).add_hit() - else: - search_results = Page.objects.none() - - # Render template - return render(request, 'search_results.html', { - 'search_query': search_query, - 'search_results': search_results, - }) - - class SearchView(TemplateView): + """ + Search View + ----------- + Displays search results. + + """ template_name = 'search_results.html' def get(self, request, *args, **kwargs): - search_query = request.GET.get('query', '') - if search_query != '': + search_results = Page.objects.none() + search_query = request.GET.get('q', '') + if search_query: search_results = Page.objects.live().search(search_query) # Log the query so Wagtail can suggest promoted results Query.get(search_query).add_hit() - else: - search_results = Page.objects.none() return render(request, self.template_name, { 'search_query': search_query,