Add task to save rendered content to db; call tasks on delay

core/asciidoc.py

@@ -5,7 +5,7 @@
 from .boostrenderer import get_body_from_html
 
 
-def convert_adoc_to_html(file_path, delete_file= True):
+def convert_adoc_to_html(file_path, delete_file=True):
  """
  Converts an AsciiDoc file to HTML.
  If delete_file is True, the temporary file will be deleted after the

core/tasks.py

@@ -1,42 +1,83 @@
-import os
-
-import subprocess
+import structlog
 
 from celery import shared_task
-
-from .asciidoc import adoc_to_html
-from .boostrenderer import get_body_from_html, get_content_from_s3
+from dateutil.parser import parse
 
 from django.core.cache import caches
 
+from .asciidoc import convert_adoc_to_html, process_adoc_to_html_content
+from .boostrenderer import get_content_from_s3
 from .models import RenderedContent
 
+logger = structlog.get_logger()
+
 
 @shared_task
 def adoc_to_html(file_path, delete_file=True):
- return adoc_to_html(file_path, delete_file=delete_file)
+ return convert_adoc_to_html(file_path, delete_file=delete_file)
+
+
+@shared_task
+def clear_rendered_content_cache_by_cache_key(cache_key):
+ """Deletes a RenderedContent object by its cache key from redis and
+ database."""
+ cache = caches["static_content"]
+ cache.delete(cache_key)
+ RenderedContent.objects.delete_by_cache_key(cache_key)
 
 
 @shared_task
-def refresh_rendered_content_from_s3(content_path, cache_key):
- """ Take a cache """
- result = get_content_from_s3(key=content_path)
- if result and result.get("content"):
- content = result.get("content")
- content_type = result.get("content_type")
- last_updated_at_raw = result.get("last_updated_at")
+def clear_rendered_content_cache_by_content_type(content_type):
+ """Deletes all RenderedContent objects for a given content type from redis
+ and database."""
+ RenderedContent.objects.clear_cache_by_content_type(content_type)
+ RenderedContent.objects.delete_by_content_type(content_type)
 
+
+@shared_task
+def refresh_content_from_s3(s3_key, cache_key):
+ """Calls S3 with the s3_key, then saves the result to the
+ RenderedContent object with the given cache_key."""
+ content_dict = get_content_from_s3(key=s3_key)
+ content = content_dict.get("content")
+ if content_dict and content:
+ content_type = content_dict.get("content_type")
  if content_type == "text/asciidoc":
- content = self.convert_adoc_to_html(content, cache_key)
- last_updated_at = (
- parse(last_updated_at_raw) if last_updated_at_raw else None
- )
+ content = process_adoc_to_html_content(content)
+ last_updated_at_raw = content_dict.get("last_updated_at")
+ last_updated_at = parse(last_updated_at_raw) if last_updated_at_raw else None
+ # Clear the cache because we're going to update it.
+ clear_rendered_content_cache_by_cache_key(cache_key)
+
+ # Update the rendered content.
+ save_rendered_content(
+ cache_key, content_type, content, last_updated_at=last_updated_at
+ )
+ # Cache the refreshed rendered content
+ cache = caches["static_content"]
+ cache.set(cache_key, {"content": content, "content_type": content_type})
 
- # Get the output from the command
- converted_html = result.stdout
 
- # Delete the temporary file
- if delete_file:
- os.remove(file_path)
+@shared_task
+def save_rendered_content(cache_key, content_type, content_html, last_updated_at=None):
+ """Saves a RenderedContent object to database."""
+ defaults = {
+ "content_type": content_type,
+ "content_html": content_html,
+ }
+
+ if last_updated_at:
+ defaults["last_updated_at"] = last_updated_at
 
- return converted_html
+ obj, created = RenderedContent.objects.update_or_create(
+ cache_key=cache_key[:255], defaults=defaults
+ )
+ logger.info(
+ "content_saved_to_rendered_content",
+ cache_key=cache_key,
+ content_type=content_type,
+ status_code=200,
+ obj_id=obj.id,
+ obj_created=created,
+ )
+ return obj

core/tests/test_asciidoc.py

@@ -0,0 +1,49 @@
+import pytest
+import tempfile
+from unittest.mock import patch
+
+from django.core.cache import caches
+from django.test import override_settings
+
+from core.asciidoc import convert_adoc_to_html, process_adoc_to_html_content
+
+
+@override_settings(
+ CACHES={
+ "static_content": {
+ "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
+ "LOCATION": "third-unique-snowflake",
+ "TIMEOUT": "60", # Cache timeout in seconds: 1 minute
+ },
+ }
+)
+def test_adoc_to_html():
+ # Get the static content cache
+ caches["static_content"]
+
+ # The content of the sample adoc file
+ sample_adoc_content = "= Document Title\n\nThis is a sample document.\n"
+
+ # Write the content to a temporary file
+ with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+ temp_file.write(sample_adoc_content.encode())
+ temp_file_path = temp_file.name
+
+ # Execute the task
+ with patch("core.asciidoc.subprocess.run") as mock_run:
+ mock_run.return_value.stdout = "html_content".encode()
+ convert_adoc_to_html(temp_file_path, delete_file=True)
+
+ # Verify that the temporary file has been deleted
+ with pytest.raises(FileNotFoundError):
+ with open(temp_file_path, "r"):
+ pass
+
+
+def test_process_adoc_to_html_content():
+ """Test the process_adoc_to_html_content function."""
+ content = "sample"
+ expected_html = '<div id="header">\n</div><div id="content">\n<div class="paragraph">\n<p>sample</p>\n</div>\n</div>' # noqa: E501
+
+ result = process_adoc_to_html_content(content)
+ assert result == expected_html

core/tests/test_tasks.py

@@ -38,7 +38,7 @@ def test_adoc_to_html():
  temp_file_path = temp_file.name
 
  # Execute the task
- with patch("core.tasks.subprocess.run") as mock_run:
+ with patch("core.asciidoc.subprocess.run") as mock_run:
  mock_run.return_value.stdout = "html_content".encode()
  adoc_to_html(temp_file_path, delete_file=True)
 

core/views.py

@@ -1,6 +1,5 @@
 import os.path
 import structlog
-import tempfile
 from dateutil.parser import parse
 
 from django.conf import settings
@@ -11,13 +10,14 @@
 from django.views.generic import TemplateView
 
 from .asciidoc import process_adoc_to_html_content
-from .boostrenderer import get_body_from_html, get_content_from_s3
+from .boostrenderer import get_content_from_s3
 from .markdown import process_md
 from .models import RenderedContent
 from .tasks import (
- adoc_to_html,
  clear_rendered_content_cache_by_cache_key,
  clear_rendered_content_cache_by_content_type,
+ refresh_content_from_s3,
+ save_rendered_content,
 )
 
 logger = structlog.get_logger()
@@ -164,39 +164,8 @@ def get(self, request, *args, **kwargs):
  return HttpResponseNotFound("Page not found")
  return super().get(request, *args, **kwargs)
 
- def get_template_names(self):
- """Returns the template name."""
- content_type = self.content_dict.get("content_type")
- if content_type == "text/asciidoc":
- return [self.template_name]
- return []
-
- def get_context_data(self, **kwargs):
- """Returns the content and content type for the template. In some cases,
- changes the content type."""
- context = super().get_context_data(**kwargs)
- content_type = self.content_dict.get("content_type")
- content = self.content_dict.get("content")
-
- if content_type == "text/asciidoc":
- content_type = "text/html"
-
- context.update({"content": content, "content_type": content_type})
-
- logger.info(
- "get_content_from_s3_view_success", key=self.kwargs.get("content_path")
- )
-
- return context
-
- def render_to_response(self, context, **response_kwargs):
- """Return the HTML response with a template, or just the content directly."""
- if self.get_template_names():
- return super().render_to_response(context, **response_kwargs)
- else:
- return HttpResponse(
- context["content"], content_type=context["content_type"]
- )
+ def cache_result(self, static_content_cache, cache_key, result):
+ static_content_cache.set(cache_key, result)
 
  def get_content(self, content_path):
  """Returns content from cache, database, or S3"""
@@ -206,11 +175,17 @@ def get_content(self, content_path):
 
  if result is None:
  result = self.get_from_database(cache_key)
+ if result:
+ # When we get a result from the database, we refresh its content
+ refresh_content_from_s3.delay(content_path, cache_key)
 
  if result is None:
- result = self.get_from_s3(content_path, cache_key)
- # Cache the result
- self.cache_result(static_content_cache, cache_key, result)
+ result = self.get_from_s3(content_path)
+ if result:
+ # Save to database
+ self.save_to_database(cache_key, result)
+ # Cache the result
+ self.cache_result(static_content_cache, cache_key, result)
 
  if result is None:
  logger.info(
@@ -222,8 +197,23 @@ def get_content(self, content_path):
 
  return result
 
- def cache_result(self, static_content_cache, cache_key, result):
- static_content_cache.set(cache_key, result)
+ def get_context_data(self, **kwargs):
+ """Returns the content and content type for the template. In some cases,
+ changes the content type."""
+ context = super().get_context_data(**kwargs)
+ content_type = self.content_dict.get("content_type")
+ content = self.content_dict.get("content")
+
+ if content_type == "text/asciidoc":
+ content_type = "text/html"
+
+ context.update({"content": content, "content_type": content_type})
+
+ logger.info(
+ "get_content_from_s3_view_success", key=self.kwargs.get("content_path")
+ )
+
+ return context
 
  def get_from_cache(self, static_content_cache, cache_key):
  cached_result = static_content_cache.get(cache_key)
@@ -232,48 +222,54 @@ def get_from_cache(self, static_content_cache, cache_key):
  def get_from_database(self, cache_key):
  try:
  content_obj = RenderedContent.objects.get(cache_key=cache_key)
- # todo: fire refresh task here
  return {
  "content": content_obj.content_html,
  "content_type": content_obj.content_type,
- "last_updated_at": content_obj.last_updated_at,
  }
  except RenderedContent.DoesNotExist:
  return None
 
- def get_from_s3(self, content_path, cache_key):
+ def get_from_s3(self, content_path):
  result = get_content_from_s3(key=content_path)
  if result and result.get("content"):
- return self.update_or_create_content(result, cache_key)
- return
+ content = result.get("content")
+ content_type = result.get("content_type")
+ if content_type == "text/asciidoc":
+ result["content"] = self.convert_adoc_to_html(content)
+ return result
 
- def update_or_create_content(self, result, cache_key):
- content = result.get("content")
+ def get_template_names(self):
+ """Returns the template name."""
+ content_type = self.content_dict.get("content_type")
+ if content_type == "text/asciidoc":
+ return [self.template_name]
+ return []
+
+ def render_to_response(self, context, **response_kwargs):
+ """Return the HTML response with a template, or just the content directly."""
+ if self.get_template_names():
+ return super().render_to_response(context, **response_kwargs)
+ else:
+ return HttpResponse(
+ context["content"], content_type=context["content_type"]
+ )
+
+ def save_to_database(self, cache_key, result):
+ """Saves the rendered asciidoc content to the database via celery."""
  content_type = result.get("content_type")
  last_updated_at_raw = result.get("last_updated_at")
 
  if content_type == "text/asciidoc":
- content = self.convert_adoc_to_html(content)
+ last_updated_at_raw = result.get("last_updated_at")
  last_updated_at = (
  parse(last_updated_at_raw) if last_updated_at_raw else None
  )
-
- defaults = {"content_html": content, "content_type": content_type}
- if last_updated_at:
- defaults["last_updated_at"] = last_updated_at
- content_obj, created = RenderedContent.objects.update_or_create(
- cache_key=cache_key, defaults=defaults
- )
- logger.info(
- "get_content_from_s3_view_saved_to_db",
- cache_key=cache_key,
- content_type=content_type,
- status_code=200,
- obj_id=content_obj.id,
- created=created,
+ save_rendered_content.delay(
+ cache_key,
+ content_type,
+ result["content"],
+ last_updated_at=last_updated_at,
  )
- result["content"] = content
- return result
 
  def convert_adoc_to_html(self, content):
  """Renders asciidoc content to HTML."""