diff --git a/docker/.env.example b/docker/.env.example index dc137c0..7641da4 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -7,3 +7,6 @@ STATIC_URL=/miller-assets/ MILLER_SCHEMA_ROOT=/contents/schema DEBUG=False LANGUAGES=en|American English|en_US|english,fr|French|fr_FR|french,de|German|de_DE|german +MILLER_CONTENTS_ENABLE_GIT=True +MILLER_CONTENTS_GIT_USERNAME=your-username-for-git +MILLER_CONTENTS_GIT_EMAIL=your-email-for-git diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index b8fa6a4..c88010e 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -34,6 +34,7 @@ services: MILLER_DATABASE_HOST: postgresdb MILLER_DATABASE_PORT: 5432 MILLER_SCHEMA_ROOT: ${MILLER_SCHEMA_ROOT} + MILLER_CONTENTS_ENABLE_GIT: ${MILLER_CONTENTS_ENABLE_GIT} MILLER_GIT_TAG: ${GIT_TAG} MILLER_GIT_BRANCH: ${GIT_BRANCH} MILLER_GIT_REVISION: ${GIT_REVISION} @@ -64,6 +65,7 @@ services: MILLER_DATABASE_HOST: postgresdb MILLER_DATABASE_PORT: 5432 MILLER_SCHEMA_ROOT: ${MILLER_SCHEMA_ROOT} + MILLER_CONTENTS_ENABLE_GIT: ${MILLER_CONTENTS_ENABLE_GIT} MILLER_GIT_TAG: ${GIT_TAG} MILLER_GIT_BRANCH: ${GIT_BRANCH} MILLER_GIT_REVISION: ${GIT_REVISION} diff --git a/miller/management/commands/document_commit.py b/miller/management/commands/document_commit.py new file mode 100644 index 0000000..2138a0c --- /dev/null +++ b/miller/management/commands/document_commit.py @@ -0,0 +1,46 @@ +from django.conf import settings +from django.core.management.base import BaseCommand +from miller.models import Document +from miller.tasks import commit_document + +class Command(BaseCommand): + """ + usage: + ENV=development pipenv run ./manage.py document_commit + or if in docker: + docker exec -it docker_miller_1 \ + python manage.py document_commit \ + , ... [--immediate] [--verbose] + """ + help = 'save YAML version of documents and commit them (if a git repo is enaled)' + + def add_arguments(self, parser): + parser.add_argument('document_pks', nargs='+', type=int) + parser.add_argument( + '--immediate', + action='store_true', + help='avoid delay tasks using celery', + ) + parser.add_argument( + '--verbose', + action='store_true', + help='use verbose logging', + ) + + def handle( + self, document_pks, immediate=False, verbose=False, *args, **options + ): + if not settings.MILLER_CONTENTS_ENABLE_GIT: + self.stderr.write('MILLER_CONTENTS_ENABLE_GIT not enabled!') + # repo = get_repo() + self.stdout.write(f'document_commit for: {document_pks}') + docs = Document.objects.filter(pk__in=document_pks) + # loop through documents + for doc in docs: + try: + if immediate: + doc.commit() + else: + commit_document.delay(document_pk=doc.pk) + except Exception as e: + self.stderr.write(e) diff --git a/miller/management/commands/story_commit.py b/miller/management/commands/story_commit.py new file mode 100644 index 0000000..385fd9d --- /dev/null +++ b/miller/management/commands/story_commit.py @@ -0,0 +1,46 @@ +from django.conf import settings +from django.core.management.base import BaseCommand +from miller.models import Story +from miller.tasks import commit_story + +class Command(BaseCommand): + """ + usage: + ENV=development pipenv run ./manage.py story_commit + or if in docker: + docker exec -it docker_miller_1 \ + python manage.py story_commit \ + , ... [--immediate] [--verbose] + """ + help = 'save YAML version of documents and commit them (if a git repo is enaled)' + + def add_arguments(self, parser): + parser.add_argument('story_pks', nargs='+', type=int) + parser.add_argument( + '--immediate', + action='store_true', + help='avoid delay tasks using celery', + ) + parser.add_argument( + '--verbose', + action='store_true', + help='use verbose logging', + ) + + def handle( + self, story_pks, immediate=False, verbose=False, *args, **options + ): + if not settings.MILLER_CONTENTS_ENABLE_GIT: + self.stderr.write('MILLER_CONTENTS_ENABLE_GIT not enabled!') + # repo = get_repo() + self.stdout.write(f'story_commit for: {story_pks}') + docs = Story.objects.filter(pk__in=story_pks) + # loop through documents + for doc in docs: + try: + if immediate: + doc.commit() + else: + commit_story.delay(story_pk=doc.pk) + except Exception as e: + self.stderr.write(e) diff --git a/miller/models/document.py b/miller/models/document.py index 643d17d..d8d0355 100644 --- a/miller/models/document.py +++ b/miller/models/document.py @@ -9,6 +9,7 @@ from ..fields import UTF8JSONField from ..snapshots import create_snapshot, create_different_sizes_from_snapshot from ..utils.models import get_search_vector_query, create_short_url +from ..utils.git import commit_instance from ..utils.media import get_video_subtitles @@ -271,4 +272,8 @@ def update_search_vector(self, verbose=False): for value, w, c in contents ] + [self.pk]) - \ No newline at end of file + def commit(self): + """ + if settings.MILLER_CONTENTS_ENABLE_GIT, write to disk + """ + commit_instance(instance=self) diff --git a/miller/models/story.py b/miller/models/story.py index ae5797f..0055d32 100644 --- a/miller/models/story.py +++ b/miller/models/story.py @@ -9,6 +9,7 @@ from . import Tag from ..utils import get_all_values_from_dict_by_key from ..utils.models import get_user_path, create_short_url, get_unique_slug +from ..utils.git import commit_instance from ..fields import UTF8JSONField @@ -93,7 +94,7 @@ class Story(models.Model): # add huge search field search_vector = SearchVectorField(null=True, blank=True) - + # enable full text search using postgres vectors stored in search_vector allow_fulltext_search = True @@ -152,3 +153,9 @@ def save_captions_from_contents(self, key='pk', parser='json'): # save captions saved = ThroughModel.objects.bulk_create([ThroughModel(document=d, story=self) for d in docs]) return saved, missing, expecting + + def commit(self): + """ + if settings.MILLER_CONTENTS_ENABLE_GIT, write to disk + """ + commit_instance(instance=self) diff --git a/miller/settings.py b/miller/settings.py index 5d9ff38..c2210d5 100644 --- a/miller/settings.py +++ b/miller/settings.py @@ -226,6 +226,8 @@ MILLER_CONTENTS_ROOT = get_env_variable('CONTENTS_ROOT', '/contents') MILLER_CONTENTS_ENABLE_GIT = get_env_variable( 'MILLER_CONTENTS_ENABLE_GIT', 'True') == 'True' +MILLER_CONTENTS_GIT_USERNAME = get_env_variable('MILLER_CONTENTS_GIT_USERNAME', 'miller') +MILLER_CONTENTS_GIT_EMAIL = get_env_variable('MILLER_CONTENTS_GIT_EMAIL', 'donotreply@miller') # snapshots and thumbnail sizes # default: max size, both heght and width must be 1200 px MILLER_SIZES_SNAPSHOT = [ diff --git a/miller/tasks.py b/miller/tasks.py index a28da7b..aabcae0 100644 --- a/miller/tasks.py +++ b/miller/tasks.py @@ -38,6 +38,32 @@ def update_document_search_vectors(self, document_pk, verbose=False): ) +@app.task( + bind=True, autoretry_for=(Exception,), exponential_backoff=2, + retry_kwargs={'max_retries': 5}, retry_jitter=True +) +def commit_document(self, document_pk, verbose=False): + logger.info(f'commit_document document(pk={document_pk})') + doc = Document.objects.get(pk=document_pk) + doc.commit() + logger.info( + f'commit_document document(pk={document_pk}) success.' + ) + + +@app.task( + bind=True, autoretry_for=(Exception,), exponential_backoff=2, + retry_kwargs={'max_retries': 5}, retry_jitter=True +) +def commit_story(self, story_pk, verbose=False): + logger.info(f'commit_story document(pk={story_pk})') + story = Story.objects.get(pk=story_pk) + story.commit() + logger.info( + f'commit_story document(pk={story_pk}) success.' + ) + + @app.task( bind=True, autoretry_for=(Exception,), exponential_backoff=2, retry_kwargs={'max_retries': 5}, retry_jitter=True @@ -62,4 +88,3 @@ def update_document_data_by_type(self, document_pk): def document_post_save_handler(sender, instance, **kwargs): logger.info(f'received @post_save document_pk: {instance.pk}') update_document_search_vectors.delay(document_pk=instance.pk) - diff --git a/miller/utils/git.py b/miller/utils/git.py new file mode 100644 index 0000000..3192e8d --- /dev/null +++ b/miller/utils/git.py @@ -0,0 +1,61 @@ +import os +import logging +from git import Repo, Actor +from django.conf import settings +from django.core import serializers + +logger = logging.getLogger(__name__) + +def get_repo(): + if not settings.MILLER_CONTENTS_ENABLE_GIT: + raise NameError('You shoud enable MILLER_CONTENTS_ENABLE_GIT in the settings file') + if not settings.MILLER_CONTENTS_ROOT: + raise NameError('You shoud set MILLER_CONTENTS_ROOT to an absolute path in the settings file') + if not os.path.exists(settings.MILLER_CONTENTS_ROOT): + raise OsError(f'path for MILLER_CONTENTS_ROOT does not exist or it is not reachable: {settings.MILLER_CONTENTS_ROOT}') # noqa: F821 + repo = Repo.init(settings.MILLER_CONTENTS_ROOT) + return repo + +def get_or_create_path_in_contents_root(folder_path): + prefixed_path = os.path.join( + settings.MILLER_CONTENTS_ROOT, + folder_path + ) + if not os.path.exists(prefixed_path): + os.makedirs(prefixed_path) + with open(os.path.join(prefixed_path, '.gitkeep'), 'w'): + pass + return prefixed_path + +def commit_filepath(filepath, username, email, message): + author = Actor(username, email) + committer = Actor(username, email) + repo = get_repo() + repo.index.add([filepath]) + commit_message = repo.index.commit(message=message, author=author, committer=committer) + short_sha = repo.git.rev_parse(commit_message, short=7) + return short_sha + + +def commit_instance( + instance, + verbose=False, + serializer='yaml' +): + logger.info( + f'commit_instance for instance pk:{instance.pk} model:{instance._meta.model.__name__}' + ) + folder_path = get_or_create_path_in_contents_root(folder_path=instance._meta.model.__name__) + contents = serializers.serialize(serializer, [instance]) + filepath = os.path.join(folder_path, f'{instance.pk}-{instance.short_url}.yml') + with open(filepath, 'w', encoding='utf-8') as f: + f.write(contents) + logger.info(f'commit_instance document: {instance.pk} {instance.short_url} written to {filepath}') + if settings.MILLER_CONTENTS_ENABLE_GIT: + short_sha = commit_filepath( + filepath=filepath, + username=instance.owner if instance.owner is not None else settings.MILLER_CONTENTS_GIT_USERNAME, + email=settings.MILLER_CONTENTS_GIT_EMAIL, + message=f'saving {instance.title}' + ) + logger.info(f'commit_instance document: {instance.pk} {instance.short_url} committed: {short_sha}')