From 9f9273eb561fbaec6fb7f0fb057f728f64844646 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Thu, 2 Jun 2022 11:08:54 -0400 Subject: [PATCH 01/12] Add explicit Jinja2 dependency --- requirements/main.in | 1 + requirements/main.txt | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/requirements/main.in b/requirements/main.in index faf0d52c..f706f67f 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -22,3 +22,4 @@ structlog celery[redis] pydantic cryptography +Jinja2 diff --git a/requirements/main.txt b/requirements/main.txt index 4928b4ad..07c4ef10 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -69,7 +69,9 @@ itsdangerous==2.0.1 # -r requirements/main.in # flask jinja2==3.1.2 - # via flask + # via + # -r requirements/main.in + # flask jmespath==1.0.0 # via # boto3 @@ -102,7 +104,7 @@ python-dateutil==2.8.2 # botocore pytz==2022.1 # via celery -redis==4.3.2 +redis==4.3.3 # via celery requests==2.27.1 # via -r requirements/main.in From b827181611dfe41c2190217c4bd1af0671c9705d Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Wed, 8 Jun 2022 10:33:09 -0400 Subject: [PATCH 02/12] Hide TestClient from Pytest --- keeper/testutils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/keeper/testutils.py b/keeper/testutils.py index 54e63d2a..cbf49c97 100644 --- a/keeper/testutils.py +++ b/keeper/testutils.py @@ -43,6 +43,8 @@ class TestClient: - `json`: the return data, parse as JSON into a Python `dict` object. """ + __test__ = False + def __init__(self, app: Flask, username: str, password: str = "") -> None: self.app = app self.auth = "Basic " + b64encode( From ad429d98d249475a36b8119b3900a86dbb797626 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Wed, 8 Jun 2022 10:27:49 -0400 Subject: [PATCH 03/12] Initial dashboard template domain This is the first sketch of a dashboard generation service+domain implementation in LTD Keeper itself. - BuiltinTemplateProvider coordinates built-in templates; when we implement S3-based templates we'll create an S3TemplateProvider with a similar API. - Context provides dataclass instances that provide information for the dashboard that are adapted from the DB models. --- keeper/dashboard/__init__.py | 1 + keeper/dashboard/context.py | 145 ++++++++++++++++++ keeper/dashboard/jinjafilters.py | 12 ++ keeper/dashboard/static/app.css | 52 +++++++ keeper/dashboard/template/base.jinja | 17 ++ .../dashboard/template/build_dashboard.jinja | 17 ++ .../template/edition_dashboard.jinja | 29 ++++ keeper/dashboard/templateproviders.py | 56 +++++++ keeper/services/createproduct.py | 1 + keeper/services/dashboard.py | 21 ++- tests/test_dashboard_service.py | 64 ++++++++ 11 files changed, 413 insertions(+), 2 deletions(-) create mode 100644 keeper/dashboard/__init__.py create mode 100644 keeper/dashboard/context.py create mode 100644 keeper/dashboard/jinjafilters.py create mode 100644 keeper/dashboard/static/app.css create mode 100644 keeper/dashboard/template/base.jinja create mode 100644 keeper/dashboard/template/build_dashboard.jinja create mode 100644 keeper/dashboard/template/edition_dashboard.jinja create mode 100644 keeper/dashboard/templateproviders.py create mode 100644 tests/test_dashboard_service.py diff --git a/keeper/dashboard/__init__.py b/keeper/dashboard/__init__.py new file mode 100644 index 00000000..1700e75f --- /dev/null +++ b/keeper/dashboard/__init__.py @@ -0,0 +1 @@ +"""Domain for edition dashboards.""" diff --git a/keeper/dashboard/context.py b/keeper/dashboard/context.py new file mode 100644 index 00000000..fc7bc7ed --- /dev/null +++ b/keeper/dashboard/context.py @@ -0,0 +1,145 @@ +"""Generate Jinja template rendering context from domain models.""" + +from __future__ import annotations + +from collections import UserList +from dataclasses import dataclass +from datetime import datetime +from typing import List, Optional, Sequence + +from keeper.models import Build, Edition, EditionKind, Product + + +@dataclass +class ProjectContext: + """Template context model for a project.""" + + title: str + """Project title.""" + + source_repo_url: str + """Url of the associated GitHub repository.""" + + url: str + """Root URL where this project is published.""" + + @classmethod + def from_product(cls, product: Product) -> ProjectContext: + return cls( + title=product.title, + source_repo_url=product.doc_repo, + url=product.published_url, + ) + + +@dataclass +class EditionContext: + """Template context model for an edition.""" + + title: str + """Human-readable label for this edition.""" + + url: str + """URL where this edition is published.""" + + date_updated: datetime + """Date when this edition was last updated.""" + + kind: EditionKind + """The edition's kind.""" + + slug: str + """The edition's slug.""" + + git_ref: Optional[str] + """The git ref that this edition tracks.""" + + @classmethod + def from_edition(cls, edition: Edition) -> EditionContext: + return cls( + title=edition.title, + url=edition.published_url, + date_updated=edition.date_rebuilt, + kind=edition.kind, + slug=edition.slug, + git_ref=edition.tracked_ref, + ) + + +class EditionContextList(UserList): + def __init__(self, contexts: Sequence[EditionContext]) -> None: + self.data: List[EditionContext] = list(contexts) + self.data.sort(key=lambda x: x.date_updated) + + @property + def main_edition(self) -> EditionContext: + for edition in self.data: + if edition.slug == "__main": + return edition + raise ValueError("No __main edition found") + + +@dataclass +class BuildContext: + """Template context model for a build.""" + + slug: str + """The URL slug for this build.""" + + url: str + """The URL for this build.""" + + git_ref: Optional[str] + """The git ref associated with this build (if appropriate.""" + + date: datetime + """Date when the build was uploaded.""" + + @classmethod + def from_build(cls, build: Build) -> BuildContext: + return cls( + slug=build.slug, + url=build.published_url, + git_ref=build.git_ref, + date=build.date_created, + ) + + +class BuildContextList(UserList): + def __init__(self, contexts: Sequence[BuildContext]) -> None: + self.data: List[BuildContext] = list(contexts) + self.data.sort(key=lambda x: x.date) + + +@dataclass +class Context: + """A class that creates Jinja template rendering context from + domain models. + """ + + project_context: ProjectContext + + edition_contexts: EditionContextList + + build_contexts: BuildContextList + + @classmethod + def create(cls, product: Product) -> Context: + project_context = ProjectContext.from_product(product) + + edition_contexts: EditionContextList = EditionContextList( + [ + EditionContext.from_edition(edition) + for edition in product.editions + ] + ) + + build_contexts: BuildContextList = BuildContextList( + [BuildContext.from_build(build) for build in product.builds] + ) + + return cls( + project_context=project_context, + edition_contexts=edition_contexts, + build_contexts=build_contexts, + ) diff --git a/keeper/dashboard/jinjafilters.py b/keeper/dashboard/jinjafilters.py new file mode 100644 index 00000000..298c76a2 --- /dev/null +++ b/keeper/dashboard/jinjafilters.py @@ -0,0 +1,12 @@ +"""Filters for Jinja2 templates.""" + +from __future__ import annotations + +__all__ = ["filter_simple_date"] + +from datetime import datetime + + +def filter_simple_date(value: datetime) -> str: + """Filter a `datetime.datetime` into a 'YYYY-MM-DD' string.""" + return value.strftime("%Y-%m-%d") diff --git a/keeper/dashboard/static/app.css b/keeper/dashboard/static/app.css new file mode 100644 index 00000000..c20a66e9 --- /dev/null +++ b/keeper/dashboard/static/app.css @@ -0,0 +1,52 @@ +// Resets +*, +*::before, +*::after { + box-sizing: border-box; +} + +* { + margin: 0; +} + +html, +body { + height: 100%; +} + +body { + line-height: 1.5; + -webkit-font-smoothing: antialiased; +} + +img, +picture, +video, +canvas, +svg { + display: block; + max-width: 100%; +} + +input, +button, +textarea, +select { + font: inherit; +} + +p, +h1, +h2, +h3, +h4, +h5, +h6 { + overflow-wrap: break-word; +} + +// System font stack +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, + Oxygen-Sans, Ubuntu, Cantarell, 'Helvetica Neue', sans-serif; +} diff --git a/keeper/dashboard/template/base.jinja b/keeper/dashboard/template/base.jinja new file mode 100644 index 00000000..9d5c9ef6 --- /dev/null +++ b/keeper/dashboard/template/base.jinja @@ -0,0 +1,17 @@ + + + + + + {% block page_title %}{% endblock page_title %} + + + + + + + {% block body %} + {% endblock body %} + + + diff --git a/keeper/dashboard/template/build_dashboard.jinja b/keeper/dashboard/template/build_dashboard.jinja new file mode 100644 index 00000000..3c1bece0 --- /dev/null +++ b/keeper/dashboard/template/build_dashboard.jinja @@ -0,0 +1,17 @@ +{% extends "base.jinja" %} + +{% block page_title %}{{ project.title }} builds{% endblock page_title %} +{% block page_description %}Find documentation builds.{% endblock page_description %} + +{% block body %} +
+
+

{{ project.title }} builds

+
+
+
+

Builds

+
+
+
+{% endblock body %} diff --git a/keeper/dashboard/template/edition_dashboard.jinja b/keeper/dashboard/template/edition_dashboard.jinja new file mode 100644 index 00000000..a116987e --- /dev/null +++ b/keeper/dashboard/template/edition_dashboard.jinja @@ -0,0 +1,29 @@ +{% extends "base.jinja" %} + +{% block page_title %}{{ project.title }} editions{% endblock page_title %} +{% block page_description %}Find documentation editions.{% endblock page_description %} + +{% block body %} +
+
+

{{ project.title }}

+
+ +
+
+

Current edition

+
+
+ {% set mainedition = editions.main_edition %} +

+
    +
  • Updated {{ mainedition.date_updated | simple_date }}
    • + {% if mainedition.git_ref %} +
  • Git ref {{ mainedition.git_ref}}
    • + {% endif %} +
+
+
+ +
+{% endblock body %} diff --git a/keeper/dashboard/templateproviders.py b/keeper/dashboard/templateproviders.py new file mode 100644 index 00000000..5a5ebb52 --- /dev/null +++ b/keeper/dashboard/templateproviders.py @@ -0,0 +1,56 @@ +"""Providers load templates from specific sources and provider a +Jinja2 rendering environment. +""" + +from __future__ import annotations + +from pathlib import Path + +import jinja2 + +from .context import BuildContextList, EditionContextList, ProjectContext +from .jinjafilters import filter_simple_date + + +class BuiltinTemplateProvider: + """A template provider for Keeper's built in dashboard templates.""" + + def __init__(self) -> None: + self.template_dir = Path(__file__).parent.joinpath("template") + self.static_dir = self.template_dir.joinpath("static") + + self.jinja_env = self._create_environment() + + def _create_environment(self) -> jinja2.Environment: + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(self.template_dir), + autoescape=jinja2.select_autoescape(["html"]), + ) + env.filters["simple_date"] = filter_simple_date + return env + + def render_edition_dashboard( + self, + *, + project_context: ProjectContext, + edition_contexts: EditionContextList, + ) -> str: + template = self.jinja_env.get_template("edition_dashboard.jinja") + return template.render( + project=project_context, + editions=edition_contexts, + asset_dir="../_dashboard-assets", + ) + + def render_build_dashboard( + self, + *, + project_context: ProjectContext, + build_contexts: BuildContextList, + ) -> str: + template = self.jinja_env.get_template("build_dashboard.jinja") + return template.render( + project=project_context, + builds=build_contexts, + asset_dir="../_dashboard-assets", + ) diff --git a/keeper/services/createproduct.py b/keeper/services/createproduct.py index 7dd25901..9de04701 100644 --- a/keeper/services/createproduct.py +++ b/keeper/services/createproduct.py @@ -57,6 +57,7 @@ def create_product( product.title = title # Compatibility with v1 table architecture. This can be removed once # these fields are dropped from the Product model + # print(f"create_product {org.root_domain}") product.root_domain = org.root_domain product.root_fastly_domain = org.fastly_domain product.bucket_name = org.bucket_name diff --git a/keeper/services/dashboard.py b/keeper/services/dashboard.py index 69c72fec..e8c76cad 100644 --- a/keeper/services/dashboard.py +++ b/keeper/services/dashboard.py @@ -1,9 +1,12 @@ -"""This service updates an edition's dashboard.""" +"""This service updates project's edition and build dashboards.""" from __future__ import annotations from typing import TYPE_CHECKING, Any +from keeper.dashboard.context import Context +from keeper.dashboard.templateproviders import BuiltinTemplateProvider + if TYPE_CHECKING: from keeper.models import Product @@ -12,5 +15,19 @@ def build_dashboard(product: Product, logger: Any) -> None: """Build a dashboard (run from a Celery task).""" - # TODO implement this service logger.debug("In build_dashboard service function.") + + context = Context.create(product) + template_provider = BuiltinTemplateProvider() + print( + template_provider.render_edition_dashboard( + project_context=context.project_context, + edition_contexts=context.edition_contexts, + ) + ) + print( + template_provider.render_build_dashboard( + project_context=context.project_context, + build_contexts=context.build_contexts, + ) + ) diff --git a/tests/test_dashboard_service.py b/tests/test_dashboard_service.py new file mode 100644 index 00000000..be31d4da --- /dev/null +++ b/tests/test_dashboard_service.py @@ -0,0 +1,64 @@ +"""Tests for keeper.dashboard and keeper.services.dashboard.""" + +from __future__ import annotations + +from unittest.mock import Mock + +from structlog import get_logger + +from keeper.models import OrganizationLayoutMode, db +from keeper.services.createbuild import create_build +from keeper.services.createorg import create_organization +from keeper.services.createproduct import create_product +from keeper.services.dashboard import build_dashboard +from keeper.testutils import MockTaskQueue, TestClient + + +def test_builtin_template(client: TestClient, mocker: Mock) -> None: + logger = get_logger("keeper") + + task_queue = mocker.patch( + "keeper.taskrunner.inspect_task_queue", return_value=None + ) + task_queue = MockTaskQueue(mocker) # noqa + + org = create_organization( + slug="test", + title="Test", + layout=OrganizationLayoutMode.path, + domain="example.org", + path_prefix="/", + bucket_name="example", + s3_public_read=False, + fastly_support=False, + aws_id=None, + aws_region=None, + aws_secret=None, + fastly_domain=None, + fastly_service_id=None, + fastly_api_key=None, + ) + db.session.add(org) + db.session.commit() + + # This print is somehow required; not exactly sure why. + print(f"test {org.root_domain}") + + product, default_edition = create_product( + org=org, + slug="myproject", + doc_repo="https://git.example.org/myproject", + title="My Project", + ) + print(product) + build, _ = create_build( + product=product, + git_ref="main", + ) + default_edition.build = build + db.session.add(build) + db.session.add(default_edition) + db.session.commit() + + build_dashboard(product, logger) + assert False From deea1502a9df4161871144d0e759281a5ed25975 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Sun, 12 Jun 2022 22:18:50 -0600 Subject: [PATCH 04/12] Include ltd-conveyor as a runtime dep We're using ltd-conveyor's s3 functions for uploading objects and directories. --- requirements/dev.txt | 10 +++++----- requirements/main.in | 1 + requirements/main.txt | 19 ++++++++++++++----- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index dd89118a..c1247318 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -43,13 +43,13 @@ markupsafe==2.1.1 # jinja2 mock==4.0.3 # via -r requirements/dev.in -mypy==0.960 +mypy==0.961 # via # -r requirements/dev.in # sqlalchemy-stubs mypy-extensions==0.4.3 # via mypy -numpydoc==1.3.1 +numpydoc==1.4.0 # via -r requirements/dev.in packaging==21.3 # via @@ -76,7 +76,7 @@ pytz==2022.1 # via # -c requirements/main.txt # babel -requests==2.27.1 +requests==2.28.0 # via # -c requirements/main.txt # responses @@ -89,7 +89,7 @@ six==1.16.0 # sphinxcontrib-httpdomain snowballstemmer==2.2.0 # via sphinx -sphinx==5.0.0 +sphinx==5.0.1 # via # -r requirements/dev.in # numpydoc @@ -118,7 +118,7 @@ tomli==2.0.1 # coverage # mypy # pytest -types-mock==4.0.14 +types-mock==4.0.15 # via -r requirements/dev.in types-python-dateutil==2.8.17 # via -r requirements/dev.in diff --git a/requirements/main.in b/requirements/main.in index f706f67f..61002fdc 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -23,3 +23,4 @@ celery[redis] pydantic cryptography Jinja2 +ltd-conveyor diff --git a/requirements/main.txt b/requirements/main.txt index 07c4ef10..ff782db4 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -12,9 +12,11 @@ async-timeout==4.0.2 # via redis billiard==3.6.4.0 # via celery -boto3==1.24.1 - # via -r requirements/main.in -botocore==1.27.1 +boto3==1.24.7 + # via + # -r requirements/main.in + # ltd-conveyor +botocore==1.27.7 # via # boto3 # s3transfer @@ -33,6 +35,7 @@ click==8.1.3 # click-plugins # click-repl # flask + # ltd-conveyor click-didyoumean==0.3.0 # via celery click-plugins==1.1.1 @@ -78,6 +81,8 @@ jmespath==1.0.0 # botocore kombu==5.2.4 # via celery +ltd-conveyor==0.8.1 + # via -r requirements/main.in mako==1.2.0 # via alembic markupsafe==2.1.1 @@ -106,8 +111,10 @@ pytz==2022.1 # via celery redis==4.3.3 # via celery -requests==2.27.1 - # via -r requirements/main.in +requests==2.28.0 + # via + # -r requirements/main.in + # ltd-conveyor s3transfer==0.6.0 # via boto3 six==1.16.0 @@ -123,6 +130,8 @@ structlog==21.5.0 # via -r requirements/main.in typing-extensions==4.2.0 # via pydantic +uritemplate==4.1.1 + # via ltd-conveyor urllib3==1.26.9 # via # botocore From 4e3a55b4178dc4ef6f2d0863ed301d5f8b80f0b0 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Tue, 14 Jun 2022 22:48:23 -0600 Subject: [PATCH 05/12] Implement dashboard service and uploads --- keeper/services/dashboard.py | 133 +++++++++++++++++++++++++++++--- tests/test_dashboard_service.py | 1 - 2 files changed, 124 insertions(+), 10 deletions(-) diff --git a/keeper/services/dashboard.py b/keeper/services/dashboard.py index e8c76cad..d82d8d64 100644 --- a/keeper/services/dashboard.py +++ b/keeper/services/dashboard.py @@ -2,8 +2,17 @@ from __future__ import annotations +from pathlib import PurePosixPath from typing import TYPE_CHECKING, Any +import boto3 +from ltdconveyor.s3 import ( + create_dir_redirect_object, + upload_dir, + upload_object, +) + +from keeper import fastly, s3 from keeper.dashboard.context import Context from keeper.dashboard.templateproviders import BuiltinTemplateProvider @@ -17,17 +26,123 @@ def build_dashboard(product: Product, logger: Any) -> None: """Build a dashboard (run from a Celery task).""" logger.debug("In build_dashboard service function.") + organization = product.organization + + aws_id = organization.aws_id + aws_secret = organization.get_aws_secret_key() + aws_region = organization.get_aws_region() + use_public_read_acl = organization.get_bucket_public_read() + + fastly_service_id = organization.fastly_service_id + fastly_key = organization.get_fastly_api_key() + + # Render dashboards using the built-in dashboard template provider; + # eventually we'll add the ability to get templates from a configured + # S3 bucket location. context = Context.create(product) template_provider = BuiltinTemplateProvider() - print( - template_provider.render_edition_dashboard( - project_context=context.project_context, - edition_contexts=context.edition_contexts, - ) + edition_html = template_provider.render_edition_dashboard( + project_context=context.project_context, + edition_contexts=context.edition_contexts, + ) + build_html = template_provider.render_build_dashboard( + project_context=context.project_context, + build_contexts=context.build_contexts, ) - print( - template_provider.render_build_dashboard( - project_context=context.project_context, - build_contexts=context.build_contexts, + + if aws_id is not None and aws_secret is not None: + s3_service = s3.open_s3_resource( + key_id=aws_id, + access_key=aws_secret.get_secret_value(), + aws_region=aws_region, + ) + + upload_dir( + product.organization.bucket_name, + f"{product.slug}/_dashboard-assets", + str(template_provider.static_dir), + upload_dir_redirect_objects=True, + surrogate_key=product.surrogate_key, + surrogate_control="max-age=31536000", + cache_control="no-cache", + acl="public-read" if use_public_read_acl else None, + aws_access_key_id=aws_id, + aws_secret_access_key=aws_secret, + ) + + upload_dashboard_html( + html=edition_html, + key="v/index.html", + product=product, + s3_service=s3_service, + ) + upload_dashboard_html( + html=build_html, + key="builds/index.html", + product=product, + s3_service=s3_service, ) + + else: + logger.warning( + "Skipping dashboard uploads because AWS credentials are not set" + ) + + if ( + organization.fastly_support + and fastly_service_id is not None + and fastly_key is not None + ): + logger.info("Starting Fastly purge_key") + fastly_service = fastly.FastlyService( + fastly_service_id, fastly_key.get_secret_value() + ) + fastly_service.purge_key(product.surrogate_key) + logger.info("Finished Fastly purge_key") + else: + logger.warning("Skipping Fastly purge because credentials are not set") + + +def upload_dashboard_html( + *, + html: str, + key: str, + product: Product, + s3_service: boto3.resources.base.ServiceResource, +) -> None: + bucket = s3_service.Bucket(product.organization.bucket_name) + + if not key.startswith("/"): + key = f"/{key}" + + object_path = f"{product.slug}{key}" + + # Have Fastly cache the dashboard for a year (or until purged) + metadata = { + "surrogate-key": product.surrogate_key, + "surrogate-control": "max-age=31536000", + } + acl = "public-read" + # Have the *browser* never cache the dashboard + cache_control = "no-cache" + + # Upload HTML object + upload_object( + object_path, + bucket, + content=html, + metadata=metadata, + acl=acl, + cache_control=cache_control, + content_type="text/html", + ) + + # Upload directory redirect object + bucket_dir_path = PurePosixPath(object_path).parent + create_dir_redirect_object( + bucket_dir_path, + bucket, + metadata=metadata, + acl=acl, + cache_control=cache_control, ) diff --git a/tests/test_dashboard_service.py b/tests/test_dashboard_service.py index be31d4da..073f9d63 100644 --- a/tests/test_dashboard_service.py +++ b/tests/test_dashboard_service.py @@ -61,4 +61,3 @@ def test_builtin_template(client: TestClient, mocker: Mock) -> None: db.session.commit() build_dashboard(product, logger) - assert False From be0b77419082d5e9c98382753d8dd6c1ac8b998d Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Wed, 15 Jun 2022 17:05:26 -0600 Subject: [PATCH 06/12] Add endpoint to regenerate dashboards --- keeper/v2api/projects.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/keeper/v2api/projects.py b/keeper/v2api/projects.py index 74053131..2088dee5 100644 --- a/keeper/v2api/projects.py +++ b/keeper/v2api/projects.py @@ -6,11 +6,13 @@ from flask import request from flask_accept import accept_fallback +from structlog import get_logger from keeper.auth import token_auth from keeper.logutils import log_route from keeper.models import Organization, Product, db from keeper.services.createproduct import create_product +from keeper.services.requestdashboardbuild import request_dashboard_build from keeper.services.updateproduct import update_product from keeper.taskrunner import launch_tasks from keeper.v2api import v2api @@ -25,6 +27,8 @@ __all__ = ["get_projects", "get_project", "create_project", "update_project"] +logger = get_logger(__name__) + @v2api.route("/orgs//projects", methods=["GET"]) @accept_fallback @@ -123,3 +127,30 @@ def update_project(org: str, slug: str) -> Tuple[str, int, Dict[str, str]]: response = ProjectResponse.from_product(product, task=task) project_url = url_for_project(product) return response.json(), 200, {"Location": project_url} + + +@v2api.route("/orgs//projects//dashboard", methods=["POST"]) +@accept_fallback +@log_route() +@token_auth.login_required +def refresh_dashboard(org: str, slug: str) -> Tuple[str, int, Dict[str, str]]: + product = ( + Product.query.join( + Organization, Organization.id == Product.organization_id + ) + .filter(Organization.slug == org) + .filter(Product.slug == slug) + .first_or_404() + ) + + try: + request_dashboard_build(product) + except Exception as e: + logger.error(f"Error building dashboard: {str(e)}") + db.session.rollback() + raise + + task = launch_tasks() + response = ProjectResponse.from_product(product, task=task) + project_url = url_for_project(product) + return response.json(), 200, {"Location": project_url} From 28baa2cdca01ce16ba8093b2e46a15fed87a3075 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Wed, 15 Jun 2022 22:58:49 -0600 Subject: [PATCH 07/12] Move upload object and upload_dir_redirect to keeper This fixes some upload issues we found for individual objects into the bucket. --- keeper/s3.py | 122 ++++++++++++++++++++++++++++++++--- keeper/services/dashboard.py | 39 ++++++----- 2 files changed, 136 insertions(+), 25 deletions(-) diff --git a/keeper/s3.py b/keeper/s3.py index 2eea2bd6..1787f7c3 100644 --- a/keeper/s3.py +++ b/keeper/s3.py @@ -17,6 +17,7 @@ List, Optional, Sequence, + Union, ) import boto3 @@ -250,16 +251,119 @@ def copy_directory( if create_directory_redirect_object: dest_dirname = dest_path.rstrip("/") - obj = bucket.Object(dest_dirname) metadata = {"dir-redirect": "true"} - put_kwargs = { - "Body": "", - "Metadata": metadata, - "CacheControl": cache_control, - } - if use_public_read_acl: - put_kwargs["ACL"] = "public-read" - obj.put(**put_kwargs) + upload_dir_redirect_object( + bucket_dir_path=dest_dirname, + bucket=bucket, + metadata=metadata, + acl="public-read" if use_public_read_acl else None, + cache_control=cache_control, + ) + + +def upload_object( + *, + bucket_path: str, + bucket: Any, + content: Union[str, bytes] = "", + metadata: Optional[Dict[str, str]] = None, + acl: Optional[str] = None, + cache_control: Optional[str] = None, + content_type: Optional[str] = None, +) -> None: + """Upload an arbitrary object to an S3 bucket. + + Parameters + ---------- + bucket_path : `str` + Destination path (also known as the key name) of the file in the + S3 bucket. + content : `str` or `bytes`, optional + Object content. + bucket : boto3 Bucket instance + S3 bucket. + metadata : `dict`, optional + Header metadata values. These keys will appear in headers as + ``x-amz-meta-*``. + acl : `str`, optional + A pre-canned access control list. See + https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl + Default is `None`, meaning that no ACL is applied to the object. + cache_control : `str`, optional + The cache-control header value. For example, ``'max-age=31536000'``. + content_type : `str`, optional + The object's content type (such as ``text/html``). If left unset, + no MIME type is passed to boto3 (which defaults to + ``binary/octet-stream``). + """ + print(f"Upload file bucket path: {bucket_path}") + obj = bucket.Object(bucket_path) + + # Object.put seems to be sensitive to None-type kwargs, so we filter first + args: Dict[str, Any] = {} + if metadata is not None and len(metadata) > 0: # avoid empty Metadata + args["Metadata"] = metadata + if acl is not None: + args["ACL"] = acl + if cache_control is not None: + args["CacheControl"] = cache_control + if content_type is not None: + args["ContentType"] = content_type + + print("Put args") + print(args) + obj.put(Body=content, **args) + + +def upload_dir_redirect_object( + *, + bucket_dir_path: str, + bucket: Any, + metadata: Optional[Dict[str, str]] = None, + acl: Optional[str] = None, + cache_control: Optional[str] = None, +) -> None: + """Create an S3 object representing a directory that's designed to + redirect a browser (via Fastly) to the ``index.html`` contained inside + that directory. + + Parameters + ---------- + bucket_dir_path : `str` + Full name of the object in the S3, which is equivalent to a POSIX + directory path, like ``dir1/dir2``. + bucket : boto3 Bucket instance + S3 bucket. + metadata : `dict`, optional + Header metadata values. These keys will appear in headers as + ``x-amz-meta-*``. + acl : `str`, optional + A pre-canned access control list. See + https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl + Default is None, meaning that no ACL is applied to the object. + cache_control : `str`, optional + The cache-control header value. For example, ``'max-age=31536000'``. + """ + # Just the name of the 'directory' itself + bucket_dir_path = bucket_dir_path.rstrip("/") + + # create a copy of metadata + if metadata is not None: + metadata = dict(metadata) + else: + metadata = {} + + # header used by LTD's Fastly Varnish config to create a 301 redirect + metadata["dir-redirect"] = "true" + + upload_object( + bucket_path=bucket_dir_path, + bucket=bucket, + content="", + metadata=metadata, + acl=acl, + cache_control=cache_control, + ) def presign_post_url_for_prefix( diff --git a/keeper/services/dashboard.py b/keeper/services/dashboard.py index d82d8d64..a96f3a50 100644 --- a/keeper/services/dashboard.py +++ b/keeper/services/dashboard.py @@ -6,21 +6,25 @@ from typing import TYPE_CHECKING, Any import boto3 -from ltdconveyor.s3 import ( - create_dir_redirect_object, - upload_dir, - upload_object, -) +from ltdconveyor.s3 import upload_dir +from structlog import get_logger -from keeper import fastly, s3 +from keeper import fastly from keeper.dashboard.context import Context from keeper.dashboard.templateproviders import BuiltinTemplateProvider +from keeper.s3 import ( + open_s3_resource, + upload_dir_redirect_object, + upload_object, +) if TYPE_CHECKING: from keeper.models import Product __all__ = ["build_dashboard"] +logger = get_logger(__name__) + def build_dashboard(product: Product, logger: Any) -> None: """Build a dashboard (run from a Celery task).""" @@ -51,7 +55,7 @@ def build_dashboard(product: Product, logger: Any) -> None: ) if aws_id is not None and aws_secret is not None: - s3_service = s3.open_s3_resource( + s3_service = open_s3_resource( key_id=aws_id, access_key=aws_secret.get_secret_value(), aws_region=aws_region, @@ -67,7 +71,7 @@ def build_dashboard(product: Product, logger: Any) -> None: cache_control="no-cache", acl="public-read" if use_public_read_acl else None, aws_access_key_id=aws_id, - aws_secret_access_key=aws_secret, + aws_secret_access_key=aws_secret.get_secret_value(), ) upload_dashboard_html( @@ -110,26 +114,29 @@ def upload_dashboard_html( product: Product, s3_service: boto3.resources.base.ServiceResource, ) -> None: - bucket = s3_service.Bucket(product.organization.bucket_name) + organization = product.organization + use_public_read_acl = organization.get_bucket_public_read() + bucket = s3_service.Bucket(organization.bucket_name) if not key.startswith("/"): key = f"/{key}" object_path = f"{product.slug}{key}" + logger.debug("Object path for html upload", path=object_path) # Have Fastly cache the dashboard for a year (or until purged) metadata = { "surrogate-key": product.surrogate_key, "surrogate-control": "max-age=31536000", } - acl = "public-read" + acl = "public-read" if use_public_read_acl else None # Have the *browser* never cache the dashboard cache_control = "no-cache" # Upload HTML object upload_object( - object_path, - bucket, + bucket_path=object_path, + bucket=bucket, content=html, metadata=metadata, acl=acl, @@ -138,10 +145,10 @@ def upload_dashboard_html( ) # Upload directory redirect object - bucket_dir_path = PurePosixPath(object_path).parent - create_dir_redirect_object( - bucket_dir_path, - bucket, + bucket_dir_path = str(PurePosixPath(object_path).parent) + upload_dir_redirect_object( + bucket_dir_path=bucket_dir_path, + bucket=bucket, metadata=metadata, acl=acl, cache_control=cache_control, From 56a842fae1f88ee80d7ac88071819d82ee470e9f Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Thu, 16 Jun 2022 12:22:28 -0600 Subject: [PATCH 08/12] Fix path for static_dir --- keeper/dashboard/templateproviders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keeper/dashboard/templateproviders.py b/keeper/dashboard/templateproviders.py index 5a5ebb52..cae34fb9 100644 --- a/keeper/dashboard/templateproviders.py +++ b/keeper/dashboard/templateproviders.py @@ -17,7 +17,7 @@ class BuiltinTemplateProvider: def __init__(self) -> None: self.template_dir = Path(__file__).parent.joinpath("template") - self.static_dir = self.template_dir.joinpath("static") + self.static_dir = Path(__file__).parent.joinpath("static") self.jinja_env = self._create_environment() From 7cafa97ba9a78c2ba0f4d1e126a9eee0ee2464aa Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Tue, 21 Jun 2022 13:09:12 -0400 Subject: [PATCH 09/12] Render built-in dashboard locally during tests You can now see the dashboard built with mock data in dashboard_dev/ by running the test_dashboard_template.py module. This enables you to quickly refine the built-in template. --- .gitignore | 3 ++ keeper/dashboard/templateproviders.py | 36 +++++++++++++++++ tests/test_dashboard_template.py | 58 +++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 tests/test_dashboard_template.py diff --git a/.gitignore b/.gitignore index 12066f9a..6e75a0fc 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,9 @@ pgdb/ integration_tests/ltd_keeper_doc_examples.txt +# Dashboard development +dashboard_dev + # Kubernetes deployment kubernetes/cloudsql-secrets.yaml kubernetes/keeper-secrets.yaml diff --git a/keeper/dashboard/templateproviders.py b/keeper/dashboard/templateproviders.py index cae34fb9..b7400ee7 100644 --- a/keeper/dashboard/templateproviders.py +++ b/keeper/dashboard/templateproviders.py @@ -4,6 +4,7 @@ from __future__ import annotations +import shutil from pathlib import Path import jinja2 @@ -54,3 +55,38 @@ def render_build_dashboard( builds=build_contexts, asset_dir="../_dashboard-assets", ) + + def render_locally( + self, + *, + directory: Path, + project_context: ProjectContext, + edition_contexts: EditionContextList, + build_contexts: BuildContextList, + clobber: bool = True, + ) -> None: + """Render the dashboard into a local directory for testing.""" + if directory.exists(): + shutil.rmtree(directory) + directory.mkdir() + assets_dir = directory.joinpath("_dashboard-assets") + # assets_dir.mkdir() + v_dir = directory.joinpath("v") + v_dir.mkdir() + builds_dir = directory.joinpath("builds") + builds_dir.mkdir() + + shutil.copytree(self.static_dir, assets_dir) + + edition_dashboard = self.render_edition_dashboard( + project_context=project_context, + edition_contexts=edition_contexts, + ) + v_html_path = v_dir.joinpath("index.html") + v_html_path.write_text(edition_dashboard) + + build_dashboard = self.render_build_dashboard( + project_context=project_context, build_contexts=build_contexts + ) + build_html_path = builds_dir.joinpath("index.html") + build_html_path.write_text(build_dashboard) diff --git a/tests/test_dashboard_template.py b/tests/test_dashboard_template.py new file mode 100644 index 00000000..1cabc6cd --- /dev/null +++ b/tests/test_dashboard_template.py @@ -0,0 +1,58 @@ +"""Test the built-in dashboard template by rendering to a local directory.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path +from typing import List + +from keeper.dashboard.context import ( + BuildContext, + BuildContextList, + EditionContext, + EditionContextList, + ProjectContext, +) +from keeper.dashboard.templateproviders import BuiltinTemplateProvider +from keeper.models import EditionKind + + +def test_templates() -> None: + output_dir = Path(__file__).parent.parent.joinpath("dashboard_dev") + + # Create mock data + project = ProjectContext( + title="LTD Test Project", + source_repo_url="https://github.com/lsst-sqre/ltd-keeper", + url="https://example.com/ltd-test/", + ) + editions: List[EditionContext] = [] + builds: List[BuildContext] = [] + + editions.append( + EditionContext( + title="Current", + url="https://example.com/ltd-test/", + date_updated=datetime(2022, 6, 21, tzinfo=timezone.utc), + kind=EditionKind.main, + slug="__main", + git_ref="main", + ) + ) + + builds.append( + BuildContext( + slug="1", + url="https://example.com/ltd-test/builds/1", + git_ref="main", + date=datetime(2022, 6, 21, tzinfo=timezone.utc), + ) + ) + + template = BuiltinTemplateProvider() + template.render_locally( + directory=output_dir, + project_context=project, + edition_contexts=EditionContextList(editions), + build_contexts=BuildContextList(builds), + ) From 11229f68c355faff339ca9d2837d9310fe6d5f10 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Fri, 24 Jun 2022 14:50:15 -0400 Subject: [PATCH 10/12] Develop MVP design for the version dashboard --- keeper/dashboard/context.py | 47 +++++++++++- keeper/dashboard/static/app.css | 48 +++++++++++- .../template/edition_dashboard.jinja | 73 +++++++++++++++---- tests/test_dashboard_template.py | 48 +++++++++++- 4 files changed, 198 insertions(+), 18 deletions(-) diff --git a/keeper/dashboard/context.py b/keeper/dashboard/context.py index fc7bc7ed..a233fbbd 100644 --- a/keeper/dashboard/context.py +++ b/keeper/dashboard/context.py @@ -54,8 +54,21 @@ class EditionContext: git_ref: Optional[str] """The git ref that this edition tracks.""" + github_url: Optional[str] + """URL to this git ref on GitHub.""" + @classmethod - def from_edition(cls, edition: Edition) -> EditionContext: + def from_edition( + cls, edition: Edition, product: Product + ) -> EditionContext: + if edition.tracked_ref and product.doc_repo: + repo_url = product.doc_repo.rstrip("/") + if repo_url[-4:] == ".git": + repo_url = repo_url[:-4] + github_url = f"{repo_url}/tree/{edition.tracked_ref}" + else: + github_url = None + return cls( title=edition.title, url=edition.published_url, @@ -63,6 +76,7 @@ def from_edition(cls, edition: Edition) -> EditionContext: kind=edition.kind, slug=edition.slug, git_ref=edition.tracked_ref, + github_url=github_url, ) @@ -73,11 +87,40 @@ def __init__(self, contexts: Sequence[EditionContext]) -> None: @property def main_edition(self) -> EditionContext: + """The main (current) edition.""" for edition in self.data: if edition.slug == "__main": return edition raise ValueError("No __main edition found") + @property + def has_releases(self) -> bool: + return len(self.releases) > 0 + + @property + def releases(self) -> List[EditionContext]: + """All edititions tagged as releases.""" + release_kinds = ( + EditionKind.release, + EditionKind.major, + EditionKind.minor, + ) + release_items = [e for e in self.data if e.kind in release_kinds] + sorted_items = sorted( + release_items, key=lambda x: x.slug, reverse=True + ) + return sorted_items + + @property + def has_drafts(self) -> bool: + return len(self.drafts) > 0 + + @property + def drafts(self) -> List[EditionContext]: + """All editions tagged as drafts.""" + draft_items = [e for e in self.data if e.kind == EditionKind.draft] + return sorted(draft_items, key=lambda x: x.date_updated, reverse=True) + @dataclass class BuildContext: @@ -129,7 +172,7 @@ def create(cls, product: Product) -> Context: edition_contexts: EditionContextList = EditionContextList( [ - EditionContext.from_edition(edition) + EditionContext.from_edition(edition=edition, product=product) for edition in product.editions ] ) diff --git a/keeper/dashboard/static/app.css b/keeper/dashboard/static/app.css index c20a66e9..d168ae95 100644 --- a/keeper/dashboard/static/app.css +++ b/keeper/dashboard/static/app.css @@ -1,4 +1,4 @@ -// Resets +/* Resets */ *, *::before, *::after { @@ -45,8 +45,52 @@ h6 { overflow-wrap: break-word; } -// System font stack +/* System font stack */ body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen-Sans, Ubuntu, Cantarell, 'Helvetica Neue', sans-serif; } + +main { + /* max-width: 16rem; */ + width: 100vw; + margin: 0 auto; + padding: 1rem; +} + +@media (min-width: 62rem) { + main { + width: 62rem; + } +} + +.main-edition-section { + margin-top: 1rem; +} + +.main-edition-section__url { + font-size: 1.2rem; + margin-bottom: 0.5rem; +} + +.version-section { + margin-top: 2rem; +} + +.version-section__listing { + list-style: none; + padding-left: 0; +} + +.version-section__listing > li { + margin-top: 1rem; +} + +.dashboard-item-metadata { + list-style: none; + display: flex; + flex-direction: row; + align-items: baseline; + gap: 1.2rem; + padding-left: 0; +} diff --git a/keeper/dashboard/template/edition_dashboard.jinja b/keeper/dashboard/template/edition_dashboard.jinja index a116987e..161b67a0 100644 --- a/keeper/dashboard/template/edition_dashboard.jinja +++ b/keeper/dashboard/template/edition_dashboard.jinja @@ -3,27 +3,74 @@ {% block page_title %}{{ project.title }} editions{% endblock page_title %} {% block page_description %}Find documentation editions.{% endblock page_description %} +{% macro edition_article(project, edition) -%} + +{%- endmacro %} + {% block body %}
+ {% set main_edition = editions.main_edition %} +
-

{{ project.title }}

+

{{ project.title }}
-
+
+

{{main_edition.url}}

+

+ Default edition last updated {{ main_edition.date_updated | simple_date }}. + {% if main_edition.git_ref %} + Based on the {{ main_edition.git_ref }} + branch/tag at {{ project.source_repo_url }}. + {% endif %} +

+
+ + {% if editions.has_releases %} +
+
+

Releases

+
+
    + {% for edition in editions.releases %} +
  • + {{ edition_article(project, edition) }} +
    • + {% endfor %} +
+
+ {% endif %} + + {% if editions.has_drafts %} +
-

Current edition

+

Drafts

-
- {% set mainedition = editions.main_edition %} -

-
    -
  • Updated {{ mainedition.date_updated | simple_date }}
    • - {% if mainedition.git_ref %} -
  • Git ref {{ mainedition.git_ref}}
    • - {% endif %} -
-
+
    + {% for edition in editions.drafts %} +
  • + {{ edition_article(project, edition) }} +
    • + {% endfor %} +
+ {% endif %}
{% endblock body %} diff --git a/tests/test_dashboard_template.py b/tests/test_dashboard_template.py index 1cabc6cd..62c4053f 100644 --- a/tests/test_dashboard_template.py +++ b/tests/test_dashboard_template.py @@ -33,10 +33,56 @@ def test_templates() -> None: EditionContext( title="Current", url="https://example.com/ltd-test/", - date_updated=datetime(2022, 6, 21, tzinfo=timezone.utc), + date_updated=datetime(2022, 6, 24, tzinfo=timezone.utc), kind=EditionKind.main, slug="__main", git_ref="main", + github_url="https://example.com/ltd-test/tree/main", + ) + ) + + editions.append( + EditionContext( + title="1.0.0", + url="https://example.com/ltd-test/v/1.0.0", + date_updated=datetime(2022, 6, 21, tzinfo=timezone.utc), + kind=EditionKind.release, + slug="1.0.0", + git_ref="1.0.0", + github_url="https://example.com/ltd-test/tree/1.0.0", + ) + ) + editions.append( + EditionContext( + title="1.1.0", + url="https://example.com/ltd-test/v/1.1.0", + date_updated=datetime(2022, 6, 22, tzinfo=timezone.utc), + kind=EditionKind.release, + slug="1.1.0", + git_ref="1.1.0", + github_url="https://example.com/ltd-test/tree/1.1.0", + ) + ) + editions.append( + EditionContext( + title="2.0.0", + url="https://example.com/ltd-test/v/2.0.0", + date_updated=datetime(2022, 6, 24, tzinfo=timezone.utc), + kind=EditionKind.release, + slug="2.0.0", + git_ref="2.0.0", + github_url="https://example.com/ltd-test/tree/2.0.0", + ) + ) + editions.append( + EditionContext( + title="my-branch", + url="https://example.com/ltd-test/v/my-branch", + date_updated=datetime(2022, 6, 24, tzinfo=timezone.utc), + kind=EditionKind.draft, + slug="my-branch", + git_ref="my-branch", + github_url="https://example.com/ltd-test/tree/my-branch", ) ) From 94d8c9e6c57ff2dba941d1b57b7247afe60bb43c Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Fri, 24 Jun 2022 15:54:48 -0400 Subject: [PATCH 11/12] Ensure the main edition doesn't appear twice Normally main editions should be marked with EditionKind.main, however if they aren't these extra tests ensure they don't appear in the releases or drafts lists. --- keeper/dashboard/context.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/keeper/dashboard/context.py b/keeper/dashboard/context.py index a233fbbd..f53c7f81 100644 --- a/keeper/dashboard/context.py +++ b/keeper/dashboard/context.py @@ -99,13 +99,17 @@ def has_releases(self) -> bool: @property def releases(self) -> List[EditionContext]: - """All edititions tagged as releases.""" + """All editions tagged as releases.""" release_kinds = ( EditionKind.release, EditionKind.major, EditionKind.minor, ) - release_items = [e for e in self.data if e.kind in release_kinds] + release_items = [ + e + for e in self.data + if (e.kind in release_kinds and e.slug != "__main") + ] sorted_items = sorted( release_items, key=lambda x: x.slug, reverse=True ) @@ -118,7 +122,11 @@ def has_drafts(self) -> bool: @property def drafts(self) -> List[EditionContext]: """All editions tagged as drafts.""" - draft_items = [e for e in self.data if e.kind == EditionKind.draft] + draft_items = [ + e + for e in self.data + if (e.kind == EditionKind.draft and e.slug != "__main") + ] return sorted(draft_items, key=lambda x: x.date_updated, reverse=True) From 2d8dc8be27b138c8141bdc969f1b4ac1930ceeda Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Fri, 24 Jun 2022 21:39:00 -0400 Subject: [PATCH 12/12] Update manifest for 2.0.0-alpha.5 --- manifests/base/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/base/kustomization.yaml b/manifests/base/kustomization.yaml index ae73996d..fc72acdc 100644 --- a/manifests/base/kustomization.yaml +++ b/manifests/base/kustomization.yaml @@ -8,4 +8,4 @@ resources: images: - name: 'lsstsqre/ltd-keeper:latest' - newTag: u-jsickcodes-deploy-2-0 + newTag: 2.0.0-alpha.5