Add changelog enforcement (#15459)

* Add changelog enforcement

* Apply suggestions from code review

Co-authored-by: Ilia Kurenkov <[email protected]>

* address review

* Apply suggestions from code review

Co-authored-by: Ilia Kurenkov <[email protected]>

---------

Co-authored-by: Ilia Kurenkov <[email protected]>

.github/workflows/pr-check.yml

@@ -0,0 +1,14 @@
+name: Check PR
+
+on: pull_request
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  run:
+    uses: ./.github/workflows/pr-quick-check.yml
+    with:
+      repo: core
+    secrets: inherit

.github/workflows/pr-quick-check.yml

@@ -0,0 +1,50 @@
+name: Quick check PR
+
+on:
+  workflow_call:
+    inputs:
+      repo:
+        required: true
+        type: string
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  PYTHON_VERSION: "3.11"
+  CHECK_SCRIPT: "ddev/src/ddev/utils/scripts/check_pr.py"
+
+jobs:
+  check:
+    name: Check PR
+    runs-on: ubuntu-22.04
+
+    steps:
+    - uses: actions/checkout@v3
+      if: inputs.repo == 'core'
+      with:
+        ref: "${{ github.event.pull_request.head.sha }}"
+
+    - name: Set up Python ${{ env.PYTHON_VERSION }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: "${{ env.PYTHON_VERSION }}"
+
+    - name: Fetch PR data
+      env:
+        GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+      run: |-
+        diff_url=$(cat "$GITHUB_EVENT_PATH" | jq -r '.pull_request.diff_url')
+        curl --header "Authorization: Bearer $GITHUB_TOKEN" -sLo /tmp/diff "$diff_url"
+        cat "$GITHUB_EVENT_PATH" | jq -r '.pull_request.number' > /tmp/number
+
+    - name: Fetch script
+      if: inputs.repo != 'core'
+      run: |-
+        mkdir -p $(dirname ${{ env.CHECK_SCRIPT }})
+        curl -sLo ${{ env.CHECK_SCRIPT }} https://raw.githubusercontent.com/DataDog/integrations-core/master/${{ env.CHECK_SCRIPT }}
+
+    - name: Check
+      run: |-
+        python ${{ env.CHECK_SCRIPT }} changelog --diff-file /tmp/diff --pr-number $(cat /tmp/number)

ddev/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+***Added***:
+
+* Add changelog enforcement (#15459)
+
 ## 3.3.0 / 2023-07-20
 
 ***Added***:

ddev/src/ddev/cli/release/__init__.py

@@ -3,13 +3,13 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 import click
 from datadog_checks.dev.tooling.commands.release.build import build
-from datadog_checks.dev.tooling.commands.release.changelog import changelog
 from datadog_checks.dev.tooling.commands.release.make import make
 from datadog_checks.dev.tooling.commands.release.tag import tag
 from datadog_checks.dev.tooling.commands.release.trello import trello
 from datadog_checks.dev.tooling.commands.release.upload import upload
 
 from ddev.cli.release.agent import agent
+from ddev.cli.release.changelog import changelog
 from ddev.cli.release.list_versions import list_versions
 from ddev.cli.release.show import show
 from ddev.cli.release.stats import stats

ddev/src/ddev/cli/release/changelog/__init__.py

@@ -0,0 +1,18 @@
+# (C) Datadog, Inc. 2023-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+import click
+
+from ddev.cli.release.changelog.fix import fix
+from ddev.cli.release.changelog.new import new
+
+
+@click.group(short_help='Manage changelogs')
+def changelog():
+    """
+    Manage changelogs.
+    """
+
+
+changelog.add_command(fix)
+changelog.add_command(new)

ddev/src/ddev/cli/release/changelog/fix.py

@@ -0,0 +1,52 @@
+# (C) Datadog, Inc. 2023-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import click
+
+if TYPE_CHECKING:
+    from ddev.cli.application import Application
+
+
+@click.command(short_help='Fix changelog entries')
+@click.pass_obj
+def fix(app: Application):
+    """
+    Fix changelog entries.
+    """
+    from ddev.utils.scripts.check_pr import get_changelog_errors
+
+    latest_commit = app.repo.git.latest_commit
+    pr = app.github.get_pull_request(latest_commit.sha)
+    if pr is not None:
+        git_diff = app.github.get_diff(pr)
+        pr_number = pr.number
+    else:
+        git_diff = app.repo.git.capture('diff', 'origin/master...')
+        pr_number = app.github.get_next_issue_number()
+
+    expected_suffix = f' (#{pr_number})'
+    fixed = 0
+    for path, line_number, _ in get_changelog_errors(git_diff, expected_suffix):
+        if line_number == 1:
+            continue
+
+        changelog = app.repo.path / path
+        lines = changelog.read_text().splitlines(keepends=True)
+        index = line_number - 1
+        original_line = lines[index]
+        new_line = original_line.rstrip()
+        if new_line.endswith(expected_suffix):
+            continue
+
+        lines[index] = f'{new_line}{expected_suffix}{original_line[len(new_line) :]}'
+        changelog.write_text(''.join(lines))
+        fixed += 1
+
+    if not fixed:
+        app.display_info('No changelog entries need fixing')
+    else:
+        app.display_success(f'Fixed {fixed} changelog entr{"ies" if fixed > 1 else "y"}')

ddev/src/ddev/cli/release/changelog/new.py

@@ -0,0 +1,101 @@
+# (C) Datadog, Inc. 2023-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import click
+
+if TYPE_CHECKING:
+    from ddev.cli.application import Application
+
+
+@click.command(short_help='Create changelog entries')
+@click.argument('entry_type', required=False)
+@click.argument('targets', nargs=-1, required=False)
+@click.option(
+    '--message',
+    '-m',
+    help='The changelog text, defaulting to the PR title followed by the most recent commit message',
+)
+@click.pass_obj
+def new(app: Application, entry_type: str | None, targets: tuple[str], message: str | None):
+    """
+    Create changelog entries.
+    """
+    from ddev.release.constants import ENTRY_TYPES
+
+    derive_message = message is None
+    latest_commit = app.repo.git.latest_commit
+    pr = app.github.get_pull_request(latest_commit.sha)
+    if pr is not None:
+        pr_number = pr.number
+        if message is None:
+            message = pr.title
+    else:
+        pr_number = app.github.get_next_issue_number()
+        if message is None:
+            message = latest_commit.subject
+
+    if entry_type is not None:
+        entry_type = entry_type.capitalize()
+        if entry_type not in ENTRY_TYPES:
+            app.abort(f'Unknown entry type: {entry_type}')
+    else:
+        entry_type = click.prompt('Entry type?', type=click.Choice(ENTRY_TYPES, case_sensitive=False))
+
+    expected_suffix = f' (#{pr_number})'
+    entry = f'* {message.rstrip()}{expected_suffix}'
+    if derive_message and (new_entry := click.edit(entry)) is not None:
+        entry = new_entry
+    entry = entry.strip()
+
+    entry_priority = ENTRY_TYPES.index(entry_type)
+    edited = 0
+
+    for target in app.repo.integrations.iter_changed_code(targets):
+        changelog = target.path / 'CHANGELOG.md'
+        lines = changelog.read_text().splitlines()
+
+        unreleased = False
+        current_entry_type: str | None = None
+        i = 0
+        for i, line in enumerate(lines):
+            if line == '## Unreleased':
+                unreleased = True
+                continue
+            elif unreleased and line.startswith('## '):
+                break
+            elif line.startswith('***'):
+                # e.g. ***Added***:
+                current_entry_type = line[3:-4]
+
+                try:
+                    current_entry_priority = ENTRY_TYPES.index(current_entry_type)
+                except ValueError:
+                    app.abort(
+                        f'{changelog.relative_to(app.repo.path)}, line {i}: unknown entry type {current_entry_type}'
+                    )
+
+                if current_entry_priority > entry_priority:
+                    break
+
+        if current_entry_type is None or current_entry_type != entry_type:
+            for line in reversed(
+                (
+                    f'***{entry_type}***:',
+                    '',
+                    entry,
+                    '',
+                )
+            ):
+                lines.insert(i, line)
+        else:
+            lines.insert(i - 1, entry)
+
+        lines.append('')
+        changelog.write_text('\n'.join(lines))
+        edited += 1
+
+    app.display_success(f'Added {edited} changelog entr{"ies" if edited > 1 else "y"}')

ddev/src/ddev/integration/core.py

@@ -67,6 +67,9 @@ def package_files(self) -> Iterator[Path]:
                 if f.endswith('.py'):
                     yield Path(root, f)
 
+    def requires_changelog_entry(self, path: Path) -> bool:
+        return self.package_directory in path.parents or (self.is_package and path == (self.path / 'pyproject.toml'))
+
     @property
     def release_tag_pattern(self) -> str:
         version_part = r'\d+\.\d+\.\d+'

ddev/src/ddev/release/__init__.py

@@ -0,0 +1,3 @@
+# (C) Datadog, Inc. 2023-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)

ddev/src/ddev/release/constants.py

@@ -0,0 +1,4 @@
+# (C) Datadog, Inc. 2023-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+ENTRY_TYPES = ('Removed', 'Changed', 'Security', 'Deprecated', 'Added', 'Fixed')

ddev/src/ddev/repo/core.py

@@ -87,48 +87,85 @@ def get(self, name: str) -> Integration:
         return integration
 
     def iter(self, selection: Iterable[str] = ()) -> Iterable[Integration]:
+        """
+        Iterate over all integrations.
+        """
         for integration in self.__iter_filtered(selection):
             if integration.is_integration:
                 yield integration
 
     def iter_all(self, selection: Iterable[str] = ()) -> Iterable[Integration]:
+        """
+        Iterate over all targets i.e. any integration or Python package.
+        """
         for integration in self.__iter_filtered(selection):
             if integration.is_valid:
                 yield integration
 
     def iter_packages(self, selection: Iterable[str] = ()) -> Iterable[Integration]:
+        """
+        Iterate over all Python packages.
+        """
         for integration in self.__iter_filtered(selection):
             if integration.is_package:
                 yield integration
 
     def iter_tiles(self, selection: Iterable[str] = ()) -> Iterable[Integration]:
+        """
+        Iterate over all tile-only integrations.
+        """
         for integration in self.__iter_filtered(selection):
             if integration.is_tile:
                 yield integration
 
     def iter_testable(self, selection: Iterable[str] = ()) -> Iterable[Integration]:
+        """
+        Iterate over all targets that can be tested.
+        """
         for integration in self.__iter_filtered(selection):
             if integration.is_testable:
                 yield integration
 
     def iter_shippable(self, selection: Iterable[str] = ()) -> Iterable[Integration]:
+        """
+        Iterate over all integrations that can be shipped by the Agent.
+        """
         for integration in self.__iter_filtered(selection):
             if integration.is_shippable:
                 yield integration
 
     def iter_agent_checks(self, selection: Iterable[str] = ()) -> Iterable[Integration]:
+        """
+        Iterate over all Python checks.
+        """
         for integration in self.__iter_filtered(selection):
             if integration.is_agent_check:
                 yield integration
 
     def iter_jmx_checks(self, selection: Iterable[str] = ()) -> Iterable[Integration]:
+        """
+        Iterate over all JMX checks.
+        """
         for integration in self.__iter_filtered(selection):
             if integration.is_jmx_check:
                 yield integration
 
     def iter_changed(self) -> Iterable[Integration]:
+        """
+        Iterate over all integrations that have changed.
+        """
         yield from self.iter_all()
 
+    def iter_changed_code(self, selection: Iterable[str] = ()) -> Iterable[Integration]:
+        """
+        Iterate over all integrations that have changes that could affect built distributions.
+        """
+        for integration in self.__iter_filtered(selection):
+            for relative_path in self.repo.git.changed_files:
+                if integration.requires_changelog_entry(self.repo.path / relative_path):
+                    yield integration
+                    break
+
     def __iter_filtered(self, selection: Iterable[str] = ()) -> Iterable[Integration]:
         selected = self.__finalize_selection(selection)
         if selected is None:

ddev/src/ddev/utils/git.py

@@ -47,7 +47,8 @@ def get_current_branch(self) -> str:
 
     @cached_property
     def latest_commit(self) -> GitCommit:
-        return GitCommit(self.capture('rev-parse', 'HEAD').strip())
+        sha, subject = self.capture('log', '-1', '--format=%H%n%s').splitlines()
+        return GitCommit(sha, subject=subject)
 
     def get_latest_commit(self) -> GitCommit:
         with suppress(AttributeError):