From 1721119c2051b4999ad51f3f5b0ba4839cbe79ce Mon Sep 17 00:00:00 2001 From: Alex Parsons Date: Wed, 7 Feb 2024 20:07:39 +0000 Subject: [PATCH] Initial commit --- .gitattributes | 8 ++ .github/workflows/template_setup.yaml | 79 ++++++++++++++++++++ .github/workflows/template_test.yaml | 29 ++++++++ .gitignore | 1 + .gitmodules | 3 + cookie-readme.md | 12 +++ cookiecutter.json | 14 ++++ hooks/post_gen_project.py | 103 ++++++++++++++++++++++++++ hooks/pre_gen_project.py | 102 +++++++++++++++++++++++++ pytest.ini | 4 + readme.md | 24 ++++++ tests/test_cookiecutter.py | 67 +++++++++++++++++ {{ cookiecutter.repo_name }} | 1 + 13 files changed, 447 insertions(+) create mode 100644 .gitattributes create mode 100644 .github/workflows/template_setup.yaml create mode 100644 .github/workflows/template_test.yaml create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 cookie-readme.md create mode 100644 cookiecutter.json create mode 100644 hooks/post_gen_project.py create mode 100644 hooks/pre_gen_project.py create mode 100644 pytest.ini create mode 100644 readme.md create mode 100644 tests/test_cookiecutter.py create mode 160000 {{ cookiecutter.repo_name }} diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b6054b9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +* text eol=lf +*.png binary +*.jpg binary +*.sqlite3 binary +*.xlsx binary +*.xls binary +*.pdf binary +*.docx binary \ No newline at end of file diff --git a/.github/workflows/template_setup.yaml b/.github/workflows/template_setup.yaml new file mode 100644 index 0000000..98ab79c --- /dev/null +++ b/.github/workflows/template_setup.yaml @@ -0,0 +1,79 @@ +name: Run cookiecutter on first push + +on: [push] + +permissions: + actions: write + contents: write + +jobs: + run-cookiecutter: + if: ${{ !endsWith(github.repository, '-auto-template') }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + fetch-depth: 0 + ref: ${{ github.head_ref }} + + - name: Install cookiecutter + run: pip3 install cookiecutter + + - uses: actions/github-script@v4 + id: fetch-repo-and-user-details + with: + script: | + const query = `query($owner:String!, $name:String!) { + repository(owner:$owner, name:$name) { + name + description + owner { + login + ... on User { + name + } + ... on Organization { + name + } + } + } + }`; + const variables = { + owner: context.repo.owner, + name: context.repo.repo + } + const result = await github.graphql(query, variables) + console.log(result) + return result + + - name: Rebuild contents using cookiecutter + env: + INFO: ${{ steps.fetch-repo-and-user-details.outputs.result }} + TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + export REPO_NAME=$(echo $INFO | jq -r '.repository.name') + git config --global user.name "Cookie Cutter" + git config --global user.email "<>" + # Run cookiecutter + pushd /tmp + cookiecutter $GITHUB_WORKSPACE --no-input \ + project_name=$REPO_NAME \ + repo_name=$REPO_NAME \ + github_id=$GITHUB_REPOSITORY \ + description="$(echo $INFO | jq -r .repository.description)" + # move into generated project and push to replace current template + cd /tmp/$REPO_NAME + git remote add origin https://$GITHUB_ACTOR:$TOKEN@github.com/$GITHUB_REPOSITORY.git + git push --force --set-upstream origin main + + - name: "enable github pages workflow option" + uses: actions/github-script@v6 + continue-on-error: true + with: + script: | + github.request('POST /repos/{owner}/{repo}/pages', { + owner: context.repo.owner, + repo: context.repo.repo, + build_type: 'workflow' + }) diff --git a/.github/workflows/template_test.yaml b/.github/workflows/template_test.yaml new file mode 100644 index 0000000..92f3e4e --- /dev/null +++ b/.github/workflows/template_test.yaml @@ -0,0 +1,29 @@ +name: Run meta pytest suite on repo + +on: + pull_request: + push: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + run-test: + if: ${{ endsWith(github.repository, '-auto-template') }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + ref: ${{ github.head_ref }} + + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install cookiecutter + run: pip install pytest cookiecutter poetry + + - name: run pytest + run: python -m pytest \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..266c348 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.venv \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..d43e19c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "{{ cookiecutter.repo_name }}"] + path = {{ cookiecutter.repo_name }} + url = https://github.com/mysociety/template_data_repo/ diff --git a/cookie-readme.md b/cookie-readme.md new file mode 100644 index 0000000..3096752 --- /dev/null +++ b/cookie-readme.md @@ -0,0 +1,12 @@ +{# This file is the template for the resulting repo's readme file #} +# {{ cookiecutter.project_name }} + +[![badge](https://mybinder.org/badge.svg)](https://mybinder.org/v2/gh/{{ cookiecutter.github_id }}/HEAD) + +{{ cookiecutter.description }} + +This repository is available online at https://github.com/{{ cookiecutter.github_id }} + +If Github Pages are enabled, the URL is: https://mysociety.github.io/{{ cookiecutter.github_id.split("/")[1] }}/ + +Instructions on using the features of this notebook (data publishing, notebook rendering, Github Pages) are available in [https://github.com/mysociety/data_common/blob/main/data-repo-readme.md](Data Common readme file). \ No newline at end of file diff --git a/cookiecutter.json b/cookiecutter.json new file mode 100644 index 0000000..601d26c --- /dev/null +++ b/cookiecutter.json @@ -0,0 +1,14 @@ +{ + "repo_name": "snake_case_repo_name", + "hyphenated": "{{ '-'.join(cookiecutter['repo_name'].lower().split()).replace('_', '-') }}", + "underscored": "{{ cookiecutter.hyphenated.replace('-', '_') }}", + "project_name": "{{ cookiecutter.hyphenated.replace('-', ' ').title() }}", + "github_id": "mysociety/{{ cookiecutter.repo_name }}", + "description": "A short description of the project.", + "_copy_without_render": [".git", + "notebooks/_render_config/default.yaml", + "src/data_common/.github", + "src/data_common", + ".github", + "docs/theme"] +} diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py new file mode 100644 index 0000000..9072dec --- /dev/null +++ b/hooks/post_gen_project.py @@ -0,0 +1,103 @@ +import shutil +import os +import subprocess +from pathlib import Path + +template_dir = r"{{ cookiecutter._template }}" +if any(x in template_dir for x in ["https:", "gh:"]): + repo_name = template_dir.split("/")[-1] + template_dir = Path.home() / ".cookiecutters" / repo_name +else: + template_dir = Path(template_dir) + +template_repo = "https://github.com/mysociety/template_data_repo" +template_branch = "main" + +helper_repo = "https://github.com/mysociety/data_common" +helper_branch = "main" + + +# this was all a submodule in the template, now it stands alone. Need to copy across the git info. +# this is made conditional because in a templating test it won't be set up this way, but also that's fine. +if os.environ.get("UPDATE_TO_LATEST", "true").lower() == "true": + Path(".git").unlink() + real_git_folder = Path(template_dir) / ".git" / "modules" / ("{" + "{ cookiecutter.repo_name }" + "}") + shutil.copytree(real_git_folder, ".git") + git_config = Path(".git", "config") + notebook_git_config = Path(".git","modules", "src", "data_common", "config") + + # remove reference to the work tree above + with open(git_config, "r") as f: + lines = f.readlines() + with open(git_config, "w") as f: + for line in lines: + if "cookiecutter.repo_name" not in line: + f.write(line) + + # remove reference to the work tree above + with open(notebook_git_config, "r") as f: + lines = f.readlines() + with open(notebook_git_config, "w") as f: + for line in lines: + if "cookiecutter.repo_name" not in line: + f.write(line) + else: + f.write(" worktree = ../../../../src/data_common\n") + + # adjust the git directory for the notebook helper + with open(Path("src","data_common",".git"), "w") as file: + file.write("gitdir: ../../.git/modules/src/data_common") + +#copy example env to env +shutil.copyfile(Path(".env-example"), + Path(".env")) + +# when doing this on windows, sometimes clones bad line endings. +# This fixes the bash file docker uses. +# replacement strings +WINDOWS_LINE_ENDING = b'\r\n' +UNIX_LINE_ENDING = b'\n' + +# relative or absolute file path, e.g.: +file_path = Path("src","data_common", "bin", "packages_setup.bash") + +with open(file_path, 'rb') as open_file: + content = open_file.read() + +content = content.replace(WINDOWS_LINE_ENDING, UNIX_LINE_ENDING) + +with open(file_path, 'wb') as open_file: + open_file.write(content) + +# Lock the upstream docker image source at point of departure from template + +data_common_tag = subprocess.check_output("git submodule status src/data_common", shell=True).strip() +data_common_tag = data_common_tag.replace(b"+", b"") +data_common_tag = data_common_tag[:7] + +data_common_tag = b"data_common:sha-" + data_common_tag + +for d in ["Dockerfile", "Dockerfile.dev"]: + + with open(d, 'rb') as open_file: + content = open_file.read() + + content = content.replace(b"data_common:latest", data_common_tag) + + with open(d, 'wb') as open_file: + open_file.write(content) + +# remove templates we haven't already copied into the higher level +bad_workflows = [Path(".github", "workflows", "template_meta_test.yaml")] + +for w in bad_workflows: + w.unlink() + +if os.environ.get("UPDATE_TO_LATEST", "true").lower() == "true": + + # remove, we don't want this project to have a default origin of the template library + os.system(f'git remote rm origin') + + # package all up in a little box + os.system("git add --all") + os.system('git commit -m "Post-templating commit"') diff --git a/hooks/pre_gen_project.py b/hooks/pre_gen_project.py new file mode 100644 index 0000000..5e11c98 --- /dev/null +++ b/hooks/pre_gen_project.py @@ -0,0 +1,102 @@ +import shutil +import os +from pathlib import Path + +template_dir = r"{{ cookiecutter._template }}" +if any(x in template_dir for x in ["https:", "gh:"]): + repo_name = template_dir.split("/")[-1] + template_dir = Path.home() / ".cookiecutters" / repo_name +else: + template_dir = Path(template_dir) + if template_dir.is_absolute() is False: + raise ValueError("If specifying a specific directory, it needs to be an absolute path") + +def amend_file(filepath: Path, replace: dict): + """ + amend a file to use cookiecutter basics + """ + with open(filepath, "r") as f: + txt = f.read() + for key, value in replace.items(): + txt = txt.replace(key, value) + with open(filepath, "w") as f: + f.write(txt) + filename = str(filepath) + for key, value in replace.items(): + filename = filename.replace(key, value) + if str(filepath) != filename: + filepath.rename(filename) + + +repo_dir = template_dir / ("{" + "{ cookiecutter.repo_name }" + "}") + +template_repo = "https://github.com/mysociety/template_data_repo" +template_branch = "main" + +helper_repo = "https://github.com/mysociety/data_common" +helper_branch = "main" + +# allow a env variable to override the template updating to latest version +# this allows testing of the template +if os.environ.get("UPDATE_TO_LATEST", "true").lower() == "true": + print("using UPDATE_TO_LATEST") + print("updating submodule") + os.system(f"cd {template_dir} && git submodule update --init --recursive") + # update to latest version + print("resetting main to latest commit") + os.system(f'cd "{repo_dir}" && git reset --hard') + + print("resetting origin to the template wiki and pullonig down latest") + os.system(f'cd "{repo_dir}" && git remote rm origin') + os.system( + f'cd "{repo_dir}" && git remote add origin "{template_repo}" && git fetch origin && git pull origin main && git checkout main' + ) + print("doing the same for the data_common repo") + os.system(f'cd "{repo_dir}" && cd src/data_common && git remote rm origin') + os.system( + f'cd "{repo_dir}" && cd src/data_common && git remote add origin "{helper_repo}" && git fetch origin && git pull origin main && git checkout main' + ) + print("done") +else: + print("UPDATE_TO_LATEST disabled.") + +source_readme = Path(template_dir, "cookie-readme.md") +dest_readme = Path(repo_dir, "readme.md") +general_readme = Path(repo_dir, "notebooks-readme.md") + +print(f"Copying {source_readme} to {dest_readme}") +shutil.copyfile(source_readme, dest_readme) + +# Amend files that have a direct reference to the original name + +replace = { + "title: template_data_repo": "title: {" + "{ cookiecutter.project_name }" + "}", + 'baseurl: "/template_data_repo"': 'baseurl: "/' + + "{" + + "{ cookiecutter.repo_name }" + + '}"', + "template_data_repo:${TAG:-latest}": "{" + "{ cookiecutter.repo_name }" + "}:${TAG:-latest}", + "template_data_repo": "{" + "{ cookiecutter.underscored }" + "}", + "Standardised template for mysociety data repositories": "{" + + "{ cookiecutter.description }" + + "}", +} + + +amend_file(Path(repo_dir, ".devcontainer", "devcontainer.json"), replace) +amend_file(Path(repo_dir, "pyproject.toml"), replace) +amend_file(Path(repo_dir, "docker-compose.yml"), replace) +amend_file(Path(repo_dir, "Dockerfile.dev"), replace) +amend_file(Path(repo_dir, "Dockerfile"), replace) +amend_file(Path(repo_dir, "tests", "test_template_data_repo.py"), replace) +amend_file(Path(repo_dir, "docs", "index.md"), replace) +amend_file(Path(repo_dir, "docs", "_config.yml"), replace) + +to_delete = [Path(repo_dir, ".github", "workflows", "docker-image.yml")] + +package_dir = Path(repo_dir, "src", "template_data_repo") +package_dir.rename(Path(repo_dir, "src", "{" + "{ cookiecutter.underscored }" + "}")) + +for f in to_delete: + if f.exists(): + f.unlink() diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..db75e7a --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +# pytest.ini +[pytest] +testpaths = + tests \ No newline at end of file diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..0e1e0f1 --- /dev/null +++ b/readme.md @@ -0,0 +1,24 @@ +# mySociety data repo template + +This is a self-formatting template for a [mySociety data repo](https://github.com/mysociety/template_data_repo). Based on [Simon Willison's approach](https://simonwillison.net/2021/Aug/28/dynamic-github-repository-templates/). + +Click the 'Use this template' button, and give a useful name (snakecase) and description, which will populate the readme.md of the new repo. + +You can also use this link: https://github.com/mysociety/python-data-auto-template/generate + +If you have done this, and you are looking at this message in the new repo, wait twenty seconds and refresh. If this message is still here, investigate the status of the Github Action in the 'Actions' tab of the new repo. + +## Templating locally + +You can also use [Cookie Cutter](https://github.com/cookiecutter/cookiecutter). + +To create a new blank notebook template: + +```bash +# if python installed but cookiecutter isn't +pip install cookiecutter +# then +python -m cookiecutter gh:mysociety/python-data-auto-template +``` + +You will be prompted on setup settings. diff --git a/tests/test_cookiecutter.py b/tests/test_cookiecutter.py new file mode 100644 index 0000000..273ec86 --- /dev/null +++ b/tests/test_cookiecutter.py @@ -0,0 +1,67 @@ +import pytest +from cookiecutter.main import cookiecutter +from pathlib import Path +import subprocess + +template_dir = Path(__file__).parent / ".." + +@pytest.fixture(scope="session") +def cookie_folder(tmp_path_factory) -> Path: + """ + Create folder to test new template in in + """ + fn = tmp_path_factory.mktemp("cookie") + return fn + + +@pytest.fixture(scope="session") +def project_folder(cookie_folder): + """ + Create a cookiecutter project + """ + context = {"repo_name":"test-project", + "description": "Demonstration of a working cloned repo"} + + cookiecutter(str(template_dir), + no_input=True, + extra_context=context, + output_dir=cookie_folder) + + return cookie_folder / "test-project" + + +@pytest.fixture(scope="session") +def activated_project(project_folder: Path): + """ + Activate venv inside project + """ + subprocess.run("python -m poetry install".split(" "), check=True, cwd=project_folder) + return project_folder + +def test_project_generation(project_folder: Path): + """ + Generate a project and check for cookiecutter errors + """ + + assert project_folder.exists() is True + +def test_internal_pytest(activated_project: Path): + """ + Within clone of project, try and run the internal meta tests + This will check basic stuff like the python library paths being valid. + """ + subprocess.run("python -m poetry run pytest".split(" "), check=True, cwd=activated_project) + + +def test_black(activated_project: Path): + """ + Within clone of project, try and run black + """ + subprocess.run("python -m poetry run black .".split(" "), check=True, cwd=activated_project) + + +def test_pyright(activated_project: Path): + """ + Within clone of project, try and run pyright + """ + subprocess.run("python -m poetry run pyright".split(" "), check=True, cwd=activated_project) \ No newline at end of file diff --git a/{{ cookiecutter.repo_name }} b/{{ cookiecutter.repo_name }} new file mode 160000 index 0000000..64fb3d9 --- /dev/null +++ b/{{ cookiecutter.repo_name }} @@ -0,0 +1 @@ +Subproject commit 64fb3d96b42aaefc21db9e43fac2d03e88604367