Use git-autoshare with git-aggregator

TT32042

13.0.Dockerfile

@@ -13,6 +13,10 @@ ENV DB_FILTER=.* \
     GEOIP_ACCOUNT_ID="" \
     GEOIP_LICENSE_KEY="" \
     GIT_AUTHOR_NAME=docker-odoo \
+    GIT_AUTHOSHARE=0 \
+    GIT_AUTOSHARE_CACHE_DIR="/home/odoo/.cache/git-autoshare/" \
+    GIT_AUTOSHARE_CONFIG_DIR="/home/odoo/.config/git-autoshare/" \
+    GIT_AUTOSHARE_ORGS_TO_CACHE="OCA,odoo,Tecnativa" \
     INITIAL_LANG="" \
     LC_ALL=C.UTF-8 \
     LIST_DB=false \
@@ -126,21 +130,23 @@ RUN build_deps=" \
         -r https://raw.githubusercontent.com/$ODOO_SOURCE/$ODOO_VERSION/requirements.txt \
         'websocket-client~=0.56' \
         astor \
-        git-aggregator \
+        git+https://github.com/Tecnativa/git-aggregator@use-git-autoshare \
+        git+https://github.com/Tecnativa/git-autoshare@bundle-git-script \
         # Install fix from https://github.com/acsone/click-odoo-contrib/pull/93
         git+https://github.com/Tecnativa/click-odoo-contrib.git@fix-active-modules-hashing \
         "pg_activity<2.0.0" \
+        debugpy \
+        geoip2 \
+        inotify \
         phonenumbers \
         plumbum \
         ptvsd \
-        debugpy \
-        pydevd-odoo \
         pudb \
+        pydevd-odoo \
         python-magic \
+        pyyaml \
         watchdog \
         wdb \
-        geoip2 \
-        inotify \
     && (python3 -m compileall -q /usr/local/lib/python3.6/ || true) \
     && apt-get purge -yqq $build_deps \
     && apt-get autopurge -yqq \

14.0.Dockerfile

@@ -12,6 +12,10 @@ ENV DB_FILTER=.* \
     GEOIP_ACCOUNT_ID="" \
     GEOIP_LICENSE_KEY="" \
     GIT_AUTHOR_NAME=docker-odoo \
+    GIT_AUTHOSHARE=0 \
+    GIT_AUTOSHARE_CACHE_DIR="/home/odoo/.cache/git-autoshare/" \
+    GIT_AUTOSHARE_CONFIG_DIR="/home/odoo/.config/git-autoshare/" \
+    GIT_AUTOSHARE_ORGS_TO_CACHE="OCA,odoo,Tecnativa" \
     INITIAL_LANG="" \
     LC_ALL=C.UTF-8 \
     LIST_DB=false \
@@ -127,14 +131,16 @@ RUN build_deps=" \
         git+https://github.com/Tecnativa/click-odoo-contrib.git@fix-active-modules-hashing \
         debugpy \
         geoip2 \
-        git-aggregator \
+        git+https://github.com/Tecnativa/git-aggregator@use-git-autoshare \
+        git+https://github.com/Tecnativa/git-autoshare@bundle-git-script \
         inotify \
         pdfminer.six \
         pg_activity \
         phonenumbers \
         plumbum \
         pudb \
         python-magic \
+        pyyaml \
         watchdog \
         wdb \
     && (python3 -m compileall -q /usr/local/lib/python3.8/ || true) \

15.0.Dockerfile

@@ -6,21 +6,25 @@ ARG GEOIP_UPDATER_VERSION=4.3.0
 ARG WKHTMLTOPDF_VERSION=0.12.5
 ARG WKHTMLTOPDF_CHECKSUM='dfab5506104447eef2530d1adb9840ee3a67f30caaad5e9bcb8743ef2f9421bd'
 ENV DB_FILTER=.* \
+    DEBUGPY_ARGS="--listen 0.0.0.0:6899 --wait-for-client" \
+    DEBUGPY_ENABLE=0 \
     DEPTH_DEFAULT=1 \
     DEPTH_MERGE=100 \
     EMAIL=https://hub.docker.com/r/tecnativa/odoo \
     GEOIP_ACCOUNT_ID="" \
     GEOIP_LICENSE_KEY="" \
     GIT_AUTHOR_NAME=docker-odoo \
+    GIT_AUTHOSHARE=0 \
+    GIT_AUTOSHARE_CACHE_DIR="/home/odoo/.cache/git-autoshare/" \
+    GIT_AUTOSHARE_CONFIG_DIR="/home/odoo/.config/git-autoshare/" \
+    GIT_AUTOSHARE_ORGS_TO_CACHE="OCA,odoo,Tecnativa" \
     INITIAL_LANG="" \
     LC_ALL=C.UTF-8 \
     LIST_DB=false \
     NODE_PATH=/usr/local/lib/node_modules:/usr/lib/node_modules \
     OPENERP_SERVER=/opt/odoo/auto/odoo.conf \
     PATH="/home/odoo/.local/bin:$PATH" \
     PIP_NO_CACHE_DIR=0 \
-    DEBUGPY_ARGS="--listen 0.0.0.0:6899 --wait-for-client" \
-    DEBUGPY_ENABLE=0 \
     PUDB_RDB_HOST=0.0.0.0 \
     PUDB_RDB_PORT=6899 \
     PYTHONOPTIMIZE=1 \
@@ -125,14 +129,16 @@ RUN build_deps=" \
         git+https://github.com/Tecnativa/click-odoo-contrib.git@fix-active-modules-hashing \
         debugpy \
         geoip2 \
-        git-aggregator \
+        git+https://github.com/Tecnativa/git-aggregator@use-git-autoshare \
+        git+https://github.com/Tecnativa/git-autoshare@bundle-git-script \
         inotify \
         pdfminer.six \
         pg_activity \
         phonenumbers \
         plumbum \
         pudb \
         python-magic \
+        pyyaml \
         watchdog \
         wdb \
     && (python3 -m compileall -q /usr/local/lib/python3.8/ || true) \

README.md

@@ -594,6 +594,50 @@ not expose the debugging port to attackers**. Usage:
 We found this one to be the most useful tool for downlading code, merging it and placing
 it somewhere.
 
+It can also take advantage of a shared repository cache in your local machine, making
+use of [git-autoshare](https://github.com/acsone/git-autoshare).
+
+To do so, you must map a local folder on the host machine to the container in the
+aggregation process and set `GIT_AUTOSHARE=1` in the container.
+
+The folder that will store the cache in the container will be
+`/home/odoo/.cache/git-autoshare/`, but you can change it by providing a different value
+through the `GIT_AUTOSHARE_CACHE_DIR` environment variable. Then, to map the local cache
+folder to the container, you can add the following line to the `volumes` section of the
+`odoo` service in the `docker-compose.yaml` file:
+
+```yaml
+- ~/.cache/git-autoshare/:/home/odoo/.cache/git-autoshare/
+```
+
+This will allow you to re-use the same cache for other git operations in the host
+machine, if you have `git-autoshare` configured.
+
+`git-autoshare` will also need a configuration file, which should be placed in
+`/home/odoo/.config/git-autoshare/repos.yml` (or another directory by changing the
+`GIT_AUTOSHARE_CONFIG_DIR` environment variable). You can check the documentation for
+more details, but you can either map a local file to the container containing this
+configuration, or pass a comma-separated list of Github Organizations through the
+`GIT_AUTOSHARE_ORGS_TO_CACHE` environment variable, which will make Doodba generate that
+file automatically with the following structure:
+
+```
+GIT_AUTOSHARE_ORGS_TO_CACHE=Tecnativa,odoo,OCA
+```
+
+`/home/odoo/.config/git-autoshare/repos.yml`:
+
+```
+github.com:
+    "*":
+        orgs:
+            - odoo
+            - OCA
+            - Tecnativa
+```
+
+This will generate a cache for all repos in those 3 Github Organizations.
+
 ### `autoaggregate`
 
 This little script wraps `git-aggregator` to make it work fine and automatically with

build.d/100-repos-aggregate

@@ -4,6 +4,8 @@ set -e
 # make sure odoo has a user.name configured, as merges would not succeed otherwise
 # (even if GIT_AUTHOR_NAME and EMAIL are set and should be used, it seems gitaggregate is not passing them to git)
 su --shell="$SHELL" odoo -c 'git config user.name 1>/dev/null || git config --global user.name "'"$GIT_AUTHOR_NAME"'"'
+# set "main" as the default branch name to supress warnings
+su --shell="$SHELL" odoo -c 'git config init.defaultBranch 1>/dev/null || git config --global init.defaultBranch main'
 
 # copy ssh directory to odoo user as well (gitaggregate may also be run as odoo user)
 if [[ ! -e ~odoo/.ssh ]] ; then

build.d/50-configure-git-autoshare

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Generates the appropiate configuration file for git-autoshare according to
+user-defined settings through environment variables
+"""
+
+import os
+from pathlib import Path
+
+import yaml
+
+GIT_AUTOSHARE_ORGS_TO_CACHE = os.environ.get(
+    "GIT_AUTOSHARE_ORGS_TO_CACHE", "OCA,odoo,Tecnativa"
+)
+GIT_AUTOSHARE_CONFIG_DIR = os.environ.get(
+    "GIT_AUTOSHARE_CONFIG_DIR", "/home/odoo/.config/git-autoshare/"
+)
+GIT_AUTOSHARE_CACHE_DIR = os.environ.get(
+    "GIT_AUTOSHARE_CACHE_DIR", "/home/odoo/.cache/git-autoshare/"
+)
+UID = int(os.environ["UID"])
+GID = int(os.environ["GID"])
+
+orgs_to_cache = GIT_AUTOSHARE_ORGS_TO_CACHE.split(",")
+git_autoshare_config_dir_path = Path(GIT_AUTOSHARE_CONFIG_DIR)
+git_autoshare_config_path = git_autoshare_config_dir_path / "repos.yml"
+
+if git_autoshare_config_path.is_file():
+    exit(0)
+else:
+    git_autoshare_config_dir_path.mkdir(parents=True, exist_ok=True)
+    os.chown(str(git_autoshare_config_dir_path), UID, GID)
+
+git_autoshare_config = {"github.com": {"*": {"orgs": orgs_to_cache,}}}
+
+with git_autoshare_config_path.open("w") as f:
+    f.write(yaml.safe_dump(git_autoshare_config))
+os.chown(str(git_autoshare_config_path), UID, GID)
+
+# Ensure cache dir exists and has correct permissions
+git_autoshare_cache_dir = Path(GIT_AUTOSHARE_CACHE_DIR)
+git_autoshare_cache_dir.mkdir(parents=True, exist_ok=True)
+os.chown(str(git_autoshare_cache_dir), UID, GID)

tests/__init__.py

@@ -711,6 +711,36 @@ def test_aggregate_permissions(self):
                 ("autoaggregate",),
             )
 
+    def test_aggregate_autoshare(self):
+        symlink_dir = join(SCAFFOLDINGS_DIR, "repo_merge")
+        for sub_env in matrix(odoo_skip={"11.0", "12.0"}):
+            self.compose_test(
+                symlink_dir,
+                dict(
+                    sub_env, UID=str(os.getuid()), GID=str(os.getgid()), GIT_AUTOSHARE=1
+                ),
+                # Ensure no cache exists prior to 1st aggregation
+                ("rm", "-rf", "/home/odoo/.cache/git-autoshare/github.com"),
+                ("test", "!", "-d", "/home/odoo/.cache/git-autoshare/github.com/ocb"),
+                # The git-autoshare config file should have been correctly generated
+                ("test", "-f", "/home/odoo/.config/git-autoshare/repos.yml"),
+                ("autoaggregate",),
+                # Cache should have been generated
+                ("test", "-d", "/home/odoo/.cache/git-autoshare/github.com/ocb"),
+                # Cloned repo should reference to cache
+                (
+                    "test",
+                    "-f",
+                    "/opt/odoo/custom/src/odoo/.git/objects/info/alternates",
+                ),
+                (
+                    "test",
+                    "(cat /opt/odoo/custom/src/odoo/.git/objects/info/alternates)",
+                    "=",
+                    "/home/odoo/.cache/git-autoshare/github.com/ocb/objects",
+                ),
+            )
+
 
 if __name__ == "__main__":
     unittest.main()

tests/scaffoldings/repo_merge/docker-compose.yaml

@@ -10,6 +10,7 @@ services:
     environment:
       PYTHONOPTIMIZE: ""
       UNACCENT: "false"
+      GIT_AUTOSHARE: "${GIT_AUTOSHARE:-0}"
     entrypoint: "" # we are not using odoo but only checking the repo state here, this skips starting the db
     volumes:
       - filestore:/var/lib/odoo:z

tests/scaffoldings/repo_merge/setup-devel.yaml

@@ -15,6 +15,7 @@ services:
       UID: "${UID:-1000}"
       GID: "${GID:-1000}"
       UMASK: "$UMASK"
+      GIT_AUTOSHARE: "${GIT_AUTOSHARE:-0}"
     user: root
     entrypoint: "" # we are only calling autoaggregate inside the tests
     volumes: