From 21931e62e9826ddc4ce8b688a94c321863ee677f Mon Sep 17 00:00:00 2001 From: Fabian Thomsen Date: Tue, 23 May 2023 10:44:49 +0200 Subject: [PATCH 1/7] Test recursive vcs import for upstream workspace --- recursive_vcs_import.py | 14 +++++++++----- templates/.gitlab-ci.template.yml | 7 +++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/recursive_vcs_import.py b/recursive_vcs_import.py index 36cbb2f..b1126ba 100755 --- a/recursive_vcs_import.py +++ b/recursive_vcs_import.py @@ -3,13 +3,15 @@ import pathlib import subprocess import sys -from typing import List +from typing import List, Optional -def findDotRepos(search_path: str) -> List[pathlib.Path]: - - return pathlib.Path(search_path).glob("**/*.repos") +def findDotRepos(search_path: str, clone_path: Optional[str]=None) -> List[pathlib.Path]: + repos = list(pathlib.Path(search_path).glob("**/*.repos")) + if clone_path is not None: + repos.extend(list(pathlib.Path(clone_path).glob("**/*.repos"))) + return repos def main(): @@ -19,7 +21,7 @@ def main(): while True: - found_repos = findDotRepos(search_path) + found_repos = findDotRepos(search_path, clone_path) remaining_repos = set(found_repos) - set(cloned_repos) if not remaining_repos: @@ -32,6 +34,8 @@ def main(): raise RuntimeError("vcs import failed") cloned_repos.append(next_repo) + + print(" ".join([str(repo) for repo in found_repos])) if __name__ == "__main__": diff --git a/templates/.gitlab-ci.template.yml b/templates/.gitlab-ci.template.yml index f9d94c5..5ffee82 100644 --- a/templates/.gitlab-ci.template.yml +++ b/templates/.gitlab-ci.template.yml @@ -137,7 +137,6 @@ run-arm64: .test: variables: - UPSTREAM_WORKSPACE: $ROS_DIR/.repos TARGET_WORKSPACE: $ROS_DIR ADDITIONAL_DEBS: git AFTER_INIT_EMBED: git config --global url.${CI_SERVER_PROTOCOL}://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}.insteadOf ${CI_SERVER_URL} @@ -146,7 +145,11 @@ run-arm64: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - apk add --update bash coreutils grep tar - git clone --branch master --depth 1 https://github.com/ros-industrial/industrial_ci.git .industrial_ci - - test -f $ROS_DIR/.repos || echo "repositories:" > $ROS_DIR/.repos + - mkdir /tmp/vcs_repos + - UPSTREAM_WORKSPACE=$(python recursive_vcs_import.py $ROS_DIR /tmp/vcs_repos | tail -1) + - if [ -z "$UPSTREAM_WORKSPACE" ]; then; else; echo "repositories:" > $ROS_DIR/.repos; UPSTREAM_WORKSPACE=$ROS_DIR/.repos fi; + - echo $UPSTREAM_WORKSPACE + - export UPSTREAM_WORKSPACE script: .industrial_ci/gitlab.sh Test dev-amd64: From e7e6468cb893a5816d7b290c3d656cd251899558 Mon Sep 17 00:00:00 2001 From: Fabian Thomsen Date: Tue, 23 May 2023 11:25:22 +0200 Subject: [PATCH 2/7] Combine before_scripts --- templates/.gitlab-ci.template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/.gitlab-ci.template.yml b/templates/.gitlab-ci.template.yml index 5ffee82..6dd8983 100644 --- a/templates/.gitlab-ci.template.yml +++ b/templates/.gitlab-ci.template.yml @@ -56,7 +56,7 @@ default: tags: - privileged - amd64 - before_script: + before_script: &default_before_script - |- if [[ ! -d $DOCKER_COMPOSE_DIR/docker-ros ]]; then git config --global url.${CI_SERVER_PROTOCOL}://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}.insteadOf ${CI_SERVER_URL} @@ -142,7 +142,7 @@ run-arm64: AFTER_INIT_EMBED: git config --global url.${CI_SERVER_PROTOCOL}://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}.insteadOf ${CI_SERVER_URL} DOCKER_RUN_OPTS: -u root:root before_script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - *default_before_script - apk add --update bash coreutils grep tar - git clone --branch master --depth 1 https://github.com/ros-industrial/industrial_ci.git .industrial_ci - mkdir /tmp/vcs_repos From 90a1f4a7c2917e1c5412844167d5bdb3db35f06d Mon Sep 17 00:00:00 2001 From: Fabian Thomsen Date: Tue, 23 May 2023 12:56:59 +0200 Subject: [PATCH 3/7] Install dependencies --- templates/.gitlab-ci.template.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/templates/.gitlab-ci.template.yml b/templates/.gitlab-ci.template.yml index 6dd8983..6e76a64 100644 --- a/templates/.gitlab-ci.template.yml +++ b/templates/.gitlab-ci.template.yml @@ -143,13 +143,14 @@ run-arm64: DOCKER_RUN_OPTS: -u root:root before_script: - *default_before_script - - apk add --update bash coreutils grep tar - - git clone --branch master --depth 1 https://github.com/ros-industrial/industrial_ci.git .industrial_ci + - apk add --update bash coreutils grep tar python3 py3-pip + - pip install vcstool - mkdir /tmp/vcs_repos - - UPSTREAM_WORKSPACE=$(python recursive_vcs_import.py $ROS_DIR /tmp/vcs_repos | tail -1) + - UPSTREAM_WORKSPACE=$(python3 $DOCKER_COMPOSE_DIR/docker-ros/recursive_vcs_import.py $ROS_DIR /tmp/vcs_repos | tail -1) - if [ -z "$UPSTREAM_WORKSPACE" ]; then; else; echo "repositories:" > $ROS_DIR/.repos; UPSTREAM_WORKSPACE=$ROS_DIR/.repos fi; - echo $UPSTREAM_WORKSPACE - export UPSTREAM_WORKSPACE + - git clone --branch master --depth 1 https://github.com/ros-industrial/industrial_ci.git .industrial_ci script: .industrial_ci/gitlab.sh Test dev-amd64: From 92ef80202d49ac1bfb907f48143a99a102dfd11e Mon Sep 17 00:00:00 2001 From: Fabian Thomsen Date: Tue, 23 May 2023 15:31:13 +0200 Subject: [PATCH 4/7] Set work directory --- templates/.gitlab-ci.template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/.gitlab-ci.template.yml b/templates/.gitlab-ci.template.yml index 6e76a64..8721de1 100644 --- a/templates/.gitlab-ci.template.yml +++ b/templates/.gitlab-ci.template.yml @@ -143,6 +143,7 @@ run-arm64: DOCKER_RUN_OPTS: -u root:root before_script: - *default_before_script + - cd - - apk add --update bash coreutils grep tar python3 py3-pip - pip install vcstool - mkdir /tmp/vcs_repos From 0775a6ee72e0083c4c2dbf2e25207917377ba934 Mon Sep 17 00:00:00 2001 From: Fabian Thomsen Date: Tue, 23 May 2023 15:59:29 +0200 Subject: [PATCH 5/7] Fix conditional --- templates/.gitlab-ci.template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/.gitlab-ci.template.yml b/templates/.gitlab-ci.template.yml index 8721de1..8840c54 100644 --- a/templates/.gitlab-ci.template.yml +++ b/templates/.gitlab-ci.template.yml @@ -148,7 +148,7 @@ run-arm64: - pip install vcstool - mkdir /tmp/vcs_repos - UPSTREAM_WORKSPACE=$(python3 $DOCKER_COMPOSE_DIR/docker-ros/recursive_vcs_import.py $ROS_DIR /tmp/vcs_repos | tail -1) - - if [ -z "$UPSTREAM_WORKSPACE" ]; then; else; echo "repositories:" > $ROS_DIR/.repos; UPSTREAM_WORKSPACE=$ROS_DIR/.repos fi; + - if [ -z "$UPSTREAM_WORKSPACE" ]; then echo "repositories:" > $ROS_DIR/.repos; UPSTREAM_WORKSPACE=$ROS_DIR/.repos; fi - echo $UPSTREAM_WORKSPACE - export UPSTREAM_WORKSPACE - git clone --branch master --depth 1 https://github.com/ros-industrial/industrial_ci.git .industrial_ci From a8bb9e4fc6486f3c45a6a752b1cbe38bdf8993e5 Mon Sep 17 00:00:00 2001 From: Fabian Thomsen Date: Tue, 23 May 2023 17:33:41 +0200 Subject: [PATCH 6/7] Dont use tmp folder for recursive .repos --- templates/.gitlab-ci.template.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/templates/.gitlab-ci.template.yml b/templates/.gitlab-ci.template.yml index 8840c54..d61064e 100644 --- a/templates/.gitlab-ci.template.yml +++ b/templates/.gitlab-ci.template.yml @@ -146,8 +146,7 @@ run-arm64: - cd - - apk add --update bash coreutils grep tar python3 py3-pip - pip install vcstool - - mkdir /tmp/vcs_repos - - UPSTREAM_WORKSPACE=$(python3 $DOCKER_COMPOSE_DIR/docker-ros/recursive_vcs_import.py $ROS_DIR /tmp/vcs_repos | tail -1) + - UPSTREAM_WORKSPACE=$(python3 $DOCKER_COMPOSE_DIR/docker-ros/recursive_vcs_import.py $ROS_DIR | tail -1) - if [ -z "$UPSTREAM_WORKSPACE" ]; then echo "repositories:" > $ROS_DIR/.repos; UPSTREAM_WORKSPACE=$ROS_DIR/.repos; fi - echo $UPSTREAM_WORKSPACE - export UPSTREAM_WORKSPACE From 9ef4594807a5347c2142e43c1d76860b175eeee9 Mon Sep 17 00:00:00 2001 From: Fabian Thomsen Date: Tue, 23 May 2023 18:07:45 +0200 Subject: [PATCH 7/7] Don't duplicate .repos files --- recursive_vcs_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recursive_vcs_import.py b/recursive_vcs_import.py index b1126ba..160460a 100755 --- a/recursive_vcs_import.py +++ b/recursive_vcs_import.py @@ -35,7 +35,7 @@ def main(): cloned_repos.append(next_repo) - print(" ".join([str(repo) for repo in found_repos])) + print(" ".join([str(repo) for repo in set(found_repos)])) if __name__ == "__main__":