diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 9fa7ec4..db1f59d 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -20,7 +20,7 @@ jobs: with: dockerfiles: ./Dockerfile image: betka - tags: latest 1 ${{ github.sha }} 0.8.1 + tags: latest 1 ${{ github.sha }} 0.9.0 - name: Push betka image to Quay.io id: push-to-quay diff --git a/Dockerfile b/Dockerfile index d3c98e2..87332ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM quay.io/fedora/fedora:37 ENV NAME=betka-fedora \ - RELEASE=0.8.1 \ + RELEASE=0.9.0 ARCH=x86_64 \ SUMMARY="Syncs changes from upstream repository to downstream" \ DESCRIPTION="Syncs changes from upstream repository to downstream" \ diff --git a/betka/core.py b/betka/core.py index 540ca65..443192b 100644 --- a/betka/core.py +++ b/betka/core.py @@ -50,7 +50,7 @@ SYNC_INTERVAL, ) from betka.utils import FileUtils -from betka.named_tuples import ProjectMR, ProjectFork +from betka.named_tuples import ProjectMR, ProjectFork, ProjectInfo requests.packages.urllib3.disable_warnings() @@ -78,6 +78,7 @@ def __init__(self, task_name=None): self.upstream_hash = None self.downstream_dir: Path = None self.downstream_git_branch: str = None + self.downstream_git_origin_branch: str = None self.repo = None self.pr_number = None self._github_api = self._gitlab_api = None @@ -99,6 +100,7 @@ def set_config(self): """ self.set_config_from_env(self.config_json["github_api_token"]) self.set_config_from_env(self.config_json["gitlab_user"]) + self.set_config_from_env(self.config_json["use_gitlab_forks"]) if "slack_webhook_url" in self.config_json: self.set_config_from_env(self.config_json["slack_webhook_url"]) self.set_config_from_env("PROJECT") @@ -201,6 +203,12 @@ def prepare_upstream_git(self): self.info("Upstream cloned directory %r", self.upstream_cloned_dir) return True + def is_fork_enabled(self): + value = self.betka_config["use_gitlab_forks"].lower() + if value in ["true", "yes"]: + return True + return False + def mandatory_variables_set(self): """ Are mandatory default 'betka' values set ? @@ -323,7 +331,7 @@ def send_result_email(self, betka_schema: Dict): ) return True - def sync_to_downstream_branches(self, branch): + def sync_to_downstream_branches(self, branch, origin_branch: str = ""): """ Sync upstream repository into relevant downstream dist-git branch based on the configuration file. @@ -337,7 +345,7 @@ def sync_to_downstream_branches(self, branch): hash=self.upstream_hash, repo=self.repo ) mr = self.gitlab_api.check_gitlab_merge_requests(branch=branch) - if not mr: + if not mr and self.is_fork_enabled(): Git.get_changes_from_distgit(url=self.gitlab_api.get_forked_ssh_url_to_repo()) Git.push_changes_to_fork(branch=branch) @@ -361,6 +369,7 @@ def sync_to_downstream_branches(self, branch): upstream_hash=self.upstream_hash, branch=branch, mr=mr, + origin_branch=origin_branch, ) self.send_result_email(betka_schema=betka_schema) @@ -493,7 +502,7 @@ def prepare(self): return False return True - def prepare_downstream_git(self, project_fork: ProjectFork) -> bool: + def prepare_fork_downstream_git(self, project_fork: ProjectFork) -> bool: """ Clone downstream dist-git repository, defined by self.ssh_url_to_repo variable @@ -514,6 +523,26 @@ def prepare_downstream_git(self, project_fork: ProjectFork) -> bool: return True + def prepare_downstream_git(self, project_info: ProjectInfo) -> bool: + + """ + Clone downstream dist-git repository, defined by self.ssh_url_to_repo variable + and set `self.downstream_dir` variable. + :returns True if downstream git directory was cloned + False if downstream git directory was not cloned + """ + self.downstream_dir = Git.clone_repo( + project_info.ssh_url_to_repo, self.betka_tmp_dir.name + ) + self.info("Downstream directory %r", self.downstream_dir) + if self.downstream_dir is None: + self.error("!!!! Cloning downstream repo %s FAILED.", self.image) + return False + os.chdir(str(self.downstream_dir)) + # This function updates fork based on the upstream + + return True + def _copy_cloned_upstream_dir(self): """ Copy cloned upstream directory stored in self.upstream_cloned_dir @@ -591,9 +620,7 @@ def _update_valid_branches(self): Git.sync_fork_with_upstream(branch_list_to_sync) return branch_list_to_sync - def _update_valid_remote_branches(self): - # Branches are taken from upstream repository like - # https://src.fedoraproject.org/container/nginx not from fork + def _get_valid_origin_branches(self): all_branches = Git.get_valid_remote_branches() self.debug(f"All remote branches {all_branches}.") # Filter our branches before checking bot-cfg.yml files @@ -601,6 +628,12 @@ def _update_valid_remote_branches(self): self.betka_config, all_branches=all_branches ) self.debug(f"Branches to sync {branch_list_to_sync}") + return branch_list_to_sync + + def _update_valid_remote_branches(self): + # Branches are taken from upstream repository like + # https://src.fedoraproject.org/container/nginx not from fork + branch_list_to_sync = self._get_valid_origin_branches() Git.sync_fork_with_upstream(branch_list_to_sync) return branch_list_to_sync @@ -617,7 +650,12 @@ def _sync_valid_branches(self, valid_branches): raise for branch in valid_branches: self.timestamp_dir: Path = None - self.downstream_git_branch = branch + if self.is_fork_enabled(): + self.downstream_git_branch = branch + self.downstream_git_origin_branch = "" + else: + self.downstream_git_branch = f"betka-{branch}" + self.downstream_git_origin_branch = branch # This loads downstream bot-cfg.yml file # and update betka's dictionary (self.config). # We need to have information up to date @@ -630,7 +668,7 @@ def _sync_valid_branches(self, valid_branches): if not self.config.get("master_checker"): continue self.create_and_copy_timestamp_dir() - self.sync_to_downstream_branches(self.downstream_git_branch) + self.sync_to_downstream_branches(self.downstream_git_branch, self.downstream_git_origin_branch) self.delete_timestamp_dir() def run_sync(self): @@ -659,7 +697,7 @@ def _run_sync(self): # variable dist_git_repos try: - self.gitlab_api.get_project_id_from_url() + project_id = self.gitlab_api.get_project_id_from_url() except requests.exceptions.HTTPError as htpe: BetkaEmails.send_email( text=f"Get project from URL {self.image} were not successful" @@ -669,31 +707,35 @@ def _run_sync(self): subject=f"[betka-sync] Get project from URL project {self.image} were not successful.", ) continue - self.gitlab_api.init_projects() - project_fork = self.gitlab_api.check_and_create_fork() - if not project_fork: - BetkaEmails.send_email( - text=f"Fork for project {self.image} were not successful" - f"by upstream2downstream-bot. See {values}\n" - f"Inform phracek@redhat.com", - receivers=["phracek@redhat.com"], - subject=f"[betka-sync] Fork for project {self.image} were not successful.", - ) - continue - + if self.is_fork_enabled(): + self.gitlab_api.init_projects() + project_fork = self.gitlab_api.check_and_create_fork() + if not project_fork: + BetkaEmails.send_email( + text=f"Fork for project {self.image} were not successful" + f"by upstream2downstream-bot. See {values}\n" + f"Inform phracek@redhat.com", + receivers=["phracek@redhat.com"], + subject=f"[betka-sync] Fork for project {self.image} were not successful.", + ) + continue + self.ssh_url_to_repo = project_fork.ssh_url_to_repo + self.debug(f"Clone URL is: {self.ssh_url_to_repo}") + os.chdir(self.betka_tmp_dir.name) + if not self.prepare_fork_downstream_git(project_fork): + continue + branch_list_to_sync = self._update_valid_remote_branches() + else: + project_info = self.gitlab_api.get_project_info() + self.ssh_url_to_repo = project_info.ssh_url_to_repo + self.debug(f"Clone URL is: {self.ssh_url_to_repo}") + os.chdir(self.betka_tmp_dir.name) + if not self.prepare_downstream_git(project_info): + continue + branch_list_to_sync = self._get_valid_origin_branches() self.info( f"Trying to sync image {self.image} to GitLab project_id {self.gitlab_api.project_id}." ) - os.chdir(self.betka_tmp_dir.name) - - self.ssh_url_to_repo = project_fork.ssh_url_to_repo - self.debug(f"Clone URL is: {self.ssh_url_to_repo}") - # after downstream is cloned then - # new cwd is self.downstream_dir - if not self.prepare_downstream_git(project_fork): - continue - #branch_list_to_sync = self._update_valid_branches() - branch_list_to_sync = self._update_valid_remote_branches() valid_branches = Git.get_valid_branches( self.image, self.downstream_dir, branch_list_to_sync diff --git a/betka/gitlab.py b/betka/gitlab.py index 9b808c3..0eb0225 100644 --- a/betka/gitlab.py +++ b/betka/gitlab.py @@ -305,6 +305,7 @@ def file_merge_request( pr_msg: str, upstream_hash: str, branch: str, + origin_branch: str, mr: Any, ) -> Dict: """ @@ -312,6 +313,7 @@ def file_merge_request( :param pr_msg: description message used in pull request :param upstream_hash: commit hash for :param branch: specify downstream branch for file a Pull Request + :param origin_branch: specify origin_branch to which file a Pull Request :param mr: named touple ProjectMR :return: schema for sending email """ @@ -323,7 +325,7 @@ def file_merge_request( logger.debug(f"Upstream {text_mr} to downstream PR not found.") mr: ProjectMR = self.create_gitlab_merge_request( - title=title, desc_msg=pr_msg, branch=branch + title=title, desc_msg=pr_msg, branch=branch, origin_branch=origin_branch ) logger.debug(f"MergeRequest is: {mr}") if mr is None: @@ -375,7 +377,7 @@ def init_projects(self) -> bool: return True def create_gitlab_merge_request( - self, title: str, desc_msg: str, branch: str + self, title: str, desc_msg: str, branch: str, origin_branch: str, ) -> ProjectMR: """ Creates the pull request for specific image @@ -392,6 +394,8 @@ def create_gitlab_merge_request( "description": desc_msg, "target_project_id": self.project_id, } + if not self.betka_config["use_gitlab_fork"]: + data["target_branch"] = origin_branch return self.create_project_mergerequest(data) def check_gitlab_merge_requests(self, branch: str): @@ -519,3 +523,4 @@ def get_project_id_from_url(self): raise HTTPError self.project_id = ret.json()["id"] logger.debug(f"Project id returned from {url} is {self.project_id}") + return self.project_id diff --git a/config.json b/config.json index c6e6bbb..682e50d 100644 --- a/config.json +++ b/config.json @@ -18,5 +18,6 @@ "gitlab_host_url": "https://gitlab.com", "gitlab_namespace": "redhat/rhel/containers", "dist_git_url": "https://src.fedoraproject.org/container", - "slack_webhook_url": "SLACK_WEBHOOK_URL" + "slack_webhook_url": "SLACK_WEBHOOK_URL", + "use_gitlab_forks": "False" } diff --git a/setup.py b/setup.py index bdf1f5c..b8ecdbe 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ def get_requirements(): setup( name="betka", - version="0.8.1", + version="0.9.0", packages=find_packages(exclude=["examples", "tests"]), url="https://github.com/sclorg/betka", license="GPLv3+", diff --git a/tests/conftest.py b/tests/conftest.py index baf0a4c..6094cea 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,6 +57,7 @@ def betka_yaml(): }, }, "downstream_master_msg": "[betka-master-sync]", + "use_gitlab_forks": "True" } @@ -84,6 +85,7 @@ def config_json(): "gitlab_url_user": "user", "dist_git_url": "https://src.fedoraproject.org/containers", "slack_webhook_url": "SLACK_WEBHOOK_URL", + "use_gitlab_forks": "True", } diff --git a/tests/integration/test_betka_upstream_sync.py b/tests/integration/test_betka_upstream_sync.py index e47c760..3171ce7 100644 --- a/tests/integration/test_betka_upstream_sync.py +++ b/tests/integration/test_betka_upstream_sync.py @@ -93,6 +93,7 @@ def setup_method(self): os.environ["GITLAB_API_TOKEN"] = "gitlabsomething" os.environ["GITHUB_API_TOKEN"] = "aklsdjfh19p3845yrp" os.environ["GITLAB_USER"] = "testymctestface" + os.environ["USE_GITLAB_FORKS"] = "true" flexmock(FileUtils).should_receive("load_config_json").and_return(config_json()) self.betka = Betka(task_name="task.betka.master_sync") self.config_json = config_json() @@ -239,6 +240,7 @@ def test_betka_run_master_sync( mock_rmtree, ): self.betka.betka_config["dist_git_repos"].pop("s2i-core") + self.betka.betka_config["use_gitlab_forks"] = "True" list_images = self.betka.get_synced_images() sync_image = "" for key, value in list_images.items(): @@ -251,6 +253,7 @@ def test_betka_run_master_sync( flexmock(self.betka).should_receive("_update_valid_remote_branches").and_return( ["fc30", "fc31"] ) + flexmock(self.betka).should_receive("prepare_fork_downstream_git").twice() # flexmock(Git).should_receive("sync_fork_with_upstream").twice() self.betka.gitlab_api.project_id = PROJECT_ID flexmock(self.betka.gitlab_api).should_receive("get_project_id_from_url").and_return(PROJECT_ID) diff --git a/tests/unit/test_betka_config_json.py b/tests/unit/test_betka_config_json.py index 4d39661..07d0f3f 100644 --- a/tests/unit/test_betka_config_json.py +++ b/tests/unit/test_betka_config_json.py @@ -64,6 +64,19 @@ def config_json_missing_generator_url(): } +def config_json_missing_use_gitlab_fork(): + return { + "api_url": "https://src.fedoraproject.org/api/0", + "get_all_pr": "https://src.fedoraproject.org/api/0/{namespace}/{repo}/pull-requests", + "git_url_repo": "https://src.fedoraproject.org/api/0/fork/{user}/{namespace}/{repo}/git/", + "namespace_containers": "container", + "github_api_token": "aklsdjfh19p3845yrp", + "gitlab_user": "testymctestface", + "gitlab_api_token": "testing", + "generator_url": "some_generator_url", + } + + class TestBetkaCore(object): def setup_method(self): self.betka = Betka() @@ -73,6 +86,7 @@ def setup_method(self): [ config_json_missing_github_api_token(), config_json_missing_generator_url(), + config_json_missing_use_gitlab_fork(), ], ) def test_betka_config_keyerror(self, config_json): diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py index 1035474..ba95e83 100644 --- a/tests/unit/test_core.py +++ b/tests/unit/test_core.py @@ -85,6 +85,19 @@ def test_get_synced_images(self, msg_upstream_url, return_value): for image, values in dict_images.items(): assert image == image in return_value + @pytest.mark.parametrize( + "status,return_value", + [ + ("True", True), + ("true", True), + ("False", False), + ("false", False), + ], + ) + def test_is_fork_enabled(self, status, return_value): + self.betka.betka_config["use_gitlab_forks"] = status + assert self.betka.is_fork_enabled() == return_value + @pytest.mark.parametrize( "msg_upstream_url,return_value", [ diff --git a/tests/unit/test_gitlab.py b/tests/unit/test_gitlab.py index 5faee8e..8496209 100644 --- a/tests/unit/test_gitlab.py +++ b/tests/unit/test_gitlab.py @@ -186,10 +186,10 @@ def test_file_merge_request_new(self): "rhel-8.6.0", "phracek", PROJECT_ID_FORK, PROJECT_ID, "https://gitlab/foo/bar", ) flexmock(self.ga).should_receive("create_gitlab_merge_request").with_args( - title="[betka-master-sync]", desc_msg=pr_msg, branch=branch_name + title="[betka-master-sync]", desc_msg=pr_msg, branch=branch_name, origin_branch="", ).and_return(mr) betka_schema = self.ga.file_merge_request( - pr_msg=pr_msg, upstream_hash=upstream_hash, branch=branch_name, mr=None + pr_msg=pr_msg, upstream_hash=upstream_hash, branch=branch_name, origin_branch="", mr=None ) assert betka_schema assert betka_schema["commit"] == upstream_hash @@ -208,10 +208,10 @@ def test_file_merge_request_failed(self): upstream_hash = "10938482734" flexmock(BetkaEmails).should_receive("send_email").once() flexmock(self.ga).should_receive("create_gitlab_merge_request").with_args( - title="[betka-master-sync]", desc_msg=pr_msg, branch=branch_name + title="[betka-master-sync]", desc_msg=pr_msg, branch=branch_name, origin_branch="", ).and_return(mr_id) betka_schema = self.ga.file_merge_request( - pr_msg=pr_msg, upstream_hash=upstream_hash, branch=branch_name, mr=mr_id + pr_msg=pr_msg, upstream_hash=upstream_hash, branch=branch_name, origin_branch="", mr=mr_id ) assert not betka_schema @@ -225,10 +225,10 @@ def test_file_merge_request_update(self): PROJECT_ID_FORK, PROJECT_ID, "https://gitlab/foo/bar", ) flexmock(self.ga).should_receive("create_gitlab_merge_request").with_args( - title="[betka-master-sync]", desc_msg=pr_msg, branch=branch_name + title="[betka-master-sync]", desc_msg=pr_msg, branch=branch_name, origin_branch="", ).and_return(mr) betka_schema = self.ga.file_merge_request( - pr_msg=pr_msg, upstream_hash=upstream_hash, branch=branch_name, mr=mr + pr_msg=pr_msg, upstream_hash=upstream_hash, branch=branch_name, origin_branch="", mr=mr ) assert betka_schema assert betka_schema["commit"] == upstream_hash