Merge pull request #125 from sclorg/fix_get_branches

Use forks only on Request

.github/workflows/build-and-push.yml

@@ -20,7 +20,7 @@ jobs:
  with:
  dockerfiles: ./Dockerfile
  image: betka
- tags: latest 1 ${{ github.sha }} 0.8.1
+ tags: latest 1 ${{ github.sha }} 0.9.0
 
  - name: Push betka image to Quay.io
  id: push-to-quay

Dockerfile

@@ -1,7 +1,7 @@
 FROM quay.io/fedora/fedora:37
 
 ENV NAME=betka-fedora \
- RELEASE=0.8.1 \
+ RELEASE=0.9.0
  ARCH=x86_64 \
  SUMMARY="Syncs changes from upstream repository to downstream" \
  DESCRIPTION="Syncs changes from upstream repository to downstream" \

betka/core.py

@@ -50,7 +50,7 @@
  SYNC_INTERVAL,
 )
 from betka.utils import FileUtils
-from betka.named_tuples import ProjectMR, ProjectFork
+from betka.named_tuples import ProjectMR, ProjectFork, ProjectInfo
 
 
 requests.packages.urllib3.disable_warnings()
@@ -78,6 +78,7 @@ def __init__(self, task_name=None):
  self.upstream_hash = None
  self.downstream_dir: Path = None
  self.downstream_git_branch: str = None
+ self.downstream_git_origin_branch: str = None
  self.repo = None
  self.pr_number = None
  self._github_api = self._gitlab_api = None
@@ -99,6 +100,7 @@ def set_config(self):
  """
  self.set_config_from_env(self.config_json["github_api_token"])
  self.set_config_from_env(self.config_json["gitlab_user"])
+ self.set_config_from_env(self.config_json["use_gitlab_forks"])
  if "slack_webhook_url" in self.config_json:
  self.set_config_from_env(self.config_json["slack_webhook_url"])
  self.set_config_from_env("PROJECT")
@@ -201,6 +203,12 @@ def prepare_upstream_git(self):
  self.info("Upstream cloned directory %r", self.upstream_cloned_dir)
  return True
 
+ def is_fork_enabled(self):
+ value = self.betka_config["use_gitlab_forks"].lower()
+ if value in ["true", "yes"]:
+ return True
+ return False
+
  def mandatory_variables_set(self):
  """
  Are mandatory default 'betka' values set ?
@@ -323,7 +331,7 @@ def send_result_email(self, betka_schema: Dict):
  )
  return True
 
- def sync_to_downstream_branches(self, branch):
+ def sync_to_downstream_branches(self, branch, origin_branch: str = ""):
  """
  Sync upstream repository into relevant downstream dist-git branch
  based on the configuration file.
@@ -337,7 +345,7 @@ def sync_to_downstream_branches(self, branch):
  hash=self.upstream_hash, repo=self.repo
  )
  mr = self.gitlab_api.check_gitlab_merge_requests(branch=branch)
- if not mr:
+ if not mr and self.is_fork_enabled():
  Git.get_changes_from_distgit(url=self.gitlab_api.get_forked_ssh_url_to_repo())
  Git.push_changes_to_fork(branch=branch)
 
@@ -361,6 +369,7 @@ def sync_to_downstream_branches(self, branch):
  upstream_hash=self.upstream_hash,
  branch=branch,
  mr=mr,
+ origin_branch=origin_branch,
  )
  self.send_result_email(betka_schema=betka_schema)
 
@@ -493,7 +502,7 @@ def prepare(self):
  return False
  return True
 
- def prepare_downstream_git(self, project_fork: ProjectFork) -> bool:
+ def prepare_fork_downstream_git(self, project_fork: ProjectFork) -> bool:
 
  """
  Clone downstream dist-git repository, defined by self.ssh_url_to_repo variable
@@ -514,6 +523,26 @@ def prepare_downstream_git(self, project_fork: ProjectFork) -> bool:
 
  return True
 
+ def prepare_downstream_git(self, project_info: ProjectInfo) -> bool:
+
+ """
+ Clone downstream dist-git repository, defined by self.ssh_url_to_repo variable
+ and set `self.downstream_dir` variable.
+ :returns True if downstream git directory was cloned
+ False if downstream git directory was not cloned
+ """
+ self.downstream_dir = Git.clone_repo(
+ project_info.ssh_url_to_repo, self.betka_tmp_dir.name
+ )
+ self.info("Downstream directory %r", self.downstream_dir)
+ if self.downstream_dir is None:
+ self.error("!!!! Cloning downstream repo %s FAILED.", self.image)
+ return False
+ os.chdir(str(self.downstream_dir))
+ # This function updates fork based on the upstream
+
+ return True
+
  def _copy_cloned_upstream_dir(self):
  """
  Copy cloned upstream directory stored in self.upstream_cloned_dir
@@ -591,16 +620,20 @@ def _update_valid_branches(self):
  Git.sync_fork_with_upstream(branch_list_to_sync)
  return branch_list_to_sync
 
- def _update_valid_remote_branches(self):
- # Branches are taken from upstream repository like
- # https://src.fedoraproject.org/container/nginx not from fork
+ def _get_valid_origin_branches(self):
  all_branches = Git.get_valid_remote_branches()
  self.debug(f"All remote branches {all_branches}.")
  # Filter our branches before checking bot-cfg.yml files
  branch_list_to_sync = Git.branches_to_synchronize(
  self.betka_config, all_branches=all_branches
  )
  self.debug(f"Branches to sync {branch_list_to_sync}")
+ return branch_list_to_sync
+
+ def _update_valid_remote_branches(self):
+ # Branches are taken from upstream repository like
+ # https://src.fedoraproject.org/container/nginx not from fork
+ branch_list_to_sync = self._get_valid_origin_branches()
  Git.sync_fork_with_upstream(branch_list_to_sync)
  return branch_list_to_sync
 
@@ -617,7 +650,12 @@ def _sync_valid_branches(self, valid_branches):
  raise
  for branch in valid_branches:
  self.timestamp_dir: Path = None
- self.downstream_git_branch = branch
+ if self.is_fork_enabled():
+ self.downstream_git_branch = branch
+ self.downstream_git_origin_branch = ""
+ else:
+ self.downstream_git_branch = f"betka-{branch}"
+ self.downstream_git_origin_branch = branch
  # This loads downstream bot-cfg.yml file
  # and update betka's dictionary (self.config).
  # We need to have information up to date
@@ -630,7 +668,7 @@ def _sync_valid_branches(self, valid_branches):
  if not self.config.get("master_checker"):
  continue
  self.create_and_copy_timestamp_dir()
- self.sync_to_downstream_branches(self.downstream_git_branch)
+ self.sync_to_downstream_branches(self.downstream_git_branch, self.downstream_git_origin_branch)
  self.delete_timestamp_dir()
 
  def run_sync(self):
@@ -659,7 +697,7 @@ def _run_sync(self):
  # variable dist_git_repos
 
  try:
- self.gitlab_api.get_project_id_from_url()
+ project_id = self.gitlab_api.get_project_id_from_url()
  except requests.exceptions.HTTPError as htpe:
  BetkaEmails.send_email(
  text=f"Get project from URL {self.image} were not successful"
@@ -669,31 +707,35 @@ def _run_sync(self):
  subject=f"[betka-sync] Get project from URL project {self.image} were not successful.",
  )
  continue
- self.gitlab_api.init_projects()
- project_fork = self.gitlab_api.check_and_create_fork()
- if not project_fork:
- BetkaEmails.send_email(
- text=f"Fork for project {self.image} were not successful"
- f"by upstream2downstream-bot. See {values}\n"
- f"Inform [email protected]",
- receivers=["[email protected]"],
- subject=f"[betka-sync] Fork for project {self.image} were not successful.",
- )
- continue
-
+ if self.is_fork_enabled():
+ self.gitlab_api.init_projects()
+ project_fork = self.gitlab_api.check_and_create_fork()
+ if not project_fork:
+ BetkaEmails.send_email(
+ text=f"Fork for project {self.image} were not successful"
+ f"by upstream2downstream-bot. See {values}\n"
+ f"Inform [email protected]",
+ receivers=["[email protected]"],
+ subject=f"[betka-sync] Fork for project {self.image} were not successful.",
+ )
+ continue
+ self.ssh_url_to_repo = project_fork.ssh_url_to_repo
+ self.debug(f"Clone URL is: {self.ssh_url_to_repo}")
+ os.chdir(self.betka_tmp_dir.name)
+ if not self.prepare_fork_downstream_git(project_fork):
+ continue
+ branch_list_to_sync = self._update_valid_remote_branches()
+ else:
+ project_info = self.gitlab_api.get_project_info()
+ self.ssh_url_to_repo = project_info.ssh_url_to_repo
+ self.debug(f"Clone URL is: {self.ssh_url_to_repo}")
+ os.chdir(self.betka_tmp_dir.name)
+ if not self.prepare_downstream_git(project_info):
+ continue
+ branch_list_to_sync = self._get_valid_origin_branches()
  self.info(
  f"Trying to sync image {self.image} to GitLab project_id {self.gitlab_api.project_id}."
  )
- os.chdir(self.betka_tmp_dir.name)
-
- self.ssh_url_to_repo = project_fork.ssh_url_to_repo
- self.debug(f"Clone URL is: {self.ssh_url_to_repo}")
- # after downstream is cloned then
- # new cwd is self.downstream_dir
- if not self.prepare_downstream_git(project_fork):
- continue
- #branch_list_to_sync = self._update_valid_branches()
- branch_list_to_sync = self._update_valid_remote_branches()
 
  valid_branches = Git.get_valid_branches(
  self.image, self.downstream_dir, branch_list_to_sync

betka/gitlab.py

@@ -305,13 +305,15 @@ def file_merge_request(
  pr_msg: str,
  upstream_hash: str,
  branch: str,
+ origin_branch: str,
  mr: Any,
  ) -> Dict:
  """
  Files a Pull Request with specific messages and text.
  :param pr_msg: description message used in pull request
  :param upstream_hash: commit hash for
  :param branch: specify downstream branch for file a Pull Request
+ :param origin_branch: specify origin_branch to which file a Pull Request
  :param mr: named touple ProjectMR
  :return: schema for sending email
  """
@@ -323,7 +325,7 @@ def file_merge_request(
  logger.debug(f"Upstream {text_mr} to downstream PR not found.")
 
  mr: ProjectMR = self.create_gitlab_merge_request(
- title=title, desc_msg=pr_msg, branch=branch
+ title=title, desc_msg=pr_msg, branch=branch, origin_branch=origin_branch
  )
  logger.debug(f"MergeRequest is: {mr}")
  if mr is None:
@@ -375,7 +377,7 @@ def init_projects(self) -> bool:
  return True
 
  def create_gitlab_merge_request(
- self, title: str, desc_msg: str, branch: str
+ self, title: str, desc_msg: str, branch: str, origin_branch: str,
  ) -> ProjectMR:
  """
  Creates the pull request for specific image
@@ -392,6 +394,8 @@ def create_gitlab_merge_request(
  "description": desc_msg,
  "target_project_id": self.project_id,
  }
+ if not self.betka_config["use_gitlab_fork"]:
+ data["target_branch"] = origin_branch
  return self.create_project_mergerequest(data)
 
  def check_gitlab_merge_requests(self, branch: str):
@@ -519,3 +523,4 @@ def get_project_id_from_url(self):
  raise HTTPError
  self.project_id = ret.json()["id"]
  logger.debug(f"Project id returned from {url} is {self.project_id}")
+ return self.project_id

config.json

@@ -18,5 +18,6 @@
  "gitlab_host_url": "https://gitlab.com",
  "gitlab_namespace": "redhat/rhel/containers",
  "dist_git_url": "https://src.fedoraproject.org/container",
- "slack_webhook_url": "SLACK_WEBHOOK_URL"
+ "slack_webhook_url": "SLACK_WEBHOOK_URL",
+ "use_gitlab_forks": "False"
 }

setup.py

@@ -14,7 +14,7 @@ def get_requirements():
 
 setup(
  name="betka",
- version="0.8.1",
+ version="0.9.0",
  packages=find_packages(exclude=["examples", "tests"]),
  url="https://github.com/sclorg/betka",
  license="GPLv3+",

tests/conftest.py

@@ -57,6 +57,7 @@ def betka_yaml():
  },
  },
  "downstream_master_msg": "[betka-master-sync]",
+ "use_gitlab_forks": "True"
  }
 
 
@@ -84,6 +85,7 @@ def config_json():
  "gitlab_url_user": "user",
  "dist_git_url": "https://src.fedoraproject.org/containers",
  "slack_webhook_url": "SLACK_WEBHOOK_URL",
+ "use_gitlab_forks": "True",
  }
 
 

tests/integration/test_betka_upstream_sync.py

@@ -93,6 +93,7 @@ def setup_method(self):
  os.environ["GITLAB_API_TOKEN"] = "gitlabsomething"
  os.environ["GITHUB_API_TOKEN"] = "aklsdjfh19p3845yrp"
  os.environ["GITLAB_USER"] = "testymctestface"
+ os.environ["USE_GITLAB_FORKS"] = "true"
  flexmock(FileUtils).should_receive("load_config_json").and_return(config_json())
  self.betka = Betka(task_name="task.betka.master_sync")
  self.config_json = config_json()
@@ -239,6 +240,7 @@ def test_betka_run_master_sync(
  mock_rmtree,
  ):
  self.betka.betka_config["dist_git_repos"].pop("s2i-core")
+ self.betka.betka_config["use_gitlab_forks"] = "True"
  list_images = self.betka.get_synced_images()
  sync_image = ""
  for key, value in list_images.items():
@@ -251,6 +253,7 @@ def test_betka_run_master_sync(
  flexmock(self.betka).should_receive("_update_valid_remote_branches").and_return(
  ["fc30", "fc31"]
  )
+ flexmock(self.betka).should_receive("prepare_fork_downstream_git").twice()
  # flexmock(Git).should_receive("sync_fork_with_upstream").twice()
  self.betka.gitlab_api.project_id = PROJECT_ID
  flexmock(self.betka.gitlab_api).should_receive("get_project_id_from_url").and_return(PROJECT_ID)

tests/unit/test_betka_config_json.py

@@ -64,6 +64,19 @@ def config_json_missing_generator_url():
  }
 
 
+def config_json_missing_use_gitlab_fork():
+ return {
+ "api_url": "https://src.fedoraproject.org/api/0",
+ "get_all_pr": "https://src.fedoraproject.org/api/0/{namespace}/{repo}/pull-requests",
+ "git_url_repo": "https://src.fedoraproject.org/api/0/fork/{user}/{namespace}/{repo}/git/",
+ "namespace_containers": "container",
+ "github_api_token": "aklsdjfh19p3845yrp",
+ "gitlab_user": "testymctestface",
+ "gitlab_api_token": "testing",
+ "generator_url": "some_generator_url",
+ }
+
+
 class TestBetkaCore(object):
  def setup_method(self):
  self.betka = Betka()
@@ -73,6 +86,7 @@ def setup_method(self):
  [
  config_json_missing_github_api_token(),
  config_json_missing_generator_url(),
+ config_json_missing_use_gitlab_fork(),
  ],
  )
  def test_betka_config_keyerror(self, config_json):

tests/unit/test_core.py

@@ -85,6 +85,19 @@ def test_get_synced_images(self, msg_upstream_url, return_value):
  for image, values in dict_images.items():
  assert image == image in return_value
 
+ @pytest.mark.parametrize(
+ "status,return_value",
+ [
+ ("True", True),
+ ("true", True),
+ ("False", False),
+ ("false", False),
+ ],
+ )
+ def test_is_fork_enabled(self, status, return_value):
+ self.betka.betka_config["use_gitlab_forks"] = status
+ assert self.betka.is_fork_enabled() == return_value
+
  @pytest.mark.parametrize(
  "msg_upstream_url,return_value",
  [