Merge pull request #101 from sclorg/fixes_caught_during_runtimt

Fixes caught by runtime

Dockerfile

@@ -1,7 +1,7 @@
 FROM quay.io/fedora/fedora:37
 
 ENV NAME=betka-fedora \
- RELEASE=3 \
+ RELEASE=4 \
  ARCH=x86_64 \
  SUMMARY="Syncs changes from upstream repository to downstream" \
  DESCRIPTION="Syncs changes from upstream repository to downstream" \

Makefile

@@ -29,7 +29,10 @@ build-test: build
  $(PODMAN) build --tag ${TEST_IMAGE_NAME} -f Dockerfile.tests .
 
 run: prepare build
- docker-compose up betka redis
+ docker-compose --verbose up betka redis
+
+run-betka: prepare build
+ ./run-podman.sh
 
 run-generator: prepare build-generator
  docker-compose up generator
@@ -44,14 +47,19 @@ image-push: build
  $(PODMAN) push ${IMAGE_NAME}
 
 send-master-sync:
- docker-compose exec betka python3 /tmp/betka-bot/upstream_master_sync.py
+ podman exec betka python3 /tmp/betka-bot/upstream_master_sync.py
+ #docker-compose exec betka python3 /tmp/betka-bot/upstream_master_sync.py
 
 clean:
  find . -name '*.pyc' -delete
 
 stop:
  docker-compose down
 
+stop-podman:
+ podman stop redis && podman rm redis
+ podman stop betka && podman rm betka
+
 image_deploy:
  $(PODMAN) build --tag=${DEPLOY_NAME} -f Dockerfile.deployment .
 

betka/config.py

@@ -10,6 +10,7 @@
 import requests
 import sys
 import yaml
+import os
 
 from typing import Any
 
@@ -108,7 +109,7 @@ def fetch_config(config_key, config_file_url):
 
  bots_config = ""
  logger.info(f"Pulling config file: {config_file_url}")
- r = requests.get(config_file_url, cookies={"gitlab": "user-cont-bot-cfg-load"})
+ r = requests.get(config_file_url, verify=False)
  if r.status_code == 200:
  bots_config = r.text
  logger.debug("Bot configuration fetched")
@@ -172,8 +173,8 @@ def load_configuration(conf_path=None, conf_str=None):
  # overwrite defaults with values in bot configuration
  dict_merge(into_dct=result, from_dct=repo_conf)
 
- # validate
- jsonschema.validate(result, BotCfg.get_schema())
+ # # validate
+ # jsonschema.validate(result, BotCfg.get_schema())
 
  logger.debug(f"Resulting bots configuration: {pretty_dict(result)}")
 

betka/core.py

@@ -31,7 +31,6 @@
 
 from os import getenv
 from datetime import datetime
-from requests import get
 from tempfile import TemporaryDirectory
 from pprint import pformat
 from pathlib import Path
@@ -109,6 +108,7 @@ def set_config(self):
  ]
  self.betka_config["generator_url"] = self.config_json["generator_url"]
  betka_url_base = self.config_json["betka_url_base"]
+ self.info(betka_url_base)
  if getenv("DEPLOYMENT") == "prod":
  self.betka_config["betka_yaml_url"] = f"{betka_url_base}betka-prod.yaml"
  else:
@@ -175,7 +175,7 @@ def get_betka_yaml_config(self):
  It is downloaded during betka start
  :return: dict
  """
- result = get(self.betka_config["betka_yaml_url"])
+ result = requests.get(self.betka_config["betka_yaml_url"], verify=False)
  result.raise_for_status()
  if result.status_code == 200:
  return yaml.safe_load(result.text)
@@ -256,6 +256,16 @@ def sync_upstream_to_downstream_directory(self) -> bool:
  if not ups_path
  else self.upstream_synced_dir / ups_path
  )
+ if not src_parent.exists():
+ self.error(f"Upstream path {ups_path} for {self.image} does not exist. "
+ f"Check betka configuration file for version validity.")
+ BetkaEmails.send_email(
+ text=f"Upstream path {ups_path} for {self.image} does not exist. "
+ f"Check betka configuration file for version validity.",
+ receivers=["[email protected]"],
+ subject=f"[betka-run-sync] Upstream path {ups_path} for {self.image} does not exist.",
+ )
+ return True
  copy_upstream2downstream(src_parent, self.downstream_dir)
  return True
 
@@ -270,7 +280,7 @@ def slack_notification(self):
  self.info("No slack webhook url provided, skipping slack notifications.")
  return False
  project_mr: ProjectMR = self.betka_schema["merge_request_dict"]
- message = f"<{project_mr.web_url}|{self.image} MR#{project_mr.iid}>: *{project_mr.title}*"
+ message = f"Sync <{project_mr.web_url}|{self.image} MR#{project_mr.iid}>: *{project_mr.title}*"
  SlackNotifications.send_webhook_notification(url=url, message=message)
  return True
 
@@ -324,9 +334,7 @@ def sync_to_downstream_branches(self, branch):
  if not self.config.get("master_checker"):
  self.info("Syncing upstream repo to downstream repo is not allowed.")
  return
- self.info(
- "Syncing upstream %r to downstream %r", self.msg_upstream_url, self.image
- )
+ self.info(f"Syncing upstream {self.msg_upstream_url} to downstream {self.image}")
  description_msg = COMMIT_MASTER_MSG.format(
  hash=self.upstream_hash, repo=self.repo
  )
@@ -443,9 +451,9 @@ def prepare(self):
  :return: bool, True - success, False - some problem occurred
  """
  self.config_json = FileUtils.load_config_json()
+ self.set_config()
  self.readme_url = self.config_json["readme_url"]
  self.refresh_betka_yaml()
- self.set_config()
  if not self.betka_config.get("dist_git_repos"):
  self.error(
  f"Global configuration file {self.betka_config['betka_yaml_url']} was not parsed properly"
@@ -544,13 +552,14 @@ def _copy_cloned_downstream_dir(self):
 
  def _get_bot_cfg(self, branch: str) -> bool:
  Git.call_git_cmd(f"checkout {branch}", msg="Change downstream branch")
+ self.debug(f"Config before getting bot-cfg.yaml {self.config}")
  try:
  self.config = self.gitlab_api.get_bot_cfg_yaml(branch=branch)
  self.debug(f"Downstream 'bot-cfg.yml' file {self.config}.")
  except jsonschema.exceptions.ValidationError as jeverror:
  self.error(
  f"Getting bot.cfg {branch} from "
- f"{self.config_json['namespace_containers']}/{self.image} "
+ f"{self.config_json['gitlab_namespace']}/{self.image} "
  f"failed. {jeverror.message}"
  )
  raise

betka/gitlab.py

@@ -174,42 +174,21 @@ def get_project_mergerequests(self) -> List[ProjectMRs]:
  for x in project_mr
  ]
 
- def get_project(self, name, group_id):
- """
- Return a Gitlab Project object for the given project name and group id.
- """
- group = self.gitlab_api.groups.get(group_id)
- projects = group.projects.list() # search=name)
- for project in projects:
- print(project)
- print(self.gitlab_api.projects.get(project.id))
-
- # if project.name == name:
- # # return the Project object, not GroupProject
- # print(self.gitlab_api.projects.get(project.id))
- # return self.gitlab_api.projects.get(project.id)
- return None
-
  def create_project_fork(self) -> ProjectFork:
  logger.debug(f"Create fork for project {self.image_config['project_id']}")
- print(
- f"Create fork for project {self.image_config['project_id']} as {self.current_user.username}/{self.image}"
- )
  assert self.target_project
  fork_data = {
  "namespace_path": f"{self.current_user.username}",
  "path": self.image,
  }
- print(fork_data)
  project_mr = self.target_project.forks.create(fork_data)
- print(project_mr)
  return ProjectFork(
  project_mr.id,
  project_mr.name,
  project_mr.ssh_url_to_repo,
  project_mr.owner["username"],
- project_mr.web_url,
  project_mr.forked_from_project["id"],
+ project_mr.web_url,
  )
 
  def load_forked_project(self):
@@ -241,7 +220,7 @@ def fork_project(self) -> Any:
  fork = self.create_project_fork()
  logger.debug(f"Fork result {fork}")
  except gitlab.exceptions.GitlabCreateError as gce:
- print(gce)
+ logger.debug(gce)
  if gce.response_code == 409:
  if (
  "namespace" in gce.error_message
@@ -255,21 +234,19 @@ def fork_project(self) -> Any:
  return fork
  logger.error(f"{gce.error_message} and {gce.response_code}")
  return None
- print(f"{fork.forked_from_id} and {self.image_config['project_id']}")
+ logger.debug(f"{fork.forked_from_id} and {self.image_config['project_id']}")
  if fork.forked_from_id != self.image_config["project_id"]:
  logger.debug("Fork project_id is different")
  return None
 
  self.fork_id = fork.id
  self.clone_url = fork.ssh_url_to_repo
  try:
- print("self.load_forked_project()")
  self.load_forked_project()
  except BetkaException:
  logger.error(f"Betka detected problem with fork for project {project_id}.")
  return None
  protected_branches = self.get_protected_branches()
- print(protected_branches)
  logger.debug(f"Protected branches are {protected_branches}")
  for brn in protected_branches:
  self.source_project.protectedbranches.delete(brn.name)
@@ -302,8 +279,13 @@ def create_project_mergerequest(self, data) -> ProjectMR:
  )
  except gitlab.exceptions.GitlabCreateError as gce:
  logger.error(f"{gce.error_message} and {gce.response_code}")
+ BetkaEmails.send_email(
+ text=f"GitLab create project merge request for project {self.image} with data {data}",
+ receivers=["[email protected]"],
+ subject="[betka-create-mergerequest] Gitlab Another MR mergerequest already exists.",
+ )
  if gce.response_code == 409:
- logger.error("Another PR already exists")
+ logger.error("Another MR already exists")
  return None
 
  def file_merge_request(
@@ -334,23 +316,25 @@ def file_merge_request(
  mr: ProjectMR = self.create_gitlab_merge_request(
  title=title, desc_msg=pr_msg, branch=branch
  )
- print(mr)
+ logger.debug(f"MergeRequest is: {mr}")
  if mr is None:
  logger.error("Merge request was not created. See logs.")
  BetkaEmails.send_email(
- text=f"Merge request for {self.image} to branch failed. see logs for reason.",
+ text=f"Merge request for {self.image} to branch {branch} failed. See logs on OpenShift for reason.",
  receivers=["[email protected]"],
  subject="[betka-run] Merge request creation failed.",
  )
  return betka_schema
  betka_schema["status"] = "created"
  mr_id = int(mr.iid)
  betka_schema["merge_request_dict"] = mr
+ betka_schema["image"] = self.image
 
  else:
  # Update pull request against the latest upstream master branch
  logger.debug(f"Sync from upstream to downstream PR={mr_id} found.")
  betka_schema["status"] = "updated"
+ betka_schema["image"] = self.image
 
  upstream_url = ""
  image_config = nested_get(self.betka_config, "dist_git_repos", self.image)
@@ -361,7 +345,7 @@ def file_merge_request(
  betka_schema["gitlab"] = self.config_json["gitlab_host_url"]
  betka_schema["commit"] = upstream_hash
  betka_schema["mr_number"] = mr_id
- betka_schema["namespace_containers"] = self.config_json["namespace_containers"]
+ betka_schema["namespace_containers"] = self.config_json["gitlab_namespace"]
  return betka_schema
 
  def init_projects(self) -> bool:
@@ -416,19 +400,19 @@ def check_gitlab_merge_requests(self, branch: str):
  f"check_gitlab_merge_requests: "
  f"This Merge Request is not valid for project {int(project_id)}"
  )
- print("Project_id different")
+ logger.debug("Project_id different")
  continue
  if mr.target_branch != branch:
  logger.debug(
  "check_gitlab_merge_requests: Target branch does not equal."
  )
- print("target_branch is different")
+ logger.debug("target_branch is different")
  continue
  if not mr.title.startswith(title):
  logger.debug(
  "check_gitlab_merge_requests: This Merge request was not filed by betka"
  )
- print(f"Title is {mr.title}")
+ logger.debug(f"Title is {mr.title}")
  continue
  logger.debug(
  f"check_gitlab_merge_requests: Downstream pull request {title} found {mr.iid}"
@@ -471,15 +455,16 @@ def get_gitlab_fork(self) -> Any:
  self.clone_url = fork.ssh_url_to_repo
  self.upstream_clone_url = fork.forked_ssh_url_to_repo
  logger.debug(f"Project fork found: {fork}")
+ self.fork_id = fork.id
+ self.load_forked_project()
  return fork
  return None
 
  # URL address is: https://gitlab.com/redhat/rhel/containers/nodejs-10/-/raw/rhel-8.6.0/bot-cfg.yml
  def cfg_url(self, branch, file="bot-cfg.yml"):
  return (
- f"{self.config_json['gitlab_host_url']}/"
- f"{self.config_json['gitlab_namespace']}/"
- f"{self.image}/-/raw/{branch}/{file}"
+ f"{self.config_json['dist_git_url']}/"
+ f"{self.image}/plain/{file}?h={branch}"
  )
 
  def get_bot_cfg_yaml(self, branch: str) -> Dict:

betka/utils.py

@@ -158,6 +158,7 @@ def list_dir_content(dir_name: Path):
  def load_config_json():
  with open(f"{HOME}/config.json") as config_file:
  data = json.load(config_file)
+ logger.info(data)
  return data
 
 
@@ -206,7 +207,7 @@ def send_webhook_notification(url: str, message: str):
  "type": "section",
  "text": {
  "type": "mrkdwn",
- "text": f"<upstream2downstream bot>: Upstream -> Downstream sync: {message}",
+ "text": f"<upstream2downstream bot>: {message}",
  },
  }
  ],

config.json

@@ -15,7 +15,8 @@
  "gitlab_fork_project": "projects/{id}/fork",
  "gitlab_access_request": "projects/{id}/access_requests",
  "gitlab_create_merge_request": "projects/{id}/merge_requests",
- "gitlab_host_url": "https://gitlab.com/",
+ "gitlab_host_url": "https://gitlab.com",
  "gitlab_namespace": "redhat/rhel/containers",
+ "dist_git_url": "https://src.fedoraproject.org/container",
  "slack_webhook_url": "SLACK_WEBHOOK_URL"
 }

files/bin/run.sh

@@ -34,7 +34,7 @@ fi
 prepare_ssh_keys
 
 # This suppresses adding authentication keys in ${HOME}/.ssh/known_host file
-echo -e "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile=/dev/null\n" >>" ${HOME}/ssh_config"
+echo -e "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile=/dev/null\n" >> "${HOME}/ssh_config"
 # For now, add both gitlab instances into known_host
 # TODO Fix this so known_hosts are not used at all.
 ssh-keyscan gitlab.com >> "${HOME}/.ssh/known_hosts"

files/home/templates/email_template

@@ -2,10 +2,10 @@ Hello!
 
 betka detected a new commit {{ template_data.commit }} in the upstream master
 branch {{ template_data.upstream_repo }} and synced the commit to the downstream dist-git repository
-as a pull request.
+as a merge request.
 
-Betka {{ template_data.status }} pull request:
-{{ template_data.gitlab }}/{{ template_data.namespace }}/{{ template_data.downstream_repo }}/-/merge_requests/{{ template_data.mr_number }}
+Betka {{ template_data.status }} merge request:
+{{ template_data.gitlab }}/{{ template_data.namespace }}/{{ template_data.image }}/-/merge_requests/{{ template_data.mr_number }}
 
 If you don't want to receive these notifications, please, write to [email protected].