Add ci-scripts (#12)

* Add ci-scripts

* add workflow file

* Update jira_script.yml

* minor updates

* Update jira_script.yml

* Update jira_script.yml

* updated script regex

* Update jira_script.yml

* address comments

* address review comments

.github/workflows/jira_script.yml

@@ -0,0 +1,29 @@
+name: Jira validation
+
+on:
+  push:
+
+
+jobs:
+  verify-jira:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Add Config File to current folder
+        run: |
+          echo "$CONFIG_CONTENT" > jira.cfg
+        env:
+          CONFIG_CONTENT: ${{ secrets.JIRA_CFG }}
+
+      - name: Install Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.8'
+
+      - name: Install tox
+        run: pip install tox
+
+      - name: Run Tox with Environment Variables
+        run: tox -e verify-bugs-are-open

ci_scripts/jira_scripts/check_jira_status.py

@@ -0,0 +1,145 @@
+import re
+
+from jira import JIRA, JIRAError
+
+from ci_scripts.utils import get_all_python_files, get_connection_params, print_status
+
+
+# Needs to be update based on the branch.
+EXPECTED_TARGET_VERSIONS = ["vfuture", "4.16", "4.15.1"]
+
+
+def get_jira_connection():
+    connection_params = get_connection_params()
+    jira_connection = JIRA(
+        token_auth=connection_params.get("token"),
+        options={"server": connection_params.get("url")},
+    )
+    return jira_connection
+
+
+def get_jira_metadata(jira_id, jira_connection):
+    retries = 0
+    max_retry = 3
+    while retries < max_retry:
+        try:
+            return jira_connection.issue(
+                id=jira_id, fields="status, issuetype, fixVersions"
+            ).fields
+        except JIRAError as jira_exception:
+            # Check for inactivity error (adjust based on your library)
+            if "Unauthorized" in str(jira_exception) or "Session timed out" in str(
+                jira_exception
+            ):
+                retries += 1
+                print(
+                    f"Failed to get issue due to inactivity, retrying ({retries}/{max_retry})"
+                )
+                if retries < max_retry:
+                    jira_connection = get_jira_connection()  # Attempt reconnection
+                else:
+                    raise  # Re-raise the error after exceeding retries
+            else:
+                raise
+
+
+def get_jira_fix_version(jira_metadata):
+    fix_version = (
+        re.search(r"([\d.]+)", jira_metadata.fixVersions[0].name)
+        if jira_metadata.fixVersions
+        else None
+    )
+    return fix_version.group(1) if fix_version else "vfuture"
+
+
+def get_all_jiras_from_file(file_content):
+    """
+    Try to find all jira tickets in the file.
+    Looking for the following patterns:
+    - jira_id=<id>>  # call in is_jira_open
+    - jira_id = <id>  # when jira is constant
+    - https://issues.redhat.com/browse/<id>  # when jira is in a link in comments
+    - pytest.mark.jira(id)  # when jira is in a marker
+
+    Args:
+        file_content (str): The content of the file.
+
+    Returns:
+        list: A list of jira tickets.
+    """
+    issue_pattern = r"([A-Z]+-[0-9]+)"
+    _pytest_jira_marker_bugs = re.findall(
+        rf"pytest.mark.jira.*?{issue_pattern}.*", file_content, re.DOTALL
+    )
+    _is_jira_open = re.findall(rf"jira_id\s*=[\s*\"\']*{issue_pattern}.*", file_content)
+    _jira_url_jiras = re.findall(
+        rf"https://issues.redhat.com/browse/{issue_pattern}.*",
+        file_content,
+    )
+    return set(_pytest_jira_marker_bugs + _is_jira_open + _jira_url_jiras)
+
+
+def get_jiras_from_all_python_files():
+    jira_found = {}
+    for filename in get_all_python_files():
+        filename_for_key = re.findall(r"openshift-virtualization-tests/.*", filename)[0]
+        with open(filename) as fd:
+            if unique_jiras := get_all_jiras_from_file(file_content=fd.read()):
+                jira_found[filename_for_key] = unique_jiras
+    return jira_found
+
+
+def main():
+    closed_statuses = get_connection_params().get("resolved_statuses")
+    closed_jiras = {}
+    mismatch_bugs_version = {}
+    jira_ids_with_errors = {}
+    jira_ids_dict = get_jiras_from_all_python_files()
+    jira_connection = get_jira_connection()
+    for filename in jira_ids_dict:
+        for jira_id in jira_ids_dict[filename]:
+            try:
+                jira_metadata = get_jira_metadata(
+                    jira_id=jira_id, jira_connection=jira_connection
+                )
+                current_jira_status = jira_metadata.status.name.lower()
+                if current_jira_status in closed_statuses:
+                    closed_jiras.setdefault(filename, []).append(
+                        f"{jira_id} [{current_jira_status}]"
+                    )
+                jira_target_release_version = get_jira_fix_version(
+                    jira_metadata=jira_metadata
+                )
+                if not jira_target_release_version.startswith(
+                    tuple(EXPECTED_TARGET_VERSIONS)
+                ):
+                    mismatch_bugs_version.setdefault(filename, []).append(
+                        f"{jira_id} [{jira_target_release_version}]"
+                    )
+            except JIRAError as exp:
+                jira_ids_with_errors.setdefault(filename, []).append(
+                    f"{jira_id} [{exp.text}]"
+                )
+                continue
+
+    if closed_jiras:
+        print(f"{len(closed_jiras)} Jira tickets are closed and need to be removed:")
+        print_status(status_dict=closed_jiras)
+
+    if mismatch_bugs_version:
+        print(
+            f"{len(mismatch_bugs_version)} Jira bugs are not matched to the current branch's expected version list:"
+            f" '{EXPECTED_TARGET_VERSIONS}' and need to be removed:"
+        )
+        print_status(status_dict=mismatch_bugs_version)
+
+    if jira_ids_with_errors:
+        print(f"{len(jira_ids_with_errors)} Jira ids had errors:")
+        print_status(status_dict=jira_ids_with_errors)
+
+    if closed_jiras or mismatch_bugs_version or jira_ids_with_errors:
+        exit(1)
+
+
+if __name__ == "__main__":
+    main()

ci_scripts/polarion/polarion_set_automated.py

@@ -0,0 +1,58 @@
+import logging
+
+from pylero.exceptions import PyleroLibException
+
+from ci_scripts.polarion.polarion_utils import (
+    PROJECT,
+    get_polarion_ids_from_diff,
+    git_diff_added_removed_lines,
+)
+
+
+logging.basicConfig(level=logging.INFO)
+LOGGER = logging.getLogger(__name__)
+AUTOMATED = "automated"
+NOT_AUTOMATED = "notautomated"
+APPROVED = "approved"
+
+
+def approve_tc(tc):
+    tc.status = APPROVED
+    tc.update()
+    LOGGER.info(f"{tc.work_item_id} {APPROVED}")
+
+
+def automate_and_approve_tc():
+    git_diff = git_diff_added_removed_lines()
+    added_ids, removed_ids = get_polarion_ids_from_diff(diff=git_diff)
+    if added_ids or removed_ids:
+        from pylero.work_item import TestCase
+
+    for _id in removed_ids:
+        if _id in added_ids:
+            continue
+
+        tc = TestCase(project_id=PROJECT, work_item_id=_id)
+        if tc.caseautomation == AUTOMATED:
+            LOGGER.info(f"{_id}: Mark as {AUTOMATED}, Setting '{NOT_AUTOMATED}'")
+            tc.caseautomation = NOT_AUTOMATED
+            approve_tc(tc=tc)
+
+    for _id in added_ids:
+        try:
+            tc = TestCase(project_id=PROJECT, work_item_id=_id)
+            if tc.caseautomation != AUTOMATED:
+                LOGGER.info(f"{_id}: Not mark as {AUTOMATED}, Setting '{AUTOMATED}'")
+                tc.caseautomation = AUTOMATED
+                approve_tc(tc=tc)
+
+            if tc.caseautomation == AUTOMATED and tc.status != APPROVED:
+                LOGGER.info(f"{_id} already {AUTOMATED}")
+                approve_tc(tc=tc)
+
+        except PyleroLibException as ex:
+            LOGGER.warning(f"{_id}: {ex}")
+
+
+if __name__ == "__main__":
+    automate_and_approve_tc()

ci_scripts/polarion/polarion_utils.py

@@ -0,0 +1,45 @@
+import logging
+import re
+import shlex
+import subprocess
+
+
+logging.basicConfig(level=logging.INFO)
+
+LOGGER = logging.getLogger(__name__)
+PROJECT = "CNV"
+
+
+def find_polarion_ids(data):
+    match_ids = set()
+    for item in data:
+        match = re.findall(rf"pytest.mark.polarion.*{PROJECT}-[0-9]+", item)
+        if match:
+            match_id = re.findall(rf"{PROJECT}-[0-9]+", match[0])
+            match_ids.add(match_id[0])
+
+    return match_ids
+
+
+def git_diff():
+    data = subprocess.check_output(shlex.split("git diff HEAD^-1"))
+    data = data.decode("utf-8")
+    return data
+
+
+def git_diff_added_removed_lines():
+    diff = {}
+    for line in git_diff().splitlines():
+        if line.startswith("+"):
+            diff.setdefault("added", []).append(line)
+
+        if line.startswith("-"):
+            diff.setdefault("removed", []).append(line)
+
+    return diff
+
+
+def get_polarion_ids_from_diff(diff):
+    added_ids = find_polarion_ids(data=diff.get("added", []))
+    removed_ids = find_polarion_ids(data=diff.get("removed", []))
+    return added_ids, removed_ids

ci_scripts/polarion/polarion_verify_tc_requirement.py

@@ -0,0 +1,47 @@
+import logging
+import sys
+
+from pylero.exceptions import PyleroLibException
+
+from ci_scripts.polarion.polarion_utils import (
+    PROJECT,
+    get_polarion_ids_from_diff,
+    git_diff_added_removed_lines,
+)
+
+
+logging.basicConfig(level=logging.INFO)
+LOGGER = logging.getLogger(__name__)
+
+
+def has_verify():
+    git_diff = git_diff_added_removed_lines()
+    added_ids, _ = get_polarion_ids_from_diff(diff=git_diff)
+    missing = []
+    if added_ids:
+        from pylero.work_item import Requirement, TestCase
+
+        for _id in added_ids:
+            has_req = False
+            LOGGER.info(f"Checking if {_id} verifies any requirement")
+            tc = TestCase(project_id=PROJECT, work_item_id=_id)
+            for link in tc.linked_work_items:
+                try:
+                    Requirement(project_id=PROJECT, work_item_id=link.work_item_id)
+                    has_req = True
+                    break
+                except PyleroLibException:
+                    continue
+
+            if not has_req:
+                LOGGER.error(f"{_id}: Is missing requirement")
+                missing.append(_id)
+
+        if missing:
+            missing_str = "\n".join(missing)
+            LOGGER.error(f"Cases with missing requirement: {missing_str}")
+            sys.exit(1)
+
+
+if __name__ == "__main__":
+    has_verify()

ci_scripts/utils.py

@@ -0,0 +1,32 @@
+import os
+from configparser import ConfigParser
+from pathlib import Path
+
+
+def get_connection_params():
+    conf_file = os.path.join(Path(".").resolve(), "jira.cfg")
+    parser = ConfigParser()
+    # Open the file with the correct encoding
+    parser.read(conf_file, encoding="utf-8")
+    params_dict = {}
+    for params in parser.items("DEFAULT"):
+        params_dict[params[0]] = params[1]
+    return params_dict
+
+
+def print_status(status_dict):
+    for key, value in status_dict.items():
+        print(f"    {key}:  {' '.join(list(set(value)))}")
+    print("\n")
+
+
+def get_all_python_files():
+    exclude_dirs = [".tox", "venv", ".pytest_cache", "site-packages", ".git"]
+    for root, _, files in os.walk(os.path.abspath(os.curdir)):
+        if [_dir for _dir in exclude_dirs if _dir in root]:
+            continue
+
+        for filename in files:
+            file_path = os.path.join(root, filename)
+            if filename.endswith(".py") and file_path != os.path.abspath(__file__):
+                yield file_path