feat: logging added to all files of github_api_helpers

dags/github_api_helpers/comments.py

@@ -1,3 +1,4 @@
+import logging
 from .smart_proxy import get
 
 
@@ -44,6 +45,7 @@ def fetch_repo_review_comments_page(
         )
     )
 
+    logging.info(f"Found {len(updated_response_data)} comments for {owner}/{repo} on page {page}. comments: {updated_response_data}")
     return updated_response_data
 
 
@@ -55,10 +57,12 @@ def get_all_repo_review_comments(owner: str, repo: str):
     :param repo: The name of the repository.
     :return: A list of all review comments for the specified repository.
     """
+    logging.info(f"Fetching all comments for {owner}/{repo}...")
     all_comments = []
     current_page = 1
 
     while True:
+        logging.info(f"Fetching page {current_page} of comments...")
         comments = fetch_repo_review_comments_page(owner, repo, current_page)
 
         if not comments:
@@ -67,6 +71,7 @@ def get_all_repo_review_comments(owner: str, repo: str):
         all_comments.extend(comments)
         current_page += 1
 
+    logging.info(f"Found a total of {len(all_comments)} comments for {owner}/{repo}.")
     return all_comments
 
 
@@ -112,6 +117,7 @@ def fetch_repo_issues_and_prs_comments_page(
         map(lambda x: {**x, **extract_type_from_comment_response(x)}, response_data)
     )
 
+    logging.info(f"Found {len(updated_response_data)} comments for {owner}/{repo} on page {page}. comments: {updated_response_data}")
     return updated_response_data
 
 
@@ -123,10 +129,12 @@ def get_all_repo_issues_and_prs_comments(owner: str, repo: str):
     :param repo: The name of the repository.
     :return: A list of all review comments for the specified repository.
     """
+    logging.info(f"Fetching all comments for {owner}/{repo}...")
     all_comments = []
     current_page = 1
 
     while True:
+        logging.info(f"Fetching page {current_page} of comments...")
         comments = fetch_repo_issues_and_prs_comments_page(owner, repo, current_page)
 
         if not comments:
@@ -135,4 +143,5 @@ def get_all_repo_issues_and_prs_comments(owner: str, repo: str):
         all_comments.extend(comments)
         current_page += 1
 
+    logging.info(f"Found a total of {len(all_comments)} comments for {owner}/{repo}.")
     return all_comments

dags/github_api_helpers/commits.py

@@ -1,4 +1,4 @@
-import requests
+import logging
 from .smart_proxy import get
 
 
@@ -18,6 +18,7 @@ def fetch_commits(owner: str, repo: str, page: int, per_page: int = 100):
     response = get(endpoint, params=params)
     response_data = response.json()
 
+    logging.info(f"Found {len(response_data)} commits for {owner}/{repo} on page {page}. Commits: {response_data}")
     return response_data
 
 
@@ -29,10 +30,12 @@ def get_all_commits(owner: str, repo: str):
     :param repo: The name of the repository.
     :return: A list of all commits for the specified repo.
     """
+    logging.info(f"Fetching all commits for {owner}/{repo}...")
     all_commits = []
     current_page = 1
 
     while True:
+        logging.info(f"Fetching page {current_page} of commits...")
         commits = fetch_commits(owner, repo, current_page)
 
         if not commits:
@@ -41,6 +44,7 @@ def get_all_commits(owner: str, repo: str):
         all_commits.extend(commits)
         current_page += 1
 
+    logging.info(f"Found a total of {len(all_commits)} commits for {owner}/{repo}.")
     return all_commits
 
 
@@ -55,7 +59,10 @@ def fetch_commit_details(owner: str, repo: str, commit_sha: str):
     """
     endpoint = f"https://api.github.com/repos/{owner}/{repo}/commits/{commit_sha}"
     response = get(endpoint)
-    return response.json()
+    response_data = response.json()
+
+    logging.info(f"Found details for commit {commit_sha} of {owner}/{repo}: {response_data}")
+    return response_data
 
 
 def fetch_commit_files(owner: str, repo: str, sha: str):
@@ -67,8 +74,11 @@ def fetch_commit_files(owner: str, repo: str, sha: str):
     :param sha: The SHA identifier of the commit.
     :return: A list of files changed in the specified commit.
     """
+    logging.info(f"Fetching files changed in commit {sha} of {owner}/{repo}...")
     commit_details = fetch_commit_details(owner, repo, sha)
     if "files" in commit_details:
+        logging.info(f"Found {len(commit_details['files'])} files changed in commit {sha} of {owner}/{repo}.")
         return commit_details["files"]
     else:
+        logging.info(f"No files changed in commit {sha} of {owner}/{repo}.")
         return []

dags/github_api_helpers/issues.py

@@ -1,4 +1,4 @@
-import requests
+import logging
 from .smart_proxy import get
 
 
@@ -27,6 +27,7 @@ def fetch_issues(owner: str, repo: str, page: int, per_page: int = 100):
     issues = [issue for issue in response_data if "pull_request" not in issue]
     is_more_issues = len(response_data) == per_page
 
+    logging.info(f"Found {len(issues)} issues for {owner}/{repo} on page {page}. Issues: {issues}")
     return issues, is_more_issues
 
 
@@ -38,10 +39,12 @@ def get_all_issues(owner: str, repo: str):
     :param repo: The name of the repository.
     :return: A list of all issues for the specified repo.
     """
+    logging.info(f"Fetching all issues for {owner}/{repo}...")
     all_issues = []
     current_page = 1
 
     while True:
+        logging.info(f"Fetching page {current_page} of issues...")
         issues, is_more_issues = fetch_issues(owner, repo, current_page)
         all_issues.extend(issues)
 
@@ -50,6 +53,7 @@ def get_all_issues(owner: str, repo: str):
 
         current_page += 1
 
+    logging.info(f"Found a total of {len(all_issues)} issues for {owner}/{repo}.")
     return all_issues
 
 
@@ -72,7 +76,10 @@ def fetch_issue_comments(
     )
     params = {"page": page, "per_page": per_page}
     response = get(endpoint, params=params)
-    return response.json()
+    response_data = response.json()
+
+    logging.info(f"Found {len(response_data)} comments for issue {issue_number} on page {page}. Comments: {response_data}")
+    return response_data
 
 
 def get_all_comments_of_issue(owner: str, repo: str, issue_number: int):
@@ -84,12 +91,16 @@ def get_all_comments_of_issue(owner: str, repo: str, issue_number: int):
     :param issue_number: The number of the issue.
     :return: A list of all comments for the specified issue.
     """
+    logging.info(f"Fetching all comments for issue {issue_number}...")
     all_comments = []
     current_page = 1
     while True:
+        logging.info(f"Fetching page {current_page} of comments...")
         comments = fetch_pull_request_comments(owner, repo, issue_number, current_page)
         if not comments:  # Break the loop if no more comments are found
             break
         all_comments.extend(comments)
         current_page += 1
+
+    logging.info(f"Found a total of {len(all_comments)} comments for issue {issue_number}.")
     return all_comments

dags/github_api_helpers/labels.py

@@ -1,5 +1,5 @@
 from .smart_proxy import get
-
+import logging
 
 def fetch_repo_labels_page(owner: str, repo: str, page: int, per_page: int = 100):
     """
@@ -17,6 +17,7 @@ def fetch_repo_labels_page(owner: str, repo: str, page: int, per_page: int = 100
     response = get(endpoint, params=params)
     response_data = response.json()
 
+    logging.info(f"Found {len(response_data)} labels for {owner}/{repo} on page {page}. Labels: {response_data}")
     return response_data
 
 
@@ -28,17 +29,19 @@ def get_all_repo_labels(owner: str, repo: str):
     :param repo: The name of the repository.
     :return: A list of labels for the specified repository.
     """
+    logging.info(f"Fetching all labels for {owner}/{repo}...")
     all_labels = []
     current_page = 1
 
     while True:
+        logging.info(f"Fetching page {current_page} of labels...")
         labels = fetch_repo_labels_page(owner, repo, current_page)
 
         if not labels:
             break  # No more labels to fetch
 
-        print("-> ", labels)
         all_labels.extend(labels)
         current_page += 1
 
+    logging.info(f"Found a total of {len(all_labels)} labels for {owner}/{repo}.")
     return all_labels

dags/github_api_helpers/orgs.py

@@ -1,4 +1,4 @@
-import requests
+import logging
 from .smart_proxy import get
 
 
@@ -9,11 +9,13 @@ def fetch_org_details(org_name: str):
     :param org_name: The name of the organization.
     :return: A dict containing the details of the specified organization.
     """
+    logging.info(f"Fetching details for organization {org_name}...")
     endpoint = f"https://api.github.com/orgs/{org_name}"
 
     response = get(endpoint)
     response_data = response.json()
 
+    logging.info(f"Found details for organization {org_name}: {response_data}")
     return response_data
 
 
@@ -32,6 +34,7 @@ def fetch_org_members_page(org: str, page: int, per_page: int = 100):
     response = get(endpoint, params=params)
     response_data = response.json()
 
+    logging.info(f"Found {len(response_data)} members for organization {org} on page {page}. Members: {response_data}")
     return response_data
 
 
@@ -42,10 +45,12 @@ def get_all_org_members(org: str):
     :param org: The name of the organization.
     :return: A list of members of the organization.
     """
+    logging.info(f"Fetching all members for organization {org}...")
     all_members = []
     current_page = 1
 
     while True:
+        logging.info(f"Fetching page {current_page} of members...")
         members = fetch_org_members_page(org, current_page)
 
         if not members:
@@ -54,4 +59,5 @@ def get_all_org_members(org: str):
         all_members.extend(members)
         current_page += 1
 
+    logging.info(f"Found a total of {len(all_members)} members for organization {org}.")
     return all_members