Merge pull request #16 from TogetherCrew/feature/ci

feat: basic CI/CD were added

.github/workflows/start.staging.yml

@@ -0,0 +1,9 @@
+name: Staging CI/CD Pipeline
+
+on: pull_request
+
+jobs:
+  ci:
+    uses: TogetherCrew/operations/.github/workflows/ci.yml@main
+    secrets:
+      CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}

.gitignore

@@ -55,6 +55,7 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 cover/
+coverage
 
 # Translations
 *.mo

Dockerfile

@@ -0,0 +1,12 @@
+# It's recommended that we use `bullseye` for Python (alpine isn't suitable as it conflcts with numpy)
+FROM python:3.10-bullseye AS base
+WORKDIR /project
+COPY . .
+RUN pip3 install -r requirements.txt
+
+FROM base AS test
+RUN chmod +x docker-entrypoint.sh
+CMD ["./docker-entrypoint.sh"]
+
+FROM base AS prod
+CMD ["echo", "aiflow dags should be running on airlfow container"]

Readme.md

@@ -2,17 +2,16 @@
 
 
 ## Running the app
-
 You can quickly launch the application using `Docker Compose`:
 
 ```bash
 docker-compose --profile flower up
 ```
 
-## Lint the code 
 
+## Lint the code
 The code can be linted by using the below command
 
 ```bash
 python -m black .
-```
+```

dags/github.py

@@ -17,42 +17,41 @@
 # under the License.
 """Example DAG demonstrating the usage of dynamic task mapping."""
 from __future__ import annotations
+
 from datetime import datetime, timedelta
 
 from airflow import DAG
 from airflow.decorators import task
-
 from github_api_helpers import (
-    get_all_repo_issues_and_prs_comments,
-    get_all_reviews_of_pull_request,
-    get_all_repo_review_comments,
-    get_all_repo_contributors,
-    get_all_pull_request_files,
-    get_all_pull_requests,
-    get_all_org_members,
-    get_all_repo_labels,
-    get_all_org_repos,
-    get_all_org_repos,
-    fetch_org_details,
     fetch_commit_files,
+    fetch_org_details,
     get_all_commits,
     get_all_issues,
+    get_all_org_members,
+    get_all_org_repos,
+    get_all_pull_request_files,
+    get_all_pull_requests,
+    get_all_repo_contributors,
+    get_all_repo_issues_and_prs_comments,
+    get_all_repo_labels,
+    get_all_repo_review_comments,
+    get_all_reviews_of_pull_request,
 )
 from neo4j_storage import (
-    save_commit_files_changes_to_neo4j,
-    save_repo_contributors_to_neo4j,
-    save_pr_files_changes_to_neo4j,
-    save_review_comment_to_neo4j,
     get_orgs_profile_from_neo4j,
-    save_pull_request_to_neo4j,
-    save_org_member_to_neo4j,
     save_comment_to_neo4j,
+    save_commit_files_changes_to_neo4j,
     save_commit_to_neo4j,
-    save_review_to_neo4j,
     save_issue_to_neo4j,
     save_label_to_neo4j,
+    save_org_member_to_neo4j,
     save_orgs_to_neo4j,
+    save_pr_files_changes_to_neo4j,
+    save_pull_request_to_neo4j,
+    save_repo_contributors_to_neo4j,
     save_repo_to_neo4j,
+    save_review_comment_to_neo4j,
+    save_review_to_neo4j,
 )
 
 with DAG(
@@ -67,18 +66,20 @@ def get_all_organization():
         orgs = get_orgs_profile_from_neo4j()
         return orgs
 
-        #! for testing
+        # !for testing
         # toghether_crew_org = {
         #     "id": 1,
         #     "name": "TogetherCrew",
-        #     "description": "TogetherCrew is a community of developers, designers, and creators who are passionate about building and learning together.",
+        #     "description": """TogetherCrew is a community of developers, designers, and creators
+        #     who are passionate about building and learning together.""",
         #     "url": "",
         #     "key": ""
         # }
         # rndao_org = {
         #     "id": 2,
         #     "name": "RnDAO",
-        #     "description": "RnDAO is a community of developers, designers, and creators who are passionate about building and learning together.",
+        #     "description": """RnDAO is a community of developers, designers, and creators
+        #     who are passionate about building and learning together.""",
         #     "url": "",
         #     "key": ""
         # }

dags/github_api_helpers/__init__.py

@@ -1,16 +1,17 @@
-from .repos import get_all_org_repos, get_all_repo_contributors
-from .commits import get_all_commits, fetch_commit_details, fetch_commit_files
-from .issues import get_all_issues, get_all_comments_of_issue
+# flake8: noqa
+from .comments import get_all_repo_issues_and_prs_comments, get_all_repo_review_comments
+from .commits import fetch_commit_details, fetch_commit_files, get_all_commits
+from .issues import get_all_comments_of_issue, get_all_issues
+from .labels import get_all_repo_labels
+from .orgs import fetch_org_details, get_all_org_members
 from .pull_requests import (
-    get_all_pull_requests,
-    get_all_commits_of_pull_request,
     get_all_comments_of_pull_request,
-    get_all_review_comments_of_pull_request,
-    get_all_reactions_of_review_comment,
+    get_all_commits_of_pull_request,
+    get_all_pull_request_files,
+    get_all_pull_requests,
     get_all_reactions_of_comment,
+    get_all_reactions_of_review_comment,
+    get_all_review_comments_of_pull_request,
     get_all_reviews_of_pull_request,
-    get_all_pull_request_files,
 )
-from .orgs import fetch_org_details, get_all_org_members
-from .labels import get_all_repo_labels
-from .comments import get_all_repo_review_comments, get_all_repo_issues_and_prs_comments
+from .repos import get_all_org_repos, get_all_repo_contributors

dags/github_api_helpers/comments.py

@@ -1,4 +1,5 @@
 import logging
+
 from .smart_proxy import get
 
 
@@ -45,7 +46,11 @@ def fetch_repo_review_comments_page(
         )
     )
 
-    logging.info(f"Found {len(updated_response_data)} comments for {owner}/{repo} on page {page}. comments: {updated_response_data}")
+    msg = f"Found {len(updated_response_data)}"
+    msg += f" review comments for {owner}/{repo} on page {page}."
+    msg += f" comments: {updated_response_data}"
+    logging.info(msg)
+
     return updated_response_data
 
 
@@ -117,7 +122,11 @@ def fetch_repo_issues_and_prs_comments_page(
         map(lambda x: {**x, **extract_type_from_comment_response(x)}, response_data)
     )
 
-    logging.info(f"Found {len(updated_response_data)} comments for {owner}/{repo} on page {page}. comments: {updated_response_data}")
+    msg = f"Found {len(updated_response_data)}"
+    msg += f" comments for {owner}/{repo} on page {page}."
+    msg += f" comments: {updated_response_data}"
+    logging.info(msg)
+
     return updated_response_data
 
 

dags/github_api_helpers/commits.py

@@ -1,4 +1,5 @@
 import logging
+
 from .smart_proxy import get
 
 
@@ -18,7 +19,9 @@ def fetch_commits(owner: str, repo: str, page: int, per_page: int = 100):
     response = get(endpoint, params=params)
     response_data = response.json()
 
-    logging.info(f"Found {len(response_data)} commits for {owner}/{repo} on page {page}. Commits: {response_data}")
+    logging.info(
+        f"Found {len(response_data)} commits for {owner}/{repo} on page {page}. Commits: {response_data}"
+    )
     return response_data
 
 
@@ -61,7 +64,9 @@ def fetch_commit_details(owner: str, repo: str, commit_sha: str):
     response = get(endpoint)
     response_data = response.json()
 
-    logging.info(f"Found details for commit {commit_sha} of {owner}/{repo}: {response_data}")
+    logging.info(
+        f"Found details for commit {commit_sha} of {owner}/{repo}: {response_data}"
+    )
     return response_data
 
 
@@ -77,7 +82,9 @@ def fetch_commit_files(owner: str, repo: str, sha: str):
     logging.info(f"Fetching files changed in commit {sha} of {owner}/{repo}...")
     commit_details = fetch_commit_details(owner, repo, sha)
     if "files" in commit_details:
-        logging.info(f"Found {len(commit_details['files'])} files changed in commit {sha} of {owner}/{repo}.")
+        logging.info(
+            f"Found {len(commit_details['files'])} files changed in commit {sha} of {owner}/{repo}."
+        )
         return commit_details["files"]
     else:
         logging.info(f"No files changed in commit {sha} of {owner}/{repo}.")

dags/github_api_helpers/issues.py

@@ -1,4 +1,5 @@
 import logging
+
 from .smart_proxy import get
 
 
@@ -27,7 +28,9 @@ def fetch_issues(owner: str, repo: str, page: int, per_page: int = 100):
     issues = [issue for issue in response_data if "pull_request" not in issue]
     is_more_issues = len(response_data) == per_page
 
-    logging.info(f"Found {len(issues)} issues for {owner}/{repo} on page {page}. Issues: {issues}")
+    logging.info(
+        f"Found {len(issues)} issues for {owner}/{repo} on page {page}. Issues: {issues}"
+    )
     return issues, is_more_issues
 
 
@@ -78,7 +81,9 @@ def fetch_issue_comments(
     response = get(endpoint, params=params)
     response_data = response.json()
 
-    logging.info(f"Found {len(response_data)} comments for issue {issue_number} on page {page}. Comments: {response_data}")
+    logging.info(
+        f"Found {len(response_data)} comments for issue {issue_number} on page {page}. Comments: {response_data}"
+    )
     return response_data
 
 
@@ -96,11 +101,13 @@ def get_all_comments_of_issue(owner: str, repo: str, issue_number: int):
     current_page = 1
     while True:
         logging.info(f"Fetching page {current_page} of comments...")
-        comments = fetch_pull_request_comments(owner, repo, issue_number, current_page)
+        comments = fetch_issue_comments(owner, repo, issue_number, current_page)
         if not comments:  # Break the loop if no more comments are found
             break
         all_comments.extend(comments)
         current_page += 1
-
-    logging.info(f"Found a total of {len(all_comments)} comments for issue {issue_number}.")
+
+    logging.info(
+        f"Found a total of {len(all_comments)} comments for issue {issue_number}."
+    )
     return all_comments

dags/github_api_helpers/labels.py

@@ -1,6 +1,8 @@
-from .smart_proxy import get
 import logging
 
+from .smart_proxy import get
+
+
 def fetch_repo_labels_page(owner: str, repo: str, page: int, per_page: int = 100):
     """
     Fetches the labels for a specific repository in GitHub.
@@ -17,7 +19,9 @@ def fetch_repo_labels_page(owner: str, repo: str, page: int, per_page: int = 100
     response = get(endpoint, params=params)
     response_data = response.json()
 
-    logging.info(f"Found {len(response_data)} labels for {owner}/{repo} on page {page}. Labels: {response_data}")
+    logging.info(
+        f"Found {len(response_data)} labels for {owner}/{repo} on page {page}. Labels: {response_data}"
+    )
     return response_data
 
 

dags/github_api_helpers/orgs.py

@@ -1,4 +1,5 @@
 import logging
+
 from .smart_proxy import get
 
 
@@ -34,7 +35,9 @@ def fetch_org_members_page(org: str, page: int, per_page: int = 100):
     response = get(endpoint, params=params)
     response_data = response.json()
 
-    logging.info(f"Found {len(response_data)} members for organization {org} on page {page}. Members: {response_data}")
+    logging.info(
+        f"Found {len(response_data)} members for organization {org} on page {page}. Members: {response_data}"
+    )
     return response_data