From c15d7cae438b5e81700ff0d19d48eeb6fac6cd1e Mon Sep 17 00:00:00 2001 From: Mohammad Twin Date: Fri, 15 Dec 2023 12:44:19 +0400 Subject: [PATCH] feat: more lint formate --- Readme.md | 2 - dags/github.py | 8 +- dags/github_api_helpers/smart_proxy.py | 1 + dags/github_old_version.py | 177 ------------------------- dags/neo4j_storage/commits.py | 2 +- dags/neo4j_storage/pull_requests.py | 5 +- 6 files changed, 10 insertions(+), 185 deletions(-) delete mode 100644 dags/github_old_version.py diff --git a/Readme.md b/Readme.md index 00b1be0b..c5ae4ffe 100644 --- a/Readme.md +++ b/Readme.md @@ -2,7 +2,6 @@ ## Running the app - You can quickly launch the application using `Docker Compose`: ```bash @@ -10,7 +9,6 @@ docker-compose --profile flower up ``` ## Lint the code - The code can be linted by using the below command ```bash diff --git a/dags/github.py b/dags/github.py index fea2a9ae..e78a427f 100644 --- a/dags/github.py +++ b/dags/github.py @@ -66,18 +66,20 @@ def get_all_organization(): orgs = get_orgs_profile_from_neo4j() return orgs - #! for testing + # !for testing # toghether_crew_org = { # "id": 1, # "name": "TogetherCrew", - # "description": "TogetherCrew is a community of developers, designers, and creators who are passionate about building and learning together.", + # "description": """TogetherCrew is a community of developers, designers, and creators + # who are passionate about building and learning together.""", # "url": "", # "key": "" # } # rndao_org = { # "id": 2, # "name": "RnDAO", - # "description": "RnDAO is a community of developers, designers, and creators who are passionate about building and learning together.", + # "description": """RnDAO is a community of developers, designers, and creators + # who are passionate about building and learning together.""", # "url": "", # "key": "" # } diff --git a/dags/github_api_helpers/smart_proxy.py b/dags/github_api_helpers/smart_proxy.py index f34001a9..c7531e17 100644 --- a/dags/github_api_helpers/smart_proxy.py +++ b/dags/github_api_helpers/smart_proxy.py @@ -2,6 +2,7 @@ import requests + def get(url: str, params=None): """ Sends a GET request With Smart Proxy. diff --git a/dags/github_old_version.py b/dags/github_old_version.py deleted file mode 100644 index 3d233df2..00000000 --- a/dags/github_old_version.py +++ /dev/null @@ -1,177 +0,0 @@ -from datetime import datetime, timedelta - -import requests -from airflow import DAG -from airflow.operators.python_operator import PythonOperator - -default_args = { - "owner": "MohammadTwin", - "start_date": datetime(2023, 11, 8), - "retries": 1, - "retry_delay": timedelta(minutes=1), -} - -dag = DAG( - "github_old_version", - default_args=default_args, - description="GitHub Data Extraction DAG", - schedule_interval=None, - catchup=False, -) - - -def get_github_repos(ti): - endpoint = "https://api.github.com/orgs/TogetherCrew/repos" - - response = requests.get(endpoint) - response_data = response.json() - - print("[response_data] ", response_data) - ti.xcom_push(key="github_repos", value=response_data) - - -get_repos_task = PythonOperator( - task_id="get_github_repos", - python_callable=get_github_repos, - provide_context=True, - dag=dag, -) - - -def get_pull_requests(owner: str, repo: str): - endpoint = f"https://api.github.com/repos/{owner}/{repo}/pulls" - - params = {"per_page": 100, "page": 1, "state": "all"} - response = requests.get(endpoint, params=params) - response_data = response.json() - - return response_data - - -def extract_pull_requests(ti): - prs_data = {} - - github_repos = ti.xcom_pull(key="github_repos", task_ids="get_github_repos") - for repo in github_repos: - prs = get_pull_requests(owner=repo["owner"]["login"], repo=repo["name"]) - prs_data[repo["id"]] = prs - - ti.xcom_push(key="github_prs", value=github_repos) - return prs_data - - -def transform_pull_requests(ti): - return None - - -def load_pull_requests(ti): - print("Loaded PR data into the destination:") - - -task_extract_pull_requests = PythonOperator( - task_id="extract_pull_requests", - python_callable=extract_pull_requests, - provide_context=True, - dag=dag, -) - -task_transform_pull_requests = PythonOperator( - task_id="transform_pull_requests", - python_callable=transform_pull_requests, - provide_context=True, - dag=dag, -) - -task_load_pull_requests = PythonOperator( - task_id="load_pull_requests", - python_callable=load_pull_requests, - provide_context=True, - dag=dag, -) - -( - get_repos_task - >> task_extract_pull_requests - >> task_transform_pull_requests - >> task_load_pull_requests -) - - -def extract_commits(ti): - github_repos = ti.xcom_pull(key="github_repos", task_ids="get_github_repos") - for repo in github_repos: - print("\n[repo] ", repo) - - return None - - -def transform_commits(ti): - return None - - -def load_commits(ti): - print("Loaded Commit data into the destination:") - - -task_extract_commits = PythonOperator( - task_id="extract_commits", - python_callable=extract_commits, - provide_context=True, - dag=dag, -) - -task_transform_commits = PythonOperator( - task_id="transform_commits", - python_callable=transform_commits, - provide_context=True, - dag=dag, -) - -task_load_commits = PythonOperator( - task_id="load_commits", - python_callable=load_commits, - provide_context=True, - dag=dag, -) - -get_repos_task >> task_extract_commits >> task_transform_commits >> task_load_commits - - -def extract_issues(ti): - github_repos = ti.xcom_pull(key="github_repos", task_ids="get_github_repos") - for repo in github_repos: - print("\n[repo] ", repo) - - return None - - -def transform_issues(ti): - return None - - -def load_issues(ti): - print("Loaded issues data into the destination:") - - -task_extract_issues = PythonOperator( - task_id="extract_issues", - python_callable=extract_issues, - provide_context=True, - dag=dag, -) - -task_transform_issues = PythonOperator( - task_id="transform_issues", - python_callable=transform_issues, - provide_context=True, - dag=dag, -) - -task_load_issues = PythonOperator( - task_id="load_issues", - python_callable=load_issues, - provide_context=True, - dag=dag, -) - -get_repos_task >> task_extract_issues >> task_transform_issues >> task_load_issues diff --git a/dags/neo4j_storage/commits.py b/dags/neo4j_storage/commits.py index 5f5e4486..354c4fc3 100644 --- a/dags/neo4j_storage/commits.py +++ b/dags/neo4j_storage/commits.py @@ -50,7 +50,7 @@ def save_commit_files_changes_to_neo4j( session.execute_write( lambda tx: tx.run( f""" - MATCH (repo:{Node.Repository.value} {{id: $repository_id}}), + MATCH (repo:{Node.Repository.value} {{id: $repository_id}}), (c:{Node.Commit.value} {{sha: $commit_sha}}) WITH repo, c UNWIND $file_changes AS file_change diff --git a/dags/neo4j_storage/pull_requests.py b/dags/neo4j_storage/pull_requests.py index 58db5200..16f13f8c 100644 --- a/dags/neo4j_storage/pull_requests.py +++ b/dags/neo4j_storage/pull_requests.py @@ -62,7 +62,7 @@ def save_pull_request_to_neo4j(pr: dict, repository_id: str): f""" MERGE (pr:{Node.PullRequest.value} {{id: $pr.id}}) SET pr += $pr, pr.repository_id = $repository_id, pr.latestSavedAt = datetime() - + WITH pr MERGE (ghu:{Node.GitHubUser.value} {{id: $repo_creator.id}}) SET ghu += $repo_creator, ghu.latestSavedAt = datetime() @@ -129,7 +129,8 @@ def save_pr_files_changes_to_neo4j(pr_id: int, repository_id: str, file_changes: session.execute_write( lambda tx: tx.run( f""" - MATCH (repo:{Node.Repository.value} {{id: $repository_id}}), (pr:{Node.PullRequest.value} {{id: $pr_id}}) + MATCH (repo:{Node.Repository.value} {{id: $repository_id}}), + (pr:{Node.PullRequest.value} {{id: $pr_id}}) WITH repo, pr UNWIND $file_changes AS file_change MERGE (f:{Node.File.value} {{sha: file_change.sha, filename: file_change.filename}})