Circleci migration 4 (#1247)

.circleci/config.yml

@@ -15,6 +15,7 @@ parameters:
         "run_daily_metadata",
         "run_defillama_stablecoins",
         "daily_l2_aggregate_later_loads",
+        "daily_l2_chain_data_uploads",
         "deploy_github_pages",
       ]
 
@@ -32,60 +33,133 @@ commands:
           name: Install project dependencies
           command: uv sync --all-extras --dev
 
-  setup-conda:
-    steps:
-      - run:
-          name: Setup Conda
-          command: |
-            wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
-            bash Miniconda3-latest-Linux-x86_64.sh -b
-
   setup-python-3-10:
     steps:
-      - setup-conda
       - run:
           name: Setup Python 3.10
           command: |
+            if [ ! -d "~/miniconda3" ]; then
+                wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
+                bash Miniconda3-latest-Linux-x86_64.sh -b
+            fi
+            export PATH=$HOME/miniconda3/bin:$PATH
             ~/miniconda3/bin/conda create -n py310 python=3.10 -y
-            ~/miniconda3/bin/conda init bash            
-            source ~/.bashrc
+            echo "Python 3.10 environment setup complete"
 
   checkout-dep-1:
+    parameters:
+      base_git_project:
+        type: string
+        default: "."
     steps:
       - run:
           name: Clone repository and checkout branch with depth 1
-          command: git clone --depth 1 "$CIRCLE_REPOSITORY_URL" --branch "$CIRCLE_BRANCH" .
+          command: |
+            git clone --depth 1 "$CIRCLE_REPOSITORY_URL" --branch "$CIRCLE_BRANCH" << parameters.base_git_project >>
+            echo "Cloned repository folder name: << parameters.base_git_project >>"
+  setup-python-3-10-project-dependencies:
+    parameters:
+      base_git_project:
+        type: string
+        default: "."
+    steps:
+      - run:
+          name: Install Python 3.10 dependencies
+          command: |
+            cd "<< parameters.base_git_project >>"
+            source ~/miniconda3/bin/activate py310
+            python --version
+            python -m pip install pipenv
+            python -m pipenv install
 
 jobs:
-  general-l2:
+  setup-python-3-10-job:
     machine:
       image: ubuntu-2004:current
     steps:
+      - checkout-dep-1:
+          base_git_project: "op-analytics"
+      - setup-python-3-10
+      - setup-python-3-10-project-dependencies:
+          base_git_project: "op-analytics"
+      - persist_to_workspace:
+          root: ~/
+          paths:
+            - miniconda3
+            - .local
+
+  convert-and-run-notebooks-to-py:
+    description: "Convert and run notebooks to Python scripts"
+    machine:
+      image: ubuntu-2004:current
+    parameters:
+      notebook_paths:
+        type: string
+    steps:
+      - attach_workspace:
+          at: ~/
       - utils/gcp-oidc-authenticate:
           project_id: GCP_PROJECT_ID
           service_account_email: GCP_SERVICE_ACCOUNT_EMAIL
           workload_identity_pool_id: GCP_WIP_ID
           workload_identity_pool_provider_id: GCP_WIP_PROVIDER_ID
-      - checkout-dep-1
+      - checkout-dep-1:
+          base_git_project: "op-analytics"
+      - run:
+          name: Generate and run Python scripts
+          command: |
+            export PATH=~/miniconda3/bin:$PATH
+            source ~/miniconda3/bin/activate py310
+
+            cd "op-analytics"
+
+            notebook_path="<< parameters.notebook_paths >>.ipynb"
+            py_script_path="<< parameters.notebook_paths >>.py"
+            directory=$(dirname "$notebook_path")
+            cd $directory
+
+            echo "Converting $notebook_path to $py_script_path"
+            python -m pipenv run jupyter nbconvert --to python $(basename ${notebook_path})
+
+            echo "Run $py_script_path"
+            python -m pipenv run python $(basename ${py_script_path})
+
+  daily-l2-aggregate-later-loads:
+    description: "Daily L2 Chain Aggregate Later Loads"
+    machine:
+      image: ubuntu-2004:current
+    steps:
+      - utils/gcp-oidc-authenticate:
+          project_id: GCP_PROJECT_ID
+          service_account_email: GCP_SERVICE_ACCOUNT_EMAIL
+          workload_identity_pool_id: GCP_WIP_ID
+          workload_identity_pool_provider_id: GCP_WIP_PROVIDER_ID
+      - checkout-dep-1:
+          base_git_project: "op-analytics"
       - setup-python-3-10
+      - setup-python-3-10-project-dependencies:
+          base_git_project: "op-analytics"
       - run:
           name: Generate and run Python scripts
           command: |
-            conda activate py310
-            python3.10 -m pip install pipenv
-            python3.10 -m pipenv install
-            python3.10 -m pipenv run jupyter nbconvert --to python op_chains_tracking/dune_opchain_uploads.ipynb other_chains_tracking/get_filtered_deployers.ipynb other_chains_tracking/total_chain_data_to_uploads.ipynb
+            export PATH=~/miniconda3/bin:$PATH
+            source ~/miniconda3/bin/activate py310
+
+            cd "op-analytics"
+
+            python -m pipenv run jupyter nbconvert --to python op_chains_tracking/dune_opchain_uploads.ipynb other_chains_tracking/get_filtered_deployers.ipynb other_chains_tracking/total_chain_data_to_uploads.ipynb
 
             cd op_chains_tracking
-            python3.10 -m pipenv run python dune_opchain_uploads.py
+            python -m pipenv run python dune_opchain_uploads.py
 
             cd ../other_chains_tracking
-            python3.10 -m pipenv run python get_filtered_deployers.py total_chain_data_to_uploads.py
+            python -m pipenv run python get_filtered_deployers.py total_chain_data_to_uploads.py
+
       - utils/get-github-access-token
       - utils/github-commit-and-push-changes:
           commit-message: "GH Action Update - Agg Downloads"
           skip-ci: true
-          folder: "."
+          folder: "op-analytics"
 
   chain-metadata:
     machine:
@@ -200,8 +274,14 @@ workflows:
     jobs:
       - defillama-protocols:
           context: op-analytics
+          filters:
+            branches:
+              only: main
       - daily-uploads-other-tasks:
           context: op-analytics
+          filters:
+            branches:
+              only: main
   run-daily-metadata:
     when:
       or:
@@ -210,6 +290,9 @@ workflows:
     jobs:
       - chain-metadata:
           context: op-analytics
+          filters:
+            branches:
+              only: main
   run-defillama-stablecoins:
     when:
       or:
@@ -218,8 +301,11 @@ workflows:
     jobs:
       - defillama-stablecoins:
           context: op-analytics
+          filters:
+            branches:
+              only: main
 
-  run-general-l2:
+  run-daily-l2-aggregate-later-loads:
     when:
       or:
         - equal:
@@ -230,11 +316,42 @@ workflows:
               "daily_l2_aggregate_later_loads",
             ]
     jobs:
-      - general-l2:
+      - daily-l2-aggregate-later-loads:
+          context:
+            - op-analytics-legacy
+            - circleci-repo-op-analytics
+            - op-analytics-gcp
+
+  daily_l2_chain_data_uploads:
+    when:
+      or:
+        - equal: [<< pipeline.schedule.name >>, "daily_l2_chain_data_uploads"]
+        - equal:
+            [<< pipeline.parameters.run_job >>, "daily_l2_chain_data_uploads"]
+    jobs:
+      - setup-python-3-10-job
+      - convert-and-run-notebooks-to-py:
+          requires:
+            - setup-python-3-10-job
           context:
             - op-analytics-legacy
             - circleci-repo-op-analytics
             - op-analytics-gcp
+          matrix:
+            parameters:
+              notebook_paths:
+                - other_chains_tracking/chain_tvl_trends
+                - other_chains_tracking/pull_l2_activity
+                - other_chains_tracking/get_ethereum_chain_list
+                - other_chains_tracking/get_dune_evms_info
+                - other_chains_tracking/get_qualified_txs
+                - other_chains_tracking/get_all_txs
+                - other_chains_tracking/get_contract_labels
+                - op_chains_tracking/get_superchain_token_list
+                - op_chains_tracking/ch_gs_uploads
+                - op_chains_tracking/dune_op_stack_chains_l1_data
+                - op_collective_economics/opcollective_feesplit/op_collective_revenue
+                - op_collective_economics/get_op_collective_net_revenue_balances
 
   github-pages:
     when: