Merge branch 'main' into cli_fixes

.github/workflows/deploy_daily_tests.yml

@@ -0,0 +1,19 @@
+name: Deploy daily tests
+
+on:
+  schedule:
+    - cron: "0 6 * * *"  # Daily at 2am EST
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+
+  DailyTests:
+    uses: ./.github/workflows/testing_dev.yml
+    secrets:
+      IPINFO_HASH_SALT: ${{ secrets.IPINFO_HASH_SALT }}
+      IPINFO_CREDENTIALS: ${{ secrets.IPINFO_CREDENTIALS }}
+      CODECOV_CREDENTIALS: ${{ secrets.CODECOV_CREDENTIALS }}

.github/workflows/publish_to_pypi_on_github_release.yml

@@ -0,0 +1,32 @@
+name: Upload Package to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade build
+          python -m pip install --upgrade twine
+          
+      - name: Build package
+        run: python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/[email protected]
+        with:
+          verbose: true
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/testing_dev.yml

@@ -8,7 +8,6 @@ on:
         required: true
       CODECOV_CREDENTIALS:
         required: true
-  workflow_dispatch:
 
 env:
   IPINFO_HASH_SALT: ${{ secrets.IPINFO_HASH_SALT }}

README.md

@@ -27,19 +27,32 @@ Developed for the [DANDI Archive](https://dandiarchive.org/).
 To iteratively parse all historical logs all at once (parallelization with 10-15 total GB recommended):
 
 ```bash
-parse_all_dandi_raw_s3_logs --base_raw_s3_log_folder_path < base log folder > --parsed_s3_log_folder_path < output folder > --excluded_ips < comma-separated list of known IPs to exclude > --maximum_number_of_workers < number of CPUs to use > --maximum_buffer_size_in_bytes < approximate amount of RAM to use >
+parse_all_dandi_raw_s3_logs \
+  --base_raw_s3_log_folder_path < base log folder > \
+  --parsed_s3_log_folder_path < output folder > \
+  --excluded_ips < comma-separated list of known IPs to exclude > \
+  --maximum_number_of_workers < number of CPUs to use > \
+  --maximum_buffer_size_in_bytes < approximate amount of RAM to use >
 ```
 
 For example, on Drogon:
 
 ```bash
-parse_all_dandi_raw_s3_logs --base_raw_s3_log_folder_path /mnt/backup/dandi/dandiarchive-logs --parsed_s3_log_folder_path /mnt/backup/dandi/dandiarchive-logs-cody/parsed_7_13_2024/GET_per_asset_id --excluded_ips < Drogon's IP > --maximum_number_of_workers 30 --maximum_buffer_size_in_bytes 15000000000
+parse_all_dandi_raw_s3_logs \
+  --base_raw_s3_log_folder_path /mnt/backup/dandi/dandiarchive-logs \
+  --parsed_s3_log_folder_path /mnt/backup/dandi/dandiarchive-logs-cody/parsed_7_13_2024/GET_per_asset_id \
+  --excluded_ips < Drogons IP > \
+  --maximum_number_of_workers 3 \
+  --maximum_buffer_size_in_bytes 15000000000
 ```
 
 To parse only a single log file at a time, such as in a CRON job:
 
 ```bash
-parse_dandi_raw_s3_log --raw_s3_log_file_path < s3 log file path > --parsed_s3_log_folder_path < output folder > --excluded_ips < comma-separated list of known IPs to exclude >
+parse_dandi_raw_s3_log \
+  --raw_s3_log_file_path < s3 log file path > \
+  --parsed_s3_log_folder_path < output folder > \
+  --excluded_ips < comma-separated list of known IPs to exclude >
 ```