New deployment

.github/utils/find.py

@@ -0,0 +1,73 @@
+import re
+import shutil
+from argparse import ArgumentParser
+from pathlib import Path
+from typing import Optional
+
+
+def _create_parser():
+    p = ArgumentParser(
+        "Find and copy files",
+        usage="python find.py -from <from> -to <to>",
+        description="""
+Used to replace the linux find function because we need some magic that the 'find' function
+doesn't provide. 
+
+Currently, only the 'copy' function is supported. During copy, the basename of the file is
+kept (all the parent paths are stripped).
+""".strip()
+    )
+
+    p.add_argument("-from",
+                   type=str,
+                   help="""
+Directory to copy from. If it does not start with '/', the folder is assumed to be 
+relative from the directory command was run from (relative path)""".strip(),
+                   default="")
+
+    p.add_argument('-to',
+                   type=str,
+                   help="""
+Directory to copy to. If it does not start with '/', the folder is assumed to be 
+relative from the directory command was run from (relative path). Note that even
+we never change the filename.""".strip(),
+                   default="")
+
+    p.add_argument('-pattern',
+                   type=str,
+                   help="Regex file pattern to match")
+
+    return p
+
+
+def find_and_copy(from_folder: Path,
+                  to_folder: Path,
+                  pattern: Optional[str]):
+    if pattern is None:
+        pat = None
+    else:
+        pat = re.compile(pattern)
+
+    print(f"Copying files from {from_folder} to {to_folder}")
+
+    for file in Path(from_folder).rglob('*'):
+        if file.is_file():
+            if pat is None or pat.search(file.as_posix()):
+                print(f"Copying file '{file}' to {to_folder / file.name} ")
+                shutil.copy(file, to_folder / file.name)
+
+
+def create_path(path: str):
+    fp = Path(path)
+    if not path.startswith('/'):
+        fp = Path.cwd() / fp
+    return fp
+
+
+if __name__ == '__main__':
+    parser = _create_parser()
+    args = parser.parse_args()
+
+    find_and_copy(create_path(getattr(args, 'from')),
+                  create_path(getattr(args, 'to')),
+                  args.pattern)

.github/workflows/test-build-deploy.yml

@@ -15,8 +15,8 @@ defaults:
     shell: bash
 
 jobs:
-  test:
-    name: Test Package
+  test-code:
+    name: Test Code
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -67,7 +67,7 @@ jobs:
 
   build-src:
     name: Build SDist (Source)
-    needs: test
+    needs: test-code
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Repository
@@ -98,7 +98,7 @@ jobs:
 
   build-wheel:
     name: Build wheels
-    needs: test
+    needs: test-code
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -181,7 +181,7 @@ jobs:
             pip install numpy wheel cython scipy
             filename="copulae-*.tar.gz"
           fi;
-          
+
           echo "Looking for file ${filename}"
           file=$(find dist -name "${filename}" -type f);
 
@@ -191,35 +191,80 @@ jobs:
           # if we can import this, all should be gucci
           python -c "import copulae"
 
-  deploy:
-    name: deploy packages
+  deploy-test:
+    name: Deploy packages to TestPyPI
     runs-on: ubuntu-latest
     needs: install-package
-    if: startsWith(github.ref, 'refs/tags/')
+    if: ${{ !startsWith(github.ref, 'refs/tags/') }}
 
     steps:
       - name: Setup Python 3.10
         uses: actions/setup-python@v5
         with:
           python-version: '3.10'
 
+      - name: Install twine
+        run: pip install twine
+
+      - name: Checkout utilities
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            .github/utils
+          sparse-checkout-cone-mode: false
+
       - name: Retrieve packages
         uses: actions/download-artifact@v4
         with:
-          name: artifact
           path: dist
 
-      - name: Install twine
-        run: pip install twine
+      - name: Find and copy packages to files/
+        run: |
+          mkdir files
+          ls -ltR
+          python .github/utils/find.py -from dist -to files -pattern "(.tar.gz|.whl)$"
 
       - name: Upload packages to testpypi
         env:
-          TWINE_USERNAME: ${{ secrets.PYPI_TEST_UID }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_TEST_PWD }}
-        run: python -m twine upload --skip-existing --repository testpypi dist/*
+          TWINE_USERNAME: ${{ secrets.PYPI_TEST_TOKEN_NAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_TEST_API_TOKEN }}
+        run: python -m twine upload --skip-existing --repository testpypi files/* --verbose
+
+  deploy:
+    name: Deploy packages to PyPI
+    runs-on: ubuntu-latest
+    needs: install-package
+    if: startsWith(github.ref, 'refs/tags/')
+
+    steps:
+      - name: Setup Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install twine
+        run: pip install twine
+
+      - name: Checkout utilities
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            .github/utils
+          sparse-checkout-cone-mode: false
+
+      - name: Retrieve packages
+        uses: actions/download-artifact@v4
+        with:
+          path: dist
+
+      - name: Find and copy packages to files/
+        run: |
+          mkdir files
+          ls -ltR
+          python .github/utils/find.py -from dist -to files -pattern "(.tar.gz|.whl)$"
 
       - name: Upload packages to pypi
         env:
-          TWINE_USERNAME: ${{ secrets.PYPI_UID }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_PWD }}
-        run: python -m twine upload --skip-existing dist/*
+          TWINE_USERNAME: ${{ secrets.PYPI_PROD_TOKEN_NAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PROD_API_TOKEN }}
+        run: python -m twine upload --skip-existing files/* --verbose