Merge pull request #2560 from dirvine/data_refactor

Data refactor

.github/workflows/python-publish-client.yml

@@ -3,15 +3,17 @@ name: Build and Publish Python Client Package
 on:
   push:
     tags:
-      - 'xxx'
+      - 'autonomi-v*'
 
+# Add top-level permissions block
 permissions:
   id-token: write
   contents: read
 
 jobs:
   macos:
     runs-on: macos-latest
+    # Add permissions to job
     permissions:
       id-token: write
       contents: read
@@ -21,43 +23,26 @@ jobs:
         target: [x86_64, aarch64]
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          components: rustfmt
-      - name: Create Python module structure
-        run: |
-          mkdir -p autonomi/python/autonomi_client
-          cat > autonomi/python/autonomi_client/__init__.py << EOL
-          from .autonomi_client import *
-          __version__ = "${{ github.ref_name }}"
-          EOL
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.target }}
-          args: --release --out dist --find-interpreter --compatibility manylinux2014
+          args: --release --out dist -i python${{ matrix.python-version }}
           sccache: 'true'
           working-directory: ./autonomi
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
-          name: wheels-${{ matrix.python-version }}-${{ matrix.target }}
-          path: autonomi/dist/*.whl
+          name: wheels-macos-${{ matrix.target }}-py${{ matrix.python-version }}
+          path: ./autonomi/dist/*.whl
           if-no-files-found: error
-          retention-days: 1
-          compression-level: 9
-        continue-on-error: true
-        timeout-minutes: 10
-        env:
-          ACTIONS_STEP_DEBUG: true
-          ACTIONS_RUNNER_DEBUG: true
 
   windows:
     runs-on: windows-latest
+    # Add permissions to job
     permissions:
       id-token: write
       contents: read
@@ -67,39 +52,22 @@ jobs:
         target: [x64]
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
           architecture: ${{ matrix.target }}
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          components: rustfmt
-      - name: Create Python module structure
-        shell: cmd
-        run: |
-          if not exist "autonomi\python\autonomi_client" mkdir autonomi\python\autonomi_client
-          echo from .autonomi_client import * > autonomi\python\autonomi_client\__init__.py
-          echo __version__ = "${{ github.ref_name }}" >> autonomi\python\autonomi_client\__init__.py
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         with:
-          args: --release --out dist --find-interpreter --compatibility manylinux2014
+          args: --release --out dist
           sccache: 'true'
           working-directory: ./autonomi
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
-          name: wheels-${{ matrix.python-version }}-${{ matrix.target }}
-          path: autonomi/dist/*.whl
+          name: wheels-windows-${{ matrix.target }}-py${{ matrix.python-version }}
+          path: ./autonomi/dist/*.whl
           if-no-files-found: error
-          retention-days: 1
-          compression-level: 9
-        continue-on-error: true
-        timeout-minutes: 10
-        env:
-          ACTIONS_STEP_DEBUG: true
-          ACTIONS_RUNNER_DEBUG: true
 
   linux:
     runs-on: ubuntu-latest
@@ -108,131 +76,188 @@ jobs:
       contents: read
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         target: [x86_64]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          components: rustfmt
-          target: x86_64-unknown-linux-gnu
-      - name: Install dependencies
-        run: |
-          python -m pip install --user cffi
-          python -m pip install --user patchelf
-          rustup component add rustfmt
-      - name: Create Python module structure
-        run: |
-          mkdir -p autonomi/python/autonomi_client
-          cat > autonomi/python/autonomi_client/__init__.py << EOL
-          from .autonomi_client import *
-          __version__ = "${{ github.ref_name }}"
-          EOL
+          architecture: x64
       - name: Build wheels
         uses: PyO3/maturin-action@v1
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
         with:
           target: ${{ matrix.target }}
-          manylinux: "2014"
-          args: --release --out dist --find-interpreter
-          sccache: 'true'
-          working-directory: ./autonomi
+          manylinux: auto
           before-script-linux: |
-            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-            source $HOME/.cargo/env
+            rustup default stable
             rustup component add rustfmt
+          args: --release --out dist -i python${{ matrix.python-version }}
+          sccache: 'true'
+          working-directory: ./autonomi
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
-          name: wheels-${{ matrix.python-version }}-${{ matrix.target }}
-          path: autonomi/dist/*.whl
+          name: wheels-linux-${{ matrix.target }}-py${{ matrix.python-version }}
+          path: ./autonomi/dist/*.whl
           if-no-files-found: error
-          retention-days: 1
-          compression-level: 9
-        continue-on-error: true
-        timeout-minutes: 10
+
+  musllinux:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      matrix:
+        target:
+          - x86_64-unknown-linux-musl
+          - aarch64-unknown-linux-musl
+          - armv7-unknown-linux-musleabihf
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
         env:
-          ACTIONS_STEP_DEBUG: true
-          ACTIONS_RUNNER_DEBUG: true
+          PYO3_CROSS_PYTHON_VERSION: ${{ matrix.python-version }}
+          PYO3_CROSS: "1"
+        with:
+          target: ${{ matrix.target }}
+          manylinux: musllinux_1_2
+          args: --release --out dist -i python${{ matrix.python-version }}
+          sccache: 'true'
+          working-directory: ./autonomi
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-musllinux-${{ matrix.target }}-py${{ matrix.python-version }}
+          path: ./autonomi/dist/*.whl
+          if-no-files-found: error
 
   sdist:
     runs-on: ubuntu-latest
+    # Add permissions to job
     permissions:
       id-token: write
       contents: read
     steps:
       - uses: actions/checkout@v4
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          components: rustfmt
-      - name: Create Python module structure
+      - name: Prepare standalone package
         run: |
-          mkdir -p autonomi/python/autonomi_client
-          cat > autonomi/python/autonomi_client/__init__.py << EOL
-          from .autonomi_client import *
-          __version__ = "${{ github.ref_name }}"
+          # Create build directory structure
+          mkdir -p build/autonomi
+          cp -r autonomi/* build/autonomi/
+          
+          # First, copy all workspace members
+          for dir in ant-* test-utils evmlib; do
+            if [ -d "$dir" ]; then
+              echo "Copying $dir to build directory"
+              cp -r "$dir" "build/$dir"
+            fi
+          done
+          
+          # Create a new workspace Cargo.toml in the build directory
+          cat > build/Cargo.toml << EOL
+          [workspace]
+          resolver = "2"
+          members = [
+              "ant-bootstrap",
+              "ant-build-info",
+              "ant-cli",
+              "ant-evm",
+              "ant-logging",
+              "ant-metrics",
+              "ant-networking",
+              "ant-node",
+              "ant-node-manager",
+              "ant-node-rpc-client",
+              "ant-protocol",
+              "ant-registers",
+              "ant-service-management",
+              "ant-token-supplies",
+              "autonomi",
+              "evmlib",
+              "test-utils"
+          ]
+
+          [workspace.lints.rust]
+          arithmetic_overflow = "forbid"
+          mutable_transmutes = "forbid"
+          no_mangle_const_items = "forbid"
+          trivial_casts = "warn"
+          trivial_numeric_casts = "warn"
+          unsafe_code = "warn"
+          unknown_crate_types = "forbid"
+          unused_extern_crates = "warn"
+          unused_import_braces = "warn"
+
+          [workspace.lints.clippy]
+          clone_on_ref_ptr = "warn"
+          unicode_not_nfc = "warn"
+          uninlined_format_args = "warn"
+          unused_async = "warn"
+          unwrap_used = "warn"
+
+          [profile.dev]
+          debug = 0
+          strip = "debuginfo"
+
+          [workspace.metadata.release]
+          pre-release-commit-message = "chore(release): release commit, tags, deps and changelog updates"
+          publish = false
+          push = false
+          tag = false
+
+          [workspace.dependencies]
+          backtrace = "=0.3.71"
           EOL
+          
+          # Update all dependency paths to be absolute
+          find build -name "Cargo.toml" -exec sed -i "s|path = \"\.\./|path = \"/home/runner/work/autonomi/autonomi/build/|g" {} \;
+          
+          # Display directory structure for debugging
+          echo "Contents of build directory:"
+          ls -la build/
+          echo "Contents of workspace Cargo.toml:"
+          cat build/Cargo.toml
       - name: Build sdist
         uses: PyO3/maturin-action@v1
         with:
           command: sdist
           args: --out dist
-          working-directory: ./autonomi
+          working-directory: build/autonomi
       - name: Upload sdist
         uses: actions/upload-artifact@v4
         with:
-          name: sdist
-          path: autonomi/dist/*.tar.gz
+          name: wheels
+          path: build/autonomi/dist/*.tar.gz
           if-no-files-found: error
-          retention-days: 1
-          compression-level: 9
-        continue-on-error: true
-        timeout-minutes: 10
-        env:
-          ACTIONS_STEP_DEBUG: true
-          ACTIONS_RUNNER_DEBUG: true
 
   release:
     name: Release
     runs-on: ubuntu-latest
-    needs: [macos, windows, linux, sdist]
+    needs: [macos, windows, linux, musllinux, sdist]
+    # Keep existing permissions
     permissions:
       id-token: write
       contents: read
     steps:
-      - name: Create dist directory
-        run: mkdir -p dist
-
-      # Download all artifacts at once
-      - name: Download all artifacts
-        uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v4
         with:
+          name: wheels
           path: dist
-
-      - name: Prepare dist directory
-        run: |
-          find dist -type f -name "*.whl" -exec mv {} dist/ \;
-          find dist -type f -name "*.tar.gz" -exec mv {} dist/ \;
-          rm -rf dist/*/
-          echo "Final dist directory contents:"
-          ls -la dist/
-
-      - name: Check if version exists
-        run: |
-          VERSION="${{ github.ref_name }}"
-          VERSION="${VERSION#v}"  # Remove 'v' prefix if present
-          if pip index versions autonomi-client | grep -q "${VERSION}"; then
-            echo "Version ${VERSION} already exists on PyPI"
-            exit 1
-          fi
-
+          merge-multiple: true
+      - name: Display structure of downloaded files
+        run: ls -R dist
       - name: Publish to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           packages-dir: dist/
           verbose: true
-          print-hash: true
+          print-hash: true