diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7019ae2..e9cfa79 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,7 +25,7 @@ jobs: run: uv sync --all-extras --dev - name: Generate Proto Files - run: bash ./generate_proto.sh + run: uv run generate_proto.py - name: Prepare Hedera Solo id: solo diff --git a/README.md b/README.md index d23bc3f..60fd6f5 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ worry about managing multiple versions of python on your machine! ```bash uv sync -./generate_proto.sh +uv run generate_proto.py ``` To update to a newer version of the protobuf libraries, edit the `generate_proto.sh` file and change the version number diff --git a/generate_proto.py b/generate_proto.py new file mode 100644 index 0000000..451407f --- /dev/null +++ b/generate_proto.py @@ -0,0 +1,107 @@ +import os +import subprocess +import shutil +import sys + +# Constants +HAPI_VERSION = "v0.57.3" +PROTOS_DIR = ".protos" +SERVICES_DIR = "src/hedera_sdk_python/hapi/services" +MIRROR_DIR = "src/hedera_sdk_python/hapi/mirror" + +def main(): + # Use Python from the virtual environment to run commands + venv_python = os.path.join('.', '.venv', 'bin', 'python') + if not os.path.exists(venv_python): + print("Virtual environment Python not found. Please ensure your virtual environment is set up correctly.") + sys.exit(1) + + # Step 1: Prepare directories + print("Setting up directories...") + for directory in [PROTOS_DIR, SERVICES_DIR, MIRROR_DIR]: + os.makedirs(directory, exist_ok=True) + + # Clean up old files + for directory in [PROTOS_DIR, SERVICES_DIR, MIRROR_DIR]: + for item in os.listdir(directory): + item_path = os.path.join(directory, item) + if os.path.isfile(item_path): + os.unlink(item_path) + elif os.path.isdir(item_path): + shutil.rmtree(item_path) + + # Create subdirectories + for subdir in ["auxiliary/tss", "event"]: + os.makedirs(os.path.join(SERVICES_DIR, subdir), exist_ok=True) + + # Touch __init__.py files + for directory in [SERVICES_DIR, MIRROR_DIR]: + open(os.path.join(directory, "__init__.py"), 'a').close() + + # Step 2: Download and extract protobuf files + print(f"Downloading Hedera protobufs version {HAPI_VERSION}...") + try: + subprocess.run(f"curl -sL 'https://github.com/hashgraph/hedera-protobufs/archive/refs/tags/{HAPI_VERSION}.tar.gz' | tar -xz -C {PROTOS_DIR} --strip-components=1", shell=True, check=True) + + # Keep 'platform', 'services', and 'mirror', remove everything else + for item in os.listdir(PROTOS_DIR): + if item not in ['platform', 'services', 'mirror']: + item_path = os.path.join(PROTOS_DIR, item) + if os.path.isdir(item_path): + shutil.rmtree(item_path) # Remove directories + elif os.path.isfile(item_path): + os.remove(item_path) # Remove files + except subprocess.CalledProcessError as e: + print(f"Error downloading or extracting protobufs: {e}") + sys.exit(1) + + # Step 3: Compile service and platform protobuf files + print("Compiling service and platform protobuf files...") + protoc_cmd = f"python -m grpc_tools.protoc --proto_path={PROTOS_DIR}/platform --proto_path={PROTOS_DIR}/services --pyi_out=./{SERVICES_DIR} --python_out=./{SERVICES_DIR} --grpc_python_out=./{SERVICES_DIR} {PROTOS_DIR}/services/*.proto {PROTOS_DIR}/services/auxiliary/tss/*.proto {PROTOS_DIR}/platform/event/*.proto" + subprocess.run(protoc_cmd, shell=True, check=True) + + # Step 4: Adjust imports for service and platform protobuf files + print("Adjusting imports for service and platform protobuf files...") + adjust_imports(SERVICES_DIR, [ + (r'^import .*_pb2 as .*__pb2', r'from . \g<0>'), + (r'^from auxiliary\.tss', 'from .auxiliary.tss'), + (r'^from event', 'from .event') + ]) + + # Step 5: Compile mirror protobuf files + print("Compiling mirror protobuf files...") + protoc_cmd = f"python -m grpc_tools.protoc --proto_path={PROTOS_DIR}/mirror --proto_path={PROTOS_DIR}/services --python_out={MIRROR_DIR} --grpc_python_out={MIRROR_DIR} {PROTOS_DIR}/mirror/*.proto" + subprocess.run(protoc_cmd, shell=True, check=True) + + # Step 6: Adjust imports for mirror protobuf files + print("Adjusting imports for mirror protobuf files...") + adjust_imports(MIRROR_DIR, [ + (r'^import basic_types_pb2 as', r'import hedera_sdk_python.hapi.services.basic_types_pb2 as'), + (r'^import timestamp_pb2 as', r'import hedera_sdk_python.hapi.services.timestamp_pb2 as'), + (r'^import consensus_submit_message_pb2 as', r'import hedera_sdk_python.hapi.services.consensus_submit_message_pb2 as'), + (r'^import consensus_service_pb2 as', r'import hedera_sdk_python.hapi.mirror.consensus_service_pb2 as'), + (r'^import mirror_network_service_pb2 as', r'import hedera_sdk_python.hapi.mirror.mirror_network_service_pb2 as') + ]) + + # Step 7: Confirm success + if os.listdir(SERVICES_DIR) and os.listdir(MIRROR_DIR): + print("All protobuf files have been generated and adjusted successfully!") + else: + print("Error: Protobuf file generation or adjustment failed.") + sys.exit(1) + +def adjust_imports(directory, replacements): + for root, _, files in os.walk(directory): + for file in files: + if file.endswith('.py'): + file_path = os.path.join(root, file) + with open(file_path, 'r') as f: + content = f.read() + for pattern, repl in replacements: + content = re.sub(pattern, repl, content, flags=re.MULTILINE) + with open(file_path, 'w') as f: + f.write(content) + +if __name__ == "__main__": + import re + main() diff --git a/generate_proto.sh b/generate_proto.sh deleted file mode 100755 index 11c7abe..0000000 --- a/generate_proto.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash - -# Source the activate script to set up the PATH for this shell session -source ./.venv/bin/activate - -hapi_version="v0.57.3" -protos_dir=".protos" -services_dir="src/hedera_sdk_python/hapi/services" -mirror_dir="src/hedera_sdk_python/hapi/mirror" - -# Step 1: Prepare directories -echo "Setting up directories..." -mkdir -p $protos_dir -rm -rf $protos_dir/* -rm -rf $services_dir/* -rm -rf $mirror_dir/* -mkdir -p $services_dir/auxiliary/tss -mkdir -p $services_dir/event -mkdir -p $mirror_dir -touch $services_dir/__init__.py -touch $mirror_dir/__init__.py - -# Step 2: Download and extract protobuf files -echo "Downloading Hedera protobufs version $hapi_version..." -curl -sL "https://github.com/hashgraph/hedera-protobufs/archive/refs/tags/${hapi_version}.tar.gz" | tar -xz -C $protos_dir --strip-components=1 -# Keep 'platform', 'services', and 'mirror', remove everything else -find "$protos_dir" -mindepth 1 -maxdepth 1 ! -name platform ! -name services ! -name mirror -exec rm -r {} + - -# Step 3: Compile service and platform protobuf files -echo "Compiling service and platform protobuf files..." -python -m grpc_tools.protoc \ - --proto_path=$protos_dir/platform \ - --proto_path=$protos_dir/services \ - --pyi_out=./$services_dir \ - --python_out=./$services_dir \ - --grpc_python_out=./$services_dir \ - $protos_dir/services/*.proto $protos_dir/services/auxiliary/tss/*.proto $protos_dir/platform/event/*.proto - -# Step 4: Adjust imports for service and platform protobuf files -echo "Adjusting imports for service and platform protobuf files..." -if [[ "$OSTYPE" == "darwin"* ]]; then - find $services_dir -type f -name "*.py" -exec sed -i '' \ - -e '/^import .*_pb2 as .*__pb2/s/^/from . /' \ - -e 's/^from auxiliary\.tss/from .auxiliary.tss/' \ - -e 's/^from event/from .event/' {} + -else - find $services_dir -type f -name "*.py" -exec sed -i \ - -e '/^import .*_pb2 as .*__pb2/s/^/from . /' \ - -e 's/^from auxiliary\.tss/from .auxiliary.tss/' \ - -e 's/^from event/from .event/' {} + -fi - -# Step 5: Compile mirror protobuf files -echo "Compiling mirror protobuf files..." -python -m grpc_tools.protoc \ - --proto_path=$protos_dir/mirror \ - --proto_path=$protos_dir/services \ - --python_out=$mirror_dir \ - --grpc_python_out=$mirror_dir \ - $protos_dir/mirror/*.proto - -# Step 6: Adjust imports for mirror protobuf files -echo "Adjusting imports for mirror protobuf files..." -if [[ "$OSTYPE" == "darwin"* ]]; then - find $mirror_dir -type f -name "*.py" -exec sed -i '' \ - -e 's/^import basic_types_pb2 as/import hedera_sdk_python.hapi.services.basic_types_pb2 as/' \ - -e 's/^import timestamp_pb2 as/import hedera_sdk_python.hapi.services.timestamp_pb2 as/' \ - -e 's/^import consensus_submit_message_pb2 as/import hedera_sdk_python.hapi.services.consensus_submit_message_pb2 as/' \ - -e 's/^import consensus_service_pb2 as/import hedera_sdk_python.hapi.mirror.consensus_service_pb2 as/' \ - -e 's/^import mirror_network_service_pb2 as/import hedera_sdk_python.hapi.mirror.mirror_network_service_pb2 as/' {} + -else - find $mirror_dir -type f -name "*.py" -exec sed -i \ - -e 's/^import basic_types_pb2 as/import hedera_sdk_python.hapi.services.basic_types_pb2 as/' \ - -e 's/^import timestamp_pb2 as/import hedera_sdk_python.hapi.services.timestamp_pb2 as/' \ - -e 's/^import consensus_submit_message_pb2 as/import hedera_sdk_python.hapi.services.consensus_submit_message_pb2 as/' \ - -e 's/^import consensus_service_pb2 as/import hedera_sdk_python.hapi.mirror.consensus_service_pb2 as/' \ - -e 's/^import mirror_network_service_pb2 as/import hedera_sdk_python.hapi.mirror.mirror_network_service_pb2 as/' {} + -fi - -# Step 7: Confirm success -if [ "$(ls -A $services_dir)" ] && [ "$(ls -A $mirror_dir)" ]; then - echo "All protobuf files have been generated and adjusted successfully!" -else - echo "Error: Protobuf file generation or adjustment failed." - exit 1 -fi