From 5a120d271c729328f716f3533401557b553609bc Mon Sep 17 00:00:00 2001 From: George Danezis <4999882+gdanezis@users.noreply.github.com> Date: Fri, 7 Jun 2024 12:21:33 +0100 Subject: [PATCH] docs: Simple python examples (#9) 3 simple python examples using the json, web and sui apis. --- .gitignore | 8 ++ examples/CONFIG/bin/README.md | 3 + examples/CONFIG/config_dir/client_config.yaml | 38 ++++++++ examples/python/README.md | 13 +++ examples/python/hello_walrus_jsonapi.py | 97 +++++++++++++++++++ examples/python/hello_walrus_sui_system.py | 48 +++++++++ examples/python/hello_walrus_webapi.py | 63 ++++++++++++ examples/python/requirements.txt | 1 + examples/python/utils.py | 27 ++++++ 9 files changed, 298 insertions(+) create mode 100644 examples/CONFIG/bin/README.md create mode 100644 examples/CONFIG/config_dir/client_config.yaml create mode 100644 examples/python/README.md create mode 100644 examples/python/hello_walrus_jsonapi.py create mode 100644 examples/python/hello_walrus_sui_system.py create mode 100644 examples/python/hello_walrus_webapi.py create mode 100644 examples/python/requirements.txt create mode 100644 examples/python/utils.py diff --git a/.gitignore b/.gitignore index 9dd8f184..18c94f47 100644 --- a/.gitignore +++ b/.gitignore @@ -12,9 +12,17 @@ # mdBook build/ +# Python +__pycache__/ +.venv/ +*.pyc + # Misc *.key .env config.yml working_dir *.log + +# Walrus binary +examples/CONFIG/bin/walrus diff --git a/examples/CONFIG/bin/README.md b/examples/CONFIG/bin/README.md new file mode 100644 index 00000000..44f07856 --- /dev/null +++ b/examples/CONFIG/bin/README.md @@ -0,0 +1,3 @@ +# Configuration + +Place the 'walrus' client binary for your system in this directory. diff --git a/examples/CONFIG/config_dir/client_config.yaml b/examples/CONFIG/config_dir/client_config.yaml new file mode 100644 index 00000000..9210fb0d --- /dev/null +++ b/examples/CONFIG/config_dir/client_config.yaml @@ -0,0 +1,38 @@ +system_pkg: 0x17108fb344dbea0d315e6f44cdd3af12a028cd568bafcbfb7f7a7b152719d15d +system_object: 0x3fb18e675ad41158f59acbdb7f574136a198cbd83be9b968c0fdeaa139c312f9 + +# You can define a custom path to your Sui wallet configuration here. If this is unset or `null`, +# the wallet is configured from `./client.yaml`, `./sui_config.yaml` (both relative to your current +# working directory), or `~/.sui/sui_config/client.yaml` in this order. +wallet_config: null + +# Default values for the client are commented out. +# +# There is no risk in playing around with these values. +# Worst case, you may not be able to store/read from Walrus. + +# communication_config: +# max_concurrent_writes: null +# max_concurrent_sliver_reads: null +# max_concurrent_metadata_reads: 3 +# reqwest_config: +# total_timeout: +# secs: 180 +# nanos: 0 +# pool_idle_timeout: null +# http2_keep_alive_timeout: +# secs: 5 +# nanos: 0 +# http2_keep_alive_interval: +# secs: 30 +# nanos: 0 +# http2_keep_alive_while_idle: true +# request_rate_config: +# max_node_connections: 10 +# max_retries: 5 +# min_backoff: +# secs: 2 +# nanos: 0 +# max_backoff: +# secs: 60 +# nanos: 0 diff --git a/examples/python/README.md b/examples/python/README.md new file mode 100644 index 00000000..e7cc7d1c --- /dev/null +++ b/examples/python/README.md @@ -0,0 +1,13 @@ +# Walrus Python Examples + +## Prerequisites + +- [Configure Sui Client](https://docs.sui.io/guides/developer/getting-started/connect) to connect + to testnet, and some testnet Sui tokens. +- Configure Walrus TODO(#12). +- Update the paths PATH_TO_WALRUS and PATH_TO_WALRUS_CONFIG and other + constant in `utils.py`. + +## Index of examples + +- ... diff --git a/examples/python/hello_walrus_jsonapi.py b/examples/python/hello_walrus_jsonapi.py new file mode 100644 index 00000000..6beb3acb --- /dev/null +++ b/examples/python/hello_walrus_jsonapi.py @@ -0,0 +1,97 @@ +# Example of uploading and downloading a file to / from the Walrus service +# Using the walrus client json input & output facilities. + +# Std lib imports +import os +import subprocess +import json +import tempfile +import base64 + +import requests + +from utils import num_to_blob_id, PATH_TO_WALRUS, PATH_TO_WALRUS_CONFIG, FULL_NODE_URL + +try: + # Create a 1MB file of random data + random_data = os.urandom(1024 * 1024) + tmp = tempfile.NamedTemporaryFile(delete=False) + tmp.write(random_data) + tmp.close() + + # Part 1. Upload the file to the Walrus service + store_json_command = f"""{{ "config" : "{PATH_TO_WALRUS_CONFIG}", + "command" : {{ "store" : + {{ "file" : "{tmp.name}", "epochs" : 2 }}}} + }}""" + result = subprocess.run( + [PATH_TO_WALRUS, "json"], + text=True, + capture_output=True, + input=store_json_command, + ) + assert result.returncode == 0 + + # Parse the response and display key information + json_result_dict = json.loads(result.stdout.strip()) + print( + f"Upload Blob ID: {json_result_dict['blob_id']} Size {len(random_data)} bytes" + ) + sui_object_id = json_result_dict["sui_object_id"] + blob_id = json_result_dict["blob_id"] + print(f"Certificate in Object ID: {sui_object_id}") + + # Part 2. Download the file from the Walrus service + read_json_command = f"""{{ "config" : "{PATH_TO_WALRUS_CONFIG}", + "command" : {{ "read" : + {{ "blob_id" : "{json_result_dict['blob_id']}" }}}} + }}""" + result = subprocess.run( + [PATH_TO_WALRUS, "json"], + text=True, + capture_output=True, + input=read_json_command, + ) + assert result.returncode == 0 + + # Parse the response and display key information + json_result_dict = json.loads(result.stdout.strip()) + downloaded_data = base64.b64decode(json_result_dict["blob"]) + assert downloaded_data == random_data + + print( + f"Download Blob ID: {json_result_dict['blob_id']} Size {len(downloaded_data)} bytes" + ) + + # Part 3. Check the availability of the blob + request = { + "jsonrpc": "2.0", + "id": 1, + "method": "sui_getObject", + "params": [ + sui_object_id, + { + "showType": True, + "showOwner": False, + "showPreviousTransaction": True, + "showDisplay": False, + "showContent": True, + "showBcs": False, + "showStorageRebate": False, + }, + ], + } + response = requests.post(FULL_NODE_URL, json=request) + object_content = response.json()["result"]["data"]["content"] + print("Object content:") + print(json.dumps(object_content, indent=4)) + + # Check that the blob ID matches the one we uploaded + blob_id_downloaded = int(object_content["fields"]["blob_id"]) + if num_to_blob_id(blob_id_downloaded) == blob_id: + print("Blob ID matches certificate!") + else: + print("Blob ID does not match") + +finally: + os.unlink(tmp.name) diff --git a/examples/python/hello_walrus_sui_system.py b/examples/python/hello_walrus_sui_system.py new file mode 100644 index 00000000..75702df8 --- /dev/null +++ b/examples/python/hello_walrus_sui_system.py @@ -0,0 +1,48 @@ +# Example of querying the Walrus system object on Sui + +# Std lib imports +import requests +import re + +from utils import PATH_TO_WALRUS_CONFIG + +system_object_id = re.findall( + r"system_object:[ ]*(.*)", open(PATH_TO_WALRUS_CONFIG).read() +)[0] +print(f"System object ID: {system_object_id}") + +# Query the Walrus system object on Sui +request = { + "jsonrpc": "2.0", + "id": 1, + "method": "sui_getObject", + "params": [ + system_object_id, + { + "showType": True, + "showOwner": False, + "showPreviousTransaction": True, + "showDisplay": False, + "showContent": True, + "showBcs": False, + "showStorageRebate": False, + }, + ], +} +response = requests.post("https://fullnode.testnet.sui.io:443", json=request) +assert response.status_code == 200 + +system_object_content = response.json()["result"]["data"]["content"]["fields"] +committee = system_object_content["current_committee"]["fields"]["bls_committee"][ + "fields" +] + +print( + f'Current walrus epoch: {system_object_content["current_committee"]["fields"]["epoch"]}' +) +print( + f'Number of members: {len(committee["members"])} Number of shards: {committee["n_shards"]}' +) +print(f'Price per unit size: {system_object_content["price_per_unit_size"]} MIST') +print(f'Total capacity size: {system_object_content["total_capacity_size"]} bytes') +print(f'Used capacity size: {system_object_content["used_capacity_size"]} bytes') diff --git a/examples/python/hello_walrus_webapi.py b/examples/python/hello_walrus_webapi.py new file mode 100644 index 00000000..0991c608 --- /dev/null +++ b/examples/python/hello_walrus_webapi.py @@ -0,0 +1,63 @@ +# Example of uploading and downloading a file to / from the Walrus service +# Using the walrus client web API facilities. +# +# Prerequisites: +# +# - Run the Walrus client in daemon mode: +# $ ../CONFIG/bin/walrus --config ../CONFIG/config_dir/client_config.yaml daemon -b 127.0.0.1:8899 +# + +# Std lib imports +import os +import time + +# External requests HTTP library +import requests + +ADDRESS = "127.0.0.1:8899" +EPOCHS = "5" + + +# Helper functions to upload a blob +def upload_blob(ADDRESS, EPOCHS, data): + # Upload the data to the Walrus service using a PUT request + store_url = f"http://{ADDRESS}/v1/store?epochs={EPOCHS}" + response = requests.put(store_url, data=data) + + # Assert the response status code + assert response.status_code == 200 + blob_id = response.text + return blob_id + + +# Helper functions to download a blob +def download_blob(ADDRESS, blob_id): + # Now read the same resource using the blob-id + read_url = f"http://{ADDRESS}/v1/{blob_id}" + response = requests.get(read_url) + + # Assert the response status code + assert response.status_code == 200 + return response.content + + +# Upload a random 1MB string then download it, and check it matches +if __name__ == "__main__": + # Generate a 1MB blob of random data + random_data = os.urandom(1024 * 1024) + + # Upload the blob to the Walrus service + start_time = time.time() + blob_id = upload_blob(ADDRESS, EPOCHS, random_data) + upload_time = time.time() + + # Now download the same blob using the blob-id + data = download_blob(ADDRESS, blob_id) + assert data == random_data + download_time = time.time() + + # Print some information about the blob + print(f"Blob ID: {blob_id}") + print(f"Size {len(random_data)} bytes") + print(f"Upload time: {upload_time - start_time:.2f}s") + print(f"Download time: {download_time - upload_time:.2f}s") diff --git a/examples/python/requirements.txt b/examples/python/requirements.txt new file mode 100644 index 00000000..d5c2bc9c --- /dev/null +++ b/examples/python/requirements.txt @@ -0,0 +1 @@ +requests>=2.22.0 diff --git a/examples/python/utils.py b/examples/python/utils.py new file mode 100644 index 00000000..fad7c7d1 --- /dev/null +++ b/examples/python/utils.py @@ -0,0 +1,27 @@ +import base64 + +# Configure these paths to match your system +FULL_NODE_URL = "https://fullnode.testnet.sui.io:443" +PATH_TO_WALRUS = "../CONFIG/bin/walrus" +PATH_TO_WALRUS_CONFIG = "../CONFIG/config_dir/client_config.yaml" + + +# Convert a numeric (u256) blob_id to a base64 encoded Blob ID +def num_to_blob_id(blob_id_num): + extracted_bytes = [] + for i in range(32): + extracted_bytes += [blob_id_num & 0xFF] + blob_id_num = blob_id_num >> 8 + assert blob_id_num == 0 + blob_id_bytes = bytes(extracted_bytes) + encoded = base64.urlsafe_b64encode(blob_id_bytes) + return encoded.decode("ascii").strip("=") + + +if __name__ == "__main__": + # A test case for the num_to_blob_id function + blob_id_num = ( + 46269954626831698189342469164469112511517843773769981308926739591706762839432 + ) + blob_id_base64 = "iIWkkUTzPZx-d1E_A7LqUynnYFD-ztk39_tP8MLdS2Y" + assert num_to_blob_id(blob_id_num) == blob_id_base64