From 7e6f6c4ede0a4a1f3d64f6080c2ca3c2c2b8fac9 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 23 Dec 2024 07:09:01 -0700 Subject: [PATCH] fix bash example client to work with latest server (#2208) https://github.com/user-attachments/assets/e91888ac-f659-44ab-9a28-350055ed787d --- .gitignore | 3 +- examples/bash/README.md | 53 +++++++++++ examples/bash/client.bash | 183 ++++++++++++++++++++++++++------------ 3 files changed, 183 insertions(+), 56 deletions(-) create mode 100644 examples/bash/README.md diff --git a/.gitignore b/.gitignore index 3da31d97bf..ba815cea6d 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,5 @@ tsconfig.tsbuildinfo wal /shapes .sst -**/deps/* \ No newline at end of file +**/deps/* +response.tmp diff --git a/examples/bash/README.md b/examples/bash/README.md new file mode 100644 index 0000000000..ec7a2dab95 --- /dev/null +++ b/examples/bash/README.md @@ -0,0 +1,53 @@ +# Electric Bash Client + +A simple bash client for consuming Electric shape logs. This client can connect to any Electric shape URL and stream updates in real-time. + +## Requirements + +- bash +- curl +- jq (for JSON processing) + +## Usage + +```bash +./client.bash 'YOUR_SHAPE_URL' +``` + +For example: +```bash +./client.bash 'http://localhost:3000/v1/shape?table=notes' +``` + +## Example Output + +When first connecting, you'll see the initial shape data: +```json +[ + { + "key": "\"public\".\"notes\"/\"1\"", + "value": { + "id": "1", + "title": "Example Note", + "created_at": "2024-12-05 01:43:05.219957+00" + }, + "headers": { + "operation": "insert", + "relation": [ + "public", + "notes" + ] + }, + "offset": "0_0" + } +] +``` + +Once caught up, the client switches to live mode and streams updates: +``` +Found control message +Control value: up-to-date +Shape is up to date, switching to live mode +``` + +Any changes to the shape will be streamed in real-time. diff --git a/examples/bash/client.bash b/examples/bash/client.bash index 0a7045bfdc..59a4997890 100755 --- a/examples/bash/client.bash +++ b/examples/bash/client.bash @@ -1,88 +1,161 @@ #!/bin/bash -# URL to download the JSON file from (without the output parameter) -BASE_URL="http://localhost:3000/v1/shape?table=todos" +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + echo "Example: $0 'http://localhost:3000/v1/shape?table=todos'" + exit 1 +fi + +# Extract base URL (everything before the query string) +BASE_URL=$(echo "$1" | sed -E 's/\?.*//') +# Extract and clean query parameters +QUERY_STRING=$(echo "$1" | sed -n 's/.*\?\(.*\)/\1/p') + +# Build cleaned query string, removing electric-specific params but keeping others +if [ -n "$QUERY_STRING" ]; then + # Split query string into individual parameters + CLEANED_PARAMS="" + IFS='&' read -ra PARAMS <<< "$QUERY_STRING" + for param in "${PARAMS[@]}"; do + KEY=$(echo "$param" | cut -d'=' -f1) + # Skip electric-specific params + case "$KEY" in + "offset"|"handle"|"live") continue ;; + *) + if [ -z "$CLEANED_PARAMS" ]; then + CLEANED_PARAMS="$param" + else + CLEANED_PARAMS="${CLEANED_PARAMS}&${param}" + fi + ;; + esac + done + + # Add question mark if we have params + if [ -n "$CLEANED_PARAMS" ]; then + BASE_URL="${BASE_URL}?${CLEANED_PARAMS}&" + else + BASE_URL="${BASE_URL}?" + fi +else + BASE_URL="${BASE_URL}?" +fi # Directory to store individual JSON files OFFSET_DIR="./json_files" -# Initialize the latest output variable +# Initialize variables LATEST_OFFSET="-1" +SHAPE_HANDLE="" +IS_LIVE_MODE=false # Create the output directory if it doesn't exist mkdir -p "$OFFSET_DIR" +# Function to extract header value from curl response +get_header_value() { + local headers="$1" + local header_name="$2" + echo "$headers" | grep -i "^$header_name:" | cut -d':' -f2- | tr -d ' \r' +} + # Function to download and process JSON data process_json() { local url="$1" - local output_file="$2" - - echo >&2 "Downloading JSON file from $url..." - curl -s -o "$output_file" "$url" - - # Check if the file was downloaded successfully - if [ ! -f "$output_file" ]; then - echo >&2 "Failed to download the JSON file." - exit 1 + local tmp_headers="headers.tmp" + local tmp_body="body.tmp" + local response_file="response.tmp" + local state_file="state.tmp" + + echo "Downloading shape log from URL: ${url}" + + # Clear any existing tmp files and create new ones + rm -f "$tmp_headers" "$tmp_body" "$response_file" "$state_file" xx* + + # Download the entire response first + curl -i -s "$url" > "$response_file" + + # Split at the double newline - everything before the JSON array + sed -n '/^\[/,$p' "$response_file" > "$tmp_body" + grep -B 1000 "^\[" "$response_file" | grep -v "^\[" > "$tmp_headers" + + # Display prettified JSON if file is not empty + if [ -s "$tmp_body" ]; then + jq '.' < "$tmp_body" fi - # Check if the file is not empty - if [ ! -s "$output_file" ]; then - echo >&2 "The downloaded JSON file is empty." - # Return the latest OFFSET - echo "$LATEST_OFFSET" - return - fi + # Extract important headers + local headers=$(cat "$tmp_headers") + local new_handle=$(get_header_value "$headers" "electric-handle") + local new_offset=$(get_header_value "$headers" "electric-offset") - echo >&2 "Successfully downloaded the JSON file." + # Always update handle from header if present + if [ -n "$new_handle" ]; then + SHAPE_HANDLE="$new_handle" + fi - # Ensure the file ends with a newline - if [ -n "$(tail -c 1 "$output_file")" ]; then - echo >> "$output_file" + # Update offset from header + if [ -n "$new_offset" ]; then + LATEST_OFFSET="$new_offset" fi - # Validate the JSON structure (optional but recommended for debugging) - if ! jq . "$output_file" > /dev/null 2>&1; then - echo >&2 "Invalid JSON format in the downloaded file." - exit 1 + # Check if headers were received + if [ ! -f "$tmp_headers" ]; then + echo >&2 "Failed to download response." + return 1 fi - # Read the JSON file line by line and save each JSON object to an individual file - while IFS= read -r line; do - # Check if the headers array contains an object with key "operation" - if echo "$line" | jq -e '.headers | map(select(.key == "operation")) | length == 0' > /dev/null; then - # echo "Skipping line without an operation: $operation" # Log skipping non-data objects - continue + # Process last 5 items in the JSON array for control messages + jq -c 'if length > 5 then .[-5:] else . end | .[]' "$tmp_body" | while IFS= read -r item; do + # Parse the JSON message + if echo "$item" | jq -e '.headers.control' >/dev/null 2>&1; then + echo "Found control message" >&2 + # Handle control messages + local control=$(echo "$item" | jq -r '.headers.control') + echo "Control value: $control" >&2 + case "$control" in + "up-to-date") + echo "true" > "$state_file" + echo >&2 "Shape is up to date, switching to live mode" + ;; + "must-refetch") + echo >&2 "Server requested refetch" + LATEST_OFFSET="-1" + IS_LIVE_MODE=false + SHAPE_HANDLE="" + ;; + esac fi + done - key=$(echo "$line" | jq -r '.key') - offset=$(echo "$line" | jq -r '.offset') - - if [ -z "$key" ]; then - echo >&2 "No key found in message: $line" # Log if no ID is found - else - echo >&2 "Extracted key: $key" # Log the extracted key - echo "$line" | jq . > "$OFFSET_DIR/json_object_$key.json" - echo >&2 "Written to file: $OFFSET_DIR/json_object_$key.json" # Log file creation - - LATEST_OFFSET="$offset" - echo >&2 "Updated latest OFFSET to: $LATEST_OFFSET" - fi - done < <(jq -c '.[]' "$output_file") + # Read the state file and update IS_LIVE_MODE + if [ -f "$state_file" ] && [ "$(cat "$state_file")" = "true" ]; then + IS_LIVE_MODE=true + fi - echo >&2 "done with jq/read loop $LATEST_OFFSET" + # Cleanup + rm -f "$tmp_headers" "$tmp_body" "$response_file" "$state_file" xx* - # Return the latest OFFSET - echo "$LATEST_OFFSET" + return 0 } -# Main loop to poll for updates every second +# Main loop to poll for updates while true; do - url="$BASE_URL&offset=$LATEST_OFFSET" - echo $url + # Construct URL with appropriate parameters + url="${BASE_URL}offset=$LATEST_OFFSET" + if [ -n "$SHAPE_HANDLE" ]; then + url="${url}&handle=$SHAPE_HANDLE" + fi + if [ "$IS_LIVE_MODE" = true ]; then + url="${url}&live=true" + fi - LATEST_OFFSET=$(process_json "$url" "shape-data.json") + if ! process_json "$url"; then + echo >&2 "Error processing response, retrying in 5 seconds..." + sleep 5 + continue + fi + # Add small delay between requests sleep 1 done -