fix bash example client to work with latest server

.gitignore

@@ -21,4 +21,5 @@ tsconfig.tsbuildinfo
 wal
 /shapes
 .sst
-**/deps/*
+**/deps/*
+response.tmp

examples/bash/README.md

@@ -0,0 +1,53 @@
+# Electric Bash Client
+
+A simple bash client for consuming Electric shape logs. This client can connect to any Electric shape URL and stream updates in real-time.
+
+## Requirements
+
+- bash
+- curl
+- jq (for JSON processing)
+
+## Usage
+
+```bash
+./client.bash 'YOUR_SHAPE_URL'
+```
+
+For example:
+```bash
+./client.bash 'http://localhost:3000/v1/shape?table=notes'
+```
+
+## Example Output
+
+When first connecting, you'll see the initial shape data:
+```json
+[
+  {
+    "key": "\"public\".\"notes\"/\"1\"",
+    "value": {
+      "id": "1",
+      "title": "Example Note",
+      "created_at": "2024-12-05 01:43:05.219957+00"
+    },
+    "headers": {
+      "operation": "insert",
+      "relation": [
+        "public",
+        "notes"
+      ]
+    },
+    "offset": "0_0"
+  }
+]
+```
+
+Once caught up, the client switches to live mode and streams updates:
+```
+Found control message
+Control value: up-to-date
+Shape is up to date, switching to live mode
+```
+
+Any changes to the shape will be streamed in real-time.

examples/bash/client.bash

@@ -1,88 +1,161 @@
 #!/bin/bash
 
-# URL to download the JSON file from (without the output parameter)
-BASE_URL="http://localhost:3000/v1/shape?table=todos"
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 <shape-url>"
+    echo "Example: $0 'http://localhost:3000/v1/shape?table=todos'"
+    exit 1
+fi
+
+# Extract base URL (everything before the query string)
+BASE_URL=$(echo "$1" | sed -E 's/\?.*//')
+# Extract and clean query parameters
+QUERY_STRING=$(echo "$1" | sed -n 's/.*\?\(.*\)/\1/p')
+
+# Build cleaned query string, removing electric-specific params but keeping others
+if [ -n "$QUERY_STRING" ]; then
+    # Split query string into individual parameters
+    CLEANED_PARAMS=""
+    IFS='&' read -ra PARAMS <<< "$QUERY_STRING"
+    for param in "${PARAMS[@]}"; do
+        KEY=$(echo "$param" | cut -d'=' -f1)
+        # Skip electric-specific params
+        case "$KEY" in
+            "offset"|"handle"|"live") continue ;;
+            *) 
+                if [ -z "$CLEANED_PARAMS" ]; then
+                    CLEANED_PARAMS="$param"
+                else
+                    CLEANED_PARAMS="${CLEANED_PARAMS}&${param}"
+                fi
+                ;;
+        esac
+    done
+
+    # Add question mark if we have params
+    if [ -n "$CLEANED_PARAMS" ]; then
+        BASE_URL="${BASE_URL}?${CLEANED_PARAMS}&"
+    else
+        BASE_URL="${BASE_URL}?"
+    fi
+else
+    BASE_URL="${BASE_URL}?"
+fi
 
 # Directory to store individual JSON files
 OFFSET_DIR="./json_files"
 
-# Initialize the latest output variable
+# Initialize variables
 LATEST_OFFSET="-1"
+SHAPE_HANDLE=""
+IS_LIVE_MODE=false
 
 # Create the output directory if it doesn't exist
 mkdir -p "$OFFSET_DIR"
 
+# Function to extract header value from curl response
+get_header_value() {
+    local headers="$1"
+    local header_name="$2"
+    echo "$headers" | grep -i "^$header_name:" | cut -d':' -f2- | tr -d ' \r'
+}
+
 # Function to download and process JSON data
 process_json() {
     local url="$1"
-    local output_file="$2"
-
-    echo >&2 "Downloading JSON file from $url..."
-    curl -s -o "$output_file" "$url"
-
-    # Check if the file was downloaded successfully
-    if [ ! -f "$output_file" ]; then
-        echo >&2 "Failed to download the JSON file."
-        exit 1
+    local tmp_headers="headers.tmp"
+    local tmp_body="body.tmp"
+    local response_file="response.tmp"
+    local state_file="state.tmp"
+
+    echo "Downloading shape log from URL: ${url}"
+
+    # Clear any existing tmp files and create new ones
+    rm -f "$tmp_headers" "$tmp_body" "$response_file" "$state_file" xx*
+
+    # Download the entire response first
+    curl -i -s "$url" > "$response_file"
+
+    # Split at the double newline - everything before the JSON array
+    sed -n '/^\[/,$p' "$response_file" > "$tmp_body"
+    grep -B 1000 "^\[" "$response_file" | grep -v "^\[" > "$tmp_headers"
+
+    # Display prettified JSON if file is not empty
+    if [ -s "$tmp_body" ]; then
+        jq '.' < "$tmp_body"
     fi
 
-    # Check if the file is not empty
-    if [ ! -s "$output_file" ]; then
-        echo >&2 "The downloaded JSON file is empty."
-        # Return the latest OFFSET
-        echo "$LATEST_OFFSET"
-        return
-    fi
+    # Extract important headers
+    local headers=$(cat "$tmp_headers")
+    local new_handle=$(get_header_value "$headers" "electric-handle")
+    local new_offset=$(get_header_value "$headers" "electric-offset")
 
-    echo >&2 "Successfully downloaded the JSON file."
+    # Always update handle from header if present
+    if [ -n "$new_handle" ]; then
+        SHAPE_HANDLE="$new_handle"
+    fi
 
-    # Ensure the file ends with a newline
-    if [ -n "$(tail -c 1 "$output_file")" ]; then
-        echo >> "$output_file"
+    # Update offset from header
+    if [ -n "$new_offset" ]; then
+        LATEST_OFFSET="$new_offset"
     fi
 
-    # Validate the JSON structure (optional but recommended for debugging)
-    if ! jq . "$output_file" > /dev/null 2>&1; then
-        echo >&2 "Invalid JSON format in the downloaded file."
-        exit 1
+    # Check if headers were received
+    if [ ! -f "$tmp_headers" ]; then
+        echo >&2 "Failed to download response."
+        return 1
     fi
 
-    # Read the JSON file line by line and save each JSON object to an individual file
-    while IFS= read -r line; do
-        # Check if the headers array contains an object with key "operation"
-        if echo "$line" | jq -e '.headers | map(select(.key == "operation")) | length == 0' > /dev/null; then
-            # echo "Skipping line without an operation: $operation"  # Log skipping non-data objects
-            continue
+    # Process each item in the JSON array
+    jq -c '.[]' "$tmp_body" | while IFS= read -r item; do
+        # Parse the JSON message
+        if echo "$item" | jq -e '.headers.control' >/dev/null 2>&1; then
+            echo "Found control message" >&2
+            # Handle control messages
+            local control=$(echo "$item" | jq -r '.headers.control')
+            echo "Control value: $control" >&2
+            case "$control" in
+                "up-to-date")
+                    echo "true" > "$state_file"
+                    echo >&2 "Shape is up to date, switching to live mode"
+                    ;;
+                "must-refetch")
+                    echo >&2 "Server requested refetch"
+                    LATEST_OFFSET="-1"
+                    IS_LIVE_MODE=false
+                    SHAPE_HANDLE=""
+                    ;;
+            esac
         fi
+    done
 
-        key=$(echo "$line" | jq -r '.key')
-        offset=$(echo "$line" | jq -r '.offset')
-
-        if [ -z "$key" ]; then
-            echo >&2 "No key found in message: $line"  # Log if no ID is found
-        else
-            echo >&2 "Extracted key: $key"  # Log the extracted key
-            echo "$line" | jq . > "$OFFSET_DIR/json_object_$key.json"
-            echo >&2 "Written to file: $OFFSET_DIR/json_object_$key.json"  # Log file creation
-
-            LATEST_OFFSET="$offset"
-            echo >&2 "Updated latest OFFSET to: $LATEST_OFFSET"
-        fi
-    done < <(jq -c '.[]' "$output_file") 
+    # Read the state file and update IS_LIVE_MODE
+    if [ -f "$state_file" ] && [ "$(cat "$state_file")" = "true" ]; then
+        IS_LIVE_MODE=true
+    fi
 
-    echo >&2 "done with jq/read loop $LATEST_OFFSET"
+    # Cleanup
+    rm -f "$tmp_headers" "$tmp_body" "$response_file" "$state_file" xx*
 
-    # Return the latest OFFSET
-    echo "$LATEST_OFFSET"
+    return 0
 }
 
-# Main loop to poll for updates every second
+# Main loop to poll for updates
 while true; do
-    url="$BASE_URL&offset=$LATEST_OFFSET"
-    echo $url
+    # Construct URL with appropriate parameters
+    url="${BASE_URL}offset=$LATEST_OFFSET"
+    if [ -n "$SHAPE_HANDLE" ]; then
+        url="${url}&handle=$SHAPE_HANDLE"
+    fi
+    if [ "$IS_LIVE_MODE" = true ]; then
+        url="${url}&live=true"
+    fi
 
-    LATEST_OFFSET=$(process_json "$url" "shape-data.json")
+    if ! process_json "$url"; then
+        echo >&2 "Error processing response, retrying in 5 seconds..."
+        sleep 5
+        continue
+    fi
 
+    # Add small delay between requests
     sleep 1
 done
-