Skip to content

[M] Deploy Ten Gateway Backend ( uat-testnet ) #601

[M] Deploy Ten Gateway Backend ( uat-testnet )

[M] Deploy Ten Gateway Backend ( uat-testnet ) #601

# Deploys TEN Gateway on Azure for Testnet
# Builds the TEN Gateway image, pushes the image to dockerhub and starts the TEN Gateway on Azure VM
# This action requires the following environment variables to be set:
# - DOCKER_BUILD_TAG_GATEWAY
# - AZURE_DEPLOY_GROUP_GATEWAY
# - L2_RPC_URL_VALIDATOR
# - GATEWAY_RATE_LIMIT_USER_COMPUTE_TIME
# - GATEWAY_RATE_LIMIT_WINDOW
# - GATEWAY_MAX_CONCURRENT_REQUESTS_PER_USER
# If we are deploying to a non primary instance all those variables should be prefixed with the instance name
# example: DEXYNTH_DOCKER_BUILD_TAG_GATEWAY
name: '[M] Deploy Ten Gateway Backend'
run-name: '[M] Deploy Ten Gateway Backend ( ${{ github.event.inputs.testnet_type }} )'
on:
workflow_dispatch:
inputs:
testnet_type:
description: "Environment"
required: true
default: "dev-testnet"
type: choice
options:
- "dev-testnet"
- "uat-testnet"
- "sepolia-testnet"
instance_type:
description: "Instance"
required: true
default: "primary"
type: choice
options:
- "primary"
- "DEXYNTH"
jobs:
validate-inputs:
runs-on: ubuntu-latest
steps:
- name: "Check if deployment is allowed"
run: |
if [[ "${{ github.event.inputs.instance_type }}" == "DEXYNTH" && "${{ github.event.inputs.testnet_type }}" != "sepolia-testnet" ]]; then
echo "Error: Dexynth can only be deployed to sepolia-testnet."
exit 1
fi
build-and-deploy:
runs-on: ubuntu-latest
environment:
name: ${{ github.event.inputs.testnet_type }}
steps:
- name: "Set up environment variables"
id: setup_env
run: |
INSTANCE_SUFFIX=""
INSTANCE_PREFIX=""
if [[ "${{ github.event.inputs.instance_type }}" != "primary" ]]; then
INSTANCE_SUFFIX="_${{ github.event.inputs.instance_type }}"
INSTANCE_SUFFIX2="-${{ github.event.inputs.instance_type }}"
INSTANCE_PREFIX="${{ github.event.inputs.instance_type }}_"
fi
echo "INSTANCE_SUFFIX=$INSTANCE_SUFFIX" >> $GITHUB_ENV
echo "INSTANCE_PREFIX=$INSTANCE_PREFIX" >> $GITHUB_ENV
# Set infrastructure variables
PUBLIC_IP="${{ github.event.inputs.testnet_type }}-OG-static${INSTANCE_SUFFIX2,,}"
DNS_NAME="obscurogateway-${{ github.event.inputs.testnet_type }}${INSTANCE_SUFFIX2,,}"
VM_NAME="${{ github.event.inputs.testnet_type }}-OG-${{ github.run_number }}${INSTANCE_SUFFIX}"
DEPLOY_GROUP="ObscuroGateway-${{ github.event.inputs.testnet_type }}-${{ github.run_number }}${INSTANCE_SUFFIX}"
VNET_NAME="ObscuroGateway-${{ github.event.inputs.testnet_type }}-01VNET${INSTANCE_SUFFIX}"
SUBNET_NAME="ObscuroGateway-${{ github.event.inputs.testnet_type }}-01Subnet${INSTANCE_SUFFIX}"
echo "PUBLIC_IP=$PUBLIC_IP" >> $GITHUB_ENV
echo "DNS_NAME=$DNS_NAME" >> $GITHUB_ENV
echo "VM_NAME=$VM_NAME" >> $GITHUB_ENV
echo "DEPLOY_GROUP=$DEPLOY_GROUP" >> $GITHUB_ENV
echo "VNET_NAME=$VNET_NAME" >> $GITHUB_ENV
echo "SUBNET_NAME=$SUBNET_NAME" >> $GITHUB_ENV
# Set instance-specific variables
declare -a VAR_NAMES=(
"DOCKER_BUILD_TAG_GATEWAY"
"AZURE_DEPLOY_GROUP_GATEWAY"
"L2_RPC_URL_VALIDATOR"
"GATEWAY_RATE_LIMIT_USER_COMPUTE_TIME"
"GATEWAY_RATE_LIMIT_WINDOW"
"GATEWAY_MAX_CONCURRENT_REQUESTS_PER_USER"
"GATEWAY_KEY_EXCHANGE_URL"
"GATEWAY_TLS_DOMAIN"
)
for VAR_NAME in "${VAR_NAMES[@]}"; do
FULL_VAR_NAME="${INSTANCE_PREFIX}${VAR_NAME}"
VAR_VALUE=$(jq -r --arg key "$FULL_VAR_NAME" '.[$key] // empty' <<< '${{ toJson(vars) }}')
if [[ -n "$VAR_VALUE" ]]; then
echo "${VAR_NAME}=${VAR_VALUE}" >> $GITHUB_ENV
else
echo "Warning: ${FULL_VAR_NAME} not found in vars" >&2
fi
done
- name: "Print environment variables"
# This is a useful record of what the environment variables were at the time the job ran, for debugging and reference
run: |
echo "INSTANCE_SUFFIX: $INSTANCE_SUFFIX"
echo "INSTANCE_PREFIX: $INSTANCE_PREFIX"
echo "PUBLIC_IP: $PUBLIC_IP"
echo "DNS_NAME: $DNS_NAME"
echo "VM_NAME: $VM_NAME"
echo "DEPLOY_GROUP: $DEPLOY_GROUP"
echo "VNET_NAME: $VNET_NAME"
echo "SUBNET_NAME: $SUBNET_NAME"
echo "DOCKER_BUILD_TAG_GATEWAY: $DOCKER_BUILD_TAG_GATEWAY"
echo "AZURE_DEPLOY_GROUP_GATEWAY: $AZURE_DEPLOY_GROUP_GATEWAY"
echo "L2_RPC_URL_VALIDATOR: $L2_RPC_URL_VALIDATOR"
echo "GATEWAY_RATE_LIMIT_USER_COMPUTE_TIME: $GATEWAY_RATE_LIMIT_USER_COMPUTE_TIME"
echo "GATEWAY_RATE_LIMIT_WINDOW: $GATEWAY_RATE_LIMIT_WINDOW"
echo "GATEWAY_MAX_CONCURRENT_REQUESTS_PER_USER: $GATEWAY_MAX_CONCURRENT_REQUESTS_PER_USER"
echo "GATEWAY_KEY_EXCHANGE_URL: $GATEWAY_KEY_EXCHANGE_URL"
echo "GATEWAY_TLS_DOMAIN: $GATEWAY_TLS_DOMAIN"
- name: "Print GitHub variables"
# This is a useful record of what the environment variables were at the time the job ran, for debugging and reference
run: |
echo "GitHub Variables = ${{ toJSON(vars) }}"
- uses: actions/checkout@v4
- name: "Extract branch name"
shell: bash
run: |
echo "Branch Name: ${GITHUB_REF_NAME}"
echo "BRANCH_NAME=${GITHUB_REF_NAME}" >> $GITHUB_ENV
- name: "Set up Docker"
uses: docker/setup-buildx-action@v1
- name: "Login to Azure docker registry"
uses: azure/docker-login@v1
with:
login-server: testnetobscuronet.azurecr.io
username: testnetobscuronet
password: ${{ secrets.REGISTRY_PASSWORD }}
- name: "Login via Azure CLI"
uses: azure/login@v1
with:
creds: ${{ secrets.AZURE_CREDENTIALS }}
- name: Build and Push Docker Image
run: |
DOCKER_BUILDKIT=1 docker build --build-arg TESTNET_TYPE=${{ github.event.inputs.testnet_type }} -t ${{ env.DOCKER_BUILD_TAG_GATEWAY }} -f ./tools/walletextension/enclave.Dockerfile .
docker push ${{ env.DOCKER_BUILD_TAG_GATEWAY }}
# This will fail some deletions due to resource dependencies ( ie. you must first delete the vm before deleting the disk)
- name: "Delete deployed VMs"
uses: azure/CLI@v1
with:
inlineScript: |
$(az resource list --tag ${{ env.AZURE_DEPLOY_GROUP_GATEWAY }}=true --query '[]."id"' -o tsv | xargs -n1 az resource delete --verbose -g Testnet --ids) || true
# This will clean up any lingering dependencies - might fail if there are no resources to cleanup
- name: "Delete VMs dependencies"
uses: azure/CLI@v1
with:
inlineScript: |
$(az resource list --tag ${{ env.AZURE_DEPLOY_GROUP_GATEWAY }}=true --query '[]."id"' -o tsv | xargs -n1 az resource delete --verbose -g Testnet --ids) || true
- name: "Ensure VM Static Public IP Exists"
uses: azure/CLI@v1
with:
inlineScript: |
az network public-ip show -g Testnet -n "${{ env.PUBLIC_IP }}" || az network public-ip create -g Testnet -n "${{ env.PUBLIC_IP }}" --allocation-method Static --sku Standard
- name: "Assign/Update DNS Name for Public IP"
uses: azure/CLI@v1
with:
inlineScript: |
existing_dns_name=$(az network public-ip show -g Testnet -n "${{ env.PUBLIC_IP }}" --query dnsSettings.domainNameLabel -o tsv)
if [ -z "$existing_dns_name" ]; then
az network public-ip update -g Testnet -n "${{ env.PUBLIC_IP }}" --dns-name "${{ env.DNS_NAME }}"
fi
- name: "Create VM for Gateway node on Azure"
uses: azure/CLI@v1
with:
inlineScript: |
az vm create -g Testnet -n "${{ env.VM_NAME }}" \
--admin-username obscurouser --admin-password "${{ secrets.OBSCURO_NODE_VM_PWD }}" \
--public-ip-address "${{ env.PUBLIC_IP }}" \
--tags deploygroup="${{ env.DEPLOY_GROUP }}" ${{ env.AZURE_DEPLOY_GROUP_GATEWAY }}=true \
--vnet-name "${{ env.VNET_NAME }}" --subnet "${{ env.SUBNET_NAME }}" \
--size Standard_DC2s_v3 --storage-sku StandardSSD_LRS --image ObscuroConfUbuntu \
--authentication-type password
- name: "Open TEN node-${{ matrix.host_id }} ports on Azure"
uses: azure/CLI@v1
with:
inlineScript: |
az vm open-port -g Testnet -n "${{ env.VM_NAME }}" --port 80,81,443
# To overcome issues with critical VM resources being unavailable, we need to wait for the VM to be ready
- name: "Allow time for VM initialization"
shell: bash
run: sleep 30
- name: "Start TEN Gateway on Azure"
uses: azure/CLI@v1
with:
inlineScript: |
az vm run-command invoke -g Testnet -n "${{ env.VM_NAME }}" \
--command-id RunShellScript \
--scripts '
set -e
mkdir -p /home/obscuro
# Wait for dpkg lock to be released
while sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do
echo "Waiting for dpkg lock to be released..."
sleep 1
done
# Proceed with package installations
sudo apt-get update
sudo apt-get install -y gcc
sudo snap refresh && sudo snap install --channel=1.18 go --classic
# Wait again before running get-docker.sh
while sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do
echo "Waiting for dpkg lock to be released before installing Docker..."
sleep 1
done
curl -fsSL https://get.docker.com -o get-docker.sh && sh ./get-docker.sh
git clone --depth 1 -b "${{ env.BRANCH_NAME }}" https://github.com/ten-protocol/go-ten.git /home/obscuro/go-obscuro
docker network create --driver bridge node_network || true
cd /home/obscuro/go-obscuro/
# Promtail Integration Start
mkdir -p /home/obscuro/metrics
cat <<EOF > /home/obscuro/metrics/promtail-config.yaml
server:
http_listen_port: 9080
grpc_listen_port: 0
positions:
filename: /tmp/positions.yaml
clients:
- url: "${{ vars.LOKI_URI }}"
batchwait: 3s
batchsize: 1048576
tls_config:
insecure_skip_verify: true
basic_auth:
username: "${{ secrets.LOKI_USER }}"
password: "${{ secrets.LOKI_PASSWORD }}"
scrape_configs:
- job_name: flog_scrape
docker_sd_configs:
- host: unix:///var/run/docker.sock
refresh_interval: 5s
relabel_configs:
- source_labels: ["__meta_docker_container_name"]
regex: "/(.*)"
target_label: "container"
- source_labels: ["__meta_docker_container_log_stream"]
target_label: "logstream"
- source_labels: ["__meta_docker_container_label_logging_jobname"]
target_label: "job"
- replacement: "${{ env.VM_NAME }}"
target_label: "node_name"
EOF
docker run -d --name promtail \
--network node_network \
-e HOSTNAME="${{ env.VM_NAME }}" \
-v /var/log:/var/log \
-v /home/obscuro/metrics:/etc/promtail \
-v /var/lib/docker/containers:/var/lib/docker/containers:ro \
-v /var/run/docker.sock:/var/run/docker.sock \
grafana/promtail:latest \
-config.file=/etc/promtail/promtail-config.yaml -config.expand-env=true
cat <<EOF > /home/obscuro/metrics/prometheus.yaml
global:
scrape_interval: 15s
evaluation_interval: 15s
remote_write:
- url: "${{ vars.PROMETHEUS_URI }}"
tls_config:
insecure_skip_verify: true
basic_auth:
username: "${{ secrets.LOKI_USER }}"
password: "${{ secrets.LOKI_PASSWORD }}"
scrape_configs:
# Node metrics
- job_name: node-${{ env.VM_NAME }}
scrape_interval: 5s # Frequent scrapes for node metrics
static_configs:
- targets:
- node_exporter:9100 # Node Exporter instance
relabel_configs:
- source_labels: [job]
target_label: 'node'
replacement: node-${{ env.VM_NAME }}
# Container metrics
- job_name: container-${{ env.VM_NAME }}
scrape_interval: 5s
static_configs:
- targets:
- cadvisor:8080 # cAdvisor instance for container metrics
relabel_configs:
- source_labels: [job]
target_label: 'node'
replacement: container-${{ env.VM_NAME }}
EOF
docker volume create prometheus-data
docker run -d --name prometheus \
--network node_network \
-p 9090:9090 \
-v /home/obscuro/metrics/prometheus.yaml:/etc/prometheus/prometheus.yml \
-v prometheus-data:/prometheus \
prom/prometheus:latest \
--config.file=/etc/prometheus/prometheus.yml
docker run -d --name node_exporter \
--network node_network \
-p 9100:9100 \
--pid="host" \
-v /:/host:ro \
quay.io/prometheus/node-exporter:latest \
--path.rootfs=/host
docker run -d --name cadvisor \
--network node_network \
-p 8080:8080 \
--privileged \
-v /:/rootfs:ro \
-v /var/run:/var/run:ro \
-v /sys:/sys:ro \
-v /var/lib/docker/:/var/lib/docker:ro \
-v /dev/disk/:/dev/disk:ro \
gcr.io/cadvisor/cadvisor:latest
# Promtail Integration End
# Create a named volume for persistence
docker volume create "${{ env.VM_NAME }}-data"
# Start Ten Gateway Container
docker run -d -p 80:80 -p 81:81 -p 443:443 --name "${{ env.VM_NAME }}" \
--device /dev/sgx_enclave --device /dev/sgx_provision \
-v "${{ env.VM_NAME }}-data:/data" \
-e OBSCURO_GATEWAY_VERSION="${{ github.run_number }}-${{ github.sha }}" \
-e OE_SIMULATION=0 \
"${{ env.DOCKER_BUILD_TAG_GATEWAY }}" \
ego run /home/ten/go-ten/tools/walletextension/main/main \
-host=0.0.0.0 -port=443 -portWS=443 -nodeHost="${{ env.L2_RPC_URL_VALIDATOR }}" -verbose=true \
-logPath=sys_out -dbType=cosmosDB -dbConnectionURL="${{ secrets.COSMOS_DB_CONNECTION_STRING }}" \
-rateLimitUserComputeTime="${{ env.GATEWAY_RATE_LIMIT_USER_COMPUTE_TIME }}" \
-rateLimitWindow="${{ env.GATEWAY_RATE_LIMIT_WINDOW }}" \
-maxConcurrentRequestsPerUser="${{ env.GATEWAY_MAX_CONCURRENT_REQUESTS_PER_USER }}" \
-keyExchangeURL="${{ env.GATEWAY_KEY_EXCHANGE_URL }}" \
-insideEnclave=true \
-enableTLS=true \
-tlsDomain="${{ env.GATEWAY_TLS_DOMAIN }}"
# After starting the container, verify the volume mount
docker exec "${{ env.VM_NAME }}" sh -c "
echo \"Checking volume mount...\";
df -h | grep /data;
echo \"Directory listing:\";
ls -la /data;
echo \"Current working directory:\";
pwd;
echo \"Directory permissions:\";
ls -la /;
echo \"Process status:\";
ps aux;
"
'