Merge pull request #1915 from Expensify/main

Update expensify_prod branch

docker/Dockerfile

@@ -0,0 +1,95 @@
+# Stage 1: Build
+FROM ubuntu:22.04 AS builder
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install apt-fast and necessary tools
+RUN apt-get update && apt-get install -y software-properties-common gnupg wget && \
+    add-apt-repository ppa:apt-fast/stable && \
+    apt-get update && apt-get -y install apt-fast
+
+# Configure apt-fast
+RUN echo debconf apt-fast/maxdownloads string 16 | debconf-set-selections && \
+    echo debconf apt-fast/dlflag boolean true | debconf-set-selections && \
+    echo debconf apt-fast/aptmanager string apt-get | debconf-set-selections
+
+# Add necessary GPG keys and repositories
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys BA9EF27F && \
+    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 4F4EA0AAE5267A6C && \
+    add-apt-repository ppa:ubuntu-toolchain-r/test && \
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+    echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" >> /etc/apt/sources.list.d/llvm.list
+
+# Update and install dependencies
+RUN apt-fast update && apt-fast install -y \
+    wget gnupg software-properties-common lsb-release ccache zlib1g-dev rsyslog cmake \
+    libpcre2-dev libpcre3-dev libsodium-dev libgpgme11-dev libstdc++-13-dev make \
+    linux-headers-generic git clang-18 lldb-18 lld-18 clangd-18 clang-tidy-18 \
+    clang-format-18 clang-tools-18 llvm-18-dev llvm-18-tools libomp-18-dev libc++-18-dev \
+    libc++abi-18-dev libclang-common-18-dev libclang-18-dev libclang-cpp18-dev libunwind-18-dev
+
+# Configure alternatives to use Clang 18 by default
+RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100 && \
+    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 100
+
+# Configure ccache
+ENV CCACHE_DIR=/ccache
+RUN mkdir -p $CCACHE_DIR && chmod 777 $CCACHE_DIR
+ENV PATH="/usr/lib/ccache:$PATH"
+
+WORKDIR /app/bedrock
+
+COPY . /app/bedrock
+
+ENV CC="clang-18"
+ENV CXX="clang++-18"
+
+# Build Bedrock
+RUN make clean
+RUN make -j$(nproc)
+
+# Add Bedrock to the PATH for the tests
+ENV PATH="/app/bedrock:$PATH"
+
+# Run tests
+WORKDIR /app/bedrock/test
+RUN ./test -threads 8
+WORKDIR /app/bedrock/test/clustertest
+RUN ./clustertest -threads 8
+
+# Clean up
+RUN apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
+    rm -rf /etc/apt/sources.list.d/*
+
+# Stage 2: Runtime
+FROM phusion/baseimage:noble-1.0.0
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Install necessary packages
+RUN apt-get update && apt-get install -y software-properties-common && \
+    apt-get install -y build-essential libpcre++ zlib1g && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
+    rm -rf /etc/apt/sources.list.d/*
+
+# Copy the compiled binary from the build stage
+COPY --from=builder /app/bedrock/bedrock /usr/local/bin/
+RUN chmod +x /usr/local/bin/bedrock
+
+RUN mkdir -p /var/db && \
+    touch /var/db/bedrock.db
+
+EXPOSE 8888
+EXPOSE 9000
+
+RUN mkdir -p /etc/service/bedrock/
+COPY docker/libstuff/bedrock.sh /etc/service/bedrock/run
+RUN chmod +x /etc/service/bedrock/run
+
+RUN adduser --disabled-password --gecos "" bedrock && \
+    adduser bedrock bedrock && \
+    chown -R bedrock:bedrock /var/db
+
+CMD ["/sbin/my_init"]

docker/README.md

@@ -0,0 +1,121 @@
+# Bedrock Docker Setup
+
+This folder contains the necessary files to build and run Bedrock in a Docker container.
+
+## Contents
+
+- `Dockerfile`: Defines the Docker image for Bedrock
+- `libstuff/bedrock.sh`: Startup script for Bedrock within the container
+- `docker-compose.yml`: Docker Compose configuration for running a Bedrock cluster
+
+## Dockerfile
+
+The Dockerfile sets up a container based on [`phusion/baseimage:noble-1.0.0`](https://github.com/phusion/baseimage-docker) 
+and includes the following key steps:
+
+1. Installs necessary packages
+2. Copies the Bedrock binary to the container
+3. Sets up the database directory
+4. Exposes ports 8888 and 9000
+5. Configures the startup script
+6. Creates a non-root user for running Bedrock
+
+## bedrock.sh
+
+This script is responsible for starting Bedrock with the appropriate parameters. It:
+
+1. Checks for a compatible Bash version
+2. Sets up default parameters
+3. Processes environment variables to configure Bedrock
+4. Starts Bedrock as the non-root user
+
+## Docker Compose
+
+The `docker-compose.yml` file defines a Bedrock cluster with three nodes. It includes:
+
+1. Three Bedrock services (node0, node1, node2)
+2. Network configuration for inter-node communication
+3. Volume mounts for data persistence
+4. Environment variable configuration for each node
+
+## Usage
+
+To build and run the Bedrock cluster using Docker Compose:
+
+```bash
+docker-compose up --build
+```
+
+This command will build the Docker image and start the Bedrock cluster as defined in the `docker-compose.yml` file.
+
+## build_and_extract.sh
+
+This script automates the process of building the Docker image and extracting the Bedrock binary. It performs the following steps:
+
+1. Builds the Docker image using the Dockerfile
+2. Creates a temporary container from the built image
+3. Copies the Bedrock binary from the container to a local output directory
+4. Removes the temporary container
+
+To use this script:
+
+```bash
+./build_and_extract.sh
+```
+
+After running the script, you'll find the Bedrock binary in the `../output` directory.
+
+To run a single Bedrock container:
+
+```bash
+docker run -d \
+  --name bedrock-node \
+  -p 8888:8888 \
+  -p 9000:9000 \
+  -e NODE_NAME=node1 \
+  -e PRIORITY=100 \
+  -e PEER_LIST=node2:9000,node3:9000 \
+  bedrock
+```
+
+## Environment Variables
+
+The following environment variables can be used to configure Bedrock:
+
+- `NODE_NAME`: Name of the Bedrock node
+- `PRIORITY`: Priority of the node in the cluster
+- `PEER_LIST`: Comma-separated list of peer nodes
+- `PLUGINS`: Plugins to enable
+- `CACHE_SIZE`: Size of the cache
+- `WORKER_THREADS`: Number of worker threads
+- `QUERY_LOG`: Query log configuration
+- `MAX_JOURNAL_SIZE`: Maximum journal size
+- `SYNCHRONOUS`: Synchronous mode setting
+
+Additional flags can be set using the `VERBOSE`, `QUIET`, and `CLEAN` environment variables.
+
+## Ports
+
+- 8888: HTTP server port
+- 9000: Peer communication port
+
+## Data Persistence
+
+The Bedrock database is stored at `/var/db/bedrock.db` within the container. In the Docker Compose setup, volumes are used to persist data for each node.
+
+## Logs
+
+Bedrock logs are written to `/var/log/bedrock.log` inside the container.
+
+Example of log output:
+```
+$ docker logs bedrock-node2
+bedrock-node2  | Oct 17 16:14:52 da62315a18ec bedrock: xxxxxx (SQLiteNode.cpp:1253) _onMESSAGE [sync] [info] {node2/LEADING} Received PING from peer 'node0'. Sending PONG.
+bedrock-node2  | Oct 17 16:14:52 da62315a18ec bedrock: xxxxxx (SQLiteNode.cpp:1261) _onMESSAGE [sync] [info] {node2/LEADING} Received PONG from peer 'node0' (0ms latency)
+bedrock-node2  | Oct 17 16:14:52 da62315a18ec bedrock: xxxxxx (SQLiteNode.cpp:1253) _onMESSAGE [sync] [info] {node2/LEADING} Received PING from peer 'node1'. Sending PONG.
+bedrock-node2  | Oct 17 16:14:52 da62315a18ec bedrock: xxxxxx (SQLiteNode.cpp:1261) _onMESSAGE [sync] [info] {node2/LEADING} Received PONG from peer 'node1' (0ms latency)
+```
+
+## Network
+
+The Bedrock nodes are connected to a custom bridge network `bedrock-cluster`, allowing for easy communication between nodes.

docker/build_and_extract.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Set the script to exit immediately if a command exits with a non-zero status.
+set -e
+
+# Define variables
+IMAGE_NAME="bedrock-build"
+CONTAINER_NAME="bedrock-build-container"
+OUTPUT_DIR="../output"
+
+# Ensure the output directory exists
+mkdir -p "$OUTPUT_DIR"
+
+echo "Building Docker image..."
+docker buildx build -t "$IMAGE_NAME" -f Dockerfile ..
+
+echo "Creating container..."
+docker create --name "$CONTAINER_NAME" "$IMAGE_NAME"
+
+echo "Copying bedrock binary from container..."
+docker cp "$CONTAINER_NAME:/usr/local/bin/bedrock" "$OUTPUT_DIR/bedrock"
+
+echo "Removing container..."
+docker rm "$CONTAINER_NAME"
+
+echo "Build complete. The bedrock binary is now in the $OUTPUT_DIR directory."

docker/docker-compose.yml

@@ -0,0 +1,70 @@
+# You can remove version: '3.8' if you are using Docker Compose v2 or later
+# but it's not an issue if you're using v2
+version: '3.8'
+
+services:
+  bedrock-node0:
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile
+    container_name: bedrock-node0
+    ports:
+      - "8888:8888"
+      - "9000:9000"
+    networks:
+      bedrock-cluster:
+        aliases:
+          - node0
+    volumes:
+      - bedrock-db-node0:/var/db
+    environment:
+      - NODE_NAME=node0
+      - PRIORITY=100
+      - PEER_LIST=node1:9000,node2:9000
+
+  bedrock-node1:
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile
+    container_name: bedrock-node1
+    ports:
+      - "8887:8888"
+      - "9001:9000"
+    networks:
+      bedrock-cluster:
+        aliases:
+          - node1
+    volumes:
+      - bedrock-db-node1:/var/db
+    environment:
+      - NODE_NAME=node1
+      - PRIORITY=101
+      - PEER_LIST=node0:9000,node2:9000
+
+  bedrock-node2:
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile
+    container_name: bedrock-node2
+    ports:
+      - "8889:8888"
+      - "9002:9000"
+    networks:
+      bedrock-cluster:
+        aliases:
+          - node2
+    volumes:
+      - bedrock-db-node2:/var/db
+    environment:
+      - NODE_NAME=node2
+      - PRIORITY=102
+      - PEER_LIST=node0:9000,node1:9000
+
+networks:
+  bedrock-cluster:
+    driver: bridge
+
+volumes:
+  bedrock-db-node0:
+  bedrock-db-node1:
+  bedrock-db-node2:

docker/libstuff/bedrock.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+set -e
+
+# Requires bash 4.2 or greater because of use of `-v` for testing variables being set
+# Returns 1 if version doesn't match
+function bashVersionCheck()
+{
+  [ -z $BASH_VERSION ] && return 1
+
+  # If it's set, check the version
+  case $BASH_VERSION in
+    5.*) return 0 ;;
+    4.1) return 1 ;;
+    4.0) return 1 ;;
+    4.*) return 0 ;;
+    ?) return 1;;
+  esac
+}
+
+if [[ $(bashVersionCheck) ]]; then
+  echo "Requires bash 4.2 or greater"
+  exit -1
+fi
+
+NO_ARG_PARAMS=("VERBOSE" "QUIET" "CLEAN")
+ONE_ARG_PARAMS=("SERVER_HOST" "NODE_NAME" "PEER_LIST" "PRIORITY" "PLUGINS" "CACHE_SIZE" "WORKER_THREADS" "QUERY_LOG" "MAX_JOURNAL_SIZE" "SYNCHRONOUS")
+
+function toLowerCase() {
+  echo "$1" | tr '[:upper:]' '[:lower:]'
+}
+
+function toSnakeCase()
+{
+  lowered=$(toLowerCase $1)
+  arr=(${lowered//_/ })
+  printf -v ccase %s "${arr[@]^}"
+  echo "${lowered:0:1}${ccase:1}"
+}
+
+PARAMS=""
+
+# Add default database path
+DB_PATH="/var/db/bedrock.db"
+PARAMS="$PARAMS -db $DB_PATH -nodeHost 0.0.0.0:9000 -serverHost 0.0.0.0:8888"
+
+for i in "${NO_ARG_PARAMS[@]}" ; do
+  if [[ -v "$i" ]]; then
+    export PARAMS="$PARAMS -$(toLowerCase ${i:0:1})"
+  fi
+done
+
+for i in "${ONE_ARG_PARAMS[@]}" ; do
+  if [[ -v "$i" ]]; then
+    export PARAMS="$PARAMS -$(toSnakeCase ${i}) ${!i}"
+  fi
+done
+
+exec /sbin/setuser bedrock /usr/local/bin/bedrock $PARAMS >> /var/log/bedrock.log 2>&1

libstuff/SLog.cpp

@@ -54,7 +54,8 @@ static const set<string> PARAMS_WHITELIST = {
     "companyName",
     "companyWebsite",
     "invoice",
-    "policyAccountID"
+    "policyAccountID",
+    "errorMessage"
 };
 
 string addLogParams(string&& message, const map<string, string>& params) {