CLN updates

conf/cln/config

@@ -8,4 +8,3 @@ bitcoin-rpcpassword=pass
 log-level=debug
 
 experimental-offers
-experimental-onion-messages

docker-compose.yml

@@ -30,7 +30,7 @@ services:
         ipv4_address: 172.30.1.3
 
   cln1:
-    image: elementsproject/lightningd:v24.05
+    build: ./docker/cln
     restart: unless-stopped
     depends_on:
       - bitcoind
@@ -103,7 +103,7 @@ services:
         ipv4_address: 172.30.2.2
 
   cln2:
-    image: elementsproject/lightningd:v24.05
+    build: ./docker/cln
     restart: unless-stopped
     depends_on:
       - bitcoind
@@ -146,7 +146,7 @@ services:
       - "ldknode2:/root/.ldknode"
 
   cln3:
-    image: elementsproject/lightningd:v24.05
+    build: ./docker/cln
     restart: unless-stopped
     depends_on:
       - bitcoind
@@ -262,4 +262,4 @@ networks:
     ipam:
       driver: default
       config:
-        - subnet: 172.30.0.0/16
+        - subnet: 172.30.0.0/16

docker/cln/Dockerfile

@@ -0,0 +1,278 @@
+# This Dockerfile is used by buildx to build ARM64, AMD64, and ARM32 Docker images from an AMD64 host.
+# To speed up the build process, we are cross-compiling rather than relying on QEMU.
+# There are four main stages:
+# * downloader: Downloads specific binaries needed for c-lightning for each architecture.
+# * builder: Cross-compiles for each architecture.
+# * builder-python: Builds Python dependencies for cln-rest with QEMU.
+# * final: Creates the runtime image.
+
+ARG BASE_DISTRO="debian:bullseye-slim"
+
+FROM --platform=$BUILDPLATFORM ${BASE_DISTRO} as base-downloader
+RUN set -ex \
+	&& apt-get update \
+	&& apt-get install -qq --no-install-recommends ca-certificates dirmngr wget qemu-user-static binfmt-support
+
+FROM base-downloader as base-downloader-linux-amd64
+ENV TARBALL_ARCH_FINAL=x86_64-linux-gnu
+
+FROM base-downloader as base-downloader-linux-arm64
+ENV TARBALL_ARCH_FINAL=aarch64-linux-gnu
+
+FROM base-downloader as base-downloader-linux-arm
+ENV TARBALL_ARCH_FINAL=arm-linux-gnueabihf
+
+FROM base-downloader-${TARGETOS}-${TARGETARCH} as downloader
+
+RUN set -ex \
+	&& apt-get update \
+	&& apt-get install -qq --no-install-recommends ca-certificates dirmngr wget
+
+WORKDIR /opt
+
+
+ENV BITCOIN_VERSION=27.1
+ENV BITCOIN_TARBALL bitcoin-${BITCOIN_VERSION}-${TARBALL_ARCH_FINAL}.tar.gz
+ENV BITCOIN_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/$BITCOIN_TARBALL
+ENV BITCOIN_ASC_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/SHA256SUMS
+
+RUN mkdir /opt/bitcoin && cd /opt/bitcoin \
+    && wget -qO $BITCOIN_TARBALL "$BITCOIN_URL" \
+    && wget -qO bitcoin "$BITCOIN_ASC_URL" \
+    && grep $BITCOIN_TARBALL bitcoin | tee SHA256SUMS \
+    && sha256sum -c SHA256SUMS \
+    && BD=bitcoin-$BITCOIN_VERSION/bin \
+    && tar -xzvf $BITCOIN_TARBALL $BD/ --strip-components=1 \
+    && rm $BITCOIN_TARBALL
+
+ENV LITECOIN_VERSION 0.16.3
+ENV LITECOIN_URL https://download.litecoin.org/litecoin-${LITECOIN_VERSION}/linux/litecoin-${LITECOIN_VERSION}-${TARBALL_ARCH_FINAL}.tar.gz
+
+# install litecoin binaries
+RUN mkdir /opt/litecoin && cd /opt/litecoin \
+    && wget -qO litecoin.tar.gz "$LITECOIN_URL" \
+    && tar -xzvf litecoin.tar.gz litecoin-$LITECOIN_VERSION/bin/litecoin-cli --strip-components=1 --exclude=*-qt \
+    && rm litecoin.tar.gz
+
+FROM --platform=linux/amd64 ${BASE_DISTRO} as base-builder
+RUN apt-get update -qq && \
+    apt-get install -qq -y --no-install-recommends \
+        autoconf \
+        automake \
+        build-essential \
+        ca-certificates \
+        curl \
+        dirmngr \
+        gettext \
+        git \
+        gnupg \
+        jq \
+        libpq-dev \
+        libtool \
+        libffi-dev \
+        pkg-config \
+        libssl-dev \
+        protobuf-compiler \
+        python3.9 \
+        python3-dev \
+        python3-mako \
+        python3-pip \
+        python3-venv \
+        python3-setuptools \
+        libev-dev \
+        libevent-dev \
+        qemu-user-static \
+        wget \
+        unzip \
+        tclsh
+
+ENV PYTHON_VERSION=3
+RUN curl -sSL https://install.python-poetry.org | python3 -
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1
+RUN pip3 install --upgrade pip setuptools wheel
+
+RUN wget -q https://zlib.net/fossils/zlib-1.2.13.tar.gz -O zlib.tar.gz && \
+    wget -q https://www.sqlite.org/2019/sqlite-src-3290000.zip -O sqlite.zip
+
+WORKDIR /opt/lightningd
+RUN git init && \
+    git remote add origin https://github.com/ElementsProject/lightning && \
+    git fetch origin pull/7461/head:part4-corner-cases && \
+    git checkout part4-corner-cases
+COPY . /tmp/lightning
+RUN git checkout part4-corner-cases
+
+# Do not build python plugins (clnrest & wss-proxy) here, python doesn't support cross compilation.
+RUN sed -i '/^clnrest\|^wss-proxy/d' pyproject.toml && \
+    /root/.local/bin/poetry export -o requirements.txt --without-hashes
+RUN pip3 install -r requirements.txt && pip3 cache purge
+WORKDIR /
+
+FROM base-builder as base-builder-linux-amd64
+
+FROM base-builder as base-builder-linux-arm64
+ENV target_host=aarch64-linux-gnu \
+    target_host_rust=aarch64-unknown-linux-gnu \
+    target_host_qemu=qemu-aarch64-static
+
+RUN apt-get install -qq -y --no-install-recommends \
+        libc6-arm64-cross \
+        gcc-${target_host} \
+        g++-${target_host}
+
+ENV AR=${target_host}-ar \
+AS=${target_host}-as \
+CC=${target_host}-gcc \
+CXX=${target_host}-g++ \
+LD=${target_host}-ld \
+STRIP=${target_host}-strip \
+QEMU_LD_PREFIX=/usr/${target_host} \
+HOST=${target_host} \
+TARGET=${target_host_rust} \
+RUSTUP_INSTALL_OPTS="--target ${target_host_rust} --default-host ${target_host_rust}" \
+PKG_CONFIG_PATH="/usr/${target_host}/lib/pkgconfig"
+
+ENV \
+ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \
+SQLITE_CONFIG="--host=${target_host} --prefix=$QEMU_LD_PREFIX"
+
+FROM base-builder as base-builder-linux-arm
+
+ENV target_host=arm-linux-gnueabihf \
+    target_host_rust=armv7-unknown-linux-gnueabihf \
+    target_host_qemu=qemu-arm-static
+
+RUN apt-get install -qq -y --no-install-recommends \
+        libc6-armhf-cross \
+        gcc-${target_host} \
+        g++-${target_host}
+
+ENV AR=${target_host}-ar \
+AS=${target_host}-as \
+CC=${target_host}-gcc \
+CXX=${target_host}-g++ \
+LD=${target_host}-ld \
+STRIP=${target_host}-strip \
+QEMU_LD_PREFIX=/usr/${target_host} \
+HOST=${target_host} \
+TARGET=${target_host_rust} \
+RUSTUP_INSTALL_OPTS="--target ${target_host_rust} --default-host ${target_host_rust}" \
+PKG_CONFIG_PATH="/usr/${target_host}/lib/pkgconfig"
+
+ENV \
+ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \
+SQLITE_CONFIG="--host=${target_host} --prefix=$QEMU_LD_PREFIX"
+
+FROM base-builder-${TARGETOS}-${TARGETARCH} as builder
+
+ENV LIGHTNINGD_VERSION=master
+
+RUN mkdir zlib && tar xvf zlib.tar.gz -C zlib --strip-components=1 \
+    && cd zlib \
+    && ./configure ${ZLIB_CONFIG} \
+    && make \
+    && make install && cd .. && \
+    rm zlib.tar.gz && \
+    rm -rf zlib
+
+RUN unzip sqlite.zip \
+    && cd sqlite-* \
+    && ./configure --enable-static --disable-readline --disable-threadsafe --disable-load-extension ${SQLITE_CONFIG} \
+    && make \
+    && make install && cd .. && rm sqlite.zip && rm -rf sqlite-*
+
+ENV RUST_PROFILE=release
+ENV PATH=$PATH:/root/.cargo/bin/
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ${RUSTUP_INSTALL_OPTS}
+RUN rustup toolchain install stable --component rustfmt --allow-downgrade
+
+COPY --from=downloader /usr/bin/${target_host_qemu} /usr/bin/${target_host_qemu}
+WORKDIR /opt/lightningd
+
+# If cross-compiling, need to tell it to cargo.
+RUN ( ! [ -n "${target_host}" ] ) || \
+    (mkdir -p .cargo && echo "[target.${target_host_rust}]\nlinker = \"${target_host}-gcc\"" > .cargo/config)
+
+# Weird errors with cargo for cln-grpc on arm7 https://github.com/ElementsProject/lightning/issues/6596
+RUN ( ! [ "${target_host}" = "arm-linux-gnueabihf" ] ) || \
+    (sed -i '/documentation = "https:\/\/docs.rs\/cln-grpc"/a include = ["**\/*.*"]' cln-grpc/Cargo.toml)
+
+# Ensure that the desired grpcio-tools & protobuf versions are installed
+# https://github.com/ElementsProject/lightning/pull/7376#issuecomment-2161102381
+RUN /root/.local/bin/poetry lock --no-update && \
+    /root/.local/bin/poetry install
+
+RUN ./configure --prefix=/tmp/lightning_install --enable-static && \
+    make && \
+    /root/.local/bin/poetry run make install
+
+# We need to build python plugins on the target's arch because python doesn't support cross build
+FROM ${BASE_DISTRO} as builder-python
+RUN apt-get update -qq && \
+    apt-get install -qq -y --no-install-recommends \
+        git \
+        curl \
+        libtool \
+        pkg-config \
+        autoconf \
+        automake \
+        build-essential \
+        libffi-dev \
+        libssl-dev \
+        python3.9 \
+        python3-dev \
+        python3-pip && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN curl -sSL https://install.python-poetry.org | python3 -
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1
+
+ENV PYTHON_VERSION=3
+WORKDIR /opt/lightningd
+
+COPY plugins/clnrest/pyproject.toml plugins/clnrest/pyproject.toml
+COPY plugins/wss-proxy/pyproject.toml plugins/wss-proxy/pyproject.toml
+
+RUN cd plugins/clnrest && \
+    /root/.local/bin/poetry export -o requirements.txt --without-hashes && \
+    pip3 install -r requirements.txt && \
+    cd /opt/lightningd
+
+RUN cd plugins/wss-proxy && \
+    /root/.local/bin/poetry export -o requirements.txt --without-hashes && \
+    pip3 install -r requirements.txt && \
+    cd /opt/lightningd && \
+    pip3 cache purge
+
+FROM ${BASE_DISTRO} as final
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+      tini \
+      socat \
+      inotify-tools \
+      jq \
+      python3.9 \
+      python3-pip \
+      libpq5 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV LIGHTNINGD_DATA=/root/.lightning
+ENV LIGHTNINGD_RPC_PORT=9835
+ENV LIGHTNINGD_PORT=9735
+ENV LIGHTNINGD_NETWORK=bitcoin
+
+RUN mkdir $LIGHTNINGD_DATA && \
+    touch $LIGHTNINGD_DATA/config
+VOLUME [ "/root/.lightning" ]
+
+COPY --from=builder /tmp/lightning_install/ /usr/local/
+COPY --from=builder-python /usr/local/lib/python3.9/dist-packages/ /usr/local/lib/python3.9/dist-packages/
+COPY --from=downloader /opt/bitcoin/bin /usr/bin
+COPY --from=downloader /opt/litecoin/bin /usr/bin
+COPY entrypoint.sh entrypoint.sh
+
+EXPOSE 9735 9835
+ENTRYPOINT  [ "/usr/bin/tini", "-g", "--", "./entrypoint.sh" ]

docker/cln/entrypoint.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+: "${EXPOSE_TCP:=false}"
+
+networkdatadir="${LIGHTNINGD_DATA}/${LIGHTNINGD_NETWORK}"
+
+set -m
+lightningd --network="${LIGHTNINGD_NETWORK}" "$@" &
+
+echo "Core-Lightning starting"
+while read -r i; do if [ "$i" = "lightning-rpc" ]; then break; fi; done \
+    < <(inotifywait -e create,open --format '%f' --quiet "${networkdatadir}" --monitor)
+
+if [ "$EXPOSE_TCP" == "true" ]; then
+    echo "Core-Lightning started, RPC available on port $LIGHTNINGD_RPC_PORT"
+
+    socat "TCP4-listen:$LIGHTNINGD_RPC_PORT,fork,reuseaddr" "UNIX-CONNECT:${networkdatadir}/lightning-rpc" &
+fi
+
+# Now run any scripts which exist in the lightning-poststart.d directory
+if [ -d "$LIGHTNINGD_DATA"/lightning-poststart.d ]; then
+    for f in "$LIGHTNINGD_DATA"/lightning-poststart.d/*; do
+	"$f"
+    done
+fi
+
+fg %-

docker/cln/plugins/clnrest/pyproject.toml

@@ -0,0 +1,22 @@
+[tool.poetry]
+name = "clnrest"
+version = "24.05rc2"
+description = "Transforms RPC calls into REST APIs"
+authors = ["ShahanaFarooqui <[email protected]>"]
+
+[tool.poetry.dependencies]
+python = "^3.8"
+json5 = "^0.9.14"
+flask = "^2.3.3"
+flask-restx = "^1.1.0"
+gunicorn = "^21.2.0"
+pyln-client = "^24.2"
+flask-socketio = "^5.3.6"
+gevent = "^23.9.0.post1"
+gevent-websocket = "^0.10.1"
+flask-cors = "^4.0.0"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

docker/cln/plugins/wss-proxy/pyproject.toml

@@ -0,0 +1,14 @@
+[tool.poetry]
+name = "wss-proxy"
+version = "24.05rc2"
+description = "Web secure socket proxy"
+authors = ["ShahanaFarooqui <[email protected]>"]
+
+[tool.poetry.dependencies]
+python = "^3.8"
+pyln-client = "^24.2"
+websockets = "^12.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"