From bf405088f8329d472db8e9cd9a279fc18bee52a9 Mon Sep 17 00:00:00 2001 From: Matt Green Date: Mon, 11 Nov 2024 20:33:00 -0800 Subject: [PATCH] Add dockerfile to run emit_measurements and kafka --- .dockerignore | 6 ++ Cargo.toml | 1 + Dockerfile | 97 +++++++++++++++++++++++++++ py-denormalized/README.md | 11 +++ py-denormalized/pyproject.toml | 7 +- py-denormalized/requirements-dev.lock | 1 + 6 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4ff6a5b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +target/ +.git/ +.gitignore +Dockerfile +.dockerignore +**/*.rs.bk diff --git a/Cargo.toml b/Cargo.toml index f3900f1..b263184 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ license = "Apache-2.0" readme = "README.md" repository = "https://github.com/probably-nothing-labs/denormalized.git" version = "0.0.1" +rust-version = "1.81.0" description = "Embeddable stream processing engine" [workspace.dependencies] diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..94c0a9c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,97 @@ +# docker build -t kafka-measurements:latest . +# Stage 1: Build the Rust application +FROM rust:1.81.0-slim-bookworm AS builder +WORKDIR /usr/src/app + +# Install build dependencies and zig +RUN apt-get update && apt-get install -y \ + cmake \ + g++ \ + libssl-dev \ + pkg-config \ + wget \ + xz-utils \ + && wget https://ziglang.org/download/0.13.0/zig-linux-x86_64-0.13.0.tar.xz \ + && tar -xf zig-linux-x86_64-0.13.0.tar.xz \ + && mv zig-linux-x86_64-0.13.0 /usr/local/zig \ + && rm zig-linux-x86_64-0.13.0.tar.xz \ + && rm -rf /var/lib/apt/lists/* + +# Add zig to PATH +ENV PATH="/usr/local/zig/:$PATH" + +# Install cargo-zigbuild +RUN cargo install --locked cargo-zigbuild && \ + rustup target add x86_64-unknown-linux-musl + +# Copy and build +COPY . . +RUN cargo zigbuild --target x86_64-unknown-linux-musl --release --example emit_measurements && \ + cp target/x86_64-unknown-linux-musl/release/examples/emit_measurements /tmp/ && \ + rm -rf /usr/src/app/* + +# Stage 2: Final image with Kafka and Rust binary +FROM confluentinc/cp-kafka:7.5.1 +USER root + +# Install minimal dependencies +RUN yum update -y && \ + yum install -y openssl-devel && \ + yum clean all && \ + rm -rf /var/cache/yum + +# Copy only the binary from builder stage +COPY --from=builder /tmp/emit_measurements /usr/local/bin/ + +# Create startup script +COPY </dev/null; do + echo "Waiting for Kafka to be ready..." + sleep 5 +done + +# Create topics with 1GB retention +echo "Creating temperature topic..." +kafka-topics --bootstrap-server localhost:9092 --create --if-not-exists --topic temperature --partitions 1 --replication-factor 1 --config retention.bytes=1073741824 + +echo "Creating humidity topic..." +kafka-topics --bootstrap-server localhost:9092 --create --if-not-exists --topic humidity --partitions 1 --replication-factor 1 --config retention.bytes=1073741824 + +# Start the Rust application +/usr/local/bin/emit_measurements +EOF + +RUN chmod +x /startup.sh + +ENV KAFKA_NODE_ID=1 +ENV KAFKA_PROCESS_ROLES=broker,controller +ENV KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 +ENV KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER +ENV KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT +ENV KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 +ENV KAFKA_CONTROLLER_QUORUM_VOTERS=1@localhost:9093 +ENV KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT +ENV KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 +ENV KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 +ENV KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1 + + + +# Expose Kafka port +EXPOSE 9092 + +CMD ["/startup.sh"] diff --git a/py-denormalized/README.md b/py-denormalized/README.md index 1c40bd6..7e5d693 100644 --- a/py-denormalized/README.md +++ b/py-denormalized/README.md @@ -3,6 +3,17 @@ denormalized-python Python bindings for [denormalized](https://github.com/probably-nothing-labs/denormalized) +Denormalized is a single node stream processing engine written in Rust. + +## Getting Started + +The easiest way to get started is to look at some of the examples in the [python/examples/](python/examples/) folder. + +- We're currently using [rye](https://rye.astral.sh/guide/) to manage dependencies. Run `rye sync` in the current directory create and install the venv +- If you have rust/cargo installed, you can run `maturin develop` to build the core rust code and install the bindings in the current virtual environment. +- Alternatively, + + ## Development Make sure you're in the `py-denormalized/` directory. diff --git a/py-denormalized/pyproject.toml b/py-denormalized/pyproject.toml index 88735ca..de58d3e 100644 --- a/py-denormalized/pyproject.toml +++ b/py-denormalized/pyproject.toml @@ -24,7 +24,12 @@ features = ["pyo3/extension-module"] module-name = "denormalized._d_internal" [tool.rye] -dev-dependencies = ["pip>=24.2", "ipython>=8.26.0", "pytest>=8.3.2"] +dev-dependencies = [ + "pip>=24.2", + "ipython>=8.26.0", + "pytest>=8.3.2", + "maturin>=1.7.4", +] # Enable docstring linting using the google style guide [tool.ruff.lint] diff --git a/py-denormalized/requirements-dev.lock b/py-denormalized/requirements-dev.lock index 1cfbc8f..c1d8781 100644 --- a/py-denormalized/requirements-dev.lock +++ b/py-denormalized/requirements-dev.lock @@ -25,6 +25,7 @@ jedi==0.19.1 # via ipython matplotlib-inline==0.1.7 # via ipython +maturin==1.7.4 numpy==2.1.0 # via pyarrow packaging==24.1