Add dockerfile to run emit_measurements and kafka

.dockerignore

@@ -0,0 +1,6 @@
+target/
+.git/
+.gitignore
+Dockerfile
+.dockerignore
+**/*.rs.bk

Cargo.toml

@@ -17,6 +17,7 @@ license = "Apache-2.0"
 readme = "README.md"
 repository = "https://github.com/probably-nothing-labs/denormalized.git"
 version = "0.0.1"
+rust-version = "1.81.0"
 description = "Embeddable stream processing engine"
 
 [workspace.dependencies]

Dockerfile

@@ -0,0 +1,97 @@
+# docker build -t kafka-measurements:latest .
+# Stage 1: Build the Rust application
+FROM rust:1.81.0-slim-bookworm AS builder
+WORKDIR /usr/src/app
+
+# Install build dependencies and zig
+RUN apt-get update && apt-get install -y \
+    cmake \
+    g++ \
+    libssl-dev \
+    pkg-config \
+    wget \
+    xz-utils \
+    && wget https://ziglang.org/download/0.13.0/zig-linux-x86_64-0.13.0.tar.xz \
+    && tar -xf zig-linux-x86_64-0.13.0.tar.xz \
+    && mv zig-linux-x86_64-0.13.0 /usr/local/zig \
+    && rm zig-linux-x86_64-0.13.0.tar.xz \
+    && rm -rf /var/lib/apt/lists/*
+
+# Add zig to PATH
+ENV PATH="/usr/local/zig/:$PATH"
+
+# Install cargo-zigbuild
+RUN cargo install --locked cargo-zigbuild && \
+    rustup target add x86_64-unknown-linux-musl
+
+# Copy and build
+COPY . .
+RUN cargo zigbuild --target x86_64-unknown-linux-musl --release --example emit_measurements && \
+    cp target/x86_64-unknown-linux-musl/release/examples/emit_measurements /tmp/ && \
+    rm -rf /usr/src/app/*
+
+# Stage 2: Final image with Kafka and Rust binary
+FROM confluentinc/cp-kafka:7.5.1
+USER root
+
+# Install minimal dependencies
+RUN yum update -y && \
+    yum install -y openssl-devel && \
+    yum clean all && \
+    rm -rf /var/cache/yum
+
+# Copy only the binary from builder stage
+COPY --from=builder /tmp/emit_measurements /usr/local/bin/
+
+# Create startup script
+COPY <<EOF /startup.sh
+#!/bin/bash
+# Generate Cluster ID if not provided
+CLUSTER_ID=\${CLUSTER_ID:-\$(kafka-storage random-uuid)}
+echo "Using Cluster ID: \$CLUSTER_ID"
+export CLUSTER_ID
+
+# Format storage directory
+echo "Formatting storage directory..."
+kafka-storage format -t \$CLUSTER_ID -c /etc/kafka/kraft/server.properties
+
+# Start Kafka with KRaft
+/etc/confluent/docker/run &
+
+# Wait for Kafka to be ready
+until kafka-topics --bootstrap-server localhost:9092 --list &>/dev/null; do
+  echo "Waiting for Kafka to be ready..."
+  sleep 5
+done
+
+# Create topics with 1GB retention
+echo "Creating temperature topic..."
+kafka-topics --bootstrap-server localhost:9092 --create --if-not-exists --topic temperature --partitions 1 --replication-factor 1 --config retention.bytes=1073741824
+
+echo "Creating humidity topic..."
+kafka-topics --bootstrap-server localhost:9092 --create --if-not-exists --topic humidity --partitions 1 --replication-factor 1 --config retention.bytes=1073741824
+
+# Start the Rust application
+/usr/local/bin/emit_measurements
+EOF
+
+RUN chmod +x /startup.sh
+
+ENV KAFKA_NODE_ID=1
+ENV KAFKA_PROCESS_ROLES=broker,controller
+ENV KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
+ENV KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER
+ENV KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+ENV KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092
+ENV KAFKA_CONTROLLER_QUORUM_VOTERS=1@localhost:9093
+ENV KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
+ENV KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+ENV KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
+ENV KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1
+
+
+
+# Expose Kafka port
+EXPOSE 9092
+
+CMD ["/startup.sh"]

py-denormalized/README.md

@@ -3,6 +3,17 @@ denormalized-python
 
 Python bindings for [denormalized](https://github.com/probably-nothing-labs/denormalized)
 
+Denormalized is a single node stream processing engine written in Rust.
+
+## Getting Started
+
+The easiest way to get started is to look at some of the examples in the [python/examples/](python/examples/) folder.
+
+- We're currently using [rye](https://rye.astral.sh/guide/) to manage dependencies. Run `rye sync` in the current directory create and install the venv
+- If you have rust/cargo installed, you can run `maturin develop` to build the core rust code and install the bindings in the current virtual environment.
+- Alternatively, 
+
+
 ## Development
 
 Make sure you're in the `py-denormalized/` directory.

py-denormalized/pyproject.toml

@@ -24,7 +24,12 @@ features = ["pyo3/extension-module"]
 module-name = "denormalized._d_internal"
 
 [tool.rye]
-dev-dependencies = ["pip>=24.2", "ipython>=8.26.0", "pytest>=8.3.2"]
+dev-dependencies = [
+    "pip>=24.2",
+    "ipython>=8.26.0",
+    "pytest>=8.3.2",
+    "maturin>=1.7.4",
+]
 
 # Enable docstring linting using the google style guide
 [tool.ruff.lint]

py-denormalized/requirements-dev.lock

@@ -25,6 +25,7 @@ jedi==0.19.1
     # via ipython
 matplotlib-inline==0.1.7
     # via ipython
+maturin==1.7.4
 numpy==2.1.0
     # via pyarrow
 packaging==24.1