diff --git a/Dockerfile b/Dockerfile
index 206a97e8..6530b614 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,11 +2,6 @@ FROM debian:12 as build-env
 
 WORKDIR /src
 
-# Copy all the source files
-# .dockerignore ignores the target dir
-# This includes the rust-toolchain.toml
-COPY . .
-
 # Install dependencies
 RUN apt-get update && \
     apt-get install -y git curl build-essential libssl-dev texinfo libcap2-bin pkg-config
@@ -15,6 +10,9 @@ RUN apt-get update && \
 # Install rustup
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
 
+# Copy only rust-toolchain.toml for better caching
+COPY ./rust-toolchain.toml ./rust-toolchain.toml
+
 # Set environment variables
 ENV PATH="/root/.cargo/bin:${PATH}"
 ENV RUSTUP_HOME="/root/.rustup"
@@ -23,7 +21,44 @@ ENV CARGO_HOME="/root/.cargo"
 # Install the toolchain
 RUN rustup component add cargo
 
+# TODO: Hacky but it works
+RUN mkdir -p ./src
+RUN mkdir -p ./crates/cognitoauth/src
+RUN mkdir -p ./crates/micro-oz/src
+RUN mkdir -p ./crates/oz-api/src
+RUN mkdir -p ./crates/postgres-docker-utils/src
+RUN mkdir -p ./crates/tx-sitter-client/src
+RUN mkdir -p ./e2e_tests/scenarios/src
+
+# Copy only Cargo.toml for better caching
+COPY ./build.rs ./build.rs
+COPY ./Cargo.toml ./Cargo.toml
+COPY ./Cargo.lock ./Cargo.lock
+COPY ./crates/cognitoauth/Cargo.toml ./crates/cognitoauth/Cargo.toml
+COPY ./crates/micro-oz/Cargo.toml ./crates/micro-oz/Cargo.toml
+COPY ./crates/oz-api/Cargo.toml ./crates/oz-api/Cargo.toml
+COPY ./crates/postgres-docker-utils/Cargo.toml ./crates/postgres-docker-utils/Cargo.toml
+COPY ./crates/tx-sitter-client/Cargo.toml ./crates/tx-sitter-client/Cargo.toml
+COPY ./e2e_tests/scenarios/Cargo.toml ./e2e_tests/scenarios/Cargo.toml
+
+RUN echo "fn main() {}" > ./src/main.rs
+RUN echo "fn main() {}" > ./crates/cognitoauth/src/main.rs
+RUN echo "fn main() {}" > ./crates/micro-oz/src/main.rs
+RUN echo "fn main() {}" > ./crates/oz-api/src/main.rs
+RUN echo "fn main() {}" > ./crates/postgres-docker-utils/src/main.rs
+RUN echo "fn main() {}" > ./crates/tx-sitter-client/src/main.rs
+RUN echo "fn main() {}" > ./e2e_tests/scenarios/src/main.rs
+
+# Prebuild dependencies
+RUN cargo fetch
+RUN cargo build --release --workspace
+
+# Copy all the source files
+# .dockerignore ignores the target dir
+COPY . .
+
 # Build the sequencer
+RUN cargo fetch
 RUN cargo build --release
 
 # cc variant because we need libgcc and others