diff --git a/cpu/Dockerfile b/cpu/Dockerfile
index 77f1093..6dbfe39 100644
--- a/cpu/Dockerfile
+++ b/cpu/Dockerfile
@@ -47,10 +47,20 @@ ENV LANG C.UTF-8
 ENV TZ UTC
 
 RUN curl -Ls https://github.com/r-lib/rig/releases/download/latest/rig-linux-latest.tar.gz | sudo tar xz -C /usr/local && \
-  rig add ${R_VERSION} && \
-  rm -rf /var/lib/apt/lists/*
+  rig add ${R_VERSION} --without-pak && \
+  rm -rf /var/lib/apt/lists/* && \
+  rm -rf /tmp/rig && \
+  apt-get clean
 
-RUN Rscript -e "file.copy('tmp/Rprofile.site', file.path(R.home(), 'etc', 'Rprofile.site'))"
+# We want to install pak into the system library, so we don't do it with rig
+RUN for inst in $(ls /opt/R/*/lib/R/bin/INSTALL); do \
+      sed -i '1s/^/#!\/bin\/bash\n/' ${inst}; \
+    done; \
+    versions="$(rig ls --json | grep '"name"' | sed 's/^.*\"name\": \"//' | sed 's/\",//')" && \
+    for ver in $versions; do \
+      R-${ver} -q -e \
+      'install.packages("pak", lib = .Library, repos = sprintf("https://r-lib.github.io/p/pak/%s/%s/%s/%s", "devel", .Platform$pkgType, R.Version()$os, R.Version()$arch))'; \
+    done
 
 # Install openBLAS which is critical for torch performance
 RUN apt-get update -qq && \
diff --git a/cuda/Dockerfile b/cuda/Dockerfile
index 3bb370f..856a35e 100644
--- a/cuda/Dockerfile
+++ b/cuda/Dockerfile
@@ -49,7 +49,18 @@ ENV LANG C.UTF-8
 ENV TZ UTC
 
 RUN curl -Ls https://github.com/r-lib/rig/releases/download/latest/rig-linux-latest.tar.gz | sudo tar xz -C /usr/local && \
-  rig add ${R_VERSION} && \
-  rm -rf /var/lib/apt/lists/*
+  rig add ${R_VERSION} --without-pak && \
+  rm -rf /var/lib/apt/lists/* && \
+  rm -rf /tmp/rig && \
+  apt-get clean
+
+# We want to install pak into the system library, so we don't do it with rig
+RUN for inst in $(ls /opt/R/*/lib/R/bin/INSTALL); do \
+      sed -i '1s/^/#!\/bin\/bash\n/' ${inst}; \
+    done; \
+    versions="$(rig ls --json | grep '"name"' | sed 's/^.*\"name\": \"//' | sed 's/\",//')" && \
+    for ver in $versions; do \
+      R-${ver} -q -e \
+      'install.packages("pak", lib = .Library, repos = sprintf("https://r-lib.github.io/p/pak/%s/%s/%s/%s", "devel", .Platform$pkgType, R.Version()$os, R.Version()$arch))'; \
+    done
 
-RUN Rscript -e "file.copy('tmp/Rprofile.site', file.path(R.home(), 'etc', 'Rprofile.site'))"