use mount_nix_store to avoid running out of disk

... on CI.

This fixes CI running out of disk:

* https://channable.semaphoreci.com/jobs/46cb6181-b263-45fc-b9df-4d1a06e5d495#L1213-L1217
* https://channable.semaphoreci.com/workflows/9a9ebc9e-0e13-446a-8189-910fc48dd4ed

This fix was suggested by Bert Peters (@bertptrs).

.semaphore/prologue.sh

@@ -41,3 +41,44 @@ sudo rm -rf \
     /opt/firefox-esr-prev \
     /usr/local/golang \
     /mnt/docker.qcow2
+
+mount_nix_store() {
+  # Before we can successfully restore to `/nix` it needs to be created and owned
+  # by the CI user. Without this, the `cache restore` command fails because it
+  # doesn't have permission to create `/nix`. (We cannot run the cache restore
+  # command as `root` as it takes settings from environment variables.)
+  # We use the local scratch SSD mounted at `/mnt` to prevent running out of disk
+  # space, as the Nix store takes up about 10 GiB when uncompressed, and a build
+  # on the smallest Semaphore instance starts out with only 13 GiB of disk space,
+  # of which 4 GiB is used by the cloned repository.
+  # The SSD has (at the time of writing) 80 GiB of space, of which 32 GiB is free
+  # when a build starts. If we really want to we can also consider wiping the SSD
+  # before using it, we don't need to do this yet.
+  sudo mkdir -p /mnt/nix /nix
+  sudo mount --bind /mnt/nix /nix
+  sudo chown -R semaphore: /nix
+}
+mount_nix_store
+
+# Attempt to restore the Semaphore cache entry for `/nix`.
+#
+# We have this in addition to Cachix because we want to avoid hitting Cachix
+# for individual store entries, as restoring `/nix` from the Semaphore cache in
+# one go is a lot faster than downloading individual cache entries from
+# Cachix's S3 + Cloudflare.
+#
+# We refresh the Nix store cache entry daily. It is populated after the first
+# successful build of the day by our main pipeline.
+#
+# Restoring the cache can fail when the cache entry is only partially matched,
+# because then it might still be in the process of being uploaded by Semaphore,
+# which can be caused by a concurrent build. Since using the Semaphore cache is
+# only an optimization, but not strictly necessary, we make sure that the build
+# doesn't fail in this case, by making sure the exit code is always 0. When
+# restoring the cache fails, we delete /nix to ensure that we are not left with
+# a partially restored Nix store.
+cache restore "nix-store-$(date -u -Idate),nix-store-$(date -u -Idate --date=yesterday),nix-store-" || {
+  sudo umount /nix
+  sudo rm -fr /mnt/nix
+  mount_nix_store
+}

.semaphore/semaphore.yml

@@ -36,13 +36,6 @@ blocks:
             # Free space on the disk in order to save the nix store and error on build failures.
             - .semaphore/prologue.sh
 
-            # Restore `/nix` cache. Create the directory first, otherwise we encounter
-            # permission errors. We do this because the Semaphore cache is faster than
-            # both Cachix and cache.nixos.org.
-            - "sudo mkdir /nix"
-            - "sudo chown -R semaphore:semaphore /nix"
-            - "cache restore nix-store-"
-
             # Restore any stack cache that we can find.
             - "cache restore stack-cache-"