Adds multiplex sandbox support and improves argument parsing

.bazelrc_shared

@@ -14,6 +14,7 @@ build --tool_java_runtime_version="remotejdk_21"
 build --strategy=ScalaCompile=worker
 build --worker_max_instances=4
 build --worker_sandboxing
+build --experimental_worker_multiplex_sandboxing
 build --verbose_failures
 
 test --test_output=all

rules/private/phases/phase_coverage_jacoco.bzl

@@ -44,7 +44,14 @@ def phase_coverage_jacoco(ctx, g):
         outputs = [in_out_pair[1] for in_out_pair in in_out_pairs],
         executable = code_coverage_configuration.instrumentation_worker.files_to_run.executable,
         input_manifests = worker_input_manifests,
-        execution_requirements = _resolve_execution_reqs(ctx, {"supports-workers": "1"}),
+        execution_requirements = _resolve_execution_reqs(
+            ctx,
+            {
+                "supports-multiplex-workers": "1",
+                "supports-workers": "1",
+                "supports-multiplex-sandboxing": "1",
+            },
+        ),
         arguments = [args],
     )
 

rules/private/phases/phase_zinc_compile.bzl

@@ -78,13 +78,17 @@ def phase_zinc_compile(ctx, g):
     execution_requirements_tags = {
         "supports-multiplex-workers": "1",
         "supports-workers": "1",
+        "supports-multiplex-sandboxing": "1",
     }
 
-    # Disable sandboxing if incremental compilation features are going to be used
-    # because they require stashing files outside the sandbox that Bazel isn't
-    # aware of.
+    # Disable several things if incremental compilation features are going to be used
+    # because incremental compilation require stashing files outside the sandbox that
+    # Bazel isn't aware of and is less deterministic than ideal.
     if zinc_configuration.incremental:
         execution_requirements_tags["no-sandbox"] = "1"
+        execution_requirements_tags["no-cache"] = "1"
+        execution_requirements_tags["no-remote"] = "1"
+        execution_requirements_tags["supports-multiplex-sandboxing"] = "0"
 
     # todo: different execution path for nosrc jar?
     ctx.actions.run(

rules/private/phases/phase_zinc_depscheck.bzl

@@ -50,7 +50,14 @@ def phase_zinc_depscheck(ctx, g):
             outputs = [deps_check],
             executable = deps_configuration.worker.files_to_run.executable,
             input_manifests = worker_input_manifests,
-            execution_requirements = _resolve_execution_reqs(ctx, {"supports-workers": "1"}),
+            execution_requirements = _resolve_execution_reqs(
+                ctx,
+                {
+                    "supports-multiplex-workers": "1",
+                    "supports-workers": "1",
+                    "supports-multiplex-sandboxing": "1",
+                },
+            ),
             arguments = [deps_args],
         )
         deps_checks[name] = deps_check

rules/scala/private/doc.bzl

@@ -62,7 +62,14 @@ def scaladoc_implementation(ctx):
     ctx.actions.run(
         arguments = [args],
         executable = ctx.attr._runner.files_to_run.executable,
-        execution_requirements = _resolve_execution_reqs(ctx, {"supports-workers": "1"}),
+        execution_requirements = _resolve_execution_reqs(
+            ctx,
+            {
+                "supports-multiplex-workers": "1",
+                "supports-workers": "1",
+                "supports-multiplex-sandboxing": "1",
+            },
+        ),
         input_manifests = input_manifests,
         inputs = depset(
             src_jars + srcs + [zinc_configuration.compiler_bridge],

rules/scala_proto/BUILD

@@ -21,6 +21,7 @@ scala_binary(
     deps = [
         "//src/main/scala/higherkindness/rules_scala/common/args",
         "//src/main/scala/higherkindness/rules_scala/common/error",
+        "//src/main/scala/higherkindness/rules_scala/common/sandbox",
         "//src/main/scala/higherkindness/rules_scala/common/worker",
         "@annex//:net_sourceforge_argparse4j_argparse4j",
         "@annex_proto//:com_github_os72_protoc_jar",

rules/scala_proto/private/ScalaProtoWorker.scala

@@ -1,31 +1,52 @@
 package annex.scala.proto
 
+import higherkindness.rules_scala.common.args.ArgsUtil
+import higherkindness.rules_scala.common.args.ArgsUtil.PathArgumentType
 import higherkindness.rules_scala.common.args.implicits._
 import higherkindness.rules_scala.common.error.AnnexWorkerError
+import higherkindness.rules_scala.common.sandbox.SandboxUtil
 import higherkindness.rules_scala.common.worker.WorkerMain
 import java.io.{File, PrintStream}
-import java.nio.file.{Files, Paths}
+import java.nio.file.{Files, Path, Paths}
 import java.util.Collections
 import net.sourceforge.argparse4j.ArgumentParsers
 import net.sourceforge.argparse4j.impl.Arguments
-import net.sourceforge.argparse4j.inf.ArgumentParser
+import net.sourceforge.argparse4j.inf.{ArgumentParser, Namespace}
 import protocbridge.{ProtocBridge, ProtocRunner}
 import scala.jdk.CollectionConverters._
 import scalapb.ScalaPbCodeGenerator
 
 object ScalaProtoWorker extends WorkerMain[Unit] {
 
+  private class ScalaProtoRequest private (
+    val isGrpc: Boolean,
+    val outputDir: Path,
+    val protoPaths: List[Path],
+    val sources: List[Path],
+  )
+
+  private object ScalaProtoRequest {
+    def apply(workDir: Path, namespace: Namespace): ScalaProtoRequest = {
+      new ScalaProtoRequest(
+        isGrpc = namespace.getBoolean("grpc"),
+        outputDir = SandboxUtil.getSandboxPath(workDir, namespace.get[Path]("output_dir")),
+        protoPaths = SandboxUtil.getSandboxPaths(workDir, namespace.getList[Path]("proto_paths")),
+        sources = SandboxUtil.getSandboxPaths(workDir, namespace.getList[Path]("sources")),
+      )
+    }
+  }
+
   private val argParser: ArgumentParser = {
     val parser = ArgumentParsers.newFor("proto").addHelp(true).fromFilePrefix("@").build
     parser
       .addArgument("--output_dir")
       .help("Output dir")
       .metavar("output_dir")
-      .`type`(Arguments.fileType.verifyCanCreate)
+      .`type`(PathArgumentType.apply())
     parser
       .addArgument("--proto_paths")
       .nargs("*")
-      .`type`(Arguments.fileType.verifyIsDirectory)
+      .`type`(PathArgumentType.apply())
       .setDefault_(Collections.emptyList)
     parser
       .addArgument("--grpc")
@@ -35,28 +56,26 @@ object ScalaProtoWorker extends WorkerMain[Unit] {
       .help("Source files")
       .metavar("source")
       .nargs("*")
-      .`type`(Arguments.fileType.verifyCanRead.verifyIsFile)
+      .`type`(PathArgumentType.apply())
       .setDefault_(Collections.emptyList)
     parser
   }
 
   override def init(args: Option[Array[String]]): Unit = ()
 
-  protected def work(ctx: Unit, args: Array[String], out: PrintStream): Unit = {
-    val namespace = argParser.parseArgs(args)
-    val sources = namespace.getList[File]("sources").asScala.toList
-    val protoPaths = namespace.getList[File]("proto_paths").asScala.toList
+  protected def work(ctx: Unit, args: Array[String], out: PrintStream, workDir: Path): Unit = {
+    val workRequest = ScalaProtoRequest(workDir, ArgsUtil.parseArgsOrFailSafe(args, argParser, out))
 
-    val scalaOut = namespace.get[File]("output_dir").toPath
+    val scalaOut = workRequest.outputDir
     Files.createDirectories(scalaOut)
-    val outOptions = if (namespace.getBoolean("grpc")) {
+    val outOptions = if (workRequest.isGrpc) {
       "grpc:"
     } else {
       ""
     }
     val params = List(s"--scala_out=${outOptions}${scalaOut}")
-      ::: protoPaths.map(dir => s"--proto_path=${dir.toString}")
-      ::: sources.map(_.getPath.toString)
+      ::: workRequest.protoPaths.map(dir => s"--proto_path=${dir.toString}")
+      ::: workRequest.sources.map(_.toString)
 
     class MyProtocRunner[ExitCode] extends ProtocRunner[Int] {
       def run(args: Seq[String], extraEnv: Seq[(String, String)]): Int = {

rules/scala_proto/private/core.bzl

@@ -49,7 +49,14 @@ def scala_proto_library_implementation(ctx):
         tools = compiler_inputs,
         input_manifests = input_manifests,
         progress_message = "Compiling %{label} protobuf into Scala source",
-        execution_requirements = _resolve_execution_reqs(ctx, {"supports-workers": supports_workers}),
+        execution_requirements = _resolve_execution_reqs(
+            ctx,
+            {
+                "supports-multiplex-workers": supports_workers,
+                "supports-workers": supports_workers,
+                "supports-multiplex-sandboxing": supports_workers,
+            },
+        ),
         arguments = [args],
     )
 

rules/scalafmt/BUILD

@@ -24,6 +24,7 @@ scala_binary(
     scala = "//src/main/scala:bootstrap",
     visibility = ["//visibility:public"],
     deps = [
+        "//src/main/scala/higherkindness/rules_scala/common/sandbox",
         "//src/main/scala/higherkindness/rules_scala/common/worker",
         "//src/main/scala/higherkindness/rules_scala/workers/common",
         "@annex//:net_sourceforge_argparse4j_argparse4j",

rules/scalafmt/private/test.bzl

@@ -50,7 +50,14 @@ def build_format(ctx):
                 input_manifests = runner_manifests,
                 inputs = [ctx.file.config, src],
                 tools = runner_inputs,
-                execution_requirements = _resolve_execution_reqs(ctx, {"supports-multiplex-workers": "1", "supports-workers": "1"}),
+                execution_requirements = _resolve_execution_reqs(
+                    ctx,
+                    {
+                        "supports-multiplex-workers": "1",
+                        "supports-workers": "1",
+                        "supports-multiplex-sandboxing": "1",
+                    },
+                ),
                 mnemonic = "ScalaFmt",
             )
             manifest_content.append("{} {}".format(src.short_path, file.short_path))

rules/scalafmt/scalafmt/ScalafmtRunner.scala

@@ -1,11 +1,15 @@
 package annex.scalafmt
 
+import higherkindness.rules_scala.common.args.ArgsUtil
+import higherkindness.rules_scala.common.args.ArgsUtil.PathArgumentType
+import higherkindness.rules_scala.common.sandbox.SandboxUtil
 import higherkindness.rules_scala.common.worker.WorkerMain
 import higherkindness.rules_scala.workers.common.Color
 import java.io.{File, PrintStream}
-import java.nio.file.Files
+import java.nio.file.{Files, Path}
 import net.sourceforge.argparse4j.ArgumentParsers
 import net.sourceforge.argparse4j.impl.Arguments
+import net.sourceforge.argparse4j.inf.{ArgumentParser, Namespace}
 import org.scalafmt.Scalafmt
 import org.scalafmt.config.ScalafmtConfig
 import org.scalafmt.sysops.FileOps
@@ -14,20 +18,38 @@ import scala.io.Codec
 
 object ScalafmtRunner extends WorkerMain[Unit] {
 
-  protected[this] def init(args: Option[Array[String]]): Unit = {}
+  private[this] class ScalafmtRequest private (
+    val configFile: Path,
+    val inputFile: Path,
+    val outputFile: Path,
+  )
 
-  protected[this] def work(worker: Unit, args: Array[String], out: PrintStream): Unit = {
+  private[this] object ScalafmtRequest {
+    def apply(workDir: Path, namespace: Namespace): ScalafmtRequest = {
+      new ScalafmtRequest(
+        configFile = SandboxUtil.getSandboxPath(workDir, namespace.get[Path]("config")),
+        inputFile = SandboxUtil.getSandboxPath(workDir, namespace.get[Path]("input")),
+        outputFile = SandboxUtil.getSandboxPath(workDir, namespace.get[Path]("output")),
+      )
+    }
+  }
 
+  private[this] val argParser: ArgumentParser = {
     val parser = ArgumentParsers.newFor("scalafmt").addHelp(true).defaultFormatWidth(80).fromFilePrefix("@").build
-    parser.addArgument("--config").required(true).`type`(Arguments.fileType)
-    parser.addArgument("input").`type`(Arguments.fileType)
-    parser.addArgument("output").`type`(Arguments.fileType)
+    parser.addArgument("--config").required(true).`type`(PathArgumentType.apply())
+    parser.addArgument("input").`type`(PathArgumentType.apply())
+    parser.addArgument("output").`type`(PathArgumentType.apply())
+    parser
+  }
+
+  protected[this] def init(args: Option[Array[String]]): Unit = {}
 
-    val namespace = parser.parseArgsOrFail(args)
+  protected[this] def work(worker: Unit, args: Array[String], out: PrintStream, workDir: Path): Unit = {
+    val workRequest = ScalafmtRequest(workDir, ArgsUtil.parseArgsOrFailSafe(args, argParser, out))
 
-    val source = FileOps.readFile(namespace.get[File]("input").toPath())(Codec.UTF8)
+    val source = FileOps.readFile(workRequest.inputFile)(Codec.UTF8)
 
-    val config = ScalafmtConfig.fromHoconFile(namespace.get[File]("config").toPath()).get
+    val config = ScalafmtConfig.fromHoconFile(workRequest.configFile).get
     @tailrec
     def format(code: String): String = {
       val formatted = Scalafmt.format(code, config).get
@@ -50,7 +72,7 @@ object ScalafmtRunner extends WorkerMain[Unit] {
         }
       }
 
-    Files.write(namespace.get[File]("output").toPath, output.getBytes)
+    Files.write(workRequest.outputFile, output.getBytes)
   }
 
 }

src/main/scala/BUILD

@@ -71,13 +71,9 @@ configure_zinc_scala(
 # IntelliJ libraries, so we can get a Scala SDK on sync.
 scala_library(
     name = "scala-sdk",
-    deps_used_whitelist = [
-        "@annex//:org_scala_lang_scala3_library_3",
-    ],
+    deps_used_whitelist = compiler_classpath_2_13,
     scala = ":zinc_3",
-    deps = [
-        "@annex//:org_scala_lang_scala3_library_3",
-    ],
+    deps = compiler_classpath_2_13,
 )
 
 compiler_classpath_3 = [
@@ -91,6 +87,13 @@ runtime_classpath_3 = [
     "@annex//:org_scala_lang_tasty_core_3",
 ]
 
+scala_library(
+    name = "scala-sdk-3",
+    deps_used_whitelist = compiler_classpath_3,
+    scala = ":zinc_3",
+    deps = compiler_classpath_3,
+)
+
 configure_bootstrap_scala(
     name = "bootstrap_3",
     compiler_classpath = compiler_classpath_3,

src/main/scala/higherkindness/rules_scala/common/args/ArgsUtil.scala

@@ -0,0 +1,35 @@
+package higherkindness.rules_scala
+package common.args
+
+import common.error.AnnexWorkerError
+import java.io.{PrintStream, PrintWriter}
+import java.nio.file.{Path, Paths}
+import net.sourceforge.argparse4j.helper.HelpScreenException
+import net.sourceforge.argparse4j.inf.{Argument, ArgumentParser, ArgumentParserException, ArgumentType, Namespace}
+import scala.language.reflectiveCalls
+
+object ArgsUtil {
+
+  /**
+   * Safely handles expected errors that pop up during parsing arguments. The argparse4j calls System.exit when things
+   * go wrong, and we don't want the worker to exit. Instead we exit the work request and print to the output stream.
+   */
+  def parseArgsOrFailSafe(args: Array[String], parser: ArgumentParser, out: PrintStream): Namespace = {
+    try {
+      parser.parseArgs(args)
+    } catch {
+      case e: HelpScreenException =>
+        parser.handleError(e, new PrintWriter(out))
+        throw new AnnexWorkerError(0)
+      case e: ArgumentParserException =>
+        parser.handleError(e, new PrintWriter(out))
+        throw new AnnexWorkerError(1)
+    }
+  }
+
+  case class PathArgumentType() extends ArgumentType[Path] {
+    override def convert(parser: ArgumentParser, arg: Argument, value: String): Path = {
+      Paths.get(value)
+    }
+  }
+}

src/main/scala/higherkindness/rules_scala/common/args/BUILD

@@ -7,6 +7,7 @@ scala_library(
     scala = "//src/main/scala:bootstrap",
     visibility = ["//visibility:public"],
     deps = [
+        "//src/main/scala/higherkindness/rules_scala/common/error",
         "@annex//:net_sourceforge_argparse4j_argparse4j",
     ],
 )