Bring back cosa_controller_img knob, drop hack status

The podman remote feature assumes that the version on the client
and the one on the host match versions, within some margin. So the
`cosa_controller_img` knob we had before was useful.

This reverts commit dbad52b. But:
1. we drop the `_hack` suffix and update the documentation
2. the default is to always use the `:main` cosa image, which most of
   the time will have a podman remote matching the FCOS version on the
   builder (except for that small period of time during a major rebase
   when FCOS is ahead of cosa)

Motivated by our pipeline hitting
containers/podman#22531
due to most FCOS streams being on f40 and cosa still being on f39.

docs/config.yaml

@@ -75,6 +75,9 @@ streams:
           - vmware
       # OPTIONAL: override default coreos-assembler image to build with
       cosa_img: quay.io/coreos-assembler/coreos-assembler:stable
+      # OPTIONAL: Override default coreos-assembler image to use for
+      # the controller pod in the build-arch job (default is :main)
+      cosa_controller_img: quay.io/coreos-assembler/coreos-assembler:next
       # OPTIONAL: override additional architectures to build for
       additional_arches:
         - s390x

jobs/build-arch.Jenkinsfile

@@ -74,6 +74,8 @@ cosa_img = cosa_img ?: pipeutils.get_cosa_img(pipecfg, params.STREAM)
 
 def stream_info = pipecfg.streams[params.STREAM]
 
+def cosa_controller_img = stream_info.cosa_controller_img ?: "quay.io/coreos-assembler/coreos-assembler:main"
+
 // Grab any environment variables we should set
 def container_env = pipeutils.get_env_vars_for_stream(pipecfg, params.STREAM)
 
@@ -110,7 +112,7 @@ lock(resource: "release-${params.VERSION}-${basearch}") {
 lock(resource: "build-${params.STREAM}-${basearch}") {
     cosaPod(cpu: "${ncpus}",
             memory: "${cosa_memory_request_mb}Mi",
-            image: cosa_img,
+            image: cosa_controller_img,
             env: container_env,
             serviceAccount: "jenkins") {
     timeout(time: timeout_mins, unit: 'MINUTES') {

jobs/debug-pod.Jenkinsfile

@@ -45,6 +45,9 @@ def stream_info = pipecfg.streams[params.STREAM]
 // Grab any environment variables we should set
 def container_env = pipeutils.get_env_vars_for_stream(pipecfg, params.STREAM)
 
+// We'll just always use main for the controller image here.
+def cosa_controller_img = "quay.io/coreos-assembler/coreos-assembler:main"
+
 // Note that the heavy lifting is done on a remote node via podman
 // --remote so we shouldn't need much memory or CPU.
 def cosa_memory_request_mb = 512
@@ -56,7 +59,7 @@ podName += UUID.randomUUID().toString().substring(0, 8)
 
 cosaPod(cpu: "${ncpus}",
         memory: "${cosa_memory_request_mb}Mi",
-        image: cosa_img,
+        image: cosa_controller_img,
         env: container_env,
         serviceAccount: "jenkins",
         name : podName) {