Refactor

* Refactor console logging
* Make output from job service more readable
* Fork sftp file system
* Add new implementation for storage
* Simplify artifact store creation
* Skip image push for local filesystem
* Rename Command to AppBundle
* Fall back to default local configuration

.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
   -   id: check-added-large-files
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.1.4
+  rev: v0.4.2
   hooks:
     - id: ruff
       args: [ --fix ]

examples/jax_gpu/launcher_docker.py

@@ -2,6 +2,7 @@
 """This showcases how to create a launcher script without ever
 requiring the user to manually build a singularity image.
 """
+
 import subprocess
 
 from absl import app

lxm3/cli/cli.py

@@ -36,19 +36,6 @@ def launch(args):
     sys.path.pop(0)
 
 
-def shell(args):
-    del args
-    from lxm3.cli.shell import main
-
-    app.run(main)
-
-
-def clean(args):
-    from lxm3.cli.clean import run_clean
-
-    run_clean(args.project, args.days, args.dry_run, args.force, args.type)
-
-
 def register_launch_parser(parsers: argparse._SubParsersAction):
     launch_parser = parsers.add_parser(
         "launch",
@@ -79,37 +66,6 @@ def register_launch_parser(parsers: argparse._SubParsersAction):
     launch_parser.set_defaults(command=launch)
 
 
-def register_shell_parser(parsers: argparse._SubParsersAction):
-    launch_parser = parsers.add_parser(
-        "shell",
-        help="Open a shell.",
-        inherited_absl_flags=None,  # type: ignore
-    )
-    launch_parser.set_defaults(command=shell)
-
-
-def register_clean_parser(parsers: argparse._SubParsersAction):
-    clean_parser = parsers.add_parser(
-        "clean",
-        help="Clean job artifacts.",
-        inherited_absl_flags=None,  # type: ignore
-    )
-    clean_parser.add_argument("--project", required=True)
-    clean_parser.add_argument(
-        "--dry_run",
-        default=False,
-        action="store_true",
-    )
-    clean_parser.add_argument(
-        "--force",
-        default=False,
-        action="store_true",
-    )
-    clean_parser.add_argument("--days", type=float)
-    clean_parser.add_argument("--type", default=None)
-    clean_parser.set_defaults(command=clean)
-
-
 def _parse_flags(argv):
     parser = argparse_flags.ArgumentParser(description="lxm3 experiment scheduler.")
     parser.set_defaults(command=lambda _: parser.print_help())
@@ -118,8 +74,6 @@ def _parse_flags(argv):
 
     register_version_parser(subparsers)
     register_launch_parser(subparsers)
-    register_shell_parser(subparsers)
-    register_clean_parser(subparsers)
 
     args = parser.parse_args(argv[1:])
     return args

lxm3/cli/shell.py

@@ -6,14 +6,14 @@
 
 from lxm3.clusters import gridengine
 from lxm3.xm_cluster import config as config_lib
-from lxm3.xm_cluster.console import console
+from lxm3.xm_cluster import console
 
 
 def main(_):
     config = config_lib.default()
     cluster_settings = config.cluster_settings()
 
-    console.log(
+    console.info(
         "Creating a client to cluster "
         f"{cluster_settings.user}@{cluster_settings.hostname} ..."
     )

lxm3/contrib/ucl.py

@@ -51,6 +51,7 @@
 To understand the rules for translating for the two clusters, refer to `:ucl_test`.
 
 """
+
 from typing import Optional
 
 from lxm3.xm_cluster import config

lxm3/experimental/image_cache.py

@@ -44,9 +44,9 @@
 
 def _log(message):
     # Consider moving this out
-    from lxm3.xm_cluster.console import console  # noqa
+    from lxm3.xm_cluster import console  # noqa
 
-    console.log(message, style="dim")
+    console.info(message, style="dim")
 
 
 @dataclasses.dataclass(frozen=True)