Merge pull request saltstack#65747 from dwoz/merge-forward

[master] Merge forward from 3006.x

changelog/65691.fixed.md

@@ -0,0 +1 @@
+Fix boto execution module loading

changelog/65692.fixed.md

@@ -0,0 +1 @@
+Removed PR 65185 changes since incomplete solution

salt/fileclient.py

@@ -42,15 +42,12 @@
 MAX_FILENAME_LENGTH = 255
 
 
-def get_file_client(opts, pillar=False, force_local=False):
+def get_file_client(opts, pillar=False):
     """
     Read in the ``file_client`` option and return the correct type of file
     server
     """
-    if force_local:
-        client = "local"
-    else:
-        client = opts.get("file_client", "remote")
+    client = opts.get("file_client", "remote")
 
     if pillar and client == "local":
         client = "pillar"

salt/loader/__init__.py

@@ -324,6 +324,10 @@ def minion_mods(
         pack_self="__salt__",
     )
 
+    # Allow the usage of salt dunder in utils modules.
+    if utils and isinstance(utils, LazyLoader):
+        utils.pack["__salt__"] = ret
+
     # Load any provider overrides from the configuration file providers option
     #  Note: Providers can be pkg, service, user or group - not to be confused
     #        with cloud providers.

salt/loader/context.py

@@ -12,6 +12,8 @@
     # Py<3.7
     import contextvars
 
+import salt.exceptions
+
 DEFAULT_CTX_VAR = "loader_ctxvar"
 
 loader_ctxvar = contextvars.ContextVar(DEFAULT_CTX_VAR)
@@ -69,7 +71,12 @@ def value(self):
             return loader.pack[self.name]
         if self.name == loader.pack_self:
             return loader
-        return loader.pack[self.name]
+        try:
+            return loader.pack[self.name]
+        except KeyError:
+            raise salt.exceptions.LoaderError(
+                f"LazyLoader does not have a packed value for: {self.name}"
+            )
 
     def get(self, key, default=None):
         return self.value().get(key, default)

salt/minion.py

@@ -115,29 +115,6 @@
 # 6. Handle publications
 
 
-def _sync_grains(opts):
-    # need sync of custom grains as may be used in pillar compilation
-    # if coming up initially and remote client, the first sync _grains
-    # doesn't have opts["master_uri"] set yet during the sync, so need
-    # to force local, otherwise will throw an exception when attempting
-    # to retrieve opts["master_uri"] when retrieving key for remote communication
-    # in addition opts sometimes does not contain extmod_whitelist and extmod_blacklist
-    # hence set those to defaults, empty dict, if not part of opts, as ref'd in
-    # salt.utils.extmod sync function
-    if opts.get("extmod_whitelist", None) is None:
-        opts["extmod_whitelist"] = {}
-
-    if opts.get("extmod_blacklist", None) is None:
-        opts["extmod_blacklist"] = {}
-
-    if opts.get("file_client", "remote") == "remote" and not opts.get(
-        "master_uri", None
-    ):
-        salt.utils.extmods.sync(opts, "grains", force_local=True)
-    else:
-        salt.utils.extmods.sync(opts, "grains")
-
-
 def resolve_dns(opts, fallback=True):
     """
     Resolves the master_ip and master_uri options
@@ -945,7 +922,6 @@ def __init__(self, opts, context=None):
         # Late setup of the opts grains, so we can log from the grains module
         import salt.loader
 
-        _sync_grains(opts)
         opts["grains"] = salt.loader.grains(opts)
         super().__init__(opts)
 
@@ -1287,6 +1263,7 @@ def __init__(
         self.ready = False
         self.jid_queue = [] if jid_queue is None else jid_queue
         self.periodic_callbacks = {}
+        self.req_channel = None
 
         if io_loop is None:
             self.io_loop = tornado.ioloop.IOLoop.current()
@@ -1397,6 +1374,16 @@ def connect_master(self, failed=False):
         """
         Return a future which will complete when you are connected to a master
         """
+        if hasattr(self, "pub_channel") and self.pub_channel:
+            self.pub_channel.on_recv(None)
+            if hasattr(self.pub_channel, "auth"):
+                self.pub_channel.auth.invalidate()
+            if hasattr(self.pub_channel, "close"):
+                self.pub_channel.close()
+        if hasattr(self, "req_channel") and self.req_channel:
+            self.req_channel.close()
+            self.req_channel = None
+
         # Consider refactoring so that eval_master does not have a subtle side-effect on the contents of the opts array
         master, self.pub_channel = yield self.eval_master(
             self.opts, self.timeout, self.safe, failed
@@ -2872,7 +2859,9 @@ def handle_event(self, package):
                             self.pub_channel.auth.invalidate()
                         if hasattr(self.pub_channel, "close"):
                             self.pub_channel.close()
-                        del self.pub_channel
+                    if hasattr(self, "req_channel") and self.req_channel:
+                        self.req_channel.close()
+                        self.req_channel = None
 
                     # if eval_master finds a new master for us, self.connected
                     # will be True again on successful master authentication
@@ -3304,6 +3293,9 @@ def destroy(self):
             self.pub_channel.on_recv(None)
             self.pub_channel.close()
             del self.pub_channel
+        if hasattr(self, "req_channel") and self.req_channel:
+            self.req_channel.close()
+            self.req_channel = None
         if hasattr(self, "periodic_callbacks"):
             for cb in self.periodic_callbacks.values():
                 cb.stop()

salt/utils/extmods.py

@@ -39,7 +39,6 @@ def sync(
     saltenv=None,
     extmod_whitelist=None,
     extmod_blacklist=None,
-    force_local=False,
 ):
     """
     Sync custom modules into the extension_modules directory
@@ -83,9 +82,7 @@ def sync(
                         "Cannot create cache module directory %s. Check permissions.",
                         mod_dir,
                     )
-            with salt.fileclient.get_file_client(
-                opts, pillar=False, force_local=force_local
-            ) as fileclient:
+            with salt.fileclient.get_file_client(opts) as fileclient:
                 for sub_env in saltenv:
                     log.info("Syncing %s for environment '%s'", form, sub_env)
                     cache = []

tests/pytests/integration/cli/test_matcher.py

@@ -502,7 +502,9 @@ def test_salt_documentation(salt_cli, salt_minion):
     """
     Test to see if we're supporting --doc
     """
-    ret = salt_cli.run("-d", "test", minion_tgt=salt_minion.id)
+    # Setting an explicity long timeout otherwise this test may fail when the
+    # system is under load.
+    ret = salt_cli.run("-d", "test", minion_tgt=salt_minion.id, _timeout=90)
     assert ret.returncode == 0
     assert "test.ping" in ret.data
 

tests/pytests/integration/cli/test_salt_call.py

@@ -429,74 +429,3 @@ def test_local_salt_call_no_function_no_retcode(salt_call_cli):
     assert "test" in ret.data
     assert ret.data["test"] == "'test' is not available."
     assert "test.echo" in ret.data
-
-
-def test_state_highstate_custom_grains(salt_master, salt_minion_factory):
-    """
-    This test ensure that custom grains in salt://_grains are loaded before pillar compilation
-    to ensure that any use of custom grains in pillar files are available, this implies that
-    a sync of grains occurs before loading the regular /etc/salt/grains or configuration file
-    grains, as well as the usual grains.
-
-    Note: cannot use salt_minion and salt_call_cli, since these will be loaded before
-    the pillar and custom_grains files are written, hence using salt_minion_factory.
-    """
-    pillar_top_sls = """
-    base:
-      '*':
-        - defaults
-        """
-
-    pillar_defaults_sls = """
-    mypillar: "{{ grains['custom_grain'] }}"
-    """
-
-    salt_top_sls = """
-    base:
-      '*':
-        - test
-        """
-
-    salt_test_sls = """
-    "donothing":
-      test.nop: []
-    """
-
-    salt_custom_grains_py = """
-    def main():
-        return {'custom_grain': 'test_value'}
-    """
-    assert salt_master.is_running()
-    with salt_minion_factory.started():
-        salt_minion = salt_minion_factory
-        salt_call_cli = salt_minion_factory.salt_call_cli()
-        with salt_minion.pillar_tree.base.temp_file(
-            "top.sls", pillar_top_sls
-        ), salt_minion.pillar_tree.base.temp_file(
-            "defaults.sls", pillar_defaults_sls
-        ), salt_minion.state_tree.base.temp_file(
-            "top.sls", salt_top_sls
-        ), salt_minion.state_tree.base.temp_file(
-            "test.sls", salt_test_sls
-        ), salt_minion.state_tree.base.temp_file(
-            "_grains/custom_grain.py", salt_custom_grains_py
-        ):
-            ret = salt_call_cli.run("--local", "state.highstate")
-            assert ret.returncode == 0
-            ret = salt_call_cli.run("--local", "pillar.items")
-            assert ret.returncode == 0
-            assert ret.data
-            pillar_items = ret.data
-            assert "mypillar" in pillar_items
-            assert pillar_items["mypillar"] == "test_value"
-
-
-def test_salt_call_versions(salt_call_cli, caplog):
-    """
-    Call test.versions without '--local' to test grains
-    are sync'd without any missing keys in opts
-    """
-    with caplog.at_level(logging.DEBUG):
-        ret = salt_call_cli.run("test.versions")
-        assert ret.returncode == 0
-        assert "Failed to sync grains module: 'master_uri'" not in caplog.messages

tests/pytests/integration/minion/test_return_retries.py

@@ -3,6 +3,8 @@
 import pytest
 from saltfactories.utils import random_string
 
+import salt.utils.files
+
 
 @pytest.fixture(scope="function")
 def salt_minion_retry(salt_master, salt_minion_id):
@@ -53,13 +55,15 @@ def test_publish_retry(salt_master, salt_minion_retry, salt_cli, salt_run_cli):
 
 
 @pytest.mark.slow_test
-def test_pillar_timeout(salt_master_factory):
-    cmd = """
-    python -c "import time; time.sleep(3.0); print('{\\"foo\\": \\"bar\\"}');\"
-    """.strip()
+def test_pillar_timeout(salt_master_factory, tmp_path):
+    cmd = 'print(\'{"foo": "bar"}\');\n'
+
+    with salt.utils.files.fopen(tmp_path / "script.py", "w") as fp:
+        fp.write(cmd)
+
     master_overrides = {
         "ext_pillar": [
-            {"cmd_json": cmd},
+            {"cmd_json": f"python {tmp_path / 'script.py'}"},
         ],
         "auto_accept": True,
         "worker_threads": 2,
@@ -103,8 +107,10 @@ def test_pillar_timeout(salt_master_factory):
     cli = master.salt_cli()
     sls_tempfile = master.state_tree.base.temp_file(f"{sls_name}.sls", sls_contents)
     with master.started(), minion1.started(), minion2.started(), minion3.started(), minion4.started(), sls_tempfile:
+        cmd = 'import time; time.sleep(6); print(\'{"foo": "bang"}\');\n'
+        with salt.utils.files.fopen(tmp_path / "script.py", "w") as fp:
+            fp.write(cmd)
         proc = cli.run("state.sls", sls_name, minion_tgt="*")
-        print(proc)
         # At least one minion should have a Pillar timeout
         assert proc.returncode == 1
         minion_timed_out = False