diff --git a/changelog/65691.fixed.md b/changelog/65691.fixed.md new file mode 100644 index 000000000000..3b5192f80c07 --- /dev/null +++ b/changelog/65691.fixed.md @@ -0,0 +1 @@ +Fix boto execution module loading diff --git a/changelog/65692.fixed.md b/changelog/65692.fixed.md new file mode 100644 index 000000000000..b4eef6c93d46 --- /dev/null +++ b/changelog/65692.fixed.md @@ -0,0 +1 @@ +Removed PR 65185 changes since incomplete solution diff --git a/salt/fileclient.py b/salt/fileclient.py index 1568143fa64a..c17b7c684341 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -42,15 +42,12 @@ MAX_FILENAME_LENGTH = 255 -def get_file_client(opts, pillar=False, force_local=False): +def get_file_client(opts, pillar=False): """ Read in the ``file_client`` option and return the correct type of file server """ - if force_local: - client = "local" - else: - client = opts.get("file_client", "remote") + client = opts.get("file_client", "remote") if pillar and client == "local": client = "pillar" diff --git a/salt/loader/__init__.py b/salt/loader/__init__.py index 50adf7ad8036..eb604362cb63 100644 --- a/salt/loader/__init__.py +++ b/salt/loader/__init__.py @@ -324,6 +324,10 @@ def minion_mods( pack_self="__salt__", ) + # Allow the usage of salt dunder in utils modules. + if utils and isinstance(utils, LazyLoader): + utils.pack["__salt__"] = ret + # Load any provider overrides from the configuration file providers option # Note: Providers can be pkg, service, user or group - not to be confused # with cloud providers. diff --git a/salt/loader/context.py b/salt/loader/context.py index 6bbfe4dbd815..560b05a8d2b3 100644 --- a/salt/loader/context.py +++ b/salt/loader/context.py @@ -12,6 +12,8 @@ # Py<3.7 import contextvars +import salt.exceptions + DEFAULT_CTX_VAR = "loader_ctxvar" loader_ctxvar = contextvars.ContextVar(DEFAULT_CTX_VAR) @@ -69,7 +71,12 @@ def value(self): return loader.pack[self.name] if self.name == loader.pack_self: return loader - return loader.pack[self.name] + try: + return loader.pack[self.name] + except KeyError: + raise salt.exceptions.LoaderError( + f"LazyLoader does not have a packed value for: {self.name}" + ) def get(self, key, default=None): return self.value().get(key, default) diff --git a/salt/minion.py b/salt/minion.py index 3619940446b8..addc91fbc86e 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -115,29 +115,6 @@ # 6. Handle publications -def _sync_grains(opts): - # need sync of custom grains as may be used in pillar compilation - # if coming up initially and remote client, the first sync _grains - # doesn't have opts["master_uri"] set yet during the sync, so need - # to force local, otherwise will throw an exception when attempting - # to retrieve opts["master_uri"] when retrieving key for remote communication - # in addition opts sometimes does not contain extmod_whitelist and extmod_blacklist - # hence set those to defaults, empty dict, if not part of opts, as ref'd in - # salt.utils.extmod sync function - if opts.get("extmod_whitelist", None) is None: - opts["extmod_whitelist"] = {} - - if opts.get("extmod_blacklist", None) is None: - opts["extmod_blacklist"] = {} - - if opts.get("file_client", "remote") == "remote" and not opts.get( - "master_uri", None - ): - salt.utils.extmods.sync(opts, "grains", force_local=True) - else: - salt.utils.extmods.sync(opts, "grains") - - def resolve_dns(opts, fallback=True): """ Resolves the master_ip and master_uri options @@ -945,7 +922,6 @@ def __init__(self, opts, context=None): # Late setup of the opts grains, so we can log from the grains module import salt.loader - _sync_grains(opts) opts["grains"] = salt.loader.grains(opts) super().__init__(opts) @@ -1287,6 +1263,7 @@ def __init__( self.ready = False self.jid_queue = [] if jid_queue is None else jid_queue self.periodic_callbacks = {} + self.req_channel = None if io_loop is None: self.io_loop = tornado.ioloop.IOLoop.current() @@ -1397,6 +1374,16 @@ def connect_master(self, failed=False): """ Return a future which will complete when you are connected to a master """ + if hasattr(self, "pub_channel") and self.pub_channel: + self.pub_channel.on_recv(None) + if hasattr(self.pub_channel, "auth"): + self.pub_channel.auth.invalidate() + if hasattr(self.pub_channel, "close"): + self.pub_channel.close() + if hasattr(self, "req_channel") and self.req_channel: + self.req_channel.close() + self.req_channel = None + # Consider refactoring so that eval_master does not have a subtle side-effect on the contents of the opts array master, self.pub_channel = yield self.eval_master( self.opts, self.timeout, self.safe, failed @@ -2872,7 +2859,9 @@ def handle_event(self, package): self.pub_channel.auth.invalidate() if hasattr(self.pub_channel, "close"): self.pub_channel.close() - del self.pub_channel + if hasattr(self, "req_channel") and self.req_channel: + self.req_channel.close() + self.req_channel = None # if eval_master finds a new master for us, self.connected # will be True again on successful master authentication @@ -3304,6 +3293,9 @@ def destroy(self): self.pub_channel.on_recv(None) self.pub_channel.close() del self.pub_channel + if hasattr(self, "req_channel") and self.req_channel: + self.req_channel.close() + self.req_channel = None if hasattr(self, "periodic_callbacks"): for cb in self.periodic_callbacks.values(): cb.stop() diff --git a/salt/utils/extmods.py b/salt/utils/extmods.py index 8601cfeedbc3..f1b8a8264483 100644 --- a/salt/utils/extmods.py +++ b/salt/utils/extmods.py @@ -39,7 +39,6 @@ def sync( saltenv=None, extmod_whitelist=None, extmod_blacklist=None, - force_local=False, ): """ Sync custom modules into the extension_modules directory @@ -83,9 +82,7 @@ def sync( "Cannot create cache module directory %s. Check permissions.", mod_dir, ) - with salt.fileclient.get_file_client( - opts, pillar=False, force_local=force_local - ) as fileclient: + with salt.fileclient.get_file_client(opts) as fileclient: for sub_env in saltenv: log.info("Syncing %s for environment '%s'", form, sub_env) cache = [] diff --git a/tests/pytests/integration/cli/test_matcher.py b/tests/pytests/integration/cli/test_matcher.py index c999c465b2ef..f6b18b51c42b 100644 --- a/tests/pytests/integration/cli/test_matcher.py +++ b/tests/pytests/integration/cli/test_matcher.py @@ -502,7 +502,9 @@ def test_salt_documentation(salt_cli, salt_minion): """ Test to see if we're supporting --doc """ - ret = salt_cli.run("-d", "test", minion_tgt=salt_minion.id) + # Setting an explicity long timeout otherwise this test may fail when the + # system is under load. + ret = salt_cli.run("-d", "test", minion_tgt=salt_minion.id, _timeout=90) assert ret.returncode == 0 assert "test.ping" in ret.data diff --git a/tests/pytests/integration/cli/test_salt_call.py b/tests/pytests/integration/cli/test_salt_call.py index f45aa3dabe02..1d770c0ffbed 100644 --- a/tests/pytests/integration/cli/test_salt_call.py +++ b/tests/pytests/integration/cli/test_salt_call.py @@ -429,74 +429,3 @@ def test_local_salt_call_no_function_no_retcode(salt_call_cli): assert "test" in ret.data assert ret.data["test"] == "'test' is not available." assert "test.echo" in ret.data - - -def test_state_highstate_custom_grains(salt_master, salt_minion_factory): - """ - This test ensure that custom grains in salt://_grains are loaded before pillar compilation - to ensure that any use of custom grains in pillar files are available, this implies that - a sync of grains occurs before loading the regular /etc/salt/grains or configuration file - grains, as well as the usual grains. - - Note: cannot use salt_minion and salt_call_cli, since these will be loaded before - the pillar and custom_grains files are written, hence using salt_minion_factory. - """ - pillar_top_sls = """ - base: - '*': - - defaults - """ - - pillar_defaults_sls = """ - mypillar: "{{ grains['custom_grain'] }}" - """ - - salt_top_sls = """ - base: - '*': - - test - """ - - salt_test_sls = """ - "donothing": - test.nop: [] - """ - - salt_custom_grains_py = """ - def main(): - return {'custom_grain': 'test_value'} - """ - assert salt_master.is_running() - with salt_minion_factory.started(): - salt_minion = salt_minion_factory - salt_call_cli = salt_minion_factory.salt_call_cli() - with salt_minion.pillar_tree.base.temp_file( - "top.sls", pillar_top_sls - ), salt_minion.pillar_tree.base.temp_file( - "defaults.sls", pillar_defaults_sls - ), salt_minion.state_tree.base.temp_file( - "top.sls", salt_top_sls - ), salt_minion.state_tree.base.temp_file( - "test.sls", salt_test_sls - ), salt_minion.state_tree.base.temp_file( - "_grains/custom_grain.py", salt_custom_grains_py - ): - ret = salt_call_cli.run("--local", "state.highstate") - assert ret.returncode == 0 - ret = salt_call_cli.run("--local", "pillar.items") - assert ret.returncode == 0 - assert ret.data - pillar_items = ret.data - assert "mypillar" in pillar_items - assert pillar_items["mypillar"] == "test_value" - - -def test_salt_call_versions(salt_call_cli, caplog): - """ - Call test.versions without '--local' to test grains - are sync'd without any missing keys in opts - """ - with caplog.at_level(logging.DEBUG): - ret = salt_call_cli.run("test.versions") - assert ret.returncode == 0 - assert "Failed to sync grains module: 'master_uri'" not in caplog.messages diff --git a/tests/pytests/integration/minion/test_return_retries.py b/tests/pytests/integration/minion/test_return_retries.py index 687ebd247ee8..c8e5f64674cf 100644 --- a/tests/pytests/integration/minion/test_return_retries.py +++ b/tests/pytests/integration/minion/test_return_retries.py @@ -3,6 +3,8 @@ import pytest from saltfactories.utils import random_string +import salt.utils.files + @pytest.fixture(scope="function") def salt_minion_retry(salt_master, salt_minion_id): @@ -53,13 +55,15 @@ def test_publish_retry(salt_master, salt_minion_retry, salt_cli, salt_run_cli): @pytest.mark.slow_test -def test_pillar_timeout(salt_master_factory): - cmd = """ - python -c "import time; time.sleep(3.0); print('{\\"foo\\": \\"bar\\"}');\" - """.strip() +def test_pillar_timeout(salt_master_factory, tmp_path): + cmd = 'print(\'{"foo": "bar"}\');\n' + + with salt.utils.files.fopen(tmp_path / "script.py", "w") as fp: + fp.write(cmd) + master_overrides = { "ext_pillar": [ - {"cmd_json": cmd}, + {"cmd_json": f"python {tmp_path / 'script.py'}"}, ], "auto_accept": True, "worker_threads": 2, @@ -103,8 +107,10 @@ def test_pillar_timeout(salt_master_factory): cli = master.salt_cli() sls_tempfile = master.state_tree.base.temp_file(f"{sls_name}.sls", sls_contents) with master.started(), minion1.started(), minion2.started(), minion3.started(), minion4.started(), sls_tempfile: + cmd = 'import time; time.sleep(6); print(\'{"foo": "bang"}\');\n' + with salt.utils.files.fopen(tmp_path / "script.py", "w") as fp: + fp.write(cmd) proc = cli.run("state.sls", sls_name, minion_tgt="*") - print(proc) # At least one minion should have a Pillar timeout assert proc.returncode == 1 minion_timed_out = False diff --git a/tests/pytests/integration/netapi/test_ssh_client.py b/tests/pytests/integration/netapi/test_ssh_client.py index 42db6d0eacdd..7b32ee83d618 100644 --- a/tests/pytests/integration/netapi/test_ssh_client.py +++ b/tests/pytests/integration/netapi/test_ssh_client.py @@ -90,7 +90,7 @@ def test_ssh_unauthenticated(client): def test_ssh_unauthenticated_raw_shell_curl(client, webserver_root, webserver_handler): - fun = "-o ProxyCommand curl {}".format(webserver_root) + fun = f"-o ProxyCommand curl {webserver_root}" low = {"client": "ssh", "tgt": "localhost", "fun": fun, "raw_shell": True} with pytest.raises(EauthAuthenticationError): @@ -102,7 +102,7 @@ def test_ssh_unauthenticated_raw_shell_curl(client, webserver_root, webserver_ha def test_ssh_unauthenticated_raw_shell_touch(client, tmp_path): badfile = tmp_path / "badfile.txt" - fun = "-o ProxyCommand touch {}".format(badfile) + fun = f"-o ProxyCommand touch {badfile}" low = {"client": "ssh", "tgt": "localhost", "fun": fun, "raw_shell": True} with pytest.raises(EauthAuthenticationError): @@ -114,7 +114,7 @@ def test_ssh_unauthenticated_raw_shell_touch(client, tmp_path): def test_ssh_authenticated_raw_shell_disabled(client, tmp_path): badfile = tmp_path / "badfile.txt" - fun = "-o ProxyCommand touch {}".format(badfile) + fun = f"-o ProxyCommand touch {badfile}" low = {"client": "ssh", "tgt": "localhost", "fun": fun, "raw_shell": True} with patch.dict(client.opts, {"netapi_allow_raw_shell": False}): @@ -149,7 +149,7 @@ def test_shell_inject_ssh_priv( "roster": "cache", "client": "ssh", "tgt": tgt, - "ssh_priv": "aaa|id>{} #".format(path), + "ssh_priv": f"aaa|id>{path} #", "fun": "test.ping", "eauth": "auto", "username": salt_auto_account.username, @@ -161,8 +161,7 @@ def test_shell_inject_ssh_priv( if ret: break assert path.exists() is False - assert not ret[tgt]["stdout"] - assert ret[tgt]["stderr"] + assert "Network is unreachable" in ret[tgt] def test_shell_inject_tgt(client, salt_ssh_roster_file, tmp_path, salt_auto_account): @@ -174,7 +173,7 @@ def test_shell_inject_tgt(client, salt_ssh_roster_file, tmp_path, salt_auto_acco low = { "roster": "cache", "client": "ssh", - "tgt": "root|id>{} #@127.0.0.1".format(path), + "tgt": f"root|id>{path} #@127.0.0.1", "roster_file": str(salt_ssh_roster_file), "rosters": "/", "fun": "test.ping", @@ -208,12 +207,14 @@ def test_shell_inject_ssh_options( "password": salt_auto_account.password, "roster_file": str(salt_ssh_roster_file), "rosters": "/", - "ssh_options": ["|id>{} #".format(path), "lol"], + "ssh_options": [f"|id>{path} #", "lol"], } ret = client.run(low) assert path.exists() is False - assert not ret["127.0.0.1"]["stdout"] - assert ret["127.0.0.1"]["stderr"] + assert ( + "Bad configuration option" in ret["127.0.0.1"] + or "no argument after keyword" in ret["127.0.0.1"] + ) def test_shell_inject_ssh_port( @@ -235,7 +236,7 @@ def test_shell_inject_ssh_port( "password": salt_auto_account.password, "roster_file": str(salt_ssh_roster_file), "rosters": "/", - "ssh_port": "hhhhh|id>{} #".format(path), + "ssh_port": f"hhhhh|id>{path} #", "ignore_host_keys": True, } ret = client.run(low) @@ -260,7 +261,7 @@ def test_shell_inject_remote_port_forwards( "fun": "test.ping", "roster_file": str(salt_ssh_roster_file), "rosters": "/", - "ssh_remote_port_forwards": "hhhhh|id>{} #, lol".format(path), + "ssh_remote_port_forwards": f"hhhhh|id>{path} #, lol", "eauth": "auto", "username": salt_auto_account.username, "password": salt_auto_account.password, @@ -288,7 +289,7 @@ def test_extra_mods(client, ssh_priv_key, rosters_dir, tmp_path, salt_auth_accou "username": salt_auth_account_1.username, "password": salt_auth_account_1.password, "regen_thin": True, - "thin_extra_mods": "';touch {};'".format(path), + "thin_extra_mods": f"';touch {path};'", } ret = client.run(low) @@ -417,7 +418,7 @@ def test_ssh_cve_2021_3197_a( "client": "ssh", "tgt": "localhost", "fun": "test.ping", - "ssh_port": '22 -o ProxyCommand="touch {}"'.format(exploited_path), + "ssh_port": f'22 -o ProxyCommand="touch {exploited_path}"', "ssh_priv": ssh_priv_key, "roster_file": "roster", "rosters": [rosters_dir], @@ -441,7 +442,7 @@ def test_ssh_cve_2021_3197_b( "tgt": "localhost", "fun": "test.ping", "ssh_port": 22, - "ssh_options": ['ProxyCommand="touch {}"'.format(exploited_path)], + "ssh_options": [f'ProxyCommand="touch {exploited_path}"'], "ssh_priv": ssh_priv_key, "roster_file": "roster", "rosters": [rosters_dir], diff --git a/tests/pytests/scenarios/multimaster/beacons/test_inotify.py b/tests/pytests/scenarios/multimaster/beacons/test_inotify.py index c16384d7b6fb..c1457adfa0e8 100644 --- a/tests/pytests/scenarios/multimaster/beacons/test_inotify.py +++ b/tests/pytests/scenarios/multimaster/beacons/test_inotify.py @@ -46,6 +46,7 @@ def setup_beacons(mm_master_1_salt_cli, salt_mm_minion_1, inotify_test_path): "inotify", beacon_data=[{"files": {str(inotify_test_path): {"mask": ["create"]}}}], minion_tgt=salt_mm_minion_1.id, + timeout=60, ) assert ret.returncode == 0 log.debug("Inotify beacon add returned: %s", ret.data or ret.stdout) @@ -95,7 +96,7 @@ def test_beacons_duplicate_53344( # Since beacons will be executed both together, we wait for the status beacon event # which means that, the inotify becacon was executed too start_time = setup_beacons - expected_tag = "salt/beacon/{}/status/*".format(salt_mm_minion_1.id) + expected_tag = f"salt/beacon/{salt_mm_minion_1.id}/status/*" expected_patterns = [ (salt_mm_master_1.id, expected_tag), (salt_mm_master_2.id, expected_tag), diff --git a/tests/pytests/unit/loader/test_loader.py b/tests/pytests/unit/loader/test_loader.py index f4a4b51a58fa..3c26b435c8c0 100644 --- a/tests/pytests/unit/loader/test_loader.py +++ b/tests/pytests/unit/loader/test_loader.py @@ -10,6 +10,7 @@ import pytest +import salt.exceptions import salt.loader import salt.loader.lazy @@ -62,3 +63,22 @@ def test_raw_mod_functions(): ret = salt.loader.raw_mod(opts, "grains", "get") for k, v in ret.items(): assert isinstance(v, salt.loader.lazy.LoadedFunc) + + +def test_named_loader_context_name_not_packed(tmp_path): + opts = { + "optimization_order": [0], + } + contents = """ + from salt.loader.dunder import loader_context + __not_packed__ = loader_context.named_context("__not_packed__") + def foobar(): + return __not_packed__["not.packed"]() + """ + with pytest.helpers.temp_file("mymod.py", contents, directory=tmp_path): + loader = salt.loader.LazyLoader([tmp_path], opts) + with pytest.raises( + salt.exceptions.LoaderError, + match="LazyLoader does not have a packed value for: __not_packed__", + ): + loader["mymod.foobar"]() diff --git a/tests/pytests/unit/loader/test_loading_modules.py b/tests/pytests/unit/loader/test_loading_modules.py new file mode 100644 index 000000000000..861e9197ecf0 --- /dev/null +++ b/tests/pytests/unit/loader/test_loading_modules.py @@ -0,0 +1,38 @@ +import pytest + +import salt.loader +import salt.loader.lazy +import salt.modules.boto_vpc +import salt.modules.virt + + +@pytest.fixture +def minion_mods(minion_opts): + utils = salt.loader.utils(minion_opts) + return salt.loader.minion_mods(minion_opts, utils=utils) + + +@pytest.mark.skipif( + not salt.modules.boto_vpc.HAS_BOTO, reason="boto must be installed." +) +def test_load_boto_vpc(minion_mods): + func = None + try: + func = minion_mods["boto_vpc.check_vpc"] + except KeyError: + pytest.fail("loader should not raise KeyError") + assert func is not None + assert isinstance(func, salt.loader.lazy.LoadedFunc) + + +@pytest.mark.skipif( + not salt.modules.virt.HAS_LIBVIRT, reason="libvirt-python must be installed." +) +def test_load_virt(minion_mods): + func = None + try: + func = minion_mods["virt.ctrl_alt_del"] + except KeyError: + pytest.fail("loader should not raise KeyError") + assert func is not None + assert isinstance(func, salt.loader.lazy.LoadedFunc) diff --git a/tools/pkg/repo/create.py b/tools/pkg/repo/create.py index 4de9ce90ad22..8e0926c1a415 100644 --- a/tools/pkg/repo/create.py +++ b/tools/pkg/repo/create.py @@ -385,13 +385,14 @@ def rpm( assert repo_path is not None assert key_id is not None - if distro == "photon": - distro_version = f"{distro_version}.0" display_name = f"{distro.capitalize()} {distro_version}" if distro_version not in _rpm_distro_info[distro]: ctx.error(f"Support for {display_name} is missing.") ctx.exit(1) + if distro == "photon": + distro_version = f"{distro_version}.0" + ctx.info("Creating repository directory structure ...") create_repo_path = create_top_level_repo_path( ctx,