diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 028778d56518f8..08128f8711b451 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -481,12 +481,6 @@ Directory and files operations or ends with an extension that is in ``PATHEXT``; and filenames that have no extension can now be found. - .. versionchanged:: 3.12.1 - On Windows, if *mode* includes ``os.X_OK``, executables with an - extension in ``PATHEXT`` will be preferred over executables without a - matching extension. - This brings behavior closer to that of Python 3.11. - .. exception:: Error This exception collects exceptions that are raised during a multi-file diff --git a/Lib/shutil.py b/Lib/shutil.py index 20ad1cb5684b25..2d2856912893bd 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -1534,21 +1534,21 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): if sys.platform == "win32": # PATHEXT is necessary to check on Windows. pathext_source = os.getenv("PATHEXT") or _WIN_DEFAULT_PATHEXT - pathext = [ext for ext in pathext_source.split(os.pathsep) if ext] + pathext = pathext_source.split(os.pathsep) + pathext = [ext.rstrip('.') for ext in pathext if ext] if use_bytes: pathext = [os.fsencode(ext) for ext in pathext] - files = ([cmd] + [cmd + ext for ext in pathext]) + files = [cmd + ext for ext in pathext] - # gh-109590. If we are looking for an executable, we need to look - # for a PATHEXT match. The first cmd is the direct match - # (e.g. python.exe instead of python) - # Check that direct match first if and only if the extension is in PATHEXT - # Otherwise check it last - suffix = os.path.splitext(files[0])[1].upper() - if mode & os.X_OK and not any(suffix == ext.upper() for ext in pathext): - files.append(files.pop(0)) + # If X_OK in mode, simulate the cmd.exe behavior: look at direct + # match if and only if the extension is in PATHEXT. + # If X_OK not in mode, simulate the first result of where.exe: + # always look at direct match before a PATHEXT match. + normcmd = cmd.upper() + if not (mode & os.X_OK) or any(normcmd.endswith(ext.upper()) for ext in pathext): + files.insert(0, cmd) else: # On other platforms you don't have things like PATHEXT to tell you # what file suffixes are executable, so just pass on cmd as-is. @@ -1557,7 +1557,7 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): seen = set() for dir in path: normdir = os.path.normcase(dir) - if not normdir in seen: + if normdir not in seen: seen.add(normdir) for thefile in files: name = os.path.join(dir, thefile) diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 7bc5d12e09c057..c553ea09e07d55 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -71,18 +71,17 @@ def wrap(*args, **kwargs): os.rename = builtin_rename return wrap -def write_file(path, content, binary=False): +def create_file(path, content=b''): """Write *content* to a file located at *path*. If *path* is a tuple instead of a string, os.path.join will be used to - make a path. If *binary* is true, the file will be opened in binary - mode. + make a path. """ if isinstance(path, tuple): path = os.path.join(*path) - mode = 'wb' if binary else 'w' - encoding = None if binary else "utf-8" - with open(path, mode, encoding=encoding) as fp: + if isinstance(content, str): + content = content.encode() + with open(path, 'xb') as fp: fp.write(content) def write_test_file(path, size): @@ -191,7 +190,7 @@ def test_rmtree_works_on_bytes(self): tmp = self.mkdtemp() victim = os.path.join(tmp, 'killme') os.mkdir(victim) - write_file(os.path.join(victim, 'somefile'), 'foo') + create_file(os.path.join(victim, 'somefile'), 'foo') victim = os.fsencode(victim) self.assertIsInstance(victim, bytes) shutil.rmtree(victim) @@ -243,7 +242,7 @@ def test_rmtree_works_on_symlinks(self): for d in dir1, dir2, dir3: os.mkdir(d) file1 = os.path.join(tmp, 'file1') - write_file(file1, 'foo') + create_file(file1, 'foo') link1 = os.path.join(dir1, 'link1') os.symlink(dir2, link1) link2 = os.path.join(dir1, 'link2') @@ -305,7 +304,7 @@ def test_rmtree_works_on_junctions(self): for d in dir1, dir2, dir3: os.mkdir(d) file1 = os.path.join(tmp, 'file1') - write_file(file1, 'foo') + create_file(file1, 'foo') link1 = os.path.join(dir1, 'link1') _winapi.CreateJunction(dir2, link1) link2 = os.path.join(dir1, 'link2') @@ -327,8 +326,8 @@ def test_rmtree_errors_onerror(self): # existing file tmpdir = self.mkdtemp() - write_file((tmpdir, "tstfile"), "") filename = os.path.join(tmpdir, "tstfile") + create_file(filename) with self.assertRaises(NotADirectoryError) as cm: shutil.rmtree(filename) self.assertEqual(cm.exception.filename, filename) @@ -359,8 +358,8 @@ def test_rmtree_errors_onexc(self): # existing file tmpdir = self.mkdtemp() - write_file((tmpdir, "tstfile"), "") filename = os.path.join(tmpdir, "tstfile") + create_file(filename) with self.assertRaises(NotADirectoryError) as cm: shutil.rmtree(filename) self.assertEqual(cm.exception.filename, filename) @@ -549,7 +548,7 @@ def raiser(fn, *args, **kwargs): os.lstat = raiser os.mkdir(TESTFN) - write_file((TESTFN, 'foo'), 'foo') + create_file((TESTFN, 'foo'), 'foo') shutil.rmtree(TESTFN) finally: os.lstat = orig_lstat @@ -622,7 +621,7 @@ def test_rmtree_with_dir_fd(self): self.addCleanup(os.close, dir_fd) os.mkdir(fullname) os.mkdir(os.path.join(fullname, 'subdir')) - write_file(os.path.join(fullname, 'subdir', 'somefile'), 'foo') + create_file(os.path.join(fullname, 'subdir', 'somefile'), 'foo') self.assertTrue(os.path.exists(fullname)) shutil.rmtree(victim, dir_fd=dir_fd) self.assertFalse(os.path.exists(fullname)) @@ -662,7 +661,7 @@ def test_rmtree_on_junction(self): src = os.path.join(TESTFN, 'cheese') dst = os.path.join(TESTFN, 'shop') os.mkdir(src) - open(os.path.join(src, 'spam'), 'wb').close() + create_file(os.path.join(src, 'spam')) _winapi.CreateJunction(src, dst) self.assertRaises(OSError, shutil.rmtree, dst) shutil.rmtree(dst, ignore_errors=True) @@ -704,9 +703,9 @@ def test_copytree_simple(self): dst_dir = os.path.join(self.mkdtemp(), 'destination') self.addCleanup(shutil.rmtree, src_dir) self.addCleanup(shutil.rmtree, os.path.dirname(dst_dir)) - write_file((src_dir, 'test.txt'), '123') + create_file((src_dir, 'test.txt'), '123') os.mkdir(os.path.join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'test.txt'), '456') shutil.copytree(src_dir, dst_dir) self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt'))) @@ -724,11 +723,11 @@ def test_copytree_dirs_exist_ok(self): self.addCleanup(shutil.rmtree, src_dir) self.addCleanup(shutil.rmtree, dst_dir) - write_file((src_dir, 'nonexisting.txt'), '123') + create_file((src_dir, 'nonexisting.txt'), '123') os.mkdir(os.path.join(src_dir, 'existing_dir')) os.mkdir(os.path.join(dst_dir, 'existing_dir')) - write_file((dst_dir, 'existing_dir', 'existing.txt'), 'will be replaced') - write_file((src_dir, 'existing_dir', 'existing.txt'), 'has been replaced') + create_file((dst_dir, 'existing_dir', 'existing.txt'), 'will be replaced') + create_file((src_dir, 'existing_dir', 'existing.txt'), 'has been replaced') shutil.copytree(src_dir, dst_dir, dirs_exist_ok=True) self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'nonexisting.txt'))) @@ -751,7 +750,7 @@ def test_copytree_symlinks(self): sub_dir = os.path.join(src_dir, 'sub') os.mkdir(src_dir) os.mkdir(sub_dir) - write_file((src_dir, 'file.txt'), 'foo') + create_file((src_dir, 'file.txt'), 'foo') src_link = os.path.join(sub_dir, 'link') dst_link = os.path.join(dst_dir, 'sub/link') os.symlink(os.path.join(src_dir, 'file.txt'), @@ -782,16 +781,16 @@ def test_copytree_with_exclude(self): src_dir = self.mkdtemp() try: dst_dir = join(self.mkdtemp(), 'destination') - write_file((src_dir, 'test.txt'), '123') - write_file((src_dir, 'test.tmp'), '123') + create_file((src_dir, 'test.txt'), '123') + create_file((src_dir, 'test.tmp'), '123') os.mkdir(join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'test.txt'), '456') os.mkdir(join(src_dir, 'test_dir2')) - write_file((src_dir, 'test_dir2', 'test.txt'), '456') + create_file((src_dir, 'test_dir2', 'test.txt'), '456') os.mkdir(join(src_dir, 'test_dir2', 'subdir')) os.mkdir(join(src_dir, 'test_dir2', 'subdir2')) - write_file((src_dir, 'test_dir2', 'subdir', 'test.txt'), '456') - write_file((src_dir, 'test_dir2', 'subdir2', 'test.py'), '456') + create_file((src_dir, 'test_dir2', 'subdir', 'test.txt'), '456') + create_file((src_dir, 'test_dir2', 'subdir2', 'test.py'), '456') # testing glob-like patterns try: @@ -850,7 +849,7 @@ def test_copytree_arg_types_of_ignore(self): os.mkdir(join(src_dir)) os.mkdir(join(src_dir, 'test_dir')) os.mkdir(os.path.join(src_dir, 'test_dir', 'subdir')) - write_file((src_dir, 'test_dir', 'subdir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'subdir', 'test.txt'), '456') invokations = [] @@ -890,9 +889,9 @@ def test_copytree_retains_permissions(self): self.addCleanup(shutil.rmtree, tmp_dir) os.chmod(src_dir, 0o777) - write_file((src_dir, 'permissive.txt'), '123') + create_file((src_dir, 'permissive.txt'), '123') os.chmod(os.path.join(src_dir, 'permissive.txt'), 0o777) - write_file((src_dir, 'restrictive.txt'), '456') + create_file((src_dir, 'restrictive.txt'), '456') os.chmod(os.path.join(src_dir, 'restrictive.txt'), 0o600) restrictive_subdir = tempfile.mkdtemp(dir=src_dir) self.addCleanup(os_helper.rmtree, restrictive_subdir) @@ -935,8 +934,7 @@ def custom_cpfun(a, b): flag = [] src = self.mkdtemp() dst = tempfile.mktemp(dir=self.mkdtemp()) - with open(os.path.join(src, 'foo'), 'w', encoding='utf-8') as f: - f.close() + create_file(os.path.join(src, 'foo')) shutil.copytree(src, dst, copy_function=custom_cpfun) self.assertEqual(len(flag), 1) @@ -971,9 +969,9 @@ def test_copytree_named_pipe(self): def test_copytree_special_func(self): src_dir = self.mkdtemp() dst_dir = os.path.join(self.mkdtemp(), 'destination') - write_file((src_dir, 'test.txt'), '123') + create_file((src_dir, 'test.txt'), '123') os.mkdir(os.path.join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'test.txt'), '456') copied = [] def _copy(src, dst): @@ -986,7 +984,7 @@ def _copy(src, dst): def test_copytree_dangling_symlinks(self): src_dir = self.mkdtemp() valid_file = os.path.join(src_dir, 'test.txt') - write_file(valid_file, 'abc') + create_file(valid_file, 'abc') dir_a = os.path.join(src_dir, 'dir_a') os.mkdir(dir_a) for d in src_dir, dir_a: @@ -1014,8 +1012,7 @@ def test_copytree_symlink_dir(self): src_dir = self.mkdtemp() dst_dir = os.path.join(self.mkdtemp(), 'destination') os.mkdir(os.path.join(src_dir, 'real_dir')) - with open(os.path.join(src_dir, 'real_dir', 'test.txt'), 'wb'): - pass + create_file(os.path.join(src_dir, 'real_dir', 'test.txt')) os.symlink(os.path.join(src_dir, 'real_dir'), os.path.join(src_dir, 'link_to_dir'), target_is_directory=True) @@ -1035,7 +1032,7 @@ def test_copytree_return_value(self): dst_dir = src_dir + "dest" self.addCleanup(shutil.rmtree, dst_dir, True) src = os.path.join(src_dir, 'foo') - write_file(src, 'foo') + create_file(src, 'foo') rv = shutil.copytree(src_dir, dst_dir) self.assertEqual(['foo'], os.listdir(rv)) @@ -1047,7 +1044,7 @@ def test_copytree_subdirectory(self): dst_dir = os.path.join(src_dir, "somevendor", "1.0") os.makedirs(src_dir) src = os.path.join(src_dir, 'pol') - write_file(src, 'pol') + create_file(src, 'pol') rv = shutil.copytree(src_dir, dst_dir) self.assertEqual(['pol'], os.listdir(rv)) @@ -1062,8 +1059,8 @@ def test_copymode_follow_symlinks(self): dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') dst_link = os.path.join(tmp_dir, 'quux') - write_file(src, 'foo') - write_file(dst, 'foo') + create_file(src, 'foo') + create_file(dst, 'foo') os.symlink(src, src_link) os.symlink(dst, dst_link) os.chmod(src, stat.S_IRWXU|stat.S_IRWXG) @@ -1095,8 +1092,8 @@ def test_copymode_symlink_to_symlink(self): dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') dst_link = os.path.join(tmp_dir, 'quux') - write_file(src, 'foo') - write_file(dst, 'foo') + create_file(src, 'foo') + create_file(dst, 'foo') os.symlink(src, src_link) os.symlink(dst, dst_link) os.chmod(src, stat.S_IRWXU|stat.S_IRWXG) @@ -1126,8 +1123,8 @@ def test_copymode_symlink_to_symlink_wo_lchmod(self): dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') dst_link = os.path.join(tmp_dir, 'quux') - write_file(src, 'foo') - write_file(dst, 'foo') + create_file(src, 'foo') + create_file(dst, 'foo') os.symlink(src, src_link) os.symlink(dst, dst_link) shutil.copymode(src_link, dst_link, follow_symlinks=False) # silent fail @@ -1141,11 +1138,11 @@ def test_copystat_symlinks(self): dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') dst_link = os.path.join(tmp_dir, 'qux') - write_file(src, 'foo') + create_file(src, 'foo') src_stat = os.stat(src) os.utime(src, (src_stat.st_atime, src_stat.st_mtime - 42.0)) # ensure different mtimes - write_file(dst, 'bar') + create_file(dst, 'bar') self.assertNotEqual(os.stat(src).st_mtime, os.stat(dst).st_mtime) os.symlink(src, src_link) os.symlink(dst, dst_link) @@ -1185,8 +1182,8 @@ def test_copystat_handles_harmless_chflags_errors(self): tmpdir = self.mkdtemp() file1 = os.path.join(tmpdir, 'file1') file2 = os.path.join(tmpdir, 'file2') - write_file(file1, 'xxx') - write_file(file2, 'xxx') + create_file(file1, 'xxx') + create_file(file2, 'xxx') def make_chflags_raiser(err): ex = OSError() @@ -1212,9 +1209,9 @@ def _chflags_raiser(path, flags, *, follow_symlinks=True): def test_copyxattr(self): tmp_dir = self.mkdtemp() src = os.path.join(tmp_dir, 'foo') - write_file(src, 'foo') + create_file(src, 'foo') dst = os.path.join(tmp_dir, 'bar') - write_file(dst, 'bar') + create_file(dst, 'bar') # no xattr == no problem shutil._copyxattr(src, dst) @@ -1228,7 +1225,7 @@ def test_copyxattr(self): os.getxattr(dst, 'user.foo')) # check errors don't affect other attrs os.remove(dst) - write_file(dst, 'bar') + create_file(dst, 'bar') os_error = OSError(errno.EPERM, 'EPERM') def _raise_on_user_foo(fname, attr, val, **kwargs): @@ -1258,15 +1255,15 @@ def _raise_on_src(fname, *, follow_symlinks=True): # test that shutil.copystat copies xattrs src = os.path.join(tmp_dir, 'the_original') srcro = os.path.join(tmp_dir, 'the_original_ro') - write_file(src, src) - write_file(srcro, srcro) + create_file(src, src) + create_file(srcro, srcro) os.setxattr(src, 'user.the_value', b'fiddly') os.setxattr(srcro, 'user.the_value', b'fiddly') os.chmod(srcro, 0o444) dst = os.path.join(tmp_dir, 'the_copy') dstro = os.path.join(tmp_dir, 'the_copy_ro') - write_file(dst, dst) - write_file(dstro, dstro) + create_file(dst, dst) + create_file(dstro, dstro) shutil.copystat(src, dst) shutil.copystat(srcro, dstro) self.assertEqual(os.getxattr(dst, 'user.the_value'), b'fiddly') @@ -1282,13 +1279,13 @@ def test_copyxattr_symlinks(self): tmp_dir = self.mkdtemp() src = os.path.join(tmp_dir, 'foo') src_link = os.path.join(tmp_dir, 'baz') - write_file(src, 'foo') + create_file(src, 'foo') os.symlink(src, src_link) os.setxattr(src, 'trusted.foo', b'42') os.setxattr(src_link, 'trusted.foo', b'43', follow_symlinks=False) dst = os.path.join(tmp_dir, 'bar') dst_link = os.path.join(tmp_dir, 'qux') - write_file(dst, 'bar') + create_file(dst, 'bar') os.symlink(dst, dst_link) shutil._copyxattr(src_link, dst_link, follow_symlinks=False) self.assertEqual(os.getxattr(dst_link, 'trusted.foo', follow_symlinks=False), b'43') @@ -1301,7 +1298,7 @@ def test_copyxattr_symlinks(self): def _copy_file(self, method): fname = 'test.txt' tmpdir = self.mkdtemp() - write_file((tmpdir, fname), 'xxx') + create_file((tmpdir, fname), 'xxx') file1 = os.path.join(tmpdir, fname) tmpdir2 = self.mkdtemp() method(file1, tmpdir2) @@ -1320,7 +1317,7 @@ def test_copy_symlinks(self): src = os.path.join(tmp_dir, 'foo') dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') - write_file(src, 'foo') + create_file(src, 'foo') os.symlink(src, src_link) if hasattr(os, 'lchmod'): os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO) @@ -1362,7 +1359,7 @@ def test_copy2_symlinks(self): src = os.path.join(tmp_dir, 'foo') dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') - write_file(src, 'foo') + create_file(src, 'foo') os.symlink(src, src_link) if hasattr(os, 'lchmod'): os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO) @@ -1396,7 +1393,7 @@ def test_copy2_xattr(self): tmp_dir = self.mkdtemp() src = os.path.join(tmp_dir, 'foo') dst = os.path.join(tmp_dir, 'bar') - write_file(src, 'foo') + create_file(src, 'foo') os.setxattr(src, 'user.foo', b'42') shutil.copy2(src, dst) self.assertEqual( @@ -1410,7 +1407,7 @@ def test_copy_return_value(self): src_dir = self.mkdtemp() dst_dir = self.mkdtemp() src = os.path.join(src_dir, 'foo') - write_file(src, 'foo') + create_file(src, 'foo') rv = fn(src, dst_dir) self.assertEqual(rv, os.path.join(dst_dir, 'foo')) rv = fn(src, os.path.join(dst_dir, 'bar')) @@ -1427,7 +1424,7 @@ def _test_copy_dir(self, copy_func): src_file = os.path.join(src_dir, 'foo') dir2 = self.mkdtemp() dst = os.path.join(src_dir, 'does_not_exist/') - write_file(src_file, 'foo') + create_file(src_file, 'foo') if sys.platform == "win32": err = PermissionError else: @@ -1447,7 +1444,7 @@ def test_copyfile_symlinks(self): dst = os.path.join(tmp_dir, 'dst') dst_link = os.path.join(tmp_dir, 'dst_link') link = os.path.join(tmp_dir, 'link') - write_file(src, 'foo') + create_file(src, 'foo') os.symlink(src, link) # don't follow shutil.copyfile(link, dst_link, follow_symlinks=False) @@ -1464,8 +1461,7 @@ def test_dont_copy_file_onto_link_to_itself(self): src = os.path.join(TESTFN, 'cheese') dst = os.path.join(TESTFN, 'shop') try: - with open(src, 'w', encoding='utf-8') as f: - f.write('cheddar') + create_file(src, 'cheddar') try: os.link(src, dst) except PermissionError as e: @@ -1484,8 +1480,7 @@ def test_dont_copy_file_onto_symlink_to_itself(self): src = os.path.join(TESTFN, 'cheese') dst = os.path.join(TESTFN, 'shop') try: - with open(src, 'w', encoding='utf-8') as f: - f.write('cheddar') + create_file(src, 'cheddar') # Using `src` here would mean we end up with a symlink pointing # to TESTFN/TESTFN/cheese, while it should point at # TESTFN/cheese. @@ -1520,7 +1515,7 @@ def test_copyfile_return_value(self): dst_dir = self.mkdtemp() dst_file = os.path.join(dst_dir, 'bar') src_file = os.path.join(src_dir, 'foo') - write_file(src_file, 'foo') + create_file(src_file, 'foo') rv = shutil.copyfile(src_file, dst_file) self.assertTrue(os.path.exists(rv)) self.assertEqual(read_file(src_file), read_file(dst_file)) @@ -1530,7 +1525,7 @@ def test_copyfile_same_file(self): # are the same. src_dir = self.mkdtemp() src_file = os.path.join(src_dir, 'foo') - write_file(src_file, 'foo') + create_file(src_file, 'foo') self.assertRaises(SameFileError, shutil.copyfile, src_file, src_file) # But Error should work too, to stay backward compatible. self.assertRaises(Error, shutil.copyfile, src_file, src_file) @@ -1547,7 +1542,7 @@ def test_copyfile_nonexistent_dir(self): src_dir = self.mkdtemp() src_file = os.path.join(src_dir, 'foo') dst = os.path.join(src_dir, 'does_not_exist/') - write_file(src_file, 'foo') + create_file(src_file, 'foo') self.assertRaises(FileNotFoundError, shutil.copyfile, src_file, dst) def test_copyfile_copy_dir(self): @@ -1558,7 +1553,7 @@ def test_copyfile_copy_dir(self): src_file = os.path.join(src_dir, 'foo') dir2 = self.mkdtemp() dst = os.path.join(src_dir, 'does_not_exist/') - write_file(src_file, 'foo') + create_file(src_file, 'foo') if sys.platform == "win32": err = PermissionError else: @@ -1584,13 +1579,13 @@ def _create_files(self, base_dir='dist'): root_dir = self.mkdtemp() dist = os.path.join(root_dir, base_dir) os.makedirs(dist, exist_ok=True) - write_file((dist, 'file1'), 'xxx') - write_file((dist, 'file2'), 'xxx') + create_file((dist, 'file1'), 'xxx') + create_file((dist, 'file2'), 'xxx') os.mkdir(os.path.join(dist, 'sub')) - write_file((dist, 'sub', 'file3'), 'xxx') + create_file((dist, 'sub', 'file3'), 'xxx') os.mkdir(os.path.join(dist, 'sub2')) if base_dir: - write_file((root_dir, 'outer'), 'xxx') + create_file((root_dir, 'outer'), 'xxx') return root_dir, base_dir @support.requires_zlib() @@ -2170,7 +2165,7 @@ def test_disk_usage(self): def test_chown(self): dirname = self.mkdtemp() filename = tempfile.mktemp(dir=dirname) - write_file(filename, 'testing chown function') + create_file(filename, 'testing chown function') with self.assertRaises(ValueError): shutil.chown(filename) @@ -2231,37 +2226,41 @@ def check_chown(path, uid=None, gid=None): class TestWhich(BaseTest, unittest.TestCase): def setUp(self): - self.temp_dir = self.mkdtemp(prefix="Tmp") + temp_dir = self.mkdtemp(prefix="Tmp") + base_dir = os.path.join(temp_dir, TESTFN + '-basedir') + os.mkdir(base_dir) + self.dir = os.path.join(base_dir, TESTFN + '-dir') + os.mkdir(self.dir) + self.other_dir = os.path.join(base_dir, TESTFN + '-dir2') + os.mkdir(self.other_dir) # Give the temp_file an ".exe" suffix for all. # It's needed on Windows and not harmful on other platforms. - self.temp_file = tempfile.NamedTemporaryFile(dir=self.temp_dir, - prefix="Tmp", - suffix=".Exe") - os.chmod(self.temp_file.name, stat.S_IXUSR) - self.addCleanup(self.temp_file.close) - self.dir, self.file = os.path.split(self.temp_file.name) + self.file = TESTFN + '.Exe' + self.filepath = os.path.join(self.dir, self.file) + self.create_file(self.filepath) self.env_path = self.dir self.curdir = os.curdir self.ext = ".EXE" - def to_text_type(self, s): - ''' - In this class we're testing with str, so convert s to a str - ''' - if isinstance(s, bytes): - return s.decode() - return s + to_text_type = staticmethod(os.fsdecode) + + def create_file(self, path): + create_file(path) + os.chmod(path, 0o755) + + def assertNormEqual(self, actual, expected): + self.assertEqual(os.path.normcase(actual), os.path.normcase(expected)) def test_basic(self): # Given an EXE in a directory, it should be returned. rv = shutil.which(self.file, path=self.dir) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) def test_absolute_cmd(self): # When given the fully qualified path to an executable that exists, # it should be returned. - rv = shutil.which(self.temp_file.name, path=self.temp_dir) - self.assertEqual(rv, self.temp_file.name) + rv = shutil.which(self.filepath, path=self.other_dir) + self.assertEqual(rv, self.filepath) def test_relative_cmd(self): # When given the relative path with a directory part to an executable @@ -2269,7 +2268,7 @@ def test_relative_cmd(self): base_dir, tail_dir = os.path.split(self.dir) relpath = os.path.join(tail_dir, self.file) with os_helper.change_cwd(path=base_dir): - rv = shutil.which(relpath, path=self.temp_dir) + rv = shutil.which(relpath, path=self.other_dir) self.assertEqual(rv, relpath) # But it shouldn't be searched in PATH directories (issue #16957). with os_helper.change_cwd(path=self.dir): @@ -2280,9 +2279,8 @@ def test_relative_cmd(self): "test is for non win32") def test_cwd_non_win32(self): # Issue #16957 - base_dir = os.path.dirname(self.dir) with os_helper.change_cwd(path=self.dir): - rv = shutil.which(self.file, path=base_dir) + rv = shutil.which(self.file, path=self.other_dir) # non-win32: shouldn't match in the current directory. self.assertIsNone(rv) @@ -2292,57 +2290,32 @@ def test_cwd_win32(self): base_dir = os.path.dirname(self.dir) with os_helper.change_cwd(path=self.dir): with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=True): - rv = shutil.which(self.file, path=base_dir) + rv = shutil.which(self.file, path=self.other_dir) # Current directory implicitly on PATH self.assertEqual(rv, os.path.join(self.curdir, self.file)) with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=False): - rv = shutil.which(self.file, path=base_dir) + rv = shutil.which(self.file, path=self.other_dir) # Current directory not on PATH self.assertIsNone(rv) @unittest.skipUnless(sys.platform == "win32", "test is for win32") def test_cwd_win32_added_before_all_other_path(self): - base_dir = pathlib.Path(os.fsdecode(self.dir)) - - elsewhere_in_path_dir = base_dir / 'dir1' - elsewhere_in_path_dir.mkdir() - match_elsewhere_in_path = elsewhere_in_path_dir / 'hello.exe' - match_elsewhere_in_path.touch() - - exe_in_cwd = base_dir / 'hello.exe' - exe_in_cwd.touch() - - with os_helper.change_cwd(path=base_dir): - with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=True): - rv = shutil.which('hello.exe', path=elsewhere_in_path_dir) - - self.assertEqual(os.path.abspath(rv), os.path.abspath(exe_in_cwd)) - - @unittest.skipUnless(sys.platform == "win32", - "test is for win32") - def test_pathext_match_before_path_full_match(self): - base_dir = pathlib.Path(os.fsdecode(self.dir)) - dir1 = base_dir / 'dir1' - dir2 = base_dir / 'dir2' - dir1.mkdir() - dir2.mkdir() - - pathext_match = dir1 / 'hello.com.exe' - path_match = dir2 / 'hello.com' - pathext_match.touch() - path_match.touch() - - test_path = os.pathsep.join([str(dir1), str(dir2)]) - assert os.path.basename(shutil.which( - 'hello.com', path=test_path, mode = os.F_OK - )).lower() == 'hello.com.exe' + other_file_path = os.path.join(self.other_dir, self.file) + self.create_file(other_file_path) + with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=True): + with os_helper.change_cwd(path=self.dir): + rv = shutil.which(self.file, path=self.other_dir) + self.assertEqual(rv, os.path.join(self.curdir, self.file)) + with os_helper.change_cwd(path=self.other_dir): + rv = shutil.which(self.file, path=self.dir) + self.assertEqual(rv, os.path.join(self.curdir, self.file)) @os_helper.skip_if_dac_override def test_non_matching_mode(self): # Set the file read-only and ask for writeable files. - os.chmod(self.temp_file.name, stat.S_IREAD) - if os.access(self.temp_file.name, os.W_OK): + os.chmod(self.filepath, stat.S_IREAD) + if os.access(self.filepath, os.W_OK): self.skipTest("can't set the file read-only") rv = shutil.which(self.file, path=self.dir, mode=os.W_OK) self.assertIsNone(rv) @@ -2364,13 +2337,13 @@ def test_pathext_checking(self): # Ask for the file without the ".exe" extension, then ensure that # it gets found properly with the extension. rv = shutil.which(self.file[:-4], path=self.dir) - self.assertEqual(rv, self.temp_file.name[:-4] + self.ext) + self.assertEqual(rv, self.filepath[:-4] + self.ext) def test_environ_path(self): with os_helper.EnvironmentVarGuard() as env: env['PATH'] = self.env_path rv = shutil.which(self.file) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) def test_environ_path_empty(self): # PATH='': no match @@ -2384,12 +2357,9 @@ def test_environ_path_empty(self): self.assertIsNone(rv) def test_environ_path_cwd(self): - expected_cwd = os.path.basename(self.temp_file.name) + expected_cwd = self.file if sys.platform == "win32": - curdir = os.curdir - if isinstance(expected_cwd, bytes): - curdir = os.fsencode(curdir) - expected_cwd = os.path.join(curdir, expected_cwd) + expected_cwd = os.path.join(self.curdir, expected_cwd) # PATH=':': explicitly looks in the current directory with os_helper.EnvironmentVarGuard() as env: @@ -2414,14 +2384,14 @@ def test_environ_path_missing(self): create=True), \ support.swap_attr(os, 'defpath', self.dir): rv = shutil.which(self.file) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) # with confstr with unittest.mock.patch('os.confstr', return_value=self.dir, \ create=True), \ support.swap_attr(os, 'defpath', ''): rv = shutil.which(self.file) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) def test_empty_path(self): base_dir = os.path.dirname(self.dir) @@ -2439,50 +2409,88 @@ def test_empty_path_no_PATH(self): @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') def test_pathext(self): - ext = self.to_text_type(".xyz") - temp_filexyz = tempfile.NamedTemporaryFile(dir=self.temp_dir, - prefix=self.to_text_type("Tmp2"), suffix=ext) - os.chmod(temp_filexyz.name, stat.S_IXUSR) - self.addCleanup(temp_filexyz.close) - - # strip path and extension - program = os.path.basename(temp_filexyz.name) - program = os.path.splitext(program)[0] - + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) with os_helper.EnvironmentVarGuard() as env: - env['PATHEXT'] = ext if isinstance(ext, str) else ext.decode() - rv = shutil.which(program, path=self.temp_dir) - self.assertEqual(rv, temp_filexyz.name) + env['PATHEXT'] = ext + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) # Issue 40592: See https://bugs.python.org/issue40592 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') def test_pathext_with_empty_str(self): - ext = self.to_text_type(".xyz") - temp_filexyz = tempfile.NamedTemporaryFile(dir=self.temp_dir, - prefix=self.to_text_type("Tmp2"), suffix=ext) - self.addCleanup(temp_filexyz.close) + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) + with os_helper.EnvironmentVarGuard() as env: + env['PATHEXT'] = ext + ';' # note the ; + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) - # strip path and extension - program = os.path.basename(temp_filexyz.name) - program = os.path.splitext(program)[0] + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_with_multidot_extension(self): + ext = '.foo.bar' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) + with os_helper.EnvironmentVarGuard() as env: + env['PATHEXT'] = ext + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_with_null_extension(self): + cmd = self.to_text_type(TESTFN2) + cmddot = cmd + self.to_text_type('.') + filepath = os.path.join(self.dir, cmd) + self.create_file(filepath) with os_helper.EnvironmentVarGuard() as env: - env['PATHEXT'] = f"{ext if isinstance(ext, str) else ext.decode()};" # note the ; - rv = shutil.which(program, path=self.temp_dir) - self.assertEqual(rv, temp_filexyz.name) + env['PATHEXT'] = '.xyz' + self.assertIsNone(shutil.which(cmd, path=self.dir)) + self.assertIsNone(shutil.which(cmddot, path=self.dir)) + env['PATHEXT'] = '.xyz;.' # note the . + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmddot, path=self.dir), + filepath + self.to_text_type('.')) + env['PATHEXT'] = '.xyz;..' # multiple dots + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmddot, path=self.dir), + filepath + self.to_text_type('.')) + + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_extension_ends_with_dot(self): + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + dot = self.to_text_type('.') + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) + with os_helper.EnvironmentVarGuard() as env: + env['PATHEXT'] = ext + '.' + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) # cmd.exe hangs here + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) + self.assertIsNone(shutil.which(cmd + dot, path=self.dir)) + self.assertIsNone(shutil.which(cmdext + dot, path=self.dir)) # See GH-75586 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') def test_pathext_applied_on_files_in_path(self): + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) with os_helper.EnvironmentVarGuard() as env: - env["PATH"] = self.temp_dir if isinstance(self.temp_dir, str) else self.temp_dir.decode() - env["PATHEXT"] = ".test" - - test_path = os.path.join(self.temp_dir, self.to_text_type("test_program.test")) - open(test_path, 'w').close() - os.chmod(test_path, 0o755) - - self.assertEqual(shutil.which(self.to_text_type("test_program")), test_path) + env["PATH"] = os.fsdecode(self.dir) + env["PATHEXT"] = ext + self.assertEqual(shutil.which(cmd), filepath) + self.assertEqual(shutil.which(cmdext), filepath) # See GH-75586 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') @@ -2498,49 +2506,107 @@ def test_win_path_needs_curdir(self): self.assertFalse(shutil._win_path_needs_curdir('dontcare', os.X_OK)) need_curdir_mock.assert_called_once_with('dontcare') - # See GH-109590 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') - def test_pathext_preferred_for_execute(self): - with os_helper.EnvironmentVarGuard() as env: - env["PATH"] = self.temp_dir if isinstance(self.temp_dir, str) else self.temp_dir.decode() - env["PATHEXT"] = ".test" - - exe = os.path.join(self.temp_dir, self.to_text_type("test.exe")) - open(exe, 'w').close() - os.chmod(exe, 0o755) + def test_same_dir_with_pathext_extension(self): + cmd = self.file # with .exe extension + # full match + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + self.filepath) + + cmd2 = cmd + self.to_text_type('.com') # with .exe.com extension + other_file_path = os.path.join(self.dir, cmd2) + self.create_file(other_file_path) + + # full match + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + self.filepath) + self.assertNormEqual(shutil.which(cmd2, path=self.dir), other_file_path) + self.assertNormEqual(shutil.which(cmd2, path=self.dir, mode=os.F_OK), + other_file_path) - # default behavior allows a direct match if nothing in PATHEXT matches - self.assertEqual(shutil.which(self.to_text_type("test.exe")), exe) - - dot_test = os.path.join(self.temp_dir, self.to_text_type("test.exe.test")) - open(dot_test, 'w').close() - os.chmod(dot_test, 0o755) - - # now we have a PATHEXT match, so it take precedence - self.assertEqual(shutil.which(self.to_text_type("test.exe")), dot_test) - - # but if we don't use os.X_OK we don't change the order based off PATHEXT - # and therefore get the direct match. - self.assertEqual(shutil.which(self.to_text_type("test.exe"), mode=os.F_OK), exe) - - # See GH-109590 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') - def test_pathext_given_extension_preferred(self): - with os_helper.EnvironmentVarGuard() as env: - env["PATH"] = self.temp_dir if isinstance(self.temp_dir, str) else self.temp_dir.decode() - env["PATHEXT"] = ".exe2;.exe" + def test_same_dir_without_pathext_extension(self): + cmd = self.file[:-4] # without .exe extension + # pathext match + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + self.filepath) + + # without extension + other_file_path = os.path.join(self.dir, cmd) + self.create_file(other_file_path) + + # pathext match if mode contains X_OK + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + # full match + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + other_file_path) + self.assertNormEqual(shutil.which(self.file, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(self.file, path=self.dir, mode=os.F_OK), + self.filepath) - exe = os.path.join(self.temp_dir, self.to_text_type("test.exe")) - open(exe, 'w').close() - os.chmod(exe, 0o755) - - exe2 = os.path.join(self.temp_dir, self.to_text_type("test.exe2")) - open(exe2, 'w').close() - os.chmod(exe2, 0o755) + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_dir_order_with_pathext_extension(self): + cmd = self.file # with .exe extension + search_path = os.pathsep.join([os.fsdecode(self.other_dir), + os.fsdecode(self.dir)]) + # full match in the second directory + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) + + cmd2 = cmd + self.to_text_type('.com') # with .exe.com extension + other_file_path = os.path.join(self.other_dir, cmd2) + self.create_file(other_file_path) + + # pathext match in the first directory + self.assertNormEqual(shutil.which(cmd, path=search_path), other_file_path) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + other_file_path) + # full match in the first directory + self.assertNormEqual(shutil.which(cmd2, path=search_path), other_file_path) + self.assertNormEqual(shutil.which(cmd2, path=search_path, mode=os.F_OK), + other_file_path) + + # full match in the first directory + search_path = os.pathsep.join([os.fsdecode(self.dir), + os.fsdecode(self.other_dir)]) + self.assertEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) - # even though .exe2 is preferred in PATHEXT, we matched directly to test.exe - self.assertEqual(shutil.which(self.to_text_type("test.exe")), exe) - self.assertEqual(shutil.which(self.to_text_type("test")), exe2) + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_dir_order_without_pathext_extension(self): + cmd = self.file[:-4] # without .exe extension + search_path = os.pathsep.join([os.fsdecode(self.other_dir), + os.fsdecode(self.dir)]) + # pathext match in the second directory + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) + + # without extension + other_file_path = os.path.join(self.other_dir, cmd) + self.create_file(other_file_path) + + # pathext match in the second directory + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + # full match in the first directory + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + other_file_path) + # full match in the second directory + self.assertNormEqual(shutil.which(self.file, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(self.file, path=search_path, mode=os.F_OK), + self.filepath) + + # pathext match in the first directory + search_path = os.pathsep.join([os.fsdecode(self.dir), + os.fsdecode(self.other_dir)]) + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) class TestWhichBytes(TestWhich): @@ -2548,18 +2614,12 @@ def setUp(self): TestWhich.setUp(self) self.dir = os.fsencode(self.dir) self.file = os.fsencode(self.file) - self.temp_file.name = os.fsencode(self.temp_file.name) - self.temp_dir = os.fsencode(self.temp_dir) + self.filepath = os.fsencode(self.filepath) + self.other_dir = os.fsencode(self.other_dir) self.curdir = os.fsencode(self.curdir) self.ext = os.fsencode(self.ext) - def to_text_type(self, s): - ''' - In this class we're testing with bytes, so convert s to a bytes - ''' - if isinstance(s, str): - return s.encode() - return s + to_text_type = staticmethod(os.fsencode) class TestMove(BaseTest, unittest.TestCase): @@ -2570,8 +2630,7 @@ def setUp(self): self.dst_dir = self.mkdtemp() self.src_file = os.path.join(self.src_dir, filename) self.dst_file = os.path.join(self.dst_dir, filename) - with open(self.src_file, "wb") as f: - f.write(b"spam") + create_file(self.src_file, b"spam") def _check_move_file(self, src, dst, real_dst): with open(src, "rb") as f: @@ -2649,8 +2708,7 @@ def test_move_dir_altsep_to_dir(self): def test_existing_file_inside_dest_dir(self): # A file with the same name inside the destination dir already exists. - with open(self.dst_file, "wb"): - pass + create_file(self.dst_file) self.assertRaises(shutil.Error, shutil.move, self.src_file, self.dst_dir) def test_dont_move_dir_in_itself(self): @@ -3065,8 +3123,7 @@ def test_empty_file(self): dstname = TESTFN + 'dst' self.addCleanup(lambda: os_helper.unlink(srcname)) self.addCleanup(lambda: os_helper.unlink(dstname)) - with open(srcname, "wb"): - pass + create_file(srcname) with open(srcname, "rb") as src: with open(dstname, "wb") as dst: @@ -3189,7 +3246,7 @@ def test_blocksize_arg(self): self.assertEqual(blocksize, os.path.getsize(TESTFN)) # ...unless we're dealing with a small file. os_helper.unlink(TESTFN2) - write_file(TESTFN2, b"hello", binary=True) + create_file(TESTFN2, b"hello") self.addCleanup(os_helper.unlink, TESTFN2 + '3') self.assertRaises(ZeroDivisionError, shutil.copyfile, TESTFN2, TESTFN2 + '3') diff --git a/Misc/NEWS.d/next/Library/2024-11-22-10-42-34.gh-issue-127035.UnbDlr.rst b/Misc/NEWS.d/next/Library/2024-11-22-10-42-34.gh-issue-127035.UnbDlr.rst new file mode 100644 index 00000000000000..6bb7abfdd50040 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-22-10-42-34.gh-issue-127035.UnbDlr.rst @@ -0,0 +1,4 @@ +Fix :mod:`shutil.which` on Windows. Now it looks at direct match if and only +if the command ends with a PATHEXT extension or X_OK is not in mode. Support +extensionless files if "." is in PATHEXT. Support PATHEXT extensions that end +with a dot.