Skip to content

Commit aa17c68

Browse files
gh-157579: Fix TemporaryDirectory cleanup fallbacks with bytes paths (GH-162116)
Co-authored-by: Stan Ulbrych <stan@python.org>
1 parent 4b8e534 commit aa17c68

4 files changed

Lines changed: 180 additions & 9 deletions

File tree

‎Lib/tempfile.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -987,10 +987,13 @@ def __init__(self, suffix=None, prefix=None, dir=None,
987987
@classmethod
988988
def _rmtree(cls, name, ignore_errors=False, repeated=False, dir_fd=None,
989989
fullname=None):
990-
if fullname is None:
991-
fullname = name
990+
991+
# work with str paths throughout
992+
name = _os.fsdecode(name)
993+
fullname = name if fullname is None else _os.fsdecode(fullname)
992994

993995
def onexc(func, path, exc, direntry=None, dir_fd=None):
996+
path = _os.fsdecode(path)
994997
# On DragonFly BSD, UF_NOUNLINK removal fails with EISDIR, not EPERM.
995998
if isinstance(exc, (PermissionError, IsADirectoryError)):
996999
if repeated and path == name:

‎Lib/test/test_shutil.py‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -566,6 +566,63 @@ def onexc(*args):
566566
self.assertTrue(onexc_called)
567567
self.assertFalse(onerror_called)
568568

569+
@unittest.skipUnless(shutil.rmtree.avoids_symlink_attacks,
570+
'requires the fd based implementation of rmtree()')
571+
def test_onexc_kwargs_dir_fd(self):
572+
os.mkdir(TESTFN)
573+
self.addCleanup(os_helper.rmtree, TESTFN)
574+
top_name = os.path.join(TESTFN, 'top')
575+
child_name = os.path.join(TESTFN, 'top', 'child')
576+
os.mkdir(top_name)
577+
os.mkdir(child_name)
578+
parent_fd = os.open(TESTFN, os.O_RDONLY)
579+
self.addCleanup(os.close, parent_fd)
580+
581+
errors = []
582+
def onexc(func, path, err, direntry=None, dir_fd=None):
583+
errors.append((func, path.replace(os.path.sep, '/')))
584+
name = path if direntry is None else direntry.name
585+
self.assertEqual(name, os.path.basename(path))
586+
self.assertTrue(os.path.samestat(
587+
os.stat(os.path.join(TESTFN, path)),
588+
os.stat(name, dir_fd=dir_fd)))
589+
590+
def failing_rmdir(path, *, dir_fd=None):
591+
raise PermissionError(errno.EACCES, 'simulated')
592+
with unittest.mock.patch('os.rmdir', failing_rmdir):
593+
shutil.rmtree('top', dir_fd=parent_fd, onexc=onexc,
594+
_onexc_kwargs=True)
595+
self.assertEqual(
596+
errors,
597+
[(failing_rmdir, 'top/child'), (failing_rmdir, 'top')])
598+
599+
child_stat = os.stat(child_name)
600+
scandir = os.scandir
601+
602+
# Fail for the top-level directory
603+
errors.clear()
604+
def failing_scandir(fd):
605+
raise PermissionError(errno.EACCES, 'simulated')
606+
with unittest.mock.patch('os.scandir', failing_scandir):
607+
shutil.rmtree('top', dir_fd=parent_fd, onexc=onexc,
608+
_onexc_kwargs=True)
609+
self.assertEqual(
610+
errors,
611+
[(failing_scandir, 'top'), (os.rmdir, 'top')])
612+
self.assertTrue(os.path.exists(top_name))
613+
614+
# Fail for the child (which is empty, so the rmdir succeeds)
615+
errors.clear()
616+
def failing_scandir_2(fd):
617+
if not os.path.samestat(os.fstat(fd), child_stat):
618+
return scandir(fd)
619+
raise PermissionError(errno.EACCES, 'simulated')
620+
with unittest.mock.patch('os.scandir', failing_scandir_2):
621+
shutil.rmtree('top', dir_fd=parent_fd, onexc=onexc,
622+
_onexc_kwargs=True)
623+
self.assertEqual(errors, [(failing_scandir_2, 'top/child')])
624+
self.assertFalse(os.path.exists(top_name))
625+
569626
def test_rmtree_does_not_choke_on_failing_lstat(self):
570627
try:
571628
orig_lstat = os.lstat

‎Lib/test/test_tempfile.py‎

Lines changed: 115 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1653,10 +1653,14 @@ def __exit__(self, *exc_info):
16531653
class TestTemporaryDirectory(BaseTestCase):
16541654
"""Test TemporaryDirectory()."""
16551655

1656-
def do_create(self, dir=None, pre="", suf="", recurse=1, dirs=1, files=1,
1656+
def do_create(self, dir=None, pre=None, suf=None, recurse=1, dirs=1, files=1,
16571657
ignore_cleanup_errors=False):
16581658
if dir is None:
16591659
dir = tempfile.gettempdir()
1660+
if pre is None:
1661+
pre = type(dir)()
1662+
if suf is None:
1663+
suf = type(dir)()
16601664
tmp = tempfile.TemporaryDirectory(
16611665
dir=dir, prefix=pre, suffix=suf,
16621666
ignore_cleanup_errors=ignore_cleanup_errors)
@@ -1666,13 +1670,17 @@ def do_create(self, dir=None, pre="", suf="", recurse=1, dirs=1, files=1,
16661670

16671671
def do_create2(self, path, recurse=1, dirs=1, files=1):
16681672
# Create subdirectories and some files
1673+
def join(name):
1674+
if isinstance(path, bytes):
1675+
name = os.fsencode(name)
1676+
return os.path.join(path, name)
16691677
if recurse:
16701678
for i in range(dirs):
1671-
name = os.path.join(path, "dir%d" % i)
1679+
name = join("dir%d" % i)
16721680
os.mkdir(name)
16731681
self.do_create2(name, recurse-1, dirs, files)
16741682
for i in range(files):
1675-
with open(os.path.join(path, "test%d.txt" % i), "wb") as f:
1683+
with open(join("test%d.txt" % i), "wb") as f:
16761684
f.write(b"Hello world!")
16771685

16781686
def test_mkdtemp_failure(self):
@@ -1698,6 +1706,15 @@ def test_explicit_cleanup(self):
16981706
finally:
16991707
os.rmdir(dir)
17001708

1709+
def test_explicit_cleanup_bytes(self):
1710+
# A TemporaryDirectory with a bytes name is deleted when cleaned up
1711+
with tempfile.TemporaryDirectory() as working_dir:
1712+
d = self.do_create(dir=os.fsencode(working_dir), recurse=2)
1713+
self.assertIsInstance(d.name, bytes)
1714+
self.assertTrue(os.path.exists(d.name))
1715+
d.cleanup()
1716+
self.assertFalse(os.path.exists(d.name))
1717+
17011718
def test_explicit_cleanup_ignore_errors(self):
17021719
"""Test that cleanup doesn't return an error when ignoring them."""
17031720
with tempfile.TemporaryDirectory() as working_dir:
@@ -1910,6 +1927,87 @@ def hook(path, *, dir_fd=None):
19101927
self.assertTrue(os.path.exists(target_file))
19111928
self.assertEqual(os.stat(target_file).st_mode, target_mode)
19121929

1930+
def _make_unopenable_subdir(self):
1931+
# Create the following structure:
1932+
# work/ - top-level
1933+
# <d>/ - tempdir to clean up (random name)
1934+
# sub/ - an unopenable dir
1935+
# subsub/
1936+
# file
1937+
# Return:
1938+
# - `work` name (str)
1939+
# - `d` (TemporaryDirectory)
1940+
# - `subsub` stat result
1941+
work = self.enterContext(os_helper.temp_dir())
1942+
d = tempfile.TemporaryDirectory(dir=work)
1943+
sub = os.path.join(d.name, 'sub')
1944+
subsub = os.path.join(sub, 'subsub')
1945+
os.mkdir(sub)
1946+
os.mkdir(subsub)
1947+
subsub_stat = os.lstat(subsub)
1948+
with open(os.path.join(subsub, 'file'), 'w') as f:
1949+
f.write("rm me pls")
1950+
os.chmod(sub, 0)
1951+
# If the test fails, make sure everything can still be removed.
1952+
self.addCleanup(d.cleanup)
1953+
@self.addCleanup
1954+
def _clean():
1955+
if os.path.isdir(sub):
1956+
os.chmod(sub, 0o700)
1957+
return work, d, subsub_stat
1958+
1959+
@os_helper.skip_unless_working_chmod
1960+
@support.requires_non_root_user
1961+
@unittest.skipUnless(tempfile._rmtree_use_dir_fd
1962+
and shutil.rmtree.avoids_symlink_attacks,
1963+
'requires the fd based cleanup')
1964+
def test_cleanup_nested_toplevel_error(self):
1965+
# rmtree() reports errors relative to the correct parent directory
1966+
work, d, subsub_stat = self._make_unopenable_subdir()
1967+
# An unrelated directory with the same name; should not be touched
1968+
unrelated = os.path.join(work, 'sub')
1969+
os.mkdir(unrelated)
1970+
1971+
rmdir = os.rmdir
1972+
failed_path = None
1973+
def failing_rmdir(path, *, dir_fd=None):
1974+
nonlocal failed_path
1975+
if path == 'sub' and dir_fd is not None and failed_path is None:
1976+
failed_path = path
1977+
raise PermissionError(errno.EPERM, 'simulated', path)
1978+
return rmdir(path, dir_fd=dir_fd)
1979+
with os_helper.change_cwd(work), mock.patch('os.rmdir', failing_rmdir):
1980+
d.cleanup()
1981+
self.assertEqual(failed_path, 'sub')
1982+
self.assertFalse(os.path.exists(d.name))
1983+
self.assertTrue(os.path.exists(unrelated))
1984+
1985+
@os_helper.skip_unless_working_chmod
1986+
@support.requires_non_root_user
1987+
@unittest.skipUnless(tempfile._rmtree_use_dir_fd
1988+
and shutil.rmtree.avoids_symlink_attacks,
1989+
'requires the fd based cleanup')
1990+
def test_cleanup_nested_scandir_error(self):
1991+
# A scandir() error is reported relative to the parent directory.
1992+
work, d, subsub_stat = self._make_unopenable_subdir()
1993+
1994+
scandir = os.scandir
1995+
failed = False
1996+
def failing_scandir(path='.'):
1997+
nonlocal failed
1998+
if (
1999+
not failed
2000+
and isinstance(path, int)
2001+
and os.path.samestat(os.fstat(path), subsub_stat)
2002+
):
2003+
failed = True
2004+
raise PermissionError(errno.EACCES, 'simulated')
2005+
return scandir(path)
2006+
with os_helper.change_cwd(work), mock.patch('os.scandir', failing_scandir):
2007+
d.cleanup()
2008+
self.assertTrue(failed)
2009+
self.assertFalse(os.path.exists(d.name))
2010+
19132011
@support.cpython_only
19142012
def test_del_on_collection(self):
19152013
# A TemporaryDirectory is deleted when garbage collected
@@ -2068,11 +2166,17 @@ def test_context_manager(self):
20682166
self.assertEqual(name, d.name)
20692167
self.assertFalse(os.path.exists(name))
20702168

2071-
def test_modes(self):
2169+
@support.subTests('bytes_name', (False, True))
2170+
def test_modes(self, bytes_name):
20722171
for mode in range(8):
20732172
mode <<= 6
20742173
with self.subTest(mode=format(mode, '03o')):
2075-
d = self.do_create(recurse=3, dirs=2, files=2)
2174+
if bytes_name:
2175+
d = self.do_create(recurse=3, dirs=2, files=2,
2176+
dir=tempfile.gettempdirb())
2177+
else:
2178+
d = self.do_create(recurse=3, dirs=2, files=2)
2179+
self.assertIsInstance(d.name, bytes if bytes_name else str)
20762180
with d:
20772181
# Change files and directories mode recursively.
20782182
for root, dirs, files in os.walk(d.name, topdown=False):
@@ -2083,13 +2187,17 @@ def test_modes(self):
20832187
self.assertFalse(os.path.exists(d.name))
20842188

20852189
@support.subTests('ignore_errors', (True, False))
2086-
def test_parent_mode_preserved(self, ignore_errors):
2190+
@support.subTests('bytes_name', (False, True))
2191+
def test_parent_mode_preserved(self, ignore_errors, bytes_name):
20872192
# Test that cleanup does not touch the parent directory,
20882193
# even if that prevents removal.
20892194
for mode in range(8):
20902195
mode <<= 6
20912196
with self.subTest(mode=format(mode, '03o')):
2092-
outer = self.do_create()
2197+
if bytes_name:
2198+
outer = self.do_create(dir=tempfile.gettempdirb())
2199+
else:
2200+
outer = self.do_create()
20932201
with outer:
20942202
d = self.do_create(dir=outer.name, dirs=2, files=2,
20952203
ignore_cleanup_errors=ignore_errors)
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
The error handler :class:`~tempfile.TemporaryDirectory` uses to clean up
2+
unreadable and inaccessible files no longer fails for temporary directories
3+
created using :class:`bytes` paths.

0 commit comments

Comments
 (0)