Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 26 additions & 5 deletions Lib/test/libregrtest/run_workers.py
Original file line number Diff line number Diff line change
Expand Up @@ -178,8 +178,9 @@ def stop(self) -> None:
self._kill()

def _run_process(self, runtests: WorkerRunTests, output_fd: int,
worker_config: StrPath,
tmp_dir: StrPath | None = None) -> int | None:
popen = create_worker_process(runtests, output_fd, tmp_dir)
popen = create_worker_process(runtests, output_fd, worker_config, tmp_dir)
self._popen = popen
self._killed = False

Expand Down Expand Up @@ -293,7 +294,8 @@ def create_worker_runtests(self, test_name: TestName,
**kwargs)

def run_tmp_files(self, worker_runtests: WorkerRunTests,
stdout_fd: int) -> tuple[int | None, list[StrPath]]:
stdout_fd: int,
worker_config: StrPath) -> tuple[int | None, list[StrPath]]:
# gh-93353: Check for leaked temporary files in the parent process,
# since the deletion of temporary files can happen late during
# Python finalization: too late for libregrtest.
Expand All @@ -305,12 +307,12 @@ def run_tmp_files(self, worker_runtests: WorkerRunTests,
tmp_dir = os.path.abspath(tmp_dir)
try:
retcode = self._run_process(worker_runtests,
stdout_fd, tmp_dir)
stdout_fd, worker_config, tmp_dir)
finally:
tmp_files = os.listdir(tmp_dir)
os_helper.rmtree(tmp_dir)
else:
retcode = self._run_process(worker_runtests, stdout_fd)
retcode = self._run_process(worker_runtests, stdout_fd, worker_config)
tmp_files = []

return (retcode, tmp_files)
Expand Down Expand Up @@ -369,9 +371,28 @@ def _runtest(self, test_name: TestName,
worker_runtests = self.create_worker_runtests(
test_name, json_file, module_name=module_name)

# Custom runners may map the source tree to a different filesystem
# root (for example, on WASI). Keep the configuration in the test
# directory and resolve its path relative to libregrtest.
config_dir = os.curdir if worker_runtests.python_cmd else None
worker_config = stack.enter_context(tempfile.NamedTemporaryFile(
'w', encoding='utf-8', prefix='test_python_', dir=config_dir,
delete_on_close=False))
worker_config.write(worker_runtests.as_json())
# Allow the worker to reopen the file on Windows.
worker_config.close()
config_path = worker_config.name
if worker_runtests.python_cmd:
try:
config_path = os.path.relpath(config_path, os.path.dirname(__file__))
except ValueError:
# On Windows the directories may be on different drives.
pass

retcode: str | int | None
retcode, tmp_files = self.run_tmp_files(worker_runtests,
stdout_file.fileno())
stdout_file.fileno(),
config_path)

stdout = self.read_stdout(stdout_file)

Expand Down
13 changes: 8 additions & 5 deletions Lib/test/libregrtest/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,10 @@


def create_worker_process(runtests: WorkerRunTests, output_fd: int,
worker_config: StrPath,
tmp_dir: StrPath | None = None) -> subprocess.Popen[str]:
worker_json = runtests.as_json()

cmd = runtests.create_python_cmd()
cmd.extend(['-m', 'test.libregrtest.worker', worker_json])
cmd.extend(['-m', 'test.libregrtest.worker', worker_config])

env = dict(os.environ)
if tmp_dir is not None:
Expand Down Expand Up @@ -122,9 +121,13 @@ def worker_process(worker_json: StrJSON) -> NoReturn:

def main() -> NoReturn:
if len(sys.argv) != 2:
print("usage: python -m test.libregrtest.worker JSON")
print("usage: python -m test.libregrtest.worker CONFIG_FILE")
sys.exit(1)
worker_json = sys.argv[1]
config_path = sys.argv[1]
if not os.path.isabs(config_path):
config_path = os.path.join(os.path.dirname(__file__), config_path)
with open(config_path, encoding='utf-8') as file:
worker_json = file.read()

tmp_dir = get_temp_dir()
# get_temp_dir() can be different in the worker and the parent process.
Expand Down
31 changes: 30 additions & 1 deletion Lib/test/test_regrtest.py
Original file line number Diff line number Diff line change
Expand Up @@ -1609,6 +1609,24 @@ def test_method4(self):
subset = ['test_method1', 'test_method3']
self.assertEqual(methods, subset)

def test_large_matchfile(self):
code = textwrap.dedent("""
import unittest

class Tests(unittest.TestCase):
def test_selected(self):
pass
def test_other(self):
pass
""")
testname = self.create_test(code=code)
filename = os.path.join(self.tmptestdir, 'matches.txt')
with open(filename, 'w', encoding='utf-8') as file:
file.write(('test_missing_' + 'x' * 80 + '\n') * 2000)
file.write('test_selected\n')
output = self.run_tests('-j2', '-v', '--matchfile', filename, testname)
self.assertEqual(self.parse_methods(output), ['test_selected'])

def test_env_changed(self):
code = textwrap.dedent("""
import unittest
Expand Down Expand Up @@ -2311,12 +2329,23 @@ def test_random_seed_workers(self):

def test_python_command(self):
code = textwrap.dedent(r"""
import os
import sys
import unittest
from test.libregrtest import worker

class WorkerTests(unittest.TestCase):
def test_dev_mode(self):
self.assertTrue(sys.flags.dev_mode)

def test_worker_config(self):
# A custom runner may map the source tree to a
# different filesystem root, as on WASI.
config = sys.orig_argv[-1]
if os.name != 'nt':
self.assertFalse(os.path.isabs(config))
config = os.path.join(os.path.dirname(worker.__file__), config)
self.assertTrue(os.path.isfile(config))
""")
tests = [self.create_test(code=code) for _ in range(3)]

Expand All @@ -2328,7 +2357,7 @@ def test_dev_mode(self):

output = self.run_tests("--python", python_cmd, "-j0", *tests)
self.check_executed_tests(output, tests,
stats=len(tests), parallel=True)
stats=2 * len(tests), parallel=True)

def test_unload_tests(self):
# Test that unloading test modules does not break tests
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Pass regrtest worker configuration through a temporary file to avoid operating
system command-line size limits when using large match files.
Loading