Skip to content

Commit de195f9

Browse files
committed
gh-139207: Pass regrtest worker configuration through a file
1 parent 6044c62 commit de195f9

4 files changed

Lines changed: 66 additions & 11 deletions

File tree

‎Lib/test/libregrtest/run_workers.py‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -178,8 +178,9 @@ def stop(self) -> None:
178178
self._kill()
179179

180180
def _run_process(self, runtests: WorkerRunTests, output_fd: int,
181+
worker_config: StrPath,
181182
tmp_dir: StrPath | None = None) -> int | None:
182-
popen = create_worker_process(runtests, output_fd, tmp_dir)
183+
popen = create_worker_process(runtests, output_fd, worker_config, tmp_dir)
183184
self._popen = popen
184185
self._killed = False
185186

@@ -293,7 +294,8 @@ def create_worker_runtests(self, test_name: TestName,
293294
**kwargs)
294295

295296
def run_tmp_files(self, worker_runtests: WorkerRunTests,
296-
stdout_fd: int) -> tuple[int | None, list[StrPath]]:
297+
stdout_fd: int,
298+
worker_config: StrPath) -> tuple[int | None, list[StrPath]]:
297299
# gh-93353: Check for leaked temporary files in the parent process,
298300
# since the deletion of temporary files can happen late during
299301
# Python finalization: too late for libregrtest.
@@ -305,12 +307,12 @@ def run_tmp_files(self, worker_runtests: WorkerRunTests,
305307
tmp_dir = os.path.abspath(tmp_dir)
306308
try:
307309
retcode = self._run_process(worker_runtests,
308-
stdout_fd, tmp_dir)
310+
stdout_fd, worker_config, tmp_dir)
309311
finally:
310312
tmp_files = os.listdir(tmp_dir)
311313
os_helper.rmtree(tmp_dir)
312314
else:
313-
retcode = self._run_process(worker_runtests, stdout_fd)
315+
retcode = self._run_process(worker_runtests, stdout_fd, worker_config)
314316
tmp_files = []
315317

316318
return (retcode, tmp_files)
@@ -369,9 +371,28 @@ def _runtest(self, test_name: TestName,
369371
worker_runtests = self.create_worker_runtests(
370372
test_name, json_file, module_name=module_name)
371373

374+
# Custom runners may map the source tree to a different filesystem
375+
# root (for example, on WASI). Keep the configuration in the test
376+
# directory and resolve its path relative to libregrtest.
377+
config_dir = os.curdir if worker_runtests.python_cmd else None
378+
worker_config = stack.enter_context(tempfile.NamedTemporaryFile(
379+
'w', encoding='utf-8', prefix='test_python_', dir=config_dir,
380+
delete_on_close=False))
381+
worker_config.write(worker_runtests.as_json())
382+
# Allow the worker to reopen the file on Windows.
383+
worker_config.close()
384+
config_path = worker_config.name
385+
if worker_runtests.python_cmd:
386+
try:
387+
config_path = os.path.relpath(config_path, os.path.dirname(__file__))
388+
except ValueError:
389+
# On Windows the directories may be on different drives.
390+
pass
391+
372392
retcode: str | int | None
373393
retcode, tmp_files = self.run_tmp_files(worker_runtests,
374-
stdout_file.fileno())
394+
stdout_file.fileno(),
395+
config_path)
375396

376397
stdout = self.read_stdout(stdout_file)
377398

‎Lib/test/libregrtest/worker.py‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,10 @@
2121

2222

2323
def create_worker_process(runtests: WorkerRunTests, output_fd: int,
24+
worker_config: StrPath,
2425
tmp_dir: StrPath | None = None) -> subprocess.Popen[str]:
25-
worker_json = runtests.as_json()
26-
2726
cmd = runtests.create_python_cmd()
28-
cmd.extend(['-m', 'test.libregrtest.worker', worker_json])
27+
cmd.extend(['-m', 'test.libregrtest.worker', worker_config])
2928

3029
env = dict(os.environ)
3130
if tmp_dir is not None:
@@ -122,9 +121,13 @@ def worker_process(worker_json: StrJSON) -> NoReturn:
122121

123122
def main() -> NoReturn:
124123
if len(sys.argv) != 2:
125-
print("usage: python -m test.libregrtest.worker JSON")
124+
print("usage: python -m test.libregrtest.worker CONFIG_FILE")
126125
sys.exit(1)
127-
worker_json = sys.argv[1]
126+
config_path = sys.argv[1]
127+
if not os.path.isabs(config_path):
128+
config_path = os.path.join(os.path.dirname(__file__), config_path)
129+
with open(config_path, encoding='utf-8') as file:
130+
worker_json = file.read()
128131

129132
tmp_dir = get_temp_dir()
130133
# get_temp_dir() can be different in the worker and the parent process.

‎Lib/test/test_regrtest.py‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1609,6 +1609,24 @@ def test_method4(self):
16091609
subset = ['test_method1', 'test_method3']
16101610
self.assertEqual(methods, subset)
16111611

1612+
def test_large_matchfile(self):
1613+
code = textwrap.dedent("""
1614+
import unittest
1615+
1616+
class Tests(unittest.TestCase):
1617+
def test_selected(self):
1618+
pass
1619+
def test_other(self):
1620+
pass
1621+
""")
1622+
testname = self.create_test(code=code)
1623+
filename = os.path.join(self.tmptestdir, 'matches.txt')
1624+
with open(filename, 'w', encoding='utf-8') as file:
1625+
file.write(('test_missing_' + 'x' * 80 + '\n') * 2000)
1626+
file.write('test_selected\n')
1627+
output = self.run_tests('-j2', '-v', '--matchfile', filename, testname)
1628+
self.assertEqual(self.parse_methods(output), ['test_selected'])
1629+
16121630
def test_env_changed(self):
16131631
code = textwrap.dedent("""
16141632
import unittest
@@ -2311,12 +2329,23 @@ def test_random_seed_workers(self):
23112329

23122330
def test_python_command(self):
23132331
code = textwrap.dedent(r"""
2332+
import os
23142333
import sys
23152334
import unittest
2335+
from test.libregrtest import worker
23162336
23172337
class WorkerTests(unittest.TestCase):
23182338
def test_dev_mode(self):
23192339
self.assertTrue(sys.flags.dev_mode)
2340+
2341+
def test_worker_config(self):
2342+
# A custom runner may map the source tree to a
2343+
# different filesystem root, as on WASI.
2344+
config = sys.orig_argv[-1]
2345+
if os.name != 'nt':
2346+
self.assertFalse(os.path.isabs(config))
2347+
config = os.path.join(os.path.dirname(worker.__file__), config)
2348+
self.assertTrue(os.path.isfile(config))
23202349
""")
23212350
tests = [self.create_test(code=code) for _ in range(3)]
23222351

@@ -2328,7 +2357,7 @@ def test_dev_mode(self):
23282357

23292358
output = self.run_tests("--python", python_cmd, "-j0", *tests)
23302359
self.check_executed_tests(output, tests,
2331-
stats=len(tests), parallel=True)
2360+
stats=2 * len(tests), parallel=True)
23322361

23332362
def test_unload_tests(self):
23342363
# Test that unloading test modules does not break tests
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Pass regrtest worker configuration through a temporary file to avoid operating
2+
system command-line size limits when using large match files.

0 commit comments

Comments
 (0)