Skip to content

Commit 48df125

Browse files
authored
[3.15] gh-161430: Omit tests expected to be skipped in regrtest (#161573)
No longer list tests expected to be skipped in the final report. Strip also "test." prefix in TestResults to make the output shorter. (cherry picked from commit 3de6d3a) Add utils.cleanup_test_name() function. (cherry picked from commit d7684fb)
1 parent 5ab502c commit 48df125

9 files changed

Lines changed: 389 additions & 28 deletions

File tree

‎Lib/test/libregrtest/results.py‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,10 @@
55

66
from .runtests import RunTests
77
from .result import State, TestResult, TestStats, Location
8+
from .skipped import expected_to_be_skipped
89
from .utils import (
910
StrPath, TestName, TestTuple, TestList, FilterDict,
10-
printlist, count, format_duration)
11+
printlist, count, format_duration, cleanup_test_name)
1112

1213
if TYPE_CHECKING:
1314
from xml.etree.ElementTree import Element
@@ -100,7 +101,7 @@ def get_exitcode(self, fail_env_changed: bool, fail_rerun: bool) -> int:
100101
return exitcode
101102

102103
def accumulate_result(self, result: TestResult, runtests: RunTests) -> None:
103-
test_name = result.test_name
104+
test_name = cleanup_test_name(result.test_name)
104105
rerun = runtests.rerun
105106
fail_env_changed = runtests.fail_env_changed
106107

@@ -207,7 +208,19 @@ def write_junit(self, filename: StrPath) -> None:
207208
for s in ET.tostringlist(root):
208209
f.write(s)
209210

211+
def split_expected_skip(self, tests):
212+
skipped = []
213+
expected = []
214+
for name in tests:
215+
if expected_to_be_skipped(name):
216+
expected.append(name)
217+
else:
218+
skipped.append(name)
219+
return (skipped, expected)
220+
210221
def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool) -> None:
222+
tests = tuple(cleanup_test_name(name) for name in tests)
223+
211224
ansi = get_colors()
212225
green = ansi.GREEN
213226
red = ansi.BOLD_RED
@@ -224,13 +237,20 @@ def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool) ->
224237
all_tests = []
225238
omitted = set(tests) - self.get_executed()
226239

240+
# Filter tests expected to be skipped
241+
skipped, expected_skip = self.split_expected_skip(self.skipped)
242+
if expected_skip:
243+
print()
244+
count_text = count(len(expected_skip), "test")
245+
print(f"{count_text} expected to be skipped (not listed)")
246+
227247
# less important
228248
all_tests.append(
229249
(sorted(omitted), "test", f"{yellow}{{}} omitted:{reset}")
230250
)
231251
if not quiet:
232252
all_tests.append(
233-
(self.skipped, "test", f"{yellow}{{}} skipped:{reset}")
253+
(skipped, "test", f"{yellow}{{}} skipped:{reset}")
234254
)
235255
all_tests.append(
236256
(

‎Lib/test/libregrtest/skipped.py‎

Lines changed: 334 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,334 @@
1+
"""
2+
Ruleset to check if a test is expected to be skipped.
3+
Tests are mostly filtered using sys.platform, except a few exceptions.
4+
"""
5+
6+
import os
7+
import sys
8+
import sysconfig
9+
from test import support
10+
from .utils import cleanup_test_name
11+
12+
13+
# On a WASI CI, the main regrtest process is run on the build Python and test
14+
# worker processes are run on the host Python. Detect WASI using
15+
# _PYTHON_HOST_PLATFORM environment variable. Below, IS_WASI should be checked
16+
# explicitly since test.support constants (such as support.has_socket_support)
17+
# are set by the build Python which is different.
18+
HOST_PLATFORM = os.environ.get('_PYTHON_HOST_PLATFORM', '')
19+
IS_WASI = support.is_wasi or HOST_PLATFORM.startswith('wasi')
20+
21+
22+
NEED_MULTIPROCESSING = {
23+
'test_multiprocessing_fork',
24+
'test_multiprocessing_forkserver',
25+
'test_multiprocessing_main_handling',
26+
'test_multiprocessing_spawn',
27+
}
28+
SELECT_TESTS = {
29+
'test_devpoll',
30+
'test_epoll',
31+
'test_kqueue',
32+
'test_poll',
33+
}
34+
NEED_TESTMULTIPHASE = {
35+
# Lib/test/test_importlib/util.py requires _testmultiphase
36+
'test_compileall',
37+
'test_import',
38+
'test_pkgutil',
39+
'test_pyclbr',
40+
'test_runpy',
41+
}
42+
NEED_WORKING_THREADING = {
43+
'test_concurrent_futures',
44+
'test_queue',
45+
'test_thread',
46+
'test_threadedtempfile',
47+
'test_threading',
48+
'test_threading_local',
49+
'test_webbrowser',
50+
}
51+
NEED_TKINTER = {
52+
'test_idle',
53+
'test_tcl',
54+
'test_tkinter',
55+
'test_ttk',
56+
'test_ttk_textonly',
57+
'test_turtle',
58+
}
59+
60+
61+
# List of tests expected to be skipped on sys.platform
62+
EXPECTED_SKIP = set()
63+
if support.Py_GIL_DISABLED:
64+
# Need GIL build
65+
EXPECTED_SKIP.add('test_interpreters')
66+
else:
67+
# Need Free Threading build
68+
EXPECTED_SKIP |= {
69+
'test_free_threading',
70+
'test_io.test_free_threading',
71+
}
72+
73+
if not support.has_socket_support or IS_WASI:
74+
# Need socket support
75+
EXPECTED_SKIP |= {
76+
'test_asyncgen',
77+
'test_asyncio',
78+
'test_docxmlrpc',
79+
'test_ftplib',
80+
'test_httplib',
81+
'test_httpservers',
82+
'test_imaplib',
83+
'test_mailbox',
84+
'test_poplib',
85+
'test_select',
86+
'test_selectors',
87+
'test_smtplib',
88+
'test_socket',
89+
'test_urllib',
90+
'test_urllib2',
91+
'test_urllib2_localnet',
92+
'test_urllib_response',
93+
'test_wsgiref',
94+
'test_xml_dom_xmlbuilder',
95+
'test_xmlrpc',
96+
}
97+
EXPECTED_SKIP |= SELECT_TESTS
98+
if not support.has_subprocess_support or IS_WASI:
99+
# Need subprocess support
100+
EXPECTED_SKIP |= {
101+
'test_argparse',
102+
'test_asyncio.test_subprocess',
103+
'test_c_stack_unwind',
104+
'test_cext',
105+
'test_clinic',
106+
'test_cmd_line',
107+
'test_embed',
108+
'test_faulthandler',
109+
'test_gdb',
110+
'test_generated_cases',
111+
'test_getopt',
112+
'test_io.test_file_eintr',
113+
'test_optparse',
114+
'test_perf_profiler',
115+
'test_regrtest',
116+
'test_repl',
117+
'test_samply_profiler',
118+
'test_subprocess',
119+
'test_tools',
120+
}
121+
if not support.has_fork_support or IS_WASI:
122+
# Need fork
123+
EXPECTED_SKIP |= {
124+
'test_fork1',
125+
'test_multiprocessing_forkserver',
126+
'test_wait3',
127+
'test_wait4',
128+
}
129+
if not(sys.platform in ("linux", "darwin") or support.MS_WINDOWS):
130+
# Need Py_SUPPORTS_REMOTE_DEBUG macro
131+
EXPECTED_SKIP.add('test_remote_pdb')
132+
if not sysconfig.is_python_build():
133+
# Skip tests on installed Python.
134+
# Tests only relevant for from-source Python builds
135+
EXPECTED_SKIP |= {
136+
# Parser/asdl.py is not installed
137+
'test_asdl_parser',
138+
# Tools/clinic/ is not installed
139+
'test_clinic',
140+
# python-gdb.py is not installed
141+
'test_gdb',
142+
# Tools/cases_generator is not installed
143+
'test_generated_cases',
144+
}
145+
if not sysconfig.get_config_var('WITH_DTRACE'):
146+
# Need ./configure --with-dtrace
147+
EXPECTED_SKIP.add('test_dtrace')
148+
149+
150+
if support.MS_WINDOWS:
151+
# Tests specific to non-Windows platforms
152+
EXPECTED_SKIP |= {
153+
'test_asyncio.test_unix_events', # UNIX only
154+
'test_dbm_gnu', # Need dbm.gnu module
155+
'test_dbm_ndbm', # Need dbm.ndbm module
156+
'test_fcntl', # Need fcntl module
157+
'test_free_threading.test_dbm_gnu', # Need dbm.gnu module
158+
'test_free_threading.test_grp', # Need grp module
159+
'test_free_threading.test_pwd', # Need pwd module
160+
'test_free_threading.test_readline', # Need readline module
161+
'test_free_threading.test_resource', # Need resource module
162+
'test_free_threading.test_syslog', # Need syslog module
163+
'test_gdb', # test_gdb doesn't work on Windows
164+
'test_grp', # Need grp module
165+
'test_ioctl', # Need fcntl module
166+
'test_openpty', # Need os.openpty()
167+
'test_pty', # Need termios module
168+
'test_pwd', # Need pwd module
169+
'test_readline', # Need readline module
170+
'test_resource', # Need resource module
171+
'test_syslog', # Need syslog module
172+
'test_termios', # Need termios module
173+
'test_threadsignals', # Can't test signal on Windows
174+
'test_tty', # Need termios and tty modules
175+
'test_wait3', # Need os.wait3()
176+
'test_wait4', # Need os.wait4()
177+
}
178+
else:
179+
EXPECTED_SKIP |= {
180+
# Tests specific to Windows
181+
'test_asyncio.test_windows_events',
182+
'test_asyncio.test_windows_utils',
183+
'test_io.test_winconsoleio',
184+
'test_launcher',
185+
'test_msvcrt',
186+
'test_os.test_windows',
187+
'test_startfile', # Need os.startfile()
188+
'test_winapi',
189+
'test_winreg',
190+
'test_winsound',
191+
'test_wmi',
192+
}
193+
if sys.platform in ("win32", 'darwin'):
194+
EXPECTED_SKIP.add('test_multiprocessing_fork')
195+
if sys.platform in ("win32", "cygwin"):
196+
EXPECTED_SKIP.add('test_multiprocessing_forkserver')
197+
198+
if not support.is_android:
199+
# Test specific to Android
200+
EXPECTED_SKIP.add('test_android')
201+
202+
203+
if not support.is_apple:
204+
# Test specific to Apple
205+
EXPECTED_SKIP.add('test_apple')
206+
207+
208+
if sys.platform == "darwin":
209+
# On macOS, running gdb fails badly:
210+
# https://github.com/python/cpython/pull/161520
211+
EXPECTED_SKIP.add('test_gdb')
212+
213+
214+
if support.is_apple_mobile:
215+
# configure disables multiple extension modules on Apple mobile
216+
EXPECTED_SKIP |= {
217+
'test_curses', # need _curses
218+
'test_dbm_gnu', # need _gdbm
219+
'test_grp', # need grp
220+
'test_pwd', # need pwd
221+
'test_readline', # need readline
222+
'test_syslog', # need syslog
223+
}
224+
# configure disables _multiprocessing on Apple mobile
225+
EXPECTED_SKIP |= NEED_MULTIPROCESSING
226+
# configure disables _tkinter on Apple mobile
227+
EXPECTED_SKIP |= NEED_TKINTER
228+
229+
230+
if support.is_emscripten:
231+
# configure disables multiple extension modules on Emscripten
232+
EXPECTED_SKIP |= {
233+
'test__interpchannels', # need _interpchannels
234+
'test__interpreters', # need _interpreters
235+
'test_crossinterp', # need _interpreters
236+
'test_curses', # need _curses
237+
'test_dbm_gnu', # need _gdbm
238+
'test_dbm_ndbm', # need _dbm
239+
'test_grp', # need grp
240+
'test_pwd', # need pwd
241+
'test_readline', # need readline
242+
'test_resource', # need resource
243+
'test_syslog', # need syslog
244+
}
245+
# configure disables _multiprocessing on Emscripten
246+
EXPECTED_SKIP |= NEED_MULTIPROCESSING
247+
248+
# Missing os.openpty() function on Emscripten
249+
EXPECTED_SKIP.add('test_openpty')
250+
251+
EXPECTED_SKIP |= {
252+
# Incompatible with Emscripten's mmap emulation
253+
'test_mmap',
254+
# Cannot create socketpair on Emscripten/WASI
255+
'test_selectors',
256+
}
257+
258+
259+
if support.is_android or support.is_apple_mobile or support.is_wasm32 or IS_WASI:
260+
EXPECTED_SKIP |= {
261+
# pty is not available on these platforms
262+
'test_pty',
263+
# venv is not available on these platforms
264+
'test_venv',
265+
}
266+
267+
268+
if IS_WASI:
269+
# configure disables multiple extension modules on WASI
270+
EXPECTED_SKIP |= {
271+
'test__interpchannels', # need _interpchannels
272+
'test__interpreters', # need _interpreters
273+
'test_crossinterp', # need _interpreters
274+
'test_curses', # need _curses
275+
'test_dbm_gnu', # need _gdbm
276+
'test_dbm_ndbm', # need _dbm
277+
'test_external_inspection', # need _remote_debugging
278+
'test_fcntl', # need fcntl
279+
'test_gc_stats', # need _remote_debugging
280+
'test_grp', # need grp
281+
'test_ioctl', # need fcntl
282+
'test_mmap', # need mmap
283+
'test_pwd', # need pwd
284+
'test_pyrepl', # need termios
285+
'test_remote_pdb', # need _remote_debugging
286+
'test_resource', # need resource
287+
'test_syslog', # need syslog
288+
'test_termios', # need termios
289+
'test_tty', # need termios
290+
'test_xxlimited', # need xxlimited
291+
}
292+
# configure disables _multiprocessing and _testmultiphase
293+
# extension modules on WASI
294+
EXPECTED_SKIP |= NEED_MULTIPROCESSING
295+
EXPECTED_SKIP |= NEED_TESTMULTIPHASE
296+
297+
# Missing os.openpty() function on WASI
298+
EXPECTED_SKIP.add('test_openpty')
299+
300+
301+
def expected_to_be_skipped(test_name):
302+
# Test EXPECTED_SKIP
303+
test_name = cleanup_test_name(test_name)
304+
if test_name in EXPECTED_SKIP:
305+
return True
306+
package = test_name.split('.')[0]
307+
if package in EXPECTED_SKIP:
308+
return True
309+
310+
# Custom ruleset
311+
if (test_name in NEED_WORKING_THREADING
312+
or package in NEED_WORKING_THREADING):
313+
# Detect whether Python can start new threads
314+
# - wasm32-wasi does not have threading.
315+
# - wasm32-emscripten can be compiled with or without pthread
316+
# support (-s USE_PTHREADS / __EMSCRIPTEN_PTHREADS__).
317+
if IS_WASI:
318+
return True
319+
from test.support import threading_helper
320+
return not threading_helper.can_start_thread
321+
322+
if test_name in SELECT_TESTS:
323+
# For example, test_epoll needs select.epoll()
324+
import select
325+
name = test_name.removeprefix('test_')
326+
return not hasattr(select, name)
327+
328+
if test_name in ('test_perf_profiler', 'test_perfmaps', 'test_samply_profiler'):
329+
# Need perf trampoline profiling: configure defines the
330+
# PY_HAVE_PERF_TRAMPOLINE macro on Linux (x86-64 and AArch64)
331+
# and on macOS.
332+
return not support.supports_perf_trampoline_profiling()
333+
334+
return False

0 commit comments

Comments
 (0)