Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
e6646d7
gh-161430: Omit tests expected to be skipped in regrtest
vstinner Nov 5, 2026
4787c7f
Fix NEED_MODULES logic
vstinner Nov 5, 2026
6f6bd83
Complete ruleset
vstinner Nov 5, 2026
2d93e09
Detect WASI using _PYTHON_HOST_PLATFORM env var
vstinner Nov 5, 2026
28431f5
Ignore more tests on WASI
vstinner Nov 5, 2026
3756938
Filter more tests
vstinner Nov 5, 2026
8aa0d9e
Add support.supports_perf_trampoline_profiling() function
vstinner Nov 5, 2026
379f303
Filter test_gdb if gdb program is not found
vstinner Nov 5, 2026
20876b5
Update for Emscripten
vstinner Nov 5, 2026
cd46dbd
Run prek --all-files
vstinner Nov 5, 2026
fc3d09a
Filter test_asdl_parser on iOS
vstinner Nov 5, 2026
9367f53
Add missing imports
vstinner Nov 5, 2026
fe10387
Skip test_interpreters on Free Threading
vstinner Nov 5, 2026
ce8761a
Skip test_free_threading sub-tests on Windows
vstinner Nov 5, 2026
59eb2ba
Skip test_xxlimited if xxlimited is missing
vstinner Nov 5, 2026
2da05cd
Add comments
vstinner Nov 5, 2026
e87dc93
Disable NEED_MODULES
vstinner Nov 5, 2026
bfd2b07
Avoid sys.is_remote_debug_enabled()
vstinner Nov 5, 2026
3a9eb25
Disable shutil.which('gdb') check
vstinner Nov 5, 2026
9f606c3
Remove commented and dead code
vstinner Nov 6, 2026
17f9ca3
Filter more tests
vstinner Nov 6, 2026
09c49f2
Update Apple mobile/Emscripten/WASI; fix Android
vstinner Nov 6, 2026
8a438c5
Missing dependencies on Android
vstinner Nov 6, 2026
0b9c802
Run prek --all-files (remove import importlib)
vstinner Nov 6, 2026
a72df3e
Invert is_android test
vstinner Nov 6, 2026
d8f7305
Android: skip test_curses
vstinner Nov 6, 2026
ff5a77b
test_gdb is skipped on installed Python
vstinner Nov 6, 2026
eb2c29c
Skip test_gdb in macOS
vstinner Nov 7, 2026
9b62623
Remove "missing dependencies" rule
vstinner Nov 7, 2026
c830030
Strip "test." prefix from TestResults
vstinner Nov 7, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion Lib/test/libregrtest/results.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

from .runtests import RunTests
from .result import State, TestResult, TestStats, Location
from .skipped import expected_to_be_skipped
from .utils import (
StrPath, TestName, TestTuple, TestList, FilterDict,
printlist, count, format_duration)
Expand Down Expand Up @@ -98,6 +99,8 @@ def get_exitcode(self, fail_env_changed: bool, fail_rerun: bool) -> int:

def accumulate_result(self, result: TestResult, runtests: RunTests) -> None:
test_name = result.test_name
# Remove "test." prefix added by some test packages
test_name = test_name.removeprefix('test.')
rerun = runtests.rerun
fail_env_changed = runtests.fail_env_changed

Expand Down Expand Up @@ -204,6 +207,16 @@ def write_junit(self, filename: StrPath) -> None:
for s in ET.tostringlist(root):
f.write(s)

def split_expected_skip(self, tests):
skipped = []
expected = []
for name in tests:
if expected_to_be_skipped(name):
expected.append(name)
else:
skipped.append(name)
return (skipped, expected)

def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool) -> None:
ansi = get_colors()
green = ansi.GREEN
Expand All @@ -221,13 +234,20 @@ def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool) ->
all_tests = []
omitted = set(tests) - self.get_executed()

# Filter tests expected to be skipped
skipped, expected_skip = self.split_expected_skip(self.skipped)
if expected_skip:
print()
count_text = count(len(expected_skip), "test")
print(f"{count_text} expected to be skipped (not listed)")

# less important
all_tests.append(
(sorted(omitted), "test", f"{yellow}{{}} omitted:{reset}")
)
if not quiet:
all_tests.append(
(self.skipped, "test", f"{yellow}{{}} skipped:{reset}")
(skipped, "test", f"{yellow}{{}} skipped:{reset}")
)
all_tests.append(
(
Expand Down
329 changes: 329 additions & 0 deletions Lib/test/libregrtest/skipped.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,329 @@
"""
Ruleset to check if a test is expected to be skipped.
Tests are mostly filtered using sys.platform, except a few exceptions.
"""

import os
import sys
import sysconfig
from test import support


# On a WASI CI, the main regrtest process is run on the build Python and test
# worker processes are run on the host Python. Detect WASI using
# _PYTHON_HOST_PLATFORM environment variable. Below, IS_WASI should be checked
# explicitly since test.support constants (such as support.has_socket_support)
# are set by the build Python which is different.
HOST_PLATFORM = os.environ.get('_PYTHON_HOST_PLATFORM', '')
IS_WASI = support.is_wasi or HOST_PLATFORM.startswith('wasi')


NEED_MULTIPROCESSING = {
'test_multiprocessing_fork',
'test_multiprocessing_forkserver',
'test_multiprocessing_main_handling',
'test_multiprocessing_spawn',
}
SELECT_TESTS = {
'test_devpoll',
'test_epoll',
'test_kqueue',
'test_poll',
}
NEED_TESTMULTIPHASE = {
# Lib/test/test_importlib/util.py requires _testmultiphase
'test_compileall',
'test_import',
'test_pkgutil',
'test_pyclbr',
'test_runpy',
}
NEED_WORKING_THREADING = {
'test_concurrent_futures',
'test_queue',
'test_thread',
'test_threadedtempfile',
'test_threading',
'test_threading_local',
'test_webbrowser',
}
NEED_TKINTER = {
'test_idle',
'test_tcl',
'test_tkinter',
'test_ttk',
'test_ttk_textonly',
'test_turtle',
}


# List of tests expected to be skipped on sys.platform
EXPECTED_SKIP = set()
if support.Py_GIL_DISABLED:
# Need GIL build
EXPECTED_SKIP.add('test_interpreters')
else:
# Need Free Threading build
EXPECTED_SKIP.add('test_free_threading')

if not support.has_socket_support or IS_WASI:
# Need socket support
EXPECTED_SKIP |= {
'test_asyncgen',
'test_asyncio',
'test_docxmlrpc',
'test_ftplib',
'test_httplib',
'test_httpservers',
'test_imaplib',
'test_mailbox',
'test_poplib',
'test_select',
'test_selectors',
'test_smtplib',
'test_socket',
'test_urllib',
'test_urllib2',
'test_urllib2_localnet',
'test_urllib_response',
'test_wsgiref',
'test_xml_dom_xmlbuilder',
'test_xmlrpc',
}
EXPECTED_SKIP |= SELECT_TESTS
if not support.has_subprocess_support or IS_WASI:
# Need subprocess support
EXPECTED_SKIP |= {
'test_argparse',
'test_asyncio.test_subprocess',
'test_c_stack_unwind',
'test_cext',
'test_clinic',
'test_cmd_line',
'test_embed',
'test_faulthandler',
'test_file_eintr',
'test_gdb',
'test_generated_cases',
'test_getopt',
'test_optparse',
'test_perf_profiler',
'test_regrtest',
'test_repl',
'test_samply_profiler',
'test_subprocess',
'test_tools',
}
if not support.has_fork_support or IS_WASI:
# Need fork
EXPECTED_SKIP |= {
'test_fork1',
'test_multiprocessing_forkserver',
'test_wait3',
'test_wait4',
}
if not(sys.platform in ("linux", "darwin") or support.MS_WINDOWS):
# Need Py_SUPPORTS_REMOTE_DEBUG macro
EXPECTED_SKIP.add('test_remote_pdb')
if not sysconfig.is_python_build():
# Skip tests on installed Python.
# Tests only relevant for from-source Python builds
EXPECTED_SKIP |= {
# Parser/asdl.py is not installed
'test_asdl_parser',
# Tools/clinic/ is not installed
'test_clinic',
# python-gdb.py is not installed
'test_gdb',
# Tools/cases_generator is not installed
'test_generated_cases',
}
if not sysconfig.get_config_var('WITH_DTRACE'):
# Need ./configure --with-dtrace
EXPECTED_SKIP.add('test_dtrace')


if support.MS_WINDOWS:
# Tests specific to non-Windows platforms
EXPECTED_SKIP |= {
'test_asyncio.test_unix_events', # UNIX only
'test_dbm_gnu', # Need dbm.gnu module
'test_dbm_ndbm', # Need dbm.ndbm module
'test_fcntl', # Need fcntl module
'test_free_threading.test_dbm_gnu', # Need dbm.gnu module
'test_free_threading.test_grp', # Need grp module
'test_free_threading.test_pwd', # Need pwd module
'test_free_threading.test_readline', # Need readline module
'test_free_threading.test_resource', # Need resource module
'test_free_threading.test_syslog', # Need syslog module
'test_gdb', # test_gdb doesn't work on Windows
'test_grp', # Need grp module
'test_ioctl', # Need fcntl module
'test_openpty', # Need os.openpty()
'test_pty', # Need termios module
'test_pwd', # Need pwd module
'test_readline', # Need readline module
'test_resource', # Need resource module
'test_syslog', # Need syslog module
'test_termios', # Need termios module
'test_threadsignals', # Can't test signal on Windows
'test_tty', # Need termios and tty modules
'test_wait3', # Need os.wait3()
'test_wait4', # Need os.wait4()
}
else:
EXPECTED_SKIP |= {
# Tests specific to Windows
'test_asyncio.test_windows_events',
'test_asyncio.test_windows_utils',
'test_msvcrt',
'test_os.test_windows',
'test_startfile', # Need os.startfile()
'test_winapi',
'test_winconsoleio',
'test_winreg',
'test_winsound',
'test_wmi',
}
if sys.platform in ("win32", 'darwin'):
EXPECTED_SKIP.add('test_multiprocessing_fork')
if sys.platform in ("win32", "cygwin"):
EXPECTED_SKIP.add('test_multiprocessing_forkserver')

if not support.is_android:
# Test specific to Android
EXPECTED_SKIP.add('test_android')


if not support.is_apple:
# Test specific to Apple
EXPECTED_SKIP.add('test_apple')


if sys.platform == "darwin":
# On macOS, running gdb fails badly:
# https://github.com/python/cpython/pull/161520
EXPECTED_SKIP.add('test_gdb')


if support.is_apple_mobile:
# configure disables multiple extension modules on Apple mobile
EXPECTED_SKIP |= {
'test_curses', # need _curses
'test_dbm_gnu', # need _gdbm
'test_grp', # need grp
'test_pwd', # need pwd
'test_readline', # need readline
'test_syslog', # need syslog
}
# configure disables _multiprocessing on Apple mobile
EXPECTED_SKIP |= NEED_MULTIPROCESSING
# configure disables _tkinter on Apple mobile
EXPECTED_SKIP |= NEED_TKINTER


if support.is_emscripten:
# configure disables multiple extension modules on Emscripten
EXPECTED_SKIP |= {
'test__interpchannels', # need _interpchannels
'test__interpreters', # need _interpreters
'test_crossinterp', # need _interpreters
'test_curses', # need _curses
'test_dbm_gnu', # need _gdbm
'test_dbm_ndbm', # need _dbm
'test_grp', # need grp
'test_pwd', # need pwd
'test_readline', # need readline
'test_resource', # need resource
'test_syslog', # need syslog
}
# configure disables _multiprocessing on Emscripten
EXPECTED_SKIP |= NEED_MULTIPROCESSING

# Missing os.openpty() function on Emscripten
EXPECTED_SKIP.add('test_openpty')

EXPECTED_SKIP |= {
# Incompatible with Emscripten's mmap emulation
'test_mmap',
# Cannot create socketpair on Emscripten/WASI
'test_selectors',
}


if support.is_android or support.is_apple_mobile or support.is_wasm32 or IS_WASI:
EXPECTED_SKIP |= {
# pty is not available on these platforms
'test_pty',
# venv is not available on these platforms
'test_venv',
}


if IS_WASI:
# configure disables multiple extension modules on WASI
EXPECTED_SKIP |= {
'test__interpchannels', # need _interpchannels
'test__interpreters', # need _interpreters
'test_crossinterp', # need _interpreters
'test_curses', # need _curses
'test_dbm_gnu', # need _gdbm
'test_dbm_ndbm', # need _dbm
'test_external_inspection', # need _remote_debugging
'test_fcntl', # need fcntl
'test_gc_stats', # need _remote_debugging
'test_grp', # need grp
'test_ioctl', # need fcntl
'test_mmap', # need mmap
'test_pwd', # need pwd
'test_pyrepl', # need termios
'test_remote_pdb', # need _remote_debugging
'test_resource', # need resource
'test_syslog', # need syslog
'test_termios', # need termios
'test_tty', # need termios
'test_xxlimited', # need xxlimited
}
# configure disables _multiprocessing and _testmultiphase
# extension modules on WASI
EXPECTED_SKIP |= NEED_MULTIPROCESSING
EXPECTED_SKIP |= NEED_TESTMULTIPHASE

# Missing os.openpty() function on WASI
EXPECTED_SKIP.add('test_openpty')


def expected_to_be_skipped(test_name):
# Test EXPECTED_SKIP
test_name = test_name.removeprefix('test.')
if test_name in EXPECTED_SKIP:
return True
package = test_name.split('.')[0]
if package in EXPECTED_SKIP:
return True

# Custom ruleset
if (test_name in NEED_WORKING_THREADING
or package in NEED_WORKING_THREADING):
# Detect whether Python can start new threads
# - wasm32-wasi does not have threading.
# - wasm32-emscripten can be compiled with or without pthread
# support (-s USE_PTHREADS / __EMSCRIPTEN_PTHREADS__).
if IS_WASI:
return True
from test.support import threading_helper
return not threading_helper.can_start_thread

if test_name in SELECT_TESTS:
# For example, test_epoll needs select.epoll()
import select
name = test_name.removeprefix('test_')
return not hasattr(select, name)

if test_name in ('test_perf_profiler', 'test_perfmaps', 'test_samply_profiler'):
# Need perf trampoline profiling: configure defines the
# PY_HAVE_PERF_TRAMPOLINE macro on Linux (x86-64 and AArch64)
# and on macOS.
return not support.supports_perf_trampoline_profiling()

return False
7 changes: 7 additions & 0 deletions Lib/test/support/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3614,3 +3614,10 @@ def inject_memory_error_cm(start=0, stop=0):
yield
finally:
_testcapi.remove_mem_hooks()


def supports_perf_trampoline_profiling():
perf_trampoline = sysconfig.get_config_var("PY_HAVE_PERF_TRAMPOLINE")
if not perf_trampoline:
return False
return int(perf_trampoline) == 1
Loading
Loading