From 24efd1bca21d671de3177fd015bc6a584f57bdc2 Mon Sep 17 00:00:00 2001 From: Larry Gritz Date: Wed, 14 Oct 2026 17:27:35 -0700 Subject: [PATCH] testing: runtest.py improvements ported from OpenImageIO Port the parts of OIIO PR 5465 and 5476 that apply to OSL's runtest.py: * run_app() takes a failureok parameter. If the first word of the command names an OSL app (oslc, oslinfo, testshade, testrender, testoptix) or an OIIO app (oiiotool, maketx, etc.), it is replaced with the full path to the built app. * New run_commands() takes a multi-line string of shell commands, one per line, and runs each through run_app(). Blank lines and `#` comments are skipped. So a run.py can list commands exactly as they would be typed, and docs can include them by reference. * The oslc, oslinfo, oiiotool, maketx, oiiodiff, testshade, testrender, and testoptix helpers are now thin wrappers around run_app(). oiiotool() also gained failureok and concat parameters. * text_diff ignores trailing whitespace differences, which can vary by platform (e.g. cmd.exe's echo). * redirect_push and redirect_pop to let a test redirect a portion of output to a different file. Assisted-by: Claude Code / Claude Opus 5.5 Signed-off-by: Larry Gritz --- testsuite/runtest.py | 229 +++++++++++++++++++++++++++++-------------- 1 file changed, 157 insertions(+), 72 deletions(-) diff --git a/testsuite/runtest.py b/testsuite/runtest.py index ec8974d1a7..5edb96c269 100755 --- a/testsuite/runtest.py +++ b/testsuite/runtest.py @@ -18,10 +18,10 @@ from optparse import OptionParser -def make_relpath (path, start=os.curdir): +def make_relpath (path: str, start: str=os.curdir) -> str: "Wrapper around os.path.relpath which always uses '/' as the separator." p = os.path.relpath (path, start) - return p if sys.platform != "Windows" else p.replace ('\\', '/') + return p if platform.system() != 'Windows' else p.replace ('\\', '/') # @@ -29,7 +29,6 @@ def make_relpath (path, start=os.curdir): # srcdir = "." -tmpdir = "." OSL_BUILD_DIR = os.environ.get("OSL_BUILD_DIR", "..") OSL_SOURCE_DIR = os.environ.get("OSL_SOURCE_DIR", "../../..") @@ -78,15 +77,40 @@ def make_relpath (path, start=os.curdir): command = "" outputs = [ "out.txt" ] # default +# Support for temporarily redirecting a section of a test's commands to a file +# other than the default out.txt. Use redirect_push(filename) before the +# commands that should go to the alternate file, and redirect_pop() right +# after to restore the previous redirect. Push/pop nest freely. The first time +# a given filename is pushed, it's truncated and added to 'outputs' so it gets +# checked against its ref; that same membership in 'outputs' is how we +# recognize a later push of the same filename (e.g. a second section of the +# test meant to append to it) and leave its contents alone rather than +# truncating again. +_redirect_stack: list[str] = [] + +def redirect_push (filename: str) -> None : + global redirect + _redirect_stack.append (redirect) + if filename not in outputs : + open (filename, "w").close () # truncate, but only the first time + outputs.append (filename) + redirect = " >> " + filename + " " + +def redirect_pop () -> None : + global redirect + if not _redirect_stack : + raise RuntimeError ("redirect_pop: no matching redirect_push") + redirect = _redirect_stack.pop () + # The image comparison thresholds are tricky to remember. Here's the key: # A test fails if more than `failpercent` of pixel values differ by more # than `failthresh` AND the difference is more than `failrelative` times the # correct pixel value, or if even one pixel differs by more than `hardfail`. failthresh = 0.004 # "Failure" threshold for any pixel value -hardfail = 0.01 # Even one pixel this wrong => hard failure failpercent = 0.02 # Ok fo this percentage of pixels to "fail" -failrelative = 0.001 # Ok to fail up to this amount vs the pixel value +hardfail = 0.012 # Even one pixel this wrong => hard failure allowfailures = 0 # Freebie failures +failrelative = 0.001 # Ok to fail up to this amount vs the pixel value # Some tests are designed for the app running to "fail" (in the sense of # terminating with an error return code), for example, a test that is designed @@ -106,16 +130,18 @@ def make_relpath (path, start=os.curdir): cleanup_on_success = True oslcargs = "-Wall" -image_extensions = [ ".tif", ".tx", ".exr", ".jpg", ".png", ".rla", - ".dpx", ".iff", ".psd" ] +image_extensions = [ ".tif", ".tiff", ".tx", ".exr", ".jpg", ".png", ".rla", + ".dpx", ".iff", ".psd", ".bmp", ".fits", ".ico", + ".jp2", ".jxl", ".sgi", ".tga", ".TGA", ".zfile" ] compile_osl_files = True splitsymbol = ';' -#print ("srcdir = " + srcdir) -#print ("tmpdir = " + tmpdir) -#print ("path = " + path) -#print ("refdir = " + refdir) +# print ("srcdir = " + srcdir) +# print ("tmpdir = " + tmpdir) +# print ("OIIO_BUILD_ROOT = " + OIIO_BUILD_ROOT) +# print ("OIIO_TESTSUITE_IMAGEDIR = {} ({})".format(OIIO_TESTSUITE_IMAGEDIR, os.path.abspath(OIIO_TESTSUITE_IMAGEDIR))) +# print ("refdir = " + refdir) print ("test source dir = ", test_source_dir) if platform.system() == 'Windows' : @@ -128,38 +154,67 @@ def make_relpath (path, start=os.curdir): if not os.path.exists(os.path.abspath("data")) : shutil.copytree (test_source_dir, os.path.abspath("data")) else : + def newsymlink(src: str, dst: str): + print("newsymlink", src, dst) + # os.path.exists returns False for broken symlinks, so remove if thats the case + if os.path.islink(dst): + os.remove(dst) + os.symlink (src, dst) if not os.path.exists("./ref") : test_source_ref_dir = os.path.join (test_source_dir, "ref") if os.path.exists(test_source_ref_dir) : os.symlink (test_source_ref_dir, "./ref") if os.path.exists (os.path.join (test_source_dir, "src")) and not os.path.exists("./src") : - os.symlink (os.path.join (test_source_dir, "src"), "./src") + newsymlink (os.path.join (test_source_dir, "src"), "./src") if not os.path.exists("./data") : - os.symlink (test_source_dir, "./data") + newsymlink (test_source_dir, "./data") + -pythonbin = sys.executable +if os.getenv("Python_EXECUTABLE") : + pythonbin = os.getenv("Python_EXECUTABLE") +else : + pythonbin = sys.executable +#print ("pythonbin = ", pythonbin) + +# Command names that run_app() will recognize as the first word of a command +# and replace with the full path to the corresponding built app. +oiio_app_list = ("oiiotool", "iinfo", "idiff", "maketx", "iconvert", "igrep", + "testtex", "iv") +osl_app_list = ("oslc", "oslinfo", "testshade", "testrender", "testoptix") +app_list = oiio_app_list + osl_app_list #print ("pythonbin = ", pythonbin) ########################################################################### # Handy functions... +# Strip trailing spaces/tabs from a line, but leave its line ending (if any) +# alone. Used to make text_diff tolerant of trailing whitespace, which can +# vary by platform (e.g. cmd.exe's `echo` bakes in a trailing space that a +# Unix shell would not) without being a meaningful difference in output. +def _rstrip_line (line: str) -> str: + ending = line[len (line.rstrip ('\r\n')):] + return line.rstrip () + ending + + # Compare two text files. Returns 0 if they are equal otherwise returns # a non-zero value and writes the differences to "diff_file". # Based on the command-line interface to difflib example from the Python # documentation -def text_diff (fromfile, tofile, diff_file=None, filter_re=None): +def text_diff (fromfile: str, tofile: str, diff_file: str=None, filter_re=None) -> int: import time try: fromdate = time.ctime (os.stat (fromfile).st_mtime) todate = time.ctime (os.stat (tofile).st_mtime) if filter_re: - filt = re.compile(filter_re) - fromlines = [l for l in open (fromfile, 'r').readlines() if filt.match(l) is not None] - tolines = [l for l in open (tofile, 'r').readlines() if filt.match(l) is not None] - else: - fromlines = open (fromfile, 'r').readlines() - tolines = open (tofile, 'r').readlines() + filt = re.compile(filter_re) + fromlines = [l for l in open (fromfile, 'r').readlines() if filt.match(l) is not None] + tolines = [l for l in open (tofile, 'r').readlines() if filt.match(l) is not None] + else: + fromlines = open (fromfile, 'r').readlines() + tolines = open (tofile, 'r').readlines() + fromlines = [_rstrip_line(l) for l in fromlines] + tolines = [_rstrip_line(l) for l in tolines] except: print ("Unexpected error:", sys.exc_info()[0]) return -1 @@ -183,16 +238,48 @@ def text_diff (fromfile, tofile, diff_file=None, filter_re=None): -def run_app (app, silent=False, concat=True) : - command = app +# Construct a command that runs `app` (a full command line), appending its +# console output to the file "out.txt" unless `silent`. If the first word of +# the command is the name of an OSL or OIIO app, substitute the full path to +# the built app. +def run_app(app: str, silent: bool=False, failureok: bool=False, + concat: bool=True) -> str: + cmd = app.strip() + # If the command starts with the name of an OIIO app, substitute the + # full path to the built app. + words = cmd.split(maxsplit=1) + if not words: + return "" + if words[0] in app_list : + cmd = app_path(words[0]) + (" " + words[1] if len(words) > 1 else "") if not silent : - command += redirect + cmd += redirect + if failureok : + cmd += " || true " if concat: - command += " ;\n" - return command - - -def osl_app (app): + cmd += " ;\n" + return cmd + + +# Take shell `commands`, split at newlines, adorn each with redirects, etc., +# then re-join with semicolons to make a single command. +# Note: `failureok` defaults to None, meaning "use the global `failureok` +# value at the time this is called", which a run.py may have set. +def run_commands(commands: str, silent: bool=False, + failureok=None, concat: bool=True) -> str : + if failureok is None : + failureok = globals()["failureok"] + result = "" + for line in commands.splitlines(): + cmd = line.strip() + # Skip empty lines or comments + if cmd == "" or cmd.startswith("#"): + continue + result += run_app(cmd, silent=silent, failureok=failureok, concat=concat) + return result + + +def osl_app (app: str) -> str: apath = os.path.join(OSL_BUILD_DIR, "bin") if (platform.system () == 'Windows'): # when we use Visual Studio, built applications are stored @@ -201,44 +288,55 @@ def osl_app (app): return os.path.join(apath, app) + " " -def oiio_app (app): +def oiio_app (app: str) -> str: if OpenImageIO_ROOT : return os.path.join (OpenImageIO_ROOT, "bin", app) + " " else : return app + " " +# Return the full path to use for the named OSL or OIIO app. +def app_path (app: str) -> str: + if app == "testshade" and 'OSL_TESTSHADE_NAME' in os.environ : + return os.environ['OSL_TESTSHADE_NAME'] + if app in ("testrender", "testoptix") : + # Disable OptiX logging to prevent messages from the library from + # appearing in the program output. + os.environ["optix_log_level"] = "0" + if app in osl_app_list : + return osl_app(app).strip() + return oiio_app(app).strip() + + # Construct a command that will compile the shader file, appending output to # the file "out.txt". -def oslc (args) : - return (osl_app("oslc") + oslcargs + " " + args + redirect + " ;\n") +def oslc (args: str) -> str: + return run_app(f"oslc {oslcargs} {args}") # Construct a command that will run oslinfo, appending output to # the file "out.txt". -def oslinfo (args) : - return (osl_app("oslinfo") + args + redirect + " ;\n") +def oslinfo (args: str) -> str: + return run_app(f"oslinfo {args}") -# Construct a command that runs oiiotool, appending console output -# to the file "out.txt". -def oiiotool (args, silent=False) : - oiiotool_cmd = (oiio_app("oiiotool") + args) - if not silent : - oiiotool_cmd += redirect - oiiotool_cmd += " ;\n" - return oiiotool_cmd +# Construct a command that will run oiiotool and append its output to out.txt +def oiiotool (args: str, silent: bool=False, concat: bool=True, + failureok: bool=False) -> str: + return run_app(f"oiiotool {args}", + silent=silent, failureok=failureok, concat=concat) # Construct a command that runs maketx, appending console output # to the file "out.txt". -def maketx (args) : - return (oiio_app("maketx") + args + redirect + " ;\n") +def maketx (args: str) -> str: + return run_app(f"maketx {args}") # Construct a command that will compare two images, appending output to # the file "out.txt". We allow a small number of pixels to have up to # 1 LSB (8 bit) error, it's very hard to make different platforms and # compilers always match to every last floating point bit. -def oiiodiff (fileA, fileB, extraargs="", silent=True, concat=True) : +def oiiodiff (fileA: str, fileB: str, extraargs: str="", + silent: bool=True, concat: bool=True) -> str: threshargs = (" -fail " + str(failthresh) + " -failpercent " + str(failpercent) + " -hardfail " + str(hardfail) @@ -247,50 +345,37 @@ def oiiodiff (fileA, fileB, extraargs="", silent=True, concat=True) : if idiff_program == "idiff" : threshargs += (" -failrelative " + str(failrelative) + " -allowfailures " + str(allowfailures)) - command = (oiio_app(idiff_program) + "-a" - + " " + threshargs - + " " + extraargs - + " " + make_relpath(fileA,tmpdir) + idiff_postfilecmd - + " " + make_relpath(fileB,tmpdir) + idiff_postfilecmd - + (" --diff" if idiff_program == "oiiotool" else "")) - if not silent : - command += redirect - if concat: - command += " ;\n" - return command + return run_app(f"{idiff_program} -a {threshargs} {extraargs}" + f" {make_relpath(fileA,tmpdir)}{idiff_postfilecmd}" + f" {make_relpath(fileB,tmpdir)}{idiff_postfilecmd}" + + (" --diff" if idiff_program == "oiiotool" else ""), + silent=silent, concat=concat) # Construct a command that run testshade with the specified arguments, # appending output to the file "out.txt". -def testshade (args) : - if os.environ.__contains__('OSL_TESTSHADE_NAME') : - testshadename = os.environ['OSL_TESTSHADE_NAME'] + " " - else : - testshadename = osl_app("testshade") - return (testshadename + args + redirect + " ;\n") +def testshade (args) -> str: + return run_app(f"testshade {args}") # Construct a command that run testrender with the specified arguments, # appending output to the file "out.txt". -def testrender (args) : - os.environ["optix_log_level"] = "0" - return (osl_app("testrender") + " " + args + redirect + " ;\n") +def testrender (args) -> str: + return run_app(f"testrender {args}") # Construct a command that run testoptix with the specified arguments, # appending output to the file "out.txt". -def testoptix (args) : - # Disable OptiX logging to prevent messages from the library from - # appearing in the program output. - os.environ["optix_log_level"] = "0" - return (osl_app("testoptix") + " " + args + redirect + " ;\n") +def testoptix (args) -> str: + return run_app(f"testoptix {args}") # Run 'command'. For each file in 'outputs', compare it to the copy # in 'ref/'. If all outputs match their reference copies, return 0 # to pass. If any outputs do not match their references return 1 to # fail. -def runtest (command, outputs, failureok=0, failthresh=0, failpercent=0, regression=None, filter_re=None) : +def runtest (command: str, outputs: list[str], failureok: int=0, + failthresh: float=0, failpercent: float=0, regression=None, filter_re=None) -> int: # print ("working dir = " + tmpdir) os.chdir (srcdir) open ("out.txt", "w").close() # truncate out.txt @@ -322,7 +407,7 @@ def runtest (command, outputs, failureok=0, failthresh=0, failpercent=0, regress sub_command = sub_command.lstrip().rstrip() #print ("running = ", sub_command) cmdret = subprocess.call (sub_command, shell=True, env=test_environ) - if cmdret != 0 and failureok == 0 : + if cmdret != 0 and not failureok : print ("#### Error: this command failed: ", sub_command) print ("FAIL") print ("Output was:\n--------")