diff --git a/testsuite/runtest.py b/testsuite/runtest.py index ec8974d1a..5edb96c26 100755 --- a/testsuite/runtest.py +++ b/testsuite/runtest.py @@ -18,10 +18,10 @@ from optparse import OptionParser -def make_relpath (path, start=os.curdir): +def make_relpath (path: str, start: str=os.curdir) -> str: "Wrapper around os.path.relpath which always uses '/' as the separator." p = os.path.relpath (path, start) - return p if sys.platform != "Windows" else p.replace ('\\', '/') + return p if platform.system() != 'Windows' else p.replace ('\\', '/') # @@ -29,7 +29,6 @@ def make_relpath (path, start=os.curdir): # srcdir = "." -tmpdir = "." OSL_BUILD_DIR = os.environ.get("OSL_BUILD_DIR", "..") OSL_SOURCE_DIR = os.environ.get("OSL_SOURCE_DIR", "../../..") @@ -78,15 +77,40 @@ def make_relpath (path, start=os.curdir): command = "" outputs = [ "out.txt" ] # default +# Support for temporarily redirecting a section of a test's commands to a file +# other than the default out.txt. Use redirect_push(filename) before the +# commands that should go to the alternate file, and redirect_pop() right +# after to restore the previous redirect. Push/pop nest freely. The first time +# a given filename is pushed, it's truncated and added to 'outputs' so it gets +# checked against its ref; that same membership in 'outputs' is how we +# recognize a later push of the same filename (e.g. a second section of the +# test meant to append to it) and leave its contents alone rather than +# truncating again. +_redirect_stack: list[str] = [] + +def redirect_push (filename: str) -> None : + global redirect + _redirect_stack.append (redirect) + if filename not in outputs : + open (filename, "w").close () # truncate, but only the first time + outputs.append (filename) + redirect = " >> " + filename + " " + +def redirect_pop () -> None : + global redirect + if not _redirect_stack : + raise RuntimeError ("redirect_pop: no matching redirect_push") + redirect = _redirect_stack.pop () + # The image comparison thresholds are tricky to remember. Here's the key: # A test fails if more than `failpercent` of pixel values differ by more # than `failthresh` AND the difference is more than `failrelative` times the # correct pixel value, or if even one pixel differs by more than `hardfail`. failthresh = 0.004 # "Failure" threshold for any pixel value -hardfail = 0.01 # Even one pixel this wrong => hard failure failpercent = 0.02 # Ok fo this percentage of pixels to "fail" -failrelative = 0.001 # Ok to fail up to this amount vs the pixel value +hardfail = 0.012 # Even one pixel this wrong => hard failure allowfailures = 0 # Freebie failures +failrelative = 0.001 # Ok to fail up to this amount vs the pixel value # Some tests are designed for the app running to "fail" (in the sense of # terminating with an error return code), for example, a test that is designed @@ -106,16 +130,18 @@ def make_relpath (path, start=os.curdir): cleanup_on_success = True oslcargs = "-Wall" -image_extensions = [ ".tif", ".tx", ".exr", ".jpg", ".png", ".rla", - ".dpx", ".iff", ".psd" ] +image_extensions = [ ".tif", ".tiff", ".tx", ".exr", ".jpg", ".png", ".rla", + ".dpx", ".iff", ".psd", ".bmp", ".fits", ".ico", + ".jp2", ".jxl", ".sgi", ".tga", ".TGA", ".zfile" ] compile_osl_files = True splitsymbol = ';' -#print ("srcdir = " + srcdir) -#print ("tmpdir = " + tmpdir) -#print ("path = " + path) -#print ("refdir = " + refdir) +# print ("srcdir = " + srcdir) +# print ("tmpdir = " + tmpdir) +# print ("OIIO_BUILD_ROOT = " + OIIO_BUILD_ROOT) +# print ("OIIO_TESTSUITE_IMAGEDIR = {} ({})".format(OIIO_TESTSUITE_IMAGEDIR, os.path.abspath(OIIO_TESTSUITE_IMAGEDIR))) +# print ("refdir = " + refdir) print ("test source dir = ", test_source_dir) if platform.system() == 'Windows' : @@ -128,38 +154,67 @@ def make_relpath (path, start=os.curdir): if not os.path.exists(os.path.abspath("data")) : shutil.copytree (test_source_dir, os.path.abspath("data")) else : + def newsymlink(src: str, dst: str): + print("newsymlink", src, dst) + # os.path.exists returns False for broken symlinks, so remove if thats the case + if os.path.islink(dst): + os.remove(dst) + os.symlink (src, dst) if not os.path.exists("./ref") : test_source_ref_dir = os.path.join (test_source_dir, "ref") if os.path.exists(test_source_ref_dir) : os.symlink (test_source_ref_dir, "./ref") if os.path.exists (os.path.join (test_source_dir, "src")) and not os.path.exists("./src") : - os.symlink (os.path.join (test_source_dir, "src"), "./src") + newsymlink (os.path.join (test_source_dir, "src"), "./src") if not os.path.exists("./data") : - os.symlink (test_source_dir, "./data") + newsymlink (test_source_dir, "./data") + -pythonbin = sys.executable +if os.getenv("Python_EXECUTABLE") : + pythonbin = os.getenv("Python_EXECUTABLE") +else : + pythonbin = sys.executable +#print ("pythonbin = ", pythonbin) + +# Command names that run_app() will recognize as the first word of a command +# and replace with the full path to the corresponding built app. +oiio_app_list = ("oiiotool", "iinfo", "idiff", "maketx", "iconvert", "igrep", + "testtex", "iv") +osl_app_list = ("oslc", "oslinfo", "testshade", "testrender", "testoptix") +app_list = oiio_app_list + osl_app_list #print ("pythonbin = ", pythonbin) ########################################################################### # Handy functions... +# Strip trailing spaces/tabs from a line, but leave its line ending (if any) +# alone. Used to make text_diff tolerant of trailing whitespace, which can +# vary by platform (e.g. cmd.exe's `echo` bakes in a trailing space that a +# Unix shell would not) without being a meaningful difference in output. +def _rstrip_line (line: str) -> str: + ending = line[len (line.rstrip ('\r\n')):] + return line.rstrip () + ending + + # Compare two text files. Returns 0 if they are equal otherwise returns # a non-zero value and writes the differences to "diff_file". # Based on the command-line interface to difflib example from the Python # documentation -def text_diff (fromfile, tofile, diff_file=None, filter_re=None): +def text_diff (fromfile: str, tofile: str, diff_file: str=None, filter_re=None) -> int: import time try: fromdate = time.ctime (os.stat (fromfile).st_mtime) todate = time.ctime (os.stat (tofile).st_mtime) if filter_re: - filt = re.compile(filter_re) - fromlines = [l for l in open (fromfile, 'r').readlines() if filt.match(l) is not None] - tolines = [l for l in open (tofile, 'r').readlines() if filt.match(l) is not None] - else: - fromlines = open (fromfile, 'r').readlines() - tolines = open (tofile, 'r').readlines() + filt = re.compile(filter_re) + fromlines = [l for l in open (fromfile, 'r').readlines() if filt.match(l) is not None] + tolines = [l for l in open (tofile, 'r').readlines() if filt.match(l) is not None] + else: + fromlines = open (fromfile, 'r').readlines() + tolines = open (tofile, 'r').readlines() + fromlines = [_rstrip_line(l) for l in fromlines] + tolines = [_rstrip_line(l) for l in tolines] except: print ("Unexpected error:", sys.exc_info()[0]) return -1 @@ -183,16 +238,48 @@ def text_diff (fromfile, tofile, diff_file=None, filter_re=None): -def run_app (app, silent=False, concat=True) : - command = app +# Construct a command that runs `app` (a full command line), appending its +# console output to the file "out.txt" unless `silent`. If the first word of +# the command is the name of an OSL or OIIO app, substitute the full path to +# the built app. +def run_app(app: str, silent: bool=False, failureok: bool=False, + concat: bool=True) -> str: + cmd = app.strip() + # If the command starts with the name of an OIIO app, substitute the + # full path to the built app. + words = cmd.split(maxsplit=1) + if not words: + return "" + if words[0] in app_list : + cmd = app_path(words[0]) + (" " + words[1] if len(words) > 1 else "") if not silent : - command += redirect + cmd += redirect + if failureok : + cmd += " || true " if concat: - command += " ;\n" - return command - - -def osl_app (app): + cmd += " ;\n" + return cmd + + +# Take shell `commands`, split at newlines, adorn each with redirects, etc., +# then re-join with semicolons to make a single command. +# Note: `failureok` defaults to None, meaning "use the global `failureok` +# value at the time this is called", which a run.py may have set. +def run_commands(commands: str, silent: bool=False, + failureok=None, concat: bool=True) -> str : + if failureok is None : + failureok = globals()["failureok"] + result = "" + for line in commands.splitlines(): + cmd = line.strip() + # Skip empty lines or comments + if cmd == "" or cmd.startswith("#"): + continue + result += run_app(cmd, silent=silent, failureok=failureok, concat=concat) + return result + + +def osl_app (app: str) -> str: apath = os.path.join(OSL_BUILD_DIR, "bin") if (platform.system () == 'Windows'): # when we use Visual Studio, built applications are stored @@ -201,44 +288,55 @@ def osl_app (app): return os.path.join(apath, app) + " " -def oiio_app (app): +def oiio_app (app: str) -> str: if OpenImageIO_ROOT : return os.path.join (OpenImageIO_ROOT, "bin", app) + " " else : return app + " " +# Return the full path to use for the named OSL or OIIO app. +def app_path (app: str) -> str: + if app == "testshade" and 'OSL_TESTSHADE_NAME' in os.environ : + return os.environ['OSL_TESTSHADE_NAME'] + if app in ("testrender", "testoptix") : + # Disable OptiX logging to prevent messages from the library from + # appearing in the program output. + os.environ["optix_log_level"] = "0" + if app in osl_app_list : + return osl_app(app).strip() + return oiio_app(app).strip() + + # Construct a command that will compile the shader file, appending output to # the file "out.txt". -def oslc (args) : - return (osl_app("oslc") + oslcargs + " " + args + redirect + " ;\n") +def oslc (args: str) -> str: + return run_app(f"oslc {oslcargs} {args}") # Construct a command that will run oslinfo, appending output to # the file "out.txt". -def oslinfo (args) : - return (osl_app("oslinfo") + args + redirect + " ;\n") +def oslinfo (args: str) -> str: + return run_app(f"oslinfo {args}") -# Construct a command that runs oiiotool, appending console output -# to the file "out.txt". -def oiiotool (args, silent=False) : - oiiotool_cmd = (oiio_app("oiiotool") + args) - if not silent : - oiiotool_cmd += redirect - oiiotool_cmd += " ;\n" - return oiiotool_cmd +# Construct a command that will run oiiotool and append its output to out.txt +def oiiotool (args: str, silent: bool=False, concat: bool=True, + failureok: bool=False) -> str: + return run_app(f"oiiotool {args}", + silent=silent, failureok=failureok, concat=concat) # Construct a command that runs maketx, appending console output # to the file "out.txt". -def maketx (args) : - return (oiio_app("maketx") + args + redirect + " ;\n") +def maketx (args: str) -> str: + return run_app(f"maketx {args}") # Construct a command that will compare two images, appending output to # the file "out.txt". We allow a small number of pixels to have up to # 1 LSB (8 bit) error, it's very hard to make different platforms and # compilers always match to every last floating point bit. -def oiiodiff (fileA, fileB, extraargs="", silent=True, concat=True) : +def oiiodiff (fileA: str, fileB: str, extraargs: str="", + silent: bool=True, concat: bool=True) -> str: threshargs = (" -fail " + str(failthresh) + " -failpercent " + str(failpercent) + " -hardfail " + str(hardfail) @@ -247,50 +345,37 @@ def oiiodiff (fileA, fileB, extraargs="", silent=True, concat=True) : if idiff_program == "idiff" : threshargs += (" -failrelative " + str(failrelative) + " -allowfailures " + str(allowfailures)) - command = (oiio_app(idiff_program) + "-a" - + " " + threshargs - + " " + extraargs - + " " + make_relpath(fileA,tmpdir) + idiff_postfilecmd - + " " + make_relpath(fileB,tmpdir) + idiff_postfilecmd - + (" --diff" if idiff_program == "oiiotool" else "")) - if not silent : - command += redirect - if concat: - command += " ;\n" - return command + return run_app(f"{idiff_program} -a {threshargs} {extraargs}" + f" {make_relpath(fileA,tmpdir)}{idiff_postfilecmd}" + f" {make_relpath(fileB,tmpdir)}{idiff_postfilecmd}" + + (" --diff" if idiff_program == "oiiotool" else ""), + silent=silent, concat=concat) # Construct a command that run testshade with the specified arguments, # appending output to the file "out.txt". -def testshade (args) : - if os.environ.__contains__('OSL_TESTSHADE_NAME') : - testshadename = os.environ['OSL_TESTSHADE_NAME'] + " " - else : - testshadename = osl_app("testshade") - return (testshadename + args + redirect + " ;\n") +def testshade (args) -> str: + return run_app(f"testshade {args}") # Construct a command that run testrender with the specified arguments, # appending output to the file "out.txt". -def testrender (args) : - os.environ["optix_log_level"] = "0" - return (osl_app("testrender") + " " + args + redirect + " ;\n") +def testrender (args) -> str: + return run_app(f"testrender {args}") # Construct a command that run testoptix with the specified arguments, # appending output to the file "out.txt". -def testoptix (args) : - # Disable OptiX logging to prevent messages from the library from - # appearing in the program output. - os.environ["optix_log_level"] = "0" - return (osl_app("testoptix") + " " + args + redirect + " ;\n") +def testoptix (args) -> str: + return run_app(f"testoptix {args}") # Run 'command'. For each file in 'outputs', compare it to the copy # in 'ref/'. If all outputs match their reference copies, return 0 # to pass. If any outputs do not match their references return 1 to # fail. -def runtest (command, outputs, failureok=0, failthresh=0, failpercent=0, regression=None, filter_re=None) : +def runtest (command: str, outputs: list[str], failureok: int=0, + failthresh: float=0, failpercent: float=0, regression=None, filter_re=None) -> int: # print ("working dir = " + tmpdir) os.chdir (srcdir) open ("out.txt", "w").close() # truncate out.txt @@ -322,7 +407,7 @@ def runtest (command, outputs, failureok=0, failthresh=0, failpercent=0, regress sub_command = sub_command.lstrip().rstrip() #print ("running = ", sub_command) cmdret = subprocess.call (sub_command, shell=True, env=test_environ) - if cmdret != 0 and failureok == 0 : + if cmdret != 0 and not failureok : print ("#### Error: this command failed: ", sub_command) print ("FAIL") print ("Output was:\n--------")