Skip and report libraries that fail instead of aborting the index

.gitignore

@@ -5,3 +5,5 @@ __pycache__
 /cache/
 index.json
 tests/tmp-cache/
+updateinfo-problems.txt
+genindex-problems.txt

Jenkinsfile

@@ -3,6 +3,7 @@ pipeline {
   parameters {
     booleanParam(name: 'UPLOAD_AND_CACHE', defaultValue: true, description: 'Also do stages upload and cache')
     string(name: 'WASM_OMC', defaultValue: 'omc', description: 'An omc whose getExternalFunctions describes what the prebuilt wasm modules are built from; empty skips them')
+    string(name: 'PROBLEMS_EMAIL', defaultValue: 'adrian.pop@liu.se', description: 'Who gets the list of skipped libraries when the job goes unstable; empty sends no mail')
   }
   options {
     newContainerPerStage()
@@ -41,6 +42,18 @@ pipeline {
           }
         }
       }
+      script {
+        // updateinfo and genindex skip a library they cannot handle instead of failing, and list it here
+        def problems = sh(returnStdout: true, script: 'cat updateinfo-problems.txt genindex-problems.txt 2>/dev/null || true').trim()
+        if (problems) {
+          unstable('some libraries were skipped, see updateinfo-problems.txt and genindex-problems.txt')
+          if (params.PROBLEMS_EMAIL) {
+            emailext subject: "${env.JOB_NAME} #${env.BUILD_NUMBER}: libraries skipped",
+              body: "${problems}\n\n${env.BUILD_URL}console\n",
+              to: params.PROBLEMS_EMAIL
+          }
+        }
+      }
       stash name: 'files', includes: 'index.json, rawdata.json, wasmdata.json, www-data/precompiled/wasm32-wasip1/**', allowEmpty: true
     }
   }

ompackagemanager/common.py

@@ -1,6 +1,22 @@
+import os
 import semantic_version
 
 
+def write_problems(path: str, title: str, problems: list[str]) -> None:
+    """Write what a command skipped to path, for Jenkins to mark the job unstable and mail it.
+
+    The file is removed when nothing was skipped, so a stale one never outlives a clean run.
+    """
+    if not problems:
+        if os.path.exists(path):
+            os.remove(path)
+        return
+    with open(path, "w") as f:
+        f.write("%s\n\n" % title)
+        for problem in problems:
+            f.write("- %s\n" % problem)
+
+
 def VersionNumber(s: str) -> semantic_version.Version:
     if s.startswith("v"):
         s = s[1:]

ompackagemanager/genindex.py

@@ -11,6 +11,9 @@ class DuplicateVersions(Exception):
     pass
 
 
+PROBLEMS_FILE = "genindex-problems.txt"
+
+
 def onlyVersioninSemVer(ver):
     ver.prerelease = []
     ver.build = []
@@ -138,14 +141,19 @@ def main():
     unusedExtraProvides = {(key, version) for (key, versions) in extraProvides.items() for version in versions}
 
     indexdata = {"libs": {}, "mirrors": ["https://libraries.openmodelica.org/cache/"]}
+    problems = []
     for firstKey in rawdata.keys():
         data = rawdata[firstKey]
         for refKey in data["refs"].keys():
             r = data["refs"][refKey]
             if "broken" in r:
                 continue
             if "libs" not in r:
-                raise Exception(firstKey + " " + refKey)
+                # A ref whose libraries did not load must not stop the index of all the others
+                problem = "%s %s has no libraries in rawdata.json, it is not in the index" % (firstKey, refKey)
+                print("WARNING: " + problem)
+                problems.append(problem)
+                continue
             for libName in r["libs"]:
                 lib = r["libs"][libName]
                 isgit = "github" in repos[firstKey] or "git" in repos[firstKey]
@@ -232,3 +240,5 @@ def main():
 
     with open("index.json", "w") as io:
         json.dump(indexdata, io, sort_keys=True, indent=0)
+
+    common.write_problems(PROBLEMS_FILE, "genindex skipped these library versions:", problems)

ompackagemanager/updateinfo.py

@@ -49,6 +49,8 @@ def allbranches(gitrepo: pygit2.Repository) -> list[Branch]:
 GIT_ATTEMPTS = 3
 GIT_RETRY_DELAY = 30  # seconds, doubled after each failed attempt
 unfetchedRepos = []  # the (url, error) of each repository whose cached clone was used instead
+failedLoads = []  # the (path, tagName, error) of each library file omc did not load
+PROBLEMS_FILE = "updateinfo-problems.txt"
 
 
 def with_retries(action, what: str):
@@ -249,14 +251,17 @@ def load_library(libname: str, tagName: str, path: str, omc: OMPython.OMCSession
     try:
         if not omc.sendExpression("loadFile(\"%s\", uses=false)" % path):
             print("Failed to load file %s in %s" % (path, tagName))
+            failedLoads.append((path, tagName, "loadFile returned false"))
             return False
         classNamesAfterLoad = omc.sendExpression("getClassNames()")
     except OMPython.OMCSessionException as e:
         print("Failed to load file %s in %s: %s" % (path, tagName, e))
+        failedLoads.append((path, tagName, str(e)))
         return False
     if libname not in classNamesAfterLoad:
         print("Did not load the library? ")
         print(classNamesAfterLoad)
+        failedLoads.append((path, tagName, "%s is not among the loaded classes %s" % (libname, classNamesAfterLoad)))
         return False
     return True
 
@@ -339,6 +344,7 @@ def main():
         os.mkdir("cache")
 
     duplicateWarnings = []
+    brokenRefs = []
 
     namesInFile = set()
     for key in data.keys():
@@ -505,6 +511,10 @@ def main():
                         if "libs" not in thisTagBackup:
                             del tagsDict[tagName]
                             refNames.discard(tagName)
+                            brokenRefs.append("%s %s: no library loaded, it is not in the index" % (key, tagName))
+                        else:
+                            brokenRefs.append("%s %s: no library loaded, the index keeps the previous entry" %
+                                              (key, tagName))
                         continue
                     thisTag["libs"] = provided
                     thisTag.update(refkeys)
@@ -532,3 +542,10 @@ def main():
         print("\n%d libraries were skipped because they are found more than once:" % len(duplicateWarnings))
         for warning in duplicateWarnings:
             print(warning)
+
+    problems = brokenRefs
+    problems += ["%s in %s did not load: %s" % (path, tagName, error) for (path, tagName, error) in failedLoads]
+    problems += ["%s could not be fetched, using the cached clone: %s" % (url, error)
+                 for (url, error) in unfetchedRepos]
+    problems += [warning.strip() for warning in duplicateWarnings]
+    common.write_problems(PROBLEMS_FILE, "updateinfo skipped these libraries:", problems)

tests/test_problems.py

@@ -0,0 +1,75 @@
+import json
+import os
+import tempfile
+import unittest
+
+import OMPython
+
+from ompackagemanager import common, genindex, updateinfo
+
+
+class InTempDir(unittest.TestCase):
+    def setUp(self):
+        self.cwd = os.getcwd()
+        self.tmp = tempfile.TemporaryDirectory()
+        os.chdir(self.tmp.name)
+
+    def tearDown(self):
+        os.chdir(self.cwd)
+        self.tmp.cleanup()
+
+
+class TestWriteProblems(InTempDir):
+    def test_writes_list(self):
+        common.write_problems("p.txt", "skipped:", ["A 1.0.0: broken", "B main: broken"])
+        with open("p.txt") as f:
+            self.assertEqual(f.read(), "skipped:\n\n- A 1.0.0: broken\n- B main: broken\n")
+
+    def test_no_problems_removes_old_file(self):
+        common.write_problems("p.txt", "skipped:", ["A 1.0.0: broken"])
+        common.write_problems("p.txt", "skipped:", [])
+        self.assertFalse(os.path.exists("p.txt"))
+
+
+class TestGenindexSkipsRefWithoutLibs(InTempDir):
+    def test_skipped_and_reported(self):
+        # Issue #61: a ref without libraries (e.g. LEPSE v1.2.1) used to raise and stop the job
+        repos = {"LEPSE": {"github": "example/LEPSE", "names": ["LEPSE"], "support": [["*", "support"]]}}
+        rawdata = {"LEPSE": {"refs": {
+            "v1.2.0": {"sha": "a" * 40, "libs": {"LEPSE": {"version": "1.2.0", "path": "LEPSE"}}},
+            "v1.2.1": {"sha": "b" * 40}}}}
+        with open("repos.json", "w") as f:
+            json.dump(repos, f)
+        with open("rawdata.json", "w") as f:
+            json.dump(rawdata, f)
+        genindex.main()
+        with open("index.json") as f:
+            index = json.load(f)
+        self.assertEqual(list(index["libs"]["LEPSE"]["versions"]), ["1.2.0"])
+        with open(genindex.PROBLEMS_FILE) as f:
+            self.assertIn("- LEPSE v1.2.1 has no libraries in rawdata.json", f.read())
+
+
+class FailingOMC:
+    def sendExpression(self, expr):
+        raise OMPython.OMCSessionException("[scripting:error:236] Expected the package to have name Co-simulation, "
+                                           "but got Co_simulation.")
+
+
+class TestUpdateinfoRecordsFailedLoads(unittest.TestCase):
+    def setUp(self):
+        updateinfo.failedLoads.clear()
+
+    def tearDown(self):
+        updateinfo.failedLoads.clear()
+
+    def test_failed_load_recorded(self):
+        self.assertFalse(updateinfo.load_library("LEPSE", "v1.2.1", "cache/LEPSE/package.mo", FailingOMC()))
+        self.assertEqual(len(updateinfo.failedLoads), 1)
+        path, tagName, error = updateinfo.failedLoads[0]
+        self.assertEqual((path, tagName), ("cache/LEPSE/package.mo", "v1.2.1"))
+        self.assertIn("Co_simulation", error)
+
+
+if __name__ == "__main__":
+    unittest.main()