From 7091b66bdfdf8d68ae33fae790f2d807293665fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Sj=C3=B6lund?= Date: Tue, 4 Aug 2026 13:49:08 +0200 Subject: [PATCH 1/2] Support simCodeTarget=wasm-jit in the test tool wasm-jit generates neither a makefile nor an executable; the model is JIT-compiled inside the omc that translated it. Detect the target from the customCommands (so `--extraflags` selects it), skip the build step and simulate with `simulate(..., resimulateExecutable=...)` in that same session instead of running an executable that does not exist. Two things turned a failure into a hang rather than a reported error: - `writeResultAndExit` called the builtin `quit()` instead of `quit_omc()`, so `SystemExit` was raised before `os._exit()` and the interpreter then waited forever for a thread stuck in a ZMQ receive. - `sendExpressionTimeout` only joined that thread with a timeout, so an omc that died mid-command was not noticed until the full `ulimitOmc` had elapsed. It now polls the process and exits as soon as it dies. The C path checks that the executable exists before running it. The default 8 GB virtual memory limit is too small for wasm-jit: omc peaks at ~5.8 GB and the JIT then reserves 4 GB of address space for the wasm memory, so every simulation failed with `mmap failed to reserve 0x104000000 bytes`. Use 16 GB for that target. Also adds the experimental wasm-jit regression job to the Jenkinsfile. Co-Authored-By: Claude Opus 5 --- .CI/Jenkinsfile | 17 +++++++++++++++ shared.py | 12 +++++++++++ test.py | 4 ++++ testmodel.py | 55 +++++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 84 insertions(+), 4 deletions(-) diff --git a/.CI/Jenkinsfile b/.CI/Jenkinsfile index 034d6a6..ac8e3a3 100644 --- a/.CI/Jenkinsfile +++ b/.CI/Jenkinsfile @@ -35,6 +35,7 @@ pipeline { booleanParam(name: 'gbode', defaultValue: false, description: 'master branch, with -d=newInst and -s gbode (ryzen-5950x-2). This is an experimental job that does not run on a fixed schedule.') booleanParam(name: 'ida', defaultValue: false, description: 'master branch, with -d=newInst and -s ida (ryzen-5950x-2). This is an experimental job that does not run on a fixed schedule.') booleanParam(name: 'generateSymbolicJacobian', defaultValue: false, description: 'master branch, with --generateSymbolicJacobian (ryzen-5950x-1). This is an experimental job that does not run on a fixed schedule.') + booleanParam(name: 'wasm_jit', defaultValue: false, description: 'master branch, with --simCodeTarget=wasm-jit (ryzen-5950x-2). This is an experimental job that does not run on a fixed schedule.') booleanParam(name: 'heavy_tests', defaultValue: false, description: 'master branch, runs one test at a time. That is, no parallel launching of tests. omc will use multiple threads for each test (-n=1 is not set unlike the other regression tests.), (ryzen-5950x-1). This is an experimental job that does not run on a fixed schedule.') } environment { @@ -373,6 +374,22 @@ pipeline { runRegressiontest('master', 'ida', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '-s ida', false, false) } } + stage('wasm-jit') { + agent { + node { + label 'ryzen-5950x-2-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.wasm_jit } + } + steps { + runRegressiontest('master', 'wasm-jit', 'setCommandLineOptions("--simCodeTarget=wasm-jit")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false) + } + } stage('generateSymbolicJacobian') { agent { node { diff --git a/shared.py b/shared.py index 2f9b7a3..204f369 100644 --- a/shared.py +++ b/shared.py @@ -3,6 +3,15 @@ import re, os, subprocess import simplejson as json +simCodeTargetRe = re.compile('--simCodeTarget=([^"\'\\s,;)]+)') + +def simCodeTargetFromCommands(target, commands): + for cmd in commands: + found = simCodeTargetRe.findall(str(cmd)) + if found: + target = found[-1] + return target + def fixData(data,abortSimulationFlag,alarmFlag,overrideDefaults,defaultCustomCommands,extrasimflags,environmentTranslation,environmentSimulation): data["configFromFile"] = dict(data) for (key,default) in overrideDefaults: @@ -24,6 +33,9 @@ def fixData(data,abortSimulationFlag,alarmFlag,overrideDefaults,defaultCustomCom else: defaultCustomCommands2 = defaultCustomCommands data["customCommands"] = (data.get("customCommands") or defaultCustomCommands2) + (data.get("extraCustomCommands") or []) + # A --simCodeTarget in the commands (e.g. from --extraflags) is what omc will + # actually use, so the rest of the testing scripts need to see it + data["simCodeTarget"] = simCodeTargetFromCommands(data["simCodeTarget"], data["customCommands"]) data["ulimitOmc"] = int(data.get("ulimitOmc") or 660) # 11 minutes to generate the C-code data["ulimitExe"] = int(data.get("ulimitExe") or 8*60) # 8 additional minutes to initialize and run the simulation data["ulimitLoadModel"] = int(data.get("ulimitLoadModel") or 3*60) # 3 minutes to load the files (could take a while if the ssd is doing backup) diff --git a/test.py b/test.py index 66bb085..d1d98c0 100755 --- a/test.py +++ b/test.py @@ -599,6 +599,10 @@ def hashReferenceFiles(s): conf["omc_thread_cmd"] = omc_threads conf["haveCppRuntime"] = haveCppRuntime conf["ulimitMemory"] = conf.get("ulimitMemory") or ulimitMemory + if conf["simCodeTarget"]=="wasm-jit": + # The JIT reserves ~4 GB of address space per wasm memory on top of what omc + # itself maps, which does not fit in the default virtual memory limit + conf["ulimitMemory"] = max(conf["ulimitMemory"], 16*1024*1024) if conf.get("fmi"): conf["haveFMI"] = fmiOK_C conf["haveFMICpp"] = fmiOK_Cpp diff --git a/testmodel.py b/testmodel.py index 121d522..b66ae28 100755 --- a/testmodel.py +++ b/testmodel.py @@ -84,7 +84,7 @@ def writeResultAndExit(exitStatus, useOsExit=False, omc=None, omc_new=None): fp.flush() sys.stdout.flush() omc = quit_omc(omc) - omc_new = quit(omc_new) + omc_new = quit_omc(omc_new) if useOsExit: os._exit(exitStatus) else: @@ -108,7 +108,23 @@ def target(res): res=[None,None] thread = threading.Thread(target=target, args=(res,)) thread.start() - thread.join(timeout) + # Poll instead of a single join: if omc dies (crash, ulimit, ...) the thread is + # stuck in a ZMQ receive that never returns, so waiting out the timeout and then + # exiting normally would hang forever on that non-daemon thread + deadline = monotonic() + timeout + while thread.is_alive() and monotonic() < deadline: + thread.join(1) + status = omc._omc_process.poll() + if thread.is_alive() and status is not None: + with open(errFile, 'a+') as fp: + fp.write("OMC exited with status %s while running: %s\n" % (status, cmd)) + try: + with open(os.path.normpath(omc._omc_log_file.name)) as omcLog: + for line in omcLog: + fp.write(line) + except IOError: + pass + writeResultAndExit(0, True, omc, omc_new) if thread.is_alive(): with open(errFile, 'a+') as fp: @@ -229,10 +245,17 @@ def target(res): with open(errFile, 'a+') as fp: fp.write("Running: %s\n" % " ".join(sys.argv)) -if conf["simCodeTarget"] not in ["Cpp","C"]: +if conf["simCodeTarget"] not in ["Cpp","C","wasm-jit"]: with open(errFile, 'a+') as fp: fp.write("Unknown simCodeTarget in %s" % conf["simCodeTarget"]) writeResultAndExit(1) +# wasm-jit builds no makefile and no executable; the model is JIT-compiled inside +# the omc that translated it and simulated there via simulate(resimulateExecutable=) +isWasmJit = conf["simCodeTarget"]=="wasm-jit" +if isWasmJit and conf.get("fmi"): + with open(errFile, 'a+') as fp: + fp.write("FMI export is not supported for simCodeTarget=wasm-jit") + writeResultAndExit(0) if conf["simCodeTarget"]=="Cpp" and not conf["haveCppRuntime"]: with open(errFile, 'a+') as fp: fp.write("C++ runtime not supported in this installation (HelloWorld failed)") @@ -433,7 +456,9 @@ def sendExpressionOldOrNew(cmd): frontend = omc.sendExpression("OpenModelica.Scripting.Internal.Time.timerTock(OpenModelica.Scripting.Internal.Time.RT_CLOCK_FRONTEND)") writeResult() -omc = quit_omc(omc) +if not isWasmJit: + # wasm-jit keeps the translated model in this session; it is needed to simulate + omc = quit_omc(omc) print(execTimeTranslateModel,frontend,backend) if backend != -1: @@ -478,6 +503,10 @@ def sendExpressionOldOrNew(cmd): execstat["phase"]=4 writeResultAndExit(0, False, omc, omc_new) execstat["phase"] = 5 + elif isWasmJit: + # Nothing to build; omc JIT-compiles the model as part of the simulation + execstat["build"] = 0.0 + execstat["phase"] = 5 else: if isWin: res = checkOutputTimeout("\"%s\\share\\omc\\scripts\\Compile.bat\" %s gcc %s parallel dynamic 24 0" % (conf["omhome"], conf["fileName"], msysEnvironment), conf["ulimitOmc"], conf) @@ -521,6 +550,18 @@ def sendExpressionOldOrNew(cmd): with open(simFile,"w") as fp: fp.write("%s %s\n" % (fmisimulator, cmd)) res = checkOutputTimeout("(rm -f %s.pipe ; mkfifo %s.pipe ; head -c 1048576 < %s.pipe >> %s & %s %s > %s.pipe 2>&1)" % (conf["fileName"],conf["fileName"],conf["fileName"],simFile,fmisimulator,cmd,conf["fileName"]), 1.05*conf["ulimitExe"], conf) + elif isWasmJit: + simflags = ("%s %s -lv LOG_STATS" % (conf["simFlags"],emit_protected)).strip() + cmd = 'simulate(%s,startTime=%g,stopTime=%g,tolerance=%g,numberOfIntervals=%d,outputFormat="%s",variableFilter="%s",fileNamePrefix="%s",simflags="%s",resimulateExecutable="%s")' % (conf["modelName"],startTime,stopTime,tolerance,numberOfIntervals,outputFormat,variableFilter,conf["fileName"],simflags,conf["fileName"]) + with open(simFile,"w") as fp: + fp.write("startTime=%g\nstopTime=%g\ntolerance=%g\nnumberOfIntervals=%d\nstepSize=%g\n" % (startTime,stopTime,tolerance,numberOfIntervals,stepSize)) + fp.write("wasm-jit simulation: %s\n" % cmd) + simres = sendExpressionTimeout(omc, cmd, conf["ulimitExe"]) or {} + with open(simFile,"a+") as fp: + fp.write(simres.get("messages") or "") + if not simres.get("resultFile"): + execstat["sim"] = monotonic()-start + writeResultAndExit(0, False, omc, omc_new) else: if isWin: cmd = (".\\%s.bat %s %s %s" % (conf["fileName"],annotationSimFlags,conf["simFlags"],emit_protected)).strip() @@ -529,6 +570,12 @@ def sendExpressionOldOrNew(cmd): if conf["simCodeTarget"]=="C": cmd = cmd + " -lv LOG_STATS" + executable = os.path.normpath("%s.bat" % conf["fileName"] if isWin else conf["fileName"]) + if not os.path.exists(executable): + with open(errFile, 'a+') as fp: + fp.write("The simulation executable %s does not exist\n" % executable) + execstat["sim"] = monotonic()-start + writeResultAndExit(0, False, omc, omc_new) with open(simFile,"w") as fp: fp.write("Environment - simulationEnvironment:\n") for e in conf["environmentSimulation"]: From 87b12c4356ad19de888f7de0b531e304eafe4538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Sj=C3=B6lund?= Date: Tue, 4 Aug 2026 16:25:24 +0200 Subject: [PATCH 2/2] Separate wasm-jit compile time from sim time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wasm-jit target JIT-compiles the model inside omc, and the test tool ran `translateModel` followed by `simulate(resimulateExecutable=)`. That path never calls `buildModel`, so `timeCompile` is not measured and the compile lands in the simulation time instead — reported as a build time of 0 and a correspondingly inflated `sim`. `--nobuildmodel` translates, builds and runs in a single `simulate()` call, which reports the phase times itself, and takes the build and simulation times from that record rather than the wall clock around the command. For VehicleInterfaces the same run splits as: | Flag | build | sim | |-------------------|---------|---------| | (none) | 0.00000 | 0.44668 | | `--nobuildmodel` | 0.21299 | 0.21309 | `--coldhot` simulates each model twice in the same omc. The second run reuses the module compiled for the first, so it is the hot number; both are printed but only the hot one is stored, since the database columns are enumerated in the insert. Both flags only apply to `simCodeTarget=wasm-jit`, and only set their config key when passed, so the confighash — and therefore the stored history — of a normal run is unchanged. `runRegressiontest` gains a `testFlags` argument, passed to `test.py` verbatim; the wasm-jit job uses it for `--nobuildmodel`. Co-Authored-By: Claude Opus 5 --- .CI/Jenkinsfile | 57 ++++++++++++++++++++++++----------------------- test.py | 16 ++++++++++++++ testmodel.py | 59 +++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 95 insertions(+), 37 deletions(-) diff --git a/.CI/Jenkinsfile b/.CI/Jenkinsfile index ac8e3a3..29e12a2 100644 --- a/.CI/Jenkinsfile +++ b/.CI/Jenkinsfile @@ -57,7 +57,7 @@ pipeline { expression { params.v1_26 } } steps { - runRegressiontest('maintenance/v1.26', 'v1.26', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', false, false) + runRegressiontest('maintenance/v1.26', 'v1.26', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', '', false, false) } } @@ -74,7 +74,7 @@ pipeline { expression { params.v1_27 } } steps { - runRegressiontest('maintenance/v1.27', 'v1.27', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', false, false) + runRegressiontest('maintenance/v1.27', 'v1.27', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', '', false, false) } } @@ -91,7 +91,7 @@ pipeline { expression { params.master } } steps { - runRegressiontest('master', 'master', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', false, false) + runRegressiontest('master', 'master', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', '', false, false) } } @@ -108,7 +108,7 @@ pipeline { expression { params.conversion_script } } steps { - runRegressiontest('master', 'conversion', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', false, true) + runRegressiontest('master', 'conversion', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', '', false, true) } } @@ -125,7 +125,7 @@ pipeline { expression { params.newInst_newBackend } } steps { - runRegressiontest('master', 'newInst-newBackend', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --newBackend")', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', false, false) + runRegressiontest('master', 'newInst-newBackend', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --newBackend")', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', '', false, false) } } @@ -142,7 +142,7 @@ pipeline { expression { params.fmi_v1_26 } } steps { - runRegressiontest('maintenance/v1.26', 'v1.26-fmi', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false) + runRegressiontest('maintenance/v1.26', 'v1.26-fmi', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) } } stage('v1.27 FMI with OMSimulator') { @@ -158,7 +158,7 @@ pipeline { expression { params.fmi_v1_27 } } steps { - runRegressiontest('maintenance/v1.27', 'v1.27-fmi', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false) + runRegressiontest('maintenance/v1.27', 'v1.27-fmi', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) } } stage('master FMI with OMSimulator') { @@ -174,7 +174,7 @@ pipeline { expression { params.fmi_master } } steps { - runRegressiontest('master', 'master-fmi', '', 'origin/master', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false) + runRegressiontest('master', 'master-fmi', '', 'origin/master', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) } } @@ -191,7 +191,7 @@ pipeline { expression { params.cs_fmu_cvode_v1_26 } } steps { - runRegressiontest('maintenance/v1.26', 'v1.26-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false) + runRegressiontest('maintenance/v1.26', 'v1.26-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) } } stage('v1.27 CVODE CS-FMUs with OMSimulator') { @@ -207,7 +207,7 @@ pipeline { expression { params.cs_fmu_cvode_v1_27 } } steps { - runRegressiontest('maintenance/v1.27', 'v1.27-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false) + runRegressiontest('maintenance/v1.27', 'v1.27-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) } } stage('master CVODE CS-FMUs with OMSimulator') { @@ -223,7 +223,7 @@ pipeline { expression { params.cs_fmu_cvode_master } } steps { - runRegressiontest('master', 'master-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', 'origin/master', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false) + runRegressiontest('master', 'master-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', 'origin/master', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) } } @@ -240,7 +240,7 @@ pipeline { expression { params.fmpy_fmi_v1_26 } } steps { - runRegressiontest('maintenance/v1.26', 'v1.26-fmi-fmpy', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false) + runRegressiontest('maintenance/v1.26', 'v1.26-fmi-fmpy', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) } } @@ -257,7 +257,7 @@ pipeline { expression { params.fmpy_fmi_v1_27 } } steps { - runRegressiontest('maintenance/v1.27', 'v1.27-fmi-fmpy', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false) + runRegressiontest('maintenance/v1.27', 'v1.27-fmi-fmpy', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) } } @@ -274,7 +274,7 @@ pipeline { expression { params.fmpy_fmi_master } } steps { - runRegressiontest('master', 'master-fmi-fmpy', '', 'origin/master', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false) + runRegressiontest('master', 'master-fmi-fmpy', '', 'origin/master', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) } } @@ -291,7 +291,7 @@ pipeline { expression { params.newInst_daeMode } } steps { - runRegressiontest('master', 'newInst-daeMode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --daeMode=true")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false) + runRegressiontest('master', 'newInst-daeMode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --daeMode=true")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) } } stage('newBackend-daeMode') { @@ -307,7 +307,7 @@ pipeline { expression { params.newBackend_daeMode } } steps { - runRegressiontest('master', 'newBackend-daeMode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --newBackend --daeMode=true")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false) + runRegressiontest('master', 'newBackend-daeMode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --newBackend --daeMode=true")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) } } stage('oldInst') { @@ -323,7 +323,7 @@ pipeline { expression { params.oldInst } } steps { - runRegressiontest('master', 'oldInst', 'setCommandLineOptions("-d=nonewInst")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false) + runRegressiontest('master', 'oldInst', 'setCommandLineOptions("-d=nonewInst")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) } } stage('cvode') { @@ -339,7 +339,7 @@ pipeline { expression { params.cvode } } steps { - runRegressiontest('master', 'cvode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '-s cvode', false, false) + runRegressiontest('master', 'cvode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '-s cvode', '', false, false) } } stage('gbode') { @@ -355,7 +355,7 @@ pipeline { expression { params.gbode } } steps { - runRegressiontest('master', 'gbode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '-s gbode -gbm=radauIIA3', false, false) + runRegressiontest('master', 'gbode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '-s gbode -gbm=radauIIA3', '', false, false) } } stage('ida') { @@ -371,7 +371,7 @@ pipeline { expression { params.ida } } steps { - runRegressiontest('master', 'ida', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '-s ida', false, false) + runRegressiontest('master', 'ida', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '-s ida', '', false, false) } } stage('wasm-jit') { @@ -387,7 +387,7 @@ pipeline { expression { params.wasm_jit } } steps { - runRegressiontest('master', 'wasm-jit', 'setCommandLineOptions("--simCodeTarget=wasm-jit")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false) + runRegressiontest('master', 'wasm-jit', 'setCommandLineOptions("--simCodeTarget=wasm-jit")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', '--nobuildmodel', false, false) } } stage('generateSymbolicJacobian') { @@ -403,7 +403,7 @@ pipeline { expression { params.generateSymbolicJacobian } } steps { - runRegressiontest('master', 'generateSymbolicJacobian', 'setCommandLineOptions("--generateSymbolicJacobian")', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', false, false) + runRegressiontest('master', 'generateSymbolicJacobian', 'setCommandLineOptions("--generateSymbolicJacobian")', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', '', false, false) } } stage('heavy_tests') { @@ -419,7 +419,7 @@ pipeline { expression { params.heavy_tests } } steps { - runRegressiontest('master', 'heavy_tests', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', false, false, 1, 'configs/heavy_tests.json') + runRegressiontest('master', 'heavy_tests', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', '', false, false, 1, 'configs/heavy_tests.json') } } @@ -436,7 +436,7 @@ pipeline { expression { params.cpp_v1_26 } } steps { - runRegressiontest('maintenance/v1.26', 'v1.26-cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false) + runRegressiontest('maintenance/v1.26', 'v1.26-cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) } } stage('C++ v1.27') { @@ -452,7 +452,7 @@ pipeline { expression { params.cpp_v1_27 } } steps { - runRegressiontest('maintenance/v1.27', 'v1.27-cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false) + runRegressiontest('maintenance/v1.27', 'v1.27-cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) } } @@ -469,7 +469,7 @@ pipeline { expression { params.cpp } } steps { - runRegressiontest('master', 'cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false) + runRegressiontest('master', 'cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) } } } } @@ -694,6 +694,7 @@ done * @parm sshConfig: SSH configuration saved on test node. * @param omcompiler: Checkout old OMCompiler submodule. Should be `false` nowadays. * @param extrasimflags: Additional simulation flags passed to test.py via flag `--extrasimflags`. + * @param testFlags: Additional flags passed to test.py verbatim, e.g. `--nobuildmodel`. * @param removePackageOrder: Passed to `installLibraries`. * @param conversionScript: Passed to `installLibraries`. * @param jobs: The number of tests/jobs to launch in parallel. @@ -702,7 +703,7 @@ done * @param libs_config_file: The config file to be used for testing. * This file specifies which libraries to test and what options to use for them. */ -def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, omcompiler, extrasimflags, boolean removePackageOrder, boolean conversionScript, int jobs=0, libs_config_file = 'configs/conf.json') { +def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, omcompiler, extrasimflags, testFlags, boolean removePackageOrder, boolean conversionScript, int jobs=0, libs_config_file = 'configs/conf.json') { sh ''' find /tmp -name "*openmodelica.hudson*" -exec rm {} ";" || true mkdir -p ~/TEST_LIBS_BACKUP @@ -911,7 +912,7 @@ def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, om cd OpenModelicaLibraryTesting # Force /usr/bin/omc as being used for generating the mos-files. Ensures consistent behavior among all tested OMC versions - stdbuf -oL -eL time ./test.py --ompython_omhome=/usr ${FMI_TESTING_FLAG} --extraflags='${extraFlags}' --extrasimflags='${extrasimflags}' --branch="${name}" --output="libraries.openmodelica.org:/var/www/libraries.openmodelica.org/branches/${name}/" --libraries='${libraryPath}/.openmodelica/libraries/' --jobs=${jobs} ${libs_config_file} ${params.OLDLIBS ? "configs/conf-old.json configs/conf-nonstandard.json" : ""} || (killall omc ; false) || exit 1 + stdbuf -oL -eL time ./test.py --ompython_omhome=/usr ${FMI_TESTING_FLAG} --extraflags='${extraFlags}' --extrasimflags='${extrasimflags}' ${testFlags} --branch="${name}" --output="libraries.openmodelica.org:/var/www/libraries.openmodelica.org/branches/${name}/" --libraries='${libraryPath}/.openmodelica/libraries/' --jobs=${jobs} ${libs_config_file} ${params.OLDLIBS ? "configs/conf-old.json configs/conf-nonstandard.json" : ""} || (killall omc ; false) || exit 1 """ sh 'date' sh "rm -f OpenModelicaLibraryTesting/${dbPrefix}-sqlite3.db.tmp" diff --git a/test.py b/test.py index d1d98c0..e1b02b7 100755 --- a/test.py +++ b/test.py @@ -34,6 +34,8 @@ parser.add_argument('--extrasimflags', default='') parser.add_argument('--ompython_omhome', default='') parser.add_argument('--noclean', action="store_true", default=False) +parser.add_argument('--nobuildmodel', action="store_true", help="Translate, build and simulate in a single simulate() call instead of translateModel() followed by simulate(resimulateExecutable=...), so the JIT compile is reported as build time rather than simulation time. Only used by simCodeTarget=wasm-jit.", default=False) +parser.add_argument('--coldhot', action="store_true", help="Simulate each model twice in the same omc; the second run reuses the compiled module. Both times are printed, but only the hot one is stored. Only used by simCodeTarget=wasm-jit.", default=False) parser.add_argument('--fmisimulator', default='') parser.add_argument('--ulimitvmem', help="Virtual memory limit (in kB) (linux only)", type=int, default=8*1024*1024) parser.add_argument('--default', action='append', help="Add a default value for some configuration key, such as --default=ulimitExe=60. The equals sign is mandatory.", default=[]) @@ -572,6 +574,11 @@ def hashReferenceFiles(s): skipped_libs = {} tests=[] for (library,conf) in configs: + # Only when asked, so a normal run's confighash is unchanged + if args.nobuildmodel: + conf["noBuildModel"] = True + if args.coldhot: + conf["coldHot"] = True c=conf.copy() del(c["configFromFile"]) if "referenceFiles" in c: @@ -901,6 +908,15 @@ def loadJsonOrEmptySet(f): #for k in sorted(stats.keys(), key=lambda c: stats[c][3]["exectime"], reverse=True): # print("%s: exectime %.2f" % (k, stats[k][3]["exectime"])) +if args.coldhot: + # Only "sim" (the hot run) is stored; the cold one is reported here + print("Cold vs hot simulation time:") + for key in sorted(stats.keys(), key=lambda k: stats[k][1]): + (name,model,libname,data)=stats[key] + if data.get("simcold") is not None: + print(" %-70s cold %8.4f hot %8.4f" % (model, data["simcold"], data.get("sim") or 0.0)) + sys.stdout.flush() + for key in stats.keys(): (name,model,libname,data)=stats[key] stats_by_libname[libname]["stats"].append(stats[key]) diff --git a/testmodel.py b/testmodel.py index b66ae28..cad39c4 100755 --- a/testmodel.py +++ b/testmodel.py @@ -214,6 +214,7 @@ def target(res): "templates":None, "build":None, "sim":None, + "simcold":None, "diff":None, "phase":0 } @@ -252,6 +253,12 @@ def target(res): # wasm-jit builds no makefile and no executable; the model is JIT-compiled inside # the omc that translated it and simulated there via simulate(resimulateExecutable=) isWasmJit = conf["simCodeTarget"]=="wasm-jit" +# --nobuildmodel: one simulate() instead of translateModel()+resimulate, so omc +# reports the build/simulation split itself +useSimulate = isWasmJit and conf.get("noBuildModel") and not conf.get("fmi") +# --coldhot: simulate again in the same session, where the module is already +# compiled, and report that run instead +useColdHot = isWasmJit and conf.get("coldHot") and not conf.get("fmi") if isWasmJit and conf.get("fmi"): with open(errFile, 'a+') as fp: fp.write("FMI export is not supported for simCodeTarget=wasm-jit") @@ -418,17 +425,25 @@ def sendExpressionOldOrNew(cmd): with open(errFile, 'a+') as fp: fp.write("Ignoring simflag %s since it seems broken on HelloWorld\n" % flagVal) +def simulateCmd(resimulate): + simflags = ("%s %s -lv LOG_STATS" % (conf["simFlags"],emit_protected)).strip() + return 'simulate(%s,startTime=%g,stopTime=%g,tolerance=%g,numberOfIntervals=%d,outputFormat="%s",variableFilter="%s",fileNamePrefix="%s",simflags="%s"%s)' % (conf["modelName"],startTime,stopTime,tolerance,numberOfIntervals,outputFormat,variableFilter,conf["fileName"],simflags,(',resimulateExecutable="%s"' % conf["fileName"]) if resimulate else "") + # TODO: Detect and handle the case where RT_CLOCK is not available in OMC total_before = omc.sendExpression("OpenModelica.Scripting.Internal.Time.timerTock(OpenModelica.Scripting.Internal.Time.RT_CLOCK_SIMULATE_TOTAL)") start=monotonic() +timeout = conf["ulimitOmc"] if conf.get("fmi"): cmd='"" <> buildModelFMU(%s,fileNamePrefix="%s",fmuType="%s",version="%s",platforms={"static"})' % (conf["modelName"],conf["fileName"].replace(".","_"),conf["fmuType"],conf["fmi"]) +elif useSimulate: + cmd=simulateCmd(resimulate=False) + timeout = conf["ulimitOmc"] + conf["ulimitExe"] else: cmd='translateModel(%s,tolerance=%g,outputFormat="%s",numberOfIntervals=%d,variableFilter="%s",fileNamePrefix="%s")' % (conf["modelName"],tolerance,outputFormat,numberOfIntervals,variableFilter,conf["fileName"]) with open(errFile, 'a+') as fp: fp.write("Running command: %s\n"%(cmd)) try: - res=sendExpressionTimeout(omc, cmd, conf["ulimitOmc"]) + res=sendExpressionTimeout(omc, cmd, timeout) except TimeoutError as e: execstat["frontend"]=monotonic()-start @@ -447,6 +462,11 @@ def sendExpressionOldOrNew(cmd): # See which translateModel phases completed execTimeTranslateModel=monotonic()-start +simres = None +if useSimulate: + simres = res or {} + # A failed translate/build is only reported in the messages of the record + res = not (simres.get("messages") or "").startswith("Failed to build model") err = omc.sendExpression("OpenModelica.Scripting.getErrorString()") total = omc.sendExpression("OpenModelica.Scripting.Internal.Time.timerTock(OpenModelica.Scripting.Internal.Time.RT_CLOCK_SIMULATE_TOTAL)")-total_before buildmodel = omc.sendExpression("OpenModelica.Scripting.Internal.Time.timerTock(OpenModelica.Scripting.Internal.Time.RT_CLOCK_BUILD_MODEL)") @@ -456,7 +476,7 @@ def sendExpressionOldOrNew(cmd): frontend = omc.sendExpression("OpenModelica.Scripting.Internal.Time.timerTock(OpenModelica.Scripting.Internal.Time.RT_CLOCK_FRONTEND)") writeResult() -if not isWasmJit: +if not isWasmJit or (useSimulate and not useColdHot): # wasm-jit keeps the translated model in this session; it is needed to simulate omc = quit_omc(omc) @@ -504,8 +524,9 @@ def sendExpressionOldOrNew(cmd): writeResultAndExit(0, False, omc, omc_new) execstat["phase"] = 5 elif isWasmJit: - # Nothing to build; omc JIT-compiles the model as part of the simulation - execstat["build"] = 0.0 + # Nothing to build; simulate() reports the JIT compile as timeCompile, while + # a resimulate leaves it in the simulation time + execstat["build"] = simres["timeCompile"] if useSimulate else 0.0 execstat["phase"] = 5 else: if isWin: @@ -528,6 +549,12 @@ def sendExpressionOldOrNew(cmd): fmisimulator = conf.get("fmisimulator") resFile = "%s_res.%s" % (conf["fileName"], outputFormat if not shared.isFMPy(fmisimulator) else 'csv') +def simElapsed(): + # omc's own time: the wall clock here covers the wrong run for both flags + if useSimulate or useColdHot: + return (simres or {}).get("timeSimulation") or 0.0 + return monotonic()-start + start=monotonic() try: # TODO: Timeout more reliably... @@ -551,17 +578,31 @@ def sendExpressionOldOrNew(cmd): fp.write("%s %s\n" % (fmisimulator, cmd)) res = checkOutputTimeout("(rm -f %s.pipe ; mkfifo %s.pipe ; head -c 1048576 < %s.pipe >> %s & %s %s > %s.pipe 2>&1)" % (conf["fileName"],conf["fileName"],conf["fileName"],simFile,fmisimulator,cmd,conf["fileName"]), 1.05*conf["ulimitExe"], conf) elif isWasmJit: - simflags = ("%s %s -lv LOG_STATS" % (conf["simFlags"],emit_protected)).strip() - cmd = 'simulate(%s,startTime=%g,stopTime=%g,tolerance=%g,numberOfIntervals=%d,outputFormat="%s",variableFilter="%s",fileNamePrefix="%s",simflags="%s",resimulateExecutable="%s")' % (conf["modelName"],startTime,stopTime,tolerance,numberOfIntervals,outputFormat,variableFilter,conf["fileName"],simflags,conf["fileName"]) + if not useSimulate: + cmd = simulateCmd(resimulate=True) with open(simFile,"w") as fp: fp.write("startTime=%g\nstopTime=%g\ntolerance=%g\nnumberOfIntervals=%d\nstepSize=%g\n" % (startTime,stopTime,tolerance,numberOfIntervals,stepSize)) fp.write("wasm-jit simulation: %s\n" % cmd) - simres = sendExpressionTimeout(omc, cmd, conf["ulimitExe"]) or {} + if not useSimulate: + simres = sendExpressionTimeout(omc, cmd, conf["ulimitExe"]) or {} with open(simFile,"a+") as fp: fp.write(simres.get("messages") or "") if not simres.get("resultFile"): - execstat["sim"] = monotonic()-start + execstat["sim"] = simElapsed() writeResultAndExit(0, False, omc, omc_new) + if useColdHot: + execstat["simcold"] = simElapsed() + cmd = simulateCmd(resimulate=True) + with open(simFile,"a+") as fp: + fp.write("wasm-jit hot simulation: %s\n" % cmd) + hotres = sendExpressionTimeout(omc, cmd, conf["ulimitExe"]) or {} + with open(simFile,"a+") as fp: + fp.write(hotres.get("messages") or "") + if hotres.get("resultFile"): + simres = hotres + else: + with open(errFile, 'a+') as fp: + fp.write("The hot simulation failed; keeping the cold time\n") else: if isWin: cmd = (".\\%s.bat %s %s %s" % (conf["fileName"],annotationSimFlags,conf["simFlags"],emit_protected)).strip() @@ -586,7 +627,7 @@ def sendExpressionOldOrNew(cmd): res = checkOutputTimeout("%s >> %s" % (cmd,simFile), conf["ulimitExe"], conf) else: res = checkOutputTimeout("(rm -f %s.pipe ; mkfifo %s.pipe ; head -c 1048576 < %s.pipe >> %s & %s > %s.pipe 2>&1)" % (conf["fileName"],conf["fileName"],conf["fileName"],simFile,cmd,conf["fileName"]), conf["ulimitExe"], conf) - execstat["sim"] = monotonic()-start + execstat["sim"] = simElapsed() execstat["phase"] = 6 except TimeoutError as e: execstat["sim"] = monotonic()-start