Add site setting to configure server-side script execution timeout (#7862)

## Rationale

The Rhino sandbox terminates any server-side JavaScript (such as trigger
scripts) after a hard-coded 60 seconds of wall-clock time, which
includes database and other Java operations invoked by the script.
Long-running but legitimate operations, such as bulk imports that fire
per-row triggers, can exceed this budget with no recourse. This change
makes the timeout a configurable site setting so admins can raise it, or
disable it entirely, without a code change.

## Related Pull Requests

None.

## Changes

- New `scriptExecutionTimeout` site setting (default 60 seconds, 0
disables) exposed on the Site Settings admin page and as a
`SiteSettings.scriptExecutionTimeout` startup property.
- The Rhino sandbox resolves the timeout once per script execution
instead of using the hard-coded constant, so setting changes take effect
immediately for new script runs; the elapsed-time comparison is promoted
to `long` to avoid overflow with large values.
- The Site Settings form binds the new field as a nullable `Integer`, so
a POST that omits the parameter leaves the stored setting unchanged
rather than silently disabling the timeout.
- Added `RhinoService.TestCase.timeoutTest` (BVT) covering both a short
timeout aborting a runaway script and 0 disabling the watchdog.

Author: labkey-martyp

api/src/org/labkey/api/settings/AppProps.java

@@ -173,6 +173,11 @@ static WriteableAppProps getWriteableInstance()
     /** Timeout in seconds for read-only HTTP requests, after which resources like DB connections and spawned processes will be killed. Set to 0 to disable. */
     int getReadOnlyHttpRequestTimeout();
 
+    int DEFAULT_SCRIPT_EXECUTION_TIMEOUT = 60;
+
+    /** Timeout in seconds for server-side JavaScript (e.g. trigger scripts), measured in wall-clock time including database and other Java operations invoked by the script. Set to 0 to disable. */
+    int getScriptExecutionTimeout();
+
     int getMaxBLOBSize();
 
     ExceptionReportingLevel getExceptionReportingLevel();

api/src/org/labkey/api/settings/AppPropsImpl.java

@@ -321,6 +321,12 @@ public int getReadOnlyHttpRequestTimeout()
         return lookupIntValue(readOnlyHttpRequestTimeout, 0);
     }
 
+    @Override
+    public int getScriptExecutionTimeout()
+    {
+        return lookupIntValue(scriptExecutionTimeout, DEFAULT_SCRIPT_EXECUTION_TIMEOUT);
+    }
+
     @Override
     public int getMaxBLOBSize()
     {

api/src/org/labkey/api/settings/SiteSettingsProperties.java

@@ -90,6 +90,14 @@ public void setValue(WriteableAppProps writeable, String value)
             writeable.setReadOnlyHttpRequestTimeout(Integer.parseInt(value));
         }
     },
+    scriptExecutionTimeout("Timeout in seconds for server-side JavaScript such as trigger scripts. Measured in wall-clock time, including database and other Java operations invoked by the script. Set to 0 to disable.")
+    {
+        @Override
+        public void setValue(WriteableAppProps writeable, String value)
+        {
+            writeable.setScriptExecutionTimeout(Integer.parseInt(value));
+        }
+    },
     maxBLOBSize("Maximum file size, in bytes, to allow in database BLOBs")
     {
         @Override

api/src/org/labkey/api/settings/WriteableAppProps.java

@@ -94,6 +94,13 @@ public void setReadOnlyHttpRequestTimeout(int timeout)
         storeIntValue(readOnlyHttpRequestTimeout, timeout);
     }
 
+    public void setScriptExecutionTimeout(int timeout)
+    {
+        if (timeout < 0)
+            throw new IllegalArgumentException("scriptExecutionTimeout must be >= 0");
+        storeIntValue(scriptExecutionTimeout, timeout);
+    }
+
     public void setMaxBLOBSize(int maxSize)
     {
         if (maxSize < 0)

core/src/org/labkey/core/admin/AdminController.java

@@ -1381,6 +1381,14 @@ public void validateCommand(SiteSettingsForm form, Errors errors)
             {
                 errors.reject(ERROR_MSG, "Memory logging frequency must be non-negative");
             }
+            if (form.getScriptExecutionTimeout() == null)
+            {
+                errors.reject(ERROR_MSG, "Script execution timeout is required; set to 0 to disable the timeout");
+            }
+            else if (form.getScriptExecutionTimeout() < 0)
+            {
+                errors.reject(ERROR_MSG, "Script execution timeout must be non-negative");
+            }
         }
 
         @Override
@@ -1415,6 +1423,7 @@ public boolean handlePost(SiteSettingsForm form, BindException errors) throws Ex
             props.setSSLPort(form.getSslPort());
             props.setMemoryUsageDumpInterval(form.getMemoryUsageDumpInterval());
             props.setReadOnlyHttpRequestTimeout(form.getReadOnlyHttpRequestTimeout());
+            props.setScriptExecutionTimeout(form.getScriptExecutionTimeout());
             props.setMaxBLOBSize(form.getMaxBLOBSize());
             props.setSelfReportExceptions(form.isSelfReportExceptions());
 
@@ -2384,6 +2393,7 @@ public static class SiteSettingsForm
         private int _sslPort;
         private int _memoryUsageDumpInterval;
         private int _readOnlyHttpRequestTimeout;
+        private Integer _scriptExecutionTimeout;
         private int _maxBLOBSize;
         private String _exceptionReportingLevel;
         private String _usageReportingLevel;
@@ -2524,6 +2534,18 @@ public int getReadOnlyHttpRequestTimeout()
             return _readOnlyHttpRequestTimeout;
         }
 
+        /** Null when the request omits or blanks the parameter; rejected in validateCommand so an omitted value can never bind to 0 and silently disable the timeout. */
+        @Nullable
+        public Integer getScriptExecutionTimeout()
+        {
+            return _scriptExecutionTimeout;
+        }
+
+        public void setScriptExecutionTimeout(@Nullable Integer timeout)
+        {
+            _scriptExecutionTimeout = timeout;
+        }
+
         public void setReadOnlyHttpRequestTimeout(int timeout)
         {
             _readOnlyHttpRequestTimeout = timeout;

core/src/org/labkey/core/admin/customizeSite.jsp

@@ -312,6 +312,11 @@ Click the Save button at any time to accept the current settings and continue.</
         "After the timeout, resources like database connections and spawned processes will be killed to abort processing the request. Set to 0 to disable the timeout.")%></label></td>
     <td><input type="text" name="<%=readOnlyHttpRequestTimeout%>" id="<%=readOnlyHttpRequestTimeout%>" size="4" value="<%=appProps.getReadOnlyHttpRequestTimeout()%>"></td>
 </tr>
+<tr>
+    <td class="labkey-form-label"><label for="<%=scriptExecutionTimeout%>">Timeout for server-side scripts, in seconds<%=helpPopup("Script execution timeout",
+        "Maximum time a server-side JavaScript invocation (such as a trigger script) may run before it is terminated. Measured in wall-clock time, including database and other Java operations invoked by the script. Set to 0 to disable the timeout.")%></label></td>
+    <td><input type="text" name="<%=scriptExecutionTimeout%>" id="<%=scriptExecutionTimeout%>" size="4" value="<%=appProps.getScriptExecutionTimeout()%>"></td>
+</tr>
 <tr>
     <td class="labkey-form-label"><label for="<%=maxBLOBSize%>">Maximum file size, in bytes, to allow in database BLOBs</label></td>
     <td><input type="text" name="<%=maxBLOBSize%>" id="<%=maxBLOBSize%>" size="10" value="<%=appProps.getMaxBLOBSize()%>"></td>

core/src/org/labkey/core/script/RhinoService.java

@@ -40,11 +40,15 @@
 import org.labkey.api.resource.Resource;
 import org.labkey.api.script.ScriptReference;
 import org.labkey.api.script.ScriptService;
+import org.labkey.api.security.User;
+import org.labkey.api.settings.AppProps;
+import org.labkey.api.settings.WriteableAppProps;
 import org.labkey.api.test.TestWhen;
 import org.labkey.api.util.HeartBeat;
 import org.labkey.api.util.JunitUtil;
 import org.labkey.api.util.MemTracker;
 import org.labkey.api.util.Path;
+import org.labkey.api.util.TestContext;
 import org.labkey.api.util.UnexpectedException;
 import org.labkey.api.view.HttpView;
 import org.mozilla.javascript.ClassShutter;
@@ -199,6 +203,44 @@ public void reportTest() throws Exception
             test("reportTest");
         }
 
+        @Test
+        public void timeoutTest() throws Exception
+        {
+            int original = AppProps.getInstance().getScriptExecutionTimeout();
+            User user = TestContext.get().getUser();
+
+            try
+            {
+                // Busy-loop bounded at 15 seconds of wall-clock time; the 1-second watchdog should abort it long before that. HeartBeat ticks once per second, so the abort lands after 2-3 real seconds.
+                setScriptExecutionTimeout(1, user);
+                try
+                {
+                    RhinoService.RHINO_FACTORY.getScriptEngine().eval("var start = new Date().getTime(); while (new Date().getTime() - start < 15000) {}");
+                    fail("Expected script to be terminated by the execution timeout");
+                }
+                catch (Exception e)
+                {
+                    assertTrue("Unexpected script error: " + e.getMessage(), e.getMessage().contains("Script execution exceeded 1 seconds"));
+                }
+
+                // 0 disables the watchdog: a loop that runs well past the 1-second timeout above should complete normally
+                setScriptExecutionTimeout(0, user);
+                Object result = RhinoService.RHINO_FACTORY.getScriptEngine().eval("var start = new Date().getTime(); while (new Date().getTime() - start < 2500) {} 'completed';");
+                assertEquals("completed", result);
+            }
+            finally
+            {
+                setScriptExecutionTimeout(original, user);
+            }
+        }
+
+        private void setScriptExecutionTimeout(int seconds, User user)
+        {
+            WriteableAppProps props = AppProps.getWriteableInstance();
+            props.setScriptExecutionTimeout(seconds);
+            props.save(user);
+        }
+
         private void test(String scriptName) throws ScriptException, NoSuchMethodException
         {
             Path js = Path.parse(ScriptService.SCRIPTS_DIR + "/validationTest/" + scriptName + ".js");
@@ -1006,9 +1048,8 @@ protected void observeInstructionCount(Context cx, int instructionCount)
     {
         SandboxContext ctx = (SandboxContext)cx;
         long currentTime = HeartBeat.currentTimeMillis();
-        final int timeout = 60;
-        if (currentTime - ctx.startTime > timeout*1000)
-            Context.reportError("Script execution exceeded " + timeout + " seconds.");
+        if (ctx.timeoutSeconds > 0 && currentTime - ctx.startTime > ctx.timeoutSeconds * 1000L)
+            Context.reportError("Script execution exceeded " + ctx.timeoutSeconds + " seconds.");
     }
 
     @Override
@@ -1044,12 +1085,15 @@ public boolean visibleToScripts(String fullClassName)
     private static class SandboxContext extends Context
     {
         private final long startTime;
+        // resolved once per context; observeInstructionCount runs far too often for a property lookup
+        private final int timeoutSeconds;
 
         private SandboxContext(SandboxContextFactory factory)
         {
             super(factory);
             setLanguageVersion(Context.VERSION_1_8);
             startTime = HeartBeat.currentTimeMillis();
+            timeoutSeconds = AppProps.getInstance().getScriptExecutionTimeout();
         }
     }