introduce an additional absolute free memory limit for data mining query manager

java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/desktop/places/whatsnew/resources/SailingAnalyticsNotes.html

@@ -5,6 +5,11 @@
 <div id="mainContent">
     <h4 class="articleHeadline">What's New - SAP Sailing Analytics</h4>
       <div class="innerContent">
+         <h5 class="articleSubheadline">December 2023</h5>
+         <ul class="bulletList">
+            <li>The memory manager for data mining queries is now less aggressive for large heap sizes and
+                therefore less likely to kill expensive queries when there is still enough free memory.</li>
+         </ul>
          <h5 class="articleSubheadline">November 2023</h5>
          <ul class="bulletList">
             <li>Including short tack and long tack in two ways: In the Data Mining module as a dimension to the GPS fixes and

java/com.sap.sse.datamining.test/src/com/sap/sse/datamining/impl/components/management/TestMemoryMonitorAction.java

@@ -17,10 +17,10 @@ public class TestMemoryMonitorAction {
     
     @Test
     public void testSimpleActionOrdering() {
-        MemoryMonitorAction low = new Test_MemoryMonitorAction(0.8);
-        MemoryMonitorAction medium = new Test_MemoryMonitorAction(0.5);
-        MemoryMonitorAction high1 = new Test_MemoryMonitorAction(0.2);
-        MemoryMonitorAction high2 = new Test_MemoryMonitorAction(0.2);
+        MemoryMonitorAction low = new Test_MemoryMonitorAction(0.8, /* 2GB freeMemoryInBytes */ 2l*1024*1024*1024);
+        MemoryMonitorAction medium = new Test_MemoryMonitorAction(0.5, /* 2GB freeMemoryInBytes */ 2*1024*1024*1024);
+        MemoryMonitorAction high1 = new Test_MemoryMonitorAction(0.2, /* 2GB freeMemoryInBytes */ 2*1024*1024*1024);
+        MemoryMonitorAction high2 = new Test_MemoryMonitorAction(0.2, /* 2GB freeMemoryInBytes */ 2*1024*1024*1024);
         List<MemoryMonitorAction> actions = Arrays.asList(low, high2, high1, medium);
         Collections.sort(actions);
 
@@ -31,15 +31,18 @@ public class TestMemoryMonitorAction {
     
     @Test
     public void testActionPerforming() {
-        Test_MemoryMonitorAction action = new Test_MemoryMonitorAction(0.2);
-
-        assertThat(action.checkMemoryAndPerformAction(0.3), is(false));
+        Test_MemoryMonitorAction action = new Test_MemoryMonitorAction(0.2, /* 2GB freeMemoryInBytes */ 2l*1024*1024*1024);
+        // enough ratio and absolute
+        assertThat(action.checkMemoryAndPerformAction(0.3, /* 2GB freeMemoryInBytes */ 2l*1024*1024*1024+1), is(false));
         assertThat(action.actionHasBeenPerformed(), is(false));
-
-        assertThat(action.checkMemoryAndPerformAction(0.2), is(false));
+        // enough ratio but too little absolute still won't fire
+        assertThat(action.checkMemoryAndPerformAction(0.3, /* 2GB freeMemoryInBytes */ 2l*1024*1024*1024-1), is(false));
         assertThat(action.actionHasBeenPerformed(), is(false));
-
-        assertThat(action.checkMemoryAndPerformAction(0.1), is(true));
+        // exactly matching ratio, and too little absolute won't fire
+        assertThat(action.checkMemoryAndPerformAction(0.2, /* 2GB freeMemoryInBytes */ 2l*1024*1024*1024-1), is(false));
+        assertThat(action.actionHasBeenPerformed(), is(false));
+        // too little ratio and too little absolute will fire
+        assertThat(action.checkMemoryAndPerformAction(0.1, /* 2GB freeMemoryInBytes */ 2l*1024*1024*1024-1), is(true));
         assertThat(action.actionHasBeenPerformed(), is(true));
     }
 

java/com.sap.sse.datamining.test/src/com/sap/sse/datamining/test/util/components/management/Test_MemoryMonitorAction.java

@@ -6,8 +6,8 @@ public class Test_MemoryMonitorAction extends AbstractMemoryMonitorAction {
 
     private boolean actionHasBeenPerformed;
 
-    public Test_MemoryMonitorAction(double freeMemoryInPercentThreshold) {
-        super(freeMemoryInPercentThreshold);
+    public Test_MemoryMonitorAction(double freeMemoryInPercentThreshold, long minFreeMemoryInBytes) {
+        super(freeMemoryInPercentThreshold, minFreeMemoryInBytes);
         actionHasBeenPerformed = false;
     }
     

java/com.sap.sse.datamining/src/com/sap/sse/datamining/components/management/MemoryMonitorAction.java

@@ -19,9 +19,10 @@ public interface MemoryMonitorAction extends Comparable<MemoryMonitorAction> {
      * and performs the action if yes.
      * 
      * @param freeMemoryRatio The current free memory as a ratio of the total memory, between 0..1
+     * @param freeMemoryInBytes TODO
      * @return <code>true</code>, if the action has been performed.
      */
-    boolean checkMemoryAndPerformAction(double freeMemoryRatio);
+    boolean checkMemoryAndPerformAction(double freeMemoryRatio, long freeMemoryInBytes);
     
     /**
      * Compares this action to the given action by its importance.

java/com.sap.sse.datamining/src/com/sap/sse/datamining/impl/DataMiningServerImpl.java

@@ -102,7 +102,7 @@ public class DataMiningServerImpl implements ModifiableDataMiningServer {
     
     private Iterable<MemoryMonitorAction> createMemoryMonitorActions() {
         Collection<MemoryMonitorAction> actions = new ArrayList<>();
-        actions.add(new AbstractMemoryMonitorAction(0.10) {
+        actions.add(new AbstractMemoryMonitorAction(0.10, /* 2GB minFreeMemoryInBytes */ 2l*1024*1024*1024) {
             @Override
             public void performAction() {
                 memoryMonitor.logWarning("Yellow Alert free memory is below " + (100*getThreshold()) + "%!");
@@ -115,7 +115,7 @@ public class DataMiningServerImpl implements ModifiableDataMiningServer {
                 }
             }
         });
-        actions.add(new AbstractMemoryMonitorAction(0.05) {
+        actions.add(new AbstractMemoryMonitorAction(0.05, /* 1GB minFreeMemoryInBytes */ 1l*1024*1024*1024) {
             @Override
             public void performAction() {
                 memoryMonitor.logSevere("Red Alert free memory is below " + (100*getThreshold()) + "%!");

java/com.sap.sse.datamining/src/com/sap/sse/datamining/impl/components/management/AbstractMemoryMonitorAction.java

@@ -2,22 +2,30 @@ package com.sap.sse.datamining.impl.components.management;
 
 import com.sap.sse.datamining.components.management.MemoryMonitorAction;
 
+/**
+ * Triggers an abort when the free memory ratio is below a given threshold and the absolute free memory
+ * is below a certain threshold, too. This should give good results for different sizes of heaps; in particular,
+ * for an "ARCHIVE" server workload with heap sizes of several hundred GBs, a fixed ratio may not be adequate alone
+ * because 10% of 500GB = 50GB may still be lots for a data mining query.
+ */
 public abstract class AbstractMemoryMonitorAction implements MemoryMonitorAction {
 
-    private final double threshold;
+    private final double thresholdFreeMemoryRatio;
+    private final long minFreeMemoryInBytes;
 
-    public AbstractMemoryMonitorAction(double freeMemoryRatioThreshold) {
-        this.threshold = freeMemoryRatioThreshold;
+    public AbstractMemoryMonitorAction(double freeMemoryRatioThreshold, long minFreeMemoryInBytes) {
+        this.thresholdFreeMemoryRatio = freeMemoryRatioThreshold;
+        this.minFreeMemoryInBytes = minFreeMemoryInBytes;
     }
 
     @Override
     public double getThreshold() {
-        return threshold;
+        return thresholdFreeMemoryRatio;
     }
 
     @Override
-    public boolean checkMemoryAndPerformAction(double freeMemoryRatio) {
-        if (freeMemoryRatio < threshold) {
+    public boolean checkMemoryAndPerformAction(double freeMemoryRatio, long freeMemoryInBytes) {
+        if (freeMemoryRatio < thresholdFreeMemoryRatio && freeMemoryInBytes < minFreeMemoryInBytes) {
             performAction();
             return true;
         }
@@ -42,7 +50,7 @@ public abstract class AbstractMemoryMonitorAction implements MemoryMonitorAction
         final int prime = 31;
         int result = 1;
         long temp;
-        temp = Double.doubleToLongBits(threshold);
+        temp = Double.doubleToLongBits(thresholdFreeMemoryRatio);
         result = prime * result + (int) (temp ^ (temp >>> 32));
         return result;
     }
@@ -56,7 +64,7 @@ public abstract class AbstractMemoryMonitorAction implements MemoryMonitorAction
         if (getClass() != obj.getClass())
             return false;
         AbstractMemoryMonitorAction other = (AbstractMemoryMonitorAction) obj;
-        if (Double.doubleToLongBits(threshold) != Double.doubleToLongBits(other.threshold))
+        if (Double.doubleToLongBits(thresholdFreeMemoryRatio) != Double.doubleToLongBits(other.thresholdFreeMemoryRatio))
             return false;
         return true;
     }

java/com.sap.sse.datamining/src/com/sap/sse/datamining/impl/components/management/QueryManagerMemoryMonitor.java

@@ -53,26 +53,26 @@ public class QueryManagerMemoryMonitor implements MemoryMonitor {
     }
     
     private void checkMemory() {
-        final long freeMemory = infoProvider.freeMemory();
-        final long totalMemory = infoProvider.totalMemory();
-        final double freeMemoryRatio = (double) freeMemory / totalMemory;
+        final long freeMemoryInBytes = infoProvider.freeMemory();
+        final long totalMemoryInBytes = infoProvider.totalMemory();
+        final double freeMemoryRatio = (double) freeMemoryInBytes / totalMemoryInBytes;
         final int numberOfRunningQueries = queryManager.getNumberOfRunningQueries();
         if (numberOfRunningQueries > 0) {
-            logStatus(freeMemory, totalMemory, freeMemoryRatio, numberOfRunningQueries);
+            logStatus(freeMemoryInBytes, totalMemoryInBytes, freeMemoryRatio, numberOfRunningQueries);
         }
         boolean actionHasBeenPerformed = false;
         final Iterator<MemoryMonitorAction> actionsIterator = actions.iterator();
         while (!actionHasBeenPerformed && actionsIterator.hasNext()) {
-            MemoryMonitorAction action = actionsIterator.next();
-            actionHasBeenPerformed = action.checkMemoryAndPerformAction(freeMemoryRatio);
+            final MemoryMonitorAction action = actionsIterator.next();
+            actionHasBeenPerformed = action.checkMemoryAndPerformAction(freeMemoryRatio, freeMemoryInBytes);
             if (actionHasBeenPerformed) {
-                //Also perform actions, that are equally important
+                // Also perform actions, that are equally important
                 while (actionsIterator.hasNext()) {
                     MemoryMonitorAction nextAction = actionsIterator.next();
                     if (action.compareTo(nextAction) != 0) {
                         break;
                     }
-                    nextAction.checkMemoryAndPerformAction(freeMemoryRatio);
+                    nextAction.checkMemoryAndPerformAction(freeMemoryRatio, freeMemoryInBytes);
                 }
             }
         }