From b23cee3e511b4cb8f1c64258ddc11b9b5b36d32e Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Thu, 14 Dec 2023 11:37:02 +0100 Subject: [PATCH] introduce an additional absolute free memory limit for data mining query manager --- .../resources/SailingAnalyticsNotes.html | 5 ++++ .../management/TestMemoryMonitorAction.java | 25 +++++++++++-------- .../management/Test_MemoryMonitorAction.java | 4 +-- .../management/MemoryMonitorAction.java | 3 ++- .../datamining/impl/DataMiningServerImpl.java | 4 +-- .../AbstractMemoryMonitorAction.java | 24 ++++++++++++------ .../management/QueryManagerMemoryMonitor.java | 16 ++++++------ 7 files changed, 49 insertions(+), 32 deletions(-) diff --git a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/desktop/places/whatsnew/resources/SailingAnalyticsNotes.html b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/desktop/places/whatsnew/resources/SailingAnalyticsNotes.html index 87850a08264..9bc938c0967 100755 --- a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/desktop/places/whatsnew/resources/SailingAnalyticsNotes.html +++ b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/desktop/places/whatsnew/resources/SailingAnalyticsNotes.html @@ -5,6 +5,11 @@

What's New - SAP Sailing Analytics

+
December 2023
+
    +
  • The memory manager for data mining queries is now less aggressive for large heap sizes and + therefore less likely to kill expensive queries when there is still enough free memory.
    • +
November 2023
  • Including short tack and long tack in two ways: In the Data Mining module as a dimension to the GPS fixes and diff --git a/java/com.sap.sse.datamining.test/src/com/sap/sse/datamining/impl/components/management/TestMemoryMonitorAction.java b/java/com.sap.sse.datamining.test/src/com/sap/sse/datamining/impl/components/management/TestMemoryMonitorAction.java index e840a165da4..a03a4a7e703 100644 --- a/java/com.sap.sse.datamining.test/src/com/sap/sse/datamining/impl/components/management/TestMemoryMonitorAction.java +++ b/java/com.sap.sse.datamining.test/src/com/sap/sse/datamining/impl/components/management/TestMemoryMonitorAction.java @@ -17,10 +17,10 @@ public class TestMemoryMonitorAction { @Test public void testSimpleActionOrdering() { - MemoryMonitorAction low = new Test_MemoryMonitorAction(0.8); - MemoryMonitorAction medium = new Test_MemoryMonitorAction(0.5); - MemoryMonitorAction high1 = new Test_MemoryMonitorAction(0.2); - MemoryMonitorAction high2 = new Test_MemoryMonitorAction(0.2); + MemoryMonitorAction low = new Test_MemoryMonitorAction(0.8, /* 2GB freeMemoryInBytes */ 2l*1024*1024*1024); + MemoryMonitorAction medium = new Test_MemoryMonitorAction(0.5, /* 2GB freeMemoryInBytes */ 2*1024*1024*1024); + MemoryMonitorAction high1 = new Test_MemoryMonitorAction(0.2, /* 2GB freeMemoryInBytes */ 2*1024*1024*1024); + MemoryMonitorAction high2 = new Test_MemoryMonitorAction(0.2, /* 2GB freeMemoryInBytes */ 2*1024*1024*1024); List actions = Arrays.asList(low, high2, high1, medium); Collections.sort(actions); @@ -31,15 +31,18 @@ public class TestMemoryMonitorAction { @Test public void testActionPerforming() { - Test_MemoryMonitorAction action = new Test_MemoryMonitorAction(0.2); - - assertThat(action.checkMemoryAndPerformAction(0.3), is(false)); + Test_MemoryMonitorAction action = new Test_MemoryMonitorAction(0.2, /* 2GB freeMemoryInBytes */ 2l*1024*1024*1024); + // enough ratio and absolute + assertThat(action.checkMemoryAndPerformAction(0.3, /* 2GB freeMemoryInBytes */ 2l*1024*1024*1024+1), is(false)); assertThat(action.actionHasBeenPerformed(), is(false)); - - assertThat(action.checkMemoryAndPerformAction(0.2), is(false)); + // enough ratio but too little absolute still won't fire + assertThat(action.checkMemoryAndPerformAction(0.3, /* 2GB freeMemoryInBytes */ 2l*1024*1024*1024-1), is(false)); assertThat(action.actionHasBeenPerformed(), is(false)); - - assertThat(action.checkMemoryAndPerformAction(0.1), is(true)); + // exactly matching ratio, and too little absolute won't fire + assertThat(action.checkMemoryAndPerformAction(0.2, /* 2GB freeMemoryInBytes */ 2l*1024*1024*1024-1), is(false)); + assertThat(action.actionHasBeenPerformed(), is(false)); + // too little ratio and too little absolute will fire + assertThat(action.checkMemoryAndPerformAction(0.1, /* 2GB freeMemoryInBytes */ 2l*1024*1024*1024-1), is(true)); assertThat(action.actionHasBeenPerformed(), is(true)); } diff --git a/java/com.sap.sse.datamining.test/src/com/sap/sse/datamining/test/util/components/management/Test_MemoryMonitorAction.java b/java/com.sap.sse.datamining.test/src/com/sap/sse/datamining/test/util/components/management/Test_MemoryMonitorAction.java index 138d3896784..9dc3cf4df6b 100644 --- a/java/com.sap.sse.datamining.test/src/com/sap/sse/datamining/test/util/components/management/Test_MemoryMonitorAction.java +++ b/java/com.sap.sse.datamining.test/src/com/sap/sse/datamining/test/util/components/management/Test_MemoryMonitorAction.java @@ -6,8 +6,8 @@ public class Test_MemoryMonitorAction extends AbstractMemoryMonitorAction { private boolean actionHasBeenPerformed; - public Test_MemoryMonitorAction(double freeMemoryInPercentThreshold) { - super(freeMemoryInPercentThreshold); + public Test_MemoryMonitorAction(double freeMemoryInPercentThreshold, long minFreeMemoryInBytes) { + super(freeMemoryInPercentThreshold, minFreeMemoryInBytes); actionHasBeenPerformed = false; } diff --git a/java/com.sap.sse.datamining/src/com/sap/sse/datamining/components/management/MemoryMonitorAction.java b/java/com.sap.sse.datamining/src/com/sap/sse/datamining/components/management/MemoryMonitorAction.java index 084909c6452..1f8a1315587 100644 --- a/java/com.sap.sse.datamining/src/com/sap/sse/datamining/components/management/MemoryMonitorAction.java +++ b/java/com.sap.sse.datamining/src/com/sap/sse/datamining/components/management/MemoryMonitorAction.java @@ -19,9 +19,10 @@ public interface MemoryMonitorAction extends Comparable { * and performs the action if yes. * * @param freeMemoryRatio The current free memory as a ratio of the total memory, between 0..1 + * @param freeMemoryInBytes TODO * @return true, if the action has been performed. */ - boolean checkMemoryAndPerformAction(double freeMemoryRatio); + boolean checkMemoryAndPerformAction(double freeMemoryRatio, long freeMemoryInBytes); /** * Compares this action to the given action by its importance. diff --git a/java/com.sap.sse.datamining/src/com/sap/sse/datamining/impl/DataMiningServerImpl.java b/java/com.sap.sse.datamining/src/com/sap/sse/datamining/impl/DataMiningServerImpl.java index 5ce82ca3bb7..1c4e1e393a8 100644 --- a/java/com.sap.sse.datamining/src/com/sap/sse/datamining/impl/DataMiningServerImpl.java +++ b/java/com.sap.sse.datamining/src/com/sap/sse/datamining/impl/DataMiningServerImpl.java @@ -102,7 +102,7 @@ public class DataMiningServerImpl implements ModifiableDataMiningServer { private Iterable createMemoryMonitorActions() { Collection actions = new ArrayList<>(); - actions.add(new AbstractMemoryMonitorAction(0.10) { + actions.add(new AbstractMemoryMonitorAction(0.10, /* 2GB minFreeMemoryInBytes */ 2l*1024*1024*1024) { @Override public void performAction() { memoryMonitor.logWarning("Yellow Alert free memory is below " + (100*getThreshold()) + "%!"); @@ -115,7 +115,7 @@ public class DataMiningServerImpl implements ModifiableDataMiningServer { } } }); - actions.add(new AbstractMemoryMonitorAction(0.05) { + actions.add(new AbstractMemoryMonitorAction(0.05, /* 1GB minFreeMemoryInBytes */ 1l*1024*1024*1024) { @Override public void performAction() { memoryMonitor.logSevere("Red Alert free memory is below " + (100*getThreshold()) + "%!"); diff --git a/java/com.sap.sse.datamining/src/com/sap/sse/datamining/impl/components/management/AbstractMemoryMonitorAction.java b/java/com.sap.sse.datamining/src/com/sap/sse/datamining/impl/components/management/AbstractMemoryMonitorAction.java index 2eb1ba19d7e..04335e34be3 100644 --- a/java/com.sap.sse.datamining/src/com/sap/sse/datamining/impl/components/management/AbstractMemoryMonitorAction.java +++ b/java/com.sap.sse.datamining/src/com/sap/sse/datamining/impl/components/management/AbstractMemoryMonitorAction.java @@ -2,22 +2,30 @@ package com.sap.sse.datamining.impl.components.management; import com.sap.sse.datamining.components.management.MemoryMonitorAction; +/** + * Triggers an abort when the free memory ratio is below a given threshold and the absolute free memory + * is below a certain threshold, too. This should give good results for different sizes of heaps; in particular, + * for an "ARCHIVE" server workload with heap sizes of several hundred GBs, a fixed ratio may not be adequate alone + * because 10% of 500GB = 50GB may still be lots for a data mining query. + */ public abstract class AbstractMemoryMonitorAction implements MemoryMonitorAction { - private final double threshold; + private final double thresholdFreeMemoryRatio; + private final long minFreeMemoryInBytes; - public AbstractMemoryMonitorAction(double freeMemoryRatioThreshold) { - this.threshold = freeMemoryRatioThreshold; + public AbstractMemoryMonitorAction(double freeMemoryRatioThreshold, long minFreeMemoryInBytes) { + this.thresholdFreeMemoryRatio = freeMemoryRatioThreshold; + this.minFreeMemoryInBytes = minFreeMemoryInBytes; } @Override public double getThreshold() { - return threshold; + return thresholdFreeMemoryRatio; } @Override - public boolean checkMemoryAndPerformAction(double freeMemoryRatio) { - if (freeMemoryRatio < threshold) { + public boolean checkMemoryAndPerformAction(double freeMemoryRatio, long freeMemoryInBytes) { + if (freeMemoryRatio < thresholdFreeMemoryRatio && freeMemoryInBytes < minFreeMemoryInBytes) { performAction(); return true; } @@ -42,7 +50,7 @@ public abstract class AbstractMemoryMonitorAction implements MemoryMonitorAction final int prime = 31; int result = 1; long temp; - temp = Double.doubleToLongBits(threshold); + temp = Double.doubleToLongBits(thresholdFreeMemoryRatio); result = prime * result + (int) (temp ^ (temp >>> 32)); return result; } @@ -56,7 +64,7 @@ public abstract class AbstractMemoryMonitorAction implements MemoryMonitorAction if (getClass() != obj.getClass()) return false; AbstractMemoryMonitorAction other = (AbstractMemoryMonitorAction) obj; - if (Double.doubleToLongBits(threshold) != Double.doubleToLongBits(other.threshold)) + if (Double.doubleToLongBits(thresholdFreeMemoryRatio) != Double.doubleToLongBits(other.thresholdFreeMemoryRatio)) return false; return true; } diff --git a/java/com.sap.sse.datamining/src/com/sap/sse/datamining/impl/components/management/QueryManagerMemoryMonitor.java b/java/com.sap.sse.datamining/src/com/sap/sse/datamining/impl/components/management/QueryManagerMemoryMonitor.java index 51dc5b51625..d962d024aa6 100644 --- a/java/com.sap.sse.datamining/src/com/sap/sse/datamining/impl/components/management/QueryManagerMemoryMonitor.java +++ b/java/com.sap.sse.datamining/src/com/sap/sse/datamining/impl/components/management/QueryManagerMemoryMonitor.java @@ -53,26 +53,26 @@ public class QueryManagerMemoryMonitor implements MemoryMonitor { } private void checkMemory() { - final long freeMemory = infoProvider.freeMemory(); - final long totalMemory = infoProvider.totalMemory(); - final double freeMemoryRatio = (double) freeMemory / totalMemory; + final long freeMemoryInBytes = infoProvider.freeMemory(); + final long totalMemoryInBytes = infoProvider.totalMemory(); + final double freeMemoryRatio = (double) freeMemoryInBytes / totalMemoryInBytes; final int numberOfRunningQueries = queryManager.getNumberOfRunningQueries(); if (numberOfRunningQueries > 0) { - logStatus(freeMemory, totalMemory, freeMemoryRatio, numberOfRunningQueries); + logStatus(freeMemoryInBytes, totalMemoryInBytes, freeMemoryRatio, numberOfRunningQueries); } boolean actionHasBeenPerformed = false; final Iterator actionsIterator = actions.iterator(); while (!actionHasBeenPerformed && actionsIterator.hasNext()) { - MemoryMonitorAction action = actionsIterator.next(); - actionHasBeenPerformed = action.checkMemoryAndPerformAction(freeMemoryRatio); + final MemoryMonitorAction action = actionsIterator.next(); + actionHasBeenPerformed = action.checkMemoryAndPerformAction(freeMemoryRatio, freeMemoryInBytes); if (actionHasBeenPerformed) { - //Also perform actions, that are equally important + // Also perform actions, that are equally important while (actionsIterator.hasNext()) { MemoryMonitorAction nextAction = actionsIterator.next(); if (action.compareTo(nextAction) != 0) { break; } - nextAction.checkMemoryAndPerformAction(freeMemoryRatio); + nextAction.checkMemoryAndPerformAction(freeMemoryRatio, freeMemoryInBytes); } } }