From 8f2dd7f0d8333121bb306e88323a8fb1ecd086ce Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Mon, 18 Feb 2019 12:27:15 +0100 Subject: [PATCH 01/24] Added unit test for WindEstimationDataClient (yet without wind_estimation_data) --- .../api/WindEstimationDataClientMock.java | 33 +++++++++++++++++ .../api/WindEstimationDataResourceTest.java | 35 +++++++++++++++++++ .../windestimation/integration/Activator.java | 6 ++-- 3 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataClientMock.java create mode 100644 java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResourceTest.java diff --git a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataClientMock.java b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataClientMock.java new file mode 100644 index 00000000000..1d8769b56b6 --- /dev/null +++ b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataClientMock.java @@ -0,0 +1,33 @@ +package com.sap.sailing.windestimation.jaxrs.api; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; + +import org.json.simple.parser.ParseException; + +import com.sap.sailing.windestimation.integration.ReplicableWindEstimationFactoryService; +import com.sap.sailing.windestimation.jaxrs.client.WindEstimationDataClient; + +/** + * + * @author Vladislav Chumak (D069712) + * + */ +public class WindEstimationDataClientMock extends WindEstimationDataClient { + + private final File file; + + public WindEstimationDataClientMock(File file, + ReplicableWindEstimationFactoryService windEstimationFactoryService) { + super(null, windEstimationFactoryService); + this.file = file; + } + + @Override + protected InputStream getContentFromResponse() throws IOException, ParseException { + return new FileInputStream(file); + } + +} diff --git a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResourceTest.java b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResourceTest.java new file mode 100644 index 00000000000..5cdb501a869 --- /dev/null +++ b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResourceTest.java @@ -0,0 +1,35 @@ +package com.sap.sailing.windestimation.jaxrs.api; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; + +import org.json.simple.parser.ParseException; +import org.junit.Ignore; +import org.junit.Test; + +import com.sap.sailing.domain.polars.NotEnoughDataHasBeenAddedException; +import com.sap.sailing.windestimation.integration.WindEstimationFactoryServiceImpl; + +/** + * + * @author Vladislav Chumak (D069712) + * + */ +public class WindEstimationDataResourceTest { + + @Test + @Ignore + public void testImportingFromMockFile() throws NotEnoughDataHasBeenAddedException, ClassNotFoundException, + IOException, ParseException, InterruptedException { + WindEstimationFactoryServiceImpl windEstimationFactoryService = new WindEstimationFactoryServiceImpl(); + assertFalse(windEstimationFactoryService.isReady()); + final WindEstimationDataClientMock client = new WindEstimationDataClientMock( + new File("resources/wind_estimation_data"), windEstimationFactoryService); + client.updateWindEstimationModels(); + assertTrue(windEstimationFactoryService.isReady()); + } + +} \ No newline at end of file diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/Activator.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/Activator.java index d43c821f7bd..a60d8530c6d 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/Activator.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/Activator.java @@ -88,10 +88,10 @@ public class Activator implements BundleActivator { private void importWindEstimationModelsFromUrl(String windEstimationModelDataSourceURL) { logger.info("Importing wind estimation data from URL: " + windEstimationModelDataSourceURL); - WindEstimationDataClient polarDataClient = new WindEstimationDataClient(windEstimationModelDataSourceURL, - service); + WindEstimationDataClient windEstimationDataClient = new WindEstimationDataClient( + windEstimationModelDataSourceURL, service); try { - polarDataClient.updateWindEstimationModels(); + windEstimationDataClient.updateWindEstimationModels(); } catch (Exception e) { logger.log(Level.SEVERE, "Exception while trying to import wind estimation model data from " + windEstimationModelDataSourceURL, e); From b159cdd4081e22ff3e473efee353f8dbcce52034 Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Mon, 18 Feb 2019 15:28:33 +0100 Subject: [PATCH 02/24] Implemented ExportedModelsGenerator --- .../ExportedModelsGenerator.java | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/ExportedModelsGenerator.java diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/ExportedModelsGenerator.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/ExportedModelsGenerator.java new file mode 100644 index 00000000000..9b3d0f5621f --- /dev/null +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/ExportedModelsGenerator.java @@ -0,0 +1,49 @@ +package com.sap.sailing.windestimation; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import com.sap.sailing.windestimation.integration.ExportedModels; +import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl; +import com.sap.sailing.windestimation.model.store.ModelDomainType; +import com.sap.sailing.windestimation.model.store.ModelStore; +import com.sap.sailing.windestimation.util.LoggingUtil; + +public class ExportedModelsGenerator { + + private static final File file = new File("windEstimationModels.dat"); + + public static void main(String[] args) throws FileNotFoundException, IOException { + ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models"); + // RegularManeuversForEstimationPersistenceManager persistenceManager = new + // RegularManeuversForEstimationPersistenceManager(); + // ModelStore modelStore = new MongoDbModelStoreImpl(persistenceManager.getDb()); + LoggingUtil.logInfo("Generation a single file with all trained models started"); + ExportedModels exportedModels = new ExportedModels(); + List nonEmptyDomainTypes = new ArrayList<>(); + for (ModelDomainType domainType : ModelDomainType.values()) { + LoggingUtil.logInfo("Started serialization of models belonging to domain type " + domainType); + Map exportedModelsForDomainType = modelStore.exportAllPersistedModels(domainType); + if (!exportedModelsForDomainType.isEmpty()) { + exportedModels.addSerializedModelsForDomainType(domainType, exportedModelsForDomainType); + LoggingUtil.logInfo("Finished serialization of " + exportedModelsForDomainType.size() + + " models belonging to domain type " + domainType); + nonEmptyDomainTypes.add(domainType); + } + } + LoggingUtil.logInfo("Serializing the exported models and writing it into a single file"); + try (ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(file))) { + oos.writeObject(exportedModels); + } + LoggingUtil.logInfo("Models have been generated"); + LoggingUtil.logInfo("The following domain types were considered: " + nonEmptyDomainTypes); + LoggingUtil.logInfo("File with all exported models was persisted in: " + file.getAbsolutePath()); + } + +} From 019a9b7a78f52275e63f78812106c182f2f45bf6 Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Mon, 18 Feb 2019 15:28:48 +0100 Subject: [PATCH 03/24] Enabled WindEstimationDataResourceTest --- .../api/WindEstimationDataClientMock.java | 26 ++++++++++++++----- .../api/WindEstimationDataResourceTest.java | 12 +++++---- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataClientMock.java b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataClientMock.java index 1d8769b56b6..c1292444471 100644 --- a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataClientMock.java +++ b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataClientMock.java @@ -1,14 +1,19 @@ package com.sap.sailing.windestimation.jaxrs.api; -import java.io.File; -import java.io.FileInputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.ObjectOutputStream; +import java.util.Map; import org.json.simple.parser.ParseException; +import com.sap.sailing.windestimation.integration.ExportedModels; import com.sap.sailing.windestimation.integration.ReplicableWindEstimationFactoryService; import com.sap.sailing.windestimation.jaxrs.client.WindEstimationDataClient; +import com.sap.sailing.windestimation.model.store.ModelDomainType; +import com.sap.sailing.windestimation.model.store.ModelStore; /** * @@ -17,17 +22,26 @@ import com.sap.sailing.windestimation.jaxrs.client.WindEstimationDataClient; */ public class WindEstimationDataClientMock extends WindEstimationDataClient { - private final File file; + private final ModelStore modelStore; - public WindEstimationDataClientMock(File file, + public WindEstimationDataClientMock(ModelStore modelStore, ReplicableWindEstimationFactoryService windEstimationFactoryService) { super(null, windEstimationFactoryService); - this.file = file; + this.modelStore = modelStore; } @Override protected InputStream getContentFromResponse() throws IOException, ParseException { - return new FileInputStream(file); + ExportedModels exportedModels = new ExportedModels(); + for (ModelDomainType domainType : ModelDomainType.values()) { + Map exportedModelsForDomainType = modelStore.exportAllPersistedModels(domainType); + exportedModels.addSerializedModelsForDomainType(domainType, exportedModelsForDomainType); + } + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + try (ObjectOutputStream oos = new ObjectOutputStream(byteArrayOutputStream)) { + oos.writeObject(exportedModels); + } + return new ByteArrayInputStream(byteArrayOutputStream.toByteArray()); } } diff --git a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResourceTest.java b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResourceTest.java index 5cdb501a869..d88b23a9d35 100644 --- a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResourceTest.java +++ b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResourceTest.java @@ -3,15 +3,15 @@ package com.sap.sailing.windestimation.jaxrs.api; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.io.File; import java.io.IOException; import org.json.simple.parser.ParseException; -import org.junit.Ignore; import org.junit.Test; import com.sap.sailing.domain.polars.NotEnoughDataHasBeenAddedException; +import com.sap.sailing.windestimation.integration.IncrementalMstHmmWindEstimationForTrackedRaceTest; import com.sap.sailing.windestimation.integration.WindEstimationFactoryServiceImpl; +import com.sap.sailing.windestimation.model.store.ClassPathReadOnlyModelStoreImpl; /** * @@ -21,13 +21,15 @@ import com.sap.sailing.windestimation.integration.WindEstimationFactoryServiceIm public class WindEstimationDataResourceTest { @Test - @Ignore public void testImportingFromMockFile() throws NotEnoughDataHasBeenAddedException, ClassNotFoundException, IOException, ParseException, InterruptedException { WindEstimationFactoryServiceImpl windEstimationFactoryService = new WindEstimationFactoryServiceImpl(); assertFalse(windEstimationFactoryService.isReady()); - final WindEstimationDataClientMock client = new WindEstimationDataClientMock( - new File("resources/wind_estimation_data"), windEstimationFactoryService); + ClassPathReadOnlyModelStoreImpl modelStore = new ClassPathReadOnlyModelStoreImpl( + "trained_wind_estimation_models", getClass().getClassLoader(), + IncrementalMstHmmWindEstimationForTrackedRaceTest.modelFilesNames); + final WindEstimationDataClientMock client = new WindEstimationDataClientMock(modelStore, + windEstimationFactoryService); client.updateWindEstimationModels(); assertTrue(windEstimationFactoryService.isReady()); } From a95994663e225af56a498d250373fc20e866e155 Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Mon, 18 Feb 2019 15:32:19 +0100 Subject: [PATCH 04/24] Made ManeuverAndWindImporter to operate in parallel --- .../importer/ManeuverAndWindImporter.java | 67 +++++++++++++------ 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java index 359302d2e17..e4e70e150ab 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java @@ -13,6 +13,9 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.apache.http.Header; import org.apache.http.HttpResponse; @@ -40,11 +43,11 @@ import com.sap.sailing.windestimation.data.persistence.maneuver.RaceWithManeuver import com.sap.sailing.windestimation.data.persistence.maneuver.RaceWithManeuverForEstimationPersistenceManager; import com.sap.sailing.windestimation.data.persistence.twdtransition.RaceWithWindSourcesPersistenceManager; import com.sap.sailing.windestimation.data.serialization.CompetitorTrackWithEstimationDataJsonDeserializer; -import com.sap.sailing.windestimation.data.serialization.ManeuverForDataAnalysisJsonSerializer; import com.sap.sailing.windestimation.data.serialization.LabeledManeuverForEstimationJsonSerializer; +import com.sap.sailing.windestimation.data.serialization.ManeuverForDataAnalysisJsonSerializer; import com.sap.sailing.windestimation.data.transformer.CompetitorTrackTransformer; -import com.sap.sailing.windestimation.data.transformer.CompleteManeuverCurveWithEstimationDataToManeuverForDataAnalysisTransformer; import com.sap.sailing.windestimation.data.transformer.CompleteManeuverCurveWithEstimationDataToLabelledManeuverForEstimationTransformer; +import com.sap.sailing.windestimation.data.transformer.CompleteManeuverCurveWithEstimationDataToManeuverForDataAnalysisTransformer; import com.sap.sailing.windestimation.util.LoggingUtil; /** @@ -70,6 +73,8 @@ public class ManeuverAndWindImporter { private final ManeuverForDataAnalysisJsonSerializer maneuverForDataAnalysisJsonSerializer; private final LabeledManeuverForEstimationJsonSerializer maneuverForEstimationJsonSerializer; private boolean skipRace; + private static final int NUMBER_OF_THREADS = 50; //high number due to HTTP requests + private final ExecutorService executorService = Executors.newFixedThreadPool(NUMBER_OF_THREADS); public ManeuverAndWindImporter() throws UnknownHostException { this.completeManeuverCurvePersistanceManager = new RaceWithCompleteManeuverCurvePersistenceManager(); @@ -94,8 +99,8 @@ public class ManeuverAndWindImporter { importer.importAllRegattas(); } - public void importAllRegattas() - throws IllegalStateException, ClientProtocolException, IOException, ParseException, URISyntaxException { + public void importAllRegattas() throws IllegalStateException, ClientProtocolException, IOException, ParseException, + URISyntaxException, InterruptedException { skipRace = startFromRegattaName != null; LoggingUtil.logInfo("Importer for CompleteManeuverCurveWithEstimationData just started"); LoggingUtil.logInfo("Dropping old database"); @@ -118,9 +123,17 @@ public class ManeuverAndWindImporter { + Math.round(100.0 * i / numberOfRegattas) + "%): \"" + regattaName + "\""); importRegatta(regattaName, importStatistics); } - LoggingUtil.logInfo("Import finished"); - importStatistics.regattasCount = regattasJson.size(); - logImportStatistics(importStatistics); + boolean success = executorService.awaitTermination(24, TimeUnit.HOURS); + executorService.shutdown(); + if (success) { + LoggingUtil.logInfo("Import finished"); + synchronized (importStatistics) { + importStatistics.regattasCount = regattasJson.size(); + logImportStatistics(importStatistics); + } + } else { + LoggingUtil.logInfo("Process was terminated"); + } } private void logImportStatistics(ImportStatistics importStatistics) { @@ -145,7 +158,9 @@ public class ManeuverAndWindImporter { try { regattaJson = (JSONObject) getHttpResponseAsJson(regattaName, null, getRegatta); } catch (Exception e) { - importStatistics.ignoredRegattas += 1; + synchronized (importStatistics) { + importStatistics.ignoredRegattas++; + } LoggingUtil.logInfo("Error while processing regatta: " + regattaName); return; } @@ -161,17 +176,25 @@ public class ManeuverAndWindImporter { if ((boolean) race.get("isTracked") && !(boolean) race.get("isLive") && (boolean) race.get("hasGpsData") && (boolean) race.get("hasWindData")) { String trackedRaceName = (String) race.get("trackedRaceName"); - LoggingUtil.logInfo("Processing race nr. " + ++i + ": \"" + trackedRaceName + "\""); - try { - importRace(regattaName, trackedRaceName, importStatistics); - } catch (Exception e) { - importStatistics.ingoredRaces += 1; - LoggingUtil - .logInfo("Error while processing race nr. " + i + ": \"" + trackedRaceName + "\""); - } + i++; + final int raceNumber = i; + LoggingUtil.logInfo("Processing race nr. " + raceNumber + ": \"" + trackedRaceName + "\""); + executorService.execute(() -> { + try { + importRace(regattaName, trackedRaceName, importStatistics); + } catch (Exception e) { + synchronized (importStatistics) { + importStatistics.ingoredRaces += 1; + } + LoggingUtil.logInfo("Error while processing race nr. " + raceNumber + ": \"" + + trackedRaceName + "\""); + } + }); } } - importStatistics.racesCount += racesJson.size(); + synchronized (importStatistics) { + importStatistics.racesCount += racesJson.size(); + } } } } @@ -217,7 +240,9 @@ public class ManeuverAndWindImporter { } LoggingUtil.logInfo( "Imported " + windFixesCount + " wind fixes from " + windSourcesJson.size() + " wind sources"); - importStatistics.racesWithHighQualityWindData++; + synchronized (importStatistics) { + importStatistics.racesWithHighQualityWindData++; + } } else { LoggingUtil.logInfo("No high quality wind fixes contained"); } @@ -274,8 +299,10 @@ public class ManeuverAndWindImporter { } LoggingUtil.logInfo( "Imported " + competitorTracks.size() + " competitor tracks with " + maneuversCount + " maneuvers"); - importStatistics.competitorTracksCount += competitorTracks.size(); - importStatistics.maneuversCount += maneuversCount; + synchronized (importStatistics) { + importStatistics.competitorTracksCount += competitorTracks.size(); + importStatistics.maneuversCount += maneuversCount; + } } private JSONObject getHttpResponseAsJson(String trackedRegattaName, String trackedRaceName, From 4c0101a317cbedecca9181a023c8086a1321f66c Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Mon, 18 Feb 2019 15:33:02 +0100 Subject: [PATCH 05/24] Added Refresh button to twa delta charts plotting --- ...ingleDimensionAggregatesPlottingFrame.java | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/SingleDimensionAggregatesPlottingFrame.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/SingleDimensionAggregatesPlottingFrame.java index ea41fc2a1f4..0e5ba2a7f2b 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/SingleDimensionAggregatesPlottingFrame.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/SingleDimensionAggregatesPlottingFrame.java @@ -3,9 +3,13 @@ package com.sap.sailing.windestimation.datavisualization; import java.awt.BasicStroke; import java.awt.Color; import java.awt.Shape; +import java.awt.event.WindowAdapter; +import java.awt.event.WindowEvent; import java.awt.geom.Ellipse2D; +import java.util.concurrent.CountDownLatch; import javax.swing.BoxLayout; +import javax.swing.JButton; import javax.swing.JFrame; import javax.swing.JPanel; import javax.swing.SwingUtilities; @@ -40,27 +44,54 @@ public class SingleDimensionAggregatesPlottingFrame extends JFrame { private XYSeries valuesSeries; private XYSeries stdSeries; private XYSeries zeroMeanStdSeries; + private final CountDownLatch windowClosedLatch = new CountDownLatch(1); public SingleDimensionAggregatesPlottingFrame( AggregatedSingleDimensionBasedTwdTransitionPersistenceManager persistenceManager) { super(persistenceManager.getCollectionName()); this.persistenceManager = persistenceManager; + addWindowListener(new WindowAdapter() { + @Override + public void windowClosed(WindowEvent event) { + windowClosedLatch.countDown(); + } + + @Override + public void windowClosing(WindowEvent e) { + setLayout(null); + setVisible(false); + dispose(); + } + }); + initializeDataWithCharts(); + } + + private void initializeDataWithCharts() { createSeries(); fillSeries(); DefaultTableXYDataset mainDataset = createMainDataset(); DefaultTableXYDataset histoDataset = createHistoDataset(); mainChart = createMainChart(mainDataset); histoChart = createHistoChart(histoDataset); - // Create Panel JPanel panel = new JPanel(); panel.setLayout(new BoxLayout(panel, BoxLayout.Y_AXIS)); ChartPanel mainChartPanel = new ChartPanel(mainChart); ChartPanel histoChartPanel = new ChartPanel(histoChart); + JButton refreshButton = new JButton("Refresh charts"); + refreshButton.addActionListener(event -> refreshCharts()); + panel.add(refreshButton); panel.add(mainChartPanel); panel.add(histoChartPanel); setContentPane(panel); } + public void refreshCharts() { + SwingUtilities.invokeLater(() -> { + initializeDataWithCharts(); + revalidate(); + }); + } + private void createSeries() { meanSeries = new XYSeries("Mean", true, false); medianSeries = new XYSeries("Median", true, false); @@ -191,4 +222,8 @@ public class SingleDimensionAggregatesPlottingFrame extends JFrame { }); } + public void awaitWindowClosed() throws InterruptedException { + windowClosedLatch.await(); + } + } From 501248a784b2696866aa883d0d0821881eaa2f9d Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Mon, 18 Feb 2019 15:47:40 +0100 Subject: [PATCH 06/24] Implemented simple models training --- .../windestimation/SimpleModelsTraining.java | 116 ++++++++++++++++++ .../AggregatedDistanceDimensionPlot.java | 8 +- .../AggregatedDurationDimensionPlot.java | 8 +- 3 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/SimpleModelsTraining.java diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/SimpleModelsTraining.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/SimpleModelsTraining.java new file mode 100644 index 00000000000..c15c543327a --- /dev/null +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/SimpleModelsTraining.java @@ -0,0 +1,116 @@ +package com.sap.sailing.windestimation; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import javax.swing.JOptionPane; + +import com.sap.sailing.windestimation.data.importer.AggregatedDistanceBasedTwdTransitionImporter; +import com.sap.sailing.windestimation.data.importer.AggregatedDurationBasedTwdTransitionImporter; +import com.sap.sailing.windestimation.data.importer.DistanceBasedTwdTransitionImporter; +import com.sap.sailing.windestimation.data.importer.DurationBasedTwdTransitionImporter; +import com.sap.sailing.windestimation.data.importer.ManeuverAndWindImporter; +import com.sap.sailing.windestimation.data.importer.PolarDataImporter; +import com.sap.sailing.windestimation.data.persistence.maneuver.ManeuverForEstimationPersistenceManager; +import com.sap.sailing.windestimation.data.persistence.maneuver.RegularManeuversForEstimationPersistenceManager; +import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedSingleDimensionBasedTwdTransitionPersistenceManager.AggregatedSingleDimensionType; +import com.sap.sailing.windestimation.datavisualization.AggregatedDistanceDimensionPlot; +import com.sap.sailing.windestimation.datavisualization.AggregatedDurationDimensionPlot; +import com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverClassifierTrainer; +import com.sap.sailing.windestimation.model.classifier.maneuver.PersistedManeuverClassifiersScorePrinter; +import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionStdRegressorTrainer; +import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionStdRegressorTrainer; +import com.sap.sailing.windestimation.util.LoggingUtil; + +/** + * + * @author Vladislav Chumak (D069712) + * + */ +public class SimpleModelsTraining { + + private static final ExecutorService executorService = Executors.newFixedThreadPool(3); + + public static void main(String[] args) throws Exception { + new ManeuverForEstimationPersistenceManager().dropCollection(); + new RegularManeuversForEstimationPersistenceManager().dropCollection(); + executeInThreadPool(() -> PolarDataImporter.main(args)); + ManeuverAndWindImporter.main(args); + awaitThreadPoolCompletion(); + executeInThreadPool(() -> { + ManeuverClassifierTrainer.main(args); + Thread.sleep(1000); + PersistedManeuverClassifiersScorePrinter.main(args); + }); + executeInThreadPool(() -> { + DurationBasedTwdTransitionImporter.main(args); + Thread.sleep(1000); + AggregatedDurationBasedTwdTransitionImporter.main(args); + }); + DistanceBasedTwdTransitionImporter.main(args); + Thread.sleep(1000); + AggregatedDistanceBasedTwdTransitionImporter.main(args); + awaitThreadPoolCompletion(); + do { + AggregatedDurationDimensionPlot.main(args); + showInfoAboutDataCleaning(AggregatedSingleDimensionType.DURATION); + AggregatedDurationDimensionPlot.awaitWindowClosed(); + } while (JOptionPane.YES_OPTION != askDataCleaningFinished(AggregatedSingleDimensionType.DURATION)); + do { + AggregatedDistanceDimensionPlot.main(args); + showInfoAboutDataCleaning(AggregatedSingleDimensionType.DISTANCE); + AggregatedDistanceDimensionPlot.awaitWindowClosed(); + } while (JOptionPane.YES_OPTION != askDataCleaningFinished(AggregatedSingleDimensionType.DISTANCE)); + DurationBasedTwdTransitionStdRegressorTrainer.main(args); + DistanceBasedTwdTransitionStdRegressorTrainer.main(args); + Thread.sleep(1000); + ExportedModelsGenerator.main(args); + LoggingUtil.logInfo("Model training finished. You can upload the generated file to a server instance."); + } + + private static int askDataCleaningFinished(AggregatedSingleDimensionType dimension) { + Object[] options = { "Continue with model training", "Restart Graphical Tool" }; + int res = JOptionPane.showOptionDialog(null, + "Have you finished the data cleaning for " + dimension + " dimension?", "Continue with model training?", + JOptionPane.DEFAULT_OPTION, JOptionPane.INFORMATION_MESSAGE, null, options, options[0]); + switch (res) { + case 0: + return JOptionPane.YES_OPTION; + case 1: + return JOptionPane.NO_OPTION; + } + return JOptionPane.CLOSED_OPTION; + } + + private static void showInfoAboutDataCleaning(AggregatedSingleDimensionType dimension) { + JOptionPane.showMessageDialog(null, "Now, clean the data for " + dimension + + " dimension. Remove instances from MongoDB collection \"" + dimension.getCollectioName() + + "\" which do not make sense. E.g. values which are represented by a small number of supporting instances (see histogram), values which cause implausible zig zag sections within zero-mean standard deviation curve and etc. Close the graphical tool, when you are done to resume model training."); + } + + private static void awaitThreadPoolCompletion() throws InterruptedException { + executorService.awaitTermination(24, TimeUnit.HOURS); + Thread.sleep(1000L); + } + + private static void executeInThreadPool(RunnableWithExceptionsCatch runnable) { + executorService.execute(() -> { + try { + runnable.run(); + } catch (Exception e) { + e.printStackTrace(); + } + }); + } + + /** + * + * @author Vladislav Chumak (D069712) + * + */ + private static interface RunnableWithExceptionsCatch { + void run() throws Exception; + } + +} diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/AggregatedDistanceDimensionPlot.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/AggregatedDistanceDimensionPlot.java index baa9f7bbaa5..ab81c226b02 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/AggregatedDistanceDimensionPlot.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/AggregatedDistanceDimensionPlot.java @@ -7,11 +7,17 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS public class AggregatedDistanceDimensionPlot { + private static SingleDimensionAggregatesPlottingFrame plot; + public static void main(String[] args) throws UnknownHostException { AggregatedSingleDimensionBasedTwdTransitionPersistenceManager persistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager( AggregatedSingleDimensionType.DISTANCE); - SingleDimensionAggregatesPlottingFrame plot = new SingleDimensionAggregatesPlottingFrame(persistenceManager); + plot = new SingleDimensionAggregatesPlottingFrame(persistenceManager); plot.renderChart(); } + public static void awaitWindowClosed() throws InterruptedException { + plot.awaitWindowClosed(); + } + } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/AggregatedDurationDimensionPlot.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/AggregatedDurationDimensionPlot.java index 4f38112b942..2903908050f 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/AggregatedDurationDimensionPlot.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/AggregatedDurationDimensionPlot.java @@ -7,11 +7,17 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS public class AggregatedDurationDimensionPlot { + private static SingleDimensionAggregatesPlottingFrame plot; + public static void main(String[] args) throws UnknownHostException { AggregatedSingleDimensionBasedTwdTransitionPersistenceManager persistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager( AggregatedSingleDimensionType.DURATION); - SingleDimensionAggregatesPlottingFrame plot = new SingleDimensionAggregatesPlottingFrame(persistenceManager); + plot = new SingleDimensionAggregatesPlottingFrame(persistenceManager); plot.renderChart(); } + + public static void awaitWindowClosed() throws InterruptedException { + plot.awaitWindowClosed(); + } } From 9c446c9a8a895acc73aeec766ae391df8b4ebcde Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Mon, 18 Feb 2019 15:48:07 +0100 Subject: [PATCH 07/24] Fixed encoding problem with regatta and race names --- .../windestimation/data/importer/ManeuverAndWindImporter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java index e4e70e150ab..24f3b4b47e3 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java @@ -365,7 +365,7 @@ public class ManeuverAndWindImporter { final Header contentEncoding = response.getEntity().getContentEncoding(); final Reader reader; if (contentEncoding == null) { - reader = new InputStreamReader(response.getEntity().getContent()); + reader = new InputStreamReader(response.getEntity().getContent(), "UTF-8"); } else { reader = new InputStreamReader(response.getEntity().getContent(), contentEncoding.getValue()); } From d6efae16ddc34c409ae8ae2772028c2dd07b3c1b Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Mon, 18 Feb 2019 18:48:10 +0100 Subject: [PATCH 08/24] Improved exception handling and thread pool config --- .../importer/ManeuverAndWindImporter.java | 50 ++++++++++++------- ...neuverNumberDependentEvaluationRunner.java | 7 +-- .../{ => model}/ExportedModelsGenerator.java | 2 +- .../{ => model}/SimpleModelsTraining.java | 7 +-- .../maneuver/ManeuverClassifierTrainer.java | 5 +- ...sistedManeuverClassifiersScorePrinter.java | 8 +-- ...BasedTwdTransitionStdRegressorTrainer.java | 5 +- ...BasedTwdTransitionStdRegressorTrainer.java | 5 +- 8 files changed, 49 insertions(+), 40 deletions(-) rename java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/{ => model}/ExportedModelsGenerator.java (98%) rename java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/{ => model}/SimpleModelsTraining.java (97%) diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java index 24f3b4b47e3..a2fabf42a8b 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java @@ -13,8 +13,9 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.http.Header; @@ -73,8 +74,8 @@ public class ManeuverAndWindImporter { private final ManeuverForDataAnalysisJsonSerializer maneuverForDataAnalysisJsonSerializer; private final LabeledManeuverForEstimationJsonSerializer maneuverForEstimationJsonSerializer; private boolean skipRace; - private static final int NUMBER_OF_THREADS = 50; //high number due to HTTP requests - private final ExecutorService executorService = Executors.newFixedThreadPool(NUMBER_OF_THREADS); + private static final int NUMBER_OF_THREADS = 50; // high number due to HTTP requests + private final ThreadPoolExecutor executorService; public ManeuverAndWindImporter() throws UnknownHostException { this.completeManeuverCurvePersistanceManager = new RaceWithCompleteManeuverCurvePersistenceManager(); @@ -85,11 +86,23 @@ public class ManeuverAndWindImporter { this.maneuverForEstimationTransformer = new CompleteManeuverCurveWithEstimationDataToLabelledManeuverForEstimationTransformer(); this.maneuverForDataAnalysisJsonSerializer = new ManeuverForDataAnalysisJsonSerializer(); this.maneuverForEstimationJsonSerializer = new LabeledManeuverForEstimationJsonSerializer(); + this.executorService = new ThreadPoolExecutor(NUMBER_OF_THREADS, NUMBER_OF_THREADS, 0L, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(NUMBER_OF_THREADS)); + this.executorService.setRejectedExecutionHandler(new RejectedExecutionHandler() { + public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { + // this will block if the queue is full + try { + executor.getQueue().put(r); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + }); } public HttpClient createNewHttpClient() { HttpParams httpParams = new BasicHttpParams(); - HttpConnectionParams.setConnectionTimeout(httpParams, 30000); + HttpConnectionParams.setConnectionTimeout(httpParams, 60000 * 5); HttpClient client = new SystemDefaultHttpClient(httpParams); return client; } @@ -312,7 +325,19 @@ public class ManeuverAndWindImporter { for (int i = 1; i <= 10; i++) { try { httpResponse = createNewHttpClient().execute(getEstimationData); - break; + if (httpResponse.getStatusLine().getStatusCode() == 200) { + JSONObject resultJson; + try { + resultJson = (JSONObject) getJsonFromResponse(httpResponse); + return resultJson; + } catch (Exception e) { + System.out.println(getEstimationData); + throw e; + } + } else { + LoggingUtil.logInfo("Connection error (" + i + "/10) while querying races of regatta \"" + + trackedRegattaName + "\" (status code not 200), retrying..."); + } } catch (Exception e) { Thread.sleep(10000); lastException = e; @@ -325,17 +350,7 @@ public class ManeuverAndWindImporter { } } } - if (httpResponse == null) { - throw lastException; - } - JSONObject resultJson; - try { - resultJson = (JSONObject) getJsonFromResponse(httpResponse); - } catch (Exception e) { - System.out.println(getEstimationData); - throw e; - } - return resultJson; + throw lastException; } private void addTransformedElementsToCompetitorTrackJson(List competitorTracks, @@ -358,7 +373,6 @@ public class ManeuverAndWindImporter { } } - // FIXME duplicated code taken from ConnectivityUtils public static Object getJsonFromResponse(HttpResponse response) throws IllegalStateException, IOException, ParseException { JSONParser jsonParser = new JSONParser(); diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/evaluation/WindEstimatorManeuverNumberDependentEvaluationRunner.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/evaluation/WindEstimatorManeuverNumberDependentEvaluationRunner.java index c5f4869cf9c..7815dd9540a 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/evaluation/WindEstimatorManeuverNumberDependentEvaluationRunner.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/evaluation/WindEstimatorManeuverNumberDependentEvaluationRunner.java @@ -12,8 +12,8 @@ import com.sap.sailing.windestimation.data.persistence.maneuver.PersistedElement import com.sap.sailing.windestimation.data.persistence.maneuver.RaceWithCompleteManeuverCurvePersistenceManager; import com.sap.sailing.windestimation.data.persistence.polars.PolarDataServiceAccessUtil; import com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverFeatures; -import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl; import com.sap.sailing.windestimation.model.store.ModelStore; +import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl; import com.sap.sailing.windestimation.util.LoggingUtil; public class WindEstimatorManeuverNumberDependentEvaluationRunner { @@ -38,18 +38,15 @@ public class WindEstimatorManeuverNumberDependentEvaluationRunner { LoggingUtil.logInfo("Loading polar data"); PolarDataService polarService = PolarDataServiceAccessUtil.getPersistedPolarService(); LoggingUtil.logInfo("Wind estimator evaluation started..."); - ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models"); - // ModelStore modelStore = new MongoDbModelStoreImpl(persistenceManager.getDb()); + ModelStore modelStore = new MongoDbModelStoreImpl(persistenceManager.getDb()); WindEstimatorFactories estimatorFactories = new WindEstimatorFactories(polarService, new ManeuverFeatures(ENABLE_POLARS, ENABLE_SCALED_SPEED, ENABLE_MARKS_INFORMATION), modelStore); - double[] avgErrorDegreesPerManeuverCount = new double[MAX_MANEUVERS]; double[] avgConfidencePerManeuverCount = new double[MAX_MANEUVERS]; double[] avgConfidenceOfCorrectEstimationsPerManeuverCount = new double[MAX_MANEUVERS]; double[] avgConfidenceOfIncorrectEstimationsPerManeuverCount = new double[MAX_MANEUVERS]; double[] accuracyPerManeuverCount = new double[MAX_MANEUVERS]; double[] emptyEstimationsPercentagePerManeuverCount = new double[MAX_MANEUVERS]; - for (int fixedNumberOfManeuvers = 1; fixedNumberOfManeuvers <= MAX_MANEUVERS; fixedNumberOfManeuvers++) { LoggingUtil.logInfo("Running evaluation with " + fixedNumberOfManeuvers + " maneuvers"); WindEstimationEvaluator evaluator = new WindEstimationEvaluatorImpl<>( diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/ExportedModelsGenerator.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/ExportedModelsGenerator.java similarity index 98% rename from java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/ExportedModelsGenerator.java rename to java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/ExportedModelsGenerator.java index 9b3d0f5621f..db0ffeb05ee 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/ExportedModelsGenerator.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/ExportedModelsGenerator.java @@ -1,4 +1,4 @@ -package com.sap.sailing.windestimation; +package com.sap.sailing.windestimation.model; import java.io.File; import java.io.FileNotFoundException; diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/SimpleModelsTraining.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java similarity index 97% rename from java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/SimpleModelsTraining.java rename to java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java index c15c543327a..85be413dac7 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/SimpleModelsTraining.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java @@ -1,4 +1,4 @@ -package com.sap.sailing.windestimation; +package com.sap.sailing.windestimation.model; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -30,7 +30,8 @@ import com.sap.sailing.windestimation.util.LoggingUtil; */ public class SimpleModelsTraining { - private static final ExecutorService executorService = Executors.newFixedThreadPool(3); + private static final int NUMBER_OF_THREADS = 2; + private static final ExecutorService executorService = Executors.newFixedThreadPool(NUMBER_OF_THREADS); public static void main(String[] args) throws Exception { new ManeuverForEstimationPersistenceManager().dropCollection(); @@ -88,7 +89,7 @@ public class SimpleModelsTraining { + " dimension. Remove instances from MongoDB collection \"" + dimension.getCollectioName() + "\" which do not make sense. E.g. values which are represented by a small number of supporting instances (see histogram), values which cause implausible zig zag sections within zero-mean standard deviation curve and etc. Close the graphical tool, when you are done to resume model training."); } - + private static void awaitThreadPoolCompletion() throws InterruptedException { executorService.awaitTermination(24, TimeUnit.HOURS); Thread.sleep(1000L); diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/ManeuverClassifierTrainer.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/ManeuverClassifierTrainer.java index cdbbc86ff6e..ed62529519d 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/ManeuverClassifierTrainer.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/ManeuverClassifierTrainer.java @@ -19,9 +19,9 @@ import com.sap.sailing.windestimation.data.persistence.maneuver.TransformedManeu import com.sap.sailing.windestimation.data.persistence.polars.PolarDataServiceAccessUtil; import com.sap.sailing.windestimation.model.classifier.LabelExtraction; import com.sap.sailing.windestimation.model.classifier.TrainableClassificationModel; -import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl; import com.sap.sailing.windestimation.model.store.ModelDomainType; import com.sap.sailing.windestimation.model.store.ModelStore; +import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl; import com.sap.sailing.windestimation.util.LoggingUtil; import com.sap.sse.common.Util.Pair; @@ -112,8 +112,7 @@ public class ManeuverClassifierTrainer { public static void main(String[] args) throws Exception { PolarDataService polarService = PolarDataServiceAccessUtil.getPersistedPolarService(); RegularManeuversForEstimationPersistenceManager persistenceManager = new RegularManeuversForEstimationPersistenceManager(); - ModelStore classifierModelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models"); - // ModelStore classifierModelStore = new MongoDbModelStore(persistenceManager.getDb()); + ModelStore classifierModelStore = new MongoDbModelStoreImpl(persistenceManager.getDb()); classifierModelStore.deleteAll(ModelDomainType.MANEUVER_CLASSIFIER); ManeuverClassifierTrainer classifierTrainer = new ManeuverClassifierTrainer(persistenceManager, classifierModelStore); diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/PersistedManeuverClassifiersScorePrinter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/PersistedManeuverClassifiersScorePrinter.java index df007fc823e..5e857143c59 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/PersistedManeuverClassifiersScorePrinter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/PersistedManeuverClassifiersScorePrinter.java @@ -13,11 +13,12 @@ import java.util.regex.Pattern; import com.sap.sailing.domain.base.BoatClass; import com.sap.sailing.domain.polars.PolarDataService; import com.sap.sailing.windestimation.data.ManeuverForEstimation; +import com.sap.sailing.windestimation.data.persistence.maneuver.RegularManeuversForEstimationPersistenceManager; import com.sap.sailing.windestimation.data.persistence.polars.PolarDataServiceAccessUtil; import com.sap.sailing.windestimation.model.classifier.TrainableClassificationModel; import com.sap.sailing.windestimation.model.exception.ModelPersistenceException; -import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl; import com.sap.sailing.windestimation.model.store.ModelStore; +import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl; import com.sap.sailing.windestimation.util.LoggingUtil; public class PersistedManeuverClassifiersScorePrinter { @@ -29,9 +30,8 @@ public class PersistedManeuverClassifiersScorePrinter { throws MalformedURLException, ClassNotFoundException, IOException, InterruptedException { PolarDataService polarService = PolarDataServiceAccessUtil.getPersistedPolarService(); Set allBoatClasses = polarService.getAllBoatClassesWithPolarSheetsAvailable(); - ModelStore classifierModelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models"); - // ModelStore classifierModelStore = new MongoDbModelStore( - // new RegularManeuversForEstimationPersistenceManager().getDb()); + ModelStore classifierModelStore = new MongoDbModelStoreImpl( + new RegularManeuversForEstimationPersistenceManager().getDb()); List> allClassifierModels = new ArrayList<>(); ManeuverClassifierModelFactory classifierModelFactory = new ManeuverClassifierModelFactory(); LoggingUtil.logInfo("### Loading all boat class classifiers:"); diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DistanceBasedTwdTransitionStdRegressorTrainer.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DistanceBasedTwdTransitionStdRegressorTrainer.java index 9aba743456c..32bbd98b57f 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DistanceBasedTwdTransitionStdRegressorTrainer.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DistanceBasedTwdTransitionStdRegressorTrainer.java @@ -5,9 +5,9 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedSingleDimensionBasedTwdTransitionPersistenceManager.AggregatedSingleDimensionType; import com.sap.sailing.windestimation.model.regressor.IncrementalSingleDimensionPolynomialRegressor; import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext.DistanceValueRange; -import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl; import com.sap.sailing.windestimation.model.store.ModelDomainType; import com.sap.sailing.windestimation.model.store.ModelStore; +import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl; /** * Trains TWD delta standard deviation by considering the distance passed between two measurements. @@ -26,8 +26,7 @@ public class DistanceBasedTwdTransitionStdRegressorTrainer extends TwdTransition public static void main(String[] args) throws Exception { AggregatedSingleDimensionBasedTwdTransitionPersistenceManager distanceBasedPersistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager( AggregatedSingleDimensionType.DISTANCE); - ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models"); - // ModelStore modelStore = new MongoDbModelStore(distanceBasedPersistenceManager.getDb()); + ModelStore modelStore = new MongoDbModelStoreImpl(distanceBasedPersistenceManager.getDb()); modelStore.deleteAll(ModelDomainType.DISTANCE_BASED_TWD_DELTA_STD_REGRESSOR); DistanceBasedTwdTransitionRegressorModelFactory distanceBasedTwdTransitionRegressorModelFactory = new DistanceBasedTwdTransitionRegressorModelFactory(); for (DistanceValueRange distanceValueRange : DistanceValueRange.values()) { diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DurationBasedTwdTransitionStdRegressorTrainer.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DurationBasedTwdTransitionStdRegressorTrainer.java index febdea8a634..d092edd36b0 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DurationBasedTwdTransitionStdRegressorTrainer.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DurationBasedTwdTransitionStdRegressorTrainer.java @@ -5,9 +5,9 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedSingleDimensionBasedTwdTransitionPersistenceManager.AggregatedSingleDimensionType; import com.sap.sailing.windestimation.model.regressor.IncrementalSingleDimensionPolynomialRegressor; import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext.DurationValueRange; -import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl; import com.sap.sailing.windestimation.model.store.ModelDomainType; import com.sap.sailing.windestimation.model.store.ModelStore; +import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl; public class DurationBasedTwdTransitionStdRegressorTrainer extends TwdTransitionAggregatedStdRegressorTrainer { @@ -20,8 +20,7 @@ public class DurationBasedTwdTransitionStdRegressorTrainer extends TwdTransition public static void main(String[] args) throws Exception { AggregatedSingleDimensionBasedTwdTransitionPersistenceManager durationBasedPersistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager( AggregatedSingleDimensionType.DURATION); - ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models"); - // ModelStore modelStore = new MongoDbModelStoreImpl(durationBasedPersistenceManager.getDb()); + ModelStore modelStore = new MongoDbModelStoreImpl(durationBasedPersistenceManager.getDb()); modelStore.deleteAll(ModelDomainType.DURATION_BASED_TWD_DELTA_STD_REGRESSOR); DurationBasedTwdTransitionRegressorModelFactory durationBasedTwdTransitionRegressorModelFactory = new DurationBasedTwdTransitionRegressorModelFactory(); for (DurationValueRange durationValueRange : DurationValueRange.values()) { From 504ec826398d44f501457299f9d40fba33763bfa Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Mon, 18 Feb 2019 18:48:38 +0100 Subject: [PATCH 09/24] Added run config for SimpleModelsTraining --- .../SimpleModelsTraining.launch | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 java/com.sap.sailing.windestimation.lab/SimpleModelsTraining.launch diff --git a/java/com.sap.sailing.windestimation.lab/SimpleModelsTraining.launch b/java/com.sap.sailing.windestimation.lab/SimpleModelsTraining.launch new file mode 100644 index 00000000000..69cd2a60688 --- /dev/null +++ b/java/com.sap.sailing.windestimation.lab/SimpleModelsTraining.launch @@ -0,0 +1,13 @@ + + + + + + + + + + + + + From b3aa09cdb9417f6763079af9a1d57430bf07b297 Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Mon, 18 Feb 2019 19:06:26 +0100 Subject: [PATCH 10/24] Added ReadMe for patched smile jars --- java/com.sap.sailing.windestimation/lib/README | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 java/com.sap.sailing.windestimation/lib/README diff --git a/java/com.sap.sailing.windestimation/lib/README b/java/com.sap.sailing.windestimation/lib/README new file mode 100644 index 00000000000..9ada6a11b62 --- /dev/null +++ b/java/com.sap.sailing.windestimation/lib/README @@ -0,0 +1,9 @@ +The library smile-score-1.5.2.jar has been patched!!! +The class files smile.feature.FeatureTransform and smile.feature.Standardizer +were removed from jar in order to overwrite those classes and make them Serializable +(see com.sap.sailing.windestimation/src/smile/feature/). The removal was necessary +due to OSGI environment. The next release version of SMILE will already include +Serialization support for the aforementioned classes, which means that the patch will +be obsolete. When 1.5.3 comes out, replace the 1.5.2 jars with the official 1.5.3 jars. +Furthermore, delete com.sap.sailing.windestimation/src/smile folder with patched +source files. \ No newline at end of file From fe0c08d34981be3171f69967d84290a21882e28c Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Mon, 18 Feb 2019 19:28:09 +0100 Subject: [PATCH 11/24] Improved verbosity and error handling of ManeuverAndWindImporter --- .../importer/ManeuverAndWindImporter.java | 53 +++++++++++-------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java index a2fabf42a8b..e7e13e9e252 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java @@ -165,7 +165,6 @@ public class ManeuverAndWindImporter { private void importRegatta(String regattaName, ImportStatistics importStatistics) throws IllegalStateException, ClientProtocolException, IOException, ParseException, URISyntaxException { String encodedRegattaName = encodeUrlPathPart(regattaName); - // TODO The encoding of some regatta names with special characters does not work well HttpGet getRegatta = new HttpGet(REST_API_BASE_URL + REST_API_REGATTAS_PATH + "/" + encodedRegattaName); JSONObject regattaJson = null; try { @@ -174,7 +173,7 @@ public class ManeuverAndWindImporter { synchronized (importStatistics) { importStatistics.ignoredRegattas++; } - LoggingUtil.logInfo("Error while processing regatta: " + regattaName); + LoggingUtil.logInfo("Error while processing regatta: " + regattaName + "\r\n" + getRegatta); return; } for (Object seriesJson : (JSONArray) regattaJson.get("series")) { @@ -199,8 +198,12 @@ public class ManeuverAndWindImporter { synchronized (importStatistics) { importStatistics.ingoredRaces += 1; } + String extraLog = ""; + if (e instanceof HttpClientException) { + extraLog = "\r\n" + ((HttpClientException) e).getRequest(); + } LoggingUtil.logInfo("Error while processing race nr. " + raceNumber + ": \"" - + trackedRaceName + "\""); + + trackedRaceName + "\"" + extraLog); } }); } @@ -251,13 +254,14 @@ public class ManeuverAndWindImporter { JSONArray windFixesJson = (JSONArray) ((JSONObject) windSourceObj).get(RaceWindJsonSerializer.FIXES); windFixesCount += windFixesJson.size(); } - LoggingUtil.logInfo( - "Imported " + windFixesCount + " wind fixes from " + windSourcesJson.size() + " wind sources"); + LoggingUtil.logInfo("Imported " + windFixesCount + " wind fixes from " + windSourcesJson.size() + + " wind sources in race \"" + trackedRaceName + "\" of regatta \"" + regattaName + "\""); synchronized (importStatistics) { importStatistics.racesWithHighQualityWindData++; } } else { - LoggingUtil.logInfo("No high quality wind fixes contained"); + LoggingUtil.logInfo("No high quality wind fixes contained in race \"" + trackedRaceName + "\" of regatta \"" + + regattaName + "\""); } } @@ -310,8 +314,8 @@ public class ManeuverAndWindImporter { } catch (Exception e) { e.printStackTrace(); } - LoggingUtil.logInfo( - "Imported " + competitorTracks.size() + " competitor tracks with " + maneuversCount + " maneuvers"); + LoggingUtil.logInfo("Imported " + competitorTracks.size() + " competitor tracks with " + maneuversCount + + " maneuvers in race \"" + trackedRaceName + "\" of regatta \"" + regattaName + "\""); synchronized (importStatistics) { importStatistics.competitorTracksCount += competitorTracks.size(); importStatistics.maneuversCount += maneuversCount; @@ -325,19 +329,8 @@ public class ManeuverAndWindImporter { for (int i = 1; i <= 10; i++) { try { httpResponse = createNewHttpClient().execute(getEstimationData); - if (httpResponse.getStatusLine().getStatusCode() == 200) { - JSONObject resultJson; - try { - resultJson = (JSONObject) getJsonFromResponse(httpResponse); - return resultJson; - } catch (Exception e) { - System.out.println(getEstimationData); - throw e; - } - } else { - LoggingUtil.logInfo("Connection error (" + i + "/10) while querying races of regatta \"" - + trackedRegattaName + "\" (status code not 200), retrying..."); - } + JSONObject resultJson = (JSONObject) getJsonFromResponse(httpResponse); + return resultJson; } catch (Exception e) { Thread.sleep(10000); lastException = e; @@ -350,7 +343,7 @@ public class ManeuverAndWindImporter { } } } - throw lastException; + throw new HttpClientException(getEstimationData.toString(), lastException); } private void addTransformedElementsToCompetitorTrackJson(List competitorTracks, @@ -399,4 +392,20 @@ public class ManeuverAndWindImporter { private int racesWithHighQualityWindData = 0; } + private static class HttpClientException extends Exception { + + private static final long serialVersionUID = 4948532287832868768L; + private final String request; + + public HttpClientException(String request, Exception e) { + super(e); + this.request = request; + } + + public String getRequest() { + return request; + } + + } + } From c72875cc0a10af1e5a143c42c77dfe76923f647f Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Mon, 18 Feb 2019 19:32:43 +0100 Subject: [PATCH 12/24] Improved javadoc --- .../integration/WindEstimationModelsUpdateOperation.java | 2 +- .../WindEstimationModelsUpdateOperationImpl.java | 6 ++++++ .../jaxrs/api/WindEstimationDataResource.java | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperation.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperation.java index c2853417f30..c0f75bb2b5b 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperation.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperation.java @@ -4,7 +4,7 @@ import com.sap.sse.replication.OperationWithResult; /** * - * Needed in the context of the initial replication of the wind estimation factory service. + * Updates wind estimation models. * * @author Vladislav Chumak (D069712) * diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperationImpl.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperationImpl.java index f8de7330282..555b187f495 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperationImpl.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperationImpl.java @@ -1,5 +1,11 @@ package com.sap.sailing.windestimation.integration; +/** + * Updates wind estimation models with provided exported models. + * + * @author Vladislav Chumak (D069712) + * + */ public class WindEstimationModelsUpdateOperationImpl implements WindEstimationModelsUpdateOperation { private static final long serialVersionUID = 6407877837605630090L; diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResource.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResource.java index 849ff4eef77..67405777eba 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResource.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResource.java @@ -22,6 +22,7 @@ import com.sap.sailing.windestimation.integration.WindEstimationModelsUpdateOper import com.sap.sailing.windestimation.jaxrs.AbstractWindEstimationDataResource; /** + * Resource which manages wind estimation model metadata. Data export and import are supported. * * @author Vladislav Chumak (D069712) * From ddf28ac4490e5315fedba9f642dfe0a8f983a0c7 Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Tue, 19 Feb 2019 20:47:12 +0100 Subject: [PATCH 13/24] Bugfixes and enhancements for model management via POST/GET/Replica --- .../domain/test/mock/MockedTrackedRace.java | 4 +- ...ockedTrackedRaceWithStartTimeAndRanks.java | 4 +- .../domain/tracking/DummyTrackedRace.java | 4 +- .../sailing/domain/tracking/TrackedRace.java | 4 +- .../TrackedRaceWithWindEssentials.java | 42 ++++++++++- .../domain/tracking/impl/TrackedRaceImpl.java | 39 ++++++---- ...ck.java => IncrementalWindEstimation.java} | 9 ++- ...intAndPositionWithToleranceComparator.java | 6 +- .../WindEstimationFactoryService.java | 2 +- ...TrackWithConfidenceForEachWindFixImpl.java | 48 ++++++++++++ .../gwt/ui/test/MockedTrackedRace.java | 4 +- ...ty on 8889, Polar Import from 8888).launch | 58 +++++++-------- .../server/impl/RacingEventServiceImpl.java | 9 ++- .../importer/ManeuverAndWindImporter.java | 9 ++- .../model/SimpleModelsTraining.java | 25 ++++--- ...stHmmWindEstimationForTrackedRaceTest.java | 1 + ...rementalMstManeuverGraphGeneratorTest.java | 1 + .../com.sap.sailing.windestimation/.classpath | 1 + .../META-INF/MANIFEST.MF | 3 +- .../resources/shiro.ini | 74 +++++++++++++++++++ ...talMstHmmWindEstimationForTrackedRace.java | 34 ++++----- ...eplicableWindEstimationFactoryService.java | 5 +- .../WindEstimationFactoryServiceImpl.java | 11 ++- .../WindEstimationModelsUpdateOperation.java | 21 +++++- ...ndEstimationModelsUpdateOperationImpl.java | 25 ------- .../jaxrs/api/WindEstimationDataResource.java | 3 +- 26 files changed, 318 insertions(+), 128 deletions(-) rename java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/{IncrementalWindEstimationTrack.java => IncrementalWindEstimation.java} (76%) rename java/{com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration => com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation}/TimePointAndPositionWithToleranceComparator.java (85%) create mode 100644 java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/WindTrackWithConfidenceForEachWindFixImpl.java create mode 100644 java/com.sap.sailing.windestimation/resources/shiro.ini delete mode 100644 java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperationImpl.java diff --git a/java/com.sap.sailing.domain.test/src/com/sap/sailing/domain/test/mock/MockedTrackedRace.java b/java/com.sap.sailing.domain.test/src/com/sap/sailing/domain/test/mock/MockedTrackedRace.java index a8eb8868a93..ae275abbc5b 100644 --- a/java/com.sap.sailing.domain.test/src/com/sap/sailing/domain/test/mock/MockedTrackedRace.java +++ b/java/com.sap.sailing.domain.test/src/com/sap/sailing/domain/test/mock/MockedTrackedRace.java @@ -83,7 +83,7 @@ import com.sap.sailing.domain.tracking.WindStore; import com.sap.sailing.domain.tracking.WindTrack; import com.sap.sailing.domain.tracking.WindWithConfidence; import com.sap.sailing.domain.tracking.impl.WindTrackImpl; -import com.sap.sailing.domain.windestimation.IncrementalWindEstimationTrack; +import com.sap.sailing.domain.windestimation.IncrementalWindEstimation; import com.sap.sse.common.Bearing; import com.sap.sse.common.Distance; import com.sap.sse.common.Duration; @@ -1235,7 +1235,7 @@ public class MockedTrackedRace implements DynamicTrackedRace { } @Override - public void setWindEstimation(IncrementalWindEstimationTrack windEstimation) { + public void setWindEstimation(IncrementalWindEstimation windEstimation) { } } diff --git a/java/com.sap.sailing.domain.test/src/com/sap/sailing/domain/test/mock/MockedTrackedRaceWithStartTimeAndRanks.java b/java/com.sap.sailing.domain.test/src/com/sap/sailing/domain/test/mock/MockedTrackedRaceWithStartTimeAndRanks.java index 823cab82297..d46099dd925 100755 --- a/java/com.sap.sailing.domain.test/src/com/sap/sailing/domain/test/mock/MockedTrackedRaceWithStartTimeAndRanks.java +++ b/java/com.sap.sailing.domain.test/src/com/sap/sailing/domain/test/mock/MockedTrackedRaceWithStartTimeAndRanks.java @@ -64,7 +64,7 @@ import com.sap.sailing.domain.tracking.WindStore; import com.sap.sailing.domain.tracking.WindTrack; import com.sap.sailing.domain.tracking.WindWithConfidence; import com.sap.sailing.domain.tracking.impl.DynamicTrackedRegattaImpl; -import com.sap.sailing.domain.windestimation.IncrementalWindEstimationTrack; +import com.sap.sailing.domain.windestimation.IncrementalWindEstimation; import com.sap.sse.common.Bearing; import com.sap.sse.common.Distance; import com.sap.sse.common.Duration; @@ -783,7 +783,7 @@ public class MockedTrackedRaceWithStartTimeAndRanks implements TrackedRace { } @Override - public void setWindEstimation(IncrementalWindEstimationTrack windEstimation) { + public void setWindEstimation(IncrementalWindEstimation windEstimation) { } } diff --git a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/DummyTrackedRace.java b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/DummyTrackedRace.java index 34e1acd5a4f..d6c59d2b5ad 100755 --- a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/DummyTrackedRace.java +++ b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/DummyTrackedRace.java @@ -38,7 +38,7 @@ import com.sap.sailing.domain.ranking.RankingMetric; import com.sap.sailing.domain.ranking.RankingMetric.RankingInfo; import com.sap.sailing.domain.tracking.impl.DynamicTrackedRegattaImpl; import com.sap.sailing.domain.tracking.impl.EmptyWindStore; -import com.sap.sailing.domain.windestimation.IncrementalWindEstimationTrack; +import com.sap.sailing.domain.windestimation.IncrementalWindEstimation; import com.sap.sse.common.Bearing; import com.sap.sse.common.Distance; import com.sap.sse.common.Duration; @@ -722,7 +722,7 @@ public class DummyTrackedRace extends TrackedRaceWithWindEssentials { } @Override - public void setWindEstimation(IncrementalWindEstimationTrack windEstimation) { + public void setWindEstimation(IncrementalWindEstimation windEstimation) { } } diff --git a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/TrackedRace.java b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/TrackedRace.java index 0f6ce033157..cccc955eaca 100644 --- a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/TrackedRace.java +++ b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/TrackedRace.java @@ -53,7 +53,7 @@ import com.sap.sailing.domain.ranking.RankingMetric; import com.sap.sailing.domain.ranking.RankingMetric.RankingInfo; import com.sap.sailing.domain.tracking.impl.NonCachingMarkPositionAtTimePointCache; import com.sap.sailing.domain.tracking.impl.TrackedRaceImpl; -import com.sap.sailing.domain.windestimation.IncrementalWindEstimationTrack; +import com.sap.sailing.domain.windestimation.IncrementalWindEstimation; import com.sap.sse.common.Bearing; import com.sap.sse.common.Distance; import com.sap.sse.common.Duration; @@ -1152,6 +1152,6 @@ public interface TrackedRace extends Serializable, IsManagedByCache windTracks; + protected transient ConcurrentMap windTracks; protected final RaceDefinition race; @@ -185,4 +192,37 @@ public abstract class TrackedRaceWithWindEssentials implements TrackedRace { protected NamedReentrantReadWriteLock getSerializationLock() { return serializationLock; } + + /** + * For serialization, maneuver based estimation wind track should be removed because its replication is redundant. + * The wind estimation track is calculated automatically when maneuver cache recalculation is triggered. Due to the + * custom deserialization of maneuver cache, this is always the case, also if replication is performed. Furthermore, + * it might introduce issues in context of already running wind estimation. + */ + private void writeObject(ObjectOutputStream s) throws IOException { + LockUtil.lockForWrite(getSerializationLock()); + try { + s.defaultWriteObject(); + Map windTracksToSerialize = new HashMap<>(); + synchronized (this.windTracks) { + for (Entry entry : windTracks.entrySet()) { + if (entry.getKey().getType() != WindSourceType.MANEUVER_BASED_ESTIMATION) { + windTracksToSerialize.put(entry.getKey(), entry.getValue()); + } + } + } + s.writeObject(windTracksToSerialize); + } finally { + LockUtil.unlockAfterWrite(getSerializationLock()); + } + } + + private void readObject(ObjectInputStream ois) throws ClassNotFoundException, IOException, PatchFailedException { + ois.defaultReadObject(); + @SuppressWarnings("unchecked") + Map windTracks = (Map) ois.readObject(); + this.windTracks = new ConcurrentHashMap<>(windTracks); + } + + } diff --git a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/impl/TrackedRaceImpl.java b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/impl/TrackedRaceImpl.java index b653199541e..387875cfb5d 100755 --- a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/impl/TrackedRaceImpl.java +++ b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/impl/TrackedRaceImpl.java @@ -146,7 +146,7 @@ import com.sap.sailing.domain.tracking.WindPositionMode; import com.sap.sailing.domain.tracking.WindStore; import com.sap.sailing.domain.tracking.WindTrack; import com.sap.sailing.domain.tracking.WindWithConfidence; -import com.sap.sailing.domain.windestimation.IncrementalWindEstimationTrack; +import com.sap.sailing.domain.windestimation.IncrementalWindEstimation; import com.sap.sse.common.Bearing; import com.sap.sse.common.Distance; import com.sap.sse.common.Duration; @@ -393,7 +393,7 @@ public abstract class TrackedRaceImpl extends TrackedRaceWithWindEssentials impl private transient PolarDataService polarDataService; - private transient volatile IncrementalWindEstimationTrack windEstimation; + private transient volatile IncrementalWindEstimation windEstimation; private transient ShortTimeAfterLastHitCache maneuverDetectorPerCompetitorCache; @@ -3611,27 +3611,34 @@ public abstract class TrackedRaceImpl extends TrackedRaceWithWindEssentials impl @Override public void setPolarDataService(PolarDataService polarDataService) { this.polarDataService = polarDataService; + if(polarDataService != null && windEstimation != null) { + updateManeuversAndWindWithNewWindEstimation(windEstimation, windEstimation); + } } @Override - public void setWindEstimation(IncrementalWindEstimationTrack windEstimation) { - IncrementalWindEstimationTrack previousWindEstimation = this.windEstimation; + public void setWindEstimation(IncrementalWindEstimation windEstimation) { + IncrementalWindEstimation previousWindEstimation = this.windEstimation; if (previousWindEstimation != windEstimation) { - if (previousWindEstimation != null) { - windTracks.remove(previousWindEstimation.getWindSource()); - } - if (windEstimation != null) { - windTracks.put(windEstimation.getWindSource(), windEstimation); - } - this.windEstimation = windEstimation; - // TODO Make more efficient by reusing the state of incremental maneuver detectors. The already computed - // complete maneuver curves can be fed directly to the windEstimation. - maneuverDetectorPerCompetitorCache.clearCache(); - shortTimeWindCache.clearCache(); - triggerManeuverCacheRecalculationForAllCompetitors(); + updateManeuversAndWindWithNewWindEstimation(windEstimation, previousWindEstimation); } } + private void updateManeuversAndWindWithNewWindEstimation(IncrementalWindEstimation windEstimation, + IncrementalWindEstimation previousWindEstimation) { + WindSource windSource = new WindSourceImpl(WindSourceType.MANEUVER_BASED_ESTIMATION); + windTracks.remove(windSource); + if (windEstimation != null) { + windTracks.put(windSource, windEstimation.getWindTrack()); + } + this.windEstimation = windEstimation; + // TODO Make more efficient by reusing the state of incremental maneuver detectors. The already computed + // complete maneuver curves can be fed directly into the windEstimation. + maneuverDetectorPerCompetitorCache.clearCache(); + shortTimeWindCache.clearCache(); + triggerManeuverCacheRecalculationForAllCompetitors(); + } + /** * Obtains the {@link #raceLogResolver}. */ diff --git a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/IncrementalWindEstimationTrack.java b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/IncrementalWindEstimation.java similarity index 76% rename from java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/IncrementalWindEstimationTrack.java rename to java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/IncrementalWindEstimation.java index 80b281cecd2..2127544d68a 100644 --- a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/IncrementalWindEstimationTrack.java +++ b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/IncrementalWindEstimation.java @@ -6,7 +6,7 @@ import com.sap.sailing.domain.tracking.WindTrack; /** * Wind estimator which interacts with maneuver detector and maintains a wind track with estimated wind fixes. All in - * all, this instance can be seen as a wind source with corresponding wind track. The underlying wind track is prone to + * all, this instance can be seen as a wind source with corresponding wind track. The managed wind track is prone to * changes which are communicated to tracked race via its * {@link TrackedRace#recordWind(com.sap.sailing.domain.common.Wind, WindSource, boolean)} and * {@link TrackedRace#removeWind(com.sap.sailing.domain.common.Wind, WindSource)}. @@ -14,11 +14,16 @@ import com.sap.sailing.domain.tracking.WindTrack; * @author Vladislav Chumak (D069712) * */ -public interface IncrementalWindEstimationTrack extends WindTrack, WindEstimationInteraction { +public interface IncrementalWindEstimation extends WindEstimationInteraction { /** * @return The wind source assigned to this wind track */ WindSource getWindSource(); + /** + * Gets the produced wind track of this wind estimation + */ + WindTrack getWindTrack(); + } diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/TimePointAndPositionWithToleranceComparator.java b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/TimePointAndPositionWithToleranceComparator.java similarity index 85% rename from java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/TimePointAndPositionWithToleranceComparator.java rename to java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/TimePointAndPositionWithToleranceComparator.java index 6e7596cdcbe..a075257a8ee 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/TimePointAndPositionWithToleranceComparator.java +++ b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/TimePointAndPositionWithToleranceComparator.java @@ -1,5 +1,6 @@ -package com.sap.sailing.windestimation.integration; +package com.sap.sailing.domain.windestimation; +import java.io.Serializable; import java.util.Comparator; import com.sap.sailing.domain.common.Position; @@ -13,8 +14,9 @@ import com.sap.sse.common.Util.Pair; * @author Vladislav Chumak (D069712) * */ -public class TimePointAndPositionWithToleranceComparator implements Comparator> { +public class TimePointAndPositionWithToleranceComparator implements Comparator>, Serializable { + private static final long serialVersionUID = -3240742353261384636L; private static final double DOUBLE_TOLERANCE = 0.0000001; @Override diff --git a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/WindEstimationFactoryService.java b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/WindEstimationFactoryService.java index 62d0336e250..2d3d3ffdabc 100644 --- a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/WindEstimationFactoryService.java +++ b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/WindEstimationFactoryService.java @@ -31,7 +31,7 @@ public interface WindEstimationFactoryService { * @see #isReady() * @see #addWindEstimationModelsChangedListenerAndReceiveUpdate(WindEstimationModelsChangedListener) */ - IncrementalWindEstimationTrack createIncrementalWindEstimationTrack(TrackedRace trackedRace); + IncrementalWindEstimation createIncrementalWindEstimationTrack(TrackedRace trackedRace); /** * Checks whether the wind estimation is ready for its use. The wind estimation maintains multiple internal machine diff --git a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/WindTrackWithConfidenceForEachWindFixImpl.java b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/WindTrackWithConfidenceForEachWindFixImpl.java new file mode 100644 index 00000000000..da5fb01136e --- /dev/null +++ b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/WindTrackWithConfidenceForEachWindFixImpl.java @@ -0,0 +1,48 @@ +package com.sap.sailing.domain.windestimation; + +import java.util.Map; + +import com.sap.sailing.domain.common.Position; +import com.sap.sailing.domain.common.Wind; +import com.sap.sailing.domain.tracking.WindWithConfidence; +import com.sap.sailing.domain.tracking.impl.WindTrackImpl; +import com.sap.sse.common.TimePoint; +import com.sap.sse.common.Util.Pair; + +/** + * Wind track implementation which uses the provided map with {@link WindWithConfidence}-instances to derive the + * confidence for each wind fix within this wind track. + * + * @author Vladislav Chumak (D069712) + * + */ +public class WindTrackWithConfidenceForEachWindFixImpl extends WindTrackImpl { + + private static final long serialVersionUID = 8135949043175649443L; + private final Map, WindWithConfidence>> windTrackWithConfidences; + + public WindTrackWithConfidenceForEachWindFixImpl(long millisecondsOverWhichToAverage, double baseConfidence, + boolean useSpeed, String nameForReadWriteLock, boolean losslessCompaction, + Map, WindWithConfidence>> windTrackWithConfidences) { + super(millisecondsOverWhichToAverage, baseConfidence, useSpeed, nameForReadWriteLock, losslessCompaction); + this.windTrackWithConfidences = windTrackWithConfidences; + } + + @Override + protected double getConfidenceOfInternalWindFixUnsynchronized(Wind windFix) { + WindWithConfidence> windWithConfidence = windTrackWithConfidences + .get(new Pair<>(windFix.getPosition(), windFix.getTimePoint())); + return super.getConfidenceOfInternalWindFixUnsynchronized(windFix) * windWithConfidence.getConfidence(); + } + + @Override + public void lockForWrite() { + super.lockForWrite(); + } + + @Override + public void unlockAfterWrite() { + super.unlockAfterWrite(); + } + +} diff --git a/java/com.sap.sailing.gwt.ui.test/src/com/sap/sailing/gwt/ui/test/MockedTrackedRace.java b/java/com.sap.sailing.gwt.ui.test/src/com/sap/sailing/gwt/ui/test/MockedTrackedRace.java index 4ed8b77ea22..55eb490c7eb 100644 --- a/java/com.sap.sailing.gwt.ui.test/src/com/sap/sailing/gwt/ui/test/MockedTrackedRace.java +++ b/java/com.sap.sailing.gwt.ui.test/src/com/sap/sailing/gwt/ui/test/MockedTrackedRace.java @@ -71,7 +71,7 @@ import com.sap.sailing.domain.tracking.WindStore; import com.sap.sailing.domain.tracking.WindTrack; import com.sap.sailing.domain.tracking.WindWithConfidence; import com.sap.sailing.domain.tracking.impl.WindTrackImpl; -import com.sap.sailing.domain.windestimation.IncrementalWindEstimationTrack; +import com.sap.sailing.domain.windestimation.IncrementalWindEstimation; import com.sap.sse.common.Bearing; import com.sap.sse.common.Distance; import com.sap.sse.common.Duration; @@ -931,7 +931,7 @@ public class MockedTrackedRace implements DynamicTrackedRace { } @Override - public void setWindEstimation(IncrementalWindEstimationTrack windEstimation) { + public void setWindEstimation(IncrementalWindEstimation windEstimation) { } } diff --git a/java/com.sap.sailing.server/SailingServer (Proxy, Jetty on 8889, Polar Import from 8888).launch b/java/com.sap.sailing.server/SailingServer (Proxy, Jetty on 8889, Polar Import from 8888).launch index 925b9efd2d3..74239ad341d 100755 --- a/java/com.sap.sailing.server/SailingServer (Proxy, Jetty on 8889, Polar Import from 8888).launch +++ b/java/com.sap.sailing.server/SailingServer (Proxy, Jetty on 8889, Polar Import from 8888).launch @@ -1,32 +1,32 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/java/com.sap.sailing.server/src/com/sap/sailing/server/impl/RacingEventServiceImpl.java b/java/com.sap.sailing.server/src/com/sap/sailing/server/impl/RacingEventServiceImpl.java index 5c61c8d2599..44e7a7b2548 100644 --- a/java/com.sap.sailing.server/src/com/sap/sailing/server/impl/RacingEventServiceImpl.java +++ b/java/com.sap.sailing.server/src/com/sap/sailing/server/impl/RacingEventServiceImpl.java @@ -124,6 +124,7 @@ import com.sap.sailing.domain.common.ScoringSchemeType; import com.sap.sailing.domain.common.TrackedRaceStatusEnum; import com.sap.sailing.domain.common.Wind; import com.sap.sailing.domain.common.WindSource; +import com.sap.sailing.domain.common.WindSourceType; import com.sap.sailing.domain.common.dto.AnniversaryType; import com.sap.sailing.domain.common.dto.EventType; import com.sap.sailing.domain.common.dto.FleetDTO; @@ -2078,12 +2079,16 @@ public class RacingEventServiceImpl implements RacingEventService, ClearStateTes @Override public void windDataReceived(Wind wind, WindSource windSource) { - replicate(new RecordWindFix(getRaceIdentifier(), windSource, wind)); + if (windSource.getType() != WindSourceType.MANEUVER_BASED_ESTIMATION) { + replicate(new RecordWindFix(getRaceIdentifier(), windSource, wind)); + } } @Override public void windDataRemoved(Wind wind, WindSource windSource) { - replicate(new RemoveWindFix(getRaceIdentifier(), windSource, wind)); + if (windSource.getType() != WindSourceType.MANEUVER_BASED_ESTIMATION) { + replicate(new RemoveWindFix(getRaceIdentifier(), windSource, wind)); + } } @Override diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java index e7e13e9e252..50aeddb8dab 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java @@ -58,6 +58,7 @@ import com.sap.sailing.windestimation.util.LoggingUtil; */ public class ManeuverAndWindImporter { + private static final int CONNECTION_TIMEOUT_MILLIS = 60000 * 5; public static final String REST_API_BASE_URL = "https://www.sapsailing.com/sailingserver/api/v1"; public static final String REST_API_REGATTAS_PATH = "/regattas"; public static final String REST_API_RACES_PATH = "/races"; @@ -74,7 +75,7 @@ public class ManeuverAndWindImporter { private final ManeuverForDataAnalysisJsonSerializer maneuverForDataAnalysisJsonSerializer; private final LabeledManeuverForEstimationJsonSerializer maneuverForEstimationJsonSerializer; private boolean skipRace; - private static final int NUMBER_OF_THREADS = 50; // high number due to HTTP requests + private static final int NUMBER_OF_THREADS = 10; // high number due to HTTP requests private final ThreadPoolExecutor executorService; public ManeuverAndWindImporter() throws UnknownHostException { @@ -102,8 +103,12 @@ public class ManeuverAndWindImporter { public HttpClient createNewHttpClient() { HttpParams httpParams = new BasicHttpParams(); - HttpConnectionParams.setConnectionTimeout(httpParams, 60000 * 5); + HttpConnectionParams.setConnectionTimeout(httpParams, CONNECTION_TIMEOUT_MILLIS); HttpClient client = new SystemDefaultHttpClient(httpParams); + client.getParams().setParameter("http.socket.timeout", CONNECTION_TIMEOUT_MILLIS); + client.getParams().setParameter("http.connection.timeout", CONNECTION_TIMEOUT_MILLIS); + client.getParams().setParameter("http.connection-manager.timeout", new Long(CONNECTION_TIMEOUT_MILLIS)); + client.getParams().setParameter("http.protocol.head-body-timeout", CONNECTION_TIMEOUT_MILLIS); return client; } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java index 85be413dac7..8aaa2488433 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java @@ -1,7 +1,8 @@ package com.sap.sailing.windestimation.model; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import javax.swing.JOptionPane; @@ -30,14 +31,16 @@ import com.sap.sailing.windestimation.util.LoggingUtil; */ public class SimpleModelsTraining { - private static final int NUMBER_OF_THREADS = 2; - private static final ExecutorService executorService = Executors.newFixedThreadPool(NUMBER_OF_THREADS); + private static final int NUMBER_OF_THREADS = 3; + private static final ExecutorService executorService = new ThreadPoolExecutor(NUMBER_OF_THREADS, NUMBER_OF_THREADS, + 0, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(NUMBER_OF_THREADS), + new ThreadPoolExecutor.CallerRunsPolicy()); public static void main(String[] args) throws Exception { new ManeuverForEstimationPersistenceManager().dropCollection(); new RegularManeuversForEstimationPersistenceManager().dropCollection(); executeInThreadPool(() -> PolarDataImporter.main(args)); - ManeuverAndWindImporter.main(args); + executeInThreadPool(() -> ManeuverAndWindImporter.main(args)); awaitThreadPoolCompletion(); executeInThreadPool(() -> { ManeuverClassifierTrainer.main(args); @@ -49,9 +52,11 @@ public class SimpleModelsTraining { Thread.sleep(1000); AggregatedDurationBasedTwdTransitionImporter.main(args); }); - DistanceBasedTwdTransitionImporter.main(args); - Thread.sleep(1000); - AggregatedDistanceBasedTwdTransitionImporter.main(args); + executeInThreadPool(() -> { + DistanceBasedTwdTransitionImporter.main(args); + Thread.sleep(1000); + AggregatedDistanceBasedTwdTransitionImporter.main(args); + }); awaitThreadPoolCompletion(); do { AggregatedDurationDimensionPlot.main(args); @@ -99,8 +104,10 @@ public class SimpleModelsTraining { executorService.execute(() -> { try { runnable.run(); - } catch (Exception e) { - e.printStackTrace(); + } catch (Throwable t) { + t.printStackTrace(); + System.exit(1); + LoggingUtil.logInfo("FAILURE: Caught unexpected exception. Model training aborted"); } }); } diff --git a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRaceTest.java b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRaceTest.java index 5fd94446243..15b4fd9a1dd 100644 --- a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRaceTest.java +++ b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRaceTest.java @@ -38,6 +38,7 @@ import com.sap.sailing.domain.tracking.WindTrack; import com.sap.sailing.domain.tracking.WindWithConfidence; import com.sap.sailing.domain.tracking.impl.DynamicTrackedRaceImpl; import com.sap.sailing.domain.tractracadapter.ReceiverType; +import com.sap.sailing.domain.windestimation.TimePointAndPositionWithToleranceComparator; import com.sap.sailing.windestimation.ManeuverBasedWindEstimationComponentImpl; import com.sap.sailing.windestimation.aggregator.ManeuverClassificationsAggregatorFactory; import com.sap.sailing.windestimation.data.CompetitorTrackWithEstimationData; diff --git a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstManeuverGraphGeneratorTest.java b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstManeuverGraphGeneratorTest.java index 0424bb9c479..7d7d72f4e6e 100644 --- a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstManeuverGraphGeneratorTest.java +++ b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstManeuverGraphGeneratorTest.java @@ -29,6 +29,7 @@ import com.sap.sailing.domain.test.OnlineTracTracBasedTest; import com.sap.sailing.domain.tracking.CompleteManeuverCurve; import com.sap.sailing.domain.tracking.impl.DynamicTrackedRaceImpl; import com.sap.sailing.domain.tractracadapter.ReceiverType; +import com.sap.sailing.domain.windestimation.TimePointAndPositionWithToleranceComparator; import com.sap.sailing.windestimation.aggregator.msthmm.DistanceAndDurationAwareWindTransitionProbabilitiesCalculator; import com.sap.sailing.windestimation.aggregator.msthmm.MstGraphLevel; import com.sap.sailing.windestimation.aggregator.msthmm.MstManeuverGraphGenerator.MstManeuverGraphComponents; diff --git a/java/com.sap.sailing.windestimation/.classpath b/java/com.sap.sailing.windestimation/.classpath index e3eae022f73..3c7b5c88b1e 100644 --- a/java/com.sap.sailing.windestimation/.classpath +++ b/java/com.sap.sailing.windestimation/.classpath @@ -3,6 +3,7 @@ + diff --git a/java/com.sap.sailing.windestimation/META-INF/MANIFEST.MF b/java/com.sap.sailing.windestimation/META-INF/MANIFEST.MF index bbaf0a88153..176c358b8c3 100644 --- a/java/com.sap.sailing.windestimation/META-INF/MANIFEST.MF +++ b/java/com.sap.sailing.windestimation/META-INF/MANIFEST.MF @@ -44,7 +44,8 @@ Import-Package: javax.ws.rs;version="1.1.1", Web-ContextPath: /windestimation Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Bundle-ActivationPolicy: lazy -Export-Package: com.sap.sailing.windestimation +Export-Package: com.sap.sailing.windestimation, + com.sap.sailing.windestimation.integration Bundle-ClassPath: lib/smile-core-1.5.2.jar, lib/smile-data-1.5.2.jar, lib/smile-graph-1.5.2.jar, diff --git a/java/com.sap.sailing.windestimation/resources/shiro.ini b/java/com.sap.sailing.windestimation/resources/shiro.ini new file mode 100644 index 00000000000..0449ff0fd0a --- /dev/null +++ b/java/com.sap.sailing.windestimation/resources/shiro.ini @@ -0,0 +1,74 @@ + +[main] +credentialsMatcher = org.apache.shiro.authc.credential.Sha256CredentialsMatcher +credentialsMatcher.storedCredentialsHexEncoded = false +credentialsMatcher.hashIterations = 1024 + +# Realm configuration: +# -------------------- + +permissionsForRoleProvider = com.sap.sailing.domain.common.security.SailingPermissionsForRoleProvider +upRealm = com.sap.sse.security.UsernamePasswordRealm +upRealm.credentialsMatcher = $credentialsMatcher +upRealm.permissionsForRoleProvider = $permissionsForRoleProvider +oauthRealm = com.sap.sse.security.OAuthRealm +oauthRealm.permissionsForRoleProvider = $permissionsForRoleProvider +bearerTokenRealm = com.sap.sse.security.BearerTokenRealm +bearerTokenRealm.permissionsForRoleProvider = $permissionsForRoleProvider + +# Putting it all together: +sessionManager = com.sap.sse.security.SecurityWebSessionManager +securityManager.sessionManager = $sessionManager +sessionDAO = org.apache.shiro.session.mgt.eis.EnterpriseCacheSessionDAO +securityManager.sessionManager.sessionDAO = $sessionDAO +subjectDAO = com.sap.sse.security.NoSessionStorageSessionDAO +securityManager.subjectDAO = $subjectDAO +# 1y timeout +securityManager.sessionManager.globalSessionTimeout = 31536000000 +cacheManager = com.sap.sse.security.SessionCacheManager +securityManager.cacheManager = $cacheManager + +# Authentication Filter Configurations +# ------------------------------------ + +# A filter that requires the user to have at least one of the roles specified as +# filter parameter. This makes it different from the default roles[...] filter +# which requires the user to have ALL of the roles specified as parameters. +anyofroles = com.sap.sse.security.AnyOfRolesFilter + +# A custom filter for GWT pages that require an authenticated user. +# Forwards unauthenticated users to the default login page. When +# addressed directly (and not based on a redirect), users who logged +# on successfully will be redirected to the UserManagement.html page +# where they can adjust their profile. +customGwt = com.sap.sse.security.CustomFilter +customGwt.loginUrl = /security/ui/Login.html +customGwt.successUrl = /UserManagement.html + +# Configuration for the default shiro HTTP form authentication filter. +# It assumes that form-based login parameters are sent to the +# /api/restsecurity/login RESTlet and forwards successful login attempts +# to the /api/restsecurity/hello service which is expected to respond +# with a JSON document containing the authenticated subject's properties. +authc.loginUrl = /api/restsecurity/login +authc.successUrl = /api/restsecurity/hello + +# This authentication filter accepts a bearer access token in the HTTP +# Authorization header field, as in +# Authorization: Bearer 1029741026501365024376093245 +# If no such bearer token is provided, the filter falls back to basic HTTP +# authentication ("Authentication: Basic ...") and then regular form-based +# authentication with POST parameters "username" and "password". +bearerToken = com.sap.sse.security.BearerTokenOrBasicOrFormAuthenticationFilter + +# Specifying filter chains for URL patterns + +[urls] +# Uncomment the following to require the permission "event:view" in order to use /api/v1/events: +#/api/v1/events = bearerToken + +# Example: Secure access to leaderboard race column factor updates; read access should require no permissions +#/api/v1/leaderboards/*/racecolumnfactors/** = bearerToken + +# Example: Secure the access (in particular the POST/write access) to posting race log events +#/rc/racelog = bearerToken \ No newline at end of file diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRace.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRace.java index ad8203bb347..c98d8e1ea15 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRace.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRace.java @@ -17,9 +17,11 @@ import com.sap.sailing.domain.maneuverdetection.TrackTimeInfo; import com.sap.sailing.domain.polars.PolarDataService; import com.sap.sailing.domain.tracking.CompleteManeuverCurve; import com.sap.sailing.domain.tracking.TrackedRace; +import com.sap.sailing.domain.tracking.WindTrack; import com.sap.sailing.domain.tracking.WindWithConfidence; -import com.sap.sailing.domain.tracking.impl.WindTrackImpl; -import com.sap.sailing.domain.windestimation.IncrementalWindEstimationTrack; +import com.sap.sailing.domain.windestimation.IncrementalWindEstimation; +import com.sap.sailing.domain.windestimation.TimePointAndPositionWithToleranceComparator; +import com.sap.sailing.domain.windestimation.WindTrackWithConfidenceForEachWindFixImpl; import com.sap.sailing.windestimation.aggregator.hmm.GraphLevelInference; import com.sap.sailing.windestimation.aggregator.msthmm.DistanceAndDurationAwareWindTransitionProbabilitiesCalculator; import com.sap.sailing.windestimation.aggregator.msthmm.MstBestPathsCalculator; @@ -48,10 +50,8 @@ import com.sap.sse.common.Util.Pair; * @author Vladislav Chumak (D069712) * */ -public class IncrementalMstHmmWindEstimationForTrackedRace extends WindTrackImpl - implements IncrementalWindEstimationTrack { +public class IncrementalMstHmmWindEstimationForTrackedRace implements IncrementalWindEstimation { - private static final long serialVersionUID = 6654969432545619955L; private static final double WIND_COURSE_TOLERANCE_IN_DEGREES_TO_IGNORE_FOR_REUSE = 1.0; private static final double DEFAULT_BASE_CONFIDENCE = 0.01; private final IncrementalMstManeuverGraphGenerator mstManeuverGraphGenerator; @@ -60,15 +60,17 @@ public class IncrementalMstHmmWindEstimationForTrackedRace extends WindTrackImpl private final Map, WindWithConfidence>> windTrackWithConfidences = new TreeMap<>( new TimePointAndPositionWithToleranceComparator()); private final TrackedRace trackedRace; - private WindSource windSource; + private final WindSource windSource; + private final WindTrackWithConfidenceForEachWindFixImpl estimatedWindTrack; public IncrementalMstHmmWindEstimationForTrackedRace(TrackedRace trackedRace, WindSource windSource, PolarDataService polarDataService, long millisecondsOverWhichToAverage, ManeuverClassifiersCache maneuverClassifiersCache, GaussianBasedTwdTransitionDistributionCache gaussianBasedTwdTransitionDistributionCache) { - super(millisecondsOverWhichToAverage, DEFAULT_BASE_CONFIDENCE, + this.estimatedWindTrack = new WindTrackWithConfidenceForEachWindFixImpl(millisecondsOverWhichToAverage, + DEFAULT_BASE_CONFIDENCE, WindSourceType.MANEUVER_BASED_ESTIMATION.useSpeed() && polarDataService != null, - IncrementalMstHmmWindEstimationForTrackedRace.class.getName(), false); + IncrementalMstHmmWindEstimationForTrackedRace.class.getName(), false, windTrackWithConfidences); this.trackedRace = trackedRace; this.windSource = windSource; DistanceAndDurationAwareWindTransitionProbabilitiesCalculator transitionProbabilitiesCalculator = new DistanceAndDurationAwareWindTransitionProbabilitiesCalculator( @@ -82,11 +84,16 @@ public class IncrementalMstHmmWindEstimationForTrackedRace extends WindTrackImpl : new PolarsBasedTwsCalculatorImpl(polarDataService)); } + @Override + public WindTrack getWindTrack() { + return estimatedWindTrack; + } + @Override public void newManeuverSpotsDetected(Competitor competitor, Iterable newManeuvers, TrackTimeInfo trackTimeInfo) { List maneuversWithEstimatedType = new ArrayList<>(); - lockForWrite(); + estimatedWindTrack.lockForWrite(); try { for (CompleteManeuverCurve newManeuverSpot : newManeuvers) { mstManeuverGraphGenerator.add(competitor, newManeuverSpot, trackTimeInfo); @@ -134,7 +141,7 @@ public class IncrementalMstHmmWindEstimationForTrackedRace extends WindTrackImpl } } } finally { - unlockAfterWrite(); + estimatedWindTrack.unlockAfterWrite(); } } @@ -146,13 +153,6 @@ public class IncrementalMstHmmWindEstimationForTrackedRace extends WindTrackImpl return true; } - @Override - protected double getConfidenceOfInternalWindFixUnsynchronized(Wind windFix) { - WindWithConfidence> windWithConfidence = windTrackWithConfidences - .get(new Pair<>(windFix.getPosition(), windFix.getTimePoint())); - return super.getConfidenceOfInternalWindFixUnsynchronized(windFix) * windWithConfidence.getConfidence(); - } - @Override public WindSource getWindSource() { return windSource; diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/ReplicableWindEstimationFactoryService.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/ReplicableWindEstimationFactoryService.java index d2206164c9f..9edf9621ef5 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/ReplicableWindEstimationFactoryService.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/ReplicableWindEstimationFactoryService.java @@ -2,6 +2,7 @@ package com.sap.sailing.windestimation.integration; import com.sap.sailing.domain.polars.PolarDataService; import com.sap.sailing.domain.windestimation.WindEstimationFactoryService; +import com.sap.sailing.windestimation.model.exception.ModelPersistenceException; import com.sap.sse.replication.ReplicableWithObjectInputStream; /** @@ -11,5 +12,7 @@ import com.sap.sse.replication.ReplicableWithObjectInputStream; * */ public interface ReplicableWindEstimationFactoryService extends WindEstimationFactoryService, - ReplicableWithObjectInputStream { + ReplicableWithObjectInputStream { + + void updateWindEstimationModels(ExportedModels exportedModels) throws ModelPersistenceException; } diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationFactoryServiceImpl.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationFactoryServiceImpl.java index 16864ae1440..839fc8ac6af 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationFactoryServiceImpl.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationFactoryServiceImpl.java @@ -11,7 +11,7 @@ import java.util.Map; import com.sap.sailing.domain.common.WindSourceType; import com.sap.sailing.domain.common.impl.WindSourceImpl; import com.sap.sailing.domain.tracking.TrackedRace; -import com.sap.sailing.domain.windestimation.IncrementalWindEstimationTrack; +import com.sap.sailing.domain.windestimation.IncrementalWindEstimation; import com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverClassifiersCache; import com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverFeatures; import com.sap.sailing.windestimation.model.exception.ModelPersistenceException; @@ -23,7 +23,7 @@ import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl; import com.sap.sse.mongodb.MongoDBService; public class WindEstimationFactoryServiceImpl extends - AbstractReplicableWithObjectInputStream + AbstractReplicableWithObjectInputStream implements ReplicableWindEstimationFactoryService { private static final boolean PRELOAD_ALL_MODELS = true; @@ -52,6 +52,7 @@ public class WindEstimationFactoryServiceImpl extends PRESERVE_LOADED_MODELS_MILLIS, MAX_MANEUVER_FEATURES); gaussianBasedTwdTransitionDistributionCache = new GaussianBasedTwdTransitionDistributionCache(MODEL_STORE, PRELOAD_ALL_MODELS, PRESERVE_LOADED_MODELS_MILLIS); + modelsReady = maneuverClassifiersCache.isReady() && gaussianBasedTwdTransitionDistributionCache.isReady(); } private boolean isMaster() { @@ -59,8 +60,8 @@ public class WindEstimationFactoryServiceImpl extends } @Override - public IncrementalWindEstimationTrack createIncrementalWindEstimationTrack(TrackedRace trackedRace) { - IncrementalWindEstimationTrack windEstimation = new IncrementalMstHmmWindEstimationForTrackedRace(trackedRace, + public IncrementalWindEstimation createIncrementalWindEstimationTrack(TrackedRace trackedRace) { + IncrementalWindEstimation windEstimation = new IncrementalMstHmmWindEstimationForTrackedRace(trackedRace, new WindSourceImpl(WindSourceType.MANEUVER_BASED_ESTIMATION), trackedRace.getPolarDataService(), trackedRace.getMillisecondsOverWhichToAverageWind(), maneuverClassifiersCache, gaussianBasedTwdTransitionDistributionCache); @@ -79,11 +80,13 @@ public class WindEstimationFactoryServiceImpl extends updateWindEstimationModels(exportedModels); } + @Override public void updateWindEstimationModels(ExportedModels exportedModels) throws ModelPersistenceException { for (ModelDomainType domainType : modelDomainTypesRequiredByWindEstimation) { Map serializedModelsForDomainType = exportedModels .getSerializedModelsForDomainType(domainType); if (serializedModelsForDomainType != null) { + MODEL_STORE.deleteAll(domainType); MODEL_STORE.importPersistedModels(serializedModelsForDomainType, domainType); } } diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperation.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperation.java index c0f75bb2b5b..fd49cb49a69 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperation.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperation.java @@ -3,13 +3,26 @@ package com.sap.sailing.windestimation.integration; import com.sap.sse.replication.OperationWithResult; /** - * - * Updates wind estimation models. + * Updates wind estimation models with provided exported models. * * @author Vladislav Chumak (D069712) * */ -public interface WindEstimationModelsUpdateOperation - extends OperationWithResult { +public class WindEstimationModelsUpdateOperation + implements OperationWithResult { + + private static final long serialVersionUID = 6407877837605630090L; + + private final ExportedModels exportedModels; + + public WindEstimationModelsUpdateOperation(ExportedModels exportedModels) { + this.exportedModels = exportedModels; + } + + @Override + public Void internalApplyTo(ReplicableWindEstimationFactoryService toState) throws Exception { + toState.updateWindEstimationModels(exportedModels); + return null; + } } diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperationImpl.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperationImpl.java deleted file mode 100644 index 555b187f495..00000000000 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperationImpl.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.sap.sailing.windestimation.integration; - -/** - * Updates wind estimation models with provided exported models. - * - * @author Vladislav Chumak (D069712) - * - */ -public class WindEstimationModelsUpdateOperationImpl implements WindEstimationModelsUpdateOperation { - - private static final long serialVersionUID = 6407877837605630090L; - - private final ExportedModels exportedModels; - - public WindEstimationModelsUpdateOperationImpl(ExportedModels exportedModels) { - this.exportedModels = exportedModels; - } - - @Override - public Void internalApplyTo(WindEstimationFactoryServiceImpl toState) throws Exception { - toState.updateWindEstimationModels(exportedModels); - return null; - } - -} diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResource.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResource.java index 67405777eba..a7611a4a688 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResource.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResource.java @@ -18,7 +18,6 @@ import com.sap.sailing.domain.common.security.Permission; import com.sap.sailing.domain.common.security.Permission.Mode; import com.sap.sailing.windestimation.integration.ExportedModels; import com.sap.sailing.windestimation.integration.WindEstimationModelsUpdateOperation; -import com.sap.sailing.windestimation.integration.WindEstimationModelsUpdateOperationImpl; import com.sap.sailing.windestimation.jaxrs.AbstractWindEstimationDataResource; /** @@ -46,7 +45,7 @@ public class WindEstimationDataResource extends AbstractWindEstimationDataResour ObjectInputStream ois = getWindEstimationFactoryServiceImpl() .createObjectInputStreamResolvingAgainstCache(inputStream); ExportedModels exportedModels = (ExportedModels) ois.readObject(); - WindEstimationModelsUpdateOperation windEstimationModelsUpdateOperation = new WindEstimationModelsUpdateOperationImpl( + WindEstimationModelsUpdateOperation windEstimationModelsUpdateOperation = new WindEstimationModelsUpdateOperation( exportedModels); getWindEstimationFactoryServiceImpl().apply(windEstimationModelsUpdateOperation); return Response.ok("Wind estimation models accepted").build(); From 44bd77f23a4c77208933dc8dc7ce8790325321e3 Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Tue, 19 Feb 2019 22:31:35 +0100 Subject: [PATCH 14/24] Adjusted and renamed run config for polar and wind estimation data import from localhost at 8888 --- ...tty on 8889, Polar & Wind estimation Import from 8888).launch} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename java/com.sap.sailing.server/{SailingServer (Proxy, Jetty on 8889, Polar Import from 8888).launch => SailingServer (Proxy, Jetty on 8889, Polar & Wind estimation Import from 8888).launch} (100%) diff --git a/java/com.sap.sailing.server/SailingServer (Proxy, Jetty on 8889, Polar Import from 8888).launch b/java/com.sap.sailing.server/SailingServer (Proxy, Jetty on 8889, Polar & Wind estimation Import from 8888).launch similarity index 100% rename from java/com.sap.sailing.server/SailingServer (Proxy, Jetty on 8889, Polar Import from 8888).launch rename to java/com.sap.sailing.server/SailingServer (Proxy, Jetty on 8889, Polar & Wind estimation Import from 8888).launch From 34644bf75885b3a722a31eb0588601e35115f019 Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Tue, 19 Feb 2019 22:32:10 +0100 Subject: [PATCH 15/24] Fixed ClassNotFound issue in replication of WindEstimationModelsUpdateOperation --- ...stractReplicableWithObjectInputStream.java | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/AbstractReplicableWithObjectInputStream.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/AbstractReplicableWithObjectInputStream.java index 4c53ede9a41..67147a638d4 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/AbstractReplicableWithObjectInputStream.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/AbstractReplicableWithObjectInputStream.java @@ -3,6 +3,7 @@ package com.sap.sailing.windestimation.integration; import java.io.IOException; import java.io.InputStream; import java.io.ObjectInputStream; +import java.io.ObjectStreamClass; import java.io.Serializable; import java.util.HashSet; import java.util.Set; @@ -105,7 +106,7 @@ public abstract class AbstractReplicableWithObjectInputStream resolveClass(ObjectStreamClass desc) throws IOException, ClassNotFoundException { + return super.resolveClass(desc); + } + + } + } From a1d89987cd07e7e230c65e4ac6f5ca0b3857dee6 Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Tue, 19 Feb 2019 22:43:55 +0100 Subject: [PATCH 16/24] Fixed failing test case by adjusting clearState to clear persistence --- .../WindEstimationFactoryServiceImpl.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationFactoryServiceImpl.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationFactoryServiceImpl.java index 839fc8ac6af..fd138198fda 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationFactoryServiceImpl.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationFactoryServiceImpl.java @@ -90,7 +90,7 @@ public class WindEstimationFactoryServiceImpl extends MODEL_STORE.importPersistedModels(serializedModelsForDomainType, domainType); } } - clearState(); + reloadWindEstimationModels(); } @Override @@ -138,15 +138,19 @@ public class WindEstimationFactoryServiceImpl extends for (ModelDomainType domainType : modelDomainTypesRequiredByWindEstimation) { MODEL_STORE.deleteAll(domainType); } - clearState(); + reloadWindEstimationModels(); + } + + @Override + public void clearState() throws Exception { + clearReplicaState(); } /** * Clears/Reloads all caches with machine learning models. Notifies ready state change listeners about the new ready * state of this wind estimation instance. */ - @Override - public void clearState() { + public void reloadWindEstimationModels() { maneuverClassifiersCache.clearCache(); gaussianBasedTwdTransitionDistributionCache.clearCache(); List modelsChangedListeners; @@ -169,7 +173,7 @@ public class WindEstimationFactoryServiceImpl extends MODEL_STORE.deleteAll(domainType); MODEL_STORE.importPersistedModels(exportedModels, domainType); } - clearState(); + reloadWindEstimationModels(); } /** From 5bb270dfea2f69a297f1f552a4bb1f1756873658 Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Wed, 20 Feb 2019 15:43:26 +0100 Subject: [PATCH 17/24] Made AbstractReplicableWithObjectInputStream more robust --- .../META-INF/MANIFEST.MF | 3 +-- .../AbstractReplicableWithObjectInputStream.java | 14 ++++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/java/com.sap.sailing.windestimation/META-INF/MANIFEST.MF b/java/com.sap.sailing.windestimation/META-INF/MANIFEST.MF index 176c358b8c3..bbaf0a88153 100644 --- a/java/com.sap.sailing.windestimation/META-INF/MANIFEST.MF +++ b/java/com.sap.sailing.windestimation/META-INF/MANIFEST.MF @@ -44,8 +44,7 @@ Import-Package: javax.ws.rs;version="1.1.1", Web-ContextPath: /windestimation Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Bundle-ActivationPolicy: lazy -Export-Package: com.sap.sailing.windestimation, - com.sap.sailing.windestimation.integration +Export-Package: com.sap.sailing.windestimation Bundle-ClassPath: lib/smile-core-1.5.2.jar, lib/smile-data-1.5.2.jar, lib/smile-graph-1.5.2.jar, diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/AbstractReplicableWithObjectInputStream.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/AbstractReplicableWithObjectInputStream.java index 67147a638d4..332c0146be0 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/AbstractReplicableWithObjectInputStream.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/AbstractReplicableWithObjectInputStream.java @@ -106,7 +106,7 @@ public abstract class AbstractReplicableWithObjectInputStream resolveClass(ObjectStreamClass desc) throws IOException, ClassNotFoundException { - return super.resolveClass(desc); + String name = desc.getName(); + try { + return Class.forName(name, /* initialize */true, Thread.currentThread().getContextClassLoader()); + } catch (ClassNotFoundException ex) { + // fall back on super class in order to load primitive types + return super.resolveClass(desc); + } } } From 629b5f9ae2bd450b14a4d3b3c14f926c151db274 Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Wed, 20 Feb 2019 17:26:33 +0100 Subject: [PATCH 18/24] Improved simple models training with corresponding wiki howto --- ...neuverNumberDependentEvaluationRunner.java | 1 + .../model/SimpleModelsTraining.java | 29 +++++++- wiki/howto/windestimation.md | 74 +++++-------------- 3 files changed, 44 insertions(+), 60 deletions(-) diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/evaluation/WindEstimatorManeuverNumberDependentEvaluationRunner.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/evaluation/WindEstimatorManeuverNumberDependentEvaluationRunner.java index 7815dd9540a..85c3c6d03f2 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/evaluation/WindEstimatorManeuverNumberDependentEvaluationRunner.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/evaluation/WindEstimatorManeuverNumberDependentEvaluationRunner.java @@ -110,6 +110,7 @@ public class WindEstimatorManeuverNumberDependentEvaluationRunner { out.write(line); } } + LoggingUtil.logInfo("CSV with evaluation results have been stored in: " + csvFile.getAbsolutePath()); } } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java index 8aaa2488433..9b4b27f6e8f 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java @@ -20,7 +20,11 @@ import com.sap.sailing.windestimation.datavisualization.AggregatedDistanceDimens import com.sap.sailing.windestimation.datavisualization.AggregatedDurationDimensionPlot; import com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverClassifierTrainer; import com.sap.sailing.windestimation.model.classifier.maneuver.PersistedManeuverClassifiersScorePrinter; +import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext; +import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext.DistanceValueRange; import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionStdRegressorTrainer; +import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext; +import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext.DurationValueRange; import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionStdRegressorTrainer; import com.sap.sailing.windestimation.util.LoggingUtil; @@ -32,9 +36,7 @@ import com.sap.sailing.windestimation.util.LoggingUtil; public class SimpleModelsTraining { private static final int NUMBER_OF_THREADS = 3; - private static final ExecutorService executorService = new ThreadPoolExecutor(NUMBER_OF_THREADS, NUMBER_OF_THREADS, - 0, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(NUMBER_OF_THREADS), - new ThreadPoolExecutor.CallerRunsPolicy()); + private static ExecutorService executorService; public static void main(String[] args) throws Exception { new ManeuverForEstimationPersistenceManager().dropCollection(); @@ -63,11 +65,15 @@ public class SimpleModelsTraining { showInfoAboutDataCleaning(AggregatedSingleDimensionType.DURATION); AggregatedDurationDimensionPlot.awaitWindowClosed(); } while (JOptionPane.YES_OPTION != askDataCleaningFinished(AggregatedSingleDimensionType.DURATION)); + showInfoAboutIntervalAdjustments(DurationBasedTwdTransitionRegressorModelContext.class, + DurationValueRange.class); do { AggregatedDistanceDimensionPlot.main(args); showInfoAboutDataCleaning(AggregatedSingleDimensionType.DISTANCE); AggregatedDistanceDimensionPlot.awaitWindowClosed(); } while (JOptionPane.YES_OPTION != askDataCleaningFinished(AggregatedSingleDimensionType.DISTANCE)); + showInfoAboutIntervalAdjustments(DistanceBasedTwdTransitionRegressorModelContext.class, + DistanceValueRange.class); DurationBasedTwdTransitionStdRegressorTrainer.main(args); DistanceBasedTwdTransitionStdRegressorTrainer.main(args); Thread.sleep(1000); @@ -92,15 +98,25 @@ public class SimpleModelsTraining { private static void showInfoAboutDataCleaning(AggregatedSingleDimensionType dimension) { JOptionPane.showMessageDialog(null, "Now, clean the data for " + dimension + " dimension. Remove instances from MongoDB collection \"" + dimension.getCollectioName() - + "\" which do not make sense. E.g. values which are represented by a small number of supporting instances (see histogram), values which cause implausible zig zag sections within zero-mean standard deviation curve and etc. Close the graphical tool, when you are done to resume model training."); + + "\" which do not make sense. E.g. values which are represented by a small number of supporting instances (see histogram), values which cause implausible zig zag sections within zero-mean standard deviation curve and etc. Close the graphical tool, when you are done to resume the model training."); + } + + private static void showInfoAboutIntervalAdjustments(Class classToAdjust, Class valueRangeEnum) { + JOptionPane.showMessageDialog(null, "Now, open the source code of the class \"" + classToAdjust.getName() + + "\". Scroll down to the definition of the inner enum \"" + valueRangeEnum.getSimpleName() + + "\", read its JavaDoc and adjust its interval definitions so that each interval can be learned by the adjusted regressor model configuration with minimal error. Press OK ONLY after you are done."); } private static void awaitThreadPoolCompletion() throws InterruptedException { + executorService.shutdown(); executorService.awaitTermination(24, TimeUnit.HOURS); Thread.sleep(1000L); } private static void executeInThreadPool(RunnableWithExceptionsCatch runnable) { + if (executorService == null || executorService.isShutdown()) { + createNewThreadPool(); + } executorService.execute(() -> { try { runnable.run(); @@ -112,6 +128,11 @@ public class SimpleModelsTraining { }); } + private static void createNewThreadPool() { + executorService = new ThreadPoolExecutor(NUMBER_OF_THREADS, NUMBER_OF_THREADS, 0, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(NUMBER_OF_THREADS), new ThreadPoolExecutor.CallerRunsPolicy()); + } + /** * * @author Vladislav Chumak (D069712) diff --git a/wiki/howto/windestimation.md b/wiki/howto/windestimation.md index f834ee77019..25c9812657f 100644 --- a/wiki/howto/windestimation.md +++ b/wiki/howto/windestimation.md @@ -2,66 +2,28 @@ This document describes the generation process of Machine Learning (ML) models which are used internally by wind estimation. It is highly recommended to proceed this howto step by step considering the order of sections. -## Overview -In total, there are the following three categories of ML models used by wind estimation: -1. **Maneuver Classifiers** -2. **Regressors** of TWD delta standard deviation for the dimension **duration** -3. **Regressors** of TWD delta standard deviation for the dimension **distance** - -Each of the model categories are composed of multiple models where each model targets a specific context. A context for a maneuver classifier is determined by the following attributes: -* Maneuver features - * Polar features enabled: yes/no - * Mark features enabled: yes/no - * Scaled speed features enabled: yes/no -* Boat class filtering for the data on which the classifier is trained, such as a specific boat class, or with all boat classes included - -The context of regressor models is represented by its assigned input interval responsibility, e.g. [0 seconds; 62 seconds) for duration, or [80 meters; 1368 meters) for distance. - -Each of the ML model categories must be trained individually. The common workflow looks as follows: -1. Get the training data from REST API of sapsailing.com -2. Preprocess data -3. Train the model category - -For each of the steps, appropriate Java classes must be executed per *Run with...->Java Application*. All referenced classes are located in *com.sap.sailing.windestimation.lab* Java project. Each class execution must finish without uncaught exceptions before proceeding to next instructions. After model training, all trained models can be collected in *./trained_wind_estimation_models*, which is normally */path/to/workspace/com.sap.sailing.windestimation/trained_wind_estimation_models* if you start the training classes in Eclipse per *Run with...->Java Application*. - -The details of the training process for each model category are described in the following sections. - ## Prerequisites To complete the training process successfully, you need to make sure that you have the following stuff: * A complete onboarding setup for SAP Sailing Analytics development * MongoDB (**3.4 or higher!**) is up and running (same MongoDB instance as required in onboarding howto) * At least 100 GB free space on the partition, where MongoDB is operating * Installed graphical MongoDB client such as MongoDB Compass (Community version) +* 16 GB RAM +* ~24 operation hours of your computer -## Get the training data from sapsailing.com -The following steps import all the data required from sapsailing.com into the local MongoDB. These steps constitute a preprequisite for training of all ML model categories: -1. Run *com.sap.sailing.windestimation.data.importer.ManeuverAndWindImporter* -2. Run *com.sap.sailing.windestimation.data.importer.PolarDataImporter* - -## Maneuver classifiers training -1. Run *com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverClassifierTrainer*. Within the this step, the maneuver data is preprocessed and all maneuver classifiers are trained for each supported context. -2. Optionally run *com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverClassifierScoring* to print the performance of the trained classifiers. After this step, a list with macro-averaged F2-score of each trained classifier will be stored in *./maneuverClassifierScores.csv* - -## Duration-based TWD delta standard deviation regressor - -1. Run *com.sap.sailing.windestimation.data.importer.DurationBasedTwdTransitionImporter* -2. Run *com.sap.sailing.windestimation.data.importer.AggregatedDurationBasedTwdTransitionImporter* -3. Run *com.sap.sailing.windestimation.datavisualization.AggregatedDurationDimensionPlot* to visualize the wind data. A Swing-based GUI-Window must open with two charts, one XY-chart where the x-axis represents **seconds**, and the y-axis represents TWD delta-based series measures (e.g. standard deviation or mean). Below the chart, a histogram for data points of the XY-Chart is provided. You can zoom-in and zoom-out in each of the chart by mouse dragging. Be aware that currently the zoom level of both charts is not synchronized -4. Open your graphical MongoDB client and connect to *windEstimation* database hosted by your local MongoDB. Open the collection with name *aggregatedDurationTwdTransition*. Within the collection you will see all the instances/data points visualized in the previous step. The total number of the points must not exceed 100. -5. Delete all the instances within the collection which do not make sense. For this, use the data visualization tool from step 3 to identify such instances. Pay a special attention to the instances in the beginnning and end. Some of the instances are not representative due to small number of supporting instances which is visualized in the histogram. Restart the data visualization tool as often as need to visualize the changed data. -6. Open the source code of the class *com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext*. Scroll down to the definition of the inner class/enum *DurationValueRange*. The enum defines the intervals for which a separate regressor model will be trained. Adjust the intervals accordingly in order to allow the regressor model to learn the data curve with minimal error. Make sure that there are at least 2 data points available within each interval. Datapoint with x = 0, y = 0 will be created automatically. -7. Run *com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionStdRegressorTrainer* -8. Verify the trained regressor functions. They are printed in the console output of the previous step. For instance, you can visualize the polynoms by means of https://www.wolframalpha.com/ - -## Distance-based TWD delta standard deviation regressor - -The steps of this sections are similar to the steps of the previous section. It is recommended to traverse through the previous section before starting with this one, because due to similarity of the steps, the similar steps in this section are described with less details and hints. - -1. Run *com.sap.sailing.windestimation.data.importer.DistanceBasedTwdTransitionImporter* -2. Run *com.sap.sailing.windestimation.data.importer.AggregatedDistanceBasedTwdTransitionImporter* with at least 10 GB JVM memory. -3. Run *com.sap.sailing.windestimation.datavisualization.AggregatedDistanceDimensionPlot* to visualize the wind data. Here, the x-axis of the XY-chart represents **meters** -4. Open your graphical MongoDB client and connect to *windEstimation* database hosted by your local MongoDB. Open collection *aggregatedDistanceTwdTransition* collection. Within the collection you will see all the instances/data points visualized in the previous step. The total number of the points must not exceed 100. -5. Delete all the instances within the collection which do not make sense. -6. Open the source code of the class *com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext*. Scroll down to the definition of the inner class/enum *DistanceValueRange*. The enum defines the intervals for which a separate regressor model will be trained. Adjust the intervals accordingly in order to allow the regressor model to learn the data curve with minimal error. -7. Run *com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionStdRegressorTrainer* -8. Verify the trained regressor functions. They are printed in the console output of the previous step. \ No newline at end of file +## Model training process +1. Run ``com.sap.sailing.windestimation.model.SimpleModelsTraining`` as normal Java Application. This program downloads all the necessary maneuver and wind data, pre-processes them and initiates training of maneuver classifiers. +2. Make sure that the launched program does not get termined by an uncaught exception. Wait until graphical info dialog shows up which requests you to perform data cleansing for duration dimension and press OK. + ![Screenshot of graphical info dialog requesting to perform data cleansing for duration dimension](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png "Screenshot of graphical info dialog requesting to perform data cleansing for duration dimension") + A Swing-based GUI-Window must open with two charts, one XY-chart where the x-axis represents **seconds**, and the y-axis represents TWD delta-based series measures (e.g. standard deviation or mean). Below the chart, a histogram for the data points of the XY-Chart is provided. You can zoom-in and zoom-out in each of the chart by mouse dragging. Be aware that currently, the zoom level of both charts is not synchronizing. + ![Screenshot of graphical wind data visualization tool for duration dimension](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png "Screenshot of graphical wind data visualization tool for duration dimension") +3. Open your graphical MongoDB client and connect to ``windEstimation`` database hosted by your local MongoDB. Open the collection with name ``aggregatedDurationTwdTransition``. Within the collection you will see all the instances/data points visualized in the previous step. The attribute used for the x-axis is represented by ``value``. + ![Screenshot of MongoDB Compass with opened aggregatedDurationTwdTransition collection](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png "Screenshot of MongoDB Compass with opened aggregatedDurationTwdTransition collection") +4. Delete all the instances within the collection which do not make sense. For this, use the data visualization tool from step 2 to identify such instances. Some of the instances are not representative due to the small number of supporting instances which is visualized in the histogram. Such instances can produce unreasonable bumps in the XY-chart. The desired output of this step is that the series curve ``Zero mean sigma`` looks smooth and always growing, like depicted below: + ![Screenshot of graphical visualization tool of duration dimension with after data cleansing](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png "Screenshot of graphical visualization tool of duration dimension with after data cleansing") + Use the ``Refresh charts`` button as often as needed to update the charts with the modified data in MongoDB. Close the graphical visualization tool window after you are done with data cleansing to resume the training process. Confirm the confirmation dialog after you have finished the data cleansing of duration dimension: + ![Screenshot of confirmation dialog for finishing the data cleansing](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png "Screenshot of confirmation dialog for finishing the data cleansing") +5. A new information dialog shows up (do not press OK yet!) requesting you to open the source code of the class ``com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext``. Open it and scroll down to the definition of the inner enum ``DurationValueRange``. The enum defines the intervals for which a separate regressor model will be trained. Read the Javadoc of ``DurationValueRange`` and adjust the intervals accordingly in order to allow the regressor model to learn the ``Zero mean sigma`` curve with minimal error. You can also configurate the polynomial which will be used for regressor training. Make sure that there are at least 2 data points available within each interval. The datapoint with x = 0, y = 0 will be created automatically. Press OK in information dialog after you are done. +6. A graphical info dialog shows up which requests you to perform data cleansing for *distance* dimension. Press OK. All steps for data cleansing for the distance dimension are very similar to the data cleansing steps step 2. until step 5. for the duration dimension. Thus, consult these steps to complete data cleansing and models configuration for the distance duration. The unit used for distance representation is **meters**. The collection name required in step 3. is ``aggregatedDistanceTwdTransition``. The class required in step 5. is ``com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext`` and its inner enum is ``DistanceValueRange``. +7. Wait until model training finishes and the program terminates normally. A new file with serialized representation of internal wind estimation models should be located in ``./windEstimationModels.dat``. The absolute path of the file must be printed in the console output of the program. You can upload the file via HTTP POST to http://sapsailing.com/windestimation/api/windestimation_data (see ``com.sap.sailing.windestimation.jaxrs.api.WindEstimationDataResource``) to update the wind estimation of a server instance. +8. Optionally, run ``com.sap.sailing.windestimation.evaluation.WindEstimatorManeuverNumberDependentEvaluationRunner`` as normal Java Application to evaluate the wind estimation with the new trained models. The evaluation score will be stored as CSV in ``./maneuverNumberDependentEvaluation.csv``. \ No newline at end of file From 90b920e190d0f5c6888315613d9f35f93f8a5fce Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Wed, 20 Feb 2019 17:37:07 +0100 Subject: [PATCH 19/24] Fixed failing WindEstimationDataResourceTest --- ...IncrementalMstHmmWindEstimationForTrackedRaceTest.java | 6 +++--- .../jaxrs/api/WindEstimationDataResourceTest.java | 8 ++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRaceTest.java b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRaceTest.java index 15b4fd9a1dd..5aaa9dbb496 100644 --- a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRaceTest.java +++ b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRaceTest.java @@ -76,14 +76,14 @@ public class IncrementalMstHmmWindEstimationForTrackedRaceTest extends OnlineTra "SERIALIZATION.modelForManeuverClassifier.NeuralNetworkClassifier.ManeuverClassification-Basic-All.clf" }; protected final SimpleDateFormat dateFormat; - private final WindEstimationFactoryServiceImpl windEstimationFactoryService; + private WindEstimationFactoryServiceImpl windEstimationFactoryService; private ClassPathReadOnlyModelStoreImpl modelStore; - public IncrementalMstHmmWindEstimationForTrackedRaceTest() - throws MalformedURLException, URISyntaxException, ModelPersistenceException { + public IncrementalMstHmmWindEstimationForTrackedRaceTest() throws Exception { dateFormat = new SimpleDateFormat("MM/dd/yyyy-HH:mm:ss"); dateFormat.setTimeZone(TimeZone.getTimeZone("GMT+2")); // will result in CEST windEstimationFactoryService = new WindEstimationFactoryServiceImpl(); + windEstimationFactoryService.clearState(); modelStore = new ClassPathReadOnlyModelStoreImpl("trained_wind_estimation_models", getClass().getClassLoader(), modelFilesNames); windEstimationFactoryService.importAllModelsFromModelStore(modelStore); diff --git a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResourceTest.java b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResourceTest.java index d88b23a9d35..8f628d61e28 100644 --- a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResourceTest.java +++ b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResourceTest.java @@ -3,12 +3,8 @@ package com.sap.sailing.windestimation.jaxrs.api; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.io.IOException; - -import org.json.simple.parser.ParseException; import org.junit.Test; -import com.sap.sailing.domain.polars.NotEnoughDataHasBeenAddedException; import com.sap.sailing.windestimation.integration.IncrementalMstHmmWindEstimationForTrackedRaceTest; import com.sap.sailing.windestimation.integration.WindEstimationFactoryServiceImpl; import com.sap.sailing.windestimation.model.store.ClassPathReadOnlyModelStoreImpl; @@ -21,9 +17,9 @@ import com.sap.sailing.windestimation.model.store.ClassPathReadOnlyModelStoreImp public class WindEstimationDataResourceTest { @Test - public void testImportingFromMockFile() throws NotEnoughDataHasBeenAddedException, ClassNotFoundException, - IOException, ParseException, InterruptedException { + public void testImportingFromMockFile() throws Exception { WindEstimationFactoryServiceImpl windEstimationFactoryService = new WindEstimationFactoryServiceImpl(); + windEstimationFactoryService.clearState(); assertFalse(windEstimationFactoryService.isReady()); ClassPathReadOnlyModelStoreImpl modelStore = new ClassPathReadOnlyModelStoreImpl( "trained_wind_estimation_models", getClass().getClassLoader(), From 6e2e3dd6f5b775b80086db1f9261e964ffda0f96 Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Thu, 21 Feb 2019 10:09:17 +0100 Subject: [PATCH 20/24] Bugfixed ThreadPool awaitTermination call in ManeuverAndWindImporter --- .../data/importer/ManeuverAndWindImporter.java | 4 ++-- .../sailing/windestimation/model/SimpleModelsTraining.java | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java index 50aeddb8dab..0ca91010d4d 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java @@ -141,8 +141,8 @@ public class ManeuverAndWindImporter { + Math.round(100.0 * i / numberOfRegattas) + "%): \"" + regattaName + "\""); importRegatta(regattaName, importStatistics); } - boolean success = executorService.awaitTermination(24, TimeUnit.HOURS); executorService.shutdown(); + boolean success = executorService.awaitTermination(2, TimeUnit.HOURS); if (success) { LoggingUtil.logInfo("Import finished"); synchronized (importStatistics) { @@ -150,7 +150,7 @@ public class ManeuverAndWindImporter { logImportStatistics(importStatistics); } } else { - LoggingUtil.logInfo("Process was terminated"); + new InterruptedException("Thread-pool was terminated after two hours waiting time"); } } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java index 9b4b27f6e8f..bb9ff01465c 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java @@ -109,7 +109,10 @@ public class SimpleModelsTraining { private static void awaitThreadPoolCompletion() throws InterruptedException { executorService.shutdown(); - executorService.awaitTermination(24, TimeUnit.HOURS); + boolean success = executorService.awaitTermination(2, TimeUnit.HOURS); + if (!success) { + new InterruptedException("Thread-pool was terminated after two hours waiting time"); + } Thread.sleep(1000L); } From 4a6e992f03fea0e07e4455a33c631dc9831c7f7b Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Thu, 21 Feb 2019 22:25:18 +0100 Subject: [PATCH 21/24] Fixed IllegalStateException during models training --- .../maneuver/ManeuverClassifierTrainer.java | 1 - .../TwdTransitionClassifierTrainer.java | 1 - ...DimensionPolynomialRegressorTrainerHelper.java | 3 +++ ...onManualTrainingDataInputRegressorTrainer.java | 3 +-- .../model/AbstractTrainableModel.java | 6 +++++- .../windestimation/model/TrainableModel.java | 15 +++++++++++++-- .../smile/AbstractSmileClassificationModel.java | 2 ++ 7 files changed, 24 insertions(+), 7 deletions(-) diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/ManeuverClassifierTrainer.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/ManeuverClassifierTrainer.java index ed62529519d..17697ea3f30 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/ManeuverClassifierTrainer.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/ManeuverClassifierTrainer.java @@ -66,7 +66,6 @@ public class ManeuverClassifierTrainer { LoggingUtil.logInfo("Training with " + trainManeuvers.size() + " maneuvers..."); double[][] x = modelContext.getXMatrix(maneuvers); int[] y = labelExtraction.getYVector(maneuvers); - classifierModel.resetTrainingStats(); classifierModel.train(x, y); LoggingUtil.logInfo("Training finished. Validating on train dataset..."); ManeuverClassifierScoring classifierScoring = new ManeuverClassifierScoring(classifierModel); diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/twdtransition/TwdTransitionClassifierTrainer.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/twdtransition/TwdTransitionClassifierTrainer.java index 3645d4e86b2..6ea95350108 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/twdtransition/TwdTransitionClassifierTrainer.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/twdtransition/TwdTransitionClassifierTrainer.java @@ -100,7 +100,6 @@ public class TwdTransitionClassifierTrainer { i++; } LoggingUtil.logInfo("Training with " + numberOfTrainingInstances + " instances..."); - classifierModel.resetTrainingStats(); classifierModel.train(x, y); LoggingUtil.logInfo("Training finished. Validating on train dataset..."); TwdTransitionClassifierScoring classifierScoring = new TwdTransitionClassifierScoring(classifierModel); diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/IncrementalSingleDimensionPolynomialRegressorTrainerHelper.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/IncrementalSingleDimensionPolynomialRegressorTrainerHelper.java index 7db214fcdbb..3cb95a314d5 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/IncrementalSingleDimensionPolynomialRegressorTrainerHelper.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/IncrementalSingleDimensionPolynomialRegressorTrainerHelper.java @@ -41,6 +41,9 @@ public class IncrementalSingleDimensionPolynomialRegressorTrainerHelper { } public void incrementRmseCalculation(double[] x, double y) { + if (!model.isModelReady()) { + model.setModelAsReadyAfterSuccessfulTraining(); + } double predictedStd = model.getValue(x); double diff = predictedStd - y; squareErrorSum += diff * diff; diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/TwdTransitionManualTrainingDataInputRegressorTrainer.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/TwdTransitionManualTrainingDataInputRegressorTrainer.java index aa358ccb5c8..fd67c7a7ba9 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/TwdTransitionManualTrainingDataInputRegressorTrainer.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/TwdTransitionManualTrainingDataInputRegressorTrainer.java @@ -21,8 +21,7 @@ public class TwdTransitionManualTrainingDataInputRegressorTrainer { throws Exception { IncrementalSingleDimensionPolynomialRegressorTrainerHelper trainerHelper = new IncrementalSingleDimensionPolynomialRegressorTrainerHelper( regressorModelStore, model); - LoggingUtil - .logInfo("########## Training of " + model.getModelContext().getId() + " started..."); + LoggingUtil.logInfo("########## Training of " + model.getModelContext().getId() + " started..."); double[] modelInput = new double[1]; for (int i = 0; i < inputOutputPairs.length; i++) { double xi = inputOutputPairs[i][0]; diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/model/AbstractTrainableModel.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/model/AbstractTrainableModel.java index f30f010d9ae..6d934bae272 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/model/AbstractTrainableModel.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/model/AbstractTrainableModel.java @@ -51,12 +51,16 @@ public abstract class AbstractTrainableModel trainInternalModel(double[][] x, int[] y); From 2486e44ae8a5f49ec45fe6d1c007729f50c9241d Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Sun, 24 Feb 2019 21:04:23 +0100 Subject: [PATCH 22/24] Bugfixes and improvements of simple models training --- ...aunch => SimpleModelsTrainingPart1.launch} | 6 +-- .../SimpleModelsTrainingPart2.launch | 12 ++++++ ...DistanceBasedAbsTwdTransitionImporter.java | 10 ++++- ...tedDistanceBasedTwdTransitionImporter.java | 10 ++++- ...DurationBasedAbsTwdTransitionImporter.java | 10 ++++- ...tedDurationBasedTwdTransitionImporter.java | 10 ++++- .../DistanceBasedTwdTransitionImporter.java | 10 +++-- .../DurationBasedTwdTransitionImporter.java | 11 +++--- ...nsionTwdTransitionAggregationImporter.java | 13 ++++--- ...RaceWithWindSourcesPersistenceManager.java | 4 +- .../WindSourcesPersistenceManager.java | 2 +- ...ingleDimensionAggregatesPlottingFrame.java | 1 + ...ng.java => SimpleModelsTrainingPart1.java} | 39 +++++++++---------- .../model/SimpleModelsTrainingPart2.java | 22 +++++++++++ 14 files changed, 112 insertions(+), 48 deletions(-) rename java/com.sap.sailing.windestimation.lab/{SimpleModelsTraining.launch => SimpleModelsTrainingPart1.launch} (82%) create mode 100644 java/com.sap.sailing.windestimation.lab/SimpleModelsTrainingPart2.launch rename java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/{SimpleModelsTraining.java => SimpleModelsTrainingPart1.java} (78%) create mode 100644 java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTrainingPart2.java diff --git a/java/com.sap.sailing.windestimation.lab/SimpleModelsTraining.launch b/java/com.sap.sailing.windestimation.lab/SimpleModelsTrainingPart1.launch similarity index 82% rename from java/com.sap.sailing.windestimation.lab/SimpleModelsTraining.launch rename to java/com.sap.sailing.windestimation.lab/SimpleModelsTrainingPart1.launch index 69cd2a60688..ac2cbb603ff 100644 --- a/java/com.sap.sailing.windestimation.lab/SimpleModelsTraining.launch +++ b/java/com.sap.sailing.windestimation.lab/SimpleModelsTrainingPart1.launch @@ -1,13 +1,13 @@ - + - + - + diff --git a/java/com.sap.sailing.windestimation.lab/SimpleModelsTrainingPart2.launch b/java/com.sap.sailing.windestimation.lab/SimpleModelsTrainingPart2.launch new file mode 100644 index 00000000000..ba6810c3b3e --- /dev/null +++ b/java/com.sap.sailing.windestimation.lab/SimpleModelsTrainingPart2.launch @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDistanceBasedAbsTwdTransitionImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDistanceBasedAbsTwdTransitionImporter.java index 7ad3616088c..ab97520b348 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDistanceBasedAbsTwdTransitionImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDistanceBasedAbsTwdTransitionImporter.java @@ -9,12 +9,18 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.SingleDimen public class AggregatedDistanceBasedAbsTwdTransitionImporter { public static void main(String[] args) throws UnknownHostException { - SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager( - SingleDimensionType.DISTANCE); + SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = createPersistenceManagerAndEnsureIndex(); SingleDimensionTwdTransitionAggregationImporter importer = new SingleDimensionTwdTransitionAggregationImporter( singleDimensionBasedTwdTransitionPersistenceManager, AggregatedSingleDimensionType.DISTANCE_ABS, new AnnealingNextThresholdCalculator(10, 1.5)); importer.runAggregation(); } + public static SingleDimensionBasedTwdTransitionPersistenceManager createPersistenceManagerAndEnsureIndex() + throws UnknownHostException { + SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager( + SingleDimensionType.DISTANCE); + return singleDimensionBasedTwdTransitionPersistenceManager; + } + } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDistanceBasedTwdTransitionImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDistanceBasedTwdTransitionImporter.java index a964e6d712e..f54c24ac7a6 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDistanceBasedTwdTransitionImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDistanceBasedTwdTransitionImporter.java @@ -9,12 +9,18 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.SingleDimen public class AggregatedDistanceBasedTwdTransitionImporter { public static void main(String[] args) throws UnknownHostException { - SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager( - SingleDimensionType.DISTANCE); + SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = createPersistenceManagerAndEnsureIndex(); SingleDimensionTwdTransitionAggregationImporter importer = new SingleDimensionTwdTransitionAggregationImporter( singleDimensionBasedTwdTransitionPersistenceManager, AggregatedSingleDimensionType.DISTANCE, new AnnealingNextThresholdCalculator(10, 1.5)); importer.runAggregation(); } + public static SingleDimensionBasedTwdTransitionPersistenceManager createPersistenceManagerAndEnsureIndex() + throws UnknownHostException { + SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager( + SingleDimensionType.DISTANCE); + return singleDimensionBasedTwdTransitionPersistenceManager; + } + } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDurationBasedAbsTwdTransitionImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDurationBasedAbsTwdTransitionImporter.java index d5e84ce3e53..ca8cc119fec 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDurationBasedAbsTwdTransitionImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDurationBasedAbsTwdTransitionImporter.java @@ -9,12 +9,18 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.SingleDimen public class AggregatedDurationBasedAbsTwdTransitionImporter { public static void main(String[] args) throws UnknownHostException { - SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager( - SingleDimensionType.DURATION); + SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = createPersistenceManagerAndEnsureIndex(); SingleDimensionTwdTransitionAggregationImporter importer = new SingleDimensionTwdTransitionAggregationImporter( singleDimensionBasedTwdTransitionPersistenceManager, AggregatedSingleDimensionType.DURATION_ABS, DurationBasedTwdTransitionImporter.getThresholdCalculator()); importer.runAggregation(); } + public static SingleDimensionBasedTwdTransitionPersistenceManager createPersistenceManagerAndEnsureIndex() + throws UnknownHostException { + SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager( + SingleDimensionType.DURATION); + return singleDimensionBasedTwdTransitionPersistenceManager; + } + } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDurationBasedTwdTransitionImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDurationBasedTwdTransitionImporter.java index baacd558f94..5fc7cc95d41 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDurationBasedTwdTransitionImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDurationBasedTwdTransitionImporter.java @@ -9,12 +9,18 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.SingleDimen public class AggregatedDurationBasedTwdTransitionImporter { public static void main(String[] args) throws UnknownHostException { - SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager( - SingleDimensionType.DURATION); + SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = createPersistenceManagerAndEnsureIndex(); SingleDimensionTwdTransitionAggregationImporter importer = new SingleDimensionTwdTransitionAggregationImporter( singleDimensionBasedTwdTransitionPersistenceManager, AggregatedSingleDimensionType.DURATION, DurationBasedTwdTransitionImporter.getThresholdCalculator()); importer.runAggregation(); } + public static SingleDimensionBasedTwdTransitionPersistenceManager createPersistenceManagerAndEnsureIndex() + throws UnknownHostException { + SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager( + SingleDimensionType.DURATION); + return singleDimensionBasedTwdTransitionPersistenceManager; + } + } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/DistanceBasedTwdTransitionImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/DistanceBasedTwdTransitionImporter.java index c33557994e5..2bf8ea62427 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/DistanceBasedTwdTransitionImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/DistanceBasedTwdTransitionImporter.java @@ -36,7 +36,8 @@ public class DistanceBasedTwdTransitionImporter { .getIterator(); iterator.hasNext();) { long percent = windSourceNumber * 100 / numberOfWindSources; WindSourceWithFixes windSource = iterator.next(); - LoggingUtil.logInfo("Processing " + windSourceNumber++ + "/" + numberOfWindSources + " (" + percent + "%)"); + LoggingUtil.logInfo("Processing wind source " + windSourceNumber++ + "/" + numberOfWindSources + " (" + + percent + "%) for distance dimension"); List entries = new ArrayList<>(); List otherWindSources = new ArrayList<>(); for (PersistedElementsIterator otherWindSourcesIterator = windSourcesPersistenceManager @@ -109,16 +110,17 @@ public class DistanceBasedTwdTransitionImporter { } while (currentFix != null && currentOtherFix != null); } if (entries.isEmpty()) { - LoggingUtil.logInfo("No TWD transitions to import"); + LoggingUtil.logInfo("No distance based TWD transitions to import"); } else { distanceBasedTwdTransitionPersistenceManager.add(entries); int totalEntries = entries.size(); totalValuesCount += totalEntries; - LoggingUtil.logInfo(totalEntries + " TWD transitions imported, " + totalValuesCount + " in total"); + LoggingUtil.logInfo( + totalEntries + " distance based TWD transitions imported, " + totalValuesCount + " in total"); } } LoggingUtil.logInfo("###################\r\nDistance based TWD transitions Import finished"); - LoggingUtil.logInfo("Totally " + totalValuesCount + " TWD transitions imported"); + LoggingUtil.logInfo("Totally " + totalValuesCount + " distance based TWD transitions imported"); } } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/DurationBasedTwdTransitionImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/DurationBasedTwdTransitionImporter.java index d5d94c4ec94..ddeaa241c13 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/DurationBasedTwdTransitionImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/DurationBasedTwdTransitionImporter.java @@ -32,7 +32,8 @@ public class DurationBasedTwdTransitionImporter { .getIterator(); iterator.hasNext();) { long percent = windSourceNumber * 100 / numberOfWindSources; WindSourceWithFixes windSource = iterator.next(); - LoggingUtil.logInfo("Processing " + windSourceNumber++ + "/" + numberOfWindSources + " (" + percent + "%)"); + LoggingUtil.logInfo("Processing wind source " + windSourceNumber++ + "/" + numberOfWindSources + " (" + + percent + "%) for duration dimension"); TimePoint timePointOfLastConsideredWindFix = null; List entries = new ArrayList<>(); int windFixIndex = 0; @@ -58,17 +59,17 @@ public class DurationBasedTwdTransitionImporter { } } if (entries.isEmpty()) { - LoggingUtil.logInfo("No TWD transitions to import"); + LoggingUtil.logInfo("No duration based TWD transitions to import"); } else { durationBasedTwdTransitionPersistenceManager.add(entries); int totalEntries = entries.size(); totalValuesCount += totalEntries; - LoggingUtil - .logInfo(totalEntries + " TWD transition entries imported, " + totalValuesCount + " in total"); + LoggingUtil.logInfo(totalEntries + " duration based TWD transition entries imported, " + + totalValuesCount + " in total"); } } LoggingUtil.logInfo("###################\r\nDuration based TWD transitions Import finished"); - LoggingUtil.logInfo("Totally " + totalValuesCount + " TWD transitions imported"); + LoggingUtil.logInfo("Totally " + totalValuesCount + " duration based TWD transitions imported"); } public static NextThresholdCalculator getThresholdCalculator() { diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/SingleDimensionTwdTransitionAggregationImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/SingleDimensionTwdTransitionAggregationImporter.java index a7e0ee3cedc..23cbe105991 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/SingleDimensionTwdTransitionAggregationImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/SingleDimensionTwdTransitionAggregationImporter.java @@ -35,7 +35,7 @@ public class SingleDimensionTwdTransitionAggregationImporter { public void runAggregation() throws UnknownHostException { long totalValuesCount = 0; List aggregates = new ArrayList<>(); - LoggingUtil.logInfo("Aggregating persisted entries"); + LoggingUtil.logInfo("Aggregating persisted entries for dimension " + dimensionType); TDoubleList entries = new TDoubleArrayList(); double currentBucketThreshold = thresholdCalculator.getInitialThresholdValue(); double nextBucketThreshold = thresholdCalculator.getNextThresholdValue(currentBucketThreshold); @@ -61,7 +61,8 @@ public class SingleDimensionTwdTransitionAggregationImporter { } entries.add(twdChange); if (entries.size() % 10000 == 0) { - LoggingUtil.logInfo((entries.size() + totalValuesCount) + " Entries aggregated"); + LoggingUtil.logInfo( + (entries.size() + totalValuesCount) + " Entries aggregated for dimension " + dimensionType); } } if (entries.size() >= MIN_NUMBER_OF_VALUES_PER_BUCKET) { @@ -69,14 +70,16 @@ public class SingleDimensionTwdTransitionAggregationImporter { aggregates.add(aggregate); totalValuesCount += entries.size(); } - LoggingUtil.logInfo("Persisting " + aggregates.size() + " aggregates"); + LoggingUtil.logInfo("Persisting " + aggregates.size() + " aggregates for dimension " + dimensionType); AggregatedSingleDimensionBasedTwdTransitionPersistenceManager aggregatedDurationBasedTwdTransitionPersistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager( dimensionType); aggregatedDurationBasedTwdTransitionPersistenceManager.dropCollection(); aggregatedDurationBasedTwdTransitionPersistenceManager.add(aggregates); LoggingUtil.logInfo("###################\r\n" + dimensionType + " based TWD transitions Import finished"); - LoggingUtil.logInfo("Totally " + totalValuesCount + " TWD transitions imported"); - LoggingUtil.logInfo("Totally " + aggregates.size() + " TWD transition aggregates imported"); + LoggingUtil.logInfo( + "Totally " + totalValuesCount + " " + dimensionType + " based TWD transitions have been processed"); + LoggingUtil.logInfo( + "Totally " + aggregates.size() + " TWD transition aggregates imported for dimension " + dimensionType); } private double getFinalBucketThreshold(double currentBucketThreshold, double nextBucketThreshold) { diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/RaceWithWindSourcesPersistenceManager.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/RaceWithWindSourcesPersistenceManager.java index d047de632b4..297e09ca2de 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/RaceWithWindSourcesPersistenceManager.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/RaceWithWindSourcesPersistenceManager.java @@ -29,8 +29,8 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa public RaceWithWindSourcesPersistenceManager() throws UnknownHostException { Document indexes = new Document(RaceWindJsonSerializer.START_TIME_POINT, 1); - indexes.put(RaceWindJsonSerializer.END_TIME_POINT, 1); - getCollection().createIndex(indexes); + indexes.put(RaceWindJsonSerializer.END_TIME_POINT, -1); + getDb().getCollection(getWindSourcesCollectionName()).createIndex(indexes); } @Override diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/WindSourcesPersistenceManager.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/WindSourcesPersistenceManager.java index 0b8fbfc27ae..d80c152d635 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/WindSourcesPersistenceManager.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/WindSourcesPersistenceManager.java @@ -23,7 +23,7 @@ public class WindSourcesPersistenceManager extends AbstractPersistenceManager { DurationBasedTwdTransitionImporter.main(args); - Thread.sleep(1000); - AggregatedDurationBasedTwdTransitionImporter.main(args); }); executeInThreadPool(() -> { DistanceBasedTwdTransitionImporter.main(args); - Thread.sleep(1000); + }); + awaitThreadPoolCompletion(); + AggregatedDurationBasedTwdTransitionImporter.createPersistenceManagerAndEnsureIndex(); + AggregatedDistanceBasedTwdTransitionImporter.createPersistenceManagerAndEnsureIndex(); + executeInThreadPool(() -> { + AggregatedDurationBasedTwdTransitionImporter.main(args); + }); + executeInThreadPool(() -> { AggregatedDistanceBasedTwdTransitionImporter.main(args); }); awaitThreadPoolCompletion(); @@ -74,18 +77,14 @@ public class SimpleModelsTraining { } while (JOptionPane.YES_OPTION != askDataCleaningFinished(AggregatedSingleDimensionType.DISTANCE)); showInfoAboutIntervalAdjustments(DistanceBasedTwdTransitionRegressorModelContext.class, DistanceValueRange.class); - DurationBasedTwdTransitionStdRegressorTrainer.main(args); - DistanceBasedTwdTransitionStdRegressorTrainer.main(args); - Thread.sleep(1000); - ExportedModelsGenerator.main(args); - LoggingUtil.logInfo("Model training finished. You can upload the generated file to a server instance."); } private static int askDataCleaningFinished(AggregatedSingleDimensionType dimension) { Object[] options = { "Continue with model training", "Restart Graphical Tool" }; int res = JOptionPane.showOptionDialog(null, - "Have you finished the data cleaning for " + dimension + " dimension?", "Continue with model training?", - JOptionPane.DEFAULT_OPTION, JOptionPane.INFORMATION_MESSAGE, null, options, options[0]); + "Have you finished the data cleansing for the " + dimension + " dimension?", + "Continue with model training?", JOptionPane.DEFAULT_OPTION, JOptionPane.INFORMATION_MESSAGE, null, + options, options[0]); switch (res) { case 0: return JOptionPane.YES_OPTION; @@ -96,22 +95,22 @@ public class SimpleModelsTraining { } private static void showInfoAboutDataCleaning(AggregatedSingleDimensionType dimension) { - JOptionPane.showMessageDialog(null, "Now, clean the data for " + dimension - + " dimension. Remove instances from MongoDB collection \"" + dimension.getCollectioName() - + "\" which do not make sense. E.g. values which are represented by a small number of supporting instances (see histogram), values which cause implausible zig zag sections within zero-mean standard deviation curve and etc. Close the graphical tool, when you are done to resume the model training."); + JOptionPane.showMessageDialog(null, "Now, clean the data for the " + dimension + + " dimension.\nRemove instances from MongoDB collection \"" + dimension.getCollectioName() + + "\"\nwhich do not make sense and cause implausible zig zag sections\nwithin \"Zero mean sigma\" curve.\n\nClose the graphical tool, when you are done to resume the model training."); } private static void showInfoAboutIntervalAdjustments(Class classToAdjust, Class valueRangeEnum) { JOptionPane.showMessageDialog(null, "Now, open the source code of the class \"" + classToAdjust.getName() - + "\". Scroll down to the definition of the inner enum \"" + valueRangeEnum.getSimpleName() - + "\", read its JavaDoc and adjust its interval definitions so that each interval can be learned by the adjusted regressor model configuration with minimal error. Press OK ONLY after you are done."); + + "\".\nScroll down to the definition of the inner enum \"" + valueRangeEnum.getSimpleName() + + "\",\nread its JavaDoc and adjust its interval definitions so that\neach interval can be learned by the adjusted regressor model configuration\nwith minimal error.\n\nPress OK after you are done."); } private static void awaitThreadPoolCompletion() throws InterruptedException { executorService.shutdown(); - boolean success = executorService.awaitTermination(2, TimeUnit.HOURS); + boolean success = executorService.awaitTermination(24, TimeUnit.HOURS); if (!success) { - new InterruptedException("Thread-pool was terminated after two hours waiting time"); + new InterruptedException("Thread-pool was terminated after 24 hours waiting time"); } Thread.sleep(1000L); } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTrainingPart2.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTrainingPart2.java new file mode 100644 index 00000000000..7388eca611d --- /dev/null +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTrainingPart2.java @@ -0,0 +1,22 @@ +package com.sap.sailing.windestimation.model; + +import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionStdRegressorTrainer; +import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionStdRegressorTrainer; +import com.sap.sailing.windestimation.util.LoggingUtil; + +/** + * + * @author Vladislav Chumak (D069712) + * + */ +public class SimpleModelsTrainingPart2 { + + public static void main(String[] args) throws Exception { + DurationBasedTwdTransitionStdRegressorTrainer.main(args); + DistanceBasedTwdTransitionStdRegressorTrainer.main(args); + Thread.sleep(1000); + ExportedModelsGenerator.main(args); + LoggingUtil.logInfo("Model training finished. You can upload the generated file to a server instance."); + } + +} From 3b29f34b22803838a728c3f693e5d4eefcac0b80 Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Sun, 24 Feb 2019 21:05:44 +0100 Subject: [PATCH 23/24] Improvement of howto for model training --- wiki/howto/windestimation.md | 28 +++--- wiki/howto/windestimationAdvanced.md | 80 ++++++++++++++++++ ...logAfterDataCleansingDurationDimension.jpg | Bin 0 -> 12695 bytes ...stingDataCleansingForDurationDimension.jpg | Bin 0 -> 20355 bytes 4 files changed, 94 insertions(+), 14 deletions(-) create mode 100644 wiki/howto/windestimationAdvanced.md create mode 100644 wiki/images/windestimation/confirmationDialogAfterDataCleansingDurationDimension.jpg create mode 100644 wiki/images/windestimation/dialogRequestingDataCleansingForDurationDimension.jpg diff --git a/wiki/howto/windestimation.md b/wiki/howto/windestimation.md index 25c9812657f..83412cc7104 100644 --- a/wiki/howto/windestimation.md +++ b/wiki/howto/windestimation.md @@ -1,29 +1,29 @@ # Training of internal Wind Estimation models -This document describes the generation process of Machine Learning (ML) models which are used internally by wind estimation. It is highly recommended to proceed this howto step by step considering the order of sections. +This document describes the generation process of Machine Learning (ML) models which are used internally by wind estimation. It is highly recommended to proceed this howto step by step considering the order of sections. At the end of this howto, you will generate a file containing the representation of internal models used by ``com.sap.sailing.windestimation`` bundle. You can use this file to update the wind estimation models of a running server instance. If you are interested in a more advanced tutorial which requires all the execution steps contained in ``SimpleModelsTrainingPart...`` classes to be executed manually, then you might be interested in [Advanced Guide for training of internal Wind Estimation models](./windestimationAdvanced.md) ## Prerequisites To complete the training process successfully, you need to make sure that you have the following stuff: * A complete onboarding setup for SAP Sailing Analytics development -* MongoDB (**3.4 or higher!**) is up and running (same MongoDB instance as required in onboarding howto) +* MongoDB (**3.4 or higher!**) is up and running (can be the same MongoDB instance as required in onboarding howto) * At least 100 GB free space on the partition, where MongoDB is operating * Installed graphical MongoDB client such as MongoDB Compass (Community version) * 16 GB RAM -* ~24 operation hours of your computer +* 24+ operating hours of your computer ## Model training process -1. Run ``com.sap.sailing.windestimation.model.SimpleModelsTraining`` as normal Java Application. This program downloads all the necessary maneuver and wind data, pre-processes them and initiates training of maneuver classifiers. -2. Make sure that the launched program does not get termined by an uncaught exception. Wait until graphical info dialog shows up which requests you to perform data cleansing for duration dimension and press OK. - ![Screenshot of graphical info dialog requesting to perform data cleansing for duration dimension](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png "Screenshot of graphical info dialog requesting to perform data cleansing for duration dimension") - A Swing-based GUI-Window must open with two charts, one XY-chart where the x-axis represents **seconds**, and the y-axis represents TWD delta-based series measures (e.g. standard deviation or mean). Below the chart, a histogram for the data points of the XY-Chart is provided. You can zoom-in and zoom-out in each of the chart by mouse dragging. Be aware that currently, the zoom level of both charts is not synchronizing. +1. Run ``com.sap.sailing.windestimation.model.SimpleModelsTrainingPart1`` as a normal Java Application. After this, all the necessary maneuver and wind data will be downloaded, pre-processed and maneuver classifiers get trained. +2. Make sure that the launched program does not get termined by an uncaught exception. Wait until graphical info dialog shows up which requests you to perform data cleansing for duration dimension. + ![Screenshot of graphical info dialog requesting to perform data cleansing for duration dimension](../images/windestimation/dialogRequestingDataCleansingForDurationDimension.jpg "Screenshot of graphical info dialog requesting to perform data cleansing for duration dimension") + Press OK. Afterwards, a graphical window must open with two charts. The top chart is an XY-chart where the x-axis represents **seconds** and the y-axis represents various TWD delta-based measures (e.g. standard deviation or mean). Below the XY-chart, a histogram for the data points of the XY-Chart is provided. You can zoom-in and zoom-out in each of the charts by mouse dragging. Be aware that currently, the zoom level of both charts is not synchronizing. ![Screenshot of graphical wind data visualization tool for duration dimension](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png "Screenshot of graphical wind data visualization tool for duration dimension") -3. Open your graphical MongoDB client and connect to ``windEstimation`` database hosted by your local MongoDB. Open the collection with name ``aggregatedDurationTwdTransition``. Within the collection you will see all the instances/data points visualized in the previous step. The attribute used for the x-axis is represented by ``value``. +3. Open your graphical MongoDB client and connect to ``windEstimation`` database hosted within your local MongoDB. Open the collection with name ``aggregatedDurationTwdTransition``. Within the collection, you will see all the instances/data points visualized in the previous step. The attribute used for the x-axis is ``value``. ![Screenshot of MongoDB Compass with opened aggregatedDurationTwdTransition collection](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png "Screenshot of MongoDB Compass with opened aggregatedDurationTwdTransition collection") -4. Delete all the instances within the collection which do not make sense. For this, use the data visualization tool from step 2 to identify such instances. Some of the instances are not representative due to the small number of supporting instances which is visualized in the histogram. Such instances can produce unreasonable bumps in the XY-chart. The desired output of this step is that the series curve ``Zero mean sigma`` looks smooth and always growing, like depicted below: +4. Delete all the instances within the collection which do not make sense. For this, use the data visualization tool from step 2 to identify such instances. Some of the instances are not representative due to the small number of supporting instances which is visualized in the histogram. Such instances can produce unreasonable bumps in the XY-chart. The desired output of this step is that the curve ``Zero mean sigma`` looks smooth and always growing, e.g. as depicted below: ![Screenshot of graphical visualization tool of duration dimension with after data cleansing](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png "Screenshot of graphical visualization tool of duration dimension with after data cleansing") - Use the ``Refresh charts`` button as often as needed to update the charts with the modified data in MongoDB. Close the graphical visualization tool window after you are done with data cleansing to resume the training process. Confirm the confirmation dialog after you have finished the data cleansing of duration dimension: - ![Screenshot of confirmation dialog for finishing the data cleansing](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png "Screenshot of confirmation dialog for finishing the data cleansing") -5. A new information dialog shows up (do not press OK yet!) requesting you to open the source code of the class ``com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext``. Open it and scroll down to the definition of the inner enum ``DurationValueRange``. The enum defines the intervals for which a separate regressor model will be trained. Read the Javadoc of ``DurationValueRange`` and adjust the intervals accordingly in order to allow the regressor model to learn the ``Zero mean sigma`` curve with minimal error. You can also configurate the polynomial which will be used for regressor training. Make sure that there are at least 2 data points available within each interval. The datapoint with x = 0, y = 0 will be created automatically. Press OK in information dialog after you are done. -6. A graphical info dialog shows up which requests you to perform data cleansing for *distance* dimension. Press OK. All steps for data cleansing for the distance dimension are very similar to the data cleansing steps step 2. until step 5. for the duration dimension. Thus, consult these steps to complete data cleansing and models configuration for the distance duration. The unit used for distance representation is **meters**. The collection name required in step 3. is ``aggregatedDistanceTwdTransition``. The class required in step 5. is ``com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext`` and its inner enum is ``DistanceValueRange``. -7. Wait until model training finishes and the program terminates normally. A new file with serialized representation of internal wind estimation models should be located in ``./windEstimationModels.dat``. The absolute path of the file must be printed in the console output of the program. You can upload the file via HTTP POST to http://sapsailing.com/windestimation/api/windestimation_data (see ``com.sap.sailing.windestimation.jaxrs.api.WindEstimationDataResource``) to update the wind estimation of a server instance. + Use the ``Refresh charts`` button as often as needed to update the charts with the modified data in MongoDB. Close the graphical visualization tool window after you are done with data cleansing to resume the training process. A confirmation dialog shows up. Confirm it by pressing *"Continue with model training"* button. + ![Screenshot of confirmation dialog for finishing the data cleansing](../images/windestimation/confirmationDialogAfterDataCleansingDurationDimension.jpg "Screenshot of confirmation dialog for finishing the data cleansing") +5. A new information dialog shows up (**do not press OK yet!**) requesting you to open the source code of the class ``com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext``. Open it and scroll down to the definition of the inner enum ``DurationValueRange``. The enum defines the intervals for which a separate regressor model will be trained. Read the Javadoc of ``DurationValueRange`` and adjust the intervals accordingly in order to allow the regressor model to learn the ``Zero mean sigma`` curve with minimal error. You can also configure the polynomial which will be used for regressor training. Make sure that there are at least 2 data points contained within each configured interval. The data point with x = 0, y = 0 will be created automatically within model training procedure. Press OK in information dialog after you are done. +6. A graphical info dialog shows up which requests you to perform data cleansing for the *distance* dimension. Press OK. All steps for data cleansing for the distance dimension are analogous to the steps of the duration dimension described from step 2. until step 5. Thus, consult these steps in order to complete the data cleansing for the distance dimension. The unit used for the distance representation is **meters**. The collection name required in step 3. is ``aggregatedDistanceTwdTransition``. The class required in step 5. is ``com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext`` and its inner enum is ``DistanceValueRange``. +7. Run ``com.sap.sailing.windestimation.model.SimpleModelsTrainingPart2`` as a normal Java Application. Wait until the model training finishes and the program terminates normally. A new file with serialized representation of internal wind estimation models should be located in ``./windEstimationModels.dat``. The absolute path of the file must be printed in the console output of the program. You can upload the file via HTTP POST to http://sapsailing.com/windestimation/api/windestimation_data (see ``com.sap.sailing.windestimation.jaxrs.api.WindEstimationDataResource``) to update the wind estimation of a server instance. If you changed the source files of ``DurationValueRange`` or ``DistanceValueRange``, then you will need to update ``com.sap.sailing.windestimation`` bundle of the server instance which is meant to receive the new wind estimation models. 8. Optionally, run ``com.sap.sailing.windestimation.evaluation.WindEstimatorManeuverNumberDependentEvaluationRunner`` as normal Java Application to evaluate the wind estimation with the new trained models. The evaluation score will be stored as CSV in ``./maneuverNumberDependentEvaluation.csv``. \ No newline at end of file diff --git a/wiki/howto/windestimationAdvanced.md b/wiki/howto/windestimationAdvanced.md new file mode 100644 index 00000000000..54dc2538b05 --- /dev/null +++ b/wiki/howto/windestimationAdvanced.md @@ -0,0 +1,80 @@ +# Training of internal Wind Estimation models + +This document describes the generation process of Machine Learning (ML) models which are used internally by wind estimation. It is highly recommended to proceed this howto step by step considering the order of sections. At the end of this howto, you will generate a file containing the representation of internal models used by ``com.sap.sailing.windestimation`` bundle. You can use this file to update the wind estimation models of a running server instance. If you are interested in a simpler tutorial with less execution steps thanks to automation, then you might be interested in [Simple Guide for training of internal Wind Estimation models](./windestimation.md) + +## Overview +In total, there are the following three categories of ML models used by wind estimation: +1. **Maneuver Classifiers** +2. **Regressors** of TWD delta standard deviation for the dimension **duration** +3. **Regressors** of TWD delta standard deviation for the dimension **distance** + +Each of the model categories are composed of multiple models where each model targets a specific context. A context for a maneuver classifier is determined by the following attributes: +* Maneuver features + * Polar features enabled: yes/no + * Mark features enabled: yes/no + * Scaled speed features enabled: yes/no +* Boat class filtering for the data on which the classifier is trained, such as a specific boat class, or with all boat classes included + +The context of regressor models is represented by its assigned input interval responsibility, e.g. [0 seconds; 62 seconds) for duration, or [80 meters; 1368 meters) for distance. + +Each of the ML model categories must be trained individually. The common workflow looks as follows: +1. Get the training data from REST API of sapsailing.com +2. Preprocess data +3. Train the model category + +For each of the steps, appropriate Java classes must be executed per *Run with...->Java Application*. All referenced classes are located in *com.sap.sailing.windestimation.lab* Java project. Each class execution must finish without termination due to uncaught exceptions before proceeding to next instruction. You can skip the training of a model category if you do not want to update the models on server of that category. After model training, A new file with serialized representation of internal wind estimation models should be located in ``./windEstimationModels.dat``, which is normally */path/to/workspace/com.sap.sailing.windestimation/trained_wind_estimation_models* if you start the training classes in Eclipse per *Run with...->Java Application*. + +The details of the training process for each model category are described in the following sections. + +## Prerequisites +To complete the training process successfully, you need to make sure that you have the following stuff: +* A complete onboarding setup for SAP Sailing Analytics development +* MongoDB (**3.4 or higher!**) is up and running +* At least 100 GB free space on the partition, where MongoDB is operating +* Installed graphical MongoDB client such as MongoDB Compass (Community version) + +## Get the training data from sapsailing.com +The following steps import all the data required from sapsailing.com into the local MongoDB. These steps constitute a preprequisite for training of all ML model categories: +1. Run *com.sap.sailing.windestimation.data.importer.ManeuverAndWindImporter* +2. Run *com.sap.sailing.windestimation.data.importer.PolarDataImporter* + +## Maneuver classifiers training +1. Run *com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverClassifierTrainer*. Within the this step, the maneuver data is preprocessed and all maneuver classifiers are trained for each supported context. +2. Optionally run *com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverClassifierScoring* to print the performance of the trained classifiers. After this step, a list with macro-averaged F2-score of each trained classifier will be stored in *./maneuverClassifierScores.csv* + +## Duration-based TWD delta standard deviation regressor + +1. Run *com.sap.sailing.windestimation.data.importer.DurationBasedTwdTransitionImporter* +2. Run *com.sap.sailing.windestimation.data.importer.AggregatedDurationBasedTwdTransitionImporter* +3. Run *com.sap.sailing.windestimation.datavisualization.AggregatedDurationDimensionPlot* to visualize the wind data. A Swing-based GUI-Window must open with two charts, one XY-chart where the x-axis represents **seconds**, and the y-axis represents TWD delta-based series measures (e.g. standard deviation or mean). Below the chart, a histogram for data points of the XY-Chart is provided. You can zoom-in and zoom-out in each of the chart by mouse dragging. Be aware that currently the zoom level of both charts is not synchronized +4. Open your graphical MongoDB client and connect to *windEstimation* database hosted by your local MongoDB. Open the collection with name *aggregatedDurationTwdTransition*. Within the collection you will see all the instances/data points visualized in the previous step. The total number of the points must not exceed 100. +5. Delete all the instances within the collection which do not make sense. For this, use the data visualization tool from step 3 to identify such instances. Some of the instances are not representative due to the small number of supporting instances which is visualized in the histogram. Such instances can produce unreasonable bumps in the XY-chart. The desired output of this step is that the curve ``Zero mean sigma`` looks smooth and always growing. Use the ``Refresh charts`` button as often as needed to update the charts with the modified data in MongoDB. +6. Open the source code of the class ``com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext``. Open it and scroll down to the definition of the inner enum ``DurationValueRange``. The enum defines the intervals for which a separate regressor model will be trained. Read the Javadoc of ``DurationValueRange`` and adjust the intervals accordingly in order to allow the regressor model to learn the ``Zero mean sigma`` curve with minimal error. You can also configure the polynomial which will be used for regressor training. Make sure that there are at least 2 data points contained within each configured interval. The data point with x = 0, y = 0 will be created automatically within model training procedure. +7. Run *com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionStdRegressorTrainer* +8. Verify the trained regressor functions. They are printed in the console output of the previous step. For instance, you can visualize the polynoms by means of https://www.wolframalpha.com/ + +## Distance-based TWD delta standard deviation regressor + +The steps of this sections are similar to the steps of the previous section. It is recommended to traverse through the previous section before starting with this one, because due to similarity of the steps, the similar steps in this section are described with less details and hints. + +1. Run *com.sap.sailing.windestimation.data.importer.DistanceBasedTwdTransitionImporter* +2. Run *com.sap.sailing.windestimation.data.importer.AggregatedDistanceBasedTwdTransitionImporter* with at least 10 GB JVM memory. +3. Run *com.sap.sailing.windestimation.datavisualization.AggregatedDistanceDimensionPlot* to visualize the wind data. Here, the x-axis of the XY-chart represents **meters** +4. Open your graphical MongoDB client and connect to *windEstimation* database hosted by your local MongoDB. Open collection *aggregatedDistanceTwdTransition* collection. Within the collection you will see all the instances/data points visualized in the previous step. The total number of the points must not exceed 100. +5. Delete all the instances within the collection which do not make sense. +6. Open the source code of the class *com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext*. Scroll down to the definition of the inner class/enum *DistanceValueRange*. The enum defines the intervals for which a separate regressor model will be trained. Adjust the intervals accordingly in order to allow the regressor model to learn the data curve with minimal error. +7. Run *com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionStdRegressorTrainer* +8. Verify the trained regressor functions. They are printed in the console output of the previous step. + +## Generate the models file + +Within this section, all the trained models produce within previous sections are aggregated and stored as one single file. + +1. Run ``com.sap.sailing.windestimation.model.ExportedModelsGenerator``. Wait until the model training finishes and the program terminates normally. A new file with serialized representation of internal wind estimation models should be located in ``./windEstimationModels.dat``. The absolute path of the file must be printed in the console output of the program. You can upload the file via HTTP POST to http://sapsailing.com/windestimation/api/windestimation_data (see ``com.sap.sailing.windestimation.jaxrs.api.WindEstimationDataResource``) to update the wind estimation of a server instance. If you changed the source files of ``DurationValueRange`` or ``DistanceValueRange``, then you will need to update ``com.sap.sailing.windestimation`` bundle of the server instance which is meant to receive the new wind estimation models. + + +## Model Evaluation + +This step is optional. However, it is recommended, to evaluate the performance of wind estimation with new trained models. + +1. Run ``com.sap.sailing.windestimation.evaluation.WindEstimatorManeuverNumberDependentEvaluationRunner`` to evaluate the wind estimation with the new trained models. The evaluation score will be stored as CSV in ``./maneuverNumberDependentEvaluation.csv``. \ No newline at end of file diff --git a/wiki/images/windestimation/confirmationDialogAfterDataCleansingDurationDimension.jpg b/wiki/images/windestimation/confirmationDialogAfterDataCleansingDurationDimension.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8250cb2201fb4765d939f14f8a9f26dafc35ffa3 GIT binary patch literal 12695 zcmeI2cT|(jx9<~r5u_I>(jh=lKp+S&9fZ(Bq*v)BKtNCw6p-Fa0!Wn-AQb7LBZ%}S zT|$wLR1ri3M6U0>N6+t`^R9F6UF-gNCM#>Q^31cpdp>*iWM=YQeYjc%&_Fe`Gyp&# z008{;0$j}lQ~@MJ#3aN-BqYQnq@*Nd6f_hR_13U$G z5CVAs1hhaxTHw_?0Q)Z|2?&9I`}?~PlMs=T5dsN*UDc)m00{|!gv6AT1Vlune|L(2 zkO)9bOF~D_%^-dsHBQRN#LS~?2rrtLo1kR%Ws`uHIRUakoeanN!ni@f#f74gaE?-bsY$x{iVb$4ol7-Kcu5)R5nBs z^Vo&dbV=S>kx-eay}|>o{c1`;OGpb)1bocY*E9V%fpgBQv#|8**Qs>3gfj}E2(TAC zNO<5s>n%C;#v*bXEm3G}c*bac2~OdO!D=xsnoHJ4EP<$+^1~fLh6>OQZ1+>4{dzBW|3J(NvuNT!$e4L9+DZB8Hq7D# zgzsXAK*E4}G!fRqbZ!|gA(_ilkTt_(mZq7`!YnX-Bk++;)X$ZkB1}CRq|QYcwiX2u z%*u`#NROkcEPU3&KqujV>u+{^3@$mRn4V_aD+$Fve^qH^*fAKESouMtsP2P}8yrh- z>r;65uMQP6SnUTEHeqtDdA^i3UVzYg%<=o2_h5Vk2lk~thB+V}^pvX4PZaEO_6adf z5Drp!7|wW&J%Oa59aZloUuxth=wLOi|NTFU*_`md?p)V6@M?*~patY~`jDrbAXWn? z8x*C9!lKv3Faiv)E(`3noPUVXXn;Sx4W&3sM`<>MTyr-j*5)I z-QCqA)B2vM41f4CG(;A&J5o)?&%cq+4<=0T!^#8Y7u;AMcLo(cDNxZ#`z3Y#AEXT9 z+)Zg8UZz>C==;naW~JG8MSMjA3{o&2chFcNVpF_4BqV!AheJhhr`S%#(s@4-Kk4oI zZFJmvGK(wRnV26U|8c3_(UHhPPUC07vzX4*l-v6ru`M{ee+pM(PKCYd?Cr0UNn4;f zanuz+NbTMXUK2(~GGPJgOY>>35CpJ0nCytPR1Y=&rNDOVpyrV;*Nd1iX@k{;;Rqd? zG^kZ1j1ol{WhTND#@|eL7&l6I&hu>K{K=-=-w7r{`8Gp_G7w~(m8z4!y&aY>K#zGk zoKp;+6Ef27lsATW^Ua|XmwZ!^3SM+aakn>2k7HJa2Pvp#ffQ~|lg%};|Wf^`& zE57i#$6HT_-8tq*{T2U|Q>K*DxOsQFb*qD#A&xI_l1GLd%G$x+4qpt-kEj`vfc*gH zyy`)RR{x$Az@#i}<{R4?7%%-QLXzts7r2hwV)jB7jy>6FDP+Ms#$F<-e{5Wh083w zkTRr~U~@yflmlrQLK!d&C@j%zbsv(WQDs{cM&88H-}!^YcX{#*W^;3JOq6?Lt$^L%_!u94qV z6T$%H4}u9@O*xGydZ7tWa=w%oNA4W>XP8zQ1^kxaBJpm>0A$KpY zgaHuJl#ez^vYYRzo0^T3Q(i!Xs6-`g`v_Z^P!uPhzYM~8GqII_7oEF2NIk67fIL~Zz z6_Pz9EQH^(^IG;+1v8@?`0lX>k$S8Hn*0P&L(=bjgDeY5Y_-fDN?Sy7vx!yE zkHS z?L=4am)ZHdsYadgrQOt$zmDDj^F{<*0dCCQsm#cJ({by_*p7`mQlqPb(f`yVRm9X4 zJ0ih+gIV|^1WwXrVFy~)(|er2XrY(cBB+~Dz(q%0B14z73aJ@7Z4DSzaAx;CtI1`o z!5SF&H<&6yGwp7l^d~ly=hnHAojV)2GMMF{VzVY?WRToBZ`o2-XW%>V>I1w?3=5Qj zdN43QwJud`@p--0bkg#iRh!gP;D?DbRd7D$=wYozAXxb+uRpZ`dR^#{r7{^B2H`rU z+H0t{y0mgVak)UTe#0K^n+&CD^wBNtGQ|$&%Q0@MM33HdrRKJ9P8jDZ34R-0?(}kU z6yLx=YHu+tXD_X$rtPHhy?*o5rLkTMR(*X`{%2+jtX{jPVPWa4g3O#F_k%PP`zYGm zy@e`s>&DVvlYEStV76;-mz_2e!|z53)6%1<*xTpyWVF`JGPJnJan15DoeV8(LM`%G z6xC0qv(8z==F4EPRGfhpbfSVVoX|o7otEEVaalf`+bl7C<2lsYzW2<+caPZ3=>X)Z z_7lEiPhyaRX==bi4S0zhmP1LDh_>`5W&E~XznDh<1TSjFhQipD_>-lm5nGV6XhHTL z;&F$SsnCK@vaj9FKQ%yAm(Rz$-0QbnON0_IYQ#d*)RqaB_edz12+l6~ytgjODOO6n zzMSw~Oqo)o&EE)VjWwh@c7+$~F!UK}jFVGp33G{-$jPs9&5T!w^c2iLjd);h5w;u3WSeI&=a@n?3nDZ0S z7U8Efr$|bdug|400BWsmGJ{1Jx+bMFidjO7AJosq;?M`0n#62cKtc970HK+2xd)pX z(~5b%#tW*pJ#p5edc2R1H!7G!OYmamzFUbW;>Bj|$H$mosrc*7149+?|DP-2%A-j| z*kX2wk5}q;gtT@%acFsF6se=Bf;Ramg&sveS8&KtEJyLP%vKo!{70e5>-$W25ukAo zKyT0|u$VE9C=NvjASPYkhZpd>dw#Z2UHKVwbHCX_prP?mRsP`>Kq2wl!t0OJml<-e zZ_!@5T!0q&zZ#F62|N$Rx~ zD;om%;xbWD%0t)Q0#ES~8dtkG-;Ap^DCVp8x3+VRxqjJoeDg1@*$@)C9ykVMNQc7E zf}sB&D-oMV9_J#=$E821jk_v?7VAD$*o+o_m$mBPxVZhu(wg^t{F`z6Zx9t6ut&3O za4Dp_7}VgHZmC~Wak29`L(J$OJa4DB;l32>C^U~SDzX|9kse=o83WT_pt7YABtA!v z^MB9ZV*>YdPM58&W~t`Wd^AJ^%o*P>Aq=IyzM!t|5+H;wX7MXcUgg)9Jwm?41$HaI zqQ_3LT9svv#lD6y#*M4;5kFoL#AF%h(|1Sn#@}<9_bAuPdcgN>Vj-+$H3qh7(z!+o z;xqX*A%dTj)jBlySK+pG4{tH5`~I}6HJ1F&kT>PFiO0QX+^&2MT2Z(Ha74%#ucTl~ zeZ9y!^;dz$&uL9QbvuR;eyYlo;~F$*q|%-#Txjd`T;n15_^#>8yGGjTD*ggZ{OIk+ zFFKJ}irGgOQoWvSt(tL{y=)}?Y27t*Dk(?bW{(2Q86@9%>s`OG9+hlT)nyJmJfk@A z$h)!a8Gn~D>q9LYb$xqEPEm4_w-WEK%Az{@zIx(nNa#c)ab5l3PkAEP+w*vBT?h)n|)@C zq%+YL=4k9iI^I*<%aAiI+HhTl9I*hzLEc(FazP?Dnp8&}=$OKiLm$V5*f^SRaPD~F zSA#QNxc!{#?+nD5vEKY~San?LGSdtz#hjG6SXiSA18=tU*Raf7SbjRb7z1U!&2`n> z@X=)BG+f@ng+>_&|JqVGGkP6(Po-jSNm#v*zT?*!Ji#n*cWxd=Q(a9EEPws_6(H~Wj2;r!(p3$dxj=lwI z5LHoW#Jo`q*Au!VEk$plnaqamW?Hd+zgUrq=d00Tj0GV5??Dr98Iu%OHq;15`Xc|* zP?>zh(&C*TtG2n6j9I^PZp2}yr`XO$ZqUcLoK4urz94dIBj|bT6@V#g7+?9OJSbn; z1sdUCBO=~TZE7xxZ?(eV$AQ$BLAxKgRcBM^)XKqued&g_Lk4?yC=2ERi zA}A&7L`hOm!I(PGuQQ>9N%u}cS!k`uiOb~VnwyYO!MsCX#>UciTZ$R%nU<#U9@g|M z+>}>da%9Z>j^dk{Cs<|8S=E$<>2S_%YV`@n=5$=K8kFhyDW3=Nmctf{Y}ck#F_;Og zycAu)<8-g}6tn5bFTKV4c}ej6qeYnfrb=}^UmX2Ahx@I~6(p-W7Q62&CH7~w9;j$* zJ(KtVMWL#^9EAy41QS2@jin%LQo^cdiyw2Nd>s>x?m+ys(1TAw5-D$#zeBf7>6IL2v86bo}#@LnHxwtE!# zLK99JwzC@2(JgW)kT9uzDD!Y$E7y$8Aaxn(dhIv^u3Q$pZECz*B$t{>OTNFF-(jyH zt@*(l0DYVwPqqrVFMI_+G?zE~wXlVHY?-ew)HKK(eU?BMHKDnMjqlfu5+ryB9N+i+ z))7TN=h*{pRm)Lo)bv?e>nWBYpEI{#8`R}~J3DvadN)6UuE0LrP66usP9)SQ zd!1tVz?tA-kSm>0+IDr)WzD-GlBK47{>xQOW%G#1uxn>-u;{fHV{pii6J|D`wfk()sUMy;gHFexw^# zWG5C%_nrbTOss`pJFbfIgZ+$I!mR0 zAF`T8*t{Nk0;`vt)Pw*y`IfKCRw|a#4W2wqFnzmjWmJmT zz>;ejxW)-mp%cj08c^))Nforcub*rVu3rXV`0q7F4uOa8Nv8AK+Ek(Auxh2Z-%Ea_ z?q7hNlAl5#@N2g#fHka0icYeYuPH6VmAH?oFc=k*qQ4k3@ugx`>^N-&e?YrgJS3j= zHrr8YD5c1bMe%G<7|*RG+zII)75nS#~2ooW#-?Qtp z$0Qq)^QNDQ-j7HDZv7ZBTe03hb{1ea0}fEcMTc-xu(~>v8zU1Sn_F1n_vu#B>#dfN zcBaMr{pU|4c!9*dQ_-Nu=smt7nzr@zYs;y_k4F~vP9sl>N^a66bB!hJfpje_gIpZ= zWi_8aM%{@E56`#M(|TJg;s9IxX>nfrRiTX7BWpCj(QKn9R?6qvOOW|XH+d5q*jr@1 zHy-}j!KD9g+PLieQ}50EjRl&u{k=Cy9Leqe6z$&IQnixGAI23RvqKfyq-^Vs1RsUe%|t~H-~xlHJN&8 zDQ}=bTR6gF54GmPe#jM<_?LUEUUmoTU5{SLN*INh*HH+E$O5w^1wxFNU&+B91d(40 zsbVoMubi1Lsc`B%SMwV{Kh!ya4Bb*piHm&XHac#e1|W_2ixK2yb+=xz_$7kvprqkL z%EW7JRK&vE9J16)J{0bqTpvb5F5fM?Dsq@(;`FXy57Q@ZAFWTW1L;E4LRF&7m03aZ zvOf=aN_XNPqZi8Ny*`W=(V zSMqf9Vf~8=n9BOnGk(*x#O_IxypC%Jbwl&aPA2{he&}9-yACb=5nU-qd>Ve~bnkyw zhZljgRz`u0$)pDO63x(IEmyyRnz*JFA!DZZT#U{vHzD2-MLz?nLfDDw(I$SsQQ$+g zW7^x;FYJz9O*=-l69%ssDr~cZH3I5P@s#>j9#75*sWp<9oPM47KZ4_{2Xnw!kJ4iT zCMKn=u5v2=gz|n@bWDgw*%-XC@L`K;gA9l7(jx!qIBQ+){rc5Wx?I4T;Mj=uIA4L0 zn}wDiw+M{-1S9Q3O-&HY_vE_xv2oFq@k>v)0Wn3l908xIYMVrG)2=x!t(+`d=WV(a zM$}ezxP62>-CVK5&>Lml(J}6RyGMSae{ooFTDS5@r!}$k}Cn%Wf@T*UHX0 zCKiqw$^YOouZ?q8YC@tQ7OZC@O1C{|Bf1GYD zQ#cNI;pA8%b{)jFMX1Y7g`IAyy9B8KVTpvpA_j{UDGu{uLN#Uwv>pfIK+34nAy@6B z^e&_wku&#AS^K;^LDq8hv^%>3c{BYlZg~8a^Xym= zHIQxN6YPe?{f!apN*SSa7p-)!$MSEZ^}J!QSKhFqBnnM=GA>&9>-{|S2S5F8b`3VT zTBtn*Cr6E|E*V?sv)7mC~X9u{`6sXiTBl)TF ziT0<;tRtN(01)Q+3KELG_QEjp-@lq30-y05n7|X)BCFxe{;d9|3?4_6qNzD=$J;!f z5r*6h<_ek)U6-4x!2NiIzfQm_)hc+AQlN+vu}!-ESKtTzX~z6pLWFnciLEEUNv}vJ z(a_Nk3%?^F4#GU1Df_n#qp_$9&FVlh&mGOZ{uGs5KimZR?mcls!oN3}LeWrI6bkcS zR(G)x*-2dLg%1H!pS-{;zhV{O$*T7=DndYQ+UH#-2?*ClzVz ze&a;F8~g6Lxx(OVQAf<&8@{naMzG=Ce*rY|KLP3d4G0QDZ?-dtl{O7E-V0y)T8*W1 z|_RWB|n*N*_Mr7{1| zeWD`7AWE<6R`QWbb@z$$spWo2pw#us?Yu=_Tpa?BbDU9GYA~Z0c)x&AhrbZzo`6-N zE5WGcS2O`Le~N^36P;|^_bN#H zc*RbFqaG-dQJe);xyrfp>yMHo3QAjhUZ7I;@$xk~ zf254WPZ~n+2~uev<)ztjQ`o&Le?|gjNJ)Yr`x1zsF~5FzWGjAKTg35b*+aU!+`Q2t zJSRA%5|!jrVf*Cqc#lWilhWKj-pZ|%26^fH+|6llLYGz=zEaEH3-giHM&PWKM4&@q zL`Y2t(XSQ9n&GH>``xq+5PgI4X|m+d*^qam5oo{58~x)DU>4rr&#`Y;F|KqRgxWcI+P&&Kwc&1-Y9~P zm%`>pA>Ub@naIp}ko0$7wOSni{(-6oq>;Ji_@^#Nm<_U)+&vbPrJRGdpH@BXHaK5; zc{wZR`bVK6-fWR+Ky6rV**00xPJ4c>Xo_`w>oquT}?CJ5LAC<8Xe*z*ioWmQEEhjxj zuv6O)E5G78+Kv(5eN}kz57d1x--~K6Y&}pwH{TF026CsrRE?)*b2Z-E=YdzOY;RC z=)5K4&PvIoh)qvS<3;YbeXxL5gGpa%i6uR6f9kp$->W;gO|p{_^66;>OP7Tpg74Rg z{fFXA2?EBpUIrmz&gDI2G+;s#yX)%R6JLL`pGj*)q2h*dyjBi#Y=L$e@}f6 zHuX%GVu0Iz_lf-UZd2R&I(6}zSnF^1V9}3~UXgFOUIFaN&>tI5)b$;MrZ`uRca!dB z{`z%d&a(`en*HMD`|RU1BOCUXX6^Px{P69Y>&Tg(yY9|jj(ls7UyIZpa)2xuVH)v+ zgIfyCgxIRpNo2rDKZovfrUFFEM$mC0-KAHgm#2EW3DZY6*8nmE1iyrkj?blFf1L14 ziZ32nf+n!+mh>N13O)3TWHVR!X-5WAi89Fu_T3yYx(9}b=DPnenH4+sT4~4qdjp%# zL9A2EJ7)U^F@scjC72=Odz|GKw+jeQM9mEu7z4&73-s31D*Ed-V<{xk8gOyd%ZQ%B zEsPk$o=_ZnuojC#OfG(0<76iE>^BnKo~>CfKdB$)5vhv`Z+YkQf~U?ZbWu^IeoeQ+ z9Jcs`3I}-|&OifWS-F_cPm&GAs`sUvz`D%?2(P`nXO1|$9uS72E0BN#fP#YKF;Cs< zohqtk{v%KKt`SG-rIGg5A13>!pw`X>=`70zp~5YXw3LcCnt+hL4BNH2O4kSet6aq@ zr#oW&{DzC#?nbugzg@>_b@dY^zxY&m&dYfEoAVziOs5aC2-95a5S}j+#Hj`<@8Gg+ z6~d_HD9pXRAG14iB5lkLB3)8lHju{e11RGnUnX(5`&)AflKMkK#)yHnL0rfZTs--} zqSk+kWk+L$hn`h@vqT?g36!0d#wNz+s2eF>ZA^Msd5qdEuIW^*w$8N~+PJU%j8AGr zRm8QIaK9vCf~m+(Ym{Wd2QuYNSo@03Ow02>j*^-ajS{rWBt^R_t_L+~IGjH3Z~hVn zqobyen63{v`{uqQj43|)lIeSEP-q5%ie3f^)7t0u+Stj zsj8sgA^a^fyXo05`Tk7y$y4=PwS4>tZauy`dW11BWipNDSks$9&-U)3%iBj!X~k3U zh+4W%Z`p=+3WNy0mqfu|{>SuRtbfgp{70Kpo0o6OKySQuVx9`NyjZXf@*y>{>8Wzp zK@T^mqrusX6s!9^>9G49=IJ<9H@({c5m^Jne!^hv=#krXtlg!UzfoJ}iyhPF z+Bs-*>Xt%&`AeyS`%a4mudj6`vYv^P^h964P~jjOq17V>AR(x4`Pt4d$4qLjEIE zWfBy`mm)s&*_qwdGe@_IcP8DDTF@+Tm|1$o~>(03q7?Ax~2uwC!_w>cTX7>=)#ZQXBVqJWcq zAiDgW}$N5hvUZ|`GZU2=HD}wh?2bDHY(Rt zIl>T^^vfxw07uET)<%qjZaJe}J1hnI{laB<_%yf44EaDsAYnl+@3roq)4NN$$|s_V zn9;#Uzj2j!&B8a~E-$m-4h$T1P)nZIc*|#?^;s-4{~p%T!dny|pO>?7?X2Vc=+~mZ zJ^w?hT`O=q-lrEeS#E9tZaX*>65`9r@8)$TO{5(&=KRkgP78VYwKI@7a4QX~=$6H+ zw6JQ1nXi<{mygD6=C`e5K*W?PQU@lo&BLo5FM2>1})ye+CXq226b zP}1HhKUPFp7^*sGsOLjU+tAzQAuU&AQBLnDM~jwmRIgiSHwqVl4p6c}AKR0DSvj3b z{2gG>A6~h79T~LWa-U^-KSQ(Lp!ixp01uy+)t*UzCCkkg0dnj- zBC(m=RHl7Ey6VyF`{GW3+h{i#6t3VU%Nu#AZu-Yzz2B@1Sb&e--#Uu)IGU8G>=~aG zaFAZji%?M?f|$QK5gt@58QHo5=uG<>m{l=-(l_UR>tj?`QDfV*bHf}@eW$`sn4HUj zzUL{+vaW_h25YuCAv@m37aUg|sW^WhnREaN5Yl-t)MuVr^e zrgojMuzX{9kuR{F@FzMCvoAwN?+JVD9#g{adB}^VkyhqJp2k4c$%#>^VWG?wj~XAz zHC+J~imm`(8_X<=&fQ7wx2%gvTE|7jx}MV2v;}Ayyrs*gp?OwS{DwW^%eUKiCw_96 z>WH%E)BXj$hsx7ccPl+u{5n0c$l;*JdgK9t+NWp^JovIlw@0cmR^O^1gB*4I&%?)Dqpmn9{jfY$9okn z?!0jWd-k-tnr(}bSxr1XGvg4=G|9t8RfV8t=43g8qt3X1oA`mX?o7<4%Wmv}4$LNk z@GYJmT^=c5Y?3E9ws{4xyg&OptR=s@aDVwxkgTFRm|oH?4n>?Y_q2}uHn$?cA_w&& z0p0-C4_*S-eKH!O~F#)e9|j#DfuWT?ItMf^m9*RHOJWP*^!%L zS&m0+eIGh^wZA`Hmr1M%nMps~1rtghd9baxu2t3_R5ws{s1K>@)YYzOC4H6T5H{lm zT0jQ4oNwZ*#f)QPJpIdaYLM(`Cb|{KFd9n+yI<=c8miR~+T@M;3g7ze_8;;Mw4pp& zQda<95iT_VmB}%`dSdqN?$Xnq63F&!PS@604SpGsvR zLw?2z1U<6GnqRd3KEJ_H{mXI1{^94j7cHw_Zq6juHvG$yLtbsxJb_Qc{$A6b#30?g z>w>g*(rpr`@36>xFH`BZqO<$r>oa|#^cY?mp4>Araz(bJ2wHNIgt^xt7~j)4u;IkB z(b-mg7W2ZQ%bY6zy3P`&{67BIax!&HtpD63n*!;^?xXkLE&dJJfAjd?htPlX_^){U z%L5zcSAgFsT6(dBZTrZ>BQ@c4i4uj-x2Sl(A{&JPHw9V*a%sW5;A=%{w?C(B<$Q08 PtB-?uhi>8Aujc*-i8$RB literal 0 HcmV?d00001 diff --git a/wiki/images/windestimation/dialogRequestingDataCleansingForDurationDimension.jpg b/wiki/images/windestimation/dialogRequestingDataCleansingForDurationDimension.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ffe69f2ac9d6f9bd789869e7a328f7e23586f149 GIT binary patch literal 20355 zcmeFYcT|(n(l;7<6QnCb5Cnt-f&$V6q?ZJxgia_TO-g`-E~4~aLI@zehTaJzG)0u& zq(dkoRi&s@70czE`@QFU>)vzMckf#7Uw3^w|K!3B=lA?w_`3#Rh3Pp{1dwrKO>zqobu~WMyPzU|{5AxqONB3g^|US2(%3uK`7_ zU*i?xa43Ic<8u0w7^#BPg-fyDkngo2Kaj**^`gOQO#>>BqqvH#EQZx4Wt z9xy=3N=0!4K*>fy#YXY>Er9QWNh*qeG{Aqg3ofatC@JVKMs-*L6jW4H6jTg!lvH$7 zG#4UKQc(kF*l5`WIi%Fh=s3Br+`yGhPH{=g89Vqj3!yaN;leVeX~m^2CQYyJXg)$? zOD^bK`{xS(^CJHwL`!oqCe3#7pMsw8A6x*a{>cOd+gC~|K`D4z@gy||myEix1FGpY zd-#VNG}1y--~N&Tm#8Q%M4(~=r~=;KDadPjh4-9Lb!j8*VzRC0NR?n?F>0*d=1fBg z`FT&B-N*OCRpdw$8$tzkU-sSvb5%2E91;>sA%e_2Y)oR?--q@GjRgkl&Wks?DFli$ z3XP>J;87i6Rn6loyFP0vl&W^KqQu6ZqM{YtXcL$|$Lzz?Ic|NvJ{{7qKZ_s(YD<=DJ`G6ywIVYLM@Wwx9@e2y37e06 zl21Cby`NjtREmIr92}0)_UgI^P!q~uZ>t(S@SPvOp*rM!_p{5&OAu=1V_V@@af=T5 z!-{{&R5SWRlh|E}zsOX#6PbaWBmAyp&t{w~aU z_*3PZfFPjk^^dZ|C*Off(*tf3fTIU~8yCxb!A520nmJ*!0>&<<{wIk>T<0>G-h7dz z4HhB&3GCkHb7pe<3ovV~6Ofj}T-HrT15XzHqPSVHWFdp#Ij1q$K( znBwt((#3}O4vx)@n;jyCs&kvOfUAjae*u)OA*Ij3PaqY)7-tWXe^o!qke$xVJpAmk zhkvX3FHsBrJ@p0t9}~V+T~J{C^Y8Bei2`okz#7qC?lo+q`SZW%R{cwVRS71xif-Q5 zvO7Vj%m3>Aw^qwR5B#?9UGD#zf&Z<6|Nm;BLB*(M#;L#X(c9vym4j^m0AA#+~cab zvz44I9vqw~+e$=49lj%ItHJfu1tREZ>sl|@{sqvzfBx_<;E${YH2c$-Q{g8M15w3K z+1)k`TVOjHKZLW5UrfxATt|dVqm$A;MJu6oX!D4yz&DFFi0Vqz!N*U!ot6%mz3h#u z6SVUpQdQH5Qqeg&DkwPd|M9cs`F^kOoL2Q)dAWGr&XFRhy`k*ui>mflpS@tWgnJ8J z9)wJ>b5zR;HSZCZX2eJ0{9Bj4)Wl4wmSy6m_fM8+?)4~PGq9XG3V=a-0)Fy~-dNjR zO8wJCqZ~xhO}zx8=1EZzMX{xfKum}@6pyZAkG8;Kk)T#dh5(Q8Nh-jjTI|I;yusgTCH0@jW`tpc_hfUU91*%7b z3jus$m>{Kypp}}bgt)42UlRWU_KLZT?G&71oNN3-$%l|{f_SOiV>Gg9AYr<-!d9Px zK>)4WSg$7Fcy6)oC9j{-H7Rw#O$RaP^D?Q&FJ2;*{$`CRf8%a(OcY}tCZkREPB-*} z%ilXAm7^4<<3ZNk(iv2*sre_3+aU6e)4#nNJ?EBO5kPzCgqSBU#^sSz3Pb(RWe`@d zn)~pYy^-f)@o$c9nB(wHB{)0)4^jicj2PiK%KxSZ_0LhOW#xC6xNpRZXlM`O(%G7l z_SDbE#T5*6X4a~#dSGkgIJ6a@Ms|P0q!e=L(p|3hx&P)^g#Aq~k zRbQXIUr_xENH&u)`Z@76^z8KVU%-Z$U-l~WAXagIxqevt)2-e%??stkH2S6n*k9r< zztdHFES@gj?@idtHlSVf^n$1QTYYvF2{e}JOkceG#LPI5xArR$z8g>(F5>6|^%JIq zW#U5^)?BYD{MsFh54=MVR_oFwwLz~2mJ!5~oV*`jx_+$Eok5s%&ix_ijn}dA+FxF> z6fa*_kOg}?%ZlOiK1uJ78KnmSniV=HXTYs!=ef!B%Mvu;H5 z-V8zolSYE6B2{CgErAkO54E_O=G>nsczi<((>&t(MMA(?>+(AzpkKX`_OQt65`Sp> zDnWO;ohXMpxV{jk9RM!^*_cRTK8#1Q=r0M6!_$Dg&{cT;Fb@?~n4v;_Jp5Wb&my~x z{{1@<8;Qm}Fts7k<32jgxr{BYQ@oCgmVb2fXR0`(zW{c0rIl@Ftc>Aufyrb4 znZ^Z8eUt~vHvZ!MiC0jG$S@KYcC(LH!qSh^PY7|HtAUKA{JF{zUwIOxSn8kf*&ySM zSwvc*F{qTwJ4-7$nMItHBU^_w$)sHp>9g{OcW*#$vHb8kbwNQfs7lzG#hp+K7tFG= z{OxV6QU-bxaoC&hqUB|{Xn{3XGSwdfOj~OU{7S}RQLt`C1vIO$+(h%Bm1o{;4)nfJ z1SEW}yy;|7bGKxCzr=r3Uo4)w5+>Q)HaDa<#HfMgWTD^S+ez^8S!U=k<=Z!Ni*`v3 z2;P}RhSoO=%dqXh*}?+F9~r!k{38LsSxtZiXJXIyNJj?B3+Y^Q@^WE}8NansXg^Fm z4|FK{YydB+%V{1%W0s^uZiQ4zw#=q%s2{Vya>eyn0E)xDeUP7RL#R~bGtSM9*}mBp z&X_s*I)MyI`cohFXQyHeWDDT>4*LW$&8J;yGOjmJVSB2sRACR0zO-W3gqIf_z%2N2 zm4z7JC4Q~jIW>GvWqUIa8&W(T5-8rBEb`ra z?EL2QuzDc;$vm&AEPBocSG=qrqM`m5U_tTc6W2=FE5^lFP7`0b`f8>sx4DKuU@bGy z<%j5NcJkMcK~U8q*+ni6J&WzKoE7L^ETpezwt9U%SF@P)y)iPFTzjXUg zr^^@ORR}~uNm2=pl<^f6iNlJozB&QI{Dyo-8`?fS_-Y4E*!c^$0J( z=q~6Cao=^6Bv(Gq<-oXbNF?#Oszg;WDhKRv)lGF(r?_z?DRx2zElXQb1RMM3d)!xB zJsq^(W>T!=UH;f0LiePpr$4Cg>E}9`D*i52N*Ork5c0|_)UBn-gxA&eKC4s)M9^|P zxNbZHZ((MdU_TKh9XF#({cQ_&IlZ0bwTq0!W9b*2@0ZhjLwqeYb$#Bri&BdTS5N`_ z+xf%DviU3LCpjBF*v-n_*1@FV31}lON#vb|H3cFe93JR<){-!p_#g}W2}v_kR(6ZZ zhmMEm7wP(8LkCLRfTGgWgehPofZoX<;zuiw#f|d(Hl^EioSz^KCD9^Qpz(Pnx=(rY znJol6yHmLN3hYtEjErA{p^X2j{=ouY0w)8D32JZM9M}qjPDZ=3!3Gl4QXmJ3)qHOV0 z&D1yRhDO4GXOhdJdtQGHp=+xEeK5LQ&oYu}X^lXEk)eE@$z`l?iiwuN4du7io)!!S zhK38%yrv`6G4GwtXQk)!oj8Ti{X3kO1^Yl+1#gy;Pct%$JLE1|T!^k(cp_VGQ*$Xm zV8KDNrGh4%y-k!EeL1+ zc|QxsV&1pKoLBI+=6w5_^=+%P3%X8ixn^Tm7Ia5%%7z?eHPeIXY#tiXK` z6o7N#(8+|xV6o{v#4(@FmUSAxXXaeCTX1r{f_wwTXi{2ak4k@oX|4Rxw_8_kBg`{Q z=5nbONtboir*1Nv4ZQMw!U&508(sv)(G~8*I+sn{RBAVi2<`@%L zIZ^2=1?0N>L<98K(^B{Ms-w)O%0)d%Mc=}Kfk)CW8YM@A@66qOrroru!aL5v z+y*BaZ5U7z@U5K5-@))8;G1x}9P(@5E@+!h)SDrAt#oM?BfSQnkvBcOlih%R=cEpn zaqV7)O2)?imB~DP(|k=K{U(-KpQsv-0u%@>1j045Po+)4;wYT8!uOmWx>S+H^;bR5 zLfMOHt&tNDG+q!XcD0#GUn7BZsu8a19VFWYCmg?8)}?}I`Z)?5*12U!e|T=vRy9wf z;_24;sB{3)ElL$dqR>-f;DwIMQ>285_isGRma!;QXdMn|z#<~=M` z=2%?zaK>wu+--L@0bcEw$O-Uc;I>#p2y18N;$LIGXhR_bAfG!71I#;K@4ud;D2AmLp2X?jJd0-u9=tqZ$;0Qsg@-F58Z7Mhl7x6zHhckuxr~WkJd=D`B~xB^R6M zRT;G`UZ#f?Sc;Pp@&@||pRn|?@+mWaqC@eaJ=P~ZyMAIkr}p5h)GB%3UNK zXBnL+MyNSOzYfY=h3^gd(SD$h)xkZNJr8&Ia?od`d=Ll~q(B`R(L$CQbDJsc+ucjn z)V&l9f<^y7d%*uO@^9w{pZ&K_oN^5o?cm!XqC3T=?4v#3H@kjkO!udF%US7kE$Lr9 znPsPk7XBPc?J5}jNaF~RRO*Vca>_gAPAPRfXQBVo6aLeJ3G)TsymT1(SecEdK3nxO zEbcGhzIlY2O6!j%)h({F!aOZkxj_rU?_|=0GHU9mQ^qEWaC<_%EYAGF7UO5q-Hy*y zH+~+|&-p2{A#>U^nIs`9+9HyoLM@_5`tvM-AM!cVVxarY^ZI>1bF0F>gGkFN_ zgT|F60)i{6uER{tQz~8E=PY(TgHGqNAB=A>lHX>{jq2iqKhGCBGd^~B$#FJAFMJv} zUpHX$9+4h|mI!(Bs3)HtY!zBTRdYG&_eLiJ2>cfS`qikk4|_)rHZIk@Z84cLn1YL! zOx~x$kiRy37UqB3AYfrah7W&xVH0(^;NDDD<9h!NZ~vRu8@#$yR4E>2HMJ&bndVLj zGT<=}@eX|TxG`WQdC`1x=##{4)@v*onP(ZUe8`ohRq#CvTw=;=$dtZdwjCC9a>X<(L8%UFc(d+pY{bc3%r2gxf`Noyno!s^-D%zqy z4IG~Y2@$CZCRdDYJEk}8!jclFQWZJdPBKex-x_jX;r8$6w8Xyf64yVC92yQ+Hx>uI zR%R=lBH7Q*C$@GBZt~`F+uMF48OE&NjYfjN3@CghCqE0EvnZ&yQ`>H3%3MATBH06r zD$jOaCcT8upLg+{izt53qIiz4i0{Xx0fhEk_O-QA~! zkrjDKQ)65%=0dx5f-ia?V?~?D5#Z5n*LYMWK?V>vAYyRHEKT`+&b6sCR zI{Q`Q25MJTKZ>vNr#HRP2R)*clVS5Q%Ba&fREI5>v6iS){|T1OK{TwNI~t_&9SV4L zV1GZFE_Ab5Y8#fmAs<$%Wg)$Ey*hZ6@g?=RDAy3~;jZ6Xkp&Bf390(=?;8%lQRZjSM zd~ay`ndc(~UHkEle*Nxe^-{0uI;%SQTmxN!8Ro4i_DxXsvY@z4ty@O=?LQ<)c7hLL zcLrmAzWIjf4%cJcgMBTlp=$g+{pX=#)RaZ%vaeek*v7*0L0V^5HalAFBjkfB$~|i(|`3Rh+ll_}Fd`j<;wnaJ+Aogsw2K)qOtKS-kOqnI85RP=wGk z(yRz1Vu}Zs1_yTS(@)yUV#fOV%Ey}+$$gV*x7U`lq?t5-=9khf{aKyZLg^$-e|%&D zBC;gKWcFfgeelecTY8>ee{!IkQAn;YqA)9`5;%W!yY!Og<--pC)cs0NOB#W%!D|I9?)SKproZN- z%!D{PlwS*ulEYWI(fe(^#XzY!e(W6g!7`qaxB1w9svMZzvR?h+;$mYZv1H$VL)q$q z!jhLrU40iXoRcwr@W5Q>`LI;NsiT!05C?JK{)RT&8hMyDWv^cz(-9r2@?#}Ms*`8u zxJKxSbZ?xY_7N8-OW)afVA*=mGa~8>7|a0W6Xv@-OgiCCNcBn4Zu$#o*#rlXNFQL! zpOuJi$`Ek_oqOx)i$8Ok5}u%i2xxmx4a<}K!^6I6A?Bs49Z)aCm%9XrC;n0gf&r%S zS~w4F7C!WPubx-7Zwy(+zOQjc^9V{^hhJSJ`=S9m_K1*!vbQ;Ev@2Y|bh`5!?jO$#Lvyf<*L${ zhL9x(K<(a;{ecvqB^+L1UKD{3Tx0iL%nNj>5P5tq5l{boY*QD-Hq9DbHc-lv;2Y)8 zTE;5v(QeLbU8^zbAba}Ah?lLWSB)JWvX6aR9`aEW^|!Y?PAQ7g7sFVVZ>_Sb~NcjQ%P zJ&LzSKQ{x?urk(Bu!w)#=Xv<#?p51v z72(@^52#+AD}lA-YP^T(R-RF>7fHo=!~EpoL) zS5fcEgzHwDb>1RZ@~ejg_;s`&EO=Y&9+r(pL6Vk?LEpoPxxTdp*&r5nu4re?0|({j ziLzqb*k(l)jw_c=u;sG^XrwSEv|;qr)HLqs z*g5~x)U62{poC5g*#$q(Zp$c8D=1v0Z}(v@$*IiQauZQEM+%s~DaoNCd;jUs(*x0v zD1h0}MwkMh#n$#L;|Yb)!TpocDySVLE`l>1Q&fggrw)iZj)d-TaeIk2* zX_i|eq%_NasiWa`5!ooSBU-!hgHlt!eBqZBJ0JhqpP?O$JT7yyeW(gpsJ_O`&?N;U zoG?vE=ck?F`hJ7q;7Lk86Gmc0!NzLM8Si;s9SqcZ>>V(gBZkBEZ%RrmS z$gc+uX$&4-5nTks_4+lMjBP6uWcVC&emZ!}p4B9sJ3i*#cC3Fp zyAG9rxEq6eF{z&cX(_$FsdJ6UYv6eZ&!}%C2gA4>6;8W zFm2)q3NU!yQL6w|Li2Z`J@WJJX2Ip@>GQ4zpmT6U=WQOCg*W4 z4$rDXsm}97fG>{N>Q&kot&_(3L$Z!|dUCr|uJ353FcO_a!MHk0Qly?!uH3E6B{6 zH*}4?!tAVbVKcY!_~yEBt$@l)d4;3%;@7SiG|e+F%}`BdF{=&wg$<9JlGQDm^9}+1 ziv2P+Zf9po{6y)u8wBe7xZUIdRy zT^R|_S#f-I?|McMvH18L<)X`KW8%v`Z2uYR@9uL4y#NM_F8=u)#_}3rrfs?XGDA{2 zZU`ui7#?q#vlvqlmLu0qZ)zlEOJhex)yQ~^v_8r6A!>NWAQYn%BuS1q?Aw4>E$RjE z1-ETyr^Z@CVqupVHN#HDHu*V2f==B}LMP|MkF)GMG)(I6zH?qlB`u%;Sjt{b%0>fq zfwUF-WV{KIwdM#SiVEG^?-PC%F4E16j?r(QXr8%>t9~oT2UddnM^S(N+ z*8}XD>&zSBj_9^8)k~LNe|z@mjwFYDXk7uC7f{x~L)GQ``$x@wA=JuqdRlIU`GKB3 z%z^P1sWE1zu8#qR(=bWH0D5+=4mD_hO1!&w2!X}EVCR?S^QeC1*88$paa@-m6NdVe zQRZZta%(-pWaCXRh&*9se`gJ|AB!d_de-@lcp4F#lk22em%vs5eG2=;#d3BN_=4=}jh6ar76cgnme8Ui_3$Y9H>1O&^jZ^urLB#qEA zPK_8};>5Gt8dxfxaz%$|Vs-Y`4izhhR}fR@rsC6l(^FgnvKKM&&2e{S6OXpRWaiUM z?ur$YaNf)euO6z(+w{ifmv3a&IR@oFOJsowlyiQ$Vcq9ndFWdC0jq% z^`-(e!`fG+CTnx+^@gR<(h^O-i^$3sGz~8|$;~14-w*f=rTf#{J{3w&@0z=7wLGy4) zCo2v|TVrX>rP=-Fn8XM8dlQchRxbm)=5y^HRCIbLve=L+k+%=n@(?h1oHTGXf`Uqb z<(=NI_G*F%@TqaU((#c2O~SV=+FAA?@CWL8cxe^3%~msQcG}-Tu!lPdcc>@8idr&f zf_$d=EHklGD6T09%F-(5Nqt}2ED=&%-j7+(Vd|-9!bm_ejq73BWDFV|>hT!)tR?KI^ zV20?h*9Wp{jze3k_f6H?p~P0MT%pXEl@xrPyNDay1_T0J5_VQpP>I?}G{|kTv3k`V z+*N%;_{D?Qno`{OB0M8lTm%Xo5^bTQ>YCL_+5N5^^z%1qFS=&W*DGS>nwx3s-CYK# z^V>4g`}cardQc%E&)!aRdznYgHOSGHE@rb()&VUiH+*==>Z9*e!KA0mC5%xTsGAlVrU2eJf>7kRC!z~Qi-)66h z#p2rH+}soy7g+QRT~?=)&gNxDbZngUI!Q~DfUKZIA8&NV8zi+C-QxM`&_JN;jSk~O zJHu$6x_-7lps|*rTN00F17aoZ9?oP`8=J)GSfU`Lw%6h?8_+_@Z8U}O!#`$y<*nV# z-#n5<&e&Kq)_EGFP0e5IJ0;bPQm4F0S}O=zuaP>2>`qn6E2bdIu1*UpS9#t0To_Db zFN#8fdMPQW9t&Te<~v+a&KxBy+_YZG>E-EAGvWB!DsfZI7(!YQaC(SK+dE|(pi9WS zkzAvs9jr^G)BL_}zT^-&QRU(4h9X`==<68`o3%a!mId%VyeOU`Eu7N>}q%6y@>N#i`@ih8jFlFD1E!cJC$_Y>zLHptMz59hkJL?%UZvP zJke0=L`gt7)~AA3M0G|_<>O#Oe`|eV0Oz6`3Gntdxj@|cZ6zJ8kp@iN$RgmK?~3W* zJp(Uht4)o-(Zn$XKd~QMHI*+eewFS~O20{%G%nhAK?khf=7DpK{qV>o{P$21`Eckdn?%O%aRj01Gni3!Hv2a~VUq>nYfaqtTWth7lB(4jKA@%uCJF$hpIialCbocfgS(aZkErfN1^wucyd*GRcg z=j`tDPsgWU#ufK2tgf=|$K6iuFITlo1oH%unuZc~jk&U)}|<_4c< zH>ql62!^o6nwo|8a}-X!6518#_*Ql=J`r{m##z=5oRPr0(t|E)%ctJ0zF+#tUI@nCR@Tdu4yhl71XTc zUegnUVWq+{_ziDX@V?Uh@$}&Gk8=F2egq8F2NHLiiO`Xj&Jc+F@omUG1-d`d zfA`N7x15eXW~rIquX*|1ZsC!|1|ksgC{i;SeYbJA(B64-bq17}F0JL1A*5)fujQ}KcyBl%7}%B5 z;vTAaW7F@a10+fw`IOYSo-zDHT#aFWXgKx)HE9hi7Tby3)T%GAYq@8A7+qfAM!c4H z_-aK8?h(VLN2kL2R?OJx0xwGJd8+-&UxBJn9HW{Fz zZFrP$O=AIV5Kn!%e`x%%wuwnZRJko?{x}O8^JKkp__Tow7&PCsAE>2` zpbw>8f5BuTj?(w`wY`t?`;89(;J93c^1d0#tBTIMKD6Rs2soAWfsAQBc%6Og_7RhT zw9Y%tO67AfgKC59q>R>kn&aj5S6;zyS8_>!!?~0alk4T`y)3gFKb$BgKABnlR3)5z zhgmDS7*^OIY&+Ck?k?FdNR>ft`fA&FKH{Q9phndE2EIv-&E*orUH^|=xxSyR7s)or z{>-TkGKB&Ye~ z|MGy2H8`gBsPUQTdRpQ={b(a-f6$Lc3N4b`J50DpG89P?Af|$E({e8Tdl)RNpv0 zdN3$r^f~_`@~UuGfqG7-y-j?BV0N>Mw-MPUl``cXOGe9aeg@@8~J6o4os z;c{+glei@fJ^XYZ7ALf5OqbTD^+T=Ew&6%(FfQ&ExuknXx?~X3c0an}fD%srjR_V= zacB8aa%>`1boWFJj6TUfP!2`2wkQn?g|6=e5e3YgCST0qS6lU^Xn&ps_yQ z(5w>dH8ax(_DCW1vS0OmG`BE3b%p2#0+N&i0_InC&8%vh-+JJhr&n%&j<7=YkQv*e zYw^N@iW^`7^WgG*qLT;HeGV_oGL!A_(}?p?h=?8vOb&E(*b3q`EzuPg<$5^Zaqq*^ z&enTV4*f5d1j{RKYEEwIDm5E6b7q-Dgkm{<|0ppN3Eq0KUd6R(RpuU|98!Y(a-R}v zeN6$W&0yUQ<7||}>$VF=1oZ`rn=hL9pjk6p^)%+@8{Z_r^7MvtVZ8xW52^Mj&3>Ah z>x94PJSOJZ1Y^qeT^wtkIAtbqhGA-NFVb5LQ!s+BPB`@4cN-BoczfRHSijtmtH@A9P>8wI_BwaB^i?$N24jW zU=oQ9ycBS0ZR6E6@*d>yW__@i5>WaBDwef~PeG%NGn%uqfRpy}j%m4auI=?!jW&aH zhVmOd+0@FzGp)C(2~|vFIABlFz%YPe0!T!1Jn1UGUsb=9UC?$zTho)&D%(^MnP6qC zr4VH2Y3E)gjsv~0$|@{D)B}~trvSaWyuGm;nzjlCzX4`JtZmD04!dgwUC~x~ z_kutgsbUZmj&{C!YA~5PgZ4F#HT#gB#x`5hCVxDVQst45MV7{dtuHe1PNicwO~@3l zca9sJ#Y9A40>YQDAv6vq4aC?wn9qu<;Jn0XENm$lD(IyWUFjHRsv!F6CBaa1T=}f* zYuR9$NF33^v1Ny)YF4B>PAO0ft797Sz)Tyjc#m;)Fd^f#k1yZ6r1{3(P4G9Q{sjWV zBnD2YVeXT`k@pbe6SBKHAK13CgbJvMQXKUB;GXo;mO(WQB+6wn(?7rYr>DK`v8R;O zrVKKr!VZK~SeahM^HEoTg-`R)AySsvH&hA@t+Si#J{Tf@oPWuptp~Caxuf|==MFG8 z5jB7sA3$2#UB%NpwbIc6?_^%#C9n8X64?6%^s(8@N5rT6-g{dDb>^93%uLtwT>d$k z^z=e~mATE%U(`R2``%v(49=cA=_rXQPF^ybQWjBQ#&kJr{ocr_#FhAt#Z?b}8xXR> zI~5p-7#Nh!s_kSsTWNvILlv9BRPXb#sUGa`g-6QU!yifMXb+7G%HZOD4Oia*1G6W%%n9!gy!u<@xwU%Ny(OhhMXOENRTt-qGFO!A zx_f*|N-pzkB*P37s?lR{xFPMG^jtG>fcp=by+szCm&+^JA4lWa-&J|>CpE6C3%D*= z4)N;9B|UpOXuDd-4%zoy^`yet52z;o;+snixZ zRYogh7Bfga882=&EO26__>|+N&Z~3cIoD-8E_k!R4SVM@b9{S^{Mj=zDf# z&D%aNC_FxQMB;T84{B{L#eMV_&%Sg~K!tH)Vi_qLZveNK_p~Y8=n*ZVI`ZM%p&7*Am(9lnD!fPwQ_+JRv3kD*g=VFplCBAJY)3ZKcV*3Enlu!2saSAG7*Pi%IVsRhq?25>aR7Wq$7reSR+Nm1MS~SdASGCd( z-OT$c(=*i5V6SgwyS?M}Xwer_gJx~Nb%)Fub>RI(jrXiFt)Ct1`_oswFA)MxdWv71 zW?w%_a}^n6kzN`VEnEK!sIIFLa!fGvemt(bWoLnW+Lj+R!|Pe7bZ*T3!>3T=$@9@l_enm7YKTC?=v-c`Je?h`~H0 z?hDq@3_r*&qUh4qcn!%?%%TguB@!BaVJURG?NAV34ONbojY5+X-e_@t-)7Tl`mviJ zyJP|proSa_;&cbU1x7lHHli7TtxLXhHx_H~h?{Q^<_m2bDvNlipb(0ue{Ul~fkxYQ zVLL7IJ6mEfya}&O z9s;A?85qs=PSu4}&&`7QjcdPDBa9K5yWb9OEH(-%npE6-clvUtxwc^SG8ez(7!w=H z5ARAg99DLl3n+Z08$caSVL=D{^cSFw#D|y~M}utmF0nEkjOh)|pMGpC@0MmBR+`jg zv@vUo_>4JhD|VxBVl@mvOAU^>8E1t1;MEsT_(E!#a4|RTYqp)`IeZ@%QBflR^sX91 z2Zi{~W*$8)TFkim5;p=@o>DGuZG3h+YHc( zk7ICi5z^M8X>WC6^)X;Itz}Uwg5WAcR{qFVk%S4qh!uBvl13k5iBK9V-Z%h&K{(B< zTczRYVW0Oi8_}}mZ7?jIo#~<2o#_Oq6_XJFz(Gbsdw?@EhZ)e^t(#*!K=T)c;M+0F zC->e!7h=r2!}5pe1WuD&&(~(6y4{e;Xd7o)kY}qPDRoqXW72s?s)aa_EaHrYvx=I9Gotw7R%AmLrxY?2zTLK47JIVdh zgNJ*7Q7OTHei=WmSNKjdm0=LM#WEpMay0fJmb2IFHE~l8lVt^^&=v%P2Zb-O%B-Zp z=YYEW-@x|`<+4~DnTxliKp;j)gOa&il_nkCD~ZBgz!Tu_Sydtwdrm*PcI%n0I`h zW}o4jEQGI{`?J}TPc?OocUrE6(QPih|K#_Rp2qz%E5~c0u0thvm1{pQ7l4%$kyDE9 zj|5&4Sy$(Pc5*bVEtuLAlRVcvBSu~v8$LE1&-(lRy8Ln#ZK(i?nw?EP#*rh~DA@pHBt>&9NV7=& z>4M)lyihd4(8JPVq?FaMmvp;|jY~1sUV{?oBq8e48^4(srlznzz~xr@KfZqXfB%h3 zm3#bu0`R%p+y81iLC>?g}h3LOdxZVwc`X164-6CEHRN3ZGR zmy-hmJF(cx{UN)DBb$F@tAE{WCNZYOOTyI5T3bZXhXGkbqK}2s z!Y)4?SMt77Vr*)Lr74e2I5LquOf2`F7w;RB(|f-+{}`ejy8lt&0`eryS%w2J4C&da z_fu@wqK_Zq@=Y@)tZ7cl$x6Jh9UT!$zWc63zGwDzh$`dC3rzR?@Qp~(SHUWs@}49~ z>Nb&4T&0-4aY;FtN1w$N9Vn49jw+b^9o8hO2^lF3R16gERNcR0!*4CL2kYg96cZ|f zK>+^c0^dH{D(h#e&!mN%Tx);adH6P!+qTc}_Oe@3Ci{4AvEbNw__B}5@08~T!+WHU z=lOH}HU8tjmg-82-)RVlKq#jon%7}Nc6$Jw&D23Nat7PAd%rW>EX2&a0mu50(wnx3 z;J3ESlTfeg4}>#2oEzoySiUW#jEA^u!XMAr-(nYOm#N^lZ8u+(IBqUU?o-#Z`_QVc z+a<=TM1ycMNHYlnJ|Z3(3xDKVrfQZXJ~|n_^Tv3yg-NFfzB{#u8#OydPtPDW z43|UuJp1AeY&D#=rg&3PUpsyau?cGr-c&P#&rV;{WdF z{jxUJhR+b#_;OnRUck{$vksi8-{Vd8=%9rAYd_>d7l;9coddLMGv-bFbkIvNS4c=h zqp!NuuEfHrGJl;&my=NRcB}w?enFEZKJ!?g%noJr2=3*I&=pksHoz5t=dux@rY)N8 zUx%=$W8=_u-gJ(2_fOc8@iZcUiiq@O9WdikQ5kt-DKGrVl5bo#BwMm?H5L)vyB@A) z*&E(ZZxJZ4!}Sy|P8)Cmof7*Ca2bhYS#Ps^GPlwy;`s$0K(dkKAUR=XzQ_P^py5BiA#)?Cq$_RPTnGcgcn21X0&+eled3$FNHk zFw}&CMYyyStYNj?d~XN*E%IU3FJ1`}K=eAZGELptbfT(-7q)7hO8h9qKJSx~&0SuA zo4UrB=;F|JT9H1ZzHGA`f-X{pROEES!zKiN$RAy;4aWCnI!mFi+>E%9#?wM5D~9+> zQ_!hCC;l<};&}KBnL@TPd-t;P(&*##>)s~y&9%P|qzX@dgMYvvQj=ZueZR7QKFw|H z^rMH@i=*P^vLNmYbA7jhp`WQp8@wTke@>)I2_=3%76pMP)b^a_8z`7x-&T*TR>Ji$2bX?(afrkpxHJkBxA< z`#G$2gX|yl(l3w0x$dqCOP6(hus?5ZiWW|bc0C&0Cb*kh$0X$TUVA1rAYB(crAKnR z+V+1;ZK;}-M6~v+WmXKfrmgac(m_Vs%ksQ=$-+Co zdd{1-k0sxj7B3B7`nb&_bw$SGSyS4!a3?-X-d_;BElthRcE(&CUA^^XPIJy{vSumd zvQL~;7!&wsw$O?0!aZ6pDp7Z~Y`IUwr#mLf7|AwjQ#1g{@WgZwCc42Ez7$0 z@#g;K-A&q^{c2u$x9+&!E1%*rTYTlS1;Wp@xb8T_|EPccD{P{duHK*83z?5MI!}M{ z`<%FDeCFHN9nAk3mY=(C3T!f(Tkn7Q>-u6T4pS}dQ}eHS3orB4{j%-7kkFDh=N|Xu z{k~!Or!`K~b?N>7XC5w#?Do!d>%H`A;mTaCsIzMhZP}UI^=i`P-6AopS`0ex4|tpm zaI5&{Zo0~7>cT0iN59H&GI}2~m;2oJ>i5#{?&~)tf~Kqh4P+pOYEf3pAoi7;z=vvA zsUt>kC*M99d&TG7lW)LQz|Ofiw$fsZgJ^m&tzD)2Cf3w!ZoBTj#e9|Lo;>M!$k69* zyC8heUBTqOV_$ANT?uPpNU@kWRYQHd`sbx@Yi$#kP5oJU{-0q``Rk@xTX($9ytnO2 z>r(k=ikTMw8J3o~hPXc{4si`rvSn#}@Nw_g_fz4F;8{D*cP`SIwNgCu{oN%e67$V= zY6t7H$8ZKK&Uxd(7{RK^C0$*eWpr|7-gnucv&){EJrz2WZ**Xqe2~xdpheL=;&YZV zl=I!+rR}f%^kc+oBQ4MTNv@SrKDPXclLgz4?pVI6WxZ@UpONvutWOs%-tIX1%v$r- zWpne{PQ`N=gczPJ&n^0PTS?uvXF>fxSEmd z3EED>kfHHF68gqytzAo_fpHc7^f~Y-0hi@nw-^;xv@VU1U)1=Y;n9-3Z_%}nmwxN1 z+wN^`yH#ECST)C=R<~cG%YhY^{d>!=;hDmns*-*|OSD!mpV_^JiVU7%xTDLEdq*reej7r$uUJrVH<#Xkju<641J!$0_c zpQ{b|q7B@ue4zeb@vBvnCj~80KNR%QZqcsju2Z6iCN*&UJG5tu=4tWfpLgcjfAOtF zI3r~1vg>@8WqqP=zSy#K>Xdg{&l%Du)hk9$*Hu+kO!0`+dR2C5$~t!vM)e5hnXoPI zc3qxm@^_O-ci-`uU6=FJ7d$-F#GtHb@m5svHMjSV`GD#*a*3>rVQSF#PI~sJO$%bB8Cd)oBd#$Cu|DDI~ yJKk5mShwv~6(7%v1r0{QeV4sLC;wA9GS$;#wR=vd_TQa9JvN03&HFk3|4jhg`{4=z literal 0 HcmV?d00001 From 940336aa73030b892d1793089bfa731fb3520be6 Mon Sep 17 00:00:00 2001 From: Vladislav Chumak Date: Tue, 26 Feb 2019 07:15:15 +0100 Subject: [PATCH 24/24] Improved model training, adjusted mongodb compound index for distance --- .../ManeuverAndWindImporter.launch | 13 --------- ...RaceWithWindSourcesPersistenceManager.java | 29 ++++--------------- .../WindSourcesPersistenceManager.java | 7 +++-- .../model/SimpleModelsTrainingPart1.java | 10 +++++-- 4 files changed, 19 insertions(+), 40 deletions(-) delete mode 100644 java/com.sap.sailing.windestimation.lab/ManeuverAndWindImporter.launch diff --git a/java/com.sap.sailing.windestimation.lab/ManeuverAndWindImporter.launch b/java/com.sap.sailing.windestimation.lab/ManeuverAndWindImporter.launch deleted file mode 100644 index 49c1b7115d1..00000000000 --- a/java/com.sap.sailing.windestimation.lab/ManeuverAndWindImporter.launch +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/RaceWithWindSourcesPersistenceManager.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/RaceWithWindSourcesPersistenceManager.java index 297e09ca2de..724c9f3472a 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/RaceWithWindSourcesPersistenceManager.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/RaceWithWindSourcesPersistenceManager.java @@ -17,20 +17,16 @@ import com.sap.sailing.server.gateway.deserialization.JsonDeserializer; import com.sap.sailing.server.gateway.serialization.impl.RaceWindJsonSerializer; import com.sap.sailing.windestimation.data.RaceWithWindSources; import com.sap.sailing.windestimation.data.persistence.maneuver.AbstractPersistenceManager; -import com.sap.sailing.windestimation.data.persistence.maneuver.PersistedElementsIterator; import com.sap.sailing.windestimation.data.serialization.RaceWithEstimationDataDeserializer; import com.sap.sailing.windestimation.data.serialization.RaceWithWindSourcesDeserializer; -import com.sap.sse.common.TimePoint; public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceManager { - private static final String COLLECTION_NAME = "racesWithWind"; - private static final String WIND_SOURCES_COLLECTION_NAME = "windSources"; + static final String COLLECTION_NAME = "racesWithWind"; + private final WindSourcesPersistenceManager windSourcesPersistenceManager; public RaceWithWindSourcesPersistenceManager() throws UnknownHostException { - Document indexes = new Document(RaceWindJsonSerializer.START_TIME_POINT, 1); - indexes.put(RaceWindJsonSerializer.END_TIME_POINT, -1); - getDb().getCollection(getWindSourcesCollectionName()).createIndex(indexes); + windSourcesPersistenceManager = new WindSourcesPersistenceManager(); } @Override @@ -41,7 +37,7 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa @Override public void dropCollection() { super.dropCollection(); - getDb().getCollection(getWindSourcesCollectionName()).drop(); + windSourcesPersistenceManager.dropCollection(); } @Override @@ -52,7 +48,7 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa JSONArray windSourceIdsJson = (JSONArray) raceJson .get(RaceWithEstimationDataDeserializer.COMPETITOR_TRACKS); JSONArray windSourcesJson = new JSONArray(); - MongoCollection windSourcesCollection = getDb().getCollection(getWindSourcesCollectionName()); + MongoCollection windSourcesCollection = windSourcesPersistenceManager.getCollection(); for (Object idObject : windSourceIdsJson) { Document dbWindSource = windSourcesCollection .find(new Document(FIELD_DB_ID, new ObjectId((String) idObject))).first(); @@ -69,10 +65,6 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa }; } - public static String getWindSourcesCollectionName() { - return COLLECTION_NAME + "." + WIND_SOURCES_COLLECTION_NAME; - } - public void add(String regattaName, String raceName, JSONObject raceWithWindSourcesJson) { JSONArray windSourcesJson = (JSONArray) raceWithWindSourcesJson.get(RaceWindJsonSerializer.WIND_SOURCES); @@ -81,7 +73,7 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa Document entry = parseJsonString(windSourceObj.toString()); dbWindSources.add(entry); } - MongoCollection windSourcesCollection = getDb().getCollection(getWindSourcesCollectionName()); + MongoCollection windSourcesCollection = windSourcesPersistenceManager.getCollection(); windSourcesCollection.insertMany(dbWindSources); BasicDBList dbWindSourceIds = new BasicDBList(); for (Document dbWindSource : dbWindSources) { @@ -96,13 +88,4 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa races.insertOne(dbObject); } - public PersistedElementsIterator getIteratorForEntriesIntersectingPeriod( - TimePoint startTimePoint, TimePoint endTimePoint, double toleranceInSeconds) { - double maxStartTime = endTimePoint.asMillis() + toleranceInSeconds * 1000; - double maxEndTime = startTimePoint.asMillis() - toleranceInSeconds * 1000; - String query = "{$and: [{'" + RaceWindJsonSerializer.START_TIME_POINT + "': {$lte: " + maxStartTime + "}}, {'" - + RaceWindJsonSerializer.END_TIME_POINT + "': {$gte: " + maxEndTime + "}}]}"; - return getIterator(query); - } - } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/WindSourcesPersistenceManager.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/WindSourcesPersistenceManager.java index d80c152d635..a49d93227b8 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/WindSourcesPersistenceManager.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/WindSourcesPersistenceManager.java @@ -21,15 +21,18 @@ import com.sap.sse.common.TimePoint; public class WindSourcesPersistenceManager extends AbstractPersistenceManager { + private static final String WIND_SOURCES_COLLECTION_NAME = "windSources"; + public WindSourcesPersistenceManager() throws UnknownHostException { - BasicDBObject indexes = new BasicDBObject(RaceWindJsonSerializer.START_TIME_POINT, 1); + BasicDBObject indexes = new BasicDBObject("_id", 1); + indexes.put(RaceWindJsonSerializer.START_TIME_POINT, 1); indexes.put(RaceWindJsonSerializer.END_TIME_POINT, -1); getCollection().createIndex(indexes); } @Override public String getCollectionName() { - return RaceWithWindSourcesPersistenceManager.getWindSourcesCollectionName(); + return RaceWithWindSourcesPersistenceManager.COLLECTION_NAME + "." + WIND_SOURCES_COLLECTION_NAME; } @Override diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTrainingPart1.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTrainingPart1.java index fc0f0801f4e..f5e9eca6722 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTrainingPart1.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTrainingPart1.java @@ -48,10 +48,10 @@ public class SimpleModelsTrainingPart1 { PersistedManeuverClassifiersScorePrinter.main(args); }); executeInThreadPool(() -> { - DurationBasedTwdTransitionImporter.main(args); + DistanceBasedTwdTransitionImporter.main(args); }); executeInThreadPool(() -> { - DistanceBasedTwdTransitionImporter.main(args); + DurationBasedTwdTransitionImporter.main(args); }); awaitThreadPoolCompletion(); AggregatedDurationBasedTwdTransitionImporter.createPersistenceManagerAndEnsureIndex(); @@ -77,6 +77,7 @@ public class SimpleModelsTrainingPart1 { } while (JOptionPane.YES_OPTION != askDataCleaningFinished(AggregatedSingleDimensionType.DISTANCE)); showInfoAboutIntervalAdjustments(DistanceBasedTwdTransitionRegressorModelContext.class, DistanceValueRange.class); + showInfoAboutRunPart2(); } private static int askDataCleaningFinished(AggregatedSingleDimensionType dimension) { @@ -94,6 +95,11 @@ public class SimpleModelsTrainingPart1 { return JOptionPane.CLOSED_OPTION; } + private static void showInfoAboutRunPart2() { + JOptionPane.showMessageDialog(null, "Now, run the class \"" + SimpleModelsTrainingPart2.class.getName() + + "\".\nThis will complete the training process."); + } + private static void showInfoAboutDataCleaning(AggregatedSingleDimensionType dimension) { JOptionPane.showMessageDialog(null, "Now, clean the data for the " + dimension + " dimension.\nRemove instances from MongoDB collection \"" + dimension.getCollectioName()