diff --git a/java/com.sap.sailing.domain.test/src/com/sap/sailing/domain/test/mock/MockedTrackedRace.java b/java/com.sap.sailing.domain.test/src/com/sap/sailing/domain/test/mock/MockedTrackedRace.java index a8eb8868a93..ae275abbc5b 100644 --- a/java/com.sap.sailing.domain.test/src/com/sap/sailing/domain/test/mock/MockedTrackedRace.java +++ b/java/com.sap.sailing.domain.test/src/com/sap/sailing/domain/test/mock/MockedTrackedRace.java @@ -83,7 +83,7 @@ import com.sap.sailing.domain.tracking.WindStore; import com.sap.sailing.domain.tracking.WindTrack; import com.sap.sailing.domain.tracking.WindWithConfidence; import com.sap.sailing.domain.tracking.impl.WindTrackImpl; -import com.sap.sailing.domain.windestimation.IncrementalWindEstimationTrack; +import com.sap.sailing.domain.windestimation.IncrementalWindEstimation; import com.sap.sse.common.Bearing; import com.sap.sse.common.Distance; import com.sap.sse.common.Duration; @@ -1235,7 +1235,7 @@ public class MockedTrackedRace implements DynamicTrackedRace { } @Override - public void setWindEstimation(IncrementalWindEstimationTrack windEstimation) { + public void setWindEstimation(IncrementalWindEstimation windEstimation) { } } diff --git a/java/com.sap.sailing.domain.test/src/com/sap/sailing/domain/test/mock/MockedTrackedRaceWithStartTimeAndRanks.java b/java/com.sap.sailing.domain.test/src/com/sap/sailing/domain/test/mock/MockedTrackedRaceWithStartTimeAndRanks.java index 823cab82297..d46099dd925 100755 --- a/java/com.sap.sailing.domain.test/src/com/sap/sailing/domain/test/mock/MockedTrackedRaceWithStartTimeAndRanks.java +++ b/java/com.sap.sailing.domain.test/src/com/sap/sailing/domain/test/mock/MockedTrackedRaceWithStartTimeAndRanks.java @@ -64,7 +64,7 @@ import com.sap.sailing.domain.tracking.WindStore; import com.sap.sailing.domain.tracking.WindTrack; import com.sap.sailing.domain.tracking.WindWithConfidence; import com.sap.sailing.domain.tracking.impl.DynamicTrackedRegattaImpl; -import com.sap.sailing.domain.windestimation.IncrementalWindEstimationTrack; +import com.sap.sailing.domain.windestimation.IncrementalWindEstimation; import com.sap.sse.common.Bearing; import com.sap.sse.common.Distance; import com.sap.sse.common.Duration; @@ -783,7 +783,7 @@ public class MockedTrackedRaceWithStartTimeAndRanks implements TrackedRace { } @Override - public void setWindEstimation(IncrementalWindEstimationTrack windEstimation) { + public void setWindEstimation(IncrementalWindEstimation windEstimation) { } } diff --git a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/DummyTrackedRace.java b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/DummyTrackedRace.java index 34e1acd5a4f..d6c59d2b5ad 100755 --- a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/DummyTrackedRace.java +++ b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/DummyTrackedRace.java @@ -38,7 +38,7 @@ import com.sap.sailing.domain.ranking.RankingMetric; import com.sap.sailing.domain.ranking.RankingMetric.RankingInfo; import com.sap.sailing.domain.tracking.impl.DynamicTrackedRegattaImpl; import com.sap.sailing.domain.tracking.impl.EmptyWindStore; -import com.sap.sailing.domain.windestimation.IncrementalWindEstimationTrack; +import com.sap.sailing.domain.windestimation.IncrementalWindEstimation; import com.sap.sse.common.Bearing; import com.sap.sse.common.Distance; import com.sap.sse.common.Duration; @@ -722,7 +722,7 @@ public class DummyTrackedRace extends TrackedRaceWithWindEssentials { } @Override - public void setWindEstimation(IncrementalWindEstimationTrack windEstimation) { + public void setWindEstimation(IncrementalWindEstimation windEstimation) { } } diff --git a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/TrackedRace.java b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/TrackedRace.java index 0f6ce033157..cccc955eaca 100644 --- a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/TrackedRace.java +++ b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/TrackedRace.java @@ -53,7 +53,7 @@ import com.sap.sailing.domain.ranking.RankingMetric; import com.sap.sailing.domain.ranking.RankingMetric.RankingInfo; import com.sap.sailing.domain.tracking.impl.NonCachingMarkPositionAtTimePointCache; import com.sap.sailing.domain.tracking.impl.TrackedRaceImpl; -import com.sap.sailing.domain.windestimation.IncrementalWindEstimationTrack; +import com.sap.sailing.domain.windestimation.IncrementalWindEstimation; import com.sap.sse.common.Bearing; import com.sap.sse.common.Distance; import com.sap.sse.common.Duration; @@ -1152,6 +1152,6 @@ public interface TrackedRace extends Serializable, IsManagedByCache windTracks; + protected transient ConcurrentMap windTracks; protected final RaceDefinition race; @@ -185,4 +192,37 @@ public abstract class TrackedRaceWithWindEssentials implements TrackedRace { protected NamedReentrantReadWriteLock getSerializationLock() { return serializationLock; } + + /** + * For serialization, maneuver based estimation wind track should be removed because its replication is redundant. + * The wind estimation track is calculated automatically when maneuver cache recalculation is triggered. Due to the + * custom deserialization of maneuver cache, this is always the case, also if replication is performed. Furthermore, + * it might introduce issues in context of already running wind estimation. + */ + private void writeObject(ObjectOutputStream s) throws IOException { + LockUtil.lockForWrite(getSerializationLock()); + try { + s.defaultWriteObject(); + Map windTracksToSerialize = new HashMap<>(); + synchronized (this.windTracks) { + for (Entry entry : windTracks.entrySet()) { + if (entry.getKey().getType() != WindSourceType.MANEUVER_BASED_ESTIMATION) { + windTracksToSerialize.put(entry.getKey(), entry.getValue()); + } + } + } + s.writeObject(windTracksToSerialize); + } finally { + LockUtil.unlockAfterWrite(getSerializationLock()); + } + } + + private void readObject(ObjectInputStream ois) throws ClassNotFoundException, IOException, PatchFailedException { + ois.defaultReadObject(); + @SuppressWarnings("unchecked") + Map windTracks = (Map) ois.readObject(); + this.windTracks = new ConcurrentHashMap<>(windTracks); + } + + } diff --git a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/impl/TrackedRaceImpl.java b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/impl/TrackedRaceImpl.java index b653199541e..387875cfb5d 100755 --- a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/impl/TrackedRaceImpl.java +++ b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/tracking/impl/TrackedRaceImpl.java @@ -146,7 +146,7 @@ import com.sap.sailing.domain.tracking.WindPositionMode; import com.sap.sailing.domain.tracking.WindStore; import com.sap.sailing.domain.tracking.WindTrack; import com.sap.sailing.domain.tracking.WindWithConfidence; -import com.sap.sailing.domain.windestimation.IncrementalWindEstimationTrack; +import com.sap.sailing.domain.windestimation.IncrementalWindEstimation; import com.sap.sse.common.Bearing; import com.sap.sse.common.Distance; import com.sap.sse.common.Duration; @@ -393,7 +393,7 @@ public abstract class TrackedRaceImpl extends TrackedRaceWithWindEssentials impl private transient PolarDataService polarDataService; - private transient volatile IncrementalWindEstimationTrack windEstimation; + private transient volatile IncrementalWindEstimation windEstimation; private transient ShortTimeAfterLastHitCache maneuverDetectorPerCompetitorCache; @@ -3611,27 +3611,34 @@ public abstract class TrackedRaceImpl extends TrackedRaceWithWindEssentials impl @Override public void setPolarDataService(PolarDataService polarDataService) { this.polarDataService = polarDataService; + if(polarDataService != null && windEstimation != null) { + updateManeuversAndWindWithNewWindEstimation(windEstimation, windEstimation); + } } @Override - public void setWindEstimation(IncrementalWindEstimationTrack windEstimation) { - IncrementalWindEstimationTrack previousWindEstimation = this.windEstimation; + public void setWindEstimation(IncrementalWindEstimation windEstimation) { + IncrementalWindEstimation previousWindEstimation = this.windEstimation; if (previousWindEstimation != windEstimation) { - if (previousWindEstimation != null) { - windTracks.remove(previousWindEstimation.getWindSource()); - } - if (windEstimation != null) { - windTracks.put(windEstimation.getWindSource(), windEstimation); - } - this.windEstimation = windEstimation; - // TODO Make more efficient by reusing the state of incremental maneuver detectors. The already computed - // complete maneuver curves can be fed directly to the windEstimation. - maneuverDetectorPerCompetitorCache.clearCache(); - shortTimeWindCache.clearCache(); - triggerManeuverCacheRecalculationForAllCompetitors(); + updateManeuversAndWindWithNewWindEstimation(windEstimation, previousWindEstimation); } } + private void updateManeuversAndWindWithNewWindEstimation(IncrementalWindEstimation windEstimation, + IncrementalWindEstimation previousWindEstimation) { + WindSource windSource = new WindSourceImpl(WindSourceType.MANEUVER_BASED_ESTIMATION); + windTracks.remove(windSource); + if (windEstimation != null) { + windTracks.put(windSource, windEstimation.getWindTrack()); + } + this.windEstimation = windEstimation; + // TODO Make more efficient by reusing the state of incremental maneuver detectors. The already computed + // complete maneuver curves can be fed directly into the windEstimation. + maneuverDetectorPerCompetitorCache.clearCache(); + shortTimeWindCache.clearCache(); + triggerManeuverCacheRecalculationForAllCompetitors(); + } + /** * Obtains the {@link #raceLogResolver}. */ diff --git a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/IncrementalWindEstimationTrack.java b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/IncrementalWindEstimation.java similarity index 76% rename from java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/IncrementalWindEstimationTrack.java rename to java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/IncrementalWindEstimation.java index 80b281cecd2..2127544d68a 100644 --- a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/IncrementalWindEstimationTrack.java +++ b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/IncrementalWindEstimation.java @@ -6,7 +6,7 @@ import com.sap.sailing.domain.tracking.WindTrack; /** * Wind estimator which interacts with maneuver detector and maintains a wind track with estimated wind fixes. All in - * all, this instance can be seen as a wind source with corresponding wind track. The underlying wind track is prone to + * all, this instance can be seen as a wind source with corresponding wind track. The managed wind track is prone to * changes which are communicated to tracked race via its * {@link TrackedRace#recordWind(com.sap.sailing.domain.common.Wind, WindSource, boolean)} and * {@link TrackedRace#removeWind(com.sap.sailing.domain.common.Wind, WindSource)}. @@ -14,11 +14,16 @@ import com.sap.sailing.domain.tracking.WindTrack; * @author Vladislav Chumak (D069712) * */ -public interface IncrementalWindEstimationTrack extends WindTrack, WindEstimationInteraction { +public interface IncrementalWindEstimation extends WindEstimationInteraction { /** * @return The wind source assigned to this wind track */ WindSource getWindSource(); + /** + * Gets the produced wind track of this wind estimation + */ + WindTrack getWindTrack(); + } diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/TimePointAndPositionWithToleranceComparator.java b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/TimePointAndPositionWithToleranceComparator.java similarity index 85% rename from java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/TimePointAndPositionWithToleranceComparator.java rename to java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/TimePointAndPositionWithToleranceComparator.java index 6e7596cdcbe..a075257a8ee 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/TimePointAndPositionWithToleranceComparator.java +++ b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/TimePointAndPositionWithToleranceComparator.java @@ -1,5 +1,6 @@ -package com.sap.sailing.windestimation.integration; +package com.sap.sailing.domain.windestimation; +import java.io.Serializable; import java.util.Comparator; import com.sap.sailing.domain.common.Position; @@ -13,8 +14,9 @@ import com.sap.sse.common.Util.Pair; * @author Vladislav Chumak (D069712) * */ -public class TimePointAndPositionWithToleranceComparator implements Comparator> { +public class TimePointAndPositionWithToleranceComparator implements Comparator>, Serializable { + private static final long serialVersionUID = -3240742353261384636L; private static final double DOUBLE_TOLERANCE = 0.0000001; @Override diff --git a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/WindEstimationFactoryService.java b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/WindEstimationFactoryService.java index 62d0336e250..2d3d3ffdabc 100644 --- a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/WindEstimationFactoryService.java +++ b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/WindEstimationFactoryService.java @@ -31,7 +31,7 @@ public interface WindEstimationFactoryService { * @see #isReady() * @see #addWindEstimationModelsChangedListenerAndReceiveUpdate(WindEstimationModelsChangedListener) */ - IncrementalWindEstimationTrack createIncrementalWindEstimationTrack(TrackedRace trackedRace); + IncrementalWindEstimation createIncrementalWindEstimationTrack(TrackedRace trackedRace); /** * Checks whether the wind estimation is ready for its use. The wind estimation maintains multiple internal machine diff --git a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/WindTrackWithConfidenceForEachWindFixImpl.java b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/WindTrackWithConfidenceForEachWindFixImpl.java new file mode 100644 index 00000000000..da5fb01136e --- /dev/null +++ b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/windestimation/WindTrackWithConfidenceForEachWindFixImpl.java @@ -0,0 +1,48 @@ +package com.sap.sailing.domain.windestimation; + +import java.util.Map; + +import com.sap.sailing.domain.common.Position; +import com.sap.sailing.domain.common.Wind; +import com.sap.sailing.domain.tracking.WindWithConfidence; +import com.sap.sailing.domain.tracking.impl.WindTrackImpl; +import com.sap.sse.common.TimePoint; +import com.sap.sse.common.Util.Pair; + +/** + * Wind track implementation which uses the provided map with {@link WindWithConfidence}-instances to derive the + * confidence for each wind fix within this wind track. + * + * @author Vladislav Chumak (D069712) + * + */ +public class WindTrackWithConfidenceForEachWindFixImpl extends WindTrackImpl { + + private static final long serialVersionUID = 8135949043175649443L; + private final Map, WindWithConfidence>> windTrackWithConfidences; + + public WindTrackWithConfidenceForEachWindFixImpl(long millisecondsOverWhichToAverage, double baseConfidence, + boolean useSpeed, String nameForReadWriteLock, boolean losslessCompaction, + Map, WindWithConfidence>> windTrackWithConfidences) { + super(millisecondsOverWhichToAverage, baseConfidence, useSpeed, nameForReadWriteLock, losslessCompaction); + this.windTrackWithConfidences = windTrackWithConfidences; + } + + @Override + protected double getConfidenceOfInternalWindFixUnsynchronized(Wind windFix) { + WindWithConfidence> windWithConfidence = windTrackWithConfidences + .get(new Pair<>(windFix.getPosition(), windFix.getTimePoint())); + return super.getConfidenceOfInternalWindFixUnsynchronized(windFix) * windWithConfidence.getConfidence(); + } + + @Override + public void lockForWrite() { + super.lockForWrite(); + } + + @Override + public void unlockAfterWrite() { + super.unlockAfterWrite(); + } + +} diff --git a/java/com.sap.sailing.gwt.ui.test/src/com/sap/sailing/gwt/ui/test/MockedTrackedRace.java b/java/com.sap.sailing.gwt.ui.test/src/com/sap/sailing/gwt/ui/test/MockedTrackedRace.java index 4ed8b77ea22..55eb490c7eb 100644 --- a/java/com.sap.sailing.gwt.ui.test/src/com/sap/sailing/gwt/ui/test/MockedTrackedRace.java +++ b/java/com.sap.sailing.gwt.ui.test/src/com/sap/sailing/gwt/ui/test/MockedTrackedRace.java @@ -71,7 +71,7 @@ import com.sap.sailing.domain.tracking.WindStore; import com.sap.sailing.domain.tracking.WindTrack; import com.sap.sailing.domain.tracking.WindWithConfidence; import com.sap.sailing.domain.tracking.impl.WindTrackImpl; -import com.sap.sailing.domain.windestimation.IncrementalWindEstimationTrack; +import com.sap.sailing.domain.windestimation.IncrementalWindEstimation; import com.sap.sse.common.Bearing; import com.sap.sse.common.Distance; import com.sap.sse.common.Duration; @@ -931,7 +931,7 @@ public class MockedTrackedRace implements DynamicTrackedRace { } @Override - public void setWindEstimation(IncrementalWindEstimationTrack windEstimation) { + public void setWindEstimation(IncrementalWindEstimation windEstimation) { } } diff --git a/java/com.sap.sailing.server/SailingServer (Proxy, Jetty on 8889, Polar & Wind estimation Import from 8888).launch b/java/com.sap.sailing.server/SailingServer (Proxy, Jetty on 8889, Polar & Wind estimation Import from 8888).launch new file mode 100755 index 00000000000..74239ad341d --- /dev/null +++ b/java/com.sap.sailing.server/SailingServer (Proxy, Jetty on 8889, Polar & Wind estimation Import from 8888).launch @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/java/com.sap.sailing.server/SailingServer (Proxy, Jetty on 8889, Polar Import from 8888).launch b/java/com.sap.sailing.server/SailingServer (Proxy, Jetty on 8889, Polar Import from 8888).launch deleted file mode 100755 index 925b9efd2d3..00000000000 --- a/java/com.sap.sailing.server/SailingServer (Proxy, Jetty on 8889, Polar Import from 8888).launch +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/java/com.sap.sailing.server/src/com/sap/sailing/server/impl/RacingEventServiceImpl.java b/java/com.sap.sailing.server/src/com/sap/sailing/server/impl/RacingEventServiceImpl.java index 5c61c8d2599..44e7a7b2548 100644 --- a/java/com.sap.sailing.server/src/com/sap/sailing/server/impl/RacingEventServiceImpl.java +++ b/java/com.sap.sailing.server/src/com/sap/sailing/server/impl/RacingEventServiceImpl.java @@ -124,6 +124,7 @@ import com.sap.sailing.domain.common.ScoringSchemeType; import com.sap.sailing.domain.common.TrackedRaceStatusEnum; import com.sap.sailing.domain.common.Wind; import com.sap.sailing.domain.common.WindSource; +import com.sap.sailing.domain.common.WindSourceType; import com.sap.sailing.domain.common.dto.AnniversaryType; import com.sap.sailing.domain.common.dto.EventType; import com.sap.sailing.domain.common.dto.FleetDTO; @@ -2078,12 +2079,16 @@ public class RacingEventServiceImpl implements RacingEventService, ClearStateTes @Override public void windDataReceived(Wind wind, WindSource windSource) { - replicate(new RecordWindFix(getRaceIdentifier(), windSource, wind)); + if (windSource.getType() != WindSourceType.MANEUVER_BASED_ESTIMATION) { + replicate(new RecordWindFix(getRaceIdentifier(), windSource, wind)); + } } @Override public void windDataRemoved(Wind wind, WindSource windSource) { - replicate(new RemoveWindFix(getRaceIdentifier(), windSource, wind)); + if (windSource.getType() != WindSourceType.MANEUVER_BASED_ESTIMATION) { + replicate(new RemoveWindFix(getRaceIdentifier(), windSource, wind)); + } } @Override diff --git a/java/com.sap.sailing.windestimation.lab/ManeuverAndWindImporter.launch b/java/com.sap.sailing.windestimation.lab/SimpleModelsTrainingPart1.launch similarity index 81% rename from java/com.sap.sailing.windestimation.lab/ManeuverAndWindImporter.launch rename to java/com.sap.sailing.windestimation.lab/SimpleModelsTrainingPart1.launch index 49c1b7115d1..ac2cbb603ff 100755 --- a/java/com.sap.sailing.windestimation.lab/ManeuverAndWindImporter.launch +++ b/java/com.sap.sailing.windestimation.lab/SimpleModelsTrainingPart1.launch @@ -1,13 +1,13 @@ - + - + - + diff --git a/java/com.sap.sailing.windestimation.lab/SimpleModelsTrainingPart2.launch b/java/com.sap.sailing.windestimation.lab/SimpleModelsTrainingPart2.launch new file mode 100644 index 00000000000..ba6810c3b3e --- /dev/null +++ b/java/com.sap.sailing.windestimation.lab/SimpleModelsTrainingPart2.launch @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDistanceBasedAbsTwdTransitionImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDistanceBasedAbsTwdTransitionImporter.java index 7ad3616088c..ab97520b348 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDistanceBasedAbsTwdTransitionImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDistanceBasedAbsTwdTransitionImporter.java @@ -9,12 +9,18 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.SingleDimen public class AggregatedDistanceBasedAbsTwdTransitionImporter { public static void main(String[] args) throws UnknownHostException { - SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager( - SingleDimensionType.DISTANCE); + SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = createPersistenceManagerAndEnsureIndex(); SingleDimensionTwdTransitionAggregationImporter importer = new SingleDimensionTwdTransitionAggregationImporter( singleDimensionBasedTwdTransitionPersistenceManager, AggregatedSingleDimensionType.DISTANCE_ABS, new AnnealingNextThresholdCalculator(10, 1.5)); importer.runAggregation(); } + public static SingleDimensionBasedTwdTransitionPersistenceManager createPersistenceManagerAndEnsureIndex() + throws UnknownHostException { + SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager( + SingleDimensionType.DISTANCE); + return singleDimensionBasedTwdTransitionPersistenceManager; + } + } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDistanceBasedTwdTransitionImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDistanceBasedTwdTransitionImporter.java index a964e6d712e..f54c24ac7a6 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDistanceBasedTwdTransitionImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDistanceBasedTwdTransitionImporter.java @@ -9,12 +9,18 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.SingleDimen public class AggregatedDistanceBasedTwdTransitionImporter { public static void main(String[] args) throws UnknownHostException { - SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager( - SingleDimensionType.DISTANCE); + SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = createPersistenceManagerAndEnsureIndex(); SingleDimensionTwdTransitionAggregationImporter importer = new SingleDimensionTwdTransitionAggregationImporter( singleDimensionBasedTwdTransitionPersistenceManager, AggregatedSingleDimensionType.DISTANCE, new AnnealingNextThresholdCalculator(10, 1.5)); importer.runAggregation(); } + public static SingleDimensionBasedTwdTransitionPersistenceManager createPersistenceManagerAndEnsureIndex() + throws UnknownHostException { + SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager( + SingleDimensionType.DISTANCE); + return singleDimensionBasedTwdTransitionPersistenceManager; + } + } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDurationBasedAbsTwdTransitionImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDurationBasedAbsTwdTransitionImporter.java index d5e84ce3e53..ca8cc119fec 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDurationBasedAbsTwdTransitionImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDurationBasedAbsTwdTransitionImporter.java @@ -9,12 +9,18 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.SingleDimen public class AggregatedDurationBasedAbsTwdTransitionImporter { public static void main(String[] args) throws UnknownHostException { - SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager( - SingleDimensionType.DURATION); + SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = createPersistenceManagerAndEnsureIndex(); SingleDimensionTwdTransitionAggregationImporter importer = new SingleDimensionTwdTransitionAggregationImporter( singleDimensionBasedTwdTransitionPersistenceManager, AggregatedSingleDimensionType.DURATION_ABS, DurationBasedTwdTransitionImporter.getThresholdCalculator()); importer.runAggregation(); } + public static SingleDimensionBasedTwdTransitionPersistenceManager createPersistenceManagerAndEnsureIndex() + throws UnknownHostException { + SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager( + SingleDimensionType.DURATION); + return singleDimensionBasedTwdTransitionPersistenceManager; + } + } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDurationBasedTwdTransitionImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDurationBasedTwdTransitionImporter.java index baacd558f94..5fc7cc95d41 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDurationBasedTwdTransitionImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/AggregatedDurationBasedTwdTransitionImporter.java @@ -9,12 +9,18 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.SingleDimen public class AggregatedDurationBasedTwdTransitionImporter { public static void main(String[] args) throws UnknownHostException { - SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager( - SingleDimensionType.DURATION); + SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = createPersistenceManagerAndEnsureIndex(); SingleDimensionTwdTransitionAggregationImporter importer = new SingleDimensionTwdTransitionAggregationImporter( singleDimensionBasedTwdTransitionPersistenceManager, AggregatedSingleDimensionType.DURATION, DurationBasedTwdTransitionImporter.getThresholdCalculator()); importer.runAggregation(); } + public static SingleDimensionBasedTwdTransitionPersistenceManager createPersistenceManagerAndEnsureIndex() + throws UnknownHostException { + SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager( + SingleDimensionType.DURATION); + return singleDimensionBasedTwdTransitionPersistenceManager; + } + } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/DistanceBasedTwdTransitionImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/DistanceBasedTwdTransitionImporter.java index c33557994e5..2bf8ea62427 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/DistanceBasedTwdTransitionImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/DistanceBasedTwdTransitionImporter.java @@ -36,7 +36,8 @@ public class DistanceBasedTwdTransitionImporter { .getIterator(); iterator.hasNext();) { long percent = windSourceNumber * 100 / numberOfWindSources; WindSourceWithFixes windSource = iterator.next(); - LoggingUtil.logInfo("Processing " + windSourceNumber++ + "/" + numberOfWindSources + " (" + percent + "%)"); + LoggingUtil.logInfo("Processing wind source " + windSourceNumber++ + "/" + numberOfWindSources + " (" + + percent + "%) for distance dimension"); List entries = new ArrayList<>(); List otherWindSources = new ArrayList<>(); for (PersistedElementsIterator otherWindSourcesIterator = windSourcesPersistenceManager @@ -109,16 +110,17 @@ public class DistanceBasedTwdTransitionImporter { } while (currentFix != null && currentOtherFix != null); } if (entries.isEmpty()) { - LoggingUtil.logInfo("No TWD transitions to import"); + LoggingUtil.logInfo("No distance based TWD transitions to import"); } else { distanceBasedTwdTransitionPersistenceManager.add(entries); int totalEntries = entries.size(); totalValuesCount += totalEntries; - LoggingUtil.logInfo(totalEntries + " TWD transitions imported, " + totalValuesCount + " in total"); + LoggingUtil.logInfo( + totalEntries + " distance based TWD transitions imported, " + totalValuesCount + " in total"); } } LoggingUtil.logInfo("###################\r\nDistance based TWD transitions Import finished"); - LoggingUtil.logInfo("Totally " + totalValuesCount + " TWD transitions imported"); + LoggingUtil.logInfo("Totally " + totalValuesCount + " distance based TWD transitions imported"); } } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/DurationBasedTwdTransitionImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/DurationBasedTwdTransitionImporter.java index d5d94c4ec94..ddeaa241c13 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/DurationBasedTwdTransitionImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/DurationBasedTwdTransitionImporter.java @@ -32,7 +32,8 @@ public class DurationBasedTwdTransitionImporter { .getIterator(); iterator.hasNext();) { long percent = windSourceNumber * 100 / numberOfWindSources; WindSourceWithFixes windSource = iterator.next(); - LoggingUtil.logInfo("Processing " + windSourceNumber++ + "/" + numberOfWindSources + " (" + percent + "%)"); + LoggingUtil.logInfo("Processing wind source " + windSourceNumber++ + "/" + numberOfWindSources + " (" + + percent + "%) for duration dimension"); TimePoint timePointOfLastConsideredWindFix = null; List entries = new ArrayList<>(); int windFixIndex = 0; @@ -58,17 +59,17 @@ public class DurationBasedTwdTransitionImporter { } } if (entries.isEmpty()) { - LoggingUtil.logInfo("No TWD transitions to import"); + LoggingUtil.logInfo("No duration based TWD transitions to import"); } else { durationBasedTwdTransitionPersistenceManager.add(entries); int totalEntries = entries.size(); totalValuesCount += totalEntries; - LoggingUtil - .logInfo(totalEntries + " TWD transition entries imported, " + totalValuesCount + " in total"); + LoggingUtil.logInfo(totalEntries + " duration based TWD transition entries imported, " + + totalValuesCount + " in total"); } } LoggingUtil.logInfo("###################\r\nDuration based TWD transitions Import finished"); - LoggingUtil.logInfo("Totally " + totalValuesCount + " TWD transitions imported"); + LoggingUtil.logInfo("Totally " + totalValuesCount + " duration based TWD transitions imported"); } public static NextThresholdCalculator getThresholdCalculator() { diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java index 359302d2e17..0ca91010d4d 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java @@ -13,6 +13,10 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import org.apache.http.Header; import org.apache.http.HttpResponse; @@ -40,11 +44,11 @@ import com.sap.sailing.windestimation.data.persistence.maneuver.RaceWithManeuver import com.sap.sailing.windestimation.data.persistence.maneuver.RaceWithManeuverForEstimationPersistenceManager; import com.sap.sailing.windestimation.data.persistence.twdtransition.RaceWithWindSourcesPersistenceManager; import com.sap.sailing.windestimation.data.serialization.CompetitorTrackWithEstimationDataJsonDeserializer; -import com.sap.sailing.windestimation.data.serialization.ManeuverForDataAnalysisJsonSerializer; import com.sap.sailing.windestimation.data.serialization.LabeledManeuverForEstimationJsonSerializer; +import com.sap.sailing.windestimation.data.serialization.ManeuverForDataAnalysisJsonSerializer; import com.sap.sailing.windestimation.data.transformer.CompetitorTrackTransformer; -import com.sap.sailing.windestimation.data.transformer.CompleteManeuverCurveWithEstimationDataToManeuverForDataAnalysisTransformer; import com.sap.sailing.windestimation.data.transformer.CompleteManeuverCurveWithEstimationDataToLabelledManeuverForEstimationTransformer; +import com.sap.sailing.windestimation.data.transformer.CompleteManeuverCurveWithEstimationDataToManeuverForDataAnalysisTransformer; import com.sap.sailing.windestimation.util.LoggingUtil; /** @@ -54,6 +58,7 @@ import com.sap.sailing.windestimation.util.LoggingUtil; */ public class ManeuverAndWindImporter { + private static final int CONNECTION_TIMEOUT_MILLIS = 60000 * 5; public static final String REST_API_BASE_URL = "https://www.sapsailing.com/sailingserver/api/v1"; public static final String REST_API_REGATTAS_PATH = "/regattas"; public static final String REST_API_RACES_PATH = "/races"; @@ -70,6 +75,8 @@ public class ManeuverAndWindImporter { private final ManeuverForDataAnalysisJsonSerializer maneuverForDataAnalysisJsonSerializer; private final LabeledManeuverForEstimationJsonSerializer maneuverForEstimationJsonSerializer; private boolean skipRace; + private static final int NUMBER_OF_THREADS = 10; // high number due to HTTP requests + private final ThreadPoolExecutor executorService; public ManeuverAndWindImporter() throws UnknownHostException { this.completeManeuverCurvePersistanceManager = new RaceWithCompleteManeuverCurvePersistenceManager(); @@ -80,12 +87,28 @@ public class ManeuverAndWindImporter { this.maneuverForEstimationTransformer = new CompleteManeuverCurveWithEstimationDataToLabelledManeuverForEstimationTransformer(); this.maneuverForDataAnalysisJsonSerializer = new ManeuverForDataAnalysisJsonSerializer(); this.maneuverForEstimationJsonSerializer = new LabeledManeuverForEstimationJsonSerializer(); + this.executorService = new ThreadPoolExecutor(NUMBER_OF_THREADS, NUMBER_OF_THREADS, 0L, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(NUMBER_OF_THREADS)); + this.executorService.setRejectedExecutionHandler(new RejectedExecutionHandler() { + public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { + // this will block if the queue is full + try { + executor.getQueue().put(r); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + }); } public HttpClient createNewHttpClient() { HttpParams httpParams = new BasicHttpParams(); - HttpConnectionParams.setConnectionTimeout(httpParams, 30000); + HttpConnectionParams.setConnectionTimeout(httpParams, CONNECTION_TIMEOUT_MILLIS); HttpClient client = new SystemDefaultHttpClient(httpParams); + client.getParams().setParameter("http.socket.timeout", CONNECTION_TIMEOUT_MILLIS); + client.getParams().setParameter("http.connection.timeout", CONNECTION_TIMEOUT_MILLIS); + client.getParams().setParameter("http.connection-manager.timeout", new Long(CONNECTION_TIMEOUT_MILLIS)); + client.getParams().setParameter("http.protocol.head-body-timeout", CONNECTION_TIMEOUT_MILLIS); return client; } @@ -94,8 +117,8 @@ public class ManeuverAndWindImporter { importer.importAllRegattas(); } - public void importAllRegattas() - throws IllegalStateException, ClientProtocolException, IOException, ParseException, URISyntaxException { + public void importAllRegattas() throws IllegalStateException, ClientProtocolException, IOException, ParseException, + URISyntaxException, InterruptedException { skipRace = startFromRegattaName != null; LoggingUtil.logInfo("Importer for CompleteManeuverCurveWithEstimationData just started"); LoggingUtil.logInfo("Dropping old database"); @@ -118,9 +141,17 @@ public class ManeuverAndWindImporter { + Math.round(100.0 * i / numberOfRegattas) + "%): \"" + regattaName + "\""); importRegatta(regattaName, importStatistics); } - LoggingUtil.logInfo("Import finished"); - importStatistics.regattasCount = regattasJson.size(); - logImportStatistics(importStatistics); + executorService.shutdown(); + boolean success = executorService.awaitTermination(2, TimeUnit.HOURS); + if (success) { + LoggingUtil.logInfo("Import finished"); + synchronized (importStatistics) { + importStatistics.regattasCount = regattasJson.size(); + logImportStatistics(importStatistics); + } + } else { + new InterruptedException("Thread-pool was terminated after two hours waiting time"); + } } private void logImportStatistics(ImportStatistics importStatistics) { @@ -139,14 +170,15 @@ public class ManeuverAndWindImporter { private void importRegatta(String regattaName, ImportStatistics importStatistics) throws IllegalStateException, ClientProtocolException, IOException, ParseException, URISyntaxException { String encodedRegattaName = encodeUrlPathPart(regattaName); - // TODO The encoding of some regatta names with special characters does not work well HttpGet getRegatta = new HttpGet(REST_API_BASE_URL + REST_API_REGATTAS_PATH + "/" + encodedRegattaName); JSONObject regattaJson = null; try { regattaJson = (JSONObject) getHttpResponseAsJson(regattaName, null, getRegatta); } catch (Exception e) { - importStatistics.ignoredRegattas += 1; - LoggingUtil.logInfo("Error while processing regatta: " + regattaName); + synchronized (importStatistics) { + importStatistics.ignoredRegattas++; + } + LoggingUtil.logInfo("Error while processing regatta: " + regattaName + "\r\n" + getRegatta); return; } for (Object seriesJson : (JSONArray) regattaJson.get("series")) { @@ -161,17 +193,29 @@ public class ManeuverAndWindImporter { if ((boolean) race.get("isTracked") && !(boolean) race.get("isLive") && (boolean) race.get("hasGpsData") && (boolean) race.get("hasWindData")) { String trackedRaceName = (String) race.get("trackedRaceName"); - LoggingUtil.logInfo("Processing race nr. " + ++i + ": \"" + trackedRaceName + "\""); - try { - importRace(regattaName, trackedRaceName, importStatistics); - } catch (Exception e) { - importStatistics.ingoredRaces += 1; - LoggingUtil - .logInfo("Error while processing race nr. " + i + ": \"" + trackedRaceName + "\""); - } + i++; + final int raceNumber = i; + LoggingUtil.logInfo("Processing race nr. " + raceNumber + ": \"" + trackedRaceName + "\""); + executorService.execute(() -> { + try { + importRace(regattaName, trackedRaceName, importStatistics); + } catch (Exception e) { + synchronized (importStatistics) { + importStatistics.ingoredRaces += 1; + } + String extraLog = ""; + if (e instanceof HttpClientException) { + extraLog = "\r\n" + ((HttpClientException) e).getRequest(); + } + LoggingUtil.logInfo("Error while processing race nr. " + raceNumber + ": \"" + + trackedRaceName + "\"" + extraLog); + } + }); } } - importStatistics.racesCount += racesJson.size(); + synchronized (importStatistics) { + importStatistics.racesCount += racesJson.size(); + } } } } @@ -215,11 +259,14 @@ public class ManeuverAndWindImporter { JSONArray windFixesJson = (JSONArray) ((JSONObject) windSourceObj).get(RaceWindJsonSerializer.FIXES); windFixesCount += windFixesJson.size(); } - LoggingUtil.logInfo( - "Imported " + windFixesCount + " wind fixes from " + windSourcesJson.size() + " wind sources"); - importStatistics.racesWithHighQualityWindData++; + LoggingUtil.logInfo("Imported " + windFixesCount + " wind fixes from " + windSourcesJson.size() + + " wind sources in race \"" + trackedRaceName + "\" of regatta \"" + regattaName + "\""); + synchronized (importStatistics) { + importStatistics.racesWithHighQualityWindData++; + } } else { - LoggingUtil.logInfo("No high quality wind fixes contained"); + LoggingUtil.logInfo("No high quality wind fixes contained in race \"" + trackedRaceName + "\" of regatta \"" + + regattaName + "\""); } } @@ -272,10 +319,12 @@ public class ManeuverAndWindImporter { } catch (Exception e) { e.printStackTrace(); } - LoggingUtil.logInfo( - "Imported " + competitorTracks.size() + " competitor tracks with " + maneuversCount + " maneuvers"); - importStatistics.competitorTracksCount += competitorTracks.size(); - importStatistics.maneuversCount += maneuversCount; + LoggingUtil.logInfo("Imported " + competitorTracks.size() + " competitor tracks with " + maneuversCount + + " maneuvers in race \"" + trackedRaceName + "\" of regatta \"" + regattaName + "\""); + synchronized (importStatistics) { + importStatistics.competitorTracksCount += competitorTracks.size(); + importStatistics.maneuversCount += maneuversCount; + } } private JSONObject getHttpResponseAsJson(String trackedRegattaName, String trackedRaceName, @@ -285,7 +334,8 @@ public class ManeuverAndWindImporter { for (int i = 1; i <= 10; i++) { try { httpResponse = createNewHttpClient().execute(getEstimationData); - break; + JSONObject resultJson = (JSONObject) getJsonFromResponse(httpResponse); + return resultJson; } catch (Exception e) { Thread.sleep(10000); lastException = e; @@ -298,17 +348,7 @@ public class ManeuverAndWindImporter { } } } - if (httpResponse == null) { - throw lastException; - } - JSONObject resultJson; - try { - resultJson = (JSONObject) getJsonFromResponse(httpResponse); - } catch (Exception e) { - System.out.println(getEstimationData); - throw e; - } - return resultJson; + throw new HttpClientException(getEstimationData.toString(), lastException); } private void addTransformedElementsToCompetitorTrackJson(List competitorTracks, @@ -331,14 +371,13 @@ public class ManeuverAndWindImporter { } } - // FIXME duplicated code taken from ConnectivityUtils public static Object getJsonFromResponse(HttpResponse response) throws IllegalStateException, IOException, ParseException { JSONParser jsonParser = new JSONParser(); final Header contentEncoding = response.getEntity().getContentEncoding(); final Reader reader; if (contentEncoding == null) { - reader = new InputStreamReader(response.getEntity().getContent()); + reader = new InputStreamReader(response.getEntity().getContent(), "UTF-8"); } else { reader = new InputStreamReader(response.getEntity().getContent(), contentEncoding.getValue()); } @@ -358,4 +397,20 @@ public class ManeuverAndWindImporter { private int racesWithHighQualityWindData = 0; } + private static class HttpClientException extends Exception { + + private static final long serialVersionUID = 4948532287832868768L; + private final String request; + + public HttpClientException(String request, Exception e) { + super(e); + this.request = request; + } + + public String getRequest() { + return request; + } + + } + } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/SingleDimensionTwdTransitionAggregationImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/SingleDimensionTwdTransitionAggregationImporter.java index a7e0ee3cedc..23cbe105991 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/SingleDimensionTwdTransitionAggregationImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/SingleDimensionTwdTransitionAggregationImporter.java @@ -35,7 +35,7 @@ public class SingleDimensionTwdTransitionAggregationImporter { public void runAggregation() throws UnknownHostException { long totalValuesCount = 0; List aggregates = new ArrayList<>(); - LoggingUtil.logInfo("Aggregating persisted entries"); + LoggingUtil.logInfo("Aggregating persisted entries for dimension " + dimensionType); TDoubleList entries = new TDoubleArrayList(); double currentBucketThreshold = thresholdCalculator.getInitialThresholdValue(); double nextBucketThreshold = thresholdCalculator.getNextThresholdValue(currentBucketThreshold); @@ -61,7 +61,8 @@ public class SingleDimensionTwdTransitionAggregationImporter { } entries.add(twdChange); if (entries.size() % 10000 == 0) { - LoggingUtil.logInfo((entries.size() + totalValuesCount) + " Entries aggregated"); + LoggingUtil.logInfo( + (entries.size() + totalValuesCount) + " Entries aggregated for dimension " + dimensionType); } } if (entries.size() >= MIN_NUMBER_OF_VALUES_PER_BUCKET) { @@ -69,14 +70,16 @@ public class SingleDimensionTwdTransitionAggregationImporter { aggregates.add(aggregate); totalValuesCount += entries.size(); } - LoggingUtil.logInfo("Persisting " + aggregates.size() + " aggregates"); + LoggingUtil.logInfo("Persisting " + aggregates.size() + " aggregates for dimension " + dimensionType); AggregatedSingleDimensionBasedTwdTransitionPersistenceManager aggregatedDurationBasedTwdTransitionPersistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager( dimensionType); aggregatedDurationBasedTwdTransitionPersistenceManager.dropCollection(); aggregatedDurationBasedTwdTransitionPersistenceManager.add(aggregates); LoggingUtil.logInfo("###################\r\n" + dimensionType + " based TWD transitions Import finished"); - LoggingUtil.logInfo("Totally " + totalValuesCount + " TWD transitions imported"); - LoggingUtil.logInfo("Totally " + aggregates.size() + " TWD transition aggregates imported"); + LoggingUtil.logInfo( + "Totally " + totalValuesCount + " " + dimensionType + " based TWD transitions have been processed"); + LoggingUtil.logInfo( + "Totally " + aggregates.size() + " TWD transition aggregates imported for dimension " + dimensionType); } private double getFinalBucketThreshold(double currentBucketThreshold, double nextBucketThreshold) { diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/RaceWithWindSourcesPersistenceManager.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/RaceWithWindSourcesPersistenceManager.java index d047de632b4..724c9f3472a 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/RaceWithWindSourcesPersistenceManager.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/RaceWithWindSourcesPersistenceManager.java @@ -17,20 +17,16 @@ import com.sap.sailing.server.gateway.deserialization.JsonDeserializer; import com.sap.sailing.server.gateway.serialization.impl.RaceWindJsonSerializer; import com.sap.sailing.windestimation.data.RaceWithWindSources; import com.sap.sailing.windestimation.data.persistence.maneuver.AbstractPersistenceManager; -import com.sap.sailing.windestimation.data.persistence.maneuver.PersistedElementsIterator; import com.sap.sailing.windestimation.data.serialization.RaceWithEstimationDataDeserializer; import com.sap.sailing.windestimation.data.serialization.RaceWithWindSourcesDeserializer; -import com.sap.sse.common.TimePoint; public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceManager { - private static final String COLLECTION_NAME = "racesWithWind"; - private static final String WIND_SOURCES_COLLECTION_NAME = "windSources"; + static final String COLLECTION_NAME = "racesWithWind"; + private final WindSourcesPersistenceManager windSourcesPersistenceManager; public RaceWithWindSourcesPersistenceManager() throws UnknownHostException { - Document indexes = new Document(RaceWindJsonSerializer.START_TIME_POINT, 1); - indexes.put(RaceWindJsonSerializer.END_TIME_POINT, 1); - getCollection().createIndex(indexes); + windSourcesPersistenceManager = new WindSourcesPersistenceManager(); } @Override @@ -41,7 +37,7 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa @Override public void dropCollection() { super.dropCollection(); - getDb().getCollection(getWindSourcesCollectionName()).drop(); + windSourcesPersistenceManager.dropCollection(); } @Override @@ -52,7 +48,7 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa JSONArray windSourceIdsJson = (JSONArray) raceJson .get(RaceWithEstimationDataDeserializer.COMPETITOR_TRACKS); JSONArray windSourcesJson = new JSONArray(); - MongoCollection windSourcesCollection = getDb().getCollection(getWindSourcesCollectionName()); + MongoCollection windSourcesCollection = windSourcesPersistenceManager.getCollection(); for (Object idObject : windSourceIdsJson) { Document dbWindSource = windSourcesCollection .find(new Document(FIELD_DB_ID, new ObjectId((String) idObject))).first(); @@ -69,10 +65,6 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa }; } - public static String getWindSourcesCollectionName() { - return COLLECTION_NAME + "." + WIND_SOURCES_COLLECTION_NAME; - } - public void add(String regattaName, String raceName, JSONObject raceWithWindSourcesJson) { JSONArray windSourcesJson = (JSONArray) raceWithWindSourcesJson.get(RaceWindJsonSerializer.WIND_SOURCES); @@ -81,7 +73,7 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa Document entry = parseJsonString(windSourceObj.toString()); dbWindSources.add(entry); } - MongoCollection windSourcesCollection = getDb().getCollection(getWindSourcesCollectionName()); + MongoCollection windSourcesCollection = windSourcesPersistenceManager.getCollection(); windSourcesCollection.insertMany(dbWindSources); BasicDBList dbWindSourceIds = new BasicDBList(); for (Document dbWindSource : dbWindSources) { @@ -96,13 +88,4 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa races.insertOne(dbObject); } - public PersistedElementsIterator getIteratorForEntriesIntersectingPeriod( - TimePoint startTimePoint, TimePoint endTimePoint, double toleranceInSeconds) { - double maxStartTime = endTimePoint.asMillis() + toleranceInSeconds * 1000; - double maxEndTime = startTimePoint.asMillis() - toleranceInSeconds * 1000; - String query = "{$and: [{'" + RaceWindJsonSerializer.START_TIME_POINT + "': {$lte: " + maxStartTime + "}}, {'" - + RaceWindJsonSerializer.END_TIME_POINT + "': {$gte: " + maxEndTime + "}}]}"; - return getIterator(query); - } - } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/WindSourcesPersistenceManager.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/WindSourcesPersistenceManager.java index 0b8fbfc27ae..a49d93227b8 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/WindSourcesPersistenceManager.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/persistence/twdtransition/WindSourcesPersistenceManager.java @@ -21,15 +21,18 @@ import com.sap.sse.common.TimePoint; public class WindSourcesPersistenceManager extends AbstractPersistenceManager { + private static final String WIND_SOURCES_COLLECTION_NAME = "windSources"; + public WindSourcesPersistenceManager() throws UnknownHostException { - BasicDBObject indexes = new BasicDBObject(RaceWindJsonSerializer.START_TIME_POINT, 1); - indexes.put(RaceWindJsonSerializer.END_TIME_POINT, 1); + BasicDBObject indexes = new BasicDBObject("_id", 1); + indexes.put(RaceWindJsonSerializer.START_TIME_POINT, 1); + indexes.put(RaceWindJsonSerializer.END_TIME_POINT, -1); getCollection().createIndex(indexes); } @Override public String getCollectionName() { - return RaceWithWindSourcesPersistenceManager.getWindSourcesCollectionName(); + return RaceWithWindSourcesPersistenceManager.COLLECTION_NAME + "." + WIND_SOURCES_COLLECTION_NAME; } @Override diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/AggregatedDistanceDimensionPlot.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/AggregatedDistanceDimensionPlot.java index baa9f7bbaa5..ab81c226b02 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/AggregatedDistanceDimensionPlot.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/AggregatedDistanceDimensionPlot.java @@ -7,11 +7,17 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS public class AggregatedDistanceDimensionPlot { + private static SingleDimensionAggregatesPlottingFrame plot; + public static void main(String[] args) throws UnknownHostException { AggregatedSingleDimensionBasedTwdTransitionPersistenceManager persistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager( AggregatedSingleDimensionType.DISTANCE); - SingleDimensionAggregatesPlottingFrame plot = new SingleDimensionAggregatesPlottingFrame(persistenceManager); + plot = new SingleDimensionAggregatesPlottingFrame(persistenceManager); plot.renderChart(); } + public static void awaitWindowClosed() throws InterruptedException { + plot.awaitWindowClosed(); + } + } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/AggregatedDurationDimensionPlot.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/AggregatedDurationDimensionPlot.java index 4f38112b942..2903908050f 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/AggregatedDurationDimensionPlot.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/AggregatedDurationDimensionPlot.java @@ -7,11 +7,17 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS public class AggregatedDurationDimensionPlot { + private static SingleDimensionAggregatesPlottingFrame plot; + public static void main(String[] args) throws UnknownHostException { AggregatedSingleDimensionBasedTwdTransitionPersistenceManager persistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager( AggregatedSingleDimensionType.DURATION); - SingleDimensionAggregatesPlottingFrame plot = new SingleDimensionAggregatesPlottingFrame(persistenceManager); + plot = new SingleDimensionAggregatesPlottingFrame(persistenceManager); plot.renderChart(); } + + public static void awaitWindowClosed() throws InterruptedException { + plot.awaitWindowClosed(); + } } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/SingleDimensionAggregatesPlottingFrame.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/SingleDimensionAggregatesPlottingFrame.java index ea41fc2a1f4..eb4abbeae26 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/SingleDimensionAggregatesPlottingFrame.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/datavisualization/SingleDimensionAggregatesPlottingFrame.java @@ -3,9 +3,13 @@ package com.sap.sailing.windestimation.datavisualization; import java.awt.BasicStroke; import java.awt.Color; import java.awt.Shape; +import java.awt.event.WindowAdapter; +import java.awt.event.WindowEvent; import java.awt.geom.Ellipse2D; +import java.util.concurrent.CountDownLatch; import javax.swing.BoxLayout; +import javax.swing.JButton; import javax.swing.JFrame; import javax.swing.JPanel; import javax.swing.SwingUtilities; @@ -40,27 +44,54 @@ public class SingleDimensionAggregatesPlottingFrame extends JFrame { private XYSeries valuesSeries; private XYSeries stdSeries; private XYSeries zeroMeanStdSeries; + private final CountDownLatch windowClosedLatch = new CountDownLatch(1); public SingleDimensionAggregatesPlottingFrame( AggregatedSingleDimensionBasedTwdTransitionPersistenceManager persistenceManager) { super(persistenceManager.getCollectionName()); this.persistenceManager = persistenceManager; + addWindowListener(new WindowAdapter() { + @Override + public void windowClosed(WindowEvent event) { + windowClosedLatch.countDown(); + } + + @Override + public void windowClosing(WindowEvent e) { + setLayout(null); + setVisible(false); + dispose(); + } + }); + initializeDataWithCharts(); + } + + private void initializeDataWithCharts() { createSeries(); fillSeries(); DefaultTableXYDataset mainDataset = createMainDataset(); DefaultTableXYDataset histoDataset = createHistoDataset(); mainChart = createMainChart(mainDataset); histoChart = createHistoChart(histoDataset); - // Create Panel JPanel panel = new JPanel(); panel.setLayout(new BoxLayout(panel, BoxLayout.Y_AXIS)); ChartPanel mainChartPanel = new ChartPanel(mainChart); ChartPanel histoChartPanel = new ChartPanel(histoChart); + JButton refreshButton = new JButton("Refresh charts"); + refreshButton.addActionListener(event -> refreshCharts()); + panel.add(refreshButton); panel.add(mainChartPanel); panel.add(histoChartPanel); setContentPane(panel); } + public void refreshCharts() { + SwingUtilities.invokeLater(() -> { + initializeDataWithCharts(); + revalidate(); + }); + } + private void createSeries() { meanSeries = new XYSeries("Mean", true, false); medianSeries = new XYSeries("Median", true, false); @@ -191,4 +222,9 @@ public class SingleDimensionAggregatesPlottingFrame extends JFrame { }); } + public void awaitWindowClosed() throws InterruptedException { + setDefaultCloseOperation(WindowConstants.DO_NOTHING_ON_CLOSE); + windowClosedLatch.await(); + } + } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/evaluation/WindEstimatorManeuverNumberDependentEvaluationRunner.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/evaluation/WindEstimatorManeuverNumberDependentEvaluationRunner.java index c5f4869cf9c..85c3c6d03f2 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/evaluation/WindEstimatorManeuverNumberDependentEvaluationRunner.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/evaluation/WindEstimatorManeuverNumberDependentEvaluationRunner.java @@ -12,8 +12,8 @@ import com.sap.sailing.windestimation.data.persistence.maneuver.PersistedElement import com.sap.sailing.windestimation.data.persistence.maneuver.RaceWithCompleteManeuverCurvePersistenceManager; import com.sap.sailing.windestimation.data.persistence.polars.PolarDataServiceAccessUtil; import com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverFeatures; -import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl; import com.sap.sailing.windestimation.model.store.ModelStore; +import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl; import com.sap.sailing.windestimation.util.LoggingUtil; public class WindEstimatorManeuverNumberDependentEvaluationRunner { @@ -38,18 +38,15 @@ public class WindEstimatorManeuverNumberDependentEvaluationRunner { LoggingUtil.logInfo("Loading polar data"); PolarDataService polarService = PolarDataServiceAccessUtil.getPersistedPolarService(); LoggingUtil.logInfo("Wind estimator evaluation started..."); - ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models"); - // ModelStore modelStore = new MongoDbModelStoreImpl(persistenceManager.getDb()); + ModelStore modelStore = new MongoDbModelStoreImpl(persistenceManager.getDb()); WindEstimatorFactories estimatorFactories = new WindEstimatorFactories(polarService, new ManeuverFeatures(ENABLE_POLARS, ENABLE_SCALED_SPEED, ENABLE_MARKS_INFORMATION), modelStore); - double[] avgErrorDegreesPerManeuverCount = new double[MAX_MANEUVERS]; double[] avgConfidencePerManeuverCount = new double[MAX_MANEUVERS]; double[] avgConfidenceOfCorrectEstimationsPerManeuverCount = new double[MAX_MANEUVERS]; double[] avgConfidenceOfIncorrectEstimationsPerManeuverCount = new double[MAX_MANEUVERS]; double[] accuracyPerManeuverCount = new double[MAX_MANEUVERS]; double[] emptyEstimationsPercentagePerManeuverCount = new double[MAX_MANEUVERS]; - for (int fixedNumberOfManeuvers = 1; fixedNumberOfManeuvers <= MAX_MANEUVERS; fixedNumberOfManeuvers++) { LoggingUtil.logInfo("Running evaluation with " + fixedNumberOfManeuvers + " maneuvers"); WindEstimationEvaluator evaluator = new WindEstimationEvaluatorImpl<>( @@ -113,6 +110,7 @@ public class WindEstimatorManeuverNumberDependentEvaluationRunner { out.write(line); } } + LoggingUtil.logInfo("CSV with evaluation results have been stored in: " + csvFile.getAbsolutePath()); } } diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/ExportedModelsGenerator.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/ExportedModelsGenerator.java new file mode 100644 index 00000000000..db0ffeb05ee --- /dev/null +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/ExportedModelsGenerator.java @@ -0,0 +1,49 @@ +package com.sap.sailing.windestimation.model; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import com.sap.sailing.windestimation.integration.ExportedModels; +import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl; +import com.sap.sailing.windestimation.model.store.ModelDomainType; +import com.sap.sailing.windestimation.model.store.ModelStore; +import com.sap.sailing.windestimation.util.LoggingUtil; + +public class ExportedModelsGenerator { + + private static final File file = new File("windEstimationModels.dat"); + + public static void main(String[] args) throws FileNotFoundException, IOException { + ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models"); + // RegularManeuversForEstimationPersistenceManager persistenceManager = new + // RegularManeuversForEstimationPersistenceManager(); + // ModelStore modelStore = new MongoDbModelStoreImpl(persistenceManager.getDb()); + LoggingUtil.logInfo("Generation a single file with all trained models started"); + ExportedModels exportedModels = new ExportedModels(); + List nonEmptyDomainTypes = new ArrayList<>(); + for (ModelDomainType domainType : ModelDomainType.values()) { + LoggingUtil.logInfo("Started serialization of models belonging to domain type " + domainType); + Map exportedModelsForDomainType = modelStore.exportAllPersistedModels(domainType); + if (!exportedModelsForDomainType.isEmpty()) { + exportedModels.addSerializedModelsForDomainType(domainType, exportedModelsForDomainType); + LoggingUtil.logInfo("Finished serialization of " + exportedModelsForDomainType.size() + + " models belonging to domain type " + domainType); + nonEmptyDomainTypes.add(domainType); + } + } + LoggingUtil.logInfo("Serializing the exported models and writing it into a single file"); + try (ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(file))) { + oos.writeObject(exportedModels); + } + LoggingUtil.logInfo("Models have been generated"); + LoggingUtil.logInfo("The following domain types were considered: " + nonEmptyDomainTypes); + LoggingUtil.logInfo("File with all exported models was persisted in: " + file.getAbsolutePath()); + } + +} diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTrainingPart1.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTrainingPart1.java new file mode 100644 index 00000000000..f5e9eca6722 --- /dev/null +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTrainingPart1.java @@ -0,0 +1,153 @@ +package com.sap.sailing.windestimation.model; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import javax.swing.JOptionPane; + +import com.sap.sailing.windestimation.data.importer.AggregatedDistanceBasedTwdTransitionImporter; +import com.sap.sailing.windestimation.data.importer.AggregatedDurationBasedTwdTransitionImporter; +import com.sap.sailing.windestimation.data.importer.DistanceBasedTwdTransitionImporter; +import com.sap.sailing.windestimation.data.importer.DurationBasedTwdTransitionImporter; +import com.sap.sailing.windestimation.data.importer.ManeuverAndWindImporter; +import com.sap.sailing.windestimation.data.importer.PolarDataImporter; +import com.sap.sailing.windestimation.data.persistence.maneuver.ManeuverForEstimationPersistenceManager; +import com.sap.sailing.windestimation.data.persistence.maneuver.RegularManeuversForEstimationPersistenceManager; +import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedSingleDimensionBasedTwdTransitionPersistenceManager.AggregatedSingleDimensionType; +import com.sap.sailing.windestimation.datavisualization.AggregatedDistanceDimensionPlot; +import com.sap.sailing.windestimation.datavisualization.AggregatedDurationDimensionPlot; +import com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverClassifierTrainer; +import com.sap.sailing.windestimation.model.classifier.maneuver.PersistedManeuverClassifiersScorePrinter; +import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext; +import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext.DistanceValueRange; +import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext; +import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext.DurationValueRange; +import com.sap.sailing.windestimation.util.LoggingUtil; + +/** + * + * @author Vladislav Chumak (D069712) + * + */ +public class SimpleModelsTrainingPart1 { + + private static final int NUMBER_OF_THREADS = 3; + private static ExecutorService executorService; + + public static void main(String[] args) throws Exception { + new ManeuverForEstimationPersistenceManager().dropCollection(); + new RegularManeuversForEstimationPersistenceManager().dropCollection(); + executeInThreadPool(() -> PolarDataImporter.main(args)); + executeInThreadPool(() -> ManeuverAndWindImporter.main(args)); + awaitThreadPoolCompletion(); + executeInThreadPool(() -> { + ManeuverClassifierTrainer.main(args); + Thread.sleep(1000); + PersistedManeuverClassifiersScorePrinter.main(args); + }); + executeInThreadPool(() -> { + DistanceBasedTwdTransitionImporter.main(args); + }); + executeInThreadPool(() -> { + DurationBasedTwdTransitionImporter.main(args); + }); + awaitThreadPoolCompletion(); + AggregatedDurationBasedTwdTransitionImporter.createPersistenceManagerAndEnsureIndex(); + AggregatedDistanceBasedTwdTransitionImporter.createPersistenceManagerAndEnsureIndex(); + executeInThreadPool(() -> { + AggregatedDurationBasedTwdTransitionImporter.main(args); + }); + executeInThreadPool(() -> { + AggregatedDistanceBasedTwdTransitionImporter.main(args); + }); + awaitThreadPoolCompletion(); + do { + AggregatedDurationDimensionPlot.main(args); + showInfoAboutDataCleaning(AggregatedSingleDimensionType.DURATION); + AggregatedDurationDimensionPlot.awaitWindowClosed(); + } while (JOptionPane.YES_OPTION != askDataCleaningFinished(AggregatedSingleDimensionType.DURATION)); + showInfoAboutIntervalAdjustments(DurationBasedTwdTransitionRegressorModelContext.class, + DurationValueRange.class); + do { + AggregatedDistanceDimensionPlot.main(args); + showInfoAboutDataCleaning(AggregatedSingleDimensionType.DISTANCE); + AggregatedDistanceDimensionPlot.awaitWindowClosed(); + } while (JOptionPane.YES_OPTION != askDataCleaningFinished(AggregatedSingleDimensionType.DISTANCE)); + showInfoAboutIntervalAdjustments(DistanceBasedTwdTransitionRegressorModelContext.class, + DistanceValueRange.class); + showInfoAboutRunPart2(); + } + + private static int askDataCleaningFinished(AggregatedSingleDimensionType dimension) { + Object[] options = { "Continue with model training", "Restart Graphical Tool" }; + int res = JOptionPane.showOptionDialog(null, + "Have you finished the data cleansing for the " + dimension + " dimension?", + "Continue with model training?", JOptionPane.DEFAULT_OPTION, JOptionPane.INFORMATION_MESSAGE, null, + options, options[0]); + switch (res) { + case 0: + return JOptionPane.YES_OPTION; + case 1: + return JOptionPane.NO_OPTION; + } + return JOptionPane.CLOSED_OPTION; + } + + private static void showInfoAboutRunPart2() { + JOptionPane.showMessageDialog(null, "Now, run the class \"" + SimpleModelsTrainingPart2.class.getName() + + "\".\nThis will complete the training process."); + } + + private static void showInfoAboutDataCleaning(AggregatedSingleDimensionType dimension) { + JOptionPane.showMessageDialog(null, "Now, clean the data for the " + dimension + + " dimension.\nRemove instances from MongoDB collection \"" + dimension.getCollectioName() + + "\"\nwhich do not make sense and cause implausible zig zag sections\nwithin \"Zero mean sigma\" curve.\n\nClose the graphical tool, when you are done to resume the model training."); + } + + private static void showInfoAboutIntervalAdjustments(Class classToAdjust, Class valueRangeEnum) { + JOptionPane.showMessageDialog(null, "Now, open the source code of the class \"" + classToAdjust.getName() + + "\".\nScroll down to the definition of the inner enum \"" + valueRangeEnum.getSimpleName() + + "\",\nread its JavaDoc and adjust its interval definitions so that\neach interval can be learned by the adjusted regressor model configuration\nwith minimal error.\n\nPress OK after you are done."); + } + + private static void awaitThreadPoolCompletion() throws InterruptedException { + executorService.shutdown(); + boolean success = executorService.awaitTermination(24, TimeUnit.HOURS); + if (!success) { + new InterruptedException("Thread-pool was terminated after 24 hours waiting time"); + } + Thread.sleep(1000L); + } + + private static void executeInThreadPool(RunnableWithExceptionsCatch runnable) { + if (executorService == null || executorService.isShutdown()) { + createNewThreadPool(); + } + executorService.execute(() -> { + try { + runnable.run(); + } catch (Throwable t) { + t.printStackTrace(); + System.exit(1); + LoggingUtil.logInfo("FAILURE: Caught unexpected exception. Model training aborted"); + } + }); + } + + private static void createNewThreadPool() { + executorService = new ThreadPoolExecutor(NUMBER_OF_THREADS, NUMBER_OF_THREADS, 0, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(NUMBER_OF_THREADS), new ThreadPoolExecutor.CallerRunsPolicy()); + } + + /** + * + * @author Vladislav Chumak (D069712) + * + */ + private static interface RunnableWithExceptionsCatch { + void run() throws Exception; + } + +} diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTrainingPart2.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTrainingPart2.java new file mode 100644 index 00000000000..7388eca611d --- /dev/null +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTrainingPart2.java @@ -0,0 +1,22 @@ +package com.sap.sailing.windestimation.model; + +import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionStdRegressorTrainer; +import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionStdRegressorTrainer; +import com.sap.sailing.windestimation.util.LoggingUtil; + +/** + * + * @author Vladislav Chumak (D069712) + * + */ +public class SimpleModelsTrainingPart2 { + + public static void main(String[] args) throws Exception { + DurationBasedTwdTransitionStdRegressorTrainer.main(args); + DistanceBasedTwdTransitionStdRegressorTrainer.main(args); + Thread.sleep(1000); + ExportedModelsGenerator.main(args); + LoggingUtil.logInfo("Model training finished. You can upload the generated file to a server instance."); + } + +} diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/ManeuverClassifierTrainer.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/ManeuverClassifierTrainer.java index cdbbc86ff6e..17697ea3f30 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/ManeuverClassifierTrainer.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/ManeuverClassifierTrainer.java @@ -19,9 +19,9 @@ import com.sap.sailing.windestimation.data.persistence.maneuver.TransformedManeu import com.sap.sailing.windestimation.data.persistence.polars.PolarDataServiceAccessUtil; import com.sap.sailing.windestimation.model.classifier.LabelExtraction; import com.sap.sailing.windestimation.model.classifier.TrainableClassificationModel; -import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl; import com.sap.sailing.windestimation.model.store.ModelDomainType; import com.sap.sailing.windestimation.model.store.ModelStore; +import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl; import com.sap.sailing.windestimation.util.LoggingUtil; import com.sap.sse.common.Util.Pair; @@ -66,7 +66,6 @@ public class ManeuverClassifierTrainer { LoggingUtil.logInfo("Training with " + trainManeuvers.size() + " maneuvers..."); double[][] x = modelContext.getXMatrix(maneuvers); int[] y = labelExtraction.getYVector(maneuvers); - classifierModel.resetTrainingStats(); classifierModel.train(x, y); LoggingUtil.logInfo("Training finished. Validating on train dataset..."); ManeuverClassifierScoring classifierScoring = new ManeuverClassifierScoring(classifierModel); @@ -112,8 +111,7 @@ public class ManeuverClassifierTrainer { public static void main(String[] args) throws Exception { PolarDataService polarService = PolarDataServiceAccessUtil.getPersistedPolarService(); RegularManeuversForEstimationPersistenceManager persistenceManager = new RegularManeuversForEstimationPersistenceManager(); - ModelStore classifierModelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models"); - // ModelStore classifierModelStore = new MongoDbModelStore(persistenceManager.getDb()); + ModelStore classifierModelStore = new MongoDbModelStoreImpl(persistenceManager.getDb()); classifierModelStore.deleteAll(ModelDomainType.MANEUVER_CLASSIFIER); ManeuverClassifierTrainer classifierTrainer = new ManeuverClassifierTrainer(persistenceManager, classifierModelStore); diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/PersistedManeuverClassifiersScorePrinter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/PersistedManeuverClassifiersScorePrinter.java index df007fc823e..5e857143c59 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/PersistedManeuverClassifiersScorePrinter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/PersistedManeuverClassifiersScorePrinter.java @@ -13,11 +13,12 @@ import java.util.regex.Pattern; import com.sap.sailing.domain.base.BoatClass; import com.sap.sailing.domain.polars.PolarDataService; import com.sap.sailing.windestimation.data.ManeuverForEstimation; +import com.sap.sailing.windestimation.data.persistence.maneuver.RegularManeuversForEstimationPersistenceManager; import com.sap.sailing.windestimation.data.persistence.polars.PolarDataServiceAccessUtil; import com.sap.sailing.windestimation.model.classifier.TrainableClassificationModel; import com.sap.sailing.windestimation.model.exception.ModelPersistenceException; -import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl; import com.sap.sailing.windestimation.model.store.ModelStore; +import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl; import com.sap.sailing.windestimation.util.LoggingUtil; public class PersistedManeuverClassifiersScorePrinter { @@ -29,9 +30,8 @@ public class PersistedManeuverClassifiersScorePrinter { throws MalformedURLException, ClassNotFoundException, IOException, InterruptedException { PolarDataService polarService = PolarDataServiceAccessUtil.getPersistedPolarService(); Set allBoatClasses = polarService.getAllBoatClassesWithPolarSheetsAvailable(); - ModelStore classifierModelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models"); - // ModelStore classifierModelStore = new MongoDbModelStore( - // new RegularManeuversForEstimationPersistenceManager().getDb()); + ModelStore classifierModelStore = new MongoDbModelStoreImpl( + new RegularManeuversForEstimationPersistenceManager().getDb()); List> allClassifierModels = new ArrayList<>(); ManeuverClassifierModelFactory classifierModelFactory = new ManeuverClassifierModelFactory(); LoggingUtil.logInfo("### Loading all boat class classifiers:"); diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/twdtransition/TwdTransitionClassifierTrainer.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/twdtransition/TwdTransitionClassifierTrainer.java index 3645d4e86b2..6ea95350108 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/twdtransition/TwdTransitionClassifierTrainer.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/twdtransition/TwdTransitionClassifierTrainer.java @@ -100,7 +100,6 @@ public class TwdTransitionClassifierTrainer { i++; } LoggingUtil.logInfo("Training with " + numberOfTrainingInstances + " instances..."); - classifierModel.resetTrainingStats(); classifierModel.train(x, y); LoggingUtil.logInfo("Training finished. Validating on train dataset..."); TwdTransitionClassifierScoring classifierScoring = new TwdTransitionClassifierScoring(classifierModel); diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DistanceBasedTwdTransitionStdRegressorTrainer.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DistanceBasedTwdTransitionStdRegressorTrainer.java index 9aba743456c..32bbd98b57f 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DistanceBasedTwdTransitionStdRegressorTrainer.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DistanceBasedTwdTransitionStdRegressorTrainer.java @@ -5,9 +5,9 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedSingleDimensionBasedTwdTransitionPersistenceManager.AggregatedSingleDimensionType; import com.sap.sailing.windestimation.model.regressor.IncrementalSingleDimensionPolynomialRegressor; import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext.DistanceValueRange; -import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl; import com.sap.sailing.windestimation.model.store.ModelDomainType; import com.sap.sailing.windestimation.model.store.ModelStore; +import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl; /** * Trains TWD delta standard deviation by considering the distance passed between two measurements. @@ -26,8 +26,7 @@ public class DistanceBasedTwdTransitionStdRegressorTrainer extends TwdTransition public static void main(String[] args) throws Exception { AggregatedSingleDimensionBasedTwdTransitionPersistenceManager distanceBasedPersistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager( AggregatedSingleDimensionType.DISTANCE); - ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models"); - // ModelStore modelStore = new MongoDbModelStore(distanceBasedPersistenceManager.getDb()); + ModelStore modelStore = new MongoDbModelStoreImpl(distanceBasedPersistenceManager.getDb()); modelStore.deleteAll(ModelDomainType.DISTANCE_BASED_TWD_DELTA_STD_REGRESSOR); DistanceBasedTwdTransitionRegressorModelFactory distanceBasedTwdTransitionRegressorModelFactory = new DistanceBasedTwdTransitionRegressorModelFactory(); for (DistanceValueRange distanceValueRange : DistanceValueRange.values()) { diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DurationBasedTwdTransitionStdRegressorTrainer.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DurationBasedTwdTransitionStdRegressorTrainer.java index febdea8a634..d092edd36b0 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DurationBasedTwdTransitionStdRegressorTrainer.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DurationBasedTwdTransitionStdRegressorTrainer.java @@ -5,9 +5,9 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedSingleDimensionBasedTwdTransitionPersistenceManager.AggregatedSingleDimensionType; import com.sap.sailing.windestimation.model.regressor.IncrementalSingleDimensionPolynomialRegressor; import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext.DurationValueRange; -import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl; import com.sap.sailing.windestimation.model.store.ModelDomainType; import com.sap.sailing.windestimation.model.store.ModelStore; +import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl; public class DurationBasedTwdTransitionStdRegressorTrainer extends TwdTransitionAggregatedStdRegressorTrainer { @@ -20,8 +20,7 @@ public class DurationBasedTwdTransitionStdRegressorTrainer extends TwdTransition public static void main(String[] args) throws Exception { AggregatedSingleDimensionBasedTwdTransitionPersistenceManager durationBasedPersistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager( AggregatedSingleDimensionType.DURATION); - ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models"); - // ModelStore modelStore = new MongoDbModelStoreImpl(durationBasedPersistenceManager.getDb()); + ModelStore modelStore = new MongoDbModelStoreImpl(durationBasedPersistenceManager.getDb()); modelStore.deleteAll(ModelDomainType.DURATION_BASED_TWD_DELTA_STD_REGRESSOR); DurationBasedTwdTransitionRegressorModelFactory durationBasedTwdTransitionRegressorModelFactory = new DurationBasedTwdTransitionRegressorModelFactory(); for (DurationValueRange durationValueRange : DurationValueRange.values()) { diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/IncrementalSingleDimensionPolynomialRegressorTrainerHelper.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/IncrementalSingleDimensionPolynomialRegressorTrainerHelper.java index 7db214fcdbb..3cb95a314d5 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/IncrementalSingleDimensionPolynomialRegressorTrainerHelper.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/IncrementalSingleDimensionPolynomialRegressorTrainerHelper.java @@ -41,6 +41,9 @@ public class IncrementalSingleDimensionPolynomialRegressorTrainerHelper { } public void incrementRmseCalculation(double[] x, double y) { + if (!model.isModelReady()) { + model.setModelAsReadyAfterSuccessfulTraining(); + } double predictedStd = model.getValue(x); double diff = predictedStd - y; squareErrorSum += diff * diff; diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/TwdTransitionManualTrainingDataInputRegressorTrainer.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/TwdTransitionManualTrainingDataInputRegressorTrainer.java index aa358ccb5c8..fd67c7a7ba9 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/TwdTransitionManualTrainingDataInputRegressorTrainer.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/TwdTransitionManualTrainingDataInputRegressorTrainer.java @@ -21,8 +21,7 @@ public class TwdTransitionManualTrainingDataInputRegressorTrainer { throws Exception { IncrementalSingleDimensionPolynomialRegressorTrainerHelper trainerHelper = new IncrementalSingleDimensionPolynomialRegressorTrainerHelper( regressorModelStore, model); - LoggingUtil - .logInfo("########## Training of " + model.getModelContext().getId() + " started..."); + LoggingUtil.logInfo("########## Training of " + model.getModelContext().getId() + " started..."); double[] modelInput = new double[1]; for (int i = 0; i < inputOutputPairs.length; i++) { double xi = inputOutputPairs[i][0]; diff --git a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRaceTest.java b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRaceTest.java index 5fd94446243..5aaa9dbb496 100644 --- a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRaceTest.java +++ b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRaceTest.java @@ -38,6 +38,7 @@ import com.sap.sailing.domain.tracking.WindTrack; import com.sap.sailing.domain.tracking.WindWithConfidence; import com.sap.sailing.domain.tracking.impl.DynamicTrackedRaceImpl; import com.sap.sailing.domain.tractracadapter.ReceiverType; +import com.sap.sailing.domain.windestimation.TimePointAndPositionWithToleranceComparator; import com.sap.sailing.windestimation.ManeuverBasedWindEstimationComponentImpl; import com.sap.sailing.windestimation.aggregator.ManeuverClassificationsAggregatorFactory; import com.sap.sailing.windestimation.data.CompetitorTrackWithEstimationData; @@ -75,14 +76,14 @@ public class IncrementalMstHmmWindEstimationForTrackedRaceTest extends OnlineTra "SERIALIZATION.modelForManeuverClassifier.NeuralNetworkClassifier.ManeuverClassification-Basic-All.clf" }; protected final SimpleDateFormat dateFormat; - private final WindEstimationFactoryServiceImpl windEstimationFactoryService; + private WindEstimationFactoryServiceImpl windEstimationFactoryService; private ClassPathReadOnlyModelStoreImpl modelStore; - public IncrementalMstHmmWindEstimationForTrackedRaceTest() - throws MalformedURLException, URISyntaxException, ModelPersistenceException { + public IncrementalMstHmmWindEstimationForTrackedRaceTest() throws Exception { dateFormat = new SimpleDateFormat("MM/dd/yyyy-HH:mm:ss"); dateFormat.setTimeZone(TimeZone.getTimeZone("GMT+2")); // will result in CEST windEstimationFactoryService = new WindEstimationFactoryServiceImpl(); + windEstimationFactoryService.clearState(); modelStore = new ClassPathReadOnlyModelStoreImpl("trained_wind_estimation_models", getClass().getClassLoader(), modelFilesNames); windEstimationFactoryService.importAllModelsFromModelStore(modelStore); diff --git a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstManeuverGraphGeneratorTest.java b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstManeuverGraphGeneratorTest.java index 0424bb9c479..7d7d72f4e6e 100644 --- a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstManeuverGraphGeneratorTest.java +++ b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/integration/IncrementalMstManeuverGraphGeneratorTest.java @@ -29,6 +29,7 @@ import com.sap.sailing.domain.test.OnlineTracTracBasedTest; import com.sap.sailing.domain.tracking.CompleteManeuverCurve; import com.sap.sailing.domain.tracking.impl.DynamicTrackedRaceImpl; import com.sap.sailing.domain.tractracadapter.ReceiverType; +import com.sap.sailing.domain.windestimation.TimePointAndPositionWithToleranceComparator; import com.sap.sailing.windestimation.aggregator.msthmm.DistanceAndDurationAwareWindTransitionProbabilitiesCalculator; import com.sap.sailing.windestimation.aggregator.msthmm.MstGraphLevel; import com.sap.sailing.windestimation.aggregator.msthmm.MstManeuverGraphGenerator.MstManeuverGraphComponents; diff --git a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataClientMock.java b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataClientMock.java new file mode 100644 index 00000000000..c1292444471 --- /dev/null +++ b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataClientMock.java @@ -0,0 +1,47 @@ +package com.sap.sailing.windestimation.jaxrs.api; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectOutputStream; +import java.util.Map; + +import org.json.simple.parser.ParseException; + +import com.sap.sailing.windestimation.integration.ExportedModels; +import com.sap.sailing.windestimation.integration.ReplicableWindEstimationFactoryService; +import com.sap.sailing.windestimation.jaxrs.client.WindEstimationDataClient; +import com.sap.sailing.windestimation.model.store.ModelDomainType; +import com.sap.sailing.windestimation.model.store.ModelStore; + +/** + * + * @author Vladislav Chumak (D069712) + * + */ +public class WindEstimationDataClientMock extends WindEstimationDataClient { + + private final ModelStore modelStore; + + public WindEstimationDataClientMock(ModelStore modelStore, + ReplicableWindEstimationFactoryService windEstimationFactoryService) { + super(null, windEstimationFactoryService); + this.modelStore = modelStore; + } + + @Override + protected InputStream getContentFromResponse() throws IOException, ParseException { + ExportedModels exportedModels = new ExportedModels(); + for (ModelDomainType domainType : ModelDomainType.values()) { + Map exportedModelsForDomainType = modelStore.exportAllPersistedModels(domainType); + exportedModels.addSerializedModelsForDomainType(domainType, exportedModelsForDomainType); + } + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + try (ObjectOutputStream oos = new ObjectOutputStream(byteArrayOutputStream)) { + oos.writeObject(exportedModels); + } + return new ByteArrayInputStream(byteArrayOutputStream.toByteArray()); + } + +} diff --git a/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResourceTest.java b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResourceTest.java new file mode 100644 index 00000000000..8f628d61e28 --- /dev/null +++ b/java/com.sap.sailing.windestimation.test/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResourceTest.java @@ -0,0 +1,33 @@ +package com.sap.sailing.windestimation.jaxrs.api; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +import com.sap.sailing.windestimation.integration.IncrementalMstHmmWindEstimationForTrackedRaceTest; +import com.sap.sailing.windestimation.integration.WindEstimationFactoryServiceImpl; +import com.sap.sailing.windestimation.model.store.ClassPathReadOnlyModelStoreImpl; + +/** + * + * @author Vladislav Chumak (D069712) + * + */ +public class WindEstimationDataResourceTest { + + @Test + public void testImportingFromMockFile() throws Exception { + WindEstimationFactoryServiceImpl windEstimationFactoryService = new WindEstimationFactoryServiceImpl(); + windEstimationFactoryService.clearState(); + assertFalse(windEstimationFactoryService.isReady()); + ClassPathReadOnlyModelStoreImpl modelStore = new ClassPathReadOnlyModelStoreImpl( + "trained_wind_estimation_models", getClass().getClassLoader(), + IncrementalMstHmmWindEstimationForTrackedRaceTest.modelFilesNames); + final WindEstimationDataClientMock client = new WindEstimationDataClientMock(modelStore, + windEstimationFactoryService); + client.updateWindEstimationModels(); + assertTrue(windEstimationFactoryService.isReady()); + } + +} \ No newline at end of file diff --git a/java/com.sap.sailing.windestimation/.classpath b/java/com.sap.sailing.windestimation/.classpath index e3eae022f73..3c7b5c88b1e 100644 --- a/java/com.sap.sailing.windestimation/.classpath +++ b/java/com.sap.sailing.windestimation/.classpath @@ -3,6 +3,7 @@ + diff --git a/java/com.sap.sailing.windestimation/lib/README b/java/com.sap.sailing.windestimation/lib/README new file mode 100644 index 00000000000..9ada6a11b62 --- /dev/null +++ b/java/com.sap.sailing.windestimation/lib/README @@ -0,0 +1,9 @@ +The library smile-score-1.5.2.jar has been patched!!! +The class files smile.feature.FeatureTransform and smile.feature.Standardizer +were removed from jar in order to overwrite those classes and make them Serializable +(see com.sap.sailing.windestimation/src/smile/feature/). The removal was necessary +due to OSGI environment. The next release version of SMILE will already include +Serialization support for the aforementioned classes, which means that the patch will +be obsolete. When 1.5.3 comes out, replace the 1.5.2 jars with the official 1.5.3 jars. +Furthermore, delete com.sap.sailing.windestimation/src/smile folder with patched +source files. \ No newline at end of file diff --git a/java/com.sap.sailing.windestimation/resources/shiro.ini b/java/com.sap.sailing.windestimation/resources/shiro.ini new file mode 100644 index 00000000000..0449ff0fd0a --- /dev/null +++ b/java/com.sap.sailing.windestimation/resources/shiro.ini @@ -0,0 +1,74 @@ + +[main] +credentialsMatcher = org.apache.shiro.authc.credential.Sha256CredentialsMatcher +credentialsMatcher.storedCredentialsHexEncoded = false +credentialsMatcher.hashIterations = 1024 + +# Realm configuration: +# -------------------- + +permissionsForRoleProvider = com.sap.sailing.domain.common.security.SailingPermissionsForRoleProvider +upRealm = com.sap.sse.security.UsernamePasswordRealm +upRealm.credentialsMatcher = $credentialsMatcher +upRealm.permissionsForRoleProvider = $permissionsForRoleProvider +oauthRealm = com.sap.sse.security.OAuthRealm +oauthRealm.permissionsForRoleProvider = $permissionsForRoleProvider +bearerTokenRealm = com.sap.sse.security.BearerTokenRealm +bearerTokenRealm.permissionsForRoleProvider = $permissionsForRoleProvider + +# Putting it all together: +sessionManager = com.sap.sse.security.SecurityWebSessionManager +securityManager.sessionManager = $sessionManager +sessionDAO = org.apache.shiro.session.mgt.eis.EnterpriseCacheSessionDAO +securityManager.sessionManager.sessionDAO = $sessionDAO +subjectDAO = com.sap.sse.security.NoSessionStorageSessionDAO +securityManager.subjectDAO = $subjectDAO +# 1y timeout +securityManager.sessionManager.globalSessionTimeout = 31536000000 +cacheManager = com.sap.sse.security.SessionCacheManager +securityManager.cacheManager = $cacheManager + +# Authentication Filter Configurations +# ------------------------------------ + +# A filter that requires the user to have at least one of the roles specified as +# filter parameter. This makes it different from the default roles[...] filter +# which requires the user to have ALL of the roles specified as parameters. +anyofroles = com.sap.sse.security.AnyOfRolesFilter + +# A custom filter for GWT pages that require an authenticated user. +# Forwards unauthenticated users to the default login page. When +# addressed directly (and not based on a redirect), users who logged +# on successfully will be redirected to the UserManagement.html page +# where they can adjust their profile. +customGwt = com.sap.sse.security.CustomFilter +customGwt.loginUrl = /security/ui/Login.html +customGwt.successUrl = /UserManagement.html + +# Configuration for the default shiro HTTP form authentication filter. +# It assumes that form-based login parameters are sent to the +# /api/restsecurity/login RESTlet and forwards successful login attempts +# to the /api/restsecurity/hello service which is expected to respond +# with a JSON document containing the authenticated subject's properties. +authc.loginUrl = /api/restsecurity/login +authc.successUrl = /api/restsecurity/hello + +# This authentication filter accepts a bearer access token in the HTTP +# Authorization header field, as in +# Authorization: Bearer 1029741026501365024376093245 +# If no such bearer token is provided, the filter falls back to basic HTTP +# authentication ("Authentication: Basic ...") and then regular form-based +# authentication with POST parameters "username" and "password". +bearerToken = com.sap.sse.security.BearerTokenOrBasicOrFormAuthenticationFilter + +# Specifying filter chains for URL patterns + +[urls] +# Uncomment the following to require the permission "event:view" in order to use /api/v1/events: +#/api/v1/events = bearerToken + +# Example: Secure access to leaderboard race column factor updates; read access should require no permissions +#/api/v1/leaderboards/*/racecolumnfactors/** = bearerToken + +# Example: Secure the access (in particular the POST/write access) to posting race log events +#/rc/racelog = bearerToken \ No newline at end of file diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/AbstractReplicableWithObjectInputStream.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/AbstractReplicableWithObjectInputStream.java index 4c53ede9a41..332c0146be0 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/AbstractReplicableWithObjectInputStream.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/AbstractReplicableWithObjectInputStream.java @@ -3,6 +3,7 @@ package com.sap.sailing.windestimation.integration; import java.io.IOException; import java.io.InputStream; import java.io.ObjectInputStream; +import java.io.ObjectStreamClass; import java.io.Serializable; import java.util.HashSet; import java.util.Set; @@ -105,7 +106,7 @@ public abstract class AbstractReplicableWithObjectInputStream resolveClass(ObjectStreamClass desc) throws IOException, ClassNotFoundException { + String name = desc.getName(); + try { + return Class.forName(name, /* initialize */true, Thread.currentThread().getContextClassLoader()); + } catch (ClassNotFoundException ex) { + // fall back on super class in order to load primitive types + return super.resolveClass(desc); + } + } + + } + } diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/Activator.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/Activator.java index d43c821f7bd..a60d8530c6d 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/Activator.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/Activator.java @@ -88,10 +88,10 @@ public class Activator implements BundleActivator { private void importWindEstimationModelsFromUrl(String windEstimationModelDataSourceURL) { logger.info("Importing wind estimation data from URL: " + windEstimationModelDataSourceURL); - WindEstimationDataClient polarDataClient = new WindEstimationDataClient(windEstimationModelDataSourceURL, - service); + WindEstimationDataClient windEstimationDataClient = new WindEstimationDataClient( + windEstimationModelDataSourceURL, service); try { - polarDataClient.updateWindEstimationModels(); + windEstimationDataClient.updateWindEstimationModels(); } catch (Exception e) { logger.log(Level.SEVERE, "Exception while trying to import wind estimation model data from " + windEstimationModelDataSourceURL, e); diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRace.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRace.java index ad8203bb347..c98d8e1ea15 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRace.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/IncrementalMstHmmWindEstimationForTrackedRace.java @@ -17,9 +17,11 @@ import com.sap.sailing.domain.maneuverdetection.TrackTimeInfo; import com.sap.sailing.domain.polars.PolarDataService; import com.sap.sailing.domain.tracking.CompleteManeuverCurve; import com.sap.sailing.domain.tracking.TrackedRace; +import com.sap.sailing.domain.tracking.WindTrack; import com.sap.sailing.domain.tracking.WindWithConfidence; -import com.sap.sailing.domain.tracking.impl.WindTrackImpl; -import com.sap.sailing.domain.windestimation.IncrementalWindEstimationTrack; +import com.sap.sailing.domain.windestimation.IncrementalWindEstimation; +import com.sap.sailing.domain.windestimation.TimePointAndPositionWithToleranceComparator; +import com.sap.sailing.domain.windestimation.WindTrackWithConfidenceForEachWindFixImpl; import com.sap.sailing.windestimation.aggregator.hmm.GraphLevelInference; import com.sap.sailing.windestimation.aggregator.msthmm.DistanceAndDurationAwareWindTransitionProbabilitiesCalculator; import com.sap.sailing.windestimation.aggregator.msthmm.MstBestPathsCalculator; @@ -48,10 +50,8 @@ import com.sap.sse.common.Util.Pair; * @author Vladislav Chumak (D069712) * */ -public class IncrementalMstHmmWindEstimationForTrackedRace extends WindTrackImpl - implements IncrementalWindEstimationTrack { +public class IncrementalMstHmmWindEstimationForTrackedRace implements IncrementalWindEstimation { - private static final long serialVersionUID = 6654969432545619955L; private static final double WIND_COURSE_TOLERANCE_IN_DEGREES_TO_IGNORE_FOR_REUSE = 1.0; private static final double DEFAULT_BASE_CONFIDENCE = 0.01; private final IncrementalMstManeuverGraphGenerator mstManeuverGraphGenerator; @@ -60,15 +60,17 @@ public class IncrementalMstHmmWindEstimationForTrackedRace extends WindTrackImpl private final Map, WindWithConfidence>> windTrackWithConfidences = new TreeMap<>( new TimePointAndPositionWithToleranceComparator()); private final TrackedRace trackedRace; - private WindSource windSource; + private final WindSource windSource; + private final WindTrackWithConfidenceForEachWindFixImpl estimatedWindTrack; public IncrementalMstHmmWindEstimationForTrackedRace(TrackedRace trackedRace, WindSource windSource, PolarDataService polarDataService, long millisecondsOverWhichToAverage, ManeuverClassifiersCache maneuverClassifiersCache, GaussianBasedTwdTransitionDistributionCache gaussianBasedTwdTransitionDistributionCache) { - super(millisecondsOverWhichToAverage, DEFAULT_BASE_CONFIDENCE, + this.estimatedWindTrack = new WindTrackWithConfidenceForEachWindFixImpl(millisecondsOverWhichToAverage, + DEFAULT_BASE_CONFIDENCE, WindSourceType.MANEUVER_BASED_ESTIMATION.useSpeed() && polarDataService != null, - IncrementalMstHmmWindEstimationForTrackedRace.class.getName(), false); + IncrementalMstHmmWindEstimationForTrackedRace.class.getName(), false, windTrackWithConfidences); this.trackedRace = trackedRace; this.windSource = windSource; DistanceAndDurationAwareWindTransitionProbabilitiesCalculator transitionProbabilitiesCalculator = new DistanceAndDurationAwareWindTransitionProbabilitiesCalculator( @@ -82,11 +84,16 @@ public class IncrementalMstHmmWindEstimationForTrackedRace extends WindTrackImpl : new PolarsBasedTwsCalculatorImpl(polarDataService)); } + @Override + public WindTrack getWindTrack() { + return estimatedWindTrack; + } + @Override public void newManeuverSpotsDetected(Competitor competitor, Iterable newManeuvers, TrackTimeInfo trackTimeInfo) { List maneuversWithEstimatedType = new ArrayList<>(); - lockForWrite(); + estimatedWindTrack.lockForWrite(); try { for (CompleteManeuverCurve newManeuverSpot : newManeuvers) { mstManeuverGraphGenerator.add(competitor, newManeuverSpot, trackTimeInfo); @@ -134,7 +141,7 @@ public class IncrementalMstHmmWindEstimationForTrackedRace extends WindTrackImpl } } } finally { - unlockAfterWrite(); + estimatedWindTrack.unlockAfterWrite(); } } @@ -146,13 +153,6 @@ public class IncrementalMstHmmWindEstimationForTrackedRace extends WindTrackImpl return true; } - @Override - protected double getConfidenceOfInternalWindFixUnsynchronized(Wind windFix) { - WindWithConfidence> windWithConfidence = windTrackWithConfidences - .get(new Pair<>(windFix.getPosition(), windFix.getTimePoint())); - return super.getConfidenceOfInternalWindFixUnsynchronized(windFix) * windWithConfidence.getConfidence(); - } - @Override public WindSource getWindSource() { return windSource; diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/ReplicableWindEstimationFactoryService.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/ReplicableWindEstimationFactoryService.java index d2206164c9f..9edf9621ef5 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/ReplicableWindEstimationFactoryService.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/ReplicableWindEstimationFactoryService.java @@ -2,6 +2,7 @@ package com.sap.sailing.windestimation.integration; import com.sap.sailing.domain.polars.PolarDataService; import com.sap.sailing.domain.windestimation.WindEstimationFactoryService; +import com.sap.sailing.windestimation.model.exception.ModelPersistenceException; import com.sap.sse.replication.ReplicableWithObjectInputStream; /** @@ -11,5 +12,7 @@ import com.sap.sse.replication.ReplicableWithObjectInputStream; * */ public interface ReplicableWindEstimationFactoryService extends WindEstimationFactoryService, - ReplicableWithObjectInputStream { + ReplicableWithObjectInputStream { + + void updateWindEstimationModels(ExportedModels exportedModels) throws ModelPersistenceException; } diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationFactoryServiceImpl.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationFactoryServiceImpl.java index 16864ae1440..fd138198fda 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationFactoryServiceImpl.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationFactoryServiceImpl.java @@ -11,7 +11,7 @@ import java.util.Map; import com.sap.sailing.domain.common.WindSourceType; import com.sap.sailing.domain.common.impl.WindSourceImpl; import com.sap.sailing.domain.tracking.TrackedRace; -import com.sap.sailing.domain.windestimation.IncrementalWindEstimationTrack; +import com.sap.sailing.domain.windestimation.IncrementalWindEstimation; import com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverClassifiersCache; import com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverFeatures; import com.sap.sailing.windestimation.model.exception.ModelPersistenceException; @@ -23,7 +23,7 @@ import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl; import com.sap.sse.mongodb.MongoDBService; public class WindEstimationFactoryServiceImpl extends - AbstractReplicableWithObjectInputStream + AbstractReplicableWithObjectInputStream implements ReplicableWindEstimationFactoryService { private static final boolean PRELOAD_ALL_MODELS = true; @@ -52,6 +52,7 @@ public class WindEstimationFactoryServiceImpl extends PRESERVE_LOADED_MODELS_MILLIS, MAX_MANEUVER_FEATURES); gaussianBasedTwdTransitionDistributionCache = new GaussianBasedTwdTransitionDistributionCache(MODEL_STORE, PRELOAD_ALL_MODELS, PRESERVE_LOADED_MODELS_MILLIS); + modelsReady = maneuverClassifiersCache.isReady() && gaussianBasedTwdTransitionDistributionCache.isReady(); } private boolean isMaster() { @@ -59,8 +60,8 @@ public class WindEstimationFactoryServiceImpl extends } @Override - public IncrementalWindEstimationTrack createIncrementalWindEstimationTrack(TrackedRace trackedRace) { - IncrementalWindEstimationTrack windEstimation = new IncrementalMstHmmWindEstimationForTrackedRace(trackedRace, + public IncrementalWindEstimation createIncrementalWindEstimationTrack(TrackedRace trackedRace) { + IncrementalWindEstimation windEstimation = new IncrementalMstHmmWindEstimationForTrackedRace(trackedRace, new WindSourceImpl(WindSourceType.MANEUVER_BASED_ESTIMATION), trackedRace.getPolarDataService(), trackedRace.getMillisecondsOverWhichToAverageWind(), maneuverClassifiersCache, gaussianBasedTwdTransitionDistributionCache); @@ -79,15 +80,17 @@ public class WindEstimationFactoryServiceImpl extends updateWindEstimationModels(exportedModels); } + @Override public void updateWindEstimationModels(ExportedModels exportedModels) throws ModelPersistenceException { for (ModelDomainType domainType : modelDomainTypesRequiredByWindEstimation) { Map serializedModelsForDomainType = exportedModels .getSerializedModelsForDomainType(domainType); if (serializedModelsForDomainType != null) { + MODEL_STORE.deleteAll(domainType); MODEL_STORE.importPersistedModels(serializedModelsForDomainType, domainType); } } - clearState(); + reloadWindEstimationModels(); } @Override @@ -135,15 +138,19 @@ public class WindEstimationFactoryServiceImpl extends for (ModelDomainType domainType : modelDomainTypesRequiredByWindEstimation) { MODEL_STORE.deleteAll(domainType); } - clearState(); + reloadWindEstimationModels(); + } + + @Override + public void clearState() throws Exception { + clearReplicaState(); } /** * Clears/Reloads all caches with machine learning models. Notifies ready state change listeners about the new ready * state of this wind estimation instance. */ - @Override - public void clearState() { + public void reloadWindEstimationModels() { maneuverClassifiersCache.clearCache(); gaussianBasedTwdTransitionDistributionCache.clearCache(); List modelsChangedListeners; @@ -166,7 +173,7 @@ public class WindEstimationFactoryServiceImpl extends MODEL_STORE.deleteAll(domainType); MODEL_STORE.importPersistedModels(exportedModels, domainType); } - clearState(); + reloadWindEstimationModels(); } /** diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperation.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperation.java index c2853417f30..fd49cb49a69 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperation.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperation.java @@ -3,13 +3,26 @@ package com.sap.sailing.windestimation.integration; import com.sap.sse.replication.OperationWithResult; /** - * - * Needed in the context of the initial replication of the wind estimation factory service. + * Updates wind estimation models with provided exported models. * * @author Vladislav Chumak (D069712) * */ -public interface WindEstimationModelsUpdateOperation - extends OperationWithResult { +public class WindEstimationModelsUpdateOperation + implements OperationWithResult { + + private static final long serialVersionUID = 6407877837605630090L; + + private final ExportedModels exportedModels; + + public WindEstimationModelsUpdateOperation(ExportedModels exportedModels) { + this.exportedModels = exportedModels; + } + + @Override + public Void internalApplyTo(ReplicableWindEstimationFactoryService toState) throws Exception { + toState.updateWindEstimationModels(exportedModels); + return null; + } } diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperationImpl.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperationImpl.java deleted file mode 100644 index f8de7330282..00000000000 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/integration/WindEstimationModelsUpdateOperationImpl.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.sap.sailing.windestimation.integration; - -public class WindEstimationModelsUpdateOperationImpl implements WindEstimationModelsUpdateOperation { - - private static final long serialVersionUID = 6407877837605630090L; - - private final ExportedModels exportedModels; - - public WindEstimationModelsUpdateOperationImpl(ExportedModels exportedModels) { - this.exportedModels = exportedModels; - } - - @Override - public Void internalApplyTo(WindEstimationFactoryServiceImpl toState) throws Exception { - toState.updateWindEstimationModels(exportedModels); - return null; - } - -} diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResource.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResource.java index 849ff4eef77..a7611a4a688 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResource.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/jaxrs/api/WindEstimationDataResource.java @@ -18,10 +18,10 @@ import com.sap.sailing.domain.common.security.Permission; import com.sap.sailing.domain.common.security.Permission.Mode; import com.sap.sailing.windestimation.integration.ExportedModels; import com.sap.sailing.windestimation.integration.WindEstimationModelsUpdateOperation; -import com.sap.sailing.windestimation.integration.WindEstimationModelsUpdateOperationImpl; import com.sap.sailing.windestimation.jaxrs.AbstractWindEstimationDataResource; /** + * Resource which manages wind estimation model metadata. Data export and import are supported. * * @author Vladislav Chumak (D069712) * @@ -45,7 +45,7 @@ public class WindEstimationDataResource extends AbstractWindEstimationDataResour ObjectInputStream ois = getWindEstimationFactoryServiceImpl() .createObjectInputStreamResolvingAgainstCache(inputStream); ExportedModels exportedModels = (ExportedModels) ois.readObject(); - WindEstimationModelsUpdateOperation windEstimationModelsUpdateOperation = new WindEstimationModelsUpdateOperationImpl( + WindEstimationModelsUpdateOperation windEstimationModelsUpdateOperation = new WindEstimationModelsUpdateOperation( exportedModels); getWindEstimationFactoryServiceImpl().apply(windEstimationModelsUpdateOperation); return Response.ok("Wind estimation models accepted").build(); diff --git a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/model/AbstractTrainableModel.java b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/model/AbstractTrainableModel.java index f30f010d9ae..6d934bae272 100644 --- a/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/model/AbstractTrainableModel.java +++ b/java/com.sap.sailing.windestimation/src/com/sap/sailing/windestimation/model/AbstractTrainableModel.java @@ -51,12 +51,16 @@ public abstract class AbstractTrainableModel trainInternalModel(double[][] x, int[] y); diff --git a/wiki/howto/windestimation.md b/wiki/howto/windestimation.md index f834ee77019..83412cc7104 100644 --- a/wiki/howto/windestimation.md +++ b/wiki/howto/windestimation.md @@ -1,67 +1,29 @@ # Training of internal Wind Estimation models -This document describes the generation process of Machine Learning (ML) models which are used internally by wind estimation. It is highly recommended to proceed this howto step by step considering the order of sections. - -## Overview -In total, there are the following three categories of ML models used by wind estimation: -1. **Maneuver Classifiers** -2. **Regressors** of TWD delta standard deviation for the dimension **duration** -3. **Regressors** of TWD delta standard deviation for the dimension **distance** - -Each of the model categories are composed of multiple models where each model targets a specific context. A context for a maneuver classifier is determined by the following attributes: -* Maneuver features - * Polar features enabled: yes/no - * Mark features enabled: yes/no - * Scaled speed features enabled: yes/no -* Boat class filtering for the data on which the classifier is trained, such as a specific boat class, or with all boat classes included - -The context of regressor models is represented by its assigned input interval responsibility, e.g. [0 seconds; 62 seconds) for duration, or [80 meters; 1368 meters) for distance. - -Each of the ML model categories must be trained individually. The common workflow looks as follows: -1. Get the training data from REST API of sapsailing.com -2. Preprocess data -3. Train the model category - -For each of the steps, appropriate Java classes must be executed per *Run with...->Java Application*. All referenced classes are located in *com.sap.sailing.windestimation.lab* Java project. Each class execution must finish without uncaught exceptions before proceeding to next instructions. After model training, all trained models can be collected in *./trained_wind_estimation_models*, which is normally */path/to/workspace/com.sap.sailing.windestimation/trained_wind_estimation_models* if you start the training classes in Eclipse per *Run with...->Java Application*. - -The details of the training process for each model category are described in the following sections. +This document describes the generation process of Machine Learning (ML) models which are used internally by wind estimation. It is highly recommended to proceed this howto step by step considering the order of sections. At the end of this howto, you will generate a file containing the representation of internal models used by ``com.sap.sailing.windestimation`` bundle. You can use this file to update the wind estimation models of a running server instance. If you are interested in a more advanced tutorial which requires all the execution steps contained in ``SimpleModelsTrainingPart...`` classes to be executed manually, then you might be interested in [Advanced Guide for training of internal Wind Estimation models](./windestimationAdvanced.md) ## Prerequisites To complete the training process successfully, you need to make sure that you have the following stuff: * A complete onboarding setup for SAP Sailing Analytics development -* MongoDB (**3.4 or higher!**) is up and running (same MongoDB instance as required in onboarding howto) +* MongoDB (**3.4 or higher!**) is up and running (can be the same MongoDB instance as required in onboarding howto) * At least 100 GB free space on the partition, where MongoDB is operating * Installed graphical MongoDB client such as MongoDB Compass (Community version) +* 16 GB RAM +* 24+ operating hours of your computer -## Get the training data from sapsailing.com -The following steps import all the data required from sapsailing.com into the local MongoDB. These steps constitute a preprequisite for training of all ML model categories: -1. Run *com.sap.sailing.windestimation.data.importer.ManeuverAndWindImporter* -2. Run *com.sap.sailing.windestimation.data.importer.PolarDataImporter* - -## Maneuver classifiers training -1. Run *com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverClassifierTrainer*. Within the this step, the maneuver data is preprocessed and all maneuver classifiers are trained for each supported context. -2. Optionally run *com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverClassifierScoring* to print the performance of the trained classifiers. After this step, a list with macro-averaged F2-score of each trained classifier will be stored in *./maneuverClassifierScores.csv* - -## Duration-based TWD delta standard deviation regressor - -1. Run *com.sap.sailing.windestimation.data.importer.DurationBasedTwdTransitionImporter* -2. Run *com.sap.sailing.windestimation.data.importer.AggregatedDurationBasedTwdTransitionImporter* -3. Run *com.sap.sailing.windestimation.datavisualization.AggregatedDurationDimensionPlot* to visualize the wind data. A Swing-based GUI-Window must open with two charts, one XY-chart where the x-axis represents **seconds**, and the y-axis represents TWD delta-based series measures (e.g. standard deviation or mean). Below the chart, a histogram for data points of the XY-Chart is provided. You can zoom-in and zoom-out in each of the chart by mouse dragging. Be aware that currently the zoom level of both charts is not synchronized -4. Open your graphical MongoDB client and connect to *windEstimation* database hosted by your local MongoDB. Open the collection with name *aggregatedDurationTwdTransition*. Within the collection you will see all the instances/data points visualized in the previous step. The total number of the points must not exceed 100. -5. Delete all the instances within the collection which do not make sense. For this, use the data visualization tool from step 3 to identify such instances. Pay a special attention to the instances in the beginnning and end. Some of the instances are not representative due to small number of supporting instances which is visualized in the histogram. Restart the data visualization tool as often as need to visualize the changed data. -6. Open the source code of the class *com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext*. Scroll down to the definition of the inner class/enum *DurationValueRange*. The enum defines the intervals for which a separate regressor model will be trained. Adjust the intervals accordingly in order to allow the regressor model to learn the data curve with minimal error. Make sure that there are at least 2 data points available within each interval. Datapoint with x = 0, y = 0 will be created automatically. -7. Run *com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionStdRegressorTrainer* -8. Verify the trained regressor functions. They are printed in the console output of the previous step. For instance, you can visualize the polynoms by means of https://www.wolframalpha.com/ - -## Distance-based TWD delta standard deviation regressor - -The steps of this sections are similar to the steps of the previous section. It is recommended to traverse through the previous section before starting with this one, because due to similarity of the steps, the similar steps in this section are described with less details and hints. - -1. Run *com.sap.sailing.windestimation.data.importer.DistanceBasedTwdTransitionImporter* -2. Run *com.sap.sailing.windestimation.data.importer.AggregatedDistanceBasedTwdTransitionImporter* with at least 10 GB JVM memory. -3. Run *com.sap.sailing.windestimation.datavisualization.AggregatedDistanceDimensionPlot* to visualize the wind data. Here, the x-axis of the XY-chart represents **meters** -4. Open your graphical MongoDB client and connect to *windEstimation* database hosted by your local MongoDB. Open collection *aggregatedDistanceTwdTransition* collection. Within the collection you will see all the instances/data points visualized in the previous step. The total number of the points must not exceed 100. -5. Delete all the instances within the collection which do not make sense. -6. Open the source code of the class *com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext*. Scroll down to the definition of the inner class/enum *DistanceValueRange*. The enum defines the intervals for which a separate regressor model will be trained. Adjust the intervals accordingly in order to allow the regressor model to learn the data curve with minimal error. -7. Run *com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionStdRegressorTrainer* -8. Verify the trained regressor functions. They are printed in the console output of the previous step. \ No newline at end of file +## Model training process +1. Run ``com.sap.sailing.windestimation.model.SimpleModelsTrainingPart1`` as a normal Java Application. After this, all the necessary maneuver and wind data will be downloaded, pre-processed and maneuver classifiers get trained. +2. Make sure that the launched program does not get termined by an uncaught exception. Wait until graphical info dialog shows up which requests you to perform data cleansing for duration dimension. + ![Screenshot of graphical info dialog requesting to perform data cleansing for duration dimension](../images/windestimation/dialogRequestingDataCleansingForDurationDimension.jpg "Screenshot of graphical info dialog requesting to perform data cleansing for duration dimension") + Press OK. Afterwards, a graphical window must open with two charts. The top chart is an XY-chart where the x-axis represents **seconds** and the y-axis represents various TWD delta-based measures (e.g. standard deviation or mean). Below the XY-chart, a histogram for the data points of the XY-Chart is provided. You can zoom-in and zoom-out in each of the charts by mouse dragging. Be aware that currently, the zoom level of both charts is not synchronizing. + ![Screenshot of graphical wind data visualization tool for duration dimension](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png "Screenshot of graphical wind data visualization tool for duration dimension") +3. Open your graphical MongoDB client and connect to ``windEstimation`` database hosted within your local MongoDB. Open the collection with name ``aggregatedDurationTwdTransition``. Within the collection, you will see all the instances/data points visualized in the previous step. The attribute used for the x-axis is ``value``. + ![Screenshot of MongoDB Compass with opened aggregatedDurationTwdTransition collection](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png "Screenshot of MongoDB Compass with opened aggregatedDurationTwdTransition collection") +4. Delete all the instances within the collection which do not make sense. For this, use the data visualization tool from step 2 to identify such instances. Some of the instances are not representative due to the small number of supporting instances which is visualized in the histogram. Such instances can produce unreasonable bumps in the XY-chart. The desired output of this step is that the curve ``Zero mean sigma`` looks smooth and always growing, e.g. as depicted below: + ![Screenshot of graphical visualization tool of duration dimension with after data cleansing](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png "Screenshot of graphical visualization tool of duration dimension with after data cleansing") + Use the ``Refresh charts`` button as often as needed to update the charts with the modified data in MongoDB. Close the graphical visualization tool window after you are done with data cleansing to resume the training process. A confirmation dialog shows up. Confirm it by pressing *"Continue with model training"* button. + ![Screenshot of confirmation dialog for finishing the data cleansing](../images/windestimation/confirmationDialogAfterDataCleansingDurationDimension.jpg "Screenshot of confirmation dialog for finishing the data cleansing") +5. A new information dialog shows up (**do not press OK yet!**) requesting you to open the source code of the class ``com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext``. Open it and scroll down to the definition of the inner enum ``DurationValueRange``. The enum defines the intervals for which a separate regressor model will be trained. Read the Javadoc of ``DurationValueRange`` and adjust the intervals accordingly in order to allow the regressor model to learn the ``Zero mean sigma`` curve with minimal error. You can also configure the polynomial which will be used for regressor training. Make sure that there are at least 2 data points contained within each configured interval. The data point with x = 0, y = 0 will be created automatically within model training procedure. Press OK in information dialog after you are done. +6. A graphical info dialog shows up which requests you to perform data cleansing for the *distance* dimension. Press OK. All steps for data cleansing for the distance dimension are analogous to the steps of the duration dimension described from step 2. until step 5. Thus, consult these steps in order to complete the data cleansing for the distance dimension. The unit used for the distance representation is **meters**. The collection name required in step 3. is ``aggregatedDistanceTwdTransition``. The class required in step 5. is ``com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext`` and its inner enum is ``DistanceValueRange``. +7. Run ``com.sap.sailing.windestimation.model.SimpleModelsTrainingPart2`` as a normal Java Application. Wait until the model training finishes and the program terminates normally. A new file with serialized representation of internal wind estimation models should be located in ``./windEstimationModels.dat``. The absolute path of the file must be printed in the console output of the program. You can upload the file via HTTP POST to http://sapsailing.com/windestimation/api/windestimation_data (see ``com.sap.sailing.windestimation.jaxrs.api.WindEstimationDataResource``) to update the wind estimation of a server instance. If you changed the source files of ``DurationValueRange`` or ``DistanceValueRange``, then you will need to update ``com.sap.sailing.windestimation`` bundle of the server instance which is meant to receive the new wind estimation models. +8. Optionally, run ``com.sap.sailing.windestimation.evaluation.WindEstimatorManeuverNumberDependentEvaluationRunner`` as normal Java Application to evaluate the wind estimation with the new trained models. The evaluation score will be stored as CSV in ``./maneuverNumberDependentEvaluation.csv``. \ No newline at end of file diff --git a/wiki/howto/windestimationAdvanced.md b/wiki/howto/windestimationAdvanced.md new file mode 100644 index 00000000000..54dc2538b05 --- /dev/null +++ b/wiki/howto/windestimationAdvanced.md @@ -0,0 +1,80 @@ +# Training of internal Wind Estimation models + +This document describes the generation process of Machine Learning (ML) models which are used internally by wind estimation. It is highly recommended to proceed this howto step by step considering the order of sections. At the end of this howto, you will generate a file containing the representation of internal models used by ``com.sap.sailing.windestimation`` bundle. You can use this file to update the wind estimation models of a running server instance. If you are interested in a simpler tutorial with less execution steps thanks to automation, then you might be interested in [Simple Guide for training of internal Wind Estimation models](./windestimation.md) + +## Overview +In total, there are the following three categories of ML models used by wind estimation: +1. **Maneuver Classifiers** +2. **Regressors** of TWD delta standard deviation for the dimension **duration** +3. **Regressors** of TWD delta standard deviation for the dimension **distance** + +Each of the model categories are composed of multiple models where each model targets a specific context. A context for a maneuver classifier is determined by the following attributes: +* Maneuver features + * Polar features enabled: yes/no + * Mark features enabled: yes/no + * Scaled speed features enabled: yes/no +* Boat class filtering for the data on which the classifier is trained, such as a specific boat class, or with all boat classes included + +The context of regressor models is represented by its assigned input interval responsibility, e.g. [0 seconds; 62 seconds) for duration, or [80 meters; 1368 meters) for distance. + +Each of the ML model categories must be trained individually. The common workflow looks as follows: +1. Get the training data from REST API of sapsailing.com +2. Preprocess data +3. Train the model category + +For each of the steps, appropriate Java classes must be executed per *Run with...->Java Application*. All referenced classes are located in *com.sap.sailing.windestimation.lab* Java project. Each class execution must finish without termination due to uncaught exceptions before proceeding to next instruction. You can skip the training of a model category if you do not want to update the models on server of that category. After model training, A new file with serialized representation of internal wind estimation models should be located in ``./windEstimationModels.dat``, which is normally */path/to/workspace/com.sap.sailing.windestimation/trained_wind_estimation_models* if you start the training classes in Eclipse per *Run with...->Java Application*. + +The details of the training process for each model category are described in the following sections. + +## Prerequisites +To complete the training process successfully, you need to make sure that you have the following stuff: +* A complete onboarding setup for SAP Sailing Analytics development +* MongoDB (**3.4 or higher!**) is up and running +* At least 100 GB free space on the partition, where MongoDB is operating +* Installed graphical MongoDB client such as MongoDB Compass (Community version) + +## Get the training data from sapsailing.com +The following steps import all the data required from sapsailing.com into the local MongoDB. These steps constitute a preprequisite for training of all ML model categories: +1. Run *com.sap.sailing.windestimation.data.importer.ManeuverAndWindImporter* +2. Run *com.sap.sailing.windestimation.data.importer.PolarDataImporter* + +## Maneuver classifiers training +1. Run *com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverClassifierTrainer*. Within the this step, the maneuver data is preprocessed and all maneuver classifiers are trained for each supported context. +2. Optionally run *com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverClassifierScoring* to print the performance of the trained classifiers. After this step, a list with macro-averaged F2-score of each trained classifier will be stored in *./maneuverClassifierScores.csv* + +## Duration-based TWD delta standard deviation regressor + +1. Run *com.sap.sailing.windestimation.data.importer.DurationBasedTwdTransitionImporter* +2. Run *com.sap.sailing.windestimation.data.importer.AggregatedDurationBasedTwdTransitionImporter* +3. Run *com.sap.sailing.windestimation.datavisualization.AggregatedDurationDimensionPlot* to visualize the wind data. A Swing-based GUI-Window must open with two charts, one XY-chart where the x-axis represents **seconds**, and the y-axis represents TWD delta-based series measures (e.g. standard deviation or mean). Below the chart, a histogram for data points of the XY-Chart is provided. You can zoom-in and zoom-out in each of the chart by mouse dragging. Be aware that currently the zoom level of both charts is not synchronized +4. Open your graphical MongoDB client and connect to *windEstimation* database hosted by your local MongoDB. Open the collection with name *aggregatedDurationTwdTransition*. Within the collection you will see all the instances/data points visualized in the previous step. The total number of the points must not exceed 100. +5. Delete all the instances within the collection which do not make sense. For this, use the data visualization tool from step 3 to identify such instances. Some of the instances are not representative due to the small number of supporting instances which is visualized in the histogram. Such instances can produce unreasonable bumps in the XY-chart. The desired output of this step is that the curve ``Zero mean sigma`` looks smooth and always growing. Use the ``Refresh charts`` button as often as needed to update the charts with the modified data in MongoDB. +6. Open the source code of the class ``com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext``. Open it and scroll down to the definition of the inner enum ``DurationValueRange``. The enum defines the intervals for which a separate regressor model will be trained. Read the Javadoc of ``DurationValueRange`` and adjust the intervals accordingly in order to allow the regressor model to learn the ``Zero mean sigma`` curve with minimal error. You can also configure the polynomial which will be used for regressor training. Make sure that there are at least 2 data points contained within each configured interval. The data point with x = 0, y = 0 will be created automatically within model training procedure. +7. Run *com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionStdRegressorTrainer* +8. Verify the trained regressor functions. They are printed in the console output of the previous step. For instance, you can visualize the polynoms by means of https://www.wolframalpha.com/ + +## Distance-based TWD delta standard deviation regressor + +The steps of this sections are similar to the steps of the previous section. It is recommended to traverse through the previous section before starting with this one, because due to similarity of the steps, the similar steps in this section are described with less details and hints. + +1. Run *com.sap.sailing.windestimation.data.importer.DistanceBasedTwdTransitionImporter* +2. Run *com.sap.sailing.windestimation.data.importer.AggregatedDistanceBasedTwdTransitionImporter* with at least 10 GB JVM memory. +3. Run *com.sap.sailing.windestimation.datavisualization.AggregatedDistanceDimensionPlot* to visualize the wind data. Here, the x-axis of the XY-chart represents **meters** +4. Open your graphical MongoDB client and connect to *windEstimation* database hosted by your local MongoDB. Open collection *aggregatedDistanceTwdTransition* collection. Within the collection you will see all the instances/data points visualized in the previous step. The total number of the points must not exceed 100. +5. Delete all the instances within the collection which do not make sense. +6. Open the source code of the class *com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext*. Scroll down to the definition of the inner class/enum *DistanceValueRange*. The enum defines the intervals for which a separate regressor model will be trained. Adjust the intervals accordingly in order to allow the regressor model to learn the data curve with minimal error. +7. Run *com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionStdRegressorTrainer* +8. Verify the trained regressor functions. They are printed in the console output of the previous step. + +## Generate the models file + +Within this section, all the trained models produce within previous sections are aggregated and stored as one single file. + +1. Run ``com.sap.sailing.windestimation.model.ExportedModelsGenerator``. Wait until the model training finishes and the program terminates normally. A new file with serialized representation of internal wind estimation models should be located in ``./windEstimationModels.dat``. The absolute path of the file must be printed in the console output of the program. You can upload the file via HTTP POST to http://sapsailing.com/windestimation/api/windestimation_data (see ``com.sap.sailing.windestimation.jaxrs.api.WindEstimationDataResource``) to update the wind estimation of a server instance. If you changed the source files of ``DurationValueRange`` or ``DistanceValueRange``, then you will need to update ``com.sap.sailing.windestimation`` bundle of the server instance which is meant to receive the new wind estimation models. + + +## Model Evaluation + +This step is optional. However, it is recommended, to evaluate the performance of wind estimation with new trained models. + +1. Run ``com.sap.sailing.windestimation.evaluation.WindEstimatorManeuverNumberDependentEvaluationRunner`` to evaluate the wind estimation with the new trained models. The evaluation score will be stored as CSV in ``./maneuverNumberDependentEvaluation.csv``. \ No newline at end of file diff --git a/wiki/images/windestimation/confirmationDialogAfterDataCleansingDurationDimension.jpg b/wiki/images/windestimation/confirmationDialogAfterDataCleansingDurationDimension.jpg new file mode 100644 index 00000000000..8250cb2201f Binary files /dev/null and b/wiki/images/windestimation/confirmationDialogAfterDataCleansingDurationDimension.jpg differ diff --git a/wiki/images/windestimation/dialogRequestingDataCleansingForDurationDimension.jpg b/wiki/images/windestimation/dialogRequestingDataCleansingForDurationDimension.jpg new file mode 100644 index 00000000000..ffe69f2ac9d Binary files /dev/null and b/wiki/images/windestimation/dialogRequestingDataCleansingForDurationDimension.jpg differ