Merge remote-tracking branch 'server/windestimation' into windestimation

Change-Id: Id300413c84e80ae2c82a1dc52b538d834aebfe43
This commit is contained in:
Axel Uhl committed 2019-02-26 09:07:41 +01:00
commit 4c88e5f287
62 files changed
+1017 -304

No files matched your search

@@ -1,13 +1,13 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication">
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
<listEntry value="/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java"/>
<listEntry value="/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTrainingPart1.java"/>
</listAttribute>
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
<listEntry value="1"/>
</listAttribute>
<booleanAttribute key="org.eclipse.jdt.launching.ATTR_EXCLUDE_TEST_CODE" value="true"/>
<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="com.sap.sailing.windestimation.data.importer.ManeuverAndWindImporter"/>
<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="com.sap.sailing.windestimation.model.SimpleModelsTrainingPart1"/>
<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="com.sap.sailing.windestimation.lab"/>
<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Xmx16G"/>
<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Xms16G -Xmx16G"/>
</launchConfiguration>
@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication">
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
<listEntry value="/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTrainingPart2.java"/>
</listAttribute>
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
<listEntry value="1"/>
</listAttribute>
<booleanAttribute key="org.eclipse.jdt.launching.ATTR_EXCLUDE_TEST_CODE" value="true"/>
<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="com.sap.sailing.windestimation.model.SimpleModelsTrainingPart2"/>
<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="com.sap.sailing.windestimation.lab"/>
</launchConfiguration>
@@ -9,12 +9,18 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.SingleDimen
public class AggregatedDistanceBasedAbsTwdTransitionImporter {
public static void main(String[] args) throws UnknownHostException {
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager(
SingleDimensionType.DISTANCE);
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = createPersistenceManagerAndEnsureIndex();
SingleDimensionTwdTransitionAggregationImporter importer = new SingleDimensionTwdTransitionAggregationImporter(
singleDimensionBasedTwdTransitionPersistenceManager, AggregatedSingleDimensionType.DISTANCE_ABS,
new AnnealingNextThresholdCalculator(10, 1.5));
importer.runAggregation();
}
public static SingleDimensionBasedTwdTransitionPersistenceManager createPersistenceManagerAndEnsureIndex()
throws UnknownHostException {
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager(
SingleDimensionType.DISTANCE);
return singleDimensionBasedTwdTransitionPersistenceManager;
}
}
@@ -9,12 +9,18 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.SingleDimen
public class AggregatedDistanceBasedTwdTransitionImporter {
public static void main(String[] args) throws UnknownHostException {
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager(
SingleDimensionType.DISTANCE);
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = createPersistenceManagerAndEnsureIndex();
SingleDimensionTwdTransitionAggregationImporter importer = new SingleDimensionTwdTransitionAggregationImporter(
singleDimensionBasedTwdTransitionPersistenceManager, AggregatedSingleDimensionType.DISTANCE,
new AnnealingNextThresholdCalculator(10, 1.5));
importer.runAggregation();
}
public static SingleDimensionBasedTwdTransitionPersistenceManager createPersistenceManagerAndEnsureIndex()
throws UnknownHostException {
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager(
SingleDimensionType.DISTANCE);
return singleDimensionBasedTwdTransitionPersistenceManager;
}
}
@@ -9,12 +9,18 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.SingleDimen
public class AggregatedDurationBasedAbsTwdTransitionImporter {
public static void main(String[] args) throws UnknownHostException {
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager(
SingleDimensionType.DURATION);
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = createPersistenceManagerAndEnsureIndex();
SingleDimensionTwdTransitionAggregationImporter importer = new SingleDimensionTwdTransitionAggregationImporter(
singleDimensionBasedTwdTransitionPersistenceManager, AggregatedSingleDimensionType.DURATION_ABS,
DurationBasedTwdTransitionImporter.getThresholdCalculator());
importer.runAggregation();
}
public static SingleDimensionBasedTwdTransitionPersistenceManager createPersistenceManagerAndEnsureIndex()
throws UnknownHostException {
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager(
SingleDimensionType.DURATION);
return singleDimensionBasedTwdTransitionPersistenceManager;
}
}
@@ -9,12 +9,18 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.SingleDimen
public class AggregatedDurationBasedTwdTransitionImporter {
public static void main(String[] args) throws UnknownHostException {
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager(
SingleDimensionType.DURATION);
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = createPersistenceManagerAndEnsureIndex();
SingleDimensionTwdTransitionAggregationImporter importer = new SingleDimensionTwdTransitionAggregationImporter(
singleDimensionBasedTwdTransitionPersistenceManager, AggregatedSingleDimensionType.DURATION,
DurationBasedTwdTransitionImporter.getThresholdCalculator());
importer.runAggregation();
}
public static SingleDimensionBasedTwdTransitionPersistenceManager createPersistenceManagerAndEnsureIndex()
throws UnknownHostException {
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager(
SingleDimensionType.DURATION);
return singleDimensionBasedTwdTransitionPersistenceManager;
}
}
@@ -36,7 +36,8 @@ public class DistanceBasedTwdTransitionImporter {
.getIterator(); iterator.hasNext();) {
long percent = windSourceNumber * 100 / numberOfWindSources;
WindSourceWithFixes windSource = iterator.next();
LoggingUtil.logInfo("Processing " + windSourceNumber++ + "/" + numberOfWindSources + " (" + percent + "%)");
LoggingUtil.logInfo("Processing wind source " + windSourceNumber++ + "/" + numberOfWindSources + " ("
+ percent + "%) for distance dimension");
List<SingleDimensionBasedTwdTransition> entries = new ArrayList<>();
List<WindSourceWithFixes> otherWindSources = new ArrayList<>();
for (PersistedElementsIterator<WindSourceWithFixes> otherWindSourcesIterator = windSourcesPersistenceManager
@@ -109,16 +110,17 @@ public class DistanceBasedTwdTransitionImporter {
} while (currentFix != null && currentOtherFix != null);
}
if (entries.isEmpty()) {
LoggingUtil.logInfo("No TWD transitions to import");
LoggingUtil.logInfo("No distance based TWD transitions to import");
} else {
distanceBasedTwdTransitionPersistenceManager.add(entries);
int totalEntries = entries.size();
totalValuesCount += totalEntries;
LoggingUtil.logInfo(totalEntries + " TWD transitions imported, " + totalValuesCount + " in total");
LoggingUtil.logInfo(
totalEntries + " distance based TWD transitions imported, " + totalValuesCount + " in total");
}
}
LoggingUtil.logInfo("###################\r\nDistance based TWD transitions Import finished");
LoggingUtil.logInfo("Totally " + totalValuesCount + " TWD transitions imported");
LoggingUtil.logInfo("Totally " + totalValuesCount + " distance based TWD transitions imported");
}
}
@@ -32,7 +32,8 @@ public class DurationBasedTwdTransitionImporter {
.getIterator(); iterator.hasNext();) {
long percent = windSourceNumber * 100 / numberOfWindSources;
WindSourceWithFixes windSource = iterator.next();
LoggingUtil.logInfo("Processing " + windSourceNumber++ + "/" + numberOfWindSources + " (" + percent + "%)");
LoggingUtil.logInfo("Processing wind source " + windSourceNumber++ + "/" + numberOfWindSources + " ("
+ percent + "%) for duration dimension");
TimePoint timePointOfLastConsideredWindFix = null;
List<SingleDimensionBasedTwdTransition> entries = new ArrayList<>();
int windFixIndex = 0;
@@ -58,17 +59,17 @@ public class DurationBasedTwdTransitionImporter {
}
}
if (entries.isEmpty()) {
LoggingUtil.logInfo("No TWD transitions to import");
LoggingUtil.logInfo("No duration based TWD transitions to import");
} else {
durationBasedTwdTransitionPersistenceManager.add(entries);
int totalEntries = entries.size();
totalValuesCount += totalEntries;
LoggingUtil
.logInfo(totalEntries + " TWD transition entries imported, " + totalValuesCount + " in total");
LoggingUtil.logInfo(totalEntries + " duration based TWD transition entries imported, "
+ totalValuesCount + " in total");
}
}
LoggingUtil.logInfo("###################\r\nDuration based TWD transitions Import finished");
LoggingUtil.logInfo("Totally " + totalValuesCount + " TWD transitions imported");
LoggingUtil.logInfo("Totally " + totalValuesCount + " duration based TWD transitions imported");
}
public static NextThresholdCalculator getThresholdCalculator() {
@@ -13,6 +13,10 @@ import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.http.Header;
import org.apache.http.HttpResponse;
@@ -40,11 +44,11 @@ import com.sap.sailing.windestimation.data.persistence.maneuver.RaceWithManeuver
import com.sap.sailing.windestimation.data.persistence.maneuver.RaceWithManeuverForEstimationPersistenceManager;
import com.sap.sailing.windestimation.data.persistence.twdtransition.RaceWithWindSourcesPersistenceManager;
import com.sap.sailing.windestimation.data.serialization.CompetitorTrackWithEstimationDataJsonDeserializer;
import com.sap.sailing.windestimation.data.serialization.ManeuverForDataAnalysisJsonSerializer;
import com.sap.sailing.windestimation.data.serialization.LabeledManeuverForEstimationJsonSerializer;
import com.sap.sailing.windestimation.data.serialization.ManeuverForDataAnalysisJsonSerializer;
import com.sap.sailing.windestimation.data.transformer.CompetitorTrackTransformer;
import com.sap.sailing.windestimation.data.transformer.CompleteManeuverCurveWithEstimationDataToManeuverForDataAnalysisTransformer;
import com.sap.sailing.windestimation.data.transformer.CompleteManeuverCurveWithEstimationDataToLabelledManeuverForEstimationTransformer;
import com.sap.sailing.windestimation.data.transformer.CompleteManeuverCurveWithEstimationDataToManeuverForDataAnalysisTransformer;
import com.sap.sailing.windestimation.util.LoggingUtil;
/**
@@ -54,6 +58,7 @@ import com.sap.sailing.windestimation.util.LoggingUtil;
*/
public class ManeuverAndWindImporter {
private static final int CONNECTION_TIMEOUT_MILLIS = 60000 * 5;
public static final String REST_API_BASE_URL = "https://www.sapsailing.com/sailingserver/api/v1";
public static final String REST_API_REGATTAS_PATH = "/regattas";
public static final String REST_API_RACES_PATH = "/races";
@@ -70,6 +75,8 @@ public class ManeuverAndWindImporter {
private final ManeuverForDataAnalysisJsonSerializer maneuverForDataAnalysisJsonSerializer;
private final LabeledManeuverForEstimationJsonSerializer maneuverForEstimationJsonSerializer;
private boolean skipRace;
private static final int NUMBER_OF_THREADS = 10; // high number due to HTTP requests
private final ThreadPoolExecutor executorService;
public ManeuverAndWindImporter() throws UnknownHostException {
this.completeManeuverCurvePersistanceManager = new RaceWithCompleteManeuverCurvePersistenceManager();
@@ -80,12 +87,28 @@ public class ManeuverAndWindImporter {
this.maneuverForEstimationTransformer = new CompleteManeuverCurveWithEstimationDataToLabelledManeuverForEstimationTransformer();
this.maneuverForDataAnalysisJsonSerializer = new ManeuverForDataAnalysisJsonSerializer();
this.maneuverForEstimationJsonSerializer = new LabeledManeuverForEstimationJsonSerializer();
this.executorService = new ThreadPoolExecutor(NUMBER_OF_THREADS, NUMBER_OF_THREADS, 0L, TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<>(NUMBER_OF_THREADS));
this.executorService.setRejectedExecutionHandler(new RejectedExecutionHandler() {
public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
// this will block if the queue is full
try {
executor.getQueue().put(r);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
});
}
public HttpClient createNewHttpClient() {
HttpParams httpParams = new BasicHttpParams();
HttpConnectionParams.setConnectionTimeout(httpParams, 30000);
HttpConnectionParams.setConnectionTimeout(httpParams, CONNECTION_TIMEOUT_MILLIS);
HttpClient client = new SystemDefaultHttpClient(httpParams);
client.getParams().setParameter("http.socket.timeout", CONNECTION_TIMEOUT_MILLIS);
client.getParams().setParameter("http.connection.timeout", CONNECTION_TIMEOUT_MILLIS);
client.getParams().setParameter("http.connection-manager.timeout", new Long(CONNECTION_TIMEOUT_MILLIS));
client.getParams().setParameter("http.protocol.head-body-timeout", CONNECTION_TIMEOUT_MILLIS);
return client;
}
@@ -94,8 +117,8 @@ public class ManeuverAndWindImporter {
importer.importAllRegattas();
}
public void importAllRegattas()
throws IllegalStateException, ClientProtocolException, IOException, ParseException, URISyntaxException {
public void importAllRegattas() throws IllegalStateException, ClientProtocolException, IOException, ParseException,
URISyntaxException, InterruptedException {
skipRace = startFromRegattaName != null;
LoggingUtil.logInfo("Importer for CompleteManeuverCurveWithEstimationData just started");
LoggingUtil.logInfo("Dropping old database");
@@ -118,9 +141,17 @@ public class ManeuverAndWindImporter {
+ Math.round(100.0 * i / numberOfRegattas) + "%): \"" + regattaName + "\"");
importRegatta(regattaName, importStatistics);
}
LoggingUtil.logInfo("Import finished");
importStatistics.regattasCount = regattasJson.size();
logImportStatistics(importStatistics);
executorService.shutdown();
boolean success = executorService.awaitTermination(2, TimeUnit.HOURS);
if (success) {
LoggingUtil.logInfo("Import finished");
synchronized (importStatistics) {
importStatistics.regattasCount = regattasJson.size();
logImportStatistics(importStatistics);
}
} else {
new InterruptedException("Thread-pool was terminated after two hours waiting time");
}
}
private void logImportStatistics(ImportStatistics importStatistics) {
@@ -139,14 +170,15 @@ public class ManeuverAndWindImporter {
private void importRegatta(String regattaName, ImportStatistics importStatistics)
throws IllegalStateException, ClientProtocolException, IOException, ParseException, URISyntaxException {
String encodedRegattaName = encodeUrlPathPart(regattaName);
// TODO The encoding of some regatta names with special characters does not work well
HttpGet getRegatta = new HttpGet(REST_API_BASE_URL + REST_API_REGATTAS_PATH + "/" + encodedRegattaName);
JSONObject regattaJson = null;
try {
regattaJson = (JSONObject) getHttpResponseAsJson(regattaName, null, getRegatta);
} catch (Exception e) {
importStatistics.ignoredRegattas += 1;
LoggingUtil.logInfo("Error while processing regatta: " + regattaName);
synchronized (importStatistics) {
importStatistics.ignoredRegattas++;
}
LoggingUtil.logInfo("Error while processing regatta: " + regattaName + "\r\n" + getRegatta);
return;
}
for (Object seriesJson : (JSONArray) regattaJson.get("series")) {
@@ -161,17 +193,29 @@ public class ManeuverAndWindImporter {
if ((boolean) race.get("isTracked") && !(boolean) race.get("isLive")
&& (boolean) race.get("hasGpsData") && (boolean) race.get("hasWindData")) {
String trackedRaceName = (String) race.get("trackedRaceName");
LoggingUtil.logInfo("Processing race nr. " + ++i + ": \"" + trackedRaceName + "\"");
try {
importRace(regattaName, trackedRaceName, importStatistics);
} catch (Exception e) {
importStatistics.ingoredRaces += 1;
LoggingUtil
.logInfo("Error while processing race nr. " + i + ": \"" + trackedRaceName + "\"");
}
i++;
final int raceNumber = i;
LoggingUtil.logInfo("Processing race nr. " + raceNumber + ": \"" + trackedRaceName + "\"");
executorService.execute(() -> {
try {
importRace(regattaName, trackedRaceName, importStatistics);
} catch (Exception e) {
synchronized (importStatistics) {
importStatistics.ingoredRaces += 1;
}
String extraLog = "";
if (e instanceof HttpClientException) {
extraLog = "\r\n" + ((HttpClientException) e).getRequest();
}
LoggingUtil.logInfo("Error while processing race nr. " + raceNumber + ": \""
+ trackedRaceName + "\"" + extraLog);
}
});
}
}
importStatistics.racesCount += racesJson.size();
synchronized (importStatistics) {
importStatistics.racesCount += racesJson.size();
}
}
}
}
@@ -215,11 +259,14 @@ public class ManeuverAndWindImporter {
JSONArray windFixesJson = (JSONArray) ((JSONObject) windSourceObj).get(RaceWindJsonSerializer.FIXES);
windFixesCount += windFixesJson.size();
}
LoggingUtil.logInfo(
"Imported " + windFixesCount + " wind fixes from " + windSourcesJson.size() + " wind sources");
importStatistics.racesWithHighQualityWindData++;
LoggingUtil.logInfo("Imported " + windFixesCount + " wind fixes from " + windSourcesJson.size()
+ " wind sources in race \"" + trackedRaceName + "\" of regatta \"" + regattaName + "\"");
synchronized (importStatistics) {
importStatistics.racesWithHighQualityWindData++;
}
} else {
LoggingUtil.logInfo("No high quality wind fixes contained");
LoggingUtil.logInfo("No high quality wind fixes contained in race \"" + trackedRaceName + "\" of regatta \""
+ regattaName + "\"");
}
}
@@ -272,10 +319,12 @@ public class ManeuverAndWindImporter {
} catch (Exception e) {
e.printStackTrace();
}
LoggingUtil.logInfo(
"Imported " + competitorTracks.size() + " competitor tracks with " + maneuversCount + " maneuvers");
importStatistics.competitorTracksCount += competitorTracks.size();
importStatistics.maneuversCount += maneuversCount;
LoggingUtil.logInfo("Imported " + competitorTracks.size() + " competitor tracks with " + maneuversCount
+ " maneuvers in race \"" + trackedRaceName + "\" of regatta \"" + regattaName + "\"");
synchronized (importStatistics) {
importStatistics.competitorTracksCount += competitorTracks.size();
importStatistics.maneuversCount += maneuversCount;
}
}
private JSONObject getHttpResponseAsJson(String trackedRegattaName, String trackedRaceName,
@@ -285,7 +334,8 @@ public class ManeuverAndWindImporter {
for (int i = 1; i <= 10; i++) {
try {
httpResponse = createNewHttpClient().execute(getEstimationData);
break;
JSONObject resultJson = (JSONObject) getJsonFromResponse(httpResponse);
return resultJson;
} catch (Exception e) {
Thread.sleep(10000);
lastException = e;
@@ -298,17 +348,7 @@ public class ManeuverAndWindImporter {
}
}
}
if (httpResponse == null) {
throw lastException;
}
JSONObject resultJson;
try {
resultJson = (JSONObject) getJsonFromResponse(httpResponse);
} catch (Exception e) {
System.out.println(getEstimationData);
throw e;
}
return resultJson;
throw new HttpClientException(getEstimationData.toString(), lastException);
}
private <ToType> void addTransformedElementsToCompetitorTrackJson(List<JSONObject> competitorTracks,
@@ -331,14 +371,13 @@ public class ManeuverAndWindImporter {
}
}
// FIXME duplicated code taken from ConnectivityUtils
public static Object getJsonFromResponse(HttpResponse response)
throws IllegalStateException, IOException, ParseException {
JSONParser jsonParser = new JSONParser();
final Header contentEncoding = response.getEntity().getContentEncoding();
final Reader reader;
if (contentEncoding == null) {
reader = new InputStreamReader(response.getEntity().getContent());
reader = new InputStreamReader(response.getEntity().getContent(), "UTF-8");
} else {
reader = new InputStreamReader(response.getEntity().getContent(), contentEncoding.getValue());
}
@@ -358,4 +397,20 @@ public class ManeuverAndWindImporter {
private int racesWithHighQualityWindData = 0;
}
private static class HttpClientException extends Exception {
private static final long serialVersionUID = 4948532287832868768L;
private final String request;
public HttpClientException(String request, Exception e) {
super(e);
this.request = request;
}
public String getRequest() {
return request;
}
}
}
@@ -35,7 +35,7 @@ public class SingleDimensionTwdTransitionAggregationImporter {
public void runAggregation() throws UnknownHostException {
long totalValuesCount = 0;
List<AggregatedSingleDimensionBasedTwdTransition> aggregates = new ArrayList<>();
LoggingUtil.logInfo("Aggregating persisted entries");
LoggingUtil.logInfo("Aggregating persisted entries for dimension " + dimensionType);
TDoubleList entries = new TDoubleArrayList();
double currentBucketThreshold = thresholdCalculator.getInitialThresholdValue();
double nextBucketThreshold = thresholdCalculator.getNextThresholdValue(currentBucketThreshold);
@@ -61,7 +61,8 @@ public class SingleDimensionTwdTransitionAggregationImporter {
}
entries.add(twdChange);
if (entries.size() % 10000 == 0) {
LoggingUtil.logInfo((entries.size() + totalValuesCount) + " Entries aggregated");
LoggingUtil.logInfo(
(entries.size() + totalValuesCount) + " Entries aggregated for dimension " + dimensionType);
}
}
if (entries.size() >= MIN_NUMBER_OF_VALUES_PER_BUCKET) {
@@ -69,14 +70,16 @@ public class SingleDimensionTwdTransitionAggregationImporter {
aggregates.add(aggregate);
totalValuesCount += entries.size();
}
LoggingUtil.logInfo("Persisting " + aggregates.size() + " aggregates");
LoggingUtil.logInfo("Persisting " + aggregates.size() + " aggregates for dimension " + dimensionType);
AggregatedSingleDimensionBasedTwdTransitionPersistenceManager aggregatedDurationBasedTwdTransitionPersistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager(
dimensionType);
aggregatedDurationBasedTwdTransitionPersistenceManager.dropCollection();
aggregatedDurationBasedTwdTransitionPersistenceManager.add(aggregates);
LoggingUtil.logInfo("###################\r\n" + dimensionType + " based TWD transitions Import finished");
LoggingUtil.logInfo("Totally " + totalValuesCount + " TWD transitions imported");
LoggingUtil.logInfo("Totally " + aggregates.size() + " TWD transition aggregates imported");
LoggingUtil.logInfo(
"Totally " + totalValuesCount + " " + dimensionType + " based TWD transitions have been processed");
LoggingUtil.logInfo(
"Totally " + aggregates.size() + " TWD transition aggregates imported for dimension " + dimensionType);
}
private double getFinalBucketThreshold(double currentBucketThreshold, double nextBucketThreshold) {
@@ -17,20 +17,16 @@ import com.sap.sailing.server.gateway.deserialization.JsonDeserializer;
import com.sap.sailing.server.gateway.serialization.impl.RaceWindJsonSerializer;
import com.sap.sailing.windestimation.data.RaceWithWindSources;
import com.sap.sailing.windestimation.data.persistence.maneuver.AbstractPersistenceManager;
import com.sap.sailing.windestimation.data.persistence.maneuver.PersistedElementsIterator;
import com.sap.sailing.windestimation.data.serialization.RaceWithEstimationDataDeserializer;
import com.sap.sailing.windestimation.data.serialization.RaceWithWindSourcesDeserializer;
import com.sap.sse.common.TimePoint;
public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceManager<RaceWithWindSources> {
private static final String COLLECTION_NAME = "racesWithWind";
private static final String WIND_SOURCES_COLLECTION_NAME = "windSources";
static final String COLLECTION_NAME = "racesWithWind";
private final WindSourcesPersistenceManager windSourcesPersistenceManager;
public RaceWithWindSourcesPersistenceManager() throws UnknownHostException {
Document indexes = new Document(RaceWindJsonSerializer.START_TIME_POINT, 1);
indexes.put(RaceWindJsonSerializer.END_TIME_POINT, 1);
getCollection().createIndex(indexes);
windSourcesPersistenceManager = new WindSourcesPersistenceManager();
}
@Override
@@ -41,7 +37,7 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa
@Override
public void dropCollection() {
super.dropCollection();
getDb().getCollection(getWindSourcesCollectionName()).drop();
windSourcesPersistenceManager.dropCollection();
}
@Override
@@ -52,7 +48,7 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa
JSONArray windSourceIdsJson = (JSONArray) raceJson
.get(RaceWithEstimationDataDeserializer.COMPETITOR_TRACKS);
JSONArray windSourcesJson = new JSONArray();
MongoCollection<Document> windSourcesCollection = getDb().getCollection(getWindSourcesCollectionName());
MongoCollection<Document> windSourcesCollection = windSourcesPersistenceManager.getCollection();
for (Object idObject : windSourceIdsJson) {
Document dbWindSource = windSourcesCollection
.find(new Document(FIELD_DB_ID, new ObjectId((String) idObject))).first();
@@ -69,10 +65,6 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa
};
}
public static String getWindSourcesCollectionName() {
return COLLECTION_NAME + "." + WIND_SOURCES_COLLECTION_NAME;
}
public void add(String regattaName, String raceName, JSONObject raceWithWindSourcesJson) {
JSONArray windSourcesJson = (JSONArray) raceWithWindSourcesJson.get(RaceWindJsonSerializer.WIND_SOURCES);
@@ -81,7 +73,7 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa
Document entry = parseJsonString(windSourceObj.toString());
dbWindSources.add(entry);
}
MongoCollection<Document> windSourcesCollection = getDb().getCollection(getWindSourcesCollectionName());
MongoCollection<Document> windSourcesCollection = windSourcesPersistenceManager.getCollection();
windSourcesCollection.insertMany(dbWindSources);
BasicDBList dbWindSourceIds = new BasicDBList();
for (Document dbWindSource : dbWindSources) {
@@ -96,13 +88,4 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa
races.insertOne(dbObject);
}
public PersistedElementsIterator<RaceWithWindSources> getIteratorForEntriesIntersectingPeriod(
TimePoint startTimePoint, TimePoint endTimePoint, double toleranceInSeconds) {
double maxStartTime = endTimePoint.asMillis() + toleranceInSeconds * 1000;
double maxEndTime = startTimePoint.asMillis() - toleranceInSeconds * 1000;
String query = "{$and: [{'" + RaceWindJsonSerializer.START_TIME_POINT + "': {$lte: " + maxStartTime + "}}, {'"
+ RaceWindJsonSerializer.END_TIME_POINT + "': {$gte: " + maxEndTime + "}}]}";
return getIterator(query);
}
}
@@ -21,15 +21,18 @@ import com.sap.sse.common.TimePoint;
public class WindSourcesPersistenceManager extends AbstractPersistenceManager<WindSourceWithFixes> {
private static final String WIND_SOURCES_COLLECTION_NAME = "windSources";
public WindSourcesPersistenceManager() throws UnknownHostException {
BasicDBObject indexes = new BasicDBObject(RaceWindJsonSerializer.START_TIME_POINT, 1);
indexes.put(RaceWindJsonSerializer.END_TIME_POINT, 1);
BasicDBObject indexes = new BasicDBObject("_id", 1);
indexes.put(RaceWindJsonSerializer.START_TIME_POINT, 1);
indexes.put(RaceWindJsonSerializer.END_TIME_POINT, -1);
getCollection().createIndex(indexes);
}
@Override
public String getCollectionName() {
return RaceWithWindSourcesPersistenceManager.getWindSourcesCollectionName();
return RaceWithWindSourcesPersistenceManager.COLLECTION_NAME + "." + WIND_SOURCES_COLLECTION_NAME;
}
@Override
@@ -7,11 +7,17 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS
public class AggregatedDistanceDimensionPlot {
private static SingleDimensionAggregatesPlottingFrame plot;
public static void main(String[] args) throws UnknownHostException {
AggregatedSingleDimensionBasedTwdTransitionPersistenceManager persistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager(
AggregatedSingleDimensionType.DISTANCE);
SingleDimensionAggregatesPlottingFrame plot = new SingleDimensionAggregatesPlottingFrame(persistenceManager);
plot = new SingleDimensionAggregatesPlottingFrame(persistenceManager);
plot.renderChart();
}
public static void awaitWindowClosed() throws InterruptedException {
plot.awaitWindowClosed();
}
}
@@ -7,11 +7,17 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS
public class AggregatedDurationDimensionPlot {
private static SingleDimensionAggregatesPlottingFrame plot;
public static void main(String[] args) throws UnknownHostException {
AggregatedSingleDimensionBasedTwdTransitionPersistenceManager persistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager(
AggregatedSingleDimensionType.DURATION);
SingleDimensionAggregatesPlottingFrame plot = new SingleDimensionAggregatesPlottingFrame(persistenceManager);
plot = new SingleDimensionAggregatesPlottingFrame(persistenceManager);
plot.renderChart();
}
public static void awaitWindowClosed() throws InterruptedException {
plot.awaitWindowClosed();
}
}
@@ -3,9 +3,13 @@ package com.sap.sailing.windestimation.datavisualization;
import java.awt.BasicStroke;
import java.awt.Color;
import java.awt.Shape;
import java.awt.event.WindowAdapter;
import java.awt.event.WindowEvent;
import java.awt.geom.Ellipse2D;
import java.util.concurrent.CountDownLatch;
import javax.swing.BoxLayout;
import javax.swing.JButton;
import javax.swing.JFrame;
import javax.swing.JPanel;
import javax.swing.SwingUtilities;
@@ -40,27 +44,54 @@ public class SingleDimensionAggregatesPlottingFrame extends JFrame {
private XYSeries valuesSeries;
private XYSeries stdSeries;
private XYSeries zeroMeanStdSeries;
private final CountDownLatch windowClosedLatch = new CountDownLatch(1);
public SingleDimensionAggregatesPlottingFrame(
AggregatedSingleDimensionBasedTwdTransitionPersistenceManager persistenceManager) {
super(persistenceManager.getCollectionName());
this.persistenceManager = persistenceManager;
addWindowListener(new WindowAdapter() {
@Override
public void windowClosed(WindowEvent event) {
windowClosedLatch.countDown();
}
@Override
public void windowClosing(WindowEvent e) {
setLayout(null);
setVisible(false);
dispose();
}
});
initializeDataWithCharts();
}
private void initializeDataWithCharts() {
createSeries();
fillSeries();
DefaultTableXYDataset mainDataset = createMainDataset();
DefaultTableXYDataset histoDataset = createHistoDataset();
mainChart = createMainChart(mainDataset);
histoChart = createHistoChart(histoDataset);
// Create Panel
JPanel panel = new JPanel();
panel.setLayout(new BoxLayout(panel, BoxLayout.Y_AXIS));
ChartPanel mainChartPanel = new ChartPanel(mainChart);
ChartPanel histoChartPanel = new ChartPanel(histoChart);
JButton refreshButton = new JButton("Refresh charts");
refreshButton.addActionListener(event -> refreshCharts());
panel.add(refreshButton);
panel.add(mainChartPanel);
panel.add(histoChartPanel);
setContentPane(panel);
}
public void refreshCharts() {
SwingUtilities.invokeLater(() -> {
initializeDataWithCharts();
revalidate();
});
}
private void createSeries() {
meanSeries = new XYSeries("Mean", true, false);
medianSeries = new XYSeries("Median", true, false);
@@ -191,4 +222,9 @@ public class SingleDimensionAggregatesPlottingFrame extends JFrame {
});
}
public void awaitWindowClosed() throws InterruptedException {
setDefaultCloseOperation(WindowConstants.DO_NOTHING_ON_CLOSE);
windowClosedLatch.await();
}
}
@@ -12,8 +12,8 @@ import com.sap.sailing.windestimation.data.persistence.maneuver.PersistedElement
import com.sap.sailing.windestimation.data.persistence.maneuver.RaceWithCompleteManeuverCurvePersistenceManager;
import com.sap.sailing.windestimation.data.persistence.polars.PolarDataServiceAccessUtil;
import com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverFeatures;
import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl;
import com.sap.sailing.windestimation.model.store.ModelStore;
import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl;
import com.sap.sailing.windestimation.util.LoggingUtil;
public class WindEstimatorManeuverNumberDependentEvaluationRunner {
@@ -38,18 +38,15 @@ public class WindEstimatorManeuverNumberDependentEvaluationRunner {
LoggingUtil.logInfo("Loading polar data");
PolarDataService polarService = PolarDataServiceAccessUtil.getPersistedPolarService();
LoggingUtil.logInfo("Wind estimator evaluation started...");
ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models");
// ModelStore modelStore = new MongoDbModelStoreImpl(persistenceManager.getDb());
ModelStore modelStore = new MongoDbModelStoreImpl(persistenceManager.getDb());
WindEstimatorFactories estimatorFactories = new WindEstimatorFactories(polarService,
new ManeuverFeatures(ENABLE_POLARS, ENABLE_SCALED_SPEED, ENABLE_MARKS_INFORMATION), modelStore);
double[] avgErrorDegreesPerManeuverCount = new double[MAX_MANEUVERS];
double[] avgConfidencePerManeuverCount = new double[MAX_MANEUVERS];
double[] avgConfidenceOfCorrectEstimationsPerManeuverCount = new double[MAX_MANEUVERS];
double[] avgConfidenceOfIncorrectEstimationsPerManeuverCount = new double[MAX_MANEUVERS];
double[] accuracyPerManeuverCount = new double[MAX_MANEUVERS];
double[] emptyEstimationsPercentagePerManeuverCount = new double[MAX_MANEUVERS];
for (int fixedNumberOfManeuvers = 1; fixedNumberOfManeuvers <= MAX_MANEUVERS; fixedNumberOfManeuvers++) {
LoggingUtil.logInfo("Running evaluation with " + fixedNumberOfManeuvers + " maneuvers");
WindEstimationEvaluator<CompleteManeuverCurveWithEstimationData> evaluator = new WindEstimationEvaluatorImpl<>(
@@ -113,6 +110,7 @@ public class WindEstimatorManeuverNumberDependentEvaluationRunner {
out.write(line);
}
}
LoggingUtil.logInfo("CSV with evaluation results have been stored in: " + csvFile.getAbsolutePath());
}
}
@@ -0,0 +1,49 @@
package com.sap.sailing.windestimation.model;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.sap.sailing.windestimation.integration.ExportedModels;
import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl;
import com.sap.sailing.windestimation.model.store.ModelDomainType;
import com.sap.sailing.windestimation.model.store.ModelStore;
import com.sap.sailing.windestimation.util.LoggingUtil;
public class ExportedModelsGenerator {
private static final File file = new File("windEstimationModels.dat");
public static void main(String[] args) throws FileNotFoundException, IOException {
ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models");
// RegularManeuversForEstimationPersistenceManager persistenceManager = new
// RegularManeuversForEstimationPersistenceManager();
// ModelStore modelStore = new MongoDbModelStoreImpl(persistenceManager.getDb());
LoggingUtil.logInfo("Generation a single file with all trained models started");
ExportedModels exportedModels = new ExportedModels();
List<ModelDomainType> nonEmptyDomainTypes = new ArrayList<>();
for (ModelDomainType domainType : ModelDomainType.values()) {
LoggingUtil.logInfo("Started serialization of models belonging to domain type " + domainType);
Map<String, byte[]> exportedModelsForDomainType = modelStore.exportAllPersistedModels(domainType);
if (!exportedModelsForDomainType.isEmpty()) {
exportedModels.addSerializedModelsForDomainType(domainType, exportedModelsForDomainType);
LoggingUtil.logInfo("Finished serialization of " + exportedModelsForDomainType.size()
+ " models belonging to domain type " + domainType);
nonEmptyDomainTypes.add(domainType);
}
}
LoggingUtil.logInfo("Serializing the exported models and writing it into a single file");
try (ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(file))) {
oos.writeObject(exportedModels);
}
LoggingUtil.logInfo("Models have been generated");
LoggingUtil.logInfo("The following domain types were considered: " + nonEmptyDomainTypes);
LoggingUtil.logInfo("File with all exported models was persisted in: " + file.getAbsolutePath());
}
}
@@ -0,0 +1,153 @@
package com.sap.sailing.windestimation.model;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import javax.swing.JOptionPane;
import com.sap.sailing.windestimation.data.importer.AggregatedDistanceBasedTwdTransitionImporter;
import com.sap.sailing.windestimation.data.importer.AggregatedDurationBasedTwdTransitionImporter;
import com.sap.sailing.windestimation.data.importer.DistanceBasedTwdTransitionImporter;
import com.sap.sailing.windestimation.data.importer.DurationBasedTwdTransitionImporter;
import com.sap.sailing.windestimation.data.importer.ManeuverAndWindImporter;
import com.sap.sailing.windestimation.data.importer.PolarDataImporter;
import com.sap.sailing.windestimation.data.persistence.maneuver.ManeuverForEstimationPersistenceManager;
import com.sap.sailing.windestimation.data.persistence.maneuver.RegularManeuversForEstimationPersistenceManager;
import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedSingleDimensionBasedTwdTransitionPersistenceManager.AggregatedSingleDimensionType;
import com.sap.sailing.windestimation.datavisualization.AggregatedDistanceDimensionPlot;
import com.sap.sailing.windestimation.datavisualization.AggregatedDurationDimensionPlot;
import com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverClassifierTrainer;
import com.sap.sailing.windestimation.model.classifier.maneuver.PersistedManeuverClassifiersScorePrinter;
import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext;
import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext.DistanceValueRange;
import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext;
import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext.DurationValueRange;
import com.sap.sailing.windestimation.util.LoggingUtil;
/**
*
* @author Vladislav Chumak (D069712)
*
*/
public class SimpleModelsTrainingPart1 {
private static final int NUMBER_OF_THREADS = 3;
private static ExecutorService executorService;
public static void main(String[] args) throws Exception {
new ManeuverForEstimationPersistenceManager().dropCollection();
new RegularManeuversForEstimationPersistenceManager().dropCollection();
executeInThreadPool(() -> PolarDataImporter.main(args));
executeInThreadPool(() -> ManeuverAndWindImporter.main(args));
awaitThreadPoolCompletion();
executeInThreadPool(() -> {
ManeuverClassifierTrainer.main(args);
Thread.sleep(1000);
PersistedManeuverClassifiersScorePrinter.main(args);
});
executeInThreadPool(() -> {
DistanceBasedTwdTransitionImporter.main(args);
});
executeInThreadPool(() -> {
DurationBasedTwdTransitionImporter.main(args);
});
awaitThreadPoolCompletion();
AggregatedDurationBasedTwdTransitionImporter.createPersistenceManagerAndEnsureIndex();
AggregatedDistanceBasedTwdTransitionImporter.createPersistenceManagerAndEnsureIndex();
executeInThreadPool(() -> {
AggregatedDurationBasedTwdTransitionImporter.main(args);
});
executeInThreadPool(() -> {
AggregatedDistanceBasedTwdTransitionImporter.main(args);
});
awaitThreadPoolCompletion();
do {
AggregatedDurationDimensionPlot.main(args);
showInfoAboutDataCleaning(AggregatedSingleDimensionType.DURATION);
AggregatedDurationDimensionPlot.awaitWindowClosed();
} while (JOptionPane.YES_OPTION != askDataCleaningFinished(AggregatedSingleDimensionType.DURATION));
showInfoAboutIntervalAdjustments(DurationBasedTwdTransitionRegressorModelContext.class,
DurationValueRange.class);
do {
AggregatedDistanceDimensionPlot.main(args);
showInfoAboutDataCleaning(AggregatedSingleDimensionType.DISTANCE);
AggregatedDistanceDimensionPlot.awaitWindowClosed();
} while (JOptionPane.YES_OPTION != askDataCleaningFinished(AggregatedSingleDimensionType.DISTANCE));
showInfoAboutIntervalAdjustments(DistanceBasedTwdTransitionRegressorModelContext.class,
DistanceValueRange.class);
showInfoAboutRunPart2();
}
private static int askDataCleaningFinished(AggregatedSingleDimensionType dimension) {
Object[] options = { "Continue with model training", "Restart Graphical Tool" };
int res = JOptionPane.showOptionDialog(null,
"Have you finished the data cleansing for the " + dimension + " dimension?",
"Continue with model training?", JOptionPane.DEFAULT_OPTION, JOptionPane.INFORMATION_MESSAGE, null,
options, options[0]);
switch (res) {
case 0:
return JOptionPane.YES_OPTION;
case 1:
return JOptionPane.NO_OPTION;
}
return JOptionPane.CLOSED_OPTION;
}
private static void showInfoAboutRunPart2() {
JOptionPane.showMessageDialog(null, "Now, run the class \"" + SimpleModelsTrainingPart2.class.getName()
+ "\".\nThis will complete the training process.");
}
private static void showInfoAboutDataCleaning(AggregatedSingleDimensionType dimension) {
JOptionPane.showMessageDialog(null, "Now, clean the data for the " + dimension
+ " dimension.\nRemove instances from MongoDB collection \"" + dimension.getCollectioName()
+ "\"\nwhich do not make sense and cause implausible zig zag sections\nwithin \"Zero mean sigma\" curve.\n\nClose the graphical tool, when you are done to resume the model training.");
}
private static void showInfoAboutIntervalAdjustments(Class<?> classToAdjust, Class<?> valueRangeEnum) {
JOptionPane.showMessageDialog(null, "Now, open the source code of the class \"" + classToAdjust.getName()
+ "\".\nScroll down to the definition of the inner enum \"" + valueRangeEnum.getSimpleName()
+ "\",\nread its JavaDoc and adjust its interval definitions so that\neach interval can be learned by the adjusted regressor model configuration\nwith minimal error.\n\nPress OK after you are done.");
}
private static void awaitThreadPoolCompletion() throws InterruptedException {
executorService.shutdown();
boolean success = executorService.awaitTermination(24, TimeUnit.HOURS);
if (!success) {
new InterruptedException("Thread-pool was terminated after 24 hours waiting time");
}
Thread.sleep(1000L);
}
private static void executeInThreadPool(RunnableWithExceptionsCatch runnable) {
if (executorService == null || executorService.isShutdown()) {
createNewThreadPool();
}
executorService.execute(() -> {
try {
runnable.run();
} catch (Throwable t) {
t.printStackTrace();
System.exit(1);
LoggingUtil.logInfo("FAILURE: Caught unexpected exception. Model training aborted");
}
});
}
private static void createNewThreadPool() {
executorService = new ThreadPoolExecutor(NUMBER_OF_THREADS, NUMBER_OF_THREADS, 0, TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<>(NUMBER_OF_THREADS), new ThreadPoolExecutor.CallerRunsPolicy());
}
/**
*
* @author Vladislav Chumak (D069712)
*
*/
private static interface RunnableWithExceptionsCatch {
void run() throws Exception;
}
}
@@ -0,0 +1,22 @@
package com.sap.sailing.windestimation.model;
import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionStdRegressorTrainer;
import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionStdRegressorTrainer;
import com.sap.sailing.windestimation.util.LoggingUtil;
/**
*
* @author Vladislav Chumak (D069712)
*
*/
public class SimpleModelsTrainingPart2 {
public static void main(String[] args) throws Exception {
DurationBasedTwdTransitionStdRegressorTrainer.main(args);
DistanceBasedTwdTransitionStdRegressorTrainer.main(args);
Thread.sleep(1000);
ExportedModelsGenerator.main(args);
LoggingUtil.logInfo("Model training finished. You can upload the generated file to a server instance.");
}
}
@@ -19,9 +19,9 @@ import com.sap.sailing.windestimation.data.persistence.maneuver.TransformedManeu
import com.sap.sailing.windestimation.data.persistence.polars.PolarDataServiceAccessUtil;
import com.sap.sailing.windestimation.model.classifier.LabelExtraction;
import com.sap.sailing.windestimation.model.classifier.TrainableClassificationModel;
import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl;
import com.sap.sailing.windestimation.model.store.ModelDomainType;
import com.sap.sailing.windestimation.model.store.ModelStore;
import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl;
import com.sap.sailing.windestimation.util.LoggingUtil;
import com.sap.sse.common.Util.Pair;
@@ -66,7 +66,6 @@ public class ManeuverClassifierTrainer {
LoggingUtil.logInfo("Training with " + trainManeuvers.size() + " maneuvers...");
double[][] x = modelContext.getXMatrix(maneuvers);
int[] y = labelExtraction.getYVector(maneuvers);
classifierModel.resetTrainingStats();
classifierModel.train(x, y);
LoggingUtil.logInfo("Training finished. Validating on train dataset...");
ManeuverClassifierScoring classifierScoring = new ManeuverClassifierScoring(classifierModel);
@@ -112,8 +111,7 @@ public class ManeuverClassifierTrainer {
public static void main(String[] args) throws Exception {
PolarDataService polarService = PolarDataServiceAccessUtil.getPersistedPolarService();
RegularManeuversForEstimationPersistenceManager persistenceManager = new RegularManeuversForEstimationPersistenceManager();
ModelStore classifierModelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models");
// ModelStore classifierModelStore = new MongoDbModelStore(persistenceManager.getDb());
ModelStore classifierModelStore = new MongoDbModelStoreImpl(persistenceManager.getDb());
classifierModelStore.deleteAll(ModelDomainType.MANEUVER_CLASSIFIER);
ManeuverClassifierTrainer classifierTrainer = new ManeuverClassifierTrainer(persistenceManager,
classifierModelStore);
@@ -13,11 +13,12 @@ import java.util.regex.Pattern;
import com.sap.sailing.domain.base.BoatClass;
import com.sap.sailing.domain.polars.PolarDataService;
import com.sap.sailing.windestimation.data.ManeuverForEstimation;
import com.sap.sailing.windestimation.data.persistence.maneuver.RegularManeuversForEstimationPersistenceManager;
import com.sap.sailing.windestimation.data.persistence.polars.PolarDataServiceAccessUtil;
import com.sap.sailing.windestimation.model.classifier.TrainableClassificationModel;
import com.sap.sailing.windestimation.model.exception.ModelPersistenceException;
import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl;
import com.sap.sailing.windestimation.model.store.ModelStore;
import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl;
import com.sap.sailing.windestimation.util.LoggingUtil;
public class PersistedManeuverClassifiersScorePrinter {
@@ -29,9 +30,8 @@ public class PersistedManeuverClassifiersScorePrinter {
throws MalformedURLException, ClassNotFoundException, IOException, InterruptedException {
PolarDataService polarService = PolarDataServiceAccessUtil.getPersistedPolarService();
Set<BoatClass> allBoatClasses = polarService.getAllBoatClassesWithPolarSheetsAvailable();
ModelStore classifierModelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models");
// ModelStore classifierModelStore = new MongoDbModelStore(
// new RegularManeuversForEstimationPersistenceManager().getDb());
ModelStore classifierModelStore = new MongoDbModelStoreImpl(
new RegularManeuversForEstimationPersistenceManager().getDb());
List<TrainableClassificationModel<ManeuverForEstimation, ManeuverClassifierModelContext>> allClassifierModels = new ArrayList<>();
ManeuverClassifierModelFactory classifierModelFactory = new ManeuverClassifierModelFactory();
LoggingUtil.logInfo("### Loading all boat class classifiers:");
@@ -100,7 +100,6 @@ public class TwdTransitionClassifierTrainer {
i++;
}
LoggingUtil.logInfo("Training with " + numberOfTrainingInstances + " instances...");
classifierModel.resetTrainingStats();
classifierModel.train(x, y);
LoggingUtil.logInfo("Training finished. Validating on train dataset...");
TwdTransitionClassifierScoring classifierScoring = new TwdTransitionClassifierScoring(classifierModel);
@@ -5,9 +5,9 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS
import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedSingleDimensionBasedTwdTransitionPersistenceManager.AggregatedSingleDimensionType;
import com.sap.sailing.windestimation.model.regressor.IncrementalSingleDimensionPolynomialRegressor;
import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext.DistanceValueRange;
import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl;
import com.sap.sailing.windestimation.model.store.ModelDomainType;
import com.sap.sailing.windestimation.model.store.ModelStore;
import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl;
/**
* Trains TWD delta standard deviation by considering the distance passed between two measurements.
@@ -26,8 +26,7 @@ public class DistanceBasedTwdTransitionStdRegressorTrainer extends TwdTransition
public static void main(String[] args) throws Exception {
AggregatedSingleDimensionBasedTwdTransitionPersistenceManager distanceBasedPersistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager(
AggregatedSingleDimensionType.DISTANCE);
ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models");
// ModelStore modelStore = new MongoDbModelStore(distanceBasedPersistenceManager.getDb());
ModelStore modelStore = new MongoDbModelStoreImpl(distanceBasedPersistenceManager.getDb());
modelStore.deleteAll(ModelDomainType.DISTANCE_BASED_TWD_DELTA_STD_REGRESSOR);
DistanceBasedTwdTransitionRegressorModelFactory distanceBasedTwdTransitionRegressorModelFactory = new DistanceBasedTwdTransitionRegressorModelFactory();
for (DistanceValueRange distanceValueRange : DistanceValueRange.values()) {
@@ -5,9 +5,9 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS
import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedSingleDimensionBasedTwdTransitionPersistenceManager.AggregatedSingleDimensionType;
import com.sap.sailing.windestimation.model.regressor.IncrementalSingleDimensionPolynomialRegressor;
import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext.DurationValueRange;
import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl;
import com.sap.sailing.windestimation.model.store.ModelDomainType;
import com.sap.sailing.windestimation.model.store.ModelStore;
import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl;
public class DurationBasedTwdTransitionStdRegressorTrainer extends TwdTransitionAggregatedStdRegressorTrainer {
@@ -20,8 +20,7 @@ public class DurationBasedTwdTransitionStdRegressorTrainer extends TwdTransition
public static void main(String[] args) throws Exception {
AggregatedSingleDimensionBasedTwdTransitionPersistenceManager durationBasedPersistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager(
AggregatedSingleDimensionType.DURATION);
ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models");
// ModelStore modelStore = new MongoDbModelStoreImpl(durationBasedPersistenceManager.getDb());
ModelStore modelStore = new MongoDbModelStoreImpl(durationBasedPersistenceManager.getDb());
modelStore.deleteAll(ModelDomainType.DURATION_BASED_TWD_DELTA_STD_REGRESSOR);
DurationBasedTwdTransitionRegressorModelFactory durationBasedTwdTransitionRegressorModelFactory = new DurationBasedTwdTransitionRegressorModelFactory();
for (DurationValueRange durationValueRange : DurationValueRange.values()) {
@@ -41,6 +41,9 @@ public class IncrementalSingleDimensionPolynomialRegressorTrainerHelper {
}
public void incrementRmseCalculation(double[] x, double y) {
if (!model.isModelReady()) {
model.setModelAsReadyAfterSuccessfulTraining();
}
double predictedStd = model.getValue(x);
double diff = predictedStd - y;
squareErrorSum += diff * diff;
@@ -21,8 +21,7 @@ public class TwdTransitionManualTrainingDataInputRegressorTrainer {
throws Exception {
IncrementalSingleDimensionPolynomialRegressorTrainerHelper trainerHelper = new IncrementalSingleDimensionPolynomialRegressorTrainerHelper(
regressorModelStore, model);
LoggingUtil
.logInfo("########## Training of " + model.getModelContext().getId() + " started...");
LoggingUtil.logInfo("########## Training of " + model.getModelContext().getId() + " started...");
double[] modelInput = new double[1];
for (int i = 0; i < inputOutputPairs.length; i++) {
double xi = inputOutputPairs[i][0];