mirror of
https://github.com/eclipse-sailing-analytics/sailing-analytics.git
synced 2026-11-16 07:10:48 +00:00
Merge remote-tracking branch 'server/windestimation' into windestimation
Change-Id: Id300413c84e80ae2c82a1dc52b538d834aebfe43
This commit is contained in:
commit
4c88e5f287
62 files changed
+1017
-304
No files matched your search
+3
-3
@@ -1,13 +1,13 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication">
|
||||
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
|
||||
<listEntry value="/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java"/>
|
||||
<listEntry value="/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTrainingPart1.java"/>
|
||||
</listAttribute>
|
||||
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
|
||||
<listEntry value="1"/>
|
||||
</listAttribute>
|
||||
<booleanAttribute key="org.eclipse.jdt.launching.ATTR_EXCLUDE_TEST_CODE" value="true"/>
|
||||
<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="com.sap.sailing.windestimation.data.importer.ManeuverAndWindImporter"/>
|
||||
<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="com.sap.sailing.windestimation.model.SimpleModelsTrainingPart1"/>
|
||||
<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="com.sap.sailing.windestimation.lab"/>
|
||||
<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Xmx16G"/>
|
||||
<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Xms16G -Xmx16G"/>
|
||||
</launchConfiguration>
|
||||
@@ -0,0 +1,12 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication">
|
||||
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
|
||||
<listEntry value="/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTrainingPart2.java"/>
|
||||
</listAttribute>
|
||||
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
|
||||
<listEntry value="1"/>
|
||||
</listAttribute>
|
||||
<booleanAttribute key="org.eclipse.jdt.launching.ATTR_EXCLUDE_TEST_CODE" value="true"/>
|
||||
<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="com.sap.sailing.windestimation.model.SimpleModelsTrainingPart2"/>
|
||||
<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="com.sap.sailing.windestimation.lab"/>
|
||||
</launchConfiguration>
|
||||
+8
-2
@@ -9,12 +9,18 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.SingleDimen
|
||||
public class AggregatedDistanceBasedAbsTwdTransitionImporter {
|
||||
|
||||
public static void main(String[] args) throws UnknownHostException {
|
||||
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager(
|
||||
SingleDimensionType.DISTANCE);
|
||||
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = createPersistenceManagerAndEnsureIndex();
|
||||
SingleDimensionTwdTransitionAggregationImporter importer = new SingleDimensionTwdTransitionAggregationImporter(
|
||||
singleDimensionBasedTwdTransitionPersistenceManager, AggregatedSingleDimensionType.DISTANCE_ABS,
|
||||
new AnnealingNextThresholdCalculator(10, 1.5));
|
||||
importer.runAggregation();
|
||||
}
|
||||
|
||||
public static SingleDimensionBasedTwdTransitionPersistenceManager createPersistenceManagerAndEnsureIndex()
|
||||
throws UnknownHostException {
|
||||
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager(
|
||||
SingleDimensionType.DISTANCE);
|
||||
return singleDimensionBasedTwdTransitionPersistenceManager;
|
||||
}
|
||||
|
||||
}
|
||||
+8
-2
@@ -9,12 +9,18 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.SingleDimen
|
||||
public class AggregatedDistanceBasedTwdTransitionImporter {
|
||||
|
||||
public static void main(String[] args) throws UnknownHostException {
|
||||
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager(
|
||||
SingleDimensionType.DISTANCE);
|
||||
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = createPersistenceManagerAndEnsureIndex();
|
||||
SingleDimensionTwdTransitionAggregationImporter importer = new SingleDimensionTwdTransitionAggregationImporter(
|
||||
singleDimensionBasedTwdTransitionPersistenceManager, AggregatedSingleDimensionType.DISTANCE,
|
||||
new AnnealingNextThresholdCalculator(10, 1.5));
|
||||
importer.runAggregation();
|
||||
}
|
||||
|
||||
public static SingleDimensionBasedTwdTransitionPersistenceManager createPersistenceManagerAndEnsureIndex()
|
||||
throws UnknownHostException {
|
||||
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager(
|
||||
SingleDimensionType.DISTANCE);
|
||||
return singleDimensionBasedTwdTransitionPersistenceManager;
|
||||
}
|
||||
|
||||
}
|
||||
+8
-2
@@ -9,12 +9,18 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.SingleDimen
|
||||
public class AggregatedDurationBasedAbsTwdTransitionImporter {
|
||||
|
||||
public static void main(String[] args) throws UnknownHostException {
|
||||
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager(
|
||||
SingleDimensionType.DURATION);
|
||||
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = createPersistenceManagerAndEnsureIndex();
|
||||
SingleDimensionTwdTransitionAggregationImporter importer = new SingleDimensionTwdTransitionAggregationImporter(
|
||||
singleDimensionBasedTwdTransitionPersistenceManager, AggregatedSingleDimensionType.DURATION_ABS,
|
||||
DurationBasedTwdTransitionImporter.getThresholdCalculator());
|
||||
importer.runAggregation();
|
||||
}
|
||||
|
||||
public static SingleDimensionBasedTwdTransitionPersistenceManager createPersistenceManagerAndEnsureIndex()
|
||||
throws UnknownHostException {
|
||||
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager(
|
||||
SingleDimensionType.DURATION);
|
||||
return singleDimensionBasedTwdTransitionPersistenceManager;
|
||||
}
|
||||
|
||||
}
|
||||
+8
-2
@@ -9,12 +9,18 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.SingleDimen
|
||||
public class AggregatedDurationBasedTwdTransitionImporter {
|
||||
|
||||
public static void main(String[] args) throws UnknownHostException {
|
||||
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager(
|
||||
SingleDimensionType.DURATION);
|
||||
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = createPersistenceManagerAndEnsureIndex();
|
||||
SingleDimensionTwdTransitionAggregationImporter importer = new SingleDimensionTwdTransitionAggregationImporter(
|
||||
singleDimensionBasedTwdTransitionPersistenceManager, AggregatedSingleDimensionType.DURATION,
|
||||
DurationBasedTwdTransitionImporter.getThresholdCalculator());
|
||||
importer.runAggregation();
|
||||
}
|
||||
|
||||
public static SingleDimensionBasedTwdTransitionPersistenceManager createPersistenceManagerAndEnsureIndex()
|
||||
throws UnknownHostException {
|
||||
SingleDimensionBasedTwdTransitionPersistenceManager singleDimensionBasedTwdTransitionPersistenceManager = new SingleDimensionBasedTwdTransitionPersistenceManager(
|
||||
SingleDimensionType.DURATION);
|
||||
return singleDimensionBasedTwdTransitionPersistenceManager;
|
||||
}
|
||||
|
||||
}
|
||||
+6
-4
@@ -36,7 +36,8 @@ public class DistanceBasedTwdTransitionImporter {
|
||||
.getIterator(); iterator.hasNext();) {
|
||||
long percent = windSourceNumber * 100 / numberOfWindSources;
|
||||
WindSourceWithFixes windSource = iterator.next();
|
||||
LoggingUtil.logInfo("Processing " + windSourceNumber++ + "/" + numberOfWindSources + " (" + percent + "%)");
|
||||
LoggingUtil.logInfo("Processing wind source " + windSourceNumber++ + "/" + numberOfWindSources + " ("
|
||||
+ percent + "%) for distance dimension");
|
||||
List<SingleDimensionBasedTwdTransition> entries = new ArrayList<>();
|
||||
List<WindSourceWithFixes> otherWindSources = new ArrayList<>();
|
||||
for (PersistedElementsIterator<WindSourceWithFixes> otherWindSourcesIterator = windSourcesPersistenceManager
|
||||
@@ -109,16 +110,17 @@ public class DistanceBasedTwdTransitionImporter {
|
||||
} while (currentFix != null && currentOtherFix != null);
|
||||
}
|
||||
if (entries.isEmpty()) {
|
||||
LoggingUtil.logInfo("No TWD transitions to import");
|
||||
LoggingUtil.logInfo("No distance based TWD transitions to import");
|
||||
} else {
|
||||
distanceBasedTwdTransitionPersistenceManager.add(entries);
|
||||
int totalEntries = entries.size();
|
||||
totalValuesCount += totalEntries;
|
||||
LoggingUtil.logInfo(totalEntries + " TWD transitions imported, " + totalValuesCount + " in total");
|
||||
LoggingUtil.logInfo(
|
||||
totalEntries + " distance based TWD transitions imported, " + totalValuesCount + " in total");
|
||||
}
|
||||
}
|
||||
LoggingUtil.logInfo("###################\r\nDistance based TWD transitions Import finished");
|
||||
LoggingUtil.logInfo("Totally " + totalValuesCount + " TWD transitions imported");
|
||||
LoggingUtil.logInfo("Totally " + totalValuesCount + " distance based TWD transitions imported");
|
||||
}
|
||||
|
||||
}
|
||||
+6
-5
@@ -32,7 +32,8 @@ public class DurationBasedTwdTransitionImporter {
|
||||
.getIterator(); iterator.hasNext();) {
|
||||
long percent = windSourceNumber * 100 / numberOfWindSources;
|
||||
WindSourceWithFixes windSource = iterator.next();
|
||||
LoggingUtil.logInfo("Processing " + windSourceNumber++ + "/" + numberOfWindSources + " (" + percent + "%)");
|
||||
LoggingUtil.logInfo("Processing wind source " + windSourceNumber++ + "/" + numberOfWindSources + " ("
|
||||
+ percent + "%) for duration dimension");
|
||||
TimePoint timePointOfLastConsideredWindFix = null;
|
||||
List<SingleDimensionBasedTwdTransition> entries = new ArrayList<>();
|
||||
int windFixIndex = 0;
|
||||
@@ -58,17 +59,17 @@ public class DurationBasedTwdTransitionImporter {
|
||||
}
|
||||
}
|
||||
if (entries.isEmpty()) {
|
||||
LoggingUtil.logInfo("No TWD transitions to import");
|
||||
LoggingUtil.logInfo("No duration based TWD transitions to import");
|
||||
} else {
|
||||
durationBasedTwdTransitionPersistenceManager.add(entries);
|
||||
int totalEntries = entries.size();
|
||||
totalValuesCount += totalEntries;
|
||||
LoggingUtil
|
||||
.logInfo(totalEntries + " TWD transition entries imported, " + totalValuesCount + " in total");
|
||||
LoggingUtil.logInfo(totalEntries + " duration based TWD transition entries imported, "
|
||||
+ totalValuesCount + " in total");
|
||||
}
|
||||
}
|
||||
LoggingUtil.logInfo("###################\r\nDuration based TWD transitions Import finished");
|
||||
LoggingUtil.logInfo("Totally " + totalValuesCount + " TWD transitions imported");
|
||||
LoggingUtil.logInfo("Totally " + totalValuesCount + " duration based TWD transitions imported");
|
||||
}
|
||||
|
||||
public static NextThresholdCalculator getThresholdCalculator() {
|
||||
|
||||
+97
-42
@@ -13,6 +13,10 @@ import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.RejectedExecutionHandler;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.http.Header;
|
||||
import org.apache.http.HttpResponse;
|
||||
@@ -40,11 +44,11 @@ import com.sap.sailing.windestimation.data.persistence.maneuver.RaceWithManeuver
|
||||
import com.sap.sailing.windestimation.data.persistence.maneuver.RaceWithManeuverForEstimationPersistenceManager;
|
||||
import com.sap.sailing.windestimation.data.persistence.twdtransition.RaceWithWindSourcesPersistenceManager;
|
||||
import com.sap.sailing.windestimation.data.serialization.CompetitorTrackWithEstimationDataJsonDeserializer;
|
||||
import com.sap.sailing.windestimation.data.serialization.ManeuverForDataAnalysisJsonSerializer;
|
||||
import com.sap.sailing.windestimation.data.serialization.LabeledManeuverForEstimationJsonSerializer;
|
||||
import com.sap.sailing.windestimation.data.serialization.ManeuverForDataAnalysisJsonSerializer;
|
||||
import com.sap.sailing.windestimation.data.transformer.CompetitorTrackTransformer;
|
||||
import com.sap.sailing.windestimation.data.transformer.CompleteManeuverCurveWithEstimationDataToManeuverForDataAnalysisTransformer;
|
||||
import com.sap.sailing.windestimation.data.transformer.CompleteManeuverCurveWithEstimationDataToLabelledManeuverForEstimationTransformer;
|
||||
import com.sap.sailing.windestimation.data.transformer.CompleteManeuverCurveWithEstimationDataToManeuverForDataAnalysisTransformer;
|
||||
import com.sap.sailing.windestimation.util.LoggingUtil;
|
||||
|
||||
/**
|
||||
@@ -54,6 +58,7 @@ import com.sap.sailing.windestimation.util.LoggingUtil;
|
||||
*/
|
||||
public class ManeuverAndWindImporter {
|
||||
|
||||
private static final int CONNECTION_TIMEOUT_MILLIS = 60000 * 5;
|
||||
public static final String REST_API_BASE_URL = "https://www.sapsailing.com/sailingserver/api/v1";
|
||||
public static final String REST_API_REGATTAS_PATH = "/regattas";
|
||||
public static final String REST_API_RACES_PATH = "/races";
|
||||
@@ -70,6 +75,8 @@ public class ManeuverAndWindImporter {
|
||||
private final ManeuverForDataAnalysisJsonSerializer maneuverForDataAnalysisJsonSerializer;
|
||||
private final LabeledManeuverForEstimationJsonSerializer maneuverForEstimationJsonSerializer;
|
||||
private boolean skipRace;
|
||||
private static final int NUMBER_OF_THREADS = 10; // high number due to HTTP requests
|
||||
private final ThreadPoolExecutor executorService;
|
||||
|
||||
public ManeuverAndWindImporter() throws UnknownHostException {
|
||||
this.completeManeuverCurvePersistanceManager = new RaceWithCompleteManeuverCurvePersistenceManager();
|
||||
@@ -80,12 +87,28 @@ public class ManeuverAndWindImporter {
|
||||
this.maneuverForEstimationTransformer = new CompleteManeuverCurveWithEstimationDataToLabelledManeuverForEstimationTransformer();
|
||||
this.maneuverForDataAnalysisJsonSerializer = new ManeuverForDataAnalysisJsonSerializer();
|
||||
this.maneuverForEstimationJsonSerializer = new LabeledManeuverForEstimationJsonSerializer();
|
||||
this.executorService = new ThreadPoolExecutor(NUMBER_OF_THREADS, NUMBER_OF_THREADS, 0L, TimeUnit.MILLISECONDS,
|
||||
new ArrayBlockingQueue<>(NUMBER_OF_THREADS));
|
||||
this.executorService.setRejectedExecutionHandler(new RejectedExecutionHandler() {
|
||||
public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
|
||||
// this will block if the queue is full
|
||||
try {
|
||||
executor.getQueue().put(r);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public HttpClient createNewHttpClient() {
|
||||
HttpParams httpParams = new BasicHttpParams();
|
||||
HttpConnectionParams.setConnectionTimeout(httpParams, 30000);
|
||||
HttpConnectionParams.setConnectionTimeout(httpParams, CONNECTION_TIMEOUT_MILLIS);
|
||||
HttpClient client = new SystemDefaultHttpClient(httpParams);
|
||||
client.getParams().setParameter("http.socket.timeout", CONNECTION_TIMEOUT_MILLIS);
|
||||
client.getParams().setParameter("http.connection.timeout", CONNECTION_TIMEOUT_MILLIS);
|
||||
client.getParams().setParameter("http.connection-manager.timeout", new Long(CONNECTION_TIMEOUT_MILLIS));
|
||||
client.getParams().setParameter("http.protocol.head-body-timeout", CONNECTION_TIMEOUT_MILLIS);
|
||||
return client;
|
||||
}
|
||||
|
||||
@@ -94,8 +117,8 @@ public class ManeuverAndWindImporter {
|
||||
importer.importAllRegattas();
|
||||
}
|
||||
|
||||
public void importAllRegattas()
|
||||
throws IllegalStateException, ClientProtocolException, IOException, ParseException, URISyntaxException {
|
||||
public void importAllRegattas() throws IllegalStateException, ClientProtocolException, IOException, ParseException,
|
||||
URISyntaxException, InterruptedException {
|
||||
skipRace = startFromRegattaName != null;
|
||||
LoggingUtil.logInfo("Importer for CompleteManeuverCurveWithEstimationData just started");
|
||||
LoggingUtil.logInfo("Dropping old database");
|
||||
@@ -118,9 +141,17 @@ public class ManeuverAndWindImporter {
|
||||
+ Math.round(100.0 * i / numberOfRegattas) + "%): \"" + regattaName + "\"");
|
||||
importRegatta(regattaName, importStatistics);
|
||||
}
|
||||
LoggingUtil.logInfo("Import finished");
|
||||
importStatistics.regattasCount = regattasJson.size();
|
||||
logImportStatistics(importStatistics);
|
||||
executorService.shutdown();
|
||||
boolean success = executorService.awaitTermination(2, TimeUnit.HOURS);
|
||||
if (success) {
|
||||
LoggingUtil.logInfo("Import finished");
|
||||
synchronized (importStatistics) {
|
||||
importStatistics.regattasCount = regattasJson.size();
|
||||
logImportStatistics(importStatistics);
|
||||
}
|
||||
} else {
|
||||
new InterruptedException("Thread-pool was terminated after two hours waiting time");
|
||||
}
|
||||
}
|
||||
|
||||
private void logImportStatistics(ImportStatistics importStatistics) {
|
||||
@@ -139,14 +170,15 @@ public class ManeuverAndWindImporter {
|
||||
private void importRegatta(String regattaName, ImportStatistics importStatistics)
|
||||
throws IllegalStateException, ClientProtocolException, IOException, ParseException, URISyntaxException {
|
||||
String encodedRegattaName = encodeUrlPathPart(regattaName);
|
||||
// TODO The encoding of some regatta names with special characters does not work well
|
||||
HttpGet getRegatta = new HttpGet(REST_API_BASE_URL + REST_API_REGATTAS_PATH + "/" + encodedRegattaName);
|
||||
JSONObject regattaJson = null;
|
||||
try {
|
||||
regattaJson = (JSONObject) getHttpResponseAsJson(regattaName, null, getRegatta);
|
||||
} catch (Exception e) {
|
||||
importStatistics.ignoredRegattas += 1;
|
||||
LoggingUtil.logInfo("Error while processing regatta: " + regattaName);
|
||||
synchronized (importStatistics) {
|
||||
importStatistics.ignoredRegattas++;
|
||||
}
|
||||
LoggingUtil.logInfo("Error while processing regatta: " + regattaName + "\r\n" + getRegatta);
|
||||
return;
|
||||
}
|
||||
for (Object seriesJson : (JSONArray) regattaJson.get("series")) {
|
||||
@@ -161,17 +193,29 @@ public class ManeuverAndWindImporter {
|
||||
if ((boolean) race.get("isTracked") && !(boolean) race.get("isLive")
|
||||
&& (boolean) race.get("hasGpsData") && (boolean) race.get("hasWindData")) {
|
||||
String trackedRaceName = (String) race.get("trackedRaceName");
|
||||
LoggingUtil.logInfo("Processing race nr. " + ++i + ": \"" + trackedRaceName + "\"");
|
||||
try {
|
||||
importRace(regattaName, trackedRaceName, importStatistics);
|
||||
} catch (Exception e) {
|
||||
importStatistics.ingoredRaces += 1;
|
||||
LoggingUtil
|
||||
.logInfo("Error while processing race nr. " + i + ": \"" + trackedRaceName + "\"");
|
||||
}
|
||||
i++;
|
||||
final int raceNumber = i;
|
||||
LoggingUtil.logInfo("Processing race nr. " + raceNumber + ": \"" + trackedRaceName + "\"");
|
||||
executorService.execute(() -> {
|
||||
try {
|
||||
importRace(regattaName, trackedRaceName, importStatistics);
|
||||
} catch (Exception e) {
|
||||
synchronized (importStatistics) {
|
||||
importStatistics.ingoredRaces += 1;
|
||||
}
|
||||
String extraLog = "";
|
||||
if (e instanceof HttpClientException) {
|
||||
extraLog = "\r\n" + ((HttpClientException) e).getRequest();
|
||||
}
|
||||
LoggingUtil.logInfo("Error while processing race nr. " + raceNumber + ": \""
|
||||
+ trackedRaceName + "\"" + extraLog);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
importStatistics.racesCount += racesJson.size();
|
||||
synchronized (importStatistics) {
|
||||
importStatistics.racesCount += racesJson.size();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -215,11 +259,14 @@ public class ManeuverAndWindImporter {
|
||||
JSONArray windFixesJson = (JSONArray) ((JSONObject) windSourceObj).get(RaceWindJsonSerializer.FIXES);
|
||||
windFixesCount += windFixesJson.size();
|
||||
}
|
||||
LoggingUtil.logInfo(
|
||||
"Imported " + windFixesCount + " wind fixes from " + windSourcesJson.size() + " wind sources");
|
||||
importStatistics.racesWithHighQualityWindData++;
|
||||
LoggingUtil.logInfo("Imported " + windFixesCount + " wind fixes from " + windSourcesJson.size()
|
||||
+ " wind sources in race \"" + trackedRaceName + "\" of regatta \"" + regattaName + "\"");
|
||||
synchronized (importStatistics) {
|
||||
importStatistics.racesWithHighQualityWindData++;
|
||||
}
|
||||
} else {
|
||||
LoggingUtil.logInfo("No high quality wind fixes contained");
|
||||
LoggingUtil.logInfo("No high quality wind fixes contained in race \"" + trackedRaceName + "\" of regatta \""
|
||||
+ regattaName + "\"");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -272,10 +319,12 @@ public class ManeuverAndWindImporter {
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
LoggingUtil.logInfo(
|
||||
"Imported " + competitorTracks.size() + " competitor tracks with " + maneuversCount + " maneuvers");
|
||||
importStatistics.competitorTracksCount += competitorTracks.size();
|
||||
importStatistics.maneuversCount += maneuversCount;
|
||||
LoggingUtil.logInfo("Imported " + competitorTracks.size() + " competitor tracks with " + maneuversCount
|
||||
+ " maneuvers in race \"" + trackedRaceName + "\" of regatta \"" + regattaName + "\"");
|
||||
synchronized (importStatistics) {
|
||||
importStatistics.competitorTracksCount += competitorTracks.size();
|
||||
importStatistics.maneuversCount += maneuversCount;
|
||||
}
|
||||
}
|
||||
|
||||
private JSONObject getHttpResponseAsJson(String trackedRegattaName, String trackedRaceName,
|
||||
@@ -285,7 +334,8 @@ public class ManeuverAndWindImporter {
|
||||
for (int i = 1; i <= 10; i++) {
|
||||
try {
|
||||
httpResponse = createNewHttpClient().execute(getEstimationData);
|
||||
break;
|
||||
JSONObject resultJson = (JSONObject) getJsonFromResponse(httpResponse);
|
||||
return resultJson;
|
||||
} catch (Exception e) {
|
||||
Thread.sleep(10000);
|
||||
lastException = e;
|
||||
@@ -298,17 +348,7 @@ public class ManeuverAndWindImporter {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (httpResponse == null) {
|
||||
throw lastException;
|
||||
}
|
||||
JSONObject resultJson;
|
||||
try {
|
||||
resultJson = (JSONObject) getJsonFromResponse(httpResponse);
|
||||
} catch (Exception e) {
|
||||
System.out.println(getEstimationData);
|
||||
throw e;
|
||||
}
|
||||
return resultJson;
|
||||
throw new HttpClientException(getEstimationData.toString(), lastException);
|
||||
}
|
||||
|
||||
private <ToType> void addTransformedElementsToCompetitorTrackJson(List<JSONObject> competitorTracks,
|
||||
@@ -331,14 +371,13 @@ public class ManeuverAndWindImporter {
|
||||
}
|
||||
}
|
||||
|
||||
// FIXME duplicated code taken from ConnectivityUtils
|
||||
public static Object getJsonFromResponse(HttpResponse response)
|
||||
throws IllegalStateException, IOException, ParseException {
|
||||
JSONParser jsonParser = new JSONParser();
|
||||
final Header contentEncoding = response.getEntity().getContentEncoding();
|
||||
final Reader reader;
|
||||
if (contentEncoding == null) {
|
||||
reader = new InputStreamReader(response.getEntity().getContent());
|
||||
reader = new InputStreamReader(response.getEntity().getContent(), "UTF-8");
|
||||
} else {
|
||||
reader = new InputStreamReader(response.getEntity().getContent(), contentEncoding.getValue());
|
||||
}
|
||||
@@ -358,4 +397,20 @@ public class ManeuverAndWindImporter {
|
||||
private int racesWithHighQualityWindData = 0;
|
||||
}
|
||||
|
||||
private static class HttpClientException extends Exception {
|
||||
|
||||
private static final long serialVersionUID = 4948532287832868768L;
|
||||
private final String request;
|
||||
|
||||
public HttpClientException(String request, Exception e) {
|
||||
super(e);
|
||||
this.request = request;
|
||||
}
|
||||
|
||||
public String getRequest() {
|
||||
return request;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+8
-5
@@ -35,7 +35,7 @@ public class SingleDimensionTwdTransitionAggregationImporter {
|
||||
public void runAggregation() throws UnknownHostException {
|
||||
long totalValuesCount = 0;
|
||||
List<AggregatedSingleDimensionBasedTwdTransition> aggregates = new ArrayList<>();
|
||||
LoggingUtil.logInfo("Aggregating persisted entries");
|
||||
LoggingUtil.logInfo("Aggregating persisted entries for dimension " + dimensionType);
|
||||
TDoubleList entries = new TDoubleArrayList();
|
||||
double currentBucketThreshold = thresholdCalculator.getInitialThresholdValue();
|
||||
double nextBucketThreshold = thresholdCalculator.getNextThresholdValue(currentBucketThreshold);
|
||||
@@ -61,7 +61,8 @@ public class SingleDimensionTwdTransitionAggregationImporter {
|
||||
}
|
||||
entries.add(twdChange);
|
||||
if (entries.size() % 10000 == 0) {
|
||||
LoggingUtil.logInfo((entries.size() + totalValuesCount) + " Entries aggregated");
|
||||
LoggingUtil.logInfo(
|
||||
(entries.size() + totalValuesCount) + " Entries aggregated for dimension " + dimensionType);
|
||||
}
|
||||
}
|
||||
if (entries.size() >= MIN_NUMBER_OF_VALUES_PER_BUCKET) {
|
||||
@@ -69,14 +70,16 @@ public class SingleDimensionTwdTransitionAggregationImporter {
|
||||
aggregates.add(aggregate);
|
||||
totalValuesCount += entries.size();
|
||||
}
|
||||
LoggingUtil.logInfo("Persisting " + aggregates.size() + " aggregates");
|
||||
LoggingUtil.logInfo("Persisting " + aggregates.size() + " aggregates for dimension " + dimensionType);
|
||||
AggregatedSingleDimensionBasedTwdTransitionPersistenceManager aggregatedDurationBasedTwdTransitionPersistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager(
|
||||
dimensionType);
|
||||
aggregatedDurationBasedTwdTransitionPersistenceManager.dropCollection();
|
||||
aggregatedDurationBasedTwdTransitionPersistenceManager.add(aggregates);
|
||||
LoggingUtil.logInfo("###################\r\n" + dimensionType + " based TWD transitions Import finished");
|
||||
LoggingUtil.logInfo("Totally " + totalValuesCount + " TWD transitions imported");
|
||||
LoggingUtil.logInfo("Totally " + aggregates.size() + " TWD transition aggregates imported");
|
||||
LoggingUtil.logInfo(
|
||||
"Totally " + totalValuesCount + " " + dimensionType + " based TWD transitions have been processed");
|
||||
LoggingUtil.logInfo(
|
||||
"Totally " + aggregates.size() + " TWD transition aggregates imported for dimension " + dimensionType);
|
||||
}
|
||||
|
||||
private double getFinalBucketThreshold(double currentBucketThreshold, double nextBucketThreshold) {
|
||||
|
||||
+6
-23
@@ -17,20 +17,16 @@ import com.sap.sailing.server.gateway.deserialization.JsonDeserializer;
|
||||
import com.sap.sailing.server.gateway.serialization.impl.RaceWindJsonSerializer;
|
||||
import com.sap.sailing.windestimation.data.RaceWithWindSources;
|
||||
import com.sap.sailing.windestimation.data.persistence.maneuver.AbstractPersistenceManager;
|
||||
import com.sap.sailing.windestimation.data.persistence.maneuver.PersistedElementsIterator;
|
||||
import com.sap.sailing.windestimation.data.serialization.RaceWithEstimationDataDeserializer;
|
||||
import com.sap.sailing.windestimation.data.serialization.RaceWithWindSourcesDeserializer;
|
||||
import com.sap.sse.common.TimePoint;
|
||||
|
||||
public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceManager<RaceWithWindSources> {
|
||||
|
||||
private static final String COLLECTION_NAME = "racesWithWind";
|
||||
private static final String WIND_SOURCES_COLLECTION_NAME = "windSources";
|
||||
static final String COLLECTION_NAME = "racesWithWind";
|
||||
private final WindSourcesPersistenceManager windSourcesPersistenceManager;
|
||||
|
||||
public RaceWithWindSourcesPersistenceManager() throws UnknownHostException {
|
||||
Document indexes = new Document(RaceWindJsonSerializer.START_TIME_POINT, 1);
|
||||
indexes.put(RaceWindJsonSerializer.END_TIME_POINT, 1);
|
||||
getCollection().createIndex(indexes);
|
||||
windSourcesPersistenceManager = new WindSourcesPersistenceManager();
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -41,7 +37,7 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa
|
||||
@Override
|
||||
public void dropCollection() {
|
||||
super.dropCollection();
|
||||
getDb().getCollection(getWindSourcesCollectionName()).drop();
|
||||
windSourcesPersistenceManager.dropCollection();
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -52,7 +48,7 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa
|
||||
JSONArray windSourceIdsJson = (JSONArray) raceJson
|
||||
.get(RaceWithEstimationDataDeserializer.COMPETITOR_TRACKS);
|
||||
JSONArray windSourcesJson = new JSONArray();
|
||||
MongoCollection<Document> windSourcesCollection = getDb().getCollection(getWindSourcesCollectionName());
|
||||
MongoCollection<Document> windSourcesCollection = windSourcesPersistenceManager.getCollection();
|
||||
for (Object idObject : windSourceIdsJson) {
|
||||
Document dbWindSource = windSourcesCollection
|
||||
.find(new Document(FIELD_DB_ID, new ObjectId((String) idObject))).first();
|
||||
@@ -69,10 +65,6 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa
|
||||
};
|
||||
}
|
||||
|
||||
public static String getWindSourcesCollectionName() {
|
||||
return COLLECTION_NAME + "." + WIND_SOURCES_COLLECTION_NAME;
|
||||
}
|
||||
|
||||
public void add(String regattaName, String raceName, JSONObject raceWithWindSourcesJson) {
|
||||
|
||||
JSONArray windSourcesJson = (JSONArray) raceWithWindSourcesJson.get(RaceWindJsonSerializer.WIND_SOURCES);
|
||||
@@ -81,7 +73,7 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa
|
||||
Document entry = parseJsonString(windSourceObj.toString());
|
||||
dbWindSources.add(entry);
|
||||
}
|
||||
MongoCollection<Document> windSourcesCollection = getDb().getCollection(getWindSourcesCollectionName());
|
||||
MongoCollection<Document> windSourcesCollection = windSourcesPersistenceManager.getCollection();
|
||||
windSourcesCollection.insertMany(dbWindSources);
|
||||
BasicDBList dbWindSourceIds = new BasicDBList();
|
||||
for (Document dbWindSource : dbWindSources) {
|
||||
@@ -96,13 +88,4 @@ public class RaceWithWindSourcesPersistenceManager extends AbstractPersistenceMa
|
||||
races.insertOne(dbObject);
|
||||
}
|
||||
|
||||
public PersistedElementsIterator<RaceWithWindSources> getIteratorForEntriesIntersectingPeriod(
|
||||
TimePoint startTimePoint, TimePoint endTimePoint, double toleranceInSeconds) {
|
||||
double maxStartTime = endTimePoint.asMillis() + toleranceInSeconds * 1000;
|
||||
double maxEndTime = startTimePoint.asMillis() - toleranceInSeconds * 1000;
|
||||
String query = "{$and: [{'" + RaceWindJsonSerializer.START_TIME_POINT + "': {$lte: " + maxStartTime + "}}, {'"
|
||||
+ RaceWindJsonSerializer.END_TIME_POINT + "': {$gte: " + maxEndTime + "}}]}";
|
||||
return getIterator(query);
|
||||
}
|
||||
|
||||
}
|
||||
+6
-3
@@ -21,15 +21,18 @@ import com.sap.sse.common.TimePoint;
|
||||
|
||||
public class WindSourcesPersistenceManager extends AbstractPersistenceManager<WindSourceWithFixes> {
|
||||
|
||||
private static final String WIND_SOURCES_COLLECTION_NAME = "windSources";
|
||||
|
||||
public WindSourcesPersistenceManager() throws UnknownHostException {
|
||||
BasicDBObject indexes = new BasicDBObject(RaceWindJsonSerializer.START_TIME_POINT, 1);
|
||||
indexes.put(RaceWindJsonSerializer.END_TIME_POINT, 1);
|
||||
BasicDBObject indexes = new BasicDBObject("_id", 1);
|
||||
indexes.put(RaceWindJsonSerializer.START_TIME_POINT, 1);
|
||||
indexes.put(RaceWindJsonSerializer.END_TIME_POINT, -1);
|
||||
getCollection().createIndex(indexes);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getCollectionName() {
|
||||
return RaceWithWindSourcesPersistenceManager.getWindSourcesCollectionName();
|
||||
return RaceWithWindSourcesPersistenceManager.COLLECTION_NAME + "." + WIND_SOURCES_COLLECTION_NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
+7
-1
@@ -7,11 +7,17 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS
|
||||
|
||||
public class AggregatedDistanceDimensionPlot {
|
||||
|
||||
private static SingleDimensionAggregatesPlottingFrame plot;
|
||||
|
||||
public static void main(String[] args) throws UnknownHostException {
|
||||
AggregatedSingleDimensionBasedTwdTransitionPersistenceManager persistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager(
|
||||
AggregatedSingleDimensionType.DISTANCE);
|
||||
SingleDimensionAggregatesPlottingFrame plot = new SingleDimensionAggregatesPlottingFrame(persistenceManager);
|
||||
plot = new SingleDimensionAggregatesPlottingFrame(persistenceManager);
|
||||
plot.renderChart();
|
||||
}
|
||||
|
||||
public static void awaitWindowClosed() throws InterruptedException {
|
||||
plot.awaitWindowClosed();
|
||||
}
|
||||
|
||||
}
|
||||
+7
-1
@@ -7,11 +7,17 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS
|
||||
|
||||
public class AggregatedDurationDimensionPlot {
|
||||
|
||||
private static SingleDimensionAggregatesPlottingFrame plot;
|
||||
|
||||
public static void main(String[] args) throws UnknownHostException {
|
||||
AggregatedSingleDimensionBasedTwdTransitionPersistenceManager persistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager(
|
||||
AggregatedSingleDimensionType.DURATION);
|
||||
SingleDimensionAggregatesPlottingFrame plot = new SingleDimensionAggregatesPlottingFrame(persistenceManager);
|
||||
plot = new SingleDimensionAggregatesPlottingFrame(persistenceManager);
|
||||
plot.renderChart();
|
||||
}
|
||||
|
||||
public static void awaitWindowClosed() throws InterruptedException {
|
||||
plot.awaitWindowClosed();
|
||||
}
|
||||
|
||||
}
|
||||
+37
-1
@@ -3,9 +3,13 @@ package com.sap.sailing.windestimation.datavisualization;
|
||||
import java.awt.BasicStroke;
|
||||
import java.awt.Color;
|
||||
import java.awt.Shape;
|
||||
import java.awt.event.WindowAdapter;
|
||||
import java.awt.event.WindowEvent;
|
||||
import java.awt.geom.Ellipse2D;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
|
||||
import javax.swing.BoxLayout;
|
||||
import javax.swing.JButton;
|
||||
import javax.swing.JFrame;
|
||||
import javax.swing.JPanel;
|
||||
import javax.swing.SwingUtilities;
|
||||
@@ -40,27 +44,54 @@ public class SingleDimensionAggregatesPlottingFrame extends JFrame {
|
||||
private XYSeries valuesSeries;
|
||||
private XYSeries stdSeries;
|
||||
private XYSeries zeroMeanStdSeries;
|
||||
private final CountDownLatch windowClosedLatch = new CountDownLatch(1);
|
||||
|
||||
public SingleDimensionAggregatesPlottingFrame(
|
||||
AggregatedSingleDimensionBasedTwdTransitionPersistenceManager persistenceManager) {
|
||||
super(persistenceManager.getCollectionName());
|
||||
this.persistenceManager = persistenceManager;
|
||||
addWindowListener(new WindowAdapter() {
|
||||
@Override
|
||||
public void windowClosed(WindowEvent event) {
|
||||
windowClosedLatch.countDown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void windowClosing(WindowEvent e) {
|
||||
setLayout(null);
|
||||
setVisible(false);
|
||||
dispose();
|
||||
}
|
||||
});
|
||||
initializeDataWithCharts();
|
||||
}
|
||||
|
||||
private void initializeDataWithCharts() {
|
||||
createSeries();
|
||||
fillSeries();
|
||||
DefaultTableXYDataset mainDataset = createMainDataset();
|
||||
DefaultTableXYDataset histoDataset = createHistoDataset();
|
||||
mainChart = createMainChart(mainDataset);
|
||||
histoChart = createHistoChart(histoDataset);
|
||||
// Create Panel
|
||||
JPanel panel = new JPanel();
|
||||
panel.setLayout(new BoxLayout(panel, BoxLayout.Y_AXIS));
|
||||
ChartPanel mainChartPanel = new ChartPanel(mainChart);
|
||||
ChartPanel histoChartPanel = new ChartPanel(histoChart);
|
||||
JButton refreshButton = new JButton("Refresh charts");
|
||||
refreshButton.addActionListener(event -> refreshCharts());
|
||||
panel.add(refreshButton);
|
||||
panel.add(mainChartPanel);
|
||||
panel.add(histoChartPanel);
|
||||
setContentPane(panel);
|
||||
}
|
||||
|
||||
public void refreshCharts() {
|
||||
SwingUtilities.invokeLater(() -> {
|
||||
initializeDataWithCharts();
|
||||
revalidate();
|
||||
});
|
||||
}
|
||||
|
||||
private void createSeries() {
|
||||
meanSeries = new XYSeries("Mean", true, false);
|
||||
medianSeries = new XYSeries("Median", true, false);
|
||||
@@ -191,4 +222,9 @@ public class SingleDimensionAggregatesPlottingFrame extends JFrame {
|
||||
});
|
||||
}
|
||||
|
||||
public void awaitWindowClosed() throws InterruptedException {
|
||||
setDefaultCloseOperation(WindowConstants.DO_NOTHING_ON_CLOSE);
|
||||
windowClosedLatch.await();
|
||||
}
|
||||
|
||||
}
|
||||
+3
-5
@@ -12,8 +12,8 @@ import com.sap.sailing.windestimation.data.persistence.maneuver.PersistedElement
|
||||
import com.sap.sailing.windestimation.data.persistence.maneuver.RaceWithCompleteManeuverCurvePersistenceManager;
|
||||
import com.sap.sailing.windestimation.data.persistence.polars.PolarDataServiceAccessUtil;
|
||||
import com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverFeatures;
|
||||
import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl;
|
||||
import com.sap.sailing.windestimation.model.store.ModelStore;
|
||||
import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl;
|
||||
import com.sap.sailing.windestimation.util.LoggingUtil;
|
||||
|
||||
public class WindEstimatorManeuverNumberDependentEvaluationRunner {
|
||||
@@ -38,18 +38,15 @@ public class WindEstimatorManeuverNumberDependentEvaluationRunner {
|
||||
LoggingUtil.logInfo("Loading polar data");
|
||||
PolarDataService polarService = PolarDataServiceAccessUtil.getPersistedPolarService();
|
||||
LoggingUtil.logInfo("Wind estimator evaluation started...");
|
||||
ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models");
|
||||
// ModelStore modelStore = new MongoDbModelStoreImpl(persistenceManager.getDb());
|
||||
ModelStore modelStore = new MongoDbModelStoreImpl(persistenceManager.getDb());
|
||||
WindEstimatorFactories estimatorFactories = new WindEstimatorFactories(polarService,
|
||||
new ManeuverFeatures(ENABLE_POLARS, ENABLE_SCALED_SPEED, ENABLE_MARKS_INFORMATION), modelStore);
|
||||
|
||||
double[] avgErrorDegreesPerManeuverCount = new double[MAX_MANEUVERS];
|
||||
double[] avgConfidencePerManeuverCount = new double[MAX_MANEUVERS];
|
||||
double[] avgConfidenceOfCorrectEstimationsPerManeuverCount = new double[MAX_MANEUVERS];
|
||||
double[] avgConfidenceOfIncorrectEstimationsPerManeuverCount = new double[MAX_MANEUVERS];
|
||||
double[] accuracyPerManeuverCount = new double[MAX_MANEUVERS];
|
||||
double[] emptyEstimationsPercentagePerManeuverCount = new double[MAX_MANEUVERS];
|
||||
|
||||
for (int fixedNumberOfManeuvers = 1; fixedNumberOfManeuvers <= MAX_MANEUVERS; fixedNumberOfManeuvers++) {
|
||||
LoggingUtil.logInfo("Running evaluation with " + fixedNumberOfManeuvers + " maneuvers");
|
||||
WindEstimationEvaluator<CompleteManeuverCurveWithEstimationData> evaluator = new WindEstimationEvaluatorImpl<>(
|
||||
@@ -113,6 +110,7 @@ public class WindEstimatorManeuverNumberDependentEvaluationRunner {
|
||||
out.write(line);
|
||||
}
|
||||
}
|
||||
LoggingUtil.logInfo("CSV with evaluation results have been stored in: " + csvFile.getAbsolutePath());
|
||||
}
|
||||
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package com.sap.sailing.windestimation.model;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectOutputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.sap.sailing.windestimation.integration.ExportedModels;
|
||||
import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl;
|
||||
import com.sap.sailing.windestimation.model.store.ModelDomainType;
|
||||
import com.sap.sailing.windestimation.model.store.ModelStore;
|
||||
import com.sap.sailing.windestimation.util.LoggingUtil;
|
||||
|
||||
public class ExportedModelsGenerator {
|
||||
|
||||
private static final File file = new File("windEstimationModels.dat");
|
||||
|
||||
public static void main(String[] args) throws FileNotFoundException, IOException {
|
||||
ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models");
|
||||
// RegularManeuversForEstimationPersistenceManager persistenceManager = new
|
||||
// RegularManeuversForEstimationPersistenceManager();
|
||||
// ModelStore modelStore = new MongoDbModelStoreImpl(persistenceManager.getDb());
|
||||
LoggingUtil.logInfo("Generation a single file with all trained models started");
|
||||
ExportedModels exportedModels = new ExportedModels();
|
||||
List<ModelDomainType> nonEmptyDomainTypes = new ArrayList<>();
|
||||
for (ModelDomainType domainType : ModelDomainType.values()) {
|
||||
LoggingUtil.logInfo("Started serialization of models belonging to domain type " + domainType);
|
||||
Map<String, byte[]> exportedModelsForDomainType = modelStore.exportAllPersistedModels(domainType);
|
||||
if (!exportedModelsForDomainType.isEmpty()) {
|
||||
exportedModels.addSerializedModelsForDomainType(domainType, exportedModelsForDomainType);
|
||||
LoggingUtil.logInfo("Finished serialization of " + exportedModelsForDomainType.size()
|
||||
+ " models belonging to domain type " + domainType);
|
||||
nonEmptyDomainTypes.add(domainType);
|
||||
}
|
||||
}
|
||||
LoggingUtil.logInfo("Serializing the exported models and writing it into a single file");
|
||||
try (ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(file))) {
|
||||
oos.writeObject(exportedModels);
|
||||
}
|
||||
LoggingUtil.logInfo("Models have been generated");
|
||||
LoggingUtil.logInfo("The following domain types were considered: " + nonEmptyDomainTypes);
|
||||
LoggingUtil.logInfo("File with all exported models was persisted in: " + file.getAbsolutePath());
|
||||
}
|
||||
|
||||
}
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
package com.sap.sailing.windestimation.model;
|
||||
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import javax.swing.JOptionPane;
|
||||
|
||||
import com.sap.sailing.windestimation.data.importer.AggregatedDistanceBasedTwdTransitionImporter;
|
||||
import com.sap.sailing.windestimation.data.importer.AggregatedDurationBasedTwdTransitionImporter;
|
||||
import com.sap.sailing.windestimation.data.importer.DistanceBasedTwdTransitionImporter;
|
||||
import com.sap.sailing.windestimation.data.importer.DurationBasedTwdTransitionImporter;
|
||||
import com.sap.sailing.windestimation.data.importer.ManeuverAndWindImporter;
|
||||
import com.sap.sailing.windestimation.data.importer.PolarDataImporter;
|
||||
import com.sap.sailing.windestimation.data.persistence.maneuver.ManeuverForEstimationPersistenceManager;
|
||||
import com.sap.sailing.windestimation.data.persistence.maneuver.RegularManeuversForEstimationPersistenceManager;
|
||||
import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedSingleDimensionBasedTwdTransitionPersistenceManager.AggregatedSingleDimensionType;
|
||||
import com.sap.sailing.windestimation.datavisualization.AggregatedDistanceDimensionPlot;
|
||||
import com.sap.sailing.windestimation.datavisualization.AggregatedDurationDimensionPlot;
|
||||
import com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverClassifierTrainer;
|
||||
import com.sap.sailing.windestimation.model.classifier.maneuver.PersistedManeuverClassifiersScorePrinter;
|
||||
import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext;
|
||||
import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext.DistanceValueRange;
|
||||
import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext;
|
||||
import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext.DurationValueRange;
|
||||
import com.sap.sailing.windestimation.util.LoggingUtil;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Vladislav Chumak (D069712)
|
||||
*
|
||||
*/
|
||||
public class SimpleModelsTrainingPart1 {
|
||||
|
||||
private static final int NUMBER_OF_THREADS = 3;
|
||||
private static ExecutorService executorService;
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
new ManeuverForEstimationPersistenceManager().dropCollection();
|
||||
new RegularManeuversForEstimationPersistenceManager().dropCollection();
|
||||
executeInThreadPool(() -> PolarDataImporter.main(args));
|
||||
executeInThreadPool(() -> ManeuverAndWindImporter.main(args));
|
||||
awaitThreadPoolCompletion();
|
||||
executeInThreadPool(() -> {
|
||||
ManeuverClassifierTrainer.main(args);
|
||||
Thread.sleep(1000);
|
||||
PersistedManeuverClassifiersScorePrinter.main(args);
|
||||
});
|
||||
executeInThreadPool(() -> {
|
||||
DistanceBasedTwdTransitionImporter.main(args);
|
||||
});
|
||||
executeInThreadPool(() -> {
|
||||
DurationBasedTwdTransitionImporter.main(args);
|
||||
});
|
||||
awaitThreadPoolCompletion();
|
||||
AggregatedDurationBasedTwdTransitionImporter.createPersistenceManagerAndEnsureIndex();
|
||||
AggregatedDistanceBasedTwdTransitionImporter.createPersistenceManagerAndEnsureIndex();
|
||||
executeInThreadPool(() -> {
|
||||
AggregatedDurationBasedTwdTransitionImporter.main(args);
|
||||
});
|
||||
executeInThreadPool(() -> {
|
||||
AggregatedDistanceBasedTwdTransitionImporter.main(args);
|
||||
});
|
||||
awaitThreadPoolCompletion();
|
||||
do {
|
||||
AggregatedDurationDimensionPlot.main(args);
|
||||
showInfoAboutDataCleaning(AggregatedSingleDimensionType.DURATION);
|
||||
AggregatedDurationDimensionPlot.awaitWindowClosed();
|
||||
} while (JOptionPane.YES_OPTION != askDataCleaningFinished(AggregatedSingleDimensionType.DURATION));
|
||||
showInfoAboutIntervalAdjustments(DurationBasedTwdTransitionRegressorModelContext.class,
|
||||
DurationValueRange.class);
|
||||
do {
|
||||
AggregatedDistanceDimensionPlot.main(args);
|
||||
showInfoAboutDataCleaning(AggregatedSingleDimensionType.DISTANCE);
|
||||
AggregatedDistanceDimensionPlot.awaitWindowClosed();
|
||||
} while (JOptionPane.YES_OPTION != askDataCleaningFinished(AggregatedSingleDimensionType.DISTANCE));
|
||||
showInfoAboutIntervalAdjustments(DistanceBasedTwdTransitionRegressorModelContext.class,
|
||||
DistanceValueRange.class);
|
||||
showInfoAboutRunPart2();
|
||||
}
|
||||
|
||||
private static int askDataCleaningFinished(AggregatedSingleDimensionType dimension) {
|
||||
Object[] options = { "Continue with model training", "Restart Graphical Tool" };
|
||||
int res = JOptionPane.showOptionDialog(null,
|
||||
"Have you finished the data cleansing for the " + dimension + " dimension?",
|
||||
"Continue with model training?", JOptionPane.DEFAULT_OPTION, JOptionPane.INFORMATION_MESSAGE, null,
|
||||
options, options[0]);
|
||||
switch (res) {
|
||||
case 0:
|
||||
return JOptionPane.YES_OPTION;
|
||||
case 1:
|
||||
return JOptionPane.NO_OPTION;
|
||||
}
|
||||
return JOptionPane.CLOSED_OPTION;
|
||||
}
|
||||
|
||||
private static void showInfoAboutRunPart2() {
|
||||
JOptionPane.showMessageDialog(null, "Now, run the class \"" + SimpleModelsTrainingPart2.class.getName()
|
||||
+ "\".\nThis will complete the training process.");
|
||||
}
|
||||
|
||||
private static void showInfoAboutDataCleaning(AggregatedSingleDimensionType dimension) {
|
||||
JOptionPane.showMessageDialog(null, "Now, clean the data for the " + dimension
|
||||
+ " dimension.\nRemove instances from MongoDB collection \"" + dimension.getCollectioName()
|
||||
+ "\"\nwhich do not make sense and cause implausible zig zag sections\nwithin \"Zero mean sigma\" curve.\n\nClose the graphical tool, when you are done to resume the model training.");
|
||||
}
|
||||
|
||||
private static void showInfoAboutIntervalAdjustments(Class<?> classToAdjust, Class<?> valueRangeEnum) {
|
||||
JOptionPane.showMessageDialog(null, "Now, open the source code of the class \"" + classToAdjust.getName()
|
||||
+ "\".\nScroll down to the definition of the inner enum \"" + valueRangeEnum.getSimpleName()
|
||||
+ "\",\nread its JavaDoc and adjust its interval definitions so that\neach interval can be learned by the adjusted regressor model configuration\nwith minimal error.\n\nPress OK after you are done.");
|
||||
}
|
||||
|
||||
private static void awaitThreadPoolCompletion() throws InterruptedException {
|
||||
executorService.shutdown();
|
||||
boolean success = executorService.awaitTermination(24, TimeUnit.HOURS);
|
||||
if (!success) {
|
||||
new InterruptedException("Thread-pool was terminated after 24 hours waiting time");
|
||||
}
|
||||
Thread.sleep(1000L);
|
||||
}
|
||||
|
||||
private static void executeInThreadPool(RunnableWithExceptionsCatch runnable) {
|
||||
if (executorService == null || executorService.isShutdown()) {
|
||||
createNewThreadPool();
|
||||
}
|
||||
executorService.execute(() -> {
|
||||
try {
|
||||
runnable.run();
|
||||
} catch (Throwable t) {
|
||||
t.printStackTrace();
|
||||
System.exit(1);
|
||||
LoggingUtil.logInfo("FAILURE: Caught unexpected exception. Model training aborted");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private static void createNewThreadPool() {
|
||||
executorService = new ThreadPoolExecutor(NUMBER_OF_THREADS, NUMBER_OF_THREADS, 0, TimeUnit.MILLISECONDS,
|
||||
new ArrayBlockingQueue<>(NUMBER_OF_THREADS), new ThreadPoolExecutor.CallerRunsPolicy());
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Vladislav Chumak (D069712)
|
||||
*
|
||||
*/
|
||||
private static interface RunnableWithExceptionsCatch {
|
||||
void run() throws Exception;
|
||||
}
|
||||
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
package com.sap.sailing.windestimation.model;
|
||||
|
||||
import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionStdRegressorTrainer;
|
||||
import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionStdRegressorTrainer;
|
||||
import com.sap.sailing.windestimation.util.LoggingUtil;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Vladislav Chumak (D069712)
|
||||
*
|
||||
*/
|
||||
public class SimpleModelsTrainingPart2 {
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
DurationBasedTwdTransitionStdRegressorTrainer.main(args);
|
||||
DistanceBasedTwdTransitionStdRegressorTrainer.main(args);
|
||||
Thread.sleep(1000);
|
||||
ExportedModelsGenerator.main(args);
|
||||
LoggingUtil.logInfo("Model training finished. You can upload the generated file to a server instance.");
|
||||
}
|
||||
|
||||
}
|
||||
+2
-4
@@ -19,9 +19,9 @@ import com.sap.sailing.windestimation.data.persistence.maneuver.TransformedManeu
|
||||
import com.sap.sailing.windestimation.data.persistence.polars.PolarDataServiceAccessUtil;
|
||||
import com.sap.sailing.windestimation.model.classifier.LabelExtraction;
|
||||
import com.sap.sailing.windestimation.model.classifier.TrainableClassificationModel;
|
||||
import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl;
|
||||
import com.sap.sailing.windestimation.model.store.ModelDomainType;
|
||||
import com.sap.sailing.windestimation.model.store.ModelStore;
|
||||
import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl;
|
||||
import com.sap.sailing.windestimation.util.LoggingUtil;
|
||||
import com.sap.sse.common.Util.Pair;
|
||||
|
||||
@@ -66,7 +66,6 @@ public class ManeuverClassifierTrainer {
|
||||
LoggingUtil.logInfo("Training with " + trainManeuvers.size() + " maneuvers...");
|
||||
double[][] x = modelContext.getXMatrix(maneuvers);
|
||||
int[] y = labelExtraction.getYVector(maneuvers);
|
||||
classifierModel.resetTrainingStats();
|
||||
classifierModel.train(x, y);
|
||||
LoggingUtil.logInfo("Training finished. Validating on train dataset...");
|
||||
ManeuverClassifierScoring classifierScoring = new ManeuverClassifierScoring(classifierModel);
|
||||
@@ -112,8 +111,7 @@ public class ManeuverClassifierTrainer {
|
||||
public static void main(String[] args) throws Exception {
|
||||
PolarDataService polarService = PolarDataServiceAccessUtil.getPersistedPolarService();
|
||||
RegularManeuversForEstimationPersistenceManager persistenceManager = new RegularManeuversForEstimationPersistenceManager();
|
||||
ModelStore classifierModelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models");
|
||||
// ModelStore classifierModelStore = new MongoDbModelStore(persistenceManager.getDb());
|
||||
ModelStore classifierModelStore = new MongoDbModelStoreImpl(persistenceManager.getDb());
|
||||
classifierModelStore.deleteAll(ModelDomainType.MANEUVER_CLASSIFIER);
|
||||
ManeuverClassifierTrainer classifierTrainer = new ManeuverClassifierTrainer(persistenceManager,
|
||||
classifierModelStore);
|
||||
|
||||
+4
-4
@@ -13,11 +13,12 @@ import java.util.regex.Pattern;
|
||||
import com.sap.sailing.domain.base.BoatClass;
|
||||
import com.sap.sailing.domain.polars.PolarDataService;
|
||||
import com.sap.sailing.windestimation.data.ManeuverForEstimation;
|
||||
import com.sap.sailing.windestimation.data.persistence.maneuver.RegularManeuversForEstimationPersistenceManager;
|
||||
import com.sap.sailing.windestimation.data.persistence.polars.PolarDataServiceAccessUtil;
|
||||
import com.sap.sailing.windestimation.model.classifier.TrainableClassificationModel;
|
||||
import com.sap.sailing.windestimation.model.exception.ModelPersistenceException;
|
||||
import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl;
|
||||
import com.sap.sailing.windestimation.model.store.ModelStore;
|
||||
import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl;
|
||||
import com.sap.sailing.windestimation.util.LoggingUtil;
|
||||
|
||||
public class PersistedManeuverClassifiersScorePrinter {
|
||||
@@ -29,9 +30,8 @@ public class PersistedManeuverClassifiersScorePrinter {
|
||||
throws MalformedURLException, ClassNotFoundException, IOException, InterruptedException {
|
||||
PolarDataService polarService = PolarDataServiceAccessUtil.getPersistedPolarService();
|
||||
Set<BoatClass> allBoatClasses = polarService.getAllBoatClassesWithPolarSheetsAvailable();
|
||||
ModelStore classifierModelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models");
|
||||
// ModelStore classifierModelStore = new MongoDbModelStore(
|
||||
// new RegularManeuversForEstimationPersistenceManager().getDb());
|
||||
ModelStore classifierModelStore = new MongoDbModelStoreImpl(
|
||||
new RegularManeuversForEstimationPersistenceManager().getDb());
|
||||
List<TrainableClassificationModel<ManeuverForEstimation, ManeuverClassifierModelContext>> allClassifierModels = new ArrayList<>();
|
||||
ManeuverClassifierModelFactory classifierModelFactory = new ManeuverClassifierModelFactory();
|
||||
LoggingUtil.logInfo("### Loading all boat class classifiers:");
|
||||
|
||||
-1
@@ -100,7 +100,6 @@ public class TwdTransitionClassifierTrainer {
|
||||
i++;
|
||||
}
|
||||
LoggingUtil.logInfo("Training with " + numberOfTrainingInstances + " instances...");
|
||||
classifierModel.resetTrainingStats();
|
||||
classifierModel.train(x, y);
|
||||
LoggingUtil.logInfo("Training finished. Validating on train dataset...");
|
||||
TwdTransitionClassifierScoring classifierScoring = new TwdTransitionClassifierScoring(classifierModel);
|
||||
|
||||
+2
-3
@@ -5,9 +5,9 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS
|
||||
import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedSingleDimensionBasedTwdTransitionPersistenceManager.AggregatedSingleDimensionType;
|
||||
import com.sap.sailing.windestimation.model.regressor.IncrementalSingleDimensionPolynomialRegressor;
|
||||
import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext.DistanceValueRange;
|
||||
import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl;
|
||||
import com.sap.sailing.windestimation.model.store.ModelDomainType;
|
||||
import com.sap.sailing.windestimation.model.store.ModelStore;
|
||||
import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl;
|
||||
|
||||
/**
|
||||
* Trains TWD delta standard deviation by considering the distance passed between two measurements.
|
||||
@@ -26,8 +26,7 @@ public class DistanceBasedTwdTransitionStdRegressorTrainer extends TwdTransition
|
||||
public static void main(String[] args) throws Exception {
|
||||
AggregatedSingleDimensionBasedTwdTransitionPersistenceManager distanceBasedPersistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager(
|
||||
AggregatedSingleDimensionType.DISTANCE);
|
||||
ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models");
|
||||
// ModelStore modelStore = new MongoDbModelStore(distanceBasedPersistenceManager.getDb());
|
||||
ModelStore modelStore = new MongoDbModelStoreImpl(distanceBasedPersistenceManager.getDb());
|
||||
modelStore.deleteAll(ModelDomainType.DISTANCE_BASED_TWD_DELTA_STD_REGRESSOR);
|
||||
DistanceBasedTwdTransitionRegressorModelFactory distanceBasedTwdTransitionRegressorModelFactory = new DistanceBasedTwdTransitionRegressorModelFactory();
|
||||
for (DistanceValueRange distanceValueRange : DistanceValueRange.values()) {
|
||||
|
||||
+2
-3
@@ -5,9 +5,9 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS
|
||||
import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedSingleDimensionBasedTwdTransitionPersistenceManager.AggregatedSingleDimensionType;
|
||||
import com.sap.sailing.windestimation.model.regressor.IncrementalSingleDimensionPolynomialRegressor;
|
||||
import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext.DurationValueRange;
|
||||
import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl;
|
||||
import com.sap.sailing.windestimation.model.store.ModelDomainType;
|
||||
import com.sap.sailing.windestimation.model.store.ModelStore;
|
||||
import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl;
|
||||
|
||||
public class DurationBasedTwdTransitionStdRegressorTrainer extends TwdTransitionAggregatedStdRegressorTrainer {
|
||||
|
||||
@@ -20,8 +20,7 @@ public class DurationBasedTwdTransitionStdRegressorTrainer extends TwdTransition
|
||||
public static void main(String[] args) throws Exception {
|
||||
AggregatedSingleDimensionBasedTwdTransitionPersistenceManager durationBasedPersistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager(
|
||||
AggregatedSingleDimensionType.DURATION);
|
||||
ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models");
|
||||
// ModelStore modelStore = new MongoDbModelStoreImpl(durationBasedPersistenceManager.getDb());
|
||||
ModelStore modelStore = new MongoDbModelStoreImpl(durationBasedPersistenceManager.getDb());
|
||||
modelStore.deleteAll(ModelDomainType.DURATION_BASED_TWD_DELTA_STD_REGRESSOR);
|
||||
DurationBasedTwdTransitionRegressorModelFactory durationBasedTwdTransitionRegressorModelFactory = new DurationBasedTwdTransitionRegressorModelFactory();
|
||||
for (DurationValueRange durationValueRange : DurationValueRange.values()) {
|
||||
|
||||
+3
@@ -41,6 +41,9 @@ public class IncrementalSingleDimensionPolynomialRegressorTrainerHelper {
|
||||
}
|
||||
|
||||
public void incrementRmseCalculation(double[] x, double y) {
|
||||
if (!model.isModelReady()) {
|
||||
model.setModelAsReadyAfterSuccessfulTraining();
|
||||
}
|
||||
double predictedStd = model.getValue(x);
|
||||
double diff = predictedStd - y;
|
||||
squareErrorSum += diff * diff;
|
||||
|
||||
+1
-2
@@ -21,8 +21,7 @@ public class TwdTransitionManualTrainingDataInputRegressorTrainer {
|
||||
throws Exception {
|
||||
IncrementalSingleDimensionPolynomialRegressorTrainerHelper trainerHelper = new IncrementalSingleDimensionPolynomialRegressorTrainerHelper(
|
||||
regressorModelStore, model);
|
||||
LoggingUtil
|
||||
.logInfo("########## Training of " + model.getModelContext().getId() + " started...");
|
||||
LoggingUtil.logInfo("########## Training of " + model.getModelContext().getId() + " started...");
|
||||
double[] modelInput = new double[1];
|
||||
for (int i = 0; i < inputOutputPairs.length; i++) {
|
||||
double xi = inputOutputPairs[i][0];
|
||||
|
||||
Reference in new issue
Block a user