diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java index 24f3b4b47e3..a2fabf42a8b 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java @@ -13,8 +13,9 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.http.Header; @@ -73,8 +74,8 @@ public class ManeuverAndWindImporter { private final ManeuverForDataAnalysisJsonSerializer maneuverForDataAnalysisJsonSerializer; private final LabeledManeuverForEstimationJsonSerializer maneuverForEstimationJsonSerializer; private boolean skipRace; - private static final int NUMBER_OF_THREADS = 50; //high number due to HTTP requests - private final ExecutorService executorService = Executors.newFixedThreadPool(NUMBER_OF_THREADS); + private static final int NUMBER_OF_THREADS = 50; // high number due to HTTP requests + private final ThreadPoolExecutor executorService; public ManeuverAndWindImporter() throws UnknownHostException { this.completeManeuverCurvePersistanceManager = new RaceWithCompleteManeuverCurvePersistenceManager(); @@ -85,11 +86,23 @@ public class ManeuverAndWindImporter { this.maneuverForEstimationTransformer = new CompleteManeuverCurveWithEstimationDataToLabelledManeuverForEstimationTransformer(); this.maneuverForDataAnalysisJsonSerializer = new ManeuverForDataAnalysisJsonSerializer(); this.maneuverForEstimationJsonSerializer = new LabeledManeuverForEstimationJsonSerializer(); + this.executorService = new ThreadPoolExecutor(NUMBER_OF_THREADS, NUMBER_OF_THREADS, 0L, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(NUMBER_OF_THREADS)); + this.executorService.setRejectedExecutionHandler(new RejectedExecutionHandler() { + public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { + // this will block if the queue is full + try { + executor.getQueue().put(r); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + }); } public HttpClient createNewHttpClient() { HttpParams httpParams = new BasicHttpParams(); - HttpConnectionParams.setConnectionTimeout(httpParams, 30000); + HttpConnectionParams.setConnectionTimeout(httpParams, 60000 * 5); HttpClient client = new SystemDefaultHttpClient(httpParams); return client; } @@ -312,7 +325,19 @@ public class ManeuverAndWindImporter { for (int i = 1; i <= 10; i++) { try { httpResponse = createNewHttpClient().execute(getEstimationData); - break; + if (httpResponse.getStatusLine().getStatusCode() == 200) { + JSONObject resultJson; + try { + resultJson = (JSONObject) getJsonFromResponse(httpResponse); + return resultJson; + } catch (Exception e) { + System.out.println(getEstimationData); + throw e; + } + } else { + LoggingUtil.logInfo("Connection error (" + i + "/10) while querying races of regatta \"" + + trackedRegattaName + "\" (status code not 200), retrying..."); + } } catch (Exception e) { Thread.sleep(10000); lastException = e; @@ -325,17 +350,7 @@ public class ManeuverAndWindImporter { } } } - if (httpResponse == null) { - throw lastException; - } - JSONObject resultJson; - try { - resultJson = (JSONObject) getJsonFromResponse(httpResponse); - } catch (Exception e) { - System.out.println(getEstimationData); - throw e; - } - return resultJson; + throw lastException; } private void addTransformedElementsToCompetitorTrackJson(List competitorTracks, @@ -358,7 +373,6 @@ public class ManeuverAndWindImporter { } } - // FIXME duplicated code taken from ConnectivityUtils public static Object getJsonFromResponse(HttpResponse response) throws IllegalStateException, IOException, ParseException { JSONParser jsonParser = new JSONParser(); diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/evaluation/WindEstimatorManeuverNumberDependentEvaluationRunner.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/evaluation/WindEstimatorManeuverNumberDependentEvaluationRunner.java index c5f4869cf9c..7815dd9540a 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/evaluation/WindEstimatorManeuverNumberDependentEvaluationRunner.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/evaluation/WindEstimatorManeuverNumberDependentEvaluationRunner.java @@ -12,8 +12,8 @@ import com.sap.sailing.windestimation.data.persistence.maneuver.PersistedElement import com.sap.sailing.windestimation.data.persistence.maneuver.RaceWithCompleteManeuverCurvePersistenceManager; import com.sap.sailing.windestimation.data.persistence.polars.PolarDataServiceAccessUtil; import com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverFeatures; -import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl; import com.sap.sailing.windestimation.model.store.ModelStore; +import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl; import com.sap.sailing.windestimation.util.LoggingUtil; public class WindEstimatorManeuverNumberDependentEvaluationRunner { @@ -38,18 +38,15 @@ public class WindEstimatorManeuverNumberDependentEvaluationRunner { LoggingUtil.logInfo("Loading polar data"); PolarDataService polarService = PolarDataServiceAccessUtil.getPersistedPolarService(); LoggingUtil.logInfo("Wind estimator evaluation started..."); - ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models"); - // ModelStore modelStore = new MongoDbModelStoreImpl(persistenceManager.getDb()); + ModelStore modelStore = new MongoDbModelStoreImpl(persistenceManager.getDb()); WindEstimatorFactories estimatorFactories = new WindEstimatorFactories(polarService, new ManeuverFeatures(ENABLE_POLARS, ENABLE_SCALED_SPEED, ENABLE_MARKS_INFORMATION), modelStore); - double[] avgErrorDegreesPerManeuverCount = new double[MAX_MANEUVERS]; double[] avgConfidencePerManeuverCount = new double[MAX_MANEUVERS]; double[] avgConfidenceOfCorrectEstimationsPerManeuverCount = new double[MAX_MANEUVERS]; double[] avgConfidenceOfIncorrectEstimationsPerManeuverCount = new double[MAX_MANEUVERS]; double[] accuracyPerManeuverCount = new double[MAX_MANEUVERS]; double[] emptyEstimationsPercentagePerManeuverCount = new double[MAX_MANEUVERS]; - for (int fixedNumberOfManeuvers = 1; fixedNumberOfManeuvers <= MAX_MANEUVERS; fixedNumberOfManeuvers++) { LoggingUtil.logInfo("Running evaluation with " + fixedNumberOfManeuvers + " maneuvers"); WindEstimationEvaluator evaluator = new WindEstimationEvaluatorImpl<>( diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/ExportedModelsGenerator.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/ExportedModelsGenerator.java similarity index 98% rename from java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/ExportedModelsGenerator.java rename to java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/ExportedModelsGenerator.java index 9b3d0f5621f..db0ffeb05ee 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/ExportedModelsGenerator.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/ExportedModelsGenerator.java @@ -1,4 +1,4 @@ -package com.sap.sailing.windestimation; +package com.sap.sailing.windestimation.model; import java.io.File; import java.io.FileNotFoundException; diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/SimpleModelsTraining.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java similarity index 97% rename from java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/SimpleModelsTraining.java rename to java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java index c15c543327a..85be413dac7 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/SimpleModelsTraining.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java @@ -1,4 +1,4 @@ -package com.sap.sailing.windestimation; +package com.sap.sailing.windestimation.model; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -30,7 +30,8 @@ import com.sap.sailing.windestimation.util.LoggingUtil; */ public class SimpleModelsTraining { - private static final ExecutorService executorService = Executors.newFixedThreadPool(3); + private static final int NUMBER_OF_THREADS = 2; + private static final ExecutorService executorService = Executors.newFixedThreadPool(NUMBER_OF_THREADS); public static void main(String[] args) throws Exception { new ManeuverForEstimationPersistenceManager().dropCollection(); @@ -88,7 +89,7 @@ public class SimpleModelsTraining { + " dimension. Remove instances from MongoDB collection \"" + dimension.getCollectioName() + "\" which do not make sense. E.g. values which are represented by a small number of supporting instances (see histogram), values which cause implausible zig zag sections within zero-mean standard deviation curve and etc. Close the graphical tool, when you are done to resume model training."); } - + private static void awaitThreadPoolCompletion() throws InterruptedException { executorService.awaitTermination(24, TimeUnit.HOURS); Thread.sleep(1000L); diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/ManeuverClassifierTrainer.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/ManeuverClassifierTrainer.java index cdbbc86ff6e..ed62529519d 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/ManeuverClassifierTrainer.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/ManeuverClassifierTrainer.java @@ -19,9 +19,9 @@ import com.sap.sailing.windestimation.data.persistence.maneuver.TransformedManeu import com.sap.sailing.windestimation.data.persistence.polars.PolarDataServiceAccessUtil; import com.sap.sailing.windestimation.model.classifier.LabelExtraction; import com.sap.sailing.windestimation.model.classifier.TrainableClassificationModel; -import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl; import com.sap.sailing.windestimation.model.store.ModelDomainType; import com.sap.sailing.windestimation.model.store.ModelStore; +import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl; import com.sap.sailing.windestimation.util.LoggingUtil; import com.sap.sse.common.Util.Pair; @@ -112,8 +112,7 @@ public class ManeuverClassifierTrainer { public static void main(String[] args) throws Exception { PolarDataService polarService = PolarDataServiceAccessUtil.getPersistedPolarService(); RegularManeuversForEstimationPersistenceManager persistenceManager = new RegularManeuversForEstimationPersistenceManager(); - ModelStore classifierModelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models"); - // ModelStore classifierModelStore = new MongoDbModelStore(persistenceManager.getDb()); + ModelStore classifierModelStore = new MongoDbModelStoreImpl(persistenceManager.getDb()); classifierModelStore.deleteAll(ModelDomainType.MANEUVER_CLASSIFIER); ManeuverClassifierTrainer classifierTrainer = new ManeuverClassifierTrainer(persistenceManager, classifierModelStore); diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/PersistedManeuverClassifiersScorePrinter.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/PersistedManeuverClassifiersScorePrinter.java index df007fc823e..5e857143c59 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/PersistedManeuverClassifiersScorePrinter.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/PersistedManeuverClassifiersScorePrinter.java @@ -13,11 +13,12 @@ import java.util.regex.Pattern; import com.sap.sailing.domain.base.BoatClass; import com.sap.sailing.domain.polars.PolarDataService; import com.sap.sailing.windestimation.data.ManeuverForEstimation; +import com.sap.sailing.windestimation.data.persistence.maneuver.RegularManeuversForEstimationPersistenceManager; import com.sap.sailing.windestimation.data.persistence.polars.PolarDataServiceAccessUtil; import com.sap.sailing.windestimation.model.classifier.TrainableClassificationModel; import com.sap.sailing.windestimation.model.exception.ModelPersistenceException; -import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl; import com.sap.sailing.windestimation.model.store.ModelStore; +import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl; import com.sap.sailing.windestimation.util.LoggingUtil; public class PersistedManeuverClassifiersScorePrinter { @@ -29,9 +30,8 @@ public class PersistedManeuverClassifiersScorePrinter { throws MalformedURLException, ClassNotFoundException, IOException, InterruptedException { PolarDataService polarService = PolarDataServiceAccessUtil.getPersistedPolarService(); Set allBoatClasses = polarService.getAllBoatClassesWithPolarSheetsAvailable(); - ModelStore classifierModelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models"); - // ModelStore classifierModelStore = new MongoDbModelStore( - // new RegularManeuversForEstimationPersistenceManager().getDb()); + ModelStore classifierModelStore = new MongoDbModelStoreImpl( + new RegularManeuversForEstimationPersistenceManager().getDb()); List> allClassifierModels = new ArrayList<>(); ManeuverClassifierModelFactory classifierModelFactory = new ManeuverClassifierModelFactory(); LoggingUtil.logInfo("### Loading all boat class classifiers:"); diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DistanceBasedTwdTransitionStdRegressorTrainer.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DistanceBasedTwdTransitionStdRegressorTrainer.java index 9aba743456c..32bbd98b57f 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DistanceBasedTwdTransitionStdRegressorTrainer.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DistanceBasedTwdTransitionStdRegressorTrainer.java @@ -5,9 +5,9 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedSingleDimensionBasedTwdTransitionPersistenceManager.AggregatedSingleDimensionType; import com.sap.sailing.windestimation.model.regressor.IncrementalSingleDimensionPolynomialRegressor; import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext.DistanceValueRange; -import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl; import com.sap.sailing.windestimation.model.store.ModelDomainType; import com.sap.sailing.windestimation.model.store.ModelStore; +import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl; /** * Trains TWD delta standard deviation by considering the distance passed between two measurements. @@ -26,8 +26,7 @@ public class DistanceBasedTwdTransitionStdRegressorTrainer extends TwdTransition public static void main(String[] args) throws Exception { AggregatedSingleDimensionBasedTwdTransitionPersistenceManager distanceBasedPersistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager( AggregatedSingleDimensionType.DISTANCE); - ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models"); - // ModelStore modelStore = new MongoDbModelStore(distanceBasedPersistenceManager.getDb()); + ModelStore modelStore = new MongoDbModelStoreImpl(distanceBasedPersistenceManager.getDb()); modelStore.deleteAll(ModelDomainType.DISTANCE_BASED_TWD_DELTA_STD_REGRESSOR); DistanceBasedTwdTransitionRegressorModelFactory distanceBasedTwdTransitionRegressorModelFactory = new DistanceBasedTwdTransitionRegressorModelFactory(); for (DistanceValueRange distanceValueRange : DistanceValueRange.values()) { diff --git a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DurationBasedTwdTransitionStdRegressorTrainer.java b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DurationBasedTwdTransitionStdRegressorTrainer.java index febdea8a634..d092edd36b0 100644 --- a/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DurationBasedTwdTransitionStdRegressorTrainer.java +++ b/java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DurationBasedTwdTransitionStdRegressorTrainer.java @@ -5,9 +5,9 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedSingleDimensionBasedTwdTransitionPersistenceManager.AggregatedSingleDimensionType; import com.sap.sailing.windestimation.model.regressor.IncrementalSingleDimensionPolynomialRegressor; import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext.DurationValueRange; -import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl; import com.sap.sailing.windestimation.model.store.ModelDomainType; import com.sap.sailing.windestimation.model.store.ModelStore; +import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl; public class DurationBasedTwdTransitionStdRegressorTrainer extends TwdTransitionAggregatedStdRegressorTrainer { @@ -20,8 +20,7 @@ public class DurationBasedTwdTransitionStdRegressorTrainer extends TwdTransition public static void main(String[] args) throws Exception { AggregatedSingleDimensionBasedTwdTransitionPersistenceManager durationBasedPersistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager( AggregatedSingleDimensionType.DURATION); - ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models"); - // ModelStore modelStore = new MongoDbModelStoreImpl(durationBasedPersistenceManager.getDb()); + ModelStore modelStore = new MongoDbModelStoreImpl(durationBasedPersistenceManager.getDb()); modelStore.deleteAll(ModelDomainType.DURATION_BASED_TWD_DELTA_STD_REGRESSOR); DurationBasedTwdTransitionRegressorModelFactory durationBasedTwdTransitionRegressorModelFactory = new DurationBasedTwdTransitionRegressorModelFactory(); for (DurationValueRange durationValueRange : DurationValueRange.values()) {