eclipse-sailing-analytics/sailing-analytics
1
0
Fork 0
mirror of https://github.com/eclipse-sailing-analytics/sailing-analytics.git synced 2026-09-24 22:48:34 +00:00
Code Activity

Improved exception handling and thread pool config

Browse Source
This commit is contained in:
Vladislav Chumak
2019-02-18 18:48:10 +01:00
parent 9c446c9a8a
commit d6efae16dd
8 changed files with 49 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/data/importer/ManeuverAndWindImporter.java
+32 -18
View File
@@ -13,8 +13,9 @@ import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.http.Header;
@@ -73,8 +74,8 @@ public class ManeuverAndWindImporter {
private final ManeuverForDataAnalysisJsonSerializer maneuverForDataAnalysisJsonSerializer;
private final LabeledManeuverForEstimationJsonSerializer maneuverForEstimationJsonSerializer;
private boolean skipRace;
private static final int NUMBER_OF_THREADS = 50; //high number due to HTTP requests
private final ExecutorService executorService = Executors.newFixedThreadPool(NUMBER_OF_THREADS);
private static final int NUMBER_OF_THREADS = 50; // high number due to HTTP requests
private final ThreadPoolExecutor executorService;
public ManeuverAndWindImporter() throws UnknownHostException {
this.completeManeuverCurvePersistanceManager = new RaceWithCompleteManeuverCurvePersistenceManager();
@@ -85,11 +86,23 @@ public class ManeuverAndWindImporter {
this.maneuverForEstimationTransformer = new CompleteManeuverCurveWithEstimationDataToLabelledManeuverForEstimationTransformer();
this.maneuverForDataAnalysisJsonSerializer = new ManeuverForDataAnalysisJsonSerializer();
this.maneuverForEstimationJsonSerializer = new LabeledManeuverForEstimationJsonSerializer();
this.executorService = new ThreadPoolExecutor(NUMBER_OF_THREADS, NUMBER_OF_THREADS, 0L, TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<>(NUMBER_OF_THREADS));
this.executorService.setRejectedExecutionHandler(new RejectedExecutionHandler() {
public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
// this will block if the queue is full
try {
executor.getQueue().put(r);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
});
}
public HttpClient createNewHttpClient() {
HttpParams httpParams = new BasicHttpParams();
HttpConnectionParams.setConnectionTimeout(httpParams, 30000);
HttpConnectionParams.setConnectionTimeout(httpParams, 60000 * 5);
HttpClient client = new SystemDefaultHttpClient(httpParams);
return client;
}
@@ -312,7 +325,19 @@ public class ManeuverAndWindImporter {
for (int i = 1; i <= 10; i++) {
try {
httpResponse = createNewHttpClient().execute(getEstimationData);
break;
if (httpResponse.getStatusLine().getStatusCode() == 200) {
JSONObject resultJson;
try {
resultJson = (JSONObject) getJsonFromResponse(httpResponse);
return resultJson;
} catch (Exception e) {
System.out.println(getEstimationData);
throw e;
}
} else {
LoggingUtil.logInfo("Connection error (" + i + "/10) while querying races of regatta \""
+ trackedRegattaName + "\" (status code not 200), retrying...");
}
} catch (Exception e) {
Thread.sleep(10000);
lastException = e;
@@ -325,17 +350,7 @@ public class ManeuverAndWindImporter {
}
}
}
if (httpResponse == null) {
throw lastException;
}
JSONObject resultJson;
try {
resultJson = (JSONObject) getJsonFromResponse(httpResponse);
} catch (Exception e) {
System.out.println(getEstimationData);
throw e;
}
return resultJson;
throw lastException;
}
private <ToType> void addTransformedElementsToCompetitorTrackJson(List<JSONObject> competitorTracks,
@@ -358,7 +373,6 @@ public class ManeuverAndWindImporter {
}
}
// FIXME duplicated code taken from ConnectivityUtils
public static Object getJsonFromResponse(HttpResponse response)
throws IllegalStateException, IOException, ParseException {
JSONParser jsonParser = new JSONParser();
java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/evaluation/WindEstimatorManeuverNumberDependentEvaluationRunner.java
+2 -5
View File
@@ -12,8 +12,8 @@ import com.sap.sailing.windestimation.data.persistence.maneuver.PersistedElement
import com.sap.sailing.windestimation.data.persistence.maneuver.RaceWithCompleteManeuverCurvePersistenceManager;
import com.sap.sailing.windestimation.data.persistence.polars.PolarDataServiceAccessUtil;
import com.sap.sailing.windestimation.model.classifier.maneuver.ManeuverFeatures;
import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl;
import com.sap.sailing.windestimation.model.store.ModelStore;
import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl;
import com.sap.sailing.windestimation.util.LoggingUtil;
public class WindEstimatorManeuverNumberDependentEvaluationRunner {
@@ -38,18 +38,15 @@ public class WindEstimatorManeuverNumberDependentEvaluationRunner {
LoggingUtil.logInfo("Loading polar data");
PolarDataService polarService = PolarDataServiceAccessUtil.getPersistedPolarService();
LoggingUtil.logInfo("Wind estimator evaluation started...");
ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models");
// ModelStore modelStore = new MongoDbModelStoreImpl(persistenceManager.getDb());
ModelStore modelStore = new MongoDbModelStoreImpl(persistenceManager.getDb());
WindEstimatorFactories estimatorFactories = new WindEstimatorFactories(polarService,
new ManeuverFeatures(ENABLE_POLARS, ENABLE_SCALED_SPEED, ENABLE_MARKS_INFORMATION), modelStore);
double[] avgErrorDegreesPerManeuverCount = new double[MAX_MANEUVERS];
double[] avgConfidencePerManeuverCount = new double[MAX_MANEUVERS];
double[] avgConfidenceOfCorrectEstimationsPerManeuverCount = new double[MAX_MANEUVERS];
double[] avgConfidenceOfIncorrectEstimationsPerManeuverCount = new double[MAX_MANEUVERS];
double[] accuracyPerManeuverCount = new double[MAX_MANEUVERS];
double[] emptyEstimationsPercentagePerManeuverCount = new double[MAX_MANEUVERS];
for (int fixedNumberOfManeuvers = 1; fixedNumberOfManeuvers <= MAX_MANEUVERS; fixedNumberOfManeuvers++) {
LoggingUtil.logInfo("Running evaluation with " + fixedNumberOfManeuvers + " maneuvers");
WindEstimationEvaluator<CompleteManeuverCurveWithEstimationData> evaluator = new WindEstimationEvaluatorImpl<>(
java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/ExportedModelsGenerator.java → java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/ExportedModelsGenerator.java
+1 -1
View File
@@ -1,4 +1,4 @@
package com.sap.sailing.windestimation;
package com.sap.sailing.windestimation.model;
import java.io.File;
import java.io.FileNotFoundException;
java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/SimpleModelsTraining.java → java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/SimpleModelsTraining.java
+4 -3
View File
@@ -1,4 +1,4 @@
package com.sap.sailing.windestimation;
package com.sap.sailing.windestimation.model;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -30,7 +30,8 @@ import com.sap.sailing.windestimation.util.LoggingUtil;
*/
public class SimpleModelsTraining {
private static final ExecutorService executorService = Executors.newFixedThreadPool(3);
private static final int NUMBER_OF_THREADS = 2;
private static final ExecutorService executorService = Executors.newFixedThreadPool(NUMBER_OF_THREADS);
public static void main(String[] args) throws Exception {
new ManeuverForEstimationPersistenceManager().dropCollection();
@@ -88,7 +89,7 @@ public class SimpleModelsTraining {
+ " dimension. Remove instances from MongoDB collection \"" + dimension.getCollectioName()
+ "\" which do not make sense. E.g. values which are represented by a small number of supporting instances (see histogram), values which cause implausible zig zag sections within zero-mean standard deviation curve and etc. Close the graphical tool, when you are done to resume model training.");
}
private static void awaitThreadPoolCompletion() throws InterruptedException {
executorService.awaitTermination(24, TimeUnit.HOURS);
Thread.sleep(1000L);
java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/ManeuverClassifierTrainer.java
+2 -3
View File
@@ -19,9 +19,9 @@ import com.sap.sailing.windestimation.data.persistence.maneuver.TransformedManeu
import com.sap.sailing.windestimation.data.persistence.polars.PolarDataServiceAccessUtil;
import com.sap.sailing.windestimation.model.classifier.LabelExtraction;
import com.sap.sailing.windestimation.model.classifier.TrainableClassificationModel;
import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl;
import com.sap.sailing.windestimation.model.store.ModelDomainType;
import com.sap.sailing.windestimation.model.store.ModelStore;
import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl;
import com.sap.sailing.windestimation.util.LoggingUtil;
import com.sap.sse.common.Util.Pair;
@@ -112,8 +112,7 @@ public class ManeuverClassifierTrainer {
public static void main(String[] args) throws Exception {
PolarDataService polarService = PolarDataServiceAccessUtil.getPersistedPolarService();
RegularManeuversForEstimationPersistenceManager persistenceManager = new RegularManeuversForEstimationPersistenceManager();
ModelStore classifierModelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models");
// ModelStore classifierModelStore = new MongoDbModelStore(persistenceManager.getDb());
ModelStore classifierModelStore = new MongoDbModelStoreImpl(persistenceManager.getDb());
classifierModelStore.deleteAll(ModelDomainType.MANEUVER_CLASSIFIER);
ManeuverClassifierTrainer classifierTrainer = new ManeuverClassifierTrainer(persistenceManager,
classifierModelStore);
java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/classifier/maneuver/PersistedManeuverClassifiersScorePrinter.java
+4 -4
View File
@@ -13,11 +13,12 @@ import java.util.regex.Pattern;
import com.sap.sailing.domain.base.BoatClass;
import com.sap.sailing.domain.polars.PolarDataService;
import com.sap.sailing.windestimation.data.ManeuverForEstimation;
import com.sap.sailing.windestimation.data.persistence.maneuver.RegularManeuversForEstimationPersistenceManager;
import com.sap.sailing.windestimation.data.persistence.polars.PolarDataServiceAccessUtil;
import com.sap.sailing.windestimation.model.classifier.TrainableClassificationModel;
import com.sap.sailing.windestimation.model.exception.ModelPersistenceException;
import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl;
import com.sap.sailing.windestimation.model.store.ModelStore;
import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl;
import com.sap.sailing.windestimation.util.LoggingUtil;
public class PersistedManeuverClassifiersScorePrinter {
@@ -29,9 +30,8 @@ public class PersistedManeuverClassifiersScorePrinter {
throws MalformedURLException, ClassNotFoundException, IOException, InterruptedException {
PolarDataService polarService = PolarDataServiceAccessUtil.getPersistedPolarService();
Set<BoatClass> allBoatClasses = polarService.getAllBoatClassesWithPolarSheetsAvailable();
ModelStore classifierModelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models");
// ModelStore classifierModelStore = new MongoDbModelStore(
// new RegularManeuversForEstimationPersistenceManager().getDb());
ModelStore classifierModelStore = new MongoDbModelStoreImpl(
new RegularManeuversForEstimationPersistenceManager().getDb());
List<TrainableClassificationModel<ManeuverForEstimation, ManeuverClassifierModelContext>> allClassifierModels = new ArrayList<>();
ManeuverClassifierModelFactory classifierModelFactory = new ManeuverClassifierModelFactory();
LoggingUtil.logInfo("### Loading all boat class classifiers:");
java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DistanceBasedTwdTransitionStdRegressorTrainer.java
+2 -3
View File
@@ -5,9 +5,9 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS
import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedSingleDimensionBasedTwdTransitionPersistenceManager.AggregatedSingleDimensionType;
import com.sap.sailing.windestimation.model.regressor.IncrementalSingleDimensionPolynomialRegressor;
import com.sap.sailing.windestimation.model.regressor.twdtransition.DistanceBasedTwdTransitionRegressorModelContext.DistanceValueRange;
import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl;
import com.sap.sailing.windestimation.model.store.ModelDomainType;
import com.sap.sailing.windestimation.model.store.ModelStore;
import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl;
/**
* Trains TWD delta standard deviation by considering the distance passed between two measurements.
@@ -26,8 +26,7 @@ public class DistanceBasedTwdTransitionStdRegressorTrainer extends TwdTransition
public static void main(String[] args) throws Exception {
AggregatedSingleDimensionBasedTwdTransitionPersistenceManager distanceBasedPersistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager(
AggregatedSingleDimensionType.DISTANCE);
ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models");
// ModelStore modelStore = new MongoDbModelStore(distanceBasedPersistenceManager.getDb());
ModelStore modelStore = new MongoDbModelStoreImpl(distanceBasedPersistenceManager.getDb());
modelStore.deleteAll(ModelDomainType.DISTANCE_BASED_TWD_DELTA_STD_REGRESSOR);
DistanceBasedTwdTransitionRegressorModelFactory distanceBasedTwdTransitionRegressorModelFactory = new DistanceBasedTwdTransitionRegressorModelFactory();
for (DistanceValueRange distanceValueRange : DistanceValueRange.values()) {
java/com.sap.sailing.windestimation.lab/src/com/sap/sailing/windestimation/model/regressor/twdtransition/DurationBasedTwdTransitionStdRegressorTrainer.java
+2 -3
View File
@@ -5,9 +5,9 @@ import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedS
import com.sap.sailing.windestimation.data.persistence.twdtransition.AggregatedSingleDimensionBasedTwdTransitionPersistenceManager.AggregatedSingleDimensionType;
import com.sap.sailing.windestimation.model.regressor.IncrementalSingleDimensionPolynomialRegressor;
import com.sap.sailing.windestimation.model.regressor.twdtransition.DurationBasedTwdTransitionRegressorModelContext.DurationValueRange;
import com.sap.sailing.windestimation.model.store.FileSystemModelStoreImpl;
import com.sap.sailing.windestimation.model.store.ModelDomainType;
import com.sap.sailing.windestimation.model.store.ModelStore;
import com.sap.sailing.windestimation.model.store.MongoDbModelStoreImpl;
public class DurationBasedTwdTransitionStdRegressorTrainer extends TwdTransitionAggregatedStdRegressorTrainer {
@@ -20,8 +20,7 @@ public class DurationBasedTwdTransitionStdRegressorTrainer extends TwdTransition
public static void main(String[] args) throws Exception {
AggregatedSingleDimensionBasedTwdTransitionPersistenceManager durationBasedPersistenceManager = new AggregatedSingleDimensionBasedTwdTransitionPersistenceManager(
AggregatedSingleDimensionType.DURATION);
ModelStore modelStore = new FileSystemModelStoreImpl("trained_wind_estimation_models");
// ModelStore modelStore = new MongoDbModelStoreImpl(durationBasedPersistenceManager.getDb());
ModelStore modelStore = new MongoDbModelStoreImpl(durationBasedPersistenceManager.getDb());
modelStore.deleteAll(ModelDomainType.DURATION_BASED_TWD_DELTA_STD_REGRESSOR);
DurationBasedTwdTransitionRegressorModelFactory durationBasedTwdTransitionRegressorModelFactory = new DurationBasedTwdTransitionRegressorModelFactory();
for (DurationValueRange durationValueRange : DurationValueRange.values()) {