a first version of a Bravo wind importer that seems to do useful things

Change-Id: Ib012a111e7a9ccdcbbeaa4e89a31d77d2c538646
This commit is contained in:
Axel Uhl
2017-09-29 16:35:15 +02:00
parent 48d0b0aee9
commit 2a06f5495e
14 changed files with 189 additions and 47 deletions
@@ -1,7 +1,9 @@
package com.sap.sailing.domain.common.sensordata;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
/**
@@ -51,8 +53,16 @@ public enum BravoSensorDataMetadata {
public static List<String> getTrackColumnNames() {
ArrayList<String> colNames = new ArrayList<>(getTrackColumnCount());
for (BravoSensorDataMetadata item : BravoSensorDataMetadata.values()) {
colNames.add(item.columnName);
colNames.add(item.getColumnName());
}
return colNames;
}
public static Map<String, Integer> getColumnNamesToIndexInDoubleFix() {
final Map<String, Integer> columnNamesToIndexInDoubleFix = new HashMap<>();
for (final BravoSensorDataMetadata column : BravoSensorDataMetadata.values()) {
columnNamesToIndexInDoubleFix.put(column.getColumnName(), column.getColumnIndex());
}
return columnNamesToIndexInDoubleFix;
}
}
@@ -19,7 +19,7 @@ import com.sap.sse.common.TimePoint;
public interface DoubleVectorFixImporter {
interface Callback {
void addFixes(Iterable<DoubleVectorFix> fix, TrackFileImportDeviceIdentifier device);
void addFixes(Iterable<DoubleVectorFix> fixes, TrackFileImportDeviceIdentifier device);
}
/**
@@ -32,12 +32,13 @@ public interface DoubleVectorFixImporter {
* @param sourceName
* the uploaded file's name. This can be used to identify the file type if the importer can import
* different formats.
* @param downsample TODO
* @throws FormatNotSupportedException
* if the uploaded file can't be parsed by the importer
* @throws IOException
* if there is a problem while reading the file
*/
void importFixes(InputStream inputStream, Callback callback, String filename, String sourceName)
void importFixes(InputStream inputStream, Callback callback, String filename, String sourceName, boolean downsample)
throws FormatNotSupportedException, IOException;
/**
@@ -0,0 +1,17 @@
package com.sap.sailing.server.gateway.test;
import static org.junit.Assert.assertEquals;
import org.junit.Test;
import com.sap.sailing.server.gateway.windimport.bravo.FunnyDegreeConverter;
public class FunnyDegreeConverterTest {
@Test
public void testDegreeConverter() {
assertEquals(41.410765, FunnyDegreeConverter.funnyLatLng(4124.645890), 0.000001);
assertEquals(2.228978, FunnyDegreeConverter.funnyLatLng(213.738670), 0.000001);
assertEquals(-41.410765, FunnyDegreeConverter.funnyLatLng(-4124.645890), 0.000001);
assertEquals(-2.228978, FunnyDegreeConverter.funnyLatLng(-213.738670), 0.000001);
}
}
@@ -96,7 +96,7 @@ public class SensorDataImportServlet extends AbstractFileUploadServlet {
}
}
}
}, fi.getName(), requestedImporterName);
}, fi.getName(), requestedImporterName, /* downsample */ true);
logger.log(Level.INFO, "Successfully imported file " + requestedImporterName);
} catch (FormatNotSupportedException e) {
logger.log(Level.INFO, "Failed to import file " + requestedImporterName);
@@ -30,6 +30,7 @@ import com.sap.sailing.domain.common.RegattaAndRaceIdentifier;
import com.sap.sailing.domain.common.RegattaNameAndRaceName;
import com.sap.sailing.domain.common.Wind;
import com.sap.sailing.domain.common.WindSource;
import com.sap.sailing.domain.trackimport.FormatNotSupportedException;
import com.sap.sailing.domain.tracking.DynamicTrackedRace;
import com.sap.sailing.server.gateway.SailingServerHttpServlet;
import com.sap.sailing.server.gateway.windimport.AbstractWindImportServlet.WindImportResult.RaceEntry;
@@ -201,7 +202,8 @@ public abstract class AbstractWindImportServlet extends SailingServerHttpServlet
response.getWriter().append(windImportResult.json().toJSONString());
}
protected abstract Iterable<Wind> importWind(Map<InputStream, String> streamsWithFilenames) throws IOException, InterruptedException;
protected abstract Iterable<Wind> importWind(Map<InputStream, String> streamsWithFilenames)
throws IOException, InterruptedException, FormatNotSupportedException;
protected abstract WindSource getWindSource(UploadRequest uploadRequest);
@@ -2,6 +2,7 @@ package com.sap.sailing.server.gateway.windimport.bravo;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -13,16 +14,24 @@ import java.util.zip.ZipInputStream;
import com.sap.sailing.domain.common.Wind;
import com.sap.sailing.domain.common.WindSource;
import com.sap.sailing.domain.common.WindSourceType;
import com.sap.sailing.domain.common.impl.DegreeBearingImpl;
import com.sap.sailing.domain.common.impl.DegreePosition;
import com.sap.sailing.domain.common.impl.KnotSpeedWithBearingImpl;
import com.sap.sailing.domain.common.impl.WindImpl;
import com.sap.sailing.domain.common.impl.WindSourceWithAdditionalID;
import com.sap.sailing.nmeaconnector.NmeaFactory;
import com.sap.sailing.domain.common.tracking.DoubleVectorFix;
import com.sap.sailing.domain.trackfiles.TrackFileImportDeviceIdentifier;
import com.sap.sailing.domain.trackimport.DoubleVectorFixImporter.Callback;
import com.sap.sailing.domain.trackimport.FormatNotSupportedException;
import com.sap.sailing.server.gateway.windimport.AbstractWindImportServlet;
import com.sap.sailing.server.trackfiles.impl.BravoDataImporterImpl;
import com.sap.sse.common.Util;
import com.sap.sse.common.impl.MillisecondsTimePoint;
public class BravoWindImportServlet extends AbstractWindImportServlet {
private static final long serialVersionUID = -4547876638456305135L;
private static final Logger logger = Logger.getLogger(BravoWindImportServlet.class.getName());
@Override
protected WindSource getWindSource(UploadRequest uploadRequest) {
final WindSource windSource;
@@ -38,7 +47,7 @@ public class BravoWindImportServlet extends AbstractWindImportServlet {
}
@Override
protected Iterable<Wind> importWind(Map<InputStream, String> inputStreamsAndFilenames) throws IOException, InterruptedException {
protected Iterable<Wind> importWind(Map<InputStream, String> inputStreamsAndFilenames) throws IOException, InterruptedException, FormatNotSupportedException {
final Iterable<Wind> result;
if (inputStreamsAndFilenames != null && inputStreamsAndFilenames.size() == 1) {
logger.info("Reading Bravo wind data from "+inputStreamsAndFilenames.values().iterator().next());
@@ -54,23 +63,59 @@ public class BravoWindImportServlet extends AbstractWindImportServlet {
return result;
}
private Iterable<Wind> readWind(String filename, InputStream inputStream) throws InterruptedException, IOException {
final Iterable<Wind> result;
private static enum Fields {
Lat, Lon, TWS, TWD;
}
private Iterable<Wind> readWind(String filename, InputStream inputStream) throws InterruptedException, IOException, FormatNotSupportedException {
final List<Wind> result = new LinkedList<>();
Map<String, Integer> columnsMap = new HashMap<>();
for (final Fields field : Fields.values()) {
columnsMap.put(field.name(), field.ordinal());
}
BravoDataImporterImpl importer = new BravoDataImporterImpl(columnsMap);
final Callback callback = new Callback() {
@Override
public void addFixes(Iterable<DoubleVectorFix> fixes, TrackFileImportDeviceIdentifier device) {
for (final DoubleVectorFix fix : fixes) {
// latitude / longitude are represented in funny NMEA-like way; the value divided by 100 as
// a floored integer represents the full degrees; the value modulo 100 represents the decimal
// minutes. Example: the pair (4124.645890, 213.738670) stands for N41°24.645890 E002°13.738670
final Wind wind = new WindImpl(new DegreePosition(funnyLatLng(fix.get(Fields.Lat.ordinal())),
funnyLatLng(fix.get(Fields.Lon.ordinal()))),
fix.getTimePoint(), new KnotSpeedWithBearingImpl(fix.get(Fields.TWS.ordinal()),
new DegreeBearingImpl(fix.get(Fields.TWD.ordinal())).reverse()));
result.add(wind);
}
}
};
if (filename.toLowerCase().endsWith("zip")) {
logger.info("Bravo file "+filename+" is a ZIP file");
final List<Wind> windList = new LinkedList<>();
final ZipInputStream zipInputStream = new ZipInputStream(inputStream);
ZipEntry entry;
while ((entry=zipInputStream.getNextEntry()) != null) {
if (entry.getName().toLowerCase().endsWith(".txt")) {
logger.info("Reading Bravo wind data from "+filename+"'s ZIP entry "+entry.getName());
Util.addAll(NmeaFactory.INSTANCE.readWind(zipInputStream), windList);
importer.importFixes(zipInputStream, callback, filename, filename, /* downsample */ false);
}
}
result = windList;
} else {
result = NmeaFactory.INSTANCE.readWind(inputStream);
importer.importFixes(inputStream, callback, filename, filename, /* downsample */ false);
}
return result;
}
/**
* latitude / longitude are represented in funny NMEA-like way; the value divided by 100 as a floored integer
* represents the full degrees; the value modulo 100 represents the decimal minutes. Example: the pair (4124.645890,
* 213.738670) stands for N41°24.645890 E002°13.738670, or as decimal degrees (41.410765, 2.228978)
*
* @param d double value in "funny" format
* @return double value as decimal degrees
*/
private double funnyLatLng(double d) {
final int intDeg = (int) (d / 100.);
final double minutes = d - intDeg;
return ((double) intDeg) + minutes/60.;
}
}
@@ -0,0 +1,25 @@
package com.sap.sailing.server.gateway.windimport.bravo;
/**
* Latitude / longitude in Bravo files are represented in funny NMEA-like way; the value divided by 100 as a floored
* integer represents the full degrees; the value modulo 100 represents the decimal minutes. Example: the pair
* (4124.645890, 213.738670) stands for N41°24.645890 E002°13.738670, or as decimal degrees (41.410765, 2.228978)
*
* @author Axel Uhl (d043530)
*
*/
public class FunnyDegreeConverter {
/**
* Latitude / longitude in Bravo files are represented in funny NMEA-like way; the value divided by 100 as a floored integer
* represents the full degrees; the value modulo 100 represents the decimal minutes. Example: the pair (4124.645890,
* 213.738670) stands for N41°24.645890 E002°13.738670, or as decimal degrees (41.410765, 2.228978)
*
* @param d double value in "funny" format
* @return double value as decimal degrees
*/
public static double funnyLatLng(double d) {
final int intDeg = (int) (d / 100.);
final double minutes = d - 100*intDeg;
return ((double) intDeg) + minutes/60.;
}
}
@@ -24,8 +24,8 @@ public class BravoDataImportTest {
private LearningBatchProcessor batchProcessor;
private DownsamplerTo1HzProcessor downsampler;
private final DoubleVectorFixImporter bravoDataImporter = new BravoDataImporterImpl() {
protected com.sap.sailing.server.trackfiles.impl.doublefix.DoubleFixProcessor createProcessor(
private final DoubleVectorFixImporter bravoDataImporter = new BravoDataImporterImpl(BravoSensorDataMetadata.getColumnNamesToIndexInDoubleFix()) {
protected com.sap.sailing.server.trackfiles.impl.doublefix.DoubleFixProcessor createDownsamplingProcessor(
DoubleVectorFixImporter.Callback callback,
TrackFileImportDeviceIdentifier trackIdentifier) {
batchProcessor = new LearningBatchProcessor(5000, 5000, callback, trackIdentifier);
@@ -73,7 +73,7 @@ public class BravoDataImportTest {
callbackCallCount++;
sumRideHeightInMeters += new BravoFixImpl(fix).getRideHeight().getMeters();
}
}, "filename", "source");
}, "filename", "source", /* downsample */ true);
Assert.assertEquals(importData.expectedFixesCount, downsampler.getCountSourceTtl());
Assert.assertEquals(importData.expectedFixesConsolidated, downsampler.getCountImportedTtl());
Assert.assertEquals(importData.expectedFixesConsolidated, callbackCallCount);
@@ -7,7 +7,8 @@ Bundle-Vendor: SAP
Bundle-RequiredExecutionEnvironment: JavaSE-1.8
Bundle-ActivationPolicy: lazy
Export-Package: com.sap.sailing.server.trackfiles,
com.sap.sailing.server.trackfiles.common
com.sap.sailing.server.trackfiles.common,
com.sap.sailing.server.trackfiles.impl
Require-Bundle:
com.sap.sailing.domain,
com.sap.sailing.domain.common,
@@ -7,6 +7,7 @@ import org.osgi.framework.BundleActivator;
import org.osgi.framework.BundleContext;
import org.osgi.framework.ServiceRegistration;
import com.sap.sailing.domain.common.sensordata.BravoSensorDataMetadata;
import com.sap.sailing.server.trackfiles.common.GPSFixImporterRegistration;
import com.sap.sailing.server.trackfiles.common.SensorDataImporterRegistration;
@@ -16,7 +17,8 @@ public class Activator implements BundleActivator {
@Override
public void start(BundleContext context) throws Exception {
registrations.addAll(GPSFixImporterRegistration.register(new RouteConverterGPSFixImporterImpl(), context));
registrations.addAll(SensorDataImporterRegistration.register(new BravoDataImporterImpl(), context));
registrations.addAll(SensorDataImporterRegistration.register(
new BravoDataImporterImpl(BravoSensorDataMetadata.getColumnNamesToIndexInDoubleFix()), context));
}
@Override
@@ -5,10 +5,11 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Serializable;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
@@ -21,6 +22,7 @@ import com.sap.sailing.domain.abstractlog.regatta.events.RegattaLogDeviceCompeti
import com.sap.sailing.domain.abstractlog.regatta.events.impl.RegattaLogDeviceCompetitorBravoMappingEventImpl;
import com.sap.sailing.domain.base.Competitor;
import com.sap.sailing.domain.common.sensordata.BravoSensorDataMetadata;
import com.sap.sailing.domain.common.tracking.impl.DoubleVectorFixImpl;
import com.sap.sailing.domain.racelogtracking.DeviceIdentifier;
import com.sap.sailing.domain.trackfiles.TrackFileImportDeviceIdentifier;
import com.sap.sailing.domain.trackfiles.TrackFileImportDeviceIdentifierImpl;
@@ -31,6 +33,7 @@ import com.sap.sailing.server.trackfiles.impl.doublefix.DoubleVectorFixData;
import com.sap.sailing.server.trackfiles.impl.doublefix.DownsamplerTo1HzProcessor;
import com.sap.sailing.server.trackfiles.impl.doublefix.LearningBatchProcessor;
import com.sap.sse.common.TimePoint;
import com.sap.sse.common.Util;
import com.sap.sse.common.impl.MillisecondsTimePoint;
/**
@@ -40,9 +43,18 @@ import com.sap.sse.common.impl.MillisecondsTimePoint;
public class BravoDataImporterImpl implements DoubleVectorFixImporter {
private final Logger LOG = Logger.getLogger(DoubleVectorFixImporter.class.getName());
private final String BOF = "jjlDATE\tjjlTIME\tEpoch";
private int trackColumnCount = BravoSensorDataMetadata.getTrackColumnCount();
private final Map<String, Integer> columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix;
public void importFixes(InputStream inputStream, Callback callback, final String filename, String sourceName)
public BravoDataImporterImpl(Map<String, Integer> columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix) {
this.columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix = columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix;
}
/**
* @param downsample
* if {@code true}, fixes will be down-sampled to a 1Hz frequency before being emitted to the
* {@code callback}. Otherwise, all fixes read will be forwarded straight to the {@link Callback}.
*/
public void importFixes(InputStream inputStream, Callback callback, final String filename, String sourceName, boolean downsample)
throws FormatNotSupportedException, IOException {
final TrackFileImportDeviceIdentifier trackIdentifier = new TrackFileImportDeviceIdentifierImpl(
UUID.randomUUID(), filename, sourceName, MillisecondsTimePoint.now());
@@ -74,12 +86,13 @@ public class BravoDataImporterImpl implements DoubleVectorFixImporter {
}
LOG.fine("Validate and parse header columns");
final Map<String, Integer> colIndices = validateAndParseHeader(headerLine);
DoubleFixProcessor downsampler = createProcessor(callback, trackIdentifier);
DoubleFixProcessor downsampler = downsample ?
createDownsamplingProcessor(callback, trackIdentifier) :
fix->callback.addFixes(Collections.singleton(new DoubleVectorFixImpl(fix.getTimepoint(), fix.getFix())), trackIdentifier);
buffer.lines().forEach(line -> {
lineNr.incrementAndGet();
downsampler.accept(parseLine(lineNr.get(), filename, line, colIndices));
});
downsampler.finish();
buffer.close();
}
@@ -93,43 +106,37 @@ public class BravoDataImporterImpl implements DoubleVectorFixImporter {
* importer.
*
* This method is protected so it can be overridden by the test case.
*
* @param metadata
* @param callback
* @param trackIdentifier
* @return
*/
protected DoubleFixProcessor createProcessor(Callback callback,
protected DoubleFixProcessor createDownsamplingProcessor(Callback callback,
final TrackFileImportDeviceIdentifier trackIdentifier) {
LearningBatchProcessor batchProcessor = new LearningBatchProcessor(5000, 5000, callback, trackIdentifier);
DoubleFixProcessor downsampler = new DownsamplerTo1HzProcessor(trackColumnCount,
DoubleFixProcessor downsampler = new DownsamplerTo1HzProcessor(getTrackColumnCount(),
batchProcessor);
return downsampler;
}
/**
* Parses the CSV line and reads the double data values in the order defined by the col enums.
*/
private DoubleVectorFixData parseLine(long lineNr, String filename, String line, Map<String, Integer> columnsInFile) {
private DoubleVectorFixData parseLine(long lineNr, String filename, String line, Map<String, Integer> columnsInFileFromHeader) {
try {
final DoubleVectorFixData result;
String[] fileContentTokens = split(line);
String epochColValue = fileContentTokens[2];
long epoch;
if (epochColValue != null && epochColValue.length() > 0) {
epochColValue = epochColValue.substring(0, epochColValue.indexOf("."));
epoch = Long.parseLong(epochColValue);
double[] trackFixData = new double[getTrackColumnCount()];
for (final Entry<String, Integer> columnNameToSearchForInFile : columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix.entrySet()) {
Integer columnsInFileIdx = columnsInFileFromHeader.get(columnNameToSearchForInFile.getKey());
trackFixData[columnNameToSearchForInFile.getValue()] = Double.parseDouble(fileContentTokens[columnsInFileIdx]);
}
result = new DoubleVectorFixData(epoch, trackFixData);
} else {
// we don't have epoch, skip the line
return null;
result = null;
}
double[] trackFixData = new double[trackColumnCount];
for (int trackColumnIdx = 0; trackColumnIdx < trackColumnCount; trackColumnIdx++) {
String columnNameToSearchForInFile = BravoSensorDataMetadata.values()[trackColumnIdx].getColumnName();
Integer columnsInFileIdx = columnsInFile.get(columnNameToSearchForInFile);
trackFixData[trackColumnIdx] = Double.parseDouble(fileContentTokens[columnsInFileIdx]);
}
return new DoubleVectorFixData(epoch, trackFixData);
return result;
} catch (Exception e) {
LOG.warning(
"Error parsing line nr " + lineNr + " in file " + filename + "with exception: " + e.getMessage());
@@ -144,9 +151,10 @@ public class BravoDataImporterImpl implements DoubleVectorFixImporter {
String header = headerTokens[j];
colIndicesInFile.put(header, j);
}
List<String> requiredColumnsInFix = BravoSensorDataMetadata.getTrackColumnNames();
if (!colIndicesInFile.keySet().containsAll(requiredColumnsInFix)) {
final Set<String> missingColumns = new HashSet<>(requiredColumnsInFix);
Iterable<String> requiredColumnsInFix = columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix.keySet();
if (!Util.containsAll(colIndicesInFile.keySet(), requiredColumnsInFix)) {
final Set<String> missingColumns = new HashSet<>();
Util.addAll(requiredColumnsInFix, missingColumns);
missingColumns.removeAll(colIndicesInFile.keySet());
LOG.log(Level.SEVERE, "Missing headers: "+missingColumns);
throw new RuntimeException("Missing headers "+missingColumns+" in import files");
@@ -170,4 +178,8 @@ public class BravoDataImporterImpl implements DoubleVectorFixImporter {
return new RegattaLogDeviceCompetitorBravoMappingEventImpl(createdAt, logicalTimePoint, author, id, mappedTo,
device, from, to);
}
private int getTrackColumnCount() {
return columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix.size();
}
}
@@ -1,7 +1,7 @@
package com.sap.sailing.server.trackfiles.impl.doublefix;
@FunctionalInterface
public interface DoubleFixProcessor {
void accept(DoubleVectorFixData fix);
void finish();
default void finish() {}
}
@@ -1,5 +1,8 @@
package com.sap.sailing.server.trackfiles.impl.doublefix;
import com.sap.sse.common.TimePoint;
import com.sap.sse.common.impl.MillisecondsTimePoint;
public final class DoubleVectorFixData {
private long timepointInMs;
private double[] fix;
@@ -25,4 +28,8 @@ public final class DoubleVectorFixData {
public double[] getFix() {
return fix;
}
public TimePoint getTimepoint() {
return new MillisecondsTimePoint(timepointInMs);
}
}
@@ -349,6 +349,26 @@ public class Util {
}
}
/**
* Checks whether for all elements from {@code what} the method {@link #contains(Iterable, Object)}
* returns {@code true}. In case {@code what} is empty or {@code null}, {@code true} is returned if and only
* if {@code ts} is not {@code null}.
*/
public static <T> boolean containsAll(Iterable<T> ts, Iterable<T> what) {
if (ts == null) {
return false;
}
if (what == null) {
return true;
}
for (final T w : what) {
if (!contains(ts, w)) {
return false;
}
}
return true;
}
public static <T> boolean isEmpty(Iterable<T> ts) {
if (ts instanceof Collection<?>) {
return ((Collection<?>) ts).isEmpty();