mirror of
https://github.com/eclipse-sailing-analytics/sailing-analytics.git
synced 2026-11-16 07:10:48 +00:00
fixed import with partial columns
Change-Id: I3119a46e39179c0f4067b4705b91ebd2cc947de3
This commit is contained in:
1 parent
8e23096394
commit
62e8a486f9
3 files changed
+42
-38
No files matched your search
+14
-16
@@ -21,19 +21,19 @@ import com.sap.sailing.domain.common.tracking.DoubleVectorFix;
|
||||
public enum ExpeditionExtendedSensorDataMetadata {
|
||||
HEEL("Heel", BravoExtendedSensorDataMetadata.HEEL), //
|
||||
TRIM("Trim", BravoExtendedSensorDataMetadata.PITCH), //
|
||||
LEEWAY("Leeway", BravoExtendedSensorDataMetadata.LEEWAY, /* expected in logfile */ true), //
|
||||
SET("Set", BravoExtendedSensorDataMetadata.SET, /* expected in logfile */ true), //
|
||||
DRIFT("Drift", BravoExtendedSensorDataMetadata.DRIFT, /* expected in logfile */ true), //
|
||||
DEPTH("Depth", BravoExtendedSensorDataMetadata.DEPTH, /* expected in logfile */ true), //
|
||||
RUDDER("Rudder", BravoExtendedSensorDataMetadata.RUDDER, /* expected in logfile */ true), //
|
||||
FORESTAY_LOAD("Forestay", BravoExtendedSensorDataMetadata.FORESTAY_LOAD, /* expected in logfile */ true), //
|
||||
FORESTAY_PRESSURE("ForestayPres", BravoExtendedSensorDataMetadata.FORESTAY_PRESSURE, /* expected in logfile */ true), //
|
||||
TACK_ANGLE(null, BravoExtendedSensorDataMetadata.TACK_ANGLE), //
|
||||
RAKE_DEG("RakeDeg", BravoExtendedSensorDataMetadata.RAKE_DEG, /* expected in logfile */ true), //
|
||||
DEFLECTOR_PERCENTAGE("DflectrPP", BravoExtendedSensorDataMetadata.DEFLECTOR_PERCENTAGE, /* expected in logfile */ true), //
|
||||
TARGET_HEEL("TG Heell", BravoExtendedSensorDataMetadata.TARGET_HEEL, /* expected in logfile */ true), //
|
||||
DEFLECTOR_MILLIMETERS("DflectrMM", BravoExtendedSensorDataMetadata.DEFLECTOR_MILLIMETERS, /* expected in logfile */ true), //
|
||||
TARGET_BOATSPEED_P("BspTr", BravoExtendedSensorDataMetadata.TARGET_BOATSPEED_P, /* expected in logfile */ true), //
|
||||
LEEWAY("Leeway", BravoExtendedSensorDataMetadata.LEEWAY), //
|
||||
SET("Set", BravoExtendedSensorDataMetadata.SET), //
|
||||
DRIFT("Drift", BravoExtendedSensorDataMetadata.DRIFT), //
|
||||
DEPTH("Depth", BravoExtendedSensorDataMetadata.DEPTH), //
|
||||
RUDDER("Rudder", BravoExtendedSensorDataMetadata.RUDDER), //
|
||||
FORESTAY_LOAD("Forestay", BravoExtendedSensorDataMetadata.FORESTAY_LOAD), //
|
||||
FORESTAY_PRESSURE("ForestayPres", BravoExtendedSensorDataMetadata.FORESTAY_PRESSURE), //
|
||||
TACK_ANGLE("TackAngle", BravoExtendedSensorDataMetadata.TACK_ANGLE), //
|
||||
RAKE_DEG("RakeDeg", BravoExtendedSensorDataMetadata.RAKE_DEG), //
|
||||
DEFLECTOR_PERCENTAGE("DflectrPP", BravoExtendedSensorDataMetadata.DEFLECTOR_PERCENTAGE), //
|
||||
TARGET_HEEL("TG Heell", BravoExtendedSensorDataMetadata.TARGET_HEEL), //
|
||||
DEFLECTOR_MILLIMETERS("DflectrMM", BravoExtendedSensorDataMetadata.DEFLECTOR_MILLIMETERS), //
|
||||
TARGET_BOATSPEED_P("BspTr", BravoExtendedSensorDataMetadata.TARGET_BOATSPEED_P), //
|
||||
;
|
||||
|
||||
private String columnName;
|
||||
@@ -100,9 +100,7 @@ public enum ExpeditionExtendedSensorDataMetadata {
|
||||
public static Map<String, Integer> getColumnNamesToIndexInDoubleFix() {
|
||||
final Map<String, Integer> columnNamesToIndexInDoubleFix = new HashMap<>();
|
||||
for (final ExpeditionExtendedSensorDataMetadata column : ExpeditionExtendedSensorDataMetadata.values()) {
|
||||
if (column.isExpectedInLogFile()) {
|
||||
columnNamesToIndexInDoubleFix.put(column.getColumnName(), column.getColumnIndex());
|
||||
}
|
||||
columnNamesToIndexInDoubleFix.put(column.getColumnName(), column.getColumnIndex());
|
||||
}
|
||||
return columnNamesToIndexInDoubleFix;
|
||||
}
|
||||
|
||||
+2
-2
@@ -57,7 +57,7 @@ public class ExpeditionDataImportTest {
|
||||
@Test
|
||||
public void simpleFileReadWithMissingColumns() throws FormatNotSupportedException, IOException {
|
||||
testImport(ImportData.FILE_EXPEDITION_PARTIAL);
|
||||
assertTrue(forestayValueSum/callbackCallCount > 0.7 && forestayValueSum/callbackCallCount < 7.0);
|
||||
assertTrue(forestayValueSum/callbackCallCount > 0.5 && forestayValueSum/callbackCallCount < 7.0);
|
||||
}
|
||||
|
||||
private enum ImportData implements ImportDataDefinition {
|
||||
@@ -69,7 +69,7 @@ public class ExpeditionDataImportTest {
|
||||
return getClass().getResourceAsStream("/2017Nov08_Expedition.csv");
|
||||
}
|
||||
},
|
||||
FILE_EXPEDITION_PARTIAL(198) {
|
||||
FILE_EXPEDITION_PARTIAL(199) {
|
||||
@Override
|
||||
public InputStream getInputStream() {
|
||||
return getClass().getResourceAsStream("/2018Feb10_clean_short_columns_removed.csv");
|
||||
|
||||
+26
-20
@@ -12,10 +12,8 @@ import java.util.Calendar;
|
||||
import java.util.Collections;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Set;
|
||||
import java.util.TimeZone;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
@@ -36,7 +34,6 @@ import com.sap.sailing.domain.trackfiles.TrackFileImportDeviceIdentifierImpl;
|
||||
import com.sap.sailing.domain.trackimport.DoubleVectorFixImporter;
|
||||
import com.sap.sailing.domain.trackimport.FormatNotSupportedException;
|
||||
import com.sap.sse.common.TimePoint;
|
||||
import com.sap.sse.common.Util;
|
||||
import com.sap.sse.common.impl.MillisecondsTimePoint;
|
||||
|
||||
/**
|
||||
@@ -62,7 +59,7 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
|
||||
private static final Pattern BOAT_CHECK_PATTERN = Pattern.compile("[1-9]?[0-9]");
|
||||
private final Map<String, Integer> columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix;
|
||||
/**
|
||||
* The maximum index into the double vector fix from the
|
||||
* The maximum index + 1 into the double vector fix from the
|
||||
* {@link #columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix}
|
||||
* values
|
||||
*/
|
||||
@@ -106,15 +103,15 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
|
||||
if (!line.trim().isEmpty()) {
|
||||
parseLine(lineNr.get(), filename, line, colIndices,
|
||||
(timePoint, lineContentTokens, columnsInFileFromHeader) -> {
|
||||
Double[] trackFixData = new Double[trackColumnCount];
|
||||
for (final Entry<String, Integer> columnNameToSearchForInFile : columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix
|
||||
.entrySet()) {
|
||||
Integer columnsInFileIdx = columnsInFileFromHeader
|
||||
.get(columnNameToSearchForInFile.getKey());
|
||||
trackFixData[columnNameToSearchForInFile
|
||||
.getValue()] = columnsInFileIdx >= lineContentTokens.length ? null
|
||||
: lineContentTokens[columnsInFileIdx].trim().isEmpty() ? null
|
||||
: Double.parseDouble(lineContentTokens[columnsInFileIdx]);
|
||||
final Double[] trackFixData = new Double[trackColumnCount];
|
||||
for (final Entry<String, Integer> columnFromFile : columnsInFileFromHeader.entrySet()) {
|
||||
final Double value = columnFromFile.getValue() >= lineContentTokens.length ? null
|
||||
: lineContentTokens[columnFromFile.getValue()].trim().isEmpty() ? null
|
||||
: Double.parseDouble(lineContentTokens[columnFromFile.getValue()]);
|
||||
final Integer indexInDoubleVectorFix = columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix.get(columnFromFile.getKey());
|
||||
if (indexInDoubleVectorFix != null) {
|
||||
trackFixData[indexInDoubleVectorFix] = value;
|
||||
}
|
||||
}
|
||||
importedFixes.set(true);
|
||||
callback.addFixes(
|
||||
@@ -162,13 +159,22 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
|
||||
* exception is thrown that reports the columns missing.
|
||||
*/
|
||||
private void validateHeader(Map<String, Integer> colIndicesInFile) throws FormatNotSupportedException {
|
||||
Iterable<String> requiredColumnsInFix = columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix.keySet();
|
||||
if (!Util.containsAll(colIndicesInFile.keySet(), requiredColumnsInFix)) {
|
||||
final Set<String> missingColumns = new HashSet<>();
|
||||
Util.addAll(requiredColumnsInFix, missingColumns);
|
||||
missingColumns.removeAll(colIndicesInFile.keySet());
|
||||
logger.log(Level.SEVERE, "Missing headers: " + missingColumns);
|
||||
throw new FormatNotSupportedException("Missing headers " + missingColumns + " in import files");
|
||||
final boolean dateTimeFormatOk;
|
||||
if (colIndicesInFile.containsKey(UTC_COLUMN)) {
|
||||
dateTimeFormatOk = true;
|
||||
} else if (colIndicesInFile.containsKey(DATE_COLUMN_1)
|
||||
|| colIndicesInFile.containsKey(DATE_COLUMN_2)) {
|
||||
dateTimeFormatOk = colIndicesInFile.containsKey(TIME_COLUMN);
|
||||
} else {
|
||||
// assume that "GPS Time" is present:
|
||||
dateTimeFormatOk = colIndicesInFile.containsKey(GPS_TIME_COLUMN);
|
||||
}
|
||||
if (!dateTimeFormatOk) {
|
||||
final String msg = "Missing date/time headers; expect either "+UTC_COLUMN+" or "+
|
||||
DATE_COLUMN_1+" with "+TIME_COLUMN+" or "+DATE_COLUMN_2+" with "+TIME_COLUMN+
|
||||
" or "+GPS_TIME_COLUMN;
|
||||
logger.log(Level.SEVERE, msg);
|
||||
throw new FormatNotSupportedException(msg);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user