fixed import with partial columns

Change-Id: I3119a46e39179c0f4067b4705b91ebd2cc947de3
This commit is contained in:
Axel Uhl committed 2018-03-08 11:12:16 +01:00
1 parent 8e23096394
commit 62e8a486f9
3 files changed
+42 -38

No files matched your search

@@ -21,19 +21,19 @@ import com.sap.sailing.domain.common.tracking.DoubleVectorFix;
public enum ExpeditionExtendedSensorDataMetadata {
HEEL("Heel", BravoExtendedSensorDataMetadata.HEEL), //
TRIM("Trim", BravoExtendedSensorDataMetadata.PITCH), //
LEEWAY("Leeway", BravoExtendedSensorDataMetadata.LEEWAY, /* expected in logfile */ true), //
SET("Set", BravoExtendedSensorDataMetadata.SET, /* expected in logfile */ true), //
DRIFT("Drift", BravoExtendedSensorDataMetadata.DRIFT, /* expected in logfile */ true), //
DEPTH("Depth", BravoExtendedSensorDataMetadata.DEPTH, /* expected in logfile */ true), //
RUDDER("Rudder", BravoExtendedSensorDataMetadata.RUDDER, /* expected in logfile */ true), //
FORESTAY_LOAD("Forestay", BravoExtendedSensorDataMetadata.FORESTAY_LOAD, /* expected in logfile */ true), //
FORESTAY_PRESSURE("ForestayPres", BravoExtendedSensorDataMetadata.FORESTAY_PRESSURE, /* expected in logfile */ true), //
TACK_ANGLE(null, BravoExtendedSensorDataMetadata.TACK_ANGLE), //
RAKE_DEG("RakeDeg", BravoExtendedSensorDataMetadata.RAKE_DEG, /* expected in logfile */ true), //
DEFLECTOR_PERCENTAGE("DflectrPP", BravoExtendedSensorDataMetadata.DEFLECTOR_PERCENTAGE, /* expected in logfile */ true), //
TARGET_HEEL("TG Heell", BravoExtendedSensorDataMetadata.TARGET_HEEL, /* expected in logfile */ true), //
DEFLECTOR_MILLIMETERS("DflectrMM", BravoExtendedSensorDataMetadata.DEFLECTOR_MILLIMETERS, /* expected in logfile */ true), //
TARGET_BOATSPEED_P("BspTr", BravoExtendedSensorDataMetadata.TARGET_BOATSPEED_P, /* expected in logfile */ true), //
LEEWAY("Leeway", BravoExtendedSensorDataMetadata.LEEWAY), //
SET("Set", BravoExtendedSensorDataMetadata.SET), //
DRIFT("Drift", BravoExtendedSensorDataMetadata.DRIFT), //
DEPTH("Depth", BravoExtendedSensorDataMetadata.DEPTH), //
RUDDER("Rudder", BravoExtendedSensorDataMetadata.RUDDER), //
FORESTAY_LOAD("Forestay", BravoExtendedSensorDataMetadata.FORESTAY_LOAD), //
FORESTAY_PRESSURE("ForestayPres", BravoExtendedSensorDataMetadata.FORESTAY_PRESSURE), //
TACK_ANGLE("TackAngle", BravoExtendedSensorDataMetadata.TACK_ANGLE), //
RAKE_DEG("RakeDeg", BravoExtendedSensorDataMetadata.RAKE_DEG), //
DEFLECTOR_PERCENTAGE("DflectrPP", BravoExtendedSensorDataMetadata.DEFLECTOR_PERCENTAGE), //
TARGET_HEEL("TG Heell", BravoExtendedSensorDataMetadata.TARGET_HEEL), //
DEFLECTOR_MILLIMETERS("DflectrMM", BravoExtendedSensorDataMetadata.DEFLECTOR_MILLIMETERS), //
TARGET_BOATSPEED_P("BspTr", BravoExtendedSensorDataMetadata.TARGET_BOATSPEED_P), //
;
private String columnName;
@@ -100,9 +100,7 @@ public enum ExpeditionExtendedSensorDataMetadata {
public static Map<String, Integer> getColumnNamesToIndexInDoubleFix() {
final Map<String, Integer> columnNamesToIndexInDoubleFix = new HashMap<>();
for (final ExpeditionExtendedSensorDataMetadata column : ExpeditionExtendedSensorDataMetadata.values()) {
if (column.isExpectedInLogFile()) {
columnNamesToIndexInDoubleFix.put(column.getColumnName(), column.getColumnIndex());
}
columnNamesToIndexInDoubleFix.put(column.getColumnName(), column.getColumnIndex());
}
return columnNamesToIndexInDoubleFix;
}
@@ -57,7 +57,7 @@ public class ExpeditionDataImportTest {
@Test
public void simpleFileReadWithMissingColumns() throws FormatNotSupportedException, IOException {
testImport(ImportData.FILE_EXPEDITION_PARTIAL);
assertTrue(forestayValueSum/callbackCallCount > 0.7 && forestayValueSum/callbackCallCount < 7.0);
assertTrue(forestayValueSum/callbackCallCount > 0.5 && forestayValueSum/callbackCallCount < 7.0);
}
private enum ImportData implements ImportDataDefinition {
@@ -69,7 +69,7 @@ public class ExpeditionDataImportTest {
return getClass().getResourceAsStream("/2017Nov08_Expedition.csv");
}
},
FILE_EXPEDITION_PARTIAL(198) {
FILE_EXPEDITION_PARTIAL(199) {
@Override
public InputStream getInputStream() {
return getClass().getResourceAsStream("/2018Feb10_clean_short_columns_removed.csv");
@@ -12,10 +12,8 @@ import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TimeZone;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -36,7 +34,6 @@ import com.sap.sailing.domain.trackfiles.TrackFileImportDeviceIdentifierImpl;
import com.sap.sailing.domain.trackimport.DoubleVectorFixImporter;
import com.sap.sailing.domain.trackimport.FormatNotSupportedException;
import com.sap.sse.common.TimePoint;
import com.sap.sse.common.Util;
import com.sap.sse.common.impl.MillisecondsTimePoint;
/**
@@ -62,7 +59,7 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
private static final Pattern BOAT_CHECK_PATTERN = Pattern.compile("[1-9]?[0-9]");
private final Map<String, Integer> columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix;
/**
* The maximum index into the double vector fix from the
* The maximum index + 1 into the double vector fix from the
* {@link #columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix}
* values
*/
@@ -106,15 +103,15 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
if (!line.trim().isEmpty()) {
parseLine(lineNr.get(), filename, line, colIndices,
(timePoint, lineContentTokens, columnsInFileFromHeader) -> {
Double[] trackFixData = new Double[trackColumnCount];
for (final Entry<String, Integer> columnNameToSearchForInFile : columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix
.entrySet()) {
Integer columnsInFileIdx = columnsInFileFromHeader
.get(columnNameToSearchForInFile.getKey());
trackFixData[columnNameToSearchForInFile
.getValue()] = columnsInFileIdx >= lineContentTokens.length ? null
: lineContentTokens[columnsInFileIdx].trim().isEmpty() ? null
: Double.parseDouble(lineContentTokens[columnsInFileIdx]);
final Double[] trackFixData = new Double[trackColumnCount];
for (final Entry<String, Integer> columnFromFile : columnsInFileFromHeader.entrySet()) {
final Double value = columnFromFile.getValue() >= lineContentTokens.length ? null
: lineContentTokens[columnFromFile.getValue()].trim().isEmpty() ? null
: Double.parseDouble(lineContentTokens[columnFromFile.getValue()]);
final Integer indexInDoubleVectorFix = columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix.get(columnFromFile.getKey());
if (indexInDoubleVectorFix != null) {
trackFixData[indexInDoubleVectorFix] = value;
}
}
importedFixes.set(true);
callback.addFixes(
@@ -162,13 +159,22 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
* exception is thrown that reports the columns missing.
*/
private void validateHeader(Map<String, Integer> colIndicesInFile) throws FormatNotSupportedException {
Iterable<String> requiredColumnsInFix = columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix.keySet();
if (!Util.containsAll(colIndicesInFile.keySet(), requiredColumnsInFix)) {
final Set<String> missingColumns = new HashSet<>();
Util.addAll(requiredColumnsInFix, missingColumns);
missingColumns.removeAll(colIndicesInFile.keySet());
logger.log(Level.SEVERE, "Missing headers: " + missingColumns);
throw new FormatNotSupportedException("Missing headers " + missingColumns + " in import files");
final boolean dateTimeFormatOk;
if (colIndicesInFile.containsKey(UTC_COLUMN)) {
dateTimeFormatOk = true;
} else if (colIndicesInFile.containsKey(DATE_COLUMN_1)
|| colIndicesInFile.containsKey(DATE_COLUMN_2)) {
dateTimeFormatOk = colIndicesInFile.containsKey(TIME_COLUMN);
} else {
// assume that "GPS Time" is present:
dateTimeFormatOk = colIndicesInFile.containsKey(GPS_TIME_COLUMN);
}
if (!dateTimeFormatOk) {
final String msg = "Missing date/time headers; expect either "+UTC_COLUMN+" or "+
DATE_COLUMN_1+" with "+TIME_COLUMN+" or "+DATE_COLUMN_2+" with "+TIME_COLUMN+
" or "+GPS_TIME_COLUMN;
logger.log(Level.SEVERE, msg);
throw new FormatNotSupportedException(msg);
}
}