diff --git a/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/sensordata/ExpeditionExtendedSensorDataMetadata.java b/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/sensordata/ExpeditionExtendedSensorDataMetadata.java index 3858832e652..a68dbc1d7b2 100755 --- a/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/sensordata/ExpeditionExtendedSensorDataMetadata.java +++ b/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/sensordata/ExpeditionExtendedSensorDataMetadata.java @@ -21,19 +21,19 @@ import com.sap.sailing.domain.common.tracking.DoubleVectorFix; public enum ExpeditionExtendedSensorDataMetadata { HEEL("Heel", BravoExtendedSensorDataMetadata.HEEL), // TRIM("Trim", BravoExtendedSensorDataMetadata.PITCH), // - LEEWAY("Leeway", BravoExtendedSensorDataMetadata.LEEWAY, /* expected in logfile */ true), // - SET("Set", BravoExtendedSensorDataMetadata.SET, /* expected in logfile */ true), // - DRIFT("Drift", BravoExtendedSensorDataMetadata.DRIFT, /* expected in logfile */ true), // - DEPTH("Depth", BravoExtendedSensorDataMetadata.DEPTH, /* expected in logfile */ true), // - RUDDER("Rudder", BravoExtendedSensorDataMetadata.RUDDER, /* expected in logfile */ true), // - FORESTAY_LOAD("Forestay", BravoExtendedSensorDataMetadata.FORESTAY_LOAD, /* expected in logfile */ true), // - FORESTAY_PRESSURE("ForestayPres", BravoExtendedSensorDataMetadata.FORESTAY_PRESSURE, /* expected in logfile */ true), // - TACK_ANGLE(null, BravoExtendedSensorDataMetadata.TACK_ANGLE), // - RAKE_DEG("RakeDeg", BravoExtendedSensorDataMetadata.RAKE_DEG, /* expected in logfile */ true), // - DEFLECTOR_PERCENTAGE("DflectrPP", BravoExtendedSensorDataMetadata.DEFLECTOR_PERCENTAGE, /* expected in logfile */ true), // - TARGET_HEEL("TG Heell", BravoExtendedSensorDataMetadata.TARGET_HEEL, /* expected in logfile */ true), // - DEFLECTOR_MILLIMETERS("DflectrMM", BravoExtendedSensorDataMetadata.DEFLECTOR_MILLIMETERS, /* expected in logfile */ true), // - TARGET_BOATSPEED_P("BspTr", BravoExtendedSensorDataMetadata.TARGET_BOATSPEED_P, /* expected in logfile */ true), // + LEEWAY("Leeway", BravoExtendedSensorDataMetadata.LEEWAY), // + SET("Set", BravoExtendedSensorDataMetadata.SET), // + DRIFT("Drift", BravoExtendedSensorDataMetadata.DRIFT), // + DEPTH("Depth", BravoExtendedSensorDataMetadata.DEPTH), // + RUDDER("Rudder", BravoExtendedSensorDataMetadata.RUDDER), // + FORESTAY_LOAD("Forestay", BravoExtendedSensorDataMetadata.FORESTAY_LOAD), // + FORESTAY_PRESSURE("ForestayPres", BravoExtendedSensorDataMetadata.FORESTAY_PRESSURE), // + TACK_ANGLE("TackAngle", BravoExtendedSensorDataMetadata.TACK_ANGLE), // + RAKE_DEG("RakeDeg", BravoExtendedSensorDataMetadata.RAKE_DEG), // + DEFLECTOR_PERCENTAGE("DflectrPP", BravoExtendedSensorDataMetadata.DEFLECTOR_PERCENTAGE), // + TARGET_HEEL("TG Heell", BravoExtendedSensorDataMetadata.TARGET_HEEL), // + DEFLECTOR_MILLIMETERS("DflectrMM", BravoExtendedSensorDataMetadata.DEFLECTOR_MILLIMETERS), // + TARGET_BOATSPEED_P("BspTr", BravoExtendedSensorDataMetadata.TARGET_BOATSPEED_P), // ; private String columnName; @@ -100,9 +100,7 @@ public enum ExpeditionExtendedSensorDataMetadata { public static Map getColumnNamesToIndexInDoubleFix() { final Map columnNamesToIndexInDoubleFix = new HashMap<>(); for (final ExpeditionExtendedSensorDataMetadata column : ExpeditionExtendedSensorDataMetadata.values()) { - if (column.isExpectedInLogFile()) { - columnNamesToIndexInDoubleFix.put(column.getColumnName(), column.getColumnIndex()); - } + columnNamesToIndexInDoubleFix.put(column.getColumnName(), column.getColumnIndex()); } return columnNamesToIndexInDoubleFix; } diff --git a/java/com.sap.sailing.server.trackfiles.test/src/com/sap/sailing/server/trackfiles/test/ExpeditionDataImportTest.java b/java/com.sap.sailing.server.trackfiles.test/src/com/sap/sailing/server/trackfiles/test/ExpeditionDataImportTest.java index 3be189db678..25d22aff7b6 100755 --- a/java/com.sap.sailing.server.trackfiles.test/src/com/sap/sailing/server/trackfiles/test/ExpeditionDataImportTest.java +++ b/java/com.sap.sailing.server.trackfiles.test/src/com/sap/sailing/server/trackfiles/test/ExpeditionDataImportTest.java @@ -57,7 +57,7 @@ public class ExpeditionDataImportTest { @Test public void simpleFileReadWithMissingColumns() throws FormatNotSupportedException, IOException { testImport(ImportData.FILE_EXPEDITION_PARTIAL); - assertTrue(forestayValueSum/callbackCallCount > 0.7 && forestayValueSum/callbackCallCount < 7.0); + assertTrue(forestayValueSum/callbackCallCount > 0.5 && forestayValueSum/callbackCallCount < 7.0); } private enum ImportData implements ImportDataDefinition { @@ -69,7 +69,7 @@ public class ExpeditionDataImportTest { return getClass().getResourceAsStream("/2017Nov08_Expedition.csv"); } }, - FILE_EXPEDITION_PARTIAL(198) { + FILE_EXPEDITION_PARTIAL(199) { @Override public InputStream getInputStream() { return getClass().getResourceAsStream("/2018Feb10_clean_short_columns_removed.csv"); diff --git a/java/com.sap.sailing.server.trackfiles/src/com/sap/sailing/server/trackfiles/impl/ExpeditionExtendedDataImporterImpl.java b/java/com.sap.sailing.server.trackfiles/src/com/sap/sailing/server/trackfiles/impl/ExpeditionExtendedDataImporterImpl.java index bcce0d5f766..346d90cb20d 100755 --- a/java/com.sap.sailing.server.trackfiles/src/com/sap/sailing/server/trackfiles/impl/ExpeditionExtendedDataImporterImpl.java +++ b/java/com.sap.sailing.server.trackfiles/src/com/sap/sailing/server/trackfiles/impl/ExpeditionExtendedDataImporterImpl.java @@ -12,10 +12,8 @@ import java.util.Calendar; import java.util.Collections; import java.util.Date; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.Map.Entry; -import java.util.Set; import java.util.TimeZone; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; @@ -36,7 +34,6 @@ import com.sap.sailing.domain.trackfiles.TrackFileImportDeviceIdentifierImpl; import com.sap.sailing.domain.trackimport.DoubleVectorFixImporter; import com.sap.sailing.domain.trackimport.FormatNotSupportedException; import com.sap.sse.common.TimePoint; -import com.sap.sse.common.Util; import com.sap.sse.common.impl.MillisecondsTimePoint; /** @@ -62,7 +59,7 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI private static final Pattern BOAT_CHECK_PATTERN = Pattern.compile("[1-9]?[0-9]"); private final Map columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix; /** - * The maximum index into the double vector fix from the + * The maximum index + 1 into the double vector fix from the * {@link #columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix} * values */ @@ -106,15 +103,15 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI if (!line.trim().isEmpty()) { parseLine(lineNr.get(), filename, line, colIndices, (timePoint, lineContentTokens, columnsInFileFromHeader) -> { - Double[] trackFixData = new Double[trackColumnCount]; - for (final Entry columnNameToSearchForInFile : columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix - .entrySet()) { - Integer columnsInFileIdx = columnsInFileFromHeader - .get(columnNameToSearchForInFile.getKey()); - trackFixData[columnNameToSearchForInFile - .getValue()] = columnsInFileIdx >= lineContentTokens.length ? null - : lineContentTokens[columnsInFileIdx].trim().isEmpty() ? null - : Double.parseDouble(lineContentTokens[columnsInFileIdx]); + final Double[] trackFixData = new Double[trackColumnCount]; + for (final Entry columnFromFile : columnsInFileFromHeader.entrySet()) { + final Double value = columnFromFile.getValue() >= lineContentTokens.length ? null + : lineContentTokens[columnFromFile.getValue()].trim().isEmpty() ? null + : Double.parseDouble(lineContentTokens[columnFromFile.getValue()]); + final Integer indexInDoubleVectorFix = columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix.get(columnFromFile.getKey()); + if (indexInDoubleVectorFix != null) { + trackFixData[indexInDoubleVectorFix] = value; + } } importedFixes.set(true); callback.addFixes( @@ -162,13 +159,22 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI * exception is thrown that reports the columns missing. */ private void validateHeader(Map colIndicesInFile) throws FormatNotSupportedException { - Iterable requiredColumnsInFix = columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix.keySet(); - if (!Util.containsAll(colIndicesInFile.keySet(), requiredColumnsInFix)) { - final Set missingColumns = new HashSet<>(); - Util.addAll(requiredColumnsInFix, missingColumns); - missingColumns.removeAll(colIndicesInFile.keySet()); - logger.log(Level.SEVERE, "Missing headers: " + missingColumns); - throw new FormatNotSupportedException("Missing headers " + missingColumns + " in import files"); + final boolean dateTimeFormatOk; + if (colIndicesInFile.containsKey(UTC_COLUMN)) { + dateTimeFormatOk = true; + } else if (colIndicesInFile.containsKey(DATE_COLUMN_1) + || colIndicesInFile.containsKey(DATE_COLUMN_2)) { + dateTimeFormatOk = colIndicesInFile.containsKey(TIME_COLUMN); + } else { + // assume that "GPS Time" is present: + dateTimeFormatOk = colIndicesInFile.containsKey(GPS_TIME_COLUMN); + } + if (!dateTimeFormatOk) { + final String msg = "Missing date/time headers; expect either "+UTC_COLUMN+" or "+ + DATE_COLUMN_1+" with "+TIME_COLUMN+" or "+DATE_COLUMN_2+" with "+TIME_COLUMN+ + " or "+GPS_TIME_COLUMN; + logger.log(Level.SEVERE, msg); + throw new FormatNotSupportedException(msg); } }