first implementation of parsing start line positions and start times from Expedition logs

Change-Id: I60b18e9633839bac65388cc4fbb0363d010fbae8

java/com.sap.sailing.server.gateway.test/src/com/sap/sailing/server/gateway/expeditionimport/ExpeditionStartDataTest.java

@@ -0,0 +1,40 @@
+package com.sap.sailing.server.gateway.expeditionimport;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.junit.Test;
+
+import com.sap.sailing.domain.trackimport.FormatNotSupportedException;
+import com.sap.sailing.server.gateway.trackfiles.impl.ExpeditionCourseInferrer;
+import com.sap.sailing.server.gateway.trackfiles.impl.ExpeditionStartData;
+import com.sap.sse.common.TimePoint;
+import com.sap.sse.common.Util;
+import com.sap.sse.common.impl.MillisecondsTimePoint;
+
+public class ExpeditionStartDataTest {
+    private static final SimpleDateFormat TIMEPOINT_FORMATTER = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
+
+    @Test
+    public void readFile() throws IOException, FormatNotSupportedException, ParseException {
+        final String filename = "2018Apr28_05.csv.gz";
+        final ExpeditionStartData result = new ExpeditionCourseInferrer().getStartData(getClass().getResourceAsStream(filename), filename);
+        assertNotNull(result);
+        final Set<TimePoint> startTimes = new HashSet<>();
+        Util.addAll(result.getStartTimes(), startTimes);
+        assertTrue(startTimes.contains(new MillisecondsTimePoint(TIMEPOINT_FORMATTER.parse("2018-04-28T14:44:59.481+0200"))));
+        assertTrue(startTimes.contains(new MillisecondsTimePoint(TIMEPOINT_FORMATTER.parse("2018-04-28T14:34:59.260+0200"))));
+        assertTrue(startTimes.contains(new MillisecondsTimePoint(TIMEPOINT_FORMATTER.parse("2018-04-28T14:14:58.819+0200"))));
+        assertTrue(startTimes.contains(new MillisecondsTimePoint(TIMEPOINT_FORMATTER.parse("2018-04-28T15:29:59.740+0200"))));
+        assertTrue(startTimes.contains(new MillisecondsTimePoint(TIMEPOINT_FORMATTER.parse("2018-04-28T14:24:58.953+0200"))));
+        assertEquals(34, Util.size(result.getStartLinePortFixes()));
+        assertEquals(34, Util.size(result.getStartLineStarboardFixes()));
+    }
+}

java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/trackfiles/impl/ExpeditionAllInOneImporter.java

@@ -64,6 +64,7 @@ import com.sap.sailing.domain.racelogtracking.RaceLogTrackingAdapter;
 import com.sap.sailing.domain.trackfiles.TrackFileImportDeviceIdentifier;
 import com.sap.sailing.domain.trackfiles.TrackFileImportDeviceIdentifierImpl;
 import com.sap.sailing.domain.trackimport.DoubleVectorFixImporter;
+import com.sap.sailing.domain.trackimport.FormatNotSupportedException;
 import com.sap.sailing.domain.trackimport.GPSFixImporter;
 import com.sap.sailing.domain.tracking.DynamicTrackedRace;
 import com.sap.sailing.domain.tracking.RaceHandle;
@@ -185,19 +186,16 @@ public class ExpeditionAllInOneImporter {
     }
 
     public ImporterResult importFiles(final String filenameWithSuffix, final FileItem fileItem,
-            final String boatClassName, ImportMode importMode, String existingRegattaName) throws AllinOneImportException {
+            final String boatClassName, ImportMode importMode, String existingRegattaName) throws AllinOneImportException, IOException, FormatNotSupportedException {
         final List<ErrorImportDTO> errors = new ArrayList<>();
         final String importTimeString = DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(LocalDateTime.now(ZoneOffset.UTC));
-
         final String filename = ExpeditionImportFilenameUtils.truncateFilenameExtentions(filenameWithSuffix);
         final String filenameWithDateTimeSuffix = filename + "_" + importTimeString;
         final String trackedRaceName = filenameWithDateTimeSuffix;
         final String windSourceId = filenameWithDateTimeSuffix;
-
-        // TODO wild guess...
         final int[] discardThresholds = new int[0];
-
         final ImportResultDTO jsonHolderForGpsFixImport = new ImportResultDTO(logger);
+        // Import GPS Fixes
         final List<Pair<String, FileItem>> filesForGpsFixImport = Arrays.asList(new Pair<>(filenameWithSuffix, fileItem));
         try {
             new TrackFilesImporter(service, serviceFinderFactory, context).importFixes(jsonHolderForGpsFixImport,
@@ -209,8 +207,7 @@ public class ExpeditionAllInOneImporter {
             throw new AllinOneImportException(e1, errors);
         }
         errors.addAll(jsonHolderForGpsFixImport.getErrorList());
-        
-
+        // Import Extended Sensor Data
         final ImportResultDTO jsonHolderForSensorFixImport = new ImportResultDTO(logger);
         final String sensorFixImporterType = DoubleVectorFixImporter.EXPEDITION_EXTENDED_TYPE;
         final Iterable<Pair<String, FileItem>> importerNamesAndFilesForSensorFixImport = Arrays
@@ -225,13 +222,11 @@ public class ExpeditionAllInOneImporter {
             throw new AllinOneImportException(e1, errors);
         }
         errors.addAll(jsonHolderForSensorFixImport.getErrorList());
-
         TimePoint firstFixAt = null;
         TimePoint lastFixAt = null;
         final ArrayList<TrackImportDTO> allData = new ArrayList<>();
         allData.addAll(jsonHolderForGpsFixImport.getImportResult());
         allData.addAll(jsonHolderForSensorFixImport.getImportResult());
-
         for (TrackImportDTO result : allData) {
             final TimePoint deviceTrackStart = result.getRange().from();
             final TimePoint deviceTrackEnd = result.getRange().to();
@@ -244,12 +239,12 @@ public class ExpeditionAllInOneImporter {
         }
         final TimePoint eventStartDate = firstFixAt;
         final TimePoint eventEndDate = lastFixAt;
-        
         final UUID eventId;
         final String leaderboardGroupName;
         final String regattaNameAndleaderboardName;
         final List<Triple<String, String, String>> raceNameRaceColumnNameFleetnameList = new ArrayList<>();
         final List<DynamicTrackedRace> trackedRaces = new ArrayList<>();
+        final ExpeditionStartData startData = new ExpeditionCourseInferrer().getStartData(fileItem.getInputStream(), filename);
         if (importMode == ImportMode.NEW_EVENT) {
             leaderboardGroupName = filenameWithDateTimeSuffix;
             regattaNameAndleaderboardName = filenameWithDateTimeSuffix;
@@ -275,17 +270,13 @@ public class ExpeditionAllInOneImporter {
                     return new ImporterResult(serverStringMessages.get(uiLocale, "allInOneErrorInvalidLeaderBoard"));
                 }
                 final RegattaLeaderboard regattaLeaderboard = (RegattaLeaderboard) leaderboard;
-                
                 final Pair<Event, LeaderboardGroup> foundEventAndLeaderboardGroup = findEventAndLeaderboardGroupForExistingLeaderboard(leaderboard);
-                
                 if (foundEventAndLeaderboardGroup == null) {
                     return new ImporterResult(serverStringMessages.get(uiLocale, "allInOneErrorInvalidLeaderBoardEventLink"));
                 }
-                
                 eventId = foundEventAndLeaderboardGroup.getA().getId();
                 ensureEventLongEnough(firstFixAt, lastFixAt, eventId);
                 leaderboardGroupName = foundEventAndLeaderboardGroup.getB().getName();
-                
                 if (importMode == ImportMode.NEW_COMPETITOR) {
                     final Iterable<RaceColumn> raceColumns = regattaLeaderboard.getRaceColumns();
                     if (Util.isEmpty(raceColumns)) {
@@ -337,16 +328,13 @@ public class ExpeditionAllInOneImporter {
                 return new ImporterResult(serverStringMessages.get(uiLocale, "allInOneErrorInvalidRegattaName"));
             }
         }
-
+        // Import Wind Data
         try {
             final WindImportResult windImportResult = new AbstractWindImporter.WindImportResult();
-            final WindSourceWithAdditionalID windSource = new WindSourceWithAdditionalID(WindSourceType.EXPEDITION,
-                    windSourceId);
+            final WindSourceWithAdditionalID windSource = new WindSourceWithAdditionalID(WindSourceType.EXPEDITION, windSourceId);
             final Map<InputStream, String> streamsWithFilenames = new HashMap<>();
             streamsWithFilenames.put(fileItem.getInputStream(), filenameWithSuffix);
-            new WindImporter().importWindToWindSourceAndTrackedRaces(service, windImportResult, windSource,
-                    trackedRaces, streamsWithFilenames);
-            
+            new WindImporter().importWindToWindSourceAndTrackedRaces(service, windImportResult, windSource, trackedRaces, streamsWithFilenames);
             return new ImporterResult(eventId, regattaNameAndleaderboardName, leaderboardGroupName,
                     regattaNameAndleaderboardName, raceNameRaceColumnNameFleetnameList,
                     jsonHolderForGpsFixImport.getImportResult(), jsonHolderForSensorFixImport.getImportResult(),

java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/trackfiles/impl/ExpeditionCourseInferrer.java

@@ -0,0 +1,102 @@
+package com.sap.sailing.server.gateway.trackfiles.impl;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Logger;
+
+import com.sap.sailing.domain.common.impl.DegreePosition;
+import com.sap.sailing.domain.common.sensordata.ExpeditionExtendedSensorDataMetadata;
+import com.sap.sailing.domain.common.tracking.GPSFix;
+import com.sap.sailing.domain.common.tracking.impl.GPSFixImpl;
+import com.sap.sailing.domain.trackimport.FormatNotSupportedException;
+import com.sap.sailing.server.trackfiles.impl.CompressedStreamsUtil;
+import com.sap.sailing.server.trackfiles.impl.ExpeditionExtendedDataImporterImpl;
+import com.sap.sailing.server.trackfiles.impl.ExpeditionImportFileHandler;
+import com.sap.sse.common.TimePoint;
+
+/**
+ * From an Expedition log file extracts a {@link ExpeditionStartData} object that has all mark "ping" positions for the
+ * two ends of the start line ("committee boat" and "pin end"), furthermore all time points identified as possible start
+ * time points because the "time to gun" value went to zero.
+ * 
+ * @author Axel Uhl (D043530)
+ *
+ */
+public class ExpeditionCourseInferrer {
+    private static final Logger logger = Logger.getLogger(ExpeditionCourseInferrer.class.getName());
+    private static final String START_LINE_PORT_END_LAT = "Port lat";
+    private static final String START_LINE_PORT_END_LON = "Port lon";
+    private static final String START_LINE_STARBOARD_END_LAT = "Stbd lat";
+    private static final String START_LINE_STARBOARD_END_LON = "Stbd lon";
+    
+    public ExpeditionStartData getStartData(InputStream inputStream, String filenameWithSuffix)
+            throws IOException, FormatNotSupportedException {
+        final List<TimePoint> startTimeCandidates = new ArrayList<>();
+        final List<GPSFix> startLinePortEndFixes = new ArrayList<>();
+        final List<GPSFix> startLineStarboardEndFixes = new ArrayList<>();
+        CompressedStreamsUtil.handlePotentiallyCompressedFiles(filenameWithSuffix, inputStream, new ExpeditionImportFileHandler() {
+            @Override
+            protected void handleExpeditionFile(String fileName, InputStream inputStream) throws IOException, FormatNotSupportedException {
+                logger.fine("Start parsing Expedition file");
+                final AtomicLong lineNr = new AtomicLong();
+                try (BufferedReader buffer = new BufferedReader(new InputStreamReader(inputStream))) {
+                    String headerLine = buffer.readLine();
+                    lineNr.incrementAndGet();
+                    logger.fine("Validate and parse header columns");
+                    final Map<String, Integer> colIndices = ExpeditionExtendedDataImporterImpl.parseHeader(headerLine);
+                    ExpeditionExtendedDataImporterImpl.validateHeader(colIndices);
+                    final Double lastTimeToGunValue[] = new Double[1];
+                    buffer.lines().forEach(line -> {
+                        lineNr.incrementAndGet();
+                        if (!line.trim().isEmpty()) {
+                            ExpeditionExtendedDataImporterImpl.parseLine(lineNr.get(), filenameWithSuffix, line, colIndices,
+                                    (timePoint, lineContentTokens, columnsInFileFromHeader) -> {
+                                        // look for a start time based on the time to gun turning negative
+                                        final Double timeToGunValue = getColumnValue(lineContentTokens, columnsInFileFromHeader,
+                                                ExpeditionExtendedSensorDataMetadata.EXPEDITION_TMTOGUN.getColumnName());
+                                        if (lastTimeToGunValue[0] != null && lastTimeToGunValue[0] > 0 && timeToGunValue != null &&
+                                                timeToGunValue <= 0) {
+                                            // found a start candidate:
+                                            startTimeCandidates.add(timePoint);
+                                        }
+                                        lastTimeToGunValue[0] = timeToGunValue;
+                                        // look for start line position pings:
+                                        final Double startLinePortEndLat = getColumnValue(lineContentTokens, columnsInFileFromHeader, START_LINE_PORT_END_LAT);
+                                        final Double startLinePortEndLon = getColumnValue(lineContentTokens, columnsInFileFromHeader, START_LINE_PORT_END_LON);
+                                        final Double startLineStarboardEndLat = getColumnValue(lineContentTokens, columnsInFileFromHeader, START_LINE_STARBOARD_END_LAT);
+                                        final Double startLineStarboardEndLon = getColumnValue(lineContentTokens, columnsInFileFromHeader, START_LINE_STARBOARD_END_LON);
+                                        addFixIfCoordinatesValid(startLinePortEndLat, startLinePortEndLon, timePoint, startLinePortEndFixes);
+                                        addFixIfCoordinatesValid(startLineStarboardEndLat, startLineStarboardEndLon, timePoint, startLineStarboardEndFixes);
+                                    });
+                        }
+                    });
+                    buffer.close();
+                }
+            }
+        });
+        return new ExpeditionStartDataImpl(startLinePortEndFixes, startLineStarboardEndFixes, startTimeCandidates);
+    }
+
+    private void addFixIfCoordinatesValid(Double lat, Double lon, TimePoint timePoint, List<GPSFix> listToAddTo) {
+        if (lat != null && lon != null) {
+            listToAddTo.add(new GPSFixImpl(new DegreePosition(lat, lon), timePoint));
+        }
+    }
+    
+    private Double getColumnValue(String[] lineContentTokens, Map<String, Integer> columnsInFileFromHeader, String columnName) {
+        return getColumnValue(lineContentTokens, columnsInFileFromHeader.get(columnName));
+    }
+
+    private Double getColumnValue(String[] lineContentTokens, final Integer columnIndex) {
+        return columnIndex == null ? null :
+            columnIndex >= lineContentTokens.length ? null
+                : lineContentTokens[columnIndex].trim().isEmpty() ? null
+                        : Double.parseDouble(lineContentTokens[columnIndex]);
+    }
+}

java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/trackfiles/impl/ExpeditionStartData.java

@@ -0,0 +1,12 @@
+package com.sap.sailing.server.gateway.trackfiles.impl;
+
+import com.sap.sailing.domain.common.tracking.GPSFix;
+import com.sap.sse.common.TimePoint;
+
+public interface ExpeditionStartData {
+    Iterable<GPSFix> getStartLinePortFixes();
+    
+    Iterable<GPSFix> getStartLineStarboardFixes();
+    
+    Iterable<TimePoint> getStartTimes();
+}

java/com.sap.sailing.server.trackfiles/src/com/sap/sailing/server/trackfiles/impl/ExpeditionExtendedDataImporterImpl.java

@@ -134,7 +134,6 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
                     });
                     buffer.close();
                 }
-                
             }
         });
         return importedFixes.get();
@@ -170,7 +169,7 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
      * key set are present in {@code colIndicesInFile}'s key set. If not, an
      * exception is thrown that reports the columns missing.
      */
-    private void validateHeader(Map<String, Integer> colIndicesInFile) throws FormatNotSupportedException {
+    public static void validateHeader(Map<String, Integer> colIndicesInFile) throws FormatNotSupportedException {
         final boolean dateTimeFormatOk;
         if (colIndicesInFile.containsKey(UTC_COLUMN)) {
             dateTimeFormatOk = true;
@@ -251,7 +250,7 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
      * </ol>
      * If none of the above is found, {@link null} is returned.
      */
-    protected static TimePoint getTimePointFromLine(Map<String, Integer> columnsInFileFromHeader,
+    public static TimePoint getTimePointFromLine(Map<String, Integer> columnsInFileFromHeader,
             String[] lineContentTokens) throws ParseException {
         final TimePoint timePoint;
         final String date;

java/com.sap.sailing.server.trackfiles/src/com/sap/sailing/server/trackfiles/impl/ExpeditionImportFileHandler.java

@@ -28,6 +28,7 @@ public abstract class ExpeditionImportFileHandler implements CompressedStreamsUt
         for (String extension : supportedExpeditionLogFileExtensions) {
             if (lowerCaseFileName.endsWith(extension)) {
                 extensionSupported = true;
+                break;
             }
         }
         if (extensionSupported) {