eclipse-sailing-analytics/sailing-analytics
1
0 Fork 0
mirror of https://github.com/eclipse-sailing-analytics/sailing-analytics.git synced 2026-11-14 22:43:57 +00:00
Code Activity

refactored Expedition import to use existing CSV logic; first successful import

Browse Source 
Change-Id: I05652a329f0115a4e4ce9a225c0aa146461e1f06
This commit is contained in:
Axel Uhl committed 2017-10-18 15:12:36 +02:00
1 parent a88b5166db
commit 3ff8e28d9a
3 files changed
+82 -29
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

java/com.sap.sailing.domain.expeditionadapter/META-INF/MANIFEST.MF
+3 -1
View File
@@ -7,5 +7,7 @@ Bundle-Vendor: SAP
Bundle-RequiredExecutionEnvironment: JavaSE-1.8
Import-Package: org.osgi.framework
Require-Bundle: com.sap.sailing.server.trackfiles,
com.sap.sailing.domain
com.sap.sailing.domain,
com.sap.sse.common,
com.sap.sailing.domain.common
Bundle-Activator: com.sap.sailing.domain.expeditionadapter.impl.Activator
java/com.sap.sailing.domain.expeditionadapter/src/com/sap/sailing/domain/expeditionadapter/impl/ExpeditionGPSFixImporter.java
+37 -6
View File
@@ -1,20 +1,52 @@
package com.sap.sailing.domain.expeditionadapter.impl;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.Date;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import com.sap.sailing.domain.common.impl.DegreeBearingImpl;
import com.sap.sailing.domain.common.impl.DegreePosition;
import com.sap.sailing.domain.common.impl.KnotSpeedWithBearingImpl;
import com.sap.sailing.domain.common.tracking.GPSFixMoving;
import com.sap.sailing.domain.common.tracking.impl.GPSFixMovingImpl;
import com.sap.sailing.domain.trackfiles.TrackFileImportDeviceIdentifier;
import com.sap.sailing.domain.trackfiles.TrackFileImportDeviceIdentifierImpl;
import com.sap.sailing.domain.trackimport.FormatNotSupportedException;
import com.sap.sailing.domain.trackimport.GPSFixImporter;
import com.sap.sailing.server.trackfiles.impl.ExpeditionExtendedDataImporterImpl;
public class ExpeditionGPSFixImporter implements GPSFixImporter {
private static final String LAT_COLUMN_HEADING = ExpeditionExtendedDataImporterImpl.GENERATED_LAT_HEADER;
private static final String LON_COLUMN_HEADING = ExpeditionExtendedDataImporterImpl.GENERATED_LON_HEADER;
private static final String COG_COLUMN_HEADING = "Ext_COG";
private static final String SOG_COLUMN_HEADING = "Ext_SOG";
@Override
public void importFixes(InputStream inputStream, Callback callback, boolean inferSpeedAndBearing, String sourceName)
public void importFixes(InputStream inputStream, Callback callback, boolean inferSpeedAndBearing, final String sourceName)
throws FormatNotSupportedException, IOException {
// TODO the following device identifier will be required for invoking the callback.addFix(...) method
// TrackFileImportDeviceIdentifier device = new TrackFileImportDeviceIdentifierImpl(sourceName, getType() + "@" + new Date());
// TODO parse the input stream using the expeditionAdapter, producing GPSFixMoving objects to pass to the callback
TrackFileImportDeviceIdentifier device = new TrackFileImportDeviceIdentifierImpl(sourceName, getType() + "@" + new Date());
final BufferedReader br = new BufferedReader(new InputStreamReader(inputStream));
final String headerLine = br.readLine();
final Map<String, Integer> columnDefinitions = ExpeditionExtendedDataImporterImpl.parseHeader(headerLine);
final AtomicInteger lineNr = new AtomicInteger(0);
br.lines().forEach(line->{
if (!line.trim().isEmpty()) {
ExpeditionExtendedDataImporterImpl.parseLine(lineNr.incrementAndGet(), sourceName, line, columnDefinitions,
(timePoint, columnValues, columns)->{
final double latDeg = Double.parseDouble(columnValues[columns.get(LAT_COLUMN_HEADING)]);
final double lonDeg = Double.parseDouble(columnValues[columns.get(LON_COLUMN_HEADING)]);
final double cogDeg = Double.parseDouble(columnValues[columns.get(COG_COLUMN_HEADING)]);
final double sogKnots = Double.parseDouble(columnValues[columns.get(SOG_COLUMN_HEADING)]);
final GPSFixMoving fix = new GPSFixMovingImpl(new DegreePosition(latDeg, lonDeg), timePoint,
new KnotSpeedWithBearingImpl(sogKnots, new DegreeBearingImpl(cogDeg)));
callback.addFix(fix, device);
});
}
});
}
@Override
@@ -26,5 +58,4 @@ public class ExpeditionGPSFixImporter implements GPSFixImporter {
public String getType() {
return "Expedition";
}
}
java/com.sap.sailing.server.trackfiles/src/com/sap/sailing/server/trackfiles/impl/ExpeditionExtendedDataImporterImpl.java
+42 -22
View File
@@ -31,7 +31,6 @@ import com.sap.sailing.domain.trackfiles.TrackFileImportDeviceIdentifier;
import com.sap.sailing.domain.trackfiles.TrackFileImportDeviceIdentifierImpl;
import com.sap.sailing.domain.trackimport.DoubleVectorFixImporter;
import com.sap.sailing.domain.trackimport.FormatNotSupportedException;
import com.sap.sailing.server.trackfiles.impl.doublefix.DoubleVectorFixData;
import com.sap.sse.common.TimePoint;
import com.sap.sse.common.Util;
import com.sap.sse.common.impl.MillisecondsTimePoint;
@@ -43,8 +42,8 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
public static final String EXPEDITION_EXTENDED_TYPE = "EXPEDITION_EXTENDED";
private static final Logger logger = Logger.getLogger(ExpeditionExtendedDataImporterImpl.class.getName());
private static final String ORIGINAL_POSITION_HEADER = "Pos[ddd.dd]";
private static final String GENERATED_LAT_HEADER = "Lat";
private static final String GENERATED_LON_HEADER = "Lon";
public static final String GENERATED_LAT_HEADER = "Lat";
public static final String GENERATED_LON_HEADER = "Lon";
private static final String DATE_COLUMN_1 = "dd/mm/yy";
private static final String DATE_COLUMN_1_PATTERN = "dd/MM/yy";
private static final String DATE_COLUMN_2 = "mm/dd/yy";
@@ -73,7 +72,7 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
}
@Override
public void importFixes(InputStream inputStream, Callback callback, String filename, String sourceName,
public void importFixes(InputStream inputStream, final Callback callback, String filename, String sourceName,
boolean downsample) throws FormatNotSupportedException, IOException {
final TrackFileImportDeviceIdentifier trackIdentifier = new TrackFileImportDeviceIdentifierImpl(
UUID.randomUUID(), filename, sourceName, MillisecondsTimePoint.now());
@@ -92,12 +91,20 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
String headerLine = buffer.readLine();
lineNr.incrementAndGet();
logger.fine("Validate and parse header columns");
final Map<String, Integer> colIndices = validateAndParseHeader(headerLine);
final Map<String, Integer> colIndices = parseHeader(headerLine);
validateHeader(colIndices);
buffer.lines().forEach(line -> {
lineNr.incrementAndGet();
if (!line.trim().isEmpty()) {
DoubleVectorFixData fix = parseLine(lineNr.get(), filename, line, colIndices);
callback.addFixes(Collections.singleton(new DoubleVectorFixImpl(fix.getTimepoint(), fix.getFix())), trackIdentifier);
parseLine(lineNr.get(), filename, line, colIndices, (timePoint, fileContentTokens, columnsInFileFromHeader)->{
double[] trackFixData = new double[trackColumnCount];
for (final Entry<String, Integer> columnNameToSearchForInFile : columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix.entrySet()) {
Integer columnsInFileIdx = columnsInFileFromHeader.get(columnNameToSearchForInFile.getKey());
trackFixData[columnNameToSearchForInFile.getValue()] = Double.parseDouble(fileContentTokens[columnsInFileIdx]);
}
callback.addFixes(Collections.singleton(new DoubleVectorFixImpl(
timePoint, trackFixData)), trackIdentifier);
});
}
});
buffer.close();
@@ -113,7 +120,7 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
* how positions are stored in Expedition files: as two comma-separated values, one for latitude, another
* for longitude, although there is only one header field.
*/
private Map<String, Integer> validateAndParseHeader(String headerLine) {
public static Map<String, Integer> parseHeader(String headerLine) {
final String[] headerTokens = split(headerLine);
Map<String, Integer> colIndicesInFile = new HashMap<>();
int columnInResultingHeader = 0;
@@ -126,6 +133,15 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
colIndicesInFile.put(header, columnInResultingHeader++);
}
}
return colIndicesInFile;
}
/**
* Ensures that all columns in {@link #columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix}'s key set
* are present in {@code colIndicesInFile}'s key set. If not, an exception is thrown that reports the columns
* missing.
*/
private void validateHeader(Map<String, Integer> colIndicesInFile) {
Iterable<String> requiredColumnsInFix = columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix.keySet();
if (!Util.containsAll(colIndicesInFile.keySet(), requiredColumnsInFix)) {
final Set<String> missingColumns = new HashSet<>();
@@ -134,19 +150,32 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
logger.log(Level.SEVERE, "Missing headers: "+missingColumns);
throw new RuntimeException("Missing headers "+missingColumns+" in import files");
}
return colIndicesInFile;
}
private String[] split(String line) {
public static String[] split(String line) {
return line.split("\\s*,\\s*");
}
/**
* Consumes a line split by the rules of the CSV format handled here; the first argument to
* {@link #accept(TimePoint, String[], Map) is the time stamp parsed from the line, the second is the tokenized
* array, each element representing one column in the current line; the third argument maps column names as found
* in the header line to their 0-based index in the array.
*
* @author Axel Uhl (d043530)
*
*/
@FunctionalInterface
public static interface LineParserCallback {
void accept(TimePoint timePoint, String[] columns, Map<String, Integer> header);
}
/**
* Parses the CSV line and reads the double data values in the order defined by the col enums.
*/
private DoubleVectorFixData parseLine(long lineNr, String filename, String line, Map<String, Integer> columnsInFileFromHeader) {
public static void parseLine(long lineNr, String filename, String line, Map<String, Integer> columnsInFileFromHeader,
LineParserCallback callback) {
try {
final DoubleVectorFixData result;
String[] fileContentTokens = split(line);
final String date;
final String dateFormatPattern;
@@ -161,20 +190,11 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
final DateFormat df = new SimpleDateFormat(dateFormatPattern+"'T'HH:mm:ssX");
final Date timestamp = df.parse(date+"T"+time+"+00:00"); // assuming UTC
if (timestamp != null) {
double[] trackFixData = new double[trackColumnCount];
for (final Entry<String, Integer> columnNameToSearchForInFile : columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix.entrySet()) {
Integer columnsInFileIdx = columnsInFileFromHeader.get(columnNameToSearchForInFile.getKey());
trackFixData[columnNameToSearchForInFile.getValue()] = Double.parseDouble(fileContentTokens[columnsInFileIdx]);
}
result = new DoubleVectorFixData(timestamp.getTime(), trackFixData);
} else {
result = null;
callback.accept(new MillisecondsTimePoint(timestamp), fileContentTokens, columnsInFileFromHeader);
}
return result;
} catch (Exception e) {
logger.warning(
"Error parsing line nr " + lineNr + " in file " + filename + "with exception: " + e.getMessage());
return null;
}
}