Introduced boolean flag for succesful imports

This commit is contained in:
Papick Garcia Taboada committed 2017-12-14 16:44:38 +01:00
1 parent 4170349796
commit 0ce6f90c37
9 files changed
+123 -84

No files matched your search

@@ -6,6 +6,8 @@ import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import com.sap.sailing.domain.common.tracking.impl.GPSFixMovingImpl;
import com.sap.sailing.domain.deckmanadapter.DeckmanAdapter;
@@ -18,16 +20,20 @@ import com.sap.sailing.domain.trackimport.GPSFixImporter;
public class DeckmanGPSFixImporter implements GPSFixImporter {
@Override
public void importFixes(InputStream inputStream, Callback callback, boolean inferSpeedAndBearing, String sourceName)
public boolean importFixes(InputStream inputStream, Callback callback, boolean inferSpeedAndBearing,
String sourceName)
throws FormatNotSupportedException, IOException {
DeckmanAdapter deckmanAdapter = DeckmanAdapterFactoryImpl.INSTANCE.createDeckmanAdapter();
TrackFileImportDeviceIdentifier device = new TrackFileImportDeviceIdentifierImpl(sourceName, getType() + "@" + new Date());
final AtomicBoolean importedFixes = new AtomicBoolean(false);
try {
for (Iterator<Record> i = deckmanAdapter.parseLogFile(new InputStreamReader(inputStream)); i.hasNext();) {
Record record = i.next();
callback.addFix(new GPSFixMovingImpl(record.getPosition(), record.getTimePoint(), record.getGpsFix().getSpeed()),
device);
importedFixes.set(true);
}
return importedFixes.get();
} catch (RuntimeException e) {
if (e.getCause() != null) {
if (e.getCause() instanceof IOException) {
@@ -7,7 +7,9 @@ import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.Date;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import com.sap.sailing.domain.common.impl.DegreeBearingImpl;
import com.sap.sailing.domain.common.impl.DegreePosition;
@@ -25,14 +27,17 @@ public class ExpeditionGPSFixImporter implements GPSFixImporter {
private static final String LON_COLUMN_HEADING = ExpeditionExtendedDataImporterImpl.COL_NAME_LON;
private static final String COG_COLUMN_HEADING = "Cog";
private static final String SOG_COLUMN_HEADING = "Sog";
@Override
public void importFixes(InputStream inputStream, Callback callback, boolean inferSpeedAndBearing, final String sourceName)
public boolean importFixes(InputStream inputStream, Callback callback, boolean inferSpeedAndBearing,
final String sourceName)
throws FormatNotSupportedException, IOException {
TrackFileImportDeviceIdentifier device = new TrackFileImportDeviceIdentifierImpl(sourceName, getType() + "@" + new Date());
final BufferedReader br = new BufferedReader(new InputStreamReader(inputStream));
final String headerLine = br.readLine();
final Map<String, Integer> columnDefinitions = ExpeditionExtendedDataImporterImpl.parseHeader(headerLine);
final AtomicInteger lineNr = new AtomicInteger(0);
final AtomicBoolean importedFixes = new AtomicBoolean(false);
br.lines().forEach(line->{
if (!line.trim().isEmpty()) {
ExpeditionExtendedDataImporterImpl.parseLine(lineNr.incrementAndGet(), sourceName, line, columnDefinitions,
@@ -44,9 +49,11 @@ public class ExpeditionGPSFixImporter implements GPSFixImporter {
final GPSFixMoving fix = new GPSFixMovingImpl(new DegreePosition(latDeg, lonDeg), timePoint,
new KnotSpeedWithBearingImpl(sogKnots, new DegreeBearingImpl(cogDeg)));
callback.addFix(fix, device);
importedFixes.set(true);
});
}
});
return importedFixes.get();
}
@Override
@@ -33,7 +33,8 @@ public interface BaseDoubleVectorFixImporter {
* @throws IOException
* if there is a problem while reading the file
*/
void importFixes(InputStream inputStream, Callback callback, String filename, String sourceName, boolean downsample)
boolean importFixes(InputStream inputStream, Callback callback, String filename, String sourceName,
boolean downsample)
throws FormatNotSupportedException, IOException;
/**
@@ -43,16 +43,20 @@ public interface GPSFixImporter {
}
/**
* Retrieves the fixes from the {@code inputStream}, and calls the
* {@code callback} with every new fix.
* @param inferSpeedAndBearing Should speed and bearing be inferred by looking
* at the previous fix, if that data is not directly present within the file?
* @throws FormatNotSupportedException If the input format cannot be read. The import process
* might then decide to try attempt importing fixes using the next suitable importer.
* Retrieves the fixes from the {@code inputStream}, and calls the {@code callback} with every new fix.
*
* @param sourceName some name that identifies the source, e.g. the file name if a file
* @param inferSpeedAndBearing
* Should speed and bearing be inferred by looking at the previous fix, if that data is not directly
* present within the file?
* @throws FormatNotSupportedException
* If the input format cannot be read. The import process might then decide to try attempt importing
* fixes using the next suitable importer.
*
* @param sourceName
* some name that identifies the source, e.g. the file name if a file
* @return returns if import was succesful or not
*/
void importFixes(InputStream inputStream, Callback callback, boolean inferSpeedAndBearing,
boolean importFixes(InputStream inputStream, Callback callback, boolean inferSpeedAndBearing,
String sourceName)
throws FormatNotSupportedException, IOException;
@@ -17,7 +17,8 @@ public class AlwaysFailingGPSFixImporter implements GPSFixImporter {
}
@Override
public void importFixes(InputStream inputStream, Callback callback, boolean inferSpeedAndBearing, String sourceName)
public boolean importFixes(InputStream inputStream, Callback callback, boolean inferSpeedAndBearing,
String sourceName)
throws FormatNotSupportedException, IOException {
int read = 0;
while (inputStream.read() != -1 && (failAfterAtMostSoManyBytes == -1 || read++ < failAfterAtMostSoManyBytes)) {
@@ -96,23 +96,26 @@ public class TrackFilesImportServlet extends AbstractFileUploadServlet {
importersToTry.addAll(getGPSFixImporters(null));
logger.log(Level.INFO, "System knows " + importersToTry.size() + " importers: "
+ importersToTry.stream().map(i -> i.getType()).collect(Collectors.joining(", ")));
parsersLoop: for (GPSFixImporter importer : importersToTry) {
boolean succeeded = false;
logger.log(Level.INFO, "Trying to import file " + fileName + " with importer " + importer.getType());
try (BufferedInputStream in = new BufferedInputStream(fileItem.getInputStream())) {
try {
importer.importFixes(in, new Callback() {
succeeded = importer.importFixes(in, new Callback() {
@Override
public void addFix(GPSFix fix, TrackFileImportDeviceIdentifier device) {
storeFix(fix, device);
jsonResult.addDeviceIndentifier(device);
}
}, true, fileName);
succeeded = true;
} catch (Exception e) {
logger.log(Level.INFO, "Failed with " + e.getClass().getSimpleName()
+ " while importing file using " + importer.getType());
jsonResult.add(importer.getClass().getName(), fileName, e);
if (importer == preferredImporter) {
jsonResult.add(importer.getClass().getName(), fileName, e);
}
}
}
if (succeeded) {
@@ -11,6 +11,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -65,7 +66,7 @@ public class BaseBravoDataImporterImpl extends AbstractDoubleVectorFixImporter {
* if {@code true}, fixes will be down-sampled to a 1Hz frequency before being emitted to the
* {@code callback}. Otherwise, all fixes read will be forwarded straight to the {@link Callback}.
*/
public void importFixes(InputStream inputStream, Callback callback, final String filename, String sourceName,
public boolean importFixes(InputStream inputStream, Callback callback, final String filename, String sourceName,
boolean downsample) throws FormatNotSupportedException, IOException {
final TrackFileImportDeviceIdentifier trackIdentifier = new TrackFileImportDeviceIdentifierImpl(
UUID.randomUUID(), filename, sourceName, MillisecondsTimePoint.now());
@@ -86,7 +87,7 @@ public class BaseBravoDataImporterImpl extends AbstractDoubleVectorFixImporter {
String headerCandidate = buffer.readLine();
lineNr.incrementAndGet();
if (headerCandidate == null) {
throw new RuntimeException("Missing required header in file " + filename);
throw new FormatNotSupportedException("Missing required header in file " + filename);
}
if (headerCandidate.startsWith(BOF)) {
LOG.fine("Found header");
@@ -100,11 +101,14 @@ public class BaseBravoDataImporterImpl extends AbstractDoubleVectorFixImporter {
: fix -> callback.addFixes(
Collections.singleton(new DoubleVectorFixImpl(fix.getTimepoint(), fix.getFix())),
trackIdentifier);
final AtomicBoolean importedFixes = new AtomicBoolean(false);
buffer.lines().forEach(line -> {
lineNr.incrementAndGet();
downsampler.accept(parseLine(lineNr.get(), filename, line, colIndices));
importedFixes.set(true);
});
downsampler.finish();
return importedFixes.get();
}
/**
@@ -6,6 +6,8 @@ import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import slash.common.type.CompactCalendar;
import slash.navigation.base.BaseNavigationPosition;
@@ -54,7 +56,8 @@ public abstract class BaseRouteConverterGPSFixImporterImpl extends BaseGPSFixImp
@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
public void importFixes(InputStream inputStream, Callback callback, boolean inferSpeedAndBearing, String sourceName)
public boolean importFixes(InputStream inputStream, Callback callback, boolean inferSpeedAndBearing,
String sourceName)
throws IOException, FormatNotSupportedException {
NavigationFormatParser parser = new NavigationFormatParser();
List<BaseRoute> routes;
@@ -79,7 +82,7 @@ public abstract class BaseRouteConverterGPSFixImporterImpl extends BaseGPSFixImp
} catch (Exception e) {
throw new IOException(e);
}
final AtomicBoolean importedFixes = new AtomicBoolean(false);
for (BaseRoute route : routes) {
List<? extends BaseNavigationPosition> positions = (List<? extends BaseNavigationPosition>) route.getPositions();
String routeName = route.getName();
@@ -87,10 +90,12 @@ public abstract class BaseRouteConverterGPSFixImporterImpl extends BaseGPSFixImp
for (BaseNavigationPosition p : positions) {
try {
addFixAndInfer(callback, inferSpeedAndBearing, convertToGPSFix(p), device);
importedFixes.set(true);
} catch (Exception e) {
e.printStackTrace();
}
}
}
return importedFixes.get();
}
}
@@ -18,6 +18,7 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.TimeZone;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -41,7 +42,8 @@ import com.sap.sse.common.impl.MillisecondsTimePoint;
/**
* Importer for CSV data files from Expedition log files, as used, e.g., by Team Phoenix.
*/
public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixImporter implements DoubleVectorFixImporter {
public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixImporter
implements DoubleVectorFixImporter {
public static final String EXPEDITION_EXTENDED_TYPE = "EXPEDITION_EXTENDED";
private static final Logger logger = Logger.getLogger(ExpeditionExtendedDataImporterImpl.class.getName());
private static final String ORIGINAL_POSITION_HEADER = "Pos[ddd.dd]";
@@ -55,7 +57,6 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
private static final String TIME_COLUMN = "hhmmss";
private static final String GPS_TIME_COLUMN = "GPS Time";
private final Map<String, Integer> columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix;
/**
* The maximum index into the double vector fix from the
* {@link #columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix} values
@@ -64,68 +65,74 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
public ExpeditionExtendedDataImporterImpl() {
super(EXPEDITION_EXTENDED_TYPE);
columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix = ExpeditionExtendedSensorDataMetadata.getColumnNamesToIndexInDoubleFix();
trackColumnCount = columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix.values().stream().max((x,y)->Integer.compare(x, y)).get()+1;
columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix = ExpeditionExtendedSensorDataMetadata
.getColumnNamesToIndexInDoubleFix();
trackColumnCount = columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix.values().stream()
.max((x, y) -> Integer.compare(x, y)).get() + 1;
}
@Override
public RegattaLogDeviceCompetitorSensorDataMappingEvent createEvent(TimePoint createdAt, TimePoint logicalTimePoint,
AbstractLogEventAuthor author, Serializable id, Competitor mappedTo, DeviceIdentifier device,
TimePoint from, TimePoint to) {
return new RegattaLogDeviceCompetitorExpeditionExtendedMappingEventImpl(createdAt, logicalTimePoint, author, id, mappedTo,
device, from, to);
return new RegattaLogDeviceCompetitorExpeditionExtendedMappingEventImpl(createdAt, logicalTimePoint, author, id,
mappedTo, device, from, to);
}
@Override
public void importFixes(InputStream inputStream, final Callback callback, String filename, String sourceName,
public boolean importFixes(InputStream inputStream, final Callback callback, String filename, String sourceName,
boolean downsample) throws FormatNotSupportedException, IOException {
final TrackFileImportDeviceIdentifier trackIdentifier = new TrackFileImportDeviceIdentifierImpl(
UUID.randomUUID(), filename, sourceName, MillisecondsTimePoint.now());
try {
logger.fine("Import Expedition CSV from " + filename);
final InputStreamReader isr;
if (sourceName.endsWith("gz")) {
logger.fine("Using gzip stream reader " + filename);
isr = new InputStreamReader(new GZIPInputStream(inputStream));
} else {
isr = new InputStreamReader(inputStream);
}
logger.fine("Start parsing Expedition file");
AtomicLong lineNr = new AtomicLong();
try (BufferedReader buffer = new BufferedReader(isr)) {
String headerLine = buffer.readLine();
lineNr.incrementAndGet();
logger.fine("Validate and parse header columns");
final Map<String, Integer> colIndices = parseHeader(headerLine);
validateHeader(colIndices);
buffer.lines().forEach(line -> {
lineNr.incrementAndGet();
if (!line.trim().isEmpty()) {
parseLine(lineNr.get(), filename, line, colIndices, (timePoint, fileContentTokens, columnsInFileFromHeader)->{
Double[] trackFixData = new Double[trackColumnCount];
for (final Entry<String, Integer> columnNameToSearchForInFile : columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix.entrySet()) {
Integer columnsInFileIdx = columnsInFileFromHeader.get(columnNameToSearchForInFile.getKey());
trackFixData[columnNameToSearchForInFile.getValue()] = columnsInFileIdx >= fileContentTokens.length ?
null : fileContentTokens[columnsInFileIdx].trim().isEmpty() ? null :
Double.parseDouble(fileContentTokens[columnsInFileIdx]);
}
callback.addFixes(Collections.singleton(new DoubleVectorFixImpl(
timePoint, trackFixData)), trackIdentifier);
});
}
});
buffer.close();
}
} catch (Exception e) {
logger.log(Level.SEVERE, "Exception parsing Expedition CSV log file " + filename, e);
logger.fine("Import Expedition CSV from " + filename);
final InputStreamReader isr;
if (sourceName.endsWith("gz")) {
logger.fine("Using gzip stream reader " + filename);
isr = new InputStreamReader(new GZIPInputStream(inputStream));
} else {
isr = new InputStreamReader(inputStream);
}
logger.fine("Start parsing Expedition file");
final AtomicLong lineNr = new AtomicLong();
final AtomicBoolean importedFixes = new AtomicBoolean(false);
try (BufferedReader buffer = new BufferedReader(isr)) {
String headerLine = buffer.readLine();
lineNr.incrementAndGet();
logger.fine("Validate and parse header columns");
final Map<String, Integer> colIndices = parseHeader(headerLine);
validateHeader(colIndices);
buffer.lines().forEach(line -> {
lineNr.incrementAndGet();
if (!line.trim().isEmpty()) {
parseLine(lineNr.get(), filename, line, colIndices,
(timePoint, fileContentTokens, columnsInFileFromHeader) -> {
Double[] trackFixData = new Double[trackColumnCount];
for (final Entry<String, Integer> columnNameToSearchForInFile : columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix
.entrySet()) {
Integer columnsInFileIdx = columnsInFileFromHeader
.get(columnNameToSearchForInFile.getKey());
trackFixData[columnNameToSearchForInFile
.getValue()] = columnsInFileIdx >= fileContentTokens.length ? null
: fileContentTokens[columnsInFileIdx].trim().isEmpty() ? null
: Double.parseDouble(fileContentTokens[columnsInFileIdx]);
}
importedFixes.set(true);
callback.addFixes(
Collections.singleton(new DoubleVectorFixImpl(timePoint, trackFixData)),
trackIdentifier);
});
}
});
buffer.close();
}
return importedFixes.get();
}
/**
* When the header contains one or more occurrences of {@link #ORIGINAL_POSITION_HEADER}, it is substituted
* by the two header columns {@link #COL_NAME_LAT} and {@link #COL_NAME_LON} because that's
* how positions are stored in Expedition files: as two comma-separated values, one for latitude, another
* for longitude, although there is only one header field.
* When the header contains one or more occurrences of {@link #ORIGINAL_POSITION_HEADER}, it is substituted by the
* two header columns {@link #COL_NAME_LAT} and {@link #COL_NAME_LON} because that's how positions are stored in
* Expedition files: as two comma-separated values, one for latitude, another for longitude, although there is only
* one header field.
*/
public static Map<String, Integer> parseHeader(String headerLine) {
final String[] headerTokens = split(headerLine);
@@ -144,18 +151,17 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
}
/**
* Ensures that all columns in {@link #columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix}'s key set
* are present in {@code colIndicesInFile}'s key set. If not, an exception is thrown that reports the columns
* missing.
* Ensures that all columns in {@link #columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix}'s key set are
* present in {@code colIndicesInFile}'s key set. If not, an exception is thrown that reports the columns missing.
*/
private void validateHeader(Map<String, Integer> colIndicesInFile) {
private void validateHeader(Map<String, Integer> colIndicesInFile) throws FormatNotSupportedException {
Iterable<String> requiredColumnsInFix = columnNamesInFileAndTheirValueIndexInResultingDoubleVectorFix.keySet();
if (!Util.containsAll(colIndicesInFile.keySet(), requiredColumnsInFix)) {
final Set<String> missingColumns = new HashSet<>();
Util.addAll(requiredColumnsInFix, missingColumns);
missingColumns.removeAll(colIndicesInFile.keySet());
logger.log(Level.SEVERE, "Missing headers: "+missingColumns);
throw new RuntimeException("Missing headers "+missingColumns+" in import files");
logger.log(Level.SEVERE, "Missing headers: " + missingColumns);
throw new FormatNotSupportedException("Missing headers " + missingColumns + " in import files");
}
}
@@ -166,8 +172,8 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
/**
* Consumes a line split by the rules of the CSV format handled here; the first argument to
* {@link #accept(TimePoint, String[], Map) is the time stamp parsed from the line, the second is the tokenized
* array, each element representing one column in the current line; the third argument maps column names as found
* in the header line to their 0-based index in the array.
* array, each element representing one column in the current line; the third argument maps column names as found in
* the header line to their 0-based index in the array.
*
* @author Axel Uhl (d043530)
*
@@ -176,12 +182,12 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
public static interface LineParserCallback {
void accept(TimePoint timePoint, String[] columns, Map<String, Integer> header);
}
/**
* Parses the CSV line and reads the double data values in the order defined by the col enums.
*/
public static void parseLine(long lineNr, String filename, String line, Map<String, Integer> columnsInFileFromHeader,
LineParserCallback callback) {
public static void parseLine(long lineNr, String filename, String line,
Map<String, Integer> columnsInFileFromHeader, LineParserCallback callback) {
try {
String[] fileContentTokens = split(line);
final TimePoint timePoint;
@@ -189,7 +195,8 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
final String dateFormatPattern;
if (columnsInFileFromHeader.containsKey(UTC_COLUMN)) {
timePoint = getTimePoint(fileContentTokens[columnsInFileFromHeader.get(UTC_COLUMN)]);
} else if (columnsInFileFromHeader.containsKey(DATE_COLUMN_1) || columnsInFileFromHeader.containsKey(DATE_COLUMN_2)) {
} else if (columnsInFileFromHeader.containsKey(DATE_COLUMN_1)
|| columnsInFileFromHeader.containsKey(DATE_COLUMN_2)) {
if (columnsInFileFromHeader.containsKey(DATE_COLUMN_1)) {
date = fileContentTokens[columnsInFileFromHeader.get(DATE_COLUMN_1)];
dateFormatPattern = DATE_COLUMN_1_PATTERN;
@@ -198,8 +205,8 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
dateFormatPattern = DATE_COLUMN_2_PATTERN;
}
final String time = fileContentTokens[columnsInFileFromHeader.get(TIME_COLUMN)];
final DateFormat df = new SimpleDateFormat(dateFormatPattern+"'T'HH:mm:ssX");
final Date timestamp = df.parse(date+"T"+time+"+00:00"); // assuming UTC
final DateFormat df = new SimpleDateFormat(dateFormatPattern + "'T'HH:mm:ssX");
final Date timestamp = df.parse(date + "T" + time + "+00:00"); // assuming UTC
timePoint = new MillisecondsTimePoint(timestamp);
} else {
// assume that "GPS Time" is present:
@@ -221,9 +228,11 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
EXCEL_EPOCH_START.set(Calendar.MILLISECOND, 0);
}
private static final BigDecimal MILLISECONDS_PER_DAY = BigDecimal.valueOf(24 * 60 * 60 * 1000);
/**
* Converts a value from a column such as "GPS Time" to a {@link TimePoint}. The value is expected to be provided
* in decimal format, representing the days since the {@link #EXCEL_EPOCH_START "Excel Epoch Start"}.
* Converts a value from a column such as "GPS Time" to a {@link TimePoint}. The value is expected to be provided in
* decimal format, representing the days since the {@link #EXCEL_EPOCH_START "Excel Epoch Start"}.
*
* @param time_ExcelEpoch
* @return
*/
@@ -239,5 +248,4 @@ public class ExpeditionExtendedDataImporterImpl extends AbstractDoubleVectorFixI
}
return timePoint;
}
}