bug6059: untested approach for obtaining resource data from primary

This commit is contained in:
Axel Uhl
2024-12-28 00:57:22 +01:00
parent 8e11afbb66
commit a7403d027f
10 changed files with 129 additions and 103 deletions
@@ -3,6 +3,7 @@ package com.sap.sailing.domain.igtimiadapter.gateway.impl;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.util.Base64;
import java.util.Base64.Encoder;
import java.util.HashMap;
@@ -123,7 +124,7 @@ public class RiotResourcesResource extends AbstractRiotServerResource {
@GET
@Produces("application/json;charset=UTF-8")
@Path(DATA)
public Response getResourcesData(@Context UriInfo ui) {
public Response getResourcesData(@Context UriInfo ui) throws MalformedURLException {
final MultivaluedMap<String, String> queryParams = ui.getQueryParameters();
final String startTimeString = queryParams.getFirst("start_time");
final TimePoint startTime = startTimeString == null ? null : TimePoint.of(Long.valueOf(startTimeString));
@@ -22,6 +22,8 @@ Import-Package: com.google.gwt.user.client.rpc.core.com.sap.sse.security.shared,
com.sap.sse.security.shared,
com.sap.sse.shared.util,
com.sap.sse.util,
org.apache.http.client;version="4.5.5",
org.json.simple.parser,
org.osgi.framework;version="1.3.0",
org.osgi.util.tracker;version="[1.5.0,2.0.0)"
Require-Bundle: com.sap.sailing.domain.igtimiadapter,
@@ -1,9 +1,13 @@
package com.sap.sailing.domain.igtimiadapter.server.riot;
import java.io.IOException;
import java.net.MalformedURLException;
import java.nio.channels.ServerSocketChannel;
import java.util.Set;
import org.apache.http.client.ClientProtocolException;
import org.json.simple.parser.ParseException;
import com.google.protobuf.InvalidProtocolBufferException;
import com.igtimi.IgtimiData.DataMsg;
import com.igtimi.IgtimiData.DataPoint;
@@ -116,8 +120,7 @@ public interface RiotServer extends Replicable<ReplicableRiotServer, RiotReplica
void removeDataAccessWindow(long dawId);
// TODO clarify what this means in the presence of replication, when run on a Replica
Iterable<Msg> getMessages(String deviceSerialNumber, TimeRange timeRange, Set<DataCase> dataCases);
Iterable<Msg> getMessages(String deviceSerialNumber, TimeRange timeRange, Set<DataCase> dataCases) throws MalformedURLException, IllegalStateException, ClientProtocolException, IOException, ParseException;
void addWebSocketClient(RiotWebsocketHandler riotWebsocketHandler);
@@ -8,6 +8,7 @@ import java.lang.Thread.State;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.SocketAddress;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.channels.SelectionKey;
import java.nio.channels.Selector;
@@ -24,6 +25,9 @@ import java.util.concurrent.ConcurrentMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.http.client.ClientProtocolException;
import org.json.simple.parser.ParseException;
import com.google.protobuf.InvalidProtocolBufferException;
import com.igtimi.IgtimiData.DataMsg;
import com.igtimi.IgtimiData.DataPoint;
@@ -35,7 +39,10 @@ import com.sap.sailing.domain.igtimiadapter.DataAccessWindow;
import com.sap.sailing.domain.igtimiadapter.DataPointTimePointExtractor;
import com.sap.sailing.domain.igtimiadapter.DataPointVisitor;
import com.sap.sailing.domain.igtimiadapter.Device;
import com.sap.sailing.domain.igtimiadapter.IgtimiConnection;
import com.sap.sailing.domain.igtimiadapter.IgtimiConnectionFactory;
import com.sap.sailing.domain.igtimiadapter.Resource;
import com.sap.sailing.domain.igtimiadapter.datatypes.Type;
import com.sap.sailing.domain.igtimiadapter.persistence.DomainObjectFactory;
import com.sap.sailing.domain.igtimiadapter.persistence.MongoObjectFactory;
import com.sap.sailing.domain.igtimiadapter.server.Activator;
@@ -510,14 +517,24 @@ public class RiotServerImpl extends AbstractReplicableWithObjectInputStream<Repl
}
@Override
public Iterable<Msg> getMessages(String deviceSerialNumber, TimeRange timeRange, Set<DataCase> dataCases) {
public Iterable<Msg> getMessages(String deviceSerialNumber, TimeRange timeRange, Set<DataCase> dataCases) throws IllegalStateException, ClientProtocolException, IOException, ParseException {
final Iterable<Msg> result;
if (getMasterDescriptor() == null) {
result = domainObjectFactory.getMessages(deviceSerialNumber, timeRange, dataCases);
} else {
// TODO fetch the data from the primary/master "somehow"
result = Collections.emptySet();
logger.severe("TODO: fetch data from primary!!!");
final int masterPort = getMasterDescriptor().getServletPort();
final String masterHostname = getMasterDescriptor().getHostname();
final SecurityService securityService = Activator.getInstance().getSecurityService();
final String currentUserBearerToken = securityService == null ? null : securityService.getAccessToken(securityService.getCurrentUser().getName());
final IgtimiConnectionFactory igtimiConnectionFactory =
IgtimiConnectionFactory.create(new URL(masterPort==443?"https":"http", masterHostname, masterPort, "/"), currentUserBearerToken);
final IgtimiConnection connection = igtimiConnectionFactory.createConnection();
final Type[] types = new Type[dataCases.size()];
int i=0;
for (final DataCase dataCase : dataCases) {
types[i++] = Type.valueOf(dataCase.getNumber());
}
result = connection.getMessages(timeRange.from(), timeRange.to(), Collections.singleton(deviceSerialNumber), types);
}
return result;
}
@@ -9,6 +9,7 @@ import org.apache.http.client.ClientProtocolException;
import org.json.simple.JSONObject;
import org.json.simple.parser.ParseException;
import com.igtimi.IgtimiStream.Msg;
import com.sap.sailing.domain.igtimiadapter.datatypes.Fix;
import com.sap.sailing.domain.igtimiadapter.datatypes.Type;
import com.sap.sailing.domain.tracking.DynamicTrack;
@@ -65,6 +66,8 @@ public interface IgtimiConnection {
Type... types) throws IllegalStateException, ClientProtocolException, IOException,
ParseException;
Iterable<Msg> getMessages(TimePoint startTime, TimePoint endTime, Iterable<String> deviceSerialNumbers, Type[] types) throws IllegalStateException, ClientProtocolException, IOException, ParseException;
/**
* Shorthand for {@link #getResourceData(TimePoint, TimePoint, Iterable, Map)} where no compression is requested for
* any type. The fixes received are forwarded to the {@link BulkFixReceiver} <code>bulkFixReceiver</code> in one call.
@@ -107,8 +110,6 @@ public interface IgtimiConnection {
*/
Iterable<Device> getDevices() throws IllegalStateException, ClientProtocolException, IOException, ParseException;
Device getDeviceBySerialNumber(String serialNumber);
void removeDevice(Device existingDevice) throws ClientProtocolException, IOException, ParseException;
/**
@@ -135,7 +136,8 @@ public interface IgtimiConnection {
DataAccessWindow createDataAccessWindow(String deviceSerialNumber, TimePoint startTime, TimePoint endTime) throws ClientProtocolException, IOException, ParseException;
/**
* Finds all data access windows that have wind data for the time span around the race, loads their wind data and
* Finds al@Override
l data access windows that have wind data for the time span around the race, loads their wind data and
* {@link DynamicTrackedRace#recordWind(com.sap.sailing.domain.tracking.Wind, com.sap.sailing.domain.common.WindSource)
* records it} in the tracked races.
*
@@ -1,8 +1,14 @@
package com.sap.sailing.domain.igtimiadapter;
import java.io.IOException;
import java.net.URL;
import java.util.function.Supplier;
import org.apache.http.client.ClientProtocolException;
import org.json.simple.parser.ParseException;
import com.sap.sailing.domain.igtimiadapter.impl.Activator;
import com.sap.sailing.domain.igtimiadapter.impl.IgtimiConnectionFactoryImpl;
import com.sap.sse.security.SecurityService;
/**
@@ -65,4 +71,21 @@ public interface IgtimiConnectionFactory {
default IgtimiConnection createConnection(Supplier<String> bearerTokenSupplierIfHasNoCredentials) {
return hasCredentials() ? createConnection() : createConnection(bearerTokenSupplierIfHasNoCredentials.get());
}
/**
* @param defaultBearerToken
* Used when {@code null} is passed to {@link #createConnection(String)} as a bearer token. If this field
* is {@code null}, too, requests to the REST API will be made through the connections returned from
* {@link #createConnection(String)} without an authenticated user.
*/
static IgtimiConnectionFactory create(URL baseUrl, String defaultBearerToken) {
return new IgtimiConnectionFactoryImpl(baseUrl, defaultBearerToken);
}
/**
* Delivers the default connection factory as per the system property configuration read by this bundle's activator.
*/
static IgtimiConnectionFactory getDefault() throws ClientProtocolException, IllegalStateException, IOException, ParseException {
return Activator.getInstance().getConnectionFactory();
}
}
@@ -49,7 +49,7 @@ public class Activator implements BundleActivator {
private static Activator INSTANCE;
private final Future<IgtimiConnectionFactory> connectionFactory;
private final IgtimiConnectionFactory connectionFactory;
private final Future<IgtimiWindTrackerFactory> windTrackerFactory;
private FullyInitializedReplicableTracker<SecurityService> securityServiceServiceTracker;
private final ExecutorService executor = Executors.newSingleThreadExecutor(new ThreadFactoryWithPriority(Thread.NORM_PRIORITY, /* daemon */ true));
@@ -59,22 +59,17 @@ public class Activator implements BundleActivator {
logger.info(getClass().getName()+" constructor");
final URL baseUrl = new URL(System.getProperty(IGTIMI_BASE_URL_PROPERTY_NAME, IGTIMI_BASE_URL_DEFAULT));
logger.info("Using base URL "+baseUrl+" for the Igtimi REST API");
connectionFactory = executor.submit(new Callable<IgtimiConnectionFactory>() {
@Override
public IgtimiConnectionFactory call() {
logger.info("Creating IgtimiConnectionFactory");
final String defaultBearerToken = System.getProperty(IGTIMI_DEFAULT_BEARER_TOKEN_PROPERTY_NAME);
if (defaultBearerToken != null) {
logger.info("A default bearer token has been provided for authentication to the Igtimi REST API at "+baseUrl);
}
return new IgtimiConnectionFactoryImpl(baseUrl, defaultBearerToken);
}
});
logger.info("Creating IgtimiConnectionFactory");
final String defaultBearerToken = System.getProperty(IGTIMI_DEFAULT_BEARER_TOKEN_PROPERTY_NAME);
if (defaultBearerToken != null) {
logger.info("A default bearer token has been provided for authentication to the Igtimi REST API at "+baseUrl);
}
connectionFactory = new IgtimiConnectionFactoryImpl(baseUrl, defaultBearerToken);
windTrackerFactory = executor.submit(new Callable<IgtimiWindTrackerFactory>() {
@Override
public IgtimiWindTrackerFactory call() throws InterruptedException, ExecutionException {
logger.info("Creating IgtimiWindTrackerFactory");
return new IgtimiWindTrackerFactory(connectionFactory.get());
return new IgtimiWindTrackerFactory(connectionFactory);
}
});
}
@@ -91,7 +86,7 @@ public class Activator implements BundleActivator {
@Override
public void run() {
try {
context.registerService(IgtimiConnectionFactory.class, connectionFactory.get(), /* properties */ null);
context.registerService(IgtimiConnectionFactory.class, connectionFactory, /* properties */ null);
context.registerService(WindTrackerFactory.class, windTrackerFactory.get(), /* properties */ null);
context.registerService(IgtimiWindTrackerFactory.class, windTrackerFactory.get(), /* properties */ null);
} catch (InterruptedException | ExecutionException e) {
@@ -118,6 +113,10 @@ public class Activator implements BundleActivator {
}
}
public IgtimiConnectionFactory getConnectionFactory() {
return connectionFactory;
}
public IgtimiWindTrackerFactory getWindTrackerFactory() {
try {
return windTrackerFactory.get();
@@ -1,18 +1,13 @@
package com.sap.sailing.domain.igtimiadapter.impl;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import com.google.protobuf.InvalidProtocolBufferException;
import com.igtimi.IgtimiData.ApparentWindAngle;
import com.igtimi.IgtimiData.ApparentWindSpeed;
import com.igtimi.IgtimiData.CourseOverGround;
@@ -45,11 +40,10 @@ import com.sap.sailing.domain.igtimiadapter.datatypes.HDG;
import com.sap.sailing.domain.igtimiadapter.datatypes.HDGM;
import com.sap.sailing.domain.igtimiadapter.datatypes.Log;
import com.sap.sailing.domain.igtimiadapter.datatypes.SOG;
import com.sap.sailing.domain.igtimiadapter.datatypes.Type;
import com.sap.sse.common.TimePoint;
import com.sap.sse.common.Util;
import com.sap.sse.common.impl.DegreeBearingImpl;
import com.sap.sse.common.impl.MillisecondsTimePoint;
import com.sun.jersey.core.util.Base64;
/**
* Can convert JSON messages from the original Riot API, such as resource data or web socket
@@ -60,77 +54,25 @@ import com.sap.sse.common.impl.MillisecondsTimePoint;
*
*/
public class FixFactory {
private static final Logger logger = Logger.getLogger(FixFactory.class.getName());
public Iterable<Fix> createFixes(JSONObject sensorsJson) {
public Iterable<Fix> createFixes(JSONObject sensorsJson) throws InvalidProtocolBufferException {
final List<Fix> result = new ArrayList<>();
for (final Entry<Object, Object> e : sensorsJson.entrySet()) {
final String deviceSerialNumber = (String) e.getKey();
final JSONObject typesJson = (JSONObject) e.getValue();
Util.addAll(createFixesForTypes(deviceSerialNumber, typesJson), result);
final JSONArray messagesAsBase64 = (JSONArray) e.getValue();
Util.addAll(createFixesForTypes(deviceSerialNumber, messagesAsBase64), result);
}
return result;
}
private Iterable<Fix> createFixesForTypes(String deviceSerialNumber, JSONObject typesJson) {
private Iterable<Fix> createFixesForTypes(String deviceSerialNumber, JSONArray messagesAsBase64) throws InvalidProtocolBufferException {
final List<Fix> result = new ArrayList<>();
for (Entry<Object, Object> fixTypeAndFixesJson : typesJson.entrySet()) {
final String fixTypeAndOptionalSensorId = (String) fixTypeAndFixesJson.getKey();
final String[] fixTypeAndOptionalColonSeparatedSensorsSubId = fixTypeAndOptionalSensorId.split(":");
try {
int fixType = Integer.valueOf(fixTypeAndOptionalColonSeparatedSensorsSubId[0]);
JSONObject fixesJson = (JSONObject) fixTypeAndFixesJson.getValue();
JSONArray timePointsMillis = (JSONArray) fixesJson.get("t");
int fixIndex = 0;
for (Object timePointMillis : timePointsMillis) {
if (timePointMillis != null) {
TimePoint timePoint = new MillisecondsTimePoint(((Number) timePointMillis).longValue());
Map<Integer, Object> valuesPerSubindex = new HashMap<>();
int i=1;
JSONArray values;
while ((values=(JSONArray) fixesJson.get(""+i)) != null) {
valuesPerSubindex.put(i, values.get(fixIndex));
i++;
}
Sensor sensor = new SensorImpl(deviceSerialNumber,
fixTypeAndOptionalColonSeparatedSensorsSubId.length < 2 ? 0
: Long.valueOf(fixTypeAndOptionalColonSeparatedSensorsSubId[1]));
final Type ft = Type.valueOf(fixType);
if (ft != null) {
final Fix fix = createFix(sensor, ft, timePoint, valuesPerSubindex);
if (fix != null) {
result.add(fix);
fixIndex++;
}
}
}
}
} catch (NumberFormatException e) {
// maybe archived_resources?
logger.warning("Couldn't parse "+fixTypeAndOptionalColonSeparatedSensorsSubId[0]+
" into a number; ignoring this record");
}
for (final Object messageAsBase64 : messagesAsBase64) {
final Msg msg = Msg.parseFrom(Base64.decode(messageAsBase64.toString()));
Util.addAll(createFixes(msg), result);
}
return result;
}
/**
* @return {@code null} in case the fix parser cannot make sense of the {@code valuesPerSubindex}, e.g., because
* it's empty.
*/
private Fix createFix(Sensor sensor, Type fixType, TimePoint timePoint, Map<Integer, Object> valuesPerSubindex) {
try {
Constructor<? extends Fix> constructor = fixType.getFixClass().getConstructor(TimePoint.class, Sensor.class, Map.class);
Fix fix = constructor.newInstance(timePoint, sensor, valuesPerSubindex);
return fix;
} catch (NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
logger.log(Level.SEVERE, "Internal error trying to find fix constructor for fix type "+
fixType+" with class "+fixType.getFixClass()+" or problem creating fix from data "+valuesPerSubindex+": "+
e.getMessage());
return null;
}
}
public Iterable<Fix> createFixes(Msg message) {
final List<Fix> fixes = new ArrayList<>(); // the fixes extracted from the message
MsgVisitor.accept(message, new MsgVisitor() {
@@ -19,7 +19,10 @@ public class IgtimiConnectionFactoryImpl implements IgtimiConnectionFactory {
* @param baseUrl
* base URL of the service where Igtimi wind data can be requested from; example:
* {@code https://wind.sapsailing.com}
* @param defaultBearerToken if no explicit
* @param defaultBearerToken
* Used when {@code null} is passed to {@link #createConnection(String)} as a bearer token. If this field
* is {@code null}, too, requests to the REST API will be made through the connections returned from
* {@link #createConnection(String)} without an authenticated user.
*/
public IgtimiConnectionFactoryImpl(URL baseUrl, String defaultBearerToken) {
this.baseUrl = baseUrl;
@@ -13,6 +13,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.function.Function;
import java.util.logging.Logger;
import org.apache.http.client.ClientProtocolException;
@@ -25,6 +26,8 @@ import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.ParseException;
import com.google.protobuf.InvalidProtocolBufferException;
import com.igtimi.IgtimiStream.Msg;
import com.sap.sailing.declination.DeclinationService;
import com.sap.sailing.domain.igtimiadapter.BulkFixReceiver;
import com.sap.sailing.domain.igtimiadapter.DataAccessWindow;
@@ -47,6 +50,7 @@ import com.sap.sse.common.TimePoint;
import com.sap.sse.common.Util;
import com.sap.sse.common.impl.MillisecondsTimePoint;
import com.sap.sse.security.util.impl.SecuredServerImpl;
import com.sun.jersey.core.util.Base64;
public class IgtimiConnectionImpl extends SecuredServerImpl implements IgtimiConnection {
private static final Logger logger = Logger.getLogger(IgtimiConnectionImpl.class.getName());
@@ -91,8 +95,21 @@ public class IgtimiConnectionImpl extends SecuredServerImpl implements IgtimiCon
@Override
public Iterable<Fix> getResourceData(final TimePoint startTime, final TimePoint endTime,
Iterable<String> deviceSerialNumbers, Map<Type, Double> typeAndCompression) throws IllegalStateException, ClientProtocolException, IOException, ParseException {
return getResourceContent(startTime, endTime, deviceSerialNumbers, typeAndCompression,
resourceDataJson->{
try {
return new FixFactory().createFixes(resourceDataJson);
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException(e);
}
});
}
private <T> Iterable<T> getResourceContent(final TimePoint startTime, final TimePoint endTime,
Iterable<String> deviceSerialNumbers, Map<Type, Double> typeAndCompression,
Function<JSONObject, Iterable<T>> messageParser) throws IllegalStateException, ClientProtocolException, IOException, ParseException {
logger.info("Requested resource data from "+startTime+" to "+endTime+" for devices "+deviceSerialNumbers+" for types "+typeAndCompression);
final List<Fix> result = new ArrayList<>();
final List<T> result = new ArrayList<>();
// Cut interval into slices that are at most one week long. See also the discussion at
// http://bugzilla.sapsailing.com/bugzilla/show_bug.cgi?id=2002 that talks about a one-month limitation
// imposed by the Igtimi API
@@ -113,7 +130,7 @@ public class IgtimiConnectionImpl extends SecuredServerImpl implements IgtimiCon
+ " to " + windowEndTime + " from devices " + deviceSerialNumbers + ": " + error
+ (reason == null ? "" : ". Reason: " + reason));
}
Util.addAll(new FixFactory().createFixes(resourceDataJson), result);
Util.addAll(messageParser.apply(resourceDataJson), result);
windowStartTime = windowEndTime.plus(1);
}
return result;
@@ -128,6 +145,29 @@ public class IgtimiConnectionImpl extends SecuredServerImpl implements IgtimiCon
}
return getResourceData(startTime, endTime, deviceSerialNumbers, typeAndCompression);
}
@Override
public Iterable<Msg> getMessages(TimePoint startTime, TimePoint endTime, Iterable<String> deviceSerialNumbers, Type[] types) throws IllegalStateException, ClientProtocolException, IOException, ParseException {
final Map<Type, Double> typeAndCompression = new HashMap<>();
for (final Type type : types) {
typeAndCompression.put(type, 0.0);
}
return getResourceContent(startTime, endTime, deviceSerialNumbers, typeAndCompression,
resourceDataJson->{
final List<Msg> messages = new ArrayList<>();
for (final Entry<Object, Object> e : resourceDataJson.entrySet()) {
final JSONArray messagesAsBase64 = (JSONArray) e.getValue();
for (final Object msgAsBase64 : messagesAsBase64) {
try {
messages.add(Msg.parseFrom(Base64.decode(msgAsBase64.toString())));
} catch (InvalidProtocolBufferException e1) {
throw new RuntimeException(e1);
}
}
}
return messages;
});
}
@Override
public Iterable<Fix> getAndNotifyResourceData(TimePoint startTime, TimePoint endTime,
@@ -176,8 +216,8 @@ public class IgtimiConnectionImpl extends SecuredServerImpl implements IgtimiCon
@Override
public Iterable<Device> getDevices() throws IllegalStateException, ClientProtocolException, IOException, ParseException {
HttpGet getResources = new HttpGet(getDevicesUrl());
JSONObject devicesJson = (JSONObject) getJsonParsedResponse(getResources).getA();
final HttpGet getResources = new HttpGet(getDevicesUrl());
final JSONObject devicesJson = (JSONObject) getJsonParsedResponse(getResources).getA();
final List<Device> result = new ArrayList<>();
for (Object deviceJson : (JSONArray) devicesJson.get("devices")) {
Device device = new DeviceDeserializer().createDeviceFromJson((JSONObject) deviceJson);
@@ -186,15 +226,9 @@ public class IgtimiConnectionImpl extends SecuredServerImpl implements IgtimiCon
return result;
}
@Override
public Device getDeviceBySerialNumber(String serialNumber) {
// TODO Auto-generated method stub
return null;
}
@Override
public void removeDevice(Device existingDevice) throws ClientProtocolException, IOException, ParseException {
HttpDelete getResources = new HttpDelete(getDeleteDeviceUrl(existingDevice.getId()));
final HttpDelete getResources = new HttpDelete(getDeleteDeviceUrl(existingDevice.getId()));
if (getJsonParsedResponse(getResources).getB() >= 400) {
throw new RuntimeException("Error deleting device with ID "+existingDevice.getId());
}