diff --git a/java/com.sap.sailing.domain.persistence/src/com/sap/sailing/domain/persistence/FieldNames.java b/java/com.sap.sailing.domain.persistence/src/com/sap/sailing/domain/persistence/FieldNames.java index cc3952eb2bb..7216f999d8c 100644 --- a/java/com.sap.sailing.domain.persistence/src/com/sap/sailing/domain/persistence/FieldNames.java +++ b/java/com.sap.sailing.domain.persistence/src/com/sap/sailing/domain/persistence/FieldNames.java @@ -206,6 +206,6 @@ public enum FieldNames { MANEUVER_DISTANCE_SAILED_POMA, MANEUVER_DISTANCE_SAILED_INMPOMA, MANEUVER_SPEED_WITH_BEARING_BEFORE_DEGREES, MANEUVER_SPEED_WITH_BEARING_BEFORE_SPEED, MANEUVER_SPEED_WITH_BEARING_AFTER_DEGREES, MANEUVER_SPEED_WITH_BEARING_AFTER_SPEED_IN_KNOTS, MANEUVER_START_POSITION_LAT_RAD, MANEUVER_START_POSITION_LNG_RAD, MANEUVER_END_POSITION_LAT_RAD, - MANEUVER_END_POSITION_LNG_RAD, MIDDLE_MAEUVER_ANGLE, MANEUVER_LOSS_DURATION + MANEUVER_END_POSITION_LNG_RAD, MIDDLE_MAEUVER_ANGLE, MANEUVER_LOSS_DURATION, MANEUVER_PAGE_INDEX ; } diff --git a/java/com.sap.sailing.domain.persistence/src/com/sap/sailing/domain/persistence/impl/DomainObjectFactoryImpl.java b/java/com.sap.sailing.domain.persistence/src/com/sap/sailing/domain/persistence/impl/DomainObjectFactoryImpl.java index fd5304c0a75..ae21aafba60 100644 --- a/java/com.sap.sailing.domain.persistence/src/com/sap/sailing/domain/persistence/impl/DomainObjectFactoryImpl.java +++ b/java/com.sap.sailing.domain.persistence/src/com/sap/sailing/domain/persistence/impl/DomainObjectFactoryImpl.java @@ -3309,13 +3309,19 @@ public class DomainObjectFactoryImpl implements DomainObjectFactory { @Override public Map loadFingerprintsForManeuverHashes() { final MongoCollection maneuversCollection = database.getCollection(CollectionNames.MANEUVERS.name()); + try { + maneuversCollection.dropIndex("maneuversbyeventraceandcompetitor"); + } catch (final Exception e) { + // index does not exist yet, nothing to drop + } maneuversCollection.createIndex(new Document() .append(FieldNames.EVENT_NAME.name(), 1) .append(FieldNames.RACE_NAME.name(), 1) - .append(FieldNames.COMPETITOR_ID.name(), 1), + .append(FieldNames.COMPETITOR_ID.name(), 1) + .append(FieldNames.MANEUVER_PAGE_INDEX.name(), 1), new IndexOptions() .unique(true) - .name("maneuversbyeventraceandcompetitor") + .name("maneuversbyeventracecompetitorandpage") .background(false)); final Map fingerprintHashMap = new HashMap<>(); for (final Document currentDocument : maneuversCollection.find()) { diff --git a/java/com.sap.sailing.domain.persistence/src/com/sap/sailing/domain/persistence/impl/MongoObjectFactoryImpl.java b/java/com.sap.sailing.domain.persistence/src/com/sap/sailing/domain/persistence/impl/MongoObjectFactoryImpl.java index 0d1c1b69b27..60e5e04fc7f 100644 --- a/java/com.sap.sailing.domain.persistence/src/com/sap/sailing/domain/persistence/impl/MongoObjectFactoryImpl.java +++ b/java/com.sap.sailing.domain.persistence/src/com/sap/sailing/domain/persistence/impl/MongoObjectFactoryImpl.java @@ -171,6 +171,7 @@ import com.sap.sse.shared.media.VideoDescriptor; public class MongoObjectFactoryImpl implements MongoObjectFactory { private static Logger logger = Logger.getLogger(MongoObjectFactoryImpl.class.getName()); + private static final int MANEUVERS_PER_PAGE = 1000; private final MongoDatabase database; private final CompetitorWithBoatRefJsonSerializer competitorWithBoatRefSerializer = CompetitorWithBoatRefJsonSerializer.create(/* serializeNonPublicCompetitorFields */ true); private final CompetitorJsonSerializer competitorSerializer = CompetitorJsonSerializer.create( @@ -2071,24 +2072,35 @@ public class MongoObjectFactoryImpl implements MongoObjectFactory { final JSONObject fingerprintjson = fingerprint.toJson(); final Document fingerprintDoc = Document.parse(fingerprintjson.toString()); for (final Entry> e : maneuvers.entrySet()) { - storeCompetitorManeuvers(maneuverCollection, raceIdentifier, fingerprintDoc, course, e.getKey(), e.getValue()); + final List competitorManeuvers = e.getValue() != null ? e.getValue() : new ArrayList<>(); + final int pageCount = Math.max(1, (int) Math.ceil((double) competitorManeuvers.size() / MANEUVERS_PER_PAGE)); + if (pageCount > 1) { + logger.warning("Competitor " + e.getKey().getName() + " in race " + raceIdentifier + + " has " + competitorManeuvers.size() + " maneuvers, splitting into " + pageCount + + " documents of up to " + MANEUVERS_PER_PAGE + " maneuvers each (bug 6226)."); + } + for (int pageIndex = 0; pageIndex < pageCount; pageIndex++) { + final int from = pageIndex * MANEUVERS_PER_PAGE; + final int to = Math.min(from + MANEUVERS_PER_PAGE, competitorManeuvers.size()); + storeCompetitorManeuvers(maneuverCollection, raceIdentifier, fingerprintDoc, course, e.getKey(), competitorManeuvers.subList(from, to), pageIndex); + } } } private void storeCompetitorManeuvers(MongoCollection maneuverCollection, RaceIdentifier raceIdentifier, - Document fingerprintDoc, Course course, Competitor competitor, List competitorManeuvers) { + Document fingerprintDoc, Course course, Competitor competitor, List competitorManeuvers, final int pageIndex) { final Document query = new Document(); DomainObjectFactoryImpl.addRaceIdentifierToQuery(query, raceIdentifier); query.put(FieldNames.COMPETITOR_ID.name(), competitor.getId()); + query.put(FieldNames.MANEUVER_PAGE_INDEX.name(), pageIndex); final Document result = new Document(); result.put(FieldNames.MANEUVER_FINGERPRINT.name(), fingerprintDoc); storeRaceIdentifier(result, raceIdentifier); result.put(FieldNames.COMPETITOR_ID.name(), competitor.getId()); + result.put(FieldNames.MANEUVER_PAGE_INDEX.name(), pageIndex); final List maneuverList = new ArrayList<>(); - if (competitorManeuvers != null) { - for (final Maneuver maneuver : competitorManeuvers) { - maneuverList.add(generateManeuverDoc(maneuver, course)); - } + for (final Maneuver maneuver : competitorManeuvers) { + maneuverList.add(generateManeuverDoc(maneuver, course)); } result.put(FieldNames.MANEUVERS.name(), maneuverList); try { diff --git a/java/com.sap.sailing.mongodb.test/src/com/sap/sailing/mongodb/test/ManeuverStorageLimitTest.java b/java/com.sap.sailing.mongodb.test/src/com/sap/sailing/mongodb/test/ManeuverStorageLimitTest.java index 01991cce289..f977aa840ed 100644 --- a/java/com.sap.sailing.mongodb.test/src/com/sap/sailing/mongodb/test/ManeuverStorageLimitTest.java +++ b/java/com.sap.sailing.mongodb.test/src/com/sap/sailing/mongodb/test/ManeuverStorageLimitTest.java @@ -1,5 +1,6 @@ package com.sap.sailing.mongodb.test; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -15,6 +16,7 @@ import org.json.simple.JSONObject; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import com.mongodb.client.FindIterable; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; @@ -26,6 +28,7 @@ import com.sap.sailing.domain.common.ManeuverType; import com.sap.sailing.domain.common.RegattaNameAndRaceName; import com.sap.sailing.domain.common.Tack; import com.sap.sailing.domain.maneuverhash.ManeuverRaceFingerprint; +import com.sap.sailing.domain.persistence.FieldNames; import com.sap.sailing.domain.persistence.impl.MongoObjectFactoryImpl; import com.sap.sailing.domain.tracking.Maneuver; import com.sap.sailing.domain.tracking.ManeuverCurveBoundaries; @@ -40,19 +43,17 @@ import com.sap.sse.mongodb.MongoDBConfiguration; /** * Regression test for bug 6226 (https://github.com/eclipse-sailing-analytics/sailing-analytics/issues/6226): - * {@link BsonMaximumSizeExceededException} storing maneuvers for large races with many competitors. + * {@link BsonMaximumSizeExceededException} storing maneuvers for large races. *

* - * The old implementation stored all competitors' maneuvers in a single MongoDB document, which exceeded the 16MB BSON - * limit for races with ~163 competitors. The fix stores one document per competitor instead. - *

- * - * This test uses a synthetic dataset (200 competitors x 500 maneuvers each) to trigger the original issue and verify - * that the new implementation handles it correctly. + * The original fix stored one document per competitor instead of one per race. This test also covers the follow-up + * case: a single competitor with enough maneuvers to exceed 16MB in one document. The implementation paginates + * across multiple documents keyed by (EVENT_NAME, RACE_NAME, COMPETITOR_ID, MANEUVER_PAGE_INDEX). */ public class ManeuverStorageLimitTest { private static final int COMPETITOR_COUNT = 200; private static final int MANEUVERS_PER_COMPETITOR = 500; + private static final int MANEUVERS_PER_PAGE = 1000; private MongoDBConfiguration dbConfiguration; private MongoDatabase database; @@ -66,9 +67,7 @@ public class ManeuverStorageLimitTest { } /** - * Verifies that the new per-competitor document storage does not exceed MongoDB's 16MB limit - * even for a large race (200 competitors, 500 maneuvers each). - * Verifies document count equals competitor count after storing. + * Verifies that competitors with fewer than 1000 maneuvers each produce one document per competitor. */ @Test public void testNewImplementationStoresOneDocumentPerCompetitor() { @@ -78,11 +77,58 @@ public class ManeuverStorageLimitTest { final Course course = mock(Course.class); new MongoObjectFactoryImpl(database).storeManeuvers(raceIdentifier, fingerprint, course, maneuvers); final MongoCollection collection = database.getCollection("MANEUVERS"); - final long documentCount = collection.countDocuments(); - assertEquals(COMPETITOR_COUNT, documentCount, + assertEquals(COMPETITOR_COUNT, collection.countDocuments(), "Expected one document per competitor in the MANEUVERS collection"); } + /** + * Verifies that a competitor with 15000 maneuvers (enough to exceed the 16MB BSON limit in one document) + * is split into 15 pages of 1000 maneuvers each, and all maneuvers are stored successfully. + */ + @Test + public void testPaginationSplitsLargeCompetitorAcrossMultipleDocuments() { + final int totalManeuvers = 15000; + final RegattaNameAndRaceName raceIdentifier = new RegattaNameAndRaceName("505 Pre-Worlds 2014 synthetic", "Race 1"); + final Competitor competitor = mock(CompetitorImpl.class); + when(competitor.getId()).thenReturn("competitor-0"); + final Map> maneuvers = new HashMap<>(); + maneuvers.put(competitor, buildManeuverList(totalManeuvers)); + final ManeuverRaceFingerprint fingerprint = buildMockFingerprint(); + final Course course = mock(Course.class); + new MongoObjectFactoryImpl(database).storeManeuvers(raceIdentifier, fingerprint, course, maneuvers); + final MongoCollection collection = database.getCollection("MANEUVERS"); + final int expectedPages = (int) Math.ceil((double) totalManeuvers / MANEUVERS_PER_PAGE); + assertEquals(expectedPages, collection.countDocuments(), "Expected " + expectedPages + " page documents for one competitor with " + totalManeuvers + " maneuvers"); + final FindIterable docs = collection.find(new Document(FieldNames.COMPETITOR_ID.name(), "competitor-0")); + int totalManeuversLoaded = 0; + for (final Document doc : docs) { + final List page = doc.getList(FieldNames.MANEUVERS.name(), Document.class); + totalManeuversLoaded += page != null ? page.size() : 0; + } + assertEquals(totalManeuvers, totalManeuversLoaded, "Expected all maneuvers to be stored across pages"); + } + + /** + * Demonstrates that the pre-pagination implementation fails with BsonMaximumSizeExceededException + * for a competitor with 15000 maneuvers (matching Axel's real-world case of 18,270). The old code + * stored all maneuvers in one document with no page splitting, reproduced here via + * OldSinglePageMongoObjectFactoryForBug6226Test. + */ + @Test + public void testOldSinglePageImplementationFailsWithBsonLimitExceeded() { + final RegattaNameAndRaceName raceIdentifier = new RegattaNameAndRaceName("505 Pre-Worlds 2014 synthetic", "Race 1"); + final Competitor competitor = mock(CompetitorImpl.class); + when(competitor.getId()).thenReturn("competitor-0"); + final Map> maneuvers = new HashMap<>(); + maneuvers.put(competitor, buildManeuverList(15000)); + final ManeuverRaceFingerprint fingerprint = buildMockFingerprint(); + final Course course = mock(Course.class); + assertThrows(BsonMaximumSizeExceededException.class, () -> + new OldSinglePageMongoObjectFactoryForBug6226Test(database).storeManeuvers(raceIdentifier, fingerprint, course, maneuvers), + "Old single-page implementation must throw BsonMaximumSizeExceededException for a competitor with 2500 maneuvers" + ); + } + private ManeuverRaceFingerprint buildMockFingerprint() { final ManeuverRaceFingerprint fingerprint = mock(ManeuverRaceFingerprint.class); final JSONObject fingerprintJson = new JSONObject(); diff --git a/java/com.sap.sailing.mongodb.test/src/com/sap/sailing/mongodb/test/OldSinglePageMongoObjectFactoryForBug6226Test.java b/java/com.sap.sailing.mongodb.test/src/com/sap/sailing/mongodb/test/OldSinglePageMongoObjectFactoryForBug6226Test.java new file mode 100644 index 00000000000..76eababc7e0 --- /dev/null +++ b/java/com.sap.sailing.mongodb.test/src/com/sap/sailing/mongodb/test/OldSinglePageMongoObjectFactoryForBug6226Test.java @@ -0,0 +1,100 @@ +package com.sap.sailing.mongodb.test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.bson.Document; + +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.ReplaceOptions; +import com.sap.sailing.domain.base.Competitor; +import com.sap.sailing.domain.base.Course; +import com.sap.sailing.domain.common.RaceIdentifier; +import com.sap.sailing.domain.maneuverhash.ManeuverRaceFingerprint; +import com.sap.sailing.domain.persistence.FieldNames; +import com.sap.sailing.domain.persistence.impl.CollectionNames; +import com.sap.sailing.domain.tracking.Maneuver; +import com.sap.sailing.domain.tracking.ManeuverCurveBoundaries; + +/** + * Reproduces the pre-pagination behaviour of {@code MongoObjectFactoryImpl.storeCompetitorManeuvers()}: + * all maneuvers for a competitor were stored in a single document with no page splitting. For competitors + * with very large maneuver counts this document exceeds MongoDB's 16MB BSON limit. + * + * This class exists solely to let {@link ManeuverStorageLimitTest} assert that the old single-page code + * path fails with {@code BsonMaximumSizeExceededException} for a competitor with 2500 maneuvers. + * It must never be used in production. + */ +class OldSinglePageMongoObjectFactoryForBug6226Test { + private final MongoDatabase database; + + OldSinglePageMongoObjectFactoryForBug6226Test(final MongoDatabase database) { + this.database = database; + } + + void storeManeuvers(final RaceIdentifier raceIdentifier, final ManeuverRaceFingerprint fingerprint, + final Course course, final Map> maneuvers) { + final MongoCollection maneuverCollection = database.getCollection(CollectionNames.MANEUVERS.name()); + final Document fingerprintDoc = Document.parse(fingerprint.toJson().toString()); + for (final Entry> e : maneuvers.entrySet()) { + storeCompetitorManeuversSinglePage(maneuverCollection, raceIdentifier, fingerprintDoc, course, e.getKey(), e.getValue()); + } + } + + private void storeCompetitorManeuversSinglePage(final MongoCollection maneuverCollection, + final RaceIdentifier raceIdentifier, final Document fingerprintDoc, final Course course, + final Competitor competitor, final List competitorManeuvers) { + final Document query = new Document(); + query.put(FieldNames.EVENT_NAME.name(), raceIdentifier.getRegattaName()); + query.put(FieldNames.RACE_NAME.name(), raceIdentifier.getRaceName()); + query.put(FieldNames.COMPETITOR_ID.name(), competitor.getId()); + final Document result = new Document(); + result.put(FieldNames.MANEUVER_FINGERPRINT.name(), fingerprintDoc); + result.put(FieldNames.EVENT_NAME.name(), raceIdentifier.getRegattaName()); + result.put(FieldNames.RACE_NAME.name(), raceIdentifier.getRaceName()); + result.put(FieldNames.COMPETITOR_ID.name(), competitor.getId()); + final List maneuverList = new ArrayList<>(); + if (competitorManeuvers != null) { + for (final Maneuver maneuver : competitorManeuvers) { + maneuverList.add(buildManeuverDoc(maneuver)); + } + } + result.put(FieldNames.MANEUVERS.name(), maneuverList); + maneuverCollection.replaceOne(query, result, new ReplaceOptions().upsert(true)); + } + + private Document buildManeuverDoc(final Maneuver maneuver) { + final Document doc = new Document(); + doc.put(FieldNames.SIMPLE_CLASS_NAME.name(), maneuver.getClass().getSimpleName()); + doc.put(FieldNames.TYPE.name(), maneuver.getType().name()); + doc.put(FieldNames.TACK.name(), maneuver.getNewTack() == null ? null : maneuver.getNewTack().name()); + doc.put(FieldNames.POSITION_LAT_RAD.name(), maneuver.getPosition().getLatRad()); + doc.put(FieldNames.POSITION_LNG_RAD.name(), maneuver.getPosition().getLngRad()); + doc.put(FieldNames.TIMEPOINT.name(), maneuver.getTimePoint().asMillis()); + doc.put(FieldNames.MAIN_CURVE_BOUNDARIES.name(), buildCurveBoundariesDoc(maneuver.getMainCurveBoundaries())); + doc.put(FieldNames.MANEUVER_CURVE_WITH_STABLE_SPEED_AND_COURSE_BOUNDERIES.name(), + buildCurveBoundariesDoc(maneuver.getManeuverCurveWithStableSpeedAndCourseBoundaries())); + doc.put(FieldNames.MAX_TURNING_RATE_IN_DEGREE_PER_SECOUND.name(), maneuver.getMaxTurningRateInDegreesPerSecond()); + doc.put(FieldNames.INDEX_OF_PASSED_WAYPOINT.name(), -1); + doc.put(FieldNames.TIME_AS_MILLIS.name(), maneuver.getDuration().asMillis()); + doc.put(FieldNames.MANEUVER_LOSS.name(), null); + return doc; + } + + private Document buildCurveBoundariesDoc(final ManeuverCurveBoundaries f) { + final Document d = new Document(); + d.put(FieldNames.MANEUVER_TIMEPOINT_BEFORE.name(), f.getTimePointBefore().asMillis()); + d.put(FieldNames.MANEUVER_TIMEPOINT_AFTER.name(), f.getTimePointAfter().asMillis()); + d.put(FieldNames.MANEUVER_SPEED_WITH_BEARING_BEFORE_DEGREES.name(), f.getSpeedWithBearingBefore().getBearing().getDegrees()); + d.put(FieldNames.MANEUVER_SPEED_WITH_BEARING_BEFORE_SPEED.name(), f.getSpeedWithBearingBefore().getKnots()); + d.put(FieldNames.MANEUVER_SPEED_WITH_BEARING_AFTER_DEGREES.name(), f.getSpeedWithBearingAfter().getBearing().getDegrees()); + d.put(FieldNames.MANEUVER_SPEED_WITH_BEARING_AFTER_SPEED_IN_KNOTS.name(), f.getSpeedWithBearingAfter().getKnots()); + d.put(FieldNames.MANEUVER_DIRECTION_CHANGE_IN_DEGREES.name(), f.getDirectionChangeInDegrees()); + d.put(FieldNames.MANEUVER_LOWEST_SPEED_IN_KNOTS.name(), f.getLowestSpeed().getKnots()); + d.put(FieldNames.MANEUVER_HIGHEST_SPEED_IN_KNOTS.name(), f.getHighestSpeed().getKnots()); + return d; + } +} diff --git a/java/com.sap.sailing.www/release_notes_admin.html b/java/com.sap.sailing.www/release_notes_admin.html index d47b4cd22db..8053961f30b 100755 --- a/java/com.sap.sailing.www/release_notes_admin.html +++ b/java/com.sap.sailing.www/release_notes_admin.html @@ -33,6 +33,12 @@ for any typical even seen so far. See also bug 6226. +

  • Further fix for the same issue: a competitor tracked over an unusually long period could accumulate + enough maneuvers to exceed the 16MB limit in their own document. Maneuver documents are now paginated — + each document holds at most 1000 maneuvers, with additional pages stored as separate documents. + A warning is logged when pagination kicks in. + See also bug 6226. +

    • May 2026