diff --git a/README.md b/README.md index c69d73ce555..11963296bb1 100644 --- a/README.md +++ b/README.md @@ -46,13 +46,13 @@ Connect to your server at ``http://localhost:8888`` and find its administration ## Docker -To build a docker image, try ``docker/makeImageForLatestRelease``. The upload to the default (private) Dockerhub repository will usually fail unless you are a collaborator for that repository, but you should see a local image tagged ``docker.sapsailing.com:443/sapsailing:...`` result from the build. To run that docker image, try something like +To build a docker image, try ``docker/makeImageForLatestRelease``. The upload to the default (private) Dockerhub repository will usually fail unless you are a collaborator for that repository, but you should see a local image tagged ``docker.sapsailing.com/sapsailing:...`` result from the build. To run that docker image, try something like ``` docker run -d -e "MEMORY=4g" -e "MONGODB_URI=mongodb://my.mongohost.org?replicaSet=rs0&retryWrites=true" -P ``` Do a "docker ps" to figure out the port exposing the web application: -CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 79f6faf19b6a docker.sapsailing.com:443/sapsailing:latest "/home/sailing/serve…" 33 seconds ago Up 32 seconds 0.0.0.0:32782->6666/tcp, 0.0.0.0:32781->7091/tcp, 0.0.0.0:32780->8000/tcp, 0.0.0.0:32779->8888/tcp, 0.0.0.0:32778->14888/tcp modest_dhawan +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 79f6faf19b6a docker.sapsailing.com/sapsailing:latest "/home/sailing/serve…" 33 seconds ago Up 32 seconds 0.0.0.0:32782->6666/tcp, 0.0.0.0:32781->7091/tcp, 0.0.0.0:32780->8000/tcp, 0.0.0.0:32779->8888/tcp, 0.0.0.0:32778->14888/tcp modest_dhawan In this example, find your web application at http://localhost:32779 which is where the port 8888 exposed by the application is exposed at on your host. In the example with telnet port 14888 mapped to localhost:32788 do a ``` @@ -62,7 +62,7 @@ to connect to the server's OSGi console. ## Docker Compose -If you have built or obtained the ``docker.sapsailing.com:443/sapsailing:latest`` image, try this: +If you have built or obtained the ``docker.sapsailing.com/sapsailing:latest`` image, try this: ``` cd docker docker-compose up diff --git a/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/sharding/ShardingType.java b/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/sharding/ShardingType.java index 1ed04cadfaa..5b0a0790e83 100644 --- a/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/sharding/ShardingType.java +++ b/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/sharding/ShardingType.java @@ -28,15 +28,10 @@ public enum ShardingType { this.prefix = prefix; } - public String encodeIfNeeded(String shardingInfo) { - if (shardingInfo.startsWith(prefix)) { - return shardingInfo; - } - return new StringBuilder().append(prefix) - .append(normalize(shardingInfo)) - .toString(); + public String encodeShardingInfo(String shardingInfo) { + return new StringBuilder().append(prefix).append(normalize(shardingInfo)).toString(); } - + private String normalize(String replace) { char[] chars = replace.toCharArray(); StringBuilder answer = new StringBuilder(); diff --git a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/sharding/ShardingContext.java b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/sharding/ShardingContext.java index e98f0873c65..05185725652 100644 --- a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/sharding/ShardingContext.java +++ b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/sharding/ShardingContext.java @@ -59,7 +59,7 @@ public class ShardingContext { * @param shardingInfo */ public static void setShardingConstraint(ShardingType shardingType, String shardingInfo) { - final String encodedShardingInfo = shardingType.encodeIfNeeded(shardingInfo); + final String encodedShardingInfo = shardingType.encodeShardingInfo(shardingInfo); ThreadLocal shardingHolder = shardingMap.computeIfAbsent(shardingType, t -> new ThreadLocal<>()); checkAndSetShardingInfo(shardingType, encodedShardingInfo, shardingHolder); } @@ -89,7 +89,7 @@ public class ShardingContext { logger.warning("No current sharding constraint for " + type.name()); return; } - final String encodedShardingInfo = type.encodeIfNeeded(shardingInfo); + final String encodedShardingInfo = type.encodeShardingInfo(shardingInfo); if (!encodedShardingInfo.equals(currentShardingInfo)) { logger.log(Level.SEVERE, "Current sharding constraint vialation for " + type.name() + ". Got " + shardingInfo + ", shard requires " + currentShardingInfo, new RuntimeException()); diff --git a/java/com.sap.sailing.gwt.ui/META-INF/MANIFEST.MF b/java/com.sap.sailing.gwt.ui/META-INF/MANIFEST.MF index 46ec09606e7..47f216e9f35 100644 --- a/java/com.sap.sailing.gwt.ui/META-INF/MANIFEST.MF +++ b/java/com.sap.sailing.gwt.ui/META-INF/MANIFEST.MF @@ -89,6 +89,7 @@ Import-Package: javax.servlet;version="3.1.0", org.osgi.framework, org.osgi.util.tracker Web-ContextPath: /gwt +Comment: The Web-ContextPath without the leading / must match com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants.WEB_CONTEXT_PATH Bundle-ClassPath: WEB-INF/classes/ TrackLifecycle: true Export-Package: com.google.gwt.user.client.rpc.core.com.sap.sailing.datamining.shared, diff --git a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/autoplay/client/app/AutoPlayClientFactoryBase.java b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/autoplay/client/app/AutoPlayClientFactoryBase.java index 05c86182d17..092365c2dae 100644 --- a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/autoplay/client/app/AutoPlayClientFactoryBase.java +++ b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/autoplay/client/app/AutoPlayClientFactoryBase.java @@ -1,6 +1,6 @@ package com.sap.sailing.gwt.autoplay.client.app; -import static com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants.mediaServiceRemotePath; +import static com.sap.sailing.landscape.common.RemoteServiceMappingConstants.mediaServiceRemotePath; import static com.sap.sse.common.HttpRequestHeaderConstants.HEADER_FORWARD_TO_MASTER; import static com.sap.sse.common.HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA; @@ -18,10 +18,10 @@ import com.sap.sailing.gwt.ui.client.MediaService; import com.sap.sailing.gwt.ui.client.MediaServiceAsync; import com.sap.sailing.gwt.ui.client.MediaServiceWrite; import com.sap.sailing.gwt.ui.client.MediaServiceWriteAsync; -import com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants; import com.sap.sailing.gwt.ui.client.SailingServiceAsync; import com.sap.sailing.gwt.ui.client.SailingServiceHelper; import com.sap.sailing.gwt.ui.client.SailingServiceWriteAsync; +import com.sap.sailing.landscape.common.RemoteServiceMappingConstants; import com.sap.sse.gwt.client.EntryPointHelper; import com.sap.sse.security.ui.client.SecureClientFactoryImpl; diff --git a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/common/communication/routing/ProvidesLeaderboardRouting.java b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/common/communication/routing/ProvidesLeaderboardRouting.java index 3c224cd0998..7742bf6948d 100644 --- a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/common/communication/routing/ProvidesLeaderboardRouting.java +++ b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/common/communication/routing/ProvidesLeaderboardRouting.java @@ -11,6 +11,6 @@ public interface ProvidesLeaderboardRouting extends ServiceRoutingProvider { String getLeaderboardName(); default String routingSuffixPath() { - return ShardingType.LEADERBOARDNAME.encodeIfNeeded(getLeaderboardName()); + return ShardingType.LEADERBOARDNAME.encodeShardingInfo(getLeaderboardName()); } } diff --git a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/communication/SailingDispatchSystemImpl.java b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/communication/SailingDispatchSystemImpl.java index 0becb810879..42158ad6d5a 100644 --- a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/communication/SailingDispatchSystemImpl.java +++ b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/communication/SailingDispatchSystemImpl.java @@ -1,6 +1,6 @@ package com.sap.sailing.gwt.home.communication; -import com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants; +import com.sap.sailing.landscape.common.RemoteServiceMappingConstants; import com.sap.sse.gwt.dispatch.client.system.DispatchSystemDefaultImpl; public class SailingDispatchSystemImpl extends DispatchSystemDefaultImpl implements diff --git a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/desktop/DesktopEntryPoint.java b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/desktop/DesktopEntryPoint.java index 64abf744423..f8067933031 100644 --- a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/desktop/DesktopEntryPoint.java +++ b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/desktop/DesktopEntryPoint.java @@ -13,10 +13,10 @@ import com.sap.sailing.gwt.home.desktop.app.DesktopClientFactory; import com.sap.sailing.gwt.home.desktop.app.TabletAndDesktopApplicationClientFactory; import com.sap.sailing.gwt.home.shared.SwitchingEntryPoint; import com.sap.sailing.gwt.home.shared.app.ApplicationHistoryMapper; -import com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants; import com.sap.sailing.gwt.ui.client.ServerConfigurationService; import com.sap.sailing.gwt.ui.client.ServerConfigurationServiceAsync; import com.sap.sailing.gwt.ui.client.StringMessages; +import com.sap.sailing.landscape.common.RemoteServiceMappingConstants; import com.sap.sse.gwt.client.EntryPointHelper; import com.sap.sse.gwt.client.mvp.AbstractMvpEntryPoint; diff --git a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/desktop/app/AbstractApplicationClientFactory.java b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/desktop/app/AbstractApplicationClientFactory.java index 6fdcd78eee9..8c1a0d68f6e 100755 --- a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/desktop/app/AbstractApplicationClientFactory.java +++ b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/desktop/app/AbstractApplicationClientFactory.java @@ -1,6 +1,6 @@ package com.sap.sailing.gwt.home.desktop.app; -import static com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants.mediaServiceRemotePath; +import static com.sap.sailing.landscape.common.RemoteServiceMappingConstants.mediaServiceRemotePath; import static com.sap.sse.common.HttpRequestHeaderConstants.HEADER_FORWARD_TO_MASTER; import static com.sap.sse.common.HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA; diff --git a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/mobile/MobileEntryPoint.java b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/mobile/MobileEntryPoint.java index 39f5042f2ba..f6903c2482b 100644 --- a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/mobile/MobileEntryPoint.java +++ b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/home/mobile/MobileEntryPoint.java @@ -13,12 +13,12 @@ import com.sap.sailing.gwt.home.shared.SwitchingEntryPoint; import com.sap.sailing.gwt.home.shared.app.ApplicationHistoryMapper; import com.sap.sailing.gwt.home.shared.app.ResettableNavigationPathDisplay; import com.sap.sailing.gwt.home.shared.app.SailingActivityManager; -import com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants; import com.sap.sailing.gwt.ui.client.SailingServiceAsync; import com.sap.sailing.gwt.ui.client.SailingServiceHelper; import com.sap.sailing.gwt.ui.client.ServerConfigurationService; import com.sap.sailing.gwt.ui.client.ServerConfigurationServiceAsync; import com.sap.sailing.gwt.ui.client.StringMessages; +import com.sap.sailing.landscape.common.RemoteServiceMappingConstants; import com.sap.sse.gwt.client.EntryPointHelper; import com.sap.sse.gwt.client.ServiceRoutingProvider; import com.sap.sse.gwt.client.mvp.AbstractMvpEntryPoint; diff --git a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/managementconsole/app/ManagementConsoleClientFactoryImpl.java b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/managementconsole/app/ManagementConsoleClientFactoryImpl.java index e426a8115f0..a9c63f52458 100644 --- a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/managementconsole/app/ManagementConsoleClientFactoryImpl.java +++ b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/managementconsole/app/ManagementConsoleClientFactoryImpl.java @@ -10,9 +10,9 @@ import com.google.gwt.user.client.rpc.ServiceDefTarget; import com.google.web.bindery.event.shared.EventBus; import com.sap.sailing.gwt.ui.client.MediaServiceWrite; import com.sap.sailing.gwt.ui.client.MediaServiceWriteAsync; -import com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants; import com.sap.sailing.gwt.ui.client.SailingServiceWriteAsync; import com.sap.sailing.gwt.ui.shared.ServerConfigurationDTO; +import com.sap.sailing.landscape.common.RemoteServiceMappingConstants; import com.sap.sse.gwt.client.DefaultErrorReporter; import com.sap.sse.gwt.client.EntryPointHelper; import com.sap.sse.gwt.client.ErrorReporter; diff --git a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/adminconsole/AdminConsoleClientFactoryImpl.java b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/adminconsole/AdminConsoleClientFactoryImpl.java index 07c8f8a3068..f5e63b24ef5 100644 --- a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/adminconsole/AdminConsoleClientFactoryImpl.java +++ b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/adminconsole/AdminConsoleClientFactoryImpl.java @@ -1,6 +1,6 @@ package com.sap.sailing.gwt.ui.adminconsole; -import static com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants.mediaServiceRemotePath; +import static com.sap.sailing.landscape.common.RemoteServiceMappingConstants.mediaServiceRemotePath; import static com.sap.sse.common.HttpRequestHeaderConstants.HEADER_FORWARD_TO_MASTER; import static com.sap.sse.gwt.client.EntryPointHelper.registerASyncService; diff --git a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/client/RemoteServiceMappingConstants.java b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/client/RemoteServiceMappingConstants.java deleted file mode 100755 index 529ad2b844a..00000000000 --- a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/client/RemoteServiceMappingConstants.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.sap.sailing.gwt.ui.client; - -/** - * Constants for remote services. The path value must fit with the corresponding path of the service in the web.xml The - * @RemoteServiceRelativePath annotation can't be used to automatically resolve the right path because it - * uses GWT.getModuleBaseURL() to calculate the path which is different for each EntryPoint. - * - * @author Frank - * - */ -public interface RemoteServiceMappingConstants { - /** - * The hosting bundle's web context path from the OSGi manifest. This is the URL prefix under which all services are - * registered based on their relative path specification in the web.xml descriptor. - */ - public static final String WEB_CONTEXT_PATH = "gwt"; - - public static final String mediaServiceRemotePath = "service/media"; - - public static final String sailingServiceRemotePath = "service/sailing"; - - public static final String serverConfigurationServiceRemotePath = "service/serverconfiguration"; - - public static final String simulatorServiceRemotePath = "service/simulator"; - - public static final String dataMiningServiceRemotePath = "service/datamining"; - - public static final String dispatchServiceRemotePath = "service/dispatch"; -} diff --git a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/client/SailingServiceHelper.java b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/client/SailingServiceHelper.java index e77c225e33f..193ab109c9e 100644 --- a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/client/SailingServiceHelper.java +++ b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/client/SailingServiceHelper.java @@ -5,6 +5,7 @@ import static com.sap.sse.common.HttpRequestHeaderConstants.HEADER_FORWARD_TO_RE import com.google.gwt.core.client.GWT; import com.google.gwt.user.client.rpc.ServiceDefTarget; +import com.sap.sailing.landscape.common.RemoteServiceMappingConstants; import com.sap.sse.gwt.client.EntryPointHelper; import com.sap.sse.gwt.client.ServiceRoutingProvider; diff --git a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/client/StringMessages_de.properties b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/client/StringMessages_de.properties index 2162015eb5a..a299e1e7e09 100755 --- a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/client/StringMessages_de.properties +++ b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/client/StringMessages_de.properties @@ -2457,4 +2457,4 @@ goToSelfServicePortalDialogText=Self-Service Portal des Abonementanbieters in ei failedFetchingSelfServicePortalSession=Self-Service Portal konnte nicht geöffnet werden. oneAlwaysStaysOne=Wertung 1 bleibt 1, unabhängig vom Spalten-Faktor otherTieBreakingLeaderboard=Andere Rangliste zum Tie-Breaking -helptextLinkingRaces=Um unverknüpfte Wettfahrten mit getrackten Rennen zu verknüpfen, müssen sie in der untenstehenden Tabelle eine Wettfahrt linksklicken und danach das entsprechende getrackte Rennen in der rechten Ansicht linksklicken. \ No newline at end of file +helptextLinkingRaces=Um unverknüpfte Wettfahrten mit getrackten Rennen zu verknüpfen, müssen sie in der untenstehenden Tabelle eine Wettfahrt linksklicken und danach das entsprechende getrackte Rennen in der rechten Ansicht linksklicken. \ No newline at end of file diff --git a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/datamining/DataMiningEntryPoint.java b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/datamining/DataMiningEntryPoint.java index e838c148f0c..a25d81d1e22 100644 --- a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/datamining/DataMiningEntryPoint.java +++ b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/datamining/DataMiningEntryPoint.java @@ -28,9 +28,9 @@ import com.sap.sailing.gwt.common.authentication.FixedSailingAuthentication; import com.sap.sailing.gwt.common.authentication.SAPSailingHeaderWithAuthentication; import com.sap.sailing.gwt.common.client.help.HelpButton; import com.sap.sailing.gwt.ui.client.AbstractSailingReadEntryPoint; -import com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants; import com.sap.sailing.gwt.ui.datamining.presentation.TabbedSailingResultsPresenter; import com.sap.sailing.gwt.ui.shared.settings.SailingSettingsConstants; +import com.sap.sailing.landscape.common.RemoteServiceMappingConstants; import com.sap.sse.datamining.shared.DataMiningSession; import com.sap.sse.datamining.shared.dto.StatisticQueryDefinitionDTO; import com.sap.sse.datamining.shared.impl.UUIDDataMiningSession; diff --git a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/raceboard/RaceBoardEntryPoint.java b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/raceboard/RaceBoardEntryPoint.java index 1ce3780086d..718adc7500f 100644 --- a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/raceboard/RaceBoardEntryPoint.java +++ b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/raceboard/RaceBoardEntryPoint.java @@ -32,10 +32,10 @@ import com.sap.sailing.gwt.ui.client.MediaServiceAsync; import com.sap.sailing.gwt.ui.client.MediaServiceWrite; import com.sap.sailing.gwt.ui.client.MediaServiceWriteAsync; import com.sap.sailing.gwt.ui.client.RaceTimesInfoProvider; -import com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants; import com.sap.sailing.gwt.ui.client.StringMessages; import com.sap.sailing.gwt.ui.shared.RaceWithCompetitorsAndBoatsDTO; import com.sap.sailing.gwt.ui.shared.RaceboardDataDTO; +import com.sap.sailing.landscape.common.RemoteServiceMappingConstants; import com.sap.sse.common.Util; import com.sap.sse.gwt.client.EntryPointHelper; import com.sap.sse.gwt.client.async.AsyncActionsExecutor; diff --git a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/simulator/SimulatorEntryPoint.java b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/simulator/SimulatorEntryPoint.java index d98cb201d83..91963f65d63 100644 --- a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/simulator/SimulatorEntryPoint.java +++ b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/simulator/SimulatorEntryPoint.java @@ -15,9 +15,9 @@ import com.sap.sailing.domain.common.security.SecuredDomainType; import com.sap.sailing.gwt.common.authentication.FixedSailingAuthentication; import com.sap.sailing.gwt.common.authentication.SAPSailingHeaderWithAuthentication; import com.sap.sailing.gwt.ui.client.AbstractSailingReadEntryPoint; -import com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants; import com.sap.sailing.gwt.ui.client.SimulatorService; import com.sap.sailing.gwt.ui.client.SimulatorServiceAsync; +import com.sap.sailing.landscape.common.RemoteServiceMappingConstants; import com.sap.sailing.simulator.util.SailingSimulatorConstants; import com.sap.sse.gwt.client.EntryPointHelper; import com.sap.sse.gwt.client.ServerInfoDTO; diff --git a/java/com.sap.sailing.landscape.common/src/com/sap/sailing/landscape/common/RemoteServiceMappingConstants.java b/java/com.sap.sailing.landscape.common/src/com/sap/sailing/landscape/common/RemoteServiceMappingConstants.java new file mode 100755 index 00000000000..d36625b082f --- /dev/null +++ b/java/com.sap.sailing.landscape.common/src/com/sap/sailing/landscape/common/RemoteServiceMappingConstants.java @@ -0,0 +1,37 @@ +package com.sap.sailing.landscape.common; + +/** + * Constants for remote services. The path value must fit with the corresponding path of the service in the web.xml The + * @RemoteServiceRelativePath annotation can't be used to automatically resolve the right path because it + * uses GWT.getModuleBaseURL() to calculate the path which is different for each EntryPoint. + * + * @author Frank + * + */ +public interface RemoteServiceMappingConstants { + /** + * The hosting bundle's web context path from the OSGi manifest's {@code Web-ContextPath} declaration. This is the + * URL prefix under which all services are registered based on their relative path specification in the + * web.xml descriptor. + */ + String WEB_CONTEXT_PATH = "gwt"; + + String mediaServiceRemotePath = "service/media"; + + String sailingServiceRemotePath = "service/sailing"; + + /** + * the prefix to use when constructing a load balancer listener rule's path condition from a sharding key. + * See also {@link ShardProcedure#getPathConditionForShardingKey(Object, String)} and + * {@link ShardProcedure#getShardingKeyFromPathCondition(String, String)}. + */ + String pathPrefixForShardingKey = "/"+WEB_CONTEXT_PATH+"/"+sailingServiceRemotePath; + + String serverConfigurationServiceRemotePath = "service/serverconfiguration"; + + String simulatorServiceRemotePath = "service/simulator"; + + String dataMiningServiceRemotePath = "service/datamining"; + + String dispatchServiceRemotePath = "service/dispatch"; +} diff --git a/java/com.sap.sailing.landscape.test/src/com/sap/sailing/landscape/test/TestProcedures.java b/java/com.sap.sailing.landscape.test/src/com/sap/sailing/landscape/test/TestProcedures.java index dc15a3412be..2287f30b7a4 100755 --- a/java/com.sap.sailing.landscape.test/src/com/sap/sailing/landscape/test/TestProcedures.java +++ b/java/com.sap.sailing.landscape.test/src/com/sap/sailing/landscape/test/TestProcedures.java @@ -27,6 +27,7 @@ import com.sap.sailing.landscape.SailingAnalyticsHost; import com.sap.sailing.landscape.SailingAnalyticsMetrics; import com.sap.sailing.landscape.SailingAnalyticsProcess; import com.sap.sailing.landscape.SailingReleaseRepository; +import com.sap.sailing.landscape.common.RemoteServiceMappingConstants; import com.sap.sailing.landscape.common.SharedLandscapeConstants; import com.sap.sailing.landscape.impl.BearerTokenReplicationCredentials; import com.sap.sailing.landscape.impl.SailingAnalyticsHostImpl; @@ -70,7 +71,7 @@ import software.amazon.awssdk.services.route53.model.RRType; * Tests for the AWS SDK landscape wrapper in bundle {@code com.sap.sse.landscape.aws}. To run these tests * successfully it is necessary to have valid AWS credentials for region {@code EU_WEST_2} that allow the * AWS user account to create keys and launch instances, etc. These are to be provided as explained - * in the documentation of {@link AwsLandscape#obtain()}. + * in the documentation of {@link AwsLandscape#obtain(String)}. * * @author Axel Uhl (D043530) * @@ -94,7 +95,7 @@ public class TestProcedures { @Before public void setUp() { privateKeyEncryptionPassphrase = ("awptyf87l"+"097384sf;,57").getBytes(); - landscape = AwsLandscape.obtain(); + landscape = AwsLandscape.obtain(RemoteServiceMappingConstants.pathPrefixForShardingKey); region = new AwsRegion(Region.EU_WEST_2, landscape); securityServiceReplicationBearerToken = System.getProperty(SECURITY_SERVICE_REPLICATION_BEARER_TOKEN); mailSmtpPassword = System.getProperty(MAIL_SMTP_PASSWORD); diff --git a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/ApplicationReplicaSetsImagesBarCell.java b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/ApplicationReplicaSetsImagesBarCell.java index 67d698899a6..2ee69ec0a31 100755 --- a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/ApplicationReplicaSetsImagesBarCell.java +++ b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/ApplicationReplicaSetsImagesBarCell.java @@ -24,6 +24,7 @@ public class ApplicationReplicaSetsImagesBarCell extends ImagesBarCell { public static final String ACTION_SWITCH_TO_REPLICA_ON_SHARED_INSTANCE = "ACTION_SWITCH_TO_REPLICA_ON_SHARED_INSTANCE"; public static final String ACTION_MOVE_MASTER_TO_OTHER_INSTANCE = "ACTION_MOVE_MASTER_TO_OTHER_INSTANCE"; public static final String ACTION_SCALE_AUTO_SCALING_REPLICAS_UP_DOWN = "ACTION_SCALE_AUTO_SCALING_REPLICAS_UP_DOWN"; + public static final String ACTION_OPEN_SHARD_MANAGEMENT = "ACTION_OPEN_SHARD_MANAGEMENT"; private final StringMessages stringMessages; private final UserService userService; @@ -63,6 +64,7 @@ public class ApplicationReplicaSetsImagesBarCell extends ImagesBarCell { if (applicationReplicaSet.getAutoScalingGroupAmiId() != null) { result.add(new ImageSpec(ACTION_UPDATE_AMI_FOR_AUTO_SCALING_REPLICAS, stringMessages.updateAmiForAutoScalingReplicas(), IconResources.INSTANCE.redGearsIcon())); result.add(new ImageSpec(ACTION_SWITCH_TO_AUTO_SCALING_REPLICAS_ONLY, stringMessages.switchToAutoScalingReplicasOnly(), IconResources.INSTANCE.scaleUpIcon())); + result.add(new ImageSpec(ACTION_OPEN_SHARD_MANAGEMENT, stringMessages.openShardManagement(), IconResources.INSTANCE.shardManagementIcon())); } result.add(new ImageSpec(ACTION_SWITCH_TO_REPLICA_ON_SHARED_INSTANCE, stringMessages.switchToReplicaOnSharedInstance(), IconResources.INSTANCE.scaleDownIcon())); if (!applicationReplicaSet.isLocalReplicaSet(userService)) { diff --git a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/LandscapeManagementPanel.java b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/LandscapeManagementPanel.java index cee014a5dfb..e3dae5c8c9b 100755 --- a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/LandscapeManagementPanel.java +++ b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/LandscapeManagementPanel.java @@ -15,7 +15,6 @@ import java.util.function.Function; import com.google.gwt.cell.client.SafeHtmlCell; import com.google.gwt.core.client.GWT; import com.google.gwt.safehtml.shared.SafeHtml; -import com.google.gwt.safehtml.shared.SafeHtmlBuilder; import com.google.gwt.user.cellview.client.Column; import com.google.gwt.user.cellview.client.TextColumn; import com.google.gwt.user.client.Timer; @@ -263,11 +262,7 @@ public class LandscapeManagementPanel extends SimplePanel { final Column, SafeHtml> versionColumn = new Column, SafeHtml>(versionCell) { @Override public SafeHtml getValue(SailingApplicationReplicaSetDTO replicaSet) { - final SafeHtmlBuilder builder = new SafeHtmlBuilder(); - final String version = replicaSet.getVersion(); - final String releaseNotesLink = getReleaseNotesLink(version); - appendEc2Link(builder, releaseNotesLink, version); - return builder.toSafeHtml(); + return new LinkBuilder().setReplicaSet(replicaSet).setPathMode(LinkBuilder.pathModes.Version).build(); } }; applicationReplicaSetsTable.addColumn(versionColumn, stringMessages.versionHeader(), (rs1, rs2)->new NaturalComparator().compare(rs1.getVersion(), rs2.getVersion())); @@ -275,12 +270,7 @@ public class LandscapeManagementPanel extends SimplePanel { final Column, SafeHtml> masterColumn = new Column, SafeHtml>(masterCell) { @Override public SafeHtml getValue(SailingApplicationReplicaSetDTO replicaSet) { - final SafeHtmlBuilder builder = new SafeHtmlBuilder(); - final String gwtStatusLink = getGwtStatusLink(replicaSet.getMaster().getHost().getPublicIpAddress(), replicaSet.getMaster().getPort()); - builder.appendHtmlConstant(""); - builder.appendEscaped(replicaSet.getMaster().getHost().getPublicIpAddress()); - builder.appendHtmlConstant(""); - return builder.toSafeHtml(); + return new LinkBuilder().setPathMode(LinkBuilder.pathModes.MasterHost).setReplicaSet(replicaSet).build(); } }; applicationReplicaSetsTable.addColumn(masterColumn, stringMessages.masterHostName(), (rs1, rs2)->new NaturalComparator().compare(rs1.getMaster().getHost().getPublicIpAddress(), rs2.getMaster().getHost().getPublicIpAddress())); @@ -289,23 +279,15 @@ public class LandscapeManagementPanel extends SimplePanel { final Column, SafeHtml> hostnameColumn = new Column, SafeHtml>(hostnameCell) { @Override public SafeHtml getValue(SailingApplicationReplicaSetDTO replicaSet) { - final SafeHtmlBuilder builder = new SafeHtmlBuilder(); - final String hostnameLink = "https://"+replicaSet.getHostname(); - builder.appendHtmlConstant(""); - builder.appendEscaped(replicaSet.getHostname()); - builder.appendHtmlConstant(""); - return builder.toSafeHtml(); - } + return new LinkBuilder().setReplicaSet(replicaSet).setPathMode(LinkBuilder.pathModes.Hostname).build() +; } }; applicationReplicaSetsTable.addColumn(hostnameColumn, stringMessages.hostname(), (rs1, rs2)->new NaturalComparator().compare(rs1.getHostname(), rs2.getMaster().getHostname())); final SafeHtmlCell masterInstanceIdCell = new SafeHtmlCell(); final Column, SafeHtml> masterInstanceIdColumn = new Column, SafeHtml>(masterInstanceIdCell) { @Override public SafeHtml getValue(SailingApplicationReplicaSetDTO replicaSet) { - final SafeHtmlBuilder builder = new SafeHtmlBuilder(); - final String instanceId = replicaSet.getMaster().getHost().getInstanceId(); - appendEc2InstanceLink(builder, instanceId); - return builder.toSafeHtml(); + return new LinkBuilder().setRegion(regionsTable.getSelectionModel().getSelectedObject()).setInstanceId(replicaSet.getMaster().getHost().getInstanceId()).setPathMode(LinkBuilder.pathModes.InstanceSearch).build(); } }; applicationReplicaSetsTable.addColumn(masterInstanceIdColumn, stringMessages.masterInstanceId(), @@ -316,21 +298,10 @@ public class LandscapeManagementPanel extends SimplePanel { final Column, SafeHtml> replicasColumn = new Column, SafeHtml>(replicasCell) { @Override public SafeHtml getValue(SailingApplicationReplicaSetDTO replicaSet) { - SafeHtmlBuilder builder = new SafeHtmlBuilder(); - for (final SailingAnalyticsProcessDTO replica : replicaSet.getReplicas()) { - final String gwtStatusLink = getGwtStatusLink(replica.getHost().getPublicIpAddress(), replica.getPort()); - builder.appendHtmlConstant(""); - builder.appendEscaped(replica.getHost().getPublicIpAddress()+":"+replica.getPort()); - builder.appendHtmlConstant(""); - final String replicaInstanceId = replica.getHost().getInstanceId(); - builder.appendEscaped(" ("); - builder.appendEscaped(replica.getServerName()); - builder.appendEscaped(", "); - appendEc2InstanceLink(builder, replicaInstanceId); - builder.appendEscaped(")"); - builder.appendHtmlConstant("
"); - } - return builder.toSafeHtml(); + LinkBuilder linkBuilder = new LinkBuilder(); + linkBuilder.setReplicaSet(replicaSet); + linkBuilder.setPathMode(LinkBuilder.pathModes.ReplicaLinks).setRegion(regionsTable.getSelectionModel().getSelectedObject()); + return linkBuilder.build(); } }; applicationReplicaSetsTable.addColumn(replicasColumn, stringMessages.replicas()); @@ -339,11 +310,7 @@ public class LandscapeManagementPanel extends SimplePanel { final Column, SafeHtml> autoScalingGroupAmiIdColumn = new Column, SafeHtml>(autoScalingGroupAmiIdCell) { @Override public SafeHtml getValue(SailingApplicationReplicaSetDTO replicaSet) { - final SafeHtmlBuilder builder = new SafeHtmlBuilder(); - if (replicaSet.getAutoScalingGroupAmiId() != null) { - appendEc2AmiLink(builder, replicaSet.getAutoScalingGroupAmiId()); - } - return builder.toSafeHtml(); + return new LinkBuilder().setReplicaSet(replicaSet).setRegion(regionsTable.getSelectionModel().getSelectedObject()).setPathMode(LinkBuilder.pathModes.AmiSearch).build(); } }; applicationReplicaSetsTable.addColumn(autoScalingGroupAmiIdColumn, stringMessages.machineImageId(), (rs1, rs2)->new NaturalComparator().compare(rs1.getAutoScalingGroupAmiId(), rs2.getAutoScalingGroupAmiId())); @@ -380,7 +347,10 @@ public class LandscapeManagementPanel extends SimplePanel { regionsTable.getSelectionModel().getSelectedObject(), Collections.singleton(applicationReplicaSetToUpgrade))); applicationReplicaSetsActionColumn.addAction(ApplicationReplicaSetsImagesBarCell.ACTION_SCALE_AUTO_SCALING_REPLICAS_UP_DOWN, applicationReplicaSetToUpgrade -> scaleAutoScalingReplicasUpDown(stringMessages, - regionsTable.getSelectionModel().getSelectedObject(), Collections.singleton(applicationReplicaSetToUpgrade))); + regionsTable.getSelectionModel().getSelectedObject(), + Collections.singleton(applicationReplicaSetToUpgrade))); + applicationReplicaSetsActionColumn.addAction(ApplicationReplicaSetsImagesBarCell.ACTION_OPEN_SHARD_MANAGEMENT, + selectedReplicaSet -> openShardManagementPanel(stringMessages, regionsTable.getSelectionModel().getSelectedObject(), selectedReplicaSet)); // see below for the finalization o the applicationRelicaSetsActionColumn; we need to have the machineImagesTable ready for the last action... final CaptionPanel applicationReplicaSetsCaptionPanel = new CaptionPanel(stringMessages.applicationReplicaSets()); final VerticalPanel applicationReplicaSetsVerticalPanel = new VerticalPanel(); @@ -489,13 +459,13 @@ public class LandscapeManagementPanel extends SimplePanel { AsyncCallback validatePassphraseCallback = new AsyncCallback() { @Override public void onSuccess(Boolean result) { - sshKeyManagementPanel.setPassphraseValidation(result.booleanValue(),stringMessages); + sshKeyManagementPanel.setPassphraseValidation(result.booleanValue(), stringMessages); addApplicationReplicaSetButton.setVisible(result); applicationReplicaSetsRefreshButton.setVisible(result); applicationReplicaSetsCaptionPanel.setVisible(result); machineImagesCaptionPanel.setVisible(result); mongoEndpointsCaptionPanel.setVisible(result); - if(result) { + if (result) { refreshApplicationReplicaSetsTable(); } } @@ -505,17 +475,34 @@ public class LandscapeManagementPanel extends SimplePanel { }; }; sshKeyManagementPanel.addSshKeySelectionChangedHandler(event->{ - validatePassphrase(stringMessages,validatePassphraseCallback); + validatePassphrase(stringMessages, validatePassphraseCallback); }); sshKeyManagementPanel.addOnPassphraseChangedListener(event -> { - validatePassphrase(stringMessages,validatePassphraseCallback); + validatePassphrase(stringMessages, validatePassphraseCallback); }); - validatePassphrase(stringMessages,validatePassphraseCallback); + validatePassphrase(stringMessages, validatePassphraseCallback); // TODO try to identify archive servers // TODO support archive server upgrade // TODO upon region selection show RabbitMQ, and Central Reverse Proxy clusters in region } + private void openShardManagementPanel(StringMessages stringMessages, String region, SailingApplicationReplicaSetDTO replicaset) { + new ShardManagementDialog(landscapeManagementService, replicaset, region, sshKeyManagementPanel.getPassphraseForPrivateKeyDecryption(), errorReporter, stringMessages, + new DialogCallback() { + @Override + public void ok(Boolean hasAnythingChanged) { + if (hasAnythingChanged) { + refreshApplicationReplicaSetsTable(); + } + } + + @Override + public void cancel() { + // there is no cancel button + } + }).show(); + } + private void disableButtonWhenLocalReplicaSetIsSelected(Button button, UserService userService) { applicationReplicaSetsTable.getSelectionModel().addSelectionChangeHandler(e->button.setEnabled( !applicationReplicaSetsTable.getSelectionModel().getSelectedSet().stream().filter(arsDTO->arsDTO.isLocalReplicaSet(userService)).findAny().isPresent())); @@ -587,7 +574,7 @@ public class LandscapeManagementPanel extends SimplePanel { @Override public void onFailure(Throwable caught) { - errorReporter.reportError(caught.getMessage()); + errorReporter.reportError(caught.getMessage() == null ? caught.getClass().getName() : caught.getMessage()); if (!replicaSetIterator.hasNext()) { applicationReplicaSetsBusy.setBusy(false); } else { @@ -762,14 +749,6 @@ public class LandscapeManagementPanel extends SimplePanel { }); } - private String getGwtStatusLink(final String host, int port) { - return (port == 443 ? "https" : "http") + "://" + host + ":" + port + "/gwt/status"; - } - - private String getReleaseNotesLink(final String version) { - return "https://releases.sapsailing.com/"+version+"/release-notes.txt"; - } - private void ensureAtLeastOneReplicaExistsStopReplicatingAndRemoveMasterFromTargetGroups( StringMessages stringMessages, String selectedObject, Iterable> applicationReplicaSetsForWhichToEnsureAtLeastOneReplicaStopReplicatingAndRemoveMasterFromTargetGroups) { @@ -1496,33 +1475,4 @@ public class LandscapeManagementPanel extends SimplePanel { RemoteServiceMappingConstants.landscapeManagementServiceRemotePath, HEADER_FORWARD_TO_MASTER); return result; } - - private String getEc2ConsoleLinkForInstanceId(String instanceId) { - return getEc2ConsoleBaseUrlForSelectedRegion()+"#Instances:search="+instanceId; - } - - private void appendEc2InstanceLink(final SafeHtmlBuilder builder, final String instanceId) { - final String ec2Link = getEc2ConsoleLinkForInstanceId(instanceId); - appendEc2Link(builder, ec2Link, instanceId); - } - - private void appendEc2Link(final SafeHtmlBuilder builder, final String ec2Link, final String text) { - builder.appendHtmlConstant(""); - builder.appendEscaped(text); - builder.appendHtmlConstant(""); - } - - private String getEc2ConsoleBaseUrlForSelectedRegion() { - return "https://"+regionsTable.getSelectionModel().getSelectedObject()+ - ".console.aws.amazon.com/ec2/v2/home?region="+regionsTable.getSelectionModel().getSelectedObject(); - } - - private String getEc2ConsoleLinkForAmiId(String amiId) { - return getEc2ConsoleBaseUrlForSelectedRegion()+"#Images:imageId="+amiId; - } - - private void appendEc2AmiLink(final SafeHtmlBuilder builder, final String amiId) { - final String ec2Link = getEc2ConsoleLinkForAmiId(amiId); - appendEc2Link(builder, ec2Link, amiId); - } } diff --git a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/LandscapeManagementWriteService.java b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/LandscapeManagementWriteService.java index 09081a3814c..72e43056991 100755 --- a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/LandscapeManagementWriteService.java +++ b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/LandscapeManagementWriteService.java @@ -1,12 +1,15 @@ package com.sap.sailing.landscape.ui.client; import java.util.ArrayList; +import java.util.Map; import com.google.gwt.user.client.rpc.RemoteService; import com.sap.sailing.domain.common.DataImportProgress; import com.sap.sailing.landscape.ui.shared.AmazonMachineImageDTO; import com.sap.sailing.landscape.ui.shared.AwsInstanceDTO; +import com.sap.sailing.landscape.ui.shared.AwsShardDTO; import com.sap.sailing.landscape.ui.shared.CompareServersResultDTO; +import com.sap.sailing.landscape.ui.shared.LeaderboardNameDTO; import com.sap.sailing.landscape.ui.shared.MongoEndpointDTO; import com.sap.sailing.landscape.ui.shared.MongoScalingInstructionsDTO; import com.sap.sailing.landscape.ui.shared.ProcessDTO; @@ -70,8 +73,8 @@ public interface LandscapeManagementWriteService extends RemoteService { ArrayList> getApplicationReplicaSets(String regionId, String optionalKeyName, byte[] privateKeyEncryptionPassphrase) throws Exception; - SerializationDummyDTO serializationDummy(ProcessDTO mongoProcessDTO, AwsInstanceDTO awsInstanceDTO, - SailingApplicationReplicaSetDTO sailingApplicationReplicationSetDTO); + SerializationDummyDTO serializationDummy(ProcessDTO mongoProcessDTO, AwsInstanceDTO awsInstanceDTO, AwsShardDTO shardDTO, + SailingApplicationReplicaSetDTO sailingApplicationReplicationSetDTO, LeaderboardNameDTO leaderboard); SailingApplicationReplicaSetDTO createApplicationReplicaSet(String regionId, String name, boolean sharedMasterInstance, String masterInstanceType, String optionalReplicaInstanceTypeOrNull, boolean dynamicLoadBalancerMapping, @@ -141,4 +144,17 @@ public interface LandscapeManagementWriteService extends RemoteService { SailingApplicationReplicaSetDTO changeAutoScalingReplicasInstanceType( SailingApplicationReplicaSetDTO replicaSet, String instanceTypeName, String optionalKeyName, byte[] privateKeyEncryptionPassphrase) throws Exception; + + ArrayList getLeaderboardNames(SailingApplicationReplicaSetDTO replicaSet, String bearerToken) throws Exception; + + void addShard(String shardName, ArrayList selectedLeaderBoardNames, SailingApplicationReplicaSetDTO replicaSet, + String bearerToken, String region, byte[] passphraseForPrivateKeyDecryption) throws Exception; + + public Map> getShards(SailingApplicationReplicaSetDTO replicaSet, String region, String bearerToken) throws Exception; + + public void removeShard(AwsShardDTO shard, SailingApplicationReplicaSetDTO replicaSet, String region, byte[] passphrase) throws Exception; + + void appendShardingKeysToShard(Iterable shardingKeysToAppend, String region, String shardName, SailingApplicationReplicaSetDTO replicaSet, String bearerToken, byte[] passphraseForPrivateKeyDecryption) throws Exception; + + void removeShardingKeysFromShard(Iterable shardingKeysToRemove, String region, String shardName, SailingApplicationReplicaSetDTO replicaSet, String bearerToken, byte[] passphraseForPrivateKeyDecryption) throws Exception; } diff --git a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/LandscapeManagementWriteServiceAsync.java b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/LandscapeManagementWriteServiceAsync.java index 078e17a7186..0027eb5f18f 100755 --- a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/LandscapeManagementWriteServiceAsync.java +++ b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/LandscapeManagementWriteServiceAsync.java @@ -1,13 +1,16 @@ package com.sap.sailing.landscape.ui.client; import java.util.ArrayList; +import java.util.Map; import com.google.gwt.user.client.rpc.AsyncCallback; import com.sap.sailing.domain.common.DataImportProgress; import com.sap.sailing.landscape.common.SharedLandscapeConstants; import com.sap.sailing.landscape.ui.shared.AmazonMachineImageDTO; import com.sap.sailing.landscape.ui.shared.AwsInstanceDTO; +import com.sap.sailing.landscape.ui.shared.AwsShardDTO; import com.sap.sailing.landscape.ui.shared.CompareServersResultDTO; +import com.sap.sailing.landscape.ui.shared.LeaderboardNameDTO; import com.sap.sailing.landscape.ui.shared.MongoEndpointDTO; import com.sap.sailing.landscape.ui.shared.MongoScalingInstructionsDTO; import com.sap.sailing.landscape.ui.shared.ProcessDTO; @@ -104,8 +107,8 @@ public interface LandscapeManagementWriteServiceAsync { Integer minimumAutoScalingGroupSizeOrNull, Integer maximumAutoScalingGroupSizeOrNull, AsyncCallback> callback); - void serializationDummy(ProcessDTO mongoProcessDTO, AwsInstanceDTO awsInstanceDTO, - SailingApplicationReplicaSetDTO sailingApplicationReplicationSetDTO, + void serializationDummy(ProcessDTO mongoProcessDTO, AwsInstanceDTO awsInstanceDTO, AwsShardDTO shardDTO, + SailingApplicationReplicaSetDTO sailingApplicationReplicationSetDTO, LeaderboardNameDTO leaderboard, AsyncCallback callback); void defineDefaultRedirect(String regionId, String hostname, RedirectDTO redirect, String keyName, @@ -193,4 +196,84 @@ public interface LandscapeManagementWriteServiceAsync { void changeAutoScalingReplicasInstanceType(SailingApplicationReplicaSetDTO replicaSet, String instanceTypeName, String optionalKeyName, byte[] privateKeyEncryptionPassphrase, AsyncCallback> callback); + + void getLeaderboardNames(SailingApplicationReplicaSetDTO replicaSet, String bearerToken, + AsyncCallback> names); + + void addShard(String shardName, ArrayList selectedLeaderBoards, + SailingApplicationReplicaSetDTO replicaSet, String bearerToken, String region, + byte[] passphraseForPrivateKeyDecryption, AsyncCallback callback); + + /** + * @param callback + * returns the shards as keys and the sharding keys (escaped / mangled leaderboard names) as values. + * Those sharding keys are what you get when mangling the {@link AwsShardDTO#getLeaderboardNames() + * leaderboard names} of the shard. + */ + void getShards(SailingApplicationReplicaSetDTO replicaset, String region, String bearerToken, + AsyncCallback>> callback); + + /** + * Removes {@code shard} from the replica set. This deletes the load balancer listener rules, and the auto scaling + * group and the target group. + * + * @param shard + * the shard to remove + * @param replicaSet + * the replica set which contains the shard. + * @param region + * replica set's region + * @param passphrase + * passphrase for the private key decription. + * @param callback + * + */ + public void removeShard(AwsShardDTO shard, SailingApplicationReplicaSetDTO replicaSet, String region, + byte[] passphrase, AsyncCallback callback); + + /** + * Appends sharding keys for each leader board in {@code selectedLeaderboards} to the shard, identified by + * {@code shardName}, from the {@code replicaset}. This function inserts rules to the replica set's load balancer + * for each {@selectedLeaderboards}'s sharding key. It the replica set's load balancer does not have enough rules + * left, a new one gets created. For inserting the rules, first every existing rule of this shard get's checked for + * space left and if there is, it gets filled with a sharding key and after that new rules are created. Throws an + * Exception if: - the shard is not found. - shards cannot be retrived - sharding rules cannot be inserted - the is + * no free load balancer or the process of moving the replica set to another load balancer failed. + * + * @param selectedLeaderBoards + * list of selected leaderboards. These are the names and not sharding keys. + * @param region + * landscape region + * @param shardName + * shard's name where the keys are supposed to be appended + * @param replicaSet + * shard's replica set + * @param bearerToken + * @param passphraseForPrivateKeyDecryption + * @param callback + */ + void appendShardingKeysToShard(Iterable selectedLeaderBoards, String region, String shardName, + SailingApplicationReplicaSetDTO replicaSet, String bearerToken, + byte[] passphraseForPrivateKeyDecryption, AsyncCallback callback); + + /** + * Removes the shardingkeys for {@selectedLeaderBoards} from a shard, identified by {@code shardName} from + * {@code replicaset}. Throws Exception if: - no shard is found - replicaset is not found - shards from replica set + * cannot be retrieved. - sharding rules cannot be updated + * + * @param selectedLeaderBoards + * Sharding keys for all selected leader boards. + * @param region + * shard's regio + * @param shardName + * Shard's name where the keys should be removed from + * @param replicaSet + * replica set which contains the shard + * @param bearerToken + * @param passphraseForPrivateKeyDecryption + * @param callback + */ + void removeShardingKeysFromShard(Iterable selectedLeaderBoards, String region, String shardName, + SailingApplicationReplicaSetDTO replicaSet, String bearerToken, + byte[] passphraseForPrivateKeyDecryption, AsyncCallback callback); } diff --git a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/LinkBuilder.java b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/LinkBuilder.java new file mode 100644 index 00000000000..deac013390f --- /dev/null +++ b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/LinkBuilder.java @@ -0,0 +1,190 @@ +package com.sap.sailing.landscape.ui.client; + +import com.google.gwt.safehtml.shared.SafeHtml; +import com.google.gwt.safehtml.shared.SafeHtmlBuilder; +import com.sap.sailing.landscape.ui.shared.SailingAnalyticsProcessDTO; +import com.sap.sailing.landscape.ui.shared.SailingApplicationReplicaSetDTO; +import com.sap.sse.common.Builder; + +public class LinkBuilder implements Builder { + + static public enum pathModes { + InstanceSearch, ImageSearch, AmiSearch, Hostname, ReplicaLinks, Version, MasterHost, TargetGroupSearch, AutoScalingGroupSearch + }; + + private pathModes pathMode; + private String region; + private String instanceId; + private SailingApplicationReplicaSetDTO replicaSet; + private String targetGroupName; + private String autoScalingGroupName; + + LinkBuilder setPathMode(pathModes mode) { + pathMode = mode; + return self(); + } + + LinkBuilder setTargetGroupName(String name) { + this.targetGroupName = name; + return self(); + } + + LinkBuilder setAutoScalingGroupName(String name) { + this.autoScalingGroupName = name; + return self(); + } + + LinkBuilder setInstanceId(String instanceId) { + this.instanceId = instanceId; + return self(); + } + + LinkBuilder setRegion(String region) { + this.region = region; + return self(); + } + + LinkBuilder setReplicaSet(SailingApplicationReplicaSetDTO replicaSet) { + this.replicaSet = replicaSet; + return self(); + } + + private String getEc2ConsoleLinkForInstanceId(String instanceId) { + return getEc2ConsoleBaseUrlForSelectedRegion() + "#Instances:search=" + instanceId; + } + + private String getEc2ConsoleLinkForTargetGroupName(String name) { + return getEc2ConsoleBaseUrlForSelectedRegion() + "#TargetGroups:search=" + name; + } + + private String getEc2ConsoleLinkForAutoScalingGroupName(String name) { + return getEc2ConsoleBaseUrlForSelectedRegion() + "#AutoScalingGroupDetails:id="+name+";view=details"; + } + + private String getEc2ConsoleLinkForAmiId(String amiId) { + return getEc2ConsoleBaseUrlForSelectedRegion() + "#Images:visibility=owned-by-me;search=" + amiId; + } + + private String getEc2ConsoleBaseUrlForSelectedRegion() { + return "https://" + region + ".console.aws.amazon.com/ec2/v2/home?region=" + region; + } + + private void appendEc2Link(final SafeHtmlBuilder builder, final String ec2Link, final String text) { + builder.appendHtmlConstant(""); + builder.appendEscaped(text); + builder.appendHtmlConstant(""); + } + + private void appendEc2InstanceLink(final SafeHtmlBuilder builder, final String instanceId) { + final String ec2Link = getEc2ConsoleLinkForInstanceId(instanceId); + appendEc2Link(builder, ec2Link, instanceId); + } + + private void appendEc2AmiLink(final SafeHtmlBuilder builder, final String amiId) { + final String ec2Link = getEc2ConsoleLinkForAmiId(amiId); + appendEc2Link(builder, ec2Link, amiId); + } + + private void appendEc2TargetGroupLink(final SafeHtmlBuilder builder, final String name) { + final String ec2Link = getEc2ConsoleLinkForTargetGroupName(name); + appendEc2Link(builder, ec2Link, name); + } + + private void appendEc2AutoScalingGroupLink(final SafeHtmlBuilder builder, final String name) { + final String ec2Link = getEc2ConsoleLinkForAutoScalingGroupName(name); + appendEc2Link(builder, ec2Link, name); + } + + private String getGwtStatusLink(final String host, int port) { + return (port == 443 ? "https" : "http") + "://" + host + ":" + port + "/gwt/status"; + } + + private String getReleaseNotesLink(final String version) { + return "https://releases.sapsailing.com/" + version + "/release-notes.txt"; + } + + private void checkAttribute(Object attr, String name) throws Exception { + if (attr == null) { + throw new Exception(name + " needs to be given!"); + } + } + + @Override + public SafeHtml build() { + SafeHtmlBuilder builder = new SafeHtmlBuilder(); + try { + switch (pathMode) { + case ReplicaLinks: + checkAttribute(replicaSet, "Replicaset"); + for (final SailingAnalyticsProcessDTO replica : replicaSet.getReplicas()) { + final String gwtStatusLink = getGwtStatusLink(replica.getHost().getPublicIpAddress(), + replica.getPort()); + builder.appendHtmlConstant(""); + builder.appendEscaped(replica.getHost().getPublicIpAddress() + ":" + replica.getPort()); + builder.appendHtmlConstant(""); + final String replicaInstanceId = replica.getHost().getInstanceId(); + builder.appendEscaped(" ("); + builder.appendEscaped(replica.getServerName()); + builder.appendEscaped(", "); + appendEc2InstanceLink(builder, replicaInstanceId); + builder.appendEscaped(")"); + builder.appendHtmlConstant("
"); + } + break; + case AmiSearch: + checkAttribute(replicaSet, "Replicaset"); + checkAttribute(region, "Region"); + if (replicaSet.getAutoScalingGroupAmiId() != null) { + appendEc2AmiLink(builder, replicaSet.getAutoScalingGroupAmiId()); + } + + break; + case Hostname: + checkAttribute(replicaSet, "Replicaset"); + final String hostnameLink = "https://" + replicaSet.getHostname(); + builder.appendHtmlConstant(""); + builder.appendEscaped(replicaSet.getHostname()); + builder.appendHtmlConstant(""); + break; + case ImageSearch: + break; + case InstanceSearch: + checkAttribute(region, "Region"); + appendEc2InstanceLink(builder, instanceId); + break; + case Version: + checkAttribute(replicaSet, "Replicaset"); + final String version = replicaSet.getVersion(); + final String releaseNotesLink = getReleaseNotesLink(version); + appendEc2Link(builder, releaseNotesLink, version); + break; + case MasterHost: + checkAttribute(replicaSet, "Replicaset"); + final String gwtStatusLink = getGwtStatusLink(replicaSet.getMaster().getHost().getPublicIpAddress(), + replicaSet.getMaster().getPort()); + builder.appendHtmlConstant(""); + builder.appendEscaped(replicaSet.getMaster().getHost().getPublicIpAddress()); + builder.appendHtmlConstant(""); + break; + case TargetGroupSearch: + checkAttribute(region, "Region"); + checkAttribute(targetGroupName, "Target Group Name"); + appendEc2TargetGroupLink(builder, targetGroupName); + break; + case AutoScalingGroupSearch: + checkAttribute(region, "Region"); + checkAttribute(autoScalingGroupName, "Auto-Scaling Group Name"); + appendEc2AutoScalingGroupLink(builder, autoScalingGroupName); + break; + default: + break; + } + } catch (Exception e) { + builder.appendHtmlConstant(""); + builder.appendEscaped(e.getMessage()); + builder.appendHtmlConstant(""); + } + + return builder.toSafeHtml(); + } +} diff --git a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/ShardManagementDialog.java b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/ShardManagementDialog.java new file mode 100644 index 00000000000..d71a750e608 --- /dev/null +++ b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/ShardManagementDialog.java @@ -0,0 +1,32 @@ +package com.sap.sailing.landscape.ui.client; + +import com.google.gwt.user.client.ui.Widget; +import com.sap.sailing.landscape.ui.client.i18n.StringMessages; +import com.sap.sailing.landscape.ui.shared.SailingApplicationReplicaSetDTO; +import com.sap.sse.gwt.client.ErrorReporter; +import com.sap.sse.gwt.client.dialog.DataEntryDialog; + +public class ShardManagementDialog extends DataEntryDialog { + private final ShardManagementPanel shardPanel; + + ShardManagementDialog(LandscapeManagementWriteServiceAsync landscapeManagementWriteServiceAsync, + SailingApplicationReplicaSetDTO applicastionReplicaSet, String region, String passphrase, + ErrorReporter errorReporter, StringMessages stringMessages, DialogCallback callback) { + super(stringMessages.shard(), stringMessages.shardingDescription(), stringMessages.close(), /* no cancel button */ null, /* validator */ null, callback); + shardPanel = new ShardManagementPanel(landscapeManagementWriteServiceAsync, errorReporter, stringMessages); + shardPanel.setRegion(region); + shardPanel.setPassphrase(passphrase); + shardPanel.setReplicaSet(applicastionReplicaSet); + shardPanel.refresh(); + } + + @Override + protected Widget getAdditionalWidget() { + return shardPanel; + } + + @Override + protected Boolean getResult() { + return shardPanel.hasAnythingChanged(); + } +} diff --git a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/ShardManagementPanel.java b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/ShardManagementPanel.java new file mode 100644 index 00000000000..566ef8fea0c --- /dev/null +++ b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/ShardManagementPanel.java @@ -0,0 +1,422 @@ +package com.sap.sailing.landscape.ui.client; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import com.google.gwt.cell.client.SafeHtmlCell; +import com.google.gwt.core.client.GWT; +import com.google.gwt.safehtml.shared.SafeHtml; +import com.google.gwt.user.cellview.client.Column; +import com.google.gwt.user.client.rpc.AsyncCallback; +import com.google.gwt.user.client.ui.Button; +import com.google.gwt.user.client.ui.CaptionPanel; +import com.google.gwt.user.client.ui.FocusWidget; +import com.google.gwt.user.client.ui.Grid; +import com.google.gwt.user.client.ui.HorizontalPanel; +import com.google.gwt.user.client.ui.Label; +import com.google.gwt.user.client.ui.SimplePanel; +import com.google.gwt.user.client.ui.TextBox; +import com.google.gwt.user.client.ui.VerticalPanel; +import com.google.gwt.user.client.ui.Widget; +import com.sap.sailing.landscape.LandscapeService; +import com.sap.sailing.landscape.ui.client.i18n.StringMessages; +import com.sap.sailing.landscape.ui.shared.AwsShardDTO; +import com.sap.sailing.landscape.ui.shared.LeaderboardNameDTO; +import com.sap.sailing.landscape.ui.shared.SailingApplicationReplicaSetDTO; +import com.sap.sse.common.Util; +import com.sap.sse.gwt.adminconsole.AdminConsoleTableResources; +import com.sap.sse.gwt.client.ErrorReporter; +import com.sap.sse.gwt.client.Notification; +import com.sap.sse.gwt.client.Notification.NotificationType; +import com.sap.sse.gwt.client.celltable.TableWrapperWithMultiSelectionAndFilter; +import com.sap.sse.gwt.client.controls.busyindicator.BusyIndicator; +import com.sap.sse.gwt.client.controls.busyindicator.SimpleBusyIndicator; +import com.sap.sse.gwt.client.dialog.DataEntryDialog; +import com.sap.sse.gwt.client.dialog.DataEntryDialog.DialogCallback; +import com.sap.sse.gwt.client.dialog.DataEntryDialog.Validator; +import com.sap.sse.landscape.aws.common.shared.ShardTargetGroupName; +import com.sap.sse.security.ui.client.component.SelectedElementsCountingButton; + +public class ShardManagementPanel extends SimplePanel { + private final ErrorReporter errorReporter; + private final LandscapeManagementWriteServiceAsync landscapeManagementService; + private final TableWrapperWithMultiSelectionAndFilter regattasTable; + private final TableWrapperWithMultiSelectionAndFilter shardTable; + private final TableWrapperWithMultiSelectionAndFilter selectedKeysTable; + private final TextBox bearerTokenText; + private SailingApplicationReplicaSetDTO replicaSet; + private final BusyIndicator busyIndicator; + private String region; + private String passphrase; + private List leaderboards; + private Map> shardsAndShardingKeys; + private final CaptionPanel leaderboardCaption, leaderboardsInShardCaption; + private final Button addButton, deleteButton; + private final SelectedElementsCountingButton removeShardButton; + private final StringMessages stringMessages; + private boolean hasAnythingChanged; + + public ShardManagementPanel(LandscapeManagementWriteServiceAsync landscapeManagementService, + ErrorReporter errorReporter, StringMessages stringMessages) { + this.stringMessages = stringMessages; + hasAnythingChanged = false; + final VerticalPanel mainPanel = new VerticalPanel(); + mainPanel.setWidth("100%"); + add(mainPanel); + this.errorReporter = errorReporter; + this.landscapeManagementService = landscapeManagementService; + final HorizontalPanel actionPanel = new HorizontalPanel(); + actionPanel.setSpacing(5); + final Button refreshButton = new Button(stringMessages.refresh()); + refreshButton.addClickHandler(event -> refresh()); + actionPanel.add(refreshButton); + regattasTable = new TableWrapperWithMultiSelectionAndFilter( + stringMessages, errorReporter, false, java.util.Optional.empty(), + GWT.create(AdminConsoleTableResources.class), java.util.Optional.empty(), java.util.Optional.empty(), + null) { + @Override + protected Iterable getSearchableStrings(LeaderboardNameDTO t) { + Set res = new HashSet(); + res.add(t.getName()); + return res; + } + }; + final Button addShard = new SelectedElementsCountingButton(stringMessages.addShard(), + regattasTable.getSelectionModel(), e -> addShard(), /* enableWhenSelectionEmpty */ true); + actionPanel.add(addShard); + shardTable = new TableWrapperWithMultiSelectionAndFilter( + stringMessages, errorReporter, false, java.util.Optional.empty(), + GWT.create(AdminConsoleTableResources.class), java.util.Optional.empty(), java.util.Optional.empty(), + null) { + @Override + protected Iterable getSearchableStrings(AwsShardDTO t) { + final Set res = new HashSet(); + res.add(t.getName()); + return res; + } + }; + removeShardButton = new SelectedElementsCountingButton(stringMessages.remove(), + shardTable.getSelectionModel(), /* element name mapper */ rs -> rs.getName(), + stringMessages::doYouReallyWantToRemoveSelectedElements, e -> removeShard()); + actionPanel.add(removeShardButton); + mainPanel.add(actionPanel); + final Grid userCredentials = new Grid(1, 2); + bearerTokenText = new TextBox(); + final Label bearerHintText = new Label(stringMessages.bearerTokenOrNullForApplicationReplicaSetToArchive("")); + userCredentials.setWidget(0, 0, bearerHintText); + userCredentials.setWidget(0, 1, bearerTokenText); + mainPanel.add(userCredentials); + shardTable.addColumn(t -> t.getName(), stringMessages.shardName()); + shardTable.addColumn(t -> String.join(", ", t.getLeaderboardNames()), stringMessages.shardingKeys()); + final SafeHtmlCell targetGroupCell = new SafeHtmlCell(); + final Column targetGroupColumn = new Column(targetGroupCell) { + @Override + public SafeHtml getValue(AwsShardDTO shard) { + return new LinkBuilder().setTargetGroupName(shard.getTargetGroupName()).setRegion(region) + .setPathMode(LinkBuilder.pathModes.TargetGroupSearch).build(); + } + }; + shardTable.addColumn(targetGroupColumn, stringMessages.targetGroup()); + final SafeHtmlCell ausoScalingGroupCell = new SafeHtmlCell(); + final Column autoScalingGroupColumn = new Column(ausoScalingGroupCell) { + @Override + public SafeHtml getValue(AwsShardDTO shard) { + return new LinkBuilder().setAutoScalingGroupName(shard.getAutoScalingGroupName()).setRegion(region) + .setPathMode(LinkBuilder.pathModes.AutoScalingGroupSearch).build(); + } + }; + shardTable.addColumn(autoScalingGroupColumn, stringMessages.autoScalingGroup()); + shardTable.getSelectionModel().addSelectionChangeHandler(event -> { + updateSelectedKeysTable(); + updateAddDeleteButton(); + }); + leaderboardCaption = new CaptionPanel(stringMessages.unshardedLeaderboards()); + regattasTable.addColumn(t -> t.getName(), stringMessages.name()); + regattasTable.getSelectionModel().addSelectionChangeHandler(event -> { + updateAddDeleteButton(); + }); + leaderboardCaption.add(regattasTable); + final HorizontalPanel tableRow = new HorizontalPanel(); + busyIndicator = new SimpleBusyIndicator(); + leaderboardsInShardCaption = new CaptionPanel(stringMessages.leaderboardsInShard()); + selectedKeysTable = new TableWrapperWithMultiSelectionAndFilter( + stringMessages, errorReporter, false, java.util.Optional.empty(), + GWT.create(AdminConsoleTableResources.class), java.util.Optional.empty(), java.util.Optional.empty(), + null) { + @Override + protected Iterable getSearchableStrings(LeaderboardNameDTO t) { + Set result = new HashSet<>(); + if (t != null && !t.getName().isEmpty()) { + result.add(t.getName()); + } + return result; + } + }; + selectedKeysTable.addColumn(t -> t.getName(), stringMessages.name()); + selectedKeysTable.getSelectionModel().addSelectionChangeHandler(event -> { + updateAddDeleteButton(); + }); + leaderboardsInShardCaption.add(selectedKeysTable); + addButton = new Button("<"); + addButton.addClickHandler(event -> addLeaderboardsToShard()); + deleteButton = new Button(">"); + deleteButton.addClickHandler(event -> removeLeaderboardsFromShard()); + final HorizontalPanel insideShardPanel = new HorizontalPanel(); + final VerticalPanel buttonPanel = new VerticalPanel(); + final SimplePanel spaceholder = new SimplePanel(); + spaceholder.setHeight("60px"); + final SimplePanel keysSpaceholder = new SimplePanel(); + buttonPanel.add(spaceholder); + buttonPanel.add(addButton); + buttonPanel.add(deleteButton); + tableRow.add(insideShardPanel); + insideShardPanel.add(shardTable); + insideShardPanel.add(keysSpaceholder); + keysSpaceholder.add(leaderboardsInShardCaption); + insideShardPanel.add(buttonPanel); + insideShardPanel.add(leaderboardCaption); + mainPanel.add(tableRow); + mainPanel.add(busyIndicator); + updateAddDeleteButton(); + } + + private void updateSelectedKeysTable() { + if (shardTable.getSelectionModel().getSelectedSet().size() == 1) { + selectedKeysTable .refresh( + Util.map(shardTable.getSelectionModel().getSelectedSet().iterator().next().getLeaderboardNames(), LeaderboardNameDTO::new)); + } else { + selectedKeysTable.clear(); + } + } + + private void updateAddDeleteButton() { + addButton.setEnabled(shardTable.getSelectionModel().getSelectedSet().size() == 1 + && regattasTable.getSelectionModel().getSelectedSet().size() > 0); + deleteButton.setEnabled(shardTable.getSelectionModel().getSelectedSet().size() == 1 + && selectedKeysTable.getSelectionModel().getSelectedSet().size() > 0); + } + + public void refresh() { + getLeaderboards(); + } + + private void display() { + final List takenLeaderboardNames = new ArrayList<>(); + for (Entry> s : shardsAndShardingKeys.entrySet()) { + for (String leaderboardName : s.getKey().getLeaderboardNames()) { + takenLeaderboardNames.add(leaderboardName); + } + } + final Iterable leaderboardsToDisplay = Util.filter(leaderboards, + leaderboardNameDTO -> !takenLeaderboardNames.contains(leaderboardNameDTO.getName())); + regattasTable.refresh(leaderboardsToDisplay); + shardTable.refresh(shardsAndShardingKeys.keySet()); + } + + private void setBusy(boolean b) { + busyIndicator.setBusy(b); + } + + private void getLeaderboards() { + setBusy(true); + landscapeManagementService.getLeaderboardNames(replicaSet, getBearerToken(), + new AsyncCallback>() { + @Override + public void onSuccess(ArrayList result) { + leaderboards = result; + getShards(); + } + + @Override + public void onFailure(Throwable caught) { + errorReporter.reportError(stringMessages.errorFetchingLeaderboardNames(caught.getMessage())); + setBusy(false); + } + }); + } + + private String getBearerToken() { + return Util.hasLength(bearerTokenText.getValue()) ? bearerTokenText.getValue(): null; + } + + private void getShards() { + setBusy(true); + landscapeManagementService.getShards(replicaSet, region, getBearerToken(), + new AsyncCallback>>() { + @Override + public void onFailure(Throwable caught) { + errorReporter.reportError(caught.getMessage()); + setBusy(false); + } + + @Override + public void onSuccess(Map> result) { + shardsAndShardingKeys = result; + setBusy(false); + display(); + } + }); + } + + private void addShard() { + final Set selectedLeaderboards = regattasTable.getSelectionModel().getSelectedSet(); + if (replicaSet != null) { + final DataEntryDialog nameRequest = new DataEntryDialog( + stringMessages.shardName(), stringMessages.enterShardName(), stringMessages.ok(), stringMessages.cancel(), + new Validator() { + @Override + public String getErrorMessage(String valueToValidate) { + String errorMessage; + if (!Util.hasLength(valueToValidate)) { + errorMessage = stringMessages.pleaseProvideANonEmptyShardName(); + } else { + try { + ShardTargetGroupName.create(replicaSet.getReplicaSetName(), valueToValidate, LandscapeService.SAILING_TARGET_GROUP_NAME_PREFIX); + errorMessage = null; + } catch (IllegalArgumentException e) { + errorMessage = stringMessages.shardNameInvalid(valueToValidate, e.getMessage()); + } + } + return errorMessage; + } + }, new DialogCallback() { + @Override + public void ok(String newShardName) { + hasAnythingChanged = true; + ArrayList l = new ArrayList<>(); + l.addAll(selectedLeaderboards); + busyIndicator.setBusy(true); + landscapeManagementService.addShard(newShardName, l, replicaSet, + getBearerToken(), region, passphrase.getBytes(), new AsyncCallback() { + @Override + public void onSuccess(Void result) { + busyIndicator.setBusy(false); + Notification.notify(stringMessages.shardCreatedSuccessfully(newShardName), NotificationType.SUCCESS); + refresh(); + } + + @Override + public void onFailure(Throwable caught) { + busyIndicator.setBusy(false); + errorReporter.reportError(caught.getMessage()); + } + }); + } + + @Override + public void cancel() { + } + }) { + private final TextBox nameTextBox = createTextBox("", /* length */ 20); + + @Override + protected Widget getAdditionalWidget() { + return nameTextBox; + } + + @Override + protected FocusWidget getInitialFocusWidget() { + return nameTextBox; + } + + @Override + protected String getResult() { + return nameTextBox.getText(); + } + }; + nameRequest.show(); + } + } + + private void removeShard() { + setBusy(true); + for (AwsShardDTO selection : shardTable.getSelectionModel().getSelectedSet()) { + hasAnythingChanged = true; + landscapeManagementService.removeShard(selection, replicaSet, region, passphrase.getBytes(), + new AsyncCallback() { + @Override + public void onFailure(Throwable caught) { + errorReporter.reportError(caught.getMessage()); + setBusy(false); + } + + @Override + public void onSuccess(Void result) { + Notification.notify(stringMessages.deletedShard(selection.getName()), NotificationType.SUCCESS); + refresh(); + } + }); + } + } + + private void addLeaderboardsToShard() { + setBusy(true); + if (!regattasTable.getSelectionModel().getSelectedSet().isEmpty() + && shardTable.getSelectionModel().getSelectedSet().size() == 1) { + final Iterable selectedLeaderboards = regattasTable.getSelectionModel().getSelectedSet(); + final AwsShardDTO shard = shardTable.getSelectionModel().getSelectedSet().iterator().next(); + hasAnythingChanged = true; + landscapeManagementService.appendShardingKeysToShard(selectedLeaderboards, region, shard.getName(), + replicaSet, getBearerToken(), passphrase.getBytes(), new AsyncCallback() { + @Override + public void onFailure(Throwable caught) { + setBusy(false); + errorReporter.reportError(caught.getMessage()); + } + + @Override + public void onSuccess(Void result) { + setBusy(false); + Notification.notify(stringMessages.successfullyAppendedShardingKeysToShard(Util.join(", ", selectedLeaderboards), shard.getName()), NotificationType.SUCCESS); + refresh(); + } + }); + } else { + } + } + + private void removeLeaderboardsFromShard() { + setBusy(true); + if (!selectedKeysTable.getSelectionModel().getSelectedSet().isEmpty() + && shardTable.getSelectionModel().getSelectedSet().size() == 1) { + final Iterable selectedLeaderboards = selectedKeysTable.getSelectionModel().getSelectedSet(); + final AwsShardDTO shard = shardTable.getSelectionModel().getSelectedSet().iterator().next(); + hasAnythingChanged = true; + landscapeManagementService.removeShardingKeysFromShard(selectedLeaderboards, region, shard.getName(), + replicaSet, getBearerToken(), passphrase.getBytes(), new AsyncCallback() { + @Override + public void onFailure(Throwable caught) { + setBusy(false); + errorReporter.reportError(caught.getMessage()); + } + + @Override + public void onSuccess(Void result) { + Notification.notify(stringMessages.successfullyRemovedLeaderboardsFromShard(Util.join(", ", selectedLeaderboards), shard.getName()), NotificationType.SUCCESS); + refresh(); + } + }); + } + } + + public void setReplicaSet(SailingApplicationReplicaSetDTO replicaset) { + replicaSet = replicaset; + } + + public void setRegion(String region) { + this.region = region; + } + + public void setPassphrase(String passphrase) { + this.passphrase = passphrase; + } + + public Boolean hasAnythingChanged() { + return hasAnythingChanged; + } + +} diff --git a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/i18n/StringMessages.java b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/i18n/StringMessages.java index 76bbce91865..d2506da5519 100755 --- a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/i18n/StringMessages.java +++ b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/i18n/StringMessages.java @@ -139,4 +139,23 @@ com.sap.sse.gwt.adminconsole.StringMessages { String errorDuringImport(String message); String errorWhileComparingServerContent(); String differencesInServerContentFound(String serverAName, String aDiffs, String serverBName, String bDiffs); + String shard(); + String shardName(); + String addShard(); + String removeShard(); + String unshardedLeaderboards(); + String shardingKeys(); + String targetGroup(); + String autoScalingGroup(); + String leaderboardsInShard(); + String enterShardName(); + String openShardManagement(); + String successfullyAppendedShardingKeysToShard(String leaderboardNames, String shardName); + String successfullyRemovedLeaderboardsFromShard(String leaderboardNames, String shardName); + String pleaseProvideANonEmptyShardName(); + String shardNameInvalid(String shardName, String message); + String shardCreatedSuccessfully(String newShardName); + String deletedShard(String shardName); + String errorFetchingLeaderboardNames(String message); + String shardingDescription(); } diff --git a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/i18n/StringMessages.properties b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/i18n/StringMessages.properties index 491b451779a..d5b14356a26 100755 --- a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/i18n/StringMessages.properties +++ b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/i18n/StringMessages.properties @@ -127,4 +127,23 @@ scaleAutoScalingReplicasUpOrDown=Scale auto-scaling replicas up/down successfullyScaledAutoScalingReplicasForReplicaSet=Successfully scaled auto-scaling replicas for replica set {0} errorDuringImport=Error during import: {0} errorWhileComparingServerContent=Error while comparing server content -differencesInServerContentFound=Differences found: server {0} has {1} while server {2} has {3} \ No newline at end of file +differencesInServerContentFound=Differences found: server {0} has {1} while server {2} has {3} +shard=Shard +shardName=Shard name +addShard=Add shard +removeShard=Delete shard +unshardedLeaderboards=Leaderboards in no shard +shardingKeys=Sharding keys +targetGroup=Target Group +autoScalingGroup=Auto-Scaling Group +leaderboardsInShard=Leaderboards in selected shard +enterShardName=Please enter a name for this shard +openShardManagement=Open sharding management panel +successfullyAppendedShardingKeysToShard=Successfully appended leaderboards {0} to shard {1} +successfullyRemovedLeaderboardsFromShard=Successfully removed leaderboards {0} from shard {1} +pleaseProvideANonEmptyShardName=Please provide a non-empty shard name +shardNameInvalid=Shard name "{0}" is invalid: {1} +shardCreatedSuccessfully=Shard "{0}" created successfully +deletedShard=Deleted shard "{0}" +errorFetchingLeaderboardNames=Error fetching leaderboard names: {0} +shardingDescription=Select zero or more leaderboards not yet in a shard, then click on "Add shard" to create a new shard handling the leaderboards selected.\nUse the arrow buttons to add leaderboards to or remove them from a selected shard.\nNote that changes are carried out immediately, not when you close this dialog. \ No newline at end of file diff --git a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/i18n/StringMessages_de.properties b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/i18n/StringMessages_de.properties index 7ca8596abf7..04cfd3eab90 100755 --- a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/i18n/StringMessages_de.properties +++ b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/client/i18n/StringMessages_de.properties @@ -126,4 +126,23 @@ scaleAutoScalingReplicasUpOrDown=Auto-skalierende Replikas hoch/herunter skalier successfullyScaledAutoScalingReplicasForReplicaSet=Auto-skalierende Replikas für Anwendungs-Cluster {0} erfolgreich skaliert errorDuringImport=Fehler beim Importieren: {0} errorWhileComparingServerContent=Fehler beim Vergleich der Server-Inhalte -differencesInServerContentFound=Unterschiede gefunden: Server {0} enthält {1} während Server {2} {3} enthält. \ No newline at end of file +differencesInServerContentFound=Unterschiede gefunden: Server {0} enthält {1} während Server {2} {3} enthält. +shard=Shard +shardName=Shard Name +addShard=Shard hinzufügen +removeShard=Shard entfernen +unshardedLeaderboards=Keinem Shard zugeordnete Ranglisten +shardingKeys=Sharding Schlüssel +targetGroup=Target Group +autoScalingGroup=Auto-Scaling Group +leaderboardsInShard=Ranglisten im ausgewählten Shard +enterShardName=Bitte Shard-Namen eingeben +openShardManagement=Shardmanagement Panel öffnen +successfullyAppendedShardingKeysToShard=Die Ranglisten {0} wurden erfolgreich zum Shard {1} hinzugefügt. +successfullyRemovedLeaderboardsFromShard=Die Ranglisten {0} wurden erfolgreich aus dem Shard {1} entfernt. +pleaseProvideANonEmptyShardName=Bitte einen nicht-leeren Shard-Namen eingeben +shardNameInvalid=Shard-Name "{0}" ist ungültig: {1} +shardCreatedSuccessfully=Shard "{0}" erfolgreich erstellt +deletedShard=Shard "{0}" gelöscht +errorFetchingLeaderboardNames=Fehler beim Bestimmen der Ranglisten-Namen: {0} +shardingDescription=Zum Anlegen eines neuen Shards optional Ranglisten auswählen, die direkt vom neuen Shard übernommen werden sollen, dann auf "Shard hinzufügen" klicken.\nMit den Pfeil-Schaltflächen Ranglisten aus dem ausgewählten Shard entfernen oder zu diesem hinzufügen.\nAchtung: die Änderungen werden sofort angewendet, nicht erst, wenn die Dialog geschlossen wird. \ No newline at end of file diff --git a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/server/LandscapeManagementWriteServiceImpl.java b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/server/LandscapeManagementWriteServiceImpl.java index b22f01b2885..20fe8f01e5e 100755 --- a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/server/LandscapeManagementWriteServiceImpl.java +++ b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/server/LandscapeManagementWriteServiceImpl.java @@ -13,6 +13,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -37,6 +38,7 @@ import com.sap.sailing.landscape.SailingAnalyticsHost; import com.sap.sailing.landscape.SailingAnalyticsMetrics; import com.sap.sailing.landscape.SailingAnalyticsProcess; import com.sap.sailing.landscape.SailingReleaseRepository; +import com.sap.sailing.landscape.common.RemoteServiceMappingConstants; import com.sap.sailing.landscape.common.SharedLandscapeConstants; import com.sap.sailing.landscape.impl.SailingAnalyticsHostImpl; import com.sap.sailing.landscape.impl.SailingAnalyticsProcessImpl; @@ -49,7 +51,9 @@ import com.sap.sailing.landscape.ui.client.LandscapeManagementWriteService; import com.sap.sailing.landscape.ui.impl.Activator; import com.sap.sailing.landscape.ui.shared.AmazonMachineImageDTO; import com.sap.sailing.landscape.ui.shared.AwsInstanceDTO; +import com.sap.sailing.landscape.ui.shared.AwsShardDTO; import com.sap.sailing.landscape.ui.shared.CompareServersResultDTO; +import com.sap.sailing.landscape.ui.shared.LeaderboardNameDTO; import com.sap.sailing.landscape.ui.shared.MongoEndpointDTO; import com.sap.sailing.landscape.ui.shared.MongoProcessDTO; import com.sap.sailing.landscape.ui.shared.MongoScalingInstructionsDTO; @@ -60,6 +64,7 @@ import com.sap.sailing.landscape.ui.shared.SailingAnalyticsProcessDTO; import com.sap.sailing.landscape.ui.shared.SailingApplicationReplicaSetDTO; import com.sap.sailing.landscape.ui.shared.SerializationDummyDTO; import com.sap.sailing.server.gateway.interfaces.CompareServersResult; +import com.sap.sailing.server.gateway.interfaces.SailingServer; import com.sap.sse.common.Duration; import com.sap.sse.common.TimePoint; import com.sap.sse.common.Util; @@ -76,6 +81,7 @@ import com.sap.sse.landscape.aws.ApplicationProcessHost; import com.sap.sse.landscape.aws.AwsApplicationReplicaSet; import com.sap.sse.landscape.aws.AwsInstance; import com.sap.sse.landscape.aws.AwsLandscape; +import com.sap.sse.landscape.aws.AwsShard; import com.sap.sse.landscape.aws.HostSupplier; import com.sap.sse.landscape.aws.common.shared.PlainRedirectDTO; import com.sap.sse.landscape.aws.common.shared.RedirectDTO; @@ -176,7 +182,7 @@ public class LandscapeManagementWriteServiceImpl extends ResultCachingProxiedRem public ArrayList getRegions() { checkLandscapeManageAwsPermission(); final ArrayList result = new ArrayList<>(); - Util.addAll(Util.map(AwsLandscape.obtain().getRegions(), r->r.getId()), result); + Util.addAll(Util.map(AwsLandscape.obtain(RemoteServiceMappingConstants.pathPrefixForShardingKey).getRegions(), r->r.getId()), result); return result; } @@ -385,7 +391,7 @@ public class LandscapeManagementWriteServiceImpl extends ResultCachingProxiedRem @Override public byte[] getEncryptedSshPrivateKey(String regionId, String keyName) throws JSchException { - final AwsLandscape landscape = AwsLandscape.obtain(); + final AwsLandscape landscape = AwsLandscape.obtain(RemoteServiceMappingConstants.pathPrefixForShardingKey); final SSHKeyPair keyPair = landscape.getSSHKeyPair(new AwsRegion(regionId, landscape), keyName); getSecurityService().checkCurrentUserReadPermission(keyPair); return keyPair.getEncryptedPrivateKey(); @@ -393,7 +399,7 @@ public class LandscapeManagementWriteServiceImpl extends ResultCachingProxiedRem @Override public byte[] getSshPublicKey(String regionId, String keyName) throws JSchException { - final AwsLandscape landscape = AwsLandscape.obtain(); + final AwsLandscape landscape = AwsLandscape.obtain(RemoteServiceMappingConstants.pathPrefixForShardingKey); final SSHKeyPair keyPair = landscape.getSSHKeyPair(new AwsRegion(regionId, landscape), keyName); getSecurityService().checkCurrentUserReadPermission(keyPair); return keyPair.getPublicKey(); @@ -603,8 +609,8 @@ public class LandscapeManagementWriteServiceImpl extends ResultCachingProxiedRem } @Override - public SerializationDummyDTO serializationDummy(ProcessDTO mongoProcessDTO, AwsInstanceDTO awsInstanceDTO, - SailingApplicationReplicaSetDTO sailingApplicationReplicationSetDTO) { + public SerializationDummyDTO serializationDummy(ProcessDTO mongoProcessDTO, AwsInstanceDTO awsInstanceDTO, AwsShardDTO shardDTO, + SailingApplicationReplicaSetDTO sailingApplicationReplicationSetDTO, LeaderboardNameDTO leaderboard) { return null; } @@ -847,4 +853,93 @@ public class LandscapeManagementWriteServiceImpl extends ResultCachingProxiedRem getLandscapeService().changeAutoScalingReplicasInstanceType(replicaSet, InstanceType.valueOf(instanceTypeName)), Optional.ofNullable(optionalKeyName), privateKeyEncryptionPassphrase); } + + @Override + public ArrayList getLeaderboardNames(SailingApplicationReplicaSetDTO replicaSet, + String bearerToken) throws Exception { + final SailingServer server = getLandscapeService().getSailingServer(replicaSet.getHostname(), bearerToken, + /* HTTPS Port */ Optional.of(443)); + return new ArrayList<>(Util.asList(Util.map(server.getLeaderboardNames(), LeaderboardNameDTO::new))); + } + + @Override + public void addShard(String shardName, ArrayList selectedLeaderBoardNames, + SailingApplicationReplicaSetDTO replicaSetDTO, String bearerToken, String region, + byte[] passphraseForPrivateKeyDecryption) throws Exception { + checkLandscapeManageAwsPermission(); + final AwsRegion awsRegion = new AwsRegion(replicaSetDTO.getMaster().getHost().getRegion(), getLandscape()); + final AwsApplicationReplicaSet> awsReplicaSet = convertFromApplicationReplicaSetDTO( + awsRegion, replicaSetDTO); + getLandscapeService().addShard(Util.map(selectedLeaderBoardNames, t -> t.getName()), awsReplicaSet, awsRegion, + bearerToken, passphraseForPrivateKeyDecryption, shardName); + } + + @Override + public Map> getShards(SailingApplicationReplicaSetDTO replicaSetDTO, + String region, String bearerToken) throws Exception { + checkLandscapeManageAwsPermission(); + final AwsRegion awsRegion = new AwsRegion(replicaSetDTO.getMaster().getHost().getRegion(), getLandscape()); + final Map> shardingKeysForShards = new HashMap<>(); + final SailingServer server = getLandscapeService().getSailingServer(replicaSetDTO.getHostname(), bearerToken, Optional.empty()); + final AwsApplicationReplicaSet> applicationServerReplicaSet = convertFromApplicationReplicaSetDTO( + awsRegion, replicaSetDTO); + final Map leaderboardNamesByShardingKeys = new HashMap<>(); + for (String leaderboard : server.getLeaderboardNames()) { + leaderboardNamesByShardingKeys.put(server.getLeaderboardShardingKey(leaderboard), leaderboard); + } + for (Entry, Iterable> entry : applicationServerReplicaSet.getShards().entrySet()) { + shardingKeysForShards.put(createAwsShardDTO(entry.getKey(), applicationServerReplicaSet.getName(), server, leaderboardNamesByShardingKeys), + entry.getValue()); + } + final Map> res = new HashMap<>(); + for (Entry> entry : shardingKeysForShards.entrySet()) { + res.put(entry.getKey(), entry.getValue()); + } + return res; + } + + @Override + public void removeShard(AwsShardDTO shard, SailingApplicationReplicaSetDTO replicaSetDTO, String region, + byte[] passphrase) throws Exception { + checkLandscapeManageAwsPermission(); + final AwsRegion awsRegion = new AwsRegion(replicaSetDTO.getMaster().getHost().getRegion(), getLandscape()); + final AwsApplicationReplicaSet> applicationServerReplicaSet = convertFromApplicationReplicaSetDTO( + awsRegion, replicaSetDTO); + getLandscapeService().removeShard(applicationServerReplicaSet, shard.getTargetgroupArn()); + } + + @Override + public void appendShardingKeysToShard(Iterable sharindKeysToAppend, String region, + String shardName, SailingApplicationReplicaSetDTO replicaSet, String bearerToken, + byte[] passphraseForPrivateKeyDecryption) throws Exception { + checkLandscapeManageAwsPermission(); + final AwsRegion awsRegion = new AwsRegion(replicaSet.getMaster().getHost().getRegion(), getLandscape()); + final AwsApplicationReplicaSet> rs = convertFromApplicationReplicaSetDTO( + awsRegion, replicaSet); + final AwsApplicationReplicaSet> applicationReplicaSet = getLandscape() + .getApplicationReplicaSet(awsRegion, rs.getServerName(), rs.getMaster(), rs.getReplicas()); + getLandscapeService().appendShardingKeysToShard(Util.map(sharindKeysToAppend, t -> t.getName()), + applicationReplicaSet, passphraseForPrivateKeyDecryption, awsRegion, shardName, bearerToken); + } + + public void removeShardingKeysFromShard(Iterable selectedShardingKeys, String region, + String shardName, SailingApplicationReplicaSetDTO replicaSet, String bearerToken, + byte[] passphraseForPrivateKeyDecryption) throws Exception { + checkLandscapeManageAwsPermission(); + final AwsRegion awsRegion = new AwsRegion(region, getLandscape()); + final AwsApplicationReplicaSet> rs = convertFromApplicationReplicaSetDTO( + awsRegion, replicaSet); + getLandscapeService().removeShardingKeysFromShard(Util.asList(Util.map(selectedShardingKeys, t -> t.getName())), + rs, passphraseForPrivateKeyDecryption, awsRegion, shardName, bearerToken); + } + + public AwsShardDTO createAwsShardDTO(AwsShard shard, String replicaSetName, SailingServer server, + Map leaderboardNamesByShardingKeys) throws Exception { + return new AwsShardDTO(Util.filter(Util.map(shard.getKeys(), + shardingKey -> leaderboardNamesByShardingKeys.get(shardingKey)), leaderboardName->leaderboardName!=null), + shard.getTargetGroup().getTargetGroupArn(), shard.getTargetGroup().getName(), + shard.getAutoScalingGroup().getAutoScalingGroup().autoScalingGroupARN(), + shard.getTargetGroup().getLoadBalancerArn(), shard.getAutoScalingGroup().getName(), + shard.getName() == null ? "" : shard.getName(), replicaSetName); + } } diff --git a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/shared/AwsShardDTO.java b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/shared/AwsShardDTO.java new file mode 100644 index 00000000000..7da9f1a2554 --- /dev/null +++ b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/shared/AwsShardDTO.java @@ -0,0 +1,72 @@ +package com.sap.sailing.landscape.ui.shared; + +import java.util.ArrayList; + +import com.sap.sse.common.Named; +import com.sap.sse.common.Util; + +public class AwsShardDTO implements Named { + private static final long serialVersionUID = 1L; + ArrayList leaderboardNames; + private String targetGroupArn; + private String targetGroupName; + private String autoScalingGroupArn; + private String autoScalingGroupName; + private String autoBalancerArn; + private String name; + private String replicaName; + + @SuppressWarnings("unused") // for GWT serialisation only + private AwsShardDTO() { + } + + public AwsShardDTO(Iterable leaderboardNames, String targetGroupArn, String targetGroupName, String autoScalinggroupArn, + String autoBalancerArn, String autoScalingGroupName, String name, String replicaName) { + this.leaderboardNames = new ArrayList<>(); + Util.addAll(leaderboardNames, this.leaderboardNames); + this.targetGroupArn = targetGroupArn; + this.targetGroupName = targetGroupName; + this.autoBalancerArn = autoBalancerArn; + this.autoScalingGroupName = autoScalingGroupName; + this.autoScalingGroupArn = autoScalinggroupArn; + this.name = name; + this.replicaName = replicaName; + } + + public String getReplicaName() { + return replicaName; + } + + public ArrayList getLeaderboardNames() { + return leaderboardNames; + } + + public String getTargetgroupArn() { + return targetGroupArn; + } + + public String getTargetGroupName() { + return targetGroupName; + } + + public String getAutosclaingGroupArn() { + return autoScalingGroupArn; + } + + public String getAutobalancerArn() { + return autoBalancerArn; + } + + public void setName(String name) { + this.name = name; + } + + public String getAutoScalingGroupName() { + return this.autoScalingGroupName; + } + + @Override + public String getName() { + return name; + } +} diff --git a/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/shared/LeaderboardNameDTO.java b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/shared/LeaderboardNameDTO.java new file mode 100644 index 00000000000..f5d7960046d --- /dev/null +++ b/java/com.sap.sailing.landscape.ui/src/com/sap/sailing/landscape/ui/shared/LeaderboardNameDTO.java @@ -0,0 +1,22 @@ +package com.sap.sailing.landscape.ui.shared; + +import com.sap.sse.common.Named; +import com.sap.sse.security.shared.dto.NamedDTO; + +/** + * This class is supposed to represent leader board names. It's just a wrapper for ensuring that we can use the + * {@link Named} functionality in the UI. + * + * @author I569653 + * + */ +public class LeaderboardNameDTO extends NamedDTO { + private static final long serialVersionUID = -173460185784328741L; + + @SuppressWarnings({ "deprecation", "unused" }) + private LeaderboardNameDTO() {} // for GWT serialisation only + + public LeaderboardNameDTO(String name) { + super(name); + } +} \ No newline at end of file diff --git a/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/LandscapeService.java b/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/LandscapeService.java index 622c6bb5394..d1ea58ef6d7 100644 --- a/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/LandscapeService.java +++ b/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/LandscapeService.java @@ -14,6 +14,7 @@ import com.sap.sailing.landscape.procedures.SailingAnalyticsReplicaConfiguration import com.sap.sailing.landscape.procedures.SailingAnalyticsReplicaConfiguration.Builder; import com.sap.sailing.landscape.procedures.StartMultiServer; import com.sap.sailing.server.gateway.interfaces.CompareServersResult; +import com.sap.sailing.server.gateway.interfaces.SailingServer; import com.sap.sse.common.Duration; import com.sap.sse.common.Util.Pair; import com.sap.sse.landscape.Release; @@ -212,7 +213,11 @@ public interface LandscapeService { * TODO bug5674: before registering the master with the TGs, spin up as many new replicas as there are currently * replicas; wait until they are all ready, then register master and new replicas in TGs and de-register old replicas. * Then terminate old auto-scaling replicas and update any unmanaged replica in-place. When the number of auto-scaling - * replicas has reached the desired size of the auto-scaling group, terminate the replicas created explicitly. + * replicas has reached the desired size of the auto-scaling group, terminate the replicas created explicitly.

+ * + * Shards are updated by spinning up replicas for the temporary transition and changing the auto scaling config. + * After that all shard replicas are getting shutdown and restarted with the new launch config. + * It's expected that the replica set has its own auto scaling group if it has shards. */ AwsApplicationReplicaSet> upgradeApplicationReplicaSet(AwsRegion region, AwsApplicationReplicaSet> replicaSet, @@ -370,7 +375,7 @@ public interface LandscapeService { InterruptedException, ExecutionException, Exception; /** - * If the {@code replicaSet} provided has an auto-scaling group, its launch configuration is adjusted such that it + * If the {@code replicaSet} provided has one or more auto-scaling groups, their launch configuration is adjusted such that it * matches the {@code optionalInstanceType}. The existing replicas managed currently by the auto-scaling group are * replaced one by one with new instances with the new configuration. This happens by setting the auto-scaling * group's new minimum size to the current number of instances managed by the auto-scaling group plus one, then @@ -385,4 +390,24 @@ public interface LandscapeService { boolean isEligibleForDeployment(SailingAnalyticsHost host, String serverName, int port, Optional waitForProcessTimeout, String optionalKeyName, byte[] privateKeyEncryptionPassphrase) throws Exception; + + SailingServer getSailingServer(String hostname, String username, String password, Optional port) + throws MalformedURLException; + + SailingServer getSailingServer(String hostname, String bearertoken, Optional port) + throws MalformedURLException; + + void removeShardingKeysFromShard(Iterable selectedleaderboards, + AwsApplicationReplicaSet> applicationReplicaSet, + byte[] passphraseForPrivateKeyDecription,AwsRegion region, String shardName, String bearertoken) throws Exception; + + public void appendShardingKeysToShard(Iterable selectedLeaderboards, + AwsApplicationReplicaSet> applicationReplicaSet, + byte[] passphraseForPrivateKeyDecription, AwsRegion region, String shardName, String bearertoken) throws Exception; + + void removeShard(AwsApplicationReplicaSet> applicationReplicaSet, String shardTargetGroupArn) throws Exception; + + void addShard(Iterable selectedLeaderboardNames, + AwsApplicationReplicaSet> applicationReplicaSet, + AwsRegion region, String bearertoken, byte[] passphraseForPrivateKeyDecription, String shardName) throws Exception; } diff --git a/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/SailingAnalyticsProcess.java b/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/SailingAnalyticsProcess.java index b124aefddeb..c0604667e9b 100755 --- a/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/SailingAnalyticsProcess.java +++ b/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/SailingAnalyticsProcess.java @@ -11,7 +11,6 @@ import com.sap.sse.landscape.aws.AwsApplicationProcess; public interface SailingAnalyticsProcess extends AwsApplicationProcess> { static Logger logger = Logger.getLogger(SailingAnalyticsProcess.class.getName()); - static String HEALTH_CHECK_PATH = "/gwt/status"; int getExpeditionUdpPort(Optional optionalTimeout, Optional optionalKeyName, byte[] privateKeyEncryptionPassphrase) throws Exception; diff --git a/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/impl/EligbleInstanceForReplicaSetFindingStrategyImpl.java b/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/impl/EligbleInstanceForReplicaSetFindingStrategyImpl.java index 47c9793a85a..1216e870631 100644 --- a/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/impl/EligbleInstanceForReplicaSetFindingStrategyImpl.java +++ b/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/impl/EligbleInstanceForReplicaSetFindingStrategyImpl.java @@ -1,6 +1,7 @@ package com.sap.sailing.landscape.impl; import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; @@ -239,7 +240,7 @@ public class EligbleInstanceForReplicaSetFindingStrategyImpl implements Eligible boolean hasManagedReplicaInDifferentAZ = false; boolean hasUnmanagedReplicaInDifferentAZ = false; for (final SailingAnalyticsProcess replica : replicaSet.getReplicas()) { - final boolean isManaged = replica.getHost().isManagedByAutoScalingGroup(replicaSet.getAutoScalingGroup()); + final boolean isManaged = replica.getHost().isManagedByAutoScalingGroup(Collections.singleton(replicaSet.getAutoScalingGroup())); final boolean isInDifferentAZ = !replica.getHost().getAvailabilityZone().equals(availabilityZone); hasManagedReplicas = hasManagedReplicas || isManaged; hasManagedReplicaInDifferentAZ = hasManagedReplicaInDifferentAZ || (isManaged && isInDifferentAZ); diff --git a/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/impl/LandscapeServiceImpl.java b/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/impl/LandscapeServiceImpl.java index 413e430484a..7b47c77d5e9 100644 --- a/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/impl/LandscapeServiceImpl.java +++ b/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/impl/LandscapeServiceImpl.java @@ -5,10 +5,13 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -36,6 +39,7 @@ import com.sap.sailing.landscape.SailingAnalyticsHost; import com.sap.sailing.landscape.SailingAnalyticsMetrics; import com.sap.sailing.landscape.SailingAnalyticsProcess; import com.sap.sailing.landscape.SailingReleaseRepository; +import com.sap.sailing.landscape.common.RemoteServiceMappingConstants; import com.sap.sailing.landscape.common.SharedLandscapeConstants; import com.sap.sailing.landscape.procedures.CreateLaunchConfigurationAndAutoScalingGroup; import com.sap.sailing.landscape.procedures.DeployProcessOnMultiServer; @@ -72,6 +76,7 @@ import com.sap.sse.landscape.aws.AwsAutoScalingGroup; import com.sap.sse.landscape.aws.AwsAvailabilityZone; import com.sap.sse.landscape.aws.AwsInstance; import com.sap.sse.landscape.aws.AwsLandscape; +import com.sap.sse.landscape.aws.AwsShard; import com.sap.sse.landscape.aws.HostSupplier; import com.sap.sse.landscape.aws.ReverseProxy; import com.sap.sse.landscape.aws.Tags; @@ -80,11 +85,15 @@ import com.sap.sse.landscape.aws.common.shared.RedirectDTO; import com.sap.sse.landscape.aws.impl.AwsApplicationReplicaSetImpl; import com.sap.sse.landscape.aws.impl.AwsRegion; import com.sap.sse.landscape.aws.impl.DNSCache; +import com.sap.sse.landscape.aws.orchestration.AddShardingKeyToShard; import com.sap.sse.landscape.aws.orchestration.AwsApplicationConfiguration; import com.sap.sse.landscape.aws.orchestration.CopyAndCompareMongoDatabase; import com.sap.sse.landscape.aws.orchestration.CreateDNSBasedLoadBalancerMapping; import com.sap.sse.landscape.aws.orchestration.CreateDynamicLoadBalancerMapping; import com.sap.sse.landscape.aws.orchestration.CreateLoadBalancerMapping; +import com.sap.sse.landscape.aws.orchestration.CreateShard; +import com.sap.sse.landscape.aws.orchestration.RemoveShardingKeyFromShard; +import com.sap.sse.landscape.aws.orchestration.ShardProcedure; import com.sap.sse.landscape.aws.orchestration.StartAwsHost; import com.sap.sse.landscape.mongodb.Database; import com.sap.sse.landscape.mongodb.MongoEndpoint; @@ -499,7 +508,7 @@ public class LandscapeServiceImpl implements LandscapeService { // only terminate replica if not running on host created by auto-scaling group final AwsAutoScalingGroup autoScalingGroup = applicationReplicaSet.getAutoScalingGroup(); for (final SailingAnalyticsProcess replica : applicationReplicaSet.getReplicas()) { - if (autoScalingGroup == null || !replica.getHost().isManagedByAutoScalingGroup(autoScalingGroup)) { + if (autoScalingGroup == null || !replica.getHost().isManagedByAutoScalingGroup(Collections.singleton(autoScalingGroup))) { logger.info("Found replica "+replica+" running on an instance not managed by auto-scaling group " + (autoScalingGroup != null ? autoScalingGroup.getName() : "") + ". Stopping..."); replica.stopAndTerminateIfLast(Landscape.WAIT_FOR_PROCESS_TIMEOUT, Optional.ofNullable(optionalKeyName), passphraseForPrivateKeyDecryption); @@ -518,6 +527,10 @@ public class LandscapeServiceImpl implements LandscapeService { final AwsRegion region = new AwsRegion(regionId, getLandscape()); final AwsAutoScalingGroup autoScalingGroup = applicationReplicaSet.getAutoScalingGroup(); final CompletableFuture autoScalingGroupRemoval; + // Remove all shards + for (AwsShard shard : applicationReplicaSet.getShards().keySet()) { + applicationReplicaSet.removeShard(shard, getLandscape()); + } terminateReplicasNotManagedByAutoScalingGroup(applicationReplicaSet, optionalKeyName, passphraseForPrivateKeyDecryption); if (autoScalingGroup != null) { // remove the launch configuration used by the auto scaling group and the auto scaling group itself; @@ -565,7 +578,7 @@ public class LandscapeServiceImpl implements LandscapeService { final Iterable> replicas = applicationReplicaSet.getMaster().getReplicas(Landscape.WAIT_FOR_PROCESS_TIMEOUT, new SailingAnalyticsHostSupplier(), processFactoryFromHostAndServerDirectory); for (final SailingAnalyticsProcess replica : replicas) { - if (replica.getHost().isManagedByAutoScalingGroup(autoScalingGroup) && replica.isAlive(Landscape.WAIT_FOR_PROCESS_TIMEOUT)) { + if (replica.getHost().isManagedByAutoScalingGroup(Collections.singleton(autoScalingGroup)) && replica.isAlive(Landscape.WAIT_FOR_PROCESS_TIMEOUT)) { logger.info("Replica "+replica+" is managed by auto-scaling group "+autoScalingGroup.getName()+" and is still alive."); return false; } @@ -625,7 +638,7 @@ public class LandscapeServiceImpl implements LandscapeService { keyId = sessionCredentials.getAccessKeyId(); secret = sessionCredentials.getSecretAccessKey(); sessionToken = sessionCredentials.getSessionToken(); - result = AwsLandscape.obtain(keyId, secret, sessionToken); + result = AwsLandscape.obtain(keyId, secret, sessionToken, RemoteServiceMappingConstants.pathPrefixForShardingKey); } else { result = null; } @@ -659,7 +672,7 @@ public class LandscapeServiceImpl implements LandscapeService { @Override public void createMfaSessionCredentials(String awsAccessKey, String awsSecret, String mfaTokenCode) { - final Credentials credentials = AwsLandscape.obtain(awsAccessKey, awsSecret).getMfaSessionCredentials(mfaTokenCode); + final Credentials credentials = AwsLandscape.obtain(awsAccessKey, awsSecret, RemoteServiceMappingConstants.pathPrefixForShardingKey).getMfaSessionCredentials(mfaTokenCode); final AwsSessionCredentialsWithExpiryImpl result = new AwsSessionCredentialsWithExpiryImpl( credentials.accessKeyId(), credentials.secretAccessKey(), credentials.sessionToken(), TimePoint.of(credentials.expiration().toEpochMilli())); @@ -843,7 +856,8 @@ public class LandscapeServiceImpl implements LandscapeService { final DNSCache dnsCache = landscape.getNewDNSCache(); final AwsApplicationReplicaSet> applicationReplicaSet = new AwsApplicationReplicaSetImpl<>(replicaSetName, masterHostname, master, /* no replicas yet */ Optional.empty(), - allLoadBalancersInRegion, allTargetGroupsInRegion, allLoadBalancerRulesInRegion, autoScalingGroups, launchConfigurations, dnsCache); + allLoadBalancersInRegion, allTargetGroupsInRegion, allLoadBalancerRulesInRegion, + autoScalingGroups, launchConfigurations, dnsCache, RemoteServiceMappingConstants.pathPrefixForShardingKey); return applicationReplicaSet; } @@ -854,16 +868,28 @@ public class LandscapeServiceImpl implements LandscapeService { String replicaReplicationBearerToken) throws MalformedURLException, IOException, TimeoutException, Exception { if (replicaSet.isLocalReplicaSet()) { - throw new IllegalArgumentException("A replica set cannot upgrade itself. Current replica set: "+ServerInfo.getName()); + throw new IllegalArgumentException( + "A replica set cannot upgrade itself. Current replica set: " + ServerInfo.getName()); + } + if (!replicaSet.getShards().isEmpty() && replicaSet.getAutoScalingGroup() == null) { + throw new IllegalStateException( + "A replica set is expected to have an auto scaling group if it has shards"); } final Release release = getRelease(releaseOrNullForLatestMaster); final String effectiveReplicaReplicationBearerToken = getEffectiveBearerToken(replicaReplicationBearerToken); final int oldAutoScalingGroupMinSize; final AwsAutoScalingGroup autoScalingGroup = replicaSet.getAutoScalingGroup(); + final Collection affectedAutoScalingGroups = new ArrayList<>(); final int autoScalingReplicaCount; + replicaSet.getShards().keySet().forEach(t -> affectedAutoScalingGroups.add(t.getAutoScalingGroup())); if (autoScalingGroup != null) { + affectedAutoScalingGroups.add(autoScalingGroup); oldAutoScalingGroupMinSize = autoScalingGroup.getAutoScalingGroup().minSize(); - autoScalingReplicaCount = autoScalingGroup.getAutoScalingGroup().desiredCapacity(); + int tempAutoScalingReplicaCount = 0; + for (AwsAutoScalingGroup asg : affectedAutoScalingGroups) { + tempAutoScalingReplicaCount = tempAutoScalingReplicaCount + asg.getAutoScalingGroup().desiredCapacity(); + } + autoScalingReplicaCount = tempAutoScalingReplicaCount; } else { oldAutoScalingGroupMinSize = -1; autoScalingReplicaCount = -1; @@ -873,7 +899,8 @@ public class LandscapeServiceImpl implements LandscapeService { final SailingAnalyticsProcess additionalReplicaStarted = ensureAtLeastOneReplicaExistsStopReplicatingAndRemoveMasterFromTargetGroups( replicaSet, optionalKeyName, privateKeyEncryptionPassphrase, effectiveReplicaReplicationBearerToken); if (replicaSet.getAutoScalingGroup() != null) { - getLandscape().updateReleaseInAutoScalingGroup(region, replicaSet.getAutoScalingGroup(), replicaSet.getName(), release); + getLandscape().updateReleaseInAutoScalingGroups(region, replicaSet.getAutoScalingGroup().getLaunchConfiguration(), + affectedAutoScalingGroups, replicaSet.getName(), release); } final SailingAnalyticsProcess master = replicaSet.getMaster(); master.refreshToRelease(release, Optional.ofNullable(optionalKeyName), privateKeyEncryptionPassphrase); @@ -882,27 +909,45 @@ public class LandscapeServiceImpl implements LandscapeService { master.waitUntilReady(Optional.of(Duration.ONE_DAY)); // wait a little longer since master may need to re-load many races logger.info("Launching upgrade replicas based on current set of replicas for replica set "+replicaSet.getName()); final Iterable> temporaryUpgradeReplicas = launchUpgradeReplicasAndWaitUntilReady( - replicaSet, release, effectiveReplicaReplicationBearerToken, Optional.ofNullable(optionalKeyName), privateKeyEncryptionPassphrase); + replicaSet, release, effectiveReplicaReplicationBearerToken, Optional.ofNullable(optionalKeyName), + privateKeyEncryptionPassphrase); + final List> temporaryUpgradeReplicasMutable = Util.asList(temporaryUpgradeReplicas); // register master again with master and public target group - logger.info("Adding master "+master+" and dedicated upgraded temporary replicas to target groups "+ - replicaSet.getPublicTargetGroup()+" and "+replicaSet.getMasterTargetGroup()); + logger.info("Adding master " + master + " and dedicated upgraded temporary replicas to target groups " + + replicaSet.getPublicTargetGroup() + " and " + replicaSet.getMasterTargetGroup()); replicaSet.getPublicTargetGroup().addTarget(master.getHost()); replicaSet.getMasterTargetGroup().addTarget(master.getHost()); - replicaSet.getPublicTargetGroup().addTargets(Util.map(temporaryUpgradeReplicas, temporaryUpgradeReplica->temporaryUpgradeReplica.getHost())); + // the following map stores the assignment of the temporary upgrade replicas to the public / shard target groups + // for later removal as the final auto-scaling replicas start to replace them: + final Map, Iterable>> tempUpgradeReplicasByTargetGroup = new HashMap<>(); + // add as many temporary upgrade replicas to each shard's target group as the target group has targets now: + for (final AwsShard shard : replicaSet.getShards().keySet()) { + final TargetGroup shardTargetGroup = shard.getTargetGroup(); + final Collection> hosts = new ArrayList<>(); + final int numberOfTargets = shardTargetGroup.getRegisteredTargets().size(); + for (int i = 0; i < numberOfTargets; i++) { + final SailingAnalyticsProcess tempUpgradeReplicaProcess = temporaryUpgradeReplicasMutable.remove(0); + shardTargetGroup.addTarget(tempUpgradeReplicaProcess.getHost()); + hosts.add(tempUpgradeReplicaProcess.getHost()); + } + tempUpgradeReplicasByTargetGroup.put(shardTargetGroup, hosts); + } + // add all other temporary upgrade replicas not used for shards to the public target group: + replicaSet.getPublicTargetGroup().addTargets(Util.map(temporaryUpgradeReplicasMutable, + temporaryUpgradeReplica -> temporaryUpgradeReplica.getHost())); + tempUpgradeReplicasByTargetGroup.put(replicaSet.getPublicTargetGroup(), Util.map(temporaryUpgradeReplicasMutable, temporaryUpgradeReplica->temporaryUpgradeReplica.getHost())); // if a replica was spun up (additionalReplicaStarted), remove from public target group and terminate: if (additionalReplicaStarted != null) { replicasToStopAfterUpgradingMaster.add(additionalReplicaStarted); if (replicaSet.getAutoScalingGroup() != null) { // run updating the auto-scaling group in the background; it's more time-critical now // to remove the old replicas from the target group - ThreadPoolUtil.INSTANCE.getDefaultBackgroundTaskThreadPoolExecutor().execute(ThreadPoolUtil.INSTANCE.associateWithSubjectIfAny(()-> - { - try { - getLandscape().updateAutoScalingGroupMinSize(replicaSet.getAutoScalingGroup(), oldAutoScalingGroupMinSize); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } - })); + for (AwsAutoScalingGroup asg : affectedAutoScalingGroups) { + ThreadPoolUtil.INSTANCE.getDefaultBackgroundTaskThreadPoolExecutor() + .execute(ThreadPoolUtil.INSTANCE.associateWithSubjectIfAny(() -> { + getLandscape().updateAutoScalingGroupMinSize(asg, oldAutoScalingGroupMinSize); + })); + } } // else, the replica was started explicitly, without an auto-scaling group; in any case, all replicas still // on the old release will now be stopped: } @@ -911,7 +956,7 @@ public class LandscapeServiceImpl implements LandscapeService { replicaSet.getPublicTargetGroup().removeTargets(Util.map(replicasToStopAfterUpgradingMaster, replica->replica.getHost())); for (final SailingAnalyticsProcess replica : replicasToStopAfterUpgradingMaster) { // if managed by auto-scaling group or if it's an "unmanaged" / "explicit" replica and it's the one launched in order to have at least one, stop/terminate; - final boolean managedByAutoScalingGroup = replica.getHost().isManagedByAutoScalingGroup(autoScalingGroup); + final boolean managedByAutoScalingGroup = replica.getHost().isManagedByAutoScalingGroup(affectedAutoScalingGroups); if (managedByAutoScalingGroup || (additionalReplicaStarted != null && replica.getHost().getInstanceId().equals(additionalReplicaStarted.getHost().getInstanceId()))) { logger.info("Stopping (and terminating if last application process on host) replicas on old release: "+replicasToStopAfterUpgradingMaster); @@ -926,21 +971,29 @@ public class LandscapeServiceImpl implements LandscapeService { } final Iterable> newUpgradedReplicas; if (autoScalingGroup != null && autoScalingReplicaCount > 0) { - logger.info("Now waiting until "+autoScalingReplicaCount+" auto-scaling replicas are ready before taking down dedicated temporary upgrade replicas"); + logger.info("Now waiting until " + autoScalingReplicaCount + + " auto-scaling replicas are ready before taking down dedicated temporary upgrade replicas"); // Note: the wait call returns *all* of the master's replicas, not only the auto-scaling ones; it's the predicate that // filters them down to the auto-scaling ones, counts them and compares them to the required count - newUpgradedReplicas = Wait.wait(()->master.getReplicas(Landscape.WAIT_FOR_PROCESS_TIMEOUT, new SailingAnalyticsHostSupplier(), new SailingAnalyticsProcessFactory(this::getLandscape)), - replicas->Util.size( - Util.filter(replicas, - replica->replica.getHost().isManagedByAutoScalingGroup(autoScalingGroup))) >= autoScalingReplicaCount, - /* retryOnException */ true, - Optional.of(Duration.ONE_DAY), /* duration between attempts */ Duration.ONE_SECOND.times(30), - Level.INFO, "Waiting for "+autoScalingReplicaCount+" auto-scaling replicas to become ready"); + newUpgradedReplicas = Wait.wait(() -> master.getReplicas(Landscape.WAIT_FOR_PROCESS_TIMEOUT, + new SailingAnalyticsHostSupplier(), new SailingAnalyticsProcessFactory(this::getLandscape)), + replicas -> { + int size = Util.size(Util.filter(replicas, + replica -> replica.getHost().isManagedByAutoScalingGroup(affectedAutoScalingGroups))); + logger.info("Until now there are " + size + " auto-scaling replicas healthy"); + return size >= autoScalingReplicaCount; + }, /* retryOnException */ true, Optional.of(Duration.ONE_DAY), + /* duration between attempts */ Duration.ONE_SECOND.times(30), Level.INFO, + "Waiting for " + autoScalingReplicaCount + " auto-scaling replicas to become ready"); } else { logger.info("No auto-scaling group or auto-scaling group did not have managed instances; using only the upgraded unmanaged replicas "+ newUpgradedUnmanagedReplicas); newUpgradedReplicas = newUpgradedUnmanagedReplicas; } + logger.info("Removing targets for all temporary upgrade replicas " + temporaryUpgradeReplicas); + for (Entry, Iterable>> entry : tempUpgradeReplicasByTargetGroup.entrySet()) { + entry.getKey().removeTargets(entry.getValue()); + } logger.info("Stopping/terminating all temporary upgrade replicas "+temporaryUpgradeReplicas); for (final SailingAnalyticsProcess upgradeReplica : temporaryUpgradeReplicas) { upgradeReplica.stopAndTerminateIfLast(Landscape.WAIT_FOR_PROCESS_TIMEOUT, Optional.ofNullable(optionalKeyName), privateKeyEncryptionPassphrase); @@ -1066,19 +1119,22 @@ public class LandscapeServiceImpl implements LandscapeService { */ @Override public SailingAnalyticsProcess ensureAtLeastOneReplicaExistsStopReplicatingAndRemoveMasterFromTargetGroups( - final AwsApplicationReplicaSet> replicaSet, String optionalKeyName, - byte[] privateKeyEncryptionPassphrase, - final String effectiveReplicaReplicationBearerToken) - throws Exception, MalformedURLException, IOException, TimeoutException, InterruptedException, - ExecutionException { + final AwsApplicationReplicaSet> replicaSet, + String optionalKeyName, byte[] privateKeyEncryptionPassphrase, + final String effectiveReplicaReplicationBearerToken) throws Exception, MalformedURLException, IOException, + TimeoutException, InterruptedException, ExecutionException { final Set> replicasToStopReplicating = new HashSet<>(); Util.addAll(replicaSet.getReplicas(), replicasToStopReplicating); final SailingAnalyticsProcess additionalReplicaStarted; - if (Util.isEmpty(replicaSet.getReplicas())) { - logger.info("No replica found for replica set " + replicaSet.getName() + if (Util.isEmpty(Util.filter(new HashSet<>(replicasToStopReplicating), replica -> + // exclude replicas belonging to a shard + !replica.getHost().isManagedByAutoScalingGroup(replicaSet.getShards().keySet().stream().map( + shard->shard.getAutoScalingGroup())::iterator)))) { + logger.info("No replica that doesn't belong to any shard was found for replica set " + replicaSet.getName() + "; spinning one up and waiting for it to become healthy"); - additionalReplicaStarted = launchReplicaAndWaitUntilHealthy(replicaSet, Optional.ofNullable(optionalKeyName), - privateKeyEncryptionPassphrase, effectiveReplicaReplicationBearerToken); + additionalReplicaStarted = launchReplicaAndWaitUntilHealthy(replicaSet, + Optional.ofNullable(optionalKeyName), privateKeyEncryptionPassphrase, + effectiveReplicaReplicationBearerToken); replicasToStopReplicating.add(additionalReplicaStarted); } else { additionalReplicaStarted = null; @@ -1205,12 +1261,12 @@ public class LandscapeServiceImpl implements LandscapeService { } /** - * Returns one replica process that is healthy, or {@code null} if no such process was found + * Returns one replica process managed by {@code autoScalingGroup} that is healthy, or {@code null} if no such process was found */ private SailingAnalyticsProcess hasHealthyAutoScalingReplica(SailingAnalyticsProcess master, AwsAutoScalingGroup autoScalingGroup) throws Exception { final HostSupplier> hostSupplier = new SailingAnalyticsHostSupplier<>(); for (final SailingAnalyticsProcess replica : master.getReplicas(Landscape.WAIT_FOR_HOST_TIMEOUT, hostSupplier, processFactoryFromHostAndServerDirectory)) { - if (replica.getHost().isManagedByAutoScalingGroup(autoScalingGroup) && replica.isReady(Landscape.WAIT_FOR_HOST_TIMEOUT)) { + if (replica.getHost().isManagedByAutoScalingGroup(Collections.singleton(autoScalingGroup)) && replica.isReady(Landscape.WAIT_FOR_HOST_TIMEOUT)) { return replica; } } @@ -1245,8 +1301,8 @@ public class LandscapeServiceImpl implements LandscapeService { if (replicaSet.getAutoScalingGroup() != null) { final AmazonMachineImage ami = optionalAmi.orElseGet( ()->getLandscape().getLatestImageWithType(region, SharedLandscapeConstants.IMAGE_TYPE_TAG_VALUE_SAILING)); - logger.info("Upgrading AMI in auto-scaling group "+replicaSet.getAutoScalingGroup().getName()+" of replica set "+replicaSet.getName()+" to "+ami.getId()); - getLandscape().updateImageInAutoScalingGroup(region, replicaSet.getAutoScalingGroup(), replicaSet.getName(), ami); + logger.info("Upgrading AMI in auto-scaling groups "+Util.join(", ", replicaSet.getAllAutoScalingGroups())+" of replica set "+replicaSet.getName()+" to "+ami.getId()); + getLandscape().updateImageInAutoScalingGroups(region, replicaSet.getAllAutoScalingGroups(), replicaSet.getName(), ami); result.add(getLandscape().getApplicationReplicaSet(region, replicaSet.getServerName(), replicaSet.getMaster(), replicaSet.getReplicas())); } else { logger.info("No auto-scaling group found for replica set "+replicaSet.getName()+" to update AMI in"); @@ -1301,7 +1357,7 @@ public class LandscapeServiceImpl implements LandscapeService { final AwsAutoScalingGroup autoScalingGroup = replicaSet.getAutoScalingGroup(); final Set> nonAutoScalingReplica = new HashSet<>(); for (final SailingAnalyticsProcess replica : replicaSet.getReplicas()) { - if (autoScalingGroup == null || !replica.getHost().isManagedByAutoScalingGroup(autoScalingGroup)) { + if (autoScalingGroup == null || !replica.getHost().isManagedByAutoScalingGroup(Collections.singleton(autoScalingGroup))) { logger.info("Found replica "+replica+" in replica set "+replicaSet.getName()+ " which is not managed by auto-scaling group"); nonAutoScalingReplica.add(replica); @@ -1411,29 +1467,31 @@ public class LandscapeServiceImpl implements LandscapeService { final AwsApplicationReplicaSet> replicaSet, InstanceType instanceType) throws Exception { final AwsApplicationReplicaSet> result; - final AwsAutoScalingGroup autoScalingGroup = replicaSet.getAutoScalingGroup(); - if (autoScalingGroup != null) { + final Iterable autoScalingGroups = replicaSet.getAllAutoScalingGroups(); + if (!Util.isEmpty(autoScalingGroups)) { + // Don't trust the replicas passed in by the client; it may be stale. Instead, obtain a new set of replicas from the master: final Iterable> oldReplicas = replicaSet.getMaster().getReplicas( Landscape.WAIT_FOR_PROCESS_TIMEOUT, new SailingAnalyticsHostSupplier(), processFactoryFromHostAndServerDirectory); - getLandscape().updateInstanceTypeInAutoScalingGroup(replicaSet.getMaster().getHost().getRegion(), autoScalingGroup, replicaSet.getName(), instanceType); - final int oldMinSize = autoScalingGroup.getAutoScalingGroup().minSize(); - final int newMinSize = autoScalingGroup.getAutoScalingGroup().desiredCapacity() + 1; - getLandscape().updateAutoScalingGroupMinSize(autoScalingGroup, newMinSize); - // Don't trust the replicas passed in by the client; it may be stale. Instead, obtain a new set of replicas from the master: Iterable> newSetOfAllReplicas = oldReplicas; final Set> terminatedReplicas = new HashSet<>(); - for (final SailingAnalyticsProcess replica : oldReplicas) { - final SailingAnalyticsHost replicaHost = replica.getHost(); - if (replicaHost.isManagedByAutoScalingGroup(autoScalingGroup)) { - logger.info("Replica "+replica+" is managed by auto-scaling group "+ - autoScalingGroup.getName()+" "); - newSetOfAllReplicas = waitUntilAtLeastSoManyAutoScalingReplicasAreReady(replicaSet, newMinSize); - getLandscape().terminate(replicaHost); - terminatedReplicas.add(replica); + getLandscape().updateInstanceTypeInAutoScalingGroup(replicaSet.getMaster().getHost().getRegion(), autoScalingGroups, replicaSet.getName(), instanceType); + for (final AwsAutoScalingGroup autoScalingGroup : autoScalingGroups) { + logger.info("Rolling upgrade of instances for auto-scaling group "+autoScalingGroup.getName()+" to new instance type"+instanceType); + final int oldMinSize = autoScalingGroup.getAutoScalingGroup().minSize(); + final int newMinSize = autoScalingGroup.getAutoScalingGroup().desiredCapacity() + 1; + getLandscape().updateAutoScalingGroupMinSize(autoScalingGroup, newMinSize); + for (final SailingAnalyticsProcess replica : oldReplicas) { + final SailingAnalyticsHost replicaHost = replica.getHost(); + if (replicaHost.isManagedByAutoScalingGroup(Collections.singleton(autoScalingGroup))) { + logger.info("Replica "+replica+" is managed by auto-scaling group "+autoScalingGroup.getName()); + newSetOfAllReplicas = waitUntilAtLeastSoManyAutoScalingReplicasAreReady(replicaSet, autoScalingGroup, newMinSize); + getLandscape().terminate(replicaHost); + terminatedReplicas.add(replica); + } } + getLandscape().updateAutoScalingGroupMinSize(autoScalingGroup, oldMinSize); } - getLandscape().updateAutoScalingGroupMinSize(autoScalingGroup, oldMinSize); result = getLandscape().getApplicationReplicaSet(replicaSet.getMaster().getHost().getRegion(), replicaSet.getServerName(), replicaSet.getMaster(), // remove terminated replicas: @@ -1448,19 +1506,19 @@ public class LandscapeServiceImpl implements LandscapeService { private Iterable> waitUntilAtLeastSoManyAutoScalingReplicasAreReady( final AwsApplicationReplicaSet> replicaSet, - final int newMinSize) throws Exception { + AwsAutoScalingGroup autoScalingGroup, final int newMinSize) throws Exception { final SailingAnalyticsProcess master = replicaSet.getMaster(); - final AwsAutoScalingGroup autoScalingGroup = replicaSet.getAutoScalingGroup(); assert autoScalingGroup != null; final Set> replicas = new HashSet<>(); if (Wait.wait(()->{ + int readyAutoScalingReplicas = 0; + try { replicas.clear(); Util.addAll(master.getReplicas( Landscape.WAIT_FOR_PROCESS_TIMEOUT, new SailingAnalyticsHostSupplier(), processFactoryFromHostAndServerDirectory), replicas); - int readyAutoScalingReplicas = 0; for (final SailingAnalyticsProcess replica : replicas) { - if (replica.getHost().isManagedByAutoScalingGroup(autoScalingGroup)) { + if (replica.getHost().isManagedByAutoScalingGroup(Collections.singleton(autoScalingGroup))) { if (replica.waitUntilReady(Landscape.WAIT_FOR_PROCESS_TIMEOUT)) { readyAutoScalingReplicas++; logger.info("Replica "+replica+" is ready; found "+readyAutoScalingReplicas+"/"+newMinSize+" so far"); @@ -1472,8 +1530,11 @@ public class LandscapeServiceImpl implements LandscapeService { } } } - return readyAutoScalingReplicas >= newMinSize; - }, Landscape.WAIT_FOR_HOST_TIMEOUT, /* duration between attempts */ Duration.ONE_SECOND.times(30), + } catch (TimeoutException timeoutException) { + logger.info("Timeout looking for replicas: "+timeoutException); + } + return readyAutoScalingReplicas >= newMinSize; + }, Landscape.WAIT_FOR_HOST_TIMEOUT, /* duration between attempts */ Duration.ONE_SECOND.times(30), Level.INFO, "Waiting until at least "+newMinSize+" auto-scaling replicas for replica set "+replicaSet.getName()+" are ready")) { return replicas; } else { @@ -1505,4 +1566,116 @@ public class LandscapeServiceImpl implements LandscapeService { } return result; } + + @Override + public SailingServer getSailingServer(String hostname, String username, String password, Optional port) + throws MalformedURLException { + final SailingServerFactory fac = sailingServerFactoryTracker.getService(); + return fac.getSailingServer(RemoteServerUtil.getBaseServerUrl(hostname, + port.isPresent() ? port.get() : /* defaults to HTTPS */ 443), username, password); + } + + @Override + public SailingServer getSailingServer(String hostname, String bearerToken, Optional port) + throws MalformedURLException { + final SailingServerFactory fac = sailingServerFactoryTracker.getService(); + return fac.getSailingServer(RemoteServerUtil.getBaseServerUrl(hostname, + port.orElse(443 /* defaults to HTTPS */)), bearerToken); + } + + private >, String, SailingAnalyticsMetrics, SailingAnalyticsProcess>> com.sap.sse.landscape.aws.orchestration.CreateShard.Builder>, String, SailingAnalyticsMetrics, SailingAnalyticsProcess> createShardBuilder() { + return CreateShard., BuilderT, String> builder(); + } + + private >, String, SailingAnalyticsMetrics, SailingAnalyticsProcess>> com.sap.sse.landscape.aws.orchestration.ShardProcedure.Builder>, String, SailingAnalyticsMetrics, SailingAnalyticsProcess> appendShardingKeyToShardBuilder() { + return AddShardingKeyToShard + ., BuilderT, String> builder(); + } + + private >, String, SailingAnalyticsMetrics, SailingAnalyticsProcess>> com.sap.sse.landscape.aws.orchestration.ShardProcedure.Builder>, String, SailingAnalyticsMetrics, SailingAnalyticsProcess> removeShardingKeyFromShardBuilder() { + return RemoveShardingKeyFromShard + ., BuilderT, String> builder(); + } + + @Override + public void removeShardingKeysFromShard(Iterable selectedleaderboards, + AwsApplicationReplicaSet> applicationReplicaSet, + byte[] passphraseForPrivateKeyDecription, AwsRegion region, String shardName, String bearerToken) + throws Exception { + final SailingServer server = getSailingServer(applicationReplicaSet.getHostname(), bearerToken, + /* HTTPS port */ Optional.of(443)); + Set shardingKeys = new HashSet<>(); + for (String leaderboardName : selectedleaderboards) { + shardingKeys.add(server.getLeaderboardShardingKey(leaderboardName)); + } + removeShardingKeyFromShardBuilder() + .setLandscape(getLandscape()) + .setRegion(region) + .setPathPrefixForShardingKey(RemoteServiceMappingConstants.pathPrefixForShardingKey) + .setShardingKeys(shardingKeys) + .setReplicaset(applicationReplicaSet) + .setShardName(shardName) + .setPassphrase(passphraseForPrivateKeyDecription) + .build() + .run(); + } + + @Override + public void appendShardingKeysToShard(Iterable selectedLeaderboards, + AwsApplicationReplicaSet> applicationReplicaSet, + byte[] passphraseForPrivateKeyDecription, AwsRegion region, String shardName, String bearerToken) + throws Exception { + final SailingServer server = getSailingServer(applicationReplicaSet.getHostname(), bearerToken, + /* HTTPS port */ Optional.of(443)); + final Set shardingkeys = new HashSet(); + for (String s : selectedLeaderboards) { + shardingkeys.add(server.getLeaderboardShardingKey(s)); + } + appendShardingKeyToShardBuilder() + .setLandscape(getLandscape()) + .setRegion(region) + .setPathPrefixForShardingKey(RemoteServiceMappingConstants.pathPrefixForShardingKey) + .setShardingKeys(shardingkeys) + .setReplicaset(applicationReplicaSet) + .setShardName(shardName) + .setPassphrase(passphraseForPrivateKeyDecription) + .build() + .run(); + } + + @Override + public void removeShard( + AwsApplicationReplicaSet> applicationReplicaSet, + String shardTargetGroupArn) throws Exception { + for (Entry, Iterable> entry : applicationReplicaSet.getShards().entrySet()) { + if (shardTargetGroupArn.equals(entry.getKey().getTargetGroup().getTargetGroupArn())) { + applicationReplicaSet.removeShard(entry.getKey(), getLandscape()); + return; + } + } + } + + @Override + public void addShard(Iterable selectedLeaderboardNames, + AwsApplicationReplicaSet> applicationReplicaSet, + AwsRegion region, String bearerToken, byte[] passphraseForPrivateKeyDecription, String shardName) + throws Exception { + final SailingServer server = getSailingServer(applicationReplicaSet.getHostname(), bearerToken, + Optional.of(443)); + final Set shardingkeys = new HashSet(); + for (final String s : selectedLeaderboardNames) { + shardingkeys.add(server.getLeaderboardShardingKey(s)); + } + createShardBuilder() + .setLandscape(getLandscape()) + .setTargetGroupNamePrefix(LandscapeService.SAILING_TARGET_GROUP_NAME_PREFIX) + .setShardingKeys(shardingkeys) + .setReplicaset(applicationReplicaSet) + .setRegion(region) + .setPathPrefixForShardingKey(RemoteServiceMappingConstants.pathPrefixForShardingKey) + .setShardName(shardName) + .setPassphrase(passphraseForPrivateKeyDecription) + .build() + .run(); + } } diff --git a/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/CreateLaunchConfigurationAndAutoScalingGroup.java b/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/CreateLaunchConfigurationAndAutoScalingGroup.java index 4da913fef59..cbafdffe5cd 100755 --- a/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/CreateLaunchConfigurationAndAutoScalingGroup.java +++ b/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/CreateLaunchConfigurationAndAutoScalingGroup.java @@ -8,6 +8,7 @@ import com.sap.sse.landscape.application.ApplicationProcessMetrics; import com.sap.sse.landscape.application.ApplicationReplicaSet; import com.sap.sse.landscape.aws.AmazonMachineImage; import com.sap.sse.landscape.aws.AwsApplicationProcess; +import com.sap.sse.landscape.aws.AwsAutoScalingGroup; import com.sap.sse.landscape.aws.AwsLandscape; import com.sap.sse.landscape.aws.Tags; import com.sap.sse.landscape.aws.TargetGroup; @@ -37,7 +38,6 @@ extends AbstractProcedureImpl implements Procedure { private static final int DEFAULT_MIN_REPLICAS = 1; private static final int DEFAULT_MAX_REPLICAS = 30; - private static final int DEFAULT_MAX_REQUESTS_PER_TARGET = 15000; /** * @@ -66,8 +66,8 @@ implements Procedure { BuilderT setMaxReplicas(int maxReplicas); /** - * Defines the scaling threshold based on the number of requests per target per minute. Defaults to 30,000 (see - * {@link CreateLaunchConfigurationAndAutoScalingGroup#DEFAULT_MAX_REQUESTS_PER_TARGET}). + * Defines the scaling threshold based on the number of requests per target per minute. Defaults to 15,000 (see + * {@link AwsAutoScalingGroup#DEFAULT_MAX_REQUESTS_PER_TARGET}). */ BuilderT setMaxRequestsPerTarget(int maxRequestsPerTarget); } @@ -85,7 +85,7 @@ implements Procedure { private Optional tags; private int minReplicas = DEFAULT_MIN_REPLICAS; private int maxReplicas = DEFAULT_MAX_REPLICAS; - private int maxRequestsPerTarget = DEFAULT_MAX_REQUESTS_PER_TARGET; + private int maxRequestsPerTarget = AwsAutoScalingGroup.DEFAULT_MAX_REQUESTS_PER_TARGET; public BuilderImpl(AwsLandscape landscape, Region region, String replicaSetName, TargetGroup targetGroup) { diff --git a/java/com.sap.sailing.server.gateway.interfaces/src/com/sap/sailing/server/gateway/interfaces/SailingServer.java b/java/com.sap.sailing.server.gateway.interfaces/src/com/sap/sailing/server/gateway/interfaces/SailingServer.java index bf9117974d6..f42046c03be 100755 --- a/java/com.sap.sailing.server.gateway.interfaces/src/com/sap/sailing/server/gateway/interfaces/SailingServer.java +++ b/java/com.sap.sailing.server.gateway.interfaces/src/com/sap/sailing/server/gateway/interfaces/SailingServer.java @@ -39,6 +39,12 @@ public interface SailingServer extends SecuredServer { URL getBaseUrl(); Iterable getLeaderboardGroupIds() throws Exception; + + Iterable getLeaderboardNames() throws Exception; + + String getLeaderboardShardingKey(String leaderboardName) throws Exception; + + String getLeaderboardFromShardingKey(String shardingKey) throws Exception; Iterable getEventIds() throws Exception; diff --git a/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/impl/SailingServerImpl.java b/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/impl/SailingServerImpl.java index 40425f37f3a..7b1274edc6e 100755 --- a/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/impl/SailingServerImpl.java +++ b/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/impl/SailingServerImpl.java @@ -4,8 +4,10 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -25,6 +27,7 @@ import org.json.simple.parser.ParseException; import com.sap.sailing.domain.base.RemoteSailingServerReference; import com.sap.sailing.domain.common.DataImportProgress; +import com.sap.sailing.domain.common.sharding.ShardingType; import com.sap.sailing.server.gateway.deserialization.impl.CompareServersResultJsonDeserializer; import com.sap.sailing.server.gateway.deserialization.impl.DataImportProgressJsonDeserializer; import com.sap.sailing.server.gateway.deserialization.impl.MasterDataImportResultJsonDeserializer; @@ -35,6 +38,7 @@ import com.sap.sailing.server.gateway.interfaces.SailingServer; import com.sap.sailing.server.gateway.jaxrs.api.CompareServersResource; import com.sap.sailing.server.gateway.jaxrs.api.EventsResource; import com.sap.sailing.server.gateway.jaxrs.api.LeaderboardGroupsResource; +import com.sap.sailing.server.gateway.jaxrs.api.LeaderboardsResource; import com.sap.sailing.server.gateway.jaxrs.api.MasterDataImportResource; import com.sap.sailing.server.gateway.jaxrs.api.RemoteServerReferenceResource; import com.sap.sailing.server.gateway.serialization.LeaderboardGroupConstants; @@ -56,11 +60,42 @@ public class SailingServerImpl extends SecuredServerImpl implements SailingServe @Override public Iterable getLeaderboardGroupIds() throws ClientProtocolException, IOException, ParseException { final URL leaderboardGroupsUrl = new URL(getBaseUrl(), GATEWAY_URL_PREFIX+LeaderboardGroupsResource.V1_LEADERBOARDGROUPS+LeaderboardGroupsResource.IDENTIFIABLE); - final HttpGet getLeaderboards = new HttpGet(leaderboardGroupsUrl.toString()); - final JSONArray jsonResponse = (JSONArray) getJsonParsedResponse(getLeaderboards).getA(); + final HttpGet getLeaderboardGroups = new HttpGet(leaderboardGroupsUrl.toString()); + final JSONArray jsonResponse = (JSONArray) getJsonParsedResponse(getLeaderboardGroups).getA(); return Util.map(jsonResponse, o->UUID.fromString(((JSONObject) o).get(LeaderboardGroupConstants.ID).toString())); } + @Override + public Iterable getLeaderboardNames() throws Exception { + final URL leaderboardsUrl = new URL(getBaseUrl(), GATEWAY_URL_PREFIX+LeaderboardsResource.V1_LEADERBOARDS); + final HttpGet getLeaderboards = new HttpGet(leaderboardsUrl.toString()); + final Pair jsonParsedResponse = getJsonParsedResponse(getLeaderboards); + final JSONArray jsonResponse = (JSONArray) jsonParsedResponse.getA(); + if (jsonResponse == null) { + throw new IllegalAccessError("Error accessing leaderboard names; response status "+jsonParsedResponse.getB()); + } + return Util.map(jsonResponse, o->o.toString()); + } + + @Override + public String getLeaderboardShardingKey(String leaderboardName) throws Exception { + // We could try to acquire this from the "leaderboards" REST API endpoint, field shardingLeaderboardName, + // but we can as well shortcut it by replicating the implementation here: + return ShardingType.LEADERBOARDNAME.encodeShardingInfo(leaderboardName); + } + + /** + * Establishes a map with all leaderboardnames and their keys and takes the value for the according shardingkey. + */ + @Override + public String getLeaderboardFromShardingKey(String shardingKey) throws Exception { + Map mapping = new HashMap<>(); + for (String leaderboard : getLeaderboardNames()) { + mapping.put(getLeaderboardFromShardingKey(leaderboard), leaderboard); + } + return mapping.get(shardingKey); + } + @Override public Iterable getEventIds() throws ClientProtocolException, IOException, ParseException { final URL eventsUrl = new URL(getBaseUrl(), GATEWAY_URL_PREFIX+EventsResource.V1_EVENTS); diff --git a/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/jaxrs/api/AbstractLeaderboardsResource.java b/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/jaxrs/api/AbstractLeaderboardsResource.java index 0a5c5a0ba9a..16b5d592131 100644 --- a/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/jaxrs/api/AbstractLeaderboardsResource.java +++ b/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/jaxrs/api/AbstractLeaderboardsResource.java @@ -138,7 +138,7 @@ public abstract class AbstractLeaderboardsResource extends AbstractSailingServer jsonLeaderboard.put("delayToLiveInMillis", leaderboard.getDelayToLiveInMillis()); jsonLeaderboard.put("resultState", resultState.name()); jsonLeaderboard.put("type", leaderboard.getLeaderboardType().name()); - jsonLeaderboard.put("shardingLeaderboardName", ShardingType.LEADERBOARDNAME.encodeIfNeeded(leaderboard.getName())); + jsonLeaderboard.put("shardingLeaderboardName", ShardingType.LEADERBOARDNAME.encodeShardingInfo(leaderboard.getName())); final ResultDiscardingRule resultDiscardingRule = leaderboard.getResultDiscardingRule(); if (resultDiscardingRule instanceof ThresholdBasedResultDiscardingRule) { final ThresholdBasedResultDiscardingRule thresholdBasedResultDiscardingRule = (ThresholdBasedResultDiscardingRule) resultDiscardingRule; diff --git a/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/jaxrs/api/LeaderboardsResource.java b/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/jaxrs/api/LeaderboardsResource.java index aade4a44576..c02b1d114b6 100755 --- a/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/jaxrs/api/LeaderboardsResource.java +++ b/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/jaxrs/api/LeaderboardsResource.java @@ -161,10 +161,11 @@ import com.sap.sse.security.shared.impl.UserGroup; import com.sap.sse.shared.json.JsonDeserializationException; import com.sap.sse.shared.util.impl.UUIDHelper; -@Path("/v1/leaderboards") +@Path(LeaderboardsResource.V1_LEADERBOARDS) public class LeaderboardsResource extends AbstractLeaderboardsResource { private static final Logger logger = Logger.getLogger(LeaderboardsResource.class.getName()); - + public static final String V1_LEADERBOARDS = "/v1/leaderboards"; + /** * When an {@link #createAutoCourse(TrackedRace, RegattaLog) automatic course inference} is requested, * the COGs of competitors are analyzed to obtain a direction and length for a start and finish line. diff --git a/java/com.sap.sailing.server.test/src/com/sap/sailing/server/util/ShardingLeaderBoardEncodingTest.java b/java/com.sap.sailing.server.test/src/com/sap/sailing/server/util/ShardingLeaderBoardEncodingTest.java index 3592153edba..7aa86e63eec 100644 --- a/java/com.sap.sailing.server.test/src/com/sap/sailing/server/util/ShardingLeaderBoardEncodingTest.java +++ b/java/com.sap.sailing.server.test/src/com/sap/sailing/server/util/ShardingLeaderBoardEncodingTest.java @@ -8,10 +8,10 @@ import com.sap.sailing.domain.common.sharding.ShardingType; public class ShardingLeaderBoardEncodingTest { @Test public void testEncoding() { - Assert.assertEquals("/leaderboard/pureascistring", ShardingType.LEADERBOARDNAME.encodeIfNeeded("pureascistring")); - Assert.assertEquals("/leaderboard/unpure_asci_string", ShardingType.LEADERBOARDNAME.encodeIfNeeded("unpure asci string")); - Assert.assertEquals("/leaderboard/c_dille", ShardingType.LEADERBOARDNAME.encodeIfNeeded("cédille")); - Assert.assertEquals("/leaderboard/Hello_World", ShardingType.LEADERBOARDNAME.encodeIfNeeded("Hello+World")); - Assert.assertEquals("/leaderboard/Hello_World_", ShardingType.LEADERBOARDNAME.encodeIfNeeded("Hello(World)")); + Assert.assertEquals("/leaderboard/pureascistring", ShardingType.LEADERBOARDNAME.encodeShardingInfo("pureascistring")); + Assert.assertEquals("/leaderboard/unpure_asci_string", ShardingType.LEADERBOARDNAME.encodeShardingInfo("unpure asci string")); + Assert.assertEquals("/leaderboard/c_dille", ShardingType.LEADERBOARDNAME.encodeShardingInfo("cédille")); + Assert.assertEquals("/leaderboard/Hello_World", ShardingType.LEADERBOARDNAME.encodeShardingInfo("Hello+World")); + Assert.assertEquals("/leaderboard/Hello_World_", ShardingType.LEADERBOARDNAME.encodeShardingInfo("Hello(World)")); } } diff --git a/java/com.sap.sailing.www/release_notes_admin.html b/java/com.sap.sailing.www/release_notes_admin.html index cc981f2d15b..feb8d02734f 100755 --- a/java/com.sap.sailing.www/release_notes_admin.html +++ b/java/com.sap.sailing.www/release_notes_admin.html @@ -23,6 +23,24 @@

Release Notes - Administration Console

+

January 2023

+
    +
  • Support for managing sharded replication in the "Landscape" panel of the "Advanced" category in the + admin console), + offered as a new icon in the "Action" column of the replica sets table: in the new dialog that pops + up you can create and remove so-called "shards" to which you can assign zero or more leaderboards + offered by that replica set. Each shard gets its own auto-scaling group and target group in addition to the regular + auto-scaling group / target group pair used for the public target group. By default a minimum of two instances + are required by each shard to ensure availability. They are created with the same launch configuration + used by the replica set's regular public target group. When a shard's instances are healthy, + rules are managed in the replica set's load balancer to route traffic regarding the leaderboards + selected for the shard to the shard's target group. This reduces traffic on the default public + target group, and the instances of the shard can focus on the sub-set of leaderboards configured + instead of having to compute all leaderboards in parallel. This helps reduce the leaderboard + re-calculation times for live events with parallel live races in several classes and tens of + competitors per leaderboard. +
    • +

December 2022

  • Added Czech language support
    • diff --git a/java/com.sap.sse.common/src/com/sap/sse/common/Util.java b/java/com.sap.sse.common/src/com/sap/sse/common/Util.java index f81f8af2b39..eb37dfe820b 100755 --- a/java/com.sap.sse.common/src/com/sap/sse/common/Util.java +++ b/java/com.sap.sse.common/src/com/sap/sse/common/Util.java @@ -785,10 +785,20 @@ public class Util { } return result; } - + + /** + * + * @param + * Type of {@code iterable} + * @param iterable + * Input Iterable + * @return + * returns List if {@code iterable} is an instance of List and if it is an instance Serializable. If not, + * an ArrayList gets constructed and filled with all items of {@code iterable} + */ public static List asList(Iterable iterable) { final List list; - if (iterable instanceof List) { + if (iterable instanceof List && iterable instanceof Serializable) { list = (List) iterable; } else { list = new ArrayList<>(); diff --git a/java/com.sap.sse.gwt/resources/com/sap/sse/gwt/client/images/shardmanagement.png b/java/com.sap.sse.gwt/resources/com/sap/sse/gwt/client/images/shardmanagement.png new file mode 100644 index 00000000000..e9d76653575 Binary files /dev/null and b/java/com.sap.sse.gwt/resources/com/sap/sse/gwt/client/images/shardmanagement.png differ diff --git a/java/com.sap.sse.gwt/src/com/sap/sse/gwt/client/IconResources.java b/java/com.sap.sse.gwt/src/com/sap/sse/gwt/client/IconResources.java index 455127c982b..894fc9ee55c 100755 --- a/java/com.sap.sse.gwt/src/com/sap/sse/gwt/client/IconResources.java +++ b/java/com.sap.sse.gwt/src/com/sap/sse/gwt/client/IconResources.java @@ -58,4 +58,7 @@ public interface IconResources extends ClientBundle { @Source("images/move.png") ImageResource moveIcon(); + + @Source("images/shardmanagement.png") + ImageResource shardManagementIcon(); } diff --git a/java/com.sap.sse.gwt/src/com/sap/sse/gwt/client/celltable/TableWrapper.java b/java/com.sap.sse.gwt/src/com/sap/sse/gwt/client/celltable/TableWrapper.java index c054b8dd652..bd985267d92 100755 --- a/java/com.sap.sse.gwt/src/com/sap/sse/gwt/client/celltable/TableWrapper.java +++ b/java/com.sap.sse.gwt/src/com/sap/sse/gwt/client/celltable/TableWrapper.java @@ -203,6 +203,9 @@ public abstract class TableWrapper, SM } } + /** + * Remove all items from this table's data model + */ public void clear() { getDataProvider().getList().clear(); } diff --git a/java/com.sap.sse.gwt/src/com/sap/sse/gwt/client/dialog/DataEntryDialog.java b/java/com.sap.sse.gwt/src/com/sap/sse/gwt/client/dialog/DataEntryDialog.java index f67413f7e81..bb28b01bf36 100755 --- a/java/com.sap.sse.gwt/src/com/sap/sse/gwt/client/dialog/DataEntryDialog.java +++ b/java/com.sap.sse.gwt/src/com/sap/sse/gwt/client/dialog/DataEntryDialog.java @@ -180,7 +180,7 @@ public abstract class DataEntryDialog { okButton.addClickHandler(new ClickHandler() { public void onClick(ClickEvent event) { // wait for any outstanding validation request and check last validation result; call OK only if the pending validation was OK - ifLastValidationRequestSuccesssful(()->{ + ifLastValidationRequestSuccessful(()->{ dataEntryDialog.hide(); if (callback != null) { callback.ok(getResult()); @@ -195,7 +195,7 @@ public abstract class DataEntryDialog { * call {@code callback}. If an action is still pending in the {@link #validationExecutor}, wait until no more * action is pending and invoke {@code callback} if the last validation state was OK. */ - protected void ifLastValidationRequestSuccesssful(Runnable callback) { + protected void ifLastValidationRequestSuccessful(Runnable callback) { validationExecutor.runAfterLastActionReturned(VALIDATION_ACTION_CATEGORY, ()->{ if (!dialogInInvalidState) { callback.run(); diff --git a/java/com.sap.sse.landscape.aws.common.test/.classpath b/java/com.sap.sse.landscape.aws.common.test/.classpath new file mode 100644 index 00000000000..3e5654f17eb --- /dev/null +++ b/java/com.sap.sse.landscape.aws.common.test/.classpath @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/java/com.sap.sse.landscape.aws.common.test/.project b/java/com.sap.sse.landscape.aws.common.test/.project new file mode 100644 index 00000000000..c95cb0828ad --- /dev/null +++ b/java/com.sap.sse.landscape.aws.common.test/.project @@ -0,0 +1,28 @@ + + + com.sap.sse.landscape.aws.common.test + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + + org.eclipse.pde.PluginNature + org.eclipse.jdt.core.javanature + + diff --git a/java/com.sap.sse.landscape.aws.common.test/.settings/org.eclipse.core.resources.prefs b/java/com.sap.sse.landscape.aws.common.test/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 00000000000..99f26c0203a --- /dev/null +++ b/java/com.sap.sse.landscape.aws.common.test/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +encoding/=UTF-8 diff --git a/java/com.sap.sse.landscape.aws.common.test/.settings/org.eclipse.jdt.core.prefs b/java/com.sap.sse.landscape.aws.common.test/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 00000000000..9f6ece88bdf --- /dev/null +++ b/java/com.sap.sse.landscape.aws.common.test/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,8 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.release=disabled +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/java/com.sap.sse.landscape.aws.common.test/META-INF/MANIFEST.MF b/java/com.sap.sse.landscape.aws.common.test/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..5d7a99da42b --- /dev/null +++ b/java/com.sap.sse.landscape.aws.common.test/META-INF/MANIFEST.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: LandscapeAWSCommonTest +Bundle-SymbolicName: com.sap.sse.landscape.aws.common.test +Bundle-Version: 1.0.0.qualifier +Bundle-Vendor: SAP +Fragment-Host: com.sap.sse.landscape.aws.common +Import-Package: org.hamcrest;version="2.2.0", + org.junit;version="4.13.2", + org.junit.function;version="4.13.2" +Automatic-Module-Name: com.sap.sse.landscape.aws.common.test +Bundle-RequiredExecutionEnvironment: JavaSE-1.8 diff --git a/java/com.sap.sse.landscape.aws.common.test/build.properties b/java/com.sap.sse.landscape.aws.common.test/build.properties new file mode 100644 index 00000000000..34d2e4d2dad --- /dev/null +++ b/java/com.sap.sse.landscape.aws.common.test/build.properties @@ -0,0 +1,4 @@ +source.. = src/ +output.. = bin/ +bin.includes = META-INF/,\ + . diff --git a/java/com.sap.sse.landscape.aws.common.test/pom.xml b/java/com.sap.sse.landscape.aws.common.test/pom.xml new file mode 100755 index 00000000000..11413b87eff --- /dev/null +++ b/java/com.sap.sse.landscape.aws.common.test/pom.xml @@ -0,0 +1,12 @@ + + + 4.0.0 + + root + com.sap.sailing + 1.0.0-SNAPSHOT + + com.sap.sse.landscape.aws.common.test + eclipse-test-plugin + diff --git a/java/com.sap.sse.landscape.aws.common.test/src/com/sap/sse/landscape/aws/common/shared/ShardTargetGroupNameTest.java b/java/com.sap.sse.landscape.aws.common.test/src/com/sap/sse/landscape/aws/common/shared/ShardTargetGroupNameTest.java new file mode 100644 index 00000000000..b2d2e613008 --- /dev/null +++ b/java/com.sap.sse.landscape.aws.common.test/src/com/sap/sse/landscape/aws/common/shared/ShardTargetGroupNameTest.java @@ -0,0 +1,58 @@ +package com.sap.sse.landscape.aws.common.shared; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +public class ShardTargetGroupNameTest { + private static final String TARGET_GROUP_NAME_PREFIX = "TG-"; + @Test + public void testCreateAndParse() { + final String replicaSetName = "abc"; + final String shardName = "def"; + final ShardTargetGroupName theShardName = ShardTargetGroupName.create(replicaSetName, shardName, TARGET_GROUP_NAME_PREFIX); + final ShardTargetGroupName parsedShardName = ShardTargetGroupName.parse(theShardName.getTargetGroupName(), null); + assertEquals(shardName, parsedShardName.getShardName()); + assertEquals(theShardName.getTargetGroupName(), parsedShardName.getTargetGroupName()); + } + + @Test + public void testCreateAndParseWithSimpleSeparatorInNames() { + final String replicaSetName = "abc"+ShardTargetGroupName.SEPARATOR+"def"; + final String shardName = "def"+ShardTargetGroupName.SEPARATOR+"ghi"; + assertThrows(IllegalArgumentException.class, ()->{ + ShardTargetGroupName.create(replicaSetName, shardName, TARGET_GROUP_NAME_PREFIX); + }); + } + + @Test + public void testCreateAndParseLongShardName() { + final String replicaSetName = "abc"; + final String shardName = "deflawencawueclkajhfalskjfhlaskjdhfllakj"; + final ShardTargetGroupName theShardName = ShardTargetGroupName.create(replicaSetName, shardName, TARGET_GROUP_NAME_PREFIX); + final ShardTargetGroupName parsedShardName = ShardTargetGroupName.parse(theShardName.getTargetGroupName(), null); + assertEquals(theShardName.getTargetGroupName(), parsedShardName.getTargetGroupName()); + final String shardNamePrefix = parsedShardName.getShardName().substring(0, + parsedShardName.getShardName().indexOf(ShardTargetGroupName.NAMESEPARATOR)); + final String shardNameSuffix = parsedShardName.getShardName() + .substring(parsedShardName.getShardName().indexOf(ShardTargetGroupName.NAMESEPARATOR) + + ShardTargetGroupName.NAMESEPARATOR.length()); + assertTrue(shardName.startsWith(shardNamePrefix)); + assertTrue(shardName.endsWith(shardNameSuffix)); + } + + @Test + public void testValidTargetGroupName() { + assertTrue(ShardTargetGroupName.isValidShardTargetGroupName("S-Humba-Trala")); + assertTrue(ShardTargetGroupName.isValidShardTargetGroupName("S-Humba-Tr--a")); + assertTrue(ShardTargetGroupName.isValidShardTargetGroupName("S-Humba-Tr--")); + assertTrue(ShardTargetGroupName.isValidShardTargetGroupName("Humba-Trala")); + assertTrue(ShardTargetGroupName.isValidShardTargetGroupName("Humba-Tr--a")); + assertTrue(ShardTargetGroupName.isValidShardTargetGroupName("Humba-Tr--")); + assertFalse(ShardTargetGroupName.isValidShardTargetGroupName("S-Humba-Tr--a-a")); + assertFalse(ShardTargetGroupName.isValidShardTargetGroupName("Humba-Tr--a-a")); + } +} diff --git a/java/com.sap.sse.landscape.aws.common/src/com/sap/sse/landscape/aws/common/shared/ShardTargetGroupName.java b/java/com.sap.sse.landscape.aws.common/src/com/sap/sse/landscape/aws/common/shared/ShardTargetGroupName.java new file mode 100644 index 00000000000..785ce6fdb5c --- /dev/null +++ b/java/com.sap.sse.landscape.aws.common/src/com/sap/sse/landscape/aws/common/shared/ShardTargetGroupName.java @@ -0,0 +1,153 @@ +package com.sap.sse.landscape.aws.common.shared; + +import com.sap.sse.common.Util; + +/** + * A wrapper class for an {@link AwsShard}'s target group name and the shard's name. {@link #parse} and create + * {@link #create} are used for encoding/decoding a shard's name by it's target group, replica set name and the target + * group tag with the key {@link #TAG_KEY} and creating a {@link ShardTargetGroupName} by a replica set and the shard + * name. Those created values should be used as names in the AWS landscape.

    + * + * Only a-z, A-Z and 0-9 are allowed in a shard name due to encoding and there are target group length restrictions. + * + * @author I569653 + * + */ +public class ShardTargetGroupName { + public final static String TAG_KEY = "shardname"; + static final String NAMESEPARATOR = "--"; + static final String SEPARATOR = "-"; + private final String shardName; + private final String targetGroupname; + + private ShardTargetGroupName(String shardName, String targetGroupname) { + this.shardName = shardName; + this.targetGroupname = targetGroupname; + } + + /** + * This method is supposed to convert a target group name into a {@link ShardTargetGroupName} via slicing the name + * into the replica set name and the shard name. And if there is a non-{@code null} {@code tagString} given, this is + * supposed to be the shard's name. So in that case it does not get parsed out of the target group name. + * + * @param shardTargetGroupName + * name read from a target group in AWS. + * @param tagString + * tag value with the key {@link TAG_KEY} + */ + public static ShardTargetGroupName parse(String shardTargetGroupName, String tagString) { + // Possible options: {SomeOptionalPrefix}{ReplicaSetName}-{Shardname} or {SomeOptionalPrefix}{ReplicaSetName}-{Shardname-Prefix}--{ShardnameSuffix} + // where the prefix/suffix must neither contain the SEPARATOR nor the NAMESEPARATOR + assert isValidShardTargetGroupName(shardTargetGroupName); + final String shardname; + if (Util.hasLength(tagString)) { + shardname = tagString; + } else { + final int lastIndexExclusiveToSearchForSeparator = + shardTargetGroupName.contains(NAMESEPARATOR) ? shardTargetGroupName.indexOf(NAMESEPARATOR) : shardTargetGroupName.length(); + final int idxHyphen1 = shardTargetGroupName.substring(0, lastIndexExclusiveToSearchForSeparator).lastIndexOf(SEPARATOR); + shardname = shardTargetGroupName.substring(idxHyphen1 + 1); + } + return new ShardTargetGroupName(shardname, shardTargetGroupName); + } + + /** + * Creates a ShardName from {@code replicaSetName} and {@code shardname}. {@code shardname} should only contain a-z, + * A-Z and numbers. This function builds a valid shard target group name from those inputs. This follows one of two + * patterns: If the {@link TargetGroup#MAX_TARGETGROUP_NAME_LENGTH} is sufficient, after the common + * {@link TargetGroup#SAILING_TARGET_GROUP_NAME_PREFIX} the replica set name and the shard name are concatenated, + * each separated by a single {@link #SEPARATOR} character. If the maximum target group name length is too short for + * this pattern, only a prefix and postfix of the shard name are used, concatenated by the {@link #NAMESEPARATOR} + * string. The prefix and postfix length are then chosen such that the maximum target group name limit is met + * exactly. If not even a single shard name character can be used without exceeding the maximum target group name + * length, an {@link IllegalArgumentException} is thrown. + * + * @param replicaSetName + * Replica set name of the shard + * @param shardName + * shard name + * @param targetGroupNamePrefix + * a prefix prepended to the {@code replicaSetName}; must not be {@code null} but may be an empty string + * @return a composite {@link ShardTargetGroupName} object that has the {@code shardName} as well as the + * {@link ShardTargetGroupName#getTargetGroupName() target group name} resulting from the construction rules + * encapsulated by this class + * @throws IllegalArgumentException + * Gets thrown if {@code shardName} is not valid or the combination is invalid. This happens if neither + * pattern can be applied. + */ + public static ShardTargetGroupName create(String replicaSetName, String shardName, String targetGroupNamePrefix) throws IllegalArgumentException { + if (!isValidTargetGroupNamePrefix(targetGroupNamePrefix)) { + throw new IllegalArgumentException("Target group name prefix "+targetGroupNamePrefix+" is not allowed"); + } + if (!shardName.matches("[a-zA-Z0-9]*")) { + throw new IllegalArgumentException("Only a-z, A-Z and 0-9 characters are allowed in shard name"); + } + if (shardName.endsWith(TargetGroupConstants.MASTER_SUFFIX) || shardName.endsWith(TargetGroupConstants.TEMP_SUFFIX)){ + throw new IllegalArgumentException(TargetGroupConstants.MASTER_SUFFIX + " and " + TargetGroupConstants.TEMP_SUFFIX + " are not allowed at the end of shard names"); + } + final String targetGroupNameBySimpleConcatenation = targetGroupNamePrefix + replicaSetName + SEPARATOR + shardName; + final String targetGroupName; + if (targetGroupNameBySimpleConcatenation.length() <= TargetGroupConstants.MAX_TARGETGROUP_NAME_LENGTH) { + targetGroupName = targetGroupNameBySimpleConcatenation; + } else { + final String targetGroupNamePrefixForElidedShardNameMiddle = targetGroupNamePrefix + replicaSetName + SEPARATOR; + if (targetGroupNamePrefixForElidedShardNameMiddle.length() + NAMESEPARATOR.length() >= TargetGroupConstants.MAX_TARGETGROUP_NAME_LENGTH) { + throw new IllegalArgumentException( + "Cannot add shard name prefix/postfix to replica set name "+replicaSetName+ + " as it would exceed the maximum target group name limit of "+TargetGroupConstants.MAX_TARGETGROUP_NAME_LENGTH+" characters"); + } + final int remainingCharactersAfterAddingNameSeparator = TargetGroupConstants.MAX_TARGETGROUP_NAME_LENGTH + - targetGroupNamePrefixForElidedShardNameMiddle.length() - NAMESEPARATOR.length(); + final int prefixLength = remainingCharactersAfterAddingNameSeparator/2 + remainingCharactersAfterAddingNameSeparator%2; // prefer prefix in odd cases + final int postfixLength = remainingCharactersAfterAddingNameSeparator/2; + targetGroupName = targetGroupNamePrefix + replicaSetName + SEPARATOR + shardName.substring(0, prefixLength) + + NAMESEPARATOR + shardName.substring(shardName.length() - postfixLength); + } + assert isValidShardTargetGroupName(targetGroupName); + return new ShardTargetGroupName(shardName, targetGroupName); + } + + public String getName() { + return shardName; + } + + public String getTargetGroupName() { + return targetGroupname; + } + + public String getShardName() { + return shardName; + } + + /** + * Checks whether {@code prefix} is a valid prefix for a target group name. For this, the prefix must not contain + * the {@link #NAMESEPARATOR} and its length must be short enough that there is at least one letter remaining for + * the replica set name and one letter for the shard name, including the {@link #SEPARATOR} used to separate the + * replica set name from the shard name. See also {@link TargetGroupConstants#MAX_TARGETGROUP_NAME_LENGTH}. + */ + public static boolean isValidTargetGroupNamePrefix(String prefix) { + return !prefix.contains(NAMESEPARATOR) + && prefix.length() + 1 + SEPARATOR.length() + 1 <= TargetGroupConstants.MAX_TARGETGROUP_NAME_LENGTH; + } + + /** + * Checks {@code name} for being a valid name for a shard's target group. The name must neither end with the + * {@link TargetGroupConstants#MASTER_SUFFIX suffix used by the "master" target group} nor with the + * {@link TargetGroupConstants#TEMP_SUFFIX suffix for temporary target groups}. Furthermore, the name is expected to + * contain at least one {@link #SEPARATOR} that separates the replica set name from the shard name; and if the shard + * name had to be shortened into a prefix / suffix separated by {@link #NAMESEPARATOR} then that prefix/suffix + * separator needs to come after the last single occurrence of the {@link #SEPARATOR} separating the shard name from + * the replica set name. + *

    + * + * No assumptions are made here about any target group name prefix which may itself contain a {@link #SEPARATOR} but + * not a {@link #NAMESEPARATOR} character. See also {@link #isValidTargetGroupNamePrefix}. + */ + public static boolean isValidShardTargetGroupName(String name) { + return name.length() <= TargetGroupConstants.MAX_TARGETGROUP_NAME_LENGTH + && !name.endsWith(TargetGroupConstants.MASTER_SUFFIX) + && !name.endsWith(TargetGroupConstants.TEMP_SUFFIX) + && name.contains(SEPARATOR) + && (!name.contains(NAMESEPARATOR) || name.indexOf(SEPARATOR, name.indexOf(NAMESEPARATOR)+NAMESEPARATOR.length()) < 0); + } +} diff --git a/java/com.sap.sse.landscape.aws.common/src/com/sap/sse/landscape/aws/common/shared/TargetGroupConstants.java b/java/com.sap.sse.landscape.aws.common/src/com/sap/sse/landscape/aws/common/shared/TargetGroupConstants.java new file mode 100644 index 00000000000..21f00e6c6b2 --- /dev/null +++ b/java/com.sap.sse.landscape.aws.common/src/com/sap/sse/landscape/aws/common/shared/TargetGroupConstants.java @@ -0,0 +1,7 @@ +package com.sap.sse.landscape.aws.common.shared; + +public interface TargetGroupConstants { + int MAX_TARGETGROUP_NAME_LENGTH = 32; + String MASTER_SUFFIX = "-m"; + String TEMP_SUFFIX = "-TMP"; +} diff --git a/java/com.sap.sse.landscape.aws.test/src/com/sap/sse/landscape/aws/ConnectivityTest.java b/java/com.sap.sse.landscape.aws.test/src/com/sap/sse/landscape/aws/ConnectivityTest.java index 9b3e6d112d3..fc71db190ed 100755 --- a/java/com.sap.sse.landscape.aws.test/src/com/sap/sse/landscape/aws/ConnectivityTest.java +++ b/java/com.sap.sse.landscape.aws.test/src/com/sap/sse/landscape/aws/ConnectivityTest.java @@ -44,6 +44,7 @@ import com.sap.sse.common.Util; import com.sap.sse.landscape.Landscape; import com.sap.sse.landscape.Release; import com.sap.sse.landscape.RotatingFileBasedLog; +import com.sap.sse.landscape.application.ApplicationProcess; import com.sap.sse.landscape.aws.impl.AwsRegion; import com.sap.sse.landscape.aws.orchestration.CreateDNSBasedLoadBalancerMapping; import com.sap.sse.landscape.impl.ReleaseRepositoryImpl; @@ -68,7 +69,7 @@ import software.amazon.awssdk.services.sts.model.Credentials; * Tests for the AWS SDK landscape wrapper in bundle {@code com.sap.sse.landscape.aws}. To run these tests * successfully it is necessary to have valid AWS credentials for region {@code EU_WEST_2} that allow the * AWS user account to create keys and launch instances, etc. These are to be provided as explained - * in the documentation of {@link AwsLandscape#obtain()}. You will need an MFA token generator, too, + * in the documentation of {@link AwsLandscape#obtain(String)}. You will need an MFA token generator, too, * such as the Google Authenticator.

    * * Run the test by providing your AWS access key ID as system property {@code com.sap.sse.landscape.aws.accesskeyid}, @@ -92,6 +93,7 @@ public class ConnectivityTest tmpLandscape = AwsLandscape.obtain( System.getProperty(AwsLandscape.ACCESS_KEY_ID_SYSTEM_PROPERTY_NAME), - System.getProperty(AwsLandscape.SECRET_ACCESS_KEY_SYSTEM_PROPERTY_NAME)); + System.getProperty(AwsLandscape.SECRET_ACCESS_KEY_SYSTEM_PROPERTY_NAME), pathPrefixForShardingKey); final Credentials credentials = tmpLandscape.getMfaSessionCredentials( System.getProperty(AwsLandscape.MFA_TOKEN_CODE_SYSTEM_PROPERTY_NAME)); logger.info("-D"+AwsLandscape.ACCESS_KEY_ID_SYSTEM_PROPERTY_NAME+"="+ credentials.accessKeyId()+" -D"+AwsLandscape.SECRET_ACCESS_KEY_SYSTEM_PROPERTY_NAME+"="+ credentials.secretAccessKey()+" -D"+AwsLandscape.SESSION_TOKEN_SYSTEM_PROPERTY_NAME+"="+ credentials.sessionToken()); - landscape = AwsLandscape.obtain(credentials.accessKeyId(), credentials.secretAccessKey(), credentials.sessionToken()); + landscape = AwsLandscape.obtain(credentials.accessKeyId(), credentials.secretAccessKey(), credentials.sessionToken(), pathPrefixForShardingKey); } region = new AwsRegion(Region.EU_WEST_2, landscape); AXELS_KEY_PASS = new String(Base64.getDecoder().decode(System.getProperty("axelskeypassphrase"))); @@ -486,7 +488,7 @@ public class ConnectivityTest targetGroup = landscape.createTargetGroup(region, targetGroupName, 80, "/gwt/status", 80, + final TargetGroup targetGroup = landscape.createTargetGroup(region, targetGroupName, 80, ApplicationProcess.HEALTH_CHECK_PATH, 80, /* loadBalancerArn */ null); try { final TargetGroup fetchedTargetGroup = landscape.getTargetGroup(region, targetGroupName, diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/ApplicationLoadBalancer.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/ApplicationLoadBalancer.java index 338992cf92b..566fbe2b5b0 100755 --- a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/ApplicationLoadBalancer.java +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/ApplicationLoadBalancer.java @@ -1,6 +1,7 @@ package com.sap.sse.landscape.aws; import java.util.Optional; +import java.util.regex.Pattern; import com.sap.sse.common.Named; import com.sap.sse.landscape.Region; @@ -31,10 +32,26 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleConditio * will only see the non-default rule set of the HTTPS listener which is used to dynamically configure the * landscape. * + * On https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-limits.html are some restrictions + * listed for the load balanancer's setup. + * * @author Axel Uhl (D043530) * */ public interface ApplicationLoadBalancer extends Named { + + static final int MAX_RULES_PER_LOADBALANCER = 100; + static final String DNS_MAPPED_ALB_NAME_PREFIX = "DNSMapped-"; + static final Pattern ALB_NAME_PATTERN = Pattern.compile(DNS_MAPPED_ALB_NAME_PREFIX+"(.*)$"); + static final int MAX_ALBS_PER_REGION = 20; + static final int MAX_CONDITIONS_PER_RULE = 5; + public static final String DEFAULT_RULE_PRIORITY = "Default"; + + /** + * The maximum {@link Rule#priority()} that can be used within a listener + */ + public static final int MAX_PRIORITY = 50000; + /** * The DNS name of this load balancer; can be used, e.g., to set a CNAME DNS record pointing * to this load balancer. @@ -89,6 +106,30 @@ public interface ApplicationLoadBalancer extends Named { */ Iterable addRulesAssigningUnusedPriorities(boolean forceContiguous, Rule... rules); + /** + * For inserting e.g. Shard at a specific priority, we must ensure that the priority is unique in the ruleset. + * This function shifts every priority starting at the highest priority one higher for making the priority at {@code index} + * free. + * @param index + * index supposed to be free + * @throws IllegalStateException + * gets thrown if shifting exceeds the limit of priorities + */ + void shiftRulesToMakeSpaceAt(int index) throws IllegalStateException; + + /** + * Returns the priority which should be used as the next sharding priority. + * @param hostname + * hostname of replica set from which the shard gets created + * @return priority of a rule with hostname as Host +1, if the rule it was found in is a redirect or the index of the Rule if it is a Forward. + * if there is no Rule with this hostname, just return the index after the last rule. This index should be used as the next shard priority. + * You may need to make space at this priority via {@link #shiftRulesToMakeSpaceAt(int)} + * @throws Exception + * if the found priority is higher than the {@code MAX_PRIORITY} + */ + + int getFirstShardingPriority(String hostname) throws IllegalStateException; + Iterable> getTargetGroups(); /** @@ -117,4 +158,8 @@ public interface ApplicationLoadBalancer extends Named { Rule getDefaultRedirectRule(String hostName, PlainRedirectDTO plainRedirectDTO); RuleCondition createHostHeaderRuleCondition(String hostname); + + Iterable getRulesForTargetGroups(Iterable> targetGroups); + + Iterable replaceTargetGroupInForwardRules(TargetGroup oldTargetGroup, TargetGroup newTargetGroup ); } diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsApplicationReplicaSet.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsApplicationReplicaSet.java index dbcf76898a7..6fe916a84f2 100755 --- a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsApplicationReplicaSet.java +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsApplicationReplicaSet.java @@ -1,18 +1,27 @@ package com.sap.sse.landscape.aws; +import java.util.HashSet; +import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutionException; import com.sap.sse.ServerInfo; import com.sap.sse.common.Duration; +import com.sap.sse.common.Util; import com.sap.sse.landscape.Process; import com.sap.sse.landscape.Region; import com.sap.sse.landscape.application.ApplicationProcess; import com.sap.sse.landscape.application.ApplicationProcessMetrics; import com.sap.sse.landscape.application.ApplicationReplicaSet; +import com.sap.sse.landscape.aws.common.shared.ShardTargetGroupName; +import com.sap.sse.landscape.aws.orchestration.AddShardingKeyToShard; +import com.sap.sse.landscape.aws.orchestration.CreateShard; +import com.sap.sse.landscape.aws.orchestration.RemoveShardingKeyFromShard; import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule; import software.amazon.awssdk.services.route53.model.RRType; +import software.amazon.awssdk.services.route53.model.ResourceRecord; import software.amazon.awssdk.services.route53.model.ResourceRecordSet; /** @@ -72,10 +81,28 @@ extends ApplicationReplicaSet { /** * The auto-scaling group is responsible for scaling the set of replicas registered with the * {@link #getPublicTargetGroup() public target group}. This is optional, so {@code null} may - * be returned. + * be returned.

    + * + * Note that in the presence of {@link #getShards() shards} those shards will each have their + * own {@link AwsShard#getAutoScalingGroup() auto-scaling group} which will share a launch + * configuration with the auto-scaling group returned by this method. */ AwsAutoScalingGroup getAutoScalingGroup() throws InterruptedException, ExecutionException; + /** + * Returns a non-{@code null} but possibly empty iterable of all auto-scaling groups for this replica set. + * These can be the default {@link #getAutoScalingGroup() auto-scaling group} for the public target group + * as well as any {@link #getShards() shards' auto-scaling groups}. + */ + default Iterable getAllAutoScalingGroups() throws InterruptedException, ExecutionException { + final Set result = new HashSet<>(); + if (getAutoScalingGroup() != null) { + result.add(getAutoScalingGroup()); + } + Util.addAll(Util.map(getShards().keySet(), shard->shard.getAutoScalingGroup()), result); + return result; + } + /** * Checks whether the {@code host} is eligible for accepting a deployment of a process that belongs to this * application replica set, either its master or a replica. In order to be eligible, the host must @@ -128,4 +155,44 @@ extends ApplicationReplicaSet { * {@link ServerInfo}. */ boolean isLocalReplicaSet(); + + /** + * Returns a {@link ShardTargetGroupName} that is created from an (user-) entered shard name ({@code shardName}). + * {@link ShardTargetGroupName} contains the target group name and the replica set name. + * + * @param shardName + * (User-) entered name for the shard. + * @param targetGroupNamePrefix + * a prefix for the target group name; must not be {@code null} but may be empty + * @return {@link ShardTargetGroupName} created from {@code shardName}. + * @throws Exception + * throws when {@code shardName} is not valid or is not parse-able to a shardName. + */ + ShardTargetGroupName getNewShardName(String shardName, String targetGroupNamePrefix) throws Exception; + + /** + * Retrieves information about sharding in this replica set, representing the situation at the point in time + * this object was created (not a live copy of the current landscape configuration). For that time point the + * map returned tells which shard handles requests for which sharding keys. All other reading traffic will + * be routed to this replica set's {@link #getPublicTargetGroup() public target group}.

    + * + * Shards can be removed using {@link #removeShard(AwsShard, AwsLandscape)}. To create and manipulate shards, use the + * {@link CreateShard}, {@link AddShardingKeyToShard}, and {@link RemoveShardingKeyFromShard} procedures. + * + * @return Keys are the {@link AwsShard shards}, values are the {@code ShardingKey}s managed by the corresponding + * key's shard. Never {@code null}, but may of course be empty. + */ + Map, Iterable> getShards(); + + /** + * Removes the {@code shard} from this replica set. This will remove the load balancer routing rules that so far + * directed traffic for the shard's keys to the shard; it will also remove the auto-scaling group for the shard's + * target group which will also terminate all instances created by that auto-scaling group so far; finally, the + * shard's target group is removed. + *

    + * + * In effect, this will make all traffic for the shard's keys default back to the {@link #getPublicTargetGroup() + * public target group}. + */ + void removeShard(AwsShard shard, AwsLandscape landscape) throws Exception; } diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsAutoScalingGroup.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsAutoScalingGroup.java index 706d9dd2680..813798b98b6 100755 --- a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsAutoScalingGroup.java +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsAutoScalingGroup.java @@ -15,6 +15,10 @@ import software.amazon.awssdk.services.autoscaling.model.LaunchConfiguration; * */ public interface AwsAutoScalingGroup extends Named { + /** + * Describes on how many requests per minute a new instance gets created. + */ + public static int DEFAULT_MAX_REQUESTS_PER_TARGET = 15000; AutoScalingGroup getAutoScalingGroup(); LaunchConfiguration getLaunchConfiguration(); diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsInstance.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsInstance.java index 99119d07109..57460a8e1e8 100755 --- a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsInstance.java +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsInstance.java @@ -35,17 +35,17 @@ public interface AwsInstance extends Host { } /** - * Finds out whether this instance is managed by an auto-scaling group.The implementation checks the + * Finds out whether this instance is managed by an auto-scaling group. The implementation checks the * {@link #AWS_AUTOSCALING_GROUP_NAME_TAG} tag's value and compares it to the {@code autoScalingGroup}'s name. */ boolean isManagedByAutoScalingGroup(); /** - * Finds out whether this instance is managed by the particular {@code autoScalingGroup} passed as parameter. The + * Finds out whether this instance is managed by any of the {@code autoScalingGroups} passed as parameter. The * implementation checks the {@link #AWS_AUTOSCALING_GROUP_NAME_TAG} tag's value and compares it to the * {@code autoScalingGroup}'s name. */ - boolean isManagedByAutoScalingGroup(AwsAutoScalingGroup autoScalingGroup); + boolean isManagedByAutoScalingGroup(Iterable autoScalingGroups); default String getId() { return getInstanceId(); diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsLandscape.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsLandscape.java index 23261f1b1b9..8d18f791d21 100755 --- a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsLandscape.java +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsLandscape.java @@ -1,6 +1,5 @@ package com.sap.sse.landscape.aws; -import java.util.Collection; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -38,6 +37,7 @@ import com.sap.sse.landscape.rabbitmq.RabbitMQEndpoint; import com.sap.sse.landscape.ssh.SSHKeyPair; import software.amazon.awssdk.services.autoscaling.model.AutoScalingGroup; +import software.amazon.awssdk.services.autoscaling.model.DeleteAutoScalingGroupResponse; import software.amazon.awssdk.services.autoscaling.model.LaunchConfiguration; import software.amazon.awssdk.services.cloudwatch.CloudWatchClient; import software.amazon.awssdk.services.ec2.Ec2Client; @@ -52,6 +52,7 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.LoadBalancer import software.amazon.awssdk.services.elasticloadbalancingv2.model.ProtocolEnum; import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule; import software.amazon.awssdk.services.elasticloadbalancingv2.model.RulePriorityPair; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.TagDescription; import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetHealth; import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetHealthDescription; import software.amazon.awssdk.services.route53.Route53Client; @@ -76,7 +77,7 @@ import software.amazon.awssdk.services.sts.model.Credentials; *

    * * Clients may also create dedicated instances of this service wrapper, using their own credentials. See - * {@link #obtain(String, String, Optional)}. + * {@link #obtain(String, String, Optional, String)}. *

    * * This object interacts with an instance of {@link AwsLandscapeState} which keeps persistent and replicable state about @@ -88,6 +89,8 @@ import software.amazon.awssdk.services.sts.model.Credentials; * @param */ public interface AwsLandscape extends Landscape { + static final long DEFAULT_DNS_TTL_SECONDS = 60l; + String ACCESS_KEY_ID_SYSTEM_PROPERTY_NAME = "com.sap.sse.landscape.aws.accesskeyid"; String SECRET_ACCESS_KEY_SYSTEM_PROPERTY_NAME = "com.sap.sse.landscape.aws.secretaccesskey"; @@ -140,8 +143,8 @@ public interface AwsLandscape extends Landscape { */ static > - AwsLandscape obtain() { - final AwsLandscape result = new AwsLandscapeImpl<>(Activator.getInstance().getLandscapeState()); + AwsLandscape obtain(String pathPrefixForShardingKey) { + final AwsLandscape result = new AwsLandscapeImpl<>(Activator.getInstance().getLandscapeState(), pathPrefixForShardingKey); return result; } @@ -152,8 +155,8 @@ public interface AwsLandscape extends Landscape { */ static > - AwsLandscape obtain(String accessKey, String secret) { - final AwsLandscape result = new AwsLandscapeImpl<>(Activator.getInstance().getLandscapeState(), accessKey, secret); + AwsLandscape obtain(String accessKey, String secret, String pathPrefixForShardingKey) { + final AwsLandscape result = new AwsLandscapeImpl<>(Activator.getInstance().getLandscapeState(), accessKey, secret, pathPrefixForShardingKey); return result; } @@ -164,8 +167,8 @@ public interface AwsLandscape extends Landscape { */ static > - AwsLandscape obtain(String accessKey, String secret, String sessionToken) { - final AwsLandscape result = new AwsLandscapeImpl<>(Activator.getInstance().getLandscapeState(), accessKey, secret, sessionToken); + AwsLandscape obtain(String accessKey, String secret, String sessionToken, String pathPrefixForShardingKey) { + final AwsLandscape result = new AwsLandscapeImpl<>(Activator.getInstance().getLandscapeState(), accessKey, secret, sessionToken, pathPrefixForShardingKey); return result; } @@ -409,6 +412,8 @@ public interface AwsLandscape extends Landscape { Iterable> getLoadBalancers(Region region); CompletableFuture, Iterable>> getTargetGroupsAsync(Region region); + + Iterable> getTargetGroups(com.sap.sse.landscape.Region region); CompletableFuture> getTargetHealthDescriptionsAsync(Region region, TargetGroup targetGroup); @@ -485,6 +490,16 @@ public interface AwsLandscape extends Landscape { */ TargetGroup createTargetGroup(Region region, String targetGroupName, int port, String healthCheckPath, int healthCheckPort, String loadBalancerArn); + /** + * Copies a target group from an existing target group. The name gets extended with {@code suffix} + * @param parent + * target group to copy from + * @param suffix + * suffix to append to the parent's name for the name of the new created target group + * @return + * newly created target group. + */ + TargetGroup copyTargetGroup(TargetGroup parent, String suffix); default TargetGroup getTargetGroup(Region region, String targetGroupName, String targetGroupArn, String loadBalancerArn, ProtocolEnum protocol, Integer port, ProtocolEnum healthCheckProtocol, @@ -509,6 +524,30 @@ public interface AwsLandscape extends Landscape { void deleteTargetGroup(TargetGroup targetGroup); Iterable getLoadBalancerListenerRules(Listener loadBalancerListener, Region region); + /** + * Modifies an existing rule that is identified by the passed {@code rule}'s ARN. Only the conditions are modified and nothing + * else gets touched. + * + * @param region + * AWS Region + * @param rule + * The Rule to modify. Only ARN and conditions are necessary. + * @return + * the modified Rule as an Iterable. + */ + Iterable modifyRuleConditions(Region region, Rule rule); + + /** + * Modifies an existing rule that is identified by the passed {@code rule}'s ARN. Only the actions are modified and nothing + * else gets touched. + * @param region + * AWS Region + * @param rule + * The Rule to modify. Only ARN and actions are necessary. + * @return + * the modified Rule as an Iterable. + */ + Iterable modifyRuleActions(Region region, Rule rule); /** * Use {@link Rule.Builder} to create {@link Rule} objects you'd like to set for the {@link Listener} passed as parameter. @@ -523,7 +562,7 @@ public interface AwsLandscape extends Landscape { void updateLoadBalancerListenerRule(Region region, Rule ruleToUpdate); - void updateLoadBalancerListenerRulePriorities(Region region, Collection newRulePriorities); + void updateLoadBalancerListenerRulePriorities(Region region, Iterable newRulePriorities); void deleteLoadBalancerListener(Region region, Listener listener); @@ -729,15 +768,77 @@ public interface AwsLandscape extends Landscape { ProcessT master, Iterable replicas) throws InterruptedException, ExecutionException, TimeoutException; CompletableFuture removeAutoScalingGroupAndLaunchConfiguration(AwsAutoScalingGroup autoScalingGroup); - + + CompletableFuture removeAutoScalingGroup(AwsAutoScalingGroup autoScalingGroup); + /** * updates minimum and desired size to {@code minSize} */ void updateAutoScalingGroupMinSize(AwsAutoScalingGroup autoScalingGroup, int minSize); - void updateReleaseInAutoScalingGroup(Region region, AwsAutoScalingGroup autoScalingGroup, String replicaSetName, Release release); + /** + * @param autoScalingGroups + * {@link AwsApplicationReplicaSet#getAllAutoScalingGroups() All} auto-scaling groups of the replica set; + * this is important because the method will eventually remove the old launch configuration used by those + * auto-scaling groups, and this will fail if there is any other auto-scaling group still using it. + */ + void updateReleaseInAutoScalingGroups(Region region, LaunchConfiguration oldLaunchConfiguration, Iterable autoScalingGroups, String replicaSetName, Release release); - void updateImageInAutoScalingGroup(Region region, AwsAutoScalingGroup autoScalingGroup, String replicaSetName, AmazonMachineImage ami); + /** + * @param autoScalingGroups + * {@link AwsApplicationReplicaSet#getAllAutoScalingGroups() All} auto-scaling groups of the replica set; + * this is important because the method will eventually remove the old launch configuration used by those + * auto-scaling groups, and this will fail if there is any other auto-scaling group still using it. + */ + void updateImageInAutoScalingGroups(Region region, Iterable autoScalingGroups, String replicaSetName, AmazonMachineImage ami); - void updateInstanceTypeInAutoScalingGroup(Region region, AwsAutoScalingGroup autoScalingGroup, String replicaSetName, InstanceType instanceType); + /** + * @param autoScalingGroups + * {@link AwsApplicationReplicaSet#getAllAutoScalingGroups() All} auto-scaling groups of the replica set; + * this is important because the method will eventually remove the old launch configuration used by those + * auto-scaling groups, and this will fail if there is any other auto-scaling group still using it. + */ + void updateInstanceTypeInAutoScalingGroup(Region region, Iterable autoScalingGroups, String replicaSetName, InstanceType instanceType); + + TargetGroup createTargetGroupWithoutLoadbalancer(Region region, String targetGroupName, int port); + + /** + * Creates a new auto-scaling group, using an existing one as a template and only deriving a new name for the auto-scaling group + * based on the {@code shareName}, configuring it to create its instances into {@code targetGroup} instead of the + * {@code autoScalingParent}'s target group, and optionally adding the {@code tags} to those copied anyhow from + * the {@code autoScalingParent}. The minimum size is copied from the {@code autoScalingParent} unless it is less than two; + * in that case, the new auto-scaling group will be configured with a minimum size of two, ensuring availability in case + * one target fails. + */ + > + void createAutoScalingGroupFromExisting(AwsAutoScalingGroup autoScalingParent, + String shardName, TargetGroup targetGroup, Optional tags); + + > + void putScalingPolicy( + int instanceWarmupTimeInSeconds, String shardname, TargetGroup targetgroup, int maxRequestPerTarget, com.sap.sse.landscape.Region region); + + Iterable getTargetGroupTags(String arn, com.sap.sse.landscape.Region region); + + /** + * + * See AWS doc for tag restrictions: + * https://docs.aws.amazon.com/elasticloadbalancing/latest/application/target-group-tags.html + * If a resource already has a tag with the same key, + * the value gets updated. + * + * @param arn + * Target group's ARN to add the tag to. + * @param key + * Key of the tag. See AWS logs for restrictions. + * @param value + * value of tag. See AWS logs for restrictions. + * @param region + * AWS Region of target group + * @return + * Returns the added Tag + */ + Tags addTargetGroupTag(String arn, String key, String value, com.sap.sse.landscape.Region region); + + String getAutoScalingGroupName(String replicaSetName); } diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsShard.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsShard.java new file mode 100644 index 00000000000..b86ab06282e --- /dev/null +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsShard.java @@ -0,0 +1,50 @@ +package com.sap.sse.landscape.aws; + +import com.sap.sse.landscape.application.Shard; +import com.sap.sse.landscape.aws.common.shared.ShardTargetGroupName; + +import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule; + + +/** + * A shard is represented by a target group that has its own auto-scaling group and which handles a number of sharding + * keys. + *

    + * + * Technically, the shard's name is represented as a tag value on the {@link #getTargetGroup() target group}, hence the + * restrictions for AWS tag values apply. The tag's key is {@link ShardTargetGroupName.TAG_KEY}. + * + *

    + * + * Routing to shards happens at the {@link #getTargetGroup() target group's} {@link TargetGroup#getLoadBalancer() load + * balancer}, or more precisely, through the load balancer's HTTPS listener's rule set. Each shard requires a number of rule pairs + * of which one rule forwards requests based on an HTTP header field requiring the request to be handled by a replica, + * and the other one based on the request using the GET request method. Another rule pair is required per three sharding keys + * to be handled, due to the restriction on the number of conditions a rule may have. + * + * @author Axel Uhl (d043530) + * + * @param + */ +public interface AwsShard extends Shard { + /** + * @return the target group receiving the traffic for this shard + */ + TargetGroup getTargetGroup(); + + /** + * The auto-scaling group managing the instances in the {@link #getTargetGroup() target group} receiving the traffic + * for this shard. + */ + AwsAutoScalingGroup getAutoScalingGroup(); + + ApplicationLoadBalancer getLoadBalancer(); + + String getReplicaSetName(); + + /** + * All ALB listener rules created to route traffic to this shard. These rules belong to a listener of + * {@link #getLoadBalancer()}. + */ + Iterable getRules(); +} diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/TargetGroup.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/TargetGroup.java index 46f0030e0ef..88ddb109804 100755 --- a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/TargetGroup.java +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/TargetGroup.java @@ -5,8 +5,10 @@ import java.util.Map; import com.sap.sse.common.Named; import com.sap.sse.landscape.Region; +import com.sap.sse.landscape.aws.common.shared.TargetGroupConstants; import software.amazon.awssdk.services.elasticloadbalancingv2.model.ProtocolEnum; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.TagDescription; import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetHealth; /** @@ -15,7 +17,7 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetHealth * @author Axel Uhl (D043530) * */ -public interface TargetGroup extends Named { +public interface TargetGroup extends Named, TargetGroupConstants { Region getRegion(); Map, TargetHealth> getRegisteredTargets(); @@ -37,6 +39,13 @@ public interface TargetGroup extends Named { */ Integer getPort(); + /** + * + * @return + * returns all tag descriptions for this target group + */ + Iterable getTagDescriptions(); + /** * @return the traffic protocol; usually either one of {@link ProtocolEnum#HTTP} or {@link ProtocolEnum#HTTPS} */ diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/ApplicationLoadBalancerImpl.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/ApplicationLoadBalancerImpl.java index 0b84d4a417e..3d5374bb89d 100644 --- a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/ApplicationLoadBalancerImpl.java +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/ApplicationLoadBalancerImpl.java @@ -1,13 +1,18 @@ package com.sap.sse.landscape.aws.impl; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Map.Entry; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.Optional; import java.util.Set; +import java.util.TreeMap; import com.sap.sse.common.Duration; import com.sap.sse.common.Util; @@ -28,15 +33,13 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.RedirectActi import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule; import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleCondition; import software.amazon.awssdk.services.elasticloadbalancingv2.model.RulePriorityPair; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupTuple; public class ApplicationLoadBalancerImpl implements ApplicationLoadBalancer { - private static final long serialVersionUID = -5297220031399131769L; + private static final Logger logger = Logger.getLogger(ApplicationLoadBalancerImpl.class.getName()); - /** - * The maximum {@link Rule#priority()} that can be used within a listener - */ - private static final int MAX_PRIORITY = 50000; + private static final long serialVersionUID = -5297220031399131769L; private final LoadBalancer loadBalancer; @@ -90,6 +93,103 @@ implements ApplicationLoadBalancer { public Iterable addRules(Rule... rulesToAdd) { return landscape.createLoadBalancerListenerRules(region, getListener(ProtocolEnum.HTTPS), rulesToAdd); } + + /** + * Returns the priority if the priority is not higher than {@code MAX_PRIORITY} + * @param priority + * requested priority + * @return + * Priority if it's valid + * @throws IllegalStateException + * If the requested priority is higher than {@code MAX_PRIORITY} + */ + private int checkNewPriority(int priority) throws IllegalStateException { + if (priority < MAX_PRIORITY) { + return priority; + } else { + throw new IllegalStateException("Priority was greater than " + MAX_PRIORITY +"!"); + } + } + + @Override + public void shiftRulesToMakeSpaceAt(int targetPrio) throws IllegalStateException { + final Iterable rules = getRules(); + final TreeMap rulesSorted = getRulesSorted(rules); + int lastPrio = targetPrio; + boolean skipNext = false; + final Collection result = new ArrayList<>(); + if (rulesSorted.get(targetPrio) != null) {// if there is a rule on prio + for (Entry entry : rulesSorted.entrySet()) { + final Integer priority = entry.getKey(); + if (priority >= targetPrio && !skipNext) { + // if prio is higher than target prio and is not supposed to be skipped + if (priority - lastPrio > 0) { + // if there is a gap between current prio and the last one. -> so this one is not supposed to be + // skipped and every rule after this + skipNext = true; + break; + } else { + lastPrio = priority + 1; + result.add(RulePriorityPair.builder().ruleArn(entry.getValue().ruleArn()) + .priority(checkNewPriority(lastPrio)).build()); + } + } + } + landscape.updateLoadBalancerListenerRulePriorities(getRegion(), result); + } + } + + @Override + public int getFirstShardingPriority(String hostname) throws IllegalStateException { + final Iterable rules = getRules(); + final TreeMap rulesSorted = getRulesSorted(rules); + final Iterator> iterSorted = rulesSorted.entrySet().iterator(); + int priorityToReturn = -1; + outher : while (iterSorted.hasNext()) { + final Rule r = iterSorted.next().getValue(); + for (RuleCondition con : r.conditions()) { + if (con.hostHeaderConfig() != null && con.hostHeaderConfig().values().contains(hostname)) { + for (Action a : r.actions()) { + if (a.forwardConfig() != null) { + priorityToReturn = Integer.parseInt(r.priority()); + break outher; + } else if (a.redirectConfig() != null) { + priorityToReturn = Integer.parseInt(r.priority()) + 1; + break outher; + } else { + continue; + } + } + } + } + if (!iterSorted.hasNext()) { + priorityToReturn = Integer.parseInt(r.priority()) + 1; + break outher; + } + } + if (priorityToReturn > MAX_PRIORITY) { + throw new IllegalStateException("Index higher than " + MAX_PRIORITY + "!"); + } + return priorityToReturn; + } + + private TreeMap getRulesSorted(Iterable rules) { + final Iterator iter = rules.iterator(); + final TreeMap rulesSorted = new TreeMap(); + // create Map with every priority + while (iter.hasNext()) { + final Rule r = iter.next(); + try { + if (!r.priority().equalsIgnoreCase(ApplicationLoadBalancer.DEFAULT_RULE_PRIORITY)) { + rulesSorted.put(Integer.valueOf(r.priority()), r); + } + } catch (Exception e) { + // Case where prio is not a number, e.g. 'Default' gets ignored. + logger.log(Level.WARNING, "Priority '" + r.priority() + "' couldn't be parsed as an Integer."); + } + } + return rulesSorted; + } @Override public Iterable addRulesAssigningUnusedPriorities(boolean forceContiguous, Rule... rules) { @@ -271,4 +371,58 @@ implements ApplicationLoadBalancer { && condition.pathPatternConfig().values().size() == 1 && condition.pathPatternConfig().values().contains("/")).findAny().isPresent(); } -} + + @Override + public Iterable getRulesForTargetGroups(Iterable> targetGroups) { + ArrayList ret = new ArrayList(); + for (Rule rule : getRules()) { + for (Action action : rule.actions()) { + if (action.type() == ActionTypeEnum.FORWARD) { + for (String arn : Util.map(action.forwardConfig().targetGroups(), s -> s.targetGroupArn())) { + for (String targetArn : Util.map(targetGroups, s -> s.getTargetGroupArn())) { + if (arn.equals(targetArn)) { + ret.add(rule); + } + } + } + } + } + } + return ret; + } + + /** + * Goes through all rules in this load balancer and replaces every target group in a forward-action that has the + * same ARN as the {@code oldTargetgroup} with the {@code newTargetgroup} + */ + @Override + public Iterable replaceTargetGroupInForwardRules(TargetGroup oldTargetGroup, + TargetGroup newTargetGroup) { + Iterable rules = getRules(); + Collection modifiedRules = new ArrayList<>(); + for (Rule r : rules) { + int i = 0; + boolean modified = false; + Action[] newActions = new Action[r.actions().size()]; + for (Action a : r.actions()) { + if (a.forwardConfig() != null && a.targetGroupArn().equals(oldTargetGroup.getTargetGroupArn())) { + newActions[i++] = createForwardToTargetGroupAction(newTargetGroup); + modified = true; + } else { + newActions[i++] = a; + } + } + if (modified) { + landscape.modifyRuleActions(region, Rule.builder().actions(newActions).ruleArn(r.ruleArn()).build()) + .forEach(t -> modifiedRules.add(t)); + } + } + return modifiedRules; + } + + private Action createForwardToTargetGroupAction(TargetGroup targetGroup) { + return Action.builder().type(ActionTypeEnum.FORWARD).forwardConfig(fc -> fc + .targetGroups(TargetGroupTuple.builder().targetGroupArn(targetGroup.getTargetGroupArn()).build())) + .build(); + } +} \ No newline at end of file diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsApplicationProcessImpl.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsApplicationProcessImpl.java index 3cb44e85c94..74c1de2c09d 100755 --- a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsApplicationProcessImpl.java +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsApplicationProcessImpl.java @@ -27,6 +27,7 @@ import com.sap.sse.landscape.aws.MongoUriParser; import com.sap.sse.landscape.aws.TargetGroup; import com.sap.sse.landscape.mongodb.Database; import com.sap.sse.replication.ReplicationStatus; +import com.sap.sse.util.IPAddressUtil; import software.amazon.awssdk.services.elasticloadbalancingv2.model.ActionTypeEnum; import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule; @@ -110,7 +111,7 @@ implements AwsApplicationProcess { private > Pair getHostAndOptionalTargetPortFromIpAddress(HostSupplier hostSupplier, final String ipAddressOrHostname) { HostT host; Integer targetPort; - final ApplicationLoadBalancer alb = landscape.getDNSMappedLoadBalancerFor(ipAddressOrHostname); + final ApplicationLoadBalancer alb = IPAddressUtil.isIPAddressLiteral(ipAddressOrHostname) ? null : landscape.getDNSMappedLoadBalancerFor(ipAddressOrHostname); if (alb != null) { logger.info("Found a hostname mapped to a load balancer; trying to find master through target group..."); host = null; diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsApplicationReplicaSetImpl.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsApplicationReplicaSetImpl.java index 6fcc00a7190..8f3f096dc6d 100755 --- a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsApplicationReplicaSetImpl.java +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsApplicationReplicaSetImpl.java @@ -1,6 +1,7 @@ package com.sap.sse.landscape.aws.impl; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Map.Entry; @@ -27,7 +28,10 @@ import com.sap.sse.landscape.aws.AwsApplicationProcess; import com.sap.sse.landscape.aws.AwsApplicationReplicaSet; import com.sap.sse.landscape.aws.AwsAutoScalingGroup; import com.sap.sse.landscape.aws.AwsLandscape; +import com.sap.sse.landscape.aws.AwsShard; import com.sap.sse.landscape.aws.TargetGroup; +import com.sap.sse.landscape.aws.common.shared.ShardTargetGroupName; +import com.sap.sse.landscape.aws.orchestration.ShardProcedure; import software.amazon.awssdk.services.autoscaling.model.AutoScalingGroup; import software.amazon.awssdk.services.autoscaling.model.LaunchConfiguration; @@ -38,6 +42,8 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.Listener; import software.amazon.awssdk.services.elasticloadbalancingv2.model.ProtocolEnum; import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule; import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleCondition; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.Tag; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.TagDescription; import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupTuple; import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetHealthDescription; import software.amazon.awssdk.services.route53.Route53AsyncClient; @@ -63,6 +69,7 @@ implements AwsApplicationReplicaSet { private static final Logger logger = Logger.getLogger(AwsApplicationReplicaSetImpl.class.getName()); private static final String ARCHIVE_SERVER_NAME = "ARCHIVE"; private static final long serialVersionUID = 6895927683667795173L; + private final Map, Iterable> shards; private final CompletableFuture autoScalingGroup; private final CompletableFuture defaultRedirectRule; private final CompletableFuture hostedZoneId; @@ -71,15 +78,17 @@ implements AwsApplicationReplicaSet { private final CompletableFuture> masterTargetGroup; private final CompletableFuture> publicTargetGroup; private final CompletableFuture resourceRecordSet; - + private final String pathPrefixForShardingKey; + public AwsApplicationReplicaSetImpl(String replicaSetAndServerName, String hostname, ProcessT master, Optional> replicas, CompletableFuture>> allLoadBalancersInRegion, CompletableFuture, Iterable>> allTargetGroupsInRegion, CompletableFuture>> allLoadBalancerRulesInRegion, CompletableFuture> allAutoScalingGroups, - CompletableFuture> allLaunchConfigurations, DNSCache dnsCache) throws InterruptedException, ExecutionException, TimeoutException { + CompletableFuture> allLaunchConfigurations, DNSCache dnsCache, String pathPrefixForShardingKey) throws InterruptedException, ExecutionException, TimeoutException { super(replicaSetAndServerName, hostname, master, replicas); + this.pathPrefixForShardingKey = pathPrefixForShardingKey; autoScalingGroup = new CompletableFuture<>(); defaultRedirectRule = new CompletableFuture<>(); hostedZoneId = new CompletableFuture<>(); @@ -88,6 +97,7 @@ implements AwsApplicationReplicaSet { masterTargetGroup = new CompletableFuture<>(); publicTargetGroup = new CompletableFuture<>(); resourceRecordSet = new CompletableFuture<>(); + shards = new HashMap<>(); try { allLoadBalancersInRegion.thenCompose(loadBalancers-> allTargetGroupsInRegion.thenCompose(targetGroupsAndTheirTargetHealthDescriptions-> @@ -106,6 +116,10 @@ implements AwsApplicationReplicaSet { throw e; } } + + public Map, Iterable> getShards() { + return shards; + } public AwsApplicationReplicaSetImpl(String replicaSetAndServerName, ProcessT master, Optional> replicas, @@ -113,10 +127,11 @@ implements AwsApplicationReplicaSet { CompletableFuture, Iterable>> allTargetGroupsInRegion, CompletableFuture>> allLoadBalancerRulesInRegion, AwsLandscape landscape, CompletableFuture> allAutoScalingGroups, - CompletableFuture> allLaunchConfigurations, DNSCache dnsCache) throws InterruptedException, ExecutionException, TimeoutException { + CompletableFuture> allLaunchConfigurations, DNSCache dnsCache, String pathPrefixForShardingKey) + throws InterruptedException, ExecutionException, TimeoutException { this(replicaSetAndServerName, /* hostname to be inferred */ null, master, replicas, allLoadBalancersInRegion, allTargetGroupsInRegion, allLoadBalancerRulesInRegion, allAutoScalingGroups, allLaunchConfigurations, - dnsCache); + dnsCache, pathPrefixForShardingKey); } /** @@ -177,9 +192,13 @@ implements AwsApplicationReplicaSet { } } if (!publicTargetGroup.isDone() - && (!Util.isEmpty(Util.filter(e.getValue(), target->Util.contains(Util.map(getReplicas(), replica->replica.getHost().getId()), target.target().id()))) - || !Util.isEmpty(Util.filter(e.getValue(), target->target.target().id().equals(getMaster().getHost().getId())))) - && hasPublicRuleForward(listenersAndTheirRules, e.getKey())) { + && (!Util.isEmpty(Util.filter(e.getValue(), + target -> Util.contains(Util.map(getReplicas(), replica -> replica.getHost().getId()), + target.target().id()))) + || !Util.isEmpty(Util.filter(e.getValue(), + target -> target.target().id().equals(getMaster().getHost().getId())))) + && hasPublicRuleForward(listenersAndTheirRules, e.getKey()) + && !hasPathCondition(listenersAndTheirRules, e.getKey())) { // not a shard target group // this replica set's master or at least one of the replicas of this replica set is registered with // the target group, and there is a rule that forwards // requests with explicit replica-markup to this target group: @@ -188,6 +207,8 @@ implements AwsApplicationReplicaSet { } } } + shards.putAll(establishShards(targetGroupsAndTheirTargetHealthDescriptions, listenersAndTheirRules, + autoScalingGroups, launchConfigurations)); if (!autoScalingGroup.isDone()) { // no auto-scaling group was found after having looked at all target groups autoScalingGroup.complete(null); } @@ -281,6 +302,67 @@ implements AwsApplicationReplicaSet { } return null; } + + private AwsAutoScalingGroup getShardAutoscalingGroup(TargetGroup targetGroup, + Iterable autoScalingGroups, Iterable launchConfigurations) { + AwsAutoScalingGroup autoscalinggroup = Util + .stream(autoScalingGroups).filter( + autoScalingGroup -> autoScalingGroup.targetGroupARNs() + .contains(targetGroup.getTargetGroupArn())) + .findFirst() + .map(asg -> new AwsAutoScalingGroupImpl(asg, + Util.filter(launchConfigurations, + lc -> Util.equalsWithNull(lc.launchConfigurationName(), asg.launchConfigurationName())) + .iterator().next(), + targetGroup.getRegion())) + .orElse(null); + return autoscalinggroup; + } + + private HashMap, Iterable> establishShards( + Map, Iterable> targetGroupsAndTheirTargetHealthDescriptions, + Map> listenersAndTheirRules, Iterable autoScalingGroups, + Iterable launchConfigurations) { + HashMap, Iterable> shardMap = new HashMap<>(); + for (final Entry, Iterable> e : targetGroupsAndTheirTargetHealthDescriptions.entrySet()) { + if ((e.getKey().getProtocol() == ProtocolEnum.HTTP || e.getKey().getProtocol() == ProtocolEnum.HTTPS) + && e.getKey().getLoadBalancerArn() != null + && e.getKey().getHealthCheckPort() == getMaster().getPort()) { + // an HTTP(S) target group that is currently active in a load balancer and forwards to this replica + // set master's port + final Set pathRules = getListenerRulesWithPathToReplica(listenersAndTheirRules, e.getKey()); + if (!pathRules.isEmpty() && ShardTargetGroupName.isValidShardTargetGroupName(e.getKey().getName())) { + // Is shard + final Set shardingKeys = getShardingKeys(listenersAndTheirRules, e.getKey()); + String tagName = null; + Iterable tagsDescs = e.getKey().getTagDescriptions(); + for (final TagDescription des : tagsDescs) { + final Iterable tag = Util.filter(des.tags(), t -> t.key().equals(ShardTargetGroupName.TAG_KEY)); + if (!Util.isEmpty(tag)) { + tagName = tag.iterator().next().value(); + break; + } + } + try { + final ShardTargetGroupName shardName = ShardTargetGroupName.parse(e.getKey().getName(), tagName); + final AwsShardImpl shard = new AwsShardImpl(getName(), + shardName.getShardName(), shardingKeys, e.getKey(), + e.getKey().getLoadBalancer(), pathRules, getShardAutoscalingGroup(e.getKey(), autoScalingGroups, launchConfigurations)); + shardMap.put(shard, shard.getKeys()); + } catch (Exception e1) { + logger.info(e1.getMessage()); + // This entry is no valid shard + } + } + } + } + return shardMap; + }; + + @Override + public ShardTargetGroupName getNewShardName(String shardName, String targetGroupNamePrefix) throws Exception{ + return ShardTargetGroupName.create(getName(), shardName, targetGroupNamePrefix); + } @Override public boolean isEligibleForDeployment(ApplicationProcessHost host, @@ -306,7 +388,7 @@ implements AwsApplicationReplicaSet { byte[] privateKeyEncryptionPassphrase) throws Exception { logger.info("Stopping all unmanaged replicas of replica set "+this); for (final ProcessT replica : getReplicas()) { - if (getAutoScalingGroup() == null || !replica.getHost().isManagedByAutoScalingGroup(getAutoScalingGroup())) { + if (getAutoScalingGroup() == null || !replica.getHost().isManagedByAutoScalingGroup(Collections.singleton(getAutoScalingGroup()))) { logger.info("Found unmanaged replica "+replica+". Removing from public target group and stopping..."); getPublicTargetGroup().removeTarget(replica.getHost()); replica.stopAndTerminateIfLast(optionalTimeout, optionalKeyName, privateKeyEncryptionPassphrase); @@ -318,6 +400,35 @@ implements AwsApplicationReplicaSet { return hasListenerRuleWithHostHeaderForward(listenersAndTheirRules, publicTargetGroupCandidate, HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA.getB()); } + /** + * Checks if any of the load balancer listeners passed has a rule that forwards to the + * {@code shardingTargetGroupCandidate} and has a {@code path-pattern} condition. + */ + private boolean hasPathCondition(Map> listenersAndTheirRules, + TargetGroup shardingTargetGroupCandidate) { + final String shardTargetGroupCandidateArn = shardingTargetGroupCandidate.getTargetGroupArn(); + for (final Entry> e : listenersAndTheirRules.entrySet()) { + if (Util.equalsWithNull(e.getKey().loadBalancerArn(), shardingTargetGroupCandidate.getLoadBalancerArn())) { + for (final Rule rule : e.getValue()) { + for (final Action action : rule.actions()) { + if (action.type() == ActionTypeEnum.FORWARD) { + for (final TargetGroupTuple targetGroupTuple : action.forwardConfig().targetGroups()) { + if (Util.equalsWithNull(targetGroupTuple.targetGroupArn(), shardTargetGroupCandidateArn)) { + for (final RuleCondition condition : rule.conditions()) { + if (Util.equalsWithNull(condition.field(), "path-pattern")) { + return true; + } + } + } + } + } + } + } + } + } + return false; + } + private boolean hasListenerRuleWithHostHeaderForward(Map> listenersAndTheirRules, TargetGroup publicTargetGroupCandidate, String hostHeaderForwardTo) { final String publicTargetGroupCandidateArn = publicTargetGroupCandidate.getTargetGroupArn(); for (final Entry> e : listenersAndTheirRules.entrySet()) { @@ -343,6 +454,84 @@ implements AwsApplicationReplicaSet { } return false; } + + /** + * Filters the {@link Rule}s that are contained in the values of {@code listenersAndTheirRules} for those + * that forward to the {@code shardTargetGroupCandidate} and whose conditions contains a {@code path-pattern} + * condition as well as a {@code http-header} condition that checks for the request to allow forwarding to + * a replica (see {@link HttpRequestHeaderConstants#HEADER_FORWARD_TO_REPLICA}). + */ + private Set getListenerRulesWithPathToReplica(Map> listenersAndTheirRules, + TargetGroup shardTargetGroupCandidate) { + final String shardTargetGroupCandidateArn = shardTargetGroupCandidate.getTargetGroupArn(); + Set res = new HashSet(); + for (final Entry> e : listenersAndTheirRules.entrySet()) { + if (Util.equalsWithNull(e.getKey().loadBalancerArn(), shardTargetGroupCandidate.getLoadBalancerArn())) { + for (final Rule rule : e.getValue()) { + for (final Action action : rule.actions()) { + if (action.type() == ActionTypeEnum.FORWARD) { + for (final TargetGroupTuple targetGroupTuple : action.forwardConfig().targetGroups()) { + if (Util.equalsWithNull(targetGroupTuple.targetGroupArn(), shardTargetGroupCandidateArn)) { + for (final RuleCondition condition : rule.conditions()) { + if (Util.equalsWithNull(condition.field(), "path-pattern")) { + for (final RuleCondition ruleCondition : rule.conditions()) { + if (Util.equalsWithNull(ruleCondition.field(), "http-header") + && Util.equalsWithNull( + ruleCondition.httpHeaderConfig().httpHeaderName(), + HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA + .getA()) + && ruleCondition.httpHeaderConfig().values().contains( + HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA + .getB())) { + res.add(rule); + } + } + } + } + } + } + } + } + } + } + } + return res; + } + + private Set getShardingKeys(Map> listenersAndTheirRules, + TargetGroup shardTargetGroupCandidate) { + final String publicTargetGroupCandidateArn = shardTargetGroupCandidate.getTargetGroupArn(); + Set shardingKeys = new HashSet<>(); + for (final Entry> e : listenersAndTheirRules.entrySet()) { + if (Util.equalsWithNull(e.getKey().loadBalancerArn(), shardTargetGroupCandidate.getLoadBalancerArn())) { + for (final Rule rule : e.getValue()) { + for (final Action action : rule.actions()) { + if (action.type() == ActionTypeEnum.FORWARD) { + for (final TargetGroupTuple targetGroupTuple : action.forwardConfig().targetGroups()) { + if (Util.equalsWithNull(targetGroupTuple.targetGroupArn(), + publicTargetGroupCandidateArn)) { + for (final RuleCondition condition : rule.conditions()) { + if (Util.equalsWithNull(condition.field(), "http-header") + && Util.equalsWithNull(condition.httpHeaderConfig().httpHeaderName(), + HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA.getA()) + && condition.httpHeaderConfig().values().contains( + HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA.getB())) { + for (final RuleCondition ruleCondition : rule.conditions()) { + if (Util.equalsWithNull(ruleCondition.field(), "path-pattern")) { + Util.addAll(Util.map(ruleCondition.values(), path->ShardProcedure.getShardingKeyFromPathCondition(path, pathPrefixForShardingKey)), shardingKeys); + } + } + } + } + } + } + } + } + } + } + } + return shardingKeys; + } private boolean hasMasterRuleForward(Map> listenersAndTheirRules, TargetGroup masterTargetGroupCandidate) { return hasListenerRuleWithHostHeaderForward(listenersAndTheirRules, masterTargetGroupCandidate, HttpRequestHeaderConstants.HEADER_FORWARD_TO_MASTER.getB()); @@ -418,4 +607,15 @@ implements AwsApplicationReplicaSet { public boolean isLocalReplicaSet() { return getName().equals(ServerInfo.getName()); } -} + + @Override + public void removeShard(AwsShard shard, AwsLandscape landscape) throws Exception { + // remove rules for targetgrouo + landscape.deleteLoadBalancerListenerRules(shard.getTargetGroup().getRegion(), + Util.toArray(getLoadBalancer().getRulesForTargetGroups(Collections.singleton(shard.getTargetGroup())), new Rule[0])); + // remove autoscaling group + landscape.removeAutoScalingGroup(shard.getAutoScalingGroup()); + // remove targetgroup + landscape.deleteTargetGroup(shard.getTargetGroup()); + } +} \ No newline at end of file diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsInstanceImpl.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsInstanceImpl.java index 5d8e5229f58..3f588892d09 100755 --- a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsInstanceImpl.java +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsInstanceImpl.java @@ -275,9 +275,9 @@ public class AwsInstanceImpl implements AwsInstance { } @Override - public boolean isManagedByAutoScalingGroup(AwsAutoScalingGroup autoScalingGroup) { - return getInstance().tags().stream().filter(tag -> autoScalingGroup != null - && tag.key().equals(AWS_AUTOSCALING_GROUP_NAME_TAG) && tag.value().equals(autoScalingGroup.getName())) + public boolean isManagedByAutoScalingGroup(Iterable autoScalingGroups) { + return getInstance().tags().stream().filter(tag -> tag.key().equals(AWS_AUTOSCALING_GROUP_NAME_TAG) + && Util.stream(autoScalingGroups).filter(autoScalingGroup -> autoScalingGroup.getName().equals(tag.value())).findAny().isPresent()) .findAny().isPresent(); } diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsLandscapeImpl.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsLandscapeImpl.java index 4c573195f78..0bae0890192 100644 --- a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsLandscapeImpl.java +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsLandscapeImpl.java @@ -95,6 +95,7 @@ import software.amazon.awssdk.services.autoscaling.AutoScalingAsyncClient; import software.amazon.awssdk.services.autoscaling.AutoScalingClient; import software.amazon.awssdk.services.autoscaling.model.AutoScalingGroup; import software.amazon.awssdk.services.autoscaling.model.CreateLaunchConfigurationRequest; +import software.amazon.awssdk.services.autoscaling.model.DeleteAutoScalingGroupResponse; import software.amazon.awssdk.services.autoscaling.model.LaunchConfiguration; import software.amazon.awssdk.services.autoscaling.model.MetricType; import software.amazon.awssdk.services.ec2.Ec2Client; @@ -149,6 +150,7 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.LoadBalancer import software.amazon.awssdk.services.elasticloadbalancingv2.model.LoadBalancerAttribute; import software.amazon.awssdk.services.elasticloadbalancingv2.model.LoadBalancerNotFoundException; import software.amazon.awssdk.services.elasticloadbalancingv2.model.LoadBalancerState; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.ModifyRuleResponse; import software.amazon.awssdk.services.elasticloadbalancingv2.model.ModifyTargetGroupAttributesRequest; import software.amazon.awssdk.services.elasticloadbalancingv2.model.ProtocolEnum; import software.amazon.awssdk.services.elasticloadbalancingv2.model.RedirectActionStatusCodeEnum; @@ -156,6 +158,7 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule; import software.amazon.awssdk.services.elasticloadbalancingv2.model.RulePriorityPair; import software.amazon.awssdk.services.elasticloadbalancingv2.model.SetRulePrioritiesRequest; import software.amazon.awssdk.services.elasticloadbalancingv2.model.SubnetMapping; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.TagDescription; import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetDescription; import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupAttribute; import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupNotFoundException; @@ -188,7 +191,7 @@ public class AwsLandscapeImpl implements AwsLandscape private static final String AUTO_SCALING_GROUP_NAME_SUFFIX = "-auto-replicas"; private static final String DEFAULT_TARGET_GROUP_PREFIX = "D"; private static final Logger logger = Logger.getLogger(AwsLandscapeImpl.class.getName()); - private static final long DEFAULT_DNS_TTL_MILLIS = 60l; + public static final long DEFAULT_DNS_TTL_SECONDS = 60l; private static final String DEFAULT_CERTIFICATE_DOMAIN = "*.sapsailing.com"; // TODO the "Java Application with Reverse Proxy" security group in eu-west-2 for experimenting; we need this security group per region private static final String DEFAULT_APPLICATION_SERVER_SECURITY_GROUP_ID_EU_WEST_1 = "sg-eaf31e85"; @@ -201,30 +204,32 @@ public class AwsLandscapeImpl implements AwsLandscape private final Optional sessionToken; private final AwsRegion globalRegion; private final AwsLandscapeState landscapeState; + private final String pathPrefixForShardingKey; - public AwsLandscapeImpl(AwsLandscapeState awsLandscapeState) { + public AwsLandscapeImpl(AwsLandscapeState awsLandscapeState, String pathPrefixForShardingKey) { this(awsLandscapeState, System.getProperty(ACCESS_KEY_ID_SYSTEM_PROPERTY_NAME), - System.getProperty(SECRET_ACCESS_KEY_SYSTEM_PROPERTY_NAME)); + System.getProperty(SECRET_ACCESS_KEY_SYSTEM_PROPERTY_NAME), pathPrefixForShardingKey); } - public AwsLandscapeImpl(AwsLandscapeState awsLandscapeState, String accessKeyId, String secretAccessKey) { - this(awsLandscapeState, accessKeyId, secretAccessKey, /* no session token */ null); + public AwsLandscapeImpl(AwsLandscapeState awsLandscapeState, String accessKeyId, String secretAccessKey, String pathPrefixForShardingKey) { + this(awsLandscapeState, accessKeyId, secretAccessKey, /* no session token */ null, pathPrefixForShardingKey); } - public AwsLandscapeImpl(AwsLandscapeState awsLandscapeState, String accessKeyId, String secretAccessKey, String sessionToken) { + public AwsLandscapeImpl(AwsLandscapeState awsLandscapeState, String accessKeyId, String secretAccessKey, String sessionToken, String pathPrefixForShardingKey) { this(accessKeyId, secretAccessKey, sessionToken, // by using MongoDBService.INSTANCE the default test configuration will be used if nothing else is configured - PersistenceFactory.INSTANCE.getDomainObjectFactory(MongoDBService.INSTANCE), PersistenceFactory.INSTANCE.getMongoObjectFactory(MongoDBService.INSTANCE), awsLandscapeState); + PersistenceFactory.INSTANCE.getDomainObjectFactory(MongoDBService.INSTANCE), PersistenceFactory.INSTANCE.getMongoObjectFactory(MongoDBService.INSTANCE), awsLandscapeState, pathPrefixForShardingKey); } public AwsLandscapeImpl(String accessKeyId, String secretAccessKey, - String sessionToken, DomainObjectFactory domainObjectFactory, MongoObjectFactory mongoObjectFactory, AwsLandscapeState landscapeState) { + String sessionToken, DomainObjectFactory domainObjectFactory, MongoObjectFactory mongoObjectFactory, AwsLandscapeState landscapeState, String pathPrefixForShardingKey) { this.accessKeyId = accessKeyId; this.secretAccessKey = secretAccessKey; this.sessionToken = Optional.ofNullable(sessionToken); this.globalRegion = new AwsRegion(Region.AWS_GLOBAL, this); this.landscapeState = landscapeState; + this.pathPrefixForShardingKey = pathPrefixForShardingKey; } private static byte[] getPrivateKeyBytes(KeyPair unencryptedKeyPair) { @@ -408,9 +413,25 @@ public class AwsLandscapeImpl implements AwsLandscape @Override public Iterable> getTargetGroupsByLoadBalancerArn(com.sap.sse.landscape.Region region, String loadBalancerArn) { return Util.map(getLoadBalancingClient(getRegion(region)).describeTargetGroupsPaginator(tg->tg.loadBalancerArn(loadBalancerArn)).targetGroups(), - tg->new AwsTargetGroupImpl<>(this, region, tg.targetGroupName(), tg.targetGroupArn(), loadBalancerArn, + tg->createTargetGroup(this, region, tg.targetGroupName(), tg.targetGroupArn(), loadBalancerArn, tg.protocol(), tg.port(), tg.healthCheckProtocol(), getHealthCheckPort(tg), tg.healthCheckPath())); } + + private TargetGroup createTargetGroup(AwsLandscape landscape, + com.sap.sse.landscape.Region region, String targetGroupName, String targetGroupArn, String loadBalancerArn, + ProtocolEnum protocol, Integer port, ProtocolEnum healthCheckProtocol, Integer healthCheckPort, + String healthCheckPath) { + return new AwsTargetGroupImpl(this, region, targetGroupName, targetGroupArn, loadBalancerArn, + protocol, port, healthCheckProtocol, healthCheckPort, healthCheckPath); + } + + @Override + public Iterable> getTargetGroups(com.sap.sse.landscape.Region region) { + return Util.map(getLoadBalancingClient(getRegion(region)).describeTargetGroupsPaginator().targetGroups(), + tg -> createTargetGroup(this, region, tg.targetGroupName(), tg.targetGroupArn(), + tg.loadBalancerArns().isEmpty() ? null : tg.loadBalancerArns().get(0), tg.protocol(), tg.port(), + tg.healthCheckProtocol(), getHealthCheckPort(tg), tg.healthCheckPath())); + } @Override public Iterable getListeners(ApplicationLoadBalancer alb) { @@ -429,6 +450,13 @@ public class AwsLandscapeImpl implements AwsLandscape public Iterable getLoadBalancerListenerRules(Listener loadBalancerListener, com.sap.sse.landscape.Region region) { return getLoadBalancingClient(getRegion(region)).describeRules(b->b.listenerArn(loadBalancerListener.listenerArn())).rules(); } + + @Override + public Iterable modifyRuleConditions(com.sap.sse.landscape.Region region, Rule rule) { + ModifyRuleResponse res = getLoadBalancingClient(getRegion(region)) + .modifyRule(t -> t.conditions(rule.conditions()).ruleArn(rule.ruleArn()).build()); + return res.rules(); + } @Override public Iterable createLoadBalancerListenerRules(com.sap.sse.landscape.Region region, @@ -461,8 +489,8 @@ public class AwsLandscapeImpl implements AwsLandscape } @Override - public void updateLoadBalancerListenerRulePriorities(com.sap.sse.landscape.Region region, Collection newRulePriorities) { - getLoadBalancingClient(getRegion(region)).setRulePriorities(SetRulePrioritiesRequest.builder().rulePriorities(newRulePriorities).build()); + public void updateLoadBalancerListenerRulePriorities(com.sap.sse.landscape.Region region, Iterable newRulePriorities) { + getLoadBalancingClient(getRegion(region)).setRulePriorities(SetRulePrioritiesRequest.builder().rulePriorities(Util.asList(newRulePriorities)).build()); } @Override @@ -622,7 +650,7 @@ public class AwsLandscapeImpl implements AwsLandscape .changeResourceRecordSets( ChangeResourceRecordSetsRequest.builder().hostedZoneId(hostedZoneId) .changeBatch(ChangeBatch.builder().changes(Change.builder().action(ChangeAction.UPSERT) - .resourceRecordSet(ResourceRecordSet.builder().name(hostname.toLowerCase()).type(type).ttl(DEFAULT_DNS_TTL_MILLIS) + .resourceRecordSet(ResourceRecordSet.builder().name(hostname.toLowerCase()).type(type).ttl(DEFAULT_DNS_TTL_SECONDS) .resourceRecords(ResourceRecord.builder().value(value).build()).build()) .build()).build()) .build()); @@ -639,7 +667,7 @@ public class AwsLandscapeImpl implements AwsLandscape return getRoute53Client().changeResourceRecordSets(ChangeResourceRecordSetsRequest.builder().hostedZoneId(hostedZoneId) .changeBatch(ChangeBatch.builder().changes(Change.builder().action(ChangeAction.DELETE) // TODO using the DEFAULT_DNS_TTL_MILLIS is a bit unclean here; if the record has been modified manually or the default has changed, removal will fail - .resourceRecordSet(ResourceRecordSet.builder().name(hostname).type(type).ttl(DEFAULT_DNS_TTL_MILLIS) + .resourceRecordSet(ResourceRecordSet.builder().name(hostname).type(type).ttl(DEFAULT_DNS_TTL_SECONDS) .resourceRecords(ResourceRecord.builder().value(value).build()).build()).build()).build()).build()). changeInfo(); } @@ -982,7 +1010,7 @@ public class AwsLandscapeImpl implements AwsLandscape final DescribeTargetGroupsResponse targetGroupResponse = loadBalancingClient.describeTargetGroups(b->b.names(targetGroupName)); final software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroup targetGroup = targetGroupResponse.targetGroups().iterator().next(); return targetGroupResponse.hasTargetGroups() - ? new AwsTargetGroupImpl<>(this, region, targetGroupName, targetGroup.targetGroupArn(), + ? createTargetGroup(this, region, targetGroupName, targetGroup.targetGroupArn(), loadBalancerArn == null ? Util.first(targetGroup.loadBalancerArns()) : loadBalancerArn, targetGroup.protocol(), targetGroup.port(), targetGroup.healthCheckProtocol(), getHealthCheckPort(targetGroup), targetGroup.healthCheckPath()) @@ -1039,7 +1067,7 @@ public class AwsLandscapeImpl implements AwsLandscape } } } - return new AwsTargetGroupImpl<>(this, region, targetGroupName, targetGroupArn, loadBalancerArn, + return createTargetGroup(this, region, targetGroupName, targetGroupArn, loadBalancerArn, targetGroup.protocol(), port, targetGroup.healthCheckProtocol(), healthCheckPort, healthCheckPath); } @@ -1291,6 +1319,20 @@ public class AwsLandscapeImpl implements AwsLandscape return tagResponse.tags().stream().map(t->t.value()).findAny(); } + public Iterable getTargetGroupTags(String arn, com.sap.sse.landscape.Region region) { + final software.amazon.awssdk.services.elasticloadbalancingv2.model.DescribeTagsResponse tagResponse = getLoadBalancingClient( + getRegion(region)).describeTags(t -> t.resourceArns(arn)); + return tagResponse.tagDescriptions(); + } + + @Override + public Tags addTargetGroupTag(String arn, String key, String value, com.sap.sse.landscape.Region region) { + Collection tags = new ArrayList<>(); + tags.add(software.amazon.awssdk.services.elasticloadbalancingv2.model.Tag.builder().key(key).value(value).build()); + getLoadBalancingClient(getRegion(region)).addTags(t -> t.resourceArns(arn).tags(tags)); + return new TagsImpl(key,value); + } + @Override public Tags getTagForMongoProcess(Tags tagsToAddTo, String replicaSetName, int port) { return tagsToAddTo.and(MONGO_REPLICA_SETS_TAG_NAME, @@ -1534,7 +1576,7 @@ public class AwsLandscapeImpl implements AwsLandscape CompletableFuture> allLaunchConfigurations, final DNSCache dnsCache) throws InterruptedException, ExecutionException, TimeoutException { final AwsApplicationReplicaSet replicaSet = new AwsApplicationReplicaSetImpl( serverName, master, Optional.ofNullable(replicas), allLoadBalancersInRegion, allTargetGroupsInRegion, - allLoadBalancerRulesInRegion, this, allAutoScalingGroups, allLaunchConfigurations, dnsCache); + allLoadBalancerRulesInRegion, this, allAutoScalingGroups, allLaunchConfigurations, dnsCache, pathPrefixForShardingKey); return replicaSet; } @@ -1726,7 +1768,7 @@ public class AwsLandscapeImpl implements AwsLandscape final Map, CompletableFuture>> futures = new HashMap<>(); for (final DescribeTargetGroupsResponse response : responses) { for (final software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroup tg : response.targetGroups()) { - final TargetGroup targetGroup = new AwsTargetGroupImpl(this, region, + final TargetGroup targetGroup = createTargetGroup(this, region, tg.targetGroupName(), tg.targetGroupArn(), Util.first(tg.loadBalancerArns()), tg.protocol(), tg.port(), tg.healthCheckProtocol(), getHealthCheckPort(tg), tg.healthCheckPath()); @@ -1756,19 +1798,20 @@ public class AwsLandscapeImpl implements AwsLandscape public Iterable getRegions() { return Util.map(Region.regions(), r->new AwsRegion(r, this)); } - + @Override - public void updateReleaseInAutoScalingGroup(com.sap.sse.landscape.Region region, AwsAutoScalingGroup autoScalingGroup, String replicaSetName, Release release) { - logger.info("Adjusting release for auto-scaling group "+autoScalingGroup.getName()+" to "+release); + public void updateReleaseInAutoScalingGroups(com.sap.sse.landscape.Region region, + LaunchConfiguration oldLaunchConfiguration, Iterable autoScalingGroups, + String replicaSetName, Release release) { + logger.info("Adjusting release for auto-scaling groups "+Util.join(", ", autoScalingGroups)+" to "+release); final String releaseName = release.getName(); final String newLaunchConfigurationName = getLaunchConfigurationName(replicaSetName, releaseName); - final LaunchConfiguration oldLaunchConfiguration = autoScalingGroup.getLaunchConfiguration(); final String oldUserData = new String(Base64.getDecoder().decode(oldLaunchConfiguration.userData().getBytes())); final String newUserData = oldUserData.replaceFirst( "(?m)^"+DefaultProcessConfigurationVariables.INSTALL_FROM_RELEASE.name()+"=(.*)$", - DefaultProcessConfigurationVariables.INSTALL_FROM_RELEASE.name()+"=\""+release.getName()+"\""); - updateLaunchConfiguration(region, autoScalingGroup, newLaunchConfigurationName, - b->b.userData(Base64.getEncoder().encodeToString(newUserData.getBytes()))); + DefaultProcessConfigurationVariables.INSTALL_FROM_RELEASE.name() + "=\"" + release.getName() + "\""); + updateLaunchConfiguration(region, oldLaunchConfiguration, autoScalingGroups, newLaunchConfigurationName, + b -> b.userData(Base64.getEncoder().encodeToString(newUserData.getBytes()))); } private CreateLaunchConfigurationRequest.Builder copyLaunchConfigurationToCreateRequestBuilder(LaunchConfiguration launchConfigurationToCopy) { @@ -1791,29 +1834,33 @@ public class AwsLandscapeImpl implements AwsLandscape } @Override - public void updateImageInAutoScalingGroup(com.sap.sse.landscape.Region region, AwsAutoScalingGroup autoScalingGroup, String replicaSetName, AmazonMachineImage ami) { - logger.info("Adjusting AMI for auto-scaling group "+autoScalingGroup.getName()+" to "+ami); + public void updateImageInAutoScalingGroups(com.sap.sse.landscape.Region region, Iterable autoScalingGroups, String replicaSetName, AmazonMachineImage ami) { + logger.info("Adjusting AMI for auto-scaling group(s) "+Util.join(", ", autoScalingGroups)+" to "+ami); final String newLaunchConfigurationName = getLaunchConfigurationName(replicaSetName, ami.getId()); - updateLaunchConfiguration(region, autoScalingGroup, newLaunchConfigurationName, b->b.imageId(ami.getId())); + updateLaunchConfiguration(region, autoScalingGroups, newLaunchConfigurationName, b->b.imageId(ami.getId())); } @Override - public void updateInstanceTypeInAutoScalingGroup(com.sap.sse.landscape.Region region, AwsAutoScalingGroup autoScalingGroup, String replicaSetName, InstanceType instanceType) { - logger.info("Adjusting instance type for auto-scaling group "+autoScalingGroup.getName()+" to "+instanceType); - final LaunchConfiguration oldLaunchConfiguration = autoScalingGroup.getLaunchConfiguration(); + public void updateInstanceTypeInAutoScalingGroup(com.sap.sse.landscape.Region region, Iterable autoScalingGroups, String replicaSetName, InstanceType instanceType) { + logger.info("Adjusting instance type for auto-scaling group(s) "+Util.join(", ", autoScalingGroups)+" to "+instanceType); + final LaunchConfiguration oldLaunchConfiguration = Util.first(autoScalingGroups).getLaunchConfiguration(); final String newLaunchConfigurationName = oldLaunchConfiguration.launchConfigurationName()+"-"+instanceType.name(); - updateLaunchConfiguration(region, autoScalingGroup, newLaunchConfigurationName, b->b.instanceType(instanceType.toString())); + updateLaunchConfiguration(region, autoScalingGroups, newLaunchConfigurationName, b->b.instanceType(instanceType.toString())); } - private void updateLaunchConfigurationForAutoScalingGroup(final AutoScalingClient autoScalingClient, - AwsAutoScalingGroup autoScalingGroup, final LaunchConfiguration oldLaunchConfiguration, + private void updateLaunchConfigurationForAutoScalingGroups(final AutoScalingClient autoScalingClient, + Iterable autoScalingGroups, final LaunchConfiguration oldLaunchConfiguration, final String newLaunchConfigurationName) { - logger.info("Telling auto-scaling group "+autoScalingGroup.getName()+" to use new launch configuration "+newLaunchConfigurationName); - autoScalingClient.updateAutoScalingGroup(b->b - .autoScalingGroupName(autoScalingGroup.getAutoScalingGroup().autoScalingGroupName()) - .launchConfigurationName(newLaunchConfigurationName)); - logger.info("Removing old launch configuration "+oldLaunchConfiguration.launchConfigurationName()); - autoScalingClient.deleteLaunchConfiguration(b->b.launchConfigurationName(oldLaunchConfiguration.launchConfigurationName())); + for (AwsAutoScalingGroup autoScalingGroup : autoScalingGroups) { + logger.info("Telling auto-scaling group " + autoScalingGroup.getName() + " to use new launch configuration " + + newLaunchConfigurationName); + autoScalingClient.updateAutoScalingGroup( + b -> b.autoScalingGroupName(autoScalingGroup.getAutoScalingGroup().autoScalingGroupName()) + .launchConfigurationName(newLaunchConfigurationName)); + } + logger.info("Removing old launch configuration " + oldLaunchConfiguration.launchConfigurationName()); + autoScalingClient.deleteLaunchConfiguration( + b -> b.launchConfigurationName(oldLaunchConfiguration.launchConfigurationName())); } /** @@ -1831,17 +1878,16 @@ public class AwsLandscapeImpl implements AwsLandscape * @see #updateLaunchConfigurationForAutoScalingGroup(AutoScalingClient, AwsAutoScalingGroup, LaunchConfiguration, * String) */ - private void updateLaunchConfiguration(com.sap.sse.landscape.Region region, AwsAutoScalingGroup autoScalingGroup, + private void updateLaunchConfiguration(com.sap.sse.landscape.Region region, LaunchConfiguration oldLaunchConfiguration, Iterable affectedAutoScalingGroups, String newLaunchConfigurationName, Consumer builderConsumer) { if (newLaunchConfigurationName == null) { - throw new NullPointerException("New launch configuration name for auto-scaling group "+autoScalingGroup.getName()+" must not be null"); + throw new NullPointerException("New launch configuration name for auto-scaling groups "+Util.join(", ", affectedAutoScalingGroups)+" must not be null"); } - logger.info("Adjusting launch configuration for auto-scaling group "+autoScalingGroup.getName()); + logger.info("Adjusting launch configuration for auto-scaling groups "+Util.join(", ", affectedAutoScalingGroups)); final AutoScalingClient autoScalingClient = getAutoScalingClient(getRegion(region)); - final LaunchConfiguration oldLaunchConfiguration = autoScalingGroup.getLaunchConfiguration(); if (newLaunchConfigurationName.equals(oldLaunchConfiguration.launchConfigurationName())) { - throw new IllegalArgumentException("New launch configuration name "+newLaunchConfigurationName+" for auto-scaling group "+ - autoScalingGroup.getName()+" equals the old one"); + throw new IllegalArgumentException("New launch configuration name "+newLaunchConfigurationName+" for auto-scaling groups "+ + Util.join(", ", affectedAutoScalingGroups)+" equals the old one"); } final CreateLaunchConfigurationRequest.Builder createLaunchConfigurationRequestBuilder = copyLaunchConfigurationToCreateRequestBuilder(oldLaunchConfiguration); builderConsumer.accept(createLaunchConfigurationRequestBuilder); @@ -1849,7 +1895,31 @@ public class AwsLandscapeImpl implements AwsLandscape final CreateLaunchConfigurationRequest createLaunchConfigurationRequest = createLaunchConfigurationRequestBuilder.build(); logger.info("Creating new launch configuration "+newLaunchConfigurationName); autoScalingClient.createLaunchConfiguration(createLaunchConfigurationRequest); - updateLaunchConfigurationForAutoScalingGroup(autoScalingClient, autoScalingGroup, oldLaunchConfiguration, newLaunchConfigurationName); + updateLaunchConfigurationForAutoScalingGroups(autoScalingClient, affectedAutoScalingGroups, oldLaunchConfiguration, newLaunchConfigurationName); + } + + private void updateLaunchConfiguration(com.sap.sse.landscape.Region region, Iterable autoScalingGroups, + String newLaunchConfigurationName, Consumer builderConsumer) { + if (Util.isEmpty(autoScalingGroups)) { + throw new IllegalArgumentException("At least one auto-scaling group must be provided for updating a launch configuration"); + } + if (newLaunchConfigurationName == null) { + throw new NullPointerException("New launch configuration name for auto-scaling group(s) "+Util.join(", ", autoScalingGroups)+" must not be null"); + } + logger.info("Adjusting launch configuration for auto-scaling group(s) "+Util.join(", ", autoScalingGroups)); + final AutoScalingClient autoScalingClient = getAutoScalingClient(getRegion(region)); + final LaunchConfiguration oldLaunchConfiguration = Util.first(autoScalingGroups).getLaunchConfiguration(); + if (newLaunchConfigurationName.equals(oldLaunchConfiguration.launchConfigurationName())) { + throw new IllegalArgumentException("New launch configuration name "+newLaunchConfigurationName+" for auto-scaling group(s) "+ + Util.join(", ", autoScalingGroups)+" equals the old one"); + } + final CreateLaunchConfigurationRequest.Builder createLaunchConfigurationRequestBuilder = copyLaunchConfigurationToCreateRequestBuilder(oldLaunchConfiguration); + builderConsumer.accept(createLaunchConfigurationRequestBuilder); + createLaunchConfigurationRequestBuilder.launchConfigurationName(newLaunchConfigurationName); + final CreateLaunchConfigurationRequest createLaunchConfigurationRequest = createLaunchConfigurationRequestBuilder.build(); + logger.info("Creating new launch configuration "+newLaunchConfigurationName); + autoScalingClient.createLaunchConfiguration(createLaunchConfigurationRequest); + updateLaunchConfigurationForAutoScalingGroups(autoScalingClient, autoScalingGroups, oldLaunchConfiguration, newLaunchConfigurationName); } @Override @@ -1892,24 +1962,14 @@ public class AwsLandscapeImpl implements AwsLandscape b.tags(awsTags); }); }); - autoScalingClient.putScalingPolicy(b->b - .autoScalingGroupName(autoScalingGroupName) - .estimatedInstanceWarmup(instanceWarmupTimeInSeconds) - .policyType("TargetTrackingScaling") - .policyName("KeepRequestsPerTargetAt"+maxRequestsPerTarget) - .targetTrackingConfiguration(t->t - .predefinedMetricSpecification(p->p - .resourceLabel("app/"+publicTargetGroup.getLoadBalancer().getName()+"/"+publicTargetGroup.getLoadBalancer().getId()+ - "/targetgroup/"+publicTargetGroup.getName()+"/"+publicTargetGroup.getId()) - .predefinedMetricType(MetricType.ALB_REQUEST_COUNT_PER_TARGET)) - .targetValue((double) maxRequestsPerTarget))); + putScalingPolicy(instanceWarmupTimeInSeconds, autoScalingGroupName, publicTargetGroup , maxRequestsPerTarget, region); } private String getLaunchConfigurationName(String replicaSetName, final String releaseName) { return replicaSetName + "-" + releaseName; } - private String getAutoScalingGroupName(String replicaSetName) { + public String getAutoScalingGroupName(String replicaSetName) { return replicaSetName+AUTO_SCALING_GROUP_NAME_SUFFIX; } @@ -1928,11 +1988,92 @@ public class AwsLandscapeImpl implements AwsLandscape public CompletableFuture removeAutoScalingGroupAndLaunchConfiguration(AwsAutoScalingGroup autoScalingGroup) { final String launchConfigurationName = autoScalingGroup.getAutoScalingGroup().launchConfigurationName(); final AutoScalingAsyncClient autoScalingAsyncClient = getAutoScalingAsyncClient(getRegion(autoScalingGroup.getRegion())); - logger.info("Removing auto-scaling group "+autoScalingGroup.getAutoScalingGroup().autoScalingGroupName()); - return autoScalingAsyncClient.deleteAutoScalingGroup(b->b.forceDelete(true).autoScalingGroupName(autoScalingGroup.getAutoScalingGroup().autoScalingGroupName())) + return removeAutoScalingGroup(autoScalingGroup) .thenAccept(response->{ logger.info("Removing launch configuration "+launchConfigurationName); autoScalingAsyncClient.deleteLaunchConfiguration(b->b.launchConfigurationName(launchConfigurationName)); }); } + + @Override + public CompletableFuture removeAutoScalingGroup(AwsAutoScalingGroup autoScalingGroup) { + final AutoScalingAsyncClient autoScalingAsyncClient = getAutoScalingAsyncClient(getRegion(autoScalingGroup.getRegion())); + logger.info("Removing auto-scaling group "+autoScalingGroup.getAutoScalingGroup().autoScalingGroupName()); + return autoScalingAsyncClient.deleteAutoScalingGroup(b->b.forceDelete(true).autoScalingGroupName(autoScalingGroup.getAutoScalingGroup().autoScalingGroupName())); + } + + @Override + public TargetGroup createTargetGroupWithoutLoadbalancer(com.sap.sse.landscape.Region region, String targetGroupName, int port) { + return createTargetGroup(region, targetGroupName, port, ApplicationProcess.HEALTH_CHECK_PATH, port, null); + } + + @Override + public > + void createAutoScalingGroupFromExisting(AwsAutoScalingGroup autoScalingParent, + String shardName, TargetGroup targetGroup, Optional tags) { + final AutoScalingClient autoScalingClient = getAutoScalingClient(getRegion(autoScalingParent.getRegion())); + final String launchConfigurationName = autoScalingParent.getAutoScalingGroup().launchConfigurationName(); + final String autoScalingGroupName = getAutoScalingGroupName(shardName); + final List availabilityZones = autoScalingParent.getAutoScalingGroup().availabilityZones(); + final int instanceWarmupTimeInSeconds = autoScalingParent.getAutoScalingGroup().defaultInstanceWarmup() != null ? autoScalingParent.getAutoScalingGroup().defaultInstanceWarmup() : 180 ; + logger.info("Creating Autoscalinggroup " + autoScalingGroupName +" for Shard "+shardName + ". Inheriting from Autoscalinggroup: " + autoScalingParent.getName()); + autoScalingClient.createAutoScalingGroup(b->{ + b + .minSize(autoScalingParent.getAutoScalingGroup().minSize() > 1 ? autoScalingParent.getAutoScalingGroup().minSize() : 2) + .maxSize(autoScalingParent.getAutoScalingGroup().maxSize()) + .healthCheckGracePeriod(instanceWarmupTimeInSeconds) + .autoScalingGroupName(autoScalingGroupName) + .availabilityZones(availabilityZones) + .targetGroupARNs(targetGroup.getTargetGroupArn()) + .launchConfigurationName(launchConfigurationName); + final List awsTags = new ArrayList<>(); + final List parentTags = autoScalingParent.getAutoScalingGroup().tags(); + for (final software.amazon.awssdk.services.autoscaling.model.TagDescription parentTag : parentTags) { + awsTags.add(software.amazon.awssdk.services.autoscaling.model.Tag.builder() + .key(parentTag.key()) + .value(parentTag.key().equals("Name") ? parentTag.value()+" ("+shardName+")" : parentTag.value()) + .propagateAtLaunch(parentTag.propagateAtLaunch()) + .build()); + } + tags.ifPresent(t->{ + for (final Entry tag : t) { + awsTags.add(software.amazon.awssdk.services.autoscaling.model.Tag.builder().key(tag.getKey()).value(tag.getValue()).build()); + } + }); + b.tags(awsTags); + }); + } + + @Override + public TargetGroup copyTargetGroup(TargetGroup parent, String suffix) { + TargetGroup child = createTargetGroupWithoutLoadbalancer(parent.getRegion(), parent.getName()+ suffix, parent.getPort()); + child.addTargets(parent.getRegisteredTargets().keySet()); + return child; + } + + @Override + public Iterable modifyRuleActions(com.sap.sse.landscape.Region region, Rule rule) { + ModifyRuleResponse res = getLoadBalancingClient(getRegion(region)) + .modifyRule(t -> t.actions(rule.actions()).ruleArn(rule.ruleArn()).build()); + return res.rules(); + } + + @Override + public > void putScalingPolicy( + int instanceWarmupTimeInSeconds, String autoScalingGroupName, TargetGroup targetgroup, + int maxRequestPerTarget, com.sap.sse.landscape.Region region) { + final AutoScalingClient autoScalingClient = getAutoScalingClient(getRegion(region)); + autoScalingClient.putScalingPolicy( + b -> b.autoScalingGroupName(autoScalingGroupName).estimatedInstanceWarmup(instanceWarmupTimeInSeconds) + .policyType("TargetTrackingScaling").policyName("KeepRequestsPerTargetAt" + maxRequestPerTarget) + .targetTrackingConfiguration(t -> t + .predefinedMetricSpecification(p -> p + .resourceLabel("app/" + targetgroup.getLoadBalancer().getName() + "/" + + targetgroup.getLoadBalancer().getId() + "/targetgroup/" + + targetgroup.getName() + "/" + targetgroup.getId()) + .predefinedMetricType(MetricType.ALB_REQUEST_COUNT_PER_TARGET)) + .targetValue((double) AwsAutoScalingGroup.DEFAULT_MAX_REQUESTS_PER_TARGET))); + } + + } diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsShardImpl.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsShardImpl.java new file mode 100644 index 00000000000..47705dcfd76 --- /dev/null +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsShardImpl.java @@ -0,0 +1,73 @@ +package com.sap.sse.landscape.aws.impl; + +import com.sap.sse.common.Util; +import com.sap.sse.landscape.aws.ApplicationLoadBalancer; +import com.sap.sse.landscape.aws.AwsAutoScalingGroup; +import com.sap.sse.landscape.aws.AwsShard; +import com.sap.sse.landscape.aws.TargetGroup; + +import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule; + +public class AwsShardImpl implements AwsShard { + + private static final long serialVersionUID = 1L; + private final Iterable keys; + private final TargetGroup targetGroup; + private final String replicaSetName; + private final String name; + private final AwsAutoScalingGroup autoScalingGroup; + private final ApplicationLoadBalancer loadBalancer; + private final Iterable rules; + + public AwsShardImpl(String replicaSetName, String shardName, Iterable keys, + TargetGroup targetgroup, ApplicationLoadBalancer loadBalancer, Iterable rules, AwsAutoScalingGroup asg) { + this.keys = keys; + this.targetGroup = targetgroup; + this.replicaSetName = replicaSetName; + this.name = shardName; + this.loadBalancer = loadBalancer; + this.autoScalingGroup = asg; + this.rules = rules; + } + + @Override + public Iterable getKeys() { + return keys; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getReplicaSetName() { + return replicaSetName; + } + + @Override + public TargetGroup getTargetGroup() { + return targetGroup; + } + + @Override + public AwsAutoScalingGroup getAutoScalingGroup() { + return autoScalingGroup; + } + + @Override + public ApplicationLoadBalancer getLoadBalancer() { + return loadBalancer; + } + + @Override + public Iterable getRules() { + return rules; + } + + @Override + public String toString() { + return "AwsShardImpl [name=" + name + ", replicaSetName=" + replicaSetName + ", keys=" + Util.joinStrings(", ", keys) + "]"; + } + +} diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsTargetGroupImpl.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsTargetGroupImpl.java index 5c4d96fa8fe..92cdbe82337 100755 --- a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsTargetGroupImpl.java +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsTargetGroupImpl.java @@ -11,6 +11,7 @@ import com.sap.sse.landscape.aws.AwsLandscape; import com.sap.sse.landscape.aws.TargetGroup; import software.amazon.awssdk.services.elasticloadbalancingv2.model.ProtocolEnum; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.TagDescription; import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetHealth; public class AwsTargetGroupImpl @@ -44,6 +45,10 @@ extends NamedImpl implements TargetGroup { private software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroup getAwsTargetGroup() { return landscape.getAwsTargetGroupByArn(getRegion(), getTargetGroupArn()); } + + public Iterable getTagDescriptions() { + return landscape.getTargetGroupTags(arn, region); + } public Region getRegion() { return region; diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/AddShardingKeyToShard.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/AddShardingKeyToShard.java new file mode 100644 index 00000000000..1fed2084a89 --- /dev/null +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/AddShardingKeyToShard.java @@ -0,0 +1,131 @@ +package com.sap.sse.landscape.aws.orchestration; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; +import java.util.logging.Logger; + +import com.sap.sse.common.Util; +import com.sap.sse.landscape.application.ApplicationProcess; +import com.sap.sse.landscape.application.ApplicationProcessMetrics; +import com.sap.sse.landscape.aws.ApplicationLoadBalancer; +import com.sap.sse.landscape.aws.AwsShard; +import com.sap.sse.landscape.aws.TargetGroup; + +import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleCondition; + +/** + * This procedure appends {@code shardingKeys} to a shard, identified by {@link #shardName} from + * the {@link #replicaSet}. This is done by adding rule conditions to the shard's rule set and after that just appending new + * rules. This could lead to moving to replica set to another load balancer, because it may get full with new sharding + * rules. + * + * @author I569653 + * + * @param + * @param + * @param + */ +public class AddShardingKeyToShard> + extends ShardProcedure { + private static final Logger logger = Logger.getLogger(AddShardingKeyToShard.class.getName()); + + public AddShardingKeyToShard(BuilderImpl builder) throws Exception { + super(builder); + } + + static class BuilderImpl, ShardingKey, MetricsT, ProcessT>, ShardingKey, MetricsT extends ApplicationProcessMetrics, ProcessT extends ApplicationProcess> + extends + ShardProcedure.BuilderImpl, ShardingKey, MetricsT, ProcessT> { + + @Override + public AddShardingKeyToShard build() throws Exception { + assert shardingKeys != null; + assert replicaSet != null; + assert region != null; + assert passphraseForPrivateKeyDecryption != null; + return new AddShardingKeyToShard(this); + } + } + + @Override + public void run() throws Exception { + AwsShard shard = null; + for (final Entry, Iterable> entry : replicaSet.getShards().entrySet()) { + if (entry.getKey().getName().equals(shardName)) { + shard = entry.getKey(); + break; + } + } + if (shard == null) { + throw new Exception("Shard "+shardName+" not found in replica set "+replicaSet.getName()); + } + logger.info("Appending " + Util.joinStrings(", ", shardingKeys) + " to " + shardName); + // list for manipulation -> elements are allowed to be removed!! + final List mutableShardingKeys = new LinkedList<>(); + mutableShardingKeys.addAll(shardingKeys); + final TargetGroup targetgroup = shard.getTargetGroup(); + final ApplicationLoadBalancer loadBalancer = shard.getLoadBalancer(); + final Collection> t = new ArrayList<>(); + t.add(targetgroup); + // check if there is a rule with space left for one or more additional conditions: + for (Rule r : shard.getRules()) { + boolean updateRule = false; + final ArrayList shardingKeys = new ArrayList<>(); + for (RuleCondition con : r.conditions()) { + // if we find a + if (con.pathPatternConfig() != null) { + // eliminate PATH_UNUSED_BY_ANY_APPLICATION in case this proxy key was found; + // it usually indicates an empty shard; when now adding one or more conditions + // it can be replaced. + Util.addAll( + Util.filter( + Util.map(con.values(), this::getShardingKeyFromPathCondition), + shardingKey->!shardingKey.equals(SHARDING_KEY_UNUSED_BY_ANY_APPLICATION)), + shardingKeys); + } + } + if (shardingKeys.isEmpty()) { + // the rule probably only has PATH_UNUSED_BY_ANY_APPLICATION and was a proxy rule, probably at the end of the list; remove + loadBalancer.deleteRules(r); + } else { // update only non-empty rule because we assume it won't be at the end of the list + while (shardingKeys.size() < ApplicationLoadBalancer.MAX_CONDITIONS_PER_RULE - NUMBER_OF_STANDARD_CONDITIONS_FOR_SHARDING_RULE + && !mutableShardingKeys.isEmpty()) { + shardingKeys.add(mutableShardingKeys.get(0)); + mutableShardingKeys.remove(0); + updateRule = true; + } + final Collection ruleConditions = getShardingRuleConditions(loadBalancer, shardingKeys); + // construct a rule only for transporting the conditions; no forwarding target is required for modifyRuleConditions + Rule proxyRuleWithNewConditions = Rule.builder().ruleArn(r.ruleArn()).conditions(ruleConditions).build(); + if (updateRule) { + getLandscape().modifyRuleConditions(region, proxyRuleWithNewConditions); + } + } + } + if (!mutableShardingKeys.isEmpty()) { + // check number of rules + final Set keysCopy = new HashSet<>(); + keysCopy.addAll(shardingKeys); + if (Util.size(loadBalancer.getRules()) + numberOfRequiredRules(Util.size(shardingKeys)) + < ApplicationLoadBalancer.MAX_RULES_PER_LOADBALANCER) { + // enough rules + addShardingRules(loadBalancer, keysCopy, targetgroup); + } else { + // not enough rules + final ApplicationLoadBalancer alb = getFreeLoadBalancerAndMoveReplicaSet(); + // set new rules + addShardingRules(alb, keysCopy, targetgroup); + } + } + } + + public static , BuilderT extends Builder, ShardingKey, MetricsT, ProcessT>, ShardingKey> Builder, ShardingKey, MetricsT, ProcessT> builder() { + return new BuilderImpl(); + } +} \ No newline at end of file diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/CreateDNSBasedLoadBalancerMapping.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/CreateDNSBasedLoadBalancerMapping.java index fb0777315c8..48a602f3f79 100755 --- a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/CreateDNSBasedLoadBalancerMapping.java +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/CreateDNSBasedLoadBalancerMapping.java @@ -6,7 +6,6 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.logging.Logger; import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.IntStream; import com.jcraft.jsch.JSchException; @@ -37,9 +36,6 @@ public class CreateDNSBasedLoadBalancerMapping> extends CreateLoadBalancerMapping implements Procedure { - private static final String DNS_MAPPED_ALB_NAME_PREFIX = "DNSMapped-"; - private static final Pattern ALB_NAME_PATTERN = Pattern.compile(DNS_MAPPED_ALB_NAME_PREFIX+"(.*)$"); - public static interface Builder, T extends CreateDNSBasedLoadBalancerMapping, ShardingKey, MetricsT extends ApplicationProcessMetrics, @@ -87,7 +83,7 @@ implements Procedure { ApplicationLoadBalancer result = null; final Set loadBalancerNames = new HashSet<>(); for (final ApplicationLoadBalancer loadBalancer : landscape.getLoadBalancers(region)) { - if (ALB_NAME_PATTERN.matcher(loadBalancer.getName()).matches()) { + if (ApplicationLoadBalancer.ALB_NAME_PATTERN.matcher(loadBalancer.getName()).matches()) { loadBalancerNames.add(loadBalancer.getName()); if (Util.size(loadBalancer.getRules()) <= MAX_RULES_PER_ALB - NUMBER_OF_RULES_PER_REPLICA_SET) { result = loadBalancer; @@ -105,18 +101,18 @@ implements Procedure { } /** - * Picks a new load balancer name following the pattern {@link #DNS_MAPPED_ALB_NAME_PREFIX}{@code [0-9]+} that is not + * Picks a new load balancer name following the pattern {@link #ApplicationLoadBalancer.DNS_MAPPED_ALB_NAME_PREFIX}{@code [0-9]+} that is not * part of {@code loadBalancerNames} and has the least number. */ private String getAvailableDNSMappedAlbName(Set loadBalancerNames) { final Set numbersTaken = new HashSet<>(); for (final String loadBalancerName : loadBalancerNames) { - final Matcher matcher = ALB_NAME_PATTERN.matcher(loadBalancerName); + final Matcher matcher = ApplicationLoadBalancer.ALB_NAME_PATTERN.matcher(loadBalancerName); if (matcher.find()) { numbersTaken.add(Integer.parseInt(matcher.group(1))); } } - return DNS_MAPPED_ALB_NAME_PREFIX + IntStream.range(0, MAX_ALBS_PER_REGION).filter(i->!numbersTaken.contains(i)).min().getAsInt(); + return ApplicationLoadBalancer.DNS_MAPPED_ALB_NAME_PREFIX + IntStream.range(0, ApplicationLoadBalancer.MAX_ALBS_PER_REGION).filter(i->!numbersTaken.contains(i)).min().getAsInt(); } @Override diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/CreateLoadBalancerMapping.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/CreateLoadBalancerMapping.java index ec62d057880..0ebdae7430c 100755 --- a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/CreateLoadBalancerMapping.java +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/CreateLoadBalancerMapping.java @@ -17,16 +17,11 @@ import com.sap.sse.landscape.aws.ApplicationLoadBalancer; import com.sap.sse.landscape.aws.AwsInstance; import com.sap.sse.landscape.aws.AwsLandscape; import com.sap.sse.landscape.aws.TargetGroup; -import com.sap.sse.landscape.aws.common.shared.PlainRedirectDTO; import com.sap.sse.shared.util.Wait; import software.amazon.awssdk.services.ec2.model.InstanceStateName; -import software.amazon.awssdk.services.elasticloadbalancingv2.model.Action; -import software.amazon.awssdk.services.elasticloadbalancingv2.model.ActionTypeEnum; import software.amazon.awssdk.services.elasticloadbalancingv2.model.LoadBalancerStateEnum; import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule; -import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleCondition; -import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupTuple; /** * For an {@link ApplicationProcess} creates a set of rules in an {@link ApplicationLoadBalancer} which drives traffic @@ -65,7 +60,7 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupT *

    * * The default health check settings for the target groups created are: - *

      + *
        Procedure *
      • healthy threshold: 2
        • *
      • unhealthy threshold: 2
        • *
      • timeout: 4s
        • @@ -77,8 +72,8 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupT */ public abstract class CreateLoadBalancerMapping> -extends ProcedureWithTargetGroup { - protected static int NUMBER_OF_RULES_PER_REPLICA_SET = 5; +extends ProcedureWithTargetGroup +implements ProcedureCreatingLoadBalancerMapping { protected static final int MAX_RULES_PER_ALB = 100; protected static final int MAX_ALBS_PER_REGION = 20; private final ProcessT process; @@ -216,44 +211,13 @@ extends ProcedureWithTargetGroup { Level.INFO, "Waiting for instance "+getHost().getId()+" to be in state RUNNING"); getLandscape().addTargetsToTargetGroup(masterTargetGroupCreated, Collections.singleton(getHost())); getLandscape().addTargetsToTargetGroup(publicTargetGroupCreated, Collections.singleton(getHost())); - getLoadBalancerUsed().addRulesAssigningUnusedPriorities(/* forceContiguous */ true, createRules()); + getLoadBalancerUsed().addRulesAssigningUnusedPriorities(/* forceContiguous */ true, + createRules(getLoadBalancerUsed(), getHostName(), masterTargetGroupCreated, publicTargetGroupCreated)); } catch (Exception e) { throw new RuntimeException(e); } } - private Rule[] createRules() { - final Rule[] rules = new Rule[NUMBER_OF_RULES_PER_REPLICA_SET]; - int ruleCount = 0; - rules[ruleCount++] = getLoadBalancerUsed().getDefaultRedirectRule(getHostName(), new PlainRedirectDTO()); - rules[ruleCount++] = Rule.builder().conditions( - RuleCondition.builder().field("http-header").httpHeaderConfig(hhcb->hhcb.httpHeaderName(HttpRequestHeaderConstants.HEADER_KEY_FORWARD_TO).values(HttpRequestHeaderConstants.HEADER_FORWARD_TO_MASTER.getB())).build(), - getLoadBalancerUsed().createHostHeaderRuleCondition(getHostName())). - actions(createForwardToTargetGroupAction(getMasterTargetGroupCreated())). - build(); - rules[ruleCount++] = Rule.builder().conditions( - RuleCondition.builder().field("http-header").httpHeaderConfig(hhcb->hhcb.httpHeaderName(HttpRequestHeaderConstants.HEADER_KEY_FORWARD_TO).values(HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA.getB())).build(), - getLoadBalancerUsed().createHostHeaderRuleCondition(getHostName())). - actions(createForwardToTargetGroupAction(getPublicTargetGroupCreated())). - build(); - rules[ruleCount++] = Rule.builder().conditions( - RuleCondition.builder().field("http-request-method").httpRequestMethodConfig(hrmcb->hrmcb.values("GET")).build(), - getLoadBalancerUsed().createHostHeaderRuleCondition(getHostName())). - actions(createForwardToTargetGroupAction(getPublicTargetGroupCreated())). - build(); - rules[ruleCount++] = Rule.builder().conditions( - getLoadBalancerUsed().createHostHeaderRuleCondition(getHostName())). - actions(createForwardToTargetGroupAction(getMasterTargetGroupCreated())). - build(); - assert ruleCount == NUMBER_OF_RULES_PER_REPLICA_SET; - return rules; - } - - private Action createForwardToTargetGroupAction(TargetGroup targetGroup) { - return Action.builder().type(ActionTypeEnum.FORWARD).forwardConfig(fc -> fc.targetGroups( - TargetGroupTuple.builder().targetGroupArn(targetGroup.getTargetGroupArn()).build())) .build(); - } - protected String getHostName() { return hostname; } diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/CreateShard.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/CreateShard.java new file mode 100644 index 00000000000..cb0cfa9a744 --- /dev/null +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/CreateShard.java @@ -0,0 +1,185 @@ +package com.sap.sse.landscape.aws.orchestration; + +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.sap.sse.common.Duration; +import com.sap.sse.common.HttpRequestHeaderConstants; +import com.sap.sse.common.Util; +import com.sap.sse.landscape.Landscape; +import com.sap.sse.landscape.application.ApplicationProcess; +import com.sap.sse.landscape.application.ApplicationProcessMetrics; +import com.sap.sse.landscape.aws.ApplicationLoadBalancer; +import com.sap.sse.landscape.aws.AwsAutoScalingGroup; +import com.sap.sse.landscape.aws.AwsInstance; +import com.sap.sse.landscape.aws.TargetGroup; +import com.sap.sse.landscape.aws.common.shared.ShardTargetGroupName; +import com.sap.sse.shared.util.Wait; + +import software.amazon.awssdk.services.elasticloadbalancingv2.model.Action; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.ActionTypeEnum; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.ForwardActionConfig; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleCondition; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupTuple; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetHealth; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetHealthStateEnum; + +/** + * This class is for creating shards out of the {@code shardingKeys}. This class creats a target group, and an + * autoscaling group and inserts rules into the {@code replicaSet}'s load balancer. If the load balancer does not have + * enough rules left in it'S HTTPS-listener, the whole replica set gets moved to another load balancer. + * + * @author I569653 + * + * @param + * @param + * @param + */ +public class CreateShard> + extends ShardProcedure { + private static int DEFAULT_INSTANCE_STARTUP_TIME = 180; + private static final Logger logger = Logger.getLogger(ShardProcedure.class.getName()); + private final String targetGroupNamePrefix; + + public CreateShard(BuilderImpl builder) throws Exception { + super(builder); + this.targetGroupNamePrefix = builder.getTargetGroupNamePrefix(); + } + + public static interface Builder< + BuilderT extends Builder, + T extends CreateShard, + ShardingKey, + MetricsT extends ApplicationProcessMetrics, + ProcessT extends ApplicationProcess> + extends ShardProcedure.Builder { + BuilderT setTargetGroupNamePrefix(String targetGroupNamePrefix); + } + + static class BuilderImpl, ShardingKey, MetricsT, ProcessT>, + ShardingKey, + MetricsT extends ApplicationProcessMetrics, + ProcessT extends ApplicationProcess> + extends ShardProcedure.BuilderImpl, ShardingKey, MetricsT, ProcessT> + implements Builder, ShardingKey, MetricsT, ProcessT> { + private String targetGroupNamePrefix = ""; + + String getTargetGroupNamePrefix() { + return targetGroupNamePrefix; + } + + public BuilderT setTargetGroupNamePrefix(String targetGroupNamePrefix) { + if (!ShardTargetGroupName.isValidTargetGroupNamePrefix(targetGroupNamePrefix)) { + throw new IllegalArgumentException("Not a valid target group name prefix: "+targetGroupNamePrefix); + } + this.targetGroupNamePrefix = targetGroupNamePrefix; + return self(); + } + + @Override + public CreateShard build() throws Exception { + assert shardingKeys != null; + assert replicaSet != null; + assert region != null; + assert passphraseForPrivateKeyDecryption != null; + return new CreateShard(this); + } + } + + @Override + public void run() throws Exception { + final ShardTargetGroupName name; + if (shardName == null) { + throw new Exception("Shardname is null, please enter a name"); + } else { + name = replicaSet.getNewShardName(shardName, targetGroupNamePrefix); + } + if (!isTargetGroupNameUnique(name.getTargetGroupName())) { + throw new Exception( + "targetgroup name with this shardname is not unique. You may change the last or first two chars"); + } + final ApplicationLoadBalancer loadBalancer = getFreeLoadBalancerAndMoveReplicaSet(); + logger.info( + "Creating Targer group for Shard " + name + ". Inheriting from Replicaset: " + replicaSet.getName()); + final TargetGroup targetGroup = getLandscape().createTargetGroupWithoutLoadbalancer(region, + name.getTargetGroupName(), replicaSet.getMaster().getPort()); + getLandscape().addTargetGroupTag(targetGroup.getTargetGroupArn(), ShardTargetGroupName.TAG_KEY, name.getName(), region); + final AwsAutoScalingGroup autoScalingGroup = replicaSet.getAutoScalingGroup(); + logger.info("Creating Autoscalinggroup for Shard " + shardName + ". Inheriting from Autoscalinggroup: " + + autoScalingGroup.getName()); + getLandscape().createAutoScalingGroupFromExisting(autoScalingGroup, shardName, targetGroup, Optional.empty()); + // create one rule to path unused by any application for linking ALB to target group. + if (loadBalancer != null) { + final Iterable rules = loadBalancer.getRules(); + if (Util.size(rules) < ApplicationLoadBalancer.MAX_RULES_PER_LOADBALANCER + - numberOfRequiredRules(Util.size(shardingKeys))) { + final int rulePrio = getHighestAvailableIndex(rules); + if (rulePrio > 0) { + Rule newRule = Rule.builder().priority("" + rulePrio) + .conditions( + loadBalancer.createHostHeaderRuleCondition(replicaSet.getHostname()), + RuleCondition.builder().field("http-header") + .httpHeaderConfig(hhcb -> hhcb + .httpHeaderName(HttpRequestHeaderConstants.HEADER_KEY_FORWARD_TO) + .values(HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA + .getB())) + .build(), + RuleCondition.builder().field("path-pattern") + .pathPatternConfig(ppc -> ppc.values(getPathConditionForShardingKey(SHARDING_KEY_UNUSED_BY_ANY_APPLICATION))).build()) + .actions(Action.builder() + .forwardConfig(ForwardActionConfig.builder() + .targetGroups(TargetGroupTuple.builder() + .targetGroupArn(targetGroup.getTargetGroupArn()).build()) + .build()) + .type(ActionTypeEnum.FORWARD).build()) + .build(); + final Iterable newRuleSet = loadBalancer.addRules(newRule); + getLandscape().putScalingPolicy(DEFAULT_INSTANCE_STARTUP_TIME, getLandscape().getAutoScalingGroupName(shardName), targetGroup, + AwsAutoScalingGroup.DEFAULT_MAX_REQUESTS_PER_TARGET, region); + // wait until instances are running + Wait.wait(()->{ + boolean ret = true; + final Map, TargetHealth> healths = getLandscape() + .getTargetHealthDescriptions(targetGroup); + if (healths.isEmpty()) { + ret = false; // if there is no Aws in target + } else { + for (Map.Entry, TargetHealth> instance : healths.entrySet()) { + if (instance.getValue().state() != TargetHealthStateEnum.HEALTHY) { + ret = false; // if this instance is unhealthy + break; + } + } + } + return ret; + }, Landscape.WAIT_FOR_HOST_TIMEOUT, Duration.ONE_SECOND.times(30), Level.INFO, "Instances not yet healty"); + // remove dummy-rule + for (Rule r : newRuleSet) { + loadBalancer.deleteRules(r); + } + final Set shardingKeysToUse; + if (shardingKeys.isEmpty()) { + shardingKeysToUse = Collections.singleton(SHARDING_KEY_UNUSED_BY_ANY_APPLICATION); + } else { + shardingKeysToUse = shardingKeys; + } + // change ALB rules to new ones + addShardingRules(loadBalancer, shardingKeysToUse, targetGroup); + } else { + throw new Exception("Unexpected Error - No prio left?"); + } + } else { + throw new Exception("Unexpected Error - Loadbalancer was null!"); + } + } + } + + public static , BuilderT extends Builder, ShardingKey, MetricsT, ProcessT>, ShardingKey> Builder, ShardingKey, MetricsT, ProcessT> builder() { + return new BuilderImpl(); + } +} \ No newline at end of file diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/ProcedureCreatingLoadBalancerMapping.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/ProcedureCreatingLoadBalancerMapping.java new file mode 100644 index 00000000000..f4936c73524 --- /dev/null +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/ProcedureCreatingLoadBalancerMapping.java @@ -0,0 +1,59 @@ +package com.sap.sse.landscape.aws.orchestration; + +import com.sap.sse.common.HttpRequestHeaderConstants; +import com.sap.sse.landscape.aws.ApplicationLoadBalancer; +import com.sap.sse.landscape.aws.TargetGroup; +import com.sap.sse.landscape.aws.common.shared.PlainRedirectDTO; + +import software.amazon.awssdk.services.elasticloadbalancingv2.model.Action; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.ActionTypeEnum; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleCondition; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupTuple; + +public interface ProcedureCreatingLoadBalancerMapping { + /** + * The default number of rules required for a replica set, including a default redirect rule, but excluding + * any additional sharding rules. + */ + int NUMBER_OF_RULES_PER_REPLICA_SET = 5; + + default Rule[] createRules(ApplicationLoadBalancer alb, String hostName, TargetGroup masterTargetGroup, + TargetGroup publicTargetGroup) { + final Rule[] rules = new Rule[NUMBER_OF_RULES_PER_REPLICA_SET]; + int ruleCount = 0; + rules[ruleCount++] = alb.getDefaultRedirectRule(hostName, new PlainRedirectDTO()); + rules[ruleCount++] = Rule.builder().conditions( + RuleCondition.builder().field("http-header") + .httpHeaderConfig(hhcb -> hhcb.httpHeaderName(HttpRequestHeaderConstants.HEADER_KEY_FORWARD_TO) + .values(HttpRequestHeaderConstants.HEADER_FORWARD_TO_MASTER.getB())) + .build(), + alb.createHostHeaderRuleCondition(hostName)) + .actions(createForwardToTargetGroupAction(masterTargetGroup)).build(); + rules[ruleCount++] = Rule.builder().conditions( + RuleCondition.builder().field("http-header") + .httpHeaderConfig(hhcb -> hhcb.httpHeaderName(HttpRequestHeaderConstants.HEADER_KEY_FORWARD_TO) + .values(HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA.getB())) + .build(), + alb.createHostHeaderRuleCondition(hostName)) + .actions(createForwardToTargetGroupAction(publicTargetGroup)).build(); + rules[ruleCount++] = Rule.builder() + .conditions( + RuleCondition.builder().field("http-request-method") + .httpRequestMethodConfig(hrmcb -> hrmcb.values("GET")).build(), + alb.createHostHeaderRuleCondition(hostName)) + .actions(createForwardToTargetGroupAction(publicTargetGroup)).build(); + rules[ruleCount++] = Rule.builder() + .conditions(alb.createHostHeaderRuleCondition(hostName)) + .actions(createForwardToTargetGroupAction(masterTargetGroup)).build(); + assert ruleCount == NUMBER_OF_RULES_PER_REPLICA_SET; + return rules; + } + + default Action createForwardToTargetGroupAction(TargetGroup targetGroup) { + return Action.builder().type(ActionTypeEnum.FORWARD).forwardConfig(fc -> fc.targetGroups( + TargetGroupTuple.builder().targetGroupArn(targetGroup.getTargetGroupArn()).build())) .build(); + } + + +} diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/ProcedureWithTargetGroup.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/ProcedureWithTargetGroup.java index dcb2bdbeeb0..458541f9a5b 100755 --- a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/ProcedureWithTargetGroup.java +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/ProcedureWithTargetGroup.java @@ -12,8 +12,6 @@ import com.sap.sse.landscape.aws.ApplicationLoadBalancer; import com.sap.sse.landscape.aws.AwsLandscape; import com.sap.sse.landscape.aws.TargetGroup; -import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule; - /** * An abstract base class for procedures dealing with the public and master target groups in a * {@link ApplicationLoadBalancer load balancer}, e.g., creating them or fetching them, or @@ -27,7 +25,6 @@ extends AbstractAwsProcedureImpl { private final ApplicationLoadBalancer loadBalancerUsed; private final String targetGroupNamePrefix; private final String serverName; - private Iterable rulesAdded; /** * If no {@link #setTargetGroupNamePrefix(String) target group name prefix} is specified, the target group names are @@ -128,8 +125,4 @@ extends AbstractAwsProcedureImpl { public ApplicationLoadBalancer getLoadBalancerUsed() { return loadBalancerUsed; } - - public Iterable getRulesAdded() { - return rulesAdded; - } } diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/RemoveShardingKeyFromShard.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/RemoveShardingKeyFromShard.java new file mode 100644 index 00000000000..6ea8d87c90e --- /dev/null +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/RemoveShardingKeyFromShard.java @@ -0,0 +1,91 @@ +package com.sap.sse.landscape.aws.orchestration; + +import java.util.HashSet; +import java.util.Set; +import java.util.Map.Entry; +import java.util.logging.Logger; + +import com.sap.sse.common.Util; +import com.sap.sse.landscape.application.ApplicationProcess; +import com.sap.sse.landscape.application.ApplicationProcessMetrics; +import com.sap.sse.landscape.aws.AwsShard; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleCondition; + +/** + * This class is supposed to remove {@code shardingKeys} from the shard, indentified by {@shardName} from + * {@code replicaSet}. This is done by rewriting all rules without {@code shardingKeys}'s path-conditions to the + * replicaSet's load balancer. In the future, there should be a algorithm to extract and resort all rules without + * removing them for ensuring that 100% of time the requests are reaching the shard's target group. + * + * @author I569653 + * + * @param + * @param + * @param + */ +public class RemoveShardingKeyFromShard> + extends ShardProcedure { + private static final Logger logger = Logger.getLogger(RemoveShardingKeyFromShard.class.getName()); + + public RemoveShardingKeyFromShard(BuilderImpl builder) throws Exception { + super(builder); + } + + static class BuilderImpl, ShardingKey, MetricsT, ProcessT>, ShardingKey, MetricsT extends ApplicationProcessMetrics, ProcessT extends ApplicationProcess> + extends + ShardProcedure.BuilderImpl, ShardingKey, MetricsT, ProcessT> { + + @Override + public RemoveShardingKeyFromShard build() throws Exception { + assert shardingKeys != null; + assert replicaSet != null; + assert region != null; + assert passphraseForPrivateKeyDecryption != null; + return new RemoveShardingKeyFromShard(this); + } + } + + @Override + public void run() throws Exception { + AwsShard shard = null; + for (Entry, Iterable> entry : replicaSet.getShards().entrySet()) { + if (entry.getKey().getName().equals(shardName)) { + shard = entry.getKey(); + break; + } + } + if (shard == null) { + throw new Exception("Shard not found!"); + } + logger.info("Removing " + Util.joinStrings(", ", shardingKeys) + " from " + shardName); + // remove conditions in rules where path is the sharding key + final Set shardingKeysFromConditions = new HashSet<>(); + for (Rule r : shard.getRules()) { + for (RuleCondition condition : r.conditions()) { + if (condition.pathPatternConfig() != null) { + shardingKeysFromConditions.addAll( + Util.asList( + Util.filter( + Util.map(condition.values(), this::getShardingKeyFromPathCondition), + shardingKey -> !shardingKeys.contains(shardingKey)))); + } else { + logger.warning("This is strange: shard "+shard.getName()+" of replica set "+shard.getReplicaSetName()+ + " has a rule "+r+" that has no path pattern condition; ignoring that rule while removing shard."); + } + } + } + if (shardingKeysFromConditions.isEmpty()) { + // if the shard runs empty (no more sharding keys defined for it), set a proxy key to keep + // the shard discoverable and its target group linked to the load balancer for continued target health checks + shardingKeysFromConditions.add(SHARDING_KEY_UNUSED_BY_ANY_APPLICATION); + } + getLandscape().deleteLoadBalancerListenerRules(region, Util.toArray(shard.getRules(), new Rule[0])); + // change ALB rules to new ones + addShardingRules(shard.getLoadBalancer(), shardingKeysFromConditions, shard.getTargetGroup()); + } + + public static , BuilderT extends Builder, ShardingKey, MetricsT, ProcessT>, ShardingKey> Builder, ShardingKey, MetricsT, ProcessT> builder() { + return new BuilderImpl(); + } +} \ No newline at end of file diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/ShardProcedure.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/ShardProcedure.java new file mode 100644 index 00000000000..437219cba9c --- /dev/null +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/orchestration/ShardProcedure.java @@ -0,0 +1,479 @@ +package com.sap.sse.landscape.aws.orchestration; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.stream.IntStream; +import java.util.stream.StreamSupport; + +import com.sap.sse.common.HttpRequestHeaderConstants; +import com.sap.sse.common.Util; +import com.sap.sse.landscape.Region; +import com.sap.sse.landscape.application.ApplicationProcess; +import com.sap.sse.landscape.application.ApplicationProcessMetrics; +import com.sap.sse.landscape.aws.ApplicationLoadBalancer; +import com.sap.sse.landscape.aws.AwsApplicationReplicaSet; +import com.sap.sse.landscape.aws.AwsLandscape; +import com.sap.sse.landscape.aws.AwsShard; +import com.sap.sse.landscape.aws.TargetGroup; + +import software.amazon.awssdk.services.elasticloadbalancingv2.model.Action; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.ActionTypeEnum; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.ForwardActionConfig; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleCondition; +import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupTuple; +/** + * This class is the base for all procedures that deal with shards. In the subclasses, all procedures are described. + * The parameter stands for a type that represents a sharding key. But those keys are normally strings because AWS deals with them as strings in + * their Rules. The function Shard.getKeys returns a list of the keys contained by the shard. Those keys are found in the shard's rules as a path-condition. + * This class implements most of the required functionality when dealing with shards like inserting rules or switching a replica set's load balancer. + * @author I569653 + * + * @param + * @param + * @param + */ +public abstract class ShardProcedure> +extends AbstractAwsProcedureImpl +implements ProcedureCreatingLoadBalancerMapping { + private static final Logger logger = Logger.getLogger(ShardProcedure.class.getName()); + final static int NUMBER_OF_STANDARD_CONDITIONS_FOR_SHARDING_RULE = 2; + @SuppressWarnings("unchecked") // this silently assumes that a String can be cast to a ShardingKey without problems + protected final ShardingKey SHARDING_KEY_UNUSED_BY_ANY_APPLICATION = (ShardingKey) "lauycaluy3cla3yrclaurlIYQL8"; + protected final String shardName; + final protected Set shardingKeys; + final AwsApplicationReplicaSet replicaSet; + final Region region; + final byte[] passphraseForPrivateKeyDecryption; + private final String pathPrefixForShardingKey; + + protected ShardProcedure(BuilderImpl builder) throws Exception { + super(builder); + this.shardName = builder.getShardName(); + this.passphraseForPrivateKeyDecryption = builder.getPassphrase(); + this.replicaSet = builder.getReplicaSet(); + this.shardingKeys = builder.getShardingKeys(); + this.region = builder.getRegion(); + this.pathPrefixForShardingKey = builder.getPathPrefixForShardingKey(); + } + + public static interface Builder< + BuilderT extends Builder, + T extends ShardProcedure, + ShardingKey, + MetricsT extends ApplicationProcessMetrics, + ProcessT extends ApplicationProcess> + extends AbstractAwsProcedureImpl.Builder { + BuilderT setPathPrefixForShardingKey(String pathPrefixForShardingKey); + + BuilderT setShardName(String name); + + BuilderT setLandscape(AwsLandscape landscape); + + BuilderT setShardingKeys(Set shardingkeys); + + BuilderT setReplicaset(AwsApplicationReplicaSet replicaset); + + BuilderT setRegion(Region region); + + BuilderT setPassphrase(byte[] passphrase); + } + + protected abstract static class BuilderImpl, + T extends ShardProcedure, + ShardingKey, + MetricsT extends ApplicationProcessMetrics, + ProcessT extends ApplicationProcess> + extends + AbstractAwsProcedureImpl.BuilderImpl + implements + Builder { + protected String shardName; + protected Set shardingKeys; + protected AwsApplicationReplicaSet replicaSet; + protected Region region; + protected byte[] passphraseForPrivateKeyDecryption; + private String pathPrefixForShardingKey; + + @Override + public BuilderT setPathPrefixForShardingKey(String pathPrefixForShardingKey) { + this.pathPrefixForShardingKey = pathPrefixForShardingKey; + return self(); + } + + @Override + public BuilderT setShardName(String name) { + this.shardName = name; + return self(); + } + + @Override + public BuilderT setShardingKeys(Set shardingkeys) { + this.shardingKeys = shardingkeys; + return self(); + } + + @Override + public BuilderT setReplicaset(AwsApplicationReplicaSet replicaset) { + this.replicaSet = replicaset; + return self(); + } + + @Override + public BuilderT setRegion(Region region) { + this.region = region; + return self(); + } + + @Override + public BuilderT setPassphrase(byte[] passphrase) { + this.passphraseForPrivateKeyDecryption = passphrase; + return self(); + } + + @SuppressWarnings("unchecked") + @Override + public BuilderT setLandscape(AwsLandscape landscape) { + super.setLandscape((AwsLandscape) landscape); + return self(); + } + + protected AwsLandscape getLandscape() { + return (AwsLandscape) super.getLandscape(); + } + + byte[] getPassphrase() { + return passphraseForPrivateKeyDecryption; + } + + Region region() { + return region; + } + + AwsApplicationReplicaSet getReplicaSet() { + return replicaSet; + } + + Set getShardingKeys() { + return shardingKeys; + } + + String getShardName() { + return shardName; + } + + Region getRegion() { + return region; + } + + String getPathPrefixForShardingKey() { + return pathPrefixForShardingKey; + } + } + + protected boolean isTargetGroupNameUnique(String name) { + final Iterable> targetGroups = getLandscape().getTargetGroups(region); + return Util.isEmpty(Util.filter(targetGroups, t -> t.getName().equals(name))); + } + + /** + * Produces conditions for a sharding load balancer rule based on the {@link #replicaSet}'s + * {@link AwsApplicationReplicaSet#getHostname() hostname}, a header-field condition that requires the request to be + * tagged for a replica, plus a path-pattern condition with the sharding keys as patterns. + * + * @param shardingKeys + * their number must not exceed {@link ApplicationLoadBalancer#MAX_CONDITIONS_PER_RULE} - + * {@link #NUMBER_OF_STANDARD_CONDITIONS_FOR_SHARDING_RULE}; pass sharding keys (as the name suggests), + * not full paths; the paths for the {@code path-pattern} condition will be constructed from the sharding + * keys by this method. See also {@link #getPathConditionForShardingKey(String, String)}. + */ + protected Collection getShardingRuleConditions(ApplicationLoadBalancer loadBalancer, + Collection shardingKeys) throws InterruptedException, ExecutionException { + if (shardingKeys.size() > ApplicationLoadBalancer.MAX_CONDITIONS_PER_RULE - NUMBER_OF_STANDARD_CONDITIONS_FOR_SHARDING_RULE) { + throw new IllegalArgumentException("too many sharding keys for the conditions of a single load balancer rule: "+shardingKeys+ + "; a maximum of "+(ApplicationLoadBalancer.MAX_CONDITIONS_PER_RULE - NUMBER_OF_STANDARD_CONDITIONS_FOR_SHARDING_RULE)+" is allowed"); + } + final Collection ruleConditions = new ArrayList<>(); + final Collection paths = Util.mapToArrayList(shardingKeys, shardingKey->getPathConditionForShardingKey(shardingKey, pathPrefixForShardingKey)); + ruleConditions.add(loadBalancer.createHostHeaderRuleCondition(replicaSet.getHostname())); + ruleConditions.add(RuleCondition.builder().field("http-header") + .httpHeaderConfig(hhcb -> hhcb.httpHeaderName(HttpRequestHeaderConstants.HEADER_KEY_FORWARD_TO) + .values(HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA.getB())) + .build()); + ruleConditions.add( + RuleCondition.builder().field("path-pattern").pathPatternConfig(hhcb -> hhcb.values(paths)).build()); + return ruleConditions; + } + + protected Iterable addShardingRules(ApplicationLoadBalancer alb, Iterable shardingKeys, + TargetGroup targetGroup) throws Exception { + // change ALB rules to new ones + final Collection rules = new ArrayList(); + final Set shardingKeyForConsumption = new HashSet<>(); + Util.addAll(shardingKeys, shardingKeyForConsumption); + final int ruleIdx = alb.getFirstShardingPriority(replicaSet.getHostname()); + while (!shardingKeyForConsumption.isEmpty()) { + alb.shiftRulesToMakeSpaceAt(ruleIdx); + final Set shardingKeysForNextRule = new HashSet<>(); + for (final Iterator i=shardingKeyForConsumption.iterator(); + shardingKeysForNextRule.size() < ApplicationLoadBalancer.MAX_CONDITIONS_PER_RULE-NUMBER_OF_STANDARD_CONDITIONS_FOR_SHARDING_RULE && i.hasNext(); ) { + shardingKeysForNextRule.add(i.next()); + i.remove(); + } + final Collection conditions = getShardingRuleConditions(alb, shardingKeysForNextRule); + rules.add(Rule.builder().priority("" + ruleIdx).conditions(conditions) + .actions(Action.builder() + .forwardConfig(ForwardActionConfig.builder() + .targetGroups(TargetGroupTuple.builder() + .targetGroupArn(targetGroup.getTargetGroupArn()).build()) + .build()) + .type(ActionTypeEnum.FORWARD).build()) + .build()); + } + return alb.addRules(Util.toArray(rules, new Rule[0])); + } + + /** + * Search through all load balancers and returns the first load balancer with enough rules left for + * the new sharding keys + the replicaSet's rules. When no load balancer is found, a new one gets created. The replicaSet gets moved to the new load balancer. + */ + protected ApplicationLoadBalancer getFreeLoadBalancerAndMoveReplicaSet() throws Exception { + int existingShardingRules = 0; + for (Entry, Iterable> s : replicaSet.getShards().entrySet()) { + existingShardingRules = existingShardingRules + Util.size(s.getKey().getRules()); + } + // shardingKeys holds those keys to add/initially create (the remove case doesn't get here); + // hence we have to add the rules required for those keys to the existing rules + // FIXME shouldn't we compute the number of required rules for the new entire set of sharding keys? What if rules were not / no longer fully filled with conditions? -->cleanup + final int requiredRules = numberOfRequiredRules(Util.size(shardingKeys)) + + (existingShardingRules + /* 5 std rules per replica set */ NUMBER_OF_RULES_PER_REPLICA_SET); + final ApplicationLoadBalancer res; + if (Util.size(replicaSet.getLoadBalancerRules()) + + numberOfRequiredRules(Util.size(shardingKeys)) < ApplicationLoadBalancer.MAX_RULES_PER_LOADBALANCER) { + res = replicaSet.getLoadBalancer(); + } else { + // Another loadbalancer + final Iterable> loadBalancers = getLandscape() + .getLoadBalancers(region); + final Iterable> loadBalancersFiltered = Util.filter(loadBalancers, + t -> { + try { + return !t.getArn().equals(replicaSet.getLoadBalancer().getArn()); + } catch (InterruptedException | ExecutionException e) { + logger.log(Level.WARNING, "Exception while trying to obtain a load balancer's ARN", e); + throw new RuntimeException(e); + } + }); + final ApplicationLoadBalancer alb = getDNSLoadbalancerWithRulesLeft(loadBalancersFiltered, + requiredRules + /* 5 default rules for the replica set */ NUMBER_OF_RULES_PER_REPLICA_SET); + if (alb != null) { + // There is an alb left with enough rules + res = alb; + } else { + final Set loadBalancerNames = new HashSet<>(); + for (ApplicationLoadBalancer lb : loadBalancers) { + loadBalancerNames.add(lb.getName()); + } + final String name = getAvailableDNSMappedAlbName(loadBalancerNames); + // Create a new alb + res = getLandscape().createLoadBalancer(name, region); + } + changeReplicaSetLoadBalancer(res, replicaSet); + } + return res; + } + + /** + * This method changes the {@code replicaSetToMove}'s load balancer to another one. This method contains a 6min + * sleep, which is necessary for ensuring that all DNS rules point to the new one. This method can fail if the + * target group names are too long because of the {@link TargetGroup#TEMP_SUFFIX} suffix for temporary target + * groups. + * + * @param targetAlb + * application load balancer to move to + * @param replicaSetToMove + * replica set to move + */ + private void changeReplicaSetLoadBalancer(ApplicationLoadBalancer targetAlb, + AwsApplicationReplicaSet replicaSetToMove) throws Exception { + // Move replicaset to this alb with all shards + // create temporary targetgroups + final Collection> tempTargetGroups = new ArrayList<>(); + final Collection> originalTargetGroups = new ArrayList<>(); + final Map, Iterable> shardingKeysPerTargetGroup = new HashMap<>(); + final Map, TargetGroup> targetGroupsToTempTargetgroups = new HashMap<>(); + final Map, TargetGroup> shardToTempTargetGroup = new HashMap<>(); + // add non sharding rules for replicaset + final Collection tempRules = new ArrayList<>(); + createTargetGroupsForMoving(shardToTempTargetGroup, tempTargetGroups, replicaSetToMove, targetGroupsToTempTargetgroups, originalTargetGroups, shardingKeysPerTargetGroup); + addRulesForMoving(targetAlb, shardToTempTargetGroup, tempRules, replicaSetToMove, targetGroupsToTempTargetgroups, originalTargetGroups, shardingKeysPerTargetGroup); + // set new DNS record -> overwrites old entry + final String hostname = replicaSetToMove.getHostname(); + getLandscape().setDNSRecordToApplicationLoadBalancer(replicaSetToMove.getHostedZoneId(), + hostname, targetAlb, /* force */ true); + // wait until new DNS record is alive + for (int i = 0; i < 6; i++) { + Thread.sleep(AwsLandscape.DEFAULT_DNS_TTL_SECONDS * /* conversion seconds to ms */ 1000); + logger.info(()->("Still waiting for DNS record " + hostname)); + } + logger.info(()->("Done waiting for DNS record " + hostname)); + // remove all old rules pointing to original TargetGroups + final Collection rulesToRemove = new ArrayList<>(); + replicaSetToMove.getLoadBalancer().getRulesForTargetGroups(originalTargetGroups) + .forEach(t -> rulesToRemove.add(t)); + rulesToRemove.add(replicaSetToMove.getDefaultRedirectRule()); + getLandscape().deleteLoadBalancerListenerRules(region, rulesToRemove.toArray(new Rule[0])); + for (Entry, TargetGroup> entry : targetGroupsToTempTargetgroups + .entrySet()) { + targetAlb.replaceTargetGroupInForwardRules(entry.getValue(), entry.getKey()); + } + for (TargetGroup t : tempTargetGroups) { + getLandscape().deleteTargetGroup(t); + } + } + + private void createTargetGroupsForMoving( + Map, TargetGroup> shardToTempTargetGroup, + Collection> tempTargetGroups, + AwsApplicationReplicaSet replicaSetToMove, + Map, TargetGroup> targetGroupsToTempTargetgroups, + Collection> originalTargetGroups, + Map, Iterable> shardingKeysPerTargetGroup) throws Exception { + final TargetGroup targetgroupMasterTemp = getLandscape() + .copyTargetGroup(replicaSetToMove.getMasterTargetGroup(), TargetGroup.TEMP_SUFFIX); + final TargetGroup targetgroupPublicTemp = getLandscape() + .copyTargetGroup(replicaSetToMove.getPublicTargetGroup(), TargetGroup.TEMP_SUFFIX); + tempTargetGroups.add(targetgroupMasterTemp); + tempTargetGroups.add(targetgroupPublicTemp); + targetGroupsToTempTargetgroups.put(replicaSetToMove.getMasterTargetGroup(), targetgroupMasterTemp); + targetGroupsToTempTargetgroups.put(replicaSetToMove.getPublicTargetGroup(), targetgroupPublicTemp); + originalTargetGroups.add(replicaSetToMove.getMasterTargetGroup()); + originalTargetGroups.add(replicaSetToMove.getPublicTargetGroup()); + for (Entry, Iterable> shardAndShardingKeys : replicaSetToMove.getShards().entrySet()) { + final TargetGroup tempShardTargetGroup = getLandscape() + .copyTargetGroup(shardAndShardingKeys.getKey().getTargetGroup(), TargetGroup.TEMP_SUFFIX); + shardToTempTargetGroup.put(shardAndShardingKeys.getKey(), tempShardTargetGroup); + shardingKeysPerTargetGroup.put(shardAndShardingKeys.getKey().getTargetGroup(), shardAndShardingKeys.getValue()); + tempTargetGroups.add(tempShardTargetGroup); + originalTargetGroups.add(shardAndShardingKeys.getKey().getTargetGroup()); + targetGroupsToTempTargetgroups.put(shardAndShardingKeys.getKey().getTargetGroup(), tempShardTargetGroup); + } + } + + private void addRulesForMoving(ApplicationLoadBalancer targetAlb, + Map, TargetGroup> shardToTempTargetGroup, Collection tempRules, + AwsApplicationReplicaSet replicaSetToMove, + Map, TargetGroup> targetGroupsToTempTargetgroups, + Collection> originalTargetGroups, + Map, Iterable> shardingKeysPerTargetGroup) throws Exception { + targetAlb + .addRulesAssigningUnusedPriorities(true, + createRules(targetAlb, replicaSet.getHostname(), + targetGroupsToTempTargetgroups.get(replicaSetToMove.getMasterTargetGroup()), + targetGroupsToTempTargetgroups.get(replicaSetToMove.getPublicTargetGroup()))) + .forEach(t -> tempRules.add(t)); + for (final Entry, Iterable> shardAndShardingKeys : replicaSetToMove.getShards().entrySet()) { + addShardingRules(targetAlb, shardingKeysPerTargetGroup.get(shardAndShardingKeys.getKey().getTargetGroup()), + shardToTempTargetGroup.get(shardAndShardingKeys.getKey())).forEach(t -> tempRules.add(t)); + } + } + + protected int numberOfRequiredRules(int numberOfShardingKeys) { + return (int) (numberOfShardingKeys / (ApplicationLoadBalancer.MAX_CONDITIONS_PER_RULE-NUMBER_OF_STANDARD_CONDITIONS_FOR_SHARDING_RULE)) + + (int) Math.signum(/* one more because casting to int rounds down */ numberOfShardingKeys % + (ApplicationLoadBalancer.MAX_CONDITIONS_PER_RULE-NUMBER_OF_STANDARD_CONDITIONS_FOR_SHARDING_RULE)); + } + + /** + * Returns a DNS-Mapped load balancer with enough rules left + * @param loadBalancers + * list of all load balancers to search through. + * @param numberOfRules + * number of required rules left in this load balancer + * @return + */ + private ApplicationLoadBalancer getDNSLoadbalancerWithRulesLeft( + Iterable> loadBalancers, int numberOfRules) { + final Iterable> loadBalancersFiltered = Util.filter(loadBalancers, + t -> t.getName().startsWith(ApplicationLoadBalancer.DNS_MAPPED_ALB_NAME_PREFIX)); + ApplicationLoadBalancer res = null; + for (ApplicationLoadBalancer loadBalancer : loadBalancersFiltered) { + if (Util.size(loadBalancer.getRules()) < ApplicationLoadBalancer.MAX_RULES_PER_LOADBALANCER - numberOfRules) { + res = loadBalancer; + break; + } + } + return res; + } + + /** + * iterates through all numbers from {@link ApplicationLoadBalancer#MAX_PRIORITY} to 1 (lowest index) and checks if + * any priority is not in the rule set. returns the first available priority. If no rules is available, it returns + * -1; + */ + protected int getHighestAvailableIndex(Iterable rules) { + for (int i = ApplicationLoadBalancer.MAX_PRIORITY; i > 1; i--) { + String y = "" + i; + if (!StreamSupport.stream(rules.spliterator(), false).anyMatch(t -> t.priority().contains(y))) { + return i; // return priority if there was no rule with the same + } + } + return -1; // if no free priority was found + } + + /** + * Picks a new load balancer name following the pattern {@link #DNS_MAPPED_ALB_NAME_PREFIX}{@code [0-9]+} that is + * not part of {@code loadBalancerNames} and has the least number. + */ + private String getAvailableDNSMappedAlbName(Set loadBalancerNames) { + final Set numbersTaken = new HashSet<>(); + for (final String loadBalancerName : loadBalancerNames) { + final Matcher matcher = ApplicationLoadBalancer.ALB_NAME_PATTERN.matcher(loadBalancerName); + if (matcher.find()) { + numbersTaken.add(Integer.parseInt(matcher.group(1))); + } + } + return ApplicationLoadBalancer.DNS_MAPPED_ALB_NAME_PREFIX + + IntStream.range(0, ApplicationLoadBalancer.MAX_ALBS_PER_REGION).filter(i -> !numbersTaken.contains(i)) + .min().getAsInt(); + } + + /** + * Path conditions are constructed by pre-pending a "*" to the sharding key. + */ + public static String getPathConditionForShardingKey(ShardingKey shardingKey, String pathPrefixForShardingKey) { + return pathPrefixForShardingKey+shardingKey.toString(); + } + + public static ShardingKey getShardingKeyFromPathCondition(String path, String pathPrefixForShardingKey) { + if (!path.startsWith(pathPrefixForShardingKey)) { + throw new IllegalStateException("path condition \""+path+"\" does not start with \""+pathPrefixForShardingKey+"\" which is unexpected"); + } + @SuppressWarnings("unchecked") // this silently assumes that a String casts into a ShardingKey without problems + final ShardingKey result = (ShardingKey) path.substring(pathPrefixForShardingKey.length()); + return result; + } + + /** + * Path conditions are constructed by pre-pending a "*" to the sharding key. + */ + protected String getPathConditionForShardingKey(ShardingKey shardingKey) { + return getPathConditionForShardingKey(shardingKey, pathPrefixForShardingKey); + } + + protected ShardingKey getShardingKeyFromPathCondition(String path) { + return getShardingKeyFromPathCondition(path, pathPrefixForShardingKey); + } +} diff --git a/java/com.sap.sse.landscape/src/com/sap/sse/landscape/application/ApplicationProcess.java b/java/com.sap.sse.landscape/src/com/sap/sse/landscape/application/ApplicationProcess.java index 531be72b681..76bda3ec90d 100755 --- a/java/com.sap.sse.landscape/src/com/sap/sse/landscape/application/ApplicationProcess.java +++ b/java/com.sap.sse.landscape/src/com/sap/sse/landscape/application/ApplicationProcess.java @@ -34,6 +34,9 @@ import com.sap.sse.util.HttpUrlConnectionHelper; public interface ApplicationProcess> extends Process { + + String HEALTH_CHECK_PATH = "/gwt/status"; + static Logger logger = Logger.getLogger(ApplicationProcess.class.getName()); static String REPLICATION_STATUS_POST_URL_PATH_AND_QUERY = ReplicationServletActions.REPLICATION_SERVLET_BASE_PATH+"?"+ReplicationServletActions.ACTION_PARAMETER_NAME+"="+ ReplicationServletActions.Action.STATUS.name(); diff --git a/java/com.sap.sse.landscape/src/com/sap/sse/landscape/application/ApplicationReplicaSet.java b/java/com.sap.sse.landscape/src/com/sap/sse/landscape/application/ApplicationReplicaSet.java index 85980b021cf..f012c77121f 100755 --- a/java/com.sap.sse.landscape/src/com/sap/sse/landscape/application/ApplicationReplicaSet.java +++ b/java/com.sap.sse.landscape/src/com/sap/sse/landscape/application/ApplicationReplicaSet.java @@ -1,9 +1,7 @@ package com.sap.sse.landscape.application; import java.io.IOException; -import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.ExecutionException; import com.sap.sse.common.Duration; @@ -69,7 +67,7 @@ ProcessT extends ApplicationProcess> extends Na /** * Moves a {@link Scope} with all its content from {@code source} into this replica set. The process may fail with * an exception, e.g., for connectivity or permission reasons, or---if the {@code failUponDiff} parameter is set to - * {@code true}--- for differences found when comparing the result in this replica set with the original content at + * {@code true}---for differences found when comparing the result in this replica set with the original content at * {@code source}. The {@code removeFromSourceUponSuccess} and {@code setRemoveReferenceInSourceUponSuccess} parameters * control how to proceed after successful import. * @@ -80,17 +78,6 @@ ProcessT extends ApplicationProcess> extends Na void removeScope(Scope scope); - /** - * Creates a "remote server reference" on this application replica set pointing to the {@code to} replica set. If a - * non-{@code null} sequence of {@link Scope}s is provided then the {@code includeOrExcludeScopes} flag decides - * whether the reference shall only include those scopes ({@code true}) or it should list all scopes - * except those listed in {@code scopes} ({@code false}) instead. - */ - void setRemoteReference(String name, ApplicationReplicaSet to, - Iterable> scopes, boolean includeOrExcludeScopes); - - void removeRemoteReference(String name); - /** * Tells this replica set whether read requests may also be addressed at the master node in case there are one or * more {@link #getReplicas() replicas} configured. If setting this to {@code true}, the {@link #getMaster() master @@ -113,26 +100,6 @@ ProcessT extends ApplicationProcess> extends Na */ boolean isReadFromMaster(); - Map>> getShardingInfo(); - - /** - * Activates sharding for the {@code shard} by configuring this replica set such that requests for the {@code shard} - * are usually submitted to any instance from the {@code processesToPrimarilyHandleShard} set. Only if no process - * within that set is available, the replica set will allow requests for {@code shard} to be handled by any other - * process in this replica set as a default. - * - * @param processesToPrimarilyHandleShard must not be {@code null} but can be empty - * - * @see #removeSharding - */ - void setSharding(Shard shard, Set> processesToPrimarilyHandleShard); - - /** - * Re-configures this replica set such that requests for {@code shard} will be spread across all processes - * of this replica set. - */ - void removeSharding(Shard shard); - /** * The fully-qualified host name by which this application replica set is publicly reachable. When resolving this * hostname through DNS, the result is expected to identify a load balancer which contains the ingress rules for diff --git a/java/com.sap.sse.landscape/src/com/sap/sse/landscape/application/Shard.java b/java/com.sap.sse.landscape/src/com/sap/sse/landscape/application/Shard.java index f35fe0f261e..cfe816bed88 100755 --- a/java/com.sap.sse.landscape/src/com/sap/sse/landscape/application/Shard.java +++ b/java/com.sap.sse.landscape/src/com/sap/sse/landscape/application/Shard.java @@ -1,17 +1,28 @@ package com.sap.sse.landscape.application; +import com.sap.sse.common.Named; + /** * Part of a {@link Scope}. A {@link Shard} cannot be moved in isolation and can hence not move across {@link Scope}s. * Sharding can be used to optionally split {@link ApplicationProcess}es in groups, each of which being responsible * primarily only for a subset of the {@link Shard}s available in the {@link ApplicationReplicaSet}. While we assume * that all {@link ApplicationProcess}es in an {@link ApplicationReplicaSet} can handle requests for all - * {@link Shard}s managed by the replica set, it may be beneficial performance-wise to have individual processes - * focus only on a subset of {@link Shard}s. This may, e.g., result in better cache utilization and hence less CPU - * consumption on the hosts running those processes. + * {@link Shard}s managed by the replica set, it may be beneficial performance-wise to have individual processes focus + * only on a subset of {@link Shard}s. This may, e.g., result in better cache utilization and hence less CPU consumption + * on the hosts running those processes. + *

        + * + * There may be restrictions for the {@link Named#getName() name} that a shard can have, for example it could be + * possible that shard names with quotes or brackets in them are not permitted; the name is expected to be human-readable + * and meaningful. * * @author Axel Uhl (D043530) * */ -public interface Shard { - ShardingKey getKey(); +public interface Shard extends Named { + /** + * @return the keys handled by this shard + * + */ + Iterable getKeys(); } diff --git a/java/com.sap.sse.landscape/src/com/sap/sse/landscape/application/impl/ApplicationReplicaSetImpl.java b/java/com.sap.sse.landscape/src/com/sap/sse/landscape/application/impl/ApplicationReplicaSetImpl.java index 1d25bd7fd3b..024c4cf6043 100755 --- a/java/com.sap.sse.landscape/src/com/sap/sse/landscape/application/impl/ApplicationReplicaSetImpl.java +++ b/java/com.sap.sse.landscape/src/com/sap/sse/landscape/application/impl/ApplicationReplicaSetImpl.java @@ -2,7 +2,6 @@ package com.sap.sse.landscape.application.impl; import java.util.Collections; import java.util.HashSet; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -15,7 +14,6 @@ import com.sap.sse.landscape.application.ApplicationProcess; import com.sap.sse.landscape.application.ApplicationProcessMetrics; import com.sap.sse.landscape.application.ApplicationReplicaSet; import com.sap.sse.landscape.application.Scope; -import com.sap.sse.landscape.application.Shard; public class ApplicationReplicaSetImpl> @@ -97,19 +95,6 @@ implements ApplicationReplicaSet { } - @Override - public void setRemoteReference(String name, ApplicationReplicaSet to, - Iterable> scopes, boolean includeOrExcludeScopes) { - // TODO Implement ApplicationReplicaSet.setRemoteReference(...) - // use /v1/remoteserverreference (RemoteServerReferenceResource) for this - } - - @Override - public void removeRemoteReference(String name) { - // TODO Implement ApplicationReplicaSet.removeRemoteReference(...) - // use /v1/remoteserverreference (RemoteServerReferenceResource) for this - } - @Override public void setReadFromMaster(boolean readFromMaster) throws IllegalStateException { // TODO Implement ApplicationReplicaSet.setReadFromMaster(...) @@ -122,23 +107,4 @@ implements ApplicationReplicaSet { // for this it would be helpful to understand the ALB / TargetGroup assignments return false; } - - @Override - public Map>> getShardingInfo() { - // TODO Implement ApplicationReplicaSet.getShardingInfo(...) - return null; - } - - @Override - public void setSharding(Shard shard, - Set> processesToPrimarilyHandleShard) { - // TODO Implement ApplicationReplicaSet.setSharding(...) - - } - - @Override - public void removeSharding(Shard shard) { - // TODO Implement ApplicationReplicaSet.removeSharding(...) - - } } diff --git a/java/com.sap.sse.security.ui/src/main/java/com/sap/sse/security/ui/client/component/SelectedElementsCountingButton.java b/java/com.sap.sse.security.ui/src/main/java/com/sap/sse/security/ui/client/component/SelectedElementsCountingButton.java index 0e8a95a1333..a667363a56a 100644 --- a/java/com.sap.sse.security.ui/src/main/java/com/sap/sse/security/ui/client/component/SelectedElementsCountingButton.java +++ b/java/com.sap.sse.security.ui/src/main/java/com/sap/sse/security/ui/client/component/SelectedElementsCountingButton.java @@ -20,13 +20,20 @@ import com.sap.sse.common.Named; * */ public class SelectedElementsCountingButton extends Button { + private final boolean enableWhenSelectionEmpty; + /** * Constructs the button without a confirmation callback installed. The {@code clickHandler} * will be invoked immediately with the selection when the button is clicked. */ public SelectedElementsCountingButton(final String html, final SetSelectionModel selectionModel, final ClickHandler clickHandler) { - this(html, selectionModel, /* no confirmation dialog */ (Supplier) null, clickHandler); + this(html, selectionModel, clickHandler, /* enableWhenSelectionEmpty */ false); + } + + public SelectedElementsCountingButton(String html, + SetSelectionModel selectionModel, ClickHandler clickHandler, boolean enableWhenSelectionEmpty) { + this(html, selectionModel, /* no confirmation dialog */ (Supplier) null, clickHandler, enableWhenSelectionEmpty); } /** @@ -60,8 +67,24 @@ public class SelectedElementsCountingButton extends Button { */ public SelectedElementsCountingButton(final String html, final SetSelectionModel selectionModel, final Supplier asker, final ClickHandler clickHandler) { + this(html, selectionModel, asker, clickHandler, /* enableWhenSelectionEmpty */ false); + } + + /** + * Allows callers to implement specific behavior for the confirmation step if a non-{@code null} {@code asker} is + * passed. A non-{@code null} {@code asker} will be invoked when the button is clicked. The {@code clickHandler} + * will be invoked after the button has been clicked and either {@code asker} was {@code null} or + * {@link Supplier#get() invoking} the {@code asker} has returned a {@code true} result. + * + * @param enableWhenSelectionEmpty + * when {@code true}, the button will also be enabled when the selection is empty; otherwise, the button + * will be disabled for an empty selection. + */ + public SelectedElementsCountingButton(final String html, final SetSelectionModel selectionModel, + final Supplier asker, final ClickHandler clickHandler, boolean enableWhenSelectionEmpty) { super(html); - setEnabled(!selectionModel.getSelectedSet().isEmpty()); + this.enableWhenSelectionEmpty = enableWhenSelectionEmpty; + setEnabled(enableWhenSelectionEmpty || !selectionModel.getSelectedSet().isEmpty()); addSelectionEventHandler(html, selectionModel); addClickHandler(asker, selectionModel, clickHandler); } @@ -70,7 +93,7 @@ public class SelectedElementsCountingButton extends Button { selectionModel.addSelectionChangeHandler(event -> { Set selectedSet = selectionModel.getSelectedSet(); setText(selectedSet.isEmpty() ? html : html + " (" + selectedSet.size() + ")"); - setEnabled(!selectedSet.isEmpty()); + setEnabled(enableWhenSelectionEmpty || !selectedSet.isEmpty()); }); } diff --git a/java/com.sap.sse.security/src/com/sap/sse/security/util/impl/SecuredServerImpl.java b/java/com.sap.sse.security/src/com/sap/sse/security/util/impl/SecuredServerImpl.java index fec33c56fc1..f080d9765a2 100644 --- a/java/com.sap.sse.security/src/com/sap/sse/security/util/impl/SecuredServerImpl.java +++ b/java/com.sap.sse.security/src/com/sap/sse/security/util/impl/SecuredServerImpl.java @@ -12,6 +12,7 @@ import java.util.UUID; import java.util.logging.Logger; import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status.Family; import org.apache.http.HttpRequest; import org.apache.http.HttpResponse; @@ -75,7 +76,8 @@ public class SecuredServerImpl implements SecuredServer { final HttpResponse response = client.execute(request); final int statusCode = response.getStatusLine().getStatusCode(); Object jsonParseResult; - if (statusCode == Response.Status.NO_CONTENT.getStatusCode()) { + if (statusCode == Response.Status.NO_CONTENT.getStatusCode() + || Response.Status.fromStatusCode(statusCode).getFamily() != Family.SUCCESSFUL) { jsonParseResult = null; } else { response.getEntity().writeTo(bos); diff --git a/java/com.sap.sse.test/src/com/sap/sse/test/TestIPAddressUtil.java b/java/com.sap.sse.test/src/com/sap/sse/test/TestIPAddressUtil.java new file mode 100644 index 00000000000..8ba4dcb4787 --- /dev/null +++ b/java/com.sap.sse.test/src/com/sap/sse/test/TestIPAddressUtil.java @@ -0,0 +1,31 @@ +package com.sap.sse.test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +import com.sap.sse.util.IPAddressUtil; + +public class TestIPAddressUtil { + @Test + public void testIPv4Matching() { + assertTrue(IPAddressUtil.isIPv4AddressLiteral("1.2.3.4")); + assertTrue(IPAddressUtil.isIPv4AddressLiteral("123.234.33.0")); + assertFalse(IPAddressUtil.isIPv4AddressLiteral("0.2.3.4")); + assertTrue(IPAddressUtil.isIPv4AddressLiteral("255.255.255.255")); + assertFalse(IPAddressUtil.isIPv4AddressLiteral("java.sun.com")); + } + + @Test + public void testIPv6Matching() { + assertTrue(IPAddressUtil.isIPv6AddressLiteral("2001:db8:3333:4444:5555:6666:7777:8888")); + assertTrue(IPAddressUtil.isIPv6AddressLiteral("2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF")); + assertTrue(IPAddressUtil.isIPv6AddressLiteral("::")); + assertTrue(IPAddressUtil.isIPv6AddressLiteral("2001:db8::")); + assertTrue(IPAddressUtil.isIPv6AddressLiteral("::1234:5678")); + assertTrue(IPAddressUtil.isIPv6AddressLiteral("2001:db8::1234:5678")); + assertFalse(IPAddressUtil.isIPv6AddressLiteral("java.sun.com")); + assertFalse(IPAddressUtil.isIPv6AddressLiteral(":")); + } +} diff --git a/java/com.sap.sse/src/com/sap/sse/util/IPAddressUtil.java b/java/com.sap.sse/src/com/sap/sse/util/IPAddressUtil.java new file mode 100644 index 00000000000..8adc1b7f61a --- /dev/null +++ b/java/com.sap.sse/src/com/sap/sse/util/IPAddressUtil.java @@ -0,0 +1,34 @@ +package com.sap.sse.util; + +import java.util.regex.Pattern; + +/** + * Helps identifying IPv4 and IPv6 address literals, using regular expressions. + *

        + * + * This can be helpful when trying to tell whether a host address is provided as + * a symbolic hostname that requires DNS resolution, or as an address literal that + * can directly be used. + * + * @author Axel Uhl (d043530) + * + */ +public class IPAddressUtil { + private static final Pattern ipv4Pattern = Pattern.compile("[1-9][0-9]{0,2}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}"); + private static final Pattern ipv6Pattern = Pattern.compile("(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))"); + + public static boolean isIPv4AddressLiteral(String address) { + return ipv4Pattern.matcher(address).matches(); + } + + public static boolean isIPv6AddressLiteral(String address) { + return ipv6Pattern.matcher(address).matches(); + } + + /** + * Tells if {@code address} is an IPv4 or IPv6 address literal. + */ + public static boolean isIPAddressLiteral(String address) { + return isIPv4AddressLiteral(address) || isIPv6AddressLiteral(address); + } +} diff --git a/java/pom.xml b/java/pom.xml index ff41b90fc2f..856a598ba56 100755 --- a/java/pom.xml +++ b/java/pom.xml @@ -93,6 +93,7 @@ com.sap.sse.landscape com.sap.sse.landscape.aws com.sap.sse.landscape.aws.common + com.sap.sse.landscape.aws.common.test com.sap.sse.landscape.aws.test com.sap.sse.landscape.aws.persistence com.sap.sailing.landscape diff --git a/wiki/bug4232-review.md b/wiki/bug4232-review.md deleted file mode 100644 index 95d9ed32576..00000000000 --- a/wiki/bug4232-review.md +++ /dev/null @@ -1,99 +0,0 @@ -**This page has the purpose to prepare and track the progress of the bug4232 review** -(See [bugzilla](https://bugzilla.sapsailing.com/bugzilla/show_bug.cgi?id=4232)) - -# Test Scenarios - -## Prerequisites -* two devices (Smartphone and/or Tablet) -* device configuration: bug4232 (so all settings are correct - or manual) -* server: https://dev.sapsailing.com -* refresh time: 20 sec (because of faster local refreshes) -* event: Extreme Sailing Series 2016 Act 8 Sydney - -The edit dialog (open with the pencil) should always be closed if a warning (yellow or red) will be displayed. In case of a red warning, the publish button is always disabled. - -One device will be called **A** and the other **B**, so they have unique names while testing. - -## While starting - -### Test 1 -* devices open the penalty fragment -* device A give Omar Air OCS, device B set DNS for Omar Air -* device A closes the penalty view (without publishing) -* device B should see a red warning sign after the automatic refresh -* publish should be disabled and can be activated by changing every red item in the list - -### Test 2 -* same as [test 1](#test-scenarios_while-starting_test-1), but this time the result should be published -* same result as [test 1](#test-scenarios_while-starting_test-1). -* publish should be disabled and can be activated by opening every red item in the list - -### Test 3 -* same as [test 1](#test-scenarios_while-starting_test-1), but after refresh device B should close the penalty view -* after refresh device A should also see the red warning sign, without the possibility to publish -* publish should be disabled and can be activated by opening every red item in the list - -### Test 4 -* both devices open the penalty fragment -* device A gives Omar Air OCS, device B sets DNS for Alinghi -* device A closes the penalty view (without publishing) -* device B should see both entries in the list after refresh - -### Test 5 -* same as [test 4](#test-scenarios_while-starting_test-4), but this time the result should be published -* same result as [test 4](#test-scenarios_while-starting_test-4). - -### Test 6 -* same as [test 4](#test-scenarios_while-starting_test-4), but after refresh device B should publish the merged result -* after the refresh device A also sees both entries - -## While finishing - -### Test 7 -* both devices open result list -* device A adds Omar Air, Alinghi and LR BAR into the list -* device A closes the result list -* after refresh device B see the three competitors in the result list - -### Test 8 -* both devices open result list -* both devices add Omar Air to the result list -* device A sets the penalty to OCS and closes the result list -* device B should see after the refresh a yellow warning sign - -### Test 9 -* both devices open result list -* both devices add Omar Air to the result list -* both devices set the penalty to different values -* device A closes the result list -* device B should see a red warning sign and the data should show the values from device A -* publish should be disabled and can be activated by opening every red item in the list - -### Test 10 -* both devices open result list -* both devices add Omar Air to the result list -* device A changes to penalty view and sets the value to DNF and changes back to result list -* device B should see a yellow warning sign after refresh - -### Test 11 -* both devices open result list -* device A add Omar Air and Alinghi -* device B add Alinghi and Omar Air -* device A closes the result list -* device B should see red warning sign and the order Omar Air and Alinghi -* publish should be disabled and can be activated by opening every red item in the list - -### Test 12 -* both devices open result list -* device A add Omar Air and Alinghi -* device B add Omar Air and LR Bar -* device A closes the result list -* device B should see 1. Omar Air, 2. Alinghi and 2. LR Bar -* publish should be disabled and can be activated by opening every item in the list and clean the duplicate rankings - -### Test 13 -* devices open the penalty fragment -* device A give Omar Air OCS, device B set DNS for Omar Air -* device A pressed the home button -* device B should see a red warning sign after the automatic refresh -* publish should be disabled and can be activated by opening every red item in the list \ No newline at end of file diff --git a/wiki/howto/windestimation.md b/wiki/howto/windestimation.md index ec9611e268e..f9dce73efa0 100644 --- a/wiki/howto/windestimation.md +++ b/wiki/howto/windestimation.md @@ -73,7 +73,7 @@ For your account that is equipped with the ``TRACKED_RACE:EXPORT`` permission yo -m 10g --rm -d \ -e MONGODB_URI="mongodb://172.17.0.1/windestimation?retryWrites=true" \ -e BEARER_TOKEN="{your-bearer-token-here}" \ - -e MEMORY=-Xmx8g \ + -e MEMORY=-Xmx6g \ docker.sapsailing.com/windestimationtraining:latest ``` If successful (and you may want to remove the ``--rm`` option otherwise to allow you to inspect logs after unsuccessful execution) you will find the output under ``/tmp/windEstimationModels.dat`` which you can upload as usual, e.g., as in diff --git a/wiki/info/landscape/docker-registry.md b/wiki/info/landscape/docker-registry.md index da007f89b14..7e70fb35abc 100644 --- a/wiki/info/landscape/docker-registry.md +++ b/wiki/info/landscape/docker-registry.md @@ -90,3 +90,15 @@ http: The Hudson build slave AWS image (AMI) has a set of valid credentials in the ``hudson`` user's account to push to the registry. + +## Garbage-Collecting Unused Content + +To run a garbage collection in the registry, try this: +``` + docker exec -it registry-registry-1 registry garbage-collect /etc/docker/registry/config.yml +``` + +If you want to delete an entire repository, e.g., because you pushed images under an incorrect repository tag, try this: +``` + docker exec -it registry-registry-1 rm -rf /var/lib/registry/docker/registry/v2/repositories/{your-repository-name} +```