Merge remote-tracking branch 'origin/master'

This commit is contained in:
Axel Uhl
2023-01-09 21:19:33 +01:00
94 changed files with 3794 additions and 577 deletions
+3 -3
View File
@@ -46,13 +46,13 @@ Connect to your server at ``http://localhost:8888`` and find its administration
## Docker
To build a docker image, try ``docker/makeImageForLatestRelease``. The upload to the default (private) Dockerhub repository will usually fail unless you are a collaborator for that repository, but you should see a local image tagged ``docker.sapsailing.com:443/sapsailing:...`` result from the build. To run that docker image, try something like
To build a docker image, try ``docker/makeImageForLatestRelease``. The upload to the default (private) Dockerhub repository will usually fail unless you are a collaborator for that repository, but you should see a local image tagged ``docker.sapsailing.com/sapsailing:...`` result from the build. To run that docker image, try something like
```
docker run -d -e "MEMORY=4g" -e "MONGODB_URI=mongodb://my.mongohost.org?replicaSet=rs0&retryWrites=true" -P <yourimage>
```
Do a "docker ps" to figure out the port exposing the web application:
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 79f6faf19b6a docker.sapsailing.com:443/sapsailing:latest "/home/sailing/serve…" 33 seconds ago Up 32 seconds 0.0.0.0:32782->6666/tcp, 0.0.0.0:32781->7091/tcp, 0.0.0.0:32780->8000/tcp, 0.0.0.0:32779->8888/tcp, 0.0.0.0:32778->14888/tcp modest_dhawan
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 79f6faf19b6a docker.sapsailing.com/sapsailing:latest "/home/sailing/serve…" 33 seconds ago Up 32 seconds 0.0.0.0:32782->6666/tcp, 0.0.0.0:32781->7091/tcp, 0.0.0.0:32780->8000/tcp, 0.0.0.0:32779->8888/tcp, 0.0.0.0:32778->14888/tcp modest_dhawan
In this example, find your web application at http://localhost:32779 which is where the port 8888 exposed by the application is exposed at on your host. In the example with telnet port 14888 mapped to localhost:32788 do a
```
@@ -62,7 +62,7 @@ to connect to the server's OSGi console.
## Docker Compose
If you have built or obtained the ``docker.sapsailing.com:443/sapsailing:latest`` image, try this:
If you have built or obtained the ``docker.sapsailing.com/sapsailing:latest`` image, try this:
```
cd docker
docker-compose up
@@ -28,15 +28,10 @@ public enum ShardingType {
this.prefix = prefix;
}
public String encodeIfNeeded(String shardingInfo) {
if (shardingInfo.startsWith(prefix)) {
return shardingInfo;
}
return new StringBuilder().append(prefix)
.append(normalize(shardingInfo))
.toString();
public String encodeShardingInfo(String shardingInfo) {
return new StringBuilder().append(prefix).append(normalize(shardingInfo)).toString();
}
private String normalize(String replace) {
char[] chars = replace.toCharArray();
StringBuilder answer = new StringBuilder();
@@ -59,7 +59,7 @@ public class ShardingContext {
* @param shardingInfo
*/
public static void setShardingConstraint(ShardingType shardingType, String shardingInfo) {
final String encodedShardingInfo = shardingType.encodeIfNeeded(shardingInfo);
final String encodedShardingInfo = shardingType.encodeShardingInfo(shardingInfo);
ThreadLocal<String> shardingHolder = shardingMap.computeIfAbsent(shardingType, t -> new ThreadLocal<>());
checkAndSetShardingInfo(shardingType, encodedShardingInfo, shardingHolder);
}
@@ -89,7 +89,7 @@ public class ShardingContext {
logger.warning("No current sharding constraint for " + type.name());
return;
}
final String encodedShardingInfo = type.encodeIfNeeded(shardingInfo);
final String encodedShardingInfo = type.encodeShardingInfo(shardingInfo);
if (!encodedShardingInfo.equals(currentShardingInfo)) {
logger.log(Level.SEVERE, "Current sharding constraint vialation for " + type.name() + ". Got "
+ shardingInfo + ", shard requires " + currentShardingInfo, new RuntimeException());
@@ -89,6 +89,7 @@ Import-Package: javax.servlet;version="3.1.0",
org.osgi.framework,
org.osgi.util.tracker
Web-ContextPath: /gwt
Comment: The Web-ContextPath without the leading / must match com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants.WEB_CONTEXT_PATH
Bundle-ClassPath: WEB-INF/classes/
TrackLifecycle: true
Export-Package: com.google.gwt.user.client.rpc.core.com.sap.sailing.datamining.shared,
@@ -1,6 +1,6 @@
package com.sap.sailing.gwt.autoplay.client.app;
import static com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants.mediaServiceRemotePath;
import static com.sap.sailing.landscape.common.RemoteServiceMappingConstants.mediaServiceRemotePath;
import static com.sap.sse.common.HttpRequestHeaderConstants.HEADER_FORWARD_TO_MASTER;
import static com.sap.sse.common.HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA;
@@ -18,10 +18,10 @@ import com.sap.sailing.gwt.ui.client.MediaService;
import com.sap.sailing.gwt.ui.client.MediaServiceAsync;
import com.sap.sailing.gwt.ui.client.MediaServiceWrite;
import com.sap.sailing.gwt.ui.client.MediaServiceWriteAsync;
import com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants;
import com.sap.sailing.gwt.ui.client.SailingServiceAsync;
import com.sap.sailing.gwt.ui.client.SailingServiceHelper;
import com.sap.sailing.gwt.ui.client.SailingServiceWriteAsync;
import com.sap.sailing.landscape.common.RemoteServiceMappingConstants;
import com.sap.sse.gwt.client.EntryPointHelper;
import com.sap.sse.security.ui.client.SecureClientFactoryImpl;
@@ -11,6 +11,6 @@ public interface ProvidesLeaderboardRouting extends ServiceRoutingProvider {
String getLeaderboardName();
default String routingSuffixPath() {
return ShardingType.LEADERBOARDNAME.encodeIfNeeded(getLeaderboardName());
return ShardingType.LEADERBOARDNAME.encodeShardingInfo(getLeaderboardName());
}
}
@@ -1,6 +1,6 @@
package com.sap.sailing.gwt.home.communication;
import com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants;
import com.sap.sailing.landscape.common.RemoteServiceMappingConstants;
import com.sap.sse.gwt.dispatch.client.system.DispatchSystemDefaultImpl;
public class SailingDispatchSystemImpl extends DispatchSystemDefaultImpl<SailingDispatchContext> implements
@@ -13,10 +13,10 @@ import com.sap.sailing.gwt.home.desktop.app.DesktopClientFactory;
import com.sap.sailing.gwt.home.desktop.app.TabletAndDesktopApplicationClientFactory;
import com.sap.sailing.gwt.home.shared.SwitchingEntryPoint;
import com.sap.sailing.gwt.home.shared.app.ApplicationHistoryMapper;
import com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants;
import com.sap.sailing.gwt.ui.client.ServerConfigurationService;
import com.sap.sailing.gwt.ui.client.ServerConfigurationServiceAsync;
import com.sap.sailing.gwt.ui.client.StringMessages;
import com.sap.sailing.landscape.common.RemoteServiceMappingConstants;
import com.sap.sse.gwt.client.EntryPointHelper;
import com.sap.sse.gwt.client.mvp.AbstractMvpEntryPoint;
@@ -1,6 +1,6 @@
package com.sap.sailing.gwt.home.desktop.app;
import static com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants.mediaServiceRemotePath;
import static com.sap.sailing.landscape.common.RemoteServiceMappingConstants.mediaServiceRemotePath;
import static com.sap.sse.common.HttpRequestHeaderConstants.HEADER_FORWARD_TO_MASTER;
import static com.sap.sse.common.HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA;
@@ -13,12 +13,12 @@ import com.sap.sailing.gwt.home.shared.SwitchingEntryPoint;
import com.sap.sailing.gwt.home.shared.app.ApplicationHistoryMapper;
import com.sap.sailing.gwt.home.shared.app.ResettableNavigationPathDisplay;
import com.sap.sailing.gwt.home.shared.app.SailingActivityManager;
import com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants;
import com.sap.sailing.gwt.ui.client.SailingServiceAsync;
import com.sap.sailing.gwt.ui.client.SailingServiceHelper;
import com.sap.sailing.gwt.ui.client.ServerConfigurationService;
import com.sap.sailing.gwt.ui.client.ServerConfigurationServiceAsync;
import com.sap.sailing.gwt.ui.client.StringMessages;
import com.sap.sailing.landscape.common.RemoteServiceMappingConstants;
import com.sap.sse.gwt.client.EntryPointHelper;
import com.sap.sse.gwt.client.ServiceRoutingProvider;
import com.sap.sse.gwt.client.mvp.AbstractMvpEntryPoint;
@@ -10,9 +10,9 @@ import com.google.gwt.user.client.rpc.ServiceDefTarget;
import com.google.web.bindery.event.shared.EventBus;
import com.sap.sailing.gwt.ui.client.MediaServiceWrite;
import com.sap.sailing.gwt.ui.client.MediaServiceWriteAsync;
import com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants;
import com.sap.sailing.gwt.ui.client.SailingServiceWriteAsync;
import com.sap.sailing.gwt.ui.shared.ServerConfigurationDTO;
import com.sap.sailing.landscape.common.RemoteServiceMappingConstants;
import com.sap.sse.gwt.client.DefaultErrorReporter;
import com.sap.sse.gwt.client.EntryPointHelper;
import com.sap.sse.gwt.client.ErrorReporter;
@@ -1,6 +1,6 @@
package com.sap.sailing.gwt.ui.adminconsole;
import static com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants.mediaServiceRemotePath;
import static com.sap.sailing.landscape.common.RemoteServiceMappingConstants.mediaServiceRemotePath;
import static com.sap.sse.common.HttpRequestHeaderConstants.HEADER_FORWARD_TO_MASTER;
import static com.sap.sse.gwt.client.EntryPointHelper.registerASyncService;
@@ -1,29 +0,0 @@
package com.sap.sailing.gwt.ui.client;
/**
* Constants for remote services. The path value must fit with the corresponding path of the service in the web.xml The
* <code>@RemoteServiceRelativePath</code> annotation can't be used to automatically resolve the right path because it
* uses <code>GWT.getModuleBaseURL()</code> to calculate the path which is different for each EntryPoint.
*
* @author Frank
*
*/
public interface RemoteServiceMappingConstants {
/**
* The hosting bundle's web context path from the OSGi manifest. This is the URL prefix under which all services are
* registered based on their relative path specification in the <code>web.xml</code> descriptor.
*/
public static final String WEB_CONTEXT_PATH = "gwt";
public static final String mediaServiceRemotePath = "service/media";
public static final String sailingServiceRemotePath = "service/sailing";
public static final String serverConfigurationServiceRemotePath = "service/serverconfiguration";
public static final String simulatorServiceRemotePath = "service/simulator";
public static final String dataMiningServiceRemotePath = "service/datamining";
public static final String dispatchServiceRemotePath = "service/dispatch";
}
@@ -5,6 +5,7 @@ import static com.sap.sse.common.HttpRequestHeaderConstants.HEADER_FORWARD_TO_RE
import com.google.gwt.core.client.GWT;
import com.google.gwt.user.client.rpc.ServiceDefTarget;
import com.sap.sailing.landscape.common.RemoteServiceMappingConstants;
import com.sap.sse.gwt.client.EntryPointHelper;
import com.sap.sse.gwt.client.ServiceRoutingProvider;
@@ -2457,4 +2457,4 @@ goToSelfServicePortalDialogText=Self-Service Portal des Abonementanbieters in ei
failedFetchingSelfServicePortalSession=Self-Service Portal konnte nicht geöffnet werden.
oneAlwaysStaysOne=Wertung 1 bleibt 1, unabhängig vom Spalten-Faktor
otherTieBreakingLeaderboard=Andere Rangliste zum Tie-Breaking
helptextLinkingRaces=Um unverknüpfte Wettfahrten mit getrackten Rennen zu verknüpfen, müssen sie in der untenstehenden Tabelle eine Wettfahrt linksklicken und danach das entsprechende getrackte Rennen in der rechten Ansicht linksklicken.
helptextLinkingRaces=Um unverknüpfte Wettfahrten mit getrackten Rennen zu verknüpfen, müssen sie in der untenstehenden Tabelle eine Wettfahrt linksklicken und danach das entsprechende getrackte Rennen in der rechten Ansicht linksklicken.
@@ -28,9 +28,9 @@ import com.sap.sailing.gwt.common.authentication.FixedSailingAuthentication;
import com.sap.sailing.gwt.common.authentication.SAPSailingHeaderWithAuthentication;
import com.sap.sailing.gwt.common.client.help.HelpButton;
import com.sap.sailing.gwt.ui.client.AbstractSailingReadEntryPoint;
import com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants;
import com.sap.sailing.gwt.ui.datamining.presentation.TabbedSailingResultsPresenter;
import com.sap.sailing.gwt.ui.shared.settings.SailingSettingsConstants;
import com.sap.sailing.landscape.common.RemoteServiceMappingConstants;
import com.sap.sse.datamining.shared.DataMiningSession;
import com.sap.sse.datamining.shared.dto.StatisticQueryDefinitionDTO;
import com.sap.sse.datamining.shared.impl.UUIDDataMiningSession;
@@ -32,10 +32,10 @@ import com.sap.sailing.gwt.ui.client.MediaServiceAsync;
import com.sap.sailing.gwt.ui.client.MediaServiceWrite;
import com.sap.sailing.gwt.ui.client.MediaServiceWriteAsync;
import com.sap.sailing.gwt.ui.client.RaceTimesInfoProvider;
import com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants;
import com.sap.sailing.gwt.ui.client.StringMessages;
import com.sap.sailing.gwt.ui.shared.RaceWithCompetitorsAndBoatsDTO;
import com.sap.sailing.gwt.ui.shared.RaceboardDataDTO;
import com.sap.sailing.landscape.common.RemoteServiceMappingConstants;
import com.sap.sse.common.Util;
import com.sap.sse.gwt.client.EntryPointHelper;
import com.sap.sse.gwt.client.async.AsyncActionsExecutor;
@@ -15,9 +15,9 @@ import com.sap.sailing.domain.common.security.SecuredDomainType;
import com.sap.sailing.gwt.common.authentication.FixedSailingAuthentication;
import com.sap.sailing.gwt.common.authentication.SAPSailingHeaderWithAuthentication;
import com.sap.sailing.gwt.ui.client.AbstractSailingReadEntryPoint;
import com.sap.sailing.gwt.ui.client.RemoteServiceMappingConstants;
import com.sap.sailing.gwt.ui.client.SimulatorService;
import com.sap.sailing.gwt.ui.client.SimulatorServiceAsync;
import com.sap.sailing.landscape.common.RemoteServiceMappingConstants;
import com.sap.sailing.simulator.util.SailingSimulatorConstants;
import com.sap.sse.gwt.client.EntryPointHelper;
import com.sap.sse.gwt.client.ServerInfoDTO;
@@ -0,0 +1,37 @@
package com.sap.sailing.landscape.common;
/**
* Constants for remote services. The path value must fit with the corresponding path of the service in the web.xml The
* <code>@RemoteServiceRelativePath</code> annotation can't be used to automatically resolve the right path because it
* uses <code>GWT.getModuleBaseURL()</code> to calculate the path which is different for each EntryPoint.
*
* @author Frank
*
*/
public interface RemoteServiceMappingConstants {
/**
* The hosting bundle's web context path from the OSGi manifest's {@code Web-ContextPath} declaration. This is the
* URL prefix under which all services are registered based on their relative path specification in the
* <code>web.xml</code> descriptor.
*/
String WEB_CONTEXT_PATH = "gwt";
String mediaServiceRemotePath = "service/media";
String sailingServiceRemotePath = "service/sailing";
/**
* the prefix to use when constructing a load balancer listener rule's path condition from a sharding key.
* See also {@link ShardProcedure#getPathConditionForShardingKey(Object, String)} and
* {@link ShardProcedure#getShardingKeyFromPathCondition(String, String)}.
*/
String pathPrefixForShardingKey = "/"+WEB_CONTEXT_PATH+"/"+sailingServiceRemotePath;
String serverConfigurationServiceRemotePath = "service/serverconfiguration";
String simulatorServiceRemotePath = "service/simulator";
String dataMiningServiceRemotePath = "service/datamining";
String dispatchServiceRemotePath = "service/dispatch";
}
@@ -27,6 +27,7 @@ import com.sap.sailing.landscape.SailingAnalyticsHost;
import com.sap.sailing.landscape.SailingAnalyticsMetrics;
import com.sap.sailing.landscape.SailingAnalyticsProcess;
import com.sap.sailing.landscape.SailingReleaseRepository;
import com.sap.sailing.landscape.common.RemoteServiceMappingConstants;
import com.sap.sailing.landscape.common.SharedLandscapeConstants;
import com.sap.sailing.landscape.impl.BearerTokenReplicationCredentials;
import com.sap.sailing.landscape.impl.SailingAnalyticsHostImpl;
@@ -70,7 +71,7 @@ import software.amazon.awssdk.services.route53.model.RRType;
* Tests for the AWS SDK landscape wrapper in bundle {@code com.sap.sse.landscape.aws}. To run these tests
* successfully it is necessary to have valid AWS credentials for region {@code EU_WEST_2} that allow the
* AWS user account to create keys and launch instances, etc. These are to be provided as explained
* in the documentation of {@link AwsLandscape#obtain()}.
* in the documentation of {@link AwsLandscape#obtain(String)}.
*
* @author Axel Uhl (D043530)
*
@@ -94,7 +95,7 @@ public class TestProcedures {
@Before
public void setUp() {
privateKeyEncryptionPassphrase = ("awptyf87l"+"097384sf;,57").getBytes();
landscape = AwsLandscape.obtain();
landscape = AwsLandscape.obtain(RemoteServiceMappingConstants.pathPrefixForShardingKey);
region = new AwsRegion(Region.EU_WEST_2, landscape);
securityServiceReplicationBearerToken = System.getProperty(SECURITY_SERVICE_REPLICATION_BEARER_TOKEN);
mailSmtpPassword = System.getProperty(MAIL_SMTP_PASSWORD);
@@ -24,6 +24,7 @@ public class ApplicationReplicaSetsImagesBarCell extends ImagesBarCell {
public static final String ACTION_SWITCH_TO_REPLICA_ON_SHARED_INSTANCE = "ACTION_SWITCH_TO_REPLICA_ON_SHARED_INSTANCE";
public static final String ACTION_MOVE_MASTER_TO_OTHER_INSTANCE = "ACTION_MOVE_MASTER_TO_OTHER_INSTANCE";
public static final String ACTION_SCALE_AUTO_SCALING_REPLICAS_UP_DOWN = "ACTION_SCALE_AUTO_SCALING_REPLICAS_UP_DOWN";
public static final String ACTION_OPEN_SHARD_MANAGEMENT = "ACTION_OPEN_SHARD_MANAGEMENT";
private final StringMessages stringMessages;
private final UserService userService;
@@ -63,6 +64,7 @@ public class ApplicationReplicaSetsImagesBarCell extends ImagesBarCell {
if (applicationReplicaSet.getAutoScalingGroupAmiId() != null) {
result.add(new ImageSpec(ACTION_UPDATE_AMI_FOR_AUTO_SCALING_REPLICAS, stringMessages.updateAmiForAutoScalingReplicas(), IconResources.INSTANCE.redGearsIcon()));
result.add(new ImageSpec(ACTION_SWITCH_TO_AUTO_SCALING_REPLICAS_ONLY, stringMessages.switchToAutoScalingReplicasOnly(), IconResources.INSTANCE.scaleUpIcon()));
result.add(new ImageSpec(ACTION_OPEN_SHARD_MANAGEMENT, stringMessages.openShardManagement(), IconResources.INSTANCE.shardManagementIcon()));
}
result.add(new ImageSpec(ACTION_SWITCH_TO_REPLICA_ON_SHARED_INSTANCE, stringMessages.switchToReplicaOnSharedInstance(), IconResources.INSTANCE.scaleDownIcon()));
if (!applicationReplicaSet.isLocalReplicaSet(userService)) {
@@ -15,7 +15,6 @@ import java.util.function.Function;
import com.google.gwt.cell.client.SafeHtmlCell;
import com.google.gwt.core.client.GWT;
import com.google.gwt.safehtml.shared.SafeHtml;
import com.google.gwt.safehtml.shared.SafeHtmlBuilder;
import com.google.gwt.user.cellview.client.Column;
import com.google.gwt.user.cellview.client.TextColumn;
import com.google.gwt.user.client.Timer;
@@ -263,11 +262,7 @@ public class LandscapeManagementPanel extends SimplePanel {
final Column<SailingApplicationReplicaSetDTO<String>, SafeHtml> versionColumn = new Column<SailingApplicationReplicaSetDTO<String>, SafeHtml>(versionCell) {
@Override
public SafeHtml getValue(SailingApplicationReplicaSetDTO<String> replicaSet) {
final SafeHtmlBuilder builder = new SafeHtmlBuilder();
final String version = replicaSet.getVersion();
final String releaseNotesLink = getReleaseNotesLink(version);
appendEc2Link(builder, releaseNotesLink, version);
return builder.toSafeHtml();
return new LinkBuilder().setReplicaSet(replicaSet).setPathMode(LinkBuilder.pathModes.Version).build();
}
};
applicationReplicaSetsTable.addColumn(versionColumn, stringMessages.versionHeader(), (rs1, rs2)->new NaturalComparator().compare(rs1.getVersion(), rs2.getVersion()));
@@ -275,12 +270,7 @@ public class LandscapeManagementPanel extends SimplePanel {
final Column<SailingApplicationReplicaSetDTO<String>, SafeHtml> masterColumn = new Column<SailingApplicationReplicaSetDTO<String>, SafeHtml>(masterCell) {
@Override
public SafeHtml getValue(SailingApplicationReplicaSetDTO<String> replicaSet) {
final SafeHtmlBuilder builder = new SafeHtmlBuilder();
final String gwtStatusLink = getGwtStatusLink(replicaSet.getMaster().getHost().getPublicIpAddress(), replicaSet.getMaster().getPort());
builder.appendHtmlConstant("<a target=\"_blank\" href=\""+gwtStatusLink+"\">");
builder.appendEscaped(replicaSet.getMaster().getHost().getPublicIpAddress());
builder.appendHtmlConstant("</a>");
return builder.toSafeHtml();
return new LinkBuilder().setPathMode(LinkBuilder.pathModes.MasterHost).setReplicaSet(replicaSet).build();
}
};
applicationReplicaSetsTable.addColumn(masterColumn, stringMessages.masterHostName(), (rs1, rs2)->new NaturalComparator().compare(rs1.getMaster().getHost().getPublicIpAddress(), rs2.getMaster().getHost().getPublicIpAddress()));
@@ -289,23 +279,15 @@ public class LandscapeManagementPanel extends SimplePanel {
final Column<SailingApplicationReplicaSetDTO<String>, SafeHtml> hostnameColumn = new Column<SailingApplicationReplicaSetDTO<String>, SafeHtml>(hostnameCell) {
@Override
public SafeHtml getValue(SailingApplicationReplicaSetDTO<String> replicaSet) {
final SafeHtmlBuilder builder = new SafeHtmlBuilder();
final String hostnameLink = "https://"+replicaSet.getHostname();
builder.appendHtmlConstant("<a target=\"_blank\" href=\""+hostnameLink+"\">");
builder.appendEscaped(replicaSet.getHostname());
builder.appendHtmlConstant("</a>");
return builder.toSafeHtml();
}
return new LinkBuilder().setReplicaSet(replicaSet).setPathMode(LinkBuilder.pathModes.Hostname).build()
; }
};
applicationReplicaSetsTable.addColumn(hostnameColumn, stringMessages.hostname(), (rs1, rs2)->new NaturalComparator().compare(rs1.getHostname(), rs2.getMaster().getHostname()));
final SafeHtmlCell masterInstanceIdCell = new SafeHtmlCell();
final Column<SailingApplicationReplicaSetDTO<String>, SafeHtml> masterInstanceIdColumn = new Column<SailingApplicationReplicaSetDTO<String>, SafeHtml>(masterInstanceIdCell) {
@Override
public SafeHtml getValue(SailingApplicationReplicaSetDTO<String> replicaSet) {
final SafeHtmlBuilder builder = new SafeHtmlBuilder();
final String instanceId = replicaSet.getMaster().getHost().getInstanceId();
appendEc2InstanceLink(builder, instanceId);
return builder.toSafeHtml();
return new LinkBuilder().setRegion(regionsTable.getSelectionModel().getSelectedObject()).setInstanceId(replicaSet.getMaster().getHost().getInstanceId()).setPathMode(LinkBuilder.pathModes.InstanceSearch).build();
}
};
applicationReplicaSetsTable.addColumn(masterInstanceIdColumn, stringMessages.masterInstanceId(),
@@ -316,21 +298,10 @@ public class LandscapeManagementPanel extends SimplePanel {
final Column<SailingApplicationReplicaSetDTO<String>, SafeHtml> replicasColumn = new Column<SailingApplicationReplicaSetDTO<String>, SafeHtml>(replicasCell) {
@Override
public SafeHtml getValue(SailingApplicationReplicaSetDTO<String> replicaSet) {
SafeHtmlBuilder builder = new SafeHtmlBuilder();
for (final SailingAnalyticsProcessDTO replica : replicaSet.getReplicas()) {
final String gwtStatusLink = getGwtStatusLink(replica.getHost().getPublicIpAddress(), replica.getPort());
builder.appendHtmlConstant("<a target=\"_blank\" href=\""+gwtStatusLink+"\">");
builder.appendEscaped(replica.getHost().getPublicIpAddress()+":"+replica.getPort());
builder.appendHtmlConstant("</a>");
final String replicaInstanceId = replica.getHost().getInstanceId();
builder.appendEscaped(" (");
builder.appendEscaped(replica.getServerName());
builder.appendEscaped(", ");
appendEc2InstanceLink(builder, replicaInstanceId);
builder.appendEscaped(")");
builder.appendHtmlConstant("<br>");
}
return builder.toSafeHtml();
LinkBuilder linkBuilder = new LinkBuilder();
linkBuilder.setReplicaSet(replicaSet);
linkBuilder.setPathMode(LinkBuilder.pathModes.ReplicaLinks).setRegion(regionsTable.getSelectionModel().getSelectedObject());
return linkBuilder.build();
}
};
applicationReplicaSetsTable.addColumn(replicasColumn, stringMessages.replicas());
@@ -339,11 +310,7 @@ public class LandscapeManagementPanel extends SimplePanel {
final Column<SailingApplicationReplicaSetDTO<String>, SafeHtml> autoScalingGroupAmiIdColumn = new Column<SailingApplicationReplicaSetDTO<String>, SafeHtml>(autoScalingGroupAmiIdCell) {
@Override
public SafeHtml getValue(SailingApplicationReplicaSetDTO<String> replicaSet) {
final SafeHtmlBuilder builder = new SafeHtmlBuilder();
if (replicaSet.getAutoScalingGroupAmiId() != null) {
appendEc2AmiLink(builder, replicaSet.getAutoScalingGroupAmiId());
}
return builder.toSafeHtml();
return new LinkBuilder().setReplicaSet(replicaSet).setRegion(regionsTable.getSelectionModel().getSelectedObject()).setPathMode(LinkBuilder.pathModes.AmiSearch).build();
}
};
applicationReplicaSetsTable.addColumn(autoScalingGroupAmiIdColumn, stringMessages.machineImageId(), (rs1, rs2)->new NaturalComparator().compare(rs1.getAutoScalingGroupAmiId(), rs2.getAutoScalingGroupAmiId()));
@@ -380,7 +347,10 @@ public class LandscapeManagementPanel extends SimplePanel {
regionsTable.getSelectionModel().getSelectedObject(), Collections.singleton(applicationReplicaSetToUpgrade)));
applicationReplicaSetsActionColumn.addAction(ApplicationReplicaSetsImagesBarCell.ACTION_SCALE_AUTO_SCALING_REPLICAS_UP_DOWN,
applicationReplicaSetToUpgrade -> scaleAutoScalingReplicasUpDown(stringMessages,
regionsTable.getSelectionModel().getSelectedObject(), Collections.singleton(applicationReplicaSetToUpgrade)));
regionsTable.getSelectionModel().getSelectedObject(),
Collections.singleton(applicationReplicaSetToUpgrade)));
applicationReplicaSetsActionColumn.addAction(ApplicationReplicaSetsImagesBarCell.ACTION_OPEN_SHARD_MANAGEMENT,
selectedReplicaSet -> openShardManagementPanel(stringMessages, regionsTable.getSelectionModel().getSelectedObject(), selectedReplicaSet));
// see below for the finalization o the applicationRelicaSetsActionColumn; we need to have the machineImagesTable ready for the last action...
final CaptionPanel applicationReplicaSetsCaptionPanel = new CaptionPanel(stringMessages.applicationReplicaSets());
final VerticalPanel applicationReplicaSetsVerticalPanel = new VerticalPanel();
@@ -489,13 +459,13 @@ public class LandscapeManagementPanel extends SimplePanel {
AsyncCallback<Boolean> validatePassphraseCallback = new AsyncCallback<Boolean>() {
@Override
public void onSuccess(Boolean result) {
sshKeyManagementPanel.setPassphraseValidation(result.booleanValue(),stringMessages);
sshKeyManagementPanel.setPassphraseValidation(result.booleanValue(), stringMessages);
addApplicationReplicaSetButton.setVisible(result);
applicationReplicaSetsRefreshButton.setVisible(result);
applicationReplicaSetsCaptionPanel.setVisible(result);
machineImagesCaptionPanel.setVisible(result);
mongoEndpointsCaptionPanel.setVisible(result);
if(result) {
if (result) {
refreshApplicationReplicaSetsTable();
}
}
@@ -505,17 +475,34 @@ public class LandscapeManagementPanel extends SimplePanel {
};
};
sshKeyManagementPanel.addSshKeySelectionChangedHandler(event->{
validatePassphrase(stringMessages,validatePassphraseCallback);
validatePassphrase(stringMessages, validatePassphraseCallback);
});
sshKeyManagementPanel.addOnPassphraseChangedListener(event -> {
validatePassphrase(stringMessages,validatePassphraseCallback);
validatePassphrase(stringMessages, validatePassphraseCallback);
});
validatePassphrase(stringMessages,validatePassphraseCallback);
validatePassphrase(stringMessages, validatePassphraseCallback);
// TODO try to identify archive servers
// TODO support archive server upgrade
// TODO upon region selection show RabbitMQ, and Central Reverse Proxy clusters in region
}
private void openShardManagementPanel(StringMessages stringMessages, String region, SailingApplicationReplicaSetDTO<String> replicaset) {
new ShardManagementDialog(landscapeManagementService, replicaset, region, sshKeyManagementPanel.getPassphraseForPrivateKeyDecryption(), errorReporter, stringMessages,
new DialogCallback<Boolean>() {
@Override
public void ok(Boolean hasAnythingChanged) {
if (hasAnythingChanged) {
refreshApplicationReplicaSetsTable();
}
}
@Override
public void cancel() {
// there is no cancel button
}
}).show();
}
private void disableButtonWhenLocalReplicaSetIsSelected(Button button, UserService userService) {
applicationReplicaSetsTable.getSelectionModel().addSelectionChangeHandler(e->button.setEnabled(
!applicationReplicaSetsTable.getSelectionModel().getSelectedSet().stream().filter(arsDTO->arsDTO.isLocalReplicaSet(userService)).findAny().isPresent()));
@@ -587,7 +574,7 @@ public class LandscapeManagementPanel extends SimplePanel {
@Override
public void onFailure(Throwable caught) {
errorReporter.reportError(caught.getMessage());
errorReporter.reportError(caught.getMessage() == null ? caught.getClass().getName() : caught.getMessage());
if (!replicaSetIterator.hasNext()) {
applicationReplicaSetsBusy.setBusy(false);
} else {
@@ -762,14 +749,6 @@ public class LandscapeManagementPanel extends SimplePanel {
});
}
private String getGwtStatusLink(final String host, int port) {
return (port == 443 ? "https" : "http") + "://" + host + ":" + port + "/gwt/status";
}
private String getReleaseNotesLink(final String version) {
return "https://releases.sapsailing.com/"+version+"/release-notes.txt";
}
private void ensureAtLeastOneReplicaExistsStopReplicatingAndRemoveMasterFromTargetGroups(
StringMessages stringMessages, String selectedObject,
Iterable<SailingApplicationReplicaSetDTO<String>> applicationReplicaSetsForWhichToEnsureAtLeastOneReplicaStopReplicatingAndRemoveMasterFromTargetGroups) {
@@ -1496,33 +1475,4 @@ public class LandscapeManagementPanel extends SimplePanel {
RemoteServiceMappingConstants.landscapeManagementServiceRemotePath, HEADER_FORWARD_TO_MASTER);
return result;
}
private String getEc2ConsoleLinkForInstanceId(String instanceId) {
return getEc2ConsoleBaseUrlForSelectedRegion()+"#Instances:search="+instanceId;
}
private void appendEc2InstanceLink(final SafeHtmlBuilder builder, final String instanceId) {
final String ec2Link = getEc2ConsoleLinkForInstanceId(instanceId);
appendEc2Link(builder, ec2Link, instanceId);
}
private void appendEc2Link(final SafeHtmlBuilder builder, final String ec2Link, final String text) {
builder.appendHtmlConstant("<a target=\"_blank\" href=\""+ec2Link+"\">");
builder.appendEscaped(text);
builder.appendHtmlConstant("</a>");
}
private String getEc2ConsoleBaseUrlForSelectedRegion() {
return "https://"+regionsTable.getSelectionModel().getSelectedObject()+
".console.aws.amazon.com/ec2/v2/home?region="+regionsTable.getSelectionModel().getSelectedObject();
}
private String getEc2ConsoleLinkForAmiId(String amiId) {
return getEc2ConsoleBaseUrlForSelectedRegion()+"#Images:imageId="+amiId;
}
private void appendEc2AmiLink(final SafeHtmlBuilder builder, final String amiId) {
final String ec2Link = getEc2ConsoleLinkForAmiId(amiId);
appendEc2Link(builder, ec2Link, amiId);
}
}
@@ -1,12 +1,15 @@
package com.sap.sailing.landscape.ui.client;
import java.util.ArrayList;
import java.util.Map;
import com.google.gwt.user.client.rpc.RemoteService;
import com.sap.sailing.domain.common.DataImportProgress;
import com.sap.sailing.landscape.ui.shared.AmazonMachineImageDTO;
import com.sap.sailing.landscape.ui.shared.AwsInstanceDTO;
import com.sap.sailing.landscape.ui.shared.AwsShardDTO;
import com.sap.sailing.landscape.ui.shared.CompareServersResultDTO;
import com.sap.sailing.landscape.ui.shared.LeaderboardNameDTO;
import com.sap.sailing.landscape.ui.shared.MongoEndpointDTO;
import com.sap.sailing.landscape.ui.shared.MongoScalingInstructionsDTO;
import com.sap.sailing.landscape.ui.shared.ProcessDTO;
@@ -70,8 +73,8 @@ public interface LandscapeManagementWriteService extends RemoteService {
ArrayList<SailingApplicationReplicaSetDTO<String>> getApplicationReplicaSets(String regionId,
String optionalKeyName, byte[] privateKeyEncryptionPassphrase) throws Exception;
SerializationDummyDTO serializationDummy(ProcessDTO mongoProcessDTO, AwsInstanceDTO awsInstanceDTO,
SailingApplicationReplicaSetDTO<String> sailingApplicationReplicationSetDTO);
SerializationDummyDTO serializationDummy(ProcessDTO mongoProcessDTO, AwsInstanceDTO awsInstanceDTO, AwsShardDTO shardDTO,
SailingApplicationReplicaSetDTO<String> sailingApplicationReplicationSetDTO, LeaderboardNameDTO leaderboard);
SailingApplicationReplicaSetDTO<String> createApplicationReplicaSet(String regionId, String name, boolean sharedMasterInstance,
String masterInstanceType, String optionalReplicaInstanceTypeOrNull, boolean dynamicLoadBalancerMapping,
@@ -141,4 +144,17 @@ public interface LandscapeManagementWriteService extends RemoteService {
SailingApplicationReplicaSetDTO<String> changeAutoScalingReplicasInstanceType(
SailingApplicationReplicaSetDTO<String> replicaSet, String instanceTypeName,
String optionalKeyName, byte[] privateKeyEncryptionPassphrase) throws Exception;
ArrayList<LeaderboardNameDTO> getLeaderboardNames(SailingApplicationReplicaSetDTO<String> replicaSet, String bearerToken) throws Exception;
void addShard(String shardName, ArrayList<LeaderboardNameDTO> selectedLeaderBoardNames, SailingApplicationReplicaSetDTO<String> replicaSet,
String bearerToken, String region, byte[] passphraseForPrivateKeyDecryption) throws Exception;
public Map<AwsShardDTO, Iterable<String>> getShards(SailingApplicationReplicaSetDTO<String> replicaSet, String region, String bearerToken) throws Exception;
public void removeShard(AwsShardDTO shard, SailingApplicationReplicaSetDTO<String> replicaSet, String region, byte[] passphrase) throws Exception;
void appendShardingKeysToShard(Iterable<LeaderboardNameDTO> shardingKeysToAppend, String region, String shardName, SailingApplicationReplicaSetDTO<String> replicaSet, String bearerToken, byte[] passphraseForPrivateKeyDecryption) throws Exception;
void removeShardingKeysFromShard(Iterable<LeaderboardNameDTO> shardingKeysToRemove, String region, String shardName, SailingApplicationReplicaSetDTO<String> replicaSet, String bearerToken, byte[] passphraseForPrivateKeyDecryption) throws Exception;
}
@@ -1,13 +1,16 @@
package com.sap.sailing.landscape.ui.client;
import java.util.ArrayList;
import java.util.Map;
import com.google.gwt.user.client.rpc.AsyncCallback;
import com.sap.sailing.domain.common.DataImportProgress;
import com.sap.sailing.landscape.common.SharedLandscapeConstants;
import com.sap.sailing.landscape.ui.shared.AmazonMachineImageDTO;
import com.sap.sailing.landscape.ui.shared.AwsInstanceDTO;
import com.sap.sailing.landscape.ui.shared.AwsShardDTO;
import com.sap.sailing.landscape.ui.shared.CompareServersResultDTO;
import com.sap.sailing.landscape.ui.shared.LeaderboardNameDTO;
import com.sap.sailing.landscape.ui.shared.MongoEndpointDTO;
import com.sap.sailing.landscape.ui.shared.MongoScalingInstructionsDTO;
import com.sap.sailing.landscape.ui.shared.ProcessDTO;
@@ -104,8 +107,8 @@ public interface LandscapeManagementWriteServiceAsync {
Integer minimumAutoScalingGroupSizeOrNull, Integer maximumAutoScalingGroupSizeOrNull,
AsyncCallback<SailingApplicationReplicaSetDTO<String>> callback);
void serializationDummy(ProcessDTO mongoProcessDTO, AwsInstanceDTO awsInstanceDTO,
SailingApplicationReplicaSetDTO<String> sailingApplicationReplicationSetDTO,
void serializationDummy(ProcessDTO mongoProcessDTO, AwsInstanceDTO awsInstanceDTO, AwsShardDTO shardDTO,
SailingApplicationReplicaSetDTO<String> sailingApplicationReplicationSetDTO, LeaderboardNameDTO leaderboard,
AsyncCallback<SerializationDummyDTO> callback);
void defineDefaultRedirect(String regionId, String hostname, RedirectDTO redirect, String keyName,
@@ -193,4 +196,84 @@ public interface LandscapeManagementWriteServiceAsync {
void changeAutoScalingReplicasInstanceType(SailingApplicationReplicaSetDTO<String> replicaSet,
String instanceTypeName, String optionalKeyName, byte[] privateKeyEncryptionPassphrase,
AsyncCallback<SailingApplicationReplicaSetDTO<String>> callback);
void getLeaderboardNames(SailingApplicationReplicaSetDTO<String> replicaSet, String bearerToken,
AsyncCallback<ArrayList<LeaderboardNameDTO>> names);
void addShard(String shardName, ArrayList<LeaderboardNameDTO> selectedLeaderBoards,
SailingApplicationReplicaSetDTO<String> replicaSet, String bearerToken, String region,
byte[] passphraseForPrivateKeyDecryption, AsyncCallback<Void> callback);
/**
* @param callback
* returns the shards as keys and the sharding keys (escaped / mangled leaderboard names) as values.
* Those sharding keys are what you get when mangling the {@link AwsShardDTO#getLeaderboardNames()
* leaderboard names} of the shard.
*/
void getShards(SailingApplicationReplicaSetDTO<String> replicaset, String region, String bearerToken,
AsyncCallback<Map<AwsShardDTO, Iterable<String>>> callback);
/**
* Removes {@code shard} from the replica set. This deletes the load balancer listener rules, and the auto scaling
* group and the target group.
*
* @param shard
* the shard to remove
* @param replicaSet
* the replica set which contains the shard.
* @param region
* replica set's region
* @param passphrase
* passphrase for the private key decription.
* @param callback
*
*/
public void removeShard(AwsShardDTO shard, SailingApplicationReplicaSetDTO<String> replicaSet, String region,
byte[] passphrase, AsyncCallback<Void> callback);
/**
* Appends sharding keys for each leader board in {@code selectedLeaderboards} to the shard, identified by
* {@code shardName}, from the {@code replicaset}. This function inserts rules to the replica set's load balancer
* for each {@selectedLeaderboards}'s sharding key. It the replica set's load balancer does not have enough rules
* left, a new one gets created. For inserting the rules, first every existing rule of this shard get's checked for
* space left and if there is, it gets filled with a sharding key and after that new rules are created. Throws an
* Exception if: - the shard is not found. - shards cannot be retrived - sharding rules cannot be inserted - the is
* no free load balancer or the process of moving the replica set to another load balancer failed.
*
* @param selectedLeaderBoards
* list of selected leaderboards. These are the names and not sharding keys.
* @param region
* landscape region
* @param shardName
* shard's name where the keys are supposed to be appended
* @param replicaSet
* shard's replica set
* @param bearerToken
* @param passphraseForPrivateKeyDecryption
* @param callback
*/
void appendShardingKeysToShard(Iterable<LeaderboardNameDTO> selectedLeaderBoards, String region, String shardName,
SailingApplicationReplicaSetDTO<String> replicaSet, String bearerToken,
byte[] passphraseForPrivateKeyDecryption, AsyncCallback<Void> callback);
/**
* Removes the shardingkeys for {@selectedLeaderBoards} from a shard, identified by {@code shardName} from
* {@code replicaset}. Throws Exception if: - no shard is found - replicaset is not found - shards from replica set
* cannot be retrieved. - sharding rules cannot be updated
*
* @param selectedLeaderBoards
* Sharding keys for all selected leader boards.
* @param region
* shard's regio
* @param shardName
* Shard's name where the keys should be removed from
* @param replicaSet
* replica set which contains the shard
* @param bearerToken
* @param passphraseForPrivateKeyDecryption
* @param callback
*/
void removeShardingKeysFromShard(Iterable<LeaderboardNameDTO> selectedLeaderBoards, String region, String shardName,
SailingApplicationReplicaSetDTO<String> replicaSet, String bearerToken,
byte[] passphraseForPrivateKeyDecryption, AsyncCallback<Void> callback);
}
@@ -0,0 +1,190 @@
package com.sap.sailing.landscape.ui.client;
import com.google.gwt.safehtml.shared.SafeHtml;
import com.google.gwt.safehtml.shared.SafeHtmlBuilder;
import com.sap.sailing.landscape.ui.shared.SailingAnalyticsProcessDTO;
import com.sap.sailing.landscape.ui.shared.SailingApplicationReplicaSetDTO;
import com.sap.sse.common.Builder;
public class LinkBuilder implements Builder<LinkBuilder, SafeHtml> {
static public enum pathModes {
InstanceSearch, ImageSearch, AmiSearch, Hostname, ReplicaLinks, Version, MasterHost, TargetGroupSearch, AutoScalingGroupSearch
};
private pathModes pathMode;
private String region;
private String instanceId;
private SailingApplicationReplicaSetDTO<String> replicaSet;
private String targetGroupName;
private String autoScalingGroupName;
LinkBuilder setPathMode(pathModes mode) {
pathMode = mode;
return self();
}
LinkBuilder setTargetGroupName(String name) {
this.targetGroupName = name;
return self();
}
LinkBuilder setAutoScalingGroupName(String name) {
this.autoScalingGroupName = name;
return self();
}
LinkBuilder setInstanceId(String instanceId) {
this.instanceId = instanceId;
return self();
}
LinkBuilder setRegion(String region) {
this.region = region;
return self();
}
LinkBuilder setReplicaSet(SailingApplicationReplicaSetDTO<String> replicaSet) {
this.replicaSet = replicaSet;
return self();
}
private String getEc2ConsoleLinkForInstanceId(String instanceId) {
return getEc2ConsoleBaseUrlForSelectedRegion() + "#Instances:search=" + instanceId;
}
private String getEc2ConsoleLinkForTargetGroupName(String name) {
return getEc2ConsoleBaseUrlForSelectedRegion() + "#TargetGroups:search=" + name;
}
private String getEc2ConsoleLinkForAutoScalingGroupName(String name) {
return getEc2ConsoleBaseUrlForSelectedRegion() + "#AutoScalingGroupDetails:id="+name+";view=details";
}
private String getEc2ConsoleLinkForAmiId(String amiId) {
return getEc2ConsoleBaseUrlForSelectedRegion() + "#Images:visibility=owned-by-me;search=" + amiId;
}
private String getEc2ConsoleBaseUrlForSelectedRegion() {
return "https://" + region + ".console.aws.amazon.com/ec2/v2/home?region=" + region;
}
private void appendEc2Link(final SafeHtmlBuilder builder, final String ec2Link, final String text) {
builder.appendHtmlConstant("<a target=\"_blank\" href=\"" + ec2Link + "\">");
builder.appendEscaped(text);
builder.appendHtmlConstant("</a>");
}
private void appendEc2InstanceLink(final SafeHtmlBuilder builder, final String instanceId) {
final String ec2Link = getEc2ConsoleLinkForInstanceId(instanceId);
appendEc2Link(builder, ec2Link, instanceId);
}
private void appendEc2AmiLink(final SafeHtmlBuilder builder, final String amiId) {
final String ec2Link = getEc2ConsoleLinkForAmiId(amiId);
appendEc2Link(builder, ec2Link, amiId);
}
private void appendEc2TargetGroupLink(final SafeHtmlBuilder builder, final String name) {
final String ec2Link = getEc2ConsoleLinkForTargetGroupName(name);
appendEc2Link(builder, ec2Link, name);
}
private void appendEc2AutoScalingGroupLink(final SafeHtmlBuilder builder, final String name) {
final String ec2Link = getEc2ConsoleLinkForAutoScalingGroupName(name);
appendEc2Link(builder, ec2Link, name);
}
private String getGwtStatusLink(final String host, int port) {
return (port == 443 ? "https" : "http") + "://" + host + ":" + port + "/gwt/status";
}
private String getReleaseNotesLink(final String version) {
return "https://releases.sapsailing.com/" + version + "/release-notes.txt";
}
private void checkAttribute(Object attr, String name) throws Exception {
if (attr == null) {
throw new Exception(name + " needs to be given!");
}
}
@Override
public SafeHtml build() {
SafeHtmlBuilder builder = new SafeHtmlBuilder();
try {
switch (pathMode) {
case ReplicaLinks:
checkAttribute(replicaSet, "Replicaset");
for (final SailingAnalyticsProcessDTO replica : replicaSet.getReplicas()) {
final String gwtStatusLink = getGwtStatusLink(replica.getHost().getPublicIpAddress(),
replica.getPort());
builder.appendHtmlConstant("<a target=\"_blank\" href=\"" + gwtStatusLink + "\">");
builder.appendEscaped(replica.getHost().getPublicIpAddress() + ":" + replica.getPort());
builder.appendHtmlConstant("</a>");
final String replicaInstanceId = replica.getHost().getInstanceId();
builder.appendEscaped(" (");
builder.appendEscaped(replica.getServerName());
builder.appendEscaped(", ");
appendEc2InstanceLink(builder, replicaInstanceId);
builder.appendEscaped(")");
builder.appendHtmlConstant("<br>");
}
break;
case AmiSearch:
checkAttribute(replicaSet, "Replicaset");
checkAttribute(region, "Region");
if (replicaSet.getAutoScalingGroupAmiId() != null) {
appendEc2AmiLink(builder, replicaSet.getAutoScalingGroupAmiId());
}
break;
case Hostname:
checkAttribute(replicaSet, "Replicaset");
final String hostnameLink = "https://" + replicaSet.getHostname();
builder.appendHtmlConstant("<a target=\"_blank\" href=\"" + hostnameLink + "\">");
builder.appendEscaped(replicaSet.getHostname());
builder.appendHtmlConstant("</a>");
break;
case ImageSearch:
break;
case InstanceSearch:
checkAttribute(region, "Region");
appendEc2InstanceLink(builder, instanceId);
break;
case Version:
checkAttribute(replicaSet, "Replicaset");
final String version = replicaSet.getVersion();
final String releaseNotesLink = getReleaseNotesLink(version);
appendEc2Link(builder, releaseNotesLink, version);
break;
case MasterHost:
checkAttribute(replicaSet, "Replicaset");
final String gwtStatusLink = getGwtStatusLink(replicaSet.getMaster().getHost().getPublicIpAddress(),
replicaSet.getMaster().getPort());
builder.appendHtmlConstant("<a target=\"_blank\" href=\"" + gwtStatusLink + "\">");
builder.appendEscaped(replicaSet.getMaster().getHost().getPublicIpAddress());
builder.appendHtmlConstant("</a>");
break;
case TargetGroupSearch:
checkAttribute(region, "Region");
checkAttribute(targetGroupName, "Target Group Name");
appendEc2TargetGroupLink(builder, targetGroupName);
break;
case AutoScalingGroupSearch:
checkAttribute(region, "Region");
checkAttribute(autoScalingGroupName, "Auto-Scaling Group Name");
appendEc2AutoScalingGroupLink(builder, autoScalingGroupName);
break;
default:
break;
}
} catch (Exception e) {
builder.appendHtmlConstant("<a target=\"_blank\" >");
builder.appendEscaped(e.getMessage());
builder.appendHtmlConstant("</a>");
}
return builder.toSafeHtml();
}
}
@@ -0,0 +1,32 @@
package com.sap.sailing.landscape.ui.client;
import com.google.gwt.user.client.ui.Widget;
import com.sap.sailing.landscape.ui.client.i18n.StringMessages;
import com.sap.sailing.landscape.ui.shared.SailingApplicationReplicaSetDTO;
import com.sap.sse.gwt.client.ErrorReporter;
import com.sap.sse.gwt.client.dialog.DataEntryDialog;
public class ShardManagementDialog extends DataEntryDialog<Boolean> {
private final ShardManagementPanel shardPanel;
ShardManagementDialog(LandscapeManagementWriteServiceAsync landscapeManagementWriteServiceAsync,
SailingApplicationReplicaSetDTO<String> applicastionReplicaSet, String region, String passphrase,
ErrorReporter errorReporter, StringMessages stringMessages, DialogCallback<Boolean> callback) {
super(stringMessages.shard(), stringMessages.shardingDescription(), stringMessages.close(), /* no cancel button */ null, /* validator */ null, callback);
shardPanel = new ShardManagementPanel(landscapeManagementWriteServiceAsync, errorReporter, stringMessages);
shardPanel.setRegion(region);
shardPanel.setPassphrase(passphrase);
shardPanel.setReplicaSet(applicastionReplicaSet);
shardPanel.refresh();
}
@Override
protected Widget getAdditionalWidget() {
return shardPanel;
}
@Override
protected Boolean getResult() {
return shardPanel.hasAnythingChanged();
}
}
@@ -0,0 +1,422 @@
package com.sap.sailing.landscape.ui.client;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import com.google.gwt.cell.client.SafeHtmlCell;
import com.google.gwt.core.client.GWT;
import com.google.gwt.safehtml.shared.SafeHtml;
import com.google.gwt.user.cellview.client.Column;
import com.google.gwt.user.client.rpc.AsyncCallback;
import com.google.gwt.user.client.ui.Button;
import com.google.gwt.user.client.ui.CaptionPanel;
import com.google.gwt.user.client.ui.FocusWidget;
import com.google.gwt.user.client.ui.Grid;
import com.google.gwt.user.client.ui.HorizontalPanel;
import com.google.gwt.user.client.ui.Label;
import com.google.gwt.user.client.ui.SimplePanel;
import com.google.gwt.user.client.ui.TextBox;
import com.google.gwt.user.client.ui.VerticalPanel;
import com.google.gwt.user.client.ui.Widget;
import com.sap.sailing.landscape.LandscapeService;
import com.sap.sailing.landscape.ui.client.i18n.StringMessages;
import com.sap.sailing.landscape.ui.shared.AwsShardDTO;
import com.sap.sailing.landscape.ui.shared.LeaderboardNameDTO;
import com.sap.sailing.landscape.ui.shared.SailingApplicationReplicaSetDTO;
import com.sap.sse.common.Util;
import com.sap.sse.gwt.adminconsole.AdminConsoleTableResources;
import com.sap.sse.gwt.client.ErrorReporter;
import com.sap.sse.gwt.client.Notification;
import com.sap.sse.gwt.client.Notification.NotificationType;
import com.sap.sse.gwt.client.celltable.TableWrapperWithMultiSelectionAndFilter;
import com.sap.sse.gwt.client.controls.busyindicator.BusyIndicator;
import com.sap.sse.gwt.client.controls.busyindicator.SimpleBusyIndicator;
import com.sap.sse.gwt.client.dialog.DataEntryDialog;
import com.sap.sse.gwt.client.dialog.DataEntryDialog.DialogCallback;
import com.sap.sse.gwt.client.dialog.DataEntryDialog.Validator;
import com.sap.sse.landscape.aws.common.shared.ShardTargetGroupName;
import com.sap.sse.security.ui.client.component.SelectedElementsCountingButton;
public class ShardManagementPanel extends SimplePanel {
private final ErrorReporter errorReporter;
private final LandscapeManagementWriteServiceAsync landscapeManagementService;
private final TableWrapperWithMultiSelectionAndFilter<LeaderboardNameDTO, StringMessages, AdminConsoleTableResources> regattasTable;
private final TableWrapperWithMultiSelectionAndFilter<AwsShardDTO, StringMessages, AdminConsoleTableResources> shardTable;
private final TableWrapperWithMultiSelectionAndFilter<LeaderboardNameDTO, StringMessages, AdminConsoleTableResources> selectedKeysTable;
private final TextBox bearerTokenText;
private SailingApplicationReplicaSetDTO<String> replicaSet;
private final BusyIndicator busyIndicator;
private String region;
private String passphrase;
private List<LeaderboardNameDTO> leaderboards;
private Map<AwsShardDTO, Iterable<String>> shardsAndShardingKeys;
private final CaptionPanel leaderboardCaption, leaderboardsInShardCaption;
private final Button addButton, deleteButton;
private final SelectedElementsCountingButton<AwsShardDTO> removeShardButton;
private final StringMessages stringMessages;
private boolean hasAnythingChanged;
public ShardManagementPanel(LandscapeManagementWriteServiceAsync landscapeManagementService,
ErrorReporter errorReporter, StringMessages stringMessages) {
this.stringMessages = stringMessages;
hasAnythingChanged = false;
final VerticalPanel mainPanel = new VerticalPanel();
mainPanel.setWidth("100%");
add(mainPanel);
this.errorReporter = errorReporter;
this.landscapeManagementService = landscapeManagementService;
final HorizontalPanel actionPanel = new HorizontalPanel();
actionPanel.setSpacing(5);
final Button refreshButton = new Button(stringMessages.refresh());
refreshButton.addClickHandler(event -> refresh());
actionPanel.add(refreshButton);
regattasTable = new TableWrapperWithMultiSelectionAndFilter<LeaderboardNameDTO, StringMessages, AdminConsoleTableResources>(
stringMessages, errorReporter, false, java.util.Optional.empty(),
GWT.create(AdminConsoleTableResources.class), java.util.Optional.empty(), java.util.Optional.empty(),
null) {
@Override
protected Iterable<String> getSearchableStrings(LeaderboardNameDTO t) {
Set<String> res = new HashSet<String>();
res.add(t.getName());
return res;
}
};
final Button addShard = new SelectedElementsCountingButton<LeaderboardNameDTO>(stringMessages.addShard(),
regattasTable.getSelectionModel(), e -> addShard(), /* enableWhenSelectionEmpty */ true);
actionPanel.add(addShard);
shardTable = new TableWrapperWithMultiSelectionAndFilter<AwsShardDTO, StringMessages, AdminConsoleTableResources>(
stringMessages, errorReporter, false, java.util.Optional.empty(),
GWT.create(AdminConsoleTableResources.class), java.util.Optional.empty(), java.util.Optional.empty(),
null) {
@Override
protected Iterable<String> getSearchableStrings(AwsShardDTO t) {
final Set<String> res = new HashSet<String>();
res.add(t.getName());
return res;
}
};
removeShardButton = new SelectedElementsCountingButton<AwsShardDTO>(stringMessages.remove(),
shardTable.getSelectionModel(), /* element name mapper */ rs -> rs.getName(),
stringMessages::doYouReallyWantToRemoveSelectedElements, e -> removeShard());
actionPanel.add(removeShardButton);
mainPanel.add(actionPanel);
final Grid userCredentials = new Grid(1, 2);
bearerTokenText = new TextBox();
final Label bearerHintText = new Label(stringMessages.bearerTokenOrNullForApplicationReplicaSetToArchive(""));
userCredentials.setWidget(0, 0, bearerHintText);
userCredentials.setWidget(0, 1, bearerTokenText);
mainPanel.add(userCredentials);
shardTable.addColumn(t -> t.getName(), stringMessages.shardName());
shardTable.addColumn(t -> String.join(", ", t.getLeaderboardNames()), stringMessages.shardingKeys());
final SafeHtmlCell targetGroupCell = new SafeHtmlCell();
final Column<AwsShardDTO, SafeHtml> targetGroupColumn = new Column<AwsShardDTO, SafeHtml>(targetGroupCell) {
@Override
public SafeHtml getValue(AwsShardDTO shard) {
return new LinkBuilder().setTargetGroupName(shard.getTargetGroupName()).setRegion(region)
.setPathMode(LinkBuilder.pathModes.TargetGroupSearch).build();
}
};
shardTable.addColumn(targetGroupColumn, stringMessages.targetGroup());
final SafeHtmlCell ausoScalingGroupCell = new SafeHtmlCell();
final Column<AwsShardDTO, SafeHtml> autoScalingGroupColumn = new Column<AwsShardDTO, SafeHtml>(ausoScalingGroupCell) {
@Override
public SafeHtml getValue(AwsShardDTO shard) {
return new LinkBuilder().setAutoScalingGroupName(shard.getAutoScalingGroupName()).setRegion(region)
.setPathMode(LinkBuilder.pathModes.AutoScalingGroupSearch).build();
}
};
shardTable.addColumn(autoScalingGroupColumn, stringMessages.autoScalingGroup());
shardTable.getSelectionModel().addSelectionChangeHandler(event -> {
updateSelectedKeysTable();
updateAddDeleteButton();
});
leaderboardCaption = new CaptionPanel(stringMessages.unshardedLeaderboards());
regattasTable.addColumn(t -> t.getName(), stringMessages.name());
regattasTable.getSelectionModel().addSelectionChangeHandler(event -> {
updateAddDeleteButton();
});
leaderboardCaption.add(regattasTable);
final HorizontalPanel tableRow = new HorizontalPanel();
busyIndicator = new SimpleBusyIndicator();
leaderboardsInShardCaption = new CaptionPanel(stringMessages.leaderboardsInShard());
selectedKeysTable = new TableWrapperWithMultiSelectionAndFilter<LeaderboardNameDTO, StringMessages, AdminConsoleTableResources>(
stringMessages, errorReporter, false, java.util.Optional.empty(),
GWT.create(AdminConsoleTableResources.class), java.util.Optional.empty(), java.util.Optional.empty(),
null) {
@Override
protected Iterable<String> getSearchableStrings(LeaderboardNameDTO t) {
Set<String> result = new HashSet<>();
if (t != null && !t.getName().isEmpty()) {
result.add(t.getName());
}
return result;
}
};
selectedKeysTable.addColumn(t -> t.getName(), stringMessages.name());
selectedKeysTable.getSelectionModel().addSelectionChangeHandler(event -> {
updateAddDeleteButton();
});
leaderboardsInShardCaption.add(selectedKeysTable);
addButton = new Button("<");
addButton.addClickHandler(event -> addLeaderboardsToShard());
deleteButton = new Button(">");
deleteButton.addClickHandler(event -> removeLeaderboardsFromShard());
final HorizontalPanel insideShardPanel = new HorizontalPanel();
final VerticalPanel buttonPanel = new VerticalPanel();
final SimplePanel spaceholder = new SimplePanel();
spaceholder.setHeight("60px");
final SimplePanel keysSpaceholder = new SimplePanel();
buttonPanel.add(spaceholder);
buttonPanel.add(addButton);
buttonPanel.add(deleteButton);
tableRow.add(insideShardPanel);
insideShardPanel.add(shardTable);
insideShardPanel.add(keysSpaceholder);
keysSpaceholder.add(leaderboardsInShardCaption);
insideShardPanel.add(buttonPanel);
insideShardPanel.add(leaderboardCaption);
mainPanel.add(tableRow);
mainPanel.add(busyIndicator);
updateAddDeleteButton();
}
private void updateSelectedKeysTable() {
if (shardTable.getSelectionModel().getSelectedSet().size() == 1) {
selectedKeysTable .refresh(
Util.map(shardTable.getSelectionModel().getSelectedSet().iterator().next().getLeaderboardNames(), LeaderboardNameDTO::new));
} else {
selectedKeysTable.clear();
}
}
private void updateAddDeleteButton() {
addButton.setEnabled(shardTable.getSelectionModel().getSelectedSet().size() == 1
&& regattasTable.getSelectionModel().getSelectedSet().size() > 0);
deleteButton.setEnabled(shardTable.getSelectionModel().getSelectedSet().size() == 1
&& selectedKeysTable.getSelectionModel().getSelectedSet().size() > 0);
}
public void refresh() {
getLeaderboards();
}
private void display() {
final List<String> takenLeaderboardNames = new ArrayList<>();
for (Entry<AwsShardDTO, Iterable<String>> s : shardsAndShardingKeys.entrySet()) {
for (String leaderboardName : s.getKey().getLeaderboardNames()) {
takenLeaderboardNames.add(leaderboardName);
}
}
final Iterable<LeaderboardNameDTO> leaderboardsToDisplay = Util.filter(leaderboards,
leaderboardNameDTO -> !takenLeaderboardNames.contains(leaderboardNameDTO.getName()));
regattasTable.refresh(leaderboardsToDisplay);
shardTable.refresh(shardsAndShardingKeys.keySet());
}
private void setBusy(boolean b) {
busyIndicator.setBusy(b);
}
private void getLeaderboards() {
setBusy(true);
landscapeManagementService.getLeaderboardNames(replicaSet, getBearerToken(),
new AsyncCallback<ArrayList<LeaderboardNameDTO>>() {
@Override
public void onSuccess(ArrayList<LeaderboardNameDTO> result) {
leaderboards = result;
getShards();
}
@Override
public void onFailure(Throwable caught) {
errorReporter.reportError(stringMessages.errorFetchingLeaderboardNames(caught.getMessage()));
setBusy(false);
}
});
}
private String getBearerToken() {
return Util.hasLength(bearerTokenText.getValue()) ? bearerTokenText.getValue(): null;
}
private void getShards() {
setBusy(true);
landscapeManagementService.getShards(replicaSet, region, getBearerToken(),
new AsyncCallback<Map<AwsShardDTO, Iterable<String>>>() {
@Override
public void onFailure(Throwable caught) {
errorReporter.reportError(caught.getMessage());
setBusy(false);
}
@Override
public void onSuccess(Map<AwsShardDTO, Iterable<String>> result) {
shardsAndShardingKeys = result;
setBusy(false);
display();
}
});
}
private void addShard() {
final Set<LeaderboardNameDTO> selectedLeaderboards = regattasTable.getSelectionModel().getSelectedSet();
if (replicaSet != null) {
final DataEntryDialog<String> nameRequest = new DataEntryDialog<String>(
stringMessages.shardName(), stringMessages.enterShardName(), stringMessages.ok(), stringMessages.cancel(),
new Validator<String>() {
@Override
public String getErrorMessage(String valueToValidate) {
String errorMessage;
if (!Util.hasLength(valueToValidate)) {
errorMessage = stringMessages.pleaseProvideANonEmptyShardName();
} else {
try {
ShardTargetGroupName.create(replicaSet.getReplicaSetName(), valueToValidate, LandscapeService.SAILING_TARGET_GROUP_NAME_PREFIX);
errorMessage = null;
} catch (IllegalArgumentException e) {
errorMessage = stringMessages.shardNameInvalid(valueToValidate, e.getMessage());
}
}
return errorMessage;
}
}, new DialogCallback<String>() {
@Override
public void ok(String newShardName) {
hasAnythingChanged = true;
ArrayList<LeaderboardNameDTO> l = new ArrayList<>();
l.addAll(selectedLeaderboards);
busyIndicator.setBusy(true);
landscapeManagementService.addShard(newShardName, l, replicaSet,
getBearerToken(), region, passphrase.getBytes(), new AsyncCallback<Void>() {
@Override
public void onSuccess(Void result) {
busyIndicator.setBusy(false);
Notification.notify(stringMessages.shardCreatedSuccessfully(newShardName), NotificationType.SUCCESS);
refresh();
}
@Override
public void onFailure(Throwable caught) {
busyIndicator.setBusy(false);
errorReporter.reportError(caught.getMessage());
}
});
}
@Override
public void cancel() {
}
}) {
private final TextBox nameTextBox = createTextBox("", /* length */ 20);
@Override
protected Widget getAdditionalWidget() {
return nameTextBox;
}
@Override
protected FocusWidget getInitialFocusWidget() {
return nameTextBox;
}
@Override
protected String getResult() {
return nameTextBox.getText();
}
};
nameRequest.show();
}
}
private void removeShard() {
setBusy(true);
for (AwsShardDTO selection : shardTable.getSelectionModel().getSelectedSet()) {
hasAnythingChanged = true;
landscapeManagementService.removeShard(selection, replicaSet, region, passphrase.getBytes(),
new AsyncCallback<Void>() {
@Override
public void onFailure(Throwable caught) {
errorReporter.reportError(caught.getMessage());
setBusy(false);
}
@Override
public void onSuccess(Void result) {
Notification.notify(stringMessages.deletedShard(selection.getName()), NotificationType.SUCCESS);
refresh();
}
});
}
}
private void addLeaderboardsToShard() {
setBusy(true);
if (!regattasTable.getSelectionModel().getSelectedSet().isEmpty()
&& shardTable.getSelectionModel().getSelectedSet().size() == 1) {
final Iterable<LeaderboardNameDTO> selectedLeaderboards = regattasTable.getSelectionModel().getSelectedSet();
final AwsShardDTO shard = shardTable.getSelectionModel().getSelectedSet().iterator().next();
hasAnythingChanged = true;
landscapeManagementService.appendShardingKeysToShard(selectedLeaderboards, region, shard.getName(),
replicaSet, getBearerToken(), passphrase.getBytes(), new AsyncCallback<Void>() {
@Override
public void onFailure(Throwable caught) {
setBusy(false);
errorReporter.reportError(caught.getMessage());
}
@Override
public void onSuccess(Void result) {
setBusy(false);
Notification.notify(stringMessages.successfullyAppendedShardingKeysToShard(Util.join(", ", selectedLeaderboards), shard.getName()), NotificationType.SUCCESS);
refresh();
}
});
} else {
}
}
private void removeLeaderboardsFromShard() {
setBusy(true);
if (!selectedKeysTable.getSelectionModel().getSelectedSet().isEmpty()
&& shardTable.getSelectionModel().getSelectedSet().size() == 1) {
final Iterable<LeaderboardNameDTO> selectedLeaderboards = selectedKeysTable.getSelectionModel().getSelectedSet();
final AwsShardDTO shard = shardTable.getSelectionModel().getSelectedSet().iterator().next();
hasAnythingChanged = true;
landscapeManagementService.removeShardingKeysFromShard(selectedLeaderboards, region, shard.getName(),
replicaSet, getBearerToken(), passphrase.getBytes(), new AsyncCallback<Void>() {
@Override
public void onFailure(Throwable caught) {
setBusy(false);
errorReporter.reportError(caught.getMessage());
}
@Override
public void onSuccess(Void result) {
Notification.notify(stringMessages.successfullyRemovedLeaderboardsFromShard(Util.join(", ", selectedLeaderboards), shard.getName()), NotificationType.SUCCESS);
refresh();
}
});
}
}
public void setReplicaSet(SailingApplicationReplicaSetDTO<String> replicaset) {
replicaSet = replicaset;
}
public void setRegion(String region) {
this.region = region;
}
public void setPassphrase(String passphrase) {
this.passphrase = passphrase;
}
public Boolean hasAnythingChanged() {
return hasAnythingChanged;
}
}
@@ -139,4 +139,23 @@ com.sap.sse.gwt.adminconsole.StringMessages {
String errorDuringImport(String message);
String errorWhileComparingServerContent();
String differencesInServerContentFound(String serverAName, String aDiffs, String serverBName, String bDiffs);
String shard();
String shardName();
String addShard();
String removeShard();
String unshardedLeaderboards();
String shardingKeys();
String targetGroup();
String autoScalingGroup();
String leaderboardsInShard();
String enterShardName();
String openShardManagement();
String successfullyAppendedShardingKeysToShard(String leaderboardNames, String shardName);
String successfullyRemovedLeaderboardsFromShard(String leaderboardNames, String shardName);
String pleaseProvideANonEmptyShardName();
String shardNameInvalid(String shardName, String message);
String shardCreatedSuccessfully(String newShardName);
String deletedShard(String shardName);
String errorFetchingLeaderboardNames(String message);
String shardingDescription();
}
@@ -127,4 +127,23 @@ scaleAutoScalingReplicasUpOrDown=Scale auto-scaling replicas up/down
successfullyScaledAutoScalingReplicasForReplicaSet=Successfully scaled auto-scaling replicas for replica set {0}
errorDuringImport=Error during import: {0}
errorWhileComparingServerContent=Error while comparing server content
differencesInServerContentFound=Differences found: server {0} has {1} while server {2} has {3}
differencesInServerContentFound=Differences found: server {0} has {1} while server {2} has {3}
shard=Shard
shardName=Shard name
addShard=Add shard
removeShard=Delete shard
unshardedLeaderboards=Leaderboards in no shard
shardingKeys=Sharding keys
targetGroup=Target Group
autoScalingGroup=Auto-Scaling Group
leaderboardsInShard=Leaderboards in selected shard
enterShardName=Please enter a name for this shard
openShardManagement=Open sharding management panel
successfullyAppendedShardingKeysToShard=Successfully appended leaderboards {0} to shard {1}
successfullyRemovedLeaderboardsFromShard=Successfully removed leaderboards {0} from shard {1}
pleaseProvideANonEmptyShardName=Please provide a non-empty shard name
shardNameInvalid=Shard name "{0}" is invalid: {1}
shardCreatedSuccessfully=Shard "{0}" created successfully
deletedShard=Deleted shard "{0}"
errorFetchingLeaderboardNames=Error fetching leaderboard names: {0}
shardingDescription=Select zero or more leaderboards not yet in a shard, then click on "Add shard" to create a new shard handling the leaderboards selected.\nUse the arrow buttons to add leaderboards to or remove them from a selected shard.\nNote that changes are carried out immediately, not when you close this dialog.
@@ -126,4 +126,23 @@ scaleAutoScalingReplicasUpOrDown=Auto-skalierende Replikas hoch/herunter skalier
successfullyScaledAutoScalingReplicasForReplicaSet=Auto-skalierende Replikas für Anwendungs-Cluster {0} erfolgreich skaliert
errorDuringImport=Fehler beim Importieren: {0}
errorWhileComparingServerContent=Fehler beim Vergleich der Server-Inhalte
differencesInServerContentFound=Unterschiede gefunden: Server {0} enthält {1} während Server {2} {3} enthält.
differencesInServerContentFound=Unterschiede gefunden: Server {0} enthält {1} während Server {2} {3} enthält.
shard=Shard
shardName=Shard Name
addShard=Shard hinzufügen
removeShard=Shard entfernen
unshardedLeaderboards=Keinem Shard zugeordnete Ranglisten
shardingKeys=Sharding Schlüssel
targetGroup=Target Group
autoScalingGroup=Auto-Scaling Group
leaderboardsInShard=Ranglisten im ausgewählten Shard
enterShardName=Bitte Shard-Namen eingeben
openShardManagement=Shardmanagement Panel öffnen
successfullyAppendedShardingKeysToShard=Die Ranglisten {0} wurden erfolgreich zum Shard {1} hinzugefügt.
successfullyRemovedLeaderboardsFromShard=Die Ranglisten {0} wurden erfolgreich aus dem Shard {1} entfernt.
pleaseProvideANonEmptyShardName=Bitte einen nicht-leeren Shard-Namen eingeben
shardNameInvalid=Shard-Name "{0}" ist ungültig: {1}
shardCreatedSuccessfully=Shard "{0}" erfolgreich erstellt
deletedShard=Shard "{0}" gelöscht
errorFetchingLeaderboardNames=Fehler beim Bestimmen der Ranglisten-Namen: {0}
shardingDescription=Zum Anlegen eines neuen Shards optional Ranglisten auswählen, die direkt vom neuen Shard übernommen werden sollen, dann auf "Shard hinzufügen" klicken.\nMit den Pfeil-Schaltflächen Ranglisten aus dem ausgewählten Shard entfernen oder zu diesem hinzufügen.\nAchtung: die Änderungen werden sofort angewendet, nicht erst, wenn die Dialog geschlossen wird.
@@ -13,6 +13,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
@@ -37,6 +38,7 @@ import com.sap.sailing.landscape.SailingAnalyticsHost;
import com.sap.sailing.landscape.SailingAnalyticsMetrics;
import com.sap.sailing.landscape.SailingAnalyticsProcess;
import com.sap.sailing.landscape.SailingReleaseRepository;
import com.sap.sailing.landscape.common.RemoteServiceMappingConstants;
import com.sap.sailing.landscape.common.SharedLandscapeConstants;
import com.sap.sailing.landscape.impl.SailingAnalyticsHostImpl;
import com.sap.sailing.landscape.impl.SailingAnalyticsProcessImpl;
@@ -49,7 +51,9 @@ import com.sap.sailing.landscape.ui.client.LandscapeManagementWriteService;
import com.sap.sailing.landscape.ui.impl.Activator;
import com.sap.sailing.landscape.ui.shared.AmazonMachineImageDTO;
import com.sap.sailing.landscape.ui.shared.AwsInstanceDTO;
import com.sap.sailing.landscape.ui.shared.AwsShardDTO;
import com.sap.sailing.landscape.ui.shared.CompareServersResultDTO;
import com.sap.sailing.landscape.ui.shared.LeaderboardNameDTO;
import com.sap.sailing.landscape.ui.shared.MongoEndpointDTO;
import com.sap.sailing.landscape.ui.shared.MongoProcessDTO;
import com.sap.sailing.landscape.ui.shared.MongoScalingInstructionsDTO;
@@ -60,6 +64,7 @@ import com.sap.sailing.landscape.ui.shared.SailingAnalyticsProcessDTO;
import com.sap.sailing.landscape.ui.shared.SailingApplicationReplicaSetDTO;
import com.sap.sailing.landscape.ui.shared.SerializationDummyDTO;
import com.sap.sailing.server.gateway.interfaces.CompareServersResult;
import com.sap.sailing.server.gateway.interfaces.SailingServer;
import com.sap.sse.common.Duration;
import com.sap.sse.common.TimePoint;
import com.sap.sse.common.Util;
@@ -76,6 +81,7 @@ import com.sap.sse.landscape.aws.ApplicationProcessHost;
import com.sap.sse.landscape.aws.AwsApplicationReplicaSet;
import com.sap.sse.landscape.aws.AwsInstance;
import com.sap.sse.landscape.aws.AwsLandscape;
import com.sap.sse.landscape.aws.AwsShard;
import com.sap.sse.landscape.aws.HostSupplier;
import com.sap.sse.landscape.aws.common.shared.PlainRedirectDTO;
import com.sap.sse.landscape.aws.common.shared.RedirectDTO;
@@ -176,7 +182,7 @@ public class LandscapeManagementWriteServiceImpl extends ResultCachingProxiedRem
public ArrayList<String> getRegions() {
checkLandscapeManageAwsPermission();
final ArrayList<String> result = new ArrayList<>();
Util.addAll(Util.map(AwsLandscape.obtain().getRegions(), r->r.getId()), result);
Util.addAll(Util.map(AwsLandscape.obtain(RemoteServiceMappingConstants.pathPrefixForShardingKey).getRegions(), r->r.getId()), result);
return result;
}
@@ -385,7 +391,7 @@ public class LandscapeManagementWriteServiceImpl extends ResultCachingProxiedRem
@Override
public byte[] getEncryptedSshPrivateKey(String regionId, String keyName) throws JSchException {
final AwsLandscape<String> landscape = AwsLandscape.obtain();
final AwsLandscape<String> landscape = AwsLandscape.obtain(RemoteServiceMappingConstants.pathPrefixForShardingKey);
final SSHKeyPair keyPair = landscape.getSSHKeyPair(new AwsRegion(regionId, landscape), keyName);
getSecurityService().checkCurrentUserReadPermission(keyPair);
return keyPair.getEncryptedPrivateKey();
@@ -393,7 +399,7 @@ public class LandscapeManagementWriteServiceImpl extends ResultCachingProxiedRem
@Override
public byte[] getSshPublicKey(String regionId, String keyName) throws JSchException {
final AwsLandscape<String> landscape = AwsLandscape.obtain();
final AwsLandscape<String> landscape = AwsLandscape.obtain(RemoteServiceMappingConstants.pathPrefixForShardingKey);
final SSHKeyPair keyPair = landscape.getSSHKeyPair(new AwsRegion(regionId, landscape), keyName);
getSecurityService().checkCurrentUserReadPermission(keyPair);
return keyPair.getPublicKey();
@@ -603,8 +609,8 @@ public class LandscapeManagementWriteServiceImpl extends ResultCachingProxiedRem
}
@Override
public SerializationDummyDTO serializationDummy(ProcessDTO mongoProcessDTO, AwsInstanceDTO awsInstanceDTO,
SailingApplicationReplicaSetDTO<String> sailingApplicationReplicationSetDTO) {
public SerializationDummyDTO serializationDummy(ProcessDTO mongoProcessDTO, AwsInstanceDTO awsInstanceDTO, AwsShardDTO shardDTO,
SailingApplicationReplicaSetDTO<String> sailingApplicationReplicationSetDTO, LeaderboardNameDTO leaderboard) {
return null;
}
@@ -847,4 +853,93 @@ public class LandscapeManagementWriteServiceImpl extends ResultCachingProxiedRem
getLandscapeService().changeAutoScalingReplicasInstanceType(replicaSet, InstanceType.valueOf(instanceTypeName)),
Optional.ofNullable(optionalKeyName), privateKeyEncryptionPassphrase);
}
@Override
public ArrayList<LeaderboardNameDTO> getLeaderboardNames(SailingApplicationReplicaSetDTO<String> replicaSet,
String bearerToken) throws Exception {
final SailingServer server = getLandscapeService().getSailingServer(replicaSet.getHostname(), bearerToken,
/* HTTPS Port */ Optional.of(443));
return new ArrayList<>(Util.asList(Util.map(server.getLeaderboardNames(), LeaderboardNameDTO::new)));
}
@Override
public void addShard(String shardName, ArrayList<LeaderboardNameDTO> selectedLeaderBoardNames,
SailingApplicationReplicaSetDTO<String> replicaSetDTO, String bearerToken, String region,
byte[] passphraseForPrivateKeyDecryption) throws Exception {
checkLandscapeManageAwsPermission();
final AwsRegion awsRegion = new AwsRegion(replicaSetDTO.getMaster().getHost().getRegion(), getLandscape());
final AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> awsReplicaSet = convertFromApplicationReplicaSetDTO(
awsRegion, replicaSetDTO);
getLandscapeService().addShard(Util.map(selectedLeaderBoardNames, t -> t.getName()), awsReplicaSet, awsRegion,
bearerToken, passphraseForPrivateKeyDecryption, shardName);
}
@Override
public Map<AwsShardDTO, Iterable<String>> getShards(SailingApplicationReplicaSetDTO<String> replicaSetDTO,
String region, String bearerToken) throws Exception {
checkLandscapeManageAwsPermission();
final AwsRegion awsRegion = new AwsRegion(replicaSetDTO.getMaster().getHost().getRegion(), getLandscape());
final Map<AwsShardDTO, Iterable<String>> shardingKeysForShards = new HashMap<>();
final SailingServer server = getLandscapeService().getSailingServer(replicaSetDTO.getHostname(), bearerToken, Optional.empty());
final AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> applicationServerReplicaSet = convertFromApplicationReplicaSetDTO(
awsRegion, replicaSetDTO);
final Map<String, String> leaderboardNamesByShardingKeys = new HashMap<>();
for (String leaderboard : server.getLeaderboardNames()) {
leaderboardNamesByShardingKeys.put(server.getLeaderboardShardingKey(leaderboard), leaderboard);
}
for (Entry<AwsShard<String>, Iterable<String>> entry : applicationServerReplicaSet.getShards().entrySet()) {
shardingKeysForShards.put(createAwsShardDTO(entry.getKey(), applicationServerReplicaSet.getName(), server, leaderboardNamesByShardingKeys),
entry.getValue());
}
final Map<AwsShardDTO, Iterable<String>> res = new HashMap<>();
for (Entry<AwsShardDTO, Iterable<String>> entry : shardingKeysForShards.entrySet()) {
res.put(entry.getKey(), entry.getValue());
}
return res;
}
@Override
public void removeShard(AwsShardDTO shard, SailingApplicationReplicaSetDTO<String> replicaSetDTO, String region,
byte[] passphrase) throws Exception {
checkLandscapeManageAwsPermission();
final AwsRegion awsRegion = new AwsRegion(replicaSetDTO.getMaster().getHost().getRegion(), getLandscape());
final AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> applicationServerReplicaSet = convertFromApplicationReplicaSetDTO(
awsRegion, replicaSetDTO);
getLandscapeService().removeShard(applicationServerReplicaSet, shard.getTargetgroupArn());
}
@Override
public void appendShardingKeysToShard(Iterable<LeaderboardNameDTO> sharindKeysToAppend, String region,
String shardName, SailingApplicationReplicaSetDTO<String> replicaSet, String bearerToken,
byte[] passphraseForPrivateKeyDecryption) throws Exception {
checkLandscapeManageAwsPermission();
final AwsRegion awsRegion = new AwsRegion(replicaSet.getMaster().getHost().getRegion(), getLandscape());
final AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> rs = convertFromApplicationReplicaSetDTO(
awsRegion, replicaSet);
final AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> applicationReplicaSet = getLandscape()
.getApplicationReplicaSet(awsRegion, rs.getServerName(), rs.getMaster(), rs.getReplicas());
getLandscapeService().appendShardingKeysToShard(Util.map(sharindKeysToAppend, t -> t.getName()),
applicationReplicaSet, passphraseForPrivateKeyDecryption, awsRegion, shardName, bearerToken);
}
public void removeShardingKeysFromShard(Iterable<LeaderboardNameDTO> selectedShardingKeys, String region,
String shardName, SailingApplicationReplicaSetDTO<String> replicaSet, String bearerToken,
byte[] passphraseForPrivateKeyDecryption) throws Exception {
checkLandscapeManageAwsPermission();
final AwsRegion awsRegion = new AwsRegion(region, getLandscape());
final AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> rs = convertFromApplicationReplicaSetDTO(
awsRegion, replicaSet);
getLandscapeService().removeShardingKeysFromShard(Util.asList(Util.map(selectedShardingKeys, t -> t.getName())),
rs, passphraseForPrivateKeyDecryption, awsRegion, shardName, bearerToken);
}
public AwsShardDTO createAwsShardDTO(AwsShard<String> shard, String replicaSetName, SailingServer server,
Map<String, String> leaderboardNamesByShardingKeys) throws Exception {
return new AwsShardDTO(Util.filter(Util.map(shard.getKeys(),
shardingKey -> leaderboardNamesByShardingKeys.get(shardingKey)), leaderboardName->leaderboardName!=null),
shard.getTargetGroup().getTargetGroupArn(), shard.getTargetGroup().getName(),
shard.getAutoScalingGroup().getAutoScalingGroup().autoScalingGroupARN(),
shard.getTargetGroup().getLoadBalancerArn(), shard.getAutoScalingGroup().getName(),
shard.getName() == null ? "" : shard.getName(), replicaSetName);
}
}
@@ -0,0 +1,72 @@
package com.sap.sailing.landscape.ui.shared;
import java.util.ArrayList;
import com.sap.sse.common.Named;
import com.sap.sse.common.Util;
public class AwsShardDTO implements Named {
private static final long serialVersionUID = 1L;
ArrayList<String> leaderboardNames;
private String targetGroupArn;
private String targetGroupName;
private String autoScalingGroupArn;
private String autoScalingGroupName;
private String autoBalancerArn;
private String name;
private String replicaName;
@SuppressWarnings("unused") // for GWT serialisation only
private AwsShardDTO() {
}
public AwsShardDTO(Iterable<String> leaderboardNames, String targetGroupArn, String targetGroupName, String autoScalinggroupArn,
String autoBalancerArn, String autoScalingGroupName, String name, String replicaName) {
this.leaderboardNames = new ArrayList<>();
Util.addAll(leaderboardNames, this.leaderboardNames);
this.targetGroupArn = targetGroupArn;
this.targetGroupName = targetGroupName;
this.autoBalancerArn = autoBalancerArn;
this.autoScalingGroupName = autoScalingGroupName;
this.autoScalingGroupArn = autoScalinggroupArn;
this.name = name;
this.replicaName = replicaName;
}
public String getReplicaName() {
return replicaName;
}
public ArrayList<String> getLeaderboardNames() {
return leaderboardNames;
}
public String getTargetgroupArn() {
return targetGroupArn;
}
public String getTargetGroupName() {
return targetGroupName;
}
public String getAutosclaingGroupArn() {
return autoScalingGroupArn;
}
public String getAutobalancerArn() {
return autoBalancerArn;
}
public void setName(String name) {
this.name = name;
}
public String getAutoScalingGroupName() {
return this.autoScalingGroupName;
}
@Override
public String getName() {
return name;
}
}
@@ -0,0 +1,22 @@
package com.sap.sailing.landscape.ui.shared;
import com.sap.sse.common.Named;
import com.sap.sse.security.shared.dto.NamedDTO;
/**
* This class is supposed to represent leader board names. It's just a wrapper for ensuring that we can use the
* {@link Named} functionality in the UI.
*
* @author I569653
*
*/
public class LeaderboardNameDTO extends NamedDTO {
private static final long serialVersionUID = -173460185784328741L;
@SuppressWarnings({ "deprecation", "unused" })
private LeaderboardNameDTO() {} // for GWT serialisation only
public LeaderboardNameDTO(String name) {
super(name);
}
}
@@ -14,6 +14,7 @@ import com.sap.sailing.landscape.procedures.SailingAnalyticsReplicaConfiguration
import com.sap.sailing.landscape.procedures.SailingAnalyticsReplicaConfiguration.Builder;
import com.sap.sailing.landscape.procedures.StartMultiServer;
import com.sap.sailing.server.gateway.interfaces.CompareServersResult;
import com.sap.sailing.server.gateway.interfaces.SailingServer;
import com.sap.sse.common.Duration;
import com.sap.sse.common.Util.Pair;
import com.sap.sse.landscape.Release;
@@ -212,7 +213,11 @@ public interface LandscapeService {
* TODO bug5674: before registering the master with the TGs, spin up as many new replicas as there are currently
* replicas; wait until they are all ready, then register master and new replicas in TGs and de-register old replicas.
* Then terminate old auto-scaling replicas and update any unmanaged replica in-place. When the number of auto-scaling
* replicas has reached the desired size of the auto-scaling group, terminate the replicas created explicitly.
* replicas has reached the desired size of the auto-scaling group, terminate the replicas created explicitly.<p>
*
* Shards are updated by spinning up replicas for the temporary transition and changing the auto scaling config.
* After that all shard replicas are getting shutdown and restarted with the new launch config.
* It's expected that the replica set has its own auto scaling group if it has shards.
*/
AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> upgradeApplicationReplicaSet(AwsRegion region,
AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> replicaSet,
@@ -370,7 +375,7 @@ public interface LandscapeService {
InterruptedException, ExecutionException, Exception;
/**
* If the {@code replicaSet} provided has an auto-scaling group, its launch configuration is adjusted such that it
* If the {@code replicaSet} provided has one or more auto-scaling groups, their launch configuration is adjusted such that it
* matches the {@code optionalInstanceType}. The existing replicas managed currently by the auto-scaling group are
* replaced one by one with new instances with the new configuration. This happens by setting the auto-scaling
* group's new minimum size to the current number of instances managed by the auto-scaling group plus one, then
@@ -385,4 +390,24 @@ public interface LandscapeService {
<ShardingKey> boolean isEligibleForDeployment(SailingAnalyticsHost<ShardingKey> host, String serverName, int port, Optional<Duration> waitForProcessTimeout,
String optionalKeyName, byte[] privateKeyEncryptionPassphrase) throws Exception;
SailingServer getSailingServer(String hostname, String username, String password, Optional<Integer> port)
throws MalformedURLException;
SailingServer getSailingServer(String hostname, String bearertoken, Optional<Integer> port)
throws MalformedURLException;
void removeShardingKeysFromShard(Iterable<String> selectedleaderboards,
AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> applicationReplicaSet,
byte[] passphraseForPrivateKeyDecription,AwsRegion region, String shardName, String bearertoken) throws Exception;
public void appendShardingKeysToShard(Iterable<String> selectedLeaderboards,
AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> applicationReplicaSet,
byte[] passphraseForPrivateKeyDecription, AwsRegion region, String shardName, String bearertoken) throws Exception;
void removeShard(AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> applicationReplicaSet, String shardTargetGroupArn) throws Exception;
void addShard(Iterable<String> selectedLeaderboardNames,
AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> applicationReplicaSet,
AwsRegion region, String bearertoken, byte[] passphraseForPrivateKeyDecription, String shardName) throws Exception;
}
@@ -11,7 +11,6 @@ import com.sap.sse.landscape.aws.AwsApplicationProcess;
public interface SailingAnalyticsProcess<ShardingKey> extends AwsApplicationProcess<ShardingKey, SailingAnalyticsMetrics, SailingAnalyticsProcess<ShardingKey>> {
static Logger logger = Logger.getLogger(SailingAnalyticsProcess.class.getName());
static String HEALTH_CHECK_PATH = "/gwt/status";
int getExpeditionUdpPort(Optional<Duration> optionalTimeout, Optional<String> optionalKeyName, byte[] privateKeyEncryptionPassphrase) throws Exception;
@@ -1,6 +1,7 @@
package com.sap.sailing.landscape.impl;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
@@ -239,7 +240,7 @@ public class EligbleInstanceForReplicaSetFindingStrategyImpl implements Eligible
boolean hasManagedReplicaInDifferentAZ = false;
boolean hasUnmanagedReplicaInDifferentAZ = false;
for (final SailingAnalyticsProcess<String> replica : replicaSet.getReplicas()) {
final boolean isManaged = replica.getHost().isManagedByAutoScalingGroup(replicaSet.getAutoScalingGroup());
final boolean isManaged = replica.getHost().isManagedByAutoScalingGroup(Collections.singleton(replicaSet.getAutoScalingGroup()));
final boolean isInDifferentAZ = !replica.getHost().getAvailabilityZone().equals(availabilityZone);
hasManagedReplicas = hasManagedReplicas || isManaged;
hasManagedReplicaInDifferentAZ = hasManagedReplicaInDifferentAZ || (isManaged && isInDifferentAZ);
@@ -5,10 +5,13 @@ import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
@@ -36,6 +39,7 @@ import com.sap.sailing.landscape.SailingAnalyticsHost;
import com.sap.sailing.landscape.SailingAnalyticsMetrics;
import com.sap.sailing.landscape.SailingAnalyticsProcess;
import com.sap.sailing.landscape.SailingReleaseRepository;
import com.sap.sailing.landscape.common.RemoteServiceMappingConstants;
import com.sap.sailing.landscape.common.SharedLandscapeConstants;
import com.sap.sailing.landscape.procedures.CreateLaunchConfigurationAndAutoScalingGroup;
import com.sap.sailing.landscape.procedures.DeployProcessOnMultiServer;
@@ -72,6 +76,7 @@ import com.sap.sse.landscape.aws.AwsAutoScalingGroup;
import com.sap.sse.landscape.aws.AwsAvailabilityZone;
import com.sap.sse.landscape.aws.AwsInstance;
import com.sap.sse.landscape.aws.AwsLandscape;
import com.sap.sse.landscape.aws.AwsShard;
import com.sap.sse.landscape.aws.HostSupplier;
import com.sap.sse.landscape.aws.ReverseProxy;
import com.sap.sse.landscape.aws.Tags;
@@ -80,11 +85,15 @@ import com.sap.sse.landscape.aws.common.shared.RedirectDTO;
import com.sap.sse.landscape.aws.impl.AwsApplicationReplicaSetImpl;
import com.sap.sse.landscape.aws.impl.AwsRegion;
import com.sap.sse.landscape.aws.impl.DNSCache;
import com.sap.sse.landscape.aws.orchestration.AddShardingKeyToShard;
import com.sap.sse.landscape.aws.orchestration.AwsApplicationConfiguration;
import com.sap.sse.landscape.aws.orchestration.CopyAndCompareMongoDatabase;
import com.sap.sse.landscape.aws.orchestration.CreateDNSBasedLoadBalancerMapping;
import com.sap.sse.landscape.aws.orchestration.CreateDynamicLoadBalancerMapping;
import com.sap.sse.landscape.aws.orchestration.CreateLoadBalancerMapping;
import com.sap.sse.landscape.aws.orchestration.CreateShard;
import com.sap.sse.landscape.aws.orchestration.RemoveShardingKeyFromShard;
import com.sap.sse.landscape.aws.orchestration.ShardProcedure;
import com.sap.sse.landscape.aws.orchestration.StartAwsHost;
import com.sap.sse.landscape.mongodb.Database;
import com.sap.sse.landscape.mongodb.MongoEndpoint;
@@ -499,7 +508,7 @@ public class LandscapeServiceImpl implements LandscapeService {
// only terminate replica if not running on host created by auto-scaling group
final AwsAutoScalingGroup autoScalingGroup = applicationReplicaSet.getAutoScalingGroup();
for (final SailingAnalyticsProcess<String> replica : applicationReplicaSet.getReplicas()) {
if (autoScalingGroup == null || !replica.getHost().isManagedByAutoScalingGroup(autoScalingGroup)) {
if (autoScalingGroup == null || !replica.getHost().isManagedByAutoScalingGroup(Collections.singleton(autoScalingGroup))) {
logger.info("Found replica "+replica+" running on an instance not managed by auto-scaling group " +
(autoScalingGroup != null ? autoScalingGroup.getName() : "") + ". Stopping...");
replica.stopAndTerminateIfLast(Landscape.WAIT_FOR_PROCESS_TIMEOUT, Optional.ofNullable(optionalKeyName), passphraseForPrivateKeyDecryption);
@@ -518,6 +527,10 @@ public class LandscapeServiceImpl implements LandscapeService {
final AwsRegion region = new AwsRegion(regionId, getLandscape());
final AwsAutoScalingGroup autoScalingGroup = applicationReplicaSet.getAutoScalingGroup();
final CompletableFuture<Void> autoScalingGroupRemoval;
// Remove all shards
for (AwsShard<String> shard : applicationReplicaSet.getShards().keySet()) {
applicationReplicaSet.removeShard(shard, getLandscape());
}
terminateReplicasNotManagedByAutoScalingGroup(applicationReplicaSet, optionalKeyName, passphraseForPrivateKeyDecryption);
if (autoScalingGroup != null) {
// remove the launch configuration used by the auto scaling group and the auto scaling group itself;
@@ -565,7 +578,7 @@ public class LandscapeServiceImpl implements LandscapeService {
final Iterable<SailingAnalyticsProcess<String>> replicas = applicationReplicaSet.getMaster().getReplicas(Landscape.WAIT_FOR_PROCESS_TIMEOUT,
new SailingAnalyticsHostSupplier<String>(), processFactoryFromHostAndServerDirectory);
for (final SailingAnalyticsProcess<String> replica : replicas) {
if (replica.getHost().isManagedByAutoScalingGroup(autoScalingGroup) && replica.isAlive(Landscape.WAIT_FOR_PROCESS_TIMEOUT)) {
if (replica.getHost().isManagedByAutoScalingGroup(Collections.singleton(autoScalingGroup)) && replica.isAlive(Landscape.WAIT_FOR_PROCESS_TIMEOUT)) {
logger.info("Replica "+replica+" is managed by auto-scaling group "+autoScalingGroup.getName()+" and is still alive.");
return false;
}
@@ -625,7 +638,7 @@ public class LandscapeServiceImpl implements LandscapeService {
keyId = sessionCredentials.getAccessKeyId();
secret = sessionCredentials.getSecretAccessKey();
sessionToken = sessionCredentials.getSessionToken();
result = AwsLandscape.obtain(keyId, secret, sessionToken);
result = AwsLandscape.obtain(keyId, secret, sessionToken, RemoteServiceMappingConstants.pathPrefixForShardingKey);
} else {
result = null;
}
@@ -659,7 +672,7 @@ public class LandscapeServiceImpl implements LandscapeService {
@Override
public void createMfaSessionCredentials(String awsAccessKey, String awsSecret, String mfaTokenCode) {
final Credentials credentials = AwsLandscape.obtain(awsAccessKey, awsSecret).getMfaSessionCredentials(mfaTokenCode);
final Credentials credentials = AwsLandscape.obtain(awsAccessKey, awsSecret, RemoteServiceMappingConstants.pathPrefixForShardingKey).getMfaSessionCredentials(mfaTokenCode);
final AwsSessionCredentialsWithExpiryImpl result = new AwsSessionCredentialsWithExpiryImpl(
credentials.accessKeyId(), credentials.secretAccessKey(), credentials.sessionToken(),
TimePoint.of(credentials.expiration().toEpochMilli()));
@@ -843,7 +856,8 @@ public class LandscapeServiceImpl implements LandscapeService {
final DNSCache dnsCache = landscape.getNewDNSCache();
final AwsApplicationReplicaSet<String,SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> applicationReplicaSet =
new AwsApplicationReplicaSetImpl<>(replicaSetName, masterHostname, master, /* no replicas yet */ Optional.empty(),
allLoadBalancersInRegion, allTargetGroupsInRegion, allLoadBalancerRulesInRegion, autoScalingGroups, launchConfigurations, dnsCache);
allLoadBalancersInRegion, allTargetGroupsInRegion, allLoadBalancerRulesInRegion,
autoScalingGroups, launchConfigurations, dnsCache, RemoteServiceMappingConstants.pathPrefixForShardingKey);
return applicationReplicaSet;
}
@@ -854,16 +868,28 @@ public class LandscapeServiceImpl implements LandscapeService {
String replicaReplicationBearerToken)
throws MalformedURLException, IOException, TimeoutException, Exception {
if (replicaSet.isLocalReplicaSet()) {
throw new IllegalArgumentException("A replica set cannot upgrade itself. Current replica set: "+ServerInfo.getName());
throw new IllegalArgumentException(
"A replica set cannot upgrade itself. Current replica set: " + ServerInfo.getName());
}
if (!replicaSet.getShards().isEmpty() && replicaSet.getAutoScalingGroup() == null) {
throw new IllegalStateException(
"A replica set is expected to have an auto scaling group if it has shards");
}
final Release release = getRelease(releaseOrNullForLatestMaster);
final String effectiveReplicaReplicationBearerToken = getEffectiveBearerToken(replicaReplicationBearerToken);
final int oldAutoScalingGroupMinSize;
final AwsAutoScalingGroup autoScalingGroup = replicaSet.getAutoScalingGroup();
final Collection<AwsAutoScalingGroup> affectedAutoScalingGroups = new ArrayList<>();
final int autoScalingReplicaCount;
replicaSet.getShards().keySet().forEach(t -> affectedAutoScalingGroups.add(t.getAutoScalingGroup()));
if (autoScalingGroup != null) {
affectedAutoScalingGroups.add(autoScalingGroup);
oldAutoScalingGroupMinSize = autoScalingGroup.getAutoScalingGroup().minSize();
autoScalingReplicaCount = autoScalingGroup.getAutoScalingGroup().desiredCapacity();
int tempAutoScalingReplicaCount = 0;
for (AwsAutoScalingGroup asg : affectedAutoScalingGroups) {
tempAutoScalingReplicaCount = tempAutoScalingReplicaCount + asg.getAutoScalingGroup().desiredCapacity();
}
autoScalingReplicaCount = tempAutoScalingReplicaCount;
} else {
oldAutoScalingGroupMinSize = -1;
autoScalingReplicaCount = -1;
@@ -873,7 +899,8 @@ public class LandscapeServiceImpl implements LandscapeService {
final SailingAnalyticsProcess<String> additionalReplicaStarted = ensureAtLeastOneReplicaExistsStopReplicatingAndRemoveMasterFromTargetGroups(
replicaSet, optionalKeyName, privateKeyEncryptionPassphrase, effectiveReplicaReplicationBearerToken);
if (replicaSet.getAutoScalingGroup() != null) {
getLandscape().updateReleaseInAutoScalingGroup(region, replicaSet.getAutoScalingGroup(), replicaSet.getName(), release);
getLandscape().updateReleaseInAutoScalingGroups(region, replicaSet.getAutoScalingGroup().getLaunchConfiguration(),
affectedAutoScalingGroups, replicaSet.getName(), release);
}
final SailingAnalyticsProcess<String> master = replicaSet.getMaster();
master.refreshToRelease(release, Optional.ofNullable(optionalKeyName), privateKeyEncryptionPassphrase);
@@ -882,27 +909,45 @@ public class LandscapeServiceImpl implements LandscapeService {
master.waitUntilReady(Optional.of(Duration.ONE_DAY)); // wait a little longer since master may need to re-load many races
logger.info("Launching upgrade replicas based on current set of replicas for replica set "+replicaSet.getName());
final Iterable<SailingAnalyticsProcess<String>> temporaryUpgradeReplicas = launchUpgradeReplicasAndWaitUntilReady(
replicaSet, release, effectiveReplicaReplicationBearerToken, Optional.ofNullable(optionalKeyName), privateKeyEncryptionPassphrase);
replicaSet, release, effectiveReplicaReplicationBearerToken, Optional.ofNullable(optionalKeyName),
privateKeyEncryptionPassphrase);
final List<SailingAnalyticsProcess<String>> temporaryUpgradeReplicasMutable = Util.asList(temporaryUpgradeReplicas);
// register master again with master and public target group
logger.info("Adding master "+master+" and dedicated upgraded temporary replicas to target groups "+
replicaSet.getPublicTargetGroup()+" and "+replicaSet.getMasterTargetGroup());
logger.info("Adding master " + master + " and dedicated upgraded temporary replicas to target groups "
+ replicaSet.getPublicTargetGroup() + " and " + replicaSet.getMasterTargetGroup());
replicaSet.getPublicTargetGroup().addTarget(master.getHost());
replicaSet.getMasterTargetGroup().addTarget(master.getHost());
replicaSet.getPublicTargetGroup().addTargets(Util.map(temporaryUpgradeReplicas, temporaryUpgradeReplica->temporaryUpgradeReplica.getHost()));
// the following map stores the assignment of the temporary upgrade replicas to the public / shard target groups
// for later removal as the final auto-scaling replicas start to replace them:
final Map<TargetGroup<String>, Iterable<AwsInstance<String>>> tempUpgradeReplicasByTargetGroup = new HashMap<>();
// add as many temporary upgrade replicas to each shard's target group as the target group has targets now:
for (final AwsShard<String> shard : replicaSet.getShards().keySet()) {
final TargetGroup<String> shardTargetGroup = shard.getTargetGroup();
final Collection<AwsInstance<String>> hosts = new ArrayList<>();
final int numberOfTargets = shardTargetGroup.getRegisteredTargets().size();
for (int i = 0; i < numberOfTargets; i++) {
final SailingAnalyticsProcess<String> tempUpgradeReplicaProcess = temporaryUpgradeReplicasMutable.remove(0);
shardTargetGroup.addTarget(tempUpgradeReplicaProcess.getHost());
hosts.add(tempUpgradeReplicaProcess.getHost());
}
tempUpgradeReplicasByTargetGroup.put(shardTargetGroup, hosts);
}
// add all other temporary upgrade replicas not used for shards to the public target group:
replicaSet.getPublicTargetGroup().addTargets(Util.map(temporaryUpgradeReplicasMutable,
temporaryUpgradeReplica -> temporaryUpgradeReplica.getHost()));
tempUpgradeReplicasByTargetGroup.put(replicaSet.getPublicTargetGroup(), Util.map(temporaryUpgradeReplicasMutable, temporaryUpgradeReplica->temporaryUpgradeReplica.getHost()));
// if a replica was spun up (additionalReplicaStarted), remove from public target group and terminate:
if (additionalReplicaStarted != null) {
replicasToStopAfterUpgradingMaster.add(additionalReplicaStarted);
if (replicaSet.getAutoScalingGroup() != null) {
// run updating the auto-scaling group in the background; it's more time-critical now
// to remove the old replicas from the target group
ThreadPoolUtil.INSTANCE.getDefaultBackgroundTaskThreadPoolExecutor().execute(ThreadPoolUtil.INSTANCE.associateWithSubjectIfAny(()->
{
try {
getLandscape().updateAutoScalingGroupMinSize(replicaSet.getAutoScalingGroup(), oldAutoScalingGroupMinSize);
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
}));
for (AwsAutoScalingGroup asg : affectedAutoScalingGroups) {
ThreadPoolUtil.INSTANCE.getDefaultBackgroundTaskThreadPoolExecutor()
.execute(ThreadPoolUtil.INSTANCE.associateWithSubjectIfAny(() -> {
getLandscape().updateAutoScalingGroupMinSize(asg, oldAutoScalingGroupMinSize);
}));
}
} // else, the replica was started explicitly, without an auto-scaling group; in any case, all replicas still
// on the old release will now be stopped:
}
@@ -911,7 +956,7 @@ public class LandscapeServiceImpl implements LandscapeService {
replicaSet.getPublicTargetGroup().removeTargets(Util.map(replicasToStopAfterUpgradingMaster, replica->replica.getHost()));
for (final SailingAnalyticsProcess<String> replica : replicasToStopAfterUpgradingMaster) {
// if managed by auto-scaling group or if it's an "unmanaged" / "explicit" replica and it's the one launched in order to have at least one, stop/terminate;
final boolean managedByAutoScalingGroup = replica.getHost().isManagedByAutoScalingGroup(autoScalingGroup);
final boolean managedByAutoScalingGroup = replica.getHost().isManagedByAutoScalingGroup(affectedAutoScalingGroups);
if (managedByAutoScalingGroup ||
(additionalReplicaStarted != null && replica.getHost().getInstanceId().equals(additionalReplicaStarted.getHost().getInstanceId()))) {
logger.info("Stopping (and terminating if last application process on host) replicas on old release: "+replicasToStopAfterUpgradingMaster);
@@ -926,21 +971,29 @@ public class LandscapeServiceImpl implements LandscapeService {
}
final Iterable<SailingAnalyticsProcess<String>> newUpgradedReplicas;
if (autoScalingGroup != null && autoScalingReplicaCount > 0) {
logger.info("Now waiting until "+autoScalingReplicaCount+" auto-scaling replicas are ready before taking down dedicated temporary upgrade replicas");
logger.info("Now waiting until " + autoScalingReplicaCount
+ " auto-scaling replicas are ready before taking down dedicated temporary upgrade replicas");
// Note: the wait call returns *all* of the master's replicas, not only the auto-scaling ones; it's the predicate that
// filters them down to the auto-scaling ones, counts them and compares them to the required count
newUpgradedReplicas = Wait.wait(()->master.getReplicas(Landscape.WAIT_FOR_PROCESS_TIMEOUT, new SailingAnalyticsHostSupplier<String>(), new SailingAnalyticsProcessFactory(this::getLandscape)),
replicas->Util.size(
Util.filter(replicas,
replica->replica.getHost().isManagedByAutoScalingGroup(autoScalingGroup))) >= autoScalingReplicaCount,
/* retryOnException */ true,
Optional.of(Duration.ONE_DAY), /* duration between attempts */ Duration.ONE_SECOND.times(30),
Level.INFO, "Waiting for "+autoScalingReplicaCount+" auto-scaling replicas to become ready");
newUpgradedReplicas = Wait.wait(() -> master.getReplicas(Landscape.WAIT_FOR_PROCESS_TIMEOUT,
new SailingAnalyticsHostSupplier<String>(), new SailingAnalyticsProcessFactory(this::getLandscape)),
replicas -> {
int size = Util.size(Util.filter(replicas,
replica -> replica.getHost().isManagedByAutoScalingGroup(affectedAutoScalingGroups)));
logger.info("Until now there are " + size + " auto-scaling replicas healthy");
return size >= autoScalingReplicaCount;
}, /* retryOnException */ true, Optional.of(Duration.ONE_DAY),
/* duration between attempts */ Duration.ONE_SECOND.times(30), Level.INFO,
"Waiting for " + autoScalingReplicaCount + " auto-scaling replicas to become ready");
} else {
logger.info("No auto-scaling group or auto-scaling group did not have managed instances; using only the upgraded unmanaged replicas "+
newUpgradedUnmanagedReplicas);
newUpgradedReplicas = newUpgradedUnmanagedReplicas;
}
logger.info("Removing targets for all temporary upgrade replicas " + temporaryUpgradeReplicas);
for (Entry<TargetGroup<String>, Iterable<AwsInstance<String>>> entry : tempUpgradeReplicasByTargetGroup.entrySet()) {
entry.getKey().removeTargets(entry.getValue());
}
logger.info("Stopping/terminating all temporary upgrade replicas "+temporaryUpgradeReplicas);
for (final SailingAnalyticsProcess<String> upgradeReplica : temporaryUpgradeReplicas) {
upgradeReplica.stopAndTerminateIfLast(Landscape.WAIT_FOR_PROCESS_TIMEOUT, Optional.ofNullable(optionalKeyName), privateKeyEncryptionPassphrase);
@@ -1066,19 +1119,22 @@ public class LandscapeServiceImpl implements LandscapeService {
*/
@Override
public SailingAnalyticsProcess<String> ensureAtLeastOneReplicaExistsStopReplicatingAndRemoveMasterFromTargetGroups(
final AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> replicaSet, String optionalKeyName,
byte[] privateKeyEncryptionPassphrase,
final String effectiveReplicaReplicationBearerToken)
throws Exception, MalformedURLException, IOException, TimeoutException, InterruptedException,
ExecutionException {
final AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> replicaSet,
String optionalKeyName, byte[] privateKeyEncryptionPassphrase,
final String effectiveReplicaReplicationBearerToken) throws Exception, MalformedURLException, IOException,
TimeoutException, InterruptedException, ExecutionException {
final Set<SailingAnalyticsProcess<String>> replicasToStopReplicating = new HashSet<>();
Util.addAll(replicaSet.getReplicas(), replicasToStopReplicating);
final SailingAnalyticsProcess<String> additionalReplicaStarted;
if (Util.isEmpty(replicaSet.getReplicas())) {
logger.info("No replica found for replica set " + replicaSet.getName()
if (Util.isEmpty(Util.filter(new HashSet<>(replicasToStopReplicating), replica ->
// exclude replicas belonging to a shard
!replica.getHost().isManagedByAutoScalingGroup(replicaSet.getShards().keySet().stream().map(
shard->shard.getAutoScalingGroup())::iterator)))) {
logger.info("No replica that doesn't belong to any shard was found for replica set " + replicaSet.getName()
+ "; spinning one up and waiting for it to become healthy");
additionalReplicaStarted = launchReplicaAndWaitUntilHealthy(replicaSet, Optional.ofNullable(optionalKeyName),
privateKeyEncryptionPassphrase, effectiveReplicaReplicationBearerToken);
additionalReplicaStarted = launchReplicaAndWaitUntilHealthy(replicaSet,
Optional.ofNullable(optionalKeyName), privateKeyEncryptionPassphrase,
effectiveReplicaReplicationBearerToken);
replicasToStopReplicating.add(additionalReplicaStarted);
} else {
additionalReplicaStarted = null;
@@ -1205,12 +1261,12 @@ public class LandscapeServiceImpl implements LandscapeService {
}
/**
* Returns one replica process that is healthy, or {@code null} if no such process was found
* Returns one replica process managed by {@code autoScalingGroup} that is healthy, or {@code null} if no such process was found
*/
private SailingAnalyticsProcess<String> hasHealthyAutoScalingReplica(SailingAnalyticsProcess<String> master, AwsAutoScalingGroup autoScalingGroup) throws Exception {
final HostSupplier<String, SailingAnalyticsHost<String>> hostSupplier = new SailingAnalyticsHostSupplier<>();
for (final SailingAnalyticsProcess<String> replica : master.getReplicas(Landscape.WAIT_FOR_HOST_TIMEOUT, hostSupplier, processFactoryFromHostAndServerDirectory)) {
if (replica.getHost().isManagedByAutoScalingGroup(autoScalingGroup) && replica.isReady(Landscape.WAIT_FOR_HOST_TIMEOUT)) {
if (replica.getHost().isManagedByAutoScalingGroup(Collections.singleton(autoScalingGroup)) && replica.isReady(Landscape.WAIT_FOR_HOST_TIMEOUT)) {
return replica;
}
}
@@ -1245,8 +1301,8 @@ public class LandscapeServiceImpl implements LandscapeService {
if (replicaSet.getAutoScalingGroup() != null) {
final AmazonMachineImage<String> ami = optionalAmi.orElseGet(
()->getLandscape().getLatestImageWithType(region, SharedLandscapeConstants.IMAGE_TYPE_TAG_VALUE_SAILING));
logger.info("Upgrading AMI in auto-scaling group "+replicaSet.getAutoScalingGroup().getName()+" of replica set "+replicaSet.getName()+" to "+ami.getId());
getLandscape().updateImageInAutoScalingGroup(region, replicaSet.getAutoScalingGroup(), replicaSet.getName(), ami);
logger.info("Upgrading AMI in auto-scaling groups "+Util.join(", ", replicaSet.getAllAutoScalingGroups())+" of replica set "+replicaSet.getName()+" to "+ami.getId());
getLandscape().updateImageInAutoScalingGroups(region, replicaSet.getAllAutoScalingGroups(), replicaSet.getName(), ami);
result.add(getLandscape().getApplicationReplicaSet(region, replicaSet.getServerName(), replicaSet.getMaster(), replicaSet.getReplicas()));
} else {
logger.info("No auto-scaling group found for replica set "+replicaSet.getName()+" to update AMI in");
@@ -1301,7 +1357,7 @@ public class LandscapeServiceImpl implements LandscapeService {
final AwsAutoScalingGroup autoScalingGroup = replicaSet.getAutoScalingGroup();
final Set<SailingAnalyticsProcess<String>> nonAutoScalingReplica = new HashSet<>();
for (final SailingAnalyticsProcess<String> replica : replicaSet.getReplicas()) {
if (autoScalingGroup == null || !replica.getHost().isManagedByAutoScalingGroup(autoScalingGroup)) {
if (autoScalingGroup == null || !replica.getHost().isManagedByAutoScalingGroup(Collections.singleton(autoScalingGroup))) {
logger.info("Found replica "+replica+" in replica set "+replicaSet.getName()+
" which is not managed by auto-scaling group");
nonAutoScalingReplica.add(replica);
@@ -1411,29 +1467,31 @@ public class LandscapeServiceImpl implements LandscapeService {
final AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> replicaSet,
InstanceType instanceType) throws Exception {
final AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> result;
final AwsAutoScalingGroup autoScalingGroup = replicaSet.getAutoScalingGroup();
if (autoScalingGroup != null) {
final Iterable<AwsAutoScalingGroup> autoScalingGroups = replicaSet.getAllAutoScalingGroups();
if (!Util.isEmpty(autoScalingGroups)) {
// Don't trust the replicas passed in by the client; it may be stale. Instead, obtain a new set of replicas from the master:
final Iterable<SailingAnalyticsProcess<String>> oldReplicas = replicaSet.getMaster().getReplicas(
Landscape.WAIT_FOR_PROCESS_TIMEOUT, new SailingAnalyticsHostSupplier<String>(),
processFactoryFromHostAndServerDirectory);
getLandscape().updateInstanceTypeInAutoScalingGroup(replicaSet.getMaster().getHost().getRegion(), autoScalingGroup, replicaSet.getName(), instanceType);
final int oldMinSize = autoScalingGroup.getAutoScalingGroup().minSize();
final int newMinSize = autoScalingGroup.getAutoScalingGroup().desiredCapacity() + 1;
getLandscape().updateAutoScalingGroupMinSize(autoScalingGroup, newMinSize);
// Don't trust the replicas passed in by the client; it may be stale. Instead, obtain a new set of replicas from the master:
Iterable<SailingAnalyticsProcess<String>> newSetOfAllReplicas = oldReplicas;
final Set<SailingAnalyticsProcess<String>> terminatedReplicas = new HashSet<>();
for (final SailingAnalyticsProcess<String> replica : oldReplicas) {
final SailingAnalyticsHost<String> replicaHost = replica.getHost();
if (replicaHost.isManagedByAutoScalingGroup(autoScalingGroup)) {
logger.info("Replica "+replica+" is managed by auto-scaling group "+
autoScalingGroup.getName()+" ");
newSetOfAllReplicas = waitUntilAtLeastSoManyAutoScalingReplicasAreReady(replicaSet, newMinSize);
getLandscape().terminate(replicaHost);
terminatedReplicas.add(replica);
getLandscape().updateInstanceTypeInAutoScalingGroup(replicaSet.getMaster().getHost().getRegion(), autoScalingGroups, replicaSet.getName(), instanceType);
for (final AwsAutoScalingGroup autoScalingGroup : autoScalingGroups) {
logger.info("Rolling upgrade of instances for auto-scaling group "+autoScalingGroup.getName()+" to new instance type"+instanceType);
final int oldMinSize = autoScalingGroup.getAutoScalingGroup().minSize();
final int newMinSize = autoScalingGroup.getAutoScalingGroup().desiredCapacity() + 1;
getLandscape().updateAutoScalingGroupMinSize(autoScalingGroup, newMinSize);
for (final SailingAnalyticsProcess<String> replica : oldReplicas) {
final SailingAnalyticsHost<String> replicaHost = replica.getHost();
if (replicaHost.isManagedByAutoScalingGroup(Collections.singleton(autoScalingGroup))) {
logger.info("Replica "+replica+" is managed by auto-scaling group "+autoScalingGroup.getName());
newSetOfAllReplicas = waitUntilAtLeastSoManyAutoScalingReplicasAreReady(replicaSet, autoScalingGroup, newMinSize);
getLandscape().terminate(replicaHost);
terminatedReplicas.add(replica);
}
}
getLandscape().updateAutoScalingGroupMinSize(autoScalingGroup, oldMinSize);
}
getLandscape().updateAutoScalingGroupMinSize(autoScalingGroup, oldMinSize);
result = getLandscape().getApplicationReplicaSet(replicaSet.getMaster().getHost().getRegion(),
replicaSet.getServerName(), replicaSet.getMaster(),
// remove terminated replicas:
@@ -1448,19 +1506,19 @@ public class LandscapeServiceImpl implements LandscapeService {
private Iterable<SailingAnalyticsProcess<String>> waitUntilAtLeastSoManyAutoScalingReplicasAreReady(
final AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> replicaSet,
final int newMinSize) throws Exception {
AwsAutoScalingGroup autoScalingGroup, final int newMinSize) throws Exception {
final SailingAnalyticsProcess<String> master = replicaSet.getMaster();
final AwsAutoScalingGroup autoScalingGroup = replicaSet.getAutoScalingGroup();
assert autoScalingGroup != null;
final Set<SailingAnalyticsProcess<String>> replicas = new HashSet<>();
if (Wait.wait(()->{
int readyAutoScalingReplicas = 0;
try {
replicas.clear();
Util.addAll(master.getReplicas(
Landscape.WAIT_FOR_PROCESS_TIMEOUT, new SailingAnalyticsHostSupplier<String>(),
processFactoryFromHostAndServerDirectory), replicas);
int readyAutoScalingReplicas = 0;
for (final SailingAnalyticsProcess<String> replica : replicas) {
if (replica.getHost().isManagedByAutoScalingGroup(autoScalingGroup)) {
if (replica.getHost().isManagedByAutoScalingGroup(Collections.singleton(autoScalingGroup))) {
if (replica.waitUntilReady(Landscape.WAIT_FOR_PROCESS_TIMEOUT)) {
readyAutoScalingReplicas++;
logger.info("Replica "+replica+" is ready; found "+readyAutoScalingReplicas+"/"+newMinSize+" so far");
@@ -1472,8 +1530,11 @@ public class LandscapeServiceImpl implements LandscapeService {
}
}
}
return readyAutoScalingReplicas >= newMinSize;
}, Landscape.WAIT_FOR_HOST_TIMEOUT, /* duration between attempts */ Duration.ONE_SECOND.times(30),
} catch (TimeoutException timeoutException) {
logger.info("Timeout looking for replicas: "+timeoutException);
}
return readyAutoScalingReplicas >= newMinSize;
}, Landscape.WAIT_FOR_HOST_TIMEOUT, /* duration between attempts */ Duration.ONE_SECOND.times(30),
Level.INFO, "Waiting until at least "+newMinSize+" auto-scaling replicas for replica set "+replicaSet.getName()+" are ready")) {
return replicas;
} else {
@@ -1505,4 +1566,116 @@ public class LandscapeServiceImpl implements LandscapeService {
}
return result;
}
@Override
public SailingServer getSailingServer(String hostname, String username, String password, Optional<Integer> port)
throws MalformedURLException {
final SailingServerFactory fac = sailingServerFactoryTracker.getService();
return fac.getSailingServer(RemoteServerUtil.getBaseServerUrl(hostname,
port.isPresent() ? port.get() : /* defaults to HTTPS */ 443), username, password);
}
@Override
public SailingServer getSailingServer(String hostname, String bearerToken, Optional<Integer> port)
throws MalformedURLException {
final SailingServerFactory fac = sailingServerFactoryTracker.getService();
return fac.getSailingServer(RemoteServerUtil.getBaseServerUrl(hostname,
port.orElse(443 /* defaults to HTTPS */)), bearerToken);
}
private <BuilderT extends CreateShard.Builder<BuilderT, CreateShard<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>>, String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>>> com.sap.sse.landscape.aws.orchestration.CreateShard.Builder<BuilderT, CreateShard<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>>, String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> createShardBuilder() {
return CreateShard.<SailingAnalyticsMetrics, SailingAnalyticsProcess<String>, BuilderT, String> builder();
}
private <BuilderT extends ShardProcedure.Builder<BuilderT, AddShardingKeyToShard<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>>, String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>>> com.sap.sse.landscape.aws.orchestration.ShardProcedure.Builder<BuilderT, AddShardingKeyToShard<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>>, String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> appendShardingKeyToShardBuilder() {
return AddShardingKeyToShard
.<SailingAnalyticsMetrics, SailingAnalyticsProcess<String>, BuilderT, String> builder();
}
private <BuilderT extends ShardProcedure.Builder<BuilderT, RemoveShardingKeyFromShard<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>>, String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>>> com.sap.sse.landscape.aws.orchestration.ShardProcedure.Builder<BuilderT, RemoveShardingKeyFromShard<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>>, String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> removeShardingKeyFromShardBuilder() {
return RemoveShardingKeyFromShard
.<SailingAnalyticsMetrics, SailingAnalyticsProcess<String>, BuilderT, String> builder();
}
@Override
public void removeShardingKeysFromShard(Iterable<String> selectedleaderboards,
AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> applicationReplicaSet,
byte[] passphraseForPrivateKeyDecription, AwsRegion region, String shardName, String bearerToken)
throws Exception {
final SailingServer server = getSailingServer(applicationReplicaSet.getHostname(), bearerToken,
/* HTTPS port */ Optional.of(443));
Set<String> shardingKeys = new HashSet<>();
for (String leaderboardName : selectedleaderboards) {
shardingKeys.add(server.getLeaderboardShardingKey(leaderboardName));
}
removeShardingKeyFromShardBuilder()
.setLandscape(getLandscape())
.setRegion(region)
.setPathPrefixForShardingKey(RemoteServiceMappingConstants.pathPrefixForShardingKey)
.setShardingKeys(shardingKeys)
.setReplicaset(applicationReplicaSet)
.setShardName(shardName)
.setPassphrase(passphraseForPrivateKeyDecription)
.build()
.run();
}
@Override
public void appendShardingKeysToShard(Iterable<String> selectedLeaderboards,
AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> applicationReplicaSet,
byte[] passphraseForPrivateKeyDecription, AwsRegion region, String shardName, String bearerToken)
throws Exception {
final SailingServer server = getSailingServer(applicationReplicaSet.getHostname(), bearerToken,
/* HTTPS port */ Optional.of(443));
final Set<String> shardingkeys = new HashSet<String>();
for (String s : selectedLeaderboards) {
shardingkeys.add(server.getLeaderboardShardingKey(s));
}
appendShardingKeyToShardBuilder()
.setLandscape(getLandscape())
.setRegion(region)
.setPathPrefixForShardingKey(RemoteServiceMappingConstants.pathPrefixForShardingKey)
.setShardingKeys(shardingkeys)
.setReplicaset(applicationReplicaSet)
.setShardName(shardName)
.setPassphrase(passphraseForPrivateKeyDecription)
.build()
.run();
}
@Override
public void removeShard(
AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> applicationReplicaSet,
String shardTargetGroupArn) throws Exception {
for (Entry<AwsShard<String>, Iterable<String>> entry : applicationReplicaSet.getShards().entrySet()) {
if (shardTargetGroupArn.equals(entry.getKey().getTargetGroup().getTargetGroupArn())) {
applicationReplicaSet.removeShard(entry.getKey(), getLandscape());
return;
}
}
}
@Override
public void addShard(Iterable<String> selectedLeaderboardNames,
AwsApplicationReplicaSet<String, SailingAnalyticsMetrics, SailingAnalyticsProcess<String>> applicationReplicaSet,
AwsRegion region, String bearerToken, byte[] passphraseForPrivateKeyDecription, String shardName)
throws Exception {
final SailingServer server = getSailingServer(applicationReplicaSet.getHostname(), bearerToken,
Optional.of(443));
final Set<String> shardingkeys = new HashSet<String>();
for (final String s : selectedLeaderboardNames) {
shardingkeys.add(server.getLeaderboardShardingKey(s));
}
createShardBuilder()
.setLandscape(getLandscape())
.setTargetGroupNamePrefix(LandscapeService.SAILING_TARGET_GROUP_NAME_PREFIX)
.setShardingKeys(shardingkeys)
.setReplicaset(applicationReplicaSet)
.setRegion(region)
.setPathPrefixForShardingKey(RemoteServiceMappingConstants.pathPrefixForShardingKey)
.setShardName(shardName)
.setPassphrase(passphraseForPrivateKeyDecription)
.build()
.run();
}
}
@@ -8,6 +8,7 @@ import com.sap.sse.landscape.application.ApplicationProcessMetrics;
import com.sap.sse.landscape.application.ApplicationReplicaSet;
import com.sap.sse.landscape.aws.AmazonMachineImage;
import com.sap.sse.landscape.aws.AwsApplicationProcess;
import com.sap.sse.landscape.aws.AwsAutoScalingGroup;
import com.sap.sse.landscape.aws.AwsLandscape;
import com.sap.sse.landscape.aws.Tags;
import com.sap.sse.landscape.aws.TargetGroup;
@@ -37,7 +38,6 @@ extends AbstractProcedureImpl<ShardingKey>
implements Procedure<ShardingKey> {
private static final int DEFAULT_MIN_REPLICAS = 1;
private static final int DEFAULT_MAX_REPLICAS = 30;
private static final int DEFAULT_MAX_REQUESTS_PER_TARGET = 15000;
/**
*
@@ -66,8 +66,8 @@ implements Procedure<ShardingKey> {
BuilderT setMaxReplicas(int maxReplicas);
/**
* Defines the scaling threshold based on the number of requests per target per minute. Defaults to 30,000 (see
* {@link CreateLaunchConfigurationAndAutoScalingGroup#DEFAULT_MAX_REQUESTS_PER_TARGET}).
* Defines the scaling threshold based on the number of requests per target per minute. Defaults to 15,000 (see
* {@link AwsAutoScalingGroup#DEFAULT_MAX_REQUESTS_PER_TARGET}).
*/
BuilderT setMaxRequestsPerTarget(int maxRequestsPerTarget);
}
@@ -85,7 +85,7 @@ implements Procedure<ShardingKey> {
private Optional<Tags> tags;
private int minReplicas = DEFAULT_MIN_REPLICAS;
private int maxReplicas = DEFAULT_MAX_REPLICAS;
private int maxRequestsPerTarget = DEFAULT_MAX_REQUESTS_PER_TARGET;
private int maxRequestsPerTarget = AwsAutoScalingGroup.DEFAULT_MAX_REQUESTS_PER_TARGET;
public BuilderImpl(AwsLandscape<ShardingKey> landscape, Region region, String replicaSetName,
TargetGroup<ShardingKey> targetGroup) {
@@ -39,6 +39,12 @@ public interface SailingServer extends SecuredServer {
URL getBaseUrl();
Iterable<UUID> getLeaderboardGroupIds() throws Exception;
Iterable<String> getLeaderboardNames() throws Exception;
String getLeaderboardShardingKey(String leaderboardName) throws Exception;
String getLeaderboardFromShardingKey(String shardingKey) throws Exception;
Iterable<UUID> getEventIds() throws Exception;
@@ -4,8 +4,10 @@ import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
@@ -25,6 +27,7 @@ import org.json.simple.parser.ParseException;
import com.sap.sailing.domain.base.RemoteSailingServerReference;
import com.sap.sailing.domain.common.DataImportProgress;
import com.sap.sailing.domain.common.sharding.ShardingType;
import com.sap.sailing.server.gateway.deserialization.impl.CompareServersResultJsonDeserializer;
import com.sap.sailing.server.gateway.deserialization.impl.DataImportProgressJsonDeserializer;
import com.sap.sailing.server.gateway.deserialization.impl.MasterDataImportResultJsonDeserializer;
@@ -35,6 +38,7 @@ import com.sap.sailing.server.gateway.interfaces.SailingServer;
import com.sap.sailing.server.gateway.jaxrs.api.CompareServersResource;
import com.sap.sailing.server.gateway.jaxrs.api.EventsResource;
import com.sap.sailing.server.gateway.jaxrs.api.LeaderboardGroupsResource;
import com.sap.sailing.server.gateway.jaxrs.api.LeaderboardsResource;
import com.sap.sailing.server.gateway.jaxrs.api.MasterDataImportResource;
import com.sap.sailing.server.gateway.jaxrs.api.RemoteServerReferenceResource;
import com.sap.sailing.server.gateway.serialization.LeaderboardGroupConstants;
@@ -56,11 +60,42 @@ public class SailingServerImpl extends SecuredServerImpl implements SailingServe
@Override
public Iterable<UUID> getLeaderboardGroupIds() throws ClientProtocolException, IOException, ParseException {
final URL leaderboardGroupsUrl = new URL(getBaseUrl(), GATEWAY_URL_PREFIX+LeaderboardGroupsResource.V1_LEADERBOARDGROUPS+LeaderboardGroupsResource.IDENTIFIABLE);
final HttpGet getLeaderboards = new HttpGet(leaderboardGroupsUrl.toString());
final JSONArray jsonResponse = (JSONArray) getJsonParsedResponse(getLeaderboards).getA();
final HttpGet getLeaderboardGroups = new HttpGet(leaderboardGroupsUrl.toString());
final JSONArray jsonResponse = (JSONArray) getJsonParsedResponse(getLeaderboardGroups).getA();
return Util.map(jsonResponse, o->UUID.fromString(((JSONObject) o).get(LeaderboardGroupConstants.ID).toString()));
}
@Override
public Iterable<String> getLeaderboardNames() throws Exception {
final URL leaderboardsUrl = new URL(getBaseUrl(), GATEWAY_URL_PREFIX+LeaderboardsResource.V1_LEADERBOARDS);
final HttpGet getLeaderboards = new HttpGet(leaderboardsUrl.toString());
final Pair<Object, Integer> jsonParsedResponse = getJsonParsedResponse(getLeaderboards);
final JSONArray jsonResponse = (JSONArray) jsonParsedResponse.getA();
if (jsonResponse == null) {
throw new IllegalAccessError("Error accessing leaderboard names; response status "+jsonParsedResponse.getB());
}
return Util.map(jsonResponse, o->o.toString());
}
@Override
public String getLeaderboardShardingKey(String leaderboardName) throws Exception {
// We could try to acquire this from the "leaderboards" REST API endpoint, field shardingLeaderboardName,
// but we can as well shortcut it by replicating the implementation here:
return ShardingType.LEADERBOARDNAME.encodeShardingInfo(leaderboardName);
}
/**
* Establishes a map with all leaderboardnames and their keys and takes the value for the according shardingkey.
*/
@Override
public String getLeaderboardFromShardingKey(String shardingKey) throws Exception {
Map<String, String> mapping = new HashMap<>();
for (String leaderboard : getLeaderboardNames()) {
mapping.put(getLeaderboardFromShardingKey(leaderboard), leaderboard);
}
return mapping.get(shardingKey);
}
@Override
public Iterable<UUID> getEventIds() throws ClientProtocolException, IOException, ParseException {
final URL eventsUrl = new URL(getBaseUrl(), GATEWAY_URL_PREFIX+EventsResource.V1_EVENTS);
@@ -138,7 +138,7 @@ public abstract class AbstractLeaderboardsResource extends AbstractSailingServer
jsonLeaderboard.put("delayToLiveInMillis", leaderboard.getDelayToLiveInMillis());
jsonLeaderboard.put("resultState", resultState.name());
jsonLeaderboard.put("type", leaderboard.getLeaderboardType().name());
jsonLeaderboard.put("shardingLeaderboardName", ShardingType.LEADERBOARDNAME.encodeIfNeeded(leaderboard.getName()));
jsonLeaderboard.put("shardingLeaderboardName", ShardingType.LEADERBOARDNAME.encodeShardingInfo(leaderboard.getName()));
final ResultDiscardingRule resultDiscardingRule = leaderboard.getResultDiscardingRule();
if (resultDiscardingRule instanceof ThresholdBasedResultDiscardingRule) {
final ThresholdBasedResultDiscardingRule thresholdBasedResultDiscardingRule = (ThresholdBasedResultDiscardingRule) resultDiscardingRule;
@@ -161,10 +161,11 @@ import com.sap.sse.security.shared.impl.UserGroup;
import com.sap.sse.shared.json.JsonDeserializationException;
import com.sap.sse.shared.util.impl.UUIDHelper;
@Path("/v1/leaderboards")
@Path(LeaderboardsResource.V1_LEADERBOARDS)
public class LeaderboardsResource extends AbstractLeaderboardsResource {
private static final Logger logger = Logger.getLogger(LeaderboardsResource.class.getName());
public static final String V1_LEADERBOARDS = "/v1/leaderboards";
/**
* When an {@link #createAutoCourse(TrackedRace, RegattaLog) automatic course inference} is requested,
* the COGs of competitors are analyzed to obtain a direction and length for a start and finish line.
@@ -8,10 +8,10 @@ import com.sap.sailing.domain.common.sharding.ShardingType;
public class ShardingLeaderBoardEncodingTest {
@Test
public void testEncoding() {
Assert.assertEquals("/leaderboard/pureascistring", ShardingType.LEADERBOARDNAME.encodeIfNeeded("pureascistring"));
Assert.assertEquals("/leaderboard/unpure_asci_string", ShardingType.LEADERBOARDNAME.encodeIfNeeded("unpure asci string"));
Assert.assertEquals("/leaderboard/c_dille", ShardingType.LEADERBOARDNAME.encodeIfNeeded("cédille"));
Assert.assertEquals("/leaderboard/Hello_World", ShardingType.LEADERBOARDNAME.encodeIfNeeded("Hello+World"));
Assert.assertEquals("/leaderboard/Hello_World_", ShardingType.LEADERBOARDNAME.encodeIfNeeded("Hello(World)"));
Assert.assertEquals("/leaderboard/pureascistring", ShardingType.LEADERBOARDNAME.encodeShardingInfo("pureascistring"));
Assert.assertEquals("/leaderboard/unpure_asci_string", ShardingType.LEADERBOARDNAME.encodeShardingInfo("unpure asci string"));
Assert.assertEquals("/leaderboard/c_dille", ShardingType.LEADERBOARDNAME.encodeShardingInfo("cédille"));
Assert.assertEquals("/leaderboard/Hello_World", ShardingType.LEADERBOARDNAME.encodeShardingInfo("Hello+World"));
Assert.assertEquals("/leaderboard/Hello_World_", ShardingType.LEADERBOARDNAME.encodeShardingInfo("Hello(World)"));
}
}
@@ -23,6 +23,24 @@
<div class="mainContent">
<h2 class="releaseHeadline">Release Notes - Administration Console</h2>
<div class="innerContent">
<h2 class="articleSubheadline">January 2023</h2>
<ul class="bulletList">
<li>Support for managing sharded replication in the "Landscape" panel of the "Advanced" category in the
<a href="https://www.sapsailing.com/gwt/AdminConsole.html#LandscapeManagementPlace:">admin console</a>),
offered as a new icon in the "Action" column of the replica sets table: in the new dialog that pops
up you can create and remove so-called "shards" to which you can assign zero or more leaderboards
offered by that replica set. Each shard gets its own auto-scaling group and target group in addition to the regular
auto-scaling group / target group pair used for the public target group. By default a minimum of two instances
are required by each shard to ensure availability. They are created with the same launch configuration
used by the replica set's regular public target group. When a shard's instances are healthy,
rules are managed in the replica set's load balancer to route traffic regarding the leaderboards
selected for the shard to the shard's target group. This reduces traffic on the default public
target group, and the instances of the shard can focus on the sub-set of leaderboards configured
instead of having to compute all leaderboards in parallel. This helps reduce the leaderboard
re-calculation times for live events with parallel live races in several classes and tens of
competitors per leaderboard.
</li>
</ul>
<h2 class="articleSubheadline">December 2022</h2>
<ul class="bulletList">
<li>Added Czech language support</li>
@@ -785,10 +785,20 @@ public class Util {
}
return result;
}
/**
*
* @param <T>
* Type of {@code iterable}
* @param iterable
* Input Iterable
* @return
* returns List<T> if {@code iterable} is an instance of List<?> and if it is an instance Serializable. If not,
* an ArrayList<T> gets constructed and filled with all items of {@code iterable}
*/
public static <T> List<T> asList(Iterable<T> iterable) {
final List<T> list;
if (iterable instanceof List<?>) {
if (iterable instanceof List<?> && iterable instanceof Serializable) {
list = (List<T>) iterable;
} else {
list = new ArrayList<>();
Binary file not shown.

After

Width:  |  Height:  |  Size: 245 B

@@ -58,4 +58,7 @@ public interface IconResources extends ClientBundle {
@Source("images/move.png")
ImageResource moveIcon();
@Source("images/shardmanagement.png")
ImageResource shardManagementIcon();
}
@@ -203,6 +203,9 @@ public abstract class TableWrapper<T, S extends RefreshableSelectionModel<T>, SM
}
}
/**
* Remove all items from this table's data model
*/
public void clear() {
getDataProvider().getList().clear();
}
@@ -180,7 +180,7 @@ public abstract class DataEntryDialog<T> {
okButton.addClickHandler(new ClickHandler() {
public void onClick(ClickEvent event) {
// wait for any outstanding validation request and check last validation result; call OK only if the pending validation was OK
ifLastValidationRequestSuccesssful(()->{
ifLastValidationRequestSuccessful(()->{
dataEntryDialog.hide();
if (callback != null) {
callback.ok(getResult());
@@ -195,7 +195,7 @@ public abstract class DataEntryDialog<T> {
* call {@code callback}. If an action is still pending in the {@link #validationExecutor}, wait until no more
* action is pending and invoke {@code callback} if the last validation state was OK.
*/
protected void ifLastValidationRequestSuccesssful(Runnable callback) {
protected void ifLastValidationRequestSuccessful(Runnable callback) {
validationExecutor.runAfterLastActionReturned(VALIDATION_ACTION_CATEGORY, ()->{
if (!dialogInInvalidState) {
callback.run();
@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8"/>
<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
<classpathentry kind="src" path="src">
<attributes>
<attribute name="test" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="bin"/>
</classpath>
@@ -0,0 +1,28 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>com.sap.sse.landscape.aws.common.test</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.pde.ManifestBuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.pde.SchemaBuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.pde.PluginNature</nature>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>
@@ -0,0 +1,2 @@
eclipse.preferences.version=1
encoding/<project>=UTF-8
@@ -0,0 +1,8 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.release=disabled
org.eclipse.jdt.core.compiler.source=1.8
@@ -0,0 +1,12 @@
Manifest-Version: 1.0
Bundle-ManifestVersion: 2
Bundle-Name: LandscapeAWSCommonTest
Bundle-SymbolicName: com.sap.sse.landscape.aws.common.test
Bundle-Version: 1.0.0.qualifier
Bundle-Vendor: SAP
Fragment-Host: com.sap.sse.landscape.aws.common
Import-Package: org.hamcrest;version="2.2.0",
org.junit;version="4.13.2",
org.junit.function;version="4.13.2"
Automatic-Module-Name: com.sap.sse.landscape.aws.common.test
Bundle-RequiredExecutionEnvironment: JavaSE-1.8
@@ -0,0 +1,4 @@
source.. = src/
output.. = bin/
bin.includes = META-INF/,\
.
+12
View File
@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>root</artifactId>
<groupId>com.sap.sailing</groupId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>com.sap.sse.landscape.aws.common.test</artifactId>
<packaging>eclipse-test-plugin</packaging>
</project>
@@ -0,0 +1,58 @@
package com.sap.sse.landscape.aws.common.shared;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import org.junit.Test;
public class ShardTargetGroupNameTest {
private static final String TARGET_GROUP_NAME_PREFIX = "TG-";
@Test
public void testCreateAndParse() {
final String replicaSetName = "abc";
final String shardName = "def";
final ShardTargetGroupName theShardName = ShardTargetGroupName.create(replicaSetName, shardName, TARGET_GROUP_NAME_PREFIX);
final ShardTargetGroupName parsedShardName = ShardTargetGroupName.parse(theShardName.getTargetGroupName(), null);
assertEquals(shardName, parsedShardName.getShardName());
assertEquals(theShardName.getTargetGroupName(), parsedShardName.getTargetGroupName());
}
@Test
public void testCreateAndParseWithSimpleSeparatorInNames() {
final String replicaSetName = "abc"+ShardTargetGroupName.SEPARATOR+"def";
final String shardName = "def"+ShardTargetGroupName.SEPARATOR+"ghi";
assertThrows(IllegalArgumentException.class, ()->{
ShardTargetGroupName.create(replicaSetName, shardName, TARGET_GROUP_NAME_PREFIX);
});
}
@Test
public void testCreateAndParseLongShardName() {
final String replicaSetName = "abc";
final String shardName = "deflawencawueclkajhfalskjfhlaskjdhfllakj";
final ShardTargetGroupName theShardName = ShardTargetGroupName.create(replicaSetName, shardName, TARGET_GROUP_NAME_PREFIX);
final ShardTargetGroupName parsedShardName = ShardTargetGroupName.parse(theShardName.getTargetGroupName(), null);
assertEquals(theShardName.getTargetGroupName(), parsedShardName.getTargetGroupName());
final String shardNamePrefix = parsedShardName.getShardName().substring(0,
parsedShardName.getShardName().indexOf(ShardTargetGroupName.NAMESEPARATOR));
final String shardNameSuffix = parsedShardName.getShardName()
.substring(parsedShardName.getShardName().indexOf(ShardTargetGroupName.NAMESEPARATOR)
+ ShardTargetGroupName.NAMESEPARATOR.length());
assertTrue(shardName.startsWith(shardNamePrefix));
assertTrue(shardName.endsWith(shardNameSuffix));
}
@Test
public void testValidTargetGroupName() {
assertTrue(ShardTargetGroupName.isValidShardTargetGroupName("S-Humba-Trala"));
assertTrue(ShardTargetGroupName.isValidShardTargetGroupName("S-Humba-Tr--a"));
assertTrue(ShardTargetGroupName.isValidShardTargetGroupName("S-Humba-Tr--"));
assertTrue(ShardTargetGroupName.isValidShardTargetGroupName("Humba-Trala"));
assertTrue(ShardTargetGroupName.isValidShardTargetGroupName("Humba-Tr--a"));
assertTrue(ShardTargetGroupName.isValidShardTargetGroupName("Humba-Tr--"));
assertFalse(ShardTargetGroupName.isValidShardTargetGroupName("S-Humba-Tr--a-a"));
assertFalse(ShardTargetGroupName.isValidShardTargetGroupName("Humba-Tr--a-a"));
}
}
@@ -0,0 +1,153 @@
package com.sap.sse.landscape.aws.common.shared;
import com.sap.sse.common.Util;
/**
* A wrapper class for an {@link AwsShard}'s target group name and the shard's name. {@link #parse} and create
* {@link #create} are used for encoding/decoding a shard's name by it's target group, replica set name and the target
* group tag with the key {@link #TAG_KEY} and creating a {@link ShardTargetGroupName} by a replica set and the shard
* name. Those created values should be used as names in the AWS landscape.<p>
*
* Only a-z, A-Z and 0-9 are allowed in a shard name due to encoding and there are target group length restrictions.
*
* @author I569653
*
*/
public class ShardTargetGroupName {
public final static String TAG_KEY = "shardname";
static final String NAMESEPARATOR = "--";
static final String SEPARATOR = "-";
private final String shardName;
private final String targetGroupname;
private ShardTargetGroupName(String shardName, String targetGroupname) {
this.shardName = shardName;
this.targetGroupname = targetGroupname;
}
/**
* This method is supposed to convert a target group name into a {@link ShardTargetGroupName} via slicing the name
* into the replica set name and the shard name. And if there is a non-{@code null} {@code tagString} given, this is
* supposed to be the shard's name. So in that case it does not get parsed out of the target group name.
*
* @param shardTargetGroupName
* name read from a target group in AWS.
* @param tagString
* tag value with the key {@link TAG_KEY}
*/
public static ShardTargetGroupName parse(String shardTargetGroupName, String tagString) {
// Possible options: {SomeOptionalPrefix}{ReplicaSetName}-{Shardname} or {SomeOptionalPrefix}{ReplicaSetName}-{Shardname-Prefix}--{ShardnameSuffix}
// where the prefix/suffix must neither contain the SEPARATOR nor the NAMESEPARATOR
assert isValidShardTargetGroupName(shardTargetGroupName);
final String shardname;
if (Util.hasLength(tagString)) {
shardname = tagString;
} else {
final int lastIndexExclusiveToSearchForSeparator =
shardTargetGroupName.contains(NAMESEPARATOR) ? shardTargetGroupName.indexOf(NAMESEPARATOR) : shardTargetGroupName.length();
final int idxHyphen1 = shardTargetGroupName.substring(0, lastIndexExclusiveToSearchForSeparator).lastIndexOf(SEPARATOR);
shardname = shardTargetGroupName.substring(idxHyphen1 + 1);
}
return new ShardTargetGroupName(shardname, shardTargetGroupName);
}
/**
* Creates a ShardName from {@code replicaSetName} and {@code shardname}. {@code shardname} should only contain a-z,
* A-Z and numbers. This function builds a valid shard target group name from those inputs. This follows one of two
* patterns: If the {@link TargetGroup#MAX_TARGETGROUP_NAME_LENGTH} is sufficient, after the common
* {@link TargetGroup#SAILING_TARGET_GROUP_NAME_PREFIX} the replica set name and the shard name are concatenated,
* each separated by a single {@link #SEPARATOR} character. If the maximum target group name length is too short for
* this pattern, only a prefix and postfix of the shard name are used, concatenated by the {@link #NAMESEPARATOR}
* string. The prefix and postfix length are then chosen such that the maximum target group name limit is met
* exactly. If not even a single shard name character can be used without exceeding the maximum target group name
* length, an {@link IllegalArgumentException} is thrown.
*
* @param replicaSetName
* Replica set name of the shard
* @param shardName
* shard name
* @param targetGroupNamePrefix
* a prefix prepended to the {@code replicaSetName}; must not be {@code null} but may be an empty string
* @return a composite {@link ShardTargetGroupName} object that has the {@code shardName} as well as the
* {@link ShardTargetGroupName#getTargetGroupName() target group name} resulting from the construction rules
* encapsulated by this class
* @throws IllegalArgumentException
* Gets thrown if {@code shardName} is not valid or the combination is invalid. This happens if neither
* pattern can be applied.
*/
public static ShardTargetGroupName create(String replicaSetName, String shardName, String targetGroupNamePrefix) throws IllegalArgumentException {
if (!isValidTargetGroupNamePrefix(targetGroupNamePrefix)) {
throw new IllegalArgumentException("Target group name prefix "+targetGroupNamePrefix+" is not allowed");
}
if (!shardName.matches("[a-zA-Z0-9]*")) {
throw new IllegalArgumentException("Only a-z, A-Z and 0-9 characters are allowed in shard name");
}
if (shardName.endsWith(TargetGroupConstants.MASTER_SUFFIX) || shardName.endsWith(TargetGroupConstants.TEMP_SUFFIX)){
throw new IllegalArgumentException(TargetGroupConstants.MASTER_SUFFIX + " and " + TargetGroupConstants.TEMP_SUFFIX + " are not allowed at the end of shard names");
}
final String targetGroupNameBySimpleConcatenation = targetGroupNamePrefix + replicaSetName + SEPARATOR + shardName;
final String targetGroupName;
if (targetGroupNameBySimpleConcatenation.length() <= TargetGroupConstants.MAX_TARGETGROUP_NAME_LENGTH) {
targetGroupName = targetGroupNameBySimpleConcatenation;
} else {
final String targetGroupNamePrefixForElidedShardNameMiddle = targetGroupNamePrefix + replicaSetName + SEPARATOR;
if (targetGroupNamePrefixForElidedShardNameMiddle.length() + NAMESEPARATOR.length() >= TargetGroupConstants.MAX_TARGETGROUP_NAME_LENGTH) {
throw new IllegalArgumentException(
"Cannot add shard name prefix/postfix to replica set name "+replicaSetName+
" as it would exceed the maximum target group name limit of "+TargetGroupConstants.MAX_TARGETGROUP_NAME_LENGTH+" characters");
}
final int remainingCharactersAfterAddingNameSeparator = TargetGroupConstants.MAX_TARGETGROUP_NAME_LENGTH
- targetGroupNamePrefixForElidedShardNameMiddle.length() - NAMESEPARATOR.length();
final int prefixLength = remainingCharactersAfterAddingNameSeparator/2 + remainingCharactersAfterAddingNameSeparator%2; // prefer prefix in odd cases
final int postfixLength = remainingCharactersAfterAddingNameSeparator/2;
targetGroupName = targetGroupNamePrefix + replicaSetName + SEPARATOR + shardName.substring(0, prefixLength)
+ NAMESEPARATOR + shardName.substring(shardName.length() - postfixLength);
}
assert isValidShardTargetGroupName(targetGroupName);
return new ShardTargetGroupName(shardName, targetGroupName);
}
public String getName() {
return shardName;
}
public String getTargetGroupName() {
return targetGroupname;
}
public String getShardName() {
return shardName;
}
/**
* Checks whether {@code prefix} is a valid prefix for a target group name. For this, the prefix must not contain
* the {@link #NAMESEPARATOR} and its length must be short enough that there is at least one letter remaining for
* the replica set name and one letter for the shard name, including the {@link #SEPARATOR} used to separate the
* replica set name from the shard name. See also {@link TargetGroupConstants#MAX_TARGETGROUP_NAME_LENGTH}.
*/
public static boolean isValidTargetGroupNamePrefix(String prefix) {
return !prefix.contains(NAMESEPARATOR)
&& prefix.length() + 1 + SEPARATOR.length() + 1 <= TargetGroupConstants.MAX_TARGETGROUP_NAME_LENGTH;
}
/**
* Checks {@code name} for being a valid name for a shard's target group. The name must neither end with the
* {@link TargetGroupConstants#MASTER_SUFFIX suffix used by the "master" target group} nor with the
* {@link TargetGroupConstants#TEMP_SUFFIX suffix for temporary target groups}. Furthermore, the name is expected to
* contain at least one {@link #SEPARATOR} that separates the replica set name from the shard name; and if the shard
* name had to be shortened into a prefix / suffix separated by {@link #NAMESEPARATOR} then that prefix/suffix
* separator needs to come after the last single occurrence of the {@link #SEPARATOR} separating the shard name from
* the replica set name.
* <p>
*
* No assumptions are made here about any target group name prefix which may itself contain a {@link #SEPARATOR} but
* not a {@link #NAMESEPARATOR} character. See also {@link #isValidTargetGroupNamePrefix}.
*/
public static boolean isValidShardTargetGroupName(String name) {
return name.length() <= TargetGroupConstants.MAX_TARGETGROUP_NAME_LENGTH
&& !name.endsWith(TargetGroupConstants.MASTER_SUFFIX)
&& !name.endsWith(TargetGroupConstants.TEMP_SUFFIX)
&& name.contains(SEPARATOR)
&& (!name.contains(NAMESEPARATOR) || name.indexOf(SEPARATOR, name.indexOf(NAMESEPARATOR)+NAMESEPARATOR.length()) < 0);
}
}
@@ -0,0 +1,7 @@
package com.sap.sse.landscape.aws.common.shared;
public interface TargetGroupConstants {
int MAX_TARGETGROUP_NAME_LENGTH = 32;
String MASTER_SUFFIX = "-m";
String TEMP_SUFFIX = "-TMP";
}
@@ -44,6 +44,7 @@ import com.sap.sse.common.Util;
import com.sap.sse.landscape.Landscape;
import com.sap.sse.landscape.Release;
import com.sap.sse.landscape.RotatingFileBasedLog;
import com.sap.sse.landscape.application.ApplicationProcess;
import com.sap.sse.landscape.aws.impl.AwsRegion;
import com.sap.sse.landscape.aws.orchestration.CreateDNSBasedLoadBalancerMapping;
import com.sap.sse.landscape.impl.ReleaseRepositoryImpl;
@@ -68,7 +69,7 @@ import software.amazon.awssdk.services.sts.model.Credentials;
* Tests for the AWS SDK landscape wrapper in bundle {@code com.sap.sse.landscape.aws}. To run these tests
* successfully it is necessary to have valid AWS credentials for region {@code EU_WEST_2} that allow the
* AWS user account to create keys and launch instances, etc. These are to be provided as explained
* in the documentation of {@link AwsLandscape#obtain()}. You will need an MFA token generator, too,
* in the documentation of {@link AwsLandscape#obtain(String)}. You will need an MFA token generator, too,
* such as the Google Authenticator.<p>
*
* Run the test by providing your AWS access key ID as system property {@code com.sap.sse.landscape.aws.accesskeyid},
@@ -92,6 +93,7 @@ public class ConnectivityTest<ProcessT extends AwsApplicationProcess<String, Sai
private static byte[] keyPass;
private static String AXELS_KEY_PASS;
private static final String AXELS_KEY_NAME = "Axel";
private static final String pathPrefixForShardingKey = "/sse/landscape/test";
@BeforeClass
public static void setUp() {
@@ -99,18 +101,18 @@ public class ConnectivityTest<ProcessT extends AwsApplicationProcess<String, Sai
landscape = AwsLandscape.obtain(
System.getProperty(AwsLandscape.ACCESS_KEY_ID_SYSTEM_PROPERTY_NAME),
System.getProperty(AwsLandscape.SECRET_ACCESS_KEY_SYSTEM_PROPERTY_NAME),
System.getProperty(AwsLandscape.SESSION_TOKEN_SYSTEM_PROPERTY_NAME));
System.getProperty(AwsLandscape.SESSION_TOKEN_SYSTEM_PROPERTY_NAME), pathPrefixForShardingKey);
} else {
final AwsLandscape<String> tmpLandscape = AwsLandscape.obtain(
System.getProperty(AwsLandscape.ACCESS_KEY_ID_SYSTEM_PROPERTY_NAME),
System.getProperty(AwsLandscape.SECRET_ACCESS_KEY_SYSTEM_PROPERTY_NAME));
System.getProperty(AwsLandscape.SECRET_ACCESS_KEY_SYSTEM_PROPERTY_NAME), pathPrefixForShardingKey);
final Credentials credentials = tmpLandscape.getMfaSessionCredentials(
System.getProperty(AwsLandscape.MFA_TOKEN_CODE_SYSTEM_PROPERTY_NAME));
logger.info("-D"+AwsLandscape.ACCESS_KEY_ID_SYSTEM_PROPERTY_NAME+"="+
credentials.accessKeyId()+" -D"+AwsLandscape.SECRET_ACCESS_KEY_SYSTEM_PROPERTY_NAME+"="+
credentials.secretAccessKey()+" -D"+AwsLandscape.SESSION_TOKEN_SYSTEM_PROPERTY_NAME+"="+
credentials.sessionToken());
landscape = AwsLandscape.obtain(credentials.accessKeyId(), credentials.secretAccessKey(), credentials.sessionToken());
landscape = AwsLandscape.obtain(credentials.accessKeyId(), credentials.secretAccessKey(), credentials.sessionToken(), pathPrefixForShardingKey);
}
region = new AwsRegion(Region.EU_WEST_2, landscape);
AXELS_KEY_PASS = new String(Base64.getDecoder().decode(System.getProperty("axelskeypassphrase")));
@@ -486,7 +488,7 @@ public class ConnectivityTest<ProcessT extends AwsApplicationProcess<String, Sai
@Test
public void createAndDeleteTargetGroupTest() {
final String targetGroupName = "TestTargetGroup-"+new Random().nextInt();
final TargetGroup<String> targetGroup = landscape.createTargetGroup(region, targetGroupName, 80, "/gwt/status", 80,
final TargetGroup<String> targetGroup = landscape.createTargetGroup(region, targetGroupName, 80, ApplicationProcess.HEALTH_CHECK_PATH, 80,
/* loadBalancerArn */ null);
try {
final TargetGroup<String> fetchedTargetGroup = landscape.getTargetGroup(region, targetGroupName,
@@ -1,6 +1,7 @@
package com.sap.sse.landscape.aws;
import java.util.Optional;
import java.util.regex.Pattern;
import com.sap.sse.common.Named;
import com.sap.sse.landscape.Region;
@@ -31,10 +32,26 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleConditio
* will only see the non-default rule set of the HTTPS listener which is used to dynamically configure the
* landscape.
*
* On https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-limits.html are some restrictions
* listed for the load balanancer's setup.
*
* @author Axel Uhl (D043530)
*
*/
public interface ApplicationLoadBalancer<ShardingKey> extends Named {
static final int MAX_RULES_PER_LOADBALANCER = 100;
static final String DNS_MAPPED_ALB_NAME_PREFIX = "DNSMapped-";
static final Pattern ALB_NAME_PATTERN = Pattern.compile(DNS_MAPPED_ALB_NAME_PREFIX+"(.*)$");
static final int MAX_ALBS_PER_REGION = 20;
static final int MAX_CONDITIONS_PER_RULE = 5;
public static final String DEFAULT_RULE_PRIORITY = "Default";
/**
* The maximum {@link Rule#priority()} that can be used within a listener
*/
public static final int MAX_PRIORITY = 50000;
/**
* The DNS name of this load balancer; can be used, e.g., to set a CNAME DNS record pointing
* to this load balancer.
@@ -89,6 +106,30 @@ public interface ApplicationLoadBalancer<ShardingKey> extends Named {
*/
Iterable<Rule> addRulesAssigningUnusedPriorities(boolean forceContiguous, Rule... rules);
/**
* For inserting e.g. Shard at a specific priority, we must ensure that the priority is unique in the ruleset.
* This function shifts every priority starting at the highest priority one higher for making the priority at {@code index}
* free.
* @param index
* index supposed to be free
* @throws IllegalStateException
* gets thrown if shifting exceeds the limit of priorities
*/
void shiftRulesToMakeSpaceAt(int index) throws IllegalStateException;
/**
* Returns the priority which should be used as the next sharding priority.
* @param hostname
* hostname of replica set from which the shard gets created
* @return priority of a rule with hostname as Host +1, if the rule it was found in is a redirect or the index of the Rule if it is a Forward.
* if there is no Rule with this hostname, just return the index after the last rule. This index should be used as the next shard priority.
* You may need to make space at this priority via {@link #shiftRulesToMakeSpaceAt(int)}
* @throws Exception
* if the found priority is higher than the {@code MAX_PRIORITY}
*/
int getFirstShardingPriority(String hostname) throws IllegalStateException;
Iterable<TargetGroup<ShardingKey>> getTargetGroups();
/**
@@ -117,4 +158,8 @@ public interface ApplicationLoadBalancer<ShardingKey> extends Named {
Rule getDefaultRedirectRule(String hostName, PlainRedirectDTO plainRedirectDTO);
RuleCondition createHostHeaderRuleCondition(String hostname);
Iterable<Rule> getRulesForTargetGroups(Iterable<TargetGroup<ShardingKey>> targetGroups);
Iterable<Rule> replaceTargetGroupInForwardRules(TargetGroup<ShardingKey> oldTargetGroup, TargetGroup<ShardingKey> newTargetGroup );
}
@@ -1,18 +1,27 @@
package com.sap.sse.landscape.aws;
import java.util.HashSet;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import com.sap.sse.ServerInfo;
import com.sap.sse.common.Duration;
import com.sap.sse.common.Util;
import com.sap.sse.landscape.Process;
import com.sap.sse.landscape.Region;
import com.sap.sse.landscape.application.ApplicationProcess;
import com.sap.sse.landscape.application.ApplicationProcessMetrics;
import com.sap.sse.landscape.application.ApplicationReplicaSet;
import com.sap.sse.landscape.aws.common.shared.ShardTargetGroupName;
import com.sap.sse.landscape.aws.orchestration.AddShardingKeyToShard;
import com.sap.sse.landscape.aws.orchestration.CreateShard;
import com.sap.sse.landscape.aws.orchestration.RemoveShardingKeyFromShard;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule;
import software.amazon.awssdk.services.route53.model.RRType;
import software.amazon.awssdk.services.route53.model.ResourceRecord;
import software.amazon.awssdk.services.route53.model.ResourceRecordSet;
/**
@@ -72,10 +81,28 @@ extends ApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> {
/**
* The auto-scaling group is responsible for scaling the set of replicas registered with the
* {@link #getPublicTargetGroup() public target group}. This is optional, so {@code null} may
* be returned.
* be returned.<p>
*
* Note that in the presence of {@link #getShards() shards} those shards will each have their
* own {@link AwsShard#getAutoScalingGroup() auto-scaling group} which will share a launch
* configuration with the auto-scaling group returned by this method.
*/
AwsAutoScalingGroup getAutoScalingGroup() throws InterruptedException, ExecutionException;
/**
* Returns a non-{@code null} but possibly empty iterable of all auto-scaling groups for this replica set.
* These can be the default {@link #getAutoScalingGroup() auto-scaling group} for the public target group
* as well as any {@link #getShards() shards' auto-scaling groups}.
*/
default Iterable<AwsAutoScalingGroup> getAllAutoScalingGroups() throws InterruptedException, ExecutionException {
final Set<AwsAutoScalingGroup> result = new HashSet<>();
if (getAutoScalingGroup() != null) {
result.add(getAutoScalingGroup());
}
Util.addAll(Util.map(getShards().keySet(), shard->shard.getAutoScalingGroup()), result);
return result;
}
/**
* Checks whether the {@code host} is eligible for accepting a deployment of a process that belongs to this
* application replica set, either its master or a replica. In order to be eligible, the host must
@@ -128,4 +155,44 @@ extends ApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> {
* {@link ServerInfo}.
*/
boolean isLocalReplicaSet();
/**
* Returns a {@link ShardTargetGroupName} that is created from an (user-) entered shard name ({@code shardName}).
* {@link ShardTargetGroupName} contains the target group name and the replica set name.
*
* @param shardName
* (User-) entered name for the shard.
* @param targetGroupNamePrefix
* a prefix for the target group name; must not be {@code null} but may be empty
* @return {@link ShardTargetGroupName} created from {@code shardName}.
* @throws Exception
* throws when {@code shardName} is not valid or is not parse-able to a shardName.
*/
ShardTargetGroupName getNewShardName(String shardName, String targetGroupNamePrefix) throws Exception;
/**
* Retrieves information about sharding in this replica set, representing the situation at the point in time
* this object was created (not a live copy of the current landscape configuration). For that time point the
* map returned tells which shard handles requests for which sharding keys. All other reading traffic will
* be routed to this replica set's {@link #getPublicTargetGroup() public target group}.<p>
*
* Shards can be removed using {@link #removeShard(AwsShard, AwsLandscape)}. To create and manipulate shards, use the
* {@link CreateShard}, {@link AddShardingKeyToShard}, and {@link RemoveShardingKeyFromShard} procedures.
*
* @return Keys are the {@link AwsShard shards}, values are the {@code ShardingKey}s managed by the corresponding
* key's shard. Never {@code null}, but may of course be empty.
*/
Map<AwsShard<ShardingKey>, Iterable<ShardingKey>> getShards();
/**
* Removes the {@code shard} from this replica set. This will remove the load balancer routing rules that so far
* directed traffic for the shard's keys to the shard; it will also remove the auto-scaling group for the shard's
* target group which will also terminate all instances created by that auto-scaling group so far; finally, the
* shard's target group is removed.
* <p>
*
* In effect, this will make all traffic for the shard's keys default back to the {@link #getPublicTargetGroup()
* public target group}.
*/
void removeShard(AwsShard<ShardingKey> shard, AwsLandscape<ShardingKey> landscape) throws Exception;
}
@@ -15,6 +15,10 @@ import software.amazon.awssdk.services.autoscaling.model.LaunchConfiguration;
*
*/
public interface AwsAutoScalingGroup extends Named {
/**
* Describes on how many requests per minute a new instance gets created.
*/
public static int DEFAULT_MAX_REQUESTS_PER_TARGET = 15000;
AutoScalingGroup getAutoScalingGroup();
LaunchConfiguration getLaunchConfiguration();
@@ -35,17 +35,17 @@ public interface AwsInstance<ShardingKey> extends Host {
}
/**
* Finds out whether this instance is managed by an auto-scaling group.The implementation checks the
* Finds out whether this instance is managed by an auto-scaling group. The implementation checks the
* {@link #AWS_AUTOSCALING_GROUP_NAME_TAG} tag's value and compares it to the {@code autoScalingGroup}'s name.
*/
boolean isManagedByAutoScalingGroup();
/**
* Finds out whether this instance is managed by the particular {@code autoScalingGroup} passed as parameter. The
* Finds out whether this instance is managed by any of the {@code autoScalingGroups} passed as parameter. The
* implementation checks the {@link #AWS_AUTOSCALING_GROUP_NAME_TAG} tag's value and compares it to the
* {@code autoScalingGroup}'s name.
*/
boolean isManagedByAutoScalingGroup(AwsAutoScalingGroup autoScalingGroup);
boolean isManagedByAutoScalingGroup(Iterable<AwsAutoScalingGroup> autoScalingGroups);
default String getId() {
return getInstanceId();
@@ -1,6 +1,5 @@
package com.sap.sse.landscape.aws;
import java.util.Collection;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
@@ -38,6 +37,7 @@ import com.sap.sse.landscape.rabbitmq.RabbitMQEndpoint;
import com.sap.sse.landscape.ssh.SSHKeyPair;
import software.amazon.awssdk.services.autoscaling.model.AutoScalingGroup;
import software.amazon.awssdk.services.autoscaling.model.DeleteAutoScalingGroupResponse;
import software.amazon.awssdk.services.autoscaling.model.LaunchConfiguration;
import software.amazon.awssdk.services.cloudwatch.CloudWatchClient;
import software.amazon.awssdk.services.ec2.Ec2Client;
@@ -52,6 +52,7 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.LoadBalancer
import software.amazon.awssdk.services.elasticloadbalancingv2.model.ProtocolEnum;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.RulePriorityPair;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TagDescription;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetHealth;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetHealthDescription;
import software.amazon.awssdk.services.route53.Route53Client;
@@ -76,7 +77,7 @@ import software.amazon.awssdk.services.sts.model.Credentials;
* <p>
*
* Clients may also create dedicated instances of this service wrapper, using their own credentials. See
* {@link #obtain(String, String, Optional)}.
* {@link #obtain(String, String, Optional, String)}.
* <p>
*
* This object interacts with an instance of {@link AwsLandscapeState} which keeps persistent and replicable state about
@@ -88,6 +89,8 @@ import software.amazon.awssdk.services.sts.model.Credentials;
* @param <MetricsT>
*/
public interface AwsLandscape<ShardingKey> extends Landscape<ShardingKey> {
static final long DEFAULT_DNS_TTL_SECONDS = 60l;
String ACCESS_KEY_ID_SYSTEM_PROPERTY_NAME = "com.sap.sse.landscape.aws.accesskeyid";
String SECRET_ACCESS_KEY_SYSTEM_PROPERTY_NAME = "com.sap.sse.landscape.aws.secretaccesskey";
@@ -140,8 +143,8 @@ public interface AwsLandscape<ShardingKey> extends Landscape<ShardingKey> {
*/
static <ShardingKey, MetricsT extends ApplicationProcessMetrics,
ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>>
AwsLandscape<ShardingKey> obtain() {
final AwsLandscape<ShardingKey> result = new AwsLandscapeImpl<>(Activator.getInstance().getLandscapeState());
AwsLandscape<ShardingKey> obtain(String pathPrefixForShardingKey) {
final AwsLandscape<ShardingKey> result = new AwsLandscapeImpl<>(Activator.getInstance().getLandscapeState(), pathPrefixForShardingKey);
return result;
}
@@ -152,8 +155,8 @@ public interface AwsLandscape<ShardingKey> extends Landscape<ShardingKey> {
*/
static <ShardingKey, MetricsT extends ApplicationProcessMetrics,
ProcessT extends AwsApplicationProcess<ShardingKey, MetricsT, ProcessT>>
AwsLandscape<ShardingKey> obtain(String accessKey, String secret) {
final AwsLandscape<ShardingKey> result = new AwsLandscapeImpl<>(Activator.getInstance().getLandscapeState(), accessKey, secret);
AwsLandscape<ShardingKey> obtain(String accessKey, String secret, String pathPrefixForShardingKey) {
final AwsLandscape<ShardingKey> result = new AwsLandscapeImpl<>(Activator.getInstance().getLandscapeState(), accessKey, secret, pathPrefixForShardingKey);
return result;
}
@@ -164,8 +167,8 @@ public interface AwsLandscape<ShardingKey> extends Landscape<ShardingKey> {
*/
static <ShardingKey, MetricsT extends ApplicationProcessMetrics,
ProcessT extends AwsApplicationProcess<ShardingKey, MetricsT, ProcessT>>
AwsLandscape<ShardingKey> obtain(String accessKey, String secret, String sessionToken) {
final AwsLandscape<ShardingKey> result = new AwsLandscapeImpl<>(Activator.getInstance().getLandscapeState(), accessKey, secret, sessionToken);
AwsLandscape<ShardingKey> obtain(String accessKey, String secret, String sessionToken, String pathPrefixForShardingKey) {
final AwsLandscape<ShardingKey> result = new AwsLandscapeImpl<>(Activator.getInstance().getLandscapeState(), accessKey, secret, sessionToken, pathPrefixForShardingKey);
return result;
}
@@ -409,6 +412,8 @@ public interface AwsLandscape<ShardingKey> extends Landscape<ShardingKey> {
Iterable<ApplicationLoadBalancer<ShardingKey>> getLoadBalancers(Region region);
CompletableFuture<Map<TargetGroup<ShardingKey>, Iterable<TargetHealthDescription>>> getTargetGroupsAsync(Region region);
Iterable<TargetGroup<ShardingKey>> getTargetGroups(com.sap.sse.landscape.Region region);
CompletableFuture<Iterable<TargetHealthDescription>> getTargetHealthDescriptionsAsync(Region region, TargetGroup<ShardingKey> targetGroup);
@@ -485,6 +490,16 @@ public interface AwsLandscape<ShardingKey> extends Landscape<ShardingKey> {
*/
TargetGroup<ShardingKey> createTargetGroup(Region region, String targetGroupName, int port,
String healthCheckPath, int healthCheckPort, String loadBalancerArn);
/**
* Copies a target group from an existing target group. The name gets extended with {@code suffix}
* @param parent
* target group to copy from
* @param suffix
* suffix to append to the parent's name for the name of the new created target group
* @return
* newly created target group.
*/
TargetGroup<ShardingKey> copyTargetGroup(TargetGroup<ShardingKey> parent, String suffix);
default TargetGroup<ShardingKey> getTargetGroup(Region region, String targetGroupName, String targetGroupArn,
String loadBalancerArn, ProtocolEnum protocol, Integer port, ProtocolEnum healthCheckProtocol,
@@ -509,6 +524,30 @@ public interface AwsLandscape<ShardingKey> extends Landscape<ShardingKey> {
<SK> void deleteTargetGroup(TargetGroup<SK> targetGroup);
Iterable<Rule> getLoadBalancerListenerRules(Listener loadBalancerListener, Region region);
/**
* Modifies an existing rule that is identified by the passed {@code rule}'s ARN. Only the conditions are modified and nothing
* else gets touched.
*
* @param region
* AWS Region
* @param rule
* The Rule to modify. Only ARN and conditions are necessary.
* @return
* the modified Rule as an Iterable.
*/
Iterable<Rule> modifyRuleConditions(Region region, Rule rule);
/**
* Modifies an existing rule that is identified by the passed {@code rule}'s ARN. Only the actions are modified and nothing
* else gets touched.
* @param region
* AWS Region
* @param rule
* The Rule to modify. Only ARN and actions are necessary.
* @return
* the modified Rule as an Iterable.
*/
Iterable<Rule> modifyRuleActions(Region region, Rule rule);
/**
* Use {@link Rule.Builder} to create {@link Rule} objects you'd like to set for the {@link Listener} passed as parameter.
@@ -523,7 +562,7 @@ public interface AwsLandscape<ShardingKey> extends Landscape<ShardingKey> {
void updateLoadBalancerListenerRule(Region region, Rule ruleToUpdate);
void updateLoadBalancerListenerRulePriorities(Region region, Collection<RulePriorityPair> newRulePriorities);
void updateLoadBalancerListenerRulePriorities(Region region, Iterable<RulePriorityPair> newRulePriorities);
void deleteLoadBalancerListener(Region region, Listener listener);
@@ -729,15 +768,77 @@ public interface AwsLandscape<ShardingKey> extends Landscape<ShardingKey> {
ProcessT master, Iterable<ProcessT> replicas) throws InterruptedException, ExecutionException, TimeoutException;
CompletableFuture<Void> removeAutoScalingGroupAndLaunchConfiguration(AwsAutoScalingGroup autoScalingGroup);
CompletableFuture<DeleteAutoScalingGroupResponse> removeAutoScalingGroup(AwsAutoScalingGroup autoScalingGroup);
/**
* updates minimum and desired size to {@code minSize}
*/
void updateAutoScalingGroupMinSize(AwsAutoScalingGroup autoScalingGroup, int minSize);
void updateReleaseInAutoScalingGroup(Region region, AwsAutoScalingGroup autoScalingGroup, String replicaSetName, Release release);
/**
* @param autoScalingGroups
* {@link AwsApplicationReplicaSet#getAllAutoScalingGroups() All} auto-scaling groups of the replica set;
* this is important because the method will eventually remove the old launch configuration used by those
* auto-scaling groups, and this will fail if there is any other auto-scaling group still using it.
*/
void updateReleaseInAutoScalingGroups(Region region, LaunchConfiguration oldLaunchConfiguration, Iterable<AwsAutoScalingGroup> autoScalingGroups, String replicaSetName, Release release);
void updateImageInAutoScalingGroup(Region region, AwsAutoScalingGroup autoScalingGroup, String replicaSetName, AmazonMachineImage<ShardingKey> ami);
/**
* @param autoScalingGroups
* {@link AwsApplicationReplicaSet#getAllAutoScalingGroups() All} auto-scaling groups of the replica set;
* this is important because the method will eventually remove the old launch configuration used by those
* auto-scaling groups, and this will fail if there is any other auto-scaling group still using it.
*/
void updateImageInAutoScalingGroups(Region region, Iterable<AwsAutoScalingGroup> autoScalingGroups, String replicaSetName, AmazonMachineImage<ShardingKey> ami);
void updateInstanceTypeInAutoScalingGroup(Region region, AwsAutoScalingGroup autoScalingGroup, String replicaSetName, InstanceType instanceType);
/**
* @param autoScalingGroups
* {@link AwsApplicationReplicaSet#getAllAutoScalingGroups() All} auto-scaling groups of the replica set;
* this is important because the method will eventually remove the old launch configuration used by those
* auto-scaling groups, and this will fail if there is any other auto-scaling group still using it.
*/
void updateInstanceTypeInAutoScalingGroup(Region region, Iterable<AwsAutoScalingGroup> autoScalingGroups, String replicaSetName, InstanceType instanceType);
TargetGroup<ShardingKey> createTargetGroupWithoutLoadbalancer(Region region, String targetGroupName, int port);
/**
* Creates a new auto-scaling group, using an existing one as a template and only deriving a new name for the auto-scaling group
* based on the {@code shareName}, configuring it to create its instances into {@code targetGroup} instead of the
* {@code autoScalingParent}'s target group, and optionally adding the {@code tags} to those copied anyhow from
* the {@code autoScalingParent}. The minimum size is copied from the {@code autoScalingParent} unless it is less than two;
* in that case, the new auto-scaling group will be configured with a minimum size of two, ensuring availability in case
* one target fails.
*/
<MetricsT extends ApplicationProcessMetrics, ProcessT extends AwsApplicationProcess<ShardingKey, MetricsT, ProcessT>>
void createAutoScalingGroupFromExisting(AwsAutoScalingGroup autoScalingParent,
String shardName, TargetGroup<ShardingKey> targetGroup, Optional<Tags> tags);
<MetricsT extends ApplicationProcessMetrics, ProcessT extends AwsApplicationProcess<ShardingKey, MetricsT, ProcessT>>
void putScalingPolicy(
int instanceWarmupTimeInSeconds, String shardname, TargetGroup<ShardingKey> targetgroup, int maxRequestPerTarget, com.sap.sse.landscape.Region region);
Iterable<TagDescription> getTargetGroupTags(String arn, com.sap.sse.landscape.Region region);
/**
*
* See AWS doc for tag restrictions:
* https://docs.aws.amazon.com/elasticloadbalancing/latest/application/target-group-tags.html
* If a resource already has a tag with the same key,
* the value gets updated.
*
* @param arn
* Target group's ARN to add the tag to.
* @param key
* Key of the tag. See AWS logs for restrictions.
* @param value
* value of tag. See AWS logs for restrictions.
* @param region
* AWS Region of target group
* @return
* Returns the added Tag
*/
Tags addTargetGroupTag(String arn, String key, String value, com.sap.sse.landscape.Region region);
String getAutoScalingGroupName(String replicaSetName);
}
@@ -0,0 +1,50 @@
package com.sap.sse.landscape.aws;
import com.sap.sse.landscape.application.Shard;
import com.sap.sse.landscape.aws.common.shared.ShardTargetGroupName;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule;
/**
* A shard is represented by a target group that has its own auto-scaling group and which handles a number of sharding
* keys.
* <p>
*
* Technically, the shard's name is represented as a tag value on the {@link #getTargetGroup() target group}, hence the
* restrictions for AWS tag values apply. The tag's key is {@link ShardTargetGroupName.TAG_KEY}.
*
* <p>
*
* Routing to shards happens at the {@link #getTargetGroup() target group's} {@link TargetGroup#getLoadBalancer() load
* balancer}, or more precisely, through the load balancer's HTTPS listener's rule set. Each shard requires a number of rule pairs
* of which one rule forwards requests based on an HTTP header field requiring the request to be handled by a replica,
* and the other one based on the request using the GET request method. Another rule pair is required per three sharding keys
* to be handled, due to the restriction on the number of conditions a rule may have.
*
* @author Axel Uhl (d043530)
*
* @param <ShardingKey>
*/
public interface AwsShard<ShardingKey> extends Shard<ShardingKey> {
/**
* @return the target group receiving the traffic for this shard
*/
TargetGroup<ShardingKey> getTargetGroup();
/**
* The auto-scaling group managing the instances in the {@link #getTargetGroup() target group} receiving the traffic
* for this shard.
*/
AwsAutoScalingGroup getAutoScalingGroup();
ApplicationLoadBalancer<ShardingKey> getLoadBalancer();
String getReplicaSetName();
/**
* All ALB listener rules created to route traffic to this shard. These rules belong to a listener of
* {@link #getLoadBalancer()}.
*/
Iterable<Rule> getRules();
}
@@ -5,8 +5,10 @@ import java.util.Map;
import com.sap.sse.common.Named;
import com.sap.sse.landscape.Region;
import com.sap.sse.landscape.aws.common.shared.TargetGroupConstants;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.ProtocolEnum;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TagDescription;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetHealth;
/**
@@ -15,7 +17,7 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetHealth
* @author Axel Uhl (D043530)
*
*/
public interface TargetGroup<ShardingKey> extends Named {
public interface TargetGroup<ShardingKey> extends Named, TargetGroupConstants {
Region getRegion();
Map<AwsInstance<ShardingKey>, TargetHealth> getRegisteredTargets();
@@ -37,6 +39,13 @@ public interface TargetGroup<ShardingKey> extends Named {
*/
Integer getPort();
/**
*
* @return
* returns all tag descriptions for this target group
*/
Iterable<TagDescription> getTagDescriptions();
/**
* @return the traffic protocol; usually either one of {@link ProtocolEnum#HTTP} or {@link ProtocolEnum#HTTPS}
*/
@@ -1,13 +1,18 @@
package com.sap.sse.landscape.aws.impl;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map.Entry;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import com.sap.sse.common.Duration;
import com.sap.sse.common.Util;
@@ -28,15 +33,13 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.RedirectActi
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleCondition;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.RulePriorityPair;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupTuple;
public class ApplicationLoadBalancerImpl<ShardingKey>
implements ApplicationLoadBalancer<ShardingKey> {
private static final long serialVersionUID = -5297220031399131769L;
private static final Logger logger = Logger.getLogger(ApplicationLoadBalancerImpl.class.getName());
/**
* The maximum {@link Rule#priority()} that can be used within a listener
*/
private static final int MAX_PRIORITY = 50000;
private static final long serialVersionUID = -5297220031399131769L;
private final LoadBalancer loadBalancer;
@@ -90,6 +93,103 @@ implements ApplicationLoadBalancer<ShardingKey> {
public Iterable<Rule> addRules(Rule... rulesToAdd) {
return landscape.createLoadBalancerListenerRules(region, getListener(ProtocolEnum.HTTPS), rulesToAdd);
}
/**
* Returns the priority if the priority is not higher than {@code MAX_PRIORITY}
* @param priority
* requested priority
* @return
* Priority if it's valid
* @throws IllegalStateException
* If the requested priority is higher than {@code MAX_PRIORITY}
*/
private int checkNewPriority(int priority) throws IllegalStateException {
if (priority < MAX_PRIORITY) {
return priority;
} else {
throw new IllegalStateException("Priority was greater than " + MAX_PRIORITY +"!");
}
}
@Override
public void shiftRulesToMakeSpaceAt(int targetPrio) throws IllegalStateException {
final Iterable<Rule> rules = getRules();
final TreeMap<Integer, Rule> rulesSorted = getRulesSorted(rules);
int lastPrio = targetPrio;
boolean skipNext = false;
final Collection<RulePriorityPair> result = new ArrayList<>();
if (rulesSorted.get(targetPrio) != null) {// if there is a rule on prio
for (Entry<Integer, Rule> entry : rulesSorted.entrySet()) {
final Integer priority = entry.getKey();
if (priority >= targetPrio && !skipNext) {
// if prio is higher than target prio and is not supposed to be skipped
if (priority - lastPrio > 0) {
// if there is a gap between current prio and the last one. -> so this one is not supposed to be
// skipped and every rule after this
skipNext = true;
break;
} else {
lastPrio = priority + 1;
result.add(RulePriorityPair.builder().ruleArn(entry.getValue().ruleArn())
.priority(checkNewPriority(lastPrio)).build());
}
}
}
landscape.updateLoadBalancerListenerRulePriorities(getRegion(), result);
}
}
@Override
public int getFirstShardingPriority(String hostname) throws IllegalStateException {
final Iterable<Rule> rules = getRules();
final TreeMap<Integer, Rule> rulesSorted = getRulesSorted(rules);
final Iterator<Entry<Integer, Rule>> iterSorted = rulesSorted.entrySet().iterator();
int priorityToReturn = -1;
outher : while (iterSorted.hasNext()) {
final Rule r = iterSorted.next().getValue();
for (RuleCondition con : r.conditions()) {
if (con.hostHeaderConfig() != null && con.hostHeaderConfig().values().contains(hostname)) {
for (Action a : r.actions()) {
if (a.forwardConfig() != null) {
priorityToReturn = Integer.parseInt(r.priority());
break outher;
} else if (a.redirectConfig() != null) {
priorityToReturn = Integer.parseInt(r.priority()) + 1;
break outher;
} else {
continue;
}
}
}
}
if (!iterSorted.hasNext()) {
priorityToReturn = Integer.parseInt(r.priority()) + 1;
break outher;
}
}
if (priorityToReturn > MAX_PRIORITY) {
throw new IllegalStateException("Index higher than " + MAX_PRIORITY + "!");
}
return priorityToReturn;
}
private TreeMap<Integer, Rule> getRulesSorted(Iterable<Rule> rules) {
final Iterator<Rule> iter = rules.iterator();
final TreeMap<Integer, Rule> rulesSorted = new TreeMap<Integer, Rule>();
// create Map with every priority
while (iter.hasNext()) {
final Rule r = iter.next();
try {
if (!r.priority().equalsIgnoreCase(ApplicationLoadBalancer.DEFAULT_RULE_PRIORITY)) {
rulesSorted.put(Integer.valueOf(r.priority()), r);
}
} catch (Exception e) {
// Case where prio is not a number, e.g. 'Default' gets ignored.
logger.log(Level.WARNING, "Priority '" + r.priority() + "' couldn't be parsed as an Integer.");
}
}
return rulesSorted;
}
@Override
public Iterable<Rule> addRulesAssigningUnusedPriorities(boolean forceContiguous, Rule... rules) {
@@ -271,4 +371,58 @@ implements ApplicationLoadBalancer<ShardingKey> {
&& condition.pathPatternConfig().values().size() == 1
&& condition.pathPatternConfig().values().contains("/")).findAny().isPresent();
}
}
@Override
public Iterable<Rule> getRulesForTargetGroups(Iterable<TargetGroup<ShardingKey>> targetGroups) {
ArrayList<Rule> ret = new ArrayList<Rule>();
for (Rule rule : getRules()) {
for (Action action : rule.actions()) {
if (action.type() == ActionTypeEnum.FORWARD) {
for (String arn : Util.map(action.forwardConfig().targetGroups(), s -> s.targetGroupArn())) {
for (String targetArn : Util.map(targetGroups, s -> s.getTargetGroupArn())) {
if (arn.equals(targetArn)) {
ret.add(rule);
}
}
}
}
}
}
return ret;
}
/**
* Goes through all rules in this load balancer and replaces every target group in a forward-action that has the
* same ARN as the {@code oldTargetgroup} with the {@code newTargetgroup}
*/
@Override
public Iterable<Rule> replaceTargetGroupInForwardRules(TargetGroup<ShardingKey> oldTargetGroup,
TargetGroup<ShardingKey> newTargetGroup) {
Iterable<Rule> rules = getRules();
Collection<Rule> modifiedRules = new ArrayList<>();
for (Rule r : rules) {
int i = 0;
boolean modified = false;
Action[] newActions = new Action[r.actions().size()];
for (Action a : r.actions()) {
if (a.forwardConfig() != null && a.targetGroupArn().equals(oldTargetGroup.getTargetGroupArn())) {
newActions[i++] = createForwardToTargetGroupAction(newTargetGroup);
modified = true;
} else {
newActions[i++] = a;
}
}
if (modified) {
landscape.modifyRuleActions(region, Rule.builder().actions(newActions).ruleArn(r.ruleArn()).build())
.forEach(t -> modifiedRules.add(t));
}
}
return modifiedRules;
}
private Action createForwardToTargetGroupAction(TargetGroup<ShardingKey> targetGroup) {
return Action.builder().type(ActionTypeEnum.FORWARD).forwardConfig(fc -> fc
.targetGroups(TargetGroupTuple.builder().targetGroupArn(targetGroup.getTargetGroupArn()).build()))
.build();
}
}
@@ -27,6 +27,7 @@ import com.sap.sse.landscape.aws.MongoUriParser;
import com.sap.sse.landscape.aws.TargetGroup;
import com.sap.sse.landscape.mongodb.Database;
import com.sap.sse.replication.ReplicationStatus;
import com.sap.sse.util.IPAddressUtil;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.ActionTypeEnum;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule;
@@ -110,7 +111,7 @@ implements AwsApplicationProcess<ShardingKey, MetricsT, ProcessT> {
private <HostT extends AwsInstance<ShardingKey>> Pair<HostT, Integer> getHostAndOptionalTargetPortFromIpAddress(HostSupplier<ShardingKey, HostT> hostSupplier, final String ipAddressOrHostname) {
HostT host;
Integer targetPort;
final ApplicationLoadBalancer<ShardingKey> alb = landscape.getDNSMappedLoadBalancerFor(ipAddressOrHostname);
final ApplicationLoadBalancer<ShardingKey> alb = IPAddressUtil.isIPAddressLiteral(ipAddressOrHostname) ? null : landscape.getDNSMappedLoadBalancerFor(ipAddressOrHostname);
if (alb != null) {
logger.info("Found a hostname mapped to a load balancer; trying to find master through target group...");
host = null;
@@ -1,6 +1,7 @@
package com.sap.sse.landscape.aws.impl;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
@@ -27,7 +28,10 @@ import com.sap.sse.landscape.aws.AwsApplicationProcess;
import com.sap.sse.landscape.aws.AwsApplicationReplicaSet;
import com.sap.sse.landscape.aws.AwsAutoScalingGroup;
import com.sap.sse.landscape.aws.AwsLandscape;
import com.sap.sse.landscape.aws.AwsShard;
import com.sap.sse.landscape.aws.TargetGroup;
import com.sap.sse.landscape.aws.common.shared.ShardTargetGroupName;
import com.sap.sse.landscape.aws.orchestration.ShardProcedure;
import software.amazon.awssdk.services.autoscaling.model.AutoScalingGroup;
import software.amazon.awssdk.services.autoscaling.model.LaunchConfiguration;
@@ -38,6 +42,8 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.Listener;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.ProtocolEnum;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleCondition;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Tag;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TagDescription;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupTuple;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetHealthDescription;
import software.amazon.awssdk.services.route53.Route53AsyncClient;
@@ -63,6 +69,7 @@ implements AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> {
private static final Logger logger = Logger.getLogger(AwsApplicationReplicaSetImpl.class.getName());
private static final String ARCHIVE_SERVER_NAME = "ARCHIVE";
private static final long serialVersionUID = 6895927683667795173L;
private final Map<AwsShard<ShardingKey>, Iterable<ShardingKey>> shards;
private final CompletableFuture<AwsAutoScalingGroup> autoScalingGroup;
private final CompletableFuture<Rule> defaultRedirectRule;
private final CompletableFuture<String> hostedZoneId;
@@ -71,15 +78,17 @@ implements AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> {
private final CompletableFuture<TargetGroup<ShardingKey>> masterTargetGroup;
private final CompletableFuture<TargetGroup<ShardingKey>> publicTargetGroup;
private final CompletableFuture<ResourceRecordSet> resourceRecordSet;
private final String pathPrefixForShardingKey;
public AwsApplicationReplicaSetImpl(String replicaSetAndServerName, String hostname, ProcessT master,
Optional<Iterable<ProcessT>> replicas,
CompletableFuture<Iterable<ApplicationLoadBalancer<ShardingKey>>> allLoadBalancersInRegion,
CompletableFuture<Map<TargetGroup<ShardingKey>, Iterable<TargetHealthDescription>>> allTargetGroupsInRegion,
CompletableFuture<Map<Listener, Iterable<Rule>>> allLoadBalancerRulesInRegion,
CompletableFuture<Iterable<AutoScalingGroup>> allAutoScalingGroups,
CompletableFuture<Iterable<LaunchConfiguration>> allLaunchConfigurations, DNSCache dnsCache) throws InterruptedException, ExecutionException, TimeoutException {
CompletableFuture<Iterable<LaunchConfiguration>> allLaunchConfigurations, DNSCache dnsCache, String pathPrefixForShardingKey) throws InterruptedException, ExecutionException, TimeoutException {
super(replicaSetAndServerName, hostname, master, replicas);
this.pathPrefixForShardingKey = pathPrefixForShardingKey;
autoScalingGroup = new CompletableFuture<>();
defaultRedirectRule = new CompletableFuture<>();
hostedZoneId = new CompletableFuture<>();
@@ -88,6 +97,7 @@ implements AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> {
masterTargetGroup = new CompletableFuture<>();
publicTargetGroup = new CompletableFuture<>();
resourceRecordSet = new CompletableFuture<>();
shards = new HashMap<>();
try {
allLoadBalancersInRegion.thenCompose(loadBalancers->
allTargetGroupsInRegion.thenCompose(targetGroupsAndTheirTargetHealthDescriptions->
@@ -106,6 +116,10 @@ implements AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> {
throw e;
}
}
public Map<AwsShard<ShardingKey>, Iterable<ShardingKey>> getShards() {
return shards;
}
public AwsApplicationReplicaSetImpl(String replicaSetAndServerName, ProcessT master,
Optional<Iterable<ProcessT>> replicas,
@@ -113,10 +127,11 @@ implements AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> {
CompletableFuture<Map<TargetGroup<ShardingKey>, Iterable<TargetHealthDescription>>> allTargetGroupsInRegion,
CompletableFuture<Map<Listener, Iterable<Rule>>> allLoadBalancerRulesInRegion,
AwsLandscape<ShardingKey> landscape, CompletableFuture<Iterable<AutoScalingGroup>> allAutoScalingGroups,
CompletableFuture<Iterable<LaunchConfiguration>> allLaunchConfigurations, DNSCache dnsCache) throws InterruptedException, ExecutionException, TimeoutException {
CompletableFuture<Iterable<LaunchConfiguration>> allLaunchConfigurations, DNSCache dnsCache, String pathPrefixForShardingKey)
throws InterruptedException, ExecutionException, TimeoutException {
this(replicaSetAndServerName, /* hostname to be inferred */ null, master, replicas, allLoadBalancersInRegion,
allTargetGroupsInRegion, allLoadBalancerRulesInRegion, allAutoScalingGroups, allLaunchConfigurations,
dnsCache);
dnsCache, pathPrefixForShardingKey);
}
/**
@@ -177,9 +192,13 @@ implements AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> {
}
}
if (!publicTargetGroup.isDone()
&& (!Util.isEmpty(Util.filter(e.getValue(), target->Util.contains(Util.map(getReplicas(), replica->replica.getHost().getId()), target.target().id())))
|| !Util.isEmpty(Util.filter(e.getValue(), target->target.target().id().equals(getMaster().getHost().getId()))))
&& hasPublicRuleForward(listenersAndTheirRules, e.getKey())) {
&& (!Util.isEmpty(Util.filter(e.getValue(),
target -> Util.contains(Util.map(getReplicas(), replica -> replica.getHost().getId()),
target.target().id())))
|| !Util.isEmpty(Util.filter(e.getValue(),
target -> target.target().id().equals(getMaster().getHost().getId()))))
&& hasPublicRuleForward(listenersAndTheirRules, e.getKey())
&& !hasPathCondition(listenersAndTheirRules, e.getKey())) { // not a shard target group
// this replica set's master or at least one of the replicas of this replica set is registered with
// the target group, and there is a rule that forwards
// requests with explicit replica-markup to this target group:
@@ -188,6 +207,8 @@ implements AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> {
}
}
}
shards.putAll(establishShards(targetGroupsAndTheirTargetHealthDescriptions, listenersAndTheirRules,
autoScalingGroups, launchConfigurations));
if (!autoScalingGroup.isDone()) { // no auto-scaling group was found after having looked at all target groups
autoScalingGroup.complete(null);
}
@@ -281,6 +302,67 @@ implements AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> {
}
return null;
}
private AwsAutoScalingGroup getShardAutoscalingGroup(TargetGroup<ShardingKey> targetGroup,
Iterable<AutoScalingGroup> autoScalingGroups, Iterable<LaunchConfiguration> launchConfigurations) {
AwsAutoScalingGroup autoscalinggroup = Util
.stream(autoScalingGroups).filter(
autoScalingGroup -> autoScalingGroup.targetGroupARNs()
.contains(targetGroup.getTargetGroupArn()))
.findFirst()
.map(asg -> new AwsAutoScalingGroupImpl(asg,
Util.filter(launchConfigurations,
lc -> Util.equalsWithNull(lc.launchConfigurationName(), asg.launchConfigurationName()))
.iterator().next(),
targetGroup.getRegion()))
.orElse(null);
return autoscalinggroup;
}
private HashMap<AwsShard<ShardingKey>, Iterable<ShardingKey>> establishShards(
Map<TargetGroup<ShardingKey>, Iterable<TargetHealthDescription>> targetGroupsAndTheirTargetHealthDescriptions,
Map<Listener, Iterable<Rule>> listenersAndTheirRules, Iterable<AutoScalingGroup> autoScalingGroups,
Iterable<LaunchConfiguration> launchConfigurations) {
HashMap<AwsShard<ShardingKey>, Iterable<ShardingKey>> shardMap = new HashMap<>();
for (final Entry<TargetGroup<ShardingKey>, Iterable<TargetHealthDescription>> e : targetGroupsAndTheirTargetHealthDescriptions.entrySet()) {
if ((e.getKey().getProtocol() == ProtocolEnum.HTTP || e.getKey().getProtocol() == ProtocolEnum.HTTPS)
&& e.getKey().getLoadBalancerArn() != null
&& e.getKey().getHealthCheckPort() == getMaster().getPort()) {
// an HTTP(S) target group that is currently active in a load balancer and forwards to this replica
// set master's port
final Set<Rule> pathRules = getListenerRulesWithPathToReplica(listenersAndTheirRules, e.getKey());
if (!pathRules.isEmpty() && ShardTargetGroupName.isValidShardTargetGroupName(e.getKey().getName())) {
// Is shard
final Set<ShardingKey> shardingKeys = getShardingKeys(listenersAndTheirRules, e.getKey());
String tagName = null;
Iterable<TagDescription> tagsDescs = e.getKey().getTagDescriptions();
for (final TagDescription des : tagsDescs) {
final Iterable<Tag> tag = Util.filter(des.tags(), t -> t.key().equals(ShardTargetGroupName.TAG_KEY));
if (!Util.isEmpty(tag)) {
tagName = tag.iterator().next().value();
break;
}
}
try {
final ShardTargetGroupName shardName = ShardTargetGroupName.parse(e.getKey().getName(), tagName);
final AwsShardImpl<ShardingKey> shard = new AwsShardImpl<ShardingKey>(getName(),
shardName.getShardName(), shardingKeys, e.getKey(),
e.getKey().getLoadBalancer(), pathRules, getShardAutoscalingGroup(e.getKey(), autoScalingGroups, launchConfigurations));
shardMap.put(shard, shard.getKeys());
} catch (Exception e1) {
logger.info(e1.getMessage());
// This entry is no valid shard
}
}
}
}
return shardMap;
};
@Override
public ShardTargetGroupName getNewShardName(String shardName, String targetGroupNamePrefix) throws Exception{
return ShardTargetGroupName.create(getName(), shardName, targetGroupNamePrefix);
}
@Override
public boolean isEligibleForDeployment(ApplicationProcessHost<ShardingKey, MetricsT, ProcessT> host,
@@ -306,7 +388,7 @@ implements AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> {
byte[] privateKeyEncryptionPassphrase) throws Exception {
logger.info("Stopping all unmanaged replicas of replica set "+this);
for (final ProcessT replica : getReplicas()) {
if (getAutoScalingGroup() == null || !replica.getHost().isManagedByAutoScalingGroup(getAutoScalingGroup())) {
if (getAutoScalingGroup() == null || !replica.getHost().isManagedByAutoScalingGroup(Collections.singleton(getAutoScalingGroup()))) {
logger.info("Found unmanaged replica "+replica+". Removing from public target group and stopping...");
getPublicTargetGroup().removeTarget(replica.getHost());
replica.stopAndTerminateIfLast(optionalTimeout, optionalKeyName, privateKeyEncryptionPassphrase);
@@ -318,6 +400,35 @@ implements AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> {
return hasListenerRuleWithHostHeaderForward(listenersAndTheirRules, publicTargetGroupCandidate, HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA.getB());
}
/**
* Checks if any of the load balancer listeners passed has a rule that forwards to the
* {@code shardingTargetGroupCandidate} and has a {@code path-pattern} condition.
*/
private boolean hasPathCondition(Map<Listener, Iterable<Rule>> listenersAndTheirRules,
TargetGroup<ShardingKey> shardingTargetGroupCandidate) {
final String shardTargetGroupCandidateArn = shardingTargetGroupCandidate.getTargetGroupArn();
for (final Entry<Listener, Iterable<Rule>> e : listenersAndTheirRules.entrySet()) {
if (Util.equalsWithNull(e.getKey().loadBalancerArn(), shardingTargetGroupCandidate.getLoadBalancerArn())) {
for (final Rule rule : e.getValue()) {
for (final Action action : rule.actions()) {
if (action.type() == ActionTypeEnum.FORWARD) {
for (final TargetGroupTuple targetGroupTuple : action.forwardConfig().targetGroups()) {
if (Util.equalsWithNull(targetGroupTuple.targetGroupArn(), shardTargetGroupCandidateArn)) {
for (final RuleCondition condition : rule.conditions()) {
if (Util.equalsWithNull(condition.field(), "path-pattern")) {
return true;
}
}
}
}
}
}
}
}
}
return false;
}
private boolean hasListenerRuleWithHostHeaderForward(Map<Listener, Iterable<Rule>> listenersAndTheirRules, TargetGroup<ShardingKey> publicTargetGroupCandidate, String hostHeaderForwardTo) {
final String publicTargetGroupCandidateArn = publicTargetGroupCandidate.getTargetGroupArn();
for (final Entry<Listener, Iterable<Rule>> e : listenersAndTheirRules.entrySet()) {
@@ -343,6 +454,84 @@ implements AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> {
}
return false;
}
/**
* Filters the {@link Rule}s that are contained in the values of {@code listenersAndTheirRules} for those
* that forward to the {@code shardTargetGroupCandidate} and whose conditions contains a {@code path-pattern}
* condition as well as a {@code http-header} condition that checks for the request to allow forwarding to
* a replica (see {@link HttpRequestHeaderConstants#HEADER_FORWARD_TO_REPLICA}).
*/
private Set<Rule> getListenerRulesWithPathToReplica(Map<Listener, Iterable<Rule>> listenersAndTheirRules,
TargetGroup<ShardingKey> shardTargetGroupCandidate) {
final String shardTargetGroupCandidateArn = shardTargetGroupCandidate.getTargetGroupArn();
Set<Rule> res = new HashSet<Rule>();
for (final Entry<Listener, Iterable<Rule>> e : listenersAndTheirRules.entrySet()) {
if (Util.equalsWithNull(e.getKey().loadBalancerArn(), shardTargetGroupCandidate.getLoadBalancerArn())) {
for (final Rule rule : e.getValue()) {
for (final Action action : rule.actions()) {
if (action.type() == ActionTypeEnum.FORWARD) {
for (final TargetGroupTuple targetGroupTuple : action.forwardConfig().targetGroups()) {
if (Util.equalsWithNull(targetGroupTuple.targetGroupArn(), shardTargetGroupCandidateArn)) {
for (final RuleCondition condition : rule.conditions()) {
if (Util.equalsWithNull(condition.field(), "path-pattern")) {
for (final RuleCondition ruleCondition : rule.conditions()) {
if (Util.equalsWithNull(ruleCondition.field(), "http-header")
&& Util.equalsWithNull(
ruleCondition.httpHeaderConfig().httpHeaderName(),
HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA
.getA())
&& ruleCondition.httpHeaderConfig().values().contains(
HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA
.getB())) {
res.add(rule);
}
}
}
}
}
}
}
}
}
}
}
return res;
}
private Set<ShardingKey> getShardingKeys(Map<Listener, Iterable<Rule>> listenersAndTheirRules,
TargetGroup<ShardingKey> shardTargetGroupCandidate) {
final String publicTargetGroupCandidateArn = shardTargetGroupCandidate.getTargetGroupArn();
Set<ShardingKey> shardingKeys = new HashSet<>();
for (final Entry<Listener, Iterable<Rule>> e : listenersAndTheirRules.entrySet()) {
if (Util.equalsWithNull(e.getKey().loadBalancerArn(), shardTargetGroupCandidate.getLoadBalancerArn())) {
for (final Rule rule : e.getValue()) {
for (final Action action : rule.actions()) {
if (action.type() == ActionTypeEnum.FORWARD) {
for (final TargetGroupTuple targetGroupTuple : action.forwardConfig().targetGroups()) {
if (Util.equalsWithNull(targetGroupTuple.targetGroupArn(),
publicTargetGroupCandidateArn)) {
for (final RuleCondition condition : rule.conditions()) {
if (Util.equalsWithNull(condition.field(), "http-header")
&& Util.equalsWithNull(condition.httpHeaderConfig().httpHeaderName(),
HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA.getA())
&& condition.httpHeaderConfig().values().contains(
HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA.getB())) {
for (final RuleCondition ruleCondition : rule.conditions()) {
if (Util.equalsWithNull(ruleCondition.field(), "path-pattern")) {
Util.addAll(Util.map(ruleCondition.values(), path->ShardProcedure.getShardingKeyFromPathCondition(path, pathPrefixForShardingKey)), shardingKeys);
}
}
}
}
}
}
}
}
}
}
}
return shardingKeys;
}
private boolean hasMasterRuleForward(Map<Listener, Iterable<Rule>> listenersAndTheirRules, TargetGroup<ShardingKey> masterTargetGroupCandidate) {
return hasListenerRuleWithHostHeaderForward(listenersAndTheirRules, masterTargetGroupCandidate, HttpRequestHeaderConstants.HEADER_FORWARD_TO_MASTER.getB());
@@ -418,4 +607,15 @@ implements AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> {
public boolean isLocalReplicaSet() {
return getName().equals(ServerInfo.getName());
}
}
@Override
public void removeShard(AwsShard<ShardingKey> shard, AwsLandscape<ShardingKey> landscape) throws Exception {
// remove rules for targetgrouo
landscape.deleteLoadBalancerListenerRules(shard.getTargetGroup().getRegion(),
Util.toArray(getLoadBalancer().getRulesForTargetGroups(Collections.singleton(shard.getTargetGroup())), new Rule[0]));
// remove autoscaling group
landscape.removeAutoScalingGroup(shard.getAutoScalingGroup());
// remove targetgroup
landscape.deleteTargetGroup(shard.getTargetGroup());
}
}
@@ -275,9 +275,9 @@ public class AwsInstanceImpl<ShardingKey> implements AwsInstance<ShardingKey> {
}
@Override
public boolean isManagedByAutoScalingGroup(AwsAutoScalingGroup autoScalingGroup) {
return getInstance().tags().stream().filter(tag -> autoScalingGroup != null
&& tag.key().equals(AWS_AUTOSCALING_GROUP_NAME_TAG) && tag.value().equals(autoScalingGroup.getName()))
public boolean isManagedByAutoScalingGroup(Iterable<AwsAutoScalingGroup> autoScalingGroups) {
return getInstance().tags().stream().filter(tag -> tag.key().equals(AWS_AUTOSCALING_GROUP_NAME_TAG)
&& Util.stream(autoScalingGroups).filter(autoScalingGroup -> autoScalingGroup.getName().equals(tag.value())).findAny().isPresent())
.findAny().isPresent();
}
@@ -95,6 +95,7 @@ import software.amazon.awssdk.services.autoscaling.AutoScalingAsyncClient;
import software.amazon.awssdk.services.autoscaling.AutoScalingClient;
import software.amazon.awssdk.services.autoscaling.model.AutoScalingGroup;
import software.amazon.awssdk.services.autoscaling.model.CreateLaunchConfigurationRequest;
import software.amazon.awssdk.services.autoscaling.model.DeleteAutoScalingGroupResponse;
import software.amazon.awssdk.services.autoscaling.model.LaunchConfiguration;
import software.amazon.awssdk.services.autoscaling.model.MetricType;
import software.amazon.awssdk.services.ec2.Ec2Client;
@@ -149,6 +150,7 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.LoadBalancer
import software.amazon.awssdk.services.elasticloadbalancingv2.model.LoadBalancerAttribute;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.LoadBalancerNotFoundException;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.LoadBalancerState;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.ModifyRuleResponse;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.ModifyTargetGroupAttributesRequest;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.ProtocolEnum;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.RedirectActionStatusCodeEnum;
@@ -156,6 +158,7 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.RulePriorityPair;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.SetRulePrioritiesRequest;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.SubnetMapping;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TagDescription;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetDescription;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupAttribute;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupNotFoundException;
@@ -188,7 +191,7 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
private static final String AUTO_SCALING_GROUP_NAME_SUFFIX = "-auto-replicas";
private static final String DEFAULT_TARGET_GROUP_PREFIX = "D";
private static final Logger logger = Logger.getLogger(AwsLandscapeImpl.class.getName());
private static final long DEFAULT_DNS_TTL_MILLIS = 60l;
public static final long DEFAULT_DNS_TTL_SECONDS = 60l;
private static final String DEFAULT_CERTIFICATE_DOMAIN = "*.sapsailing.com";
// TODO <config> the "Java Application with Reverse Proxy" security group in eu-west-2 for experimenting; we need this security group per region
private static final String DEFAULT_APPLICATION_SERVER_SECURITY_GROUP_ID_EU_WEST_1 = "sg-eaf31e85";
@@ -201,30 +204,32 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
private final Optional<String> sessionToken;
private final AwsRegion globalRegion;
private final AwsLandscapeState landscapeState;
private final String pathPrefixForShardingKey;
public AwsLandscapeImpl(AwsLandscapeState awsLandscapeState) {
public AwsLandscapeImpl(AwsLandscapeState awsLandscapeState, String pathPrefixForShardingKey) {
this(awsLandscapeState,
System.getProperty(ACCESS_KEY_ID_SYSTEM_PROPERTY_NAME),
System.getProperty(SECRET_ACCESS_KEY_SYSTEM_PROPERTY_NAME));
System.getProperty(SECRET_ACCESS_KEY_SYSTEM_PROPERTY_NAME), pathPrefixForShardingKey);
}
public AwsLandscapeImpl(AwsLandscapeState awsLandscapeState, String accessKeyId, String secretAccessKey) {
this(awsLandscapeState, accessKeyId, secretAccessKey, /* no session token */ null);
public AwsLandscapeImpl(AwsLandscapeState awsLandscapeState, String accessKeyId, String secretAccessKey, String pathPrefixForShardingKey) {
this(awsLandscapeState, accessKeyId, secretAccessKey, /* no session token */ null, pathPrefixForShardingKey);
}
public AwsLandscapeImpl(AwsLandscapeState awsLandscapeState, String accessKeyId, String secretAccessKey, String sessionToken) {
public AwsLandscapeImpl(AwsLandscapeState awsLandscapeState, String accessKeyId, String secretAccessKey, String sessionToken, String pathPrefixForShardingKey) {
this(accessKeyId, secretAccessKey, sessionToken,
// by using MongoDBService.INSTANCE the default test configuration will be used if nothing else is configured
PersistenceFactory.INSTANCE.getDomainObjectFactory(MongoDBService.INSTANCE), PersistenceFactory.INSTANCE.getMongoObjectFactory(MongoDBService.INSTANCE), awsLandscapeState);
PersistenceFactory.INSTANCE.getDomainObjectFactory(MongoDBService.INSTANCE), PersistenceFactory.INSTANCE.getMongoObjectFactory(MongoDBService.INSTANCE), awsLandscapeState, pathPrefixForShardingKey);
}
public AwsLandscapeImpl(String accessKeyId, String secretAccessKey,
String sessionToken, DomainObjectFactory domainObjectFactory, MongoObjectFactory mongoObjectFactory, AwsLandscapeState landscapeState) {
String sessionToken, DomainObjectFactory domainObjectFactory, MongoObjectFactory mongoObjectFactory, AwsLandscapeState landscapeState, String pathPrefixForShardingKey) {
this.accessKeyId = accessKeyId;
this.secretAccessKey = secretAccessKey;
this.sessionToken = Optional.ofNullable(sessionToken);
this.globalRegion = new AwsRegion(Region.AWS_GLOBAL, this);
this.landscapeState = landscapeState;
this.pathPrefixForShardingKey = pathPrefixForShardingKey;
}
private static byte[] getPrivateKeyBytes(KeyPair unencryptedKeyPair) {
@@ -408,9 +413,25 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
@Override
public Iterable<TargetGroup<ShardingKey>> getTargetGroupsByLoadBalancerArn(com.sap.sse.landscape.Region region, String loadBalancerArn) {
return Util.map(getLoadBalancingClient(getRegion(region)).describeTargetGroupsPaginator(tg->tg.loadBalancerArn(loadBalancerArn)).targetGroups(),
tg->new AwsTargetGroupImpl<>(this, region, tg.targetGroupName(), tg.targetGroupArn(), loadBalancerArn,
tg->createTargetGroup(this, region, tg.targetGroupName(), tg.targetGroupArn(), loadBalancerArn,
tg.protocol(), tg.port(), tg.healthCheckProtocol(), getHealthCheckPort(tg), tg.healthCheckPath()));
}
private TargetGroup<ShardingKey> createTargetGroup(AwsLandscape<ShardingKey> landscape,
com.sap.sse.landscape.Region region, String targetGroupName, String targetGroupArn, String loadBalancerArn,
ProtocolEnum protocol, Integer port, ProtocolEnum healthCheckProtocol, Integer healthCheckPort,
String healthCheckPath) {
return new AwsTargetGroupImpl<ShardingKey>(this, region, targetGroupName, targetGroupArn, loadBalancerArn,
protocol, port, healthCheckProtocol, healthCheckPort, healthCheckPath);
}
@Override
public Iterable<TargetGroup<ShardingKey>> getTargetGroups(com.sap.sse.landscape.Region region) {
return Util.map(getLoadBalancingClient(getRegion(region)).describeTargetGroupsPaginator().targetGroups(),
tg -> createTargetGroup(this, region, tg.targetGroupName(), tg.targetGroupArn(),
tg.loadBalancerArns().isEmpty() ? null : tg.loadBalancerArns().get(0), tg.protocol(), tg.port(),
tg.healthCheckProtocol(), getHealthCheckPort(tg), tg.healthCheckPath()));
}
@Override
public Iterable<Listener> getListeners(ApplicationLoadBalancer<ShardingKey> alb) {
@@ -429,6 +450,13 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
public Iterable<Rule> getLoadBalancerListenerRules(Listener loadBalancerListener, com.sap.sse.landscape.Region region) {
return getLoadBalancingClient(getRegion(region)).describeRules(b->b.listenerArn(loadBalancerListener.listenerArn())).rules();
}
@Override
public Iterable<Rule> modifyRuleConditions(com.sap.sse.landscape.Region region, Rule rule) {
ModifyRuleResponse res = getLoadBalancingClient(getRegion(region))
.modifyRule(t -> t.conditions(rule.conditions()).ruleArn(rule.ruleArn()).build());
return res.rules();
}
@Override
public Iterable<Rule> createLoadBalancerListenerRules(com.sap.sse.landscape.Region region,
@@ -461,8 +489,8 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
}
@Override
public void updateLoadBalancerListenerRulePriorities(com.sap.sse.landscape.Region region, Collection<RulePriorityPair> newRulePriorities) {
getLoadBalancingClient(getRegion(region)).setRulePriorities(SetRulePrioritiesRequest.builder().rulePriorities(newRulePriorities).build());
public void updateLoadBalancerListenerRulePriorities(com.sap.sse.landscape.Region region, Iterable<RulePriorityPair> newRulePriorities) {
getLoadBalancingClient(getRegion(region)).setRulePriorities(SetRulePrioritiesRequest.builder().rulePriorities(Util.asList(newRulePriorities)).build());
}
@Override
@@ -622,7 +650,7 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
.changeResourceRecordSets(
ChangeResourceRecordSetsRequest.builder().hostedZoneId(hostedZoneId)
.changeBatch(ChangeBatch.builder().changes(Change.builder().action(ChangeAction.UPSERT)
.resourceRecordSet(ResourceRecordSet.builder().name(hostname.toLowerCase()).type(type).ttl(DEFAULT_DNS_TTL_MILLIS)
.resourceRecordSet(ResourceRecordSet.builder().name(hostname.toLowerCase()).type(type).ttl(DEFAULT_DNS_TTL_SECONDS)
.resourceRecords(ResourceRecord.builder().value(value).build()).build())
.build()).build())
.build());
@@ -639,7 +667,7 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
return getRoute53Client().changeResourceRecordSets(ChangeResourceRecordSetsRequest.builder().hostedZoneId(hostedZoneId)
.changeBatch(ChangeBatch.builder().changes(Change.builder().action(ChangeAction.DELETE)
// TODO using the DEFAULT_DNS_TTL_MILLIS is a bit unclean here; if the record has been modified manually or the default has changed, removal will fail
.resourceRecordSet(ResourceRecordSet.builder().name(hostname).type(type).ttl(DEFAULT_DNS_TTL_MILLIS)
.resourceRecordSet(ResourceRecordSet.builder().name(hostname).type(type).ttl(DEFAULT_DNS_TTL_SECONDS)
.resourceRecords(ResourceRecord.builder().value(value).build()).build()).build()).build()).build()).
changeInfo();
}
@@ -982,7 +1010,7 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
final DescribeTargetGroupsResponse targetGroupResponse = loadBalancingClient.describeTargetGroups(b->b.names(targetGroupName));
final software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroup targetGroup = targetGroupResponse.targetGroups().iterator().next();
return targetGroupResponse.hasTargetGroups()
? new AwsTargetGroupImpl<>(this, region, targetGroupName, targetGroup.targetGroupArn(),
? createTargetGroup(this, region, targetGroupName, targetGroup.targetGroupArn(),
loadBalancerArn == null ? Util.first(targetGroup.loadBalancerArns()) : loadBalancerArn,
targetGroup.protocol(), targetGroup.port(),
targetGroup.healthCheckProtocol(), getHealthCheckPort(targetGroup), targetGroup.healthCheckPath())
@@ -1039,7 +1067,7 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
}
}
}
return new AwsTargetGroupImpl<>(this, region, targetGroupName, targetGroupArn, loadBalancerArn,
return createTargetGroup(this, region, targetGroupName, targetGroupArn, loadBalancerArn,
targetGroup.protocol(), port, targetGroup.healthCheckProtocol(), healthCheckPort, healthCheckPath);
}
@@ -1291,6 +1319,20 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
return tagResponse.tags().stream().map(t->t.value()).findAny();
}
public Iterable<TagDescription> getTargetGroupTags(String arn, com.sap.sse.landscape.Region region) {
final software.amazon.awssdk.services.elasticloadbalancingv2.model.DescribeTagsResponse tagResponse = getLoadBalancingClient(
getRegion(region)).describeTags(t -> t.resourceArns(arn));
return tagResponse.tagDescriptions();
}
@Override
public Tags addTargetGroupTag(String arn, String key, String value, com.sap.sse.landscape.Region region) {
Collection<software.amazon.awssdk.services.elasticloadbalancingv2.model.Tag> tags = new ArrayList<>();
tags.add(software.amazon.awssdk.services.elasticloadbalancingv2.model.Tag.builder().key(key).value(value).build());
getLoadBalancingClient(getRegion(region)).addTags(t -> t.resourceArns(arn).tags(tags));
return new TagsImpl(key,value);
}
@Override
public Tags getTagForMongoProcess(Tags tagsToAddTo, String replicaSetName, int port) {
return tagsToAddTo.and(MONGO_REPLICA_SETS_TAG_NAME,
@@ -1534,7 +1576,7 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
CompletableFuture<Iterable<LaunchConfiguration>> allLaunchConfigurations, final DNSCache dnsCache) throws InterruptedException, ExecutionException, TimeoutException {
final AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaSet = new AwsApplicationReplicaSetImpl<ShardingKey, MetricsT, ProcessT>(
serverName, master, Optional.ofNullable(replicas), allLoadBalancersInRegion, allTargetGroupsInRegion,
allLoadBalancerRulesInRegion, this, allAutoScalingGroups, allLaunchConfigurations, dnsCache);
allLoadBalancerRulesInRegion, this, allAutoScalingGroups, allLaunchConfigurations, dnsCache, pathPrefixForShardingKey);
return replicaSet;
}
@@ -1726,7 +1768,7 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
final Map<TargetGroup<ShardingKey>, CompletableFuture<Iterable<TargetHealthDescription>>> futures = new HashMap<>();
for (final DescribeTargetGroupsResponse response : responses) {
for (final software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroup tg : response.targetGroups()) {
final TargetGroup<ShardingKey> targetGroup = new AwsTargetGroupImpl<ShardingKey>(this, region,
final TargetGroup<ShardingKey> targetGroup = createTargetGroup(this, region,
tg.targetGroupName(), tg.targetGroupArn(), Util.first(tg.loadBalancerArns()),
tg.protocol(), tg.port(), tg.healthCheckProtocol(), getHealthCheckPort(tg),
tg.healthCheckPath());
@@ -1756,19 +1798,20 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
public Iterable<com.sap.sse.landscape.Region> getRegions() {
return Util.map(Region.regions(), r->new AwsRegion(r, this));
}
@Override
public void updateReleaseInAutoScalingGroup(com.sap.sse.landscape.Region region, AwsAutoScalingGroup autoScalingGroup, String replicaSetName, Release release) {
logger.info("Adjusting release for auto-scaling group "+autoScalingGroup.getName()+" to "+release);
public void updateReleaseInAutoScalingGroups(com.sap.sse.landscape.Region region,
LaunchConfiguration oldLaunchConfiguration, Iterable<AwsAutoScalingGroup> autoScalingGroups,
String replicaSetName, Release release) {
logger.info("Adjusting release for auto-scaling groups "+Util.join(", ", autoScalingGroups)+" to "+release);
final String releaseName = release.getName();
final String newLaunchConfigurationName = getLaunchConfigurationName(replicaSetName, releaseName);
final LaunchConfiguration oldLaunchConfiguration = autoScalingGroup.getLaunchConfiguration();
final String oldUserData = new String(Base64.getDecoder().decode(oldLaunchConfiguration.userData().getBytes()));
final String newUserData = oldUserData.replaceFirst(
"(?m)^"+DefaultProcessConfigurationVariables.INSTALL_FROM_RELEASE.name()+"=(.*)$",
DefaultProcessConfigurationVariables.INSTALL_FROM_RELEASE.name()+"=\""+release.getName()+"\"");
updateLaunchConfiguration(region, autoScalingGroup, newLaunchConfigurationName,
b->b.userData(Base64.getEncoder().encodeToString(newUserData.getBytes())));
DefaultProcessConfigurationVariables.INSTALL_FROM_RELEASE.name() + "=\"" + release.getName() + "\"");
updateLaunchConfiguration(region, oldLaunchConfiguration, autoScalingGroups, newLaunchConfigurationName,
b -> b.userData(Base64.getEncoder().encodeToString(newUserData.getBytes())));
}
private CreateLaunchConfigurationRequest.Builder copyLaunchConfigurationToCreateRequestBuilder(LaunchConfiguration launchConfigurationToCopy) {
@@ -1791,29 +1834,33 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
}
@Override
public void updateImageInAutoScalingGroup(com.sap.sse.landscape.Region region, AwsAutoScalingGroup autoScalingGroup, String replicaSetName, AmazonMachineImage<ShardingKey> ami) {
logger.info("Adjusting AMI for auto-scaling group "+autoScalingGroup.getName()+" to "+ami);
public void updateImageInAutoScalingGroups(com.sap.sse.landscape.Region region, Iterable<AwsAutoScalingGroup> autoScalingGroups, String replicaSetName, AmazonMachineImage<ShardingKey> ami) {
logger.info("Adjusting AMI for auto-scaling group(s) "+Util.join(", ", autoScalingGroups)+" to "+ami);
final String newLaunchConfigurationName = getLaunchConfigurationName(replicaSetName, ami.getId());
updateLaunchConfiguration(region, autoScalingGroup, newLaunchConfigurationName, b->b.imageId(ami.getId()));
updateLaunchConfiguration(region, autoScalingGroups, newLaunchConfigurationName, b->b.imageId(ami.getId()));
}
@Override
public void updateInstanceTypeInAutoScalingGroup(com.sap.sse.landscape.Region region, AwsAutoScalingGroup autoScalingGroup, String replicaSetName, InstanceType instanceType) {
logger.info("Adjusting instance type for auto-scaling group "+autoScalingGroup.getName()+" to "+instanceType);
final LaunchConfiguration oldLaunchConfiguration = autoScalingGroup.getLaunchConfiguration();
public void updateInstanceTypeInAutoScalingGroup(com.sap.sse.landscape.Region region, Iterable<AwsAutoScalingGroup> autoScalingGroups, String replicaSetName, InstanceType instanceType) {
logger.info("Adjusting instance type for auto-scaling group(s) "+Util.join(", ", autoScalingGroups)+" to "+instanceType);
final LaunchConfiguration oldLaunchConfiguration = Util.first(autoScalingGroups).getLaunchConfiguration();
final String newLaunchConfigurationName = oldLaunchConfiguration.launchConfigurationName()+"-"+instanceType.name();
updateLaunchConfiguration(region, autoScalingGroup, newLaunchConfigurationName, b->b.instanceType(instanceType.toString()));
updateLaunchConfiguration(region, autoScalingGroups, newLaunchConfigurationName, b->b.instanceType(instanceType.toString()));
}
private void updateLaunchConfigurationForAutoScalingGroup(final AutoScalingClient autoScalingClient,
AwsAutoScalingGroup autoScalingGroup, final LaunchConfiguration oldLaunchConfiguration,
private void updateLaunchConfigurationForAutoScalingGroups(final AutoScalingClient autoScalingClient,
Iterable<AwsAutoScalingGroup> autoScalingGroups, final LaunchConfiguration oldLaunchConfiguration,
final String newLaunchConfigurationName) {
logger.info("Telling auto-scaling group "+autoScalingGroup.getName()+" to use new launch configuration "+newLaunchConfigurationName);
autoScalingClient.updateAutoScalingGroup(b->b
.autoScalingGroupName(autoScalingGroup.getAutoScalingGroup().autoScalingGroupName())
.launchConfigurationName(newLaunchConfigurationName));
logger.info("Removing old launch configuration "+oldLaunchConfiguration.launchConfigurationName());
autoScalingClient.deleteLaunchConfiguration(b->b.launchConfigurationName(oldLaunchConfiguration.launchConfigurationName()));
for (AwsAutoScalingGroup autoScalingGroup : autoScalingGroups) {
logger.info("Telling auto-scaling group " + autoScalingGroup.getName() + " to use new launch configuration "
+ newLaunchConfigurationName);
autoScalingClient.updateAutoScalingGroup(
b -> b.autoScalingGroupName(autoScalingGroup.getAutoScalingGroup().autoScalingGroupName())
.launchConfigurationName(newLaunchConfigurationName));
}
logger.info("Removing old launch configuration " + oldLaunchConfiguration.launchConfigurationName());
autoScalingClient.deleteLaunchConfiguration(
b -> b.launchConfigurationName(oldLaunchConfiguration.launchConfigurationName()));
}
/**
@@ -1831,17 +1878,16 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
* @see #updateLaunchConfigurationForAutoScalingGroup(AutoScalingClient, AwsAutoScalingGroup, LaunchConfiguration,
* String)
*/
private void updateLaunchConfiguration(com.sap.sse.landscape.Region region, AwsAutoScalingGroup autoScalingGroup,
private void updateLaunchConfiguration(com.sap.sse.landscape.Region region, LaunchConfiguration oldLaunchConfiguration, Iterable<AwsAutoScalingGroup> affectedAutoScalingGroups,
String newLaunchConfigurationName, Consumer<CreateLaunchConfigurationRequest.Builder> builderConsumer) {
if (newLaunchConfigurationName == null) {
throw new NullPointerException("New launch configuration name for auto-scaling group "+autoScalingGroup.getName()+" must not be null");
throw new NullPointerException("New launch configuration name for auto-scaling groups "+Util.join(", ", affectedAutoScalingGroups)+" must not be null");
}
logger.info("Adjusting launch configuration for auto-scaling group "+autoScalingGroup.getName());
logger.info("Adjusting launch configuration for auto-scaling groups "+Util.join(", ", affectedAutoScalingGroups));
final AutoScalingClient autoScalingClient = getAutoScalingClient(getRegion(region));
final LaunchConfiguration oldLaunchConfiguration = autoScalingGroup.getLaunchConfiguration();
if (newLaunchConfigurationName.equals(oldLaunchConfiguration.launchConfigurationName())) {
throw new IllegalArgumentException("New launch configuration name "+newLaunchConfigurationName+" for auto-scaling group "+
autoScalingGroup.getName()+" equals the old one");
throw new IllegalArgumentException("New launch configuration name "+newLaunchConfigurationName+" for auto-scaling groups "+
Util.join(", ", affectedAutoScalingGroups)+" equals the old one");
}
final CreateLaunchConfigurationRequest.Builder createLaunchConfigurationRequestBuilder = copyLaunchConfigurationToCreateRequestBuilder(oldLaunchConfiguration);
builderConsumer.accept(createLaunchConfigurationRequestBuilder);
@@ -1849,7 +1895,31 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
final CreateLaunchConfigurationRequest createLaunchConfigurationRequest = createLaunchConfigurationRequestBuilder.build();
logger.info("Creating new launch configuration "+newLaunchConfigurationName);
autoScalingClient.createLaunchConfiguration(createLaunchConfigurationRequest);
updateLaunchConfigurationForAutoScalingGroup(autoScalingClient, autoScalingGroup, oldLaunchConfiguration, newLaunchConfigurationName);
updateLaunchConfigurationForAutoScalingGroups(autoScalingClient, affectedAutoScalingGroups, oldLaunchConfiguration, newLaunchConfigurationName);
}
private void updateLaunchConfiguration(com.sap.sse.landscape.Region region, Iterable<AwsAutoScalingGroup> autoScalingGroups,
String newLaunchConfigurationName, Consumer<CreateLaunchConfigurationRequest.Builder> builderConsumer) {
if (Util.isEmpty(autoScalingGroups)) {
throw new IllegalArgumentException("At least one auto-scaling group must be provided for updating a launch configuration");
}
if (newLaunchConfigurationName == null) {
throw new NullPointerException("New launch configuration name for auto-scaling group(s) "+Util.join(", ", autoScalingGroups)+" must not be null");
}
logger.info("Adjusting launch configuration for auto-scaling group(s) "+Util.join(", ", autoScalingGroups));
final AutoScalingClient autoScalingClient = getAutoScalingClient(getRegion(region));
final LaunchConfiguration oldLaunchConfiguration = Util.first(autoScalingGroups).getLaunchConfiguration();
if (newLaunchConfigurationName.equals(oldLaunchConfiguration.launchConfigurationName())) {
throw new IllegalArgumentException("New launch configuration name "+newLaunchConfigurationName+" for auto-scaling group(s) "+
Util.join(", ", autoScalingGroups)+" equals the old one");
}
final CreateLaunchConfigurationRequest.Builder createLaunchConfigurationRequestBuilder = copyLaunchConfigurationToCreateRequestBuilder(oldLaunchConfiguration);
builderConsumer.accept(createLaunchConfigurationRequestBuilder);
createLaunchConfigurationRequestBuilder.launchConfigurationName(newLaunchConfigurationName);
final CreateLaunchConfigurationRequest createLaunchConfigurationRequest = createLaunchConfigurationRequestBuilder.build();
logger.info("Creating new launch configuration "+newLaunchConfigurationName);
autoScalingClient.createLaunchConfiguration(createLaunchConfigurationRequest);
updateLaunchConfigurationForAutoScalingGroups(autoScalingClient, autoScalingGroups, oldLaunchConfiguration, newLaunchConfigurationName);
}
@Override
@@ -1892,24 +1962,14 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
b.tags(awsTags);
});
});
autoScalingClient.putScalingPolicy(b->b
.autoScalingGroupName(autoScalingGroupName)
.estimatedInstanceWarmup(instanceWarmupTimeInSeconds)
.policyType("TargetTrackingScaling")
.policyName("KeepRequestsPerTargetAt"+maxRequestsPerTarget)
.targetTrackingConfiguration(t->t
.predefinedMetricSpecification(p->p
.resourceLabel("app/"+publicTargetGroup.getLoadBalancer().getName()+"/"+publicTargetGroup.getLoadBalancer().getId()+
"/targetgroup/"+publicTargetGroup.getName()+"/"+publicTargetGroup.getId())
.predefinedMetricType(MetricType.ALB_REQUEST_COUNT_PER_TARGET))
.targetValue((double) maxRequestsPerTarget)));
putScalingPolicy(instanceWarmupTimeInSeconds, autoScalingGroupName, publicTargetGroup , maxRequestsPerTarget, region);
}
private String getLaunchConfigurationName(String replicaSetName, final String releaseName) {
return replicaSetName + "-" + releaseName;
}
private String getAutoScalingGroupName(String replicaSetName) {
public String getAutoScalingGroupName(String replicaSetName) {
return replicaSetName+AUTO_SCALING_GROUP_NAME_SUFFIX;
}
@@ -1928,11 +1988,92 @@ public class AwsLandscapeImpl<ShardingKey> implements AwsLandscape<ShardingKey>
public CompletableFuture<Void> removeAutoScalingGroupAndLaunchConfiguration(AwsAutoScalingGroup autoScalingGroup) {
final String launchConfigurationName = autoScalingGroup.getAutoScalingGroup().launchConfigurationName();
final AutoScalingAsyncClient autoScalingAsyncClient = getAutoScalingAsyncClient(getRegion(autoScalingGroup.getRegion()));
logger.info("Removing auto-scaling group "+autoScalingGroup.getAutoScalingGroup().autoScalingGroupName());
return autoScalingAsyncClient.deleteAutoScalingGroup(b->b.forceDelete(true).autoScalingGroupName(autoScalingGroup.getAutoScalingGroup().autoScalingGroupName()))
return removeAutoScalingGroup(autoScalingGroup)
.thenAccept(response->{
logger.info("Removing launch configuration "+launchConfigurationName);
autoScalingAsyncClient.deleteLaunchConfiguration(b->b.launchConfigurationName(launchConfigurationName));
});
}
@Override
public CompletableFuture<DeleteAutoScalingGroupResponse> removeAutoScalingGroup(AwsAutoScalingGroup autoScalingGroup) {
final AutoScalingAsyncClient autoScalingAsyncClient = getAutoScalingAsyncClient(getRegion(autoScalingGroup.getRegion()));
logger.info("Removing auto-scaling group "+autoScalingGroup.getAutoScalingGroup().autoScalingGroupName());
return autoScalingAsyncClient.deleteAutoScalingGroup(b->b.forceDelete(true).autoScalingGroupName(autoScalingGroup.getAutoScalingGroup().autoScalingGroupName()));
}
@Override
public TargetGroup<ShardingKey> createTargetGroupWithoutLoadbalancer(com.sap.sse.landscape.Region region, String targetGroupName, int port) {
return createTargetGroup(region, targetGroupName, port, ApplicationProcess.HEALTH_CHECK_PATH, port, null);
}
@Override
public <MetricsT extends ApplicationProcessMetrics, ProcessT extends AwsApplicationProcess<ShardingKey, MetricsT, ProcessT>>
void createAutoScalingGroupFromExisting(AwsAutoScalingGroup autoScalingParent,
String shardName, TargetGroup<ShardingKey> targetGroup, Optional<Tags> tags) {
final AutoScalingClient autoScalingClient = getAutoScalingClient(getRegion(autoScalingParent.getRegion()));
final String launchConfigurationName = autoScalingParent.getAutoScalingGroup().launchConfigurationName();
final String autoScalingGroupName = getAutoScalingGroupName(shardName);
final List<String> availabilityZones = autoScalingParent.getAutoScalingGroup().availabilityZones();
final int instanceWarmupTimeInSeconds = autoScalingParent.getAutoScalingGroup().defaultInstanceWarmup() != null ? autoScalingParent.getAutoScalingGroup().defaultInstanceWarmup() : 180 ;
logger.info("Creating Autoscalinggroup " + autoScalingGroupName +" for Shard "+shardName + ". Inheriting from Autoscalinggroup: " + autoScalingParent.getName());
autoScalingClient.createAutoScalingGroup(b->{
b
.minSize(autoScalingParent.getAutoScalingGroup().minSize() > 1 ? autoScalingParent.getAutoScalingGroup().minSize() : 2)
.maxSize(autoScalingParent.getAutoScalingGroup().maxSize())
.healthCheckGracePeriod(instanceWarmupTimeInSeconds)
.autoScalingGroupName(autoScalingGroupName)
.availabilityZones(availabilityZones)
.targetGroupARNs(targetGroup.getTargetGroupArn())
.launchConfigurationName(launchConfigurationName);
final List<software.amazon.awssdk.services.autoscaling.model.Tag> awsTags = new ArrayList<>();
final List<software.amazon.awssdk.services.autoscaling.model.TagDescription> parentTags = autoScalingParent.getAutoScalingGroup().tags();
for (final software.amazon.awssdk.services.autoscaling.model.TagDescription parentTag : parentTags) {
awsTags.add(software.amazon.awssdk.services.autoscaling.model.Tag.builder()
.key(parentTag.key())
.value(parentTag.key().equals("Name") ? parentTag.value()+" ("+shardName+")" : parentTag.value())
.propagateAtLaunch(parentTag.propagateAtLaunch())
.build());
}
tags.ifPresent(t->{
for (final Entry<String, String> tag : t) {
awsTags.add(software.amazon.awssdk.services.autoscaling.model.Tag.builder().key(tag.getKey()).value(tag.getValue()).build());
}
});
b.tags(awsTags);
});
}
@Override
public TargetGroup<ShardingKey> copyTargetGroup(TargetGroup<ShardingKey> parent, String suffix) {
TargetGroup<ShardingKey> child = createTargetGroupWithoutLoadbalancer(parent.getRegion(), parent.getName()+ suffix, parent.getPort());
child.addTargets(parent.getRegisteredTargets().keySet());
return child;
}
@Override
public Iterable<Rule> modifyRuleActions(com.sap.sse.landscape.Region region, Rule rule) {
ModifyRuleResponse res = getLoadBalancingClient(getRegion(region))
.modifyRule(t -> t.actions(rule.actions()).ruleArn(rule.ruleArn()).build());
return res.rules();
}
@Override
public <MetricsT extends ApplicationProcessMetrics, ProcessT extends AwsApplicationProcess<ShardingKey, MetricsT, ProcessT>> void putScalingPolicy(
int instanceWarmupTimeInSeconds, String autoScalingGroupName, TargetGroup<ShardingKey> targetgroup,
int maxRequestPerTarget, com.sap.sse.landscape.Region region) {
final AutoScalingClient autoScalingClient = getAutoScalingClient(getRegion(region));
autoScalingClient.putScalingPolicy(
b -> b.autoScalingGroupName(autoScalingGroupName).estimatedInstanceWarmup(instanceWarmupTimeInSeconds)
.policyType("TargetTrackingScaling").policyName("KeepRequestsPerTargetAt" + maxRequestPerTarget)
.targetTrackingConfiguration(t -> t
.predefinedMetricSpecification(p -> p
.resourceLabel("app/" + targetgroup.getLoadBalancer().getName() + "/"
+ targetgroup.getLoadBalancer().getId() + "/targetgroup/"
+ targetgroup.getName() + "/" + targetgroup.getId())
.predefinedMetricType(MetricType.ALB_REQUEST_COUNT_PER_TARGET))
.targetValue((double) AwsAutoScalingGroup.DEFAULT_MAX_REQUESTS_PER_TARGET)));
}
}
@@ -0,0 +1,73 @@
package com.sap.sse.landscape.aws.impl;
import com.sap.sse.common.Util;
import com.sap.sse.landscape.aws.ApplicationLoadBalancer;
import com.sap.sse.landscape.aws.AwsAutoScalingGroup;
import com.sap.sse.landscape.aws.AwsShard;
import com.sap.sse.landscape.aws.TargetGroup;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule;
public class AwsShardImpl<ShardingKey> implements AwsShard<ShardingKey> {
private static final long serialVersionUID = 1L;
private final Iterable<ShardingKey> keys;
private final TargetGroup<ShardingKey> targetGroup;
private final String replicaSetName;
private final String name;
private final AwsAutoScalingGroup autoScalingGroup;
private final ApplicationLoadBalancer<ShardingKey> loadBalancer;
private final Iterable<Rule> rules;
public AwsShardImpl(String replicaSetName, String shardName, Iterable<ShardingKey> keys,
TargetGroup<ShardingKey> targetgroup, ApplicationLoadBalancer<ShardingKey> loadBalancer, Iterable<Rule> rules, AwsAutoScalingGroup asg) {
this.keys = keys;
this.targetGroup = targetgroup;
this.replicaSetName = replicaSetName;
this.name = shardName;
this.loadBalancer = loadBalancer;
this.autoScalingGroup = asg;
this.rules = rules;
}
@Override
public Iterable<ShardingKey> getKeys() {
return keys;
}
@Override
public String getName() {
return name;
}
@Override
public String getReplicaSetName() {
return replicaSetName;
}
@Override
public TargetGroup<ShardingKey> getTargetGroup() {
return targetGroup;
}
@Override
public AwsAutoScalingGroup getAutoScalingGroup() {
return autoScalingGroup;
}
@Override
public ApplicationLoadBalancer<ShardingKey> getLoadBalancer() {
return loadBalancer;
}
@Override
public Iterable<Rule> getRules() {
return rules;
}
@Override
public String toString() {
return "AwsShardImpl [name=" + name + ", replicaSetName=" + replicaSetName + ", keys=" + Util.joinStrings(", ", keys) + "]";
}
}
@@ -11,6 +11,7 @@ import com.sap.sse.landscape.aws.AwsLandscape;
import com.sap.sse.landscape.aws.TargetGroup;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.ProtocolEnum;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TagDescription;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetHealth;
public class AwsTargetGroupImpl<ShardingKey>
@@ -44,6 +45,10 @@ extends NamedImpl implements TargetGroup<ShardingKey> {
private software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroup getAwsTargetGroup() {
return landscape.getAwsTargetGroupByArn(getRegion(), getTargetGroupArn());
}
public Iterable<TagDescription> getTagDescriptions() {
return landscape.getTargetGroupTags(arn, region);
}
public Region getRegion() {
return region;
@@ -0,0 +1,131 @@
package com.sap.sse.landscape.aws.orchestration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
import java.util.logging.Logger;
import com.sap.sse.common.Util;
import com.sap.sse.landscape.application.ApplicationProcess;
import com.sap.sse.landscape.application.ApplicationProcessMetrics;
import com.sap.sse.landscape.aws.ApplicationLoadBalancer;
import com.sap.sse.landscape.aws.AwsShard;
import com.sap.sse.landscape.aws.TargetGroup;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleCondition;
/**
* This procedure appends {@code shardingKeys} to a shard, identified by {@link #shardName} from
* the {@link #replicaSet}. This is done by adding rule conditions to the shard's rule set and after that just appending new
* rules. This could lead to moving to replica set to another load balancer, because it may get full with new sharding
* rules.
*
* @author I569653
*
* @param <ShardingKey>
* @param <MetricsT>
* @param <ProcessT>
*/
public class AddShardingKeyToShard<ShardingKey, MetricsT extends ApplicationProcessMetrics, ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>>
extends ShardProcedure<ShardingKey, MetricsT, ProcessT> {
private static final Logger logger = Logger.getLogger(AddShardingKeyToShard.class.getName());
public AddShardingKeyToShard(BuilderImpl<?, ShardingKey, MetricsT, ProcessT> builder) throws Exception {
super(builder);
}
static class BuilderImpl<BuilderT extends Builder<BuilderT, AddShardingKeyToShard<ShardingKey, MetricsT, ProcessT>, ShardingKey, MetricsT, ProcessT>, ShardingKey, MetricsT extends ApplicationProcessMetrics, ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>>
extends
ShardProcedure.BuilderImpl<BuilderT, AddShardingKeyToShard<ShardingKey, MetricsT, ProcessT>, ShardingKey, MetricsT, ProcessT> {
@Override
public AddShardingKeyToShard<ShardingKey, MetricsT, ProcessT> build() throws Exception {
assert shardingKeys != null;
assert replicaSet != null;
assert region != null;
assert passphraseForPrivateKeyDecryption != null;
return new AddShardingKeyToShard<ShardingKey, MetricsT, ProcessT>(this);
}
}
@Override
public void run() throws Exception {
AwsShard<ShardingKey> shard = null;
for (final Entry<AwsShard<ShardingKey>, Iterable<ShardingKey>> entry : replicaSet.getShards().entrySet()) {
if (entry.getKey().getName().equals(shardName)) {
shard = entry.getKey();
break;
}
}
if (shard == null) {
throw new Exception("Shard "+shardName+" not found in replica set "+replicaSet.getName());
}
logger.info("Appending " + Util.joinStrings(", ", shardingKeys) + " to " + shardName);
// list for manipulation -> elements are allowed to be removed!!
final List<ShardingKey> mutableShardingKeys = new LinkedList<>();
mutableShardingKeys.addAll(shardingKeys);
final TargetGroup<ShardingKey> targetgroup = shard.getTargetGroup();
final ApplicationLoadBalancer<ShardingKey> loadBalancer = shard.getLoadBalancer();
final Collection<TargetGroup<ShardingKey>> t = new ArrayList<>();
t.add(targetgroup);
// check if there is a rule with space left for one or more additional conditions:
for (Rule r : shard.getRules()) {
boolean updateRule = false;
final ArrayList<ShardingKey> shardingKeys = new ArrayList<>();
for (RuleCondition con : r.conditions()) {
// if we find a
if (con.pathPatternConfig() != null) {
// eliminate PATH_UNUSED_BY_ANY_APPLICATION in case this proxy key was found;
// it usually indicates an empty shard; when now adding one or more conditions
// it can be replaced.
Util.addAll(
Util.filter(
Util.map(con.values(), this::getShardingKeyFromPathCondition),
shardingKey->!shardingKey.equals(SHARDING_KEY_UNUSED_BY_ANY_APPLICATION)),
shardingKeys);
}
}
if (shardingKeys.isEmpty()) {
// the rule probably only has PATH_UNUSED_BY_ANY_APPLICATION and was a proxy rule, probably at the end of the list; remove
loadBalancer.deleteRules(r);
} else { // update only non-empty rule because we assume it won't be at the end of the list
while (shardingKeys.size() < ApplicationLoadBalancer.MAX_CONDITIONS_PER_RULE - NUMBER_OF_STANDARD_CONDITIONS_FOR_SHARDING_RULE
&& !mutableShardingKeys.isEmpty()) {
shardingKeys.add(mutableShardingKeys.get(0));
mutableShardingKeys.remove(0);
updateRule = true;
}
final Collection<RuleCondition> ruleConditions = getShardingRuleConditions(loadBalancer, shardingKeys);
// construct a rule only for transporting the conditions; no forwarding target is required for modifyRuleConditions
Rule proxyRuleWithNewConditions = Rule.builder().ruleArn(r.ruleArn()).conditions(ruleConditions).build();
if (updateRule) {
getLandscape().modifyRuleConditions(region, proxyRuleWithNewConditions);
}
}
}
if (!mutableShardingKeys.isEmpty()) {
// check number of rules
final Set<ShardingKey> keysCopy = new HashSet<>();
keysCopy.addAll(shardingKeys);
if (Util.size(loadBalancer.getRules()) + numberOfRequiredRules(Util.size(shardingKeys))
< ApplicationLoadBalancer.MAX_RULES_PER_LOADBALANCER) {
// enough rules
addShardingRules(loadBalancer, keysCopy, targetgroup);
} else {
// not enough rules
final ApplicationLoadBalancer<ShardingKey> alb = getFreeLoadBalancerAndMoveReplicaSet();
// set new rules
addShardingRules(alb, keysCopy, targetgroup);
}
}
}
public static <MetricsT extends ApplicationProcessMetrics, ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>, BuilderT extends Builder<BuilderT, AddShardingKeyToShard<ShardingKey, MetricsT, ProcessT>, ShardingKey, MetricsT, ProcessT>, ShardingKey> Builder<BuilderT, AddShardingKeyToShard<ShardingKey, MetricsT, ProcessT>, ShardingKey, MetricsT, ProcessT> builder() {
return new BuilderImpl<BuilderT, ShardingKey, MetricsT, ProcessT>();
}
}
@@ -6,7 +6,6 @@ import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.IntStream;
import com.jcraft.jsch.JSchException;
@@ -37,9 +36,6 @@ public class CreateDNSBasedLoadBalancerMapping<ShardingKey, MetricsT extends App
ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>>
extends CreateLoadBalancerMapping<ShardingKey, MetricsT, ProcessT>
implements Procedure<ShardingKey> {
private static final String DNS_MAPPED_ALB_NAME_PREFIX = "DNSMapped-";
private static final Pattern ALB_NAME_PATTERN = Pattern.compile(DNS_MAPPED_ALB_NAME_PREFIX+"(.*)$");
public static interface Builder<BuilderT extends Builder<BuilderT, T, ShardingKey, MetricsT, ProcessT>,
T extends CreateDNSBasedLoadBalancerMapping<ShardingKey, MetricsT, ProcessT>,
ShardingKey, MetricsT extends ApplicationProcessMetrics,
@@ -87,7 +83,7 @@ implements Procedure<ShardingKey> {
ApplicationLoadBalancer<ShardingKey> result = null;
final Set<String> loadBalancerNames = new HashSet<>();
for (final ApplicationLoadBalancer<ShardingKey> loadBalancer : landscape.getLoadBalancers(region)) {
if (ALB_NAME_PATTERN.matcher(loadBalancer.getName()).matches()) {
if (ApplicationLoadBalancer.ALB_NAME_PATTERN.matcher(loadBalancer.getName()).matches()) {
loadBalancerNames.add(loadBalancer.getName());
if (Util.size(loadBalancer.getRules()) <= MAX_RULES_PER_ALB - NUMBER_OF_RULES_PER_REPLICA_SET) {
result = loadBalancer;
@@ -105,18 +101,18 @@ implements Procedure<ShardingKey> {
}
/**
* Picks a new load balancer name following the pattern {@link #DNS_MAPPED_ALB_NAME_PREFIX}{@code [0-9]+} that is not
* Picks a new load balancer name following the pattern {@link #ApplicationLoadBalancer.DNS_MAPPED_ALB_NAME_PREFIX}{@code [0-9]+} that is not
* part of {@code loadBalancerNames} and has the least number.
*/
private String getAvailableDNSMappedAlbName(Set<String> loadBalancerNames) {
final Set<Integer> numbersTaken = new HashSet<>();
for (final String loadBalancerName : loadBalancerNames) {
final Matcher matcher = ALB_NAME_PATTERN.matcher(loadBalancerName);
final Matcher matcher = ApplicationLoadBalancer.ALB_NAME_PATTERN.matcher(loadBalancerName);
if (matcher.find()) {
numbersTaken.add(Integer.parseInt(matcher.group(1)));
}
}
return DNS_MAPPED_ALB_NAME_PREFIX + IntStream.range(0, MAX_ALBS_PER_REGION).filter(i->!numbersTaken.contains(i)).min().getAsInt();
return ApplicationLoadBalancer.DNS_MAPPED_ALB_NAME_PREFIX + IntStream.range(0, ApplicationLoadBalancer.MAX_ALBS_PER_REGION).filter(i->!numbersTaken.contains(i)).min().getAsInt();
}
@Override
@@ -17,16 +17,11 @@ import com.sap.sse.landscape.aws.ApplicationLoadBalancer;
import com.sap.sse.landscape.aws.AwsInstance;
import com.sap.sse.landscape.aws.AwsLandscape;
import com.sap.sse.landscape.aws.TargetGroup;
import com.sap.sse.landscape.aws.common.shared.PlainRedirectDTO;
import com.sap.sse.shared.util.Wait;
import software.amazon.awssdk.services.ec2.model.InstanceStateName;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Action;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.ActionTypeEnum;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.LoadBalancerStateEnum;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleCondition;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupTuple;
/**
* For an {@link ApplicationProcess} creates a set of rules in an {@link ApplicationLoadBalancer} which drives traffic
@@ -65,7 +60,7 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupT
* <p>
*
* The default health check settings for the target groups created are:
* <ul>
* <ul>Procedure<ShardingKey>
* <li>healthy threshold: 2</li>
* <li>unhealthy threshold: 2</li>
* <li>timeout: 4s</li>
@@ -77,8 +72,8 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupT
*/
public abstract class CreateLoadBalancerMapping<ShardingKey, MetricsT extends ApplicationProcessMetrics,
ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>>
extends ProcedureWithTargetGroup<ShardingKey> {
protected static int NUMBER_OF_RULES_PER_REPLICA_SET = 5;
extends ProcedureWithTargetGroup<ShardingKey>
implements ProcedureCreatingLoadBalancerMapping<ShardingKey> {
protected static final int MAX_RULES_PER_ALB = 100;
protected static final int MAX_ALBS_PER_REGION = 20;
private final ProcessT process;
@@ -216,44 +211,13 @@ extends ProcedureWithTargetGroup<ShardingKey> {
Level.INFO, "Waiting for instance "+getHost().getId()+" to be in state RUNNING");
getLandscape().addTargetsToTargetGroup(masterTargetGroupCreated, Collections.singleton(getHost()));
getLandscape().addTargetsToTargetGroup(publicTargetGroupCreated, Collections.singleton(getHost()));
getLoadBalancerUsed().addRulesAssigningUnusedPriorities(/* forceContiguous */ true, createRules());
getLoadBalancerUsed().addRulesAssigningUnusedPriorities(/* forceContiguous */ true,
createRules(getLoadBalancerUsed(), getHostName(), masterTargetGroupCreated, publicTargetGroupCreated));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private Rule[] createRules() {
final Rule[] rules = new Rule[NUMBER_OF_RULES_PER_REPLICA_SET];
int ruleCount = 0;
rules[ruleCount++] = getLoadBalancerUsed().getDefaultRedirectRule(getHostName(), new PlainRedirectDTO());
rules[ruleCount++] = Rule.builder().conditions(
RuleCondition.builder().field("http-header").httpHeaderConfig(hhcb->hhcb.httpHeaderName(HttpRequestHeaderConstants.HEADER_KEY_FORWARD_TO).values(HttpRequestHeaderConstants.HEADER_FORWARD_TO_MASTER.getB())).build(),
getLoadBalancerUsed().createHostHeaderRuleCondition(getHostName())).
actions(createForwardToTargetGroupAction(getMasterTargetGroupCreated())).
build();
rules[ruleCount++] = Rule.builder().conditions(
RuleCondition.builder().field("http-header").httpHeaderConfig(hhcb->hhcb.httpHeaderName(HttpRequestHeaderConstants.HEADER_KEY_FORWARD_TO).values(HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA.getB())).build(),
getLoadBalancerUsed().createHostHeaderRuleCondition(getHostName())).
actions(createForwardToTargetGroupAction(getPublicTargetGroupCreated())).
build();
rules[ruleCount++] = Rule.builder().conditions(
RuleCondition.builder().field("http-request-method").httpRequestMethodConfig(hrmcb->hrmcb.values("GET")).build(),
getLoadBalancerUsed().createHostHeaderRuleCondition(getHostName())).
actions(createForwardToTargetGroupAction(getPublicTargetGroupCreated())).
build();
rules[ruleCount++] = Rule.builder().conditions(
getLoadBalancerUsed().createHostHeaderRuleCondition(getHostName())).
actions(createForwardToTargetGroupAction(getMasterTargetGroupCreated())).
build();
assert ruleCount == NUMBER_OF_RULES_PER_REPLICA_SET;
return rules;
}
private Action createForwardToTargetGroupAction(TargetGroup<ShardingKey> targetGroup) {
return Action.builder().type(ActionTypeEnum.FORWARD).forwardConfig(fc -> fc.targetGroups(
TargetGroupTuple.builder().targetGroupArn(targetGroup.getTargetGroupArn()).build())) .build();
}
protected String getHostName() {
return hostname;
}
@@ -0,0 +1,185 @@
package com.sap.sse.landscape.aws.orchestration;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import com.sap.sse.common.Duration;
import com.sap.sse.common.HttpRequestHeaderConstants;
import com.sap.sse.common.Util;
import com.sap.sse.landscape.Landscape;
import com.sap.sse.landscape.application.ApplicationProcess;
import com.sap.sse.landscape.application.ApplicationProcessMetrics;
import com.sap.sse.landscape.aws.ApplicationLoadBalancer;
import com.sap.sse.landscape.aws.AwsAutoScalingGroup;
import com.sap.sse.landscape.aws.AwsInstance;
import com.sap.sse.landscape.aws.TargetGroup;
import com.sap.sse.landscape.aws.common.shared.ShardTargetGroupName;
import com.sap.sse.shared.util.Wait;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Action;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.ActionTypeEnum;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.ForwardActionConfig;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleCondition;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupTuple;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetHealth;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetHealthStateEnum;
/**
* This class is for creating shards out of the {@code shardingKeys}. This class creats a target group, and an
* autoscaling group and inserts rules into the {@code replicaSet}'s load balancer. If the load balancer does not have
* enough rules left in it'S HTTPS-listener, the whole replica set gets moved to another load balancer.
*
* @author I569653
*
* @param <ShardingKey>
* @param <MetricsT>
* @param <ProcessT>
*/
public class CreateShard<ShardingKey, MetricsT extends ApplicationProcessMetrics, ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>>
extends ShardProcedure<ShardingKey, MetricsT, ProcessT> {
private static int DEFAULT_INSTANCE_STARTUP_TIME = 180;
private static final Logger logger = Logger.getLogger(ShardProcedure.class.getName());
private final String targetGroupNamePrefix;
public CreateShard(BuilderImpl<?, ShardingKey, MetricsT, ProcessT> builder) throws Exception {
super(builder);
this.targetGroupNamePrefix = builder.getTargetGroupNamePrefix();
}
public static interface Builder<
BuilderT extends Builder<BuilderT, T, ShardingKey, MetricsT, ProcessT>,
T extends CreateShard<ShardingKey, MetricsT, ProcessT>,
ShardingKey,
MetricsT extends ApplicationProcessMetrics,
ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>>
extends ShardProcedure.Builder<BuilderT, T, ShardingKey, MetricsT, ProcessT> {
BuilderT setTargetGroupNamePrefix(String targetGroupNamePrefix);
}
static class BuilderImpl<BuilderT extends Builder<BuilderT, CreateShard<ShardingKey, MetricsT, ProcessT>, ShardingKey, MetricsT, ProcessT>,
ShardingKey,
MetricsT extends ApplicationProcessMetrics,
ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>>
extends ShardProcedure.BuilderImpl<BuilderT, CreateShard<ShardingKey, MetricsT, ProcessT>, ShardingKey, MetricsT, ProcessT>
implements Builder<BuilderT, CreateShard<ShardingKey, MetricsT, ProcessT>, ShardingKey, MetricsT, ProcessT> {
private String targetGroupNamePrefix = "";
String getTargetGroupNamePrefix() {
return targetGroupNamePrefix;
}
public BuilderT setTargetGroupNamePrefix(String targetGroupNamePrefix) {
if (!ShardTargetGroupName.isValidTargetGroupNamePrefix(targetGroupNamePrefix)) {
throw new IllegalArgumentException("Not a valid target group name prefix: "+targetGroupNamePrefix);
}
this.targetGroupNamePrefix = targetGroupNamePrefix;
return self();
}
@Override
public CreateShard<ShardingKey, MetricsT, ProcessT> build() throws Exception {
assert shardingKeys != null;
assert replicaSet != null;
assert region != null;
assert passphraseForPrivateKeyDecryption != null;
return new CreateShard<ShardingKey, MetricsT, ProcessT>(this);
}
}
@Override
public void run() throws Exception {
final ShardTargetGroupName name;
if (shardName == null) {
throw new Exception("Shardname is null, please enter a name");
} else {
name = replicaSet.getNewShardName(shardName, targetGroupNamePrefix);
}
if (!isTargetGroupNameUnique(name.getTargetGroupName())) {
throw new Exception(
"targetgroup name with this shardname is not unique. You may change the last or first two chars");
}
final ApplicationLoadBalancer<ShardingKey> loadBalancer = getFreeLoadBalancerAndMoveReplicaSet();
logger.info(
"Creating Targer group for Shard " + name + ". Inheriting from Replicaset: " + replicaSet.getName());
final TargetGroup<ShardingKey> targetGroup = getLandscape().createTargetGroupWithoutLoadbalancer(region,
name.getTargetGroupName(), replicaSet.getMaster().getPort());
getLandscape().addTargetGroupTag(targetGroup.getTargetGroupArn(), ShardTargetGroupName.TAG_KEY, name.getName(), region);
final AwsAutoScalingGroup autoScalingGroup = replicaSet.getAutoScalingGroup();
logger.info("Creating Autoscalinggroup for Shard " + shardName + ". Inheriting from Autoscalinggroup: "
+ autoScalingGroup.getName());
getLandscape().createAutoScalingGroupFromExisting(autoScalingGroup, shardName, targetGroup, Optional.empty());
// create one rule to path unused by any application for linking ALB to target group.
if (loadBalancer != null) {
final Iterable<Rule> rules = loadBalancer.getRules();
if (Util.size(rules) < ApplicationLoadBalancer.MAX_RULES_PER_LOADBALANCER
- numberOfRequiredRules(Util.size(shardingKeys))) {
final int rulePrio = getHighestAvailableIndex(rules);
if (rulePrio > 0) {
Rule newRule = Rule.builder().priority("" + rulePrio)
.conditions(
loadBalancer.createHostHeaderRuleCondition(replicaSet.getHostname()),
RuleCondition.builder().field("http-header")
.httpHeaderConfig(hhcb -> hhcb
.httpHeaderName(HttpRequestHeaderConstants.HEADER_KEY_FORWARD_TO)
.values(HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA
.getB()))
.build(),
RuleCondition.builder().field("path-pattern")
.pathPatternConfig(ppc -> ppc.values(getPathConditionForShardingKey(SHARDING_KEY_UNUSED_BY_ANY_APPLICATION))).build())
.actions(Action.builder()
.forwardConfig(ForwardActionConfig.builder()
.targetGroups(TargetGroupTuple.builder()
.targetGroupArn(targetGroup.getTargetGroupArn()).build())
.build())
.type(ActionTypeEnum.FORWARD).build())
.build();
final Iterable<Rule> newRuleSet = loadBalancer.addRules(newRule);
getLandscape().putScalingPolicy(DEFAULT_INSTANCE_STARTUP_TIME, getLandscape().getAutoScalingGroupName(shardName), targetGroup,
AwsAutoScalingGroup.DEFAULT_MAX_REQUESTS_PER_TARGET, region);
// wait until instances are running
Wait.wait(()->{
boolean ret = true;
final Map<AwsInstance<ShardingKey>, TargetHealth> healths = getLandscape()
.getTargetHealthDescriptions(targetGroup);
if (healths.isEmpty()) {
ret = false; // if there is no Aws in target
} else {
for (Map.Entry<AwsInstance<ShardingKey>, TargetHealth> instance : healths.entrySet()) {
if (instance.getValue().state() != TargetHealthStateEnum.HEALTHY) {
ret = false; // if this instance is unhealthy
break;
}
}
}
return ret;
}, Landscape.WAIT_FOR_HOST_TIMEOUT, Duration.ONE_SECOND.times(30), Level.INFO, "Instances not yet healty");
// remove dummy-rule
for (Rule r : newRuleSet) {
loadBalancer.deleteRules(r);
}
final Set<ShardingKey> shardingKeysToUse;
if (shardingKeys.isEmpty()) {
shardingKeysToUse = Collections.singleton(SHARDING_KEY_UNUSED_BY_ANY_APPLICATION);
} else {
shardingKeysToUse = shardingKeys;
}
// change ALB rules to new ones
addShardingRules(loadBalancer, shardingKeysToUse, targetGroup);
} else {
throw new Exception("Unexpected Error - No prio left?");
}
} else {
throw new Exception("Unexpected Error - Loadbalancer was null!");
}
}
}
public static <MetricsT extends ApplicationProcessMetrics, ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>, BuilderT extends Builder<BuilderT, CreateShard<ShardingKey, MetricsT, ProcessT>, ShardingKey, MetricsT, ProcessT>, ShardingKey> Builder<BuilderT, CreateShard<ShardingKey, MetricsT, ProcessT>, ShardingKey, MetricsT, ProcessT> builder() {
return new BuilderImpl<BuilderT, ShardingKey, MetricsT, ProcessT>();
}
}
@@ -0,0 +1,59 @@
package com.sap.sse.landscape.aws.orchestration;
import com.sap.sse.common.HttpRequestHeaderConstants;
import com.sap.sse.landscape.aws.ApplicationLoadBalancer;
import com.sap.sse.landscape.aws.TargetGroup;
import com.sap.sse.landscape.aws.common.shared.PlainRedirectDTO;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Action;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.ActionTypeEnum;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleCondition;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupTuple;
public interface ProcedureCreatingLoadBalancerMapping<ShardingKey> {
/**
* The default number of rules required for a replica set, including a default redirect rule, but excluding
* any additional sharding rules.
*/
int NUMBER_OF_RULES_PER_REPLICA_SET = 5;
default Rule[] createRules(ApplicationLoadBalancer<ShardingKey> alb, String hostName, TargetGroup<ShardingKey> masterTargetGroup,
TargetGroup<ShardingKey> publicTargetGroup) {
final Rule[] rules = new Rule[NUMBER_OF_RULES_PER_REPLICA_SET];
int ruleCount = 0;
rules[ruleCount++] = alb.getDefaultRedirectRule(hostName, new PlainRedirectDTO());
rules[ruleCount++] = Rule.builder().conditions(
RuleCondition.builder().field("http-header")
.httpHeaderConfig(hhcb -> hhcb.httpHeaderName(HttpRequestHeaderConstants.HEADER_KEY_FORWARD_TO)
.values(HttpRequestHeaderConstants.HEADER_FORWARD_TO_MASTER.getB()))
.build(),
alb.createHostHeaderRuleCondition(hostName))
.actions(createForwardToTargetGroupAction(masterTargetGroup)).build();
rules[ruleCount++] = Rule.builder().conditions(
RuleCondition.builder().field("http-header")
.httpHeaderConfig(hhcb -> hhcb.httpHeaderName(HttpRequestHeaderConstants.HEADER_KEY_FORWARD_TO)
.values(HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA.getB()))
.build(),
alb.createHostHeaderRuleCondition(hostName))
.actions(createForwardToTargetGroupAction(publicTargetGroup)).build();
rules[ruleCount++] = Rule.builder()
.conditions(
RuleCondition.builder().field("http-request-method")
.httpRequestMethodConfig(hrmcb -> hrmcb.values("GET")).build(),
alb.createHostHeaderRuleCondition(hostName))
.actions(createForwardToTargetGroupAction(publicTargetGroup)).build();
rules[ruleCount++] = Rule.builder()
.conditions(alb.createHostHeaderRuleCondition(hostName))
.actions(createForwardToTargetGroupAction(masterTargetGroup)).build();
assert ruleCount == NUMBER_OF_RULES_PER_REPLICA_SET;
return rules;
}
default Action createForwardToTargetGroupAction(TargetGroup<ShardingKey> targetGroup) {
return Action.builder().type(ActionTypeEnum.FORWARD).forwardConfig(fc -> fc.targetGroups(
TargetGroupTuple.builder().targetGroupArn(targetGroup.getTargetGroupArn()).build())) .build();
}
}
@@ -12,8 +12,6 @@ import com.sap.sse.landscape.aws.ApplicationLoadBalancer;
import com.sap.sse.landscape.aws.AwsLandscape;
import com.sap.sse.landscape.aws.TargetGroup;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule;
/**
* An abstract base class for procedures dealing with the public and master target groups in a
* {@link ApplicationLoadBalancer load balancer}, e.g., creating them or fetching them, or
@@ -27,7 +25,6 @@ extends AbstractAwsProcedureImpl<ShardingKey> {
private final ApplicationLoadBalancer<ShardingKey> loadBalancerUsed;
private final String targetGroupNamePrefix;
private final String serverName;
private Iterable<Rule> rulesAdded;
/**
* If no {@link #setTargetGroupNamePrefix(String) target group name prefix} is specified, the target group names are
@@ -128,8 +125,4 @@ extends AbstractAwsProcedureImpl<ShardingKey> {
public ApplicationLoadBalancer<ShardingKey> getLoadBalancerUsed() {
return loadBalancerUsed;
}
public Iterable<Rule> getRulesAdded() {
return rulesAdded;
}
}
@@ -0,0 +1,91 @@
package com.sap.sse.landscape.aws.orchestration;
import java.util.HashSet;
import java.util.Set;
import java.util.Map.Entry;
import java.util.logging.Logger;
import com.sap.sse.common.Util;
import com.sap.sse.landscape.application.ApplicationProcess;
import com.sap.sse.landscape.application.ApplicationProcessMetrics;
import com.sap.sse.landscape.aws.AwsShard;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleCondition;
/**
* This class is supposed to remove {@code shardingKeys} from the shard, indentified by {@shardName} from
* {@code replicaSet}. This is done by rewriting all rules without {@code shardingKeys}'s path-conditions to the
* replicaSet's load balancer. In the future, there should be a algorithm to extract and resort all rules without
* removing them for ensuring that 100% of time the requests are reaching the shard's target group.
*
* @author I569653
*
* @param <ShardingKey>
* @param <MetricsT>
* @param <ProcessT>
*/
public class RemoveShardingKeyFromShard<ShardingKey, MetricsT extends ApplicationProcessMetrics, ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>>
extends ShardProcedure<ShardingKey, MetricsT, ProcessT> {
private static final Logger logger = Logger.getLogger(RemoveShardingKeyFromShard.class.getName());
public RemoveShardingKeyFromShard(BuilderImpl<?, ShardingKey, MetricsT, ProcessT> builder) throws Exception {
super(builder);
}
static class BuilderImpl<BuilderT extends Builder<BuilderT, RemoveShardingKeyFromShard<ShardingKey, MetricsT, ProcessT>, ShardingKey, MetricsT, ProcessT>, ShardingKey, MetricsT extends ApplicationProcessMetrics, ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>>
extends
ShardProcedure.BuilderImpl<BuilderT, RemoveShardingKeyFromShard<ShardingKey, MetricsT, ProcessT>, ShardingKey, MetricsT, ProcessT> {
@Override
public RemoveShardingKeyFromShard<ShardingKey, MetricsT, ProcessT> build() throws Exception {
assert shardingKeys != null;
assert replicaSet != null;
assert region != null;
assert passphraseForPrivateKeyDecryption != null;
return new RemoveShardingKeyFromShard<ShardingKey, MetricsT, ProcessT>(this);
}
}
@Override
public void run() throws Exception {
AwsShard<ShardingKey> shard = null;
for (Entry<AwsShard<ShardingKey>, Iterable<ShardingKey>> entry : replicaSet.getShards().entrySet()) {
if (entry.getKey().getName().equals(shardName)) {
shard = entry.getKey();
break;
}
}
if (shard == null) {
throw new Exception("Shard not found!");
}
logger.info("Removing " + Util.joinStrings(", ", shardingKeys) + " from " + shardName);
// remove conditions in rules where path is the sharding key
final Set<ShardingKey> shardingKeysFromConditions = new HashSet<>();
for (Rule r : shard.getRules()) {
for (RuleCondition condition : r.conditions()) {
if (condition.pathPatternConfig() != null) {
shardingKeysFromConditions.addAll(
Util.asList(
Util.filter(
Util.map(condition.values(), this::getShardingKeyFromPathCondition),
shardingKey -> !shardingKeys.contains(shardingKey))));
} else {
logger.warning("This is strange: shard "+shard.getName()+" of replica set "+shard.getReplicaSetName()+
" has a rule "+r+" that has no path pattern condition; ignoring that rule while removing shard.");
}
}
}
if (shardingKeysFromConditions.isEmpty()) {
// if the shard runs empty (no more sharding keys defined for it), set a proxy key to keep
// the shard discoverable and its target group linked to the load balancer for continued target health checks
shardingKeysFromConditions.add(SHARDING_KEY_UNUSED_BY_ANY_APPLICATION);
}
getLandscape().deleteLoadBalancerListenerRules(region, Util.toArray(shard.getRules(), new Rule[0]));
// change ALB rules to new ones
addShardingRules(shard.getLoadBalancer(), shardingKeysFromConditions, shard.getTargetGroup());
}
public static <MetricsT extends ApplicationProcessMetrics, ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>, BuilderT extends Builder<BuilderT, RemoveShardingKeyFromShard<ShardingKey, MetricsT, ProcessT>, ShardingKey, MetricsT, ProcessT>, ShardingKey> Builder<BuilderT, RemoveShardingKeyFromShard<ShardingKey, MetricsT, ProcessT>, ShardingKey, MetricsT, ProcessT> builder() {
return new BuilderImpl<BuilderT, ShardingKey, MetricsT, ProcessT>();
}
}
@@ -0,0 +1,479 @@
package com.sap.sse.landscape.aws.orchestration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.stream.IntStream;
import java.util.stream.StreamSupport;
import com.sap.sse.common.HttpRequestHeaderConstants;
import com.sap.sse.common.Util;
import com.sap.sse.landscape.Region;
import com.sap.sse.landscape.application.ApplicationProcess;
import com.sap.sse.landscape.application.ApplicationProcessMetrics;
import com.sap.sse.landscape.aws.ApplicationLoadBalancer;
import com.sap.sse.landscape.aws.AwsApplicationReplicaSet;
import com.sap.sse.landscape.aws.AwsLandscape;
import com.sap.sse.landscape.aws.AwsShard;
import com.sap.sse.landscape.aws.TargetGroup;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Action;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.ActionTypeEnum;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.ForwardActionConfig;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleCondition;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupTuple;
/**
* This class is the base for all procedures that deal with shards. In the subclasses, all procedures are described.
* The parameter <ShardingKey> stands for a type that represents a sharding key. But those keys are normally strings because AWS deals with them as strings in
* their Rules. The function Shard.getKeys returns a list of the keys contained by the shard. Those keys are found in the shard's rules as a path-condition.
* This class implements most of the required functionality when dealing with shards like inserting rules or switching a replica set's load balancer.
* @author I569653
*
* @param <ShardingKey>
* @param <MetricsT>
* @param <ProcessT>
*/
public abstract class ShardProcedure<ShardingKey,
MetricsT extends ApplicationProcessMetrics,
ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>>
extends AbstractAwsProcedureImpl<ShardingKey>
implements ProcedureCreatingLoadBalancerMapping<ShardingKey> {
private static final Logger logger = Logger.getLogger(ShardProcedure.class.getName());
final static int NUMBER_OF_STANDARD_CONDITIONS_FOR_SHARDING_RULE = 2;
@SuppressWarnings("unchecked") // this silently assumes that a String can be cast to a ShardingKey without problems
protected final ShardingKey SHARDING_KEY_UNUSED_BY_ANY_APPLICATION = (ShardingKey) "lauycaluy3cla3yrclaurlIYQL8";
protected final String shardName;
final protected Set<ShardingKey> shardingKeys;
final AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaSet;
final Region region;
final byte[] passphraseForPrivateKeyDecryption;
private final String pathPrefixForShardingKey;
protected ShardProcedure(BuilderImpl<?,?, ShardingKey, MetricsT, ProcessT> builder) throws Exception {
super(builder);
this.shardName = builder.getShardName();
this.passphraseForPrivateKeyDecryption = builder.getPassphrase();
this.replicaSet = builder.getReplicaSet();
this.shardingKeys = builder.getShardingKeys();
this.region = builder.getRegion();
this.pathPrefixForShardingKey = builder.getPathPrefixForShardingKey();
}
public static interface Builder<
BuilderT extends Builder<BuilderT, T, ShardingKey, MetricsT, ProcessT>,
T extends ShardProcedure<ShardingKey, MetricsT, ProcessT>,
ShardingKey,
MetricsT extends ApplicationProcessMetrics,
ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>>
extends AbstractAwsProcedureImpl.Builder<BuilderT, T, ShardingKey> {
BuilderT setPathPrefixForShardingKey(String pathPrefixForShardingKey);
BuilderT setShardName(String name);
BuilderT setLandscape(AwsLandscape<String> landscape);
BuilderT setShardingKeys(Set<ShardingKey> shardingkeys);
BuilderT setReplicaset(AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaset);
BuilderT setRegion(Region region);
BuilderT setPassphrase(byte[] passphrase);
}
protected abstract static class BuilderImpl<BuilderT extends Builder<BuilderT, T, ShardingKey, MetricsT, ProcessT>,
T extends ShardProcedure<ShardingKey,MetricsT,ProcessT>,
ShardingKey,
MetricsT extends ApplicationProcessMetrics,
ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>>
extends
AbstractAwsProcedureImpl.BuilderImpl<BuilderT, T, ShardingKey>
implements
Builder<BuilderT, T, ShardingKey, MetricsT, ProcessT> {
protected String shardName;
protected Set<ShardingKey> shardingKeys;
protected AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaSet;
protected Region region;
protected byte[] passphraseForPrivateKeyDecryption;
private String pathPrefixForShardingKey;
@Override
public BuilderT setPathPrefixForShardingKey(String pathPrefixForShardingKey) {
this.pathPrefixForShardingKey = pathPrefixForShardingKey;
return self();
}
@Override
public BuilderT setShardName(String name) {
this.shardName = name;
return self();
}
@Override
public BuilderT setShardingKeys(Set<ShardingKey> shardingkeys) {
this.shardingKeys = shardingkeys;
return self();
}
@Override
public BuilderT setReplicaset(AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaset) {
this.replicaSet = replicaset;
return self();
}
@Override
public BuilderT setRegion(Region region) {
this.region = region;
return self();
}
@Override
public BuilderT setPassphrase(byte[] passphrase) {
this.passphraseForPrivateKeyDecryption = passphrase;
return self();
}
@SuppressWarnings("unchecked")
@Override
public BuilderT setLandscape(AwsLandscape<String> landscape) {
super.setLandscape((AwsLandscape<ShardingKey>) landscape);
return self();
}
protected AwsLandscape<ShardingKey> getLandscape() {
return (AwsLandscape<ShardingKey>) super.getLandscape();
}
byte[] getPassphrase() {
return passphraseForPrivateKeyDecryption;
}
Region region() {
return region;
}
AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> getReplicaSet() {
return replicaSet;
}
Set<ShardingKey> getShardingKeys() {
return shardingKeys;
}
String getShardName() {
return shardName;
}
Region getRegion() {
return region;
}
String getPathPrefixForShardingKey() {
return pathPrefixForShardingKey;
}
}
protected boolean isTargetGroupNameUnique(String name) {
final Iterable<TargetGroup<ShardingKey>> targetGroups = getLandscape().getTargetGroups(region);
return Util.isEmpty(Util.filter(targetGroups, t -> t.getName().equals(name)));
}
/**
* Produces conditions for a sharding load balancer rule based on the {@link #replicaSet}'s
* {@link AwsApplicationReplicaSet#getHostname() hostname}, a header-field condition that requires the request to be
* tagged for a replica, plus a path-pattern condition with the sharding keys as patterns.
*
* @param shardingKeys
* their number must not exceed {@link ApplicationLoadBalancer#MAX_CONDITIONS_PER_RULE} -
* {@link #NUMBER_OF_STANDARD_CONDITIONS_FOR_SHARDING_RULE}; pass sharding keys (as the name suggests),
* not full paths; the paths for the {@code path-pattern} condition will be constructed from the sharding
* keys by this method. See also {@link #getPathConditionForShardingKey(String, String)}.
*/
protected Collection<RuleCondition> getShardingRuleConditions(ApplicationLoadBalancer<ShardingKey> loadBalancer,
Collection<ShardingKey> shardingKeys) throws InterruptedException, ExecutionException {
if (shardingKeys.size() > ApplicationLoadBalancer.MAX_CONDITIONS_PER_RULE - NUMBER_OF_STANDARD_CONDITIONS_FOR_SHARDING_RULE) {
throw new IllegalArgumentException("too many sharding keys for the conditions of a single load balancer rule: "+shardingKeys+
"; a maximum of "+(ApplicationLoadBalancer.MAX_CONDITIONS_PER_RULE - NUMBER_OF_STANDARD_CONDITIONS_FOR_SHARDING_RULE)+" is allowed");
}
final Collection<RuleCondition> ruleConditions = new ArrayList<>();
final Collection<String> paths = Util.mapToArrayList(shardingKeys, shardingKey->getPathConditionForShardingKey(shardingKey, pathPrefixForShardingKey));
ruleConditions.add(loadBalancer.createHostHeaderRuleCondition(replicaSet.getHostname()));
ruleConditions.add(RuleCondition.builder().field("http-header")
.httpHeaderConfig(hhcb -> hhcb.httpHeaderName(HttpRequestHeaderConstants.HEADER_KEY_FORWARD_TO)
.values(HttpRequestHeaderConstants.HEADER_FORWARD_TO_REPLICA.getB()))
.build());
ruleConditions.add(
RuleCondition.builder().field("path-pattern").pathPatternConfig(hhcb -> hhcb.values(paths)).build());
return ruleConditions;
}
protected Iterable<Rule> addShardingRules(ApplicationLoadBalancer<ShardingKey> alb, Iterable<ShardingKey> shardingKeys,
TargetGroup<ShardingKey> targetGroup) throws Exception {
// change ALB rules to new ones
final Collection<Rule> rules = new ArrayList<Rule>();
final Set<ShardingKey> shardingKeyForConsumption = new HashSet<>();
Util.addAll(shardingKeys, shardingKeyForConsumption);
final int ruleIdx = alb.getFirstShardingPriority(replicaSet.getHostname());
while (!shardingKeyForConsumption.isEmpty()) {
alb.shiftRulesToMakeSpaceAt(ruleIdx);
final Set<ShardingKey> shardingKeysForNextRule = new HashSet<>();
for (final Iterator<ShardingKey> i=shardingKeyForConsumption.iterator();
shardingKeysForNextRule.size() < ApplicationLoadBalancer.MAX_CONDITIONS_PER_RULE-NUMBER_OF_STANDARD_CONDITIONS_FOR_SHARDING_RULE && i.hasNext(); ) {
shardingKeysForNextRule.add(i.next());
i.remove();
}
final Collection<RuleCondition> conditions = getShardingRuleConditions(alb, shardingKeysForNextRule);
rules.add(Rule.builder().priority("" + ruleIdx).conditions(conditions)
.actions(Action.builder()
.forwardConfig(ForwardActionConfig.builder()
.targetGroups(TargetGroupTuple.builder()
.targetGroupArn(targetGroup.getTargetGroupArn()).build())
.build())
.type(ActionTypeEnum.FORWARD).build())
.build());
}
return alb.addRules(Util.toArray(rules, new Rule[0]));
}
/**
* Search through all load balancers and returns the first load balancer with enough rules left for
* the new sharding keys + the replicaSet's rules. When no load balancer is found, a new one gets created. The replicaSet gets moved to the new load balancer.
*/
protected ApplicationLoadBalancer<ShardingKey> getFreeLoadBalancerAndMoveReplicaSet() throws Exception {
int existingShardingRules = 0;
for (Entry<AwsShard<ShardingKey>, Iterable<ShardingKey>> s : replicaSet.getShards().entrySet()) {
existingShardingRules = existingShardingRules + Util.size(s.getKey().getRules());
}
// shardingKeys holds those keys to add/initially create (the remove case doesn't get here);
// hence we have to add the rules required for those keys to the existing rules
// FIXME shouldn't we compute the number of required rules for the new entire set of sharding keys? What if rules were not / no longer fully filled with conditions? -->cleanup
final int requiredRules = numberOfRequiredRules(Util.size(shardingKeys))
+ (existingShardingRules + /* 5 std rules per replica set */ NUMBER_OF_RULES_PER_REPLICA_SET);
final ApplicationLoadBalancer<ShardingKey> res;
if (Util.size(replicaSet.getLoadBalancerRules())
+ numberOfRequiredRules(Util.size(shardingKeys)) < ApplicationLoadBalancer.MAX_RULES_PER_LOADBALANCER) {
res = replicaSet.getLoadBalancer();
} else {
// Another loadbalancer
final Iterable<ApplicationLoadBalancer<ShardingKey>> loadBalancers = getLandscape()
.getLoadBalancers(region);
final Iterable<ApplicationLoadBalancer<ShardingKey>> loadBalancersFiltered = Util.filter(loadBalancers,
t -> {
try {
return !t.getArn().equals(replicaSet.getLoadBalancer().getArn());
} catch (InterruptedException | ExecutionException e) {
logger.log(Level.WARNING, "Exception while trying to obtain a load balancer's ARN", e);
throw new RuntimeException(e);
}
});
final ApplicationLoadBalancer<ShardingKey> alb = getDNSLoadbalancerWithRulesLeft(loadBalancersFiltered,
requiredRules + /* 5 default rules for the replica set */ NUMBER_OF_RULES_PER_REPLICA_SET);
if (alb != null) {
// There is an alb left with enough rules
res = alb;
} else {
final Set<String> loadBalancerNames = new HashSet<>();
for (ApplicationLoadBalancer<ShardingKey> lb : loadBalancers) {
loadBalancerNames.add(lb.getName());
}
final String name = getAvailableDNSMappedAlbName(loadBalancerNames);
// Create a new alb
res = getLandscape().createLoadBalancer(name, region);
}
changeReplicaSetLoadBalancer(res, replicaSet);
}
return res;
}
/**
* This method changes the {@code replicaSetToMove}'s load balancer to another one. This method contains a 6min
* sleep, which is necessary for ensuring that all DNS rules point to the new one. This method can fail if the
* target group names are too long because of the {@link TargetGroup#TEMP_SUFFIX} suffix for temporary target
* groups.
*
* @param targetAlb
* application load balancer to move to
* @param replicaSetToMove
* replica set to move
*/
private void changeReplicaSetLoadBalancer(ApplicationLoadBalancer<ShardingKey> targetAlb,
AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaSetToMove) throws Exception {
// Move replicaset to this alb with all shards
// create temporary targetgroups
final Collection<TargetGroup<ShardingKey>> tempTargetGroups = new ArrayList<>();
final Collection<TargetGroup<ShardingKey>> originalTargetGroups = new ArrayList<>();
final Map<TargetGroup<ShardingKey>, Iterable<ShardingKey>> shardingKeysPerTargetGroup = new HashMap<>();
final Map<TargetGroup<ShardingKey>, TargetGroup<ShardingKey>> targetGroupsToTempTargetgroups = new HashMap<>();
final Map<AwsShard<ShardingKey>, TargetGroup<ShardingKey>> shardToTempTargetGroup = new HashMap<>();
// add non sharding rules for replicaset
final Collection<Rule> tempRules = new ArrayList<>();
createTargetGroupsForMoving(shardToTempTargetGroup, tempTargetGroups, replicaSetToMove, targetGroupsToTempTargetgroups, originalTargetGroups, shardingKeysPerTargetGroup);
addRulesForMoving(targetAlb, shardToTempTargetGroup, tempRules, replicaSetToMove, targetGroupsToTempTargetgroups, originalTargetGroups, shardingKeysPerTargetGroup);
// set new DNS record -> overwrites old entry
final String hostname = replicaSetToMove.getHostname();
getLandscape().setDNSRecordToApplicationLoadBalancer(replicaSetToMove.getHostedZoneId(),
hostname, targetAlb, /* force */ true);
// wait until new DNS record is alive
for (int i = 0; i < 6; i++) {
Thread.sleep(AwsLandscape.DEFAULT_DNS_TTL_SECONDS * /* conversion seconds to ms */ 1000);
logger.info(()->("Still waiting for DNS record " + hostname));
}
logger.info(()->("Done waiting for DNS record " + hostname));
// remove all old rules pointing to original TargetGroups
final Collection<Rule> rulesToRemove = new ArrayList<>();
replicaSetToMove.getLoadBalancer().getRulesForTargetGroups(originalTargetGroups)
.forEach(t -> rulesToRemove.add(t));
rulesToRemove.add(replicaSetToMove.getDefaultRedirectRule());
getLandscape().deleteLoadBalancerListenerRules(region, rulesToRemove.toArray(new Rule[0]));
for (Entry<TargetGroup<ShardingKey>, TargetGroup<ShardingKey>> entry : targetGroupsToTempTargetgroups
.entrySet()) {
targetAlb.replaceTargetGroupInForwardRules(entry.getValue(), entry.getKey());
}
for (TargetGroup<ShardingKey> t : tempTargetGroups) {
getLandscape().deleteTargetGroup(t);
}
}
private void createTargetGroupsForMoving(
Map<AwsShard<ShardingKey>, TargetGroup<ShardingKey>> shardToTempTargetGroup,
Collection<TargetGroup<ShardingKey>> tempTargetGroups,
AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaSetToMove,
Map<TargetGroup<ShardingKey>, TargetGroup<ShardingKey>> targetGroupsToTempTargetgroups,
Collection<TargetGroup<ShardingKey>> originalTargetGroups,
Map<TargetGroup<ShardingKey>, Iterable<ShardingKey>> shardingKeysPerTargetGroup) throws Exception {
final TargetGroup<ShardingKey> targetgroupMasterTemp = getLandscape()
.copyTargetGroup(replicaSetToMove.getMasterTargetGroup(), TargetGroup.TEMP_SUFFIX);
final TargetGroup<ShardingKey> targetgroupPublicTemp = getLandscape()
.copyTargetGroup(replicaSetToMove.getPublicTargetGroup(), TargetGroup.TEMP_SUFFIX);
tempTargetGroups.add(targetgroupMasterTemp);
tempTargetGroups.add(targetgroupPublicTemp);
targetGroupsToTempTargetgroups.put(replicaSetToMove.getMasterTargetGroup(), targetgroupMasterTemp);
targetGroupsToTempTargetgroups.put(replicaSetToMove.getPublicTargetGroup(), targetgroupPublicTemp);
originalTargetGroups.add(replicaSetToMove.getMasterTargetGroup());
originalTargetGroups.add(replicaSetToMove.getPublicTargetGroup());
for (Entry<AwsShard<ShardingKey>, Iterable<ShardingKey>> shardAndShardingKeys : replicaSetToMove.getShards().entrySet()) {
final TargetGroup<ShardingKey> tempShardTargetGroup = getLandscape()
.copyTargetGroup(shardAndShardingKeys.getKey().getTargetGroup(), TargetGroup.TEMP_SUFFIX);
shardToTempTargetGroup.put(shardAndShardingKeys.getKey(), tempShardTargetGroup);
shardingKeysPerTargetGroup.put(shardAndShardingKeys.getKey().getTargetGroup(), shardAndShardingKeys.getValue());
tempTargetGroups.add(tempShardTargetGroup);
originalTargetGroups.add(shardAndShardingKeys.getKey().getTargetGroup());
targetGroupsToTempTargetgroups.put(shardAndShardingKeys.getKey().getTargetGroup(), tempShardTargetGroup);
}
}
private void addRulesForMoving(ApplicationLoadBalancer<ShardingKey> targetAlb,
Map<AwsShard<ShardingKey>, TargetGroup<ShardingKey>> shardToTempTargetGroup, Collection<Rule> tempRules,
AwsApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> replicaSetToMove,
Map<TargetGroup<ShardingKey>, TargetGroup<ShardingKey>> targetGroupsToTempTargetgroups,
Collection<TargetGroup<ShardingKey>> originalTargetGroups,
Map<TargetGroup<ShardingKey>, Iterable<ShardingKey>> shardingKeysPerTargetGroup) throws Exception {
targetAlb
.addRulesAssigningUnusedPriorities(true,
createRules(targetAlb, replicaSet.getHostname(),
targetGroupsToTempTargetgroups.get(replicaSetToMove.getMasterTargetGroup()),
targetGroupsToTempTargetgroups.get(replicaSetToMove.getPublicTargetGroup())))
.forEach(t -> tempRules.add(t));
for (final Entry<AwsShard<ShardingKey>, Iterable<ShardingKey>> shardAndShardingKeys : replicaSetToMove.getShards().entrySet()) {
addShardingRules(targetAlb, shardingKeysPerTargetGroup.get(shardAndShardingKeys.getKey().getTargetGroup()),
shardToTempTargetGroup.get(shardAndShardingKeys.getKey())).forEach(t -> tempRules.add(t));
}
}
protected int numberOfRequiredRules(int numberOfShardingKeys) {
return (int) (numberOfShardingKeys / (ApplicationLoadBalancer.MAX_CONDITIONS_PER_RULE-NUMBER_OF_STANDARD_CONDITIONS_FOR_SHARDING_RULE))
+ (int) Math.signum(/* one more because casting to int rounds down */ numberOfShardingKeys %
(ApplicationLoadBalancer.MAX_CONDITIONS_PER_RULE-NUMBER_OF_STANDARD_CONDITIONS_FOR_SHARDING_RULE));
}
/**
* Returns a DNS-Mapped load balancer with enough rules left
* @param loadBalancers
* list of all load balancers to search through.
* @param numberOfRules
* number of required rules left in this load balancer
* @return
*/
private ApplicationLoadBalancer<ShardingKey> getDNSLoadbalancerWithRulesLeft(
Iterable<ApplicationLoadBalancer<ShardingKey>> loadBalancers, int numberOfRules) {
final Iterable<ApplicationLoadBalancer<ShardingKey>> loadBalancersFiltered = Util.filter(loadBalancers,
t -> t.getName().startsWith(ApplicationLoadBalancer.DNS_MAPPED_ALB_NAME_PREFIX));
ApplicationLoadBalancer<ShardingKey> res = null;
for (ApplicationLoadBalancer<ShardingKey> loadBalancer : loadBalancersFiltered) {
if (Util.size(loadBalancer.getRules()) < ApplicationLoadBalancer.MAX_RULES_PER_LOADBALANCER - numberOfRules) {
res = loadBalancer;
break;
}
}
return res;
}
/**
* iterates through all numbers from {@link ApplicationLoadBalancer#MAX_PRIORITY} to 1 (lowest index) and checks if
* any priority is not in the rule set. returns the first available priority. If no rules is available, it returns
* -1;
*/
protected int getHighestAvailableIndex(Iterable<Rule> rules) {
for (int i = ApplicationLoadBalancer.MAX_PRIORITY; i > 1; i--) {
String y = "" + i;
if (!StreamSupport.stream(rules.spliterator(), false).anyMatch(t -> t.priority().contains(y))) {
return i; // return priority if there was no rule with the same
}
}
return -1; // if no free priority was found
}
/**
* Picks a new load balancer name following the pattern {@link #DNS_MAPPED_ALB_NAME_PREFIX}{@code [0-9]+} that is
* not part of {@code loadBalancerNames} and has the least number.
*/
private String getAvailableDNSMappedAlbName(Set<String> loadBalancerNames) {
final Set<Integer> numbersTaken = new HashSet<>();
for (final String loadBalancerName : loadBalancerNames) {
final Matcher matcher = ApplicationLoadBalancer.ALB_NAME_PATTERN.matcher(loadBalancerName);
if (matcher.find()) {
numbersTaken.add(Integer.parseInt(matcher.group(1)));
}
}
return ApplicationLoadBalancer.DNS_MAPPED_ALB_NAME_PREFIX
+ IntStream.range(0, ApplicationLoadBalancer.MAX_ALBS_PER_REGION).filter(i -> !numbersTaken.contains(i))
.min().getAsInt();
}
/**
* Path conditions are constructed by pre-pending a "*" to the sharding key.
*/
public static <ShardingKey> String getPathConditionForShardingKey(ShardingKey shardingKey, String pathPrefixForShardingKey) {
return pathPrefixForShardingKey+shardingKey.toString();
}
public static <ShardingKey> ShardingKey getShardingKeyFromPathCondition(String path, String pathPrefixForShardingKey) {
if (!path.startsWith(pathPrefixForShardingKey)) {
throw new IllegalStateException("path condition \""+path+"\" does not start with \""+pathPrefixForShardingKey+"\" which is unexpected");
}
@SuppressWarnings("unchecked") // this silently assumes that a String casts into a ShardingKey without problems
final ShardingKey result = (ShardingKey) path.substring(pathPrefixForShardingKey.length());
return result;
}
/**
* Path conditions are constructed by pre-pending a "*" to the sharding key.
*/
protected String getPathConditionForShardingKey(ShardingKey shardingKey) {
return getPathConditionForShardingKey(shardingKey, pathPrefixForShardingKey);
}
protected ShardingKey getShardingKeyFromPathCondition(String path) {
return getShardingKeyFromPathCondition(path, pathPrefixForShardingKey);
}
}
@@ -34,6 +34,9 @@ import com.sap.sse.util.HttpUrlConnectionHelper;
public interface ApplicationProcess<ShardingKey, MetricsT extends ApplicationProcessMetrics,
ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>>
extends Process<RotatingFileBasedLog, MetricsT> {
String HEALTH_CHECK_PATH = "/gwt/status";
static Logger logger = Logger.getLogger(ApplicationProcess.class.getName());
static String REPLICATION_STATUS_POST_URL_PATH_AND_QUERY = ReplicationServletActions.REPLICATION_SERVLET_BASE_PATH+"?"+ReplicationServletActions.ACTION_PARAMETER_NAME+"="+
ReplicationServletActions.Action.STATUS.name();
@@ -1,9 +1,7 @@
package com.sap.sse.landscape.application;
import java.io.IOException;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import com.sap.sse.common.Duration;
@@ -69,7 +67,7 @@ ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>> extends Na
/**
* Moves a {@link Scope} with all its content from {@code source} into this replica set. The process may fail with
* an exception, e.g., for connectivity or permission reasons, or---if the {@code failUponDiff} parameter is set to
* {@code true}--- for differences found when comparing the result in this replica set with the original content at
* {@code true}---for differences found when comparing the result in this replica set with the original content at
* {@code source}. The {@code removeFromSourceUponSuccess} and {@code setRemoveReferenceInSourceUponSuccess} parameters
* control how to proceed after successful import.
*
@@ -80,17 +78,6 @@ ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>> extends Na
void removeScope(Scope<ShardingKey> scope);
/**
* Creates a "remote server reference" on this application replica set pointing to the {@code to} replica set. If a
* non-{@code null} sequence of {@link Scope}s is provided then the {@code includeOrExcludeScopes} flag decides
* whether the reference shall only <em>include</em> those scopes ({@code true}) or it should list all scopes
* <em>except those listed in {@code scopes}</em> ({@code false}) instead.
*/
void setRemoteReference(String name, ApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> to,
Iterable<Scope<ShardingKey>> scopes, boolean includeOrExcludeScopes);
void removeRemoteReference(String name);
/**
* Tells this replica set whether read requests may also be addressed at the master node in case there are one or
* more {@link #getReplicas() replicas} configured. If setting this to {@code true}, the {@link #getMaster() master
@@ -113,26 +100,6 @@ ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>> extends Na
*/
boolean isReadFromMaster();
Map<ShardingKey, Set<ApplicationProcess<ShardingKey, MetricsT, ProcessT>>> getShardingInfo();
/**
* Activates sharding for the {@code shard} by configuring this replica set such that requests for the {@code shard}
* are usually submitted to any instance from the {@code processesToPrimarilyHandleShard} set. Only if no process
* within that set is available, the replica set will allow requests for {@code shard} to be handled by any other
* process in this replica set as a default.
*
* @param processesToPrimarilyHandleShard must not be {@code null} but can be empty
*
* @see #removeSharding
*/
void setSharding(Shard<ShardingKey> shard, Set<ApplicationProcess<ShardingKey, MetricsT, ProcessT>> processesToPrimarilyHandleShard);
/**
* Re-configures this replica set such that requests for {@code shard} will be spread across all processes
* of this replica set.
*/
void removeSharding(Shard<ShardingKey> shard);
/**
* The fully-qualified host name by which this application replica set is publicly reachable. When resolving this
* hostname through DNS, the result is expected to identify a load balancer which contains the ingress rules for
@@ -1,17 +1,28 @@
package com.sap.sse.landscape.application;
import com.sap.sse.common.Named;
/**
* Part of a {@link Scope}. A {@link Shard} cannot be moved in isolation and can hence not move across {@link Scope}s.
* Sharding can be used to optionally split {@link ApplicationProcess}es in groups, each of which being responsible
* primarily only for a subset of the {@link Shard}s available in the {@link ApplicationReplicaSet}. While we assume
* that all {@link ApplicationProcess}es in an {@link ApplicationReplicaSet} <em>can</em> handle requests for all
* {@link Shard}s managed by the replica set, it may be beneficial performance-wise to have individual processes
* focus only on a subset of {@link Shard}s. This may, e.g., result in better cache utilization and hence less CPU
* consumption on the hosts running those processes.
* {@link Shard}s managed by the replica set, it may be beneficial performance-wise to have individual processes focus
* only on a subset of {@link Shard}s. This may, e.g., result in better cache utilization and hence less CPU consumption
* on the hosts running those processes.
* <p>
*
* There may be restrictions for the {@link Named#getName() name} that a shard can have, for example it could be
* possible that shard names with quotes or brackets in them are not permitted; the name is expected to be human-readable
* and meaningful.
*
* @author Axel Uhl (D043530)
*
*/
public interface Shard<ShardingKey> {
ShardingKey getKey();
public interface Shard<ShardingKey> extends Named {
/**
* @return the keys handled by this shard
*
*/
Iterable<ShardingKey> getKeys();
}
@@ -2,7 +2,6 @@ package com.sap.sse.landscape.application.impl;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
@@ -15,7 +14,6 @@ import com.sap.sse.landscape.application.ApplicationProcess;
import com.sap.sse.landscape.application.ApplicationProcessMetrics;
import com.sap.sse.landscape.application.ApplicationReplicaSet;
import com.sap.sse.landscape.application.Scope;
import com.sap.sse.landscape.application.Shard;
public class ApplicationReplicaSetImpl<ShardingKey, MetricsT extends ApplicationProcessMetrics,
ProcessT extends ApplicationProcess<ShardingKey, MetricsT, ProcessT>>
@@ -97,19 +95,6 @@ implements ApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> {
}
@Override
public void setRemoteReference(String name, ApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> to,
Iterable<Scope<ShardingKey>> scopes, boolean includeOrExcludeScopes) {
// TODO Implement ApplicationReplicaSet<ShardingKey,MetricsT,ProcessT>.setRemoteReference(...)
// use /v1/remoteserverreference (RemoteServerReferenceResource) for this
}
@Override
public void removeRemoteReference(String name) {
// TODO Implement ApplicationReplicaSet<ShardingKey,MetricsT,ProcessT>.removeRemoteReference(...)
// use /v1/remoteserverreference (RemoteServerReferenceResource) for this
}
@Override
public void setReadFromMaster(boolean readFromMaster) throws IllegalStateException {
// TODO Implement ApplicationReplicaSet<ShardingKey,MetricsT,ProcessT>.setReadFromMaster(...)
@@ -122,23 +107,4 @@ implements ApplicationReplicaSet<ShardingKey, MetricsT, ProcessT> {
// for this it would be helpful to understand the ALB / TargetGroup assignments
return false;
}
@Override
public Map<ShardingKey, Set<ApplicationProcess<ShardingKey, MetricsT, ProcessT>>> getShardingInfo() {
// TODO Implement ApplicationReplicaSet<ShardingKey,MetricsT,ProcessT>.getShardingInfo(...)
return null;
}
@Override
public void setSharding(Shard<ShardingKey> shard,
Set<ApplicationProcess<ShardingKey, MetricsT, ProcessT>> processesToPrimarilyHandleShard) {
// TODO Implement ApplicationReplicaSet<ShardingKey,MetricsT,ProcessT>.setSharding(...)
}
@Override
public void removeSharding(Shard<ShardingKey> shard) {
// TODO Implement ApplicationReplicaSet<ShardingKey,MetricsT,ProcessT>.removeSharding(...)
}
}
@@ -20,13 +20,20 @@ import com.sap.sse.common.Named;
*
*/
public class SelectedElementsCountingButton<T extends Named> extends Button {
private final boolean enableWhenSelectionEmpty;
/**
* Constructs the button without a confirmation callback installed. The {@code clickHandler}
* will be invoked immediately with the selection when the button is clicked.
*/
public SelectedElementsCountingButton(final String html, final SetSelectionModel<T> selectionModel,
final ClickHandler clickHandler) {
this(html, selectionModel, /* no confirmation dialog */ (Supplier<Boolean>) null, clickHandler);
this(html, selectionModel, clickHandler, /* enableWhenSelectionEmpty */ false);
}
public SelectedElementsCountingButton(String html,
SetSelectionModel<T> selectionModel, ClickHandler clickHandler, boolean enableWhenSelectionEmpty) {
this(html, selectionModel, /* no confirmation dialog */ (Supplier<Boolean>) null, clickHandler, enableWhenSelectionEmpty);
}
/**
@@ -60,8 +67,24 @@ public class SelectedElementsCountingButton<T extends Named> extends Button {
*/
public SelectedElementsCountingButton(final String html, final SetSelectionModel<T> selectionModel,
final Supplier<Boolean> asker, final ClickHandler clickHandler) {
this(html, selectionModel, asker, clickHandler, /* enableWhenSelectionEmpty */ false);
}
/**
* Allows callers to implement specific behavior for the confirmation step if a non-{@code null} {@code asker} is
* passed. A non-{@code null} {@code asker} will be invoked when the button is clicked. The {@code clickHandler}
* will be invoked after the button has been clicked and either {@code asker} was {@code null} or
* {@link Supplier#get() invoking} the {@code asker} has returned a {@code true} result.
*
* @param enableWhenSelectionEmpty
* when {@code true}, the button will also be enabled when the selection is empty; otherwise, the button
* will be disabled for an empty selection.
*/
public SelectedElementsCountingButton(final String html, final SetSelectionModel<T> selectionModel,
final Supplier<Boolean> asker, final ClickHandler clickHandler, boolean enableWhenSelectionEmpty) {
super(html);
setEnabled(!selectionModel.getSelectedSet().isEmpty());
this.enableWhenSelectionEmpty = enableWhenSelectionEmpty;
setEnabled(enableWhenSelectionEmpty || !selectionModel.getSelectedSet().isEmpty());
addSelectionEventHandler(html, selectionModel);
addClickHandler(asker, selectionModel, clickHandler);
}
@@ -70,7 +93,7 @@ public class SelectedElementsCountingButton<T extends Named> extends Button {
selectionModel.addSelectionChangeHandler(event -> {
Set<T> selectedSet = selectionModel.getSelectedSet();
setText(selectedSet.isEmpty() ? html : html + " (" + selectedSet.size() + ")");
setEnabled(!selectedSet.isEmpty());
setEnabled(enableWhenSelectionEmpty || !selectedSet.isEmpty());
});
}
@@ -12,6 +12,7 @@ import java.util.UUID;
import java.util.logging.Logger;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status.Family;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
@@ -75,7 +76,8 @@ public class SecuredServerImpl implements SecuredServer {
final HttpResponse response = client.execute(request);
final int statusCode = response.getStatusLine().getStatusCode();
Object jsonParseResult;
if (statusCode == Response.Status.NO_CONTENT.getStatusCode()) {
if (statusCode == Response.Status.NO_CONTENT.getStatusCode()
|| Response.Status.fromStatusCode(statusCode).getFamily() != Family.SUCCESSFUL) {
jsonParseResult = null;
} else {
response.getEntity().writeTo(bos);
@@ -0,0 +1,31 @@
package com.sap.sse.test;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import org.junit.Test;
import com.sap.sse.util.IPAddressUtil;
public class TestIPAddressUtil {
@Test
public void testIPv4Matching() {
assertTrue(IPAddressUtil.isIPv4AddressLiteral("1.2.3.4"));
assertTrue(IPAddressUtil.isIPv4AddressLiteral("123.234.33.0"));
assertFalse(IPAddressUtil.isIPv4AddressLiteral("0.2.3.4"));
assertTrue(IPAddressUtil.isIPv4AddressLiteral("255.255.255.255"));
assertFalse(IPAddressUtil.isIPv4AddressLiteral("java.sun.com"));
}
@Test
public void testIPv6Matching() {
assertTrue(IPAddressUtil.isIPv6AddressLiteral("2001:db8:3333:4444:5555:6666:7777:8888"));
assertTrue(IPAddressUtil.isIPv6AddressLiteral("2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF"));
assertTrue(IPAddressUtil.isIPv6AddressLiteral("::"));
assertTrue(IPAddressUtil.isIPv6AddressLiteral("2001:db8::"));
assertTrue(IPAddressUtil.isIPv6AddressLiteral("::1234:5678"));
assertTrue(IPAddressUtil.isIPv6AddressLiteral("2001:db8::1234:5678"));
assertFalse(IPAddressUtil.isIPv6AddressLiteral("java.sun.com"));
assertFalse(IPAddressUtil.isIPv6AddressLiteral(":"));
}
}
@@ -0,0 +1,34 @@
package com.sap.sse.util;
import java.util.regex.Pattern;
/**
* Helps identifying IPv4 and IPv6 address literals, using regular expressions.
* <p>
*
* This can be helpful when trying to tell whether a host address is provided as
* a symbolic hostname that requires DNS resolution, or as an address literal that
* can directly be used.
*
* @author Axel Uhl (d043530)
*
*/
public class IPAddressUtil {
private static final Pattern ipv4Pattern = Pattern.compile("[1-9][0-9]{0,2}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}");
private static final Pattern ipv6Pattern = Pattern.compile("(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))");
public static boolean isIPv4AddressLiteral(String address) {
return ipv4Pattern.matcher(address).matches();
}
public static boolean isIPv6AddressLiteral(String address) {
return ipv6Pattern.matcher(address).matches();
}
/**
* Tells if {@code address} is an IPv4 or IPv6 address literal.
*/
public static boolean isIPAddressLiteral(String address) {
return isIPv4AddressLiteral(address) || isIPv6AddressLiteral(address);
}
}
+1
View File
@@ -93,6 +93,7 @@
<module>com.sap.sse.landscape</module>
<module>com.sap.sse.landscape.aws</module>
<module>com.sap.sse.landscape.aws.common</module>
<module>com.sap.sse.landscape.aws.common.test</module>
<module>com.sap.sse.landscape.aws.test</module>
<module>com.sap.sse.landscape.aws.persistence</module>
<module>com.sap.sailing.landscape</module>
-99
View File
@@ -1,99 +0,0 @@
**This page has the purpose to prepare and track the progress of the bug4232 review**
(See [bugzilla](https://bugzilla.sapsailing.com/bugzilla/show_bug.cgi?id=4232))
# Test Scenarios
## Prerequisites
* two devices (Smartphone and/or Tablet)
* device configuration: bug4232 (so all settings are correct - or manual)
* server: https://dev.sapsailing.com
* refresh time: 20 sec (because of faster local refreshes)
* event: Extreme Sailing Series 2016 Act 8 Sydney
The edit dialog (open with the pencil) should always be closed if a warning (yellow or red) will be displayed. In case of a red warning, the publish button is always disabled.
One device will be called **A** and the other **B**, so they have unique names while testing.
## While starting
### Test 1
* devices open the penalty fragment
* device A give Omar Air OCS, device B set DNS for Omar Air
* device A closes the penalty view (without publishing)
* device B should see a red warning sign after the automatic refresh
* publish should be disabled and can be activated by changing every red item in the list
### Test 2
* same as [test 1](#test-scenarios_while-starting_test-1), but this time the result should be published
* same result as [test 1](#test-scenarios_while-starting_test-1).
* publish should be disabled and can be activated by opening every red item in the list
### Test 3
* same as [test 1](#test-scenarios_while-starting_test-1), but after refresh device B should close the penalty view
* after refresh device A should also see the red warning sign, without the possibility to publish
* publish should be disabled and can be activated by opening every red item in the list
### Test 4
* both devices open the penalty fragment
* device A gives Omar Air OCS, device B sets DNS for Alinghi
* device A closes the penalty view (without publishing)
* device B should see both entries in the list after refresh
### Test 5
* same as [test 4](#test-scenarios_while-starting_test-4), but this time the result should be published
* same result as [test 4](#test-scenarios_while-starting_test-4).
### Test 6
* same as [test 4](#test-scenarios_while-starting_test-4), but after refresh device B should publish the merged result
* after the refresh device A also sees both entries
## While finishing
### Test 7
* both devices open result list
* device A adds Omar Air, Alinghi and LR BAR into the list
* device A closes the result list
* after refresh device B see the three competitors in the result list
### Test 8
* both devices open result list
* both devices add Omar Air to the result list
* device A sets the penalty to OCS and closes the result list
* device B should see after the refresh a yellow warning sign
### Test 9
* both devices open result list
* both devices add Omar Air to the result list
* both devices set the penalty to different values
* device A closes the result list
* device B should see a red warning sign and the data should show the values from device A
* publish should be disabled and can be activated by opening every red item in the list
### Test 10
* both devices open result list
* both devices add Omar Air to the result list
* device A changes to penalty view and sets the value to DNF and changes back to result list
* device B should see a yellow warning sign after refresh
### Test 11
* both devices open result list
* device A add Omar Air and Alinghi
* device B add Alinghi and Omar Air
* device A closes the result list
* device B should see red warning sign and the order Omar Air and Alinghi
* publish should be disabled and can be activated by opening every red item in the list
### Test 12
* both devices open result list
* device A add Omar Air and Alinghi
* device B add Omar Air and LR Bar
* device A closes the result list
* device B should see 1. Omar Air, 2. Alinghi and 2. LR Bar
* publish should be disabled and can be activated by opening every item in the list and clean the duplicate rankings
### Test 13
* devices open the penalty fragment
* device A give Omar Air OCS, device B set DNS for Omar Air
* device A pressed the home button
* device B should see a red warning sign after the automatic refresh
* publish should be disabled and can be activated by opening every red item in the list
+1 -1
View File
@@ -73,7 +73,7 @@ For your account that is equipped with the ``TRACKED_RACE:EXPORT`` permission yo
-m 10g --rm -d \
-e MONGODB_URI="mongodb://172.17.0.1/windestimation?retryWrites=true" \
-e BEARER_TOKEN="{your-bearer-token-here}" \
-e MEMORY=-Xmx8g \
-e MEMORY=-Xmx6g \
docker.sapsailing.com/windestimationtraining:latest
```
If successful (and you may want to remove the ``--rm`` option otherwise to allow you to inspect logs after unsuccessful execution) you will find the output under ``/tmp/windEstimationModels.dat`` which you can upload as usual, e.g., as in
+12
View File
@@ -90,3 +90,15 @@ http:
The Hudson build slave AWS image (AMI) has a set of valid credentials in the ``hudson`` user's account to
push to the registry.
## Garbage-Collecting Unused Content
To run a garbage collection in the registry, try this:
```
docker exec -it registry-registry-1 registry garbage-collect /etc/docker/registry/config.yml
```
If you want to delete an entire repository, e.g., because you pushed images under an incorrect repository tag, try this:
```
docker exec -it registry-registry-1 rm -rf /var/lib/registry/docker/registry/v2/repositories/{your-repository-name}
```