diff --git a/java/com.amazon.aws.aws-java-api.updatesite/features/aws-sdk/feature.xml b/java/com.amazon.aws.aws-java-api.updatesite/features/aws-sdk/feature.xml index 9f7ae179fa2..1a7f58d5dde 100755 --- a/java/com.amazon.aws.aws-java-api.updatesite/features/aws-sdk/feature.xml +++ b/java/com.amazon.aws.aws-java-api.updatesite/features/aws-sdk/feature.xml @@ -2,21 +2,21 @@ diff --git a/java/com.amazon.aws.aws-java-api.updatesite/site.xml b/java/com.amazon.aws.aws-java-api.updatesite/site.xml index 7fd6f30502b..6ffe012824e 100755 --- a/java/com.amazon.aws.aws-java-api.updatesite/site.xml +++ b/java/com.amazon.aws.aws-java-api.updatesite/site.xml @@ -1,6 +1,6 @@ - + diff --git a/java/com.amazon.aws.aws-java-api/build.gradle b/java/com.amazon.aws.aws-java-api/build.gradle index 862c0802743..dd140de2709 100644 --- a/java/com.amazon.aws.aws-java-api/build.gradle +++ b/java/com.amazon.aws.aws-java-api/build.gradle @@ -5,13 +5,13 @@ repositories { } dependencies { - implementation platform('software.amazon.awssdk:bom:2.14.24') + implementation platform('software.amazon.awssdk:bom:2.15.16') implementation 'software.amazon.awssdk:s3' implementation 'software.amazon.awssdk:ec2' implementation 'software.amazon.awssdk:route53' implementation 'software.amazon.awssdk:cloudwatch' implementation 'software.amazon.awssdk:elasticloadbalancingv2' - implementation 'software.amazon.awssdk:firehose' + implementation 'software.amazon.awssdk:autoscaling' implementation group: 'software.amazon.awssdk', name: 'sdk-core', classifier: 'sources' implementation group: 'software.amazon.awssdk', name: 'arns', classifier: 'sources' implementation group: 'software.amazon.awssdk', name: 'aws-core', classifier: 'sources' @@ -24,7 +24,7 @@ dependencies { implementation group: 'software.amazon.awssdk', name: 'route53', classifier: 'sources' implementation group: 'software.amazon.awssdk', name: 'cloudwatch', classifier: 'sources' implementation group: 'software.amazon.awssdk', name: 'elasticloadbalancingv2', classifier: 'sources' - implementation group: 'software.amazon.awssdk', name: 'firehose', classifier: 'sources' + implementation group: 'software.amazon.awssdk', name: 'autoscaling', classifier: 'sources' } task downloadLibs(type: Copy) { into file('lib') diff --git a/java/com.amazon.aws.aws-java-api/pom.xml b/java/com.amazon.aws.aws-java-api/pom.xml index 3dded51a9ed..83c4b8e5f66 100644 --- a/java/com.amazon.aws.aws-java-api/pom.xml +++ b/java/com.amazon.aws.aws-java-api/pom.xml @@ -8,6 +8,6 @@ 1.0.0-SNAPSHOT com.amazon.aws.aws-java-api - 2.14.24 + 2.15.16 eclipse-plugin diff --git a/java/com.sap.sailing.landscape.test/src/com/sap/sailing/landscape/test/TestProcedures.java b/java/com.sap.sailing.landscape.test/src/com/sap/sailing/landscape/test/TestProcedures.java new file mode 100755 index 00000000000..08c69109944 --- /dev/null +++ b/java/com.sap.sailing.landscape.test/src/com/sap/sailing/landscape/test/TestProcedures.java @@ -0,0 +1,120 @@ +package com.sap.sailing.landscape.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.Optional; +import java.util.Random; +import java.util.UUID; + +import org.junit.Before; +import org.junit.Test; + +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.SftpException; +import com.sap.sailing.landscape.SailingAnalyticsHost; +import com.sap.sailing.landscape.SailingAnalyticsMaster; +import com.sap.sailing.landscape.SailingAnalyticsMetrics; +import com.sap.sailing.landscape.SailingAnalyticsReplica; +import com.sap.sailing.landscape.SailingReleaseRepository; +import com.sap.sailing.landscape.procedures.StartSailingAnalyticsMaster; +import com.sap.sse.common.Duration; +import com.sap.sse.landscape.Release; +import com.sap.sse.landscape.application.ApplicationProcess; +import com.sap.sse.landscape.application.impl.ApplicationProcessImpl; +import com.sap.sse.landscape.aws.AwsInstance; +import com.sap.sse.landscape.aws.AwsLandscape; +import com.sap.sse.landscape.aws.Tags; +import com.sap.sse.landscape.aws.impl.AwsRegion; +import com.sap.sse.landscape.aws.orchestration.CreateDNSBasedLoadBalancerMapping; +import com.sap.sse.landscape.aws.orchestration.StartAwsHost; + +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.ec2.model.Instance; +import software.amazon.awssdk.services.ec2.model.InstanceType; +import software.amazon.awssdk.services.ec2.model.Tag; +import software.amazon.awssdk.services.route53.model.RRType; + +/** + * Tests for the AWS SDK landscape wrapper in bundle {@code com.sap.sse.landscape.aws}. To run these tests + * successfully it is necessary to have valid AWS credentials for region {@code EU_WEST_2} that allow the + * AWS user account to create keys and launch instances, etc. These are to be provided as explained + * in the documentation of {@link AwsLandscape#obtain()}. + * + * @author Axel Uhl (D043530) + * + */ +public class TestProcedures { + private static final Optional optionalTimeout = Optional.of(Duration.ONE_MINUTE.times(5)); + private AwsLandscape, SailingAnalyticsReplica> landscape; + private AwsRegion region; + + @Before + public void setUp() { + landscape = AwsLandscape.obtain(); + region = new AwsRegion(Region.EU_WEST_2); + } + + @Test + public void testConnectivity() throws JSchException, IOException, SftpException, NumberFormatException, InterruptedException, URISyntaxException { + final String TARGET_GROUP_NAME_PREFIX = "S-test-"; + final String hostedZoneName = "wiesen-weg.de"; + final String hostname = "S-test-"+new Random().nextInt()+"."+hostedZoneName; + final String keyName = "MyKey-"+UUID.randomUUID(); + final Release release = SailingReleaseRepository.INSTANCE.getLatestMasterRelease(); + final StartAwsHost, SailingAnalyticsReplica, SailingAnalyticsHost> startSailingAnalyticsMaster = new StartSailingAnalyticsMaster( + hostname, region, landscape, InstanceType.T2_SMALL, keyName, Optional.of(release), /* databaseConfiguration */ null, + /* rabbitConfiguration */ null, /* replicationConfiguration */ null, "axel.uhl@sap.com", Optional.empty()); + startSailingAnalyticsMaster.run(); + final AwsInstance host = landscape.launchHost(landscape + .getImage(region, "ami-01b4b27a5699e33e6"), + InstanceType.T3_SMALL, landscape.getAvailabilityZoneByName(region, "eu-west-2b"), keyName, Collections.singleton(()->"sg-0b2afd48960251280"), + Optional.of(Tags.with("Name", "MyHost").and("Hello", "World"))); + try { + assertNotNull(host); + final Instance instance = landscape.getInstance(host.getInstanceId(), region); + boolean foundName = false; + boolean foundHello = false; + for (final Tag tag : instance.tags()) { + if (tag.key().equals("Name") && tag.value().equals("MyHost")) { + foundName = true; + } + if (tag.key().equals("Hello") && tag.value().equals("World")) { + foundHello = true; + } + } + assertTrue(foundName); + assertTrue(foundHello); + // check env.sh access + final ApplicationProcess, SailingAnalyticsReplica> process = new ApplicationProcessImpl<>(8888, host, "/home/sailing/servers/server"); + final String envSh = process.getEnvSh(optionalTimeout); + assertFalse(envSh.isEmpty()); + assertTrue(envSh.contains("SERVER_NAME=")); + assertEquals(14888, process.getTelnetPortToOSGiConsole(optionalTimeout)); + final AwsLandscape, SailingAnalyticsReplica> castLandscape = (AwsLandscape, SailingAnalyticsReplica>) landscape; + final CreateDNSBasedLoadBalancerMapping, SailingAnalyticsReplica, AwsInstance> createDNSBasedLoadBalancerMappingProcedure = + new CreateDNSBasedLoadBalancerMapping, SailingAnalyticsReplica, AwsInstance>( + process, hostname, TARGET_GROUP_NAME_PREFIX, castLandscape, optionalTimeout); + final String wiesenWegId = landscape.getDNSHostedZoneId(hostedZoneName); + try { + createDNSBasedLoadBalancerMappingProcedure.run(); + assertNotNull(createDNSBasedLoadBalancerMappingProcedure.getLoadBalancerUsed()); + assertNotNull(createDNSBasedLoadBalancerMappingProcedure.getMasterTargetGroupCreated()); + assertEquals(TARGET_GROUP_NAME_PREFIX+process.getServerName(optionalTimeout), createDNSBasedLoadBalancerMappingProcedure.getPublicTargetGroupCreated().getName()); + } finally { + if (createDNSBasedLoadBalancerMappingProcedure.getLoadBalancerUsed() != null) { + createDNSBasedLoadBalancerMappingProcedure.getLoadBalancerUsed().delete(); + landscape.removeDNSRecord(wiesenWegId, hostname, RRType.CNAME, createDNSBasedLoadBalancerMappingProcedure.getLoadBalancerUsed().getDNSName()); + } + } + } finally { + landscape.terminate(host); + landscape.deleteKeyPair(region, keyName); + } + } +} diff --git a/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/StartMultiServer.java b/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/StartMultiServer.java index 765fbf14529..2a41bce0263 100755 --- a/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/StartMultiServer.java +++ b/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/StartMultiServer.java @@ -8,12 +8,12 @@ import com.sap.sailing.landscape.SailingAnalyticsHost; import com.sap.sailing.landscape.SailingAnalyticsMaster; import com.sap.sailing.landscape.SailingAnalyticsMetrics; import com.sap.sailing.landscape.SailingAnalyticsReplica; -import com.sap.sse.landscape.Landscape; -import com.sap.sse.landscape.MachineImage; import com.sap.sse.landscape.Release; import com.sap.sse.landscape.SecurityGroup; import com.sap.sse.landscape.aws.AwsAvailabilityZone; +import com.sap.sse.landscape.aws.AwsLandscape; import com.sap.sse.landscape.aws.Tags; +import com.sap.sse.landscape.aws.impl.AmazonMachineImage; import com.sap.sse.landscape.mongodb.Database; import com.sap.sse.landscape.rabbitmq.RabbitMQEndpoint; @@ -31,10 +31,10 @@ import software.amazon.awssdk.services.ec2.model.InstanceType; */ public class StartMultiServer> extends StartSailingAnalyticsHost> { - public StartMultiServer(String name, MachineImage machineImage, - Landscape, SailingAnalyticsReplica> landscape, + public StartMultiServer(String name, AmazonMachineImage machineImage, + AwsLandscape, SailingAnalyticsReplica> landscape, InstanceType instanceType, AwsAvailabilityZone availabilityZone, String keyName, - Iterable securityGroups, Release release, Database databaseConfiguration, + Iterable securityGroups, Optional release, Database databaseConfiguration, RabbitMQEndpoint rabbitConfiguration, String outputReplicationExchangeName, Optional replicationConfiguration, String commaSeparatedEmailAddressesToNotifyOfStartup, Optional tags, String[] userData) @@ -44,4 +44,12 @@ public class StartMultiServer + */ public abstract class StartSailingAnalyticsHost> extends StartAwsHost, SailingAnalyticsReplica, SailingAnalyticsHost> implements Procedure, SailingAnalyticsReplica> { + private final static String IMAGE_TYPE_TAG_VALUE_SAILING = "sailing-analytics-server"; + + /** + * Configures the start host procedure using the latest default machine image for the Sailing Analytics application, + * tagged with the value {@link #IMAGE_TYPE_TAG_VALUE_SAILING} for the image type tag key. A random availability + * zone is picked from the {@code region} specified, and the + * {@link SailingReleaseRepository#getLatestMasterRelease() latest master build release} is chosen as the release to + * deploy. + * + * @param name + * the name is used as the server name, is the basis for the server-group's name + */ + public StartSailingAnalyticsHost(String name, + AwsLandscape, SailingAnalyticsReplica> landscape, + AwsRegion region, + InstanceType instanceType, String keyName, Database databaseConfiguration, + RabbitMQEndpoint rabbitConfiguration, String outputReplicationExchangeName, + Optional replicationConfiguration, + String commaSeparatedEmailAddressesToNotifyOfStartup, Optional tags, String... additionalUserData) throws URISyntaxException { + this(name, getLatestSailingMachineImage(landscape, region), landscape, instanceType, + getRandomAvailabilityZone(region, landscape), keyName, + Optional.of(SailingReleaseRepository.INSTANCE.getLatestMasterRelease()), databaseConfiguration, rabbitConfiguration, + outputReplicationExchangeName, replicationConfiguration, commaSeparatedEmailAddressesToNotifyOfStartup, + tags, additionalUserData); + } + + protected static AmazonMachineImage getLatestSailingMachineImage( + AwsLandscape, SailingAnalyticsReplica> landscape, + AwsRegion region) { + return getLatestImageOfType(IMAGE_TYPE_TAG_VALUE_SAILING, landscape, region); + } + + public StartSailingAnalyticsHost(String name, Optional release, + AwsLandscape, SailingAnalyticsReplica> landscape, + AwsRegion region, + InstanceType instanceType, String keyName, Database databaseConfiguration, + RabbitMQEndpoint rabbitConfiguration, String outputReplicationExchangeName, + Optional replicationConfiguration, + String commaSeparatedEmailAddressesToNotifyOfStartup, Optional tags, String... additionalUserData) throws URISyntaxException { + this(name, getLatestSailingMachineImage(landscape, region), landscape, instanceType, + getRandomAvailabilityZone(region, landscape), keyName, + release, databaseConfiguration, rabbitConfiguration, + outputReplicationExchangeName, replicationConfiguration, commaSeparatedEmailAddressesToNotifyOfStartup, + tags, additionalUserData); + } + /** * @param name - * the name is used as the server name, is the basis for the server-group's name and is used as the name - * for the messaging exchange to which the new host will send out its replication operations. + * the name is used as the server name, is the basis for the server-group's name + */ + public StartSailingAnalyticsHost(String name, + AmazonMachineImage machineImage, + AwsLandscape, SailingAnalyticsReplica> landscape, + InstanceType instanceType, AwsAvailabilityZone availabilityZone, String keyName, + Optional release, Database databaseConfiguration, RabbitMQEndpoint rabbitConfiguration, + String outputReplicationExchangeName, Optional replicationConfiguration, + String commaSeparatedEmailAddressesToNotifyOfStartup, Optional tags, String... userData) { + super(machineImage, release, landscape, instanceType, availabilityZone, keyName, addNameTag(name, tags), userData); + } + + /** + * @param name + * the name is used as the server name, is the basis for the server-group's name */ public StartSailingAnalyticsHost(String name, MachineImage machineImage, - Landscape, SailingAnalyticsReplica> landscape, + AwsLandscape, SailingAnalyticsReplica> landscape, InstanceType instanceType, AwsAvailabilityZone availabilityZone, String keyName, - Iterable securityGroups, Release release, Database databaseConfiguration, + Iterable securityGroups, Optional release, Database databaseConfiguration, RabbitMQEndpoint rabbitConfiguration, String outputReplicationExchangeName, Optional replicationConfiguration, String commaSeparatedEmailAddressesToNotifyOfStartup, Optional tags, String... additionalUserData) throws URISyntaxException { - super(machineImage, landscape, instanceType, availabilityZone, keyName, securityGroups, Optional.of(tags.orElse(Tags.empty()).and("Name", name)), additionalUserData); - addUserData(release); + super(machineImage, release, landscape, instanceType, availabilityZone, keyName, securityGroups, addNameTag(name, tags), additionalUserData); addUserData(databaseConfiguration); addUserData(rabbitConfiguration); addUserData(ProcessConfigurationVariable.SERVER_NAME, name); diff --git a/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/StartSailingAnalyticsMaster.java b/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/StartSailingAnalyticsMaster.java index 86f5bd52677..6b9664c11c8 100755 --- a/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/StartSailingAnalyticsMaster.java +++ b/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/StartSailingAnalyticsMaster.java @@ -1,7 +1,6 @@ package com.sap.sailing.landscape.procedures; import java.net.URISyntaxException; -import java.util.Collections; import java.util.Optional; import com.sap.sailing.landscape.ReplicationConfiguration; @@ -9,36 +8,57 @@ import com.sap.sailing.landscape.SailingAnalyticsHost; import com.sap.sailing.landscape.SailingAnalyticsMaster; import com.sap.sailing.landscape.SailingAnalyticsMetrics; import com.sap.sailing.landscape.SailingAnalyticsReplica; -import com.sap.sse.landscape.Landscape; import com.sap.sse.landscape.MachineImage; +import com.sap.sse.landscape.ProcessConfigurationVariable; import com.sap.sse.landscape.Release; import com.sap.sse.landscape.SecurityGroup; -import com.sap.sse.landscape.ProcessConfigurationVariable; import com.sap.sse.landscape.aws.AwsAvailabilityZone; +import com.sap.sse.landscape.aws.AwsLandscape; import com.sap.sse.landscape.aws.Tags; +import com.sap.sse.landscape.aws.impl.AwsRegion; import com.sap.sse.landscape.mongodb.Database; import com.sap.sse.landscape.rabbitmq.RabbitMQReplicaSet; import software.amazon.awssdk.services.ec2.model.InstanceType; +/** + * TODO handle the naming problem: base name drives instance "Name" tag generation ("SL ... (Master)"), exchange name, + * database name and SERVER_NAME. When moving up the inheritance hierarchy, name is interpreted in some places as the instance name + * which obviously doesn't equal the "Name" tag value. So, we have to clearly distinguish these. + * + * @author Axel Uhl (D043530) + * + * @param + */ public class StartSailingAnalyticsMaster extends StartSailingAnalyticsHost> { - public StartSailingAnalyticsMaster(String name, MachineImage machineImage, - Landscape, SailingAnalyticsReplica> landscape, - InstanceType instanceType, AwsAvailabilityZone availabilityZone, String keyName, Release release, - Database databaseConfiguration, RabbitMQReplicaSet rabbitConfiguration, - Optional replicationConfiguration, + public StartSailingAnalyticsMaster(String name, AwsRegion region, + AwsLandscape, SailingAnalyticsReplica> landscape, + InstanceType instanceType, String keyName, Optional release, Database databaseConfiguration, + RabbitMQReplicaSet rabbitConfiguration, Optional replicationConfiguration, String commaSeparatedEmailAddressesToNotifyOfStartup, Optional tags) throws URISyntaxException { - this(name, machineImage, landscape, instanceType, availabilityZone, keyName, - Collections - .singleton(landscape.getDefaultSecurityGroupForApplicationHosts(availabilityZone.getRegion())), + this(name, + getLatestSailingMachineImage(landscape, region), landscape, instanceType, getRandomAvailabilityZone(region, landscape), keyName, + getDefaultSecurityGroupForApplicationHosts(landscape, region), release, databaseConfiguration, rabbitConfiguration, replicationConfiguration, commaSeparatedEmailAddressesToNotifyOfStartup, tags); } public StartSailingAnalyticsMaster(String name, MachineImage machineImage, - Landscape, SailingAnalyticsReplica> landscape, + AwsRegion region, + AwsLandscape, SailingAnalyticsReplica> landscape, + InstanceType instanceType, String keyName, Optional release, + Database databaseConfiguration, RabbitMQReplicaSet rabbitConfiguration, + Optional replicationConfiguration, + String commaSeparatedEmailAddressesToNotifyOfStartup, Optional tags) throws URISyntaxException { + this(name, machineImage, landscape, instanceType, getRandomAvailabilityZone(region, landscape), keyName, + getDefaultSecurityGroupForApplicationHosts(landscape, region), + release, databaseConfiguration, rabbitConfiguration, replicationConfiguration, commaSeparatedEmailAddressesToNotifyOfStartup, tags); + } + + public StartSailingAnalyticsMaster(String name, MachineImage machineImage, + AwsLandscape, SailingAnalyticsReplica> landscape, InstanceType instanceType, AwsAvailabilityZone availabilityZone, String keyName, - Iterable securityGroups, Release release, Database databaseConfiguration, + Iterable securityGroups, Optional release, Database databaseConfiguration, RabbitMQReplicaSet rabbitConfiguration, Optional replicationConfiguration, String commaSeparatedEmailAddressesToNotifyOfStartup, Optional tags) throws URISyntaxException { super(name, machineImage, landscape, instanceType, availabilityZone, keyName, securityGroups, release, diff --git a/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/StartSailingAnalyticsReplica.java b/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/StartSailingAnalyticsReplica.java index 8bdb70150eb..5194a7db7ef 100755 --- a/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/StartSailingAnalyticsReplica.java +++ b/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/StartSailingAnalyticsReplica.java @@ -9,23 +9,32 @@ import com.sap.sailing.landscape.SailingAnalyticsHost; import com.sap.sailing.landscape.SailingAnalyticsMaster; import com.sap.sailing.landscape.SailingAnalyticsMetrics; import com.sap.sailing.landscape.SailingAnalyticsReplica; -import com.sap.sse.landscape.Landscape; -import com.sap.sse.landscape.MachineImage; +import com.sap.sse.landscape.ProcessConfigurationVariable; import com.sap.sse.landscape.Release; import com.sap.sse.landscape.SecurityGroup; -import com.sap.sse.landscape.ProcessConfigurationVariable; import com.sap.sse.landscape.aws.AwsAvailabilityZone; +import com.sap.sse.landscape.aws.AwsLandscape; import com.sap.sse.landscape.aws.Tags; +import com.sap.sse.landscape.aws.impl.AmazonMachineImage; import com.sap.sse.landscape.mongodb.Database; import com.sap.sse.landscape.rabbitmq.RabbitMQEndpoint; import software.amazon.awssdk.services.ec2.model.InstanceType; +/** + * TODO handle the naming problem: base name drives instance "Name" tag generation ("SL ... (Master)"), exchange name, + * database name and SERVER_NAME. When moving up the inheritance hierarchy, name is interpreted in some places as the instance name + * which obviously doesn't equal the "Name" tag value. So, we have to clearly distinguish these. + * + * @author Axel Uhl (D043530) + * + * @param + */ public class StartSailingAnalyticsReplica extends StartSailingAnalyticsHost> { - public StartSailingAnalyticsReplica(String name, MachineImage machineImage, - Landscape, SailingAnalyticsReplica> landscape, - InstanceType instanceType, AwsAvailabilityZone availabilityZone, String keyName, Release release, + public StartSailingAnalyticsReplica(String name, AmazonMachineImage machineImage, + AwsLandscape, SailingAnalyticsReplica> landscape, + InstanceType instanceType, AwsAvailabilityZone availabilityZone, String keyName, Optional release, Database databaseConfiguration, RabbitMQEndpoint rabbitConfiguration, Optional replicationConfiguration, SailingAnalyticsMaster master, String commaSeparatedEmailAddressesToNotifyOfStartup, Optional tags, String[] userData) @@ -37,10 +46,10 @@ public class StartSailingAnalyticsReplica commaSeparatedEmailAddressesToNotifyOfStartup, tags, userData); } - public StartSailingAnalyticsReplica(String name, MachineImage machineImage, - Landscape, SailingAnalyticsReplica> landscape, + public StartSailingAnalyticsReplica(String name, AmazonMachineImage machineImage, + AwsLandscape, SailingAnalyticsReplica> landscape, InstanceType instanceType, AwsAvailabilityZone availabilityZone, String keyName, - Iterable securityGroups, Release release, Database databaseConfiguration, + Iterable securityGroups, Optional release, Database databaseConfiguration, RabbitMQEndpoint rabbitConfiguration, Optional replicationConfiguration, SailingAnalyticsMaster master, String commaSeparatedEmailAddressesToNotifyOfStartup, Optional tags, String... userData) throws URISyntaxException { diff --git a/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/UpgradeAmi.java b/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/UpgradeAmi.java index b36970ee9a4..ed420b01855 100755 --- a/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/UpgradeAmi.java +++ b/java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/procedures/UpgradeAmi.java @@ -2,15 +2,16 @@ package com.sap.sailing.landscape.procedures; import java.util.Optional; -import com.sap.sailing.landscape.SailingAnalyticsHost; -import com.sap.sailing.landscape.SailingAnalyticsMaster; -import com.sap.sailing.landscape.SailingAnalyticsMetrics; -import com.sap.sailing.landscape.SailingAnalyticsReplica; -import com.sap.sse.landscape.Landscape; import com.sap.sse.landscape.MachineImage; import com.sap.sse.landscape.SecurityGroup; +import com.sap.sse.landscape.application.ApplicationMasterProcess; +import com.sap.sse.landscape.application.ApplicationProcessMetrics; +import com.sap.sse.landscape.application.ApplicationReplicaProcess; import com.sap.sse.landscape.aws.AwsAvailabilityZone; +import com.sap.sse.landscape.aws.AwsInstance; +import com.sap.sse.landscape.aws.AwsLandscape; import com.sap.sse.landscape.aws.Tags; +import com.sap.sse.landscape.aws.impl.AmazonMachineImage; import com.sap.sse.landscape.aws.orchestration.StartAwsHost; import com.sap.sse.landscape.orchestration.Procedure; @@ -26,20 +27,23 @@ import software.amazon.awssdk.services.ec2.model.InstanceType; * @param * @param */ -public class UpgradeAmi> +public class UpgradeAmi, +ReplicaProcessT extends ApplicationReplicaProcess, +HostT extends AwsInstance> extends - StartAwsHost, SailingAnalyticsReplica, SailingAnalyticsHost> implements - Procedure, SailingAnalyticsReplica> { + StartAwsHost implements + Procedure { private static final String IMAGE_UPGRADE = "image-upgrade"; private MachineImage upgradedAmi; - public UpgradeAmi(MachineImage machineImage, - Landscape, SailingAnalyticsReplica> landscape, + public UpgradeAmi(AmazonMachineImage machineImage, + AwsLandscape landscape, InstanceType instanceType, AwsAvailabilityZone availabilityZone, String keyName, Iterable securityGroups) { - super(machineImage, landscape, instanceType, availabilityZone, keyName, securityGroups, - Optional.of(Tags.with("Name", IMAGE_UPGRADE+" for "+machineImage.getId())), /* user data */ IMAGE_UPGRADE); + super(machineImage, Optional.empty(), landscape, instanceType, availabilityZone, keyName, + securityGroups, Optional.of(Tags.with("Name", IMAGE_UPGRADE+" for "+machineImage.getId())), /* user data */ IMAGE_UPGRADE); } @Override @@ -47,7 +51,7 @@ public class UpgradeAmi - + diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsLandscape.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsLandscape.java index 116a37af289..55bd242dd3f 100755 --- a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsLandscape.java +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsLandscape.java @@ -102,6 +102,8 @@ extends Landscape { String... userData); AmazonMachineImage getImage(Region region, String imageId); + + AmazonMachineImage getLatestImageWithTag(Region region, String tagName, String tagValue); KeyPairInfo getKeyPairInfo(Region region, String keyName); diff --git a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsLandscapeImpl.java b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsLandscapeImpl.java index e5427248605..ec45211c20c 100755 --- a/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsLandscapeImpl.java +++ b/java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsLandscapeImpl.java @@ -4,6 +4,7 @@ import java.io.ByteArrayOutputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -59,6 +60,8 @@ import software.amazon.awssdk.services.ec2.model.DescribeImagesRequest; import software.amazon.awssdk.services.ec2.model.DescribeImagesResponse; import software.amazon.awssdk.services.ec2.model.DescribeInstancesRequest; import software.amazon.awssdk.services.ec2.model.DescribeKeyPairsRequest; +import software.amazon.awssdk.services.ec2.model.Filter; +import software.amazon.awssdk.services.ec2.model.Image; import software.amazon.awssdk.services.ec2.model.ImportKeyPairRequest; import software.amazon.awssdk.services.ec2.model.Instance; import software.amazon.awssdk.services.ec2.model.InstanceType; @@ -423,6 +426,19 @@ ReplicaProcessT extends ApplicationReplicaProcess(response.images().iterator().next(), region); } + @Override + public AmazonMachineImage getLatestImageWithTag(com.sap.sse.landscape.Region region, String tagName, String tagValue) { + final DescribeImagesResponse response = getEc2Client(getRegion(region)) + .describeImages(DescribeImagesRequest.builder().filters(Filter.builder().name("tag:"+tagName).values(tagValue).build()).build()); + return new AmazonMachineImage<>(response.images().stream().min(getMachineImageCreationDateComparator()).get(), region); + } + + private Comparator getMachineImageCreationDateComparator() { + return (ami1, ami2)->{ + return ami1.creationDate().compareTo(ami2.creationDate()); + }; + } + private AwsCredentials getCredentials() { return AwsBasicCredentials.create(accessKeyId, secretAccessKey); } @@ -650,6 +666,7 @@ ReplicaProcessT extends ApplicationReplicaProcess> extends StartHost { private static final Logger logger = Logger.getLogger(StartAwsHost.class.getName()); + + /** + * The {@link AwsLandscape#getLatestImageWithTag(com.sap.sse.landscape.Region, String, String)} method is + * used to obtain default images for specific AWS host starting procedures that subclass this class. The + * Amazon Machine Images (AMIs) for this are then expected to be tagged with a tag named as specified by this + * constant ("image-type"). The tag value then must match what the subclass wants. + * + * @see #getLatestImageOfType(String) + */ + private final static String IMAGE_TYPE_TAG_NAME = "image-type"; + private final List userData; private final InstanceType instanceType; private final AwsAvailabilityZone availabilityZone; @@ -52,10 +71,10 @@ extends StartHost private final Optional tags; private AwsInstance host; - public StartAwsHost(MachineImage machineImage, Landscape landscape, - InstanceType instanceType, - AwsAvailabilityZone availabilityZone, String keyName, Iterable securityGroups, - Optional tags, String... userData) { + public StartAwsHost(MachineImage machineImage, Optional release, + AwsLandscape landscape, + InstanceType instanceType, AwsAvailabilityZone availabilityZone, String keyName, + Iterable securityGroups, Optional tags, String... userData) { super(machineImage, landscape); this.userData = new ArrayList<>(); for (final String ud : userData) { @@ -66,8 +85,47 @@ extends StartHost this.keyName = keyName; this.tags = tags; this.securityGroups = securityGroups; + release.ifPresent(this::addUserData); } + public StartAwsHost(AmazonMachineImage machineImage, + Optional release, + AwsLandscape landscape, InstanceType instanceType, AwsAvailabilityZone availabilityZone, String keyName, + Optional tags, String[] userData) { + this(machineImage, release, landscape, instanceType, availabilityZone, + keyName, + Collections.singleton(landscape.getDefaultSecurityGroupForApplicationHosts(availabilityZone.getRegion())), tags, userData); + } + + protected static , + ReplicaProcessT extends ApplicationReplicaProcess> + AmazonMachineImage getLatestImageOfType(String imageType, AwsLandscape landscape, Region region) { + return landscape.getLatestImageWithTag(region, IMAGE_TYPE_TAG_NAME, imageType); + } + + protected static , + ReplicaProcessT extends ApplicationReplicaProcess> + AwsAvailabilityZone getRandomAvailabilityZone(AwsRegion region, AwsLandscape landscape) { + final Iterable azs = landscape.getAvailabilityZones(region); + return (AwsAvailabilityZone) Util.get(azs, new Random().nextInt(Util.size(azs))); + } + + protected static , + ReplicaProcessT extends ApplicationReplicaProcess> + Set getDefaultSecurityGroupForApplicationHosts(Landscape landscape, Region region) { + return Collections.singleton(landscape.getDefaultSecurityGroupForApplicationHosts(region)); + } + + protected static Optional addNameTag(String name, Optional tags) { + return Optional.of(tags.orElse(Tags.empty()).and("Name", name)); + } + @Override public AwsLandscape getLandscape() { return (AwsLandscape) super.getLandscape();