Merge branch 'bug5787'

This commit is contained in:
Axel Uhl
2023-05-05 18:19:29 +02:00
13 changed files with 587 additions and 141 deletions
@@ -2,21 +2,21 @@
<feature
id="com.amazon.aws.aws-java-api"
label="AWS API"
version="2.18.8"
version="2.20.59"
provider-name="Amazon">
<plugin
id="com.amazon.aws.aws-java-api"
download-size="0"
install-size="0"
version="2.18.8"
version="2.20.59"
unpack="false"/>
<plugin
id="com.amazon.aws.aws-java-api.source"
download-size="0"
install-size="0"
version="2.18.8"
version="2.20.59"
unpack="false"/>
</feature>
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<feature url="features/aws-sdk/com.amazon.aws.aws-java-api_2.18.8.jar" id="com.amazon.aws.aws-java-api" version="2.18.8">
<feature url="features/aws-sdk/com.amazon.aws.aws-java-api_2.20.59.jar" id="com.amazon.aws.aws-java-api" version="2.20.59">
<category name="aws-java-api"/>
</feature>
<category-def name="com.amazon.aws.aws-java-api" label="aws-java-api"/>
@@ -5,7 +5,7 @@ repositories {
}
dependencies {
implementation platform('software.amazon.awssdk:bom:2.18.8')
implementation platform('software.amazon.awssdk:bom:2.20.59')
implementation 'software.amazon.awssdk:s3'
implementation 'software.amazon.awssdk:lambda'
implementation 'software.amazon.awssdk:ec2'
+1 -1
View File
@@ -16,7 +16,7 @@
</properties>
<groupId>com.amazon.aws</groupId>
<artifactId>com.amazon.aws.aws-java-api</artifactId>
<version>2.18.8</version>
<version>2.20.59</version>
<packaging>eclipse-plugin</packaging>
<build>
<plugins>
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?pde version="3.8"?>
<target name="Race Analysis Target" sequenceNumber="218">
<target name="Race Analysis Target" sequenceNumber="219">
<locations>
<location includeAllPlatforms="false" includeConfigurePhase="false" includeMode="slicer" includeSource="true" type="InstallableUnit">
<unit id="javax.ws.rs" version="1.1.1.v20130318-1750"/>
@@ -147,7 +147,7 @@
<repository location="https://download.eclipse.org/tools/orbit/downloads/drops/R20210223232630/repository"/>
</location>
<location includeAllPlatforms="false" includeConfigurePhase="false" includeMode="slicer" includeSource="true" type="InstallableUnit">
<unit id="com.amazon.aws.aws-java-api.feature.group" version="2.18.8"/>
<unit id="com.amazon.aws.aws-java-api.feature.group" version="2.20.59"/>
<repository location="https://p2.sapsailing.com/p2/aws-sdk/"/>
</location>
<location includeAllPlatforms="false" includeConfigurePhase="false" includeMode="slicer" includeSource="true" type="InstallableUnit">
+2 -2
View File
@@ -742,14 +742,14 @@
id="com.amazon.aws.aws-java-api"
download-size="0"
install-size="0"
version="2.18.8"
version="2.20.59"
unpack="false"/>
<plugin
id="com.amazon.aws.aws-java-api.source"
download-size="0"
install-size="0"
version="2.18.8"
version="2.20.59"
unpack="false"/>
<plugin
@@ -7,6 +7,8 @@ Bundle-Vendor: SAP
Fragment-Host: com.sap.sse.landscape.aws.common
Import-Package: org.hamcrest;version="2.2.0",
org.junit;version="4.13.2",
org.junit.function;version="4.13.2"
org.junit.function;version="4.13.2",
software.amazon.awssdk.services.elasticloadbalancingv2.model
Automatic-Module-Name: com.sap.sse.landscape.aws.common.test
Bundle-RequiredExecutionEnvironment: JavaSE-1.8
Require-Bundle: com.sap.sse.landscape.aws
@@ -0,0 +1,232 @@
package com.sap.sse.landscape.aws.common.shared;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.stream.IntStream;
import org.junit.Before;
import org.junit.Test;
import com.sap.sse.common.Util;
import com.sap.sse.common.Util.Pair;
import com.sap.sse.landscape.aws.ApplicationLoadBalancer;
import com.sap.sse.landscape.aws.impl.LoadBalancerRuleInserter;
import com.sap.sse.landscape.aws.impl.LoadBalancerRuleInserter.LoadBalancerAdapter;
import com.sap.sse.landscape.aws.impl.LoadBalancerRuleInserter.RuleAdapter;
public class LoadBalancerRuleInserterTest {
private static final String RULE_NAME_PREFIX = "Rule ";
private static final String DEFAULT_RULE_NAME = "default";
private LoadBalancerAdapter<TestRuleAdapter> loadBalancerAdapter;
private LoadBalancerRuleInserter<String, TestRuleAdapter> ruleInserter;
private static class TestRuleAdapter implements RuleAdapter<TestRuleAdapter> {
private final boolean isDefault;
private final int priority;
private final String ruleArn;
public TestRuleAdapter(boolean isDefault, int priority, String ruleArn) {
super();
this.isDefault = isDefault;
this.priority = priority;
this.ruleArn = ruleArn;
}
@Override
public boolean isDefault() {
return isDefault;
}
@Override
public String priority() {
return "" + priority;
}
@Override
public String ruleArn() {
return ruleArn;
}
@Override
public TestRuleAdapter copyWithNewPriority(int priorityToUseForRuleCopy) {
return new TestRuleAdapter(isDefault, priorityToUseForRuleCopy, ruleArn);
}
}
private static class TestLoadBalancerAdapter implements LoadBalancerAdapter<TestRuleAdapter> {
private static final long serialVersionUID = -4853040303405813921L;
private Iterable<TestRuleAdapter> rules;
@Override
public String getName() {
return "Test Load Balancer Adapter";
}
@Override
public Iterable<TestRuleAdapter> getRules() {
return rules;
}
@Override
public void updateLoadBalancerListenerRulePriorities(
List<Pair<Integer, TestRuleAdapter>> newPrioritiesForExistingRules) {
rules = Util.map(newPrioritiesForExistingRules, p -> p.getB().copyWithNewPriority(p.getA()));
}
@Override
public void addRules(List<TestRuleAdapter> rulesToAdd) {
rules = Util.concat(Arrays.asList(rules, rulesToAdd));
}
}
@Before
public void setUp() {
loadBalancerAdapter = new TestLoadBalancerAdapter();
loadBalancerAdapter.addRules(Arrays.asList(new TestRuleAdapter(/* isDefault */ true, 0, DEFAULT_RULE_NAME)));
ruleInserter = new LoadBalancerRuleInserter<>(loadBalancerAdapter, ApplicationLoadBalancer.MAX_PRIORITY,
ApplicationLoadBalancer.MAX_RULES_PER_LOADBALANCER);
assertUniqueAscendingPriorities();
}
@Test
public void simpleRuleInsertionTest() {
assertEquals(1, Util.size(getRulesSortedByPriority()));
addRules(1, true, 1);
assertEquals(2, Util.size(getRulesSortedByPriority()));
assertUniqueAscendingPriorities();
}
@Test
public void additionalRuleInsertionTest() {
assertEquals(1, Util.size(getRulesSortedByPriority()));
addRules(1, true, 1);
assertEquals(2, Util.size(getRulesSortedByPriority()));
addRules(1, true, 2);
assertUniqueAscendingPriorities();
}
@Test
public void testMassInsert() {
addRules(1, true, 1, 3, 2);
assertEquals(4, Util.size(getRulesSortedByPriority()));
assertUniqueAscendingPriorities();
assertRuleOrder(1, 3, 2);
}
@Test
public void testInsertBefore() {
addRules(1, true, 1, 2);
addRulesBefore(1, true, /* insert before: */ 2, /* insert rule numbers: */ 3, 4);
assertRuleOrder(1, 3, 4, 2);
}
@Test
public void testContiguousInsertBeforeWithEnoughSpace() {
addRules(1, true, 1, 2);
loadBalancerAdapter.addRules(Arrays.asList(new TestRuleAdapter(/* isDefault */ false, 5, RULE_NAME_PREFIX+5)));
addRulesBefore(1, true, /* insert before: */ 5, /* insert rule numbers: */ 3, 4);
assertRuleOrder(1, 2, 3, 4, 5);
}
@Test
public void testNonContiguousInsertBeforeWithoutEnoughSpace() {
addRules(1, true, 1);
loadBalancerAdapter.addRules(Arrays.asList(new TestRuleAdapter(/* isDefault */ false, 3, RULE_NAME_PREFIX+3)));
loadBalancerAdapter.addRules(Arrays.asList(new TestRuleAdapter(/* isDefault */ false, 5, RULE_NAME_PREFIX+5)));
addRulesBefore(1, false, /* insert before: */ 5, /* insert rule numbers: */ 2, 4);
assertRuleOrder(1, 2, 3, 4, 5);
}
@Test
public void testInsertInHole() {
addRules(1, true, 1);
loadBalancerAdapter.addRules(Arrays.asList(new TestRuleAdapter(/* isDefault */ false, 4, RULE_NAME_PREFIX+4)));
addRules(2, true, 2, 3);
assertRuleOrder(1, 2, 3, 4);
}
@Test
public void testInsertAfterTooSmallAHole() {
addRules(1, true, 1);
loadBalancerAdapter.addRules(Arrays.asList(new TestRuleAdapter(/* isDefault */ false, 4, RULE_NAME_PREFIX+4)));
addRules(2, true, 2, 3, 5);
assertRuleOrder(1, 4, 2, 3, 5);
}
@Test
public void testNonContiguousInsertWithTooSmallAHole() {
addRules(1, true, 1);
loadBalancerAdapter.addRules(Arrays.asList(new TestRuleAdapter(/* isDefault */ false, 4, RULE_NAME_PREFIX+4)));
addRules(2, false, 2, 3, 5);
assertRuleOrder(1, 2, 3, 4, 5);
}
@Test
public void testExceptionForTooManyRules() {
try {
addRules(1, true, Util.toArray(IntStream.range(0, 101).boxed()::iterator, new Integer[0]));
fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
// expected
}
}
@Test
public void testExceptionForTooManyRulesWhenAdding() {
addRules(1, true, Util.toArray(IntStream.range(0, 50).boxed()::iterator, new Integer[0]));
try {
addRules(1, true, Util.toArray(IntStream.range(50, 101).boxed()::iterator, new Integer[0]));
fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
// expected
}
}
@Test
public void testExceptionForRuleToInsertBeforeNotFound() {
addRules(1, true, 1);
try {
addRulesBefore(1, true, Optional.of(new TestRuleAdapter(/* isDefault */ false, 2, RULE_NAME_PREFIX+2)), 2);
fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
// expected
}
}
private void addRules(int priority, boolean forceContiguous, Integer... ruleNumbers) {
addRulesBefore(priority, forceContiguous, /* insertBefore */ Optional.empty(), ruleNumbers);
}
private void addRulesBefore(int priority, boolean forceContiguous, int insertBeforeRuleNumber, Integer... ruleNumbers) {
addRulesBefore(priority, forceContiguous, Util.stream(getRulesSortedByPriority())
.filter(r -> r.ruleArn().equals(RULE_NAME_PREFIX + insertBeforeRuleNumber)).findFirst(), ruleNumbers);
}
private void addRulesBefore(int priority, boolean forceContiguous, Optional<TestRuleAdapter> insertBefore, Integer... ruleNumbers) {
ruleInserter.addRulesAssigningUnusedPriorities(forceContiguous, insertBefore,
Util.map(Arrays.asList(ruleNumbers), ruleNumber->new TestRuleAdapter(/* isDefault */ false, priority, RULE_NAME_PREFIX + ruleNumber)));
}
private void assertRuleOrder(Integer... ruleNumbers) {
assertEquals(Util.asList(Util.map(Arrays.asList(ruleNumbers), ruleNumber->RULE_NAME_PREFIX+ruleNumber)),
// ignore default rule at index 0 in comparison; look at numbered rules only
Util.asList(Util.map(getRulesSortedByPriority(), TestRuleAdapter::ruleArn)).subList(1, Util.size(getRulesSortedByPriority())));
}
private Iterable<TestRuleAdapter> getRulesSortedByPriority() {
return ruleInserter.getRulesSortedByPriority();
}
private void assertUniqueAscendingPriorities() {
int lastPriority = -1;
for (final TestRuleAdapter rule : getRulesSortedByPriority()) {
assertTrue(Integer.valueOf(rule.priority()) > lastPriority);
lastPriority = Integer.valueOf(rule.priority());
}
}
}
@@ -53,6 +53,8 @@ public interface ApplicationLoadBalancer<ShardingKey> extends Named {
*/
public static final int MAX_PRIORITY = 50000;
AwsLandscape<ShardingKey> getLandscape();
/**
* The DNS name of this load balancer; can be used, e.g., to set a CNAME DNS record pointing
* to this load balancer.
@@ -106,21 +108,17 @@ public interface ApplicationLoadBalancer<ShardingKey> extends Named {
* original order to the resulting sequence. Otherwise, the existing rules are "compressed" by re-numbering their
* priorities to make space for the new rules at the end of the list.
*
* @param insertBefore
* if empty, the new {@code rules} can be inserted anywhere in the list of rules; if a {@link Rule} is
* specified and that rule is part of this ALB's {@link #getRules() rules},
* {@link #shiftRulesToMakeSpaceAt(int) make space} at that point and insert the new {@code rules} there.
* If the rule specified cannot be found in this ALB's {@link #getRules()}, an {@link IllegalArgumentException}
* is thrown. The rules are compared by their {@link Rule#ruleArn() ARN}, not by their Java object identity.
*
* @return copies of the original rules, with unused {@link Rule#priority() priorities} assigned, as passed already
* to {@link #addRules(Rule...)}.
*/
Iterable<Rule> addRulesAssigningUnusedPriorities(boolean forceContiguous, Rule... rules);
/**
* For inserting e.g. Shard at a specific priority, we must ensure that the priority is unique in the ruleset.
* This function shifts every priority starting at the highest priority one higher for making the priority at {@code index}
* free.
* @param index
* index supposed to be free
* @throws IllegalStateException
* gets thrown if shifting exceeds the limit of priorities
*/
void shiftRulesToMakeSpaceAt(int index) throws IllegalStateException;
Iterable<Rule> addRulesAssigningUnusedPriorities(boolean forceContiguous, Optional<Rule> insertBefore, Rule... rules);
/**
* Returns the priority which should be used as the next sharding priority.
@@ -1,18 +1,17 @@
package com.sap.sse.landscape.aws.impl;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map.Entry;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import com.sap.sse.common.Duration;
import com.sap.sse.common.Util;
@@ -21,6 +20,7 @@ import com.sap.sse.landscape.aws.ApplicationLoadBalancer;
import com.sap.sse.landscape.aws.AwsLandscape;
import com.sap.sse.landscape.aws.TargetGroup;
import com.sap.sse.landscape.aws.common.shared.PlainRedirectDTO;
import com.sap.sse.landscape.aws.impl.LoadBalancerRuleInserter.ALBRuleAdapter;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Action;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.ActionTypeEnum;
@@ -32,7 +32,6 @@ import software.amazon.awssdk.services.elasticloadbalancingv2.model.RedirectActi
import software.amazon.awssdk.services.elasticloadbalancingv2.model.RedirectActionStatusCodeEnum;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.RuleCondition;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.RulePriorityPair;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.TargetGroupTuple;
public class ApplicationLoadBalancerImpl<ShardingKey>
@@ -52,6 +51,11 @@ implements ApplicationLoadBalancer<ShardingKey> {
this.loadBalancer = loadBalancer;
this.landscape = landscape;
}
@Override
public AwsLandscape<ShardingKey> getLandscape() {
return landscape;
}
@Override
public String getName() {
@@ -104,51 +108,6 @@ implements ApplicationLoadBalancer<ShardingKey> {
return landscape.createLoadBalancerListenerRules(region, getListener(ProtocolEnum.HTTPS), rulesToAdd);
}
/**
* Returns the priority if the priority is not higher than {@code MAX_PRIORITY}
* @param priority
* requested priority
* @return
* Priority if it's valid
* @throws IllegalStateException
* If the requested priority is higher than {@code MAX_PRIORITY}
*/
private int checkNewPriority(int priority) throws IllegalStateException {
if (priority < MAX_PRIORITY) {
return priority;
} else {
throw new IllegalStateException("Priority was greater than " + MAX_PRIORITY +"!");
}
}
@Override
public void shiftRulesToMakeSpaceAt(int targetPrio) throws IllegalStateException {
final Iterable<Rule> rules = getRules();
final TreeMap<Integer, Rule> rulesSorted = getRulesSorted(rules);
int lastPrio = targetPrio;
boolean skipNext = false;
final Collection<RulePriorityPair> result = new ArrayList<>();
if (rulesSorted.get(targetPrio) != null) {// if there is a rule on prio
for (Entry<Integer, Rule> entry : rulesSorted.entrySet()) {
final Integer priority = entry.getKey();
if (priority >= targetPrio && !skipNext) {
// if prio is higher than target prio and is not supposed to be skipped
if (priority - lastPrio > 0) {
// if there is a gap between current prio and the last one. -> so this one is not supposed to be
// skipped and every rule after this
skipNext = true;
break;
} else {
lastPrio = priority + 1;
result.add(RulePriorityPair.builder().ruleArn(entry.getValue().ruleArn())
.priority(checkNewPriority(lastPrio)).build());
}
}
}
landscape.updateLoadBalancerListenerRulePriorities(getRegion(), result);
}
}
@Override
public int getFirstShardingPriority(String hostname) throws IllegalStateException {
final Iterable<Rule> rules = getRules();
@@ -202,65 +161,11 @@ implements ApplicationLoadBalancer<ShardingKey> {
}
@Override
public Iterable<Rule> addRulesAssigningUnusedPriorities(boolean forceContiguous, Rule... rules) {
final Iterable<Rule> existingRules = getRules();
if (Util.size(existingRules)-1 + rules.length > MAX_PRIORITY) { // -1 due to the default rule being part of existingRules
throw new IllegalArgumentException("The "+rules.length+" new rules won't find enough unused priority numbers because there are already "+
(Util.size(existingRules)-1)+" of them and together they would exceed the maximum of "+MAX_PRIORITY+" by "+
(Util.size(existingRules)-1 + rules.length - MAX_PRIORITY));
}
final List<Rule> result = new ArrayList<>(rules.length);
final List<Rule> sortedExistingNonDefaultRules = new ArrayList<>(Util.size(existingRules)-1);
Util.addAll(Util.filter(existingRules, r->!r.isDefault()), sortedExistingNonDefaultRules);
Collections.sort(sortedExistingNonDefaultRules, (r1, r2)->Integer.valueOf(r1.priority()).compareTo(Integer.valueOf(r2.priority())));
final int stepwidth;
if (forceContiguous) {
stepwidth = rules.length;
} else {
stepwidth = 1;
}
int rulesIndex = 0;
int previousPriority = 0;
final Iterator<Rule> existingRulesIter = sortedExistingNonDefaultRules.iterator();
while (rulesIndex < rules.length) {
// find next available slot
int nextPriority = MAX_PRIORITY+1; // if no further rule exists, the usable gap ends after MAX_PRIORITY
while (existingRulesIter.hasNext() && (nextPriority=Integer.valueOf(existingRulesIter.next().priority())) <= previousPriority+stepwidth) {
// not enough space for stepwidth many rules; keep on searching
previousPriority = nextPriority;
if (!existingRulesIter.hasNext()) {
nextPriority = MAX_PRIORITY+1;
}
}
if (previousPriority+stepwidth > MAX_PRIORITY) {
if (forceContiguous) {
previousPriority = squeezeExistingRulesAndReturnLastUsedPriority(sortedExistingNonDefaultRules);
nextPriority = MAX_PRIORITY+1;
// we previously checked already that there is enough room for the new set of rules
assert previousPriority + rules.length <= MAX_PRIORITY;
} else {
throw new IllegalStateException(
"The " + rules.length + " new rules don't fit into the existing rule set of load balancer "
+ getName() + " without exceeding the maximum priority of " + MAX_PRIORITY);
}
}
while (rulesIndex < rules.length && ++previousPriority < nextPriority) {
final int priorityToUseForNextRule = previousPriority;
result.add(rules[rulesIndex++].copy(b->b.priority(""+priorityToUseForNextRule)));
}
}
addRules(result.toArray(new Rule[0]));
return result;
}
private int squeezeExistingRulesAndReturnLastUsedPriority(final List<Rule> sortedExistingNonDefaultRules) {
final List<RulePriorityPair> newPrioritiesForExistingRules = new LinkedList<>();
int priority = 0;
for (final Rule existingRule : sortedExistingNonDefaultRules) {
newPrioritiesForExistingRules.add(RulePriorityPair.builder().ruleArn(existingRule.ruleArn()).priority(++priority).build());
}
landscape.updateLoadBalancerListenerRulePriorities(getRegion(), newPrioritiesForExistingRules);
return priority;
public Iterable<Rule> addRulesAssigningUnusedPriorities(boolean forceContiguous, Optional<Rule> insertBefore, Rule... rules) {
final List<Rule> rulesAsList = Arrays.asList(rules);
return Util.map(LoadBalancerRuleInserter.create(this, MAX_PRIORITY, MAX_RULES_PER_LOADBALANCER).addRulesAssigningUnusedPriorities(
forceContiguous, insertBefore.map(LoadBalancerRuleInserter::createRuleAdapter),
Util.map(rulesAsList, LoadBalancerRuleInserter::createRuleAdapter)), ALBRuleAdapter::getRule);
}
@Override
@@ -314,11 +219,26 @@ implements ApplicationLoadBalancer<ShardingKey> {
return Util.stream(getRules()).filter(r->isDefaultRedirectRule(r, hostname)).findAny()
.map(defaultRedirectRule->updateDefaultRedirectRule(defaultRedirectRule.ruleArn(), hostname, pathWithLeadingSlash, query))
.orElseGet(()->{
final Rule defaultRedirectRule = createDefaultRedirectRule(hostname, pathWithLeadingSlash, query);
addRulesAssigningUnusedPriorities(/* forceContiguous */ false, defaultRedirectRule);
return defaultRedirectRule;
return insertAndReturnDefaultRedirectRule(hostname, pathWithLeadingSlash, query);
});
}
/**
* Checks the load balancer rule set size. If already at its maximum, an {@link IllegalStateException} is thrown because
* adding a default rule is not possible, and this method does not handle moving an appplication replica set to a different
* load balancer. If there is space for at least one more rule, searches this load balancer's rule set for the first rule
* for hostname header {@code hostname}. It then uses {@link #shiftRulesToMakeSpaceAt(int)} for that position to make
* space for the new default redirect rule.
*
* @return the new default redirect rule that was inserted into this load balancer's HTTPS listener's rule set
*/
private Rule insertAndReturnDefaultRedirectRule(String hostname, String pathWithLeadingSlash, Optional<String> query) {
final Rule defaultRedirectRule = createDefaultRedirectRule(hostname, pathWithLeadingSlash, query);
addRulesAssigningUnusedPriorities(/* forceContiguous */ false,
Util.stream(getRules()).filter(rule->rule.conditions().stream().anyMatch(condition->condition.field().equals("host-header") && condition.hostHeaderConfig().values().contains(hostname))).findFirst(),
defaultRedirectRule);
return defaultRedirectRule;
}
@Override
public Rule getDefaultRedirectRule(String hostName, PlainRedirectDTO redirect) {
@@ -0,0 +1,291 @@
package com.sap.sse.landscape.aws.impl;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Optional;
import com.sap.sse.common.Named;
import com.sap.sse.common.Util;
import com.sap.sse.common.Util.Pair;
import com.sap.sse.landscape.aws.ApplicationLoadBalancer;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Rule;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.RulePriorityPair;
/**
* Manages a set of rules from an Application Load Balancer. This includes adding a set of rules either contiguously or
* optionally non-contiguously to an existing set of rules that are ordered by their priorities. There are limits on the
* maximum number of rules and on the maximum priority that can be used. Trying to add too many rules so that one of
* these limits would have to be violated to accommodate will throw an exception.
* <p>
*
* For purposes of testability, the class is designed such that the actual {@link Rule} and {@link RulePriorityPair}
* classes are abstracted by an interface. Default adapters are available for the actual ALB-related classes, but tests
* may provide their own mocks instead so that it is not required to test this class against a real AWS environment.
*
* @author Axel Uhl (d043530)
*
*/
public class LoadBalancerRuleInserter<ShardingKey, RA extends com.sap.sse.landscape.aws.impl.LoadBalancerRuleInserter.RuleAdapter<RA>> {
private final LoadBalancerAdapter<RA> loadBalancerAdapter;
private final int maxPriority;
private final int maxRulesPerLoadBalancer;
public static interface RuleAdapter<RA extends RuleAdapter<RA>> {
boolean isDefault();
String priority();
String ruleArn();
RA copyWithNewPriority(int priorityToUseForRuleCopy);
}
public static interface LoadBalancerAdapter<RA extends RuleAdapter<RA>> extends Named {
Iterable<RA> getRules();
void updateLoadBalancerListenerRulePriorities(List<Pair<Integer, RA>> newPrioritiesForExistingRules);
void addRules(List<RA> result);
}
public static class ALBRuleAdapter implements RuleAdapter<ALBRuleAdapter> {
private final Rule rule;
private ALBRuleAdapter(Rule rule) {
this.rule = rule;
}
@Override
public boolean isDefault() {
return rule.isDefault();
}
@Override
public String priority() {
return rule.priority();
}
@Override
public String ruleArn() {
return rule.ruleArn();
}
@Override
public ALBRuleAdapter copyWithNewPriority(int priorityToUseForRuleCopy) {
return new ALBRuleAdapter(rule.copy(b->b.priority(""+priorityToUseForRuleCopy).build()));
}
Rule getRule() {
return rule;
}
}
private static class ALBAdapter<ShardingKey> implements LoadBalancerAdapter<ALBRuleAdapter> {
private static final long serialVersionUID = -4337640786328697695L;
private final ApplicationLoadBalancer<ShardingKey> alb;
private ALBAdapter(ApplicationLoadBalancer<ShardingKey> alb) {
super();
this.alb = alb;
}
@Override
public String getName() {
return alb.getName();
}
@Override
public Iterable<ALBRuleAdapter> getRules() {
return Util.map(alb.getRules(), r->createRuleAdapter(r));
}
@Override
public void updateLoadBalancerListenerRulePriorities(
List<Pair<Integer, ALBRuleAdapter>> newPrioritiesForExistingRules) {
alb.getLandscape().updateLoadBalancerListenerRulePriorities(alb.getRegion(),
Util.map(newPrioritiesForExistingRules, p->RulePriorityPair.builder().priority(p.getA()).ruleArn(p.getB().ruleArn()).build()));
}
@Override
public void addRules(List<ALBRuleAdapter> rules) {
alb.addRules(Util.toArray(Util.map(rules, ra->ra.getRule()), new Rule[0]));
}
}
public LoadBalancerRuleInserter(LoadBalancerAdapter<RA> loadBalancerAdapter, int maxPriority, int maxRulesPerLoadBalancer) {
super();
this.loadBalancerAdapter = loadBalancerAdapter;
this.maxPriority = maxPriority;
this.maxRulesPerLoadBalancer = maxRulesPerLoadBalancer;
}
public static <ShardingKey> LoadBalancerRuleInserter<ShardingKey, ALBRuleAdapter> create(
ApplicationLoadBalancer<ShardingKey> alb, int maxPriority, int maxRulesPerLoadBalancer) {
return new LoadBalancerRuleInserter<>(createLoadBalancerAdapter(alb), maxPriority, maxRulesPerLoadBalancer);
}
public static ALBRuleAdapter createRuleAdapter(Rule rule) {
return new ALBRuleAdapter(rule);
}
static <ShardingKey> ALBAdapter<ShardingKey> createLoadBalancerAdapter(ApplicationLoadBalancer<ShardingKey> alb) {
return new ALBAdapter<ShardingKey>(alb);
}
/**
* As the rule {@link Rule#priority() priorities} within a load balancer's listener have to be unique, this method
* supports adding rules by assigning yet unused priorities to them. It keeps the order in which the {@code rules}
* are passed. If {@code forceContiguous} is {@code false}, for each rule the next available priority is chosen and
* assigned by creating a copy of the {@link Rule} object and adding it to the resulting sequence. This can lead to
* existing rules interleaving with the rules to add while ensuring that the {@code rules} have priorities in
* numerically ascending order, consistent with the order in which they were passed to this method.
* <p>
*
* If {@code forceContiguous} is {@code true}, the rules that result will have contiguously increasing priority
* values, hence not interleaving with other existing rules. If there are enough contiguous unused priorities
* available, they are selected and assigned by creating copies of the {@link Rule} objects and adding them in their
* original order to the resulting sequence. Otherwise, the existing rules are "compressed" by re-numbering their
* priorities to make space for the new rules at the end of the list.
*
* @param forceContiguous
* if {@code true}, the {@code rules} must be inserted as a contiguous sequence in the overall sequence
* of rules ordered by priority; otherwise it is permissible to look for unused priorities anywhere along
* the sequence and to use existing gaps in the priority sequence, possibly separating the {@code rules}
* into one or more sub-sequences scattered throughout the resulting rules sequence
* @param insertBefore
* if empty, the new {@code rules} can be inserted anywhere in the list of rules, of course adhering to
* the {@code forceContiguous} rules explained above; if a {@link RuleAdapter} is specified and that rule
* is part of this ALB's {@link LoadBalancerAdapter#getRules() rules}, find unused priorities before that
* rule according to the {@code forceContiguous} rules, and if necessary
* {@link #shiftRulesToMakeSpaceAt(int) make space} at that point and insert the {@code rules} rules not
* yet inserted prior to that point there. If the rule specified cannot be found in this ALB's
* {@link #getRules()}, an {@link IllegalArgumentException} is thrown. The rules are compared by their
* {@link Rule#ruleArn() ARN}, not by their Java object identity.
*
* @return {@link RuleAdapter#copyWithNewPriority(int) copies} of the original rules, with unused
* {@link RuleAdapter#priority() priorities} assigned, as passed already to
* {@link LoadBalancerAdapter#addRules(List)}.
*/
public List<RA> addRulesAssigningUnusedPriorities(boolean forceContiguous, Optional<RA> insertBefore,
Iterable<RA> rules) {
final Iterable<RA> existingRules = loadBalancerAdapter.getRules();
if (insertBefore.isPresent() && !Util.contains(Util.map(existingRules, RuleAdapter::ruleArn), insertBefore.get().ruleArn())) {
throw new IllegalArgumentException("Didn't find rule to insert before: "+insertBefore.get());
}
final int rulesSize = Util.size(rules);
int remainingNumberOfRulesToAdd = rulesSize;
if (Util.size(existingRules)-1 + rulesSize > maxPriority) { // -1 due to the default rule being part of existingRules
throw new IllegalArgumentException("The "+rulesSize+" new rules won't find enough unused priority numbers because there are already "+
(Util.size(existingRules)-1)+" of them and together they would exceed the maximum of "+maxPriority+" by "+
(Util.size(existingRules)-1 + rulesSize - maxPriority));
}
if (Util.size(existingRules) + rulesSize > maxRulesPerLoadBalancer) {
throw new IllegalArgumentException("The " + rulesSize + " new rules would make the ALB " + loadBalancerAdapter.getName()
+ " exceed its maximum number of rules (" + maxPriority + ") by "
+ (Util.size(existingRules) + rulesSize - maxRulesPerLoadBalancer));
}
final List<RA> result = new ArrayList<>(rulesSize);
final List<RA> sortedExistingNonDefaultRules = new ArrayList<>(Util.size(existingRules)-1);
Util.addAll(Util.filter(existingRules, r->!r.isDefault()), sortedExistingNonDefaultRules);
Collections.sort(sortedExistingNonDefaultRules, (r1, r2)->Integer.valueOf(r1.priority()).compareTo(Integer.valueOf(r2.priority())));
final int stepwidth;
if (forceContiguous) {
stepwidth = rulesSize;
} else {
stepwidth = 1;
}
final Iterator<RA> rulesIterator = rules.iterator();
int previousPriority = 0;
final Iterator<RA> existingRulesIter = sortedExistingNonDefaultRules.iterator();
boolean foundRuleToInsertBefore = false;
while (rulesIterator.hasNext()) {
// find next available slot
int nextPriority = maxPriority+1; // if no further rule exists, the usable gap ends after MAX_PRIORITY
final RuleAdapter<?>[] nextRule = new RuleAdapter<?>[1]; // using an array to make it final so we can access it inside the mapping function below
// search for space until
// - we reach the end of the existing rules list, or
while (existingRulesIter.hasNext() &&
// - we find enough space, or
((nextPriority=Integer.valueOf((nextRule[0]=existingRulesIter.next()).priority())) <= previousPriority+stepwidth) &&
// - the rule before which all new rules must be inserted is found
(!insertBefore.isPresent() || !(foundRuleToInsertBefore=insertBefore.map(rule->rule.ruleArn().equals(nextRule[0].ruleArn())).orElse(false)))) {
// not enough space for stepwidth many rules; keep on searching
previousPriority = nextPriority;
if (!existingRulesIter.hasNext()) {
nextPriority = maxPriority+1;
}
}
if (foundRuleToInsertBefore) {
// We must insert all remaining rules from rulesIterator here, making space if necessary
shiftRulesToMakeSpaceAt(nextPriority, remainingNumberOfRulesToAdd);
nextPriority += remainingNumberOfRulesToAdd;
}
if (previousPriority+stepwidth > maxPriority) {
if (forceContiguous) {
previousPriority = squeezeExistingRulesAndReturnLastUsedPriority(sortedExistingNonDefaultRules);
nextPriority = maxPriority+1;
// we previously checked already that there is enough room for the new set of rules
assert previousPriority + rulesSize <= maxPriority;
} else {
throw new IllegalStateException(
"The " + rulesSize + " new rules don't fit into the existing rule set of load balancer "
+ loadBalancerAdapter.getName() + " without exceeding the maximum priority of " + maxPriority);
}
}
while (rulesIterator.hasNext() && ++previousPriority < nextPriority) {
final int priorityToUseForNextRule = previousPriority;
result.add(rulesIterator.next().copyWithNewPriority(priorityToUseForNextRule));
remainingNumberOfRulesToAdd--;
}
}
loadBalancerAdapter.addRules(result);
return result;
}
private int squeezeExistingRulesAndReturnLastUsedPriority(final List<RA> sortedExistingNonDefaultRules) {
final List<Pair<Integer, RA>> newPrioritiesForExistingRules = new LinkedList<>();
int priority = 0;
for (final RA existingRule : sortedExistingNonDefaultRules) {
newPrioritiesForExistingRules.add(new Pair<>(++priority, existingRule));
}
loadBalancerAdapter.updateLoadBalancerListenerRulePriorities(newPrioritiesForExistingRules);
return priority;
}
public Iterable<RA> getRulesSortedByPriority() {
return Util.stream(loadBalancerAdapter.getRules()).sorted((r1, r2)->Integer.compare(Integer.valueOf(r1.priority()), Integer.valueOf(r2.priority())))::iterator;
}
/**
* For inserting rules before some other rule we may need to shift other rules' priorities to make enough space in
* the sequence of ascending unique priorities. This function shifts every priority starting at the highest priority
* one higher for making the priority at {@code targetPrio} free.
*
* @param targetPriority
* index supposed to be free
* @param howManySlots
* tells how many free slots are needed; must be greater than or equal to 1
* @throws IllegalStateException
* gets thrown if shifting exceeds the limit of priorities
*/
public void shiftRulesToMakeSpaceAt(int targetPriority, int howManySlots) throws IllegalStateException {
final Iterable<RA> rulesSorted = getRulesSortedByPriority();
int minimumPriorityForNextRule = targetPriority + howManySlots;
final List<Pair<Integer, RA>> result = new ArrayList<>();
for (final RA rule : rulesSorted) {
final int priority = Integer.valueOf(rule.priority());
final int newPriorityForRule;
// if targetPriority not reached or rule priority is already greater than required then the rule can be copied unchanged:
if (priority < targetPriority || priority >= minimumPriorityForNextRule) {
newPriorityForRule = priority;
} else {
newPriorityForRule = minimumPriorityForNextRule;
minimumPriorityForNextRule = newPriorityForRule + 1;
}
result.add(new Pair<>(newPriorityForRule, rule));
}
loadBalancerAdapter.updateLoadBalancerListenerRulePriorities(result);
}
}
@@ -225,7 +225,7 @@ implements ProcedureCreatingLoadBalancerMapping<ShardingKey> {
getLandscape().addTargetsToTargetGroup(masterTargetGroupCreated, Collections.singleton(getHost()));
getLandscape().addTargetsToTargetGroup(publicTargetGroupCreated, Collections.singleton(getHost()));
getLoadBalancerUsed().addRulesAssigningUnusedPriorities(/* forceContiguous */ true,
createRules(getLoadBalancerUsed(), getHostName(), masterTargetGroupCreated, publicTargetGroupCreated));
Optional.empty(), createRules(getLoadBalancerUsed(), getHostName(), masterTargetGroupCreated, publicTargetGroupCreated));
} catch (Exception e) {
throw new RuntimeException(e);
}
@@ -7,6 +7,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.logging.Level;
@@ -26,6 +27,7 @@ import com.sap.sse.landscape.aws.AwsApplicationReplicaSet;
import com.sap.sse.landscape.aws.AwsLandscape;
import com.sap.sse.landscape.aws.AwsShard;
import com.sap.sse.landscape.aws.TargetGroup;
import com.sap.sse.landscape.aws.impl.LoadBalancerRuleInserter;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.Action;
import software.amazon.awssdk.services.elasticloadbalancingv2.model.ActionTypeEnum;
@@ -216,7 +218,7 @@ implements ProcedureCreatingLoadBalancerMapping<ShardingKey> {
Util.addAll(shardingKeys, shardingKeyForConsumption);
final int ruleIdx = alb.getFirstShardingPriority(replicaSet.getHostname());
while (!shardingKeyForConsumption.isEmpty()) {
alb.shiftRulesToMakeSpaceAt(ruleIdx);
LoadBalancerRuleInserter.create(alb, ApplicationLoadBalancer.MAX_PRIORITY, ApplicationLoadBalancer.MAX_RULES_PER_LOADBALANCER).shiftRulesToMakeSpaceAt(ruleIdx, 1);
final Set<ShardingKey> shardingKeysForNextRule = new HashSet<>();
for (final Iterator<ShardingKey> i=shardingKeyForConsumption.iterator();
shardingKeysForNextRule.size() < ApplicationLoadBalancer.MAX_CONDITIONS_PER_RULE-NUMBER_OF_STANDARD_CONDITIONS_FOR_SHARDING_RULE && i.hasNext(); ) {
@@ -251,7 +253,7 @@ implements ProcedureCreatingLoadBalancerMapping<ShardingKey> {
final int requiredRules = numberOfRequiredRules(Util.size(shardingKeys))
+ (existingShardingRules + /* 5 std rules per replica set */ NUMBER_OF_RULES_PER_REPLICA_SET);
final ApplicationLoadBalancer<ShardingKey> res;
if (Util.size(replicaSet.getLoadBalancerRules())
if (Util.size(replicaSet.getLoadBalancer().getRules())
+ numberOfRequiredRules(Util.size(shardingKeys)) < ApplicationLoadBalancer.MAX_RULES_PER_LOADBALANCER) {
res = replicaSet.getLoadBalancer();
} else {
@@ -261,7 +263,8 @@ implements ProcedureCreatingLoadBalancerMapping<ShardingKey> {
final Iterable<ApplicationLoadBalancer<ShardingKey>> loadBalancersFiltered = Util.filter(loadBalancers,
t -> {
try {
return !t.getArn().equals(replicaSet.getLoadBalancer().getArn());
return t.getVpcId().equals(replicaSet.getLoadBalancer().getVpcId())
&& !t.getArn().equals(replicaSet.getLoadBalancer().getArn());
} catch (InterruptedException | ExecutionException e) {
logger.log(Level.WARNING, "Exception while trying to obtain a load balancer's ARN", e);
throw new RuntimeException(e);
@@ -374,8 +377,8 @@ implements ProcedureCreatingLoadBalancerMapping<ShardingKey> {
Collection<TargetGroup<ShardingKey>> originalTargetGroups,
Map<TargetGroup<ShardingKey>, Iterable<ShardingKey>> shardingKeysPerTargetGroup) throws Exception {
targetAlb
.addRulesAssigningUnusedPriorities(true,
createRules(targetAlb, replicaSet.getHostname(),
.addRulesAssigningUnusedPriorities(/* forceContiguous */ true,
Optional.empty(), createRules(targetAlb, replicaSet.getHostname(),
targetGroupsToTempTargetgroups.get(replicaSetToMove.getMasterTargetGroup()),
targetGroupsToTempTargetgroups.get(replicaSetToMove.getPublicTargetGroup())))
.forEach(t -> tempRules.add(t));