first shot at waiting for new auto-replicas and terminating upgrade replicas

configuration/on-site-scripts/upgrade-landscape.sh

@@ -156,5 +156,7 @@ for REGION in $( cat `dirname $0`/regions.txt ); do
       exit ${EXIT_CODE}
     fi
   done
+  echo " * Waiting for new auto-replicas to become available in region ${REGION}, then terminating upgrade replicas..."
+  `dirname $0`/wait-for-new-auto-replicas-and-terminate-upgrade-replicas.sh -g ${REGION} &
 done
 echo " * DONE"

configuration/on-site-scripts/wait-for-new-auto-replicas-and-terminate-upgrade-replicas.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+VPC="Tokyo2020"
+TARGET_GROUP_NAME="S-ded-tokyo2020"
+UPGRADE_REPLICA_NAME="SL Tokyo2020 (Upgrade Replica)"
+AUTO_REPLICA_NAME="SL Tokyo2020 (auto-replica)"
+
+if [ $# -eq 0 ]; then
+    echo "$0 [ -g <AWS-region> ]"
+    echo ""
+    echo "-g AWS Region, e.g., eu-west-1; if not provided, your default AWS region will be used"
+    echo
+    echo "Example: $0 -g ap-southeast-2"
+    echo
+    echo "Counts the upgrade replicas currently healthy in the ${TARGET_GROUP_NAME} target group"
+    echo "and waits until as many auto-replicas are healthy in the same target group."
+    echo "Then, all upgrade replicas are removed from the target group and then terminated."
+    exit 2
+fi
+
+options='g:'
+while getopts $options option
+do
+    case $option in
+	g) REGION=$OPTARG;;
+        \?) echo "Invalid option"
+            exit 4;;
+    esac
+done
+if [ -n "${REGION}" ]; then
+  export AWS_DEFAULT_REGION=${REGION}
+fi
+TARGET_GROUP_ARN=$( aws elbv2 describe-target-groups --names ${TARGET_GROUP_NAME} | jq -r '.TargetGroups[].TargetGroupArn' )
+echo "Found target group with name ${TARGET_GROUP_NAME} and ARN ${TARGET_GROUP_ARN}"
+
+UPGRADE_REPLICA_IDS=$( aws ec2 describe-instances --filters Name=tag:Name,Values="${UPGRADE_REPLICA_NAME}" | jq -r '.Reservations[].Instances[].InstanceId' )
+NUMBER_OF_UPGRADE_REPLICAS=$( echo ${UPGRADE_REPLICA_IDS} | wc )
+echo "Found ${NUMBER_OF_UPGRADE_REPLICAS} upgrade replicas. Waiting until we see this many auto-replicas named ${AUTO_REPLICA_NAME}..."
+TOTAL_NUMBER_OF_TOTAL_REPLICAS_TO_WAIT_FOR=$(( 2 * ${NUMBER_OF_UPGRADE_REPLICAS} ))
+NUMBER_OF_TOTAL_HEALTHY_REPLICAS=$( aws --region eu-west-2 elbv2 describe-target-health --target-group-arn ${TARGET_GROUP_ARN} | jq '.TargetHealthDescriptions | map(select(.TargetHealth.State=="healthy")) | length' )
+while [ ${NUMBER_OF_TOTAL_HEALTHY_REPLICAS} -lt ${TOTAL_NUMBER_OF_TOTAL_REPLICAS_TO_WAIT_FOR} ]; do
+  echo "Found ${NUMBER_OF_TOTAL_HEALTHY_REPLICAS} healthy replicas so far; waiting until we see ${TOTAL_NUMBER_OF_TOTAL_REPLICAS_TO_WAIT_FOR}..."
+  sleep 10
+done
+echo "Found a total of ${TOTAL_NUMBER_OF_TOTAL_REPLICAS_TO_WAIT_FOR} healthy replicas in target group ${TARGET_GROUP_NAME}. Terminating ${UPGRADE_REPLICA_NAME} replicas..."
+for UPGRADE_REPLICA_ID in ${UPGRADE_REPLICA_IDS}; do
+  echo "Terminating instance with ID ${UPGRADE_REPLICA_ID}..."
+  aws ec2 terminate-instance --instance-ids ${UPGRADE_REPLICA_ID}
+done