diff --git a/configuration/on-site-scripts/monitor-tokyo2020-reachability-from-regions b/configuration/on-site-scripts/monitor-tokyo2020-reachability-from-regions index 201a8c5220b..4d645b9d6dc 100755 --- a/configuration/on-site-scripts/monitor-tokyo2020-reachability-from-regions +++ b/configuration/on-site-scripts/monitor-tokyo2020-reachability-from-regions @@ -1,20 +1,30 @@ #!/bin/bash -# Discover replicas -IPs="" -for i in $( cat `dirname $0`/regions.txt ); do - IPs="$( aws --region $i ec2 describe-instances --filters Name=instance-state-name,Values=running Name=tag:sailing-analytics-server,Values=tokyo2020 | jq .Reservations[].Instances[].PublicIpAddress -r )" - if [ -z "${IPs}" ]; then - echo "Couldn't find a running replica in region $i" >&2 - message="Couldn't find a running replica in region $i" - echo "${message}" | notify-operators "${message}" - else - read first others <<<"${IPs}" - if ! ssh -o StrictHostKeyChecking=no sailing@${first} "curl https://tokyo2020.sapsailing.com/gwt/status 2>/dev/null >/dev/null"; then - echo "Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}" >&2 - message="Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}" +# Probe for valid AWS MFA token +EXPIRY_MARKER_FILE=/tmp/aws-mfa-token-expired +aws ec2 describe-instances >/dev/null 2>/dev/null +EXIT_CODE=$? +if [ "${EXIT_CODE}" = "254" -a ! -f "${EXPIRY_MARKER_FILE}" ]; then + message="AWS MFA token seems to have expired. aws ec2 describe-instances exited with code ${EXIT_CODE}. Not messaging you again. Try another awsmfa as root@sapsailing.com's tmux." + echo "${message}" | notify-operators "AWS MFA token expired" + touch "${EXPIRY_MARKER_FILE}" +else + # Discover replicas + IPs="" + for i in $( cat `dirname $0`/regions.txt ); do + IPs="$( aws --region $i ec2 describe-instances --filters Name=instance-state-name,Values=running Name=tag:sailing-analytics-server,Values=tokyo2020 | jq .Reservations[].Instances[].PublicIpAddress -r )" + if [ -z "${IPs}" ]; then + echo "Couldn't find a running replica in region $i" >&2 + message="Couldn't find a running replica in region $i" echo "${message}" | notify-operators "${message}" else - echo "Access from region ${i}, IP ${first} OK." >&2 + read first others <<<"${IPs}" + if ! ssh -o StrictHostKeyChecking=no sailing@${first} "curl https://tokyo2020.sapsailing.com/gwt/status 2>/dev/null >/dev/null"; then + echo "Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}" >&2 + message="Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}" + echo "${message}" | notify-operators "${message}" + else + echo "Access from region ${i}, IP ${first} OK." >&2 + fi fi - fi -done + done +fi