tokyo2020 scripts: use a marker file to warn only once of an expired AWS MFA token

configuration/on-site-scripts/monitor-tokyo2020-reachability-from-regions

@@ -1,20 +1,30 @@
 #!/bin/bash
-# Discover replicas
-IPs=""
-for i in $( cat `dirname $0`/regions.txt ); do
-  IPs="$( aws --region $i ec2 describe-instances --filters Name=instance-state-name,Values=running Name=tag:sailing-analytics-server,Values=tokyo2020 | jq .Reservations[].Instances[].PublicIpAddress -r )"
-  if [ -z "${IPs}" ]; then
-    echo "Couldn't find a running replica in region $i" >&2
-    message="Couldn't find a running replica in region $i"
-    echo "${message}" | notify-operators "${message}"
-  else
-    read first others <<<"${IPs}"
-    if ! ssh -o StrictHostKeyChecking=no sailing@${first} "curl https://tokyo2020.sapsailing.com/gwt/status 2>/dev/null >/dev/null"; then
-      echo "Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}" >&2
-      message="Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}"
+# Probe for valid AWS MFA token
+EXPIRY_MARKER_FILE=/tmp/aws-mfa-token-expired
+aws ec2 describe-instances >/dev/null 2>/dev/null
+EXIT_CODE=$?
+if [ "${EXIT_CODE}" = "254" -a ! -f "${EXPIRY_MARKER_FILE}" ]; then
+  message="AWS MFA token seems to have expired. aws ec2 describe-instances exited with code ${EXIT_CODE}. Not messaging you again. Try another awsmfa as root@sapsailing.com's tmux."
+  echo "${message}" | notify-operators "AWS MFA token expired"
+  touch "${EXPIRY_MARKER_FILE}"
+else
+  # Discover replicas
+  IPs=""
+  for i in $( cat `dirname $0`/regions.txt ); do
+    IPs="$( aws --region $i ec2 describe-instances --filters Name=instance-state-name,Values=running Name=tag:sailing-analytics-server,Values=tokyo2020 | jq .Reservations[].Instances[].PublicIpAddress -r )"
+    if [ -z "${IPs}" ]; then
+      echo "Couldn't find a running replica in region $i" >&2
+      message="Couldn't find a running replica in region $i"
       echo "${message}" | notify-operators "${message}"
     else
-      echo "Access from region ${i}, IP ${first} OK." >&2
+      read first others <<<"${IPs}"
+      if ! ssh -o StrictHostKeyChecking=no sailing@${first} "curl https://tokyo2020.sapsailing.com/gwt/status 2>/dev/null >/dev/null"; then
+	echo "Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}" >&2
+	message="Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}"
+	echo "${message}" | notify-operators "${message}"
+      else
+	echo "Access from region ${i}, IP ${first} OK." >&2
+      fi
     fi
-  fi
-done
+  done
+fi