added script monitor-paris2024-reachability-from-regions for paris2024 on-site

2026-09-19 04:05:36 +00:00 · 2024-03-19 18:05:49 +01:00
parent 154a8bded8
commit 0c3ea97750
1 changed files with 34 additions and 0 deletions
@@ -0,0 +1,34 @@
+#!/bin/bash
+# Probe for valid AWS MFA token
+EXPIRY_MARKER_FILE=/tmp/aws-mfa-token-expired
+aws ec2 describe-instances >/dev/null 2>/dev/null
+EXIT_CODE=$?
+if [ "${EXIT_CODE}" = "255" ]; then
+  if [ ! -f "${EXPIRY_MARKER_FILE}" ]; then
+    message="AWS MFA token seems to have expired. aws ec2 describe-instances exited with code ${EXIT_CODE}. Not messaging you again. Try another awsmfa as root@sapsailing.com's tmux."
+    echo "${message}" | notify-operators "AWS MFA token expired"
+    touch "${EXPIRY_MARKER_FILE}"
+  else
+    echo "AWS MFA token expired, but we had sent e-mail already."
+  fi
+else
+  # Discover replicas
+  IPs=""
+  for i in $( cat `dirname $0`/regions.txt ); do
+    IPs="$( aws --region $i ec2 describe-instances --filters Name=instance-state-name,Values=running Name=tag:sailing-analytics-server,Values=paris2024 | jq .Reservations[].Instances[].PublicIpAddress -r )"
+    if [ -z "${IPs}" ]; then
+      echo "Couldn't find a running replica in region $i" >&2
+      message="Couldn't find a running replica in region $i"
+      echo "${message}" | notify-operators "${message}"
+    else
+      read first others <<<"${IPs}"
+      if ! ssh -o StrictHostKeyChecking=no root@${first} "curl https://paris2024.sapsailing.com/gwt/status 2>/dev/null >/dev/null"; then
+	echo "Problem reaching paris2024.sapsailing.com from instance ${first} in region ${i}" >&2
+	message="Problem reaching paris2024.sapsailing.com from instance ${first} in region ${i}"
+	echo "${message}" | notify-operators "${message}"
+      else
+	echo "Access from region ${i}, IP ${first} OK." >&2
+      fi
+    fi
+  done
+fi