eclipse-sailing-analytics/sailing-analytics
1
0
Fork 0
mirror of https://github.com/eclipse-sailing-analytics/sailing-analytics.git synced 2026-09-21 05:05:31 +00:00
Code Activity

Merge remote-tracking branch 'origin/master'

Browse Source
This commit is contained in:
Generic Wiki User
2021-07-29 01:48:18 +00:00
parent 072e4705f0 ff33cd5d37
commit 277f34f769
2 changed files with 27 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
configuration/on-site-scripts/launch-replicas-in-all-regions.sh
+1
View File
@@ -70,3 +70,4 @@ for REGION in $( cat `dirname $0`/regions.txt ); do
echo "Waiting a minute now after having asked for replica launch in region ${REGION} to avoid overloading master with initial load requests"
sleep 60
done
wait
configuration/on-site-scripts/monitor-tokyo2020-reachability-from-regions
+26 -16
View File
@@ -1,20 +1,30 @@
#!/bin/bash
# Discover replicas
IPs=""
for i in $( cat `dirname $0`/regions.txt ); do
IPs="$( aws --region $i ec2 describe-instances --filters Name=instance-state-name,Values=running Name=tag:sailing-analytics-server,Values=tokyo2020 | jq .Reservations[].Instances[].PublicIpAddress -r )"
if [ -z "${IPs}" ]; then
echo "Couldn't find a running replica in region $i" >&2
message="Couldn't find a running replica in region $i"
echo "${message}" | notify-operators "${message}"
else
read first others <<<"${IPs}"
if ! ssh -o StrictHostKeyChecking=no root@${first} "curl https://tokyo2020.sapsailing.com/gwt/status 2>/dev/null >/dev/null"; then
echo "Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}" >&2
message="Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}"
# Probe for valid AWS MFA token
EXPIRY_MARKER_FILE=/tmp/aws-mfa-token-expired
aws ec2 describe-instances >/dev/null 2>/dev/null
EXIT_CODE=$?
if [ "${EXIT_CODE}" = "254" -a ! -f "${EXPIRY_MARKER_FILE}" ]; then
message="AWS MFA token seems to have expired. aws ec2 describe-instances exited with code ${EXIT_CODE}. Not messaging you again. Try another awsmfa as root@sapsailing.com's tmux."
echo "${message}" | notify-operators "AWS MFA token expired"
touch "${EXPIRY_MARKER_FILE}"
else
# Discover replicas
IPs=""
for i in $( cat `dirname $0`/regions.txt ); do
IPs="$( aws --region $i ec2 describe-instances --filters Name=instance-state-name,Values=running Name=tag:sailing-analytics-server,Values=tokyo2020 | jq .Reservations[].Instances[].PublicIpAddress -r )"
if [ -z "${IPs}" ]; then
echo "Couldn't find a running replica in region $i" >&2
message="Couldn't find a running replica in region $i"
echo "${message}" | notify-operators "${message}"
else
echo "Access from region ${i}, IP ${first} OK." >&2
read first others <<<"${IPs}"
if ! ssh -o StrictHostKeyChecking=no root@${first} "curl https://tokyo2020.sapsailing.com/gwt/status 2>/dev/null >/dev/null"; then
echo "Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}" >&2
message="Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}"
echo "${message}" | notify-operators "${message}"
else
echo "Access from region ${i}, IP ${first} OK." >&2
fi
fi
fi
done
done
fi