From 975f420cfb87719c3a553db1c7c77caa59515087 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sun, 25 Jul 2021 23:37:15 +0900 Subject: [PATCH 1/3] fixed calls to notify-operators where stdin was missing in on-site-scripts --- configuration/on-site-scripts/monitor-disk-usage | 2 +- configuration/on-site-scripts/monitor-mongo-disk-full | 3 ++- .../monitor-tokyo2020-reachability-from-regions | 6 ++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/configuration/on-site-scripts/monitor-disk-usage b/configuration/on-site-scripts/monitor-disk-usage index 017d28c3b60..6ba012b06a6 100755 --- a/configuration/on-site-scripts/monitor-disk-usage +++ b/configuration/on-site-scripts/monitor-disk-usage @@ -4,5 +4,5 @@ DIR_TO_CHECK=/var/lib/mongodb THRESHOLD_FOR_ALERT=90 usage_percent=$( df "${DIR_TO_CHECK}" | tail -n 1 | sed -e 's/^.* \([0-9]*\)% .*$/\1/' ) if [ $usage_percent -gt $THRESHOLD_FOR_ALERT ]; then - echo "Disk holding ${DIR_TO_CHECK} on `hostname` used more then ${THRESHOLD_FOR_ALERT} percent" notify-operators "DISK USAGE ALERT" + echo "Disk holding ${DIR_TO_CHECK} on `hostname` used more then ${THRESHOLD_FOR_ALERT} percent" | notify-operators "DISK USAGE ALERT" fi diff --git a/configuration/on-site-scripts/monitor-mongo-disk-full b/configuration/on-site-scripts/monitor-mongo-disk-full index 936795b5fa5..5481309a829 100644 --- a/configuration/on-site-scripts/monitor-mongo-disk-full +++ b/configuration/on-site-scripts/monitor-mongo-disk-full @@ -2,5 +2,6 @@ WARN_THRESHOLD_PERCENT=80 mongo_volume_fill=$( ssh ec2-user@tokyo-ssh.sapsailing.com df /var/lib/mongo | awk '{ print $5; }' | tail -n 1 | sed -e 's/%$//' ) if [ ${mongo_volume_fill} -gt ${WARN_THRESHOLD_PERCENT} ]; then - notify-operators "MongoDB volume on tokyo-ssh.sapsailing.com almost full; currently at ${mongo_volume_fill}%" + subjectAndBody="MongoDB volume on tokyo-ssh.sapsailing.com almost full; currently at ${mongo_volume_fill}%" + echo "${subjectAndBody}" | notify-operators "${subjectAndBody}" fi diff --git a/configuration/on-site-scripts/monitor-tokyo2020-reachability-from-regions b/configuration/on-site-scripts/monitor-tokyo2020-reachability-from-regions index 4f17f664074..201a8c5220b 100755 --- a/configuration/on-site-scripts/monitor-tokyo2020-reachability-from-regions +++ b/configuration/on-site-scripts/monitor-tokyo2020-reachability-from-regions @@ -5,12 +5,14 @@ for i in $( cat `dirname $0`/regions.txt ); do IPs="$( aws --region $i ec2 describe-instances --filters Name=instance-state-name,Values=running Name=tag:sailing-analytics-server,Values=tokyo2020 | jq .Reservations[].Instances[].PublicIpAddress -r )" if [ -z "${IPs}" ]; then echo "Couldn't find a running replica in region $i" >&2 - notify-operators "Couldn't find a running replica in region $i" + message="Couldn't find a running replica in region $i" + echo "${message}" | notify-operators "${message}" else read first others <<<"${IPs}" if ! ssh -o StrictHostKeyChecking=no sailing@${first} "curl https://tokyo2020.sapsailing.com/gwt/status 2>/dev/null >/dev/null"; then echo "Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}" >&2 - notify-operators "Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}" + message="Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}" + echo "${message}" | notify-operators "${message}" else echo "Access from region ${i}, IP ${first} OK." >&2 fi From 7eafbbc9959018fffac0c4fbe2d52199a417622b Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sun, 25 Jul 2021 23:50:09 +0900 Subject: [PATCH 2/3] If a race-competitor update is received from TracTrac without valid race, log and ignore --- .../impl/CompetitorChangeReceiver.java | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/java/com.sap.sailing.domain.tractracadapter/src/com/sap/sailing/domain/tractracadapter/impl/CompetitorChangeReceiver.java b/java/com.sap.sailing.domain.tractracadapter/src/com/sap/sailing/domain/tractracadapter/impl/CompetitorChangeReceiver.java index eb3a495d73f..0fa1733396b 100755 --- a/java/com.sap.sailing.domain.tractracadapter/src/com/sap/sailing/domain/tractracadapter/impl/CompetitorChangeReceiver.java +++ b/java/com.sap.sailing.domain.tractracadapter/src/com/sap/sailing/domain/tractracadapter/impl/CompetitorChangeReceiver.java @@ -94,15 +94,20 @@ implements OfficialCompetitorUpdateProvider { @Override protected void handleEvent(Triple event) { final IRace race = event.getA().getRace(); - final DynamicTrackedRace trackedRace = getTrackedRace(race); - if (trackedRace != null) { - final IRaceCompetitor raceCompetitor = event.getA(); - if (reconciler != null && raceCompetitor != null) { - reconciler.reconcileCompetitorStatus(raceCompetitor, trackedRace); - } + if (race == null) { + logger.warning("Received a competitor update for a null race for competitor "+event.getA().getCompetitor().getName()+ + " in "+this+". Ignoring."); } else { - logger.warning("Couldn't find tracked race for race " + race.getName() - + ". Dropping competitor change event " + event); + final DynamicTrackedRace trackedRace = getTrackedRace(race); + if (trackedRace != null) { + final IRaceCompetitor raceCompetitor = event.getA(); + if (reconciler != null && raceCompetitor != null) { + reconciler.reconcileCompetitorStatus(raceCompetitor, trackedRace); + } + } else { + logger.warning("Couldn't find tracked race for race " + race.getName() + + ". Dropping competitor change event " + event); + } } } From d7829c04b96ed0229a7bb4b6197232b58d88075a Mon Sep 17 00:00:00 2001 From: Alessandro Stoltenberg Date: Mon, 26 Jul 2021 09:01:08 +0900 Subject: [PATCH 3/3] Olympic-Setup: Describing ssh_config and sshd_config changes to recover quickly from network failures --- wiki/info/landscape/olympic-setup.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/wiki/info/landscape/olympic-setup.md b/wiki/info/landscape/olympic-setup.md index 159b6860f22..495a800b77a 100644 --- a/wiki/info/landscape/olympic-setup.md +++ b/wiki/info/landscape/olympic-setup.md @@ -163,6 +163,25 @@ The port forwards vary for exceptional situations, such as when the Internet con The tunnel configurations are established and configured using a set of scripts, each to be found under ``/usr/local/bin`` on each of the two laptops. +#### ssh_config and sshd_config tweaks + +In order to recover quickly from failures we changed ``/etc/ssh/ssh_config`` on both of the P1s and added the following parameters: +``` +ExitOnForwardFailure yes +ConnectTimeout 10 +ServerAliveCountMax 3 +ServerAliveInterval 10 +``` +For the server side on tokyo-ssh and on the both P1s the following parameters have been added to ``/etc/ssh/sshd_config``: +``` +ClientAliveInterval 3 +ClientAliveCountMax 3 +``` + +ExitOnForwardFailure will force ssh to exit if one of the port forwards fails. ConnectTimeout manages the time in seconds until an initial connection fails. AliveInterval (client and server) manages the time in seconds after ssh/sshd are sending client and server alive probes. CountMax is the number of retries for those probes. + +The settings have been verified by executing a network change on both the laptops, the ssh tunnel returns after a couple of seconds. + #### Regular Operations: master on sap-p1-1, replica on sap-p1-2, with Internet / Cloud connection On sap-p1-1 two SSH connections are maintained, with the following default port forwards, assuming sap-p1-1 is the local master: