Merge remote-tracking branch 'origin/master'

configuration/on-site-scripts/monitor-disk-usage

@@ -4,5 +4,5 @@ DIR_TO_CHECK=/var/lib/mongodb
 THRESHOLD_FOR_ALERT=90
 usage_percent=$( df "${DIR_TO_CHECK}" | tail -n 1 | sed -e 's/^.* \([0-9]*\)% .*$/\1/' )
 if [ $usage_percent -gt $THRESHOLD_FOR_ALERT ]; then
-  echo "Disk holding ${DIR_TO_CHECK} on `hostname` used more then ${THRESHOLD_FOR_ALERT} percent" notify-operators "DISK USAGE ALERT"
+  echo "Disk holding ${DIR_TO_CHECK} on `hostname` used more then ${THRESHOLD_FOR_ALERT} percent" | notify-operators "DISK USAGE ALERT"
 fi

configuration/on-site-scripts/monitor-mongo-disk-full

@@ -2,5 +2,6 @@
 WARN_THRESHOLD_PERCENT=80
 mongo_volume_fill=$( ssh ec2-user@tokyo-ssh.sapsailing.com df /var/lib/mongo | awk '{ print $5; }' | tail -n 1 | sed -e 's/%$//' )
 if [ ${mongo_volume_fill} -gt ${WARN_THRESHOLD_PERCENT} ]; then
-  notify-operators "MongoDB volume on tokyo-ssh.sapsailing.com almost full; currently at ${mongo_volume_fill}%"
+  subjectAndBody="MongoDB volume on tokyo-ssh.sapsailing.com almost full; currently at ${mongo_volume_fill}%"
+  echo "${subjectAndBody}" | notify-operators "${subjectAndBody}"
 fi

configuration/on-site-scripts/monitor-tokyo2020-reachability-from-regions

@@ -5,12 +5,14 @@ for i in $( cat `dirname $0`/regions.txt ); do
   IPs="$( aws --region $i ec2 describe-instances --filters Name=instance-state-name,Values=running Name=tag:sailing-analytics-server,Values=tokyo2020 | jq .Reservations[].Instances[].PublicIpAddress -r )"
   if [ -z "${IPs}" ]; then
     echo "Couldn't find a running replica in region $i" >&2
-    notify-operators "Couldn't find a running replica in region $i"
+    message="Couldn't find a running replica in region $i"
+    echo "${message}" | notify-operators "${message}"
   else
     read first others <<<"${IPs}"
     if ! ssh -o StrictHostKeyChecking=no sailing@${first} "curl https://tokyo2020.sapsailing.com/gwt/status 2>/dev/null >/dev/null"; then
       echo "Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}" >&2
-      notify-operators "Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}"
+      message="Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}"
+      echo "${message}" | notify-operators "${message}"
     else
       echo "Access from region ${i}, IP ${first} OK." >&2
     fi

java/com.sap.sailing.domain.tractracadapter/src/com/sap/sailing/domain/tractracadapter/impl/CompetitorChangeReceiver.java

@@ -94,15 +94,20 @@ implements OfficialCompetitorUpdateProvider {
     @Override
     protected void handleEvent(Triple<IRaceCompetitor, Void, Void> event) {
         final IRace race = event.getA().getRace();
-        final DynamicTrackedRace trackedRace = getTrackedRace(race);
-        if (trackedRace != null) {
-            final IRaceCompetitor raceCompetitor = event.getA();
-            if (reconciler != null && raceCompetitor != null) {
-                reconciler.reconcileCompetitorStatus(raceCompetitor, trackedRace);
-            }
+        if (race == null) {
+            logger.warning("Received a competitor update for a null race for competitor "+event.getA().getCompetitor().getName()+
+                    " in "+this+". Ignoring.");
         } else {
-            logger.warning("Couldn't find tracked race for race " + race.getName()
-                    + ". Dropping competitor change event " + event);
+            final DynamicTrackedRace trackedRace = getTrackedRace(race);
+            if (trackedRace != null) {
+                final IRaceCompetitor raceCompetitor = event.getA();
+                if (reconciler != null && raceCompetitor != null) {
+                    reconciler.reconcileCompetitorStatus(raceCompetitor, trackedRace);
+                }
+            } else {
+                logger.warning("Couldn't find tracked race for race " + race.getName()
+                        + ". Dropping competitor change event " + event);
+            }
         }
     }
 

wiki/info/landscape/olympic-setup.md

@@ -163,6 +163,25 @@ The port forwards vary for exceptional situations, such as when the Internet con
 
 The tunnel configurations are established and configured using a set of scripts, each to be found under ``/usr/local/bin`` on each of the two laptops.
 
+#### ssh_config and sshd_config tweaks
+
+In order to recover quickly from failures we changed ``/etc/ssh/ssh_config`` on both of the P1s and added the following parameters:
+```
+ExitOnForwardFailure yes
+ConnectTimeout 10
+ServerAliveCountMax 3
+ServerAliveInterval 10
+```
+For the server side on tokyo-ssh and on the both P1s the following parameters have been added to ``/etc/ssh/sshd_config``:
+```
+ClientAliveInterval 3
+ClientAliveCountMax 3
+```
+
+ExitOnForwardFailure will force ssh to exit if one of the port forwards fails. ConnectTimeout manages the time in seconds until an initial connection fails. AliveInterval (client and server) manages the time in seconds after ssh/sshd are sending client and server alive probes. CountMax is the number of retries for those probes. 
+
+The settings have been verified by executing a network change on both the laptops, the ssh tunnel returns after a couple of seconds.
+
 #### Regular Operations: master on sap-p1-1, replica on sap-p1-2, with Internet / Cloud connection 
 
 On sap-p1-1 two SSH connections are maintained, with the following default port forwards, assuming sap-p1-1 is the local master: