diff --git a/configuration/on-site-scripts/monitor-disk-usage b/configuration/on-site-scripts/monitor-disk-usage index 017d28c3b60..6ba012b06a6 100755 --- a/configuration/on-site-scripts/monitor-disk-usage +++ b/configuration/on-site-scripts/monitor-disk-usage @@ -4,5 +4,5 @@ DIR_TO_CHECK=/var/lib/mongodb THRESHOLD_FOR_ALERT=90 usage_percent=$( df "${DIR_TO_CHECK}" | tail -n 1 | sed -e 's/^.* \([0-9]*\)% .*$/\1/' ) if [ $usage_percent -gt $THRESHOLD_FOR_ALERT ]; then - echo "Disk holding ${DIR_TO_CHECK} on `hostname` used more then ${THRESHOLD_FOR_ALERT} percent" notify-operators "DISK USAGE ALERT" + echo "Disk holding ${DIR_TO_CHECK} on `hostname` used more then ${THRESHOLD_FOR_ALERT} percent" | notify-operators "DISK USAGE ALERT" fi diff --git a/configuration/on-site-scripts/monitor-mongo-disk-full b/configuration/on-site-scripts/monitor-mongo-disk-full index 936795b5fa5..5481309a829 100755 --- a/configuration/on-site-scripts/monitor-mongo-disk-full +++ b/configuration/on-site-scripts/monitor-mongo-disk-full @@ -2,5 +2,6 @@ WARN_THRESHOLD_PERCENT=80 mongo_volume_fill=$( ssh ec2-user@tokyo-ssh.sapsailing.com df /var/lib/mongo | awk '{ print $5; }' | tail -n 1 | sed -e 's/%$//' ) if [ ${mongo_volume_fill} -gt ${WARN_THRESHOLD_PERCENT} ]; then - notify-operators "MongoDB volume on tokyo-ssh.sapsailing.com almost full; currently at ${mongo_volume_fill}%" + subjectAndBody="MongoDB volume on tokyo-ssh.sapsailing.com almost full; currently at ${mongo_volume_fill}%" + echo "${subjectAndBody}" | notify-operators "${subjectAndBody}" fi diff --git a/configuration/on-site-scripts/monitor-tokyo2020-reachability-from-regions b/configuration/on-site-scripts/monitor-tokyo2020-reachability-from-regions index 4f17f664074..201a8c5220b 100755 --- a/configuration/on-site-scripts/monitor-tokyo2020-reachability-from-regions +++ b/configuration/on-site-scripts/monitor-tokyo2020-reachability-from-regions @@ -5,12 +5,14 @@ for i in $( cat `dirname $0`/regions.txt ); do IPs="$( aws --region $i ec2 describe-instances --filters Name=instance-state-name,Values=running Name=tag:sailing-analytics-server,Values=tokyo2020 | jq .Reservations[].Instances[].PublicIpAddress -r )" if [ -z "${IPs}" ]; then echo "Couldn't find a running replica in region $i" >&2 - notify-operators "Couldn't find a running replica in region $i" + message="Couldn't find a running replica in region $i" + echo "${message}" | notify-operators "${message}" else read first others <<<"${IPs}" if ! ssh -o StrictHostKeyChecking=no sailing@${first} "curl https://tokyo2020.sapsailing.com/gwt/status 2>/dev/null >/dev/null"; then echo "Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}" >&2 - notify-operators "Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}" + message="Problem reaching tokyo2020.sapsailing.com from instance ${first} in region ${i}" + echo "${message}" | notify-operators "${message}" else echo "Access from region ${i}, IP ${first} OK." >&2 fi diff --git a/java/com.sap.sailing.domain.tractracadapter/src/com/sap/sailing/domain/tractracadapter/impl/CompetitorChangeReceiver.java b/java/com.sap.sailing.domain.tractracadapter/src/com/sap/sailing/domain/tractracadapter/impl/CompetitorChangeReceiver.java index eb3a495d73f..0fa1733396b 100755 --- a/java/com.sap.sailing.domain.tractracadapter/src/com/sap/sailing/domain/tractracadapter/impl/CompetitorChangeReceiver.java +++ b/java/com.sap.sailing.domain.tractracadapter/src/com/sap/sailing/domain/tractracadapter/impl/CompetitorChangeReceiver.java @@ -94,15 +94,20 @@ implements OfficialCompetitorUpdateProvider { @Override protected void handleEvent(Triple event) { final IRace race = event.getA().getRace(); - final DynamicTrackedRace trackedRace = getTrackedRace(race); - if (trackedRace != null) { - final IRaceCompetitor raceCompetitor = event.getA(); - if (reconciler != null && raceCompetitor != null) { - reconciler.reconcileCompetitorStatus(raceCompetitor, trackedRace); - } + if (race == null) { + logger.warning("Received a competitor update for a null race for competitor "+event.getA().getCompetitor().getName()+ + " in "+this+". Ignoring."); } else { - logger.warning("Couldn't find tracked race for race " + race.getName() - + ". Dropping competitor change event " + event); + final DynamicTrackedRace trackedRace = getTrackedRace(race); + if (trackedRace != null) { + final IRaceCompetitor raceCompetitor = event.getA(); + if (reconciler != null && raceCompetitor != null) { + reconciler.reconcileCompetitorStatus(raceCompetitor, trackedRace); + } + } else { + logger.warning("Couldn't find tracked race for race " + race.getName() + + ". Dropping competitor change event " + event); + } } } diff --git a/wiki/info/landscape/olympic-setup.md b/wiki/info/landscape/olympic-setup.md index 159b6860f22..495a800b77a 100644 --- a/wiki/info/landscape/olympic-setup.md +++ b/wiki/info/landscape/olympic-setup.md @@ -163,6 +163,25 @@ The port forwards vary for exceptional situations, such as when the Internet con The tunnel configurations are established and configured using a set of scripts, each to be found under ``/usr/local/bin`` on each of the two laptops. +#### ssh_config and sshd_config tweaks + +In order to recover quickly from failures we changed ``/etc/ssh/ssh_config`` on both of the P1s and added the following parameters: +``` +ExitOnForwardFailure yes +ConnectTimeout 10 +ServerAliveCountMax 3 +ServerAliveInterval 10 +``` +For the server side on tokyo-ssh and on the both P1s the following parameters have been added to ``/etc/ssh/sshd_config``: +``` +ClientAliveInterval 3 +ClientAliveCountMax 3 +``` + +ExitOnForwardFailure will force ssh to exit if one of the port forwards fails. ConnectTimeout manages the time in seconds until an initial connection fails. AliveInterval (client and server) manages the time in seconds after ssh/sshd are sending client and server alive probes. CountMax is the number of retries for those probes. + +The settings have been verified by executing a network change on both the laptops, the ssh tunnel returns after a couple of seconds. + #### Regular Operations: master on sap-p1-1, replica on sap-p1-2, with Internet / Cloud connection On sap-p1-1 two SSH connections are maintained, with the following default port forwards, assuming sap-p1-1 is the local master: