diff --git a/configuration/environments_scripts/reverse_proxy/files/var/www/cgi-bin/reverseProxyHealthcheck.sh b/configuration/environments_scripts/reverse_proxy/files/var/www/cgi-bin/reverseProxyHealthcheck.sh index 396c9473c13..0c8a3a8f1f0 100755 --- a/configuration/environments_scripts/reverse_proxy/files/var/www/cgi-bin/reverseProxyHealthcheck.sh +++ b/configuration/environments_scripts/reverse_proxy/files/var/www/cgi-bin/reverseProxyHealthcheck.sh @@ -18,7 +18,7 @@ status() { } randomise() { # $1: A number to slightly randomise - echo $(($RANDOM % 10 + $1 )) + echo $(($RANDOM % 20 + $1 )) } # The names of the variables in the macros file. ARCHIVE_IP_NAME="ARCHIVE_IP" @@ -37,14 +37,14 @@ ARCHIVE_IP_FILENAME="${CACHE_LOCATION}/archive_ip" ARCHIVE_AZ_FILENAME="${CACHE_LOCATION}/archive_az" LAST_TARGET_GROUP_HEALTHCHECK_RESULT_FILENAME="${CACHE_LOCATION}/last_target_group_healthcheck_result" # The time it takes before the cache is reset. -CACHE_TIMEOUT_SECONDS=120 +CACHE_TIMEOUT_SECONDS=125 # Target group healthcheck timeout TARGET_GROUP_HEALTHCHECK_TIMEOUT_SECONDS=30 # IPs and AZs of instance MY_IP=$( ec2-metadata --local-ipv4 | grep -o "${IP_REGEX}") MY_AZ=$( ec2-metadata --availability-zone | grep -o "[a-zA-Z]\+-[a-zA-Z]\+-[0-9a-z]\+\>") ARCHIVE_IP=$(grep -m 1 "^Define ${ARCHIVE_IP_NAME}\> .*" ${MACROS_PATH} | grep -o "${IP_REGEX}") -ARCHIVE_TAG_KEY="temp-key" +ARCHIVE_TAG_KEY="sailing-analytics-server" ARCHIVE_TAG_VALUE="ARCHIVE" if [[ "$1" == "cleanup" ]]; then rm -rf ${CACHE_LOCATION} @@ -55,7 +55,7 @@ fi curl --silent --location --fail "http://${MY_IP}/internal-server-status" >/dev/null if [[ "$?" -ne 0 ]]; then status "500 Reverse proxy itself is unhealthy" - outputMessage "Reverse proxy is unhealthy" + outputMessage "Unhealthy: Reverse proxy is unhealthy." exit 0 fi current_time=$(date +"%s") @@ -65,8 +65,8 @@ if [[ ! -e "$ID_TO_AZ_FILENAME" || ! -e "$ARCHIVE_AZ_FILENAME" || "$(cat ${ARCHI echo "$instances" | jq -r ".Reservations | .[] | .Instances | .[]" | jq -r '"\(.InstanceId) \(.Placement.AvailabilityZone)"' > ${ID_TO_AZ_FILENAME} tmp_az=$(echo "$instances" | jq -r '.Reservations | .[] | .Instances | .[] | select(.Tags | any(.Key=="'${ARCHIVE_TAG_KEY}'" and .Value=="'${ARCHIVE_TAG_VALUE}'")) | "\(.Placement.AvailabilityZone)"') if [[ -z "$tmp_az" ]]; then - status "500 Unable to retrieve archive AZ" - outputMessage "Cannot retrieve archive AZ, ensure the correct tags are in place." + status "200" + outputMessage "Healthy: Forced healthy, as retrieval of archive AZ failed. Ensure the correct tags are in place." exit 1 fi echo "$tmp_az" > ${ARCHIVE_AZ_FILENAME} @@ -77,7 +77,7 @@ then archive_az=$(cat ${ARCHIVE_AZ_FILENAME}) if [[ "$archive_az" == "$MY_AZ" ]]; then status "200" - outputMessage "Healthy: in the same az as the archive" + outputMessage "Healthy: In the same az as the archive." else if [[ ! -e "${LAST_TARGET_GROUP_HEALTHCHECK_RESULT_FILENAME}" || "$(($current_time - $(stat --format '%Y' ${LAST_TARGET_GROUP_HEALTHCHECK_RESULT_FILENAME}) ))" -gt "$(randomise $TARGET_GROUP_HEALTHCHECK_TIMEOUT_SECONDS)" ]]; then # This branch runs if the cached timestamp, of the last target group healthcheck, doesn't exist, or if it exceeds TARGET_GROUP_HEALTHCHECK_TIMEOUT_SECONDS. @@ -91,25 +91,25 @@ then if [[ "$healthy_target_exists_in_same_az_as_archive" == "true" ]]; then echo "unhealthy" > ${LAST_TARGET_GROUP_HEALTHCHECK_RESULT_FILENAME} status "503 Not in the same az" - outputMessage "Not in the same AZ; healthy instance in the same AZ." + outputMessage "Unhealthy: Not in the same AZ as the archive; healthy instance in the same AZ as the archive. (Result cached)" else echo "healthy" > ${LAST_TARGET_GROUP_HEALTHCHECK_RESULT_FILENAME} status "200" - outputMessage "No healthy instance in the same AZ; forcing healthy for a short while." + outputMessage "Healthy: No healthy instance in the same AZ; forcing healthy for ${TARGET_GROUP_HEALTHCHECK_TIMEOUT_SECONDS} seconds. (Result cached)" fi else # This branch runs in the case that there is a timestamp file but the target group healthcheck has been performed within the TARGET_GROUP_HEALTHCHECK_TIMEOUT_SECONDS. if [[ -e "${LAST_TARGET_GROUP_HEALTHCHECK_RESULT_FILENAME}" && "$(cat $LAST_TARGET_GROUP_HEALTHCHECK_RESULT_FILENAME)" == "healthy" ]]; then status "200" - outputMessage "Using stored healthcheck, value healthy" + outputMessage "Healthy: Using cached healthcheck value 'healthy' (despite residing in a different AZ to the ARCHIVE)." else status "503 Not in the same az" - outputMessage "Unhealthy: Not in the same az as the archive." + outputMessage "Unhealthy: Not in the same az as the archive and the value cached is unhealthy." fi fi fi else # We don't check if failover archive is in the same az status "200" - outputMessage "failover at play: force healthy" + outputMessage "Healthy: Failover at play, force healthy." fi \ No newline at end of file