From dd11d7b3e6c06257ae80ec45cedbcd586ae69a5d Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Thu, 9 May 2024 18:09:50 +0200 Subject: [PATCH 001/134] b5912: Add additional modules needed for email sending to bugzilla. --- .../central_reverse_proxy/setup-central-reverse-proxy.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh index 6369f048b9d..e279a8d0cc3 100755 --- a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh @@ -118,6 +118,8 @@ ssh -A "root@${IP}" "bash -s" << BUGZILLAEOF &>/dev/null /usr/bin/perl install-module.pl Daemon::Generic /usr/bin/perl install-module.pl File::MimeInfo::Magic /usr/bin/perl install-module.pl File::Copy::Recursive +/usr/bin/perl install-module.pl MIME::Base64 +/usr/bin/perl install-module.pl Authen::SASL scp -p root@sapsailing.com:/usr/share/bugzilla/data/params.json /usr/share/bugzilla/data/params.json BUGZILLAEOF read -n 1 -p "Bugzilla installation complete, when ready press a key to continue." key_pressed From cf6106e0972195c20ec68678767ed8cf820184ab Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Fri, 10 May 2024 12:17:58 +0200 Subject: [PATCH 002/134] b5912: Fix issue with different tags. Remove umount for /var/log/old on sailing servers (as they don't have this mount). --- .../target-group-tag-route53-nfs-elasticIP-setup.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh index 1ade30928a8..4da718c2043 100755 --- a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh +++ b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh @@ -12,7 +12,8 @@ target_groups=$(aws elbv2 describe-target-groups) LOCAL_IPV4=$(ssh root@"$1" "ec2-metadata --local-ipv4 | sed \"s/local-ipv4: *//\"") INSTANCE_ID=$(ssh root@"$1" "ec2-metadata --instance-id | sed \"s/instance-id: *//\"") ELASTIC_IP="54.229.94.254" -TAGS=("allReverseProxies" "CentralReverseProxy") +TAGS=("CentralReverseProxy" "ReverseProxy") +TARGET_GROUP_TAGS=("allReverseProxies" "CentralReverseProxy") extract_public_ip() { jq -r ' .Instances | .[] | .PublicIpAddress' | grep -v null } @@ -31,7 +32,7 @@ echo "Registering with nlb" aws elbv2 register-targets --target-group-arn "$nlbArn" --targets Id="${LOCAL_IPV4}",Port=80 echo "Fetching tags" describe_tags=$(aws elbv2 describe-tags --resource-arns $(echo "$target_groups" | jq -r '.TargetGroups | .[] | .TargetGroupArn')) -for tag in "${TAGS[@]}"; do +for tag in "${TARGET_GROUP_TAGS[@]}"; do echo "Adding to target groups with $tag" for tgArn in $(echo "$describe_tags" | jq -r '.TagDescriptions | .[] | select(.Tags | any(.Key=="'"$tag"'") ) | .ResourceArn'); do if [[ "$tgArn" != "$nlbArn" ]]; then @@ -50,7 +51,7 @@ echo "Describing instances for remounting." describe_instances=$(aws ec2 describe-instances) for instanceIp in $(echo "$describe_instances" | select_instances_by_tag "sailing-analytics-server" | extract_public_ip); do echo "Remounting on $instanceIp" - ssh root@"${instanceIp}" "umount -l -f /var/log/old; umount -l -f /home/scores; mount -a" + ssh root@"${instanceIp}" " umount -l -f /home/scores; mount -a" done for instanceIp in $(echo "$describe_instances" | select_instances_by_tag "DisposableProxy" | extract_public_ip); do echo "Remounting on $instanceIp" From 90659289783b5c0c3d19c3885bfcaa9e4d00ab69 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Fri, 10 May 2024 12:40:15 +0200 Subject: [PATCH 003/134] b5912: Add setup_swap to imageupgrade functions. --- .../setup-central-reverse-proxy.sh | 1 + .../repo/usr/local/bin/imageupgrade_functions.sh | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh index e279a8d0cc3..276412cb9a9 100755 --- a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh @@ -65,6 +65,7 @@ scp -o StrictHostKeyChecking=no -p root@"$IMAGEUPGRADE_FUNCTIONS_IP":"$IMAGEUPGR setup_cloud_cfg_and_root_login # setup files build_crontab_and_setup_files -c -n "${IMAGE_TYPE}" "${GIT_COPY_USER}" "${RELATIVE_PATH_TO_GIT}" # -c & -n mean only files are copied over. +setup_swap 5000 cd /home for folder in * ; do [[ -d "\$folder" ]] || continue diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index 107a95668a2..4ffd0f12630 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -303,3 +303,16 @@ setup_apachetop() { make install popd } + +setup_swap() { + # $1: size of swapspace in megabytes. + local swapfile_location=/var/cache/swapfile + pushd . + sudo dd if=/dev/zero of="$swapfile_location" bs=1M count="$1" + sudo chmod 600 "$swapfile_location" + sudo chown root:root "$swapfile_location" + sudo mkswap "$swapfile_location" + sudo su - -c "echo \"$swapfile_location none swap pri=0 0 0\" >> /etc/fstab" + sudo swapon -a + popd +} \ No newline at end of file From 9fe9840581e990143d9a1969f8020372ede65a69 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Mon, 13 May 2024 14:03:52 +0200 Subject: [PATCH 004/134] b5912: Fix max 20 resources for describe tags issue. --- .../target-group-tag-route53-nfs-elasticIP-setup.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh index 4da718c2043..39f00667830 100755 --- a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh +++ b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh @@ -31,7 +31,7 @@ nlbArn=$(aws elbv2 describe-tags --resource-arns $(echo "$target_groups" | jq -r echo "Registering with nlb" aws elbv2 register-targets --target-group-arn "$nlbArn" --targets Id="${LOCAL_IPV4}",Port=80 echo "Fetching tags" -describe_tags=$(aws elbv2 describe-tags --resource-arns $(echo "$target_groups" | jq -r '.TargetGroups | .[] | .TargetGroupArn')) +describe_tags=$(aws elbv2 describe-tags --resource-arns $(echo "$target_groups" | jq -r '.TargetGroups | .[] | select(.TargetGroupName | startswith("S-") | not ) | .TargetGroupArn')) for tag in "${TARGET_GROUP_TAGS[@]}"; do echo "Adding to target groups with $tag" for tgArn in $(echo "$describe_tags" | jq -r '.TagDescriptions | .[] | select(.Tags | any(.Key=="'"$tag"'") ) | .ResourceArn'); do @@ -45,6 +45,7 @@ done # alter records using batch file. sed -i "s/LOGFILES_INTERNAL_IP/$LOCAL_IPV4/" batch-for-route53-dns-record-update.json sed -i "s/SMTP_INTERNAL_IP/$LOCAL_IPV4/" batch-for-route53-dns-record-update.json +read -n 1 -p "Check the instance has the correct tags and is in the correct target group. Furthermore, check the batch file has been modified correctly. Press a key to continue.." key_pressed ###### DO NOT ENABLE WHILST TESTING: aws route53 change-resource-record-sets --hosted-zone-id Z2JYWXYWLLRLTE --change-batch file://batch-for-route53-dns-record-update.json # reload the nfs mountpoints. echo "Describing instances for remounting." From b76a5a8281fdcc1ee83f3dcf84edbb3f4082993c Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Mon, 13 May 2024 15:17:07 +0200 Subject: [PATCH 005/134] b5912: Improve build_crontab... in image upgrade functions to use an optional -h as hostname. --- .../build-crontab-and-cp-files | 2 +- .../setup-central-reverse-proxy-part-2.sh | 2 +- .../usr/local/bin/imageupgrade_functions.sh | 50 ++++++++++++++++--- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/configuration/environments_scripts/build-crontab-and-cp-files b/configuration/environments_scripts/build-crontab-and-cp-files index b92dfc1b879..92d3bd23053 100755 --- a/configuration/environments_scripts/build-crontab-and-cp-files +++ b/configuration/environments_scripts/build-crontab-and-cp-files @@ -39,7 +39,7 @@ do exit 4;; esac done -shift $((OPTIND-1)) # shift the arguments along so -s is no longer $1 +shift $((OPTIND-1)) # shift the arguments along. OPTIND is the position of the next argument to process. ENV_TYPE="$1" GIT_USER="$2" RELATIVE_GIT_DIR_NAME="$3" diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh index 34de879425a..894ebac7cbe 100755 --- a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh @@ -21,7 +21,7 @@ sudo systemctl start crond.service . imageupgrade_functions.sh cp -r "$TEMPORARY_HOME_COPY_LOCATION"/home / rm -rf "$TEMPORARY_HOME_COPY_LOCATION" -build_crontab_and_setup_files -f "${IMAGE_TYPE}" "${GIT_COPY_USER}" "${RELATIVE_PATH_TO_GIT}" # files have already been copied so -f is used. +build_crontab_and_setup_files -h localhost -f "${IMAGE_TYPE}" "${GIT_COPY_USER}" "${RELATIVE_PATH_TO_GIT}" # files have already been copied so -f is used. chown trac:static /var/www/static # setup nfs systemctl enable nfs-server diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index 4ffd0f12630..924c60533b8 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -79,16 +79,54 @@ update_root_crontab() { } build_crontab_and_setup_files() { - # There must be at least 1 and all args are passed to build-crontab-and-cp-files. See the documentation of this file for more info. - if [[ "$#" -lt 1 ]]; then - echo "Number of arguments is invalid. There must be at least 1 and all args are passed to build-crontab-and-cp-files." + if [[ "$#" -eq 0 ]]; then + echo "Number of arguments is invalid. Please use the options and arguments as follows." + echo "Options for this function:" + echo " -h is the hostname to fetch the configuration/environments_scripts from." + echo "The remaining (optional) args and options, if correct, are passed to the build_crontab_and_setup_files script." + echo " -c means no crontab file is created" + echo " -n means that if a crontab has been created, it isn't actually installed. This is useful for testing." + echo " -f means no files are copied over, which is useful if you have already copied files accross or don't want to override existing files" + echo "Then there are the arguments, where the order matters:" + echo " ENVIRONMENT_TYPE - the directory name in environments_scripts which will be used." + echo " USER_WITH_COPY_OF_REPO - a user which will exist on the environment type, which has a checked out copy of the git workspace." + echo " RELATIVE_PATH_OF_GIT_DIR_WITHIN_USER - the relative path within the USER_WITH_COPY_OF_REPO to get to the git workspace." else + TEMP=$(getopt -o fnch: -n 'options checker' -- "$@") + [[ "$?" -eq 0 ]] || exit 2 + eval set -- "$TEMP" + PASS_OPTIONS=() + HOSTNAME="sapsailing.com" + while :; do + case "$1" in + -c|-f|-n) + PASS_OPTIONS+=("$1") + ;; + -h) + if [[ "$2" ]]; then + HOSTNAME="$2" + shift + else + echo "hostname option requires argument" + fi + ;; + --) + shift + break + ;; + *) + echo "no more options" + break + esac + shift + done TEMP_ENVIRONMENTS_SCRIPTS=$(mktemp -d /root/environments_scripts_XXX) - scp -o StrictHostKeyChecking=no -pr "wiki@sapsailing.com:~/gitwiki/configuration/environments_scripts/*" "${TEMP_ENVIRONMENTS_SCRIPTS}" - [[ "$?" -eq 0 ]] || scp -o StrictHostKeyChecking=no -pr "root@sapsailing.com:/home/wiki/gitwiki/configuration/environments_scripts/*" "${TEMP_ENVIRONMENTS_SCRIPTS}" # For initial setup as not all landscape managers have direct wiki access. + echo "Attempting access to the wiki, typically used by image upgrade. Otherwise, try root. NOTE: If the first command fails, there will be a warning message." + scp -o StrictHostKeyChecking=no -pr "wiki@$HOSTNAME:~/gitwiki/configuration/environments_scripts/*" "${TEMP_ENVIRONMENTS_SCRIPTS}" + [[ "$?" -eq 0 ]] || scp -o StrictHostKeyChecking=no -pr "root@$HOSTNAME:/home/wiki/gitwiki/configuration/environments_scripts/*" "${TEMP_ENVIRONMENTS_SCRIPTS}" # For initial setup as not all landscape managers have direct wiki access. chown root:root "$TEMP_ENVIRONMENTS_SCRIPTS" cd "${TEMP_ENVIRONMENTS_SCRIPTS}" - ./build-crontab-and-cp-files $@ + ./build-crontab-and-cp-files "${PASS_OPTIONS[@]}" "$@" cd .. rm -rf "$TEMP_ENVIRONMENTS_SCRIPTS" fi From fa5e1e86b194e2c4b0ce4a0bc49c68a684e4f921 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Mon, 13 May 2024 15:34:32 +0200 Subject: [PATCH 006/134] b5912: COmment, disable stricthostchecking and silent curl. --- .../central_reverse_proxy/setup-central-reverse-proxy.sh | 5 +++-- .../target-group-tag-route53-nfs-elasticIP-setup.sh | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh index 276412cb9a9..cace6ab8ae9 100755 --- a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh @@ -26,7 +26,8 @@ # currently attached to and mounted on the current Central Reverse Proxy will # need to be unmounted, detached, attached to the new instance, and mounted there. if [[ "$#" -ne 4 ]]; then - echo "IP and bearer token required. Please check comment description for further details." + echo "4 arguments required. Please check comment description for further details." + echo "Example usage: setup-central-reverse-proxy.sh 1.2.3.4 0OcJ1938QE5it875kjlQe7HnzQ6740jsnMEVzowjZrs= 18.170.25.225 /home/sailing/code" fi IP=$1 BEARER_TOKEN=$2 @@ -48,7 +49,7 @@ yum update -y yum install -y httpd mod_proxy_html tmux nfs-utils git whois jq cronie iptables mailx nmap icu mariadb105-server tree #icu is a c/c++ library that provides unicode and globalisation support for software development. # docker setup yum install -y docker -sudo curl -L "https://github.com/docker/compose/releases/download/v2.26.1/docker-compose-\$(uname -s)-\$(uname -m)" -o /usr/local/bin/docker-compose +sudo curl --silent -L "https://github.com/docker/compose/releases/download/v2.26.1/docker-compose-\$(uname -s)-\$(uname -m)" -o /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose yum install -y perl perl-CGI perl-Template-Toolkit perl-CPAN perl-DBD-MySQL mod_perl perl-GD gcc-c++ # ruby and gollum for wiki diff --git a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh index 39f00667830..7c2413529e4 100755 --- a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh +++ b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh @@ -9,8 +9,8 @@ if [[ "$#" -ne 1 ]]; then exit 2 fi target_groups=$(aws elbv2 describe-target-groups) -LOCAL_IPV4=$(ssh root@"$1" "ec2-metadata --local-ipv4 | sed \"s/local-ipv4: *//\"") -INSTANCE_ID=$(ssh root@"$1" "ec2-metadata --instance-id | sed \"s/instance-id: *//\"") +LOCAL_IPV4=$(ssh -o StrictHostKeyChecking=false root@"$1" "ec2-metadata --local-ipv4 | sed \"s/local-ipv4: *//\"") +INSTANCE_ID=$(ssh -o StrictHostKeyChecking=false root@"$1" "ec2-metadata --instance-id | sed \"s/instance-id: *//\"") ELASTIC_IP="54.229.94.254" TAGS=("CentralReverseProxy" "ReverseProxy") TARGET_GROUP_TAGS=("allReverseProxies" "CentralReverseProxy") @@ -52,11 +52,11 @@ echo "Describing instances for remounting." describe_instances=$(aws ec2 describe-instances) for instanceIp in $(echo "$describe_instances" | select_instances_by_tag "sailing-analytics-server" | extract_public_ip); do echo "Remounting on $instanceIp" - ssh root@"${instanceIp}" " umount -l -f /home/scores; mount -a" + ssh -o StrictHostKeyChecking=false root@"${instanceIp}" " umount -l -f /home/scores; mount -a" done for instanceIp in $(echo "$describe_instances" | select_instances_by_tag "DisposableProxy" | extract_public_ip); do echo "Remounting on $instanceIp" - ssh root@"${instanceIp}" "umount -l -f /var/log/old; mount -a" + ssh -o StrictHostKeyChecking=false root@"${instanceIp}" "umount -l -f /var/log/old; mount -a" done # Alter the elastic IP. # WARNING: Will terminate connections via the existing public ip. From d51e2817dfcc031fdd2dc6fde2fd272a0e25177b Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Mon, 13 May 2024 16:13:27 +0200 Subject: [PATCH 007/134] b5912: initiate exit if build crontab fails. --- .../repo/usr/local/bin/imageupgrade_functions.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index 924c60533b8..60034179119 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -126,7 +126,9 @@ build_crontab_and_setup_files() { [[ "$?" -eq 0 ]] || scp -o StrictHostKeyChecking=no -pr "root@$HOSTNAME:/home/wiki/gitwiki/configuration/environments_scripts/*" "${TEMP_ENVIRONMENTS_SCRIPTS}" # For initial setup as not all landscape managers have direct wiki access. chown root:root "$TEMP_ENVIRONMENTS_SCRIPTS" cd "${TEMP_ENVIRONMENTS_SCRIPTS}" - ./build-crontab-and-cp-files "${PASS_OPTIONS[@]}" "$@" + if ! ./build-crontab-and-cp-files "${PASS_OPTIONS[@]}" "$@"; then + exit 1 + fi cd .. rm -rf "$TEMP_ENVIRONMENTS_SCRIPTS" fi From 9e3720b5ba63113da756aef1cc4f1a81b1044260 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Mon, 13 May 2024 16:14:30 +0200 Subject: [PATCH 008/134] b5912: add additional exits for incorrect args and if each block passed over ssh fails. --- .../central_reverse_proxy/setup-central-reverse-proxy.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh index cace6ab8ae9..62fe686e7e1 100755 --- a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh @@ -28,6 +28,7 @@ if [[ "$#" -ne 4 ]]; then echo "4 arguments required. Please check comment description for further details." echo "Example usage: setup-central-reverse-proxy.sh 1.2.3.4 0OcJ1938QE5it875kjlQe7HnzQ6740jsnMEVzowjZrs= 18.170.25.225 /home/sailing/code" + exit 2 fi IP=$1 BEARER_TOKEN=$2 @@ -90,6 +91,7 @@ scp -o StrictHostKeyChecking=no root@sapsailing.com:/usr/share/bugzilla/localco echo "Bugzilla has been copied. Now setting up bugzilla modules." echo "This can take 5 minutes or so. The output is muted to prevent excessive warnings and clutter." SECONDEOF +[[ "$?" -eq 0 ]] || exit 1 # ensure program doesn't continue if a major error occurs. ssh -A "root@${IP}" "bash -s" << BUGZILLAEOF &>/dev/null # essentials bugzilla /usr/bin/perl install-module.pl DateTime @@ -124,6 +126,7 @@ ssh -A "root@${IP}" "bash -s" << BUGZILLAEOF &>/dev/null /usr/bin/perl install-module.pl Authen::SASL scp -p root@sapsailing.com:/usr/share/bugzilla/data/params.json /usr/share/bugzilla/data/params.json BUGZILLAEOF +[[ "$?" -eq 0 ]] || exit 1 # ensure program doesn't continue if a major error occurs. read -n 1 -p "Bugzilla installation complete, when ready press a key to continue." key_pressed # use the localconfig file to setup the bugzilla # t forces tty allocation. @@ -183,7 +186,7 @@ UUID=ff598428-d380-4429-a690-3809157506b7 /var/log/old ext3 defaults,noatime,com UUID=d371e530-c189-4012-ae57-45d67a690554 /var/log/old/cache ext4 defaults,noatime,commit=30 0 0" >>/etc/fstab systemctl enable --now docker THIRDEOF - +[[ "$?" -eq 0 ]] || exit 1 # ensure program doesn't continue if a major error occurs. echo "" echo "" echo "Your turn! READ CAREFULLY! The instance is now prepared." From 067a47b475a9cb7d65251d3ac12e97e95bb8fecc Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Tue, 14 May 2024 00:00:36 +0200 Subject: [PATCH 009/134] b5912: Add timeout equal to DNS timeout to the "part 3" script which is run locally. --- .../target-group-tag-route53-nfs-elasticIP-setup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh index 7c2413529e4..8bd518a697d 100755 --- a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh +++ b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh @@ -48,6 +48,8 @@ sed -i "s/SMTP_INTERNAL_IP/$LOCAL_IPV4/" batch-for-route53-dns-record-update.jso read -n 1 -p "Check the instance has the correct tags and is in the correct target group. Furthermore, check the batch file has been modified correctly. Press a key to continue.." key_pressed ###### DO NOT ENABLE WHILST TESTING: aws route53 change-resource-record-sets --hosted-zone-id Z2JYWXYWLLRLTE --change-batch file://batch-for-route53-dns-record-update.json # reload the nfs mountpoints. +echo "Waiting 60 seconds for records to change" +sleep 60 echo "Describing instances for remounting." describe_instances=$(aws ec2 describe-instances) for instanceIp in $(echo "$describe_instances" | select_instances_by_tag "sailing-analytics-server" | extract_public_ip); do From 2aa3d041649288fdf114fe6e24fddf6e1f3bd012 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Tue, 14 May 2024 00:01:27 +0200 Subject: [PATCH 010/134] b5912: Copy ssh directory and perform sshd check before restarting the sshd. --- .../setup-central-reverse-proxy-part-2.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh index 894ebac7cbe..fd1afb3ebb0 100755 --- a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh @@ -28,12 +28,15 @@ systemctl enable nfs-server echo "/var/log/old 172.31.0.0/16(rw,nohide,no_root_squash) /home/scores 172.31.0.0/16(rw,nohide,no_root_squash)" >>/etc/exports systemctl start nfs-server -# scp -p -o StrictHostKeyChecking=no -r root@sapsailing.com:/etc/ssh /etc # causes some issue with MACs +scp -p -o StrictHostKeyChecking=no -r root@sapsailing.com:/etc/ssh /etc # append hostname to sysconfig echo "HOSTNAME=sapsailing.com" >> /etc/sysconfig/network sed -i "s/\(127.0.0.1 *\)/\1 sapsailing.com /" /etc/hosts hostname sapsailing.com hostnamectl set-hostname sapsailing.com +if sshd -t; then + systemctl restart sshd +fi EOF ssh -A -f root@"$IP" "cd /var/log/old/cache/docker/registry && nohup docker-compose up &>/dev/null &" &> /dev/null echo "Please now run the script target-group-tag-route53-nfs-elasticIP-setup.sh which configures the EC2 instance tags, adds to the " From 291a0e2b629fc9c2f3caf7368feda1e2a9bf28bb Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Tue, 14 May 2024 00:04:50 +0200 Subject: [PATCH 011/134] b5912: Alter when params.json is copied and log the bugzilla module install to a separate file. Params.json's parent directory is created by the checksetup script, so must be run afterwards to overwrite the setup. --- .../central_reverse_proxy/setup-central-reverse-proxy.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh index 62fe686e7e1..0180fff42fc 100755 --- a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh @@ -89,10 +89,10 @@ mv bugzilla-5.0.4 /usr/share/bugzilla cd /usr/share/bugzilla/ scp -o StrictHostKeyChecking=no root@sapsailing.com:/usr/share/bugzilla/localconfig . echo "Bugzilla has been copied. Now setting up bugzilla modules." -echo "This can take 5 minutes or so. The output is muted to prevent excessive warnings and clutter." +echo "This can take 5 minutes or so. The output is muted (but sent to log.txt) to prevent excessive warnings and clutter in the terminal." SECONDEOF -[[ "$?" -eq 0 ]] || exit 1 # ensure program doesn't continue if a major error occurs. -ssh -A "root@${IP}" "bash -s" << BUGZILLAEOF &>/dev/null +ssh -A "root@${IP}" "bash -s" << BUGZILLAEOF &>log.txt +cd /usr/share/bugzilla/ # essentials bugzilla /usr/bin/perl install-module.pl DateTime /usr/bin/perl install-module.pl DateTime::TimeZone @@ -124,7 +124,6 @@ ssh -A "root@${IP}" "bash -s" << BUGZILLAEOF &>/dev/null /usr/bin/perl install-module.pl File::Copy::Recursive /usr/bin/perl install-module.pl MIME::Base64 /usr/bin/perl install-module.pl Authen::SASL -scp -p root@sapsailing.com:/usr/share/bugzilla/data/params.json /usr/share/bugzilla/data/params.json BUGZILLAEOF [[ "$?" -eq 0 ]] || exit 1 # ensure program doesn't continue if a major error occurs. read -n 1 -p "Bugzilla installation complete, when ready press a key to continue." key_pressed @@ -134,6 +133,7 @@ ssh root@"${IP}" -A -t 'cd /usr/share/bugzilla/; ./checksetup.pl' ssh -A "root@${IP}" "cpan install Geo::IP" ssh -A "root@${IP}" "bash -s" << THIRDEOF . imageupgrade_functions.sh +scp -p root@sapsailing.com:/usr/share/bugzilla/data/params.json /usr/share/bugzilla/data/params.json echo $BEARER_TOKEN > /root/ssh-key-reader.token # awstats - depends on some of the previous perl modules. scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/usr/share/GeoIP /usr/share/GeoIP From 4328b0db9654e7d2a1daf652ce8311d14d5461bc Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Tue, 14 May 2024 00:05:55 +0200 Subject: [PATCH 012/134] b5912: Improve clarity of msg and add keyboard press to continue. --- .../target-group-tag-route53-nfs-elasticIP-setup.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh index 8bd518a697d..9cf87dc242c 100755 --- a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh +++ b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh @@ -48,8 +48,10 @@ sed -i "s/SMTP_INTERNAL_IP/$LOCAL_IPV4/" batch-for-route53-dns-record-update.jso read -n 1 -p "Check the instance has the correct tags and is in the correct target group. Furthermore, check the batch file has been modified correctly. Press a key to continue.." key_pressed ###### DO NOT ENABLE WHILST TESTING: aws route53 change-resource-record-sets --hosted-zone-id Z2JYWXYWLLRLTE --change-batch file://batch-for-route53-dns-record-update.json # reload the nfs mountpoints. -echo "Waiting 60 seconds for records to change" +echo "Waiting 60 seconds for records to change. The program will await a key press after this time." +echo "Please check the route53 DNS records have been correctly updated." sleep 60 +read -n 1 -p " Press a key to continue.." key_pressed echo "Describing instances for remounting." describe_instances=$(aws ec2 describe-instances) for instanceIp in $(echo "$describe_instances" | select_instances_by_tag "sailing-analytics-server" | extract_public_ip); do From 2316f97c25a57206abeaa09c34a8e6d206576024 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Tue, 14 May 2024 00:06:57 +0200 Subject: [PATCH 013/134] b5912: Copy icons and error documents across. --- .../central_reverse_proxy/setup-central-reverse-proxy.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh index 0180fff42fc..be75a4c0572 100755 --- a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh @@ -180,6 +180,9 @@ sudo systemctl enable postfix sudo systemctl restart postfix mkdir --parents /root/temporary_home_copy/home mv /home/* /root/temporary_home_copy/home +scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/var/www/icons /var/www +scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/var/www/errorpages /var/www +scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/var/www/error /var/www echo "UUID=23d42c52-85ee-4f6d-bdfe-c62f69bb689f /home ext4 defaults,noatime,commit=30 0 0 UUID=0b15f5cb-fd3e-48e6-8195-be248cd7726d /var/www/static ext3 defaults,noatime,commit=30 0 0 UUID=ff598428-d380-4429-a690-3809157506b7 /var/log/old ext3 defaults,noatime,commit=30 0 0 From 538ed6d45c2a2de6be750246d6265918fc62910d Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Tue, 14 May 2024 00:09:23 +0200 Subject: [PATCH 014/134] b5912: Add status check after the major SSH blocks, so that if a major error occurs, then the program does not continue. This increases the chances of diagnosing the issue and being able to continue with the setup, without starting from scratch. --- .../central_reverse_proxy/setup-central-reverse-proxy.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh index be75a4c0572..1959cb06391 100755 --- a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh @@ -25,6 +25,10 @@ # This will do all necessary set-up up to the point where the large volumes # currently attached to and mounted on the current Central Reverse Proxy will # need to be unmounted, detached, attached to the new instance, and mounted there. +terminationCheck() { + # $1 status returned by the last command. + [[ "$1" -eq 0 ]] || { echo "Non-zero exit code. Exiting to avoid damage" && exit 1; } +} if [[ "$#" -ne 4 ]]; then echo "4 arguments required. Please check comment description for further details." echo "Example usage: setup-central-reverse-proxy.sh 1.2.3.4 0OcJ1938QE5it875kjlQe7HnzQ6740jsnMEVzowjZrs= 18.170.25.225 /home/sailing/code" @@ -91,6 +95,7 @@ scp -o StrictHostKeyChecking=no root@sapsailing.com:/usr/share/bugzilla/localco echo "Bugzilla has been copied. Now setting up bugzilla modules." echo "This can take 5 minutes or so. The output is muted (but sent to log.txt) to prevent excessive warnings and clutter in the terminal." SECONDEOF +terminationCheck "$?" ssh -A "root@${IP}" "bash -s" << BUGZILLAEOF &>log.txt cd /usr/share/bugzilla/ # essentials bugzilla @@ -125,7 +130,7 @@ cd /usr/share/bugzilla/ /usr/bin/perl install-module.pl MIME::Base64 /usr/bin/perl install-module.pl Authen::SASL BUGZILLAEOF -[[ "$?" -eq 0 ]] || exit 1 # ensure program doesn't continue if a major error occurs. +terminationCheck "$?" read -n 1 -p "Bugzilla installation complete, when ready press a key to continue." key_pressed # use the localconfig file to setup the bugzilla # t forces tty allocation. @@ -189,7 +194,7 @@ UUID=ff598428-d380-4429-a690-3809157506b7 /var/log/old ext3 defaults,noatime,com UUID=d371e530-c189-4012-ae57-45d67a690554 /var/log/old/cache ext4 defaults,noatime,commit=30 0 0" >>/etc/fstab systemctl enable --now docker THIRDEOF -[[ "$?" -eq 0 ]] || exit 1 # ensure program doesn't continue if a major error occurs. +terminationCheck "$?" echo "" echo "" echo "Your turn! READ CAREFULLY! The instance is now prepared." From 080ea20e0773f5f78787bf0e6daff4bd171883a2 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Tue, 14 May 2024 00:09:55 +0200 Subject: [PATCH 015/134] b5912: Add insightful echo statement. --- .../repo/usr/local/bin/imageupgrade_functions.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index 60034179119..a2c39d5148d 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -346,6 +346,7 @@ setup_apachetop() { setup_swap() { # $1: size of swapspace in megabytes. + echo "Creating swapswpace of $1 MBs" local swapfile_location=/var/cache/swapfile pushd . sudo dd if=/dev/zero of="$swapfile_location" bs=1M count="$1" From 1e6810ea582435e795a73fab5b0ec8bdd5de16b7 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Tue, 14 May 2024 16:09:31 +0200 Subject: [PATCH 016/134] b5912: Exit setup if the build crontab stage fails. --- .../environments_scripts/build-crontab-and-cp-files | 2 +- .../setup-central-reverse-proxy-part-2.sh | 4 +++- .../central_reverse_proxy/setup-central-reverse-proxy.sh | 6 +++++- .../target-group-tag-route53-nfs-elasticIP-setup.sh | 2 +- .../repo/usr/local/bin/imageupgrade_functions.sh | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/configuration/environments_scripts/build-crontab-and-cp-files b/configuration/environments_scripts/build-crontab-and-cp-files index 92d3bd23053..0bf42a51ecd 100755 --- a/configuration/environments_scripts/build-crontab-and-cp-files +++ b/configuration/environments_scripts/build-crontab-and-cp-files @@ -35,7 +35,7 @@ do c) CREATE_CRONTABS="false";; f) COPY_FILES="false";; n) INSTALL_CRONTAB="false";; - \?) echo "Invalid option" + \?) echo "Invalid option" # getops replaces unknown options with a question mark, so we have to escape the question mark. exit 4;; esac done diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh index fd1afb3ebb0..0932ddbc422 100755 --- a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh @@ -21,7 +21,9 @@ sudo systemctl start crond.service . imageupgrade_functions.sh cp -r "$TEMPORARY_HOME_COPY_LOCATION"/home / rm -rf "$TEMPORARY_HOME_COPY_LOCATION" -build_crontab_and_setup_files -h localhost -f "${IMAGE_TYPE}" "${GIT_COPY_USER}" "${RELATIVE_PATH_TO_GIT}" # files have already been copied so -f is used. +if ! build_crontab_and_setup_files -h localhost -f "${IMAGE_TYPE}" "${GIT_COPY_USER}" "${RELATIVE_PATH_TO_GIT}"; then # files have already been copied so -f is used. + exit 1 +fi chown trac:static /var/www/static # setup nfs systemctl enable nfs-server diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh index 1959cb06391..041e055422e 100755 --- a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh @@ -27,6 +27,8 @@ # need to be unmounted, detached, attached to the new instance, and mounted there. terminationCheck() { # $1 status returned by the last command. + # Exits the program if the passed argument is non zero. + # Used to ensure the program doesn't continue if a major error occurs. [[ "$1" -eq 0 ]] || { echo "Non-zero exit code. Exiting to avoid damage" && exit 1; } } if [[ "$#" -ne 4 ]]; then @@ -70,7 +72,9 @@ scp -o StrictHostKeyChecking=no -p root@"$IMAGEUPGRADE_FUNCTIONS_IP":"$IMAGEUPGR . imageupgrade_functions.sh setup_cloud_cfg_and_root_login # setup files -build_crontab_and_setup_files -c -n "${IMAGE_TYPE}" "${GIT_COPY_USER}" "${RELATIVE_PATH_TO_GIT}" # -c & -n mean only files are copied over. +if ! build_crontab_and_setup_files -c -n "${IMAGE_TYPE}" "${GIT_COPY_USER}" "${RELATIVE_PATH_TO_GIT}"; then # -c & -n mean only files are copied over. + exit 1 +fi setup_swap 5000 cd /home for folder in * ; do diff --git a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh index 9cf87dc242c..265d178c535 100755 --- a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh +++ b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh @@ -51,7 +51,7 @@ read -n 1 -p "Check the instance has the correct tags and is in the correct tar echo "Waiting 60 seconds for records to change. The program will await a key press after this time." echo "Please check the route53 DNS records have been correctly updated." sleep 60 -read -n 1 -p " Press a key to continue.." key_pressed +read -n 1 -p "Press a key to continue.." key_pressed echo "Describing instances for remounting." describe_instances=$(aws ec2 describe-instances) for instanceIp in $(echo "$describe_instances" | select_instances_by_tag "sailing-analytics-server" | extract_public_ip); do diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index a2c39d5148d..b930982d309 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -127,7 +127,7 @@ build_crontab_and_setup_files() { chown root:root "$TEMP_ENVIRONMENTS_SCRIPTS" cd "${TEMP_ENVIRONMENTS_SCRIPTS}" if ! ./build-crontab-and-cp-files "${PASS_OPTIONS[@]}" "$@"; then - exit 1 + return 1 fi cd .. rm -rf "$TEMP_ENVIRONMENTS_SCRIPTS" From c3a8ab54c188f1caa9ffcdd6483f60a3bb6136e8 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Tue, 14 May 2024 16:13:06 +0200 Subject: [PATCH 017/134] b5912: Only copy keys if they don't already exist, so we never ireparably overwrite existing keys. --- .../repo/usr/local/bin/imageupgrade_functions.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index b930982d309..222d24fc365 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -182,6 +182,7 @@ setup_keys() { chmod 700 "${user_home_dir}/.ssh" for key in "${user}"/ssh/*; do [[ -f "$key" ]] || continue + [[ ! -f "$user_home_dir"/.ssh/"$(basename "$key")" ]] || continue \cp --preserve --dereference "$key" "$user_home_dir"/.ssh done for key in "${user}"/ssh/authorized_keys/*; do From 1fad02d65f4f38c026235deb0fa9b5f1821d65ed Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Tue, 14 May 2024 17:21:44 +0200 Subject: [PATCH 018/134] b5912: Add optional permission/ownership settings to setup_keys. --- .../usr/local/bin/imageupgrade_functions.sh | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index 222d24fc365..2f52424dfc5 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -136,6 +136,31 @@ build_crontab_and_setup_files() { setup_keys() { #1: Environment type. + # Optional parameter is -p which indicates that no permissions will be set or overwritten. + TEMP=$(getopt -o p -n 'options' -- "$@") + [[ "$?" -eq 0 ]] || return 2 + SET_PERMISSIONS="true" + eval set -- "$TEMP" + while true; do + case "$1" in + -p) + SET_PERMISSIONS="false" + ;; + --) + shift + break + ;; + *) + echo "Option not recognised" + return 2 + ;; + esac + shift + done + if [[ "$#" -ne 1 ]]; then + echo "Please specify the environment type and use the optional -p flag to indicate that no permissions will be set or overwritten." + return 2 + fi pushd . TEMP_KEY_DIR=$(mktemp -d /root/keysXXXXX) REGION=$(TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" --silent -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"` \ @@ -149,7 +174,6 @@ setup_keys() { # aws setup if [[ -d "${user}/aws" ]]; then mkdir --parents "${user_home_dir}/.aws" - chmod 755 "${user_home_dir}"/.aws # Setup credentials if [[ -d "${user}/aws/credentials" && ! -e "${user_home_dir}/.aws/credentials" ]]; then > "${user_home_dir}"/.aws/credentials @@ -173,13 +197,15 @@ setup_keys() { done fi fi - chown -R ${user}:${user} "${user_home_dir}/.aws" - chmod 600 "${user_home_dir}"/.aws/* + if [[ "$SET_PERMISSIONS" == "true" ]]; then + chmod 755 "${user_home_dir}"/.aws + chown -R ${user}:${user} "${user_home_dir}/.aws" + chmod 600 "${user_home_dir}"/.aws/* + fi fi # ssh setup if [[ -d "${user}/ssh" ]]; then mkdir --parents "${user_home_dir}/.ssh" - chmod 700 "${user_home_dir}/.ssh" for key in "${user}"/ssh/*; do [[ -f "$key" ]] || continue [[ ! -f "$user_home_dir"/.ssh/"$(basename "$key")" ]] || continue @@ -189,8 +215,11 @@ setup_keys() { [[ -f "$key" ]] || continue cat "${key}" >> ${user_home_dir}/.ssh/authorized_keys done - chown -R ${user}:${user} "${user_home_dir}/.ssh" - chmod 600 "${user_home_dir}"/.ssh/* + if [[ "$SET_PERMISSIONS" == "true" ]]; then + chmod 700 "${user_home_dir}/.ssh" + chown -R ${user}:${user} "${user_home_dir}/.ssh" + chmod 600 "${user_home_dir}"/.ssh/* + fi fi fi done From 445ae472dba03f242315137865640fcbebb6dc6b Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Tue, 14 May 2024 17:22:02 +0200 Subject: [PATCH 019/134] b5912: Return instead of exit. --- .../repo/usr/local/bin/imageupgrade_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index 2f52424dfc5..419c01180f9 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -93,7 +93,7 @@ build_crontab_and_setup_files() { echo " RELATIVE_PATH_OF_GIT_DIR_WITHIN_USER - the relative path within the USER_WITH_COPY_OF_REPO to get to the git workspace." else TEMP=$(getopt -o fnch: -n 'options checker' -- "$@") - [[ "$?" -eq 0 ]] || exit 2 + [[ "$?" -eq 0 ]] || return 2 eval set -- "$TEMP" PASS_OPTIONS=() HOSTNAME="sapsailing.com" From b5d700159bb445a14dff54bdc8bb0aa2b69a389f Mon Sep 17 00:00:00 2001 From: Thomas S Date: Thu, 16 May 2024 13:35:37 +0000 Subject: [PATCH 020/134] b5912: Make build_crontab_.... in imageupgrade_functions.sh sudo capable --- .../usr/local/bin/imageupgrade_functions.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index 419c01180f9..48e95929660 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -120,15 +120,18 @@ build_crontab_and_setup_files() { esac shift done - TEMP_ENVIRONMENTS_SCRIPTS=$(mktemp -d /root/environments_scripts_XXX) + TEMP_ENVIRONMENTS_SCRIPTS=$(mktemp -d /var/tmp/environments_scripts_XXX) echo "Attempting access to the wiki, typically used by image upgrade. Otherwise, try root. NOTE: If the first command fails, there will be a warning message." - scp -o StrictHostKeyChecking=no -pr "wiki@$HOSTNAME:~/gitwiki/configuration/environments_scripts/*" "${TEMP_ENVIRONMENTS_SCRIPTS}" - [[ "$?" -eq 0 ]] || scp -o StrictHostKeyChecking=no -pr "root@$HOSTNAME:/home/wiki/gitwiki/configuration/environments_scripts/*" "${TEMP_ENVIRONMENTS_SCRIPTS}" # For initial setup as not all landscape managers have direct wiki access. - chown root:root "$TEMP_ENVIRONMENTS_SCRIPTS" + scp -o StrictHostKeyChecking=no -pr wiki@"$HOSTNAME":~/gitwiki/configuration/environments_scripts/* "${TEMP_ENVIRONMENTS_SCRIPTS}" + [[ "$?" -eq 0 ]] || scp -o StrictHostKeyChecking=no -pr root@"$HOSTNAME":/home/wiki/gitwiki/configuration/environments_scripts/* "${TEMP_ENVIRONMENTS_SCRIPTS}" # For initial setup as not all landscape managers have direct wiki access. cd "${TEMP_ENVIRONMENTS_SCRIPTS}" - if ! ./build-crontab-and-cp-files "${PASS_OPTIONS[@]}" "$@"; then - return 1 - fi + # Add all args to array, otherwise, if PASS_OPTIONS is empty, and we also pass $@ then argument $1 is in fact null, which would cause errors. + for option in "$@"; do + PASS_OPTIONS+=( "$option") + done + if ! sudo ./build-crontab-and-cp-files "${PASS_OPTIONS[@]}"; then + return 1 + fi cd .. rm -rf "$TEMP_ENVIRONMENTS_SCRIPTS" fi @@ -386,4 +389,4 @@ setup_swap() { sudo su - -c "echo \"$swapfile_location none swap pri=0 0 0\" >> /etc/fstab" sudo swapon -a popd -} \ No newline at end of file +} From ae01d0a918252f1acfa571700db7fc8687075fc2 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Thu, 16 May 2024 19:22:57 +0200 Subject: [PATCH 021/134] b5912: Fix formatting and move setup keys to part 2, making use of the new -p option to disable permission setting. This is vital for the trac user. --- .../setup-central-reverse-proxy-part-2.sh | 1 + .../setup-central-reverse-proxy.sh | 1 - .../repo/usr/local/bin/imageupgrade_functions.sh | 14 +++++++------- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh index 0932ddbc422..5c1dd25b92f 100755 --- a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh @@ -24,6 +24,7 @@ rm -rf "$TEMPORARY_HOME_COPY_LOCATION" if ! build_crontab_and_setup_files -h localhost -f "${IMAGE_TYPE}" "${GIT_COPY_USER}" "${RELATIVE_PATH_TO_GIT}"; then # files have already been copied so -f is used. exit 1 fi +setup_keys -p "${IMAGE_TYPE}" chown trac:static /var/www/static # setup nfs systemctl enable nfs-server diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh index 041e055422e..542266c6d83 100755 --- a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh @@ -165,7 +165,6 @@ EOF echo "net.ipv4.ip_conntrac_max = 131072" >> /etc/sysctl.conf # setup fail2ban setup_fail2ban -setup_keys "${IMAGE_TYPE}" # setup logrotate.d/httpd # echo "Patching $HTTP_LOGROTATE_ABSOLUTE so that old logs go to /var/log/old/$IP" >>/var/log/sailing.out # mkdir --parents "/var/log/old/REVERSE_PROXIES/${IP}" diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index 48e95929660..a0c2aa38362 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -125,13 +125,13 @@ build_crontab_and_setup_files() { scp -o StrictHostKeyChecking=no -pr wiki@"$HOSTNAME":~/gitwiki/configuration/environments_scripts/* "${TEMP_ENVIRONMENTS_SCRIPTS}" [[ "$?" -eq 0 ]] || scp -o StrictHostKeyChecking=no -pr root@"$HOSTNAME":/home/wiki/gitwiki/configuration/environments_scripts/* "${TEMP_ENVIRONMENTS_SCRIPTS}" # For initial setup as not all landscape managers have direct wiki access. cd "${TEMP_ENVIRONMENTS_SCRIPTS}" - # Add all args to array, otherwise, if PASS_OPTIONS is empty, and we also pass $@ then argument $1 is in fact null, which would cause errors. - for option in "$@"; do - PASS_OPTIONS+=( "$option") - done - if ! sudo ./build-crontab-and-cp-files "${PASS_OPTIONS[@]}"; then - return 1 - fi + # Add all args to array, otherwise, if PASS_OPTIONS is empty, and we also pass $@ then argument $1 is in fact null, which would cause errors. + for option in "$@"; do + PASS_OPTIONS+=( "$option") + done + if ! sudo ./build-crontab-and-cp-files "${PASS_OPTIONS[@]}"; then + return 1 + fi cd .. rm -rf "$TEMP_ENVIRONMENTS_SCRIPTS" fi From f612608eda625c897f45e5e1bcfdc40001275c86 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Fri, 17 May 2024 10:56:50 +0200 Subject: [PATCH 022/134] b5938: only append key if it doesn't already exist in the authorized keys. --- .../repo/usr/local/bin/imageupgrade_functions.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index a0c2aa38362..9ad5bc0d55e 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -216,7 +216,9 @@ setup_keys() { done for key in "${user}"/ssh/authorized_keys/*; do [[ -f "$key" ]] || continue - cat "${key}" >> ${user_home_dir}/.ssh/authorized_keys + if ! grep -q "$(cat "$key")" "${user_home_dir}"/.ssh/authorized_keys; then + cat "${key}" >> "${user_home_dir}"/.ssh/authorized_keys + fi done if [[ "$SET_PERMISSIONS" == "true" ]]; then chmod 700 "${user_home_dir}/.ssh" From ffdf0f7f2f8d5523482fed3ca4bcfe44c1bc4226 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Sun, 19 May 2024 22:52:54 +0200 Subject: [PATCH 023/134] b5912: Copy awstats database across. --- .../central_reverse_proxy/setup-central-reverse-proxy.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh index 542266c6d83..937f62d5191 100755 --- a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh @@ -151,6 +151,7 @@ wget http://prdownloads.sourceforge.net/awstats/awstats-7.0.tar.gz tar -zvxf awstats-7.0.tar.gz mv awstats-7.0/ /usr/share/awstats mkdir /var/lib/awstats +scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/var/lib/awstats /var/lib scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/etc/awstats /etc/awstats chmod 755 /root cd ~ From e133c146c38c32ca35fb4f3eca501d1fd21e006f Mon Sep 17 00:00:00 2001 From: Thomas S Date: Thu, 23 May 2024 21:02:21 +0000 Subject: [PATCH 024/134] b5912: Formatting --- .../repo/usr/local/bin/imageupgrade_functions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index 55eb71a90f1..587a6f84420 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -127,10 +127,10 @@ build_crontab_and_setup_files() { cd "${TEMP_ENVIRONMENTS_SCRIPTS}" # Add all args to array, otherwise, if PASS_OPTIONS is empty, and we also pass $@ then argument $1 is in fact null, which would cause errors. for option in "$@"; do - PASS_OPTIONS+=( "$option") + PASS_OPTIONS+=( "$option") done if ! sudo ./build-crontab-and-cp-files "${PASS_OPTIONS[@]}"; then - return 1 + return 1 fi cd .. rm -rf "$TEMP_ENVIRONMENTS_SCRIPTS" From ac8327d860000e086be02713ffd63798f0f5cb4e Mon Sep 17 00:00:00 2001 From: Thomas S Date: Thu, 23 May 2024 21:05:21 +0000 Subject: [PATCH 025/134] b5912: Use var/tmp instead of /root so any user can run. In reality they must have sudo permissions for the other parts of this function. --- .../repo/usr/local/bin/imageupgrade_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index 587a6f84420..5e60b74a0ad 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -165,7 +165,7 @@ setup_keys() { return 2 fi pushd . - TEMP_KEY_DIR=$(mktemp -d /root/keysXXXXX) + TEMP_KEY_DIR=$(mktemp -d /var/tmp/keysXXXXX) REGION=$(TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" --silent -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"` \ && curl -H "X-aws-ec2-metadata-token: $TOKEN" --silent http://169.254.169.254/latest/meta-data/placement/region) scp -o StrictHostKeyChecking=no -pr root@sapsailing.com:/root/new_version_key_vault/"${1}"/* "${TEMP_KEY_DIR}" From 05747765c241a896880d4e438bcd7e70244aea5b Mon Sep 17 00:00:00 2001 From: Thomas S Date: Thu, 23 May 2024 21:17:40 +0000 Subject: [PATCH 026/134] b5912: Rework setup_keys to work with any sudo user --- .../repo/usr/local/bin/imageupgrade_functions.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index 5e60b74a0ad..915f60899c1 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -166,9 +166,19 @@ setup_keys() { fi pushd . TEMP_KEY_DIR=$(mktemp -d /var/tmp/keysXXXXX) + scp -o StrictHostKeyChecking=no -pr root@sapsailing.com:/root/new_version_key_vault/"${1}"/* "${TEMP_KEY_DIR}" + sudo su - -c "source imageupgrade_functions.sh; __setup_keys_using_local_copy $TEMP_KEY_DIR $SET_PERMISSIONS" + popd + rm -rf "${TEMP_KEY_DIR}" +} +__setup_keys_using_local_copy() { + # $1 the local location of the key_vault subdirectory corresponding to the image type this is run on. + # $2 a "true" or "false" string, indicating whether to override existing permissions. + # "true" indicates the permissions and ownership of the .ssh and .aws folders will not be set. + TEMP_KEY_DIR="$1" + SET_PERMISSIONS="$2" REGION=$(TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" --silent -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"` \ && curl -H "X-aws-ec2-metadata-token: $TOKEN" --silent http://169.254.169.254/latest/meta-data/placement/region) - scp -o StrictHostKeyChecking=no -pr root@sapsailing.com:/root/new_version_key_vault/"${1}"/* "${TEMP_KEY_DIR}" cd "${TEMP_KEY_DIR}" for user in *; do [[ -e "$user" ]] || continue @@ -228,8 +238,6 @@ setup_keys() { fi fi done - popd - rm -rf "${TEMP_KEY_DIR}" } clean_root_ssh_dir_and_tmp() { From ec950a44981623df2086c7d3bcb26c9c05e87994 Mon Sep 17 00:00:00 2001 From: Thomas S Date: Thu, 23 May 2024 21:20:51 +0000 Subject: [PATCH 027/134] b5912: Add echo statement when key already exists --- .../repo/usr/local/bin/imageupgrade_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index 915f60899c1..ff65b66c497 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -221,7 +221,7 @@ __setup_keys_using_local_copy() { mkdir --parents "${user_home_dir}/.ssh" for key in "${user}"/ssh/*; do [[ -f "$key" ]] || continue - [[ ! -f "$user_home_dir"/.ssh/"$(basename "$key")" ]] || continue + [[ ! -f "$user_home_dir"/.ssh/"$(basename "$key")" ]] || { echo "$key not touched, as it is already present in ssh dir of $user. You may want to override it." && continue; } \cp --preserve --dereference "$key" "$user_home_dir"/.ssh done for key in "${user}"/ssh/authorized_keys/*; do From acbe4ad76d3ae419773b38cd39ebb9293863d8ab Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sat, 25 May 2024 00:19:33 +0200 Subject: [PATCH 028/134] fixed syntax/indentation problem in docker-compose-17.yml --- docker/docker-compose-17.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose-17.yml b/docker/docker-compose-17.yml index 3661e89b978..6de89609bf9 100644 --- a/docker/docker-compose-17.yml +++ b/docker/docker-compose-17.yml @@ -1,7 +1,7 @@ version: "3.10" services: sailing-analytics: - image: "ghcr.io/sap/sailing-analytics:latest-17" + image: "ghcr.io/sap/sailing-analytics:latest-17" ports: - "8888:8888" - "6666:6666" From 54194673a74dd4ed417b6c74a31e1ee15b8d97ef Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sat, 25 May 2024 00:51:11 +0200 Subject: [PATCH 029/134] testing workflow to fetch Git from sapsailing.com --- .github/workflows/test-fetch-sapsailing.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/test-fetch-sapsailing.yml diff --git a/.github/workflows/test-fetch-sapsailing.yml b/.github/workflows/test-fetch-sapsailing.yml new file mode 100644 index 00000000000..dc21ee39f06 --- /dev/null +++ b/.github/workflows/test-fetch-sapsailing.yml @@ -0,0 +1,15 @@ +name: Test Git Fetch from sapsailing.com +on: + workflow_dispatch: {} +jobs: + fetch-git-from-sapsailing-com: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Create SSH key and fetch + shell: bash + run: | + echo "${{ secrets.SSH_PRIVATE_KEY_FOR_TRAC_SAPSAILING_COM }}" >~/.ssh/id_ed25519 + git remote add sapsailing ssh://trac@sapsailing.com/home/trac/git + git fetch sapsailing From 2b8d0bc16e492e457c6e7e13d087cf4ea6664529 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sat, 25 May 2024 00:54:19 +0200 Subject: [PATCH 030/134] output git remotes while testing --- .github/workflows/test-fetch-sapsailing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-fetch-sapsailing.yml b/.github/workflows/test-fetch-sapsailing.yml index dc21ee39f06..fbdb5a63211 100644 --- a/.github/workflows/test-fetch-sapsailing.yml +++ b/.github/workflows/test-fetch-sapsailing.yml @@ -12,4 +12,5 @@ jobs: run: | echo "${{ secrets.SSH_PRIVATE_KEY_FOR_TRAC_SAPSAILING_COM }}" >~/.ssh/id_ed25519 git remote add sapsailing ssh://trac@sapsailing.com/home/trac/git + git remote -v git fetch sapsailing From 409921b43a1401becda99b4c36a4c0d4cf90f2e2 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sat, 25 May 2024 00:57:34 +0200 Subject: [PATCH 031/134] fix file access modes for SSH key used in runner --- .github/workflows/test-fetch-sapsailing.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-fetch-sapsailing.yml b/.github/workflows/test-fetch-sapsailing.yml index fbdb5a63211..a70bef470ae 100644 --- a/.github/workflows/test-fetch-sapsailing.yml +++ b/.github/workflows/test-fetch-sapsailing.yml @@ -10,7 +10,10 @@ jobs: - name: Create SSH key and fetch shell: bash run: | + mkdir -p ~/.ssh + chmod 700 ~/.ssh echo "${{ secrets.SSH_PRIVATE_KEY_FOR_TRAC_SAPSAILING_COM }}" >~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 git remote add sapsailing ssh://trac@sapsailing.com/home/trac/git git remote -v git fetch sapsailing From e82abee47a811168bc3350a24207543dc2f2d3f1 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sat, 25 May 2024 01:01:26 +0200 Subject: [PATCH 032/134] disable host key checking for fetching from sapsailing.com --- .github/workflows/test-fetch-sapsailing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-fetch-sapsailing.yml b/.github/workflows/test-fetch-sapsailing.yml index a70bef470ae..b891c6023ca 100644 --- a/.github/workflows/test-fetch-sapsailing.yml +++ b/.github/workflows/test-fetch-sapsailing.yml @@ -16,4 +16,4 @@ jobs: chmod 600 ~/.ssh/id_ed25519 git remote add sapsailing ssh://trac@sapsailing.com/home/trac/git git remote -v - git fetch sapsailing + GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git fetch sapsailing From 10196beccd01e4f9f37440bed7ee0bc2f372f746 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sat, 25 May 2024 01:12:04 +0200 Subject: [PATCH 033/134] test merging master from sapsailing.com into github main every 5min --- ...tch-sapsailing.yml => merge-sapsailing-com-master.yml} | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) rename .github/workflows/{test-fetch-sapsailing.yml => merge-sapsailing-com-master.yml} (75%) diff --git a/.github/workflows/test-fetch-sapsailing.yml b/.github/workflows/merge-sapsailing-com-master.yml similarity index 75% rename from .github/workflows/test-fetch-sapsailing.yml rename to .github/workflows/merge-sapsailing-com-master.yml index b891c6023ca..acfe7fbfb79 100644 --- a/.github/workflows/test-fetch-sapsailing.yml +++ b/.github/workflows/merge-sapsailing-com-master.yml @@ -1,8 +1,10 @@ -name: Test Git Fetch from sapsailing.com +name: Merge master from sapsailing.com on: + schedule: + - cron: "*/5 * * * *" # every five minutes workflow_dispatch: {} jobs: - fetch-git-from-sapsailing-com: + merge-master-from-sapsailing-com: runs-on: ubuntu-latest steps: - name: Checkout @@ -17,3 +19,5 @@ jobs: git remote add sapsailing ssh://trac@sapsailing.com/home/trac/git git remote -v GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git fetch sapsailing + git merge sapsailing.com/master + git push From 55c27ddaa69930e7c486a57467a8675b8e198149 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sat, 25 May 2024 01:14:15 +0200 Subject: [PATCH 034/134] renamed create-docker-image workflow to be more human-readable --- .github/workflows/create-docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-docker-image.yml b/.github/workflows/create-docker-image.yml index 3a9085c6239..4ade5237ca5 100644 --- a/.github/workflows/create-docker-image.yml +++ b/.github/workflows/create-docker-image.yml @@ -1,4 +1,4 @@ -name: create-docker-image +name: Create Docker Image on: workflow_run: workflows: [release] From 3dadecdd4af6fa4f071a67067ca3fb04db8bb047 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Fri, 24 May 2024 23:22:48 +0000 Subject: [PATCH 035/134] Updated the number of SAP customers in the Wiki home --- Home.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Home.md b/Home.md index 0dc4836e2a8..46a80d79bd1 100644 --- a/Home.md +++ b/Home.md @@ -6,7 +6,7 @@ This is the Wik Like businesses, sailors need the latest information to make strategic decisions - but they need it even faster. One wrong tack, a false estimation of the current, or the slightest wind shift can cost the skipper the entire race. As premium sponsor of the Kieler Woche 2011, and co-sponsor of Sailing Team Germany (STG), SAP is showing how innovative IT solutions providing real time data analysis can give teams the competitive edge. -SAP is at the center of today’s technology revolution, developing innovations that not only help businesses run like never before, but also improve the lives of people everywhere. As market leader in enterprise application software, SAP SE (NYSE: SAP) helps companies of all sizes and industries run better. From back office to boardroom, warehouse to storefront, desktop to mobile device – SAP empowers people and organizations to work together more efficiently and use business insight more effectively to stay ahead of the competition. SAP applications and services enable more than 258,000 customers to operate profitably, adapt continuously, and grow sustainably. +SAP is at the center of today’s technology revolution, developing innovations that not only help businesses run like never before, but also improve the lives of people everywhere. As market leader in enterprise application software, SAP SE (NYSE: SAP) helps companies of all sizes and industries run better. From back office to boardroom, warehouse to storefront, desktop to mobile device – SAP empowers people and organizations to work together more efficiently and use business insight more effectively to stay ahead of the competition. SAP applications and services enable more than 400****,000 customers to operate profitably, adapt continuously, and grow sustainably. # Table of Contents From b8fee19d6f7237b2908b1ed0767063d33fc3db32 Mon Sep 17 00:00:00 2001 From: Generic Wiki User Date: Fri, 24 May 2024 23:23:48 +0000 Subject: [PATCH 036/134] adjusted the number of SAP customers to 400,000 --- Home.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Home.md b/Home.md index 46a80d79bd1..3d2498e3971 100644 --- a/Home.md +++ b/Home.md @@ -6,7 +6,7 @@ This is the Wik Like businesses, sailors need the latest information to make strategic decisions - but they need it even faster. One wrong tack, a false estimation of the current, or the slightest wind shift can cost the skipper the entire race. As premium sponsor of the Kieler Woche 2011, and co-sponsor of Sailing Team Germany (STG), SAP is showing how innovative IT solutions providing real time data analysis can give teams the competitive edge. -SAP is at the center of today’s technology revolution, developing innovations that not only help businesses run like never before, but also improve the lives of people everywhere. As market leader in enterprise application software, SAP SE (NYSE: SAP) helps companies of all sizes and industries run better. From back office to boardroom, warehouse to storefront, desktop to mobile device – SAP empowers people and organizations to work together more efficiently and use business insight more effectively to stay ahead of the competition. SAP applications and services enable more than 400****,000 customers to operate profitably, adapt continuously, and grow sustainably. +SAP is at the center of today’s technology revolution, developing innovations that not only help businesses run like never before, but also improve the lives of people everywhere. As market leader in enterprise application software, SAP SE (NYSE: SAP) helps companies of all sizes and industries run better. From back office to boardroom, warehouse to storefront, desktop to mobile device – SAP empowers people and organizations to work together more efficiently and use business insight more effectively to stay ahead of the competition. SAP applications and services enable more than 400,000 customers to operate profitably, adapt continuously, and grow sustainably. # Table of Contents From a35e7d35aae2dadcf35d2a3bd0bf313eb2e37648 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sat, 25 May 2024 01:25:44 +0200 Subject: [PATCH 037/134] fixed typo in merge-sapsailing-com-master workflow --- .github/workflows/merge-sapsailing-com-master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-sapsailing-com-master.yml b/.github/workflows/merge-sapsailing-com-master.yml index acfe7fbfb79..b30c3709314 100644 --- a/.github/workflows/merge-sapsailing-com-master.yml +++ b/.github/workflows/merge-sapsailing-com-master.yml @@ -19,5 +19,5 @@ jobs: git remote add sapsailing ssh://trac@sapsailing.com/home/trac/git git remote -v GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git fetch sapsailing - git merge sapsailing.com/master + git merge sapsailing/master git push From 42f24a7ba0c3a580ba573504d77e1a0afa6519c4 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sat, 25 May 2024 01:39:36 +0200 Subject: [PATCH 038/134] also fetch full origin history before trying to merge sapsailing/master --- .github/workflows/merge-sapsailing-com-master.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/merge-sapsailing-com-master.yml b/.github/workflows/merge-sapsailing-com-master.yml index b30c3709314..1f898e1ccc8 100644 --- a/.github/workflows/merge-sapsailing-com-master.yml +++ b/.github/workflows/merge-sapsailing-com-master.yml @@ -19,5 +19,6 @@ jobs: git remote add sapsailing ssh://trac@sapsailing.com/home/trac/git git remote -v GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git fetch sapsailing + git fetch origin git merge sapsailing/master git push From c6002b8a6b3723ab50b324b54d97f0a2c640ca89 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sat, 25 May 2024 01:44:49 +0200 Subject: [PATCH 039/134] fetch full history before trying to merge sapsailing/master --- .github/workflows/merge-sapsailing-com-master.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/merge-sapsailing-com-master.yml b/.github/workflows/merge-sapsailing-com-master.yml index 1f898e1ccc8..2fd1017cbae 100644 --- a/.github/workflows/merge-sapsailing-com-master.yml +++ b/.github/workflows/merge-sapsailing-com-master.yml @@ -9,6 +9,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + fetch-depth: 0 # fetch the whole thing to make sure the histories merge - name: Create SSH key and fetch shell: bash run: | @@ -19,6 +21,5 @@ jobs: git remote add sapsailing ssh://trac@sapsailing.com/home/trac/git git remote -v GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git fetch sapsailing - git fetch origin git merge sapsailing/master git push From 8eecba69147a73efee3f56b52cdbe31fb27aa386 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sat, 25 May 2024 01:46:18 +0200 Subject: [PATCH 040/134] updated step name in merge-sapsailing-com-master workflow --- .github/workflows/merge-sapsailing-com-master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-sapsailing-com-master.yml b/.github/workflows/merge-sapsailing-com-master.yml index 2fd1017cbae..37e0ff07547 100644 --- a/.github/workflows/merge-sapsailing-com-master.yml +++ b/.github/workflows/merge-sapsailing-com-master.yml @@ -11,7 +11,7 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 # fetch the whole thing to make sure the histories merge - - name: Create SSH key and fetch + - name: Create SSH key and fetch repo from sapsailing.com shell: bash run: | mkdir -p ~/.ssh From 2cbe21a909362e62840629f30fa1b40019a37c02 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sat, 25 May 2024 01:50:38 +0200 Subject: [PATCH 041/134] define user.[email|name] for merging commit --- .github/workflows/merge-sapsailing-com-master.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/merge-sapsailing-com-master.yml b/.github/workflows/merge-sapsailing-com-master.yml index 37e0ff07547..dcee95b3881 100644 --- a/.github/workflows/merge-sapsailing-com-master.yml +++ b/.github/workflows/merge-sapsailing-com-master.yml @@ -21,5 +21,7 @@ jobs: git remote add sapsailing ssh://trac@sapsailing.com/home/trac/git git remote -v GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git fetch sapsailing + git config --global user.email "github@sapsailing.com" + git config --global user.name "Github Auto-Merger" git merge sapsailing/master git push From 1391c53f2d96f5b320b5158d49c7a088f501000c Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sat, 25 May 2024 02:05:48 +0200 Subject: [PATCH 042/134] added a comment in a Github workflow --- .github/workflows/merge-sapsailing-com-master.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/merge-sapsailing-com-master.yml b/.github/workflows/merge-sapsailing-com-master.yml index dcee95b3881..a3fbddb1b50 100644 --- a/.github/workflows/merge-sapsailing-com-master.yml +++ b/.github/workflows/merge-sapsailing-com-master.yml @@ -1,3 +1,7 @@ +# One of the reasons to have this workflow is that the Gollum-based Wiki +# at wiki.sapsailing.com is operating on a Git workspace at wiki@sapsailing.com:gitwiki +# which has been cloned from ssh://trac@sapsailing.com/home/trac/git. Its changes +# shall be synchronized to the Github repo at github.com/SAP/sailing-analytics name: Merge master from sapsailing.com on: schedule: From 0b9237684ed42456dbdca7d29a82dcc626348d02 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sat, 25 May 2024 02:08:19 +0200 Subject: [PATCH 043/134] reduced auto-merge frequency for sapsailing.com:master to once an hour --- .github/workflows/merge-sapsailing-com-master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-sapsailing-com-master.yml b/.github/workflows/merge-sapsailing-com-master.yml index a3fbddb1b50..3cd5c2b4517 100644 --- a/.github/workflows/merge-sapsailing-com-master.yml +++ b/.github/workflows/merge-sapsailing-com-master.yml @@ -5,7 +5,7 @@ name: Merge master from sapsailing.com on: schedule: - - cron: "*/5 * * * *" # every five minutes + - cron: "53 * * * *" # every hour, at :53 minutes workflow_dispatch: {} jobs: merge-master-from-sapsailing-com: From 7167fccd76561cb8d8c0e342c8c404e9d81da710 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Sat, 25 May 2024 09:36:12 +0000 Subject: [PATCH 044/134] Add test --- Home.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Home.md b/Home.md index 3d2498e3971..b82a2b9db90 100644 --- a/Home.md +++ b/Home.md @@ -179,3 +179,5 @@ SAP is at the center of today’s technology revolution, developing innovations * [GIT Repository (SAP)](ssh://git.wdf.sap.corp:29418/SAPSail/sapsailingcapture.git) * [Maven Repository Browser](https://maven.sapsailing.com/maven/) (see [[how to setup repository for Android builds|wiki/info/mobile/racecommittee-app-environment]]) * [Main Sailing Website](https://www.sapsailing.com) + +test From 0cc44d22a9ba7f3768b4c78fee0142f0616a3973 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Sat, 25 May 2024 09:44:13 +0000 Subject: [PATCH 045/134] Fix formatting --- wiki/howto/onboarding.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki/howto/onboarding.md b/wiki/howto/onboarding.md index ecad9682510..a56207d4eec 100644 --- a/wiki/howto/onboarding.md +++ b/wiki/howto/onboarding.md @@ -257,7 +257,7 @@ Install the GWT Browser Plugin for the GWT Development mode. As of 2016-08-31 Fi If you want a hudson job to run when you push your branch then you can run a script in `configuration` called . Run options for a branch titled `createHudsonJobForBug.sh`. For you bug branch titled `bug`, create a build job, which will create a release, by running the script like so: `./createHudsonJobForBug.sh `. If on Windows, you may need to disable any web shields in antivirus software, to allow `curl` to function. If on Mac, you may need to install gnu-sed. -###Issues when playing around with AWS +### Issues when playing around with AWS - The problem: **aws cli (used for aws ec2 describe-tags) hangs in eu-west-2** in all AZs on new instances I created, using a target group which permitted all outbound connections and inbound https, http and ssh connections. I tried permitting everything but that didn’t work. When I attached (at Axel’s suggestion) the Java Application with Reverse Proxy security group, it worked but — even if I duplicated this security group, and applied that copy instead — it still didn’t work. Curl issue solution: it turns out that the network interface only permits certain outbound and inbounds from certain target groups. The path to the solution: On my instance in eu-west-2a, I ran aws --debug ec2 describe-tags (you may need to do aws configure first). This is much akin to verbose mode of other unix commands. I noticed it hang on a request to ec2.eu-west-2.amazonaws.com. If you do `dig -t any ec2.eu-west-2.amazonaws.com` you see 3 ip addresses, which — as you will see later — are IPs in each of the eu-west-2 availability zones. When I ran curl -v ec2.eu-west-2.amazonaws.com (the v flag is verbose), one of the IPs from dig was used (namely the one in eu-west-2a, where the instance resides) and it hangs. I then went to endpoints for the VPC and noticed a service for the service `com.amazonaws.eu-west-2.ec2`. It had the default security group, which turned out to only allow inbound rules from the default or Java Application with Reverse Proxy target group. From 05b56201ef8a44f2babbcfce53170a973d3bce40 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sat, 25 May 2024 22:02:35 +0000 Subject: [PATCH 046/134] Removed "test" again from Wiki homepage --- Home.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/Home.md b/Home.md index b82a2b9db90..3d2498e3971 100644 --- a/Home.md +++ b/Home.md @@ -179,5 +179,3 @@ SAP is at the center of today’s technology revolution, developing innovations * [GIT Repository (SAP)](ssh://git.wdf.sap.corp:29418/SAPSail/sapsailingcapture.git) * [Maven Repository Browser](https://maven.sapsailing.com/maven/) (see [[how to setup repository for Android builds|wiki/info/mobile/racecommittee-app-environment]]) * [Main Sailing Website](https://www.sapsailing.com) - -test From 0054f33033667dcc8c714895585cd7f6e34848b0 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Mon, 27 May 2024 08:20:24 +0200 Subject: [PATCH 047/134] b5912: Add Home.md to allowed paths. --- .../central_reverse_proxy/files/home/wiki/app.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/files/home/wiki/app.rb b/configuration/environments_scripts/central_reverse_proxy/files/home/wiki/app.rb index 9161333c52b..10ef9ee067d 100644 --- a/configuration/environments_scripts/central_reverse_proxy/files/home/wiki/app.rb +++ b/configuration/environments_scripts/central_reverse_proxy/files/home/wiki/app.rb @@ -35,7 +35,6 @@ class App < Precious::App return end @_auth = Rack::Auth::Basic::Request.new(request.env) - puts "here" if self.env['PATH_INFO'].split('/').length >=2 && (self.env['PATH_INFO'].split('/')[1] != 'wiki' && self.env['PATH_INFO'].split('/')[2] != 'wiki' && @@ -43,6 +42,7 @@ class App < Precious::App self.env['PATH_INFO'].split('/')[2] != 'home' && self.env['PATH_INFO'].split('/')[1] != 'Home' && self.env['PATH_INFO'].split('/')[2] != 'Home' && + self.env['PATH_INFO'].split('/')[1] != 'Home.md' && self.env['PATH_INFO'].split('/')[1] != 'search' && self.env['PATH_INFO'].split('/')[2] != 'search' && self.env['PATH_INFO'].split('/')[1] != 'edit' && From 497970ad96c90903a83c22a7227b9b76d3a0d412 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Mon, 27 May 2024 10:51:00 +0200 Subject: [PATCH 048/134] bug5912: more useful prompt message after patching batch file for DNS record update --- .../target-group-tag-route53-nfs-elasticIP-setup.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh index 265d178c535..f469c69c101 100755 --- a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh +++ b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh @@ -45,7 +45,9 @@ done # alter records using batch file. sed -i "s/LOGFILES_INTERNAL_IP/$LOCAL_IPV4/" batch-for-route53-dns-record-update.json sed -i "s/SMTP_INTERNAL_IP/$LOCAL_IPV4/" batch-for-route53-dns-record-update.json -read -n 1 -p "Check the instance has the correct tags and is in the correct target group. Furthermore, check the batch file has been modified correctly. Press a key to continue.." key_pressed +read -n 1 -p "Check the instance has the correct tags and is in the correct target group. +Furthermore, check the batch file batch-for-route53-dns-record-update.json has been modified correctly. +Press a key to continue.." key_pressed ###### DO NOT ENABLE WHILST TESTING: aws route53 change-resource-record-sets --hosted-zone-id Z2JYWXYWLLRLTE --change-batch file://batch-for-route53-dns-record-update.json # reload the nfs mountpoints. echo "Waiting 60 seconds for records to change. The program will await a key press after this time." From d3febc08f519343ac4067637ff7ef4cf43061cf2 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Mon, 27 May 2024 10:52:20 +0200 Subject: [PATCH 049/134] bug5912: removed extra spaces --- .../target-group-tag-route53-nfs-elasticIP-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh index f469c69c101..feea2ca3282 100755 --- a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh +++ b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh @@ -58,7 +58,7 @@ echo "Describing instances for remounting." describe_instances=$(aws ec2 describe-instances) for instanceIp in $(echo "$describe_instances" | select_instances_by_tag "sailing-analytics-server" | extract_public_ip); do echo "Remounting on $instanceIp" - ssh -o StrictHostKeyChecking=false root@"${instanceIp}" " umount -l -f /home/scores; mount -a" + ssh -o StrictHostKeyChecking=false root@"${instanceIp}" "umount -l -f /home/scores; mount -a" done for instanceIp in $(echo "$describe_instances" | select_instances_by_tag "DisposableProxy" | extract_public_ip); do echo "Remounting on $instanceIp" From b20c2453f6eb5ffea0c92b8c8010ce295e1e81ad Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Mon, 27 May 2024 11:08:58 +0200 Subject: [PATCH 050/134] bug5912: removed more extra spaces --- .../repo/usr/local/bin/imageupgrade_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index ff65b66c497..24c681f8fa4 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -176,7 +176,7 @@ __setup_keys_using_local_copy() { # $2 a "true" or "false" string, indicating whether to override existing permissions. # "true" indicates the permissions and ownership of the .ssh and .aws folders will not be set. TEMP_KEY_DIR="$1" - SET_PERMISSIONS="$2" + SET_PERMISSIONS="$2" REGION=$(TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" --silent -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"` \ && curl -H "X-aws-ec2-metadata-token: $TOKEN" --silent http://169.254.169.254/latest/meta-data/placement/region) cd "${TEMP_KEY_DIR}" From 81624a4cbc5b718299cf770b2c17d2e7bc8422ad Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Mon, 27 May 2024 09:52:33 +0000 Subject: [PATCH 051/134] Fix case --- wiki/info/landscape/amazon-ec2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki/info/landscape/amazon-ec2.md b/wiki/info/landscape/amazon-ec2.md index 6a542c8b01a..900ec7b6e60 100644 --- a/wiki/info/landscape/amazon-ec2.md +++ b/wiki/info/landscape/amazon-ec2.md @@ -37,7 +37,7 @@ To improve availability and reliability, we have a disposable environment type a The IPs for all reverse proxies will automatically be added to ALB target groups with the tag key `allReverseProxies`, including the `CentralWebServerHTTP-Dyn` target group (in the dynamic ALB in eu-west-1) and all the `DDNSMapped-x-HTTP` (in all the DDNSMapped servers). These are the target groups for the default rules and it ensures availability to the ARCHIVE especially. -Disposables instances are tagged with `disposableProxy` to indicate it hosts no vital services. `ReverseProxy` also identifies any reverse proxies. The health check for the target groups would change to trigger a script which returns different error codes: healthy/200 if in the same AZ as the archive (or if the failover archive is in use), whilst unhealthy/503 if in different AZs. This will reduce cross-AZ, archive traffic costs, but maintain availability and load balancing. +Disposables instances are tagged with `DisposableProxy` to indicate it hosts no vital services. `ReverseProxy` also identifies any reverse proxies. The health check for the target groups would change to trigger a script which returns different error codes: healthy/200 if in the same AZ as the archive (or if the failover archive is in use), whilst unhealthy/503 if in different AZs. This will reduce cross-AZ, archive traffic costs, but maintain availability and load balancing. For security groups of the central reverse proxy, we want Webserver, as well as Disposable Reverse Proxy. The disposables just have the latter. From 13132def93bd480f004008db8c3e180f061bde68 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Mon, 27 May 2024 11:53:46 +0200 Subject: [PATCH 052/134] Alter syncgit to sync main and master branches in the github and trac user remote. --- .../repo/usr/local/bin/syncgit | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/configuration/environments_scripts/repo/usr/local/bin/syncgit b/configuration/environments_scripts/repo/usr/local/bin/syncgit index a5f10a035a1..8b03c485810 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/syncgit +++ b/configuration/environments_scripts/repo/usr/local/bin/syncgit @@ -1,16 +1,35 @@ #!/bin/sh +# Syncs the necessary branches of a repo at a specified location or, as default, +# /home/wiki/gitwiki. The assumed checked-out branch is master and the tracking branch +# is assumed to be origin master. ADMIN_EMAIL="axel.uhl@sap.com jan.hamann@sapsailing.com" if [ $# -eq 0 ]; then GIT_PATH="/home/wiki/gitwiki" else - GIT_PATH=$1 + GIT_PATH="$1" +fi +cd "$GIT_PATH" +git fetch github main:main >/tmp/wiki-git.out 2>/tmp/wiki-git.err +if [ "$?" != "0" ]; then + cat /tmp/wiki-git.out /tmp/wiki-git.err | mail -s "Wiki git problem: could not fetch github/main into local main" $ADMIN_EMAIL +fi +git push origin main:main >/tmp/wiki-git.out 2>/tmp/wiki-git.err +if [ "$?" != "0" ]; then + cat /tmp/wiki-git.out /tmp/wiki-git.err | mail -s "Wiki git problem: could not push main into sapsailing/main" $ADMIN_EMAIL +fi +git pull github main >/tmp/wiki-git.out 2>/tmp/wiki-git.err +if [ "$?" != "1" ]; then + cat /tmp/wiki-git.out /tmp/wiki-git.err | mail -s "Wiki git problem: could not merge github/main into sapsailing/master" $ADMIN_EMAIL fi -cd $GIT_PATH git pull >/tmp/wiki-git.out 2>/tmp/wiki-git.err if [ "$?" != "0" ]; then - cat /tmp/wiki-git.out /tmp/wiki-git.err | mail -s "Wiki git problem" $ADMIN_EMAIL + cat /tmp/wiki-git.out /tmp/wiki-git.err | mail -s "Wiki git problem: could not pull origin/master" $ADMIN_EMAIL +fi +git push origin master:main >>/tmp/wiki-git.out 2>/tmp/wiki-git.err +if [ "$?" != "0" ]; then + cat /tmp/wiki-git.out /tmp/wiki-git.err | mail -s "Wiki git problem: could not push master to origin/main" $ADMIN_EMAIL fi git push >>/tmp/wiki-git.out 2>/tmp/wiki-git.err if [ "$?" != "0" ]; then - cat /tmp/wiki-git.out /tmp/wiki-git.err | mail -s "Wiki git problem" $ADMIN_EMAIL + cat /tmp/wiki-git.out /tmp/wiki-git.err | mail -s "Wiki git problem: could not push master to origin/master" $ADMIN_EMAIL fi From 7c5c75215e85d9bd1f13bbbe3f23ac566640eaff Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Mon, 27 May 2024 10:07:32 +0000 Subject: [PATCH 053/134] Ensure image displays --- wiki/info/landscape/amazon-ec2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki/info/landscape/amazon-ec2.md b/wiki/info/landscape/amazon-ec2.md index 900ec7b6e60..27b3cd636fb 100644 --- a/wiki/info/landscape/amazon-ec2.md +++ b/wiki/info/landscape/amazon-ec2.md @@ -102,7 +102,7 @@ The webserver must be tagged with key ``CentralReverseProxy`` where the value is The following diagram explains the disposable reverse proxies role a little better. - + ### DNS and Application Load Balancers (ALBs) From 191db7ea3d80ff3229a0743bee4a20ddf681e695 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Mon, 27 May 2024 16:32:14 +0200 Subject: [PATCH 054/134] Create .pipeline/README Explanation of the Azure .pipeline/ folder --- .pipeline/README | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .pipeline/README diff --git a/.pipeline/README b/.pipeline/README new file mode 100644 index 00000000000..9baf656193e --- /dev/null +++ b/.pipeline/README @@ -0,0 +1,7 @@ +This folder contains an Azure pipeline definition that is used for building our +Android companion apps, in particular the Sailing Race Manager app and the +Buoy Pinger app. See also the mobile/ folder for their source code. The +gradle build that is used at the core of this build pipeline is controlled +by the ``build.gradle``and ``gradle.properties`` file in the root folder. +A local Android app build can be achieved by running the ``gradlew`` +wrapper script from the root folder. From ebff2f07817a48a724af16f864f88eb5d6937cf1 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Mon, 27 May 2024 16:58:00 +0200 Subject: [PATCH 055/134] Create .reuse/README Explaining the need for this folder --- .reuse/README | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .reuse/README diff --git a/.reuse/README b/.reuse/README new file mode 100644 index 00000000000..0d5afba287d --- /dev/null +++ b/.reuse/README @@ -0,0 +1,3 @@ +This folder is used by the workflow defined in ``.github/workflows/reuse-tool-license-updates.yml`` +and produces machine-readable licensing information for the entire repository into the dep5 file +in this same folder. From 79c5e65af0be14a1e4781012a14da375d324f10d Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Mon, 27 May 2024 21:35:59 +0200 Subject: [PATCH 056/134] b5912: Restart sshd daemon if healthy. --- .../setup-central-reverse-proxy-part-2.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh index 5c1dd25b92f..5040d09e256 100755 --- a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh @@ -32,6 +32,9 @@ echo "/var/log/old 172.31.0.0/16(rw,nohide,no_root_squash) /home/scores 172.31.0.0/16(rw,nohide,no_root_squash)" >>/etc/exports systemctl start nfs-server scp -p -o StrictHostKeyChecking=no -r root@sapsailing.com:/etc/ssh /etc +if sshd -t; then + systemctl restart sshd +fi # append hostname to sysconfig echo "HOSTNAME=sapsailing.com" >> /etc/sysconfig/network sed -i "s/\(127.0.0.1 *\)/\1 sapsailing.com /" /etc/hosts From a81d242dd13287153f35fddfbbcbdf3a0c168562 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Mon, 27 May 2024 21:39:38 +0200 Subject: [PATCH 057/134] b5912: Automatically start part 3 of the script. --- .../setup-central-reverse-proxy-part-2.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh index 5040d09e256..e96d0f415ab 100755 --- a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh @@ -45,10 +45,14 @@ if sshd -t; then fi EOF ssh -A -f root@"$IP" "cd /var/log/old/cache/docker/registry && nohup docker-compose up &>/dev/null &" &> /dev/null -echo "Please now run the script target-group-tag-route53-nfs-elasticIP-setup.sh which configures the EC2 instance tags, adds to the " +echo "PLEASE READ ALL OF THE FOLLOWING..." +echo "When ready, press a key to continue, which will run the script target-group-tag-route53-nfs-elasticIP-setup.sh." +echo "This configures the EC2 instance tags, adds to the " echo "necessary target groups, modifies a few records in route53 (logs.internal.sapsailing.com" -echo "and smtp.internal.sapsailing.com), remounts those dependent on this, and sets the elastic IP." -echo "You will need to have the aws cli installed and have the necessary permissions to make these alterations manually." -echo "In particular, make sure you have an active session token in your shell's environment, e.g., obtained" -echo "through the awsmfalogon.sh script." -echo "Have a great day!" +echo "and smtp.internal.sapsailing.com), remounts those instances that are dependent on this, and sets the elastic IP." +echo "You will need to have the AWS CLI installed and have credentials that allow you to run all the commands in the script." +echo "Make sure you have an active session token in your shell's environment, e.g., obtained" +echo "through the awsmfalogon.sh script. If you haven't already authenticated then stop this script; authenticate manually;" +echo "and then run the script named above." +read -n 1 -p "Press a key to continue" key_pressed +"$(dirname $0)"/target-group-tag-route53-nfs-elasticIP-setup.sh \ No newline at end of file From 6221072dcd5d91ac3582a89699ef08138c8de393 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Mon, 27 May 2024 21:45:24 +0200 Subject: [PATCH 058/134] b5912: Add certbot to docker group. --- .../central_reverse_proxy/users/certbot/groups | 1 + 1 file changed, 1 insertion(+) create mode 100644 configuration/environments_scripts/central_reverse_proxy/users/certbot/groups diff --git a/configuration/environments_scripts/central_reverse_proxy/users/certbot/groups b/configuration/environments_scripts/central_reverse_proxy/users/certbot/groups new file mode 100644 index 00000000000..6d0eac4b37a --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/users/certbot/groups @@ -0,0 +1 @@ +docker \ No newline at end of file From d604710dd00ae7ed1d96121f8dbf3b601a276970 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Tue, 28 May 2024 12:58:08 +0200 Subject: [PATCH 059/134] adjusted rexml version to 3.2.7 consistently --- Gemfile.lock | 2 +- wiki/howto/development/i18n.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 17d9585c19d..f007689a036 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -205,7 +205,7 @@ GEM claide (>= 1.0.2, < 2.0) colored2 (~> 3.1) nanaimo (~> 0.3.0) - rexml (~> 3.2.4) + rexml (~> 3.2.7) xcpretty (0.3.0) rouge (~> 2.0.7) xcpretty-travis-formatter (1.0.1) diff --git a/wiki/howto/development/i18n.md b/wiki/howto/development/i18n.md index 5a362fcf1be..0d2ac42be07 100644 --- a/wiki/howto/development/i18n.md +++ b/wiki/howto/development/i18n.md @@ -58,13 +58,13 @@ The primary repository for the app lives at ``trac@sapsailing.com:/home/trac/sai ## Triggering and Integrating Translations -We entertain a Git repo clone at ``https://github.tools.sap/SAP-Sailing-Analytics/sapsailing``. When we push new commits to the ``translation`` branch of that repository (see [https://github.tools.sap/SAP-Sailing-Analytics/sapsailing/tree/translation](https://github.tools.sap/SAP-Sailing-Analytics/sapsailing/tree/translation)) then LXLabs will grab these changes in regular intervals (as of this writing typically every Friday evening) and look for changes in files that belong to any message collection. +We keep our Git repo at ``https://github.com/SAP/sailing-analytics``. When we push new commits to the ``translation`` branch of that repository (see [https://github.com/SAP/sailing-analytics/tree/translation](https://github.com/SAP/sailing-analytics/tree/translation)) then LXLabs will grab these changes in regular intervals (as of this writing typically every Friday evening) and look for changes in files that belong to any message collection. -Message collections are declared in the file ``translation_v2.json`` in the root of our Git workspace. Adding new collections to this file requires us to inform LXLabs (e.g., vyacheslav.sklyarenko@sap.com, called "Slava"). They need to make the corresponding update in the so-called [Translation Enablement Workbench](https://tew.ingress.prod.lp.shoot.live.k8s-hana.ondemand.com/#/param/WzIwMiw1NiwiLyIsInRhc2tRdWV1ZSJd). +Message collections are declared in the file ``translation_v2.json`` in the root of our Git workspace. Adding new collections to this file requires us to inform LXLabs (e.g., vyacheslav.sklyarenko@sap.com, called "Slava"). They need to make the corresponding update in the so-called [Translation Enablement Workbench](https://tew.ingress.prod.lp.shoot.live.k8s-hana.ondemand.com/). During the translation process, translators may have questions regarding new or changed message strings. This will results in tasks being assigned by the Translation Enablement Workbench, and an e-mail to the assignee will result. Currently, the default assignee for these tasks is axel.uhl@sap.com. We then need to answer these questions as soon as possible to unblock the translation process so that ideally we get the translation in the same sweep as those for other languages, in the same round. -When translations are done, they are pushed to the same ``translation`` branch of ``https://github.tools.sap/SAP-Sailing-Analytics/sapsailing`` by LXLabs and then usually contain the translation of all changed or added message strings. LXLabs goes by the default locale ("en") and ignores all we do in the German ("de") locale. +When translations are done, they are pushed to the same ``translation`` branch of ``https://github.com/SAP/sailing-analytics`` by LXLabs and then usually contain the translation of all changed or added message strings. LXLabs goes by the default locale ("en") and ignores all we do in the German ("de") locale. We have a Github webhook installed (see [https://github.tools.sap/SAP-Sailing-Analytics/sapsailing/settings/hooks/285417](https://github.tools.sap/SAP-Sailing-Analytics/sapsailing/settings/hooks/285417)) which triggers a CGI-BIN script that is installed on sapsailing.com at ``/var/www/cgi-bin/github.cgi`` and is versioned under ``configuration/httpd/cgi-bin/github.cgi`` to which ``/var/www/cgi-bin/github.cgi`` is a symbolic link, and which reacts to pushes from the user with the e-mail address ``tmsatsls+github.tools.sap_service-tip-git@sap.com``, pushing to branch ``translation``. In this case, the Git workspace installed at ``/home/wiki/gitwiki`` for the user ``wiki`` is used to fetch the latest translations, using the Git remote ``sapsailing`` which is expected to use a Personal Access Token (PAT) that is granted read access to the repository. The changes are then pushed to the ``translation`` branch on ``ssh://trac@sapsailing.com/home/trac/git`` which is expected to trigger the [hudson job for the translation branch](https://hudson.sapsailing.com/job/SAPSailingAnalytics-translation/). This, in turn, will run a full build, and if successful, merge the ``translation`` branch into ``master`` and push it back to ``ssh://trac@sapsailing.com/home/trac/git`` which will then trigger the [master's build job](https://hudson.sapsailing.com/job/SAPSailingAnalytics-master/). From a3b429eb9fd5089c86ba63bc500a8f297e2b20af Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Tue, 28 May 2024 13:09:56 +0200 Subject: [PATCH 060/134] upgraded rexml to 3.2.8 --- Gemfile.lock | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index f007689a036..ef19a6f1335 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -171,7 +171,8 @@ GEM trailblazer-option (>= 0.1.1, < 0.2.0) uber (< 0.2.0) retriable (3.1.2) - rexml (3.2.7) + rexml (3.2.8) + strscan (>= 3.0.9) rouge (2.0.7) ruby2_keywords (0.0.5) rubyzip (2.3.2) @@ -184,6 +185,7 @@ GEM simctl (1.6.10) CFPropertyList naturally + strscan (3.1.0) terminal-notifier (2.0.0) terminal-table (3.0.2) unicode-display_width (>= 1.1.1, < 3) From e2b219a3d8696201ae0dbb701c7e93d5356d8009 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Tue, 28 May 2024 13:15:03 +0200 Subject: [PATCH 061/134] upgraded xcodeproj Ruby bundle for Android app build --- Gemfile.lock | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index ef19a6f1335..1c5c4825e06 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,9 @@ GEM remote: https://rubygems.org/ specs: - CFPropertyList (3.0.6) + CFPropertyList (3.0.7) + base64 + nkf rexml addressable (2.8.5) public_suffix (>= 2.0.2, < 6.0) @@ -24,6 +26,7 @@ GEM aws-sigv4 (1.6.0) aws-eventstream (~> 1, >= 1.0.2) babosa (1.0.4) + base64 (0.2.0) claide (1.1.0) colored (1.2) colored2 (3.1.2) @@ -161,6 +164,7 @@ GEM multipart-post (2.3.0) nanaimo (0.3.0) naturally (2.2.1) + nkf (0.2.0) optparse (0.1.1) os (1.1.4) plist (3.7.0) @@ -201,13 +205,13 @@ GEM unicode-display_width (2.5.0) webrick (1.8.1) word_wrap (1.0.0) - xcodeproj (1.23.0) + xcodeproj (1.24.0) CFPropertyList (>= 2.3.3, < 4.0) atomos (~> 0.1.3) claide (>= 1.0.2, < 2.0) colored2 (~> 3.1) nanaimo (~> 0.3.0) - rexml (~> 3.2.7) + rexml (~> 3.2.8) xcpretty (0.3.0) rouge (~> 2.0.7) xcpretty-travis-formatter (1.0.1) From 890316e279f356d429c09f28b38c1ce535a1d3c0 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Tue, 28 May 2024 13:28:05 +0200 Subject: [PATCH 062/134] downgrade rexml dependency to ~> 3.2.4 (again) --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 1c5c4825e06..67a87ee1234 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -211,7 +211,7 @@ GEM claide (>= 1.0.2, < 2.0) colored2 (~> 3.1) nanaimo (~> 0.3.0) - rexml (~> 3.2.8) + rexml (~> 3.2.4) xcpretty (0.3.0) rouge (~> 2.0.7) xcpretty-travis-formatter (1.0.1) From c2a5ae524378cfe0e4ac998a432ab80f28fc5573 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Tue, 28 May 2024 12:25:05 +0000 Subject: [PATCH 063/134] Created domain-switchover (markdown) --- wiki/planning/domain-switchover.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 wiki/planning/domain-switchover.md diff --git a/wiki/planning/domain-switchover.md b/wiki/planning/domain-switchover.md new file mode 100644 index 00000000000..b702825f17e --- /dev/null +++ b/wiki/planning/domain-switchover.md @@ -0,0 +1,16 @@ +# Domain Switchover + +## Locations that reference sapsailing.com + +- The ChargeBee subscription module use for payment services has a callback method pointing to sapsailing.com where updates to the users are processed and then replicated across the landscape. +- SharedLandscapeConsants.DEFAULT_DOMAIN_NAME is set to "sapsailing.com" but can be overridden; for things to work, however, we would need to add the deviating domain as a "hosted zone" to our Route53 account so that the DNS-maintaining logic will run without exceptions. +- The certificate used in the ALBs is created for \*.sapsailing.com; to run under a different domain name, a certificate matching the domain name will be required. The ALB set-up in AwsLandscapeImpl.createLoadBalancerHttpsListener uses the default certificate domain "\*.sapsailing.com" from a constant; this would need to be made flexible, e.g., through a system property, if we were to use any form of landscape automation under a different domain name. +- Shared security configuration as in + env.sh:#ADDITIONAL_JAVA_ARGS="$ADDITIONAL_JAVA_ARGS -Dsecurity.sharedAcrossSubdomainsOf=sapsailing.com -Dsecurity.baseUrlForCrossDomainStorage=https://security-service.sapsailing.com -Dgwt.acceptableCrossDomainStorageRequestOriginRegexp=https?://(.\*\.)?sapsailing\.com(:[0-9]\*)?$" +- mail sending and Amazon SES +- REPLICATE_MASTER_SERVLET_HOST is sometimes hardcoded. +- Igtimi Authentication +- Google Maps API is SAP provided +- A number of bash scripts have sapsailing.com hardcoded, especially the setup scripts and imageupgrade_functions.sh. Ideally we will need to factor out into a constant that all bash scripts can access. +- Releases and p2 respository references. +- MongoDB references and rabbit references, or any other private IP references. \ No newline at end of file From 31b05d0c628677648f5c1c064e94347eca3a64bb Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Tue, 28 May 2024 12:37:56 +0000 Subject: [PATCH 064/134] Minor edits. --- wiki/planning/domain-switchover.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/wiki/planning/domain-switchover.md b/wiki/planning/domain-switchover.md index b702825f17e..dbd549858c9 100644 --- a/wiki/planning/domain-switchover.md +++ b/wiki/planning/domain-switchover.md @@ -7,10 +7,10 @@ - The certificate used in the ALBs is created for \*.sapsailing.com; to run under a different domain name, a certificate matching the domain name will be required. The ALB set-up in AwsLandscapeImpl.createLoadBalancerHttpsListener uses the default certificate domain "\*.sapsailing.com" from a constant; this would need to be made flexible, e.g., through a system property, if we were to use any form of landscape automation under a different domain name. - Shared security configuration as in env.sh:#ADDITIONAL_JAVA_ARGS="$ADDITIONAL_JAVA_ARGS -Dsecurity.sharedAcrossSubdomainsOf=sapsailing.com -Dsecurity.baseUrlForCrossDomainStorage=https://security-service.sapsailing.com -Dgwt.acceptableCrossDomainStorageRequestOriginRegexp=https?://(.\*\.)?sapsailing\.com(:[0-9]\*)?$" -- mail sending and Amazon SES +- Mail sending and Amazon SES. - REPLICATE_MASTER_SERVLET_HOST is sometimes hardcoded. -- Igtimi Authentication -- Google Maps API is SAP provided +- Igtimi Authentication. +- Google Maps API is SAP provided. - A number of bash scripts have sapsailing.com hardcoded, especially the setup scripts and imageupgrade_functions.sh. Ideally we will need to factor out into a constant that all bash scripts can access. - Releases and p2 respository references. -- MongoDB references and rabbit references, or any other private IP references. \ No newline at end of file +- MongoDB references and rabbit references, or any other private IP references in route53. \ No newline at end of file From ccb8c0d0f1b0f78e0a66f3cf580f5d9cdd387861 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Tue, 28 May 2024 14:39:24 +0200 Subject: [PATCH 065/134] b5912: rename key vault --- .../central_reverse_proxy/setup-central-reverse-proxy.sh | 2 +- .../repo/usr/local/bin/imageupgrade_functions.sh | 2 +- wiki/info/landscape/amazon-ec2.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh index 937f62d5191..c33515f8154 100755 --- a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh @@ -157,7 +157,7 @@ chmod 755 /root cd ~ # Copies across the key vault and other relevant secrets from the existing # Central Reverse Proxy's /root folder: -rsync -a root@sapsailing.com:/root/{dev-secrets,github_tools_sap.pat,hudson-aws-credentials,key_vault,mail.properties,new_version_key_vault,secrets,ssh-key-reader.token} /root +rsync -a root@sapsailing.com:/root/{dev-secrets,github_tools_sap.pat,hudson-aws-credentials,key_vault,mail.properties,key_vault_old,secrets,ssh-key-reader.token} /root scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/etc/letsencrypt /etc # add basic test page which won't cause redirect error code if used as a health check. cat < /var/www/html/index.html diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index 24c681f8fa4..89a6e3b240c 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -166,7 +166,7 @@ setup_keys() { fi pushd . TEMP_KEY_DIR=$(mktemp -d /var/tmp/keysXXXXX) - scp -o StrictHostKeyChecking=no -pr root@sapsailing.com:/root/new_version_key_vault/"${1}"/* "${TEMP_KEY_DIR}" + scp -o StrictHostKeyChecking=no -pr root@sapsailing.com:/root/key_vault/"${1}"/* "${TEMP_KEY_DIR}" sudo su - -c "source imageupgrade_functions.sh; __setup_keys_using_local_copy $TEMP_KEY_DIR $SET_PERMISSIONS" popd rm -rf "${TEMP_KEY_DIR}" diff --git a/wiki/info/landscape/amazon-ec2.md b/wiki/info/landscape/amazon-ec2.md index 6a542c8b01a..5412529e74b 100644 --- a/wiki/info/landscape/amazon-ec2.md +++ b/wiki/info/landscape/amazon-ec2.md @@ -278,7 +278,7 @@ In all of the following sub-sections the text will assume that you have provided In several of the scenarios, both, AdminConsole and REST API, you will have the option to provide security bearer tokens that are used to authenticate requests to processes running the SAP Sailing Analytics. If you omit those, the credentials of the session used to authenticate your sailing user will be used. (Note, that for local test set-ups disconnected from the standard security realm used by all of the sapsailing.com-deployed processes, these credentials may not be accepted by the processes you're trying to control. In this case, please provide explicit bearer tokens instead.) We distinguish between the credentials required to replicate the information shared across the landscape, usually from ``security-service.sapsailing.com``, and those used by a replica in one of your application replica sets to authenticate for credentials to replicate the application replica set's master. -There is now a single point of truth for the various ssh and AWS keys, and possibly others in the future. This can be found at `/root/new_version_key_vault` on the central reverse proxy. There you will find directories for different environments' key setups, named consistently with the environment types under `${GIT_HOME}/configuration/environments_scripts` (the directory names are the environment type). One can use the `setup_keys` function in `imageupgrade_functions.sh` to setup the keys. There is 1 parameter, the environment type. +There is now a single point of truth for the various ssh and AWS keys, and possibly others in the future. This can be found at `/root/key_vault` on the central reverse proxy. There you will find directories for different environments' key setups, named consistently with the environment types under `${GIT_HOME}/configuration/environments_scripts` (the directory names are the environment type). One can use the `setup_keys` function in `imageupgrade_functions.sh` to setup the keys. There is 1 parameter, the environment type. The structure of the vault is important for the efficacy of the script and should appear as below. There is an explanation afterwards. ``` From 1ee4745b0ff90d54a890c7bab35f358f9958e602 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Tue, 28 May 2024 17:09:21 +0200 Subject: [PATCH 066/134] b5912: enable wiki creation of articles. Remove logging. --- .../central_reverse_proxy/files/home/wiki/app.rb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/files/home/wiki/app.rb b/configuration/environments_scripts/central_reverse_proxy/files/home/wiki/app.rb index 10ef9ee067d..0b61c2df0a7 100644 --- a/configuration/environments_scripts/central_reverse_proxy/files/home/wiki/app.rb +++ b/configuration/environments_scripts/central_reverse_proxy/files/home/wiki/app.rb @@ -17,7 +17,6 @@ class App < Precious::App helpers do def authenticate! - puts "authentication" public_urls=IO.readlines 'public.txt' public_urls.each {|url| if self.env['PATH_INFO'] == url.slice(0, url.length-1) @@ -27,7 +26,6 @@ class App < Precious::App if self.env['PATH_INFO'].start_with?('/wiki/images') || self.env['PATH_INFO'].start_with?('/favicon.ico') - puts "Allowing " + self.env['PATH_INFO'] return end } @@ -48,11 +46,10 @@ class App < Precious::App self.env['PATH_INFO'].split('/')[1] != 'edit' && self.env['PATH_INFO'].split('/')[2] != 'edit' && self.env['PATH_INFO'].split('/')[1] != 'preview' && - self.env['PATH_INFO'].split('/')[2] != 'preview') + self.env['PATH_INFO'].split('/')[2] != 'preview' && + (self.env['PATH_INFO'].split('/')[1] != 'gollum'|| self.env['PATH_INFO'].split('/')[2] != 'create')) throw(:halt, [403, 'Forbidden - You can not access anything outside wiki/ path.']) end - puts settings - puts settings.authorized_users if @_auth.provided? end if @_auth.provided? && @_auth.basic? && @_auth.credentials && @user = detected_user(@_auth.credentials) @@ -69,7 +66,6 @@ class App < Precious::App end def users - puts settings @_users ||= settings.authorized_users.map {|u| User.new(*u) } end From 32accafcb1e5edac380d6330d205050ced96b69f Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Tue, 28 May 2024 18:20:17 +0200 Subject: [PATCH 067/134] updated Wiki for domain-switchover.md, dynamically discover hosted zone ID for sapsailing.com --- ...t-group-tag-route53-nfs-elasticIP-setup.sh | 5 +++- wiki/planning/domain-switchover.md | 28 +++++++++++++------ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh index 1ade30928a8..a4adc022678 100755 --- a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh +++ b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh @@ -44,7 +44,10 @@ done # alter records using batch file. sed -i "s/LOGFILES_INTERNAL_IP/$LOCAL_IPV4/" batch-for-route53-dns-record-update.json sed -i "s/SMTP_INTERNAL_IP/$LOCAL_IPV4/" batch-for-route53-dns-record-update.json -###### DO NOT ENABLE WHILST TESTING: aws route53 change-resource-record-sets --hosted-zone-id Z2JYWXYWLLRLTE --change-batch file://batch-for-route53-dns-record-update.json +HOSTED_ZONE_ID=$( aws route53 list-hosted-zones | \ + jq -r '.HostedZones[] | select(.Name == "sapsailing.com.").Id' | \ + sed -e 's|/hostedzone/||' ) +###### DO NOT ENABLE WHILST TESTING: aws route53 change-resource-record-sets --hosted-zone-id ${HOSTED_ZONE_ID} --change-batch file://batch-for-route53-dns-record-update.json # reload the nfs mountpoints. echo "Describing instances for remounting." describe_instances=$(aws ec2 describe-instances) diff --git a/wiki/planning/domain-switchover.md b/wiki/planning/domain-switchover.md index dbd549858c9..1bbd6b3f9c3 100644 --- a/wiki/planning/domain-switchover.md +++ b/wiki/planning/domain-switchover.md @@ -2,15 +2,27 @@ ## Locations that reference sapsailing.com -- The ChargeBee subscription module use for payment services has a callback method pointing to sapsailing.com where updates to the users are processed and then replicated across the landscape. -- SharedLandscapeConsants.DEFAULT_DOMAIN_NAME is set to "sapsailing.com" but can be overridden; for things to work, however, we would need to add the deviating domain as a "hosted zone" to our Route53 account so that the DNS-maintaining logic will run without exceptions. +- The ChargeBee subscription module used for payment services has a callback method pointing to ``sapsailing.com`` where updates to the users are processed and then replicated across the landscape. + +- ``SharedLandscapeConsants.DEFAULT_DOMAIN_NAME`` is set to "sapsailing.com" but can be overridden; for things to work, however, we would need to add the deviating domain as a "hosted zone" to our Route53 account so that the DNS-maintaining logic will run without exceptions. + - The certificate used in the ALBs is created for \*.sapsailing.com; to run under a different domain name, a certificate matching the domain name will be required. The ALB set-up in AwsLandscapeImpl.createLoadBalancerHttpsListener uses the default certificate domain "\*.sapsailing.com" from a constant; this would need to be made flexible, e.g., through a system property, if we were to use any form of landscape automation under a different domain name. + - Shared security configuration as in - env.sh:#ADDITIONAL_JAVA_ARGS="$ADDITIONAL_JAVA_ARGS -Dsecurity.sharedAcrossSubdomainsOf=sapsailing.com -Dsecurity.baseUrlForCrossDomainStorage=https://security-service.sapsailing.com -Dgwt.acceptableCrossDomainStorageRequestOriginRegexp=https?://(.\*\.)?sapsailing\.com(:[0-9]\*)?$" + env.sh:#ADDITIONAL_JAVA_ARGS="$ADDITIONAL_JAVA_ARGS -Dsecurity.sharedAcrossSubdomainsOf=sapsailing.com -Dsecurity.baseUrlForCrossDomainStorage=https://security-service.sapsailing.com -Dgwt.acceptableCrossDomainStorageRequestOriginRegexp=https?://(.\*\.)?sapsailing\.com(:[0-9]\*)?$" + +- The ``target-group-tag-route53-nfs-elasticIP-setup.sh`` script used as the last stage of the setup process of a new central reverse proxy. It queries the Route 53 ID of the "sapsailing.com" hosted zone which would have to become a parameter or would need to be changed in the script. + - Mail sending and Amazon SES. -- REPLICATE_MASTER_SERVLET_HOST is sometimes hardcoded. -- Igtimi Authentication. -- Google Maps API is SAP provided. + +- ``REPLICATE_MASTER_SERVLET_HOST`` is sometimes hardcoded. + +- Igtimi Authentication: the Igtimi REST API and authentication scheme uses a callback URL that we have configured on the igtimi.com web site, pointing to sapsailing.com. + +- Google Maps API is SAP provided in our current production environment, and a different API token/key is required once SAP stops its support. + - A number of bash scripts have sapsailing.com hardcoded, especially the setup scripts and imageupgrade_functions.sh. Ideally we will need to factor out into a constant that all bash scripts can access. -- Releases and p2 respository references. -- MongoDB references and rabbit references, or any other private IP references in route53. \ No newline at end of file + +- Releases and p2 respository references, where "releases.sapsailing.com" will be replaced over time by Github releases which the ``refreshInstance.sh`` script can then download. + +- MongoDB references and rabbit references, or any other private IP references in route53. Such references can in particular be found in the ``environments/`` folder on ``releases.sapsailing.com`` for the default parameterization of certain application process types, such as ``security-service-master`` or ``archive-server``. \ No newline at end of file From 2f524fc4a25027fd98e7a5fd87bc4c66ba7eb7eb Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Tue, 28 May 2024 18:31:01 +0200 Subject: [PATCH 068/134] wiki updates for i18n and the domain-switchover considerations --- wiki/howto/development/i18n.md | 4 +++- wiki/planning/domain-switchover.md | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/wiki/howto/development/i18n.md b/wiki/howto/development/i18n.md index 0d2ac42be07..9717f86ec3c 100644 --- a/wiki/howto/development/i18n.md +++ b/wiki/howto/development/i18n.md @@ -66,7 +66,9 @@ During the translation process, translators may have questions regarding new or When translations are done, they are pushed to the same ``translation`` branch of ``https://github.com/SAP/sailing-analytics`` by LXLabs and then usually contain the translation of all changed or added message strings. LXLabs goes by the default locale ("en") and ignores all we do in the German ("de") locale. -We have a Github webhook installed (see [https://github.tools.sap/SAP-Sailing-Analytics/sapsailing/settings/hooks/285417](https://github.tools.sap/SAP-Sailing-Analytics/sapsailing/settings/hooks/285417)) which triggers a CGI-BIN script that is installed on sapsailing.com at ``/var/www/cgi-bin/github.cgi`` and is versioned under ``configuration/httpd/cgi-bin/github.cgi`` to which ``/var/www/cgi-bin/github.cgi`` is a symbolic link, and which reacts to pushes from the user with the e-mail address ``tmsatsls+github.tools.sap_service-tip-git@sap.com``, pushing to branch ``translation``. In this case, the Git workspace installed at ``/home/wiki/gitwiki`` for the user ``wiki`` is used to fetch the latest translations, using the Git remote ``sapsailing`` which is expected to use a Personal Access Token (PAT) that is granted read access to the repository. The changes are then pushed to the ``translation`` branch on ``ssh://trac@sapsailing.com/home/trac/git`` which is expected to trigger the [hudson job for the translation branch](https://hudson.sapsailing.com/job/SAPSailingAnalytics-translation/). This, in turn, will run a full build, and if successful, merge the ``translation`` branch into ``master`` and push it back to ``ssh://trac@sapsailing.com/home/trac/git`` which will then trigger the [master's build job](https://hudson.sapsailing.com/job/SAPSailingAnalytics-master/). +We have a Github webhook installed (see [https://github.com/SAP/sailing-analytics/settings/hooks/480929064](https://github.com/SAP/sailing-analytics/settings/hooks/480929064)) which triggers a CGI-BIN script that is installed on sapsailing.com at ``/var/www/cgi-bin/github.cgi`` and is versioned under ``configuration/httpd/cgi-bin/github.cgi`` to which ``/var/www/cgi-bin/github.cgi`` is a symbolic link, and which reacts to pushes from the user with the e-mail address ``tmsatsls+github.tools.sap_service-tip-git@sap.com``, pushing to branch ``translation``. In this case, the Git workspace installed at ``/home/wiki/gitwiki`` for the user ``wiki`` is used to fetch the latest translations, using the Git remote ``sapsailing`` which is expected to use a Personal Access Token (PAT) that is granted read access to the repository. The changes are then pushed to the ``translation`` branch on ``ssh://trac@sapsailing.com/home/trac/git`` which is expected to trigger the [hudson job for the translation branch](https://hudson.sapsailing.com/job/SAPSailingAnalytics-translation/). This, in turn, will run a full build, and if successful, merge the ``translation`` branch into ``master`` and push it back to ``ssh://trac@sapsailing.com/home/trac/git`` which will then trigger the [master's build job](https://hudson.sapsailing.com/job/SAPSailingAnalytics-master/). + +TODO Github Publication: With the "translation" branch being pushed to github.com, a build for the translation branch will start automatically using Github Actions. If we renamed our "SAPSailingAnalytics-translation" job to just "translation" then the magic should work and we should get a trigger for the "translation" job when the Github Actions build is done. First building in a separate job before merging into master partly goes back to the times when LXLabs had issues exporting Java properties file syntax when placeholders and single-quote characters were used in the translations; a common problem in the French translations. However, this has improved massively, and we haven't seen any translation-incurred build errors in a long time. Yet, we stick to the process. diff --git a/wiki/planning/domain-switchover.md b/wiki/planning/domain-switchover.md index 1bbd6b3f9c3..599497bc902 100644 --- a/wiki/planning/domain-switchover.md +++ b/wiki/planning/domain-switchover.md @@ -25,4 +25,6 @@ - Releases and p2 respository references, where "releases.sapsailing.com" will be replaced over time by Github releases which the ``refreshInstance.sh`` script can then download. -- MongoDB references and rabbit references, or any other private IP references in route53. Such references can in particular be found in the ``environments/`` folder on ``releases.sapsailing.com`` for the default parameterization of certain application process types, such as ``security-service-master`` or ``archive-server``. \ No newline at end of file +- MongoDB references and rabbit references, or any other private IP references in route53. Such references can in particular be found in the ``environments/`` folder on ``releases.sapsailing.com`` for the default parameterization of certain application process types, such as ``security-service-master`` or ``archive-server``. + +- branch.io is used for "deep linking" for the apps. We have an integration with our Route53 hosted zone "sapsailing.com" for the specific domain names, allowing smart payload transfer through from-scratch installations, e.g., from a QR code. The branch.io account would need to be transferred and would need to have the configuration changed; or a new account would be required. The apps' deep linking capabilities are expressed in their manifests, and hence the apps would need to be adjusted to work with a different domain. \ No newline at end of file From 7ba8d768f8a466e57d51ed64e814cc1466f5ed79 Mon Sep 17 00:00:00 2001 From: Generic Wiki User Date: Wed, 29 May 2024 08:06:41 +0000 Subject: [PATCH 069/134] pushing translation merges to github now --- .../central_reverse_proxy/files/var/www/cgi-bin/github.cgi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/files/var/www/cgi-bin/github.cgi b/configuration/environments_scripts/central_reverse_proxy/files/var/www/cgi-bin/github.cgi index 75a7a891e7f..6a29f922e39 100755 --- a/configuration/environments_scripts/central_reverse_proxy/files/var/www/cgi-bin/github.cgi +++ b/configuration/environments_scripts/central_reverse_proxy/files/var/www/cgi-bin/github.cgi @@ -5,7 +5,7 @@ BODY=$( cat ) echo "${BODY}" >/tmp/github-hook-body REF=$( echo "${BODY}" | jq -r '.ref' ) PUSHER=$( echo "${BODY}" | jq -r '.pusher.email' ) -logger -t github-cgi "ref ia $REF, pusher was $PUSHER" +logger -t github-cgi "ref is $REF, pusher was $PUSHER" # For testing: #if [ "${PUSHER}" = "axel.uhl@sap.com" -a "${REF}" = "refs/heads/translation" ]; then if [ "${PUSHER}" = "tmsatsls+github.tools.sap_service-tip-git@sap.com" -a "${REF}" = "refs/heads/translation" ]; then @@ -13,7 +13,7 @@ if [ "${PUSHER}" = "tmsatsls+github.tools.sap_service-tip-git@sap.com" -a "${REF echo "Fetching translation branch from github.tools.sap and pushing it to ssh://trac@sapsailing.com/home/trac/git" logger -t github-cgi "fetching translation branch from github.tools.sap and pushing it to ssh://trac@sapsailing.com/home/trac/git" cd /home/wiki/gitwiki - sudo -u wiki git fetch sapsailing translation:translation 2>&1 + sudo -u wiki git fetch github translation:translation 2>&1 sudo -u wiki git push origin translation:translation 2>&1 else echo "Either pusher was not tmsatsls+github.tools.sap_service-tip-git@sap.com or ref was not refs/heads/translation. Not triggering." From a41d5b16a8a00c83b7e585e7d5dcc895c76d2ab5 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Wed, 29 May 2024 10:21:39 +0200 Subject: [PATCH 070/134] adding a workflow that merges main into docker-17 after successful build --- .../workflows/merge-main-into-docker-17.yml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/merge-main-into-docker-17.yml diff --git a/.github/workflows/merge-main-into-docker-17.yml b/.github/workflows/merge-main-into-docker-17.yml new file mode 100644 index 00000000000..aae18f5fbfb --- /dev/null +++ b/.github/workflows/merge-main-into-docker-17.yml @@ -0,0 +1,24 @@ +name: Merge main branch into docker-17 after successful build +on: + workflow_run: + workflows: [release] + types: [completed] + branches: [main] + workflow_dispatch: {} +jobs: + merge-main-into-docker-17: + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: docker-17 + #fetch-depth: 0 # fetch the whole thing to make sure the histories merge + - name: Merge main into docker-17 + uses: devmasx/merge-branch@master + with: + type: now + from_branch: main + target_branch: docker-17 + github_token: ${{ secrets.GITHUB_TOKEN }} From 1b541d349efaf11e8b68fba2d026fc1ac676deaf Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Wed, 29 May 2024 10:37:07 +0200 Subject: [PATCH 071/134] try pushing with bash to trigger release workflow after merge --- .github/workflows/merge-main-into-docker-17.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/merge-main-into-docker-17.yml b/.github/workflows/merge-main-into-docker-17.yml index aae18f5fbfb..71ffce924a0 100644 --- a/.github/workflows/merge-main-into-docker-17.yml +++ b/.github/workflows/merge-main-into-docker-17.yml @@ -15,10 +15,11 @@ jobs: with: ref: docker-17 #fetch-depth: 0 # fetch the whole thing to make sure the histories merge - - name: Merge main into docker-17 - uses: devmasx/merge-branch@master - with: - type: now - from_branch: main - target_branch: docker-17 - github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Merge main into docker-17 and push + shell: bash + run: | + git config --global user.email "github@sapsailing.com" + git config --global user.name "Github Auto-Merger" + git fetch origin main:main + git merge main + git push From a5f0f812f6df9c03332d27bd23de06a1177120ef Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Wed, 29 May 2024 10:40:09 +0200 Subject: [PATCH 072/134] use full repo history for main-->docker-17 merge --- .github/workflows/merge-main-into-docker-17.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-main-into-docker-17.yml b/.github/workflows/merge-main-into-docker-17.yml index 71ffce924a0..296f7eb52ba 100644 --- a/.github/workflows/merge-main-into-docker-17.yml +++ b/.github/workflows/merge-main-into-docker-17.yml @@ -14,7 +14,7 @@ jobs: uses: actions/checkout@v4 with: ref: docker-17 - #fetch-depth: 0 # fetch the whole thing to make sure the histories merge + fetch-depth: 0 # fetch the whole thing to make sure the histories merge - name: Merge main into docker-17 and push shell: bash run: | From dc0622a8ccb3f64f3d86b8747cc6c84485ca4f34 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Wed, 29 May 2024 10:57:08 +0200 Subject: [PATCH 073/134] use new PAT to run push, so it triggers the release workflow --- .github/workflows/merge-main-into-docker-17.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/merge-main-into-docker-17.yml b/.github/workflows/merge-main-into-docker-17.yml index 296f7eb52ba..5ac9d0bdeef 100644 --- a/.github/workflows/merge-main-into-docker-17.yml +++ b/.github/workflows/merge-main-into-docker-17.yml @@ -15,11 +15,12 @@ jobs: with: ref: docker-17 fetch-depth: 0 # fetch the whole thing to make sure the histories merge - - name: Merge main into docker-17 and push - shell: bash - run: | - git config --global user.email "github@sapsailing.com" - git config --global user.name "Github Auto-Merger" - git fetch origin main:main - git merge main - git push + - name: Merge main into docker-17 + uses: devmasx/merge-branch@master + env: + GITHUB_TOKEN: ${{ secrets.REPO_TOKEN_FOR_MERGE_AND_PUSH }} + with: + type: now + from_branch: main + target_branch: docker-17 + github_token: ${{ secrets.GITHUB_TOKEN }} From 2eee53418fc44db2c99a7aa6a4d3d544ceb660dd Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Wed, 29 May 2024 11:03:09 +0200 Subject: [PATCH 074/134] use the new PAT at the right place for triggering release after merge --- .github/workflows/merge-main-into-docker-17.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/merge-main-into-docker-17.yml b/.github/workflows/merge-main-into-docker-17.yml index 5ac9d0bdeef..361c380fd81 100644 --- a/.github/workflows/merge-main-into-docker-17.yml +++ b/.github/workflows/merge-main-into-docker-17.yml @@ -17,10 +17,8 @@ jobs: fetch-depth: 0 # fetch the whole thing to make sure the histories merge - name: Merge main into docker-17 uses: devmasx/merge-branch@master - env: - GITHUB_TOKEN: ${{ secrets.REPO_TOKEN_FOR_MERGE_AND_PUSH }} with: type: now from_branch: main target_branch: docker-17 - github_token: ${{ secrets.GITHUB_TOKEN }} + github_token: ${{ secrets.REPO_TOKEN_FOR_MERGE_AND_PUSH }} From b87097e3d432dc41025d4e0da6c9bfd0b915a3b7 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Wed, 29 May 2024 12:16:19 +0200 Subject: [PATCH 075/134] b5912: Improve comments of build-crontab-and-cp-files --- .../build-crontab-and-cp-files | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/configuration/environments_scripts/build-crontab-and-cp-files b/configuration/environments_scripts/build-crontab-and-cp-files index 9c8a9ab4b61..dec616dbf89 100755 --- a/configuration/environments_scripts/build-crontab-and-cp-files +++ b/configuration/environments_scripts/build-crontab-and-cp-files @@ -3,18 +3,19 @@ # Purpose: The first parameter is an image type. This script iterates over the image types' USER folders, # concatenating all the symbolic links (referencing locations within configuration/crontabs) into one crontab # file for each user. It also takes the name of the user containing a copy of the git repo as well as the name of the -# dir within that user to go to. The created files for each user go into that user's home dir and are installed for -# that user too. +# dir within that user to go to. The created crontab for each user go into that user's home dir and are installed for +# that user too. # Within these crontab stubs, are certain string literals, which are replaced: PATH_OF_GIT_HOME_DIR_TO_REPLACE becomes # the absolute path to the git home dir, and PATH_OF_HOME_DIR_TO_REPLACE is changed to the path of the home directory -# in which the crontab will be installed. -# Furthermore, within the users folder optional uid, gid and groups files an exist. uid should store the user's uid, -# gid the group id and groups should have secondary groups (one per line). The secondary groups are added after all users are -# created. +# in which the crontab will be installed (ie. that user's home). +# Furthermore, within the users folder optional uid, gid and groups files can exist. uid should store the user's uid, +# gid the group id and groups should have secondary groups (one per line). The user is added to the secondary groups +# after all users are created. # Useful files are also copied across from the "files" dir within each image type. # Finally, the script ends by performing a deamon reload and reenabling all the systemd units in the files directory, # so that the correct wants, requires, etc. are installed/linked. Any systemd units, within the mimicked filesystem, -# will be automatically enabled. +# will be automatically enabled. +# For options, see the help below. if [[ $# -lt 3 || $# -gt 6 ]]; then echo "$0 " @@ -39,7 +40,7 @@ do exit 4;; esac done -shift $((OPTIND-1)) # shift the arguments along. OPTIND is the position of the next argument to process. +shift $((OPTIND-1)) # shift the arguments along so there are no options in the arguments anymore. ENV_TYPE="$1" GIT_USER="$2" RELATIVE_GIT_DIR_NAME="$3" @@ -59,9 +60,10 @@ if [[ -d "groups" ]]; then fi if [[ -d "users" ]]; then cd "users" - GIT_PATH="$(eval echo $(printf "~%q" "$GIT_USER"))/${RELATIVE_GIT_DIR_NAME}" # The path to the git repo that contains the files needed. - for dir in $(ls -d */ ); do - USERNAME=$(echo "$dir" | sed "s/\/$//") # Dirname is the username. The trailing slash is removed. + GIT_PATH="$(eval echo $(printf "~%q" "$GIT_USER"))/${RELATIVE_GIT_DIR_NAME}" # The absolute path to a git repo in the environment type. + for dir in *; do + [[ -d "$dir" ]] || continue + USERNAME="$dir" user_UID="" user_GID="" echo "STARTING WORK ON USER $USERNAME" @@ -111,8 +113,9 @@ if [[ -d "users" ]]; then echo "****** finished user ******" done # Add to groups after all the groups exist. - for dir in $(ls -d */ ); do - USERNAME=$(echo "$dir" | sed "s/\/$//") # Dirname is the username. The trailing slash is removed. + for dir in *; do + [[ -d "$dir" ]] || continue + USERNAME="$dir" [[ -e "$USERNAME"/groups ]] || continue echo "ADDING $USERNAME TO THE GROUPS SPECIFIED IN THEIR 'groups' FILE" for group in $(cat "$USERNAME"/groups); do From 24cfe91654a05e5b875b7f3ecdae77a4ab63696c Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Wed, 29 May 2024 15:41:53 +0200 Subject: [PATCH 076/134] put PAT into GH_TOKEN into env to trigger dependent workflow --- .github/workflows/merge-main-into-docker-17.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/merge-main-into-docker-17.yml b/.github/workflows/merge-main-into-docker-17.yml index 361c380fd81..06a46100166 100644 --- a/.github/workflows/merge-main-into-docker-17.yml +++ b/.github/workflows/merge-main-into-docker-17.yml @@ -17,6 +17,8 @@ jobs: fetch-depth: 0 # fetch the whole thing to make sure the histories merge - name: Merge main into docker-17 uses: devmasx/merge-branch@master + env: + GH_TOKEN: ${{ secrets.REPO_TOKEN_FOR_MERGE_AND_PUSH }} with: type: now from_branch: main From dc36ded6801e38c66e963878c0f9e606c8433259 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Wed, 29 May 2024 18:01:29 +0200 Subject: [PATCH 077/134] b5912: Minor edits + start tmux management panel. --- .../setup-central-reverse-proxy-part-2.sh | 5 +++++ .../repo/usr/local/bin/imageupgrade_functions.sh | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh index e96d0f415ab..8358aa912f2 100755 --- a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh @@ -11,6 +11,10 @@ # where {image-type} identifies the "environment" from configuration/environments_scripts # which usually would be "central_reverse_proxy" here, passed in from the previous stage's # script. +if [[ "$#" -ne 2 ]]; then + echo "2 arguments required. Check comment of scripts for more details." + exit 2 +fi IP=$1 IMAGE_TYPE="$2" GIT_COPY_USER="wiki" @@ -18,6 +22,7 @@ RELATIVE_PATH_TO_GIT="gitwiki" # the relative path to the repo within the git_co TEMPORARY_HOME_COPY_LOCATION="/root/temporary_home_copy" # home nested within this. ssh -A "root@${IP}" "bash -s" << EOF sudo systemctl start crond.service +sudo systemctl start tmux-management-panel.service . imageupgrade_functions.sh cp -r "$TEMPORARY_HOME_COPY_LOCATION"/home / rm -rf "$TEMPORARY_HOME_COPY_LOCATION" diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index 89a6e3b240c..f7074fe4e11 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -404,7 +404,7 @@ setup_apachetop() { setup_swap() { # $1: size of swapspace in megabytes. - echo "Creating swapswpace of $1 MBs" + echo "Creating swapspace of $1 MBs" local swapfile_location=/var/cache/swapfile pushd . sudo dd if=/dev/zero of="$swapfile_location" bs=1M count="$1" From fb00b2478aba8654c0c759faf4a41a453a4b6dda Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Wed, 29 May 2024 18:06:36 +0200 Subject: [PATCH 078/134] b5912: New approach for fetching az. --- .../repo/usr/local/bin/imageupgrade_functions.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index f7074fe4e11..12fc0544e3e 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -177,8 +177,7 @@ __setup_keys_using_local_copy() { # "true" indicates the permissions and ownership of the .ssh and .aws folders will not be set. TEMP_KEY_DIR="$1" SET_PERMISSIONS="$2" - REGION=$(TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" --silent -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"` \ - && curl -H "X-aws-ec2-metadata-token: $TOKEN" --silent http://169.254.169.254/latest/meta-data/placement/region) + REGION="$( ec2-metadata | grep "^placement:" | sed -e 's/^.*: \(.*\).$/\1/')" cd "${TEMP_KEY_DIR}" for user in *; do [[ -e "$user" ]] || continue From 9d797ab3c8866ef00bd8e3d6932d38088276aed8 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Wed, 29 May 2024 18:24:07 +0200 Subject: [PATCH 079/134] Add central setup procedure. --- wiki/info/landscape/amazon-ec2.md | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/wiki/info/landscape/amazon-ec2.md b/wiki/info/landscape/amazon-ec2.md index 27b3cd636fb..925cd7f5cb4 100644 --- a/wiki/info/landscape/amazon-ec2.md +++ b/wiki/info/landscape/amazon-ec2.md @@ -31,7 +31,7 @@ Further ALBs may exist in addition to the default ALB and the NLB for ``sapsaili ### Apache httpd, the central reverse proxy (Webserver) and disposable reverse proxies A key pillar of our architecture is the central reverse proxy, which handles traffic for the wiki, bugzilla, awstats, releases, p2, Git, jobs, static and is the target of the catch all rule in the Dynamic ALB. -Any traffic to the Hudson build server subdomain *does not* go through the central webserver. Instead, it gets directed by route 53 to a `DDNSMapped` load balancer (which all route any port 80 traffic to 443), which has a rule pointing to a target group, that contains only the Hudson server. +Any traffic to the Hudson build server subdomain *does not* go through the central webserver. Instead, it gets directed by route 53 to a `DDNSMapped` load balancer (which all route any port 80 traffic to 443), which has a rule pointing to a target group, that contains only the Hudson server. The setup procedure can be found below. To improve availability and reliability, we have a disposable environment type and AMI. The instances from this AMI are only for serving requests to the archive but are lightweight and can be quickly started and shutdown, using the landscape management console. @@ -39,7 +39,7 @@ The IPs for all reverse proxies will automatically be added to ALB target groups and all the `DDNSMapped-x-HTTP` (in all the DDNSMapped servers). These are the target groups for the default rules and it ensures availability to the ARCHIVE especially. Disposables instances are tagged with `DisposableProxy` to indicate it hosts no vital services. `ReverseProxy` also identifies any reverse proxies. The health check for the target groups would change to trigger a script which returns different error codes: healthy/200 if in the same AZ as the archive (or if the failover archive is in use), whilst unhealthy/503 if in different AZs. This will reduce cross-AZ, archive traffic costs, but maintain availability and load balancing. -For security groups of the central reverse proxy, we want Webserver, as well as Disposable Reverse Proxy. The disposables just have the latter. +For security groups of the central reverse proxy, we want Webserver, as well as Reverse Proxy. The disposables just have the latter. There is hope to also deploy the httpd on already existing instances, which have free resources and a certain tag permitting this co-deployment. @@ -94,7 +94,7 @@ Use Status 172.31.19.129 internal-server-status Use Status 127.0.0.1 internal-server-status ``` -The second obviously requires maintenance as the internal IP changes, e.g., when instantiating a new Webserver copy by creating an image and restoring from the image. When upgrading / moving / copying the webserver you may try to be smart and copy the contents of ``/etc/ssh``, in particular the ``ssh_host_...`` files that contain the host keys. As you switch, users will then not have to upgrade their ``known_hosts`` file, and even internal accounts such as the Wiki account or the sailing accounts on other hosts that clone the git, or the build infrastructure won't be affected. +The second obviously requires maintenance as the internal IP changes, e.g., when instantiating a new Webserver copy by creating an image and restoring from the image. This maintenance is managed by a service unit. When upgrading / moving / copying the webserver you may try to be smart and copy the contents of ``/etc/ssh``, in particular the ``ssh_host_...`` files that contain the host keys. As you switch, users will then not have to upgrade their ``known_hosts`` file, and even internal accounts such as the Wiki account or the sailing accounts on other hosts that clone the git, or the build infrastructure won't be affected. After (re-)booting the webserver, check that all services have come up before adding the instance to its respective target groups. For example, ensure that the Wiki "Gollum" service has been launched (see ``/home/wiki/serve.sh``). Furthermore, ensure that the Docker daemon is running and that it runs the Docker registry containers (``registry-ui-1`` and ``registry-registry-1``). See [here](https://wiki.sapsailing.com/wiki/info/landscape/docker-registry) for how this is set up. @@ -104,6 +104,23 @@ The following diagram explains the disposable reverse proxies role a little bett +## Setting up the Central Reverse Proxy + +A lot of the above procedure has since been combined into a series of setup scripts found under `configuration/environments_scripts/central_reverse_proxy`. The script requires that you have added an SSH key with maximum access to your SSH authentication agent. This can be done with +``` +eval `ssh-agent` +ssh-add +``` + +You will also need the AWS CLI and can run `./awsmfalogon.sh` to authenticate and gain a session token. Next, you should notify the community that internal services, such as Bugzilla will temporarily be down (make sure to notify them afterwards too, so they can continue their work). + +Using the landscape tab of the admin console, ensure there is a disposable in the same AZ as the archive (this ensures we can still route traffic to the archive). + +Next, remove the central reverse proxy from all target groups tagged with `allReverseProxies`. Then when at the path described above, launch `./setup-central-reverse-proxy.sh` and follow the necessary instructions. You +will need to unmount and detach volumes from the old instance and then reattach and mount on the new webserver. +Then `setup-central-reverse-proxy-part-2.sh` runs to finish any setup that requires these mounts. Finally, `target-group-tag-route53-nfs-elasticIP-setup.sh` +will run to configure the target groups, tags and route 53. You will need to then remove the old reverse proxy from the target groups tagged with `CentralReverseProxy`. + ### DNS and Application Load Balancers (ALBs) We distinguish between DNS-mapped and non-DNS-mapped content. The basic services offered by the web server as listed above are DNS-mapped, with the DNS entries being CNAME records pointing to an ALB (DNSMapped-0-1286577811.eu-west-1.elb.amazonaws.com) which handles SSL offloading with the Amazon-managed certificate and forwards those requests to the web server. Furthermore, longer-running application replica sets can have a sub-domain declared in Route53's DNS, pointing to an ALB which then forwards to the public and master target groups for this replica set based on hostname, header fields and request method. A default redirect for the ``/`` path can also be defined, obsoleting previous Apache httpd reverse proxy redirects for non-archived ALB-mapped content. From c501053f76dd6d1052119d59c5a42f77f1bdcd54 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Wed, 29 May 2024 18:30:19 +0200 Subject: [PATCH 080/134] Add clarification regarding awsmfalogon usage in central setup. --- wiki/info/landscape/amazon-ec2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki/info/landscape/amazon-ec2.md b/wiki/info/landscape/amazon-ec2.md index 925cd7f5cb4..1f338a2482b 100644 --- a/wiki/info/landscape/amazon-ec2.md +++ b/wiki/info/landscape/amazon-ec2.md @@ -112,7 +112,7 @@ eval `ssh-agent` ssh-add ``` -You will also need the AWS CLI and can run `./awsmfalogon.sh` to authenticate and gain a session token. Next, you should notify the community that internal services, such as Bugzilla will temporarily be down (make sure to notify them afterwards too, so they can continue their work). +You will also need the AWS CLI and must run `./awsmfalogon.sh` before running the first script below, to authenticate and gain a session token. Next, you should notify the community that internal services, such as Bugzilla will temporarily be down (make sure to notify them afterwards too, so they can continue their work). Using the landscape tab of the admin console, ensure there is a disposable in the same AZ as the archive (this ensures we can still route traffic to the archive). From e6327a830538a5d00938735dcf2e73d5d0107201 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Fri, 31 May 2024 14:23:34 +0200 Subject: [PATCH 081/134] more Wiki entries describing the tasks for switching to a different domain --- wiki/planning/domain-switchover.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/wiki/planning/domain-switchover.md b/wiki/planning/domain-switchover.md index 599497bc902..0bebf403df1 100644 --- a/wiki/planning/domain-switchover.md +++ b/wiki/planning/domain-switchover.md @@ -13,9 +13,11 @@ - The ``target-group-tag-route53-nfs-elasticIP-setup.sh`` script used as the last stage of the setup process of a new central reverse proxy. It queries the Route 53 ID of the "sapsailing.com" hosted zone which would have to become a parameter or would need to be changed in the script. -- Mail sending and Amazon SES. +- Mail sending and Amazon SES: we have Amazon WorkMail accounts, particularly for support@sapsailing.com and also a few other @sapsailing.com addresses, such as marcus.baur@sapsailing.com and jan.hamann@sapsailing.com. There are also technical accounts such as noreply@sapsailing.com. The mail address used for mail sent by the application as well as the mail server to use for sending, including its authentication parameters, can be set in the ``mail.properties`` file. The secrets for this currently come from ``root@sapsailing.com:secrets``. -- ``REPLICATE_MASTER_SERVLET_HOST`` is sometimes hardcoded. +- ``sapsailing.com`` is hardcoded in some of the environments files currently found under ``https://releases.sapsailing.com/environments`` + +- The ``refreshInstance.sh`` script contains around eight references to ``releases.sapsailing.com``. Some of these are about where to download releases from and shall be replaced by logic that obtains those releases from Github. The others are concerned with environments, as referenced by the ``USE_ENVIRONMENT`` variable and are currently expected to be found under ``https://releases.sapsailing.com/environments`` which is just a static share on our central reverse proxy. If that is configured for a different domain then so has the refreshInstance.sh script. - Igtimi Authentication: the Igtimi REST API and authentication scheme uses a callback URL that we have configured on the igtimi.com web site, pointing to sapsailing.com. @@ -23,7 +25,7 @@ - A number of bash scripts have sapsailing.com hardcoded, especially the setup scripts and imageupgrade_functions.sh. Ideally we will need to factor out into a constant that all bash scripts can access. -- Releases and p2 respository references, where "releases.sapsailing.com" will be replaced over time by Github releases which the ``refreshInstance.sh`` script can then download. +- Releases and p2 respository references, where "releases.sapsailing.com" will be replaced over time by Github releases which the ``refreshInstance.sh`` script can then download. The p2 repositories for the AWS SDK combined repository, as well as our base p2 repo that forms an important part of our target platform are currently expected to be found at ``https://p2.sapsailing.com/p2/aws-sdk`` and ``https://p2.sapsailing.com/p2/sailing``, respectively. Also, we have the latest SDBG installable hosted there under ``https://p2.sapsailing.com/p2/sdbg``. The scripts under ``java/com.sap.sailing.targetplatform/scripts`` and ``java/com.sap.sailing.feature.p2build/scripts`` as well as the target platform definitions under ``java/com.sap.sailing.targetplatform/definitions`` reference those p2 repositories. Should this set of p2 repositories be migrated to a different domain, the corresponding scripts and target platform definitions need to be adjusted, and the builds including the builds with an adjusted local target platform, need to be tested thoroughly. - MongoDB references and rabbit references, or any other private IP references in route53. Such references can in particular be found in the ``environments/`` folder on ``releases.sapsailing.com`` for the default parameterization of certain application process types, such as ``security-service-master`` or ``archive-server``. From 31c065e3e20a336db00ab6a3ed99d161ac15cc6d Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sun, 2 Jun 2024 00:54:21 +0200 Subject: [PATCH 082/134] switched to github.com-specifig LXLabs e-mail address for translation git hook --- .../central_reverse_proxy/files/var/www/cgi-bin/github.cgi | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/configuration/environments_scripts/central_reverse_proxy/files/var/www/cgi-bin/github.cgi b/configuration/environments_scripts/central_reverse_proxy/files/var/www/cgi-bin/github.cgi index 6a29f922e39..4ba836711d5 100755 --- a/configuration/environments_scripts/central_reverse_proxy/files/var/www/cgi-bin/github.cgi +++ b/configuration/environments_scripts/central_reverse_proxy/files/var/www/cgi-bin/github.cgi @@ -8,7 +8,11 @@ PUSHER=$( echo "${BODY}" | jq -r '.pusher.email' ) logger -t github-cgi "ref is $REF, pusher was $PUSHER" # For testing: #if [ "${PUSHER}" = "axel.uhl@sap.com" -a "${REF}" = "refs/heads/translation" ]; then -if [ "${PUSHER}" = "tmsatsls+github.tools.sap_service-tip-git@sap.com" -a "${REF}" = "refs/heads/translation" ]; then +tmsatsls+github.com_service-tip-git@sap.com +# The filter in case this is to be used for github.tools.sap: +#if [ "${PUSHER}" = "tmsatsls+github.tools.sap_service-tip-git@sap.com" -a "${REF}" = "refs/heads/translation" ]; then +# The filter for github.com: +if [ "${PUSHER}" = "tmsatsls+github.com_service-tip-git@sap.com" -a "${REF}" = "refs/heads/translation" ]; then echo "Identified a push to refs/heads/translation by ${PUSHER}." echo "Fetching translation branch from github.tools.sap and pushing it to ssh://trac@sapsailing.com/home/trac/git" logger -t github-cgi "fetching translation branch from github.tools.sap and pushing it to ssh://trac@sapsailing.com/home/trac/git" From 62855aa118f1d4d6138c1d18b75bbb3d1c55f038 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sun, 2 Jun 2024 12:47:27 +0200 Subject: [PATCH 083/134] bug6013: fixing NPE --- .../domain/ranking/NonPerformanceCurveRankingMetric.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/ranking/NonPerformanceCurveRankingMetric.java b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/ranking/NonPerformanceCurveRankingMetric.java index d35fd3bc602..09f9a50aaab 100755 --- a/java/com.sap.sailing.domain/src/com/sap/sailing/domain/ranking/NonPerformanceCurveRankingMetric.java +++ b/java/com.sap.sailing.domain/src/com/sap/sailing/domain/ranking/NonPerformanceCurveRankingMetric.java @@ -192,7 +192,7 @@ public abstract class NonPerformanceCurveRankingMetric extends AbstractRankingMe final Competitor who = legWho.getCompetitor(); final Competitor to = legTo.getCompetitor(); if (who == to) { - if (isAssumedToHaveFinishedLeg(timePoint, legWho)) { + if (isAssumedToHaveFinishedLeg(timePoint, legWho) && legWho.getFinishTime() != null) { result = timePoint.until(legWho.getFinishTime()); // negative time; reached end of leg some time ago } else { result = Duration.NULL; // still racing in the leg; reaches its own position in no time From de22cd3a31e1388c5f6a45034be520f8190cae55 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Sun, 2 Jun 2024 20:56:11 +0200 Subject: [PATCH 084/134] Update merge-main-into-docker-17.yml: commit message for auto-merge --- .github/workflows/merge-main-into-docker-17.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/merge-main-into-docker-17.yml b/.github/workflows/merge-main-into-docker-17.yml index 06a46100166..4c69da3bf00 100644 --- a/.github/workflows/merge-main-into-docker-17.yml +++ b/.github/workflows/merge-main-into-docker-17.yml @@ -23,4 +23,5 @@ jobs: type: now from_branch: main target_branch: docker-17 + message: Auto-merging main into docker-17 after successful release build github_token: ${{ secrets.REPO_TOKEN_FOR_MERGE_AND_PUSH }} From 6a50f0ce6497c262e11de32ba8d11e173a5fad15 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Mon, 3 Jun 2024 11:03:25 +0200 Subject: [PATCH 085/134] For logrotation of httpd, systemctl is found under /usr/bin, not /bin --- configuration/environments_scripts/repo/etc/logrotate.d/httpd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/environments_scripts/repo/etc/logrotate.d/httpd b/configuration/environments_scripts/repo/etc/logrotate.d/httpd index 6b1f0b00876..8b0d93d9d6b 100644 --- a/configuration/environments_scripts/repo/etc/logrotate.d/httpd +++ b/configuration/environments_scripts/repo/etc/logrotate.d/httpd @@ -8,7 +8,7 @@ compress delaycompress postrotate - /bin/systemctl reload httpd.service > /dev/null 2>/dev/null || true + /usr/bin/systemctl reload httpd.service > /dev/null 2>/dev/null || true mv /var/log/logrotate-target/*.gz /var/log/old endscript } From aa127b8644906437d748290c010160ebc99c5f30 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Mon, 3 Jun 2024 17:03:15 +0200 Subject: [PATCH 086/134] security-service-master environment also accepts *.sailing.omegatiming.com requests now for cross-domain storage --- .../environments/security-service-master | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/configuration/environments/security-service-master b/configuration/environments/security-service-master index 74bbc94438e..19588f4f4d2 100755 --- a/configuration/environments/security-service-master +++ b/configuration/environments/security-service-master @@ -1,5 +1,19 @@ +TELNET_PORT=14888 +SERVER_PORT=8888 +MONGODB_HOST=dbserver.internal.sapsailing.com +MONGODB_PORT=10202 +EXPEDITION_PORT=2010 +DEPLOY_TO=security_service +REPLICATE_MASTER_SERVLET_PORT=8888 +REPLICATE_MASTER_QUEUE_HOST=rabbit.internal.sapsailing.com +REPLICATE_MASTER_QUEUE_PORT=5672 +REPLICATION_HOST=rabbit.internal.sapsailing.com +ADDITIONAL_JAVA_ARGS="$ADDITIONAL_JAVA_ARGS -Dorg.eclipse.jetty.LEVEL=OFF -Dorg.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog -Drestore.tracked.races=true -Dpolardata.source.url=https://www.sapsailing.com -XX:ThreadPriorityPolicy=2" +SERVER_STARTUP_NOTIFY=axel.uhl@sap.com +BUILD_COMPLETE_NOTIFY=axel.uhl@sap.com + # To enable the use of the shared SecurityService and SharedSailingData from security-service.sapsailing.com: -ADDITIONAL_JAVA_ARGS="$ADDITIONAL_JAVA_ARGS -Dsecurity.sharedAcrossSubdomainsOf=sapsailing.com -Dsecurity.baseUrlForCrossDomainStorage=https://security-service.sapsailing.com -Dgwt.acceptableCrossDomainStorageRequestOriginRegexp=https?://(.*\.)?sapsailing\.com(:[0-9]*)?$" +ADDITIONAL_JAVA_ARGS="$ADDITIONAL_JAVA_ARGS -Dsecurity.sharedAcrossSubdomainsOf=sapsailing.com -Dsecurity.baseUrlForCrossDomainStorage=https://security-service.sapsailing.com -Dgwt.acceptableCrossDomainStorageRequestOriginRegexp=https?://((.*\.)?sapsailing\.com)|((.*\.)?sailing\.omegatiming\.com)(:[0-9]*)?$" # For the security-service master instance, don't activate replication from security-service #REPLICATE_ON_START=com.sap.sse.security.impl.SecurityServiceImpl,com.sap.sailing.shared.server.impl.SharedSailingDataImpl,com.sap.sse.landscape.aws.impl.AwsLandscapeStateImpl #REPLICATE_MASTER_SERVLET_HOST=security-service.sapsailing.com From 6725225c5f126b35d9b2ae77c19fba1bb91ede55 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Tue, 4 Jun 2024 10:23:00 +0200 Subject: [PATCH 087/134] use GIT_REPOSITORY constant in sailing script after it has been defined --- .../sailing_server/files/usr/local/bin/sailing | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configuration/environments_scripts/sailing_server/files/usr/local/bin/sailing b/configuration/environments_scripts/sailing_server/files/usr/local/bin/sailing index 294b4d8b7c3..04f86e9785d 100755 --- a/configuration/environments_scripts/sailing_server/files/usr/local/bin/sailing +++ b/configuration/environments_scripts/sailing_server/files/usr/local/bin/sailing @@ -31,11 +31,11 @@ echo "Executing with $1 at `date`" >>/var/log/sailing.err start_servers() { /usr/local/bin/update_authorized_keys_for_landscape_managers $( cat ${SSH_KEY_READER_BEARER_TOKEN} ) https://security-service.sapsailing.com /root 2>&1 >>/var/log/sailing.err - cp /home/sailing/code/configuration/cp_root_mail_properties /usr/local/bin + cp ${GIT_REPOSITORY}/configuration/cp_root_mail_properties /usr/local/bin chown root /usr/local/bin/cp_root_mail_properties chgrp root /usr/local/bin/cp_root_mail_properties chmod 755 /usr/local/bin/cp_root_mail_properties - cp /home/sailing/code/configuration/cp_root_mail_properties_sudoers /etc/sudoers.d + cp ${GIT_REPOSITORY}/configuration/cp_root_mail_properties_sudoers /etc/sudoers.d if which $EC2_METADATA_CMD && $EC2_METADATA_CMD -d | sed "s/user-data\: //g" | grep "^image-upgrade$"; then echo "Found image-upgrade in EC2 user data; upgrading image, then probably shutting down for AMI creation depending on the no-shutdown user data string..." >>/var/log/sailing.err $GIT_REPOSITORY/configuration/imageupgrade.sh From 92c4dccd4b2889fd5553ac1a74d54fcdcfff8019 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Tue, 4 Jun 2024 14:37:16 +0200 Subject: [PATCH 088/134] cache CIDR to IP list resolution to save CPU --- .../www/cgi-bin/reverseProxyHealthcheck.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/configuration/environments_scripts/repo/var/www/cgi-bin/reverseProxyHealthcheck.sh b/configuration/environments_scripts/repo/var/www/cgi-bin/reverseProxyHealthcheck.sh index e721f964aac..76b5c80f0d7 100755 --- a/configuration/environments_scripts/repo/var/www/cgi-bin/reverseProxyHealthcheck.sh +++ b/configuration/environments_scripts/repo/var/www/cgi-bin/reverseProxyHealthcheck.sh @@ -101,11 +101,28 @@ randomise() { echo $(($RANDOM % 5 + $1 )) } +getListOfIpsForCidr() { + # $1: a CIDR, such as 172.10.0.0/32 + # Outputs the list of IP addresses that match the CIDR. + # Tries to read this from a file in ${CACHE_LOCATION} + # whose name is inferred from the CIDR parameter. If not + # found, nmap is used to produce that list, and it is + # written to the file in the ${CACHE_LOCATION} so it can + # be found the next time requested + cidr="${1}" + FILE_WITH_IPS_FOR_CIDR=${CACHE_LOCATION}/$( echo "${cidr}" | tr '/' '_' ) + if [ -f ${FILE_WITH_IPS_FOR_CIDR} ]; then + cat ${FILE_WITH_IPS_FOR_CIDR} + else + nmap -sL -n ${cidr} | head -n -1 | tail -n +2 | sed -e 's/^.*\<\([0-9]*\.[0-9]*\.[0-9]*\.[0-9]*\)$/\1/' | tee ${FILE_WITH_IPS_FOR_CIDR} + fi +} + getAzCidr() { # $1: path to a file containing one CIDR per line # $2: an IP address to obtain the subnet CIDR for for cidr in $( cat "${1}" ); do - if nmap -sL -n ${cidr} | head -n -1 | tail -n +2 | grep -q ${2}; then + if getListOfIpsForCidr ${cidr} | grep -q ${2}; then echo ${cidr} fi done From 3dd38150880ec533f224f72a5df648f0ac1e81e5 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Wed, 5 Jun 2024 00:24:18 +0200 Subject: [PATCH 089/134] new Dockerfile set-up for ARM --- docker/Dockerfile-raspi.tpl | 18 --------------- docker/makeImageForLatestRelease-arm | 31 ++++++++++++++++++++++++++ docker/makeImageForLatestRelease-raspi | 25 --------------------- 3 files changed, 31 insertions(+), 43 deletions(-) delete mode 100755 docker/Dockerfile-raspi.tpl create mode 100755 docker/makeImageForLatestRelease-arm delete mode 100755 docker/makeImageForLatestRelease-raspi diff --git a/docker/Dockerfile-raspi.tpl b/docker/Dockerfile-raspi.tpl deleted file mode 100755 index 6cda42db1e4..00000000000 --- a/docker/Dockerfile-raspi.tpl +++ /dev/null @@ -1,18 +0,0 @@ -FROM donaldduck70/openjdk:1.8.0_242 -LABEL maintainer=axel.uhl@gmx.de -# Download and extract the release -WORKDIR /home/sailing/servers/server -RUN apt-get update \ - && apt-get install -y vim telnet dnsutils net-tools jq -COPY vimrc /root/.vimrc -RUN wget -O /tmp/rds.pem https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem \ - && keytool -importcert -alias AWSRDS -file /tmp/rds.pem -keystore /usr/lib/jvm/java-8-openjdk-arm64/jre/lib/security/cacerts -noprompt -storepass changeit \ - && rm /tmp/rds.pem -RUN wget -O /tmp/RELEASE.tar.gz http://releases.sapsailing.com/RELEASE/RELEASE.tar.gz \ - && tar xzvpf /tmp/RELEASE.tar.gz \ - && rm /tmp/RELEASE.tar.gz -COPY env.sh . -COPY start . -COPY JavaSE-11.profile . -EXPOSE 8888 14888 8000 7091 6666 -CMD [ "/home/sailing/servers/server/start", "fg" ] diff --git a/docker/makeImageForLatestRelease-arm b/docker/makeImageForLatestRelease-arm new file mode 100755 index 00000000000..45ee6485b36 --- /dev/null +++ b/docker/makeImageForLatestRelease-arm @@ -0,0 +1,31 @@ +#!/bin/bash +release_prefix=$1 +GITROOT="`dirname $0`/.." +DOCKERDIR="${GITROOT}/docker" +DOCKERFILE="$DOCKERDIR/Dockerfile" +if [ "${release_prefix}" = "" ]; then + SET_LATEST=1 + release_prefix="main-" +fi +pushd "${DOCKERDIR}" +RELEASE_TAR_GZ_FILENAME=$( ${GITROOT}/configuration/github-download-release-assets.sh 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 "${release_prefix}" ) +if [ "${RELEASE_TAR_GZ_FILENAME}" = "" ]; then + echo "No release with prefix ${release_prefix} found" >&2 +else + release=$( echo ${RELEASE_TAR_GZ_FILENAME} | sed -e 's/\.tar\.gz//' ) + echo Release is ${release} + echo "Copying files from GITROOT $GITROOT into Docker workspace" + cp "$GITROOT/java/target/env.sh" "$DOCKERDIR" + cp "$GITROOT/java/target/start" "$DOCKERDIR" + cp "$GITROOT/java/target/configuration/JavaSE-11.profile" "$DOCKERDIR" + cd "$DOCKERDIR" + docker build --build-arg RELEASE=${release} -t ghcr.io/sap/sailing-analytics:${release}-arm . + echo "Cleaning up..." + rm start env.sh JavaSE-11.profile ${RELEASE_TAR_GZ_FILENAME} release-notes.txt + docker push ghcr.io/sap/sailing-analytics:${release}-arm + if [ "$SET_LATEST" = "1" ]; then + docker tag ghcr.io/sap/sailing-analytics:${release}-arm ghcr.io/sap/sailing-analytics:latest-arm + docker push ghcr.io/sap/sailing-analytics:latest-arm + fi +fi +popd diff --git a/docker/makeImageForLatestRelease-raspi b/docker/makeImageForLatestRelease-raspi deleted file mode 100755 index a5b0477f3fb..00000000000 --- a/docker/makeImageForLatestRelease-raspi +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -release=$1 -if [ "${release}" = "" ]; then - SET_LATEST=1 - release=$(wget -O - http://releases.sapsailing.com/ 2>/dev/null | grep build- | tail -1 | sed -e 's/^.*\(build-[0-9]*\).*$/\1/') - echo Latest release is ${release} -fi -GITROOT="`dirname $0`/.." -DOCKERDIR="${GITROOT}/docker" -DOCKERFILE="$DOCKERDIR/Dockerfile-raspi" -echo "Copying files from GITROOT $GITROOT into Docker workspace" -cp "$GITROOT/java/target/env.sh" "$DOCKERDIR" -cp "$GITROOT/java/target/start" "$DOCKERDIR" -cp "$GITROOT/java/target/configuration/JavaSE-11.profile" "$DOCKERDIR" -cat ${DOCKERFILE}.tpl | sed -e "s/RELEASE/${release}/g" >${DOCKERFILE} -cd "$DOCKERDIR" -docker build -t sapsailing:${release}-raspi -f ${DOCKERFILE} . -echo "Cleaning up..." -rm start env.sh JavaSE-11.profile ${Dockerfile} -docker tag sapsailing:${release}-raspi donaldduck70/sapsailing:${release}-raspi -docker push donaldduck70/sapsailing:${release}-raspi -if [ "$SET_LATEST" = "1" ]; then - docker tag sapsailing:${release}-raspi donaldduck70/sapsailing:latest-raspi - docker push donaldduck70/sapsailing:latest-raspi -fi From 75f4f2e228395b8b1e58026526aad677a284256f Mon Sep 17 00:00:00 2001 From: Generic Wiki User Date: Wed, 5 Jun 2024 00:30:01 +0000 Subject: [PATCH 090/134] use multi-platform build for Docker images (amd64 / arm64) --- docker/makeImageForLatestRelease | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/makeImageForLatestRelease b/docker/makeImageForLatestRelease index b7aa741a07c..2d5e8e5b5fa 100755 --- a/docker/makeImageForLatestRelease +++ b/docker/makeImageForLatestRelease @@ -19,7 +19,7 @@ else cp "$GITROOT/java/target/start" "$DOCKERDIR" cp "$GITROOT/java/target/configuration/JavaSE-11.profile" "$DOCKERDIR" cd "$DOCKERDIR" - docker build --build-arg RELEASE=${release} -t ghcr.io/sap/sailing-analytics:${release} . + docker buildx build --builder=container --platform=linux/amd64,linux/arm64 --build-arg RELEASE=${release} -t ghcr.io/sap/sailing-analytics:${release} --push . echo "Cleaning up..." rm start env.sh JavaSE-11.profile ${RELEASE_TAR_GZ_FILENAME} release-notes.txt docker push ghcr.io/sap/sailing-analytics:${release} From 21a892e0d35f6f7164dcf822b5dac525e9648352 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Wed, 5 Jun 2024 02:33:35 +0200 Subject: [PATCH 091/134] try to build multi-platform Docker images --- docker/makeImageForLatestRelease | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/makeImageForLatestRelease b/docker/makeImageForLatestRelease index b7aa741a07c..4deddce6bf8 100755 --- a/docker/makeImageForLatestRelease +++ b/docker/makeImageForLatestRelease @@ -19,7 +19,8 @@ else cp "$GITROOT/java/target/start" "$DOCKERDIR" cp "$GITROOT/java/target/configuration/JavaSE-11.profile" "$DOCKERDIR" cd "$DOCKERDIR" - docker build --build-arg RELEASE=${release} -t ghcr.io/sap/sailing-analytics:${release} . + docker buildx create --name=container --driver=docker-container --use --bootstrap + docker buildx build --builder=container --platform=linux/amd64,linux/arm64 --build-arg RELEASE=${release} -t ghcr.io/sap/sailing-analytics:${release} . echo "Cleaning up..." rm start env.sh JavaSE-11.profile ${RELEASE_TAR_GZ_FILENAME} release-notes.txt docker push ghcr.io/sap/sailing-analytics:${release} From 99e3e6c49681a18eb41423ef72c0fec6a9a043ad Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Wed, 5 Jun 2024 11:36:32 +0200 Subject: [PATCH 092/134] try to build docker image for multiple platforms --- .github/workflows/create-docker-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/create-docker-image.yml b/.github/workflows/create-docker-image.yml index 4ade5237ca5..d1715ffb496 100644 --- a/.github/workflows/create-docker-image.yml +++ b/.github/workflows/create-docker-image.yml @@ -56,6 +56,7 @@ jobs: tags: ghcr.io/sap/sailing-analytics:${{ env.RELEASE }}${{ env.BRANCH == 'main' && github.event.inputs.release == '' && ',ghcr.io/sap/sailing-analytics:latest' || env.BRANCH == 'docker-17' && github.event.inputs.release == '' && ',ghcr.io/sap/sailing-analytics:latest-17' || '' }} file: ${{ env.BRANCH == 'docker-17' && 'docker/Dockerfile_sapsailing_on_sapmachine17' || 'docker/Dockerfile' }} context: docker/ + platforms: linux/amd64,linux/arm64 push: true - name: Clean up copied files shell: bash From e7a9839240b871900ebb109e537f2305abfd35cb Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Wed, 5 Jun 2024 11:54:58 +0200 Subject: [PATCH 093/134] add qemu support for multi-platform docker image build --- .github/workflows/create-docker-image.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/create-docker-image.yml b/.github/workflows/create-docker-image.yml index d1715ffb496..e6a182a7e2f 100644 --- a/.github/workflows/create-docker-image.yml +++ b/.github/workflows/create-docker-image.yml @@ -18,6 +18,10 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ github.event_name == 'workflow_dispatch' && github.workflow_dispatch.ref || github.event.workflow_run.head_branch }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Login to ghcr.io to upload sailing-analytics image uses: docker/login-action@v3 with: From 75c61730a9de04280869e752f9999a3184d383b0 Mon Sep 17 00:00:00 2001 From: Thomas Stokes Date: Wed, 5 Jun 2024 11:22:49 +0100 Subject: [PATCH 094/134] b5912: Improve setup_keys comment. --- .../repo/usr/local/bin/imageupgrade_functions.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index 12fc0544e3e..6ae5609167e 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -138,7 +138,8 @@ build_crontab_and_setup_files() { } setup_keys() { - #1: Environment type. + # Installs the necessary aws and ssh keys for a given environment type, by copying from the key vault. + # $1: Environment type. # Optional parameter is -p which indicates that no permissions will be set or overwritten. TEMP=$(getopt -o p -n 'options' -- "$@") [[ "$?" -eq 0 ]] || return 2 From 681bec61cf75e6464f2236d423a0344c70ea184c Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Wed, 5 Jun 2024 15:20:16 +0200 Subject: [PATCH 095/134] bug6007: implemented logging for event and regatta creation --- .../sailing/gwt/ui/server/SailingServiceWriteImpl.java | 8 +++++--- .../sailing/server/gateway/jaxrs/api/EventsResource.java | 1 + .../trackfiles/impl/ExpeditionAllInOneImporter.java | 2 ++ .../operationaltransformation/AddSpecificRegatta.java | 2 +- .../sap/sailing/server/impl/RacingEventServiceImpl.java | 8 ++++---- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/server/SailingServiceWriteImpl.java b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/server/SailingServiceWriteImpl.java index b5cc1ba9693..2e4179f7686 100644 --- a/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/server/SailingServiceWriteImpl.java +++ b/java/com.sap.sailing.gwt.ui/src/main/java/com/sap/sailing/gwt/ui/server/SailingServiceWriteImpl.java @@ -1529,6 +1529,7 @@ public class SailingServiceWriteImpl extends SailingServiceImpl implements Saili new Callable() { @Override public EventDTO call() throws Exception { + logger.info("User "+SecurityUtils.getSubject().getPrincipal()+" is trying to create event "+eventName); final TimePoint startTimePoint = startDate != null ? new MillisecondsTimePoint(startDate) : null; final TimePoint endTimePoint = endDate != null ? new MillisecondsTimePoint(endDate) : null; final URL officialWebsiteURL = officialWebsiteURLAsString != null @@ -1711,14 +1712,15 @@ public class SailingServiceWriteImpl extends SailingServiceImpl implements Saili RegattaCreationParametersDTO seriesNamesWithFleetNamesAndFleetOrderingAndMedal, boolean persistent, ScoringSchemeType scoringSchemeType, List courseAreaIds, Double buoyZoneRadiusInHullLengths, boolean useStartTimeInference, boolean controlTrackingFromStartAndFinishTimes, boolean autoRestartTrackingUponCompetitorSetChange, RankingMetrics rankingMetricType) { + logger.info("User "+SecurityUtils.getSubject().getPrincipal()+" is trying to create regatta "+regattaName); return getSecurityService().setOwnershipCheckPermissionForObjectCreationAndRevertOnError( SecuredDomainType.REGATTA, Regatta.getTypeRelativeObjectIdentifier(regattaName), regattaName, new Callable() { @Override public RegattaDTO call() throws Exception { - TimePoint startTimePoint = startDate != null ? new MillisecondsTimePoint(startDate) : null; - TimePoint endTimePoint = endDate != null ? new MillisecondsTimePoint(endDate) : null; - Regatta regatta = getService().apply(new AddSpecificRegatta(regattaName, boatClassName, + final TimePoint startTimePoint = startDate != null ? new MillisecondsTimePoint(startDate) : null; + final TimePoint endTimePoint = endDate != null ? new MillisecondsTimePoint(endDate) : null; + final Regatta regatta = getService().apply(new AddSpecificRegatta(regattaName, boatClassName, canBoatsOfCompetitorsChangePerRace, competitorRegistrationType, registrationLinkSecret, startTimePoint, endTimePoint, UUID.randomUUID(), seriesNamesWithFleetNamesAndFleetOrderingAndMedal, persistent, diff --git a/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/jaxrs/api/EventsResource.java b/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/jaxrs/api/EventsResource.java index 23599f7f736..b53fbd533dc 100644 --- a/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/jaxrs/api/EventsResource.java +++ b/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/jaxrs/api/EventsResource.java @@ -608,6 +608,7 @@ public class EventsResource extends AbstractSailingServerResource { Callable> doCreationAction = new Callable>() { @Override public Util.Triple call() throws Exception { + logger.info("User "+SecurityUtils.getSubject().getPrincipal()+" is trying to create event "+eventName); Event event = getSecurityService().setOwnershipCheckPermissionForObjectCreationAndRevertOnError( SecuredDomainType.EVENT, EventBaseImpl.getTypeRelativeObjectIdentifier(eventId), eventName, ()->getService().apply(new CreateEvent(eventName, eventDescription, startDate, endDate, venueName, isPublic, eventId, diff --git a/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/trackfiles/impl/ExpeditionAllInOneImporter.java b/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/trackfiles/impl/ExpeditionAllInOneImporter.java index 12fa5926002..cb3d1a1dbad 100644 --- a/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/trackfiles/impl/ExpeditionAllInOneImporter.java +++ b/java/com.sap.sailing.server.gateway/src/com/sap/sailing/server/gateway/trackfiles/impl/ExpeditionAllInOneImporter.java @@ -27,6 +27,7 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.apache.commons.fileupload.FileItem; +import org.apache.shiro.SecurityUtils; import org.apache.shiro.authz.AuthorizationException; import org.apache.shiro.subject.Subject; import org.json.simple.parser.ParseException; @@ -329,6 +330,7 @@ public class ExpeditionAllInOneImporter { final TimePointsOfFirstAndLastFix firstAndLastFixAt = importFixes(filenameWithSuffix, fileItem, jsonHolderForGpsFixImport, jsonHolderForSensorFixImport, errors); final TimePoint eventStartDate = firstAndLastFixAt.getFirstFixAt(); final TimePoint eventEndDate = firstAndLastFixAt.getLastFixAt(); + logger.info("Trying to create event "+eventName+" and regatta "+regattaName+" on behalf of user "+SecurityUtils.getSubject().getPrincipal()); final Iterable> trackedRacesAndRaceColumnNamesAndFleetNames = createEventStructureWithASingleRaceAndTrackIt( filenameWithSuffix, boatClassName, errors, importTimeString, filename, diff --git a/java/com.sap.sailing.server.interface/src/com/sap/sailing/server/operationaltransformation/AddSpecificRegatta.java b/java/com.sap.sailing.server.interface/src/com/sap/sailing/server/operationaltransformation/AddSpecificRegatta.java index 49dbf4162ac..5b639100cea 100755 --- a/java/com.sap.sailing.server.interface/src/com/sap/sailing/server/operationaltransformation/AddSpecificRegatta.java +++ b/java/com.sap.sailing.server.interface/src/com/sap/sailing/server/operationaltransformation/AddSpecificRegatta.java @@ -71,7 +71,7 @@ public class AddSpecificRegatta extends AbstractAddRegattaOperation { @Override public Regatta internalApplyTo(RacingEventService toState) throws Exception { - Regatta regatta = toState.createRegatta(getRegattaName(), getBoatClassName(), + final Regatta regatta = toState.createRegatta(getRegattaName(), getBoatClassName(), canBoatsOfCompetitorsChangePerRace, competitorRegistrationType, registrationLinkSecret, getStartDate(), getEndDate(), getId(), createSeries(toState), persistent, scoringScheme, courseAreaIds, buoyZoneRadiusInHullLengths, useStartTimeInference, controlTrackingFromStartAndFinishTimes, diff --git a/java/com.sap.sailing.server/src/com/sap/sailing/server/impl/RacingEventServiceImpl.java b/java/com.sap.sailing.server/src/com/sap/sailing/server/impl/RacingEventServiceImpl.java index aededa2f460..1cf524d8aeb 100644 --- a/java/com.sap.sailing.server/src/com/sap/sailing/server/impl/RacingEventServiceImpl.java +++ b/java/com.sap.sailing.server/src/com/sap/sailing/server/impl/RacingEventServiceImpl.java @@ -1879,12 +1879,12 @@ Replicator { if (useStartTimeInference && controlTrackingFromStartAndFinishTimes) { throw new IllegalArgumentException("Cannot set both of useStartTimeInference and controlTrackingFromStartAndFinishTimes to true"); } - com.sap.sse.common.Util.Pair regattaWithCreatedFlag = getOrCreateRegattaWithoutReplication( + final com.sap.sse.common.Util.Pair regattaWithCreatedFlag = getOrCreateRegattaWithoutReplication( fullRegattaName, boatClassName, canBoatsOfCompetitorsChangePerRace, competitorRegistrationType, registrationLinkSecret, startDate, endDate, id, series, persistent, scoringScheme, courseAreaIds, buoyZoneRadiusInHullLengths, useStartTimeInference, controlTrackingFromStartAndFinishTimes, autoRestartTrackingUponCompetitorSetChange, rankingMetricConstructor); - Regatta regatta = regattaWithCreatedFlag.getA(); + final Regatta regatta = regattaWithCreatedFlag.getA(); if (regattaWithCreatedFlag.getB()) { onRegattaLikeAdded(regatta); replicateSpecificRegattaWithoutRaceColumns(regatta); @@ -1916,13 +1916,13 @@ Replicator { ScoringScheme scoringScheme, Iterable courseAreaIds, Double buoyZoneRadiusInHullLengths, boolean useStartTimeInference, boolean controlTrackingFromStartAndFinishTimes, boolean autoRestartTrackingUponCompetitorSetChange, RankingMetricConstructor rankingMetricConstructor) { - Regatta regatta = new RegattaImpl(getRaceLogStore(), getRegattaLogStore(), fullRegattaName, + final Regatta regatta = new RegattaImpl(getRaceLogStore(), getRegattaLogStore(), fullRegattaName, getBaseDomainFactory().getOrCreateBoatClass(boatClassName), canBoatsOfCompetitorsChangePerRace, competitorRegistrationType, startDate, endDate, series, persistent, scoringScheme, id, Util.map(courseAreaIds, courseAreaId->getBaseDomainFactory().getExistingCourseAreaById(courseAreaId)), buoyZoneRadiusInHullLengths, useStartTimeInference, controlTrackingFromStartAndFinishTimes, autoRestartTrackingUponCompetitorSetChange, rankingMetricConstructor, registrationLinkSecret); - boolean wasCreated = addAndConnectRegatta(persistent, regatta); + final boolean wasCreated = addAndConnectRegatta(persistent, regatta); if (wasCreated) { logger.info("Created regatta " + regatta.getName() + " (" + hashCode() + ") on " + this); } From 09472b3db03e34f80a015d86545c70f51beef22a Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Wed, 5 Jun 2024 15:44:18 +0200 Subject: [PATCH 096/134] set com.sap.sse.security.common to lazy activation --- java/com.sap.sse.security.common/META-INF/MANIFEST.MF | 1 + 1 file changed, 1 insertion(+) diff --git a/java/com.sap.sse.security.common/META-INF/MANIFEST.MF b/java/com.sap.sse.security.common/META-INF/MANIFEST.MF index 0642893650f..b674c9ab4c8 100644 --- a/java/com.sap.sse.security.common/META-INF/MANIFEST.MF +++ b/java/com.sap.sse.security.common/META-INF/MANIFEST.MF @@ -12,3 +12,4 @@ Export-Package: com.sap.sse.security.shared, com.sap.sse.security.shared.subscription.chargebee Require-Bundle: com.sap.sse.common Automatic-Module-Name: com.sap.sse.security.common +Bundle-ActivationPolicy: lazy From 22e3624a729cdfd55a7d695f6a2679bf709d6950 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Wed, 5 Jun 2024 15:45:05 +0200 Subject: [PATCH 097/134] bug5791: fixed test for non-empty variables in env-default-rules.sh; fixed test for empty chargebee values in ChargebeeConfiguration --- .../chargebee/ChargebeeConfiguration.java | 4 +++- java/target/env-default-rules.sh | 22 +++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/java/com.sap.sse.security/src/com/sap/sse/security/subscription/chargebee/ChargebeeConfiguration.java b/java/com.sap.sse.security/src/com/sap/sse/security/subscription/chargebee/ChargebeeConfiguration.java index e621d25e002..f0647b813a6 100644 --- a/java/com.sap.sse.security/src/com/sap/sse/security/subscription/chargebee/ChargebeeConfiguration.java +++ b/java/com.sap.sse.security/src/com/sap/sse/security/subscription/chargebee/ChargebeeConfiguration.java @@ -2,6 +2,8 @@ package com.sap.sse.security.subscription.chargebee; import java.util.logging.Logger; +import com.sap.sse.common.Util; + /** * Subscription configuration. The system will get these information from server application startup arguments: * {@code chargebee.site}, {@code chargebee.apikey} @@ -29,7 +31,7 @@ public class ChargebeeConfiguration { String site = System.getProperty(CHARGEBEE_SITE); String apiKey = System.getProperty(CHARGEBEE_APIKEY); logger.info("Chargebee site: " + site); - if (site != null && apiKey != null) { + if (Util.hasLength(site) && Util.hasLength(apiKey)) { instance = new ChargebeeConfiguration(site, apiKey); logger.info("Activating Chargebee Configuration and service"); } else { diff --git a/java/target/env-default-rules.sh b/java/target/env-default-rules.sh index e013ae0406a..6228c4d58a1 100755 --- a/java/target/env-default-rules.sh +++ b/java/target/env-default-rules.sh @@ -8,7 +8,7 @@ if [ -z "${SERVER_NAME}" ]; then SERVER_NAME=MASTER fi -if [ -n "$AUTO_REPLICATE" ]; then +if [ -n "${AUTO_REPLICATE}" ]; then if [ -z "${REPLICATE_ON_START}" ]; then # To start replication upon startup provide the fully-qualified names of the Replicable service classes # for which to trigger replication. If you activate this make sure to @@ -29,7 +29,7 @@ fi # The name of the message queuing fan-out exchange that this server will use in its role as replication master. # Make sure this is unique so that no other master is writing to this exchange at any time. if [ -z "${REPLICATION_CHANNEL}" ]; then - if [ -n "$AUTO_REPLICATE" ]; then + if [ -n "${AUTO_REPLICATE}" ]; then # This seems to be a replica; use a dedicated outbound channel for "transitive replication" REPLICATION_CHANNEL=${SERVER_NAME}-${INSTANCE_NAME} else @@ -46,7 +46,7 @@ if [ -z "${SERVER_PORT}" ]; then SERVER_PORT=8888 fi if [ -z "${MONGODB_NAME}" ]; then - if [ -n "$AUTO_REPLICATE" ]; then + if [ -n "${AUTO_REPLICATE}" ]; then # This seems to be a replica; use a "phony" DB for all replicas: MONGODB_NAME=${SERVER_NAME}-replica else @@ -58,7 +58,7 @@ if [ -z "${MONGODB_PORT}" ]; then MONGODB_PORT=27017 fi if [ -z "${MONGODB_HOST}" -a -z "${MONGODB_URI}" ]; then - if [ -n "$AUTO_REPLICATE" ]; then + if [ -n "${AUTO_REPLICATE}" ]; then # An auto-replication replica by default assumes it has a local MongoDB replica set running on localhost, # called "replica" and running on the default port 27017: MONGODB_URI="mongodb://localhost/${MONGODB_NAME}?replicaSet=replica&retryWrites=true&readPreference=nearest" @@ -110,24 +110,24 @@ if [ -z "${MEMORY}" ]; then fi export MEMORY="${MEMORY_PER_INSTANCE_IN_MB}m" fi -if [ -n ${CHARGEBEE_SITE} -a -n ${CHARGEBEE_APIKEY} ]; then +if [ -n "${CHARGEBEE_SITE}" -a -n "${CHARGEBEE_APIKEY}" ]; then ADDITIONAL_JAVA_ARGS="${ADDITIONAL_JAVA_ARGS} -Dchargebee.site=${CHARGEBEE_SITE} -Dchargebee.apikey=${CHARGEBEE_APIKEY}" fi -if [ -n ${MANAGE2SAIL_ACCESS_TOKEN} ]; then +if [ -n "${MANAGE2SAIL_ACCESS_TOKEN}" ]; then ADDITIONAL_JAVA_ARGS="${ADDITIONAL_JAVA_ARGS} -Dmanage2sail.accesstoken=${MANAGE2SAIL_ACCESS_TOKEN}" fi -if [ -n ${IGTIMI_CLIENT_ID} ]; then +if [ -n "${IGTIMI_CLIENT_ID}" ]; then ADDITIONAL_JAVA_ARGS="${ADDITIONAL_JAVA_ARGS} -Digtimi.client.id=${IGTIMI_CLIENT_ID}" fi -if [ -n ${IGTIMI_CLIENT_SECRET} ]; then +if [ -n "${IGTIMI_CLIENT_SECRET}" ]; then ADDITIONAL_JAVA_ARGS="${ADDITIONAL_JAVA_ARGS} -Digtimi.client.secret=${IGTIMI_CLIENT_SECRET}" fi -if [ -n ${GOOGLE_MAPS_AUTHENTICATION_PARAMS} ]; then +if [ -n "${GOOGLE_MAPS_AUTHENTICATION_PARAMS}" ]; then ADDITIONAL_JAVA_ARGS="${ADDITIONAL_JAVA_ARGS} -Dgoogle.maps.authenticationparams=${GOOGLE_MAPS_AUTHENTICATION_PARAMS}" fi -if [ -n ${WINDESTIMATION_BEARER_TOKEN} ]; then +if [ -n "${WINDESTIMATION_BEARER_TOKEN}" ]; then ADDITIONAL_JAVA_ARGS="${ADDITIONAL_JAVA_ARGS} -Dwindestimation.source.bearertoken=${WINDESTIMATION_BEARER_TOKEN}" fi -if [ -n ${POLAR_DATA_BEARER_TOKEN} ]; then +if [ -n "${POLAR_DATA_BEARER_TOKEN}" ]; then ADDITIONAL_JAVA_ARGS="${ADDITIONAL_JAVA_ARGS} -Dpolardata.source.bearertoken=${POLAR_DATA_BEARER_TOKEN}" fi From 2771428a1e27168f59fc6a5e86d72cff31c72da9 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Wed, 5 Jun 2024 16:55:25 +0200 Subject: [PATCH 098/134] build multi-platform (amd64 / arm64) docker images --- docker/makeImageForLatestRelease | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docker/makeImageForLatestRelease b/docker/makeImageForLatestRelease index 4deddce6bf8..daaccfeb9b4 100755 --- a/docker/makeImageForLatestRelease +++ b/docker/makeImageForLatestRelease @@ -20,13 +20,8 @@ else cp "$GITROOT/java/target/configuration/JavaSE-11.profile" "$DOCKERDIR" cd "$DOCKERDIR" docker buildx create --name=container --driver=docker-container --use --bootstrap - docker buildx build --builder=container --platform=linux/amd64,linux/arm64 --build-arg RELEASE=${release} -t ghcr.io/sap/sailing-analytics:${release} . + docker buildx build --builder=container --platform=linux/amd64,linux/arm64 --build-arg RELEASE=${release} -t ghcr.io/sap/sailing-analytics:${release} --push . echo "Cleaning up..." rm start env.sh JavaSE-11.profile ${RELEASE_TAR_GZ_FILENAME} release-notes.txt - docker push ghcr.io/sap/sailing-analytics:${release} - if [ "$SET_LATEST" = "1" ]; then - docker tag ghcr.io/sap/sailing-analytics:${release} ghcr.io/sap/sailing-analytics:latest - docker push ghcr.io/sap/sailing-analytics:latest - fi fi popd From 8dbaa125ff7b233740873321d5ccf6b58f01d127 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Wed, 5 Jun 2024 15:53:12 +0000 Subject: [PATCH 099/134] Deleted wiki/passwords.md --- wiki/passwords.md | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 wiki/passwords.md diff --git a/wiki/passwords.md b/wiki/passwords.md deleted file mode 100644 index 412b3d151ce..00000000000 --- a/wiki/passwords.md +++ /dev/null @@ -1,9 +0,0 @@ ------BEGIN PGP MESSAGE----- -Version: GnuPG v1.4.5 (GNU/Linux) - -jA0EAwMCpdXEg2lB49NgyaannCeG3Sc67wTHIxMbx8Jxxp9jc8zEYPEo9FmIgfda -lDarXZBzMwtp0ZslrwV5NVdf/ozQDnKCxNq/016jl4zKrmCDT+hnnKKBtSvhBo0v -yPteBdq+9zul/pphLdAx70L8VDmzS96K8W+MY5XKGNu+lWjoupp6Z6hZsHdq/bXb -KKaN44y8Su8bAcuSQAxoCFeujjqEDJYdsmu1OhG+De06bGNTCDfx -=uuLi ------END PGP MESSAGE----- From 5ab776018b971eb0ae1931fe77caa11f81f61e9e Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Wed, 5 Jun 2024 15:53:25 +0000 Subject: [PATCH 100/134] Deleted wiki/liveraces.md --- wiki/liveraces.md | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 wiki/liveraces.md diff --git a/wiki/liveraces.md b/wiki/liveraces.md deleted file mode 100644 index efdc2877a00..00000000000 --- a/wiki/liveraces.md +++ /dev/null @@ -1,3 +0,0 @@ -# Live races - -https://bugzilla.sapsailing.com/bugzilla/show_bug.cgi?id=3720 \ No newline at end of file From fb5e3672ec36f3383592889d5de9ae4874b506f9 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Wed, 5 Jun 2024 18:13:53 +0200 Subject: [PATCH 101/134] improvements of the README.md file --- README.md | 69 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 6bdade97156..a8a9ef5d0fa 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## About this Project -The Sailing Analytics, formerly known as the "SAP Sailing Analytics," are a solution for portraying and analyzing sailing regattas, supporting training scenarios, and powering the vast archive at https://sapsailing.com. +The Sailing Analytics, formerly known as the "SAP Sailing Analytics," are a solution for portraying and analyzing sailing regattas, supporting training scenarios, and powering the vast archive at https://sapsailing.com. The solution consists of a cloud application with a web-based user interface, as well as three companion apps that integrate with the cloud application. This repository has the code for the cloud-based web application, and two of the three mobile apps (Buoy Pinger and Race Manager). The third companion app (Sail Insight) is found in [another repository](https://github.com/SailTracks/sailinsight). ## Description @@ -16,9 +16,30 @@ Sailing provides the perfect platform for SAP to showcase solutions and help the SAP has a longstanding involvement with sailing and has established a portfolio spanning across teams and regattas. +## Quick Start with Docker Compose + +If you have built or obtained the ``ghcr.io/sap/sailing-analytics:latest`` multi-platform image (currently available for linux/amd64 and linux/arm64 architectures), try this: +``` + cd docker + docker-compose up +``` +Based on the ``docker/docker-compose.yml`` definition you should end up with three running Docker containers: +- a MongoDB server, listening on default port 27017 +- a RabbitMQ server, listening on default port +- a Sailing Analytics server, listening for HTTP requests on port 8888 and for telnet connections to the OSGi console on port 14888 + +Try a request to [``http://127.0.0.1:8888/index.html``](http://127.0.0.1:8888/index.html) or [``http://127.0.0.1:8888/gwt/status``](http://127.0.0.1:8888/gwt/status) to see if things worked. + +To use Java 17, use the ``docker-compose-17.yml`` file instead: + +``` + cd docker + docker-compose -f docker-compose-17.yml up +``` + ## Requirements -The software can be run on any Linux or Windows machine with ``bash`` installed; it has also been compiled successfully for the ARM platform and was deployed to a Raspberry Pi computer. As a database, MongoDB is required, tested with releases 4.4, 5.0, and 6.0. For use in a replicated scenario (scale-out, high availability), RabbitMQ is required. Docker Compose can be used to tie these three components together, as Docker images are produced on a regular basis. +The software can be run on any Linux or Windows machine with ``bash`` installed; it has also been compiled successfully for the ARM platform and was deployed to a Raspberry Pi computer. As a database, MongoDB is required, tested with releases 4.4, 5.0, and 6.0. For use in a replicated scenario (scale-out, high availability), RabbitMQ is required. A simple Docker Compose set-up can be used to tie these three components together, e.g., for a quick local test and to familiarize yourself with the application, as Docker images are produced on a regular basis. Compute node and database sizing depends on several aspects of your workloads, such as whether live or replay data is to be served, how many different classes with separate leaderboard are racing concurrently, how many competitors are racing in each class, or how many concurrent viewers produce how many requests and which type (e.g., analytical, data mining, or watching a live race). @@ -28,11 +49,13 @@ To quantify this at least approximately, here are a few examples for typical nod - Large multi-class event with 15 classes with their separate leaderboards, concurrently racing on six course areas: 16GB of RAM, 16 CPUs - Archive of 30,000 races with a few thousand visitors per day with varying analytical and replay workloads: 64GB RAM, 2TB NVMe swap, 8 CPUs -A single node typically handles up to 500 concurrent viewers for live events. You will want to scale out accordingly, using the replication pattern offered by the solution which uses RabbitMQ for transaction log shipping. +A single node typically handles up to 500-1000 concurrent viewers for live events. You will want to scale out accordingly, using the replication pattern offered by the solution which uses RabbitMQ for transaction log shipping. ## Contributing -To start contributing, read the onboarding document at the following URL: [https://wiki.sapsailing.com/wiki/howto/onboarding](https://wiki.sapsailing.com/wiki/howto/onboarding). The project welcomes contributions in the form of pull requests, for example, enhancements of the Data Mining functionality, including any sailing-specific metric or dimension you may think of and that you find is still missing so far; or additional features for the race viewer; or a map visualization that does not require a Google Map but uses Open Street Map / Open Layers; landscape automation; improved start sequence analytics; major UI improvements for the administrative layer ("AdminConsole"), etc. +To start contributing, read the onboarding document at the following URL: [https://wiki.sapsailing.com/wiki/howto/onboarding](https://wiki.sapsailing.com/wiki/howto/onboarding). Further documentation can be found in our Wiki at [https://wiki.sapsailing.com](https://wiki.sapsailing.com), serving the content from this repo's ``wiki/`` folder. + +The project welcomes contributions in the form of pull requests, for example, enhancements of the Data Mining functionality, including any sailing-specific metric or dimension you may think of and that you find is still missing so far; or additional features for the race viewer; or a map visualization that does not require a Google Map but uses Open Street Map / Open Layers; landscape automation; improved start sequence analytics; major UI improvements for the administrative layer ("AdminConsole"), etc. The issue tracker at [https://bugzilla.sapsailing.com](https://bugzilla.sapsailing.com) is currently used for any sort of issue and enhancement request tracking. Help to migrate this smoothly to Github Issues would be much appreciated, ideally keeping issue numbers stable due to many references to those Bugzilla bug numbers, be it in the source code, the Wiki, or the build infrastructure. @@ -49,16 +72,29 @@ See [here](https://www.sapsailing.com/gwt/Home.html#/imprint/:) for a list of co ## Building and Running -To build, invoke +This assumes you have completed the onboarding (see again [here](https://wiki.sapsailing.com/wiki/howto/onboarding)) successfully. To build, then invoke + ``` configuration/buildAndUpdateProduct.sh build ``` -If the build was successful you can install the product locally by invoking +This will build the Android companion apps first, then the web application. If the build was successful you can install the product locally by invoking + ``` configuration/buildAndUpdateProduct.sh install [ -s ] ``` + The default server name is taken to be your current branch name, e.g., ``master``. The install goes to ``${HOME}/servers/{server-name}``. You will find a ``start`` script there which you can use to launch the product. +Or you create a release by running + +``` + configuration/buildAndUpdateProduct.sh -L -u -n release +``` +which produces a tarball under ``dist/{release-name}-{timestamp}/{release-name}-{timestamp}.tar.gz`` which can then be used for ``scp``-based downloads with the ``refreshInstance.sh`` script under ``java/target``. + +Run the ``buildAndUpdateProduct.sh`` without any arguments to see the sub-commands and options available. + + ## Downloading, Installing and Running an Official Release You need to have Java 8 installed. Get one from [here](https://tools.eu1.hana.ondemand.com/#cloud). Either ensure that this JVM's ``java`` executable in on the ``PATH`` or set ``JAVA_HOME`` appropriately. @@ -124,27 +160,6 @@ Then patch or copy ``docker/Dockerfile`` so that it has (of course with the version tag adjusted to what you used above when you built/tagged the image). This will give you an SAP JVM 8 under ``/opt/sapjvm_8`` in the container which in particular includes the useful ``jvmmon`` utility specific to the SAP JVM 8. -## Docker Compose - -If you have built or obtained the ``ghcr.io/sap/sailing-analytics:latest`` image, try this: -``` - cd docker - docker-compose up -``` -Based on the ``docker/docker-compose.yml`` definition you should end up with three running Docker containers: -- a MongoDB server, listening on default port 27017 -- a RabbitMQ server, listening on default port -- a Sailing Analytics server, listening for HTTP requests on port 8888 and for telnet connections to the OSGi console on port 14888 - -Try a request to [``http://127.0.0.1:8888/index.html``](http://127.0.0.1:8888/index.html) or [``http://127.0.0.1:8888/gwt/status``](http://127.0.0.1:8888/gwt/status) to see if things worked. - -To use Java 17, use the ``docker-compose-17.yml`` file instead: - -``` - cd docker - docker-compose -f docker-compose-17.yml up -``` - ## Configuration Options, Environment Variables The server process can be configured in various ways. The corresponding environment variables you may use during installation with ``refreshInstance.sh`` and for setting up your Docker environment can be found in the following files: From 12f985cd70d2f2bbdbb190336d44012f27a0543b Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Thu, 6 Jun 2024 00:56:36 +0200 Subject: [PATCH 102/134] adding annotations to multi-platform docker images --- .github/workflows/create-docker-image.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/create-docker-image.yml b/.github/workflows/create-docker-image.yml index e6a182a7e2f..e16241c9f0c 100644 --- a/.github/workflows/create-docker-image.yml +++ b/.github/workflows/create-docker-image.yml @@ -53,11 +53,20 @@ jobs: mv ${RELEASE_TAR_GZ_FILENAME} docker/ fi echo "RELEASE=${RELEASE}" >>${GITHUB_ENV} + - name: Create Annotations + id: meta + uses: docker/metadata-action@v5 + with: + labels: | + maintainer=axel.uhl@sap.com + org.opencontainers.image.title=Sailing Analytics + org.opencontainers.image.description=The Sailing Analytics Web Application - name: Build & push Docker image uses: docker/build-push-action@v5 with: build-args: RELEASE=${{ env.RELEASE }} tags: ghcr.io/sap/sailing-analytics:${{ env.RELEASE }}${{ env.BRANCH == 'main' && github.event.inputs.release == '' && ',ghcr.io/sap/sailing-analytics:latest' || env.BRANCH == 'docker-17' && github.event.inputs.release == '' && ',ghcr.io/sap/sailing-analytics:latest-17' || '' }} + annotations: ${{ steps.meta.outputs.annotations }} file: ${{ env.BRANCH == 'docker-17' && 'docker/Dockerfile_sapsailing_on_sapmachine17' || 'docker/Dockerfile' }} context: docker/ platforms: linux/amd64,linux/arm64 From ebdbfd1132ae9705a598e3cf1d254a46fdadbe43 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Thu, 6 Jun 2024 01:06:04 +0200 Subject: [PATCH 103/134] using "manifest:" as prefix for docker image tag / annotation setting --- .github/workflows/create-docker-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-docker-image.yml b/.github/workflows/create-docker-image.yml index e16241c9f0c..c6b0817ce77 100644 --- a/.github/workflows/create-docker-image.yml +++ b/.github/workflows/create-docker-image.yml @@ -59,8 +59,8 @@ jobs: with: labels: | maintainer=axel.uhl@sap.com - org.opencontainers.image.title=Sailing Analytics - org.opencontainers.image.description=The Sailing Analytics Web Application + manifest:org.opencontainers.image.title=Sailing Analytics + manifest:org.opencontainers.image.description=The Sailing Analytics Web Application - name: Build & push Docker image uses: docker/build-push-action@v5 with: From ede9848658afa42abe6cb746037e4dcf46115d61 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Thu, 6 Jun 2024 01:12:28 +0200 Subject: [PATCH 104/134] provide annotations explicitly to buildx --- .github/workflows/create-docker-image.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/create-docker-image.yml b/.github/workflows/create-docker-image.yml index c6b0817ce77..b38cac5d722 100644 --- a/.github/workflows/create-docker-image.yml +++ b/.github/workflows/create-docker-image.yml @@ -53,20 +53,15 @@ jobs: mv ${RELEASE_TAR_GZ_FILENAME} docker/ fi echo "RELEASE=${RELEASE}" >>${GITHUB_ENV} - - name: Create Annotations - id: meta - uses: docker/metadata-action@v5 - with: - labels: | - maintainer=axel.uhl@sap.com - manifest:org.opencontainers.image.title=Sailing Analytics - manifest:org.opencontainers.image.description=The Sailing Analytics Web Application - name: Build & push Docker image uses: docker/build-push-action@v5 with: build-args: RELEASE=${{ env.RELEASE }} tags: ghcr.io/sap/sailing-analytics:${{ env.RELEASE }}${{ env.BRANCH == 'main' && github.event.inputs.release == '' && ',ghcr.io/sap/sailing-analytics:latest' || env.BRANCH == 'docker-17' && github.event.inputs.release == '' && ',ghcr.io/sap/sailing-analytics:latest-17' || '' }} - annotations: ${{ steps.meta.outputs.annotations }} + annotations: | + maintainer=axel.uhl@sap.com + manifest:org.opencontainers.image.title=Sailing Analytics + manifest:org.opencontainers.image.description=The Sailing Analytics Web Application file: ${{ env.BRANCH == 'docker-17' && 'docker/Dockerfile_sapsailing_on_sapmachine17' || 'docker/Dockerfile' }} context: docker/ platforms: linux/amd64,linux/arm64 From d24733179a0e42d61ec1eb70dcf336c865181478 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Thu, 6 Jun 2024 01:28:05 +0200 Subject: [PATCH 105/134] set description on index, not manifests --- .github/workflows/create-docker-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-docker-image.yml b/.github/workflows/create-docker-image.yml index b38cac5d722..a9336d16e86 100644 --- a/.github/workflows/create-docker-image.yml +++ b/.github/workflows/create-docker-image.yml @@ -60,8 +60,8 @@ jobs: tags: ghcr.io/sap/sailing-analytics:${{ env.RELEASE }}${{ env.BRANCH == 'main' && github.event.inputs.release == '' && ',ghcr.io/sap/sailing-analytics:latest' || env.BRANCH == 'docker-17' && github.event.inputs.release == '' && ',ghcr.io/sap/sailing-analytics:latest-17' || '' }} annotations: | maintainer=axel.uhl@sap.com - manifest:org.opencontainers.image.title=Sailing Analytics - manifest:org.opencontainers.image.description=The Sailing Analytics Web Application + index:org.opencontainers.image.title=Sailing Analytics + index:org.opencontainers.image.description=The Sailing Analytics Web Application file: ${{ env.BRANCH == 'docker-17' && 'docker/Dockerfile_sapsailing_on_sapmachine17' || 'docker/Dockerfile' }} context: docker/ platforms: linux/amd64,linux/arm64 From 92c186430d248a594ace0e45c97fe09069e0a8be Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Thu, 6 Jun 2024 13:12:27 +0200 Subject: [PATCH 106/134] Upgraded TracAPI to 3.15.8, supporting DCT, DNE, and RCT --- ...CompetitorStatusWithRaceLogReconciler.java | 135 ++++++++++-------- .../release_notes_admin.html | 5 + .../META-INF/MANIFEST.MF | 2 +- java/com.tractrac.clientmodule/README.txt | 15 ++ .../javadoc/allclasses-frame.html | 6 +- .../javadoc/allclasses-noframe.html | 6 +- .../tractrac/model/lib/api/IIdentifiable.html | 8 +- .../com/tractrac/model/lib/api/INamed.html | 8 +- .../tractrac/model/lib/api/ModelLocator.html | 8 +- .../api/attachment/AbstractAttachable.html | 8 +- .../model/lib/api/attachment/IAttachable.html | 8 +- .../lib/api/attachment/IAttachmentKey.html | 8 +- .../api/attachment/IAttachmentManager.html | 8 +- .../class-use/AbstractAttachable.html | 8 +- .../api/attachment/class-use/IAttachable.html | 8 +- .../attachment/class-use/IAttachmentKey.html | 8 +- .../class-use/IAttachmentManager.html | 8 +- .../lib/api/attachment/package-frame.html | 6 +- .../lib/api/attachment/package-summary.html | 8 +- .../lib/api/attachment/package-tree.html | 8 +- .../model/lib/api/attachment/package-use.html | 8 +- .../lib/api/class-use/IIdentifiable.html | 8 +- .../model/lib/api/class-use/INamed.html | 8 +- .../model/lib/api/class-use/ModelLocator.html | 8 +- .../model/lib/api/data/IControlPassing.html | 8 +- .../model/lib/api/data/IControlPassings.html | 8 +- .../model/lib/api/data/IMessageData.html | 8 +- .../model/lib/api/data/IPosition.html | 43 ++++-- .../model/lib/api/data/IPositionFactory.html | 30 ++-- .../model/lib/api/data/IPositionOffset.html | 8 +- .../model/lib/api/data/IPositionSnapped.html | 10 +- .../model/lib/api/data/IStartStopData.html | 8 +- .../model/lib/api/data/ITimeData.html | 8 +- .../api/data/class-use/IControlPassing.html | 8 +- .../api/data/class-use/IControlPassings.html | 8 +- .../lib/api/data/class-use/IMessageData.html | 8 +- .../lib/api/data/class-use/IPosition.html | 18 +-- .../api/data/class-use/IPositionFactory.html | 8 +- .../api/data/class-use/IPositionOffset.html | 8 +- .../api/data/class-use/IPositionSnapped.html | 8 +- .../api/data/class-use/IStartStopData.html | 8 +- .../lib/api/data/class-use/ITimeData.html | 8 +- .../model/lib/api/data/package-frame.html | 6 +- .../model/lib/api/data/package-summary.html | 8 +- .../model/lib/api/data/package-tree.html | 8 +- .../model/lib/api/data/package-use.html | 8 +- .../lib/api/event/CreateModelException.html | 8 +- .../model/lib/api/event/DataSource.html | 8 +- .../model/lib/api/event/EventType.html | 8 +- .../model/lib/api/event/ICompetitor.html | 8 +- .../model/lib/api/event/ICompetitorClass.html | 8 +- .../tractrac/model/lib/api/event/IEvent.html | 8 +- .../model/lib/api/event/IEventFactory.html | 8 +- .../tractrac/model/lib/api/event/IRace.html | 8 +- .../model/lib/api/event/IRaceCompetitor.html | 8 +- .../model/lib/api/event/IRaceSerie.html | 8 +- .../tractrac/model/lib/api/event/ITeam.html | 8 +- .../api/event/RaceCompetitorStatusType.html | 56 ++++++-- .../lib/api/event/RaceLoadingException.html | 8 +- .../model/lib/api/event/RaceStatusType.html | 8 +- .../lib/api/event/RaceVisibilityType.html | 8 +- .../model/lib/api/event/StartTimeType.html | 8 +- .../event/class-use/CreateModelException.html | 8 +- .../lib/api/event/class-use/DataSource.html | 8 +- .../lib/api/event/class-use/EventType.html | 8 +- .../lib/api/event/class-use/ICompetitor.html | 8 +- .../api/event/class-use/ICompetitorClass.html | 8 +- .../model/lib/api/event/class-use/IEvent.html | 8 +- .../api/event/class-use/IEventFactory.html | 8 +- .../model/lib/api/event/class-use/IRace.html | 8 +- .../api/event/class-use/IRaceCompetitor.html | 8 +- .../lib/api/event/class-use/IRaceSerie.html | 8 +- .../model/lib/api/event/class-use/ITeam.html | 8 +- .../class-use/RaceCompetitorStatusType.html | 8 +- .../event/class-use/RaceLoadingException.html | 8 +- .../api/event/class-use/RaceStatusType.html | 8 +- .../event/class-use/RaceVisibilityType.html | 8 +- .../api/event/class-use/StartTimeType.html | 8 +- .../model/lib/api/event/package-frame.html | 6 +- .../model/lib/api/event/package-summary.html | 8 +- .../model/lib/api/event/package-tree.html | 8 +- .../model/lib/api/event/package-use.html | 8 +- .../model/lib/api/metadata/IMetadata.html | 8 +- .../lib/api/metadata/IMetadataContainer.html | 8 +- .../lib/api/metadata/IMetadataFactory.html | 8 +- .../api/metadata/IPropertiesContainer.html | 8 +- .../lib/api/metadata/class-use/IMetadata.html | 8 +- .../class-use/IMetadataContainer.html | 8 +- .../metadata/class-use/IMetadataFactory.html | 8 +- .../class-use/IPropertiesContainer.html | 8 +- .../model/lib/api/metadata/package-frame.html | 6 +- .../lib/api/metadata/package-summary.html | 8 +- .../model/lib/api/metadata/package-tree.html | 8 +- .../model/lib/api/metadata/package-use.html | 8 +- .../tractrac/model/lib/api/package-frame.html | 6 +- .../model/lib/api/package-summary.html | 8 +- .../tractrac/model/lib/api/package-tree.html | 8 +- .../tractrac/model/lib/api/package-use.html | 8 +- .../model/lib/api/route/IControl.html | 8 +- .../model/lib/api/route/IControlPoint.html | 8 +- .../model/lib/api/route/IControlRoute.html | 8 +- .../model/lib/api/route/IPathRoute.html | 8 +- .../lib/api/route/IPathRouteFactory.html | 8 +- .../model/lib/api/route/IPathSegment.html | 8 +- .../tractrac/model/lib/api/route/IRoute.html | 8 +- .../model/lib/api/route/ISegment.html | 8 +- .../lib/api/route/class-use/IControl.html | 8 +- .../api/route/class-use/IControlPoint.html | 8 +- .../api/route/class-use/IControlRoute.html | 8 +- .../lib/api/route/class-use/IPathRoute.html | 8 +- .../route/class-use/IPathRouteFactory.html | 8 +- .../lib/api/route/class-use/IPathSegment.html | 8 +- .../model/lib/api/route/class-use/IRoute.html | 8 +- .../lib/api/route/class-use/ISegment.html | 8 +- .../model/lib/api/route/package-frame.html | 6 +- .../model/lib/api/route/package-summary.html | 8 +- .../model/lib/api/route/package-tree.html | 8 +- .../model/lib/api/route/package-use.html | 8 +- .../model/lib/api/sensor/ISensorData.html | 8 +- .../lib/api/sensor/class-use/ISensorData.html | 8 +- .../model/lib/api/sensor/package-frame.html | 6 +- .../model/lib/api/sensor/package-summary.html | 8 +- .../model/lib/api/sensor/package-tree.html | 8 +- .../model/lib/api/sensor/package-use.html | 8 +- .../model/lib/api/spatial/ICoordinate.html | 8 +- .../lib/api/spatial/ICoordinateSequence.html | 8 +- .../model/lib/api/spatial/IExtent.html | 8 +- .../model/lib/api/spatial/IGeoCoordinate.html | 8 +- .../api/spatial/class-use/ICoordinate.html | 8 +- .../class-use/ICoordinateSequence.html | 8 +- .../lib/api/spatial/class-use/IExtent.html | 8 +- .../api/spatial/class-use/IGeoCoordinate.html | 8 +- .../model/lib/api/spatial/package-frame.html | 6 +- .../lib/api/spatial/package-summary.html | 8 +- .../model/lib/api/spatial/package-tree.html | 8 +- .../model/lib/api/spatial/package-use.html | 8 +- .../lib/api/IEventSubscriber.html | 8 +- .../subscription/lib/api/IRaceSubscriber.html | 8 +- .../subscription/lib/api/ISubscriber.html | 8 +- .../lib/api/ISubscriberFactory.html | 8 +- .../lib/api/ISubscriberListener.html | 8 +- .../SubscriberInitializationException.html | 8 +- .../lib/api/SubscriptionLocator.html | 8 +- .../lib/api/class-use/IEventSubscriber.html | 8 +- .../lib/api/class-use/IRaceSubscriber.html | 8 +- .../lib/api/class-use/ISubscriber.html | 8 +- .../lib/api/class-use/ISubscriberFactory.html | 8 +- .../api/class-use/ISubscriberListener.html | 8 +- .../SubscriberInitializationException.html | 8 +- .../api/class-use/SubscriptionLocator.html | 8 +- .../ICompetitorSensorDataListener.html | 8 +- .../api/competitor/ICompetitorsListener.html | 8 +- .../lib/api/competitor/IPositionListener.html | 8 +- .../competitor/IPositionOffsetListener.html | 8 +- .../competitor/IPositionSnappedListener.html | 8 +- .../ICompetitorSensorDataListener.html | 8 +- .../class-use/ICompetitorsListener.html | 8 +- .../class-use/IPositionListener.html | 8 +- .../class-use/IPositionOffsetListener.html | 8 +- .../class-use/IPositionSnappedListener.html | 8 +- .../lib/api/competitor/package-frame.html | 6 +- .../lib/api/competitor/package-summary.html | 8 +- .../lib/api/competitor/package-tree.html | 8 +- .../lib/api/competitor/package-use.html | 8 +- .../api/control/IControlPassingsListener.html | 8 +- .../IControlPointPositionListener.html | 8 +- .../IControlPointSensorDataListener.html | 8 +- .../control/IControlRouteChangeListener.html | 8 +- .../lib/api/control/IControlsListener.html | 8 +- .../class-use/IControlPassingsListener.html | 8 +- .../IControlPointPositionListener.html | 8 +- .../IControlPointSensorDataListener.html | 8 +- .../IControlRouteChangeListener.html | 8 +- .../control/class-use/IControlsListener.html | 8 +- .../lib/api/control/package-frame.html | 6 +- .../lib/api/control/package-summary.html | 8 +- .../lib/api/control/package-tree.html | 8 +- .../lib/api/control/package-use.html | 8 +- .../api/event/IConnectionStatusListener.html | 8 +- .../lib/api/event/IEventMessageListener.html | 8 +- .../api/event/ILiveDataEvent.StatusType.html | 8 +- .../lib/api/event/ILiveDataEvent.html | 8 +- .../subscription/lib/api/event/IMessage.html | 8 +- .../lib/api/event/IServerTimeListener.html | 8 +- .../lib/api/event/IStoredDataEvent.Type.html | 8 +- .../lib/api/event/IStoredDataEvent.html | 8 +- .../class-use/IConnectionStatusListener.html | 8 +- .../class-use/IEventMessageListener.html | 8 +- .../class-use/ILiveDataEvent.StatusType.html | 8 +- .../api/event/class-use/ILiveDataEvent.html | 8 +- .../lib/api/event/class-use/IMessage.html | 8 +- .../event/class-use/IServerTimeListener.html | 8 +- .../class-use/IStoredDataEvent.Type.html | 8 +- .../api/event/class-use/IStoredDataEvent.html | 8 +- .../lib/api/event/package-frame.html | 6 +- .../lib/api/event/package-summary.html | 8 +- .../lib/api/event/package-tree.html | 8 +- .../lib/api/event/package-use.html | 8 +- .../subscription/lib/api/package-frame.html | 6 +- .../subscription/lib/api/package-summary.html | 8 +- .../subscription/lib/api/package-tree.html | 8 +- .../subscription/lib/api/package-use.html | 8 +- .../lib/api/race/IRaceCompetitorListener.html | 8 +- .../lib/api/race/IRaceMessageListener.html | 8 +- .../IRaceStartStopTimesChangeListener.html | 8 +- .../lib/api/race/IRacesListener.html | 8 +- .../race/IStartStopTimesChangeListener.html | 8 +- .../class-use/IRaceCompetitorListener.html | 8 +- .../race/class-use/IRaceMessageListener.html | 8 +- .../IRaceStartStopTimesChangeListener.html | 8 +- .../api/race/class-use/IRacesListener.html | 8 +- .../IStartStopTimesChangeListener.html | 8 +- .../lib/api/race/package-frame.html | 6 +- .../lib/api/race/package-summary.html | 8 +- .../lib/api/race/package-tree.html | 8 +- .../lib/api/race/package-use.html | 8 +- .../lib/api/route/IRoutesListener.html | 8 +- .../api/route/class-use/IRoutesListener.html | 8 +- .../lib/api/route/package-frame.html | 6 +- .../lib/api/route/package-summary.html | 8 +- .../lib/api/route/package-tree.html | 8 +- .../lib/api/route/package-use.html | 8 +- .../javadoc/constant-values.html | 8 +- .../javadoc/deprecated-list.html | 8 +- .../javadoc/help-doc.html | 8 +- .../javadoc/index-all.html | 17 ++- .../javadoc/index.html | 4 +- .../javadoc/overview-frame.html | 6 +- .../javadoc/overview-summary.html | 10 +- .../javadoc/overview-tree.html | 8 +- .../javadoc/serialized-form.html | 8 +- .../lib/TracAPI-src.jar | Bin 4265216 -> 4265419 bytes .../com.tractrac.clientmodule/lib/TracAPI.jar | Bin 9209390 -> 9210339 bytes java/com.tractrac.clientmodule/pom.xml | 2 +- 234 files changed, 1085 insertions(+), 980 deletions(-) diff --git a/java/com.sap.sailing.domain.tractracadapter/src/com/sap/sailing/domain/tractracadapter/impl/RaceAndCompetitorStatusWithRaceLogReconciler.java b/java/com.sap.sailing.domain.tractracadapter/src/com/sap/sailing/domain/tractracadapter/impl/RaceAndCompetitorStatusWithRaceLogReconciler.java index 65d99fb9902..20d91b82124 100755 --- a/java/com.sap.sailing.domain.tractracadapter/src/com/sap/sailing/domain/tractracadapter/impl/RaceAndCompetitorStatusWithRaceLogReconciler.java +++ b/java/com.sap.sailing.domain.tractracadapter/src/com/sap/sailing/domain/tractracadapter/impl/RaceAndCompetitorStatusWithRaceLogReconciler.java @@ -311,68 +311,81 @@ public class RaceAndCompetitorStatusWithRaceLogReconciler { */ private MaxPointsReason getMaxPointsReason(RaceCompetitorStatusType raceCompetitorStatusType) { MaxPointsReason result; - switch (raceCompetitorStatusType) { - // TODO we expect NSC and perhaps also TLE to show up; support the mapping to MaxPointsReason.NSC / TLE, respectively when they appear - case ABANDONED: - result = MaxPointsReason.NONE; // TODO bug 5154: find out what ABANDONED means and if/how we can translate it to a MaxPointsReason - break; - case BFD: - result = MaxPointsReason.BFD; - break; - case DISQUALIFIED: - result = MaxPointsReason.DSQ; - break; - case DNC: - result = MaxPointsReason.DNC; - break; - case DNF: - result = MaxPointsReason.DNF; - break; - case DONT_RACE: - result = MaxPointsReason.DNS; // TODO bug 5154: find out what DONT_RACE means and if/how we can translate it to a MaxPointsReason; is it DNS? - break; - case FIN: - result = MaxPointsReason.NONE; // TODO bug 5154: find out what FIN means and if/how we can translate it to a MaxPointsReason; does it mean the competitor finished properly? - break; - case FINISH_CONFIRMED: - result = MaxPointsReason.NONE; // TODO bug 5154: find out what FINISH_CONFIRMED means and if/how we can translate it to a MaxPointsReason; does it mean the competitor finished properly? - break; - case MIS: - result = MaxPointsReason.NONE; // TODO bug 5154: find out what MIS means and if/how we can translate it to a MaxPointsReason; does it mean the competitor is "missing?" - break; - case NO_COLLECT: - result = MaxPointsReason.NONE; // TODO bug 5154: find out what NO_COLLECT means and if/how we can translate it to a MaxPointsReason - break; - case NO_DATA: + if (raceCompetitorStatusType == null) { result = MaxPointsReason.NONE; - break; - case OCS: - result = MaxPointsReason.OCS; - break; - case RACING: - result = MaxPointsReason.NONE; - break; - case RETIRED: - result = MaxPointsReason.RET; - break; - case UFD: - result = MaxPointsReason.UFD; - break; - case NSC: - result = MaxPointsReason.NSC; - break; - case TLE: - result = MaxPointsReason.TLE; - break; - case SCP: - result = MaxPointsReason.SCP; - break; - case STP: - result = MaxPointsReason.STP; - break; - default: - result = MaxPointsReason.NONE; - break; + } else { + switch (raceCompetitorStatusType) { + // TODO we expect NSC and perhaps also TLE to show up; support the mapping to MaxPointsReason.NSC / TLE, respectively when they appear + case ABANDONED: + result = MaxPointsReason.NONE; // TODO bug 5154: find out what ABANDONED means and if/how we can translate it to a MaxPointsReason + break; + case BFD: + result = MaxPointsReason.BFD; + break; + case DISQUALIFIED: + result = MaxPointsReason.DSQ; + break; + case DNC: + result = MaxPointsReason.DNC; + break; + case DNF: + result = MaxPointsReason.DNF; + break; + case DONT_RACE: + result = MaxPointsReason.DNS; // TODO bug 5154: find out what DONT_RACE means and if/how we can translate it to a MaxPointsReason; is it DNS? + break; + case FIN: + result = MaxPointsReason.NONE; // TODO bug 5154: find out what FIN means and if/how we can translate it to a MaxPointsReason; does it mean the competitor finished properly? + break; + case FINISH_CONFIRMED: + result = MaxPointsReason.NONE; // TODO bug 5154: find out what FINISH_CONFIRMED means and if/how we can translate it to a MaxPointsReason; does it mean the competitor finished properly? + break; + case MIS: + result = MaxPointsReason.NONE; // TODO bug 5154: find out what MIS means and if/how we can translate it to a MaxPointsReason; does it mean the competitor is "missing?" + break; + case NO_COLLECT: + result = MaxPointsReason.NONE; // TODO bug 5154: find out what NO_COLLECT means and if/how we can translate it to a MaxPointsReason + break; + case NO_DATA: + result = MaxPointsReason.NONE; + break; + case OCS: + result = MaxPointsReason.OCS; + break; + case RACING: + result = MaxPointsReason.NONE; + break; + case RETIRED: + result = MaxPointsReason.RET; + break; + case UFD: + result = MaxPointsReason.UFD; + break; + case NSC: + result = MaxPointsReason.NSC; + break; + case TLE: + result = MaxPointsReason.TLE; + break; + case SCP: + result = MaxPointsReason.SCP; + break; + case STP: + result = MaxPointsReason.STP; + break; + case DCT: + result = MaxPointsReason.DCT; + break; + case DNE: + result = MaxPointsReason.DNE; + break; + case RCT: + result = MaxPointsReason.RCT; + break; + default: + result = MaxPointsReason.NONE; + break; + } } return result; } diff --git a/java/com.sap.sailing.www/release_notes_admin.html b/java/com.sap.sailing.www/release_notes_admin.html index 2dfcfc5e79e..fe5a03f9f82 100755 --- a/java/com.sap.sailing.www/release_notes_admin.html +++ b/java/com.sap.sailing.www/release_notes_admin.html @@ -23,6 +23,11 @@

Release Notes - Administration Console

+

June 2024

+
    +
  • Upgraded to TracAPI 3.15.8, supporting more TracTrac invalid result markers (IRMs), + in particular DCT, DNE, and RCT.
  • +

May 2024

  • Reading the wind estimation data from a server during start-up, using the diff --git a/java/com.tractrac.clientmodule/META-INF/MANIFEST.MF b/java/com.tractrac.clientmodule/META-INF/MANIFEST.MF index 24dc0e2c189..acb784024fe 100644 --- a/java/com.tractrac.clientmodule/META-INF/MANIFEST.MF +++ b/java/com.tractrac.clientmodule/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: TracTrac Client module Bundle-SymbolicName: com.tractrac.clientmodule -Bundle-Version: 3.15.7 +Bundle-Version: 3.15.8 Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Bundle-ClassPath: ., lib/TracAPI.jar diff --git a/java/com.tractrac.clientmodule/README.txt b/java/com.tractrac.clientmodule/README.txt index 82effdbf3b5..76564739ed8 100644 --- a/java/com.tractrac.clientmodule/README.txt +++ b/java/com.tractrac.clientmodule/README.txt @@ -15,6 +15,21 @@ It contains also some files: - test.sh -> script that compiles the code in the src folder, creates the test.jar library and execute the code of the example. - Manifest.txt -> manifest used to create the test.jar file +******************************************** + TracAPI 3.15.8 +******************************************** +This is a final version. It keeps the backward compatibility. + + Release date: 06/06/2024 + + 1) Features + + - Adding new values to the RaceCompetitorStatusType: RCT(Retired after causing a tangle), + DCT(Disqualified after causing a tangle) and DNE(Disqualification not excludable). + (Requested by Radek Masnika, 30/05/2024) + - Adding the method IPosition.getHACC to retrieve the horizontal accuracy (Requested by Chris Terkelsen, + 05/05/2024) + ******************************************** TracAPI 3.15.7 ******************************************** diff --git a/java/com.tractrac.clientmodule/javadoc/allclasses-frame.html b/java/com.tractrac.clientmodule/javadoc/allclasses-frame.html index 1a429f9dcbb..5cd2d4386ee 100644 --- a/java/com.tractrac.clientmodule/javadoc/allclasses-frame.html +++ b/java/com.tractrac.clientmodule/javadoc/allclasses-frame.html @@ -2,10 +2,10 @@ - + -All Classes (Subscription - Applications - TracAPI 3.15.7 API) - +All Classes (Subscription - Applications - TracAPI 3.15.8 API) + diff --git a/java/com.tractrac.clientmodule/javadoc/allclasses-noframe.html b/java/com.tractrac.clientmodule/javadoc/allclasses-noframe.html index 6131472a992..d9c9a2b0f47 100644 --- a/java/com.tractrac.clientmodule/javadoc/allclasses-noframe.html +++ b/java/com.tractrac.clientmodule/javadoc/allclasses-noframe.html @@ -2,10 +2,10 @@ - + -All Classes (Subscription - Applications - TracAPI 3.15.7 API) - +All Classes (Subscription - Applications - TracAPI 3.15.8 API) + diff --git a/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/IIdentifiable.html b/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/IIdentifiable.html index 4591e218d42..04a387d163c 100644 --- a/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/IIdentifiable.html +++ b/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/IIdentifiable.html @@ -2,10 +2,10 @@ - + -IIdentifiable (Subscription - Applications - TracAPI 3.15.7 API) - +IIdentifiable (Subscription - Applications - TracAPI 3.15.8 API) + @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ diff --git a/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/attachment/package-summary.html b/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/attachment/package-summary.html index f38bf573fb2..53b8afe8627 100644 --- a/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/attachment/package-summary.html +++ b/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/attachment/package-summary.html @@ -2,10 +2,10 @@ - + -com.tractrac.model.lib.api.attachment (Subscription - Applications - TracAPI 3.15.7 API) - +com.tractrac.model.lib.api.attachment (Subscription - Applications - TracAPI 3.15.8 API) + @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,13 +13,13 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ diff --git a/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/data/package-summary.html b/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/data/package-summary.html index acc0b8f38d6..23eadf2d93c 100644 --- a/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/data/package-summary.html +++ b/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/data/package-summary.html @@ -2,10 +2,10 @@ - + -com.tractrac.model.lib.api.data (Subscription - Applications - TracAPI 3.15.7 API) - +com.tractrac.model.lib.api.data (Subscription - Applications - TracAPI 3.15.8 API) + @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ diff --git a/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/event/package-summary.html b/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/event/package-summary.html index 464da6a0e31..da6e13f66e2 100644 --- a/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/event/package-summary.html +++ b/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/event/package-summary.html @@ -2,10 +2,10 @@ - + -com.tractrac.model.lib.api.event (Subscription - Applications - TracAPI 3.15.7 API) - +com.tractrac.model.lib.api.event (Subscription - Applications - TracAPI 3.15.8 API) + @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ diff --git a/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/metadata/package-summary.html b/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/metadata/package-summary.html index 6a83c7db0d2..430960eb796 100644 --- a/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/metadata/package-summary.html +++ b/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/metadata/package-summary.html @@ -2,10 +2,10 @@ - + -com.tractrac.model.lib.api.metadata (Subscription - Applications - TracAPI 3.15.7 API) - +com.tractrac.model.lib.api.metadata (Subscription - Applications - TracAPI 3.15.8 API) + @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ diff --git a/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/package-summary.html b/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/package-summary.html index 502d20fc149..fba9592da8e 100644 --- a/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/package-summary.html +++ b/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/package-summary.html @@ -2,10 +2,10 @@ - + -com.tractrac.model.lib.api (Subscription - Applications - TracAPI 3.15.7 API) - +com.tractrac.model.lib.api (Subscription - Applications - TracAPI 3.15.8 API) + @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ diff --git a/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/route/package-summary.html b/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/route/package-summary.html index 91d7b95490c..ef6a9fc0a54 100644 --- a/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/route/package-summary.html +++ b/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/route/package-summary.html @@ -2,10 +2,10 @@ - + -com.tractrac.model.lib.api.route (Subscription - Applications - TracAPI 3.15.7 API) - +com.tractrac.model.lib.api.route (Subscription - Applications - TracAPI 3.15.8 API) + @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ diff --git a/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/sensor/package-summary.html b/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/sensor/package-summary.html index 6d9b318a0d2..54d0c218084 100644 --- a/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/sensor/package-summary.html +++ b/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/sensor/package-summary.html @@ -2,10 +2,10 @@ - + -com.tractrac.model.lib.api.sensor (Subscription - Applications - TracAPI 3.15.7 API) - +com.tractrac.model.lib.api.sensor (Subscription - Applications - TracAPI 3.15.8 API) + @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ diff --git a/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/spatial/package-summary.html b/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/spatial/package-summary.html index 703cbe2fd28..2e80d64a1b4 100644 --- a/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/spatial/package-summary.html +++ b/java/com.tractrac.clientmodule/javadoc/com/tractrac/model/lib/api/spatial/package-summary.html @@ -2,10 +2,10 @@ - + -com.tractrac.model.lib.api.spatial (Subscription - Applications - TracAPI 3.15.7 API) - +com.tractrac.model.lib.api.spatial (Subscription - Applications - TracAPI 3.15.8 API) + @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ diff --git a/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/competitor/package-summary.html b/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/competitor/package-summary.html index fddca636f86..c0e6bafa0fe 100644 --- a/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/competitor/package-summary.html +++ b/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/competitor/package-summary.html @@ -2,10 +2,10 @@ - + -com.tractrac.subscription.lib.api.competitor (Subscription - Applications - TracAPI 3.15.7 API) - +com.tractrac.subscription.lib.api.competitor (Subscription - Applications - TracAPI 3.15.8 API) + @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ diff --git a/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/control/package-summary.html b/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/control/package-summary.html index 23ed86b046b..4e19cbc1880 100644 --- a/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/control/package-summary.html +++ b/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/control/package-summary.html @@ -2,10 +2,10 @@ - + -com.tractrac.subscription.lib.api.control (Subscription - Applications - TracAPI 3.15.7 API) - +com.tractrac.subscription.lib.api.control (Subscription - Applications - TracAPI 3.15.8 API) + @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ diff --git a/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/event/package-summary.html b/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/event/package-summary.html index ee48383dfbf..48d63a69c96 100644 --- a/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/event/package-summary.html +++ b/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/event/package-summary.html @@ -2,10 +2,10 @@ - + -com.tractrac.subscription.lib.api.event (Subscription - Applications - TracAPI 3.15.7 API) - +com.tractrac.subscription.lib.api.event (Subscription - Applications - TracAPI 3.15.8 API) + @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ diff --git a/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/package-summary.html b/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/package-summary.html index 3e24dd397a3..60b3126713f 100644 --- a/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/package-summary.html +++ b/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/package-summary.html @@ -2,10 +2,10 @@ - + -com.tractrac.subscription.lib.api (Subscription - Applications - TracAPI 3.15.7 API) - +com.tractrac.subscription.lib.api (Subscription - Applications - TracAPI 3.15.8 API) + @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ diff --git a/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/race/package-summary.html b/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/race/package-summary.html index a9972433463..1ec8d45f0b4 100644 --- a/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/race/package-summary.html +++ b/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/race/package-summary.html @@ -2,10 +2,10 @@ - + -com.tractrac.subscription.lib.api.race (Subscription - Applications - TracAPI 3.15.7 API) - +com.tractrac.subscription.lib.api.race (Subscription - Applications - TracAPI 3.15.8 API) + @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ diff --git a/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/route/package-summary.html b/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/route/package-summary.html index 4942fdfcbb3..b8f46e8cb25 100644 --- a/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/route/package-summary.html +++ b/java/com.tractrac.clientmodule/javadoc/com/tractrac/subscription/lib/api/route/package-summary.html @@ -2,10 +2,10 @@ - + -com.tractrac.subscription.lib.api.route (Subscription - Applications - TracAPI 3.15.7 API) - +com.tractrac.subscription.lib.api.route (Subscription - Applications - TracAPI 3.15.8 API) + @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ diff --git a/java/com.tractrac.clientmodule/javadoc/overview-summary.html b/java/com.tractrac.clientmodule/javadoc/overview-summary.html index 5be896c3bf3..b761dfdd6ca 100644 --- a/java/com.tractrac.clientmodule/javadoc/overview-summary.html +++ b/java/com.tractrac.clientmodule/javadoc/overview-summary.html @@ -2,10 +2,10 @@ - + -Overview (Subscription - Applications - TracAPI 3.15.7 API) - +Overview (Subscription - Applications - TracAPI 3.15.8 API) + @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@