Merge remote-tracking branch 'origin/bug5912' into bug5912

2026-09-15 10:18:45 +00:00 · 2024-05-08 15:30:58 +02:00
parent 32e37167fe 4022c5f208
commit dfa62ea6f9
12 changed files with 78 additions and 30 deletions
@@ -85,24 +85,29 @@ if [[ -d "users" ]]; then
            adduser "$USERNAME"
        fi
        id "$USERNAME"
-        if [[ "$?" -eq 0 &&  "$CREATE_CRONTABS" == "true" ]]; then  # 9 is the exit code indicating the username already exists.
+        if [[ "$?" -eq 0 ]]; then
            HOME_DIR=$(eval echo $(printf "~%q" "$USERNAME"))   # The path to the home dir of the user whose cronjob will be installed.
-            # Clear the crontab file before assembling it from the snippets:
-            > $HOME_DIR/crontab
-            for crontab in ${USERNAME}/crontab*; do
-                [[ -e $crontab ]] || continue
-                cat "${crontab}">> $HOME_DIR/crontab
-                echo "">> $HOME_DIR/crontab
-            done
-            chown "$USERNAME":"$USERNAME" $HOME_DIR/crontab
-            sed -i "s|PATH_OF_GIT_HOME_DIR_TO_REPLACE|${GIT_PATH}|g" "$HOME_DIR"/crontab # Sets correct path to the git repo within the crontab.
-            sed -i "s|PATH_OF_HOME_DIR_TO_REPLACE|${HOME_DIR}|g" "$HOME_DIR"/crontab # Sets the correct path to the home dir of the user whose crontab will be installed.
-            sed -i '/^$/d' $HOME_DIR/crontab # purges random empty lines.
-            echo "">>$HOME_DIR/crontab  # Adds a newline
-            if [[ "$INSTALL_CRONTAB" == "true" ]]; then
-                crontab -u ${USERNAME} $HOME_DIR/crontab # Install the crontab in the given user's home dir.
+            # Sets permissions of home dir.
+            [[ -e "$USERNAME"/permissions ]] && chmod "$(cat "$USERNAME"/permissions)" "$HOME_DIR"
+            if [[ "$CREATE_CRONTABS" == "true" ]]; then  # 9 is the exit code indicating the username already exists.
+                # Clear the crontab file before assembling it from the snippets:
+                > $HOME_DIR/crontab
+                for crontab in ${USERNAME}/crontab*; do
+                    [[ -e $crontab ]] || continue
+                    cat "${crontab}">> $HOME_DIR/crontab
+                    echo "">> $HOME_DIR/crontab
+                done
+                chown "$USERNAME":"$USERNAME" $HOME_DIR/crontab
+                sed -i "s|PATH_OF_GIT_HOME_DIR_TO_REPLACE|${GIT_PATH}|g" "$HOME_DIR"/crontab # Sets correct path to the git repo within the crontab.
+                sed -i "s|PATH_OF_HOME_DIR_TO_REPLACE|${HOME_DIR}|g" "$HOME_DIR"/crontab # Sets the correct path to the home dir of the user whose crontab will be installed.
+                sed -i '/^$/d' $HOME_DIR/crontab # purges random empty lines.
+                echo "">>$HOME_DIR/crontab  # Adds a newline
+                if [[ "$INSTALL_CRONTAB" == "true" ]]; then
+                    crontab -u ${USERNAME} $HOME_DIR/crontab # Install the crontab in the given user's home dir.
+                fi
            fi
        fi
+        
        echo "****** finished user ******"
    done
    # Add to groups after all the groups exist.
@@ -0,0 +1,2 @@
+ulimit -n 30000
+ulimit -u 40000
@@ -0,0 +1,12 @@
+[Unit]
+Description=Removes welcome.conf.
+Requires=-.mount network-online.target
+After=-.mount network-online.target
+Before=httpd.service get-latest-httpd-conf.service
+[Install]
+RequiredBy=multi-user.target
+WantedBy=httpd.service
+[Service]
+Type=oneshot
+RemainAfterExit=true
+ExecStart=/bin/bash -c "cd /etc/httpd; [[ -e conf.d/welcome.conf ]] && rm -f conf.d/welcome.conf"
@@ -19,24 +19,23 @@ TEMPORARY_HOME_COPY_LOCATION="/root/temporary_home_copy" # home nested within th
 ssh -A "root@${IP}" "bash -s" << EOF
 sudo systemctl start crond.service
 . imageupgrade_functions.sh
-cp -r --preserve "$TEMPORARY_HOME_COPY_LOCATION"/home /
+cp -r "$TEMPORARY_HOME_COPY_LOCATION"/home /
 rm -rf "$TEMPORARY_HOME_COPY_LOCATION"
 build_crontab_and_setup_files -f "${IMAGE_TYPE}" "${GIT_COPY_USER}" "${RELATIVE_PATH_TO_GIT}"  # files have already been copied so -f is used.
 chown trac:static /var/www/static
-setup_keys "${IMAGE_TYPE}"
 # setup nfs
 systemctl enable nfs-server
 echo "/var/log/old 172.31.0.0/16(rw,nohide,no_root_squash)
 /home/scores 172.31.0.0/16(rw,nohide,no_root_squash)" >>/etc/exports
 systemctl start nfs-server
-cd /var/log/old/cache/docker/registry && docker-compose-up
-scp -p -o StrictHostKeyChecking=no -r root@sapsailing.com:/etc/ssh /etc
+# scp -p -o StrictHostKeyChecking=no -r root@sapsailing.com:/etc/ssh /etc   # causes some issue with MACs
 # append hostname to sysconfig
 echo "HOSTNAME=sapsailing.com" >> /etc/sysconfig/network
 sed -i "s/\(127.0.0.1 *\)/\1 sapsailing.com /" /etc/hosts
 hostname sapsailing.com
 hostnamectl set-hostname sapsailing.com
 EOF
+ssh -A -f root@"$IP" "cd /var/log/old/cache/docker/registry && nohup docker-compose up &>/dev/null &" &> /dev/null
 echo "Please now run the script target-group-tag-route53-nfs-elasticIP-setup.sh which configures the EC2 instance tags, adds to the "
 echo "necessary target groups, modifies a few records in route53 (logs.internal.sapsailing.com"
 echo "and smtp.internal.sapsailing.com), remounts those dependent on this, and sets the elastic IP."
@@ -23,11 +23,13 @@
 # This will do all necessary set-up up to the point where the large volumes
 # currently attached to and mounted on the current Central Reverse Proxy will
 # need to be unmounted, detached, attached to the new instance, and mounted there.
-if [[ "$#" -ne 2 ]]; then
+if [[ "$#" -ne 4 ]]; then
    echo "IP and bearer token required. Please check comment description for further details."
 fi
 IP=$1
 BEARER_TOKEN=$2
+IMAGEUPGRADE_FUNCTIONS_IP="$3"   # can be a domain name, such as sapsailing.com
+IMAGEUPGRADE_FUNCTIONS_PATH_ON_INSTANCE_TO_GIT="$4"
 IMAGE_TYPE="central_reverse_proxy"
 HTTP_LOGROTATE_ABSOLUTE=/etc/logrotate.d/httpd
 GIT_COPY_USER="wiki"
@@ -50,11 +52,11 @@ yum install -y perl perl-CGI perl-Template-Toolkit  perl-CPAN perl-DBD-MySQL mod
 # ruby and gollum for wiki
 yum group install -y "Development Tools"
 yum install -y ruby ruby-devel libicu libicu-devel zlib zlib-devel git cmake openssl-devel libyaml-devel
-gem install gollum
+gem install gollum -v 5.3.2
 gem update --system 3.5.7
 cd /home
 # The following line is for production use:
-scp -o StrictHostKeyChecking=no -p "root@sapsailing.com:/home/wiki/gitwiki/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh" /usr/local/bin
+scp -o StrictHostKeyChecking=no -p root@"$IMAGEUPGRADE_FUNCTIONS_IP":"$IMAGEUPGRADE_FUNCTIONS_PATH_ON_INSTANCE_TO_GIT"/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh /usr/local/bin
 # The following line is for test use, copying from a test instance with a check-out Git workspace:
 # scp -o StrictHostKeyChecking=no -p "root@13.40.100.54:/home/sailing/code/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh" /usr/local/bin
 . imageupgrade_functions.sh
@@ -131,7 +133,7 @@ chmod 755 /root
 cd ~
 # Copies across the key vault and other relevant secrets from the existing
 # Central Reverse Proxy's /root folder:
-rsync -a dev-secrets github_tools_sap.pat hudson-aws-credentials key_vault mail.properties new_version_key_vault secrets ssh-key-reader.token root@sapsailing.com:/root /
+rsync -a  root@sapsailing.com:/root/{dev-secrets,github_tools_sap.pat,hudson-aws-credentials,key_vault,mail.properties,new_version_key_vault,secrets,ssh-key-reader.token} /root
 scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/etc/letsencrypt /etc
 # add basic test page which won't cause redirect error code if used as a health check.
 cat <<EOF > /var/www/html/index.html
@@ -140,6 +142,7 @@ EOF
 echo "net.ipv4.ip_conntrac_max = 131072" >> /etc/sysctl.conf
 # setup fail2ban
 setup_fail2ban
+setup_keys "${IMAGE_TYPE}"
 # setup logrotate.d/httpd 
 # echo "Patching $HTTP_LOGROTATE_ABSOLUTE so that old logs go to /var/log/old/$IP" >>/var/log/sailing.out
 # mkdir --parents "/var/log/old/REVERSE_PROXIES/${IP}"
@@ -154,6 +157,7 @@ chown root:root /etc/httpd/conf/pass*
 # create mountpoints (see part 2 for ownership changes)
 mkdir /var/log/old
 mkdir /var/www/static
+download_and_install_latest_sap_jvm_8
 # enable units which build-crontab doesn't 
 systemctl enable httpd
 systemctl start httpd
@@ -144,7 +144,8 @@ setup_keys() {
                    [[ -f "$key" ]] || continue
                    \cp --preserve --dereference "$key" "$user_home_dir"/.ssh
                done
-                for key in $(find ${user}/ssh/authorized_keys -type f); do
+                for key in "${user}"/ssh/authorized_keys/*; do
+                    [[ -f "$key" ]] || continue
                    cat "${key}" >>  ${user_home_dir}/.ssh/authorized_keys
                done
                chown -R  ${user}:${user} "${user_home_dir}/.ssh"
@@ -98,11 +98,11 @@ setFailoverIfNotSet() {
 logger -t archive "begin check"
 # --fail option ensures that, if a server error is returned (ie. 5xx/4xx status code), then the status code (stored in $?) will be non zero.
 # -L follows redirects
-curl -s -L --fail --connect-timeout ${TIMEOUT1_IN_SECONDS} "http://${archiveIp}:${ARCHIVE_PORT}/gwt/status" >> /dev/null
+curl -s -L --fail --max-time ${TIMEOUT1_IN_SECONDS} "http://${archiveIp}:${ARCHIVE_PORT}/gwt/status" >> /dev/null
 if [[ $? -ne 0 ]]
 then
    logger -t archive "first check failed"
-    curl -s -L --fail --connect-timeout ${TIMEOUT2_IN_SECONDS} "http://${archiveIp}:${ARCHIVE_PORT}/gwt/status" >> /dev/null
+    curl -s -L --fail --max-time ${TIMEOUT2_IN_SECONDS} "http://${archiveIp}:${ARCHIVE_PORT}/gwt/status" >> /dev/null
    if [[ $? -ne 0 ]]
    then
        setFailoverIfNotSet
@@ -282,13 +282,15 @@ There is now a single point of truth for the various ssh and AWS keys, and possi

 The structure of the vault is important for the efficacy of the script and should appear as below. There is an explanation afterwards.
 ```
-key_vault
+.
 ├── aws_credentials
+│   ├── axel-central-reverse-proxy-credentials
 │   └── disposable-reverse-proxy-automation
 ├── central_reverse_proxy
 │   ├── httpdConf
 │   │   ├── aws
-│   │   │   └── credentials -> ../../../aws_credentials/disposable-reverse-proxy-automation
+│   │   │   └── credentials
+│   │   │       └── disposable-reverse-proxy-automation -> ../../../../aws_credentials/disposable-reverse-proxy-automation
 │   │   └── ssh
 │   │       ├── authorized_keys
 │   │       │   ├── id_ed25519.pub@root@central_reverse_proxy -> ../../../root/ssh/id_ed25519.pub
@@ -296,23 +298,46 @@ key_vault
 │   │       ├── id_ed25519
 │   │       └── id_ed25519.pub
 │   ├── root
+│   │   ├── aws
+│   │   │   ├── config
+│   │   │   │   └── axel
+│   │   │   └── credentials
+│   │   │       ├── axel-central-reverse-proxy-credentials -> ../../../../aws_credentials/axel-central-reverse-proxy-credentials
+│   │   │       └── disposable-reverse-proxy-automation -> ../../../../aws_credentials/disposable-reverse-proxy-automation
 │   │   └── ssh
 │   │       ├── authorized_keys
 │   │       │   └── id_ed25519.pub@httpdConf@central_reverse_proxy -> ../../../httpdConf/ssh/id_ed25519.pub
 │   │       ├── id_ed25519
 │   │       └── id_ed25519.pub
+│   └── wiki
+│       └── ssh
+│           └── authorized_keys
+│               └── id_ed25519.pub@root@reverse_proxy -> ../../../../reverse_proxy/root/ssh/id_ed25519.pub
+├── README
+...
 ```
+
 1. So we have the aws_credentials directory, storing the credentials for specific AWS users.
+
 2. We also have directories named after the environment types (matching the directory names in GIT_HOME/configuration/environments_scripts).
+
 3. Nested within these, we have directories for each user that will require some keys, for the given environment type.
+
 4. For each user, we have optional directories "ssh" & "aws" (the naming is important).
-5. The aws folder should contain only credentials files which are sym links to the aws_credentials folder.
-6. If the setup_keys script is run, the contents of the aws folder will be copied, across to the respective .aws folder on the instance the script runs on, within the correct user's home directory. The config file will be created with the correct region. Although, it will *only* be the default profile.
+
+5. The aws folder can contain a config and credentials folder. The credentials folder contains sym links to aws_credentials, whilst the config folder contains config header and format. 
+
+6. If the setup_keys script is run, the contents of the credentials and config folders are concatenated into the credentials and config file respectively, which are found in .aws home dir of the user that the folders are nested within. The region is automatically added and so shouldn't be defined in the config. Note, that the general idea is to use the "default" header and define access control for that instance type through AWS credentials, reducing the number  of different profiles, as well as complexity of scripts.
+
 7. The ssh folder will contain the ssh keys of the user; they are named based on the type of the key.
+
 8. Furthermore, the folder will contain an authorized_keys directory, which holds references to the keys (elsewhere in the vault), which should be authorized to access the user. In the above example, the symbolic link named `id_ed25519.pub@httpdConf@central_reverse_proxy` means that the key referenced will be in the authorized keys
 for root, so the id_ed25519 key of the httpdConf user on the central reverse proxy will be able to access the root user.
+
 9. The name of these links doesn't matter, but by convention we will use the format used in the image above (`key_type@user@env_type`), using @ as a separator.
+
 10. The script will copy across the keys in the ssh folder (ignoring sym links or directories).
+
 11. The script will append every public key that is linked in the authorized_keys folder, to the authorized_keys file of the respective user. 

 ### Creating a New Application Replica Set