mirror of
https://github.com/eclipse-sailing-analytics/sailing-analytics.git
synced 2026-11-04 01:59:35 +00:00
Merge branch 'master' of ssh://sapsailing.com/home/trac/git into bug5938
This commit is contained in:
Thomas Stokes committed
commit
cd8542d1d7
6 files changed
+77
-110
No files matched your search
@@ -22,10 +22,14 @@ else
|
||||
echo "Running on an AWS EC2 instance as user ${USER} / $(whoami), starting setup..."
|
||||
# Install secrets
|
||||
scp root@sapsailing.com:dev-secrets /tmp
|
||||
scp root@sapsailing.com:hudson-aws-credentials /tmp
|
||||
sudo mv /tmp/dev-secrets /root/secrets
|
||||
sudo chown root /root/secrets
|
||||
sudo chgrp root /root/secrets
|
||||
sudo chmod 600 /root/secrets
|
||||
mkdir /root/.aws
|
||||
sudo mv /tmp/hudson-aws-credentials /root/.aws/credentials
|
||||
sudo chown root:root /root/secrets /root/.aws/credentials
|
||||
sudo chmod 600 /root/secrets /root/.aws/credentials
|
||||
# Make eu-west-1 the default region for any aws CLI interaction:
|
||||
sudo su - -c "aws configure set default.region eu-west-1"
|
||||
# Create "hudson" user and clear its directory again which is to become a mount point
|
||||
sudo adduser hudson
|
||||
sudo su - hudson -c "rm -rf /home/hudson/* /home/hudson/.* 2>/dev/null"
|
||||
@@ -41,6 +45,13 @@ else
|
||||
sudo ln -s /home/sailing/code/configuration/hudson_instance_setup/hudson.service /etc/systemd/system
|
||||
# Link Hudson system-wide config file:
|
||||
sudo ln -s /home/sailing/code/configuration/hudson_instance_setup/sysconfig-hudson /etc/sysconfig/hudson
|
||||
# Link additional script files needed for Hudson build server control:
|
||||
sudo ln -s /home/sailing/code/configuration/launchhudsonslave /usr/local/bin
|
||||
sudo ln -s /home/sailing/code/configuration/launchhudsonslave-java11 /usr/local/bin
|
||||
sudo ln -s /home/sailing/code/configuration/aws-automation/getLatestImageOfType.sh /usr/local/bin
|
||||
# Enable NFS server
|
||||
sudo systemctl enable nfs-server.service
|
||||
sudo systemctl start nfs-server.service
|
||||
# Enable the service:
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable hudson.service
|
||||
@@ -48,6 +59,13 @@ else
|
||||
sudo su - -c "cat <<EOF >>/etc/exports
|
||||
/home/hudson/android-sdk-linux 172.31.0.0/16(rw,nohide,no_root_squash)
|
||||
EOF
|
||||
"
|
||||
# Allow "hudson" user to launch EC2 instances:
|
||||
sudo su - -c "cat <<EOF >>/etc/sudoers.d/hudsoncanlaunchec2instances
|
||||
hudson ALL = (root) NOPASSWD: /usr/local/bin/launchhudsonslave
|
||||
hudson ALL = (root) NOPASSWD: /usr/local/bin/launchhudsonslave-java11
|
||||
hudson ALL = (root) NOPASSWD: /usr/local/bin/getLatestImageOfType.sh
|
||||
EOF
|
||||
"
|
||||
# Install DEV server
|
||||
sudo su - sailing -c "mkdir /home/sailing/servers/DEV
|
||||
@@ -62,6 +80,10 @@ EOF
|
||||
sudo cp /root/mail.properties /home/sailing/servers/DEV/configuration
|
||||
sudo chown sailing /home/sailing/servers/DEV/configuration/mail.properties
|
||||
sudo chgrp sailing /home/sailing/servers/DEV/configuration/mail.properties
|
||||
# Start the sailing.service with empty/no user data, so the next boot is recognized as a re-boot
|
||||
sudo systemctl start sailing.service
|
||||
sudo systemctl stop sailing.service
|
||||
sudo mount -a
|
||||
else
|
||||
echo "Not running on an AWS instance; refusing to run setup!" >&2
|
||||
echo "To prepare an instance running in AWS, provide its external IP as argument to this script." >&2
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# hudson ALL = (root) NOPASSWD: /usr/local/bin/launchhudsonslave
|
||||
# hudson ALL = (root) NOPASSWD: /usr/local/bin/launchhudsonslave-java11
|
||||
# hudson ALL = (root) NOPASSWD: /usr/local/bin/getLatestImageOfType.sh
|
||||
AWS=/usr/bin/aws
|
||||
AWS=/bin/aws
|
||||
REGION=eu-west-1
|
||||
HUDSON_SLAVE_AMI_ID=$( /usr/local/bin/getLatestImageOfType.sh hudson-slave )
|
||||
echo Launching instance from AMI ${HUDSON_SLAVE_AMI_ID} ...
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#!/bin/bash
|
||||
MONGODBS="mongodb://dbserver.internal.sapsailing.com:10201/?replicaSet=archive mongodb://dbserver.internal.sapsailing.com:10202/?replicaSet=slow mongodb://dbserver.internal.sapsailing.com:10203/?replicaSet=live"
|
||||
MAILINGLIST="/home/trac/mailinglists/unhealthy-mongo-list"
|
||||
#MAILINGLIST="/home/trac/mailinglists/unhealthy-mongo-list"
|
||||
MAILINGLIST=$(cat /var/cache/landscapeManagersMailingList)
|
||||
for db in $MONGODBS; do
|
||||
echo "rs.status()" | mongo --quiet "$db" | grep '\("set"\)\|\("name"\)\|\("health"\)\|\("stateStr"\)' | sed -e 's/^.* : "\?//' -e 's/"\?,$//' | (
|
||||
read replicaset
|
||||
|
||||
@@ -119,8 +119,8 @@ EOF
|
||||
sudo mkswap /var/cache/swapfile
|
||||
# And while adding to /etc/fstab, also add the NFS mount of /home/scores:
|
||||
sudo mkdir /home/scores
|
||||
sudo su - -c 'echo "/var/cache/swapfile none swap pri=0 0 0" >>/etc/fstab
|
||||
logfiles.internal.sapsailing.com:/home/scores /home/scores nfs tcp,intr,timeo=100,retry=0'
|
||||
sudo su - -c 'echo "/var/cache/swapfile none swap pri=0 0 0
|
||||
logfiles.internal.sapsailing.com:/home/scores /home/scores nfs tcp,intr,timeo=100,retry=0" >>/etc/fstab'
|
||||
else
|
||||
echo "Not running on an AWS instance; refusing to run setup!" >&2
|
||||
echo "To prepare an instance running in AWS, provide its external IP as argument to this script." >&2
|
||||
|
||||
@@ -5,6 +5,7 @@ Bundle-SymbolicName: com.sap.sailing.server.interface
|
||||
Bundle-Version: 1.0.0.qualifier
|
||||
Bundle-Vendor: SAP
|
||||
Automatic-Module-Name: com.sap.sailing.server.interface
|
||||
Bundle-ActivationPolicy: lazy
|
||||
Bundle-RequiredExecutionEnvironment: JavaSE-1.8
|
||||
Require-Bundle: com.sap.sailing.domain;bundle-version="1.0.0",
|
||||
com.sap.sse.common;bundle-version="1.0.0",
|
||||
|
||||
@@ -1,112 +1,55 @@
|
||||
# Setting up an image for the hudson.sapsailing.com server
|
||||
|
||||
This is an add-on to the regular EC2 image set-up described [here](https://wiki.sapsailing.com/wiki/info/landscape/creating-ec2-image-from-scratch). An Android SDK needs to be installed.
|
||||
|
||||
|
||||
* Create a ``hudson`` user/group
|
||||
* Make sure ``/home/hudson`` is a separate mount; probably just mount the existing volume of a previous installation
|
||||
* Install an Android SDK under ``/home/hudson/android-sdk-linux``; if you simply re-used an old ``/home/hudson`` mount this should already be in place.
|
||||
* Install Eclipse to ``/home/hudson/eclipse`` to allow sharing it in case a large AWS instance is needed, e.g., for heap dump analysis.
|
||||
* export ``/home/hudson/android-sdk-linux`` and ``/home/hudson/eclipse`` as follows in ``/etc/exports``:
|
||||
Like when setting up a regular sailing application server instance, start with a fresh Amazon Linux 2 image and create an instance with a 16GB root volume. Use the "Sailing Analytics Server" security group and something like a ``t3.small`` instance type for creating the image. Then, invoke the script
|
||||
```
|
||||
/home/hudson/android-sdk-linux 172.31.0.0/16(rw,nohide,no_root_squash)
|
||||
/home/hudson/eclipse 172.31.0.0/16(rw,nohide,no_root_squash)
|
||||
configuration/hudson_instance_setup/setup-hudson-server.sh {external-IP-of-new-instance}
|
||||
```
|
||||
This will first run the regular sailing server set-up which allows the instance to run the ``dev.sapsailing.com`` Sailing Analytics instance later. Then, the script will continue to obtain the Hudson WAR file from ``https://static.sapsailing.com/hudson.war.patched-with-mail-1.6.2`` and deploy it to ``/usr/lib/hudson``, obtain and install the default system-wide Hudson configuration, get adjusted dev server secrets from ``ssh://root@sapsailing.com/root/dev-secrets`` as well as ``mail.properties``, and install a ``hudson.service`` unit under ``/etc/systemd/system``. A ``hudson`` user is created, and its ``/home/hudson`` home directory is emptied so it can act as a mount point. A latest version of the SAP Sailing Analytics is installed to ``/home/sailing/servers/DEV``.
|
||||
|
||||
The ``/home/hudson/android-sdk-linux`` folder that is later expected to be mounted into the ``/home/hudson`` mount point is exported through NFS by appending a corresponding entry to ``/etc/exports``. The script will also allow the ``hudson`` user to run the ``/usr/local/bin/launchhudsonslave`` script with ``sudo``. In order to elastically scale our build / CI infrastructure, we use AWS to provide Hudson build slaves on demand. The Hudson Master (https://hudson.sapsailing.com) has a script obtained from our git at ``./configuration/launchhudsonslave`` which takes an Amazon Machine Image (AMI), launches it in our default region (eu-west-1) and connects to it. The AWS credentials are stored in the ``root`` account on ``hudson.sapsailing.com``, and the ``hudson`` user is granted access to the script by means of an ``/etc/sudoers.d`` entry.
|
||||
|
||||
When the script has finished, proceed as follows:
|
||||
|
||||
* make a volume available that holds the ``/home/hudson`` content, including the ``android-sdk-linux`` folder. This can happen, e.g., by creating a snapshot of the existing ``/home/hudson`` volume of the running "Build/Dev/Test" server, or you may search the weekly backup snapshots for the latest version to start with. Make sure to create the volume in the same availability zone (AZ) as your instance is running in.
|
||||
* Based on how full the volume already is, consider re-sizing it by creating the volume larger than the snapshot.
|
||||
* Attach it.
|
||||
* In the instance, as ``root`` user, call ``dmesg``. This will show the new volume that just got attached, as well as its partition names.
|
||||
* If you chose the volume bigger than the snapshot, use ``resize2fs`` to resize accordingly.
|
||||
* Enter an ``/etc/fstab`` entry for the volume, e.g., like this:
|
||||
```
|
||||
UUID={the-UUID-of-the-partition-to-mount} /home/hudson ext4 defaults,noatime,commit=30 0 0
|
||||
```
|
||||
To find out ``{the-UUID-of-the-partition-to-mount}``, use ``blkid``.
|
||||
* Mount with ``mount -a``
|
||||
* Adjust ownerships in case the new instance's ``hudson`` user/group IDs have changed compared to the old instance:
|
||||
```
|
||||
sudo chmod -R hudson:hudson /home/hudson
|
||||
```
|
||||
* If you'd like to keep in sync with the latest version of a still running live Hudson environment, keep copying its ``/home/hudson`` contents with ``rsync -av root@dev.internal.sapsailing.com:/home/hudson/ /home/hudson/`` until you switch
|
||||
* Ensure you have EC2 / EBS snapshot backups for the volumes by tagging them as follows: ``WeeklySailingInfrastructureBackup=Yes`` for ``/`` and ``/home/hudson``.
|
||||
|
||||
``/home/hudson/repo`` has the Hudson build repository. The Hudson WAR file is under ``/usr/lib/hudson/hudson.war`` which has to be taken from [here](https://static.sapsailing.com/hudson.war.patched-with-mail-1.6.2). ``/etc/init.d/hudson``, linked to from ``/etc/rc0.d/K29hudson``, ``/etc/rc1.d/K29hudson``, ``/etc/rc2.d/K29hudson``, ``/etc/rc3.d/S81hudson``, ``/etc/rc4.d/K29hudson``, ``/etc/rc5.d/S81hudson``, and ``/etc/rc6.d/K29hudson``, takes care of spinning up Hudson during instance re-boot. It can be obtained from [here](https://static.sapsailing.com/etc-init.d-hudson). Hudson systemwide configuration is under ``/etc/sysconfig/hudson``:
|
||||
## In-Place Start-Up
|
||||
|
||||
You can then either use the instance right away by starting the two essential services, as follows:
|
||||
```
|
||||
## Path: Development/Hudson
|
||||
## Description: Configuration for the Hudson continuous build server
|
||||
## Type: string
|
||||
## Default: "/var/lib/hudson"
|
||||
## ServiceRestart: hudson
|
||||
#
|
||||
# Directory where Hudson store its configuration and working
|
||||
# files (checkouts, build reports, artifacts, ...).
|
||||
#
|
||||
HUDSON_HOME="/home/hudson/repo"
|
||||
|
||||
## Type: string
|
||||
## Default: ""
|
||||
## ServiceRestart: hudson
|
||||
#
|
||||
# Java executable to run Hudson
|
||||
# When left empty, we'll try to find the suitable Java.
|
||||
#
|
||||
|
||||
HUDSON_JAVA_CMD="/opt/sapjvm_8/bin/java"
|
||||
# The following line choses JavaSE-1.7
|
||||
#HUDSON_JAVA_CMD="/opt/jdk1.7.0_02/bin/java"
|
||||
# The following line choses JavaSE-1.8
|
||||
#HUDSON_JAVA_CMD="/opt/jdk1.8.0_20/bin/java"
|
||||
|
||||
## Type: string
|
||||
## Default: "hudson"
|
||||
## ServiceRestart: hudson
|
||||
#
|
||||
# Unix user account that runs the Hudson daemon
|
||||
# Be careful when you change this, as you need to update
|
||||
# permissions of $HUDSON_HOME and /var/log/hudson.
|
||||
#
|
||||
HUDSON_USER="hudson"
|
||||
|
||||
## Type: string
|
||||
## Default: "-Djava.awt.headless=true"
|
||||
## ServiceRestart: hudson
|
||||
#
|
||||
# Options to pass to java when running Hudson.
|
||||
#
|
||||
HUDSON_JAVA_OPTIONS="-Djava.awt.headless=true -Xmx2G -Dhudson.slaves.ChannelPinger.pingInterval=60 -Dhudson.slaves.ChannelPinger.pingIntervalSeconds=60 -Dhudson.slaves.ChannelPinger.pingTimeoutSeconds=60"
|
||||
|
||||
## Type: integer(0:65535)
|
||||
## Default: 8080
|
||||
## ServiceRestart: hudson
|
||||
#
|
||||
# Port Hudson is listening on.
|
||||
#
|
||||
HUDSON_PORT="8080"
|
||||
|
||||
## Type: integer(1:9)
|
||||
## Default: 5
|
||||
## ServiceRestart: hudson
|
||||
#
|
||||
# Debug level for logs -- the higher the value, the more verbose.
|
||||
# 5 is INFO.
|
||||
#
|
||||
HUDSON_DEBUG_LEVEL="5"
|
||||
|
||||
## Type: yesno
|
||||
## Default: no
|
||||
## ServiceRestart: hudson
|
||||
#
|
||||
# Whether to enable access logging or not.
|
||||
#
|
||||
HUDSON_ENABLE_ACCESS_LOG="no"
|
||||
|
||||
## Type: integer
|
||||
## Default: 100
|
||||
## ServiceRestart: hudson
|
||||
#
|
||||
# Maximum number of HTTP worker threads.
|
||||
#
|
||||
HUDSON_HANDLER_MAX="100"
|
||||
|
||||
## Type: integer
|
||||
## Default: 20
|
||||
## ServiceRestart: hudson
|
||||
#
|
||||
# Maximum number of idle HTTP worker threads.
|
||||
#
|
||||
HUDSON_HANDLER_IDLE="20"
|
||||
|
||||
## Type: string
|
||||
## Default: ""
|
||||
## ServiceRestart: hudson
|
||||
#
|
||||
# Pass arbitrary arguments to Hudson.
|
||||
# Full option list: java -jar hudson.war --help
|
||||
#
|
||||
HUDSON_ARGS=""
|
||||
sudo systemctl start hudson.service
|
||||
sudo systemctl start sailing.service
|
||||
```
|
||||
## Creating an AMI to Launch
|
||||
|
||||
Or you stop the instance, either from within using ``shutdown -h now`` or from the AWS Console, then create an image (AMI) that you can use to create a new instance at any time. Keep in mind, though, that keeping such an image around incurs cost for the relatively large ``/home/hudson`` volume's snapshot. As the volume is part of the weekly backup strategy anyhow, and due to the fair degree of automation during producing a new version of this instance type, this may not be necessary.
|
||||
|
||||
The AMI should be labeled accordingly, and so should the snapshots. For allowing for automated image upgrades, ensure the AMI and snapshots follow a common naming and versioning pattern. Name your AMI something like "Build/Dev/Test x.y" with x and y being major and minor version numbers, such as 2.0. Then, your snapshots shall be named "Build/Dev/Test x.y ({volume-name})" where {volume-name} can be any human-readable identifier that lets you recognize the volume. Examples for {volume-name} may be "Root" or "Home" or "HudsonHome" or similar.
|
||||
|
||||
Then terminate your instance used only for image creation and launch from the AMI, then, as in "In-Place Start-Up", place the instance into the "Hudson" target group and adjust the elastic IP to point to the new instance.
|
||||
|
||||
## Steps to Perform to Activate the New Server
|
||||
|
||||
* Replace the old "Build/Test/Dev" instance in the "Hudson" and "S-DEV" and "S-DEV-m" target groups with the new one
|
||||
* Switch the elastic IP ``52.17.217.83`` to the new instance, too
|
||||
* Change the ``dev.internal.sapsailing.com`` Route53 DNS entry from the old instance's internal IP address to the new instance's internal IP address
|
||||
|
||||
|
||||
## Testing the New and Terminating the Old Instance
|
||||
|
||||
After a while of testing the new environment successfully you may choose to terminate the old instance. Make sure the large ``/home/hudson`` volume is deleted with it, and it not, delete it manually.
|
||||
Reference in new issue
Block a user