eclipse-sailing-analytics/sailing-analytics
1
0
Fork 0
mirror of https://github.com/eclipse-sailing-analytics/sailing-analytics.git synced 2026-09-15 10:18:45 +00:00
Code Activity

Merge remote-tracking branch 'origin/master'

Browse Source
This commit is contained in:
Axel Uhl
2024-05-08 23:22:47 +02:00
parent d62202ef0e 2550e84860
commit f90bc87cd2
66 changed files with 1955 additions and 484 deletions
Download Patch File Download Diff File Expand all files Collapse all files
home.md → Home.md
+1 -1
View File
@@ -117,7 +117,7 @@ SAP is at the center of today’s technology revolution, developing innovations
* [[Java De(Serialization) and Circular Dependencies|wiki/howto/development/java-de-serialization-and-circular-dependencies]]
* [[JMX Support|wiki/howto/development/jmx]]
* [[Working with GWT Locally|wiki/howto/development/local-gwt]]
* [[Log File Analysis|wiki/howto/development/log-file-analysis]]
* [[Log File Analysis|wiki/howto/development/Log-File-Analysis]]
* [[Old Log Compression|wiki/howto/development/Log-File-Compression]]
* [[UI Tests with Selenium|wiki/howto/development/selenium-ui-tests]]
* [[Profiling|wiki/howto/development/profiling]]
configuration/aws-automation/awsmfalogon.sh
-15
View File
@@ -1,15 +0,0 @@
# Use like this:
# . awsmfalogon.sh {mfaDeviceArn} {tokenCode}
# It will set the necessary environment variables that will allow the "aws" client to work
# with a session token
# Use with a bash alias definition like this, replacing {ARN-of-your-MFA-device} with the ARN of your MFA device:
# alias awsmfa='echo -n "Token: "; read aws_mfa_token; . awsmfalogon.sh "{ARN-of-your-MFA-device}" ${aws_mfa_token}'
# Then, you can invoke the alias "awsmfa" on your bash command line, and you will get prompted for an MFA token
# which, when entered, will add the necessary environment variables to your bash session that will allow your aws
# client to function with a valid session key.
mfaDeviceArn=$1
tokenCode=$2
jsonOutput="$(aws sts get-session-token --serial-number "${mfaDeviceArn}" --token-code ${tokenCode})"
export AWS_ACCESS_KEY_ID=$( echo "${jsonOutput}" | jq --raw-output '.Credentials.AccessKeyId' )
export AWS_SECRET_ACCESS_KEY=$( echo "${jsonOutput}" | jq --raw-output '.Credentials.SecretAccessKey' )
export AWS_SESSION_TOKEN=$( echo "${jsonOutput}" | jq --raw-output '.Credentials.SessionToken' )
configuration/aws-automation/awsmfalogon.sh
Symlink
+1
View File
@@ -0,0 +1 @@
../environments_scripts/repo/usr/local/bin/awsmfalogon.sh
configuration/environments_scripts/build-crontab-and-cp-files
+93 -17
View File
@@ -8,26 +8,33 @@
# Within these crontab stubs, are certain string literals, which are replaced: PATH_OF_GIT_HOME_DIR_TO_REPLACE becomes
# the absolute path to the git home dir, and PATH_OF_HOME_DIR_TO_REPLACE is changed to the path of the home directory
# in which the crontab will be installed.
# Furthermore, within the users folder optional uid, gid and groups files an exist. uid should store the user's uid,
# gid the group id and groups should have secondary groups (one per line). The secondary groups are added after all users are
# created.
# Useful files are also copied across from the "files" dir within each image type.
# Finally, the script ends by performing a deamon reload and reenabling all the systemd units in the files directory,
# so that the correct wants, requires, etc. are installed/linked. Any systemd units, within the mimicked filesystem,
# will be automatically enabled.
if [[ $# -lt 3 || $# -gt 5 ]]; then
if [[ $# -lt 3 || $# -gt 6 ]]; then
echo "$0 <ENVIRONMENT_TYPE> <USER_WITH_COPY_OF_REPO> <RELATIVE_PATH_OF_GIT_DIR_WITHIN_USER> "
echo ""
echo "Where USER_WITH_COPY_OF_REPO is a user that contains a checked out copy of the main git."
echo "And where RELATIVE_PATH_OF_GIT_DIR_WITHIN_USER is the path to the git repo from the USER_WITH_COPY_OF_REPO's home directory."
echo "Use the s(imple) flag to only do the crontab and not copy any files across. The n(o install) flag can be used to setup the crontabs but not install them."
echo "Use the f(iles) flag to disable file copying. Use the c(rontab) flag to disable the creation of users and the creation of crontab files for those users."
echo "If the c flag is not used then the n(o install) flag can be used to setup the crontabs but not install them."
exit 2
fi
INSTALL_CRONTAB="true"
options='sn'
COPY_FILES="true"
CREATE_CRONTABS="true";
options='fnc'
while getopts $options option
do
case $option in
c) CREATE_CRONTABS="false";;
f) COPY_FILES="false";;
n) INSTALL_CRONTAB="false";;
s) ONLY_CRONTAB="true";;
\?) echo "Invalid option"
exit 4;;
esac
@@ -37,27 +44,96 @@ ENV_TYPE="$1"
GIT_USER="$2"
RELATIVE_GIT_DIR_NAME="$3"
cd "$(dirname "$0")/${ENV_TYPE}"
if [[ -d "groups" ]]; then
cd "groups"
for group in * ; do
[[ -e "$group" ]] || continue
echo "CREATING GROUP $group"
if [[ -e "$group"/gid ]]; then
groupadd --gid $(cat "$group"/gid) "$group"
else
groupadd "$group"
fi
done
cd ..
fi
if [[ -d "users" ]]; then
cd "users"
GIT_PATH="$(eval echo $(printf "~%q" "$GIT_USER"))/${RELATIVE_GIT_DIR_NAME}" # The path to the git repo that contains the files needed.
for dir in $(ls -d */ ); do
USERNAME=$(echo $dir | sed "s/\/$//") # Dirname is the username. The trailing slash is removed.
HOME_DIR=$(eval echo $(printf "~%q" "$USERNAME")) # The path to the home dir of the user whose cronjob will be installed.
# Clear the crontab file before assembling it from the snippets:
> $HOME_DIR/crontab
for crontab in $(ls ${USERNAME}/crontab*); do
cat "${crontab}">> $HOME_DIR/crontab
done
sed -i "s|PATH_OF_GIT_HOME_DIR_TO_REPLACE|${GIT_PATH}|g" $HOME_DIR/crontab # Sets correct path to the git repo within the crontab.
sed -i "s|PATH_OF_HOME_DIR_TO_REPLACE|${HOME_DIR}|g" $HOME_DIR/crontab # Sets the correct path to the home dir of the user whose crontab will be installed.
echo "">>$HOME_DIR/crontab # Adds a newline
if [[ "$INSTALL_CRONTAB" == "true" ]]; then
crontab -u ${USERNAME} $HOME_DIR/crontab # Install the crontab in the given user's home dir.
USERNAME=$(echo "$dir" | sed "s/\/$//") # Dirname is the username. The trailing slash is removed.
user_UID=""
user_GID=""
echo "STARTING WORK ON USER $USERNAME"
if [[ -f "$USERNAME"/uid ]]; then
user_UID=$(cat "$USERNAME"/uid)
echo "UID => $user_UID"
fi
if [[ -f "$USERNAME"/gid ]]; then
user_GID=$(cat "$USERNAME"/gid)
echo "GID => $user_GID"
groupadd --gid "$user_GID" "$USERNAME"
[[ "$?" -eq 0 ]] || echo "Group id or name already exists." >&2
fi
if [[ -n "$user_UID" && -n "$user_GID" ]]; then
adduser --uid "$user_UID" --gid "$user_GID" "$USERNAME"
elif [[ -n "$user_UID" ]]; then
adduser --uid "$user_UID" "$USERNAME"
elif [[ -n "$user_GID" ]]; then
adduser --gid "$user_GID" "$USERNAME"
else
adduser "$USERNAME"
fi
id "$USERNAME"
if [[ "$?" -eq 0 ]]; then
HOME_DIR=$(eval echo $(printf "~%q" "$USERNAME")) # The path to the home dir of the user whose cronjob will be installed.
# Sets permissions of home dir.
[[ -e "$USERNAME"/permissions ]] && chmod "$(cat "$USERNAME"/permissions)" "$HOME_DIR"
if [[ "$CREATE_CRONTABS" == "true" ]]; then # 9 is the exit code indicating the username already exists.
# Clear the crontab file before assembling it from the snippets:
> $HOME_DIR/crontab
for crontab in ${USERNAME}/crontab*; do
[[ -e $crontab ]] || continue
cat "${crontab}">> $HOME_DIR/crontab
echo "">> $HOME_DIR/crontab
done
chown "$USERNAME":"$USERNAME" $HOME_DIR/crontab
sed -i "s|PATH_OF_GIT_HOME_DIR_TO_REPLACE|${GIT_PATH}|g" "$HOME_DIR"/crontab # Sets correct path to the git repo within the crontab.
sed -i "s|PATH_OF_HOME_DIR_TO_REPLACE|${HOME_DIR}|g" "$HOME_DIR"/crontab # Sets the correct path to the home dir of the user whose crontab will be installed.
sed -i '/^$/d' $HOME_DIR/crontab # purges random empty lines.
echo "">>$HOME_DIR/crontab # Adds a newline
if [[ "$INSTALL_CRONTAB" == "true" ]]; then
crontab -u ${USERNAME} $HOME_DIR/crontab # Install the crontab in the given user's home dir.
fi
fi
fi
echo "****** finished user ******"
done
# Add to groups after all the groups exist.
for dir in $(ls -d */ ); do
USERNAME=$(echo "$dir" | sed "s/\/$//") # Dirname is the username. The trailing slash is removed.
[[ -e "$USERNAME"/groups ]] || continue
echo "ADDING $USERNAME TO THE GROUPS SPECIFIED IN THEIR 'groups' FILE"
for group in $(cat "$USERNAME"/groups); do
gpasswd --add "$USERNAME" "$group"
done
done
cd .. # exits users folder, which is essential for the next commands
fi
if [[ "$ONLY_CRONTAB" != "true" && -d "files" ]]; then
# Add users to groups, specified in ENVIRONMENT_TYPE/groups/GROUP/users. Used if groups exist which aren't directly related to a user.
if [[ -d "groups" ]]; then
cd "groups"
for group in * ; do
[[ -e "$group"/users ]] || continue
echo "ADDING USERS IN 'users' FILE TO GROUP $group"
for user in $(cat "$group"/users); do
gpasswd -a "$user" "$group"
done
done
cd ..
fi
if [[ "$COPY_FILES" == "true" && -d "files" ]]; then
cd "files"
\cp -rL * / # copies all files accross, realising any symbolic links. The backslash escapes the alias cp -i.
systemctl daemon-reload
configuration/environments_scripts/central_reverse_proxy/batch-for-route53-dns-record-update.json
+32
View File
@@ -0,0 +1,32 @@
{
"Comment": "Alter logfiles record ",
"Changes": [
{
"Action": "UPSERT",
"ResourceRecordSet": {
"Name": "logfiles.internal.sapsailing.com",
"Type": "A",
"TTL": 60,
"ResourceRecords": [
{
"Value": "LOGFILES_INTERNAL_IP"
}
]
}
},
{
"Action": "UPSERT",
"ResourceRecordSet": {
"Name": "smtp.internal.sapsailing.com",
"Type": "A",
"TTL": 60,
"ResourceRecords": [
{
"Value": "SMTP_INTERNAL_IP"
}
]
}
}
]
}
configuration/environments_scripts/central_reverse_proxy/files/etc/cron.daily/logrotate
Executable
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
/usr/sbin/logrotate /etc/logrotate.conf
EXITVALUE=$?
if [ $EXITVALUE != 0 ]; then
/usr/bin/logger -t logrotate "ALERT exited abnormally with [$EXITVALUE]"
fi
exit 0
configuration/environments_scripts/central_reverse_proxy/files/etc/cron.weekly/awstats
Executable
+4
View File
@@ -0,0 +1,4 @@
#!/bin/bash
su -l -c '/usr/share/awstats/tools/awstats_updateall.pl now -configdir="/etc/awstats" -awstatsprog="/usr/share/awstats/wwwroot/cgi-bin/awstats.pl" >>/var/log/awstats-cron.out 2>>/var/log/awstats-cron.err'
#exec /usr/share/awstats/tools/awstats_updateall.pl now -configdir="/etc/awstats" -awstatsprog="/usr/share/awstats/wwwroot/cgi-bin/awstats.pl" >>/var/log/awstats-cron.out 2>>/var/log/awstats-cron.err
exit 0
configuration/environments_scripts/central_reverse_proxy/files/etc/cron.weekly/letsencrypt
Executable
+4
View File
@@ -0,0 +1,4 @@
#!/bin/bash
# Check if sail-insight.com cert needs renewing
/usr/bin/sudo -u certbot docker run --rm --name certbot -v "/etc/letsencrypt:/etc/letsencrypt" -v "/var/lib/letsencrypt:/var/lib/letsencrypt" -v "/home/trac/sail-insight-website/:/home/trac/sail-insight-website" certbot/certbot renew && service httpd reload
exit 0
configuration/environments_scripts/central_reverse_proxy/files/etc/cron.weekly/unique_ips_per_referrer
Executable
+4
View File
@@ -0,0 +1,4 @@
#!/bin/bash
cd /var/log/old
unique_ips_per_referrer_fast `find . -name 'access_log*'` bundesliga2015_elb_access_log.gz tw2015_elb_access_log.gz >/dev/null 2>/dev/null
exit 0
configuration/environments_scripts/central_reverse_proxy/files/etc/goaccess.conf
+720
View File
@@ -0,0 +1,720 @@
######################################
# Time Format Options (required)
######################################
#
# The hour (24-hour clock) [00,23]; leading zeros are permitted but not required.
# The minute [00,59]; leading zeros are permitted but not required.
# The seconds [00,60]; leading zeros are permitted but not required.
# See `man strftime` for more details
#
# The following time format works with any of the
# Apache/NGINX's log formats below.
#
time-format %H:%M:%S
#
# Google Cloud Storage or
# The time in microseconds since the Unix epoch.
#
#time-format %f
# Squid native log format
#
#time-format %s
######################################
# Date Format Options (required)
######################################
#
# The date-format variable followed by a space, specifies
# the log format date containing any combination of regular
# characters and special format specifiers. They all begin with a
# percentage (%) sign. See `man strftime`
#
# The following date format works with any of the
# Apache/NGINX's log formats below.
#
date-format %d/%b/%Y
#
# AWS | Amazon CloudFront (Download Distribution)
# AWS | Elastic Load Balancing
# W3C (IIS)
#
#date-format %Y-%m-%d
#
# Google Cloud Storage or
# The time in microseconds since the Unix epoch.
#
#date-format %f
# Squid native log format
#
#date-format %s
######################################
# Log Format Options (required)
######################################
#
# The log-format variable followed by a space or \t for
# tab-delimited, specifies the log format string.
#
# NOTE: If the time/date is a timestamp in seconds or microseconds
# %x must be used instead of %d & %t to represent the date & time.
# NCSA Combined Log Format
#log-format %h %^[%d:%t %^] "%r" %s %b "%R" "%u"
# NCSA Combined with virtual host name as prefix:
log-format %v %h %^[%d:%t %^] "%r" %s %b "%R" "%u"
# NCSA Combined Log Format with Virtual Host
#log-format %v:%^ %h %^[%d:%t %^] "%r" %s %b "%R" "%u"
# Common Log Format (CLF)
#log-format %h %^[%d:%t %^] "%r" %s %b
# Common Log Format (CLF) with Virtual Host
#log-format %v:%^ %h %^[%d:%t %^] "%r" %s %b
# W3C
#log-format %d %t %h %^ %^ %^ %^ %r %^ %s %b %^ %^ %u %R
# Squid native log format
#log-format %^ %^ %^ %v %^: %x.%^ %~%L %h %^/%s %b %m %U
# AWS | Amazon CloudFront (Download Distribution)
#log-format %d\t%t\t%^\t%b\t%h\t%m\t%^\t%r\t%s\t%R\t%u\t%^
# Google Cloud Storage
#log-format "%x","%h",%^,%^,"%m","%U","%s",%^,"%b","%D",%^,"%R","%u"
# AWS | Elastic Load Balancing
#log-format %dT%t.%^ %^ %h:%^ %^ %T %^ %^ %^ %s %^ %b "%r" "%u"
# AWSS3 | Amazon Simple Storage Service (S3)
#log-format %^[%d:%t %^] %h %^"%r" %s %^ %b %^ %L %^ "%R" "%u"
# Virtualmin Log Format with Virtual Host
#log-format %h %^ %v %^[%d:%t %^] "%r" %s %b "%R" "%u"
# In addition to specifying the raw log/date/time formats, for
# simplicity, any of the following predefined log format names can be
# supplied to the log/date/time-format variables. GoAccess can also
# handle one predefined name in one variable and another predefined
# name in another variable.
#
#log-format COMBINED
#log-format VCOMBINED
#log-format COMMON
#log-format VCOMMON
#log-format W3C
#log-format SQUID
#log-format CLOUDFRONT
#log-format CLOUDSTORAGE
#log-format AWSELB
#log-format AWSS3
######################################
# UI Options
######################################
# Choose among color schemes
# 1 : Monochrome
# 2 : Green
# 3 : Monokai (if 256-colors supported)
#
#color-scheme 3
# Prompt log/date configuration window on program start.
#
config-dialog false
# Color highlight active panel.
#
hl-header true
# Specify a custom CSS file in the HTML report.
#
#html-custom-css /path/file.css
# Specify a custom JS file in the HTML report.
#
#html-custom-js /path/file.js
# Set default HTML preferences.
#
# NOTE: A valid JSON object is required.
# DO NOT USE A MULTILINE JSON OBJECT.
# The parser will only parse the value next to `html-prefs` (single line)
# It allows the ability to customize each panel plot. See example below.
#
#html-prefs {"theme":"bright","perPage":5,"layout":"horizontal","showTables":true,"visitors":{"plot":{"chartType":"bar"}}}
# Set HTML report page title and header.
#
#html-report-title My Awesome Web Stats
# Format JSON output using tabs and newlines.
#
json-pretty-print false
# Turn off colored output. This is the default output on
# terminals that do not support colors.
# true : for no color output
# false : use color-scheme
#
no-color false
# Don't write column names in the terminal output. By default, it displays
# column names for each available metric in every panel.
#
no-column-names false
# Disable summary metrics on the CSV output.
#
no-csv-summary false
# Disable progress metrics.
#
no-progress false
# Disable scrolling through panels on TAB.
#
no-tab-scroll false
# Do not show the last updated field displayed in the HTML generated report.
#
#no-html-last-updated true
# Enable mouse support on main dashboard.
#
with-mouse false
# Maximum number of items to show per panel.
# Note: Only the CSV and JSON outputs allow a maximum greater than the
# default value of 366.
#
#max-items 366
# Custom colors for the terminal output
# Tailor GoAccess to suit your own tastes.
#
# Color Syntax:
# DEFINITION space/tab colorFG#:colorBG# [[attributes,] PANEL]
#
# FG# = foreground color number [-1...255] (-1 = default terminal color)
# BG# = background color number [-1...255] (-1 = default terminal color)
#
# Optionally:
#
# It is possible to apply color attributes, such as:
# bold,underline,normal,reverse,blink.
# Multiple attributes are comma separated
#
# If desired, it is possible to apply custom colors per panel, that is, a
# metric in the REQUESTS panel can be of color A, while the same metric in the
# BROWSERS panel can be of color B.
#
# The following is a 256 color scheme (hybrid palette)
#
#color COLOR_MTRC_HITS color110:color-1
#color COLOR_MTRC_VISITORS color173:color-1
#color COLOR_MTRC_DATA color221:color-1
#color COLOR_MTRC_BW color167:color-1
#color COLOR_MTRC_AVGTS color143:color-1
#color COLOR_MTRC_CUMTS color247:color-1
#color COLOR_MTRC_MAXTS color186:color-1
#color COLOR_MTRC_PROT color109:color-1
#color COLOR_MTRC_MTHD color139:color-1
#color COLOR_MTRC_HITS_PERC color186:color-1
#color COLOR_MTRC_HITS_PERC_MAX color139:color-1
#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 VISITORS
#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 OS
#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 BROWSERS
#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 VISIT_TIMES
#color COLOR_MTRC_VISITORS_PERC color186:color-1
#color COLOR_MTRC_VISITORS_PERC_MAX color139:color-1
#color COLOR_PANEL_COLS color243:color-1
#color COLOR_BARS color250:color-1
#color COLOR_ERROR color231:color167
#color COLOR_SELECTED color7:color167
#color COLOR_PANEL_ACTIVE color7:color237
#color COLOR_PANEL_HEADER color250:color235
#color COLOR_PANEL_DESC color242:color-1
#color COLOR_OVERALL_LBLS color243:color-1
#color COLOR_OVERALL_VALS color167:color-1
#color COLOR_OVERALL_PATH color186:color-1
#color COLOR_ACTIVE_LABEL color139:color235 bold underline
#color COLOR_BG color250:color-1
#color COLOR_DEFAULT color243:color-1
#color COLOR_PROGRESS color7:color110
######################################
# Server Options
######################################
# Specify IP address to bind server to.
#
#addr 0.0.0.0
# Run GoAccess as daemon (if --real-time-html enabled).
#
#daemonize false
# Ensure clients send the specified origin header upon the WebSocket
# handshake.
#
#origin http://example.org
# The port to which the connection is being attempted to connect.
# By default GoAccess' WebSocket server listens on port 7890
# See man page or http://gwsocket.io for details.
#
#port 7890
# Enable real-time HTML output.
#
#real-time-html true
# Path to TLS/SSL certificate.
# Note that ssl-cert and ssl-key need to be used to enable TLS/SSL.
#
#ssl-cert /path/ssl/domain.crt
# Path to TLS/SSL private key.
# Note that ssl-cert and ssl-key need to be used to enable TLS/SSL.
#
#ssl-key /path/ssl/domain.key
# URL to which the WebSocket server responds. This is the URL supplied
# to the WebSocket constructor on the client side.
#
# Optionally, it is possible to specify the WebSocket URI scheme, such as ws://
# or wss:// for unencrypted and encrypted connections.
# e.g., ws-url wss://goaccess.io
#
# If GoAccess is running behind a proxy, you could set the client side
# to connect to a different port by specifying the host followed by a
# colon and the port.
# e.g., ws-url goaccess.io:9999
#
# By default, it will attempt to connect to localhost. If GoAccess is
# running on a remote server, the host of the remote server should be
# specified here. Also, make sure it is a valid host and NOT an http
# address.
#
#ws-url goaccess.io
# Path to read named pipe (FIFO).
#
#fifo-in /tmp/wspipein.fifo
# Path to write named pipe (FIFO).
#
#fifo-in /tmp/wspipeout.fifo
######################################
# File Options
######################################
# Specify the path to the input log file. If set, it will take
# priority over -f from the command line.
#
#log-file /var/log/apache2/access.log
# Send all debug messages to the specified file.
#
#debug-file debug.log
# Specify a custom configuration file to use. If set, it will take
# priority over the global configuration file (if any).
#
#config-file <filename>
# Log invalid requests to the specified file.
#
#invalid-requests <filename>
# Do not load the global configuration file.
#
#no-global-config false
######################################
# Parse Options
######################################
# Enable a list of user-agents by host. For faster parsing, do not
# enable this flag.
#
agent-list false
# Enable IP resolver on HTML|JSON|CSV output.
#
with-output-resolver false
# Exclude an IPv4 or IPv6 from being counted.
# Ranges can be included as well using a dash in between
# the IPs (start-end).
#
#exclude-ip 127.0.0.1
#exclude-ip 192.168.0.1-192.168.0.100
#exclude-ip ::1
#exclude-ip 0:0:0:0:0:ffff:808:804-0:0:0:0:0:ffff:808:808
# Include HTTP request method if found. This will create a
# request key containing the request method + the actual request.
#
# <yes|no> [default: yes]
#
http-method yes
# Include HTTP request protocol if found. This will create a
# request key containing the request protocol + the actual request.
#
# <yes|no> [default: yes]
#
http-protocol yes
# Write output to stdout given one of the following files and the
# corresponding extension for the output format:
#
# /path/file.csv - Comma-separated values (CSV)
# /path/file.json - JSON (JavaScript Object Notation)
# /path/file.html - HTML
#
#output-format /path/file.html
# Ignore request's query string.
# i.e., www.google.com/page.htm?query => www.google.com/page.htm
#
# Note: Removing the query string can greatly decrease memory
# consumption, especially on timestamped requests.
#
no-query-string false
# Disable IP resolver on terminal output.
#
no-term-resolver false
# Treat non-standard status code 444 as 404.
#
444-as-404 false
# Add 4xx client errors to the unique visitors count.
#
4xx-to-unique-count false
# Include static files that contain a query string in the static files
# panel.
# e.g., /fonts/fontawesome-webfont.woff?v=4.0.3
#
all-static-files false
# Date specificity. Possible values: `date` (default), or `hr`.
#
#date-spec hr
# Decode double-encoded values.
#
double-decode false
# Enable parsing/displaying the given panel.
#
#enable-panel VISITORS
#enable-panel REQUESTS
#enable-panel REQUESTS_STATIC
#enable-panel NOT_FOUND
#enable-panel HOSTS
#enable-panel OS
#enable-panel BROWSERS
#enable-panel VISIT_TIMES
#enable-panel VIRTUAL_HOSTS
#enable-panel REFERRERS
#enable-panel REFERRING_SITES
#enable-panel KEYPHRASES
#enable-panel STATUS_CODES
#enable-panel REMOTE_USER
#enable-panel GEO_LOCATION
# Hour specificity. Possible values: `hr` (default), or `min` (tenth
# of a minute).
#
#hour-spec min
# Ignore crawlers from being counted.
# This will ignore robots listed under browsers.c
# Note that it will count them towards the total
# number of requests, but excluded from any of the panels.
#
ignore-crawlers false
# Parse and display crawlers only.
# This will ignore robots listed under browsers.c
# Note that it will count them towards the total
# number of requests, but excluded from any of the panels.
#
crawlers-only false
# Ignore parsing and displaying the given panel.
#
#ignore-panel VISITORS
#ignore-panel REQUESTS
#ignore-panel REQUESTS_STATIC
#ignore-panel NOT_FOUND
#ignore-panel HOSTS
#ignore-panel OS
#ignore-panel BROWSERS
#ignore-panel VISIT_TIMES
#ignore-panel VIRTUAL_HOSTS
ignore-panel REFERRERS
#ignore-panel REFERRING_SITES
ignore-panel KEYPHRASES
#ignore-panel STATUS_CODES
#ignore-panel REMOTE_USER
#ignore-panel GEO_LOCATION
# Ignore referers from being counted.
# This supports wild cards. For instance,
# '*' matches 0 or more characters (including spaces)
# '?' matches exactly one character
#
#ignore-referer *.domain.com
#ignore-referer ww?.domain.*
# Ignore parsing and displaying one or multiple status code(s)
#
#ignore-status 400
#ignore-status 502
# Number of lines from the access log to test against the provided
# log/date/time format. By default, the parser is set to test 10
# lines. If set to 0, the parser won't test any lines and will parse
# the whole access log.
#
#num-tests 10
# Parse log and exit without outputting data.
#
#process-and-exit false
# Display real OS names. e.g, Windows XP, Snow Leopard.
#
real-os true
# Sort panel on initial load.
# Sort options are separated by comma.
# Options are in the form: PANEL,METRIC,ORDER
#
# Available metrics:
# BY_HITS - Sort by hits
# BY_VISITORS - Sort by unique visitors
# BY_DATA - Sort by data
# BY_BW - Sort by bandwidth
# BY_AVGTS - Sort by average time served
# BY_CUMTS - Sort by cumulative time served
# BY_MAXTS - Sort by maximum time served
# BY_PROT - Sort by http protocol
# BY_MTHD - Sort by http method
# Available orders:
# ASC
# DESC
#
#sort-panel VISITORS,BY_DATA,ASC
#sort-panel REQUESTS,BY_HITS,ASC
#sort-panel REQUESTS_STATIC,BY_HITS,ASC
#sort-panel NOT_FOUND,BY_HITS,ASC
#sort-panel HOSTS,BY_HITS,ASC
#sort-panel OS,BY_HITS,ASC
#sort-panel BROWSERS,BY_HITS,ASC
#sort-panel VISIT_TIMES,BY_DATA,DESC
#sort-panel VIRTUAL_HOSTS,BY_HITS,ASC
#sort-panel REFERRERS,BY_HITS,ASC
#sort-panel REFERRING_SITES,BY_HITS,ASC
#sort-panel KEYPHRASES,BY_HITS,ASC
#sort-panel STATUS_CODES,BY_HITS,ASC
#sort-panel REMOTE_USER,BY_HITS,ASC
#sort-panel GEO_LOCATION,BY_HITS,ASC
# Consider the following extensions as static files
# The actual '.' is required and extensions are case sensitive
# For a full list, uncomment the less common static extensions below.
#
static-file .css
static-file .js
static-file .jpg
static-file .png
static-file .gif
static-file .ico
static-file .jpeg
static-file .pdf
static-file .txt
static-file .csv
static-file .zip
static-file .mp3
static-file .mp4
static-file .mpeg
static-file .mpg
static-file .exe
static-file .swf
static-file .woff
static-file .woff2
static-file .xls
static-file .xlsx
static-file .doc
static-file .docx
static-file .ppt
static-file .pptx
static-file .iso
static-file .gz
static-file .rar
static-file .svg
static-file .bmp
static-file .tar
static-file .tgz
static-file .tiff
static-file .tif
static-file .ttf
static-file .flv
#static-file .less
#static-file .ac3
#static-file .avi
#static-file .bz2
#static-file .class
#static-file .cue
#static-file .dae
#static-file .dat
#static-file .dts
#static-file .ejs
#static-file .eot
#static-file .eps
#static-file .img
#static-file .jar
#static-file .map
#static-file .mid
#static-file .midi
#static-file .mkv
#static-file .odp
#static-file .ods
#static-file .odt
#static-file .ogg
#static-file .otf
#static-file .pict
#static-file .pls
#static-file .ps
#static-file .qt
#static-file .rm
#static-file .svgz
#static-file .wav
#static-file .webp
######################################
# GeoIP Options
# Only if configured with --enable-geoip
######################################
# Standard GeoIP database for less memory usage.
#
#std-geoip false
# Specify path to GeoIP database file. i.e., GeoLiteCity.dat
# .dat file needs to be downloaded from maxmind.com.
#
# For IPv4 City database:
# wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz
# gunzip GeoLiteCity.dat.gz
#
# For IPv6 City database:
# wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz
# gunzip GeoLiteCityv6.dat.gz
#
# For IPv6 Country database:
# wget -N http://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz
# gunzip GeoIPv6.dat.gz
#
# Note: `geoip-city-data` is an alias of `geoip-database`
#
#geoip-database /usr/local/share/GeoIP/GeoLiteCity.dat
######################################
# Tokyo Cabinet Options
# Only if configured with --enable-tcb=btree
######################################
# GoAccess has the ability to process logs incrementally through the on-disk
# B+Tree database.
#
# It works in the following way:
# - A data set must be persisted first with --keep-db-files, then the same data
# set can be loaded with --load-from-disk.
# - If new data is passed (piped or through a log file), it will append it to
# the original data set.
# - To preserve the data at all times, --keep-db-files must be used.
# - If --load-from-disk is used without --keep-db-files, database files will be
# deleted upon closing the program.
# On-disk B+ Tree
# Persist parsed data into disk. This should be set to
# the first dataset prior to use `load-from-disk`.
# Setting it to false will delete all database files
# when exiting the program.
#keep-db-files true
# On-disk B+ Tree
# Load previously stored data from disk.
# Database files need to exist. See `keep-db-files`.
#load-from-disk false
# On-disk B+ Tree
# Path where the on-disk database files are stored.
# The default value is the /tmp/ directory
# Note the trailing forward-slash.
#
#db-path /tmp/
# On-disk B+ Tree
# Set the size in bytes of the extra mapped memory.
# The default value is 0.
#
#xmmap 0
# On-disk B+ Tree
# Max number of leaf nodes to be cached.
# Specifies the maximum number of leaf nodes to be cached.
# If it is not more than 0, the default value is specified.
# The default value is 1024.
#
#cache-lcnum 1024
# On-disk B+ Tree
# Specifies the maximum number of non-leaf nodes to be cached.
# If it is not more than 0, the default value is specified.
# The default value is 512.
#
#cache-ncnum 512
# On-disk B+ Tree
# Specifies the number of members in each leaf page.
# If it is not more than 0, the default value is specified.
# The default value is 128.
#
#tune-lmemb 128
# On-disk B+ Tree
# Specifies the number of members in each non-leaf page.
# If it is not more than 0, the default value is specified.
# The default value is 256.
#
#tune-nmemb 256
# On-disk B+ Tree
# Specifies the number of elements of the bucket array.
# If it is not more than 0, the default value is specified.
# The default value is 32749.
# Suggested size of the bucket array is about from 1 to 4
# times of the number of all pages to be stored.
#
#tune-bnum 32749
# On-disk B+ Tree
# Specifies that each page is compressed with ZLIB|BZ2 encoding.
# Disabled by default.
#
#compression zlib
configuration/environments_scripts/central_reverse_proxy/files/etc/logrotate.d/httpd
Symlink
+1
View File
@@ -0,0 +1 @@
../../../../repo/etc/logrotate.d/httpd
configuration/environments_scripts/central_reverse_proxy/files/etc/profile.d/sailing.sh
+2
View File
@@ -0,0 +1,2 @@
ulimit -n 30000
ulimit -u 40000
configuration/environments_scripts/central_reverse_proxy/files/etc/sudoers.d/trac_push_to_github_tools_sap
+1 -1
View File
@@ -1,2 +1,2 @@
# Enables the trac user, on all hosts, to execute commands as the root user. It can do this without a password but only to run the given command.
# Enables the trac user, on all hosts, to execute commands as the root user. It can do this without a password but only to run the given command.
trac ALL = (root) NOPASSWD: /usr/local/bin/push_hyperspace_to_github_tools_sap
configuration/environments_scripts/central_reverse_proxy/files/etc/sudoers.d/www-for-wiki-git
+2 -2
View File
@@ -1,3 +1,3 @@
# Enables the apache user, on all hosts, to execute commands as the wiki user. This can be done without a password for
# running git, and all of its commands.
# Enables the apache user, on all hosts, to execute commands as the wiki user. This can be done without a password for
# running git, and all of its commands. This is used for the cgi-bin script on the central.
apache ALL = (wiki) NOPASSWD: /usr/bin/git
configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/create-internal-status-file.service
Symlink
+1
View File
@@ -0,0 +1 @@
../../../../../repo/etc/systemd/system/create-internal-status-file.service
configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service
Symlink
+1
View File
@@ -0,0 +1 @@
../../../../../repo/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service
configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/remove-welcome-conf.service
+12
View File
@@ -0,0 +1,12 @@
[Unit]
Description=Removes welcome.conf.
Requires=-.mount network-online.target
After=-.mount network-online.target
Before=httpd.service get-latest-httpd-conf.service
[Install]
RequiredBy=multi-user.target
WantedBy=httpd.service
[Service]
Type=oneshot
RemainAfterExit=true
ExecStart=/bin/bash -c "cd /etc/httpd; [[ -e conf.d/welcome.conf ]] && rm -f conf.d/welcome.conf"
configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/tmux-management-panel.service
+12
View File
@@ -0,0 +1,12 @@
[Unit]
Description=Starts up the tmux management panel
Requires=-.mount
After=-.mount
[Install]
RequiredBy=multi-user.target
[Service]
Type=oenshot
RemainAfterExit=true
ExecStart=/usr/local/bin/tmuxManagementConsole.sh
configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/wiki.service
+13
View File
@@ -0,0 +1,13 @@
[Unit]
Description=Starts up the wiki
Requires=-.mount
After=-.mount
[Install]
RequiredBy=multi-user.target
[Service]
Type=exec
RemainAfterExit=true
ExecStart=su - wiki -c "cd /home/wiki && rackup -p 4567 /home/wiki/config.ru"
ExecStop=/bin/kill -SIGTERM "$MAINPID"
configuration/environments_scripts/central_reverse_proxy/files/etc/tmux.conf
+38
View File
@@ -0,0 +1,38 @@
# C-b is not acceptable -- Vim uses it
set-option -g prefix C-b
bind-key C-b last-window
# Start numbering at 0
set -g base-index 0
# Allows for faster key repetition
set -s escape-time 0
# Set status bar
set -g status-bg black
set -g status-fg white
set -g status-left ""
set -g status-right "#[fg=green]#H"
# set a short history limit to avoid using too much memory
set -g history-limit 1000
# Rather than constraining window size to the maximum size of any client
# connected to the *session*, constrain window size to the maximum size of any
# client connected to *that window*. Much more reasonable.
setw -g aggressive-resize on
# Allows us to use C-b a <command> to send commands to a TMUX session inside
# another TMUX session
bind-key a send-prefix
# Activity monitoring
setw -g monitor-activity off
#set -g visual-activity on
# Example of using a shell command in the status line
#set -g status-right "#[fg=yellow]#(uptime | cut -d ',' -f 2-)"
# Highlight active window
set-window-option -g window-status-current-style bg=red
set-window-option -g default-shell /bin/bash
configuration/environments_scripts/central_reverse_proxy/files/home/httpdConf/repo.git/hooks/post-receive
+9 -8
View File
@@ -19,7 +19,7 @@ sync_repo() {
# Gets all public ips for the instances with the chosen tag and iterates over the IPs.
for IP in $(aws ec2 describe-instances --filters Name=tag-key,Values="${1}" | jq -r '.Reservations[].Instances[].PublicIpAddress'); do
# strictHostKey... means no authenticity check. The sync-repo... script must be installed in the root user's home.
echo $IP
echo "Getting the latest changes on branch $2 for $IP ($1)"
ssh -o "StrictHostKeyChecking=no" -o "ConnectTimeout=10" root@${IP} "cd ~ && sync-repo-and-execute-cmd.sh '${DIR}' '${COMMAND}' '${2}'";
done;
}
@@ -35,12 +35,12 @@ check_conflict() {
merge_main_into_branch() {
#$1 Branch name. Assumes you are in the git repo.
git checkout "${1}"
check_conflict "$?" "Conflict checking out ${1}"
check_conflict "$?" "Conflict checking out ${1}, in the checked-out workspace."
git pull
check_conflict "$?" "Conflict pulling ${1}"
echo "Merging ${MAIN_BRANCH_NAME} into ${1}"
check_conflict "$?" "Conflict pulling ${1}, in the checked-out workspace."
echo "Merging ${MAIN_BRANCH_NAME} into ${1}, in the checked-out workspace."
git merge ${MAIN_BRANCH_NAME}
check_conflict "$?" "Conflict merging ${MAIN_BRANCH_NAME} into ${1}"
check_conflict "$?" "Conflict merging ${MAIN_BRANCH_NAME} into ${1}, in the checked-out workspace."
git push
}
while read oldrev newrev refname; do # These vars are passed on stdin to this hook.
@@ -55,14 +55,15 @@ while read oldrev newrev refname; do # These vars are passed on stdin to this
echo "The central pulls the central branch"
sync_repo "${CENTRAL_TAG}" "${CENTRAL_BRANCH_NAME}"
elif [[ "$branch_name" == "$MAIN_BRANCH_NAME" ]]; then
echo "Performing manipulation in the checked-out workspace on the central, used for merges and manipulation."
cd ~
unset GIT_DIR
cd ${CHECKED_OUT_WORKSPACE_NAME}
# Get the latest main
git checkout ${MAIN_BRANCH_NAME}
check_conflict "$?" "Uncommitted files in the working directory"
check_conflict "$?" "Uncommitted files in the working directory, in the checked-out workspace."
git pull
check_conflict "$?" "Merge conflict in main branch"
check_conflict "$?" "Merge conflict in main branch from the checked-out workspace."
if [[ "$(git rev-parse ${MAIN_BRANCH_NAME})" != "$(git rev-parse origin/${MAIN_BRANCH_NAME})" ]]; then
echo "Merge induced by pull. Pushing latest changes in ${MAIN_BRANCH_NAME} to origin" # In case of a merge from diverging branches. This seems cyclic as we are pushing to the same branch, but should resolve, when "Everything up-to-date".
git push
@@ -75,4 +76,4 @@ while read oldrev newrev refname; do # These vars are passed on stdin to this
fi
fi
fi
done
done
configuration/environments_scripts/central_reverse_proxy/files/home/trac/git/hooks/post-receive
+11 -11
View File
@@ -1,11 +1,11 @@
#!/bin/bash
# This hook executes once for the receive operation. It takes no arguments, but for each ref to be updated it receives on
# standard input a line of the format:
#
# <old-value> SP <new-value> SP <ref-name> LF
#
REFS=$( cat )
curl https://postreceive:ohfa7083.98@hudson.sapsailing.com/git/notifyCommit?url=ssh://trac@sapsailing.com/home/trac/git
if echo "$REFS" | awk '{ print $3; }' | grep "\<hyperspace\>"; then
sudo /usr/local/bin/push_hyperspace_to_github_tools_sap
fi
#!/bin/bash
# This hook executes once for the receive operation. It takes no arguments, but for each ref to be updated it receives on
# standard input a line of the format:
#
# <old-value> SP <new-value> SP <ref-name> LF
#
REFS=$( cat )
curl https://postreceive:ohfa7083.98@hudson.sapsailing.com/git/notifyCommit?url=ssh://trac@sapsailing.com/home/trac/git
if echo "$REFS" | awk '{ print $3; }' | grep "\<hyperspace\>"; then
sudo /usr/local/bin/push_hyperspace_to_github_tools_sap
fi
configuration/environments_scripts/central_reverse_proxy/files/home/wiki/app.rb
+100
View File
@@ -0,0 +1,100 @@
require 'gollum/app'
require 'digest/sha1'
#__DIR__ = File.expand_path(File.dirname(__FILE__))
#$: << __DIR__
class App < Precious::App
User = Struct.new(:name, :email, :password_hash, :can_write)
before { authenticate! }
before /edit/ do authorize_write! ; end
before do
session['gollum.author'] = {
:name => "%s" % settings.loggedInUser,
:email => "%s@example.com" % settings.loggedInUser,
}
end
helpers do
def authenticate!
puts "authentication"
public_urls=IO.readlines 'public.txt'
public_urls.each {|url|
if self.env['PATH_INFO'] == url.slice(0, url.length-1)
puts "Allowing " + url
return
end
if self.env['PATH_INFO'].start_with?('/wiki/images') ||
self.env['PATH_INFO'].start_with?('/favicon.ico')
puts "Allowing " + self.env['PATH_INFO']
return
end
}
if self.env['PATH_INFO'].split('/')[1] == 'gollum' && self.env['PATH_INFO'].split('/')[2] == 'assets'
return
end
@_auth = Rack::Auth::Basic::Request.new(request.env)
puts "here"
if self.env['PATH_INFO'].split('/').length >=2 &&
(self.env['PATH_INFO'].split('/')[1] != 'wiki' &&
self.env['PATH_INFO'].split('/')[2] != 'wiki' &&
self.env['PATH_INFO'].split('/')[1] != 'home' &&
self.env['PATH_INFO'].split('/')[2] != 'home' &&
self.env['PATH_INFO'].split('/')[1] != 'Home' &&
self.env['PATH_INFO'].split('/')[2] != 'Home' &&
self.env['PATH_INFO'].split('/')[1] != 'search' &&
self.env['PATH_INFO'].split('/')[2] != 'search' &&
self.env['PATH_INFO'].split('/')[1] != 'edit' &&
self.env['PATH_INFO'].split('/')[2] != 'edit' &&
self.env['PATH_INFO'].split('/')[1] != 'preview' &&
self.env['PATH_INFO'].split('/')[2] != 'preview')
throw(:halt, [403, 'Forbidden - You can not access anything outside wiki/ path.'])
end
puts settings
puts settings.authorized_users
if @_auth.provided?
end
if @_auth.provided? && @_auth.basic? && @_auth.credentials && @user = detected_user(@_auth.credentials)
Precious::App.set(:loggedInUser, @user.name)
return @user
else
response['WWW-Authenticate'] = %(Basic realm="Gollum Wiki")
throw(:halt, [401, "Not authorized\n"])
end
end
def authorize_write!
throw(:halt, [403, "Forbidden\n"]) unless @user.can_write
end
def users
puts settings
@_users ||= settings.authorized_users.map {|u| User.new(*u) }
end
def detected_user(credentials)
users.detect do |u|
[u.email, u.password_hash] ==
[credentials[0], Digest::SHA1.hexdigest(credentials[1])]
end
end
end
def commit_
{
:message => params[:message],
# :name => @user.name,
:email => @user.email
}
end
end
##set author
#class Precious::App
# before do
# session['gollum.author'] = {
# :name => "%s" % settings.loggedInUser,
# :email => "%s@example.com" % settings.loggedInUser,
# }
# end
#End
configuration/environments_scripts/central_reverse_proxy/files/home/wiki/config.ru
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env ruby
#require 'rubygems'
#require 'gollum/app'
## Define list of authorized users.
## Each user must have a username, password, name and email.
##
## Instead of a password you can also define a password_digest, which is the
## SHA-256 hash of a password.
##
## Example:
#users = YAML.load %q{
#---
#- username: rick
# password: asdf754&1129-@lUZw
# name: Rick Sanchez
# email: rick@example.com
#- username: morty
# password_digest: 5994471abb01112afcc18159f6cc74b4f511b99806da59b3caf5a9c173cacfc5
# name: Morty Smith
# email: morty@example.com
#}
#
## Allow unauthenticated users to read the wiki (disabled by default).
#options = { allow_unauthenticated_readonly: true }
#
## Allow only authenticated users to change the wiki.
## (NOTE: This must be loaded *before* Precious::App!)
#use Gollum::Auth, users, options
#
## That's it. The rest is for gollum only.
#gollum_path = "/home/wiki/gitwiki"
#wiki_options = {:universal_toc => false}
#Precious::App.set(:gollum_path, gollum_path)
#Precious::App.set(:wiki_options, wiki_options)
#run Precious::App
require 'rubygems'
require 'gollum/app'
require_relative 'app'
#require 'ruby-prof'
#require 'gollum/auth' # Don't forget to load the gem!
Gollum::Page.send :remove_const, :FORMAT_NAMES if defined? Gollum::Page::FORMAT_NAMES
#Gollum::Markup.formats.clear
#Gollum::Markup.formats[:markdown] = {
# :name => "MarkDown",
# :extensions => "md",
# :regexp => /md|mkdn?|mdown|markdown/
#}
gollum_path = "/home/wiki/gitwiki"
wiki_options = {:universal_toc => false}
Precious::App.set(:gollum_path, gollum_path)
Precious::App.set(:wiki_options, wiki_options)
Precious::App.set(:authorized_users, YAML.load_file(File.expand_path('users.yml', File.expand_path(File.dirname(__FILE__)))))
Precious::App.set(:loggedInUser, "anonymous");
App.set(:default_markup, :markdown) # set your favorite markup language
run App
#require 'rubygems'
#require 'gollum/app'
#
#gollum_path = "/home/wiki/gitwiki"
#wiki_options = {:universal_toc => false}
#Precious::App.set(:gollum_path, gollum_path)
#Precious::App.set(:default_markup, :markdown) # set your favorite markup language
#Precious::App.set(:wiki_options, wiki_options)
#run Precious::App
configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/awsmfalogon.sh
Symlink
+1
View File
@@ -0,0 +1 @@
../../../../../repo/usr/local/bin/awsmfalogon.sh
configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/setupHttpdGitLocal.sh
Symlink
+1
View File
@@ -0,0 +1 @@
../../../../../repo/usr/local/bin/setupHttpdGitLocal.sh
configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/tmuxManagementConsole.sh
Executable
+29
View File
@@ -0,0 +1,29 @@
#!/bin/bash
command -v tmux >/dev/null 2>&1 || { echo "I require tmux but it's not installed. Aborting." >&2; exit 1; }
sn=sailing
SERVERS_DIR=/home/trac/servers
TMUX_ACTIVE=`tmux has-session -t $sn 2>/dev/null`
if [ $? -eq 0 ]; then
echo "Session exists...not configuring a new one"
else
echo "Session does not exist...creating a new one with name $sn"
cd /home/trac/git
tmux new-session -s "$sn" -n "BUILD" -d
counter=0
tmux new-window -t "$sn:$[counter+1]" -n "GOAccess" "bash -c 'goaccess -f /var/log/httpd/access_log'; bash"
cd /opt/
tmux new-window -t "$sn:$[counter+2]" -n "ATop" "bash -c 'apachetop -f /var/log/httpd/access_log'; bash"
cd /home/trac/servers
tmux new-window -t "$sn:$[counter+3]" -n "Logs" "bash -c 'ls -lah'; bash"
tmux select-window -t "$sn:0"
fi
configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/unique_ips_per_referrer
Executable
+55
View File
@@ -0,0 +1,55 @@
#!/bin/bash
CACHE=/var/log/old/cache/unique-ips-per-referrer
VISITED_FILES=$CACHE/visited
STATS=$CACHE/stats
MONTHS=$STATS/months
mkdir -p "$STATS"
# groups Apache / httpd log entries into "${referrer}.ips" files, where ${referrer} is the
# referrer URL identifying the "event"; the lines written to the .ips files hold the IP
# address of the requestor, the date (not the time) and the user agent string.
# Sorting for unique entries should give a count similar to what goaccess is using to
# determine "unique visitors."
# A sample line:
# 505Worlds2012.sapsailing.com 66.249.73.53 - - [12/Jan/2014:03:33:11 +0000] "GET /robots.txt HTTP/1.1" 404 238 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
echo Starting "$0" at `date` on file set "$*"
for i in $*; do
grep -q "^$i\$" $VISITED_FILES
if [ "$?" = "0" ]; then
echo "Already visited $i; ignoring (edit $VISITED_FILES to change this)."
else
echo "Analyzing log file $i"
if [ ${i: -3} == ".gz" ]; then
gzip -cd $i
else
cat $i
fi | recode ISO-8859-1..UTF-8 | sed -e 's/^\([^ ]*\) \([^ ]*\) \([^ ]*\) \([^ ]*\) \[\([^:]*\):\([^]]*\)\] \"[^"]*\" [^ ]* [^ ]* \"[^"]*\" \"\([^"]*\)\"/\1 \2 \5 \7/' | while read referrer hit; do
echo "$hit" >>${STATS}/${referrer}.ips
done
echo "$i" >>$VISITED_FILES
fi
done
for i in ${STATS}/*.ips; do
echo -n "`basename $i .ips` "
cat $i | sort -u | wc | awk '{ print $1; }'
# TODO consider producing an AWStats configuration file for each virtual host name and splitting all new logs into virtual host name-specific log files
done | tee $STATS/unique-ips-days-useragents-per-event
echo "Wrote total results to $STATS/unique-ips-days-useragents-per-event"
# Now filter and group by month
cat ${STATS}/*.ips | awk '{ print $2; }' | sed -e 's/^[0-9]*\///' | sort -u >$MONTHS
for month in `cat $MONTHS`; do
for i in ${STATS}/*.ips; do
echo -n "`basename $i .ips` "
cat $i | grep "^[^ ]* [0-9]*/$month .*" | sort -u | wc | awk '{ print $1; }'
# TODO consider producing an AWStats configuration file for each virtual host name and splitting all new logs into virtual host name-specific log files
done | tee $STATS/unique-ips-days-useragents-per-event-`echo $month | tr / -`
echo "Wrote per-month results to $STATS/unique-ips-days-useragents-per-event-`echo $month | tr / -`"
done
echo "Done at `date`."
configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/unique_ips_per_referrer_fast
Executable
+16
View File
@@ -0,0 +1,16 @@
#!/bin/bash
JAVA_HOME=/opt/sapjvm_8
VARLOGOLD=/var/log/old
# Production:
CACHE=$VARLOGOLD/cache/unique-ips-per-referrer
# Test:
#CACHE=$VARLOGOLD/cache/unique-ips-per-referrer/test
JAR_FILE=$VARLOGOLD/com.sap.sse.jar
$JAVA_HOME/bin/java -Xmx8G -jar $JAR_FILE $CACHE $*
# Do the sorting of all .unique files now:
for i in `find $CACHE -name '*.unique'`; do
cat $i | awk '{ print $2 " " $1; }' | sort -rn >${i}.sorted
done
configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/unique_ips_per_referrer_generate_month_results_only
Executable
+16
View File
@@ -0,0 +1,16 @@
#!/bin/bash
CACHE=/var/log/old/cache/unique-ips-per-referrer
VISITED_FILES=$CACHE/visited
STATS=$CACHE/stats
MONTHS=$STATS/months
for month in `cat $MONTHS`; do
for i in ${STATS}/*.ips; do
echo -n "`basename $i .ips` "
cat $i | grep "^[^ ]* [0-9]*/$month .*" | sort -u | wc | awk '{ print $1; }'
# TODO consider producing an AWStats configuration file for each virtual host name and splitting all new logs into virtual host name-specific log files
done | tee $STATS/unique-ips-days-useragents-per-event-`echo $month | tr / -`
echo "Wrote per-month results to $STATS/unique-ips-days-useragents-per-event-`echo $month | tr / -`"
done
echo "Done."
configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/unique_ips_per_referrer_generate_results_only
Executable
+26
View File
@@ -0,0 +1,26 @@
#!/bin/bash
CACHE=/var/log/old/cache/unique-ips-per-referrer
VISITED_FILES=$CACHE/visited
STATS=$CACHE/stats
MONTHS=$STATS/months
for i in ${STATS}/*.ips; do
echo -n "`basename $i .ips` "
cat $i | sort -u | wc | awk '{ print $1; }'
# TODO consider producing an AWStats configuration file for each virtual host name and splitting all new logs into virtual host name-specific log files
done | tee $STATS/unique-ips-days-useragents-per-event
echo "Wrote total results to $STATS/unique-ips-days-useragents-per-event"
# Now filter and group by month
cat ${STATS}/*.ips | awk '{ print $2; }' | sed -e 's/^[0-9]*\///' | sort -u >$MONTHS
for month in `cat $MONTHS`; do
for i in ${STATS}/*.ips; do
echo -n "`basename $i .ips` "
cat $i | grep "^[^ ]* [0-9]*/$month .*" | sort -u | wc | awk '{ print $1; }'
# TODO consider producing an AWStats configuration file for each virtual host name and splitting all new logs into virtual host name-specific log files
done | tee $STATS/unique-ips-days-useragents-per-event-`echo $month | tr / -`
echo "Wrote per-month results to $STATS/unique-ips-days-useragents-per-event-$month"
done
echo "Done."
configuration/environments_scripts/central_reverse_proxy/files/var/www/cgi-bin/reverseProxyHealthcheck.sh
Symlink
+1
View File
@@ -0,0 +1 @@
../../../../../repo/var/www/cgi-bin/reverseProxyHealthcheck.sh
configuration/environments_scripts/central_reverse_proxy/groups/static/gid
+1
View File
@@ -0,0 +1 @@
1006
configuration/environments_scripts/central_reverse_proxy/groups/static/users
+2
View File
@@ -0,0 +1,2 @@
wiki
trac
configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh
Executable
+45
View File
@@ -0,0 +1,45 @@
#!/bin/bash
# PART 2
# Assumes to be run after or be invoked by setup-central-reverse-proxy.sh which
# is assumed to have prepared content for user home folders in /root/temporary_home_copy,
# has set up the software packages, and that the user has mounted the volumes with their
# original content to /home, /var/log, et cetera.
#
# Call as follows:
# setup-central-reverse-proxy-part-2.sh {external-ip-of-new-instance} {image-type}
# where {image-type} identifies the "environment" from configuration/environments_scripts
# which usually would be "central_reverse_proxy" here, passed in from the previous stage's
# script.
IP=$1
IMAGE_TYPE="$2"
GIT_COPY_USER="wiki"
RELATIVE_PATH_TO_GIT="gitwiki" # the relative path to the repo within the git_copy_user
TEMPORARY_HOME_COPY_LOCATION="/root/temporary_home_copy" # home nested within this.
ssh -A "root@${IP}" "bash -s" << EOF
sudo systemctl start crond.service
. imageupgrade_functions.sh
cp -r "$TEMPORARY_HOME_COPY_LOCATION"/home /
rm -rf "$TEMPORARY_HOME_COPY_LOCATION"
build_crontab_and_setup_files -f "${IMAGE_TYPE}" "${GIT_COPY_USER}" "${RELATIVE_PATH_TO_GIT}" # files have already been copied so -f is used.
chown trac:static /var/www/static
# setup nfs
systemctl enable nfs-server
echo "/var/log/old 172.31.0.0/16(rw,nohide,no_root_squash)
/home/scores 172.31.0.0/16(rw,nohide,no_root_squash)" >>/etc/exports
systemctl start nfs-server
# scp -p -o StrictHostKeyChecking=no -r root@sapsailing.com:/etc/ssh /etc # causes some issue with MACs
# append hostname to sysconfig
echo "HOSTNAME=sapsailing.com" >> /etc/sysconfig/network
sed -i "s/\(127.0.0.1 *\)/\1 sapsailing.com /" /etc/hosts
hostname sapsailing.com
hostnamectl set-hostname sapsailing.com
EOF
ssh -A -f root@"$IP" "cd /var/log/old/cache/docker/registry && nohup docker-compose up &>/dev/null &" &> /dev/null
echo "Please now run the script target-group-tag-route53-nfs-elasticIP-setup.sh which configures the EC2 instance tags, adds to the "
echo "necessary target groups, modifies a few records in route53 (logs.internal.sapsailing.com"
echo "and smtp.internal.sapsailing.com), remounts those dependent on this, and sets the elastic IP."
echo "You will need to have the aws cli installed and have the necessary permissions to make these alterations manually."
echo "In particular, make sure you have an active session token in your shell's environment, e.g., obtained"
echo "through the awsmfalogon.sh script."
echo "Have a great day!"
configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh
Executable
+201
View File
@@ -0,0 +1,201 @@
#!/bin/bash
# Setup script for a "Central Reverse HTTP Proxy" that runs Apache httpd,
# provides the Git repository, runs Bugzilla and Gollum for wiki access,
# furthermore AWStats, goaccess and apachetop support within a tmux session,
# releases.sapsailing.com, jobs.sapsailing.com content, and a Docker
# infrastructure for a self-hosted Docker image registry.
#
# Start by launching a new instance, e.g., of type m3.xlarge, in the same AZ
# as the current Webserver / Central Reverse Proxy. This will become important
# as you will need to detach volumes from the latter to attach them to the
# new instance.
#
# Then, call this script with the new instance's external IP address as the first,
# and with a "bearer token" as a second argument, authenticating a user at
# security-service.sapsailing.com which needs to have the following permissions:
# USER:READ:*
# SSH_KEY_READ:*
# which will be used to determine the landscape management users, for example. In addition,
# add the IP/hostname of an instance with the Git repository checked out, containing
# the file configuration/imageupgrade_functions.sh, and the path to the repo on that instance.
#
# Example usage:
# setup-central-reverse-proxy.sh 1.2.3.4 0OcJ1938QE5it875kjlQe7HnzQ6740jsnMEVzowjZrs= 18.170.25.225 /home/sailing/code
# This will do all necessary set-up up to the point where the large volumes
# currently attached to and mounted on the current Central Reverse Proxy will
# need to be unmounted, detached, attached to the new instance, and mounted there.
if [[ "$#" -ne 4 ]]; then
echo "IP and bearer token required. Please check comment description for further details."
fi
IP=$1
BEARER_TOKEN=$2
IMAGEUPGRADE_FUNCTIONS_IP="$3" # can be a domain name, such as sapsailing.com
IMAGEUPGRADE_FUNCTIONS_PATH_ON_INSTANCE_TO_GIT="$4"
IMAGE_TYPE="central_reverse_proxy"
HTTP_LOGROTATE_ABSOLUTE=/etc/logrotate.d/httpd
GIT_COPY_USER="wiki"
RELATIVE_PATH_TO_GIT="gitwiki" # the relative path to the repo within the git_copy_user
# The aws credentials will have to be manually installed in the aws user.
ssh -A "ec2-user@${IP}" "bash -s" << FIRSTEOF
# Correct authorized keys. May not be necessary if update_authorized_keys is running.
sudo su - -c "cat ~ec2-user/.ssh/authorized_keys > /root/.ssh/authorized_keys"
FIRSTEOF
# writes std error to local text file
ssh -A "root@${IP}" "bash -s" << SECONDEOF >log.txt
# update instance
yum update -y
yum install -y httpd mod_proxy_html tmux nfs-utils git whois jq cronie iptables mailx nmap icu mariadb105-server tree #icu is a c/c++ library that provides unicode and globalisation support for software development.
# docker setup
yum install -y docker
sudo curl -L "https://github.com/docker/compose/releases/download/v2.26.1/docker-compose-\$(uname -s)-\$(uname -m)" -o /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose
yum install -y perl perl-CGI perl-Template-Toolkit perl-CPAN perl-DBD-MySQL mod_perl perl-GD gcc-c++
# ruby and gollum for wiki
yum group install -y "Development Tools"
yum install -y ruby ruby-devel libicu libicu-devel zlib zlib-devel git cmake openssl-devel libyaml-devel
gem install gollum -v 5.3.2
gem update --system 3.5.7
cd /home
# The following line is for production use:
scp -o StrictHostKeyChecking=no -p root@"$IMAGEUPGRADE_FUNCTIONS_IP":"$IMAGEUPGRADE_FUNCTIONS_PATH_ON_INSTANCE_TO_GIT"/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh /usr/local/bin
# The following line is for test use, copying from a test instance with a check-out Git workspace:
# scp -o StrictHostKeyChecking=no -p "root@13.40.100.54:/home/sailing/code/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh" /usr/local/bin
. imageupgrade_functions.sh
setup_cloud_cfg_and_root_login
# setup files
build_crontab_and_setup_files -c -n "${IMAGE_TYPE}" "${GIT_COPY_USER}" "${RELATIVE_PATH_TO_GIT}" # -c & -n mean only files are copied over.
cd /home
for folder in * ; do
[[ -d "\$folder" ]] || continue
grep "\$folder" /etc/passwd || continue
chown -R "\$folder":"\$folder" "\$folder"
done
# setup mail
setup_mail_sending
# setup sshd config
setup_sshd_resilience
# setup goaccess and apachetop
setup_apachetop
setup_goaccess
# copy bugzilla
scp -o StrictHostKeyChecking=no root@sapsailing.com:/var/www/static/bugzilla-5.0.4.tar.gz /usr/local/src
cd /usr/local/src
tar -xzvf bugzilla-5.0.4.tar.gz
mv bugzilla-5.0.4 /usr/share/bugzilla
cd /usr/share/bugzilla/
scp -o StrictHostKeyChecking=no root@sapsailing.com:/usr/share/bugzilla/localconfig .
# essentials bugzilla
/usr/bin/perl install-module.pl DateTime
/usr/bin/perl install-module.pl DateTime::TimeZone
/usr/bin/perl install-module.pl Email::Sender
/usr/bin/perl install-module.pl Email::MIME
/usr/bin/perl install-module.pl List::MoreUtils
/usr/bin/perl install-module.pl Math::Random::ISAAC
/usr/bin/perl install-module.pl JSON::XS
# important bugzilla
/usr/bin/perl install-module.pl Email::Address
/usr/bin/perl install-module.pl autodie
/usr/bin/perl install-module.pl Class::XSAccessor
# nice to have for buzilla
/usr/bin/perl install-module.pl Date::Parse
/usr/bin/perl install-module.pl Email::Send
/usr/bin/perl install-module.pl DBI
/usr/bin/perl install-module.pl IO::Socket::SSL
/usr/bin/perl install-module.pl Chart::Lines
/usr/bin/perl install-module.pl Template::Plugin::GD::Image
/usr/bin/perl install-module.pl GD::Text
/usr/bin/perl install-module.pl GD::Graph
/usr/bin/perl install-module.pl PatchReader
/usr/bin/perl install-module.pl Authen::Radius
/usr/bin/perl install-module.pl JSON::RPC
/usr/bin/perl install-module.pl TheSchwartz
/usr/bin/perl install-module.pl Daemon::Generic
/usr/bin/perl install-module.pl File::MimeInfo::Magic
/usr/bin/perl install-module.pl File::Copy::Recursive
# use the localconfig file to setup the bugzilla
SECONDEOF
read -n 1 -p "Check bugzilla localconfig file and then press a key to continue" key_pressed
# t forces tty allocation.
ssh root@"${IP}" -A -t 'cd /usr/share/bugzilla/; ./checksetup.pl'
ssh -A "root@${IP}" "cpan install Geo::IP"
ssh -A "root@${IP}" "bash -s" << THIRDEOF >>log.txt
. imageupgrade_functions.sh
echo $BEARER_TOKEN > /root/ssh-key-reader.token
# awstats - depends on some of the previous perl modules.
scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/usr/share/GeoIP /usr/share/GeoIP
cd /usr/local/src
wget http://prdownloads.sourceforge.net/awstats/awstats-7.0.tar.gz
tar -zvxf awstats-7.0.tar.gz
mv awstats-7.0/ /usr/share/awstats
mkdir /var/lib/awstats
scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/etc/awstats /etc/awstats
chmod 755 /root
cd ~
# Copies across the key vault and other relevant secrets from the existing
# Central Reverse Proxy's /root folder:
rsync -a root@sapsailing.com:/root/{dev-secrets,github_tools_sap.pat,hudson-aws-credentials,key_vault,mail.properties,new_version_key_vault,secrets,ssh-key-reader.token} /root
scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/etc/letsencrypt /etc
# add basic test page which won't cause redirect error code if used as a health check.
cat <<EOF > /var/www/html/index.html
<!DOCTYPE html><html lang="en"><head><title>Health check</title><meta charset="UTF-8"></head><body><h1>Test page</h1></body></html>
EOF
echo "net.ipv4.ip_conntrac_max = 131072" >> /etc/sysctl.conf
# setup fail2ban
setup_fail2ban
setup_keys "${IMAGE_TYPE}"
# setup logrotate.d/httpd
# echo "Patching $HTTP_LOGROTATE_ABSOLUTE so that old logs go to /var/log/old/$IP" >>/var/log/sailing.out
# mkdir --parents "/var/log/old/REVERSE_PROXIES/${IP}"
# sed -i "s|/var/log/old|/var/log/old/REVERSE_PROXIES/${IP}|" $HTTP_LOGROTATE_ABSOLUTE
# logrotate.conf setup
sed -i 's/rotate 4/rotate 20 \n\nolddir \/var\/log\/logrotate-target/' /etc/logrotate.conf
sed -i "s/^#compress/compress/" /etc/logrotate.conf
# setup httpd git
(/usr/local/bin/setupHttpdGitLocal.sh "httpdConf@sapsailing.com:repo.git" central "Central Reverse Proxy")
scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/etc/httpd/conf/pass* /etc/httpd/conf/
chown root:root /etc/httpd/conf/pass*
# create mountpoints (see part 2 for ownership changes)
mkdir /var/log/old
mkdir /var/www/static
download_and_install_latest_sap_jvm_8
# enable units which build-crontab doesn't
systemctl enable httpd
systemctl start httpd
sudo systemctl enable crond.service
sudo systemctl enable postfix
sudo systemctl restart postfix
mkdir --parents /root/temporary_home_copy/home
mv /home/* /root/temporary_home_copy/home
echo "UUID=f03cc464-c3c0-452a-87da-e0eadc4c497f /var/log ext4 defaults,noatime,commit=30 0 0
UUID=23d42c52-85ee-4f6d-bdfe-c62f69bb689f /home ext4 defaults,noatime,commit=30 0 0
UUID=0b15f5cb-fd3e-48e6-8195-be248cd7726d /var/www/static ext3 defaults,noatime,commit=30 0 0
UUID=ff598428-d380-4429-a690-3809157506b7 /var/log/old ext3 defaults,noatime,commit=30 0 0
UUID=d371e530-c189-4012-ae57-45d67a690554 /var/log/old/cache ext4 defaults,noatime,commit=30 0 0" >>/etc/fstab
THIRDEOF
echo "Your turn! READ CAREFULLY! The instance is now prepared."
echo "Please remove the existing central reverse proxy from all target groups tagged with \"CentralReverseProxy\""
echo "or \"allReverseProxies\" (draining can take 5 mins)."
echo "Also ensure there is at least 1 healthy disposable in the SAME availability zone as the archive,"
echo "so there is no risk of all the targets being briefly unhealthy."
echo "Then unmount the volumes /var/log, /home, /var/www/static, /var/log/old and /var/log/old/cache from the existing reverse proxy,"
echo "detach, reattach to the new instance and remount as follows:"
echo "The detaching and attaching can be done in the AWS EC2 console by going to the webserver"
echo "and clicking on the volumes in question (found within the storage tab)."
echo "Then click Detach from within the Actions column. The mounting can be done using"
echo " umount -l -f <location>"
echo "on the existing instance; the remounting can be done with"
echo " mount -a"
echo "on the new instance."
echo "For further details, checkout this wiki page https://wiki.sapsailing.com/wiki/info/landscape/amazon-ec2#amazon-ec2-for-sap-sailing-analytics_landscape-overview_apache-httpd-the-central-reverse-proxy-webserver-and-disposable-reverse-proxies"
echo "Check that all these volumes were mounted successfully, e.g. by invoking"
echo " mount"
echo "without any arguments. If everything looks good, please press a key to trigger part 2, which"
echo "sets up the hostname, copies /etc/ssh and configures the users and crontabs."
read -n 1 -p "Press a key to continue" key_pressed
"$(dirname $0)"/setup-central-reverse-proxy-part-2.sh "$IP" "$IMAGE_TYPE"
# anything in etc
#not available: perl-HTML-Template /usr/bin/perl install-module.pl GD
configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh
Executable
+61
View File
@@ -0,0 +1,61 @@
#!/bin/bash
# This script is to be run after part 1 and part 2. It should be run locally and
# requires the user to have the AWS CLI installed, as well as credentials. They must also have run the awsmfalogon.sh to authenticate.
# It will add the necessary tags, alter route 53 records, remount the nfs mounts that depend on these changes, add to the
# necessary target groups and switch the elastic IP.
if [[ "$#" -ne 1 ]]; then
echo "Please pass the remote IP address as the only parameter."
echo "Please check comment description for usage."
exit 2
fi
target_groups=$(aws elbv2 describe-target-groups)
LOCAL_IPV4=$(ssh root@"$1" "ec2-metadata --local-ipv4 | sed \"s/local-ipv4: *//\"")
INSTANCE_ID=$(ssh root@"$1" "ec2-metadata --instance-id | sed \"s/instance-id: *//\"")
ELASTIC_IP="54.229.94.254"
TAGS=("allReverseProxies" "CentralReverseProxy")
extract_public_ip() {
jq -r ' .Instances | .[] | .PublicIpAddress' | grep -v null
}
select_instances_by_tag() {
# $1: tag
jq -r '.Reservations | .[] | select(.Instances | .[] | .Tags| any (.Key=="'"$1"'"))'
}
cd $(dirname "$0")
# give the instance the necessary tags.
for tag in "${TAGS[@]}"; do
aws ec2 create-tags --resources "$INSTANCE_ID" --tags Key="$tag",Value=""
done
# The nlb is the exception case as we use the load balancer arn to further identify it.
nlbArn=$(aws elbv2 describe-tags --resource-arns $(echo "$target_groups" | jq -r '.TargetGroups | .[] | select(.LoadBalancerArns | .[] | contains("loadbalancer/net") ) | .TargetGroupArn') | jq -r '.TagDescriptions | .[] | select(.Tags | any(.Key=="allReverseProxies") ) | .ResourceArn')
echo "Registering with nlb"
aws elbv2 register-targets --target-group-arn "$nlbArn" --targets Id="${LOCAL_IPV4}",Port=80
echo "Fetching tags"
describe_tags=$(aws elbv2 describe-tags --resource-arns $(echo "$target_groups" | jq -r '.TargetGroups | .[] | .TargetGroupArn'))
for tag in "${TAGS[@]}"; do
echo "Adding to target groups with $tag"
for tgArn in $(echo "$describe_tags" | jq -r '.TagDescriptions | .[] | select(.Tags | any(.Key=="'"$tag"'") ) | .ResourceArn'); do
if [[ "$tgArn" != "$nlbArn" ]]; then
echo "Registering in $tgArn as it has the correct tag"
aws elbv2 register-targets --target-group-arn "$tgArn" --targets Id="${INSTANCE_ID}"
[[ "$?" -eq 0 ]] || echo "Register target not successful"
fi
done
done
# alter records using batch file.
sed -i "s/LOGFILES_INTERNAL_IP/$internal_ip/" batch-for-route53-dns-record-update.json
sed -i "s/SMTP_INTERNAL_IP/$internal_ip/" batch-for-route53-dns-record-update.json
###### DO NOT ENABLE WHILST TESTING: aws route53 change-resource-record-sets --hosted-zone-id Z2JYWXYWLLRLTE --change-batch file://batch-for-route53-dns-record-update.json
# reload the nfs mountpoints.
echo "Describing instances for remounting."
describe_instances=$(aws ec2 describe-instances)
for instanceIp in $(echo "$describe_instances" | select_instances_by_tag "sailing-analytics-server" | extract_public_ip); do
echo "Remounting on $instanceIp"
ssh root@"${instanceIp}" "umount -l -f /var/log/old; umount -l -f /home/scores; mount -a"
done
for instanceIp in $(echo "$describe_instances" | select_instances_by_tag "DisposableProxy" | extract_public_ip); do
echo "Remounting on $instanceIp"
ssh root@"${instanceIp}" "umount -l -f /var/log/old; mount -a"
done
# Alter the elastic IP.
# WARNING: Will terminate connections via the existing public ip.
aws ec2 associate-address --instance-id "${INSTANCE_ID}" --public-ip "${ELASTIC_IP}"
configuration/environments_scripts/central_reverse_proxy/users/certbot/gid
+1
View File
@@ -0,0 +1 @@
1016
configuration/environments_scripts/central_reverse_proxy/users/certbot/uid
+1
View File
@@ -0,0 +1 @@
1014
configuration/environments_scripts/central_reverse_proxy/users/httpdConf/gid
+1
View File
@@ -0,0 +1 @@
1017
configuration/environments_scripts/central_reverse_proxy/users/httpdConf/uid
+1
View File
@@ -0,0 +1 @@
1015
configuration/environments_scripts/central_reverse_proxy/users/root/crontab-update-authorized-keys-httpdConf-user
+1
View File
@@ -0,0 +1 @@
* * * * * export PATH=/bin:/usr/bin:/usr/local/bin; sleep $(( $RANDOM * 60 / 32768 )); update_authorized_keys_for_landscape_managers_if_changed $( cat PATH_OF_HOME_DIR_TO_REPLACE/ssh-key-reader.token ) https://security-service.sapsailing.com /home/httpdConf # 2>&1 >>/var/log/sailing.err
configuration/environments_scripts/central_reverse_proxy/users/scores/gid
+1
View File
@@ -0,0 +1 @@
1013
configuration/environments_scripts/central_reverse_proxy/users/scores/uid
+1
View File
@@ -0,0 +1 @@
1012
configuration/environments_scripts/central_reverse_proxy/users/trac/uid
+1
View File
@@ -0,0 +1 @@
1004
configuration/environments_scripts/central_reverse_proxy/users/wiki/groups
+1
View File
@@ -0,0 +1 @@
trac
configuration/environments_scripts/central_reverse_proxy/users/wiki/uid
+1
View File
@@ -0,0 +1 @@
1003
configuration/environments_scripts/repo/etc/logrotate.d/httpd
+15
View File
@@ -0,0 +1,15 @@
/var/log/httpd/*log {
missingok
notifempty
sharedscripts
olddir /var/log/logrotate-target
# use date as a suffix of the rotated file
dateext
compress
delaycompress
postrotate
/bin/systemctl reload httpd.service > /dev/null 2>/dev/null || true
mv /var/log/logrotate-target/*.gz /var/log/old
endscript
}
configuration/environments_scripts/repo/etc/systemd/system/create-internal-status-file.service
+14
View File
@@ -0,0 +1,14 @@
[Unit]
Description=Creates httpd internal-status file.
Requires=-.mount network-online.target
After=-.mount network-online.target get-latest-httpd-conf.service
Before=httpd.service
# Staring before means no "systemctl start httpd" is needed.
[Install]
RequiredBy=multi-user.target
WantedBy=httpd.service
[Service]
Type=oneshot
RemainAfterExit=true
ExecStart=setupHttpdGitLocal.sh
configuration/environments_scripts/repo/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service
+19
View File
@@ -0,0 +1,19 @@
[Unit]
Description=Force logrotate upon shutdown and clean files in var/cache
Requires=-.mount network-online.target
After=-.mount network-online.target
Before=shutdown.target
RequiresMountsFor=/
SurviveFinalKillSignal=yes
[Install]
RequiredBy=graphical.target
WantedBy=shutdown.target
[Service]
Type=oneshot
RemainAfterExit=yes
KillMode=none
ExecStart=/bin/true
ExecStopPost=/bin/bash -c "logrotate --force -v /etc/logrotate.conf; /var/www/cgi-bin/reverseProxyHealthcheck.sh cleanup"
TimeoutStopSec=50
configuration/environments_scripts/repo/usr/local/bin/awsmfalogon.sh
Executable
+15
View File
@@ -0,0 +1,15 @@
# Use like this:
# . awsmfalogon.sh {mfaDeviceArn} {tokenCode}
# It will set the necessary environment variables that will allow the "aws" client to work
# with a session token
# Use with a bash alias definition like this, replacing {ARN-of-your-MFA-device} with the ARN of your MFA device:
# alias awsmfa='echo -n "Token: "; read aws_mfa_token; . awsmfalogon.sh "{ARN-of-your-MFA-device}" ${aws_mfa_token}'
# Then, you can invoke the alias "awsmfa" on your bash command line, and you will get prompted for an MFA token
# which, when entered, will add the necessary environment variables to your bash session that will allow your aws
# client to function with a valid session key.
mfaDeviceArn=$1
tokenCode=$2
jsonOutput="$(aws sts get-session-token --serial-number "${mfaDeviceArn}" --token-code ${tokenCode})"
export AWS_ACCESS_KEY_ID=$( echo "${jsonOutput}" | jq --raw-output '.Credentials.AccessKeyId' )
export AWS_SECRET_ACCESS_KEY=$( echo "${jsonOutput}" | jq --raw-output '.Credentials.SecretAccessKey' )
export AWS_SESSION_TOKEN=$( echo "${jsonOutput}" | jq --raw-output '.Credentials.SessionToken' )
configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh
+77 -19
View File
@@ -79,20 +79,16 @@ update_root_crontab() {
}
build_crontab_and_setup_files() {
#1: Environment type.
local ENVIRONMENT_TYPE="$1"
#2 git copy user
local GIT_COPY_USER="$2"
#3 relative path to git within the git user
local RELATIVE_PATH_TO_GIT="$3"
if [[ "$#" -lt 3 || "$#" -gt 5 ]]; then
echo "Number of arguments is invalid"
# There must be at least 1 and all args are passed to build-crontab-and-cp-files. See the documentation of this file for more info.
if [[ "$#" -lt 1 ]]; then
echo "Number of arguments is invalid. There must be at least 1 and all args are passed to build-crontab-and-cp-files."
else
TEMP_ENVIRONMENTS_SCRIPTS=$(mktemp -d /root/environments_scripts_XXX)
scp -o StrictHostKeyChecking=no -pr "wiki@sapsailing.com:~/gitwiki/configuration/environments_scripts/*" "${TEMP_ENVIRONMENTS_SCRIPTS}"
[[ "$?" -eq 0 ]] || scp -o StrictHostKeyChecking=no -pr "root@sapsailing.com:/home/wiki/gitwiki/configuration/environments_scripts/*" "${TEMP_ENVIRONMENTS_SCRIPTS}" # For initial setup as not all landscape managers have direct wiki access.
chown root:root "$TEMP_ENVIRONMENTS_SCRIPTS"
cd "${TEMP_ENVIRONMENTS_SCRIPTS}"
./build-crontab-and-cp-files "${ENVIRONMENT_TYPE}" "${GIT_COPY_USER}" "${RELATIVE_PATH_TO_GIT}"
./build-crontab-and-cp-files $@
cd ..
rm -rf "$TEMP_ENVIRONMENTS_SCRIPTS"
fi
@@ -100,21 +96,43 @@ build_crontab_and_setup_files() {
setup_keys() {
#1: Environment type.
pushd .
TEMP_KEY_DIR=$(mktemp -d /root/keysXXXXX)
REGION=$(TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" --silent -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"` \
&& curl -H "X-aws-ec2-metadata-token: $TOKEN" --silent http://169.254.169.254/latest/meta-data/placement/region)
scp -o StrictHostKeyChecking=no -pr root@sapsailing.com:/root/key_vault/"${1}"/* "${TEMP_KEY_DIR}"
scp -o StrictHostKeyChecking=no -pr root@sapsailing.com:/root/new_version_key_vault/"${1}"/* "${TEMP_KEY_DIR}"
cd "${TEMP_KEY_DIR}"
for user in $(ls); do
if id -u "$user"; then
for user in *; do
[[ -e "$user" ]] || continue
if id -u "$user" > /dev/null; then
user_home_dir=$(getent passwd $(id -u "$user") | cut -d: -f6) # getent searches for passwd based on user id, which the "id" command supplies.
# aws setup
if [[ -d "${user}/aws" ]]; then
mkdir --parents "${user_home_dir}/.aws"
chmod 755 "${user_home_dir}/.aws"
\cp -r --preserve --dereference "${user}"/aws/* "${user_home_dir}/.aws"
echo "[default]" >> "${user_home_dir}/.aws/config"
echo "region = ${REGION}" >> "${user_home_dir}"/.aws/config
chmod 755 "${user_home_dir}"/.aws
# Setup credentials
if [[ -d "${user}/aws/credentials" && ! -e "${user_home_dir}/.aws/credentials" ]]; then
> "${user_home_dir}"/.aws/credentials
for credentials in "${user}"/aws/credentials/*; do
[[ -f "$credentials" ]] || continue
cat "$credentials" >> "${user_home_dir}"/.aws/credentials
echo "" >> "${user_home_dir}"/.aws/credentials
done
fi
# Setup config
if [[ ! -e "${user_home_dir}/.aws/config" ]]; then
echo "[default]" >> "${user_home_dir}/.aws/config"
echo "region = ${REGION}" >> "${user_home_dir}"/.aws/config
echo "" >> "${user_home_dir}"/.aws/config
if [[ -d "${user}/aws/config" ]]; then
for config in "${user}"/aws/config/*; do
[[ -f "$config" ]] || continue
cat "$config" >> "${user_home_dir}"/.aws/config
echo "region = ${REGION}" >> "${user_home_dir}"/.aws/config
echo "" >> "${user_home_dir}"/.aws/config
done
fi
fi
chown -R ${user}:${user} "${user_home_dir}/.aws"
chmod 600 "${user_home_dir}"/.aws/*
fi
@@ -122,8 +140,12 @@ setup_keys() {
if [[ -d "${user}/ssh" ]]; then
mkdir --parents "${user_home_dir}/.ssh"
chmod 700 "${user_home_dir}/.ssh"
\cp --preserve --dereference $(find ${user}/ssh -maxdepth 1 -type f) "${user_home_dir}/.ssh"
for key in $(find ${user}/ssh/authorized_keys -type f); do
for key in "${user}"/ssh/*; do
[[ -f "$key" ]] || continue
\cp --preserve --dereference "$key" "$user_home_dir"/.ssh
done
for key in "${user}"/ssh/authorized_keys/*; do
[[ -f "$key" ]] || continue
cat "${key}" >> ${user_home_dir}/.ssh/authorized_keys
done
chown -R ${user}:${user} "${user_home_dir}/.ssh"
@@ -131,7 +153,7 @@ setup_keys() {
fi
fi
done
cd /
popd
rm -rf "${TEMP_KEY_DIR}"
}
@@ -166,8 +188,10 @@ setup_cloud_cfg_and_root_login() {
}
setup_fail2ban() {
pushd .
if [[ ! -f "/etc/systemd/system/fail2ban.service" ]]; then
yum install 2to3 -y
cd /usr/local/src
wget https://github.com/fail2ban/fail2ban/archive/refs/tags/1.0.2.tar.gz
tar -xvf 1.0.2.tar.gz
cd fail2ban-1.0.2/
@@ -189,8 +213,10 @@ setup_fail2ban() {
logpath = /var/log/fail2ban.log
maxretry = 5
EOF
touch /var/log/fail2ban.log
service fail2ban start
yum remove -y firewalld
popd
}
setup_mail_sending() {
@@ -244,3 +270,35 @@ identify_suitable_partition_for_ephemeral_volume() {
done 2>/dev/null | head -n 1 )
echo $EPHEMERAL_VOLUME_NAME
}
setup_goaccess() {
# Compatible with Amazon Linux 2023
pushd .
cd /usr/local/src
wget https://tar.goaccess.io/goaccess-1.9.1.tar.gz
tar -xzvf goaccess-1.9.1.tar.gz
cd goaccess-1.9.1/
yum install -y gcc-c++
yum install -y libmaxminddb-devel ncurses-devel
./configure --enable-utf8
make
make install
scp root@sapsailing.com:/etc/goaccess.conf /usr/local/etc/goaccess/goaccess.conf
# once we switch from amazon linux 1:
# scp root@sapsailing.com:/usr/local/etc/goaccess/goaccess.conf /usr/local/etc/goaccess/goaccess.conf
popd
}
setup_apachetop() {
# Compatible with Amazon Linux 2023
pushd .
yum install -y gcc-c++
yum install -y ncurses-devel readline-devel
cd /usr/local/src
wget https://github.com/tessus/apachetop/releases/download/0.23.2/apachetop-0.23.2.tar.gz
tar -xvzf apachetop-0.23.2.tar.gz
cd apachetop-0.23.2
./configure
make
make install
popd
}
configuration/environments_scripts/reverse_proxy/files/root/setupHttpdGitLocal.sh → configuration/environments_scripts/repo/usr/local/bin/setupHttpdGitLocal.sh
+4 -2
View File
@@ -1,5 +1,7 @@
#!/bin/bash
REMOTE=$1
BRANCH=$2
GIT_USERNAME=$3
STATUS_DEFINITION_FILE="internal-server-status.conf"
SELF_IP=$( ec2-metadata --local-ipv4 | grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+\>")
cd /etc/httpd
@@ -12,9 +14,9 @@ if ! git status; then
git init
git remote add origin "${REMOTE}"
GIT_SSH_COMMAND="ssh -A -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git fetch
git checkout disposable
git checkout "$BRANCH"
fi
echo "Use Status ${SELF_IP} internal-server-status" > /etc/httpd/conf.d/${STATUS_DEFINITION_FILE}
cd /etc/httpd
git config user.name "Disposable Reverse Proxy"
git config user.name "$GIT_USERNAME"
git config user.email "$(hostname)"
configuration/environments_scripts/repo/var/www/cgi-bin/reverseProxyHealthcheck.sh
Executable
+194
View File
@@ -0,0 +1,194 @@
#!/bin/bash
# Purpose: A smart health check for a Apache httpd reverse proxy server that takes three
# things into account:
# - /internal-server-status local technical health check
# - the availability zone (AZ) of the current live ARCHIVE application server
# - the health of all reverse proxies in a target group whose ARN is passed in the
# QUERY_STRINGS environments variable (assuming this script runs as a CGI-BIN script)
# as the "arn" query parameter, as in "...?arn=arn:aws:elasticloadbalancing:eu-west-1:017363970217:targetgroup/CentralWebServerHTTP-Dyn/32e57ea39e5fb165)
# The goal is to reduce cross-AZ traffic which produces extra cost and adds latency when
# a load balancer routes traffic to a reverse proxy in one AZ, and the application runs
# in a different AZ. So we'd like to have reverse proxies report an "unhealthy" status
# particularly if they run in an AZ different from the one in which the ARCHIVE application
# server runs, but only if there is at least one healthy reverse proxy in the ARCHIVE application
# server's AZ in the target group in question.
#
# To keep the health check swift, this script tries to keep the number of AWS API
# requests low. The first quick check is that for the local technical health, using the
# "/internal-server-status" endpoint. If that fails, this script will return an "unhealthy"
# status 500 and exit with code 1.
#
# When "/internal-server-status" reported a healthy status, the CIDRs of the VPC's subnets representing
# the AZs are cached persistently in a file once, using
#
# aws ec2 describe-subnets | jq -r '.Subnets[].CidrBlock'
#
# This produces, e.g.,
#
# 172.31.16.0/20
# 172.31.32.0/20
# 172.31.0.0/20
#
# These CIDRs are not assumed to change during the life-cycle of this instance, so this needs
# to happen only one time. Using
#
# nmap -sL -n <net> | head -n -1 | tail -n +2 | grep -q <IP>"
#
# we can check quickly whether an <IP> address is within a <net> CIDR.
#
# The ARCHIVE configuration with its production and failover instances can
# be determined from the /etc/httpd/conf.d/000-macros.conf file, telling the
# internal IP address of the ARCHIVE server currently used. From this, the AZ CIDR
# can be determined using the "nmap" approach described above.
#
# The same can be done for this instance's internal IP address.
#
# If the local instance is in the same AZ as the current live ARCHIVE server,
# we'll report "healthy" because our traffic to the ARCHIVE will not be cross-AZ.
#
# Otherwise (different AZ than live ARCHIVE server), we need to find out if at
# least one reverse proxy that is in the live ARCHIVE server's AZ is healthy.
# To determine the other targets in the target group specified through the "arn="
# query parameter in the QUERY_STRING variable, an
#
# aws elbv2 describe-target-health
#
# call is made, and the instance IDs returned are mapped to their internal IP addresses
# using an "aws ec2 describe-instances" call. This is the list of all reverse proxies
# registered with the target group at that time. Since this list is not considered to
# change very often, and because the negative effects of an outdated list are mild
# (in the worst case causing an instance in the wrong AZ to report "healthy", causing
# some temporary cross-AZ traffic), we cache the results and update this list only every
# few minutes and not upon every health check.
#
# From the list of targets we focus on those that are in the same AZ as the live ARCHIVE
# and check their health using the "/internal-server-status" endpoint. As soon as one
# healthy reverse proxy in the same AZ as the live ARCHIVE is found, this script will
# return an unhealthy 503 status and exit with code 2. If no healthy target in the live
# ARCHIVE's AZ is found, healthy (200) is reported and the script exits with code 0.
#
# (Keep in mind that with a load balancer that has cross-AZ balancing activated,
# requests may come in to a load balancer node in one AZ, and the only health reverse
# proxy lives in a different AZ; while this kind of cross-AZ traffic also will add
# latency, it doesn't add cost as it is considered "intra-loadbalancer traffic.")
#
# The user it is run by must have aws credentials that don't need mfa. Install to /usr/share/httpd/.aws.
#
# Optional parameter: "cleanup"; this will clean the cache and exit
#
# Exit status:
# 0 means all necessary checks could be performed and we're healthy (status 200)
# 1 means we were technically not healthy because the internal-server-status check failed (500)
# 2 means all checks could be performed, but we're not healthy (503) because not in the right AZ
# and there is at least one healthy target in the correct AZ
# 3 means there was a problem determining our health; we may still report status 200 (healthy)
outputMessage() {
# parameter 1: the message to display on the site
echo "Content-type: text/html"
echo ""
echo $1
}
status() {
# parameter 1: status code and messages
echo "Status: $1"
}
randomise() {
# $1: A number to slightly randomise
echo $(($RANDOM % 5 + $1 ))
}
getAzCidr() {
# $1: path to a file containing one CIDR per line
# $2: an IP address to obtain the subnet CIDR for
for cidr in $( cat "${1}" ); do
if nmap -sL -n ${cidr} | head -n -1 | tail -n +2 | grep -q ${2}; then
echo ${cidr}
fi
done
}
# The regex to extract the ip from a line ending in an ip.
IP_REGEX="[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+$"
MY_IP=$( ec2-metadata --local-ipv4 | grep -o "${IP_REGEX}")
# First check, using /internal-server-status, if we're technically healthy, and abort if not.
curl --silent --location --fail "http://${MY_IP}/internal-server-status" >/dev/null
if [[ "$?" -ne 0 ]]; then
status "500 Reverse proxy itself is unhealthy"
outputMessage "Unhealthy: Reverse proxy is unhealthy."
exit 1
fi
# Create cache location if it doesn't exist:
CACHE_LOCATION="/var/cache/httpd/healthcheck" # Folder storing all the cached info, which the Apache user can access.
mkdir --parents ${CACHE_LOCATION}
if [[ "$1" == "cleanup" ]]; then
rm -rf ${CACHE_LOCATION}
status "200"
outputMessage "cleanup complete"
exit 0
fi
# Ensure we have a cached copy of the AZs' CIDRs:
AZ_CIDRS_FILENAME="${CACHE_LOCATION}/az_cidrs"
if [ ! -f "${AZ_CIDRS_FILENAME}" ]; then
aws ec2 describe-subnets | jq -r '.Subnets[].CidrBlock' >"${AZ_CIDRS_FILENAME}"
fi
# The names of the variables in the macros file.
MACROS_PATH="/etc/httpd/conf.d/000-macros.conf"
ARCHIVE_IP_NAME="ARCHIVE_IP"
ARCHIVE_FAILOVER_IP_NAME="ARCHIVE_FAILOVER_IP"
PRODUCTION_ARCHIVE_NAME="PRODUCTION_ARCHIVE"
# Extracts which IP is in production.
PRODUCTION_ARCHIVE=$(sed -n -e "s/^Define ${PRODUCTION_ARCHIVE_NAME}\> \(.*\)$/\1/p" ${MACROS_PATH})
ARCHIVE_IP=$(grep -m 1 "^Define ${ARCHIVE_IP_NAME}\> .*" ${MACROS_PATH} | grep -o "${IP_REGEX}")
ARCHIVE_FAILOVER_IP=$(grep -m 1 "^Define ${ARCHIVE_FAILOVER_IP_NAME}\> .*" ${MACROS_PATH} | grep -o "${IP_REGEX}")
if [[ "$PRODUCTION_ARCHIVE" == "\${${ARCHIVE_IP_NAME}}" ]]; then
PRODUCTION_ARCHIVE_IP=${ARCHIVE_IP}
else
PRODUCTION_ARCHIVE_IP=${ARCHIVE_FAILOVER_IP}
fi
PRODUCTION_ARCHIVE_CIDR=$( getAzCidr "${AZ_CIDRS_FILENAME}" ${PRODUCTION_ARCHIVE_IP} )
# AZ of instance
MY_AZ_CIDR=$( getAzCidr "${AZ_CIDRS_FILENAME}" ${MY_IP} )
# Check if in the same AZ as the live ARCHIVE server and report healthy in that case
if [ "${PRODUCTION_ARCHIVE_CIDR}" = "${MY_AZ_CIDR}" ]; then
status "200"
outputMessage "Healthy: In the same AZ as the archive."
exit 0
else
# Otherwise, get cached (or updated, if cache expired) list of reverse proxies in target group and
# search for targets in same AZ as live ARCHIVE:
TARGET_GROUP_ARN="${QUERY_STRING//arn=/}"
TARGET_GROUP_NAME=$( basename $( dirname "${TARGET_GROUP_ARN}" ) )
LAST_TARGET_GROUP_IPS="${CACHE_LOCATION}/last_target_ips_${TARGET_GROUP_NAME}"
# Target group healthcheck timeout
TARGET_GROUP_HEALTHCHECK_TIMEOUT_SECONDS=300
current_time=$( date +%s )
if [[ ! -e "${LAST_TARGET_GROUP_IPS}" || "$(($current_time - $(stat --format '%Y' ${LAST_TARGET_GROUP_IPS}) ))" -gt "$(randomise $TARGET_GROUP_HEALTHCHECK_TIMEOUT_SECONDS)" ]]; then
# This branch runs if the cached timestamp, of the last target group healthcheck, doesn't exist, or if it exceeds TARGET_GROUP_HEALTHCHECK_TIMEOUT_SECONDS.
INSTANCE_IDS=$( aws elbv2 describe-target-health --target-group-arn ${TARGET_GROUP_ARN} | jq -r '.TargetHealthDescriptions[].Target.Id' )
INSTANCE_PRIVATE_IPS=$( aws ec2 describe-instances --instance-ids $( echo "${INSTANCE_IDS}" | tr '\n' ' ' ) | jq -r '.Reservations[].Instances[].NetworkInterfaces[].PrivateIpAddress' )
echo "${INSTANCE_PRIVATE_IPS}" >"${LAST_TARGET_GROUP_IPS}"
else
INSTANCE_PRIVATE_IPS=$( cat "${LAST_TARGET_GROUP_IPS}" )
fi
for INSTANCE_PRIVATE_IP in ${INSTANCE_PRIVATE_IPS}; do
INSTANCE_CIDR=$( getAzCidr "${AZ_CIDRS_FILENAME}" $INSTANCE_PRIVATE_IP )
if [ "${INSTANCE_CIDR}" = "${PRODUCTION_ARCHIVE_CIDR}" ]; then
# found a reverse proxy in the same AZ as the current live/production ARCHIVE; check its health:
curl --silent --location --fail "http://${INSTANCE_PRIVATE_IP}/internal-server-status" >/dev/null
if [[ "$?" = "0" ]]; then
# the reverse proxy in the same AZ as the current live/production ARCHIVE is healthy; then we're not:
status "503 Not in the same AZ"
outputMessage "Unhealthy: Not in the same AZ as the archive; healthy instance in the same AZ as the archive."
exit 2
fi
fi
done
# No healthy reverse proxy found in the same AZ as live/production ARCHIVE, so we'll report healthy
status "200"
outputMessage "Healthy: No healthy instance in the same AZ as the archive"
exit 0
fi
configuration/environments_scripts/reverse_proxy/files/etc/logrotate.d/httpd
-15
View File
@@ -1,15 +0,0 @@
/var/log/httpd/*log {
missingok
notifempty
sharedscripts
olddir /var/log/logrotate-target
# use date as a suffix of the rotated file
dateext
compress
delaycompress
postrotate
/bin/systemctl reload httpd.service > /dev/null 2>/dev/null || true
mv /var/log/logrotate-target/*.gz /var/log/old
endscript
}
configuration/environments_scripts/reverse_proxy/files/etc/logrotate.d/httpd
Symlink
+1
View File
@@ -0,0 +1 @@
../../../../repo/etc/logrotate.d/httpd
configuration/environments_scripts/reverse_proxy/files/etc/systemd/system/create-internal-status-file.service
-14
View File
@@ -1,14 +0,0 @@
[Unit]
Description=Creates httpd internal-status file.
Requires=-.mount network-online.target
After=-.mount network-online.target get-latest-httpd-conf.service
Before=httpd.service
# Staring before means no "systemctl start httpd" is needed.
[Install]
RequiredBy=multi-user.target
WantedBy=httpd.service
[Service]
Type=oneshot
RemainAfterExit=true
ExecStart=/root/setupHttpdGitLocal.sh
configuration/environments_scripts/reverse_proxy/files/etc/systemd/system/create-internal-status-file.service
Symlink
+1
View File
@@ -0,0 +1 @@
../../../../../repo/etc/systemd/system/create-internal-status-file.service
configuration/environments_scripts/reverse_proxy/files/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service
-19
View File
@@ -1,19 +0,0 @@
[Unit]
Description=Force logrotate upon shutdown and clean files in var/cache
Requires=-.mount network-online.target
After=-.mount network-online.target
Before=shutdown.target
RequiresMountsFor=/
SurviveFinalKillSignal=yes
[Install]
RequiredBy=graphical.target
WantedBy=shutdown.target
[Service]
Type=oneshot
RemainAfterExit=yes
KillMode=none
ExecStart=/bin/true
ExecStopPost=/bin/bash -c "logrotate --force -v /etc/logrotate.conf; /var/www/cgi-bin/reverseProxyHealthcheck.sh cleanup"
TimeoutStopSec=50
configuration/environments_scripts/reverse_proxy/files/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service
Symlink
+1
View File
@@ -0,0 +1 @@
../../../../../repo/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service
configuration/environments_scripts/reverse_proxy/files/usr/local/bin/awsmfalogon.sh
Symlink
+1
View File
@@ -0,0 +1 @@
../../../../../repo/usr/local/bin/awsmfalogon.sh
configuration/environments_scripts/reverse_proxy/files/usr/local/bin/setupHttpdGitLocal.sh
Symlink
+1
View File
@@ -0,0 +1 @@
../../../../../repo/usr/local/bin/setupHttpdGitLocal.sh
configuration/environments_scripts/reverse_proxy/files/var/www/cgi-bin/reverseProxyHealthcheck.sh
-194
View File
@@ -1,194 +0,0 @@
#!/bin/bash
# Purpose: A smart health check for a Apache httpd reverse proxy server that takes three
# things into account:
# - /internal-server-status local technical health check
# - the availability zone (AZ) of the current live ARCHIVE application server
# - the health of all reverse proxies in a target group whose ARN is passed in the
# QUERY_STRINGS environments variable (assuming this script runs as a CGI-BIN script)
# as the "arn" query parameter, as in "...?arn=arn:aws:elasticloadbalancing:eu-west-1:017363970217:targetgroup/CentralWebServerHTTP-Dyn/32e57ea39e5fb165)
# The goal is to reduce cross-AZ traffic which produces extra cost and adds latency when
# a load balancer routes traffic to a reverse proxy in one AZ, and the application runs
# in a different AZ. So we'd like to have reverse proxies report an "unhealthy" status
# particularly if they run in an AZ different from the one in which the ARCHIVE application
# server runs, but only if there is at least one healthy reverse proxy in the ARCHIVE application
# server's AZ in the target group in question.
#
# To keep the health check swift, this script tries to keep the number of AWS API
# requests low. The first quick check is that for the local technical health, using the
# "/internal-server-status" endpoint. If that fails, this script will return an "unhealthy"
# status 500 and exit with code 1.
#
# When "/internal-server-status" reported a healthy status, the CIDRs of the VPC's subnets representing
# the AZs are cached persistently in a file once, using
#
# aws ec2 describe-subnets | jq -r '.Subnets[].CidrBlock'
#
# This produces, e.g.,
#
# 172.31.16.0/20
# 172.31.32.0/20
# 172.31.0.0/20
#
# These CIDRs are not assumed to change during the life-cycle of this instance, so this needs
# to happen only one time. Using
#
# nmap -sL -n <net> | head -n -1 | tail -n +2 | grep -q <IP>"
#
# we can check quickly whether an <IP> address is within a <net> CIDR.
#
# The ARCHIVE configuration with its production and failover instances can
# be determined from the /etc/httpd/conf.d/000-macros.conf file, telling the
# internal IP address of the ARCHIVE server currently used. From this, the AZ CIDR
# can be determined using the "nmap" approach described above.
#
# The same can be done for this instance's internal IP address.
#
# If the local instance is in the same AZ as the current live ARCHIVE server,
# we'll report "healthy" because our traffic to the ARCHIVE will not be cross-AZ.
#
# Otherwise (different AZ than live ARCHIVE server), we need to find out if at
# least one reverse proxy that is in the live ARCHIVE server's AZ is healthy.
# To determine the other targets in the target group specified through the "arn="
# query parameter in the QUERY_STRING variable, an
#
# aws elbv2 describe-target-health
#
# call is made, and the instance IDs returned are mapped to their internal IP addresses
# using an "aws ec2 describe-instances" call. This is the list of all reverse proxies
# registered with the target group at that time. Since this list is not considered to
# change very often, and because the negative effects of an outdated list are mild
# (in the worst case causing an instance in the wrong AZ to report "healthy", causing
# some temporary cross-AZ traffic), we cache the results and update this list only every
# few minutes and not upon every health check.
#
# From the list of targets we focus on those that are in the same AZ as the live ARCHIVE
# and check their health using the "/internal-server-status" endpoint. As soon as one
# healthy reverse proxy in the same AZ as the live ARCHIVE is found, this script will
# return an unhealthy 503 status and exit with code 2. If no healthy target in the live
# ARCHIVE's AZ is found, healthy (200) is reported and the script exits with code 0.
#
# (Keep in mind that with a load balancer that has cross-AZ balancing activated,
# requests may come in to a load balancer node in one AZ, and the only health reverse
# proxy lives in a different AZ; while this kind of cross-AZ traffic also will add
# latency, it doesn't add cost as it is considered "intra-loadbalancer traffic.")
#
# The user it is run by must have aws credentials that don't need mfa. Install to /usr/share/httpd/.aws.
#
# Optional parameter: "cleanup"; this will clean the cache and exit
#
# Exit status:
# 0 means all necessary checks could be performed and we're healthy (status 200)
# 1 means we were technically not healthy because the internal-server-status check failed (500)
# 2 means all checks could be performed, but we're not healthy (503) because not in the right AZ
# and there is at least one healthy target in the correct AZ
# 3 means there was a problem determining our health; we may still report status 200 (healthy)
outputMessage() {
# parameter 1: the message to display on the site
echo "Content-type: text/html"
echo ""
echo $1
}
status() {
# parameter 1: status code and messages
echo "Status: $1"
}
randomise() {
# $1: A number to slightly randomise
echo $(($RANDOM % 5 + $1 ))
}
getAzCidr() {
# $1: path to a file containing one CIDR per line
# $2: an IP address to obtain the subnet CIDR for
for cidr in $( cat "${1}" ); do
if nmap -sL -n ${cidr} | head -n -1 | tail -n +2 | grep -q ${2}; then
echo ${cidr}
fi
done
}
# The regex to extract the ip from a line ending in an ip.
IP_REGEX="[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+$"
MY_IP=$( ec2-metadata --local-ipv4 | grep -o "${IP_REGEX}")
# First check, using /internal-server-status, if we're technically healthy, and abort if not.
curl --silent --location --fail "http://${MY_IP}/internal-server-status" >/dev/null
if [[ "$?" -ne 0 ]]; then
status "500 Reverse proxy itself is unhealthy"
outputMessage "Unhealthy: Reverse proxy is unhealthy."
exit 1
fi
# Create cache location if it doesn't exist:
CACHE_LOCATION="/var/cache/httpd/healthcheck" # Folder storing all the cached info, which the Apache user can access.
mkdir --parents ${CACHE_LOCATION}
if [[ "$1" == "cleanup" ]]; then
rm -rf ${CACHE_LOCATION}
status "200"
outputMessage "cleanup complete"
exit 0
fi
# Ensure we have a cached copy of the AZs' CIDRs:
AZ_CIDRS_FILENAME="${CACHE_LOCATION}/az_cidrs"
if [ ! -f "${AZ_CIDRS_FILENAME}" ]; then
aws ec2 describe-subnets | jq -r '.Subnets[].CidrBlock' >"${AZ_CIDRS_FILENAME}"
fi
# The names of the variables in the macros file.
MACROS_PATH="/etc/httpd/conf.d/000-macros.conf"
ARCHIVE_IP_NAME="ARCHIVE_IP"
ARCHIVE_FAILOVER_IP_NAME="ARCHIVE_FAILOVER_IP"
PRODUCTION_ARCHIVE_NAME="PRODUCTION_ARCHIVE"
# Extracts which IP is in production.
PRODUCTION_ARCHIVE=$(sed -n -e "s/^Define ${PRODUCTION_ARCHIVE_NAME}\> \(.*\)$/\1/p" ${MACROS_PATH})
ARCHIVE_IP=$(grep -m 1 "^Define ${ARCHIVE_IP_NAME}\> .*" ${MACROS_PATH} | grep -o "${IP_REGEX}")
ARCHIVE_FAILOVER_IP=$(grep -m 1 "^Define ${ARCHIVE_FAILOVER_IP_NAME}\> .*" ${MACROS_PATH} | grep -o "${IP_REGEX}")
if [[ "$PRODUCTION_ARCHIVE" == "\${${ARCHIVE_IP_NAME}}" ]]; then
PRODUCTION_ARCHIVE_IP=${ARCHIVE_IP}
else
PRODUCTION_ARCHIVE_IP=${ARCHIVE_FAILOVER_IP}
fi
PRODUCTION_ARCHIVE_CIDR=$( getAzCidr "${AZ_CIDRS_FILENAME}" ${PRODUCTION_ARCHIVE_IP} )
# AZ of instance
MY_AZ_CIDR=$( getAzCidr "${AZ_CIDRS_FILENAME}" ${MY_IP} )
# Check if in the same AZ as the live ARCHIVE server and report healthy in that case
if [ "${PRODUCTION_ARCHIVE_CIDR}" = "${MY_AZ_CIDR}" ]; then
status "200"
outputMessage "Healthy: In the same AZ as the archive."
exit 0
else
# Otherwise, get cached (or updated, if cache expired) list of reverse proxies in target group and
# search for targets in same AZ as live ARCHIVE:
TARGET_GROUP_ARN="${QUERY_STRING//arn=/}"
TARGET_GROUP_NAME=$( basename $( dirname "${TARGET_GROUP_ARN}" ) )
LAST_TARGET_GROUP_IPS="${CACHE_LOCATION}/last_target_ips_${TARGET_GROUP_NAME}"
# Target group healthcheck timeout
TARGET_GROUP_HEALTHCHECK_TIMEOUT_SECONDS=300
current_time=$( date +%s )
if [[ ! -e "${LAST_TARGET_GROUP_IPS}" || "$(($current_time - $(stat --format '%Y' ${LAST_TARGET_GROUP_IPS}) ))" -gt "$(randomise $TARGET_GROUP_HEALTHCHECK_TIMEOUT_SECONDS)" ]]; then
# This branch runs if the cached timestamp, of the last target group healthcheck, doesn't exist, or if it exceeds TARGET_GROUP_HEALTHCHECK_TIMEOUT_SECONDS.
INSTANCE_IDS=$( aws elbv2 describe-target-health --target-group-arn ${TARGET_GROUP_ARN} | jq -r '.TargetHealthDescriptions[].Target.Id' )
INSTANCE_PRIVATE_IPS=$( aws ec2 describe-instances --instance-ids $( echo "${INSTANCE_IDS}" | tr '\n' ' ' ) | jq -r '.Reservations[].Instances[].NetworkInterfaces[].PrivateIpAddress' )
echo "${INSTANCE_PRIVATE_IPS}" >"${LAST_TARGET_GROUP_IPS}"
else
INSTANCE_PRIVATE_IPS=$( cat "${LAST_TARGET_GROUP_IPS}" )
fi
for INSTANCE_PRIVATE_IP in ${INSTANCE_PRIVATE_IPS}; do
INSTANCE_CIDR=$( getAzCidr "${AZ_CIDRS_FILENAME}" $INSTANCE_PRIVATE_IP )
if [ "${INSTANCE_CIDR}" = "${PRODUCTION_ARCHIVE_CIDR}" ]; then
# found a reverse proxy in the same AZ as the current live/production ARCHIVE; check its health:
curl --silent --location --fail "http://${INSTANCE_PRIVATE_IP}/internal-server-status" >/dev/null
if [[ "$?" = "0" ]]; then
# the reverse proxy in the same AZ as the current live/production ARCHIVE is healthy; then we're not:
status "503 Not in the same AZ"
outputMessage "Unhealthy: Not in the same AZ as the archive; healthy instance in the same AZ as the archive."
exit 2
fi
fi
done
# No healthy reverse proxy found in the same AZ as live/production ARCHIVE, so we'll report healthy
status "200"
outputMessage "Healthy: No healthy instance in the same AZ as the archive"
exit 0
fi
configuration/environments_scripts/reverse_proxy/files/var/www/cgi-bin/reverseProxyHealthcheck.sh
Symlink
+1
View File
@@ -0,0 +1 @@
../../../../../repo/var/www/cgi-bin/reverseProxyHealthcheck.sh
configuration/environments_scripts/reverse_proxy/setup-disposable-reverse-proxy.sh
+4 -1
View File
@@ -48,6 +48,9 @@ systemctl enable httpd
echo "net.ipv4.ip_conntrac_max = 131072" >> /etc/sysctl.conf
# setup fail2ban
setup_fail2ban
# goaccess and apachetop
setup_goaccess
setup_apachetop
# mount nvme if available
mountnvmeswap
# setup logrotate.d/httpd
@@ -59,7 +62,7 @@ sed -i "s|/var/log/old|/var/log/old/REVERSE_PROXIES/${IP}|" $HTTP_LOGROTATE_ABS
sed -i 's/rotate 4/rotate 20 \n\nolddir \/var\/log\/logrotate-target/' /etc/logrotate.conf
sed -i "s/^#compress/compress/" /etc/logrotate.conf
# setup git
/root/setupHttpdGitLocal.sh "httpdConf@sapsailing.com:repo.git"
setupHttpdGitLocal.sh "httpdConf@sapsailing.com:repo.git" disposable "Disposable Reverse Proxy"
# Final enabling and starting of services.
systemctl start httpd
sudo systemctl start crond.service
configuration/tmuxManagementConsole.sh
-44
View File
@@ -1,44 +0,0 @@
#!/bin/bash
command -v tmux >/dev/null 2>&1 || { echo "I require tmux but it's not installed. Aborting." >&2; exit 1; }
sn=sailing
SERVERS_DIR=/home/trac/servers
TMUX_ACTIVE=`tmux has-session -t $sn 2>/dev/null`
if [ $? -eq 0 ]; then
echo "Session exists...not configuring a new one"
else
echo "Session does not exist...creating a new one with name $sn"
cd /home/trac/git
tmux new-session -s "$sn" -n "BUILD" -d
counter=1
for dir in dev test prod1 prod2; do
cd $SERVERS_DIR/$dir
tmux new-window -t "$sn:$counter" -n `basename $dir` "bash -c './start'; bash"
counter=$[counter + 1]
done
cd /home/trac/servers/prod1
tmux new-window -t "$sn:$counter" -n "UDP" "bash -c './udpmirror -v 2012 localhost 2010 localhost 2011 localhost 2013 localhost 2014'; bash"
cd /opt/mongodb/bin
tmux new-window -t "$sn:$[counter+1]" -n "GOAccess" "bash -c 'goaccess -f /var/log/httpd/access_log'; bash"
cd /opt/
tmux new-window -t "$sn:$[counter+2]" -n "ATop" "bash -c 'apachetop -f /var/log/httpd/access_log'; bash"
cd /home/trac/servers
tmux new-window -t "$sn:$[counter+3]" -n "Logs" "bash -c 'ls -lah'; bash"
cd /home/trac/servers/prod1
tmux new-window -t "$sn:$[counter+4]" -n "STListener" "bash -c './swisstiminglistener 3500 3501'; bash"
tmux select-window -t "$sn:0"
fi
if [[ "$1" != "unattended" ]]; then
tmux -2 attach-session -t "$sn"
fi
configuration/tmuxManagementConsole.sh
Symlink
+1
View File
@@ -0,0 +1 @@
environments_scripts/central_reverse_proxy/files/usr/local/bin/tmuxManagementConsole.sh
configuration/unique_ips_per_referrer
-55
View File
@@ -1,55 +0,0 @@
#!/bin/bash
CACHE=/var/log/old/cache/unique-ips-per-referrer
VISITED_FILES=$CACHE/visited
STATS=$CACHE/stats
MONTHS=$STATS/months
mkdir -p "$STATS"
# groups Apache / httpd log entries into "${referrer}.ips" files, where ${referrer} is the
# referrer URL identifying the "event"; the lines written to the .ips files hold the IP
# address of the requestor, the date (not the time) and the user agent string.
# Sorting for unique entries should give a count similar to what goaccess is using to
# determine "unique visitors."
# A sample line:
# 505Worlds2012.sapsailing.com 66.249.73.53 - - [12/Jan/2014:03:33:11 +0000] "GET /robots.txt HTTP/1.1" 404 238 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
echo Starting "$0" at `date` on file set "$*"
for i in $*; do
grep -q "^$i\$" $VISITED_FILES
if [ "$?" = "0" ]; then
echo "Already visited $i; ignoring (edit $VISITED_FILES to change this)."
else
echo "Analyzing log file $i"
if [ ${i: -3} == ".gz" ]; then
gzip -cd $i
else
cat $i
fi | recode ISO-8859-1..UTF-8 | sed -e 's/^\([^ ]*\) \([^ ]*\) \([^ ]*\) \([^ ]*\) \[\([^:]*\):\([^]]*\)\] \"[^"]*\" [^ ]* [^ ]* \"[^"]*\" \"\([^"]*\)\"/\1 \2 \5 \7/' | while read referrer hit; do
echo "$hit" >>${STATS}/${referrer}.ips
done
echo "$i" >>$VISITED_FILES
fi
done
for i in ${STATS}/*.ips; do
echo -n "`basename $i .ips` "
cat $i | sort -u | wc | awk '{ print $1; }'
# TODO consider producing an AWStats configuration file for each virtual host name and splitting all new logs into virtual host name-specific log files
done | tee $STATS/unique-ips-days-useragents-per-event
echo "Wrote total results to $STATS/unique-ips-days-useragents-per-event"
# Now filter and group by month
cat ${STATS}/*.ips | awk '{ print $2; }' | sed -e 's/^[0-9]*\///' | sort -u >$MONTHS
for month in `cat $MONTHS`; do
for i in ${STATS}/*.ips; do
echo -n "`basename $i .ips` "
cat $i | grep "^[^ ]* [0-9]*/$month .*" | sort -u | wc | awk '{ print $1; }'
# TODO consider producing an AWStats configuration file for each virtual host name and splitting all new logs into virtual host name-specific log files
done | tee $STATS/unique-ips-days-useragents-per-event-`echo $month | tr / -`
echo "Wrote per-month results to $STATS/unique-ips-days-useragents-per-event-`echo $month | tr / -`"
done
echo "Done at `date`."
configuration/unique_ips_per_referrer
Symlink
+1
View File
@@ -0,0 +1 @@
environments_scripts/central_reverse_proxy/files/usr/local/bin/unique_ips_per_referrer
configuration/unique_ips_per_referrer_fast
-16
View File
@@ -1,16 +0,0 @@
#!/bin/bash
JAVA_HOME=/opt/sapjvm_8
VARLOGOLD=/var/log/old
# Production:
CACHE=$VARLOGOLD/cache/unique-ips-per-referrer
# Test:
#CACHE=$VARLOGOLD/cache/unique-ips-per-referrer/test
JAR_FILE=$VARLOGOLD/com.sap.sse.jar
$JAVA_HOME/bin/java -Xmx8G -jar $JAR_FILE $CACHE $*
# Do the sorting of all .unique files now:
for i in `find $CACHE -name '*.unique'`; do
cat $i | awk '{ print $2 " " $1; }' | sort -rn >${i}.sorted
done
configuration/unique_ips_per_referrer_fast
Symlink
+1
View File
@@ -0,0 +1 @@
environments_scripts/central_reverse_proxy/files/usr/local/bin/unique_ips_per_referrer_fast
configuration/unique_ips_per_referrer_generate_month_results_only
-16
View File
@@ -1,16 +0,0 @@
#!/bin/bash
CACHE=/var/log/old/cache/unique-ips-per-referrer
VISITED_FILES=$CACHE/visited
STATS=$CACHE/stats
MONTHS=$STATS/months
for month in `cat $MONTHS`; do
for i in ${STATS}/*.ips; do
echo -n "`basename $i .ips` "
cat $i | grep "^[^ ]* [0-9]*/$month .*" | sort -u | wc | awk '{ print $1; }'
# TODO consider producing an AWStats configuration file for each virtual host name and splitting all new logs into virtual host name-specific log files
done | tee $STATS/unique-ips-days-useragents-per-event-`echo $month | tr / -`
echo "Wrote per-month results to $STATS/unique-ips-days-useragents-per-event-`echo $month | tr / -`"
done
echo "Done."
configuration/unique_ips_per_referrer_generate_month_results_only
Symlink
+1
View File
@@ -0,0 +1 @@
environments_scripts/central_reverse_proxy/files/usr/local/bin/unique_ips_per_referrer_generate_month_results_only
configuration/unique_ips_per_referrer_generate_results_only
-26
View File
@@ -1,26 +0,0 @@
#!/bin/bash
CACHE=/var/log/old/cache/unique-ips-per-referrer
VISITED_FILES=$CACHE/visited
STATS=$CACHE/stats
MONTHS=$STATS/months
for i in ${STATS}/*.ips; do
echo -n "`basename $i .ips` "
cat $i | sort -u | wc | awk '{ print $1; }'
# TODO consider producing an AWStats configuration file for each virtual host name and splitting all new logs into virtual host name-specific log files
done | tee $STATS/unique-ips-days-useragents-per-event
echo "Wrote total results to $STATS/unique-ips-days-useragents-per-event"
# Now filter and group by month
cat ${STATS}/*.ips | awk '{ print $2; }' | sed -e 's/^[0-9]*\///' | sort -u >$MONTHS
for month in `cat $MONTHS`; do
for i in ${STATS}/*.ips; do
echo -n "`basename $i .ips` "
cat $i | grep "^[^ ]* [0-9]*/$month .*" | sort -u | wc | awk '{ print $1; }'
# TODO consider producing an AWStats configuration file for each virtual host name and splitting all new logs into virtual host name-specific log files
done | tee $STATS/unique-ips-days-useragents-per-event-`echo $month | tr / -`
echo "Wrote per-month results to $STATS/unique-ips-days-useragents-per-event-$month"
done
echo "Done."
configuration/unique_ips_per_referrer_generate_results_only
Symlink
+1
View File
@@ -0,0 +1 @@
environments_scripts/central_reverse_proxy/files/usr/local/bin/unique_ips_per_referrer_generate_results_only
java/target/swisstiminglistener
-8
View File
@@ -1,8 +0,0 @@
#!/bin/bash
MONGO_VERSION=2.6.2
cd plugins
jar xvf com.mongodb.driver_${MONGO_VERSION}.jar lib/
cd ..
classpath=`find plugins -name 'com.sap.sailing.domain.common_*.jar' | sort | tail -1`:`find plugins -name 'org.eclipse.osgi_*.jar' | sort | tail -1`:`find plugins -name 'com.sap.sailing.mongodb_*.jar' | sort | tail -1`:`find plugins -name 'com.sap.sailing.domain_*.jar' | sort | tail -1`:`find plugins -name 'com.sap.sailing.udpconnector_*.jar' | sort | tail -1`:`find plugins -name 'com.sap.sailing.domain.swisstimingadapter_*.jar' | sort | tail -1`:`find plugins -name 'com.sap.sailing.domain.swisstimingadapter.persistence_*.jar' | sort | tail -1`:plugins/lib/mongo-${MONGO_VERSION}.jar
echo Using classpath $classpath
java -cp "$classpath" -Dmongo.port=10200 -Djava.util.logging.config.file=swisstiminglistenerLog.properties com.sap.sailing.domain.swisstimingadapter.persistence.StoreAndForward $*