diff --git a/.github/workflows/create-docker-image.yml b/.github/workflows/create-docker-image.yml new file mode 100644 index 00000000000..a9336d16e86 --- /dev/null +++ b/.github/workflows/create-docker-image.yml @@ -0,0 +1,71 @@ +name: Create Docker Image +on: + workflow_run: + workflows: [release] + types: [completed] + branches: [main, docker-17] + workflow_dispatch: + inputs: + release: + description: Release + default: '' +jobs: + create-docker-image-job: + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.workflow_dispatch.ref || github.event.workflow_run.head_branch }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to ghcr.io to upload sailing-analytics image + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Copy a few files + shell: bash + run: | + cp "java/target/env.sh" docker/ + cp "java/target/start" docker/ + cp "java/target/configuration/JavaSE-11.profile" docker/ + - name: Determine release + shell: bash + run: | + echo "BRANCH=$( git branch --show-current ) + RELEASE_PREFIX=$( if [ "${{ github.event.inputs.release }}" = "" ]; then + echo "$( git branch --show-current )-" + else + echo "${{ github.event.inputs.release }}" + fi ) + BEARER_TOKEN=${{ secrets.GITHUB_TOKEN }}" >>${GITHUB_ENV} + - name: Download release + shell: bash + run: | + RELEASE_TAR_GZ_FILENAME=$( configuration/github-download-release-assets.sh ${{ secrets.GITHUB_TOKEN }} ${{ env.RELEASE_PREFIX }} ) + RELEASE=$( echo ${RELEASE_TAR_GZ_FILENAME} | sed -e 's/\.tar\.gz$//' ) + if [ -n ${RELEASE_TAR_GZ_FILENAME} ]; then + mv ${RELEASE_TAR_GZ_FILENAME} docker/ + fi + echo "RELEASE=${RELEASE}" >>${GITHUB_ENV} + - name: Build & push Docker image + uses: docker/build-push-action@v5 + with: + build-args: RELEASE=${{ env.RELEASE }} + tags: ghcr.io/sap/sailing-analytics:${{ env.RELEASE }}${{ env.BRANCH == 'main' && github.event.inputs.release == '' && ',ghcr.io/sap/sailing-analytics:latest' || env.BRANCH == 'docker-17' && github.event.inputs.release == '' && ',ghcr.io/sap/sailing-analytics:latest-17' || '' }} + annotations: | + maintainer=axel.uhl@sap.com + index:org.opencontainers.image.title=Sailing Analytics + index:org.opencontainers.image.description=The Sailing Analytics Web Application + file: ${{ env.BRANCH == 'docker-17' && 'docker/Dockerfile_sapsailing_on_sapmachine17' || 'docker/Dockerfile' }} + context: docker/ + platforms: linux/amd64,linux/arm64 + push: true + - name: Clean up copied files + shell: bash + run: rm docker/start docker/env.sh docker/JavaSE-11.profile docker/${{ env.RELEASE }}.tar.gz release-notes.txt diff --git a/.github/workflows/merge-main-into-docker-17.yml b/.github/workflows/merge-main-into-docker-17.yml new file mode 100644 index 00000000000..4c69da3bf00 --- /dev/null +++ b/.github/workflows/merge-main-into-docker-17.yml @@ -0,0 +1,27 @@ +name: Merge main branch into docker-17 after successful build +on: + workflow_run: + workflows: [release] + types: [completed] + branches: [main] + workflow_dispatch: {} +jobs: + merge-main-into-docker-17: + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: docker-17 + fetch-depth: 0 # fetch the whole thing to make sure the histories merge + - name: Merge main into docker-17 + uses: devmasx/merge-branch@master + env: + GH_TOKEN: ${{ secrets.REPO_TOKEN_FOR_MERGE_AND_PUSH }} + with: + type: now + from_branch: main + target_branch: docker-17 + message: Auto-merging main into docker-17 after successful release build + github_token: ${{ secrets.REPO_TOKEN_FOR_MERGE_AND_PUSH }} diff --git a/.github/workflows/merge-sapsailing-com-master.yml b/.github/workflows/merge-sapsailing-com-master.yml new file mode 100644 index 00000000000..3cd5c2b4517 --- /dev/null +++ b/.github/workflows/merge-sapsailing-com-master.yml @@ -0,0 +1,31 @@ +# One of the reasons to have this workflow is that the Gollum-based Wiki +# at wiki.sapsailing.com is operating on a Git workspace at wiki@sapsailing.com:gitwiki +# which has been cloned from ssh://trac@sapsailing.com/home/trac/git. Its changes +# shall be synchronized to the Github repo at github.com/SAP/sailing-analytics +name: Merge master from sapsailing.com +on: + schedule: + - cron: "53 * * * *" # every hour, at :53 minutes + workflow_dispatch: {} +jobs: + merge-master-from-sapsailing-com: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 # fetch the whole thing to make sure the histories merge + - name: Create SSH key and fetch repo from sapsailing.com + shell: bash + run: | + mkdir -p ~/.ssh + chmod 700 ~/.ssh + echo "${{ secrets.SSH_PRIVATE_KEY_FOR_TRAC_SAPSAILING_COM }}" >~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + git remote add sapsailing ssh://trac@sapsailing.com/home/trac/git + git remote -v + GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git fetch sapsailing + git config --global user.email "github@sapsailing.com" + git config --global user.name "Github Auto-Merger" + git merge sapsailing/master + git push diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000000..c96eedb4448 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,149 @@ +name: release +on: + push: + paths-ignore: + - 'wiki/**' + - '.github/workflows/*' + - '**/*.md' + workflow_dispatch: + inputs: + skip_tests: + description: Skip tests (set to "true" to skip all tests) + default: 'false' + +jobs: + build: + runs-on: ubuntu-latest-64cpu + #runs-on: ubuntu-latest-4cpu + steps: + - name: Allocate swap space + shell: bash + run: | + sudo dd if=/dev/zero of=/mnt/large-swapfile bs=1G count=10 + sudo chmod 600 /mnt/large-swapfile + sudo mkswap /mnt/large-swapfile + sudo chown root:root /mnt/large-swapfile + sudo swapon /mnt/large-swapfile + - name: Collect Workflow Telemetry + uses: catchpoint/workflow-telemetry-action@v2 + with: + comment_on_pr: false + - name: Check out the repository to the runner + uses: actions/checkout@v4 + with: + fetch-depth: '500' + - name: Install JDK 8 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' # See 'Supported distributions' for available options + java-version: '8' + mvn-toolchain-id: 'JavaSE-1.8' + - name: Setup Android SDK + uses: android-actions/setup-android@v3 + with: + log-accepted-android-sdk-licenses: 'false' + cmdline-tools-version: 9123335 # This corresponds to human-friendly version number 8.0 + - name: Start MongoDB + uses: supercharge/mongodb-github-action@1.10.0 + with: + mongodb-version: 4.4 + - name: Start RabbitMQ + uses: namoshek/rabbitmq-github-action@v1 + with: + version: '3.8.9' + ports: '5672:5672' + - shell: bash + env: # Or as an environment variable + APP_PARAMETERS: "-Daws.region=eu-west-1 -Dgoogle.maps.authenticationparams=${{ secrets.GOOGLE_MAPS_AUTHENTICATIONPARAMS }} -Daws.s3.test.s3AccessId=${{ secrets.AWS_S3_TEST_S3ACCESSID }} -Daws.s3.test.s3AccessKey=${{ secrets.AWS_S3_TEST_S3ACCESSKEY }}" + JAVA8_HOME: ${{env.JAVA_HOME_8_X64}} + run: | + ./configuration/buildAndUpdateProduct.sh -x 60 ${{ github.event.inputs.skip_tests == 'true' && '-t' || '' }} build 2>&1 + - name: Upload build log + uses: actions/upload-artifact@v4 + if: always() + with: + name: build.log + path: build.log + retention-days: 90 + - name: Collect Test Reports + uses: dorny/test-reporter@v1 + if: always() + with: + name: Maven Tests + path: '**/TEST-*.xml' + max-annotations: '1' + list-suites: failed + list-tests: failed + reporter: java-junit + fail-on-error: true + - name: Build Release + if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/docker-17' || startsWith(github.ref, 'refs/heads/releases/') }} + shell: bash + run: | + ./configuration/buildAndUpdateProduct.sh -u -L ${{ github.event.inputs.skip_tests == 'true' && '-n untested' || '' }} release + - name: Determine release name + run: | + echo "SIMPLE_VERSION_INFO=$( basename dist/* )" >>${GITHUB_ENV} + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-results + path: | + **/surefire-reports/** + retention-days: 90 + - name: Upload distribution artifact + uses: actions/upload-artifact@v4 + if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/docker-17' || startsWith(github.ref, 'refs/heads/releases/') }} + with: + name: ${{ env.SIMPLE_VERSION_INFO }} + path: dist/**/* + retention-days: 90 + - name: Create Release + id: create_release + if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/docker-17' || startsWith(github.ref, 'refs/heads/releases/') }} + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ env.SIMPLE_VERSION_INFO }} + name: ${{ env.SIMPLE_VERSION_INFO }} + draft: false + prerelease: false + - name: Upload Release Asset tar.gz + id: upload-release-asset-tar-gz + if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/docker-17' || startsWith(github.ref, 'refs/heads/releases/') }} + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps + asset_path: ./dist/${{ env.SIMPLE_VERSION_INFO }}/${{ env.SIMPLE_VERSION_INFO }}.tar.gz + asset_name: ${{ env.SIMPLE_VERSION_INFO }}.tar.gz + asset_content_type: application/x-tar + - name: Upload Release Asset release_notes.txt + id: upload-release-asset-release-notes-txt + if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/docker-17' || startsWith(github.ref, 'refs/heads/releases/') }} + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps + asset_path: ./dist/${{ env.SIMPLE_VERSION_INFO }}/release-notes.txt + asset_name: release-notes.txt + asset_content_type: text/plain + - name: Trigger Hudson job + if: always() + shell: bash + run: | + if [ "${{ github.ref }}" = "refs/heads/main" ]; then + if [ "${{ github.event.inputs.skip_tests }}" = "true" ]; then + JOB=SAPSailingAnalytics-master-fasttrack-no-tests + else + JOB=SAPSailingAnalytics-master + fi + elif [[ "${{ github.ref }}" =~ refs/heads/releases/.* ]]; then + JOB=$( echo "${{ github.ref }}" | sed -e 's/^refs\/heads\/releases\///' ) + else + JOB=$( echo "${{ github.ref }}" | sed -e 's/^refs\/heads\///' ) + fi + echo "Identified Hudson job: ${JOB}" + curl https://${{ secrets.HUDSON_JOB_USERNAME }}:${{ secrets.HUDSON_JOB_PASSWORD}}@hudson.sapsailing.com/job/${JOB}/build?token=${{ secrets.HUDSON_JOB_TOKEN }} diff --git a/.github/workflows/reuse-tool-license-updates.yml b/.github/workflows/reuse-tool-license-updates.yml new file mode 100644 index 00000000000..e7b66618483 --- /dev/null +++ b/.github/workflows/reuse-tool-license-updates.yml @@ -0,0 +1,15 @@ +name: "Launch Metadata Creation Tool for REUSE" +on: + workflow_dispatch: ~ + +jobs: + create_metadata_proposal: + runs-on: ubuntu-latest + name: "Metadata Creation Tool" + steps: + - uses: SAP/metadata-creation-tool-for-reuse@main + with: + repository_url: "${{ github.server_url }}/${{ github.repository }}" + access_token: "${{ secrets.REUSE_ACCESS_TOKEN }}" + copyright_owner: "SAP SE or an SAP affiliate company" + upstream_contact: "Axel Uhl " diff --git a/.pipeline/README b/.pipeline/README new file mode 100644 index 00000000000..9baf656193e --- /dev/null +++ b/.pipeline/README @@ -0,0 +1,7 @@ +This folder contains an Azure pipeline definition that is used for building our +Android companion apps, in particular the Sailing Race Manager app and the +Buoy Pinger app. See also the mobile/ folder for their source code. The +gradle build that is used at the core of this build pipeline is controlled +by the ``build.gradle``and ``gradle.properties`` file in the root folder. +A local Android app build can be achieved by running the ``gradlew`` +wrapper script from the root folder. diff --git a/.reuse/README b/.reuse/README new file mode 100644 index 00000000000..0d5afba287d --- /dev/null +++ b/.reuse/README @@ -0,0 +1,3 @@ +This folder is used by the workflow defined in ``.github/workflows/reuse-tool-license-updates.yml`` +and produces machine-readable licensing information for the entire repository into the dep5 file +in this same folder. diff --git a/.reuse/dep5 b/.reuse/dep5 new file mode 100644 index 00000000000..2f2b5f1da52 --- /dev/null +++ b/.reuse/dep5 @@ -0,0 +1,60 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: sailing-analytics +Upstream-Contact: SAP Open Source Program Office ospo@sap.com +Source: https://github.com/SAP/sailing-analytics + +Files: java/com.google.gwt.servlet/* +Copyright: 2024 com.google.gwt.servlet contributors. +License: Apache-2.0 + +Files: java/net.sf.marineapi/doc/* +Copyright: 2024 doc contributors. +License: GPL-3.0-only + +Files: java/com.google.gwt.user/* +Copyright: 2024 com.google.gwt.user contributors. +License: Apache-2.0 + +Files: java/com.googlecode.java-diff-utils.test/* +Copyright: 2024 com.googlecode.java-diff-utils.test contributors. +License: Apache-2.0 + +Files: * +Copyright: 2024 SAP SE or an SAP affiliate company and sailing-analytics contributors. +License: Apache-2.0 + +Files: java/com.google.gwt.dev/* +Copyright: 2024 com.google.gwt.dev contributors. +License: Apache-2.0 + +Files: java/org.mp4parser.isoparser/* +Copyright: 2024 org.mp4parser.isoparser contributors. +License: Apache-2.0 + +Files: java/elemental2/* +Copyright: 2024 elemental2 contributors. +License: Apache-2.0 + +Files: java/org.json.simple/* +Copyright: 2024 org.json.simple contributors. +License: MIT + +Files: java/org.openqa.selenium.osgi/* +Copyright: 2024 org.openqa.selenium.osgi contributors. +License: Apache-2.0 + +Files: java/com.google.gwt.ajaxloader/* +Copyright: 2024 com.google.gwt.ajaxloader contributors. +License: Apache-2.0 + +Files: java/com.googlecode.java-diff-utils/* +Copyright: 2024 com.googlecode.java-diff-utils contributors. +License: Apache-2.0 + +Files: java/com.sap.sailing.gwt.ui/qrcode/* +Copyright: 2024 qrcode contributors. +License: MIT + +Files: ios/SAPTracker/src/xcode/SAPTracker/flags/* +Copyright: 2024 flags contributors. +License: MIT diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000000..bb9b3d7d89c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,171 @@ +# Contributing to the Sailing Analytics + +Thank you for your interest in the Sailing Analytics! We welcome contributions in all forms. + +Here are some of the ways you can contribute to Sailing Analytics + +Note: This is a living document. Like the Sailing Analytics, this document will be improved over time with the help of our community. Feedback welcome. + +---- + +## Reporting Issues + +Bug reports welcome! We are currently still using [Bugzilla](https://bugzilla.sapsailing.com) for tracking user stories and bugs. + +### Determining if an Issue Should be Created + +Analyzing reports is a bit of work, so please ask yourself these questions before creating an issue: + +* Does the bug really apply to Sailing Analytics + + There may be reasons the problem is occurring that have nothing to do with the Sailing Anlaytics code. Consider the following: + + * The problem may be caused by code that is not part of Sailing Analytics. + * The problem may be caused by data being sent to Sailing Analytics through REST API calls. + * The behavior may be different from what you expect, but may be working as designed. Or the feature may need improvement. In this case, it would be an enhancement request. Feature requests can also be submitted to [Bugzilla](https://bugzilla.sapsailing.com). + +* Is the problem reproducible in the latest release? + + * If you are not using the latest libraries or source code, please try to reproduce with the latest code first. + * Make sure the problem is consistently reproducible using repeatable steps. +* Has this bug already been reported? + * Please search the issue tracker for a similar bug before reporting the issue as a new bug. + +If the bug does indeed apply to Sailing Analytics code, please create an issue. + +### Providing the Right Information for the Issue + +* Summarize the issue well. The following are some guidelines: + + * Precisely state what you expected as compared to the actual behavior you're seeing. + * Include only those details that apply to the issue. + * Be concise, but include details important for helping us in understanding the problem and finding the root cause. + * State the version you are testing, which browser you are using, and on which device. + + If it is possible to indicate what you've seen in other browser and device combinations, that is even better! For example, does the issue occur in all browsers, or just one browser on one particular operating system? (For example, "only in Firefox on Windows 10.) + * If possible, include the last version where the bug was not present. + * If the bug is more visual, please attach a screenshot and mark up the problem. + * Generally, provide as much detail as necessary, but balance our need for information with how obvious the problem is. + +* Provide detailed, step-by-step instructions for reproducing the issue with an example, including, if possible: + + * A URL to your example + * Screenshots if it helps us understand better. + * Do not include more than one bug per issue created. This helps us to analyze bugs more easily. + +Please report issues in English. + +### About Reporting Security Issues + +If you find a security issue, we'd like to encourage you to tell us directly instead of creating a public issue. This way we can fix it before it can be exploited. + +* Researchers/non-Customers: please send the relevant information to support@sapsailing.com. + +### How We Process Issues + +New issues are reviewed regularly for validity and prioritization. Confirmed issues are given the "Approved" label, and the rest are either sent back to the reporter with a request for more details, or closed with an explanation. + +Validated issues are then moved into one of these buckets: + +* Priority issues will be assigned to one of our developers +* Issues that are less urgent will be left open as "contribution welcome" +* Certain issues may be moved to our internal issue tracking system if we don't think they belong to the Sailing Analytics. + +Issues are closed when the fix is merged to develop. The release that contains the fix will be noted in the comments and the release changelog. + +### How We Use Bugzilla Priorities + +Our Bugzilla set-up supports five priorities (1-5): + +* "1" is meant for show stoppers where, e.g., the website locks up, wrong data is being reported or the application crashes. Issues with thie priority need immediate action to unblock a live event using the solution. +* "2" is used for things that definitely need fixing but that are not necessarily holding up general operations at a live event. +* "3" identifies nice to have things we would like to see fixed at some point, but there is no specific urgency. Topics with this priority may be good candidates to start working on the project. +* "4" and "5" are what you use to make sure an idea is recorded and not forgotten, but other than when searching for a neat onboarding task for someone who would like to join the project, tasks with these priorities are hardly ever looked at. + +### Issue Reporting Disclaimer + +Feedback, especially bug reports, are always welcome. However, our capacity as a team is limited -- we cannot answer specific project or consultation requests, nor can we invest time in fleshing out what might be a bug. Because of this, we reserve the right to close or not process issue reports that do not contain enough information. We also do not guarantee that every well-documented issue will be fixed. + +That being said, we will try our very best to ensure the Sailing Analytics codebase is of high quality. + +---- + +## Contributing Code + +We welcome contributions to the Sailing Analytics codebase. Before you start your first contribution, here are some things you should know: + +1. You must be aware of the Apache License (which describes contributions), and you must agree to the [Contributors License Agreement](LICENSE.md). This is common practice for most open source projects. + + * For company contributors, special rules apply. See the respective section below for details. + +1. Contributions must be compliant with the project code style, quality, and standards. We also follow them :-) + + The `Contribution Content Guidelines` section below gives more details on the coding guidelines. + +1. Not all contributions will be accepted. + * The code you are submitting must fit the overall vision and direction of the Sailing Analytics and really improve it. Bug fixes are simple cases, for example, but new features may work better as third-party extensions. + * Major feature implementations should be discussed on the issue tracker. + +### Contributor License Agreement + +When you contribute anything to the Sailing Analytics (code, documentation, analysis, anything), be aware that your contribution is covered by the same [Apache 2.0 License](http://www.apache.org/licenses/LICENSE-2.0) that is applied to the Sailing Analytics itself. + +This applies to all contributors, including those contributing on behalf of a company. + +### Developer Certificate of Origin (DCO) + +Due to legal reasons, contributors will be asked to accept a DCO when they create the first pull request to this project. This happens in an automated fashion during the submission process. SAP uses [the standard DCO text of the Linux Foundation](https://developercertificate.org/). + +#### Company Contributors + +If employees of a company contribute code, in **addition** to the individual agreement above, there needs to be one company agreement submitted. This is mainly for the protection of the contributing employees. + +A company representative authorized to do so must fill out the following [Corporate Contributor License Agreement](/docs/SAP%20Corporate%20Contributor%20License%20Agreement.pdf) form. The form contains a list of employees who are authorized to contribute on behalf of your company. When this list changes, please let us know. + +Submit the form to us through one of the following methods: + +* Email to [opensource@sap.com](mailto:opensource@sap.com) and [support@sapsailing.com](mailto:support@sapsailing.com) +* Fax to +49 6227 78-45813 +* Mail to: + + Industry Standards & Open Source Team + Dietmar-Hopp-Allee 16 + 69190 Walldorf, Germany + +### Contribution Content Guidelines + + A contribution will be considered for inclusion in the Sailing Analytics if it meets the following criteria: + +* The contribution fits the overall vision and direction of the Sailing Analytics +* The contribution truly improves the solution +* The contribution follows the applicable guidelines and standards. + + The "guidelines and standards" requirement could fill entire books and still lack a 100% clear definition, but rest assured that you will receive feedback if something is not right. That being said, please consult the [Onboarding Document](https://wiki.sapsailing.com/wiki/howto/onboarding). + +### Contribution Process + + 1. Make sure the change would be welcome, as described above. + + 1. Create a fork of the Sailing Analytics sources. + + 1. Work on the change in your fork (either on the `master` branch or on a feature branch, typically named after the Bugzilla issue, such as ``bug1234``). + + 1. Commit and push your changes. + + 1. Give reasonable commit messages; a short first line, and if you'd like to write additional, more verbose comments, finish the first line with a semicolon, add an empty line, then add more comments. + + 1. If your change fixes an issue reported in Bugzilla, add the following line to the commit message: + + ```bug{issueNumber}: ...``` + + 1. Create a pull request so that we can review your change. + 1. Wait for our code review and approval, possibly enhancing your change on request. + + Note: This may take time, depending on the required effort for reviewing, testing, and clarification. Sailing Analytics developers are also working their regular duties. + +1. After the change has been approved, we will inform you in a comment. + +1. Due to internal SAP processes, your pull request cannot be merged directly into the branch. It will be merged internally, and will also immediately appear in the public repository. +1. We will close the pull request. At that point, you can delete your branch. + +We look forward to hearing from you! diff --git a/Gemfile.lock b/Gemfile.lock index 5c242f80487..67a87ee1234 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,9 @@ GEM remote: https://rubygems.org/ specs: - CFPropertyList (3.0.6) + CFPropertyList (3.0.7) + base64 + nkf rexml addressable (2.8.5) public_suffix (>= 2.0.2, < 6.0) @@ -24,6 +26,7 @@ GEM aws-sigv4 (1.6.0) aws-eventstream (~> 1, >= 1.0.2) babosa (1.0.4) + base64 (0.2.0) claide (1.1.0) colored (1.2) colored2 (3.1.2) @@ -161,6 +164,7 @@ GEM multipart-post (2.3.0) nanaimo (0.3.0) naturally (2.2.1) + nkf (0.2.0) optparse (0.1.1) os (1.1.4) plist (3.7.0) @@ -171,7 +175,8 @@ GEM trailblazer-option (>= 0.1.1, < 0.2.0) uber (< 0.2.0) retriable (3.1.2) - rexml (3.2.6) + rexml (3.2.8) + strscan (>= 3.0.9) rouge (2.0.7) ruby2_keywords (0.0.5) rubyzip (2.3.2) @@ -184,6 +189,7 @@ GEM simctl (1.6.10) CFPropertyList naturally + strscan (3.1.0) terminal-notifier (2.0.0) terminal-table (3.0.2) unicode-display_width (>= 1.1.1, < 3) @@ -199,7 +205,7 @@ GEM unicode-display_width (2.5.0) webrick (1.8.1) word_wrap (1.0.0) - xcodeproj (1.23.0) + xcodeproj (1.24.0) CFPropertyList (>= 2.3.3, < 4.0) atomos (~> 0.1.3) claide (>= 1.0.2, < 2.0) diff --git a/home.md b/Home.md similarity index 97% rename from home.md rename to Home.md index 035732c5339..3d2498e3971 100644 --- a/home.md +++ b/Home.md @@ -6,7 +6,7 @@ This is the Wik Like businesses, sailors need the latest information to make strategic decisions - but they need it even faster. One wrong tack, a false estimation of the current, or the slightest wind shift can cost the skipper the entire race. As premium sponsor of the Kieler Woche 2011, and co-sponsor of Sailing Team Germany (STG), SAP is showing how innovative IT solutions providing real time data analysis can give teams the competitive edge. -SAP is at the center of today’s technology revolution, developing innovations that not only help businesses run like never before, but also improve the lives of people everywhere. As market leader in enterprise application software, SAP SE (NYSE: SAP) helps companies of all sizes and industries run better. From back office to boardroom, warehouse to storefront, desktop to mobile device – SAP empowers people and organizations to work together more efficiently and use business insight more effectively to stay ahead of the competition. SAP applications and services enable more than 258,000 customers to operate profitably, adapt continuously, and grow sustainably. +SAP is at the center of today’s technology revolution, developing innovations that not only help businesses run like never before, but also improve the lives of people everywhere. As market leader in enterprise application software, SAP SE (NYSE: SAP) helps companies of all sizes and industries run better. From back office to boardroom, warehouse to storefront, desktop to mobile device – SAP empowers people and organizations to work together more efficiently and use business insight more effectively to stay ahead of the competition. SAP applications and services enable more than 400,000 customers to operate profitably, adapt continuously, and grow sustainably. # Table of Contents @@ -68,7 +68,7 @@ SAP is at the center of today’s technology revolution, developing innovations * [[Data consumption of mobile apps|wiki/info/mobile/data-consumption]] * [[Mobile Sailing Analytics|wiki/info/mobile/mobilesailinganalytics]] * [[Push Notifications|wiki/info/mobile/push-notifications]] -* [[NMEA|wiki/info/mobile/nmea]] +* [[NMEA|wiki/info/mobile/NMEA]] ### API @@ -117,7 +117,7 @@ SAP is at the center of today’s technology revolution, developing innovations * [[Java De(Serialization) and Circular Dependencies|wiki/howto/development/java-de-serialization-and-circular-dependencies]] * [[JMX Support|wiki/howto/development/jmx]] * [[Working with GWT Locally|wiki/howto/development/local-gwt]] -* [[Log File Analysis|wiki/howto/development/log-file-analysis]] +* [[Log File Analysis|wiki/howto/development/Log-File-Analysis]] * [[Old Log Compression|wiki/howto/development/Log-File-Compression]] * [[UI Tests with Selenium|wiki/howto/development/selenium-ui-tests]] * [[Profiling|wiki/howto/development/profiling]] @@ -132,7 +132,7 @@ SAP is at the center of today’s technology revolution, developing innovations * [[Operating Igtimi WindBots|wiki/howto/eventmanagers/windbot-operations]] * [[Linking Race Videos|wiki/howto/eventmanagers/linking-race-videos]] * [[Import official results|wiki/howto/eventmanagers/results-import]] -* [[Pairing lists|wiki/howto/eventmanagers/pairing-lists]] +* [[Pairing lists|wiki/howto/eventmanagers/Pairing-Lists]] * [[Manage media content|wiki/howto/eventmanagers/Manage-media-content]] * [[Import Event into Archive|wiki/howto/eventmanagers/import-into-archive]] @@ -167,7 +167,7 @@ SAP is at the center of today’s technology revolution, developing innovations * [[505 worlds Kiel 2014 event page|wiki/events/505-worlds-kiel-2014]] * [[Kieler Woche event page|wiki/events/kieler-woche-2015]] * [[Charleston Race Week 2016|wiki/events/Charleston-Race-Week-2016]] -* [[Sailing Leagues 2016|wiki/events/sailing-Leagues-2016]] +* [[Sailing Leagues 2016|wiki/events/Sailing-Leagues-2016]] * [[Media Content|wiki/events/Sailing-events-media-content]] ## Planning diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000000..ef37159e3b0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [2024] [SAP SE or an SAP affiliate company and sailing-analytics contributors] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSES/Apache-2.0.txt b/LICENSES/Apache-2.0.txt new file mode 100644 index 00000000000..137069b8238 --- /dev/null +++ b/LICENSES/Apache-2.0.txt @@ -0,0 +1,73 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + + You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/LICENSES/MIT.txt b/LICENSES/MIT.txt new file mode 100644 index 00000000000..2071b23b0e0 --- /dev/null +++ b/LICENSES/MIT.txt @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index c4680e2dd9d..1b670ac07c4 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,100 @@ # SAP Sailing Analytics -The SAP Sailing Analytics platform as seen on [sapsailing.com](https://sapsailing.com) +## About this Project + +The Sailing Analytics, formerly known as the "SAP Sailing Analytics," are a solution for portraying and analyzing sailing regattas, supporting training scenarios, and powering the vast archive at https://sapsailing.com. The solution consists of a cloud application with a web-based user interface, as well as three companion apps that integrate with the cloud application. This repository has the code for the cloud-based web application, and two of the three mobile apps (Buoy Pinger and Race Manager). The third companion app (Sail Insight) is found in [another repository](https://github.com/SailTracks/sailinsight). + +## Description + +This is the software running the SAP Sailing Analytics platform as seen on [sapsailing.com](https://sapsailing.com) Sailing provides the perfect platform for SAP to showcase solutions and help the sport run like never before. SAP’s involvement in the sport has transformed the sailing experience by providing tools, which: -Help sailors analyze performance and optimize strategy -Bring fans closer to the action -Provide the media with information and insights to deliver a greater informed commentary +- Help sailors analyze performance and optimize strategy +- Bring fans closer to the action +- Provide the media with information and insights to deliver a greater informed commentary + SAP has a longstanding involvement with sailing and has established a portfolio spanning across teams and regattas. -Start contributing: [https://wiki.sapsailing.com/wiki/howto/onboarding](https://wiki.sapsailing.com/wiki/howto/onboarding) +## Quick Start with Docker Compose + +If you have built or obtained the ``ghcr.io/sap/sailing-analytics:latest`` multi-platform image (currently available for linux/amd64 and linux/arm64 architectures), try this: +``` + cd docker + docker-compose up +``` +Based on the ``docker/docker-compose.yml`` definition you should end up with three running Docker containers: +- a MongoDB server, listening on default port 27017 +- a RabbitMQ server, listening on default port +- a Sailing Analytics server, listening for HTTP requests on port 8888 and for telnet connections to the OSGi console on port 14888 + +Try a request to [``http://127.0.0.1:8888/index.html``](http://127.0.0.1:8888/index.html) or [``http://127.0.0.1:8888/gwt/status``](http://127.0.0.1:8888/gwt/status) to see if things worked. + +To use Java 17, use the ``docker-compose-17.yml`` file instead: + +``` + cd docker + docker-compose -f docker-compose-17.yml up +``` + +## Requirements + +The software can be run on any Linux or Windows machine with ``bash`` installed; it has also been compiled successfully for the ARM platform and was deployed to a Raspberry Pi computer. As a database, MongoDB is required, tested with releases 4.4, 5.0, and 6.0. For use in a replicated scenario (scale-out, high availability), RabbitMQ is required. A simple Docker Compose set-up can be used to tie these three components together, e.g., for a quick local test and to familiarize yourself with the application, as Docker images are produced on a regular basis. + +Compute node and database sizing depends on several aspects of your workloads, such as whether live or replay data is to be served, how many different classes with separate leaderboard are racing concurrently, how many competitors are racing in each class, or how many concurrent viewers produce how many requests and which type (e.g., analytical, data mining, or watching a live race). + +To quantify this at least approximately, here are a few examples for typical node sizes that can handle such types of events reasonably well: + +- National sailing league event; six boats, 18 competitors, a single live leaderboard: 8GB of RAM and 4-8 CPUs +- Large multi-class event with 15 classes with their separate leaderboards, concurrently racing on six course areas: 16GB of RAM, 16 CPUs +- Archive of 30,000 races with a few thousand visitors per day with varying analytical and replay workloads: 64GB RAM, 2TB NVMe swap, 8 CPUs + +A single node typically handles up to 500-1000 concurrent viewers for live events. You will want to scale out accordingly, using the replication pattern offered by the solution which uses RabbitMQ for transaction log shipping. + +## Contributing + +To start contributing, read the onboarding document at the following URL: [https://wiki.sapsailing.com/wiki/howto/onboarding](https://wiki.sapsailing.com/wiki/howto/onboarding). Further documentation can be found in our Wiki at [https://wiki.sapsailing.com](https://wiki.sapsailing.com), serving the content from this repo's ``wiki/`` folder. + +The project welcomes contributions in the form of pull requests, for example, enhancements of the Data Mining functionality, including any sailing-specific metric or dimension you may think of and that you find is still missing so far; or additional features for the race viewer; or a map visualization that does not require a Google Map but uses Open Street Map / Open Layers; landscape automation; improved start sequence analytics; major UI improvements for the administrative layer ("AdminConsole"), etc. + +The issue tracker at [https://bugzilla.sapsailing.com](https://bugzilla.sapsailing.com) is currently used for any sort of issue and enhancement request tracking. Help to migrate this smoothly to Github Issues would be much appreciated, ideally keeping issue numbers stable due to many references to those Bugzilla bug numbers, be it in the source code, the Wiki, or the build infrastructure. + + +## Code of Conduct + +We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone. By participating in this project, you agree to abide by its [Code of Conduct](https://github.com/SAP/.github/blob/main/CODE_OF_CONDUCT.md) at all times. + +## Licensing + +Copyright 2024 SAP SE or an SAP affiliate company and sailing-analytics contributors. Please see our [LICENSE](LICENSE) for copyright and license information. Detailed information including third-party components and their licensing/copyright information is available [via the REUSE tool](https://api.reuse.software/info/github.com/SAP/sailing-analytics). + +See [here](https://www.sapsailing.com/gwt/Home.html#/imprint/:) for a list of components used by the project, as well as their licenses, also to be found in the file ``java/com.sap.sailing.gwt.ui/imprint.json``. ## Building and Running -To build, invoke +This assumes you have completed the onboarding (see again [here](https://wiki.sapsailing.com/wiki/howto/onboarding)) successfully. To build, then invoke + ``` configuration/buildAndUpdateProduct.sh build ``` -If the build was successful you can install the product locally by invoking +This will build the Android companion apps first, then the web application. If the build was successful you can install the product locally by invoking + ``` configuration/buildAndUpdateProduct.sh install [ -s ] ``` + The default server name is taken to be your current branch name, e.g., ``master``. The install goes to ``${HOME}/servers/{server-name}``. You will find a ``start`` script there which you can use to launch the product. +Or you create a release by running + +``` + configuration/buildAndUpdateProduct.sh -L -u -n release +``` +which produces a tarball under ``dist/{release-name}-{timestamp}/{release-name}-{timestamp}.tar.gz`` which can then be used for ``scp``-based downloads with the ``refreshInstance.sh`` script under ``java/target``. + +Run the ``buildAndUpdateProduct.sh`` without any arguments to see the sub-commands and options available. + + ## Downloading, Installing and Running an Official Release You need to have Java 8 installed. Get one from [here](https://tools.eu1.hana.ondemand.com/#cloud). Either ensure that this JVM's ``java`` executable in on the ``PATH`` or set ``JAVA_HOME`` appropriately. @@ -33,8 +105,17 @@ At [https://releases.sapsailing.com](https://releases.sapsailing.com) you find o cd sailinganalytics echo "MONGODB_URI=mongodb://localhost/winddb" | ${GIT_ROOT}/java/target/refreshInstance.sh auto-install-from-stdin ``` + This will download and install the latest release and configure it such that it will connect to a MongoDB server running locally (``localhost``) and listening on the default port ``27017``, using the database called ``winddb``. +In addition to the necessary ``MONGODB_URI`` variable you may need to inject a few secrets into your runtime environment: + +- ``MANAGE2SAIL_ACCESS_TOKEN`` access token for result and regatta structure import from the Manage2Sail regatta management system +- ``IGTIMI_CLIENT_ID`` / ``IGTIMI_CLIENT_SECRET`` credentials for ``igtimi.com`` in case you have one or more WindBot devices that you would like to integrate with +- ``GOOGLE_MAPS_AUTHENTICATION_PARAMS`` as in ``"key=..."`` or ``"client=..."``, required to display the Google Map in the race viewer. Obtain a Google Maps key from the Google Cloud Developer console, e.g., [here](https://console.cloud.google.com/apis/dashboard). + +In the ``configuration/mail.properties`` file make the necessary adjustments in case you would like to enable the application to send out e-mails, e.g., for user notifications or invitations. + Launch the server from the directory to which you just installed it: ``` ./start @@ -47,10 +128,13 @@ Connect to your server at ``http://localhost:8888`` and find its administration ## Docker -To build a docker image, try ``docker/makeImageForLatestRelease``. The upload to the default (private) Dockerhub repository will usually fail unless you are a collaborator for that repository, but you should see a local image tagged ``docker.sapsailing.com/sapsailing:...`` result from the build. To run that docker image, try something like +To build a docker image, try ``docker/makeImageForLatestRelease``. The upload to the default Github Package Registry (``ghcr.io``) will usually fail unless you are a collaborator for that repository, but you should see a local image tagged ``ghcr.io/sap/sailing-analytics:...`` resulting from the build. To run that docker image, try something like ``` docker run -d -e "MEMORY=4g" -e "MONGODB_URI=mongodb://my.mongohost.org?replicaSet=rs0&retryWrites=true" -P ``` + +As explained above for the non-Docker scenario, add any variable assignments for secrets you need to pass to the runtime by adding more ``-e`` arguments. + Do a "docker ps" to figure out the port exposing the web application: CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 79f6faf19b6a docker.sapsailing.com/sapsailing:latest "/home/sailing/serve…" 33 seconds ago Up 32 seconds 0.0.0.0:32782->6666/tcp, 0.0.0.0:32781->7091/tcp, 0.0.0.0:32780->8000/tcp, 0.0.0.0:32779->8888/tcp, 0.0.0.0:32778->14888/tcp modest_dhawan @@ -61,19 +145,31 @@ In this example, find your web application at http://localhost:32779 which is wh ``` to connect to the server's OSGi console. -## Docker Compose - -If you have built or obtained the ``docker.sapsailing.com/sapsailing:latest`` image, try this: +Alternatively, use an "environment" definition that sets useful defaults, e.g., like this: ``` - cd docker - docker-compose up + docker run -P -it --rm -e "SERVER_NAME=test77" -e "USE_ENVIRONMENT=live-master-server" -e "REPLICATE_MASTER_BEARER_TOKEN=BRxGpF0nr68Z4m/f13/MgiYhdRB3xoDCYd+rLc17rTs=" ghcr.io/sap/sailing-analytics:latest \ + bash -c "rm env.sh; echo \" +SERVER_NAME=test77 +USE_ENVIRONMENT=live-master-server +REPLICATE_MASTER_BEARER_TOKEN=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\" | ./refreshInstance.sh auto-install-from-stdin; ./start fg" ``` -Based on the ``docker/docker-compose.yml`` definition you should end up with three running Docker containers: -- a MongoDB server, listening on default port 27017 -- a RabbitMQ server, listening on default port -- a Sailing Analytics server, listening for HTTP requests on port 8888 and for telnet connections to the OSGi console on port 14888 -Try a request to [``http://127.0.0.1:8888/index.html``](http://127.0.0.1:8888/index.html) or [``http://127.0.0.1:8888/gwt/status``](http://127.0.0.1:8888/gwt/status) to see if things worked. +This will download and install the latest release and set up a primary/master instance, here as an example for server name "test77" and synchronize security data from ``security-service.sapsailing.com`` using the bearer token specified as ``REPLICA_MASTER_BEARER_TOKEN``. + +The default Docker image built by ``docker/makeImageForLatestRelease`` uses the ``eclipse-temurin:8-jdk`` base image. If you'd like to use the SAP JVM 8 which gives you great profiling and reversible on-the-fly debugging capabilities, you may build a Docker image for it, using ``docker/Dockerfile_sapjvm``. Using it to build an image will accept version 3.2 of the SAP tools developer license on your behalf. Build the image, e.g., like this: + +``` + cd ${GITROOT}/docker + docker build --build-arg SAPJVM_VERSION=8.1.099 -t sapjvm8:8.1.099 -f Dockerfile_sapjvm . +``` + +Then patch or copy ``docker/Dockerfile`` so that it has + +``` + FROM sapjvm8:8.1.099 +``` + +(of course with the version tag adjusted to what you used above when you built/tagged the image). This will give you an SAP JVM 8 under ``/opt/sapjvm_8`` in the container which in particular includes the useful ``jvmmon`` utility specific to the SAP JVM 8. ## Configuration Options, Environment Variables diff --git a/cfg/VERSION b/cfg/VERSION index 39811d8de7e..526f3d02dd2 100644 --- a/cfg/VERSION +++ b/cfg/VERSION @@ -1 +1 @@ -1.4.118 +1.4.119 diff --git a/configuration/RunFortifyBuild b/configuration/RunFortifyBuild index 5bc1d23e132..f86e796f550 100755 --- a/configuration/RunFortifyBuild +++ b/configuration/RunFortifyBuild @@ -1,3 +1,4 @@ #!/bin/bash +# Requires a checked out copy of the git workspace at /home/sailing/code sourceanalyzer -b com.sap.sailing.domain -Xmx4000m -debug -logfile FortifyScan.log -extdirs /home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins -cp "/home/sailing/code/java/com.google.gwt.osgi/lib/gwt-user.jar:/home/sailing/gwt-2.7.0/validation-api-1.0.0.GA.jar:/home/sailing/gwt-2.7.0/gwt-codeserver.jar:/home/sailing/gwt-2.7.0/validation-api-1.0.0.GA-sources.jar:/home/sailing/gwt-2.7.0/gwt-user.jar:/home/sailing/gwt-2.7.0/gwt-dev.jar:/home/sailing/code/java/com.google.gwt.dev/lib/gwt-dev.jar:/home/sailing/code/java/com.tractrac.clientmodule/lib/TracAPI.jar:/home/sailing/code/java/com.sap.sailing.dashboards.gwt/WEB-INF/lib/gwt-ajaxloader.jar:/home/sailing/code/java/com.sap.sailing.dashboards.gwt/WEB-INF/lib/gwt-maps-api-3.9.0-build-17.jar:/home/sailing/code/java/com.googlecode.mgwt/lib/mgwt-2.0.0.jar:/home/sailing/code/java/com.sap.sailing.gwt.ui/WEB-INF/lib/gwt-maps-api-3.9.0-build-17.jar:/home/sailing/code/java/com.sap.sailing.gwt.ui/WEB-INF/lib/gwt-ajaxloader.jar" -source 1.8 "/home/sailing/code/java/com.sap.sailing.ess40.resultimport" "/home/sailing/code/java/com.sap.sailing.manage2sail" "/home/sailing/code/java/com.sap.sailing.manage2sail.resultimport" "/home/sailing/code/java/com.tractrac.clientmodule" "/home/sailing/code/java/com.sap.sse.filestorage" "/home/sailing/code/java/com.sap.sailing.domain.swisstimingreplayadapter" "/home/sailing/code/java/com.sap.sse.mongodb" "/home/sailing/code/java/com.sap.sailing.udpconnector" "/home/sailing/code/java/com.sap.sailing.server.gateway" "/home/sailing/code/java/com.sap.sse" "/home/sailing/code/java/com.sap.sse.datamining" "/home/sailing/code/java/com.sap.sailing.domain.racelogtrackingadapter" "/home/sailing/code/java/com.sap.sailing.datamining.shared" "/home/sailing/code/java/com.sap.sailing.domain.igtimiadapter.persistence" "/home/sailing/code/java/com.sap.sailing.monitoring" "/home/sailing/code/java/com.sap.sailing.domain" "/home/sailing/code/java/com.sap.sse.datamining.annotations" "/home/sailing/code/java/com.sap.sailing.domain.swisstimingadapter.persistence" "/home/sailing/code/java/com.sap.sse.threadmanager" "/home/sailing/code/java/com.sap.sailing.xrr.structureimport" "/home/sailing/code/java/com.sap.sailing.freg.resultimport" "/home/sailing/code/java/com.sap.sailing.domain.igtimiadapter" "/home/sailing/code/java/com.sap.sailing.server.gateway.ess40" "/home/sailing/code/java/com.sap.sailing.xrr.schema" "/home/sailing/code/java/com.sap.sse.common" "/home/sailing/code/java/com.sap.sailing.expeditionconnector" "/home/sailing/code/java/com.sap.sailing.declination" "/home/sailing/code/java/com.sap.sailing.geocoding" "/home/sailing/code/java/com.sap.sailing.server.gateway.serialization" "/home/sailing/code/java/com.sap.sailing.kiworesultimport" "/home/sailing/code/java/com.sap.sailing.polars.datamining" "/home/sailing/code/java/com.sap.sailing.simulator" "/home/sailing/code/java/com.sap.sse.replication" "/home/sailing/code/java/com.sap.sailing.www" "/home/sailing/code/java/com.sap.sse.gwt.adminconsole" "/home/sailing/code/java/com.sap.sailing.dashboards.gwt" "/home/sailing/code/java/com.sap.sailing.news" "/home/sailing/code/java/com.sap.sailing.polars.datamining.shared" "/home/sailing/code/java/com.sap.sailing.barbados.resultimport" "/home/sailing/code/java/com.sap.sse.security.common" "/home/sailing/code/java/com.sap.sse.security.ui" "/home/sailing/code/java/com.sap.sse.shared.android" "/home/sailing/code/java/org.json.simple" "/home/sailing/code/java/com.sap.sailing.resultimport" "/home/sailing/code/java/com.sap.sse.security" "/home/sailing/code/java/com.sap.sailing.gwt.ui" "/home/sailing/code/java/com.sap.sailing.domain.tractracadapter" "/home/sailing/code/java/com.sap.sailing.server.trackfiles" "/home/sailing/code/java/com.googlecode.java-diff-utils" "/home/sailing/code/java/com.sap.sse.operationaltransformation" "/home/sailing/code/java/com.sap.sailing.domain.swisstimingadapter" "/home/sailing/code/java/com.sap.sse.gwt.theme" "/home/sailing/code/java/com.sap.sailing.domain.persistence" "/home/sailing/code/java/com.sap.sailing.sailwave.resultimport" "/home/sailing/code/java/com.sap.sailing.xmlexport" "/home/sailing/code/java/com.sap.sse.mail" "/home/sailing/code/java/com.sap.sailing.velum.resultimport" "/home/sailing/code/java/com.sap.sse.security.userstore.mongodb" "/home/sailing/code/java/com.sap.sailing.server.gateway.serialization.shared.android" "/home/sailing/code/java/com.sap.sse.datamining.shared" "/home/sailing/code/java/org.moxieapps.gwt.highcharts" "/home/sailing/code/java/com.sap.sailing.autoload" "/home/sailing/code/java/com.googlecode.mgwt" "/home/sailing/code/java/com.sap.sailing.datamining" "/home/sailing/code/java/com.sap.sailing.polars" "/home/sailing/code/java/com.sap.sailing.www.events" "/home/sailing/code/java/com.sap.sailing.domain.common" "/home/sailing/code/java/com.sap.sailing.domain.shared.android" "/home/sailing/code/java/com.sap.sailing.domain.deckmanadapter" "/home/sailing/code/java/com.sap.sailing.server" "/home/sailing/code/java/com.sap.sse.gwt" "/home/sailing/code/java/com.sap.sailing.xrr.resultimport" "/home/sailing/code/java/com.sap.sailing.domain.tractracadapter.persistence" #sourceanalyzer -b com.sap.sailing -Xmx10g -debug -logfile FortifyScan.log "-show-build-warnings" "-machine-output" "-cp" "/home/sailing/code/java/com.sap.sse.gwt.theme/WEB-INF/classes:/home/sailing/code/java/com.sap.sailing.dashboards.gwt/WEB-INF/classes:/home/sailing/code/java/com.sap.sailing.gwt.ui/WEB-INF/classes:/home/sailing/code/java/com.sap.sse.security.ui/WEB-INF/classes:/home/sailing/code/java/com.sap.sailing.www.events/WEB-INF/classes:/home/sailing/code/java/com.google.gwt.osgi/lib/gwt-user.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.eclipse.osgi_3.10.2.v20150203-1939.jar:/home/sailing/gwt-2.7.0/validation-api-1.0.0.GA.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.eclipse.jetty.osgi.npn.fragment_9.2.2.v20140723.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/javax.servlet_3.1.0.v20140303-1611.jar:/home/sailing/gwt-2.7.0/gwt-codeserver.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.servicemix.bundles.zxing_3.1.0.1.jar:/home/sailing/gwt-2.7.0/validation-api-1.0.0.GA-sources.jar:/home/sailing/gwt-2.7.0/gwt-user.jar:/home/sailing/gwt-2.7.0/gwt-dev.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.commons.math_2.1.0.v201105210652.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/com.fasterxml.jackson.core.jackson-databind_2.5.0.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.hamcrest.core_1.1.0.v20090501071000.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.junit4_4.8.2.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/com.fasterxml.jackson.core.jackson-annotations_2.5.0.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/com.rabbitmq.client_2.8.4.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/lz4-java_1.3.0.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.servicemix.bundles.aws-java-sdk_1.9.8.1.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.commons.logging_1.1.3.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/javax.validation_1.0.0.GA_v201205091237.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.eclipse.jetty.webapp_9.2.2.v20140723.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/com.fasterxml.jackson.core.jackson-core_2.5.0.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.eclipse.jetty.server_9.2.2.v20140723.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.mongodb.mongo-java-driver_2.13.0.RELEASE.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.httpcomponents.httpclient_4.2.6.v201311072007.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.commons.io_2.2.0.v201405211200.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/com.sun.mail.javax.mail_1.4.5.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.eclipse.jetty.io_9.2.2.v20140723.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/javax.ws.rs_1.1.1.v20130318-1750.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.jvnet.mimepull_1.9.4.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.shiro.ehcache_1.2.3.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.commons.lang_2.6.0.v201404270220.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.shiro.core_1.2.2.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.eclipse.jetty.websocket.servlet_9.2.2.v20140723.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.commons.beanutils_1.9.0.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/com.sun.jersey.contribs.jersey-multipart_1.17.0.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.eclipse.jetty.websocket.client_9.2.2.v20140723.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.objectweb.asm_5.0.1.v201404251740.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.jdom_1.1.1.v201101151400.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/routeconverter_2.12.2.201406032322.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/jackson-jaxrs_1.9.13.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.httpcomponents.httpcore_4.2.5.v201311072007.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.shiro.web_1.2.2.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.commons.fileupload_1.2.2.v20111214-1400.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.servicemix.bundles.ehcache_2.6.9.1.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.eclipse.jetty.util_9.2.2.v20140723.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.servicemix.bundles.scribe_1.3.3.1.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.eclipse.jetty.websocket.server_9.2.2.v20140723.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/slf4j.api_1.6.4.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/com.sun.jersey_1.17.0.v20130314-2020.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.eclipse.jetty.websocket.api_9.2.2.v20140723.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.eclipse.jetty.websocket.common_9.2.2.v20140723.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.hyperic.sigar_1.6.4.1.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/jcl.over.slf4j_1.6.4.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.poi_3.9.0.v201405241750.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.dom4j_1.6.1.v201312101416.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.xmlbeans_2.3.0.v201405241900.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.poi.ooxml.schemas_3.9.0.v201405241750.jar:/home/sailing/workspace/.metadata/.plugins/org.eclipse.pde.core/.bundle_pool/plugins/org.apache.poi.ooxml_3.9.0.v201405241905.jar:/home/sailing/code/java/com.google.gwt.dev/lib/gwt-dev.jar:/home/sailing/code/java/com.tractrac.clientmodule/lib/TracAPI.jar:/home/sailing/code/java/com.sap.sailing.dashboards.gwt/WEB-INF/lib/gwt-ajaxloader.jar:/home/sailing/code/java/com.sap.sailing.dashboards.gwt/WEB-INF/lib/gwt-maps-api-3.9.0-build-17.jar:/home/sailing/code/java/com.googlecode.mgwt/lib/mgwt-2.0.0.jar:/home/sailing/code/java/com.sap.sailing.gwt.ui/WEB-INF/lib/gwt-maps-api-3.9.0-build-17.jar:/home/sailing/code/java/com.sap.sailing.gwt.ui/WEB-INF/lib/gwt-ajaxloader.jar" "-source" "1.8" "/home/sailing/code/java/com.sap.sailing.ess40.resultimport" "/home/sailing/code/java/com.sap.sailing.manage2sail" "/home/sailing/code/java/com.sap.sailing.manage2sail.resultimport" "/home/sailing/code/java/com.tractrac.clientmodule" "/home/sailing/code/java/com.sap.sse.filestorage" "/home/sailing/code/java/com.sap.sailing.domain.swisstimingreplayadapter" "/home/sailing/code/java/com.sap.sse.mongodb" "/home/sailing/code/java/com.sap.sailing.udpconnector" "/home/sailing/code/java/com.sap.sailing.server.gateway" "/home/sailing/code/java/com.sap.sse" "/home/sailing/code/java/com.sap.sse.datamining" "/home/sailing/code/java/com.sap.sailing.domain.racelogtrackingadapter" "/home/sailing/code/java/com.sap.sailing.datamining.shared" "/home/sailing/code/java/com.sap.sailing.domain.igtimiadapter.persistence" "/home/sailing/code/java/com.sap.sailing.monitoring" "/home/sailing/code/java/com.sap.sailing.domain" "/home/sailing/code/java/com.sap.sse.datamining.annotations" "/home/sailing/code/java/com.sap.sailing.domain.swisstimingadapter.persistence" "/home/sailing/code/java/com.sap.sse.threadmanager" "/home/sailing/code/java/com.sap.sailing.xrr.structureimport" "/home/sailing/code/java/com.sap.sailing.freg.resultimport" "/home/sailing/code/java/com.sap.sailing.domain.igtimiadapter" "/home/sailing/code/java/com.sap.sailing.server.gateway.ess40" "/home/sailing/code/java/com.sap.sailing.xrr.schema" "/home/sailing/code/java/com.sap.sse.common" "/home/sailing/code/java/com.sap.sailing.expeditionconnector" "/home/sailing/code/java/com.sap.sailing.declination" "/home/sailing/code/java/com.sap.sailing.geocoding" "/home/sailing/code/java/com.sap.sailing.server.gateway.serialization" "/home/sailing/code/java/com.sap.sailing.kiworesultimport" "/home/sailing/code/java/com.sap.sailing.polars.datamining" "/home/sailing/code/java/com.sap.sailing.simulator" "/home/sailing/code/java/com.sap.sse.replication" "/home/sailing/code/java/com.sap.sailing.www" "/home/sailing/code/java/com.sap.sse.gwt.adminconsole" "/home/sailing/code/java/com.sap.sailing.dashboards.gwt" "/home/sailing/code/java/com.sap.sailing.news" "/home/sailing/code/java/com.sap.sailing.polars.datamining.shared" "/home/sailing/code/java/com.sap.sailing.barbados.resultimport" "/home/sailing/code/java/com.sap.sse.security.common" "/home/sailing/code/java/com.sap.sse.security.ui" "/home/sailing/code/java/com.sap.sse.shared.android" "/home/sailing/code/java/org.json.simple" "/home/sailing/code/java/com.sap.sailing.resultimport" "/home/sailing/code/java/com.sap.sse.security" "/home/sailing/code/java/com.sap.sailing.gwt.ui" "/home/sailing/code/java/com.sap.sailing.domain.tractracadapter" "/home/sailing/code/java/com.sap.sailing.server.trackfiles" "/home/sailing/code/java/com.googlecode.java-diff-utils" "/home/sailing/code/java/com.sap.sse.operationaltransformation" "/home/sailing/code/java/com.sap.sailing.domain.swisstimingadapter" "/home/sailing/code/java/com.sap.sse.gwt.theme" "/home/sailing/code/java/com.sap.sailing.domain.persistence" "/home/sailing/code/java/com.sap.sailing.sailwave.resultimport" "/home/sailing/code/java/com.sap.sailing.xmlexport" "/home/sailing/code/java/com.sap.sse.mail" "/home/sailing/code/java/com.sap.sailing.velum.resultimport" "/home/sailing/code/java/com.sap.sse.security.userstore.mongodb" "/home/sailing/code/java/com.sap.sailing.server.gateway.serialization.shared.android" "/home/sailing/code/java/com.sap.sse.datamining.shared" "/home/sailing/code/java/org.moxieapps.gwt.highcharts" "/home/sailing/code/java/com.sap.sailing.autoload" "/home/sailing/code/java/com.googlecode.mgwt" "/home/sailing/code/java/com.sap.sailing.datamining" "/home/sailing/code/java/com.sap.sailing.polars" "/home/sailing/code/java/com.sap.sailing.www.events" "/home/sailing/code/java/com.sap.sailing.domain.common" "/home/sailing/code/java/com.sap.sailing.domain.shared.android" "/home/sailing/code/java/com.sap.sailing.domain.deckmanadapter" "/home/sailing/code/java/com.sap.sailing.server" "/home/sailing/code/java/com.sap.sse.gwt" "/home/sailing/code/java/com.sap.sailing.xrr.resultimport" "/home/sailing/code/java/com.sap.sailing.domain.tractracadapter.persistence" diff --git a/configuration/RunFortifyScan b/configuration/RunFortifyScan index cc09b144cbe..021ab12aa34 100755 --- a/configuration/RunFortifyScan +++ b/configuration/RunFortifyScan @@ -1,3 +1,4 @@ #!/bin/bash +# Requires a checked out copy of the git workspace at /home/sailing/code #sourceanalyzer -b com.sap.sailing.domain -scan sourceanalyzer -b com.sap.sailing.domain -Xmx6000m -scan "-f" "/home/sailing/code/java/com.sap.sailing_scan.fpr" diff --git a/configuration/aws-automation/awsmfalogon.sh b/configuration/aws-automation/awsmfalogon.sh deleted file mode 100755 index 797ece6d697..00000000000 --- a/configuration/aws-automation/awsmfalogon.sh +++ /dev/null @@ -1,15 +0,0 @@ -# Use like this: -# . awsmfalogon.sh {mfaDeviceArn} {tokenCode} -# It will set the necessary environment variables that will allow the "aws" client to work -# with a session token -# Use with a bash alias definition like this, replacing {ARN-of-your-MFA-device} with the ARN of your MFA device: -# alias awsmfa='echo -n "Token: "; read aws_mfa_token; . awsmfalogon.sh "{ARN-of-your-MFA-device}" ${aws_mfa_token}' -# Then, you can invoke the alias "awsmfa" on your bash command line, and you will get prompted for an MFA token -# which, when entered, will add the necessary environment variables to your bash session that will allow your aws -# client to function with a valid session key. -mfaDeviceArn=$1 -tokenCode=$2 -jsonOutput="$(aws sts get-session-token --serial-number "${mfaDeviceArn}" --token-code ${tokenCode})" -export AWS_ACCESS_KEY_ID=$( echo "${jsonOutput}" | jq --raw-output '.Credentials.AccessKeyId' ) -export AWS_SECRET_ACCESS_KEY=$( echo "${jsonOutput}" | jq --raw-output '.Credentials.SecretAccessKey' ) -export AWS_SESSION_TOKEN=$( echo "${jsonOutput}" | jq --raw-output '.Credentials.SessionToken' ) diff --git a/configuration/aws-automation/awsmfalogon.sh b/configuration/aws-automation/awsmfalogon.sh new file mode 120000 index 00000000000..c7c0df2a274 --- /dev/null +++ b/configuration/aws-automation/awsmfalogon.sh @@ -0,0 +1 @@ +../environments_scripts/repo/usr/local/bin/awsmfalogon.sh \ No newline at end of file diff --git a/configuration/aws-automation/lib/scenario_shared_instance.sh b/configuration/aws-automation/lib/scenario_shared_instance.sh index 9649c518059..9ec6166c37c 100644 --- a/configuration/aws-automation/lib/scenario_shared_instance.sh +++ b/configuration/aws-automation/lib/scenario_shared_instance.sh @@ -65,7 +65,7 @@ function shared_instance_execute() { local readme_file="$servers_dir/README" local comment_out_line_in_env_with_pattern='JAVA_HOME=\/opt\/jdk1.8.0_20' local comment_in_line_in_env_with_pattern='sapjvm_gc' - local refreshInstance_file='/home/sailing/code/java/target/refreshInstance.sh' + local refreshInstance_file='/usr/local/bin/refreshInstance.sh' header "Sub instance setup" diff --git a/configuration/buildAndUpdateProduct.sh b/configuration/buildAndUpdateProduct.sh index a17b2236f06..a3564f18fb5 100755 --- a/configuration/buildAndUpdateProduct.sh +++ b/configuration/buildAndUpdateProduct.sh @@ -127,6 +127,7 @@ if [ $# -eq 0 ]; then echo "buildAndUpdateProduct [-b -u -g -t -a -r -o -c -p -v -m -n -l -x -j ] [build|install|all|hot-deploy|remote-deploy|local-deploy|release]" echo "" echo "-g Disable GWT compile, no gwt files will be generated, old ones will be preserved." + echo "-G Build forked GWT and gwt-maven-plugin locally instead of downloading Github release." echo "-b Build GWT permutation only for one browser and English language." echo "-t Disable tests" echo "-a Disable mobile projects (RaceCommittee App, e.g., in case no AndroidSDK is installed)" @@ -140,6 +141,7 @@ if [ $# -eq 0 ]; then echo " com.sap.sailing.monitoring. Only works if there is a fully built server available." echo " This parameter can also hold the name of the release if you are using the release command." echo "-l Telnet port the OSGi server is running. Optional but enables fully automatic hot-deploy." + echo "-L in conjunction with the release sub-command, build the release only locally to dist/ and do not upload" echo "-s Name of server you want to use as target for install, hot-deploy or remote-reploy. This overrides default behaviour." echo "-w Target for remote-deploy and release. Must comply with the following format: user@server." echo "-u Run without confirmation messages. Use with extreme care." @@ -182,11 +184,12 @@ echo SERVERS_HOME is $SERVERS_HOME echo BRANCH is $active_branch echo VERSION is $VERSION_INFO -options=':bgtocpaArvm:n:l:s:w:x:j:u' +options=':bgtocpaArvmLG:n:l:s:w:x:j:u' while getopts $options option do case $option in g) gwtcompile=0;; + G) BUILD_GWT_FORK=1;; t) testing=0;; b) onegwtpermutationonly=1;; o) offline=1;; @@ -199,6 +202,7 @@ do m) MAVEN_SETTINGS=$OPTARG;; n) OSGI_BUNDLE_NAME=$OPTARG;; l) OSGI_TELNET_PORT=$OPTARG;; + L) LOCAL_RELEASE_ONLY=1;; s) TARGET_SERVER_NAME=$OPTARG HAS_OVERWRITTEN_TARGET=1;; w) REMOTE_SERVER_LOGIN=$OPTARG;; @@ -358,12 +362,16 @@ INSTALL_FROM_RELEASE=$SIMPLE_VERSION_INFO echo "Packaged release $PROJECT_HOME/dist/$SIMPLE_VERSION_INFO.tar.gz! I've put an env.sh that matches the current branch to $PROJECT_HOME/dist/$SIMPLE_VERSION_INFO/env.sh!" - echo "Checking the remote connection..." - REMOTE_HOME=`ssh $REMOTE_SERVER_LOGIN 'echo $HOME/releases'` - echo "Now uploading release to $REMOTE_SERVER_LOGIN:$REMOTE_HOME. Can take quite a while!" + if [ "${LOCAL_RELEASE_ONLY}" != "1" ]; then + echo "Checking the remote connection..." + REMOTE_HOME=`ssh $REMOTE_SERVER_LOGIN 'echo $HOME/releases'` + echo "Now uploading release to $REMOTE_SERVER_LOGIN:$REMOTE_HOME. Can take quite a while!" - `which scp` -r $PROJECT_HOME/dist/$SIMPLE_VERSION_INFO $REMOTE_SERVER_LOGIN:$REMOTE_HOME/ - echo "Uploaded release to $REMOTE_HOME! Make sure to also put an updated env.sh if needed to the right place ($REMOTE_HOME/environment in most cases)" + `which scp` -r $PROJECT_HOME/dist/$SIMPLE_VERSION_INFO $REMOTE_SERVER_LOGIN:$REMOTE_HOME/ + echo "Uploaded release to $REMOTE_HOME! Make sure to also put an updated env.sh if needed to the right place ($REMOTE_HOME/environment in most cases)" + else + echo "Release available at $PROJECT_HOME/dist/$SIMPLE_VERSION_INFO/, tarball at $PROJECT_HOME/dist/$SIMPLE_VERSION_INFO/$SIMPLE_VERSION_INFO.tar.gz" + fi fi if [[ "$@" == "local-deploy" ]]; then @@ -659,10 +667,13 @@ if [[ "$@" == "build" ]] || [[ "$@" == "all" ]]; then echo "ANDROID_HOME=$ANDROID_HOME" PATH=$PATH:$ANDROID_HOME/tools/bin PATH=$PATH:$ANDROID_HOME/platform-tools - SDK_MANAGER="$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" + SDK_MANAGER="$ANDROID_HOME/cmdline-tools/8.0/bin/sdkmanager" if [ \! -x "$SDK_MANAGER" ]; then SDK_MANAGER="$ANDROID_HOME/tools/bin/sdkmanager.bat" fi + echo "SDK_MANAGER=${SDK_MANAGER}" + echo "cmdline-tools:" + ls -l "$ANDROID_HOME/cmdline-tools/" BUILD_TOOLS_VERSION=`grep "buildTools = " build.gradle | cut -d "\"" -f 2` echo "BUILD_TOOLS_VERSION=$BUILD_TOOLS_VERSION" @@ -748,8 +759,13 @@ if [[ "$@" == "build" ]] || [[ "$@" == "all" ]]; then extra="$extra -P with-not-android-relevant,!with-mobile" if [ $gwtcompile -eq 1 ]; then - echo "Building and installing forked GWT version..." - JAVA_HOME="${JAVA8_HOME}" `dirname $0`/install-gwt "${PROJECT_HOME}" + if [ "${BUILD_GWT_FORK}" = "1" ]; then + echo "Building and installing forked GWT version..." + JAVA_HOME="${JAVA8_HOME}" `dirname $0`/install-gwt "${PROJECT_HOME}" + else + echo "Downloading and installing forked GWT version..." + `dirname $0`/install-gwt-from-fork-releases https://github.com/SAP/gwt-forward-serialization-rpc https://github.com/SAP/gwt-maven-plugin-forward-serialization-rpc 2.11.0 . + fi fi echo "Using following command: mvn $extra -DargLine=\"$APP_PARAMETERS\" -fae -s $MAVEN_SETTINGS $clean install" echo "Maven version used: `mvn --version`" diff --git a/configuration/createHudsonJobForBug.sh b/configuration/createHudsonJobForBug.sh index e8c616275ad..3ad7d0b654c 100755 --- a/configuration/createHudsonJobForBug.sh +++ b/configuration/createHudsonJobForBug.sh @@ -1,30 +1,15 @@ #!/bin/bash - if [ $# -eq 0 ]; then - echo "$0 [-t -r ] " + echo "$0 " echo "" - echo "-t Disable tests" - echo "-r Build release" echo echo "Constructs a Hudson job for the given bugid" - echo "Example: $0 -r 4221" - echo "Builds a Hudson job for bug branch bug4221, linking to the Bugzilla bug and building a release" + echo "Example: $0 4221" + echo "Builds a Hudson job for bug branch bug4221, linking to the Bugzilla bug and copying a release" + echo "from Github to https://releases.sapsailing.com if the Github Actions Workflow built one" exit 2 fi -options='tr' -while getopts $options option -do - case $option in - t) SKIP_TESTS=1;; - r) BUILD_RELEASE=1;; - \?) echo "Invalid option" - exit 4;; - esac -done - -shift $((OPTIND-1)) - BUG_ID="$1" CONFIGFILE=$(mktemp mylocalconfigXXXX.xml) @@ -42,25 +27,47 @@ curl -s -X GET $COPY_TEMPLATE_CONFIG_URL -u "$USERNAME:$PASSWORD" -o "$CONFIGFIL # On macosx is gnu-sed needed if [[ "$OSTYPE" == *"$OS_FOR_GSED"* ]]; then echo "Using gsed" - gsed -i'' -e 's|..*|This is the build job for \<a href=\"'$BUGZILLA_BASE'/show_bug.cgi?id='$BUG_ID'\"\>Bug '$BUG_ID'\</a\>|' -e 's|true|false|' "$CONFIGFILE" + gsed -i'' -e 's|..*|This is the CI job for \<a href=\"'$BUGZILLA_BASE'/show_bug.cgi?id='$BUG_ID'\"\>Bug '$BUG_ID'\</a\>|' -e 's|true|false|' "$CONFIGFILE" else - sed -i -e 's|..*|This is the build job for \<a href=\"'$BUGZILLA_BASE'/show_bug.cgi?id='$BUG_ID'\"\>Bug '$BUG_ID'\</a\>|' -e 's|true|false|' "$CONFIGFILE" + sed -i -e 's|..*|This is the CI job for \<a href=\"'$BUGZILLA_BASE'/show_bug.cgi?id='$BUG_ID'\"\>Bug '$BUG_ID'\</a\>|' -e 's|true|false|' "$CONFIGFILE" fi -if [ "$SKIP_TESTS" = "1" ]; then - if [[ "$OSTYPE" == *"$OS_FOR_GSED"* ]]; then - gsed -i'' -e 's|\([^<]*\)build *|\1-t build|' "$CONFIGFILE" - else - sed -i -e 's|\([^<]*\)build *|\1-t build|' "$CONFIGFILE" - fi -fi -if [ "$BUILD_RELEASE" = "1" ]; then - if [[ "$OSTYPE" == *"$OS_FOR_GSED"* ]]; then - gsed -i'' -e 's|\([^<]*\)|\1 \&\& PATH=/usr/local/bin:${PATH} configuration/buildAndUpdateProduct.sh -n bug'$BUG_ID' -w trac@sapsailing.com -u release|' "$CONFIGFILE" - else - sed -i -e 's|\([^<]*\)|\1 \&\& PATH=/usr/local/bin:${PATH} configuration/buildAndUpdateProduct.sh -n bug'$BUG_ID' -w trac@sapsailing.com -u release|' "$CONFIGFILE" - fi +# On macosx is gnu-sed needed +if [[ "$OSTYPE" == *"$OS_FOR_GSED"* ]]; then + echo "Using gsed" + gsed -i'' -n -e ':loop +//b InCommand +p +$b +N +D +b loop +:InCommand +s/BRANCH/'bug${BUG_ID}'/g +/<\/command>/p +/<\/command>/b End +N +b InCommand +:End +' "$CONFIGFILE" +else + sed -i -n -e ':loop +//b InCommand +p +$b +N +D +b loop +:InCommand +s/BRANCH/'bug${BUG_ID}'/g +/<\/command>/p +/<\/command>/b End +N +b InCommand +:End +' "$CONFIGFILE" fi + if [[ "$OSTYPE" == *"$OS_FOR_GSED"* ]]; then gsed -i'' -e '/$/{ N diff --git a/configuration/crontab b/configuration/crontab deleted file mode 100644 index ae1bd42a6b8..00000000000 --- a/configuration/crontab +++ /dev/null @@ -1,3 +0,0 @@ -* * * * * export PATH=/bin:/usr/bin:/usr/local/bin; sleep $(( $RANDOM * 60 / 32768 )); update_authorized_keys_for_landscape_managers_if_changed $( cat /root/ssh-key-reader.token ) https://security-service.sapsailing.com /root 2>&1 >>/var/log/sailing.err -# NOTICE: Please try to reference the customised crontabs at $GIT_HOME/configuration/crontabs or use -# the build_crontab_and_setup_files command in imageupgrade_functions.sh. This file has been maintained for continuity, but is deprecated. \ No newline at end of file diff --git a/configuration/crontabs/README b/configuration/crontabs/README index b9467328242..d32ecb4c2de 100644 --- a/configuration/crontabs/README +++ b/configuration/crontabs/README @@ -1,7 +1,27 @@ This is the crontab repo and it contains one line crontabs for all the different environments. These files are concatenated by the build-crontab-and-cp-files script. Any time the crontab should contain -a user's home directory, instead write PATH_OF_HOME_DIR_TO_REPLACE; if the crontab should -contain the path to the git directory, instead write PATH_OF_GIT_HOME_DIR_TO_REPLACE. -These are replaced by the build-crontab-and-cp-filesscript. +a user's home directory, instead write PATH_OF_HOME_DIR_TO_REPLACE. +These are replaced by the build-crontab-and-cp-files script found under +configuration/environments_scripts. + +Further replacements in the crontab snippets can be controlled by the names used for the +files or symbolic links used in the users/ directories, as follows: + + crontab-snippet@{PARAM1}={VALUE1}@{PARAM2}={VALUE2}... + +will replace all occurrences of PARAM1 in the snippet by VALUE1, etc. The values can be +empty. If a value shall contain an @ sign, use @@ instead. For a / (slash) character +use an underscore _ character, and to produce a single underscore, double it (__), as +illustrated by the following example: + + crontab-snippet@EMAIL_ADDRESS=john.doe@@example.com@HOME_DIR=_home_john__doe_ + +It is good practice to name a reusable crontab snippet such that its parameters are +listed in its name, separated from the name and from each other with @-signs, as in + + crontab-snippet@USER@EMAIL_ADDRESS + +which would indicate that two placeholders exist in the snippet that a name for a symbolic +link referencing this snippet shall configure. Note: theses files are symbolically linked to, so beware of the ramifications of changes. \ No newline at end of file diff --git a/configuration/crontabs/crontab-download-new-archived-trac-trac-events b/configuration/crontabs/crontab-download-new-archived-trac-trac-events deleted file mode 100644 index d3d1c7e19a0..00000000000 --- a/configuration/crontabs/crontab-download-new-archived-trac-trac-events +++ /dev/null @@ -1 +0,0 @@ -15 12 * * * PATH_OF_GIT_HOME_DIR_TO_REPLACE/configuration/downloadNewArchivedTracTracEvents.sh PATH_OF_HOME_DIR_TO_REPLACE/static/TracTracTracks "PATH_OF_GIT_HOME_DIR_TO_REPLACE">PATH_OF_HOME_DIR_TO_REPLACE/downloadNewArchivedTracTracEvents.out 2>PATH_OF_HOME_DIR_TO_REPLACE/downloadNewArchivedTracTracEvents.err \ No newline at end of file diff --git a/configuration/crontabs/crontab-mongo-health-check b/configuration/crontabs/crontab-mongo-health-check index 05bc0722551..08c83b89c9c 100644 --- a/configuration/crontabs/crontab-mongo-health-check +++ b/configuration/crontabs/crontab-mongo-health-check @@ -1 +1 @@ -* * * * * PATH_OF_GIT_HOME_DIR_TO_REPLACE/configuration/notify-unhealthy-mongodb 2>PATH_OF_HOME_DIR_TO_REPLACE/notify-unhealthy-mongodb.err >PATH_OF_HOME_DIR_TO_REPLACE/notify-unhealthy-mongodb.out \ No newline at end of file +* * * * * export PATH=/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin; notify-unhealthy-mongodb 2>/var/log/notify-unhealthy-mongodb.err >/var/log/notify-unhealthy-mongodb.out diff --git a/configuration/crontabs/crontab-syncgit b/configuration/crontabs/crontab-syncgit deleted file mode 100644 index dd123304b91..00000000000 --- a/configuration/crontabs/crontab-syncgit +++ /dev/null @@ -1 +0,0 @@ -*/10 * * * * export PATH=/bin:/usr/bin:/usr/local/bin; syncgit PATH_OF_GIT_HOME_DIR_TO_REPLACE diff --git a/configuration/crontabs/crontab-update-authorized-keys b/configuration/crontabs/crontab-update-authorized-keys deleted file mode 100644 index b5a2ea7f7ac..00000000000 --- a/configuration/crontabs/crontab-update-authorized-keys +++ /dev/null @@ -1 +0,0 @@ -* * * * * export PATH=/bin:/usr/bin:/usr/local/bin; sleep $(( $RANDOM * 60 / 32768 )); update_authorized_keys_for_landscape_managers_if_changed $( cat PATH_OF_HOME_DIR_TO_REPLACE/ssh-key-reader.token ) https://security-service.sapsailing.com PATH_OF_HOME_DIR_TO_REPLACE # 2>&1 >>/var/log/sailing.err diff --git a/configuration/crontabs/crontab-update-authorized-keys-ec2-user b/configuration/crontabs/crontab-update-authorized-keys@USER_HOME_DIR similarity index 65% rename from configuration/crontabs/crontab-update-authorized-keys-ec2-user rename to configuration/crontabs/crontab-update-authorized-keys@USER_HOME_DIR index 72573ec8c47..6d444dd2b83 100644 --- a/configuration/crontabs/crontab-update-authorized-keys-ec2-user +++ b/configuration/crontabs/crontab-update-authorized-keys@USER_HOME_DIR @@ -1 +1 @@ -* * * * * export PATH=/bin:/usr/bin:/usr/local/bin; sleep $(( $RANDOM * 60 / 32768 )); update_authorized_keys_for_landscape_managers_if_changed $( cat PATH_OF_HOME_DIR_TO_REPLACE/ssh-key-reader.token ) https://security-service.sapsailing.com /home/ec2-user # 2>&1 >>/var/log/sailing.err +* * * * * export PATH=/bin:/usr/bin:/usr/local/bin; sleep $(( $RANDOM * 60 / 32768 )); update_authorized_keys_for_landscape_managers_if_changed $( cat PATH_OF_HOME_DIR_TO_REPLACE/ssh-key-reader.token ) https://security-service.sapsailing.com USER_HOME_DIR # 2>&1 >>/var/log/sailing.err diff --git a/configuration/crontabs/crontab-update-trac-trac-urls b/configuration/crontabs/crontab-update-trac-trac-urls index 2726ffd027d..2f1ef3557d6 100644 --- a/configuration/crontabs/crontab-update-trac-trac-urls +++ b/configuration/crontabs/crontab-update-trac-trac-urls @@ -1 +1 @@ -10 12 * * * PATH_OF_GIT_HOME_DIR_TO_REPLACE/configuration/update-tractrac-urls-to-archive.sh "PATH_OF_GIT_HOME_DIR_TO_REPLACE" >PATH_OF_HOME_DIR_TO_REPLACE/update-tractrac-urls-to-archive.out 2>PATH_OF_HOME_DIR_TO_REPLACE/update-tractrac-urls-to-archive.err \ No newline at end of file +10 12 * * * export PATH=/bin:/usr/bin:/usr/local/bin; update-tractrac-urls-to-archive.sh "/root/code" >/var/log/update-tractrac-urls-to-archive.out 2>/var/log/update-tractrac-urls-to-archive.err \ No newline at end of file diff --git a/configuration/downloadNewArchivedTracTracEvents.sh b/configuration/downloadNewArchivedTracTracEvents.sh deleted file mode 120000 index 013a90068ed..00000000000 --- a/configuration/downloadNewArchivedTracTracEvents.sh +++ /dev/null @@ -1 +0,0 @@ -environments_scripts/central_reverse_proxy/files/usr/local/bin/downloadNewArchivedTracTracEvents.sh \ No newline at end of file diff --git a/configuration/environments/security-service-master b/configuration/environments/security-service-master index 74bbc94438e..19588f4f4d2 100755 --- a/configuration/environments/security-service-master +++ b/configuration/environments/security-service-master @@ -1,5 +1,19 @@ +TELNET_PORT=14888 +SERVER_PORT=8888 +MONGODB_HOST=dbserver.internal.sapsailing.com +MONGODB_PORT=10202 +EXPEDITION_PORT=2010 +DEPLOY_TO=security_service +REPLICATE_MASTER_SERVLET_PORT=8888 +REPLICATE_MASTER_QUEUE_HOST=rabbit.internal.sapsailing.com +REPLICATE_MASTER_QUEUE_PORT=5672 +REPLICATION_HOST=rabbit.internal.sapsailing.com +ADDITIONAL_JAVA_ARGS="$ADDITIONAL_JAVA_ARGS -Dorg.eclipse.jetty.LEVEL=OFF -Dorg.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog -Drestore.tracked.races=true -Dpolardata.source.url=https://www.sapsailing.com -XX:ThreadPriorityPolicy=2" +SERVER_STARTUP_NOTIFY=axel.uhl@sap.com +BUILD_COMPLETE_NOTIFY=axel.uhl@sap.com + # To enable the use of the shared SecurityService and SharedSailingData from security-service.sapsailing.com: -ADDITIONAL_JAVA_ARGS="$ADDITIONAL_JAVA_ARGS -Dsecurity.sharedAcrossSubdomainsOf=sapsailing.com -Dsecurity.baseUrlForCrossDomainStorage=https://security-service.sapsailing.com -Dgwt.acceptableCrossDomainStorageRequestOriginRegexp=https?://(.*\.)?sapsailing\.com(:[0-9]*)?$" +ADDITIONAL_JAVA_ARGS="$ADDITIONAL_JAVA_ARGS -Dsecurity.sharedAcrossSubdomainsOf=sapsailing.com -Dsecurity.baseUrlForCrossDomainStorage=https://security-service.sapsailing.com -Dgwt.acceptableCrossDomainStorageRequestOriginRegexp=https?://((.*\.)?sapsailing\.com)|((.*\.)?sailing\.omegatiming\.com)(:[0-9]*)?$" # For the security-service master instance, don't activate replication from security-service #REPLICATE_ON_START=com.sap.sse.security.impl.SecurityServiceImpl,com.sap.sailing.shared.server.impl.SharedSailingDataImpl,com.sap.sse.landscape.aws.impl.AwsLandscapeStateImpl #REPLICATE_MASTER_SERVLET_HOST=security-service.sapsailing.com diff --git a/configuration/environments_scripts/README b/configuration/environments_scripts/README index f34fa7b7941..b1919be3082 100644 --- a/configuration/environments_scripts/README +++ b/configuration/environments_scripts/README @@ -1,16 +1,36 @@ -The environments_scripts directory contains more directories, which map to environment types, each holding scripts and files useful to that environment. Each environment type also contains a "users" folder, which contains subfolders for each user on the system. -The build-crontab-and-cp-files script uses the contents of these subfolders to create and install a customised crontab for each user in the folder. It does this by concatenating one liner crontabs into an uber crontab. -Any time the crontab should contain a user's home directory, instead write PATH_OF_HOME_DIR_TO_REPLACE; if the crontab should -contain the path to the git directory, instead write PATH_OF_GIT_HOME_DIR_TO_REPLACE. -These are replaced by the build-crontab-and-cp-files script with the correct paths +The environments_scripts directory contains more directories, which map to environment +types, each holding scripts and files useful to that environment. +Each environment type also contains a "users" folder, which contains +subfolders for each user on the system. +The build-crontab-and-cp-files script uses the contents of these subfolders to +create and install a customised crontab for each user in the folder. +It does this by concatenating one liner crontab files/links following the naming +pattern "crontab*" into an uber crontab. + +For crontab snippet placeholder substitutions, please see +${GIT_ROOT}/configuration/crontabs/README. environment_scripts (directory) | -|_environement_type (directory) +|_{environment_type} (directory) | |_usefulScripts + |_files (directory) + | | + | |_{contents to copy to environment, starting from / folder} + | |_users (directory) | |_user1 (directory) | |_symbolicLinks (to configuration/crontabs) + + +The build-crontab-and-cp-files script is not installed to /usr/local/bin, +so in order to actually perform this crontab setup and file copying from +on an instance, you will need to use the build_crontab_and_setup-files function +from imageupgrade_functions.sh, which copies the latest environment_scripts +folder onto the local instance and then executes the build-crontab-and-cp-files +script, within the directory. This allows for a single dependency and calling method. +Furthermore, it means the latest version of the build-crontab-and-cp-files script +is always used. \ No newline at end of file diff --git a/configuration/environments_scripts/build-crontab-and-cp-files b/configuration/environments_scripts/build-crontab-and-cp-files index 62bd4aef131..d8318e3d0e1 100755 --- a/configuration/environments_scripts/build-crontab-and-cp-files +++ b/configuration/environments_scripts/build-crontab-and-cp-files @@ -3,66 +3,142 @@ # Purpose: The first parameter is an image type. This script iterates over the image types' USER folders, # concatenating all the symbolic links (referencing locations within configuration/crontabs) into one crontab # file for each user. It also takes the name of the user containing a copy of the git repo as well as the name of the -# dir within that user to go to. The created files for each user go into that user's home dir and are installed for -# that user too. -# Within these crontab stubs, are certain string literals, which are replaced: PATH_OF_GIT_HOME_DIR_TO_REPLACE becomes -# the absolute path to the git home dir, and PATH_OF_HOME_DIR_TO_REPLACE is changed to the path of the home directory -# in which the crontab will be installed. +# dir within that user to go to. The created crontab for each user go into that user's home dir and are installed for +# that user too. +# Within these crontab stubs, are certain string literals, which are replaced: PATH_OF_HOME_DIR_TO_REPLACE is changed +# to the path of the home directory in which the crontab will be installed (ie. that user's home). +# Furthermore, within the users folder optional uid, gid and groups files an exist. uid should store the user's uid, +# gid the group id and groups should have secondary groups (one per line). The user is added to the secondary groups +# after all users are created. # Useful files are also copied across from the "files" dir within each image type. # Finally, the script ends by performing a deamon reload and reenabling all the systemd units in the files directory, # so that the correct wants, requires, etc. are installed/linked. Any systemd units, within the mimicked filesystem, -# will be automatically enabled. - -if [[ $# -lt 3 || $# -gt 5 ]]; then - echo "$0 " - echo "" - echo "Where USER_WITH_COPY_OF_REPO is a user that contains a checked out copy of the main git." - echo "And where RELATIVE_PATH_OF_GIT_DIR_WITHIN_USER is the path to the git repo from the USER_WITH_COPY_OF_REPO's home directory." - echo "Use the s(imple) flag to only do the crontab and not copy any files across. The n(o install) flag can be used to setup the crontabs but not install them." - exit 2 -fi +# will be automatically enabled. +# For options, see the help below. +# This script should NOT be installed into /usr/local/bin. See the README for more. +. repo/usr/local/bin/imageupgrade_functions.sh INSTALL_CRONTAB="true" -options='sn' +COPY_FILES="true" +CREATE_CRONTABS="true" +options='fnc' while getopts $options option do case $option in + c) CREATE_CRONTABS="false";; + f) COPY_FILES="false";; n) INSTALL_CRONTAB="false";; - s) ONLY_CRONTAB="true";; - \?) echo "Invalid option" + \?) echo "Invalid option" # getops replaces unknown options with a question mark, so we have to escape the question mark. exit 4;; esac done -shift $((OPTIND-1)) # shift the arguments along so -s is no longer $1 +shift $((OPTIND-1)) # shift the arguments along so there are no options in the arguments anymore. +if [[ "$#" -ne 1 ]]; then + echo "$0 [ -f ] [ -n ] [ -c ] " + echo "" + echo "Use the f(iles) flag to disable file copying. Use the c(rontab) flag to disable the creation of users and the creation of crontab files for those users." + echo "If the c flag is not used then the n(o install) flag can be used to setup the crontabs but not install them." + exit 2 +fi ENV_TYPE="$1" -GIT_USER="$2" -RELATIVE_GIT_DIR_NAME="$3" cd "$(dirname "$0")/${ENV_TYPE}" +if [[ -d "groups" ]]; then + cd "groups" + for group in * ; do + [[ -e "$group" ]] || continue + echo "CREATING GROUP $group" + if [[ -e "$group"/gid ]]; then + groupadd --gid $(cat "$group"/gid) "$group" + else + groupadd "$group" + fi + done + cd .. +fi if [[ -d "users" ]]; then cd "users" - GIT_PATH="$(eval echo $(printf "~%q" "$GIT_USER"))/${RELATIVE_GIT_DIR_NAME}" # The path to the git repo that contains the files needed. - for dir in $(ls -d */ ); do - USERNAME=$(echo $dir | sed "s/\/$//") # Dirname is the username. The trailing slash is removed. - HOME_DIR=$(eval echo $(printf "~%q" "$USERNAME")) # The path to the home dir of the user whose cronjob will be installed. - # Clear the crontab file before assembling it from the snippets: - > $HOME_DIR/crontab - for crontab in $(ls ${USERNAME}/crontab*); do - cat "${crontab}">> $HOME_DIR/crontab - done - sed -i "s|PATH_OF_GIT_HOME_DIR_TO_REPLACE|${GIT_PATH}|g" $HOME_DIR/crontab # Sets correct path to the git repo within the crontab. - sed -i "s|PATH_OF_HOME_DIR_TO_REPLACE|${HOME_DIR}|g" $HOME_DIR/crontab # Sets the correct path to the home dir of the user whose crontab will be installed. - echo "">>$HOME_DIR/crontab # Adds a newline - if [[ "$INSTALL_CRONTAB" == "true" ]]; then - crontab -u ${USERNAME} $HOME_DIR/crontab # Install the crontab in the given user's home dir. + for dir in *; do + [[ -d "$dir" ]] || continue + USERNAME="$dir" + user_UID="" + user_GID="" + echo "STARTING WORK ON USER $USERNAME" + if [[ -f "$USERNAME"/uid ]]; then + user_UID=$(cat "$USERNAME"/uid) + echo "UID => $user_UID" fi + if [[ -f "$USERNAME"/gid ]]; then + user_GID=$(cat "$USERNAME"/gid) + echo "GID => $user_GID" + groupadd --gid "$user_GID" "$USERNAME" + [[ "$?" -eq 0 ]] || echo "Group id or name already exists." >&2 + fi + if [[ -n "$user_UID" && -n "$user_GID" ]]; then + adduser --uid "$user_UID" --gid "$user_GID" "$USERNAME" + elif [[ -n "$user_UID" ]]; then + adduser --uid "$user_UID" "$USERNAME" + elif [[ -n "$user_GID" ]]; then + adduser --gid "$user_GID" "$USERNAME" + else + adduser "$USERNAME" + fi + id "$USERNAME" + if [[ "$?" -eq 0 ]]; then + HOME_DIR=$(eval echo $(printf "~%q" "$USERNAME")) # The path to the home dir of the user whose cronjob will be installed. + # Sets permissions of home dir. + [[ -e "$USERNAME"/permissions ]] && chmod "$(cat "$USERNAME"/permissions)" "$HOME_DIR" + if [[ "$CREATE_CRONTABS" == "true" ]]; then # 9 is the exit code indicating the username already exists. + # Clear the crontab file before assembling it from the snippets: + > $HOME_DIR/crontab + echo "# Note that this file should not be edited manually. Please instead make use of the build_crontab_and_setup_files command in imageupgrade_functions.sh">>"$HOME_DIR"/crontab + echo "# which concatenates the crontabs found at users/*/* into this file. Please edit these crontab stubs in the git repo to make permanent changes.">>"$HOME_DIR"/crontab + echo "">>"$HOME_DIR"/crontab + for crontab in ${USERNAME}/crontab*; do + [[ -e $crontab ]] || continue + cat "${crontab}" | eval $( get_replacements_from_crontab_filename_as_sed_command "${crontab}" ) >> $HOME_DIR/crontab + echo "">> $HOME_DIR/crontab + done + chown "$USERNAME":"$USERNAME" $HOME_DIR/crontab + sed -i "s|PATH_OF_HOME_DIR_TO_REPLACE|${HOME_DIR}|g" "$HOME_DIR"/crontab # Sets the correct path to the home dir of the user whose crontab will be installed. + sed -i '/^$/d' $HOME_DIR/crontab # purges random empty lines. + echo "">>$HOME_DIR/crontab # Adds a newline + if [[ "$INSTALL_CRONTAB" == "true" ]]; then + crontab -u ${USERNAME} $HOME_DIR/crontab # Install the crontab in the given user's home dir. + fi + fi + fi + echo "****** finished user ******" + done + # Add to groups after all the groups exist. + for dir in *; do + [[ -d "$dir" ]] || continue + USERNAME="$dir" + [[ -e "$USERNAME"/groups ]] || continue + echo "ADDING $USERNAME TO THE GROUPS SPECIFIED IN THEIR 'groups' FILE" + for group in $(cat "$USERNAME"/groups); do + gpasswd --add "$USERNAME" "$group" + done done cd .. # exits users folder, which is essential for the next commands fi -if [[ "$ONLY_CRONTAB" != "true" && -d "files" ]]; then +# Add users to groups, specified in ENVIRONMENT_TYPE/groups/GROUP/users. Used if groups exist which aren't directly related to a user. +if [[ -d "groups" ]]; then + cd "groups" + for group in * ; do + [[ -e "$group"/users ]] || continue + echo "ADDING USERS IN 'users' FILE TO GROUP $group" + for user in $(cat "$group"/users); do + gpasswd -a "$user" "$group" + done + done + cd .. +fi +if [[ "$COPY_FILES" == "true" && -d "files" ]]; then cd "files" - \cp -rL * / # copies all files accross, realising any symbolic links. The backslash escapes the alias cp -i. + \cp --remove-destination -rL * / # copies all files accross, realising any symbolic links. The backslash escapes the alias cp -i. systemctl daemon-reload - for unit in $(find etc/systemd/system -type f); do + for unit in etc/systemd/system/*; do + [[ -f "$unit" ]] || continue systemctl reenable /"${unit}" # to recreate all the dependency links. done cd .. -fi \ No newline at end of file +fi diff --git a/configuration/environments_scripts/build_server/files/etc/sudoers.d/hudsoncanlaunchec2instances b/configuration/environments_scripts/build_server/files/etc/sudoers.d/hudsoncanlaunchec2instances new file mode 100644 index 00000000000..3b99c63f2c9 --- /dev/null +++ b/configuration/environments_scripts/build_server/files/etc/sudoers.d/hudsoncanlaunchec2instances @@ -0,0 +1,3 @@ +hudson ALL = (root) NOPASSWD: /usr/local/bin/launchhudsonslave +hudson ALL = (root) NOPASSWD: /usr/local/bin/launchhudsonslave-java11 +hudson ALL = (root) NOPASSWD: /usr/local/bin/getLatestImageOfType.sh diff --git a/configuration/environments_scripts/build_server/files/etc/systemd/system/hudson.service b/configuration/environments_scripts/build_server/files/etc/systemd/system/hudson.service index 5b0cbe59a6f..18fc29edde1 100644 --- a/configuration/environments_scripts/build_server/files/etc/systemd/system/hudson.service +++ b/configuration/environments_scripts/build_server/files/etc/systemd/system/hudson.service @@ -9,5 +9,5 @@ RequiredBy=multi-user.target [Service] Type=oneshot RemainAfterExit=true -ExecStart=/etc/init.d/hudson start -ExecStop=/etc/init.d/hudson stop +ExecStart=/usr/local/bin/hudson start +ExecStop=/usr/local/bin/hudson stop diff --git a/configuration/environments_scripts/build_server/files/usr/local/bin/hudson b/configuration/environments_scripts/build_server/files/usr/local/bin/hudson index fdb4e1072df..86572c9aba0 100755 --- a/configuration/environments_scripts/build_server/files/usr/local/bin/hudson +++ b/configuration/environments_scripts/build_server/files/usr/local/bin/hudson @@ -60,13 +60,14 @@ RETVAL=0 case "$1" in start) echo -n "Starting Hudson " + # Note, the pidfile isn't written to below but it is checked to see if the program is already running. daemon --user "$HUDSON_USER" --pidfile "$HUDSON_PID_FILE" "$JAVA_CMD" "$PARAMS" &> /var/tmp/hudson.log & RETVAL=$? if [ $RETVAL = 0 ]; then success echo > "$HUDSON_PID_FILE" # just in case we fail to find it MY_SESSION_ID=`/bin/ps h -o sess -p $$` - # get PID + # get PID. Note the ww means unlimited width. /bin/ps hww -u "$HUDSON_USER" -o sess,ppid,pid,cmd | \ while read sess ppid pid cmd; do [ "$ppid" = 1 ] || continue diff --git a/configuration/hudson_instance_setup/setup-hudson-server.sh b/configuration/environments_scripts/build_server/setup-hudson-server.sh similarity index 56% rename from configuration/hudson_instance_setup/setup-hudson-server.sh rename to configuration/environments_scripts/build_server/setup-hudson-server.sh index 666f969b61a..b532fb05fa9 100755 --- a/configuration/hudson_instance_setup/setup-hudson-server.sh +++ b/configuration/environments_scripts/build_server/setup-hudson-server.sh @@ -7,14 +7,11 @@ # of the instance you just launched in AWS as only argument. This will then # turn the instance into an application server for the SAP Sailing Analytics # application. When the script is done you may log in to look around and check -# things. When done, shut down the instance (Stop, not Terminate) and create -# an image off of it, naming it, e.g., "SAP Sailing Analytics 2.0" and -# also tagging its root volume snapshot as, e.g., "SAP Sailing Analytics 2.0 (Root)". -# If you want to use the resulting image in production, also tag it with -# tag key "image-type" and tag value "sailing-analytics-server". +# things, such as the landscape managers keys, aws credentials, and that +# the necessary scripts are in /usr/local/bin. if [ $# != 0 ]; then SERVER=$1 - $(dirname $0)/../sailing_server_setup/setup-sailing-server.sh ${SERVER} + $(dirname $0)/../sailing_server/setup-sailing-server.sh ${SERVER} scp "${0}" ec2-user@${SERVER}: ssh -A ec2-user@${SERVER} ./$( basename "${0}" ) else @@ -22,16 +19,16 @@ else echo "Running on an AWS EC2 instance as user ${USER} / $(whoami), starting setup..." # Install secrets scp root@sapsailing.com:dev-secrets /tmp - scp root@sapsailing.com:hudson-aws-credentials /tmp sudo mv /tmp/dev-secrets /root/secrets - sudo mkdir /root/.aws - sudo mv /tmp/hudson-aws-credentials /root/.aws/credentials - sudo chown root:root /root/secrets /root/.aws/credentials - sudo chmod 600 /root/secrets /root/.aws/credentials + sudo chown root:root /root/secrets + sudo chmod 600 /root/secrets + . imageupgrade_functions.sh + if ! build_crontab_and_setup_files build_server; then + exit 1 + fi # Make eu-west-1 the default region for any aws CLI interaction: sudo su - -c "aws configure set default.region eu-west-1" - # Create "hudson" user and clear its directory again which is to become a mount point - sudo adduser hudson + # Clear "hudson" user's directory again which is to become a mount point sudo su - hudson -c "rm -rf /home/hudson/* /home/hudson/.* 2>/dev/null" sudo mkdir /usr/lib/hudson sudo chown hudson /usr/lib/hudson @@ -39,19 +36,12 @@ else sudo chgrp hudson /var/log/hudson sudo chmod g+w /var/log/hudson sudo wget -O /usr/lib/hudson/hudson.war "https://static.sapsailing.com/hudson.war.patched-with-mail-1.6.2" - # Link hudson file to /etc/init.d - sudo ln -s /home/sailing/code/configuration/hudson_instance_setup/hudson /etc/init.d - # Link hudson service to /etc/systemd/system - sudo ln -s /home/sailing/code/configuration/hudson_instance_setup/hudson.service /etc/systemd/system - # Link Hudson system-wide config file: - sudo ln -s /home/sailing/code/configuration/hudson_instance_setup/sysconfig-hudson /etc/sysconfig/hudson - # Link additional script files needed for Hudson build server control: - sudo ln -s /home/sailing/code/configuration/launchhudsonslave /usr/local/bin - sudo ln -s /home/sailing/code/configuration/launchhudsonslave-java11 /usr/local/bin - sudo ln -s /home/sailing/code/configuration/aws-automation/getLatestImageOfType.sh /usr/local/bin # Enable NFS server sudo systemctl enable nfs-server.service sudo systemctl start nfs-server.service + # Enable the mongod.service on which the sailing service depends: + sudo systemctl enable mongod.service + sudo systemctl start mongod.service # Enable the service: sudo systemctl daemon-reload sudo systemctl enable hudson.service @@ -59,18 +49,11 @@ else sudo su - -c "cat <>/etc/exports /home/hudson/android-sdk-linux 172.31.0.0/16(rw,nohide,no_root_squash) EOF -" - # Allow "hudson" user to launch EC2 instances: - sudo su - -c "cat <>/etc/sudoers.d/hudsoncanlaunchec2instances -hudson ALL = (root) NOPASSWD: /usr/local/bin/launchhudsonslave -hudson ALL = (root) NOPASSWD: /usr/local/bin/launchhudsonslave-java11 -hudson ALL = (root) NOPASSWD: /usr/local/bin/getLatestImageOfType.sh -EOF " # Install DEV server sudo su - sailing -c "mkdir /home/sailing/servers/DEV cd /home/sailing/servers/DEV -cat <&2 echo "To prepare an instance running in AWS, provide its external IP as argument to this script." >&2 diff --git a/configuration/environments_scripts/build_server/users/hudson/uid b/configuration/environments_scripts/build_server/users/hudson/uid new file mode 100644 index 00000000000..5ed13cbaef2 --- /dev/null +++ b/configuration/environments_scripts/build_server/users/hudson/uid @@ -0,0 +1 @@ +1002 \ No newline at end of file diff --git a/configuration/environments_scripts/build_server/users/root/crontab-update-authorized-keys b/configuration/environments_scripts/build_server/users/root/crontab-update-authorized-keys deleted file mode 120000 index d9b0fed0726..00000000000 --- a/configuration/environments_scripts/build_server/users/root/crontab-update-authorized-keys +++ /dev/null @@ -1 +0,0 @@ -../../../../crontabs/crontab-update-authorized-keys \ No newline at end of file diff --git a/configuration/environments_scripts/build_server/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_root b/configuration/environments_scripts/build_server/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_root new file mode 120000 index 00000000000..cf597d8e543 --- /dev/null +++ b/configuration/environments_scripts/build_server/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_root @@ -0,0 +1 @@ +../../../../crontabs/crontab-update-authorized-keys@USER_HOME_DIR \ No newline at end of file diff --git a/configuration/environments_scripts/central_mongo_setup/files/etc/logrotate.d/mongodb b/configuration/environments_scripts/central_mongo_setup/files/etc/logrotate.d/mongodb new file mode 120000 index 00000000000..00ba3e765ea --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/files/etc/logrotate.d/mongodb @@ -0,0 +1 @@ +../../../../repo/etc/logrotate.d/mongodb \ No newline at end of file diff --git a/configuration/environments_scripts/central_mongo_setup/files/etc/mongodb/mongodb-archive.conf b/configuration/environments_scripts/central_mongo_setup/files/etc/mongodb/mongodb-archive.conf new file mode 100644 index 00000000000..56130374cf6 --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/files/etc/mongodb/mongodb-archive.conf @@ -0,0 +1,15 @@ +net: + port: 10201 + bindIp: 0.0.0.0 +storage: + dbPath: "/var/lib/mongodb/archive" + directoryPerDB: true + wiredTiger: + engineConfig: + cacheSizeGB: 2 +systemLog: + destination: file + path: "/var/log/mongodb/archive.log" +replication: + replSetName: "archive" + diff --git a/configuration/environments_scripts/central_mongo_setup/files/etc/mongodb/mongodb-hidden-live-replica.conf b/configuration/environments_scripts/central_mongo_setup/files/etc/mongodb/mongodb-hidden-live-replica.conf new file mode 100644 index 00000000000..35fd69e3510 --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/files/etc/mongodb/mongodb-hidden-live-replica.conf @@ -0,0 +1,15 @@ +net: + port: 10203 + bindIp: 0.0.0.0 +storage: + dbPath: "/var/lib/mongodb/hidden-live-replica" + directoryPerDB: true + wiredTiger: + engineConfig: + cacheSizeGB: 2 +systemLog: + destination: file + path: "/var/log/mongodb/hidden-live-replica.log" +replication: + replSetName: "live" + diff --git a/configuration/environments_scripts/central_mongo_setup/files/etc/mongodb/mongodb-slow.conf b/configuration/environments_scripts/central_mongo_setup/files/etc/mongodb/mongodb-slow.conf new file mode 100644 index 00000000000..1cab7780c17 --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/files/etc/mongodb/mongodb-slow.conf @@ -0,0 +1,15 @@ +net: + port: 10202 + bindIp: 0.0.0.0 +storage: + dbPath: "/var/lib/mongodb/slow" + directoryPerDB: true + wiredTiger: + engineConfig: + cacheSizeGB: 2 +systemLog: + destination: file + path: "/var/log/mongodb/slow.log" +replication: + replSetName: "slow" + diff --git a/configuration/environments_scripts/central_mongo_setup/files/etc/systemd/system/chownvarlibmongo.service b/configuration/environments_scripts/central_mongo_setup/files/etc/systemd/system/chownvarlibmongo.service new file mode 100644 index 00000000000..870de7dd902 --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/files/etc/systemd/system/chownvarlibmongo.service @@ -0,0 +1,13 @@ +[Unit] +Description=Ensures all files under /var/lib/mongo are owned by mongod user/group +Requires=ephemeralvolume.service +After=ephemeralvolume.service +Before=mongod.service + +[Install] +RequiredBy=mongod.service + +[Service] +Type=oneshot +RemainAfterExit=true +ExecStart=/bin/chown -R mongod:mongod /var/lib/mongodb/ diff --git a/configuration/environments_scripts/central_mongo_setup/files/etc/systemd/system/imageupgrade.service b/configuration/environments_scripts/central_mongo_setup/files/etc/systemd/system/imageupgrade.service new file mode 120000 index 00000000000..32775d542f4 --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/files/etc/systemd/system/imageupgrade.service @@ -0,0 +1 @@ +../../../../../repo/etc/systemd/system/imageupgrade.service \ No newline at end of file diff --git a/configuration/environments_scripts/central_mongo_setup/files/etc/systemd/system/mongod-archive.service b/configuration/environments_scripts/central_mongo_setup/files/etc/systemd/system/mongod-archive.service new file mode 100644 index 00000000000..7f3ae27a696 --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/files/etc/systemd/system/mongod-archive.service @@ -0,0 +1,36 @@ +[Unit] +Description=MongoDB Database Server +Documentation=https://docs.mongodb.org/manual +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=mongod +Group=mongod +ExecStart=/bin/mongod --config /etc/mongodb/mongod-archive.conf +ExecReload=/bin/kill -HUP $MAINPID +Restart=always +PIDFile=/var/run/mongodb/mongod-archive.pid +RuntimeDirectory=mongodb +# file size +LimitFSIZE=infinity +# cpu time +LimitCPU=infinity +# virtual memory size +LimitAS=infinity +# open files +LimitNOFILE=64000 +# processes/threads +LimitNPROC=64000 +# locked memory +LimitMEMLOCK=infinity +# total threads (user+kernel) +TasksMax=infinity +TasksAccounting=false + +# Recommended limits for mongod as specified in +# https://docs.mongodb.com/manual/reference/ulimit/#recommended-ulimit-settings + +[Install] +WantedBy=multi-user.target diff --git a/configuration/environments_scripts/central_mongo_setup/files/etc/systemd/system/mongod-hidden-live-replica.service b/configuration/environments_scripts/central_mongo_setup/files/etc/systemd/system/mongod-hidden-live-replica.service new file mode 100644 index 00000000000..d45f5eb0788 --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/files/etc/systemd/system/mongod-hidden-live-replica.service @@ -0,0 +1,36 @@ +[Unit] +Description=MongoDB Database Server +Documentation=https://docs.mongodb.org/manual +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=mongod +Group=mongod +ExecStart=/bin/mongod --config /etc/mongodb/mongod-hidden-live-replica.conf +ExecReload=/bin/kill -HUP $MAINPID +Restart=always +PIDFile=/var/run/mongodb/mongod-hidden-live-replicaslow.pid +RuntimeDirectory=mongodb +# file size +LimitFSIZE=infinity +# cpu time +LimitCPU=infinity +# virtual memory size +LimitAS=infinity +# open files +LimitNOFILE=64000 +# processes/threads +LimitNPROC=64000 +# locked memory +LimitMEMLOCK=infinity +# total threads (user+kernel) +TasksMax=infinity +TasksAccounting=false + +# Recommended limits for mongod as specified in +# https://docs.mongodb.com/manual/reference/ulimit/#recommended-ulimit-settings + +[Install] +WantedBy=multi-user.target diff --git a/configuration/environments_scripts/central_mongo_setup/files/etc/systemd/system/mongod-slow.service b/configuration/environments_scripts/central_mongo_setup/files/etc/systemd/system/mongod-slow.service new file mode 100644 index 00000000000..08249c1b8fd --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/files/etc/systemd/system/mongod-slow.service @@ -0,0 +1,36 @@ +[Unit] +Description=MongoDB Database Server +Documentation=https://docs.mongodb.org/manual +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=mongod +Group=mongod +ExecStart=/bin/mongod --config /etc/mongodb/mongod-slow.conf +ExecReload=/bin/kill -HUP $MAINPID +Restart=always +PIDFile=/var/run/mongodb/mongod-slow.pid +RuntimeDirectory=mongodb +# file size +LimitFSIZE=infinity +# cpu time +LimitCPU=infinity +# virtual memory size +LimitAS=infinity +# open files +LimitNOFILE=64000 +# processes/threads +LimitNPROC=64000 +# locked memory +LimitMEMLOCK=infinity +# total threads (user+kernel) +TasksMax=infinity +TasksAccounting=false + +# Recommended limits for mongod as specified in +# https://docs.mongodb.com/manual/reference/ulimit/#recommended-ulimit-settings + +[Install] +WantedBy=multi-user.target diff --git a/configuration/archiveDb.sh b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/archiveDb.sh similarity index 94% rename from configuration/archiveDb.sh rename to configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/archiveDb.sh index 404fdbbca27..e2714c74d4e 100755 --- a/configuration/archiveDb.sh +++ b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/archiveDb.sh @@ -12,7 +12,8 @@ else fi DB=$1 -DUMPBASEDIR=/mnt/mongodb-slow +DUMPBASEDIR=/var/lib/mongodb/slow/.dump +mkdir -p "${DUMPBASEDIR}" if [ "$CONFIRMATION" = "" ]; then echo -n "This will drop the database $DB in the archive DB before importing. Are you sure (y/n)? " @@ -44,4 +45,4 @@ else echo "Hashes of old and new are empty of differ: ${ORIGINAL_HASH} and ${ARCHIVE_HASH}" exit 1 fi - +rm -rf ${DUMPBASEDIR}" diff --git a/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/finduserbyemail b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/finduserbyemail new file mode 100755 index 00000000000..872faedffa9 --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/finduserbyemail @@ -0,0 +1,18 @@ +#!/bin/sh +useremail=$1 +for port in 10201 10202 10203; do + for db in `echo "rs.secondaryOk() + show dbs + quit()" | mongo --port $port | tail -n +3 | grep -v "^bye$" | awk '{print $1;}'`; do + match_1=`echo "rs.secondaryOk() + db.USERS.find({EMAIL: \"$useremail\"})" | mongo --quiet --port $port $db` + match_2=`echo "rs.secondaryOk() + db.COMPETITORS.find({email: \"$useremail\"})" | mongo --quiet --port $port $db` + if [ "$match_1" != "" -o "$match_2" != "" ]; then + echo ${db} on port $port: $match_1 $match_2 + fi + done +done +# Run this from a device with access to root@sapsailing.com to easily find a user by their email, where $1 is the user: +# ssh -A root@sapsailing.com ssh -A ec2-user@dbserver.internal.sapsailing.com finduserbyemail $1 + diff --git a/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/imageupgrade b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/imageupgrade new file mode 100755 index 00000000000..08aa49ddc9d --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/imageupgrade @@ -0,0 +1,22 @@ +#!/bin/bash + +# Upgrades the AWS EC2 MongoDB instance that this script is assumed to be executed on. +# The steps are as follows: + +. imageupgrade_functions.sh + +clean_mongo_pid() { + rm -f /var/run/mongodb/mongod.pid +} + +LOGON_USER_HOME=/home/ec2-user +METADATA=$( get_ec2_user_data ) +echo "Metadata: ${METADATA}" +if echo "${METADATA}" | grep -q "^image-upgrade$"; then + echo "Image upgrade..." + run_yum_update + build_crontab_and_setup_files central_mongo_setup + clean_startup_logs + clean_mongo_pid + finalize +fi diff --git a/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/imageupgrade_functions.sh new file mode 120000 index 00000000000..5fa6a14de15 --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/imageupgrade_functions.sh @@ -0,0 +1 @@ +../../../../../repo/usr/local/bin/imageupgrade_functions.sh \ No newline at end of file diff --git a/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/notify-operators b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/notify-operators new file mode 120000 index 00000000000..420f074aecd --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/notify-operators @@ -0,0 +1 @@ +../../../../../repo/usr/local/bin/notify-operators \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/files/home/trac/bin/notify-unhealthy-mongodb b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/notify-unhealthy-mongodb similarity index 90% rename from configuration/environments_scripts/central_reverse_proxy/files/home/trac/bin/notify-unhealthy-mongodb rename to configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/notify-unhealthy-mongodb index d16fe4bd149..a1399d4ceb1 100755 --- a/configuration/environments_scripts/central_reverse_proxy/files/home/trac/bin/notify-unhealthy-mongodb +++ b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/notify-unhealthy-mongodb @@ -9,10 +9,11 @@ for db in $MONGODBS; do read health read state if [ "$health" != "1" ]; then - echo "MongoDB replica set $replicaset unhealthy, node $server is in state $state\ + echo "MongoDB replica set $replicaset unhealthy, node $server is in state $state\ Check status of $db urgently. To unsubscribe, change $MAILINGLIST on `hostname`." | mail -s "MongoDB Replica Set \"$replicaset\" Unhealthy" `cat "$MAILINGLIST"` fi done ) done + diff --git a/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/update-tractrac-urls-to-archive.sh b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/update-tractrac-urls-to-archive.sh new file mode 100755 index 00000000000..b25d6de1c24 --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/update-tractrac-urls-to-archive.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Fetches the latest trac trac urls from the Mongo archive replica set, currently residing on the central mongo instance, +# and commits the changes to the master branch of the main Git. Note that local state is prioritised if there is a conflict. +# Parameter 1 is an optional absolute path to the Git root. If unspecified, then the default is /home/wiki/gitwiki. +if [[ $# -eq 0 ]]; then + GIT_ROOT=/home/wiki/gitwiki +else + GIT_ROOT=$1 +fi +PATH_TO_TRAC_TRAC_URLS="configuration/tractrac-json-urls" +urls=$(mongo --quiet "mongodb://dbserver.internal.sapsailing.com:10201/winddb?replicaSet=archive" --eval 'db.TRACTRAC_CONFIGURATIONS.find({}, {TT_CONFIG_JSON_URL : 1}).toArray()' | grep -v ObjectId | jq -r '.[].TT_CONFIG_JSON_URL' ) +if [[ "$?" -ne 0 || "$urls" == "null" ]]; then + echo "Mongo db returns null for tractrac url discovery" | notify-operators "MongoDB/tractrac urls issue" + exit 1 +else + echo "${urls}" | sort -u >"${GIT_ROOT}/${PATH_TO_TRAC_TRAC_URLS}" + cd "${GIT_ROOT}" + git config pull.rebase false # ensure that diverging branches merge. + branch_head=$(git rev-parse HEAD) + stash_stack_size=$(git stash list | wc -l) + git stash + new_stash_stack_size=$(git stash list | wc -l) + git pull + if [[ "$stash_stack_size" -eq "$new_stash_stack_size" ]]; then + echo "NO CHANGES STASHED" # This implies there were no changes to the tractrac urls or other changes in the workspace. + else + if git stash apply; then + echo "STASH APPLIED CLEANLY" + git stash drop + git add "${GIT_ROOT}/${PATH_TO_TRAC_TRAC_URLS}" + git commit -m "Updated tractrac-json-urls" + git push + else + # prioritise existing state and local changes. + echo "PRIORITISING LOCAL STATE, RESETTING HEAD AND APPLYING STASH." + git reset --hard "$branch_head" + git stash apply + echo "The stash is still stored in the stack." + fi + fi +fi diff --git a/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/update_authorized_keys_for_landscape_managers b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/update_authorized_keys_for_landscape_managers new file mode 120000 index 00000000000..c1f7891521d --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/update_authorized_keys_for_landscape_managers @@ -0,0 +1 @@ +../../../../../repo/usr/local/bin/update_authorized_keys_for_landscape_managers \ No newline at end of file diff --git a/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/update_authorized_keys_for_landscape_managers_if_changed b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/update_authorized_keys_for_landscape_managers_if_changed new file mode 120000 index 00000000000..9dee9ebd77c --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/update_authorized_keys_for_landscape_managers_if_changed @@ -0,0 +1 @@ +../../../../../repo/usr/local/bin/update_authorized_keys_for_landscape_managers_if_changed \ No newline at end of file diff --git a/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/update_landscape_managers_mailing_list.sh b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/update_landscape_managers_mailing_list.sh new file mode 120000 index 00000000000..f057a2a0de1 --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/files/usr/local/bin/update_landscape_managers_mailing_list.sh @@ -0,0 +1 @@ +../../../../../repo/usr/local/bin/update_landscape_managers_mailing_list.sh \ No newline at end of file diff --git a/configuration/environments_scripts/central_mongo_setup/setup-central-mongo-instance.sh b/configuration/environments_scripts/central_mongo_setup/setup-central-mongo-instance.sh new file mode 100755 index 00000000000..5a48847336e --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/setup-central-mongo-instance.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Usage: Launch an Amazon EC2 instance from an Amazon Linux 2 AMI with +# 16GB of root partition size and the "Database and Messaging" security group +# using an SSH key for which you have a working private key available. +# Make sure to launch it in the same AZ as your current "MongoDB Central" +# instance, or you won't be able to attach the existing data volumes! +# Then, run this script on your local computer, using the external IP address +# of the instance you just launched in AWS as only argument. This will then +# prepare the instance as a central MongoDB instance. +# When the script is done you may log in to look around and check +# things. When done, stop the running "MongoDB Central" instance and detach +# its MongoDB data volumes as instructed below; attach them to the new instance, +# then run "mount -a" and start the MongoDB processes using the service units: +# - mongod-archive.service +# - mongod-hidden-live-replica.service +# - mongod-slow.service +# as per the instructions printed to the console at the end of this script. +# Name your instance "MongoDB Central" and assign the Route53 entry for +# dbserver.internal.sapsailing.com to its internal IP address. +if [ $# != 0 ]; then + SERVER=$1 + scp "${0}" ec2-user@${SERVER}: + ssh -A ec2-user@${SERVER} ./$( basename "${0}" ) +else + if ec2-metadata | grep -q instance-id; then + echo "Running on an AWS EC2 instance as user ${USER} / $(whoami), starting setup..." + # Install standard packages: + sudo yum -y update + sudo yum -y install nvme-cli chrony cronie cronie-anacron jq mailx + # Copy imageupgrade_function.sh + scp -o StrictHostKeyChecking=no -p root@sapsailing.com:/home/wiki/gitwiki/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh . + sudo mv imageupgrade_functions.sh /usr/local/bin + # build-crontab + . imageupgrade_functions.sh + build_crontab_and_setup_files central_mongo_setup + # obtain root SSH key from key vault: + setup_keys "central_mongo_setup" + # Create some swap space for the case mountnvmeswap hasn't created any + setup_swap 6000 + # Install MongoDB 4.4 and configure as replica set "live" + setup_mongo_4_4 + # Disable default mongod service unit derived from /etc/init.d/mongod: + sudo systemctl disable mongod.service + # Prepare for the MongoDB volume mounts: + sudo mkdir -p /var/lib/mongodb/hidden-live-replica + sudo mkdir -p /var/lib/mongodb/slow + sudo mkdir -p /var/lib/mongodb/archive + sudo chown -R mongod:mongod /var/lib/mongodb + sudo su - -c "cat << EOF >>/etc/fstab +UUID=d59156c5-7833-425a-9143-3a83b01f32b9 /var/lib/mongodb/hidden-live-replica xfs defaults,relatime 1 1 +UUID=5586bc6e-d399-42e8-a0a0-bf2d65f8c703 /var/lib/mongodb/archive xfs defaults,relatime 1 1 +UUID=0c73fde1-4719-405a-ab0e-bffd0aaf8d84 /var/lib/mongodb/slow xfs defaults,relatime 1 1 +EOF +" + scp root@sapsailing.com:ssh-key-reader.token /tmp + sudo mv /tmp/ssh-key-reader.token /root + sudo chown root:root /root/ssh-key-reader.token + sudo chmod 600 /root/ssh-key-reader.token + setup_sshd_resilience + echo "Part 1 has completed. Now carry out the following steps in the AWS Console:" + echo " - stop the \"MongoDB Central\" instance" + echo " - detach the following volumes from it:" + echo " * Hidden MongoDB Live Replica encrypted" + echo " * MongoDB Archive winddb encrypted" + echo " * MongoDB Archive slow encrypted" + echo " - attach these volumes to the new instance" + echo " - issue the following commands on the new instance:" + echo " mount -a" + echo " chmod -R mongod:mongod /var/lib/mongodb" + echo " systemctl start mongod-archive.service" + echo " systemctl start mongod-hidden-live-replica.service" + echo " systemctl start mongod-slow.service" + echo " - assign the Route53 DNS record \"dbserver.internal.sapsailing.com\" to this" + echo " instance's internal IP address $( ec2-metadata --local-ipv4 | sed -e 's/^local-ipv4: //' )" + echo " - name the new instance \"MongoDB Central\" and terminate the old instance" + echo " - tag the instance with mongo-replica-sets = live:10203,archive:10201,slow:10202" + else + echo "Not running on an AWS instance; refusing to run setup!" >&2 + echo "To prepare an instance running in AWS, provide its external IP as argument to this script." >&2 + exit 2 + fi +fi diff --git a/configuration/environments_scripts/central_reverse_proxy/users/trac/crontab-mongo-health-check b/configuration/environments_scripts/central_mongo_setup/users/root/crontab-mongo-health-check similarity index 100% rename from configuration/environments_scripts/central_reverse_proxy/users/trac/crontab-mongo-health-check rename to configuration/environments_scripts/central_mongo_setup/users/root/crontab-mongo-health-check diff --git a/configuration/environments_scripts/central_mongo_setup/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_home_ec2-user b/configuration/environments_scripts/central_mongo_setup/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_home_ec2-user new file mode 120000 index 00000000000..cf597d8e543 --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_home_ec2-user @@ -0,0 +1 @@ +../../../../crontabs/crontab-update-authorized-keys@USER_HOME_DIR \ No newline at end of file diff --git a/configuration/environments_scripts/central_mongo_setup/users/root/crontab-update-landscape-managers-mailing-list b/configuration/environments_scripts/central_mongo_setup/users/root/crontab-update-landscape-managers-mailing-list new file mode 120000 index 00000000000..b46a13c0aae --- /dev/null +++ b/configuration/environments_scripts/central_mongo_setup/users/root/crontab-update-landscape-managers-mailing-list @@ -0,0 +1 @@ +../../../../crontabs/crontab-update-landscape-managers-mailing-list \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/users/wiki/crontab-update-trac-trac-urls b/configuration/environments_scripts/central_mongo_setup/users/root/crontab-update-trac-trac-urls similarity index 100% rename from configuration/environments_scripts/central_reverse_proxy/users/wiki/crontab-update-trac-trac-urls rename to configuration/environments_scripts/central_mongo_setup/users/root/crontab-update-trac-trac-urls diff --git a/configuration/environments_scripts/central_reverse_proxy/batch-for-route53-dns-record-update.json b/configuration/environments_scripts/central_reverse_proxy/batch-for-route53-dns-record-update.json new file mode 100644 index 00000000000..db9d41fd719 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/batch-for-route53-dns-record-update.json @@ -0,0 +1,32 @@ +{ + "Comment": "Alter logfiles record ", + "Changes": [ + { + "Action": "UPSERT", + "ResourceRecordSet": { + "Name": "logfiles.internal.sapsailing.com", + "Type": "A", + "TTL": 60, + "ResourceRecords": [ + { + "Value": "LOGFILES_INTERNAL_IP" + } + ] + } + }, + { + "Action": "UPSERT", + "ResourceRecordSet": { + "Name": "smtp.internal.sapsailing.com", + "Type": "A", + "TTL": 60, + "ResourceRecords": [ + { + "Value": "SMTP_INTERNAL_IP" + } + ] + } + } + ] + } + \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/files/etc/awstats/awstats.model.conf.sample b/configuration/environments_scripts/central_reverse_proxy/files/etc/awstats/awstats.model.conf.sample new file mode 100644 index 00000000000..952a13de8c8 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/etc/awstats/awstats.model.conf.sample @@ -0,0 +1,1550 @@ +# AWSTATS CONFIGURE FILE 7.0 +#----------------------------------------------------------------------------- +# Copy this file into awstats.www.mydomain.conf and edit this new config file +# to setup AWStats (See documentation in docs/ directory). +# The config file must be in /etc/awstats, /usr/local/etc/awstats or /etc (for +# Unix/Linux) or same directory than awstats.pl (Windows, Mac, Unix/Linux...) +# To include an environment variable in any parameter (AWStats will replace +# it with its value when reading it), follow the example: +# Parameter="__ENVNAME__" +# Note that environment variable AWSTATS_CURRENT_CONFIG is always defined with +# the config value in an AWStats running session and can be used like others. +#----------------------------------------------------------------------------- +# $Revision: 1.350 $ - $Author: eldy $ - $Date: 2010/09/29 19:16:21 $ + + + +#----------------------------------------------------------------------------- +# MAIN SETUP SECTION (Required to make AWStats work) +#----------------------------------------------------------------------------- + +# "LogFile" contains the web, ftp or mail server log file to analyze. +# Possible values: A full path, or a relative path from awstats.pl directory. +# Example: "/var/log/apache/access.log" +# Example: "../logs/mycombinedlog.log" +# You can also use tags in this filename if you need a dynamic file name +# depending on date or time (Replacement is made by AWStats at the beginning +# of its execution). This is available tags : +# %YYYY-n is replaced with 4 digits year we were n hours ago +# %YY-n is replaced with 2 digits year we were n hours ago +# %MM-n is replaced with 2 digits month we were n hours ago +# %MO-n is replaced with 3 letters month we were n hours ago +# %DD-n is replaced with day we were n hours ago +# %HH-n is replaced with hour we were n hours ago +# %NS-n is replaced with number of seconds at 00:00 since 1970 +# %WM-n is replaced with the week number in month (1-5) +# %Wm-n is replaced with the week number in month (0-4) +# %WY-n is replaced with the week number in year (01-52) +# %Wy-n is replaced with the week number in year (00-51) +# %DW-n is replaced with the day number in week (1-7, 1=sunday) +# use n=24 if you need (1-7, 1=monday) +# %Dw-n is replaced with the day number in week (0-6, 0=sunday) +# use n=24 if you need (0-6, 0=monday) +# Use 0 for n if you need current year, month, day, hour... +# Example: "/var/log/access_log.%YYYY-0%MM-0%DD-0.log" +# Example: "C:/WINNT/system32/LogFiles/W3SVC1/ex%YY-24%MM-24%DD-24.log" +# You can also use a pipe if log file come from a pipe : +# Example: "gzip -d outputpath/output.html"), enter +# path of icon directory relative to the output directory 'outputpath'. +# Example: "/awstatsicons" +# Example: "../icon" +# Default: "/icon" (means you must copy icon directories in "/mywwwroot/icon") +# +DirIcons="/awstatsicons" + + +# When this parameter is set to 1, AWStats adds a button on report page to +# allow to "update" statistics from a web browser. Warning, when "update" is +# made from a browser, AWStats is run as a CGI by the web server user defined +# in your web server (user "nobody" by default with Apache, "IUSR_XXX" with +# IIS), so the "DirData" directory and all already existing history files +# awstatsMMYYYY[.xxx].txt must be writable by this user. Change permissions if +# necessary to "Read/Write" (and "Modify" for Windows NTFS file systems). +# Warning: Update process can be long so you might experience "time out" +# browser errors if you don't launch AWStats frequently enough. +# When set to 0, update is only made when AWStats is run from the command +# line interface (or a task scheduler). +# Possible values: 0 or 1 +# Default: 0 +# +AllowToUpdateStatsFromBrowser=0 + + +# AWStats saves and sorts its database on a month basis (except if using +# databasebreak option from command line). +# However, if you choose the -month=all from command line or +# value '-Year-' from CGI combo form to have a report for all year, AWStats +# needs to reload all data for full year (each month), and sort them, +# requiring a large amount of time, memory and CPU. This might be a problem +# for web hosting providers that offer AWStats for large sites, on shared +# servers, to non CPU cautious customers. +# For this reason, the 'full year' is only enabled on Command Line by default. +# You can change this by setting this parameter to 0, 1, 2 or 3. +# Possible values: +# 0 - Never allowed +# 1 - Allowed on CLI only, -Year- value in combo is not visible +# 2 - Allowed on CLI only, -Year- value in combo is visible but not allowed +# 3 - Possible on CLI and CGI +# Default: 2 +# +AllowFullYearView=2 + + + +#----------------------------------------------------------------------------- +# OPTIONAL SETUP SECTION (Not required but increase AWStats features) +#----------------------------------------------------------------------------- + +# When the update process runs, AWStats can set a lock file in TEMP or TMP +# directory. This lock is to avoid to have 2 update processes running at the +# same time to prevent unknown conflicts problems and avoid DoS attacks when +# AllowToUpdateStatsFromBrowser is set to 1. +# Because, when you use lock file, you can experience sometimes problems in +# lock file not correctly removed (killed process for example requires that +# you remove the file manualy), this option is not enabled by default (Do +# not enable this option with no console server access). +# Change : Effective immediatly +# Possible values: 0 or 1 +# Default: 0 +# +EnableLockForUpdate=1 + + +# AWStats can do reverse DNS lookups through a static DNS cache file that was +# previously created manually. If no path is given in static DNS cache file +# name, AWStats will search DirData directory. This file is never changed. +# This option is not used if DNSLookup=0. +# Note: DNS cache file format is 'minsince1970 ipaddress resolved_hostname' +# or just 'ipaddress resolved_hostname' +# Change : Effective for new updates only +# Example: "/mydnscachedir/dnscache" +# Default: "dnscache.txt" +# +DNSStaticCacheFile="dnscache.txt" + + +# AWStats can do reverse DNS lookups through a DNS cache file that was created +# by a previous run of AWStats. This file is erased and recreated after each +# statistics update process. You don't need to create and/or edit it. +# AWStats will read and save this file in DirData directory. +# This option is used only if DNSLookup=1. +# Note: If a DNSStaticCacheFile is available, AWStats will check for DNS +# lookup in DNSLastUpdateCacheFile after checking into DNSStaticCacheFile. +# Change : Effective for new updates only +# Example: "/mydnscachedir/dnscachelastupdate" +# Default: "dnscachelastupdate.txt" +# +DNSLastUpdateCacheFile="dnscachelastupdate.txt" + + +# You can specify specific IP addresses that should NOT be looked up in DNS. +# This option is used only if DNSLookup=1. +# Note: Use space between each value. +# Note: You can use regular expression values writing value with REGEX[value]. +# Change : Effective for new updates only +# Example: "123.123.123.123 REGEX[^192\.168\.]" +# Default: "" +# +SkipDNSLookupFor="" + + +# The following two parameters allow you to protect a config file from being +# read by AWStats when called from a browser if web user has not been +# authenticated. Your AWStats program must be in a web protected "realm" (With +# Apache, you can use .htaccess files to do so. With other web servers, see +# your server setup manual). +# Change : Effective immediatly +# Possible values: 0 or 1 +# Default: 0 +# +AllowAccessFromWebToAuthenticatedUsersOnly=0 + + +# This parameter gives the list of all authorized authenticated users to view +# statistics for this domain/config file. This parameter is used only if +# AllowAccessFromWebToAuthenticatedUsersOnly is set to 1. +# Change : Effective immediatly +# Example: "user1 user2" +# Example: "__REMOTE_USER__" +# Default: "" +# +AllowAccessFromWebToFollowingAuthenticatedUsers="" + + +# When this parameter is defined to something, the IP address of the user that +# reads its statistics from a browser (when AWStats is used as a CGI) is +# checked and must match one of the IP address values or ranges. +# Change : Effective immediatly +# Example: "127.0.0.1 123.123.123.1-123.123.123.255" +# Default: "" +# +AllowAccessFromWebToFollowingIPAddresses="" + + +# If the "DirData" directory (see above) does not exist, AWStats return an +# error. However, you can ask AWStats to create it. +# This option can be used by some Web Hosting Providers that has defined a +# dynamic value for DirData (for example DirData="/home/__REMOTE_USER__") and +# don't want to have to create a new directory each time they add a new user. +# Change : Effective immediatly +# Possible values: 0 or 1 +# Default: 0 +# +CreateDirDataIfNotExists=0 + + +# You can choose in which format the Awstats history database is saved. +# Note: Using "xml" format make AWStats building database files three times +# larger than using "text" format. +# Change : Database format is switched after next update +# Possible values: text or xml +# Default: text +# +BuildHistoryFormat=text + + +# If you prefer having the report output pages be built as XML compliant pages +# instead of simple HTML pages, you can set this to 'xhtml' (May not work +# properly with old browsers). +# Change : Effective immediatly +# Possible values: html or xhtml +# Default: html +# +BuildReportFormat=html + + +# AWStats databases can be updated from command line of from a browser (when +# used as a cgi program). So AWStats database files need write permission +# for both command line user and default web server user (nobody for Unix, +# IUSR_xxx for IIS/Windows,...). +# To avoid permission problems between update process (run by an admin user) +# and CGI process (ran by a low level user), AWStats can save its database +# files with read and write permissions for everyone. +# By default, AWStats keeps default user permissions on updated files. If you +# set AllowToUpdateStatsFromBrowser to 1, you can change this parameter to 1. +# Change : Effective for new updates only +# Possible values: 0 or 1 +# Default: 0 +# +SaveDatabaseFilesWithPermissionsForEveryone=0 + + +# AWStats can purge log file, after analyzing it. Note that AWStats is able +# to detect new lines in a log file, to process only them, so you can launch +# AWStats as often as you want, even with this parameter to 0. +# With 0, no purge is made, so you must use a scheduled task or a web server +# that make this purge frequently. +# With 1, the purge of the log file is made each time AWStats update is run. +# This parameter doesn't work with IIS (This web server doesn't let its log +# file to be purged). +# Change : Effective for new updates only +# Possible values: 0 or 1 +# Default: 0 +# +PurgeLogFile=0 + + +# When PurgeLogFile is setup to 1, AWStats will clean your log file after +# processing it. You can however keep an archive file of all processed log +# records by setting this parameter (For example if you want to use another +# log analyzer). The archived log file is saved in "DirData" with name +# awstats_archive.configname[.suffix].log +# This parameter is not used if PurgeLogFile=0 +# Change : Effective for new updates only +# Possible values: 0, 1, or tags (See LogFile parameter) for suffix +# Example: 1 +# Example: %YYYY%MM%DD +# Default: 0 +# +ArchiveLogRecords=0 + + +# Each time you run the update process, AWStats overwrites the 'historic file' +# for the month (awstatsMMYYYY[.*].txt) with the updated one. +# When write errors occurs (IO, disk full,...), this historic file can be +# corrupted and must be deleted. Because this file contains information of all +# past processed log files, you will loose old stats if removed. So you can +# ask AWStats to save last non corrupted file in a .bak file. This file is +# stored in "DirData" directory with other 'historic files'. +# Change : Effective for new updates only +# Possible values: 0 or 1 +# Default: 0 +# +KeepBackupOfHistoricFiles=0 + + +# Default index page name for your web server. +# Change : Effective for new updates only +# Example: "index.php index.html default.html" +# Default: "index.php index.html" +# +DefaultFile="index.php index.html" + + +# Do not include access from clients that match following criteria. +# If your log file contains IP addresses in host field, you must enter here +# matching IP addresses criteria. +# If DNS lookup is already done in your log file, you must enter here hostname +# criteria, else enter ip address criteria. +# The opposite parameter of "SkipHosts" is "OnlyHosts". +# Note: Use space between each value. This parameter is not case sensitive. +# Note: You can use regular expression values writing value with REGEX[value]. +# Change : Effective for new updates only +# Example: "127.0.0.1 REGEX[^192\.168\.] REGEX[^10\.]" +# Example: "localhost REGEX[^.*\.localdomain$]" +# Default: "" +# +SkipHosts="127.0.0.1" + + +# Do not include access from clients with a user agent that match following +# criteria. If you want to exclude a robot, you should update the robots.pm +# file instead of this parameter. +# The opposite parameter of "SkipUserAgents" is "OnlyUserAgents". +# Note: Use space between each value. This parameter is not case sensitive. +# Note: You can use regular expression values writing value with REGEX[value]. +# Change : Effective for new updates only +# Example: "konqueror REGEX[ua_test_v\d\.\d]" +# Default: "" +# +SkipUserAgents="" + + +# Use SkipFiles to ignore access to URLs that match one of following entries. +# You can enter a list of not important URLs (like framed menus, hidden pages, +# etc...) to exclude them from statistics. You must enter here exact relative +# URL as found in log file, or a matching REGEX value. Check apply on URL with +# all its query paramaters. +# For example, to ignore /badpage.php, just add "/badpage.php". To ignore all +# pages in a particular directory, add "REGEX[^\/directorytoexclude]". +# The opposite parameter of "SkipFiles" is "OnlyFiles". +# Note: Use space between each value. This parameter is or not case sensitive +# depending on URLNotCaseSensitive parameter. +# Note: You can use regular expression values writing value with REGEX[value]. +# Change : Effective for new updates only +# Example: "/badpage.php /page.php?param=x REGEX[^\/excludedirectory]" +# Default: "" +# +SkipFiles="" + + +# Use SkipReferrersBlackList if you want to exclude records coming from a SPAM +# referrer. Parameter must receive a local file name containing rules applied +# on referrer field. If parameter is empty, no filter is applied. +# An example of such a file is available in lib/blacklist.txt +# Change : Effective for new updates only +# Example: "/mylibpath/blacklist.txt" +# Default: "" +# +# WARNING!! Using this feature make AWStats running very slower (5 times slower +# with black list file provided with AWStats ! +# +SkipReferrersBlackList="" + + +# Include in stats, only accesses from hosts that match one of following +# entries. For example, if you want AWStats to filter access to keep only +# stats for visits from particular hosts, you can add those host names in +# this parameter. +# If DNS lookup is already done in your log file, you must enter here hostname +# criteria, else enter ip address criteria. +# The opposite parameter of "OnlyHosts" is "SkipHosts". +# Note: Use space between each value. This parameter is not case sensitive. +# Note: You can use regular expression values writing value with REGEX[value]. +# Change : Effective for new updates only +# Example: "127.0.0.1 REGEX[^192\.168\.] REGEX[^10\.]" +# Default: "" +# +OnlyHosts="" + + +# Include in stats, only accesses from user agent that match one of following +# entries. For example, if you want AWStats to filter access to keep only +# stats for visits from particular browsers, you can add their user agents +# string in this parameter. +# The opposite parameter of "OnlyUserAgents" is "SkipUserAgents". +# Note: Use space between each value. This parameter is not case sensitive. +# Note: You can use regular expression values writing value with REGEX[value]. +# Change : Effective for new updates only +# Example: "msie" +# Default: "" +# +OnlyUserAgents="" + + +# Include in stats, only accesses from authenticated users that match one of +# following entries. For example, if you want AWStats to filter access to keep +# only stats for authenticated users, you can add those users names in +# this parameter. Useful for statistics for per user ftp logs. +# Note: Use space between each value. This parameter is not case sensitive. +# Note: You can use regular expression values writing value with REGEX[value]. +# Change : Effective for new updates only +# Example: "john bob REGEX[^testusers]" +# Default: "" +# +OnlyUsers="" + + +# Include in stats, only accesses to URLs that match one of following entries. +# For example, if you want AWStats to filter access to keep only stats that +# match a particular string, like a particular directory, you can add this +# directory name in this parameter. +# The opposite parameter of "OnlyFiles" is "SkipFiles". +# Note: Use space between each value. This parameter is or not case sensitive +# depending on URLNotCaseSensitive parameter. +# Note: You can use regular expression values writing value with REGEX[value]. +# Change : Effective for new updates only +# Example: "REGEX[marketing_directory] REGEX[office\/.*\.(csv|sxw)$]" +# Default: "" +# +OnlyFiles="" + + +# Add here a list of kind of url (file extension) that must be counted as +# "Hit only" and not as a "Hit" and "Page/Download". You can set here all +# image extensions as they are hit downloaded that must be counted but they +# are not viewed pages. URLs with such extensions are not included in the TOP +# Pages/URL report. +# Note: If you want to exclude particular URLs from stats (No Pages and no +# Hits reported), you must use SkipFiles parameter. +# Change : Effective for new updates only +# Example: "css js class gif jpg jpeg png bmp ico rss xml swf zip arj rar gz z bz2 wav mp3 wma mpg avi" +# Example: "" +# Default: "css js class gif jpg jpeg png bmp ico rss xml swf" +# +NotPageList="css js class gif jpg jpeg png bmp ico rss xml swf" + + +# By default, AWStats considers that records found in web log file are +# successful hits if HTTP code returned by server is a valid HTTP code (200 +# and 304). Any other code are reported in HTTP status chart. +# Note that HTTP 'control codes', like redirection (302, 305) are not added by +# default in this list as they are not pages seen by a visitor but are +# protocol exchange codes to tell the browser to ask another page. Because +# this other page will be counted and seen with a 200 or 304 code, if you +# add such codes, you will have 2 pages viewed reported for only one in facts. +# Change : Effective for new updates only +# Example: "200 304 302 305" +# Default: "200 304" +# +ValidHTTPCodes="200 304" + + +# By default, AWStats considers that records found in mail log file are +# successful mail transfers if field that represent return code in analyzed +# log file match values defined by this parameter. +# Change : Effective for new updates only +# Example: "1 250 200" +# Default: "1 250" +# +ValidSMTPCodes="1 250" + + +# Some web servers on some Operating systems (IIS-Windows) consider that a +# login with same value but different case are the same login. To tell AWStats +# to also consider them as one, set this parameter to 1. +# Change : Effective for new updates only +# Possible values: 0 or 1 +# Default: 0 +# +AuthenticatedUsersNotCaseSensitive=0 + + +# Some web servers on some Operating systems (IIS-Windows) considers that two +# URLs with same value but different case are the same URL. To tell AWStats to +# also considers them as one, set this parameter to 1. +# Change : Effective for new updates only +# Possible values: 0 or 1 +# Default: 0 +# +URLNotCaseSensitive=0 + + +# Keep or remove the anchor string you can find in some URLs. +# Change : Effective for new updates only +# Possible values: 0 or 1 +# Default: 0 +# +URLWithAnchor=0 + + +# In URL links, "?" char is used to add parameter's list in URLs. Syntax is: +# /mypage.html?param1=value1¶m2=value2 +# However, some servers/sites use also other chars to isolate dynamic part of +# their URLs. You can complete this list with all such characters. +# Change : Effective for new updates only +# Example: "?;," +# Default: "?;" +# +URLQuerySeparators="?;" + + +# Keep or remove the query string to the URL in the statistics for individual +# pages. This is primarily used to differentiate between the URLs of dynamic +# pages. If set to 1, mypage.html?id=x and mypage.html?id=y are counted as two +# different pages. +# Warning, when set to 1, memory required to run AWStats is dramatically +# increased if you have a lot of changing URLs (for example URLs with a random +# id inside). Such web sites should not set this option to 1 or use seriously +# the next parameter URLWithQueryWithOnlyFollowingParameters (or eventually +# URLWithQueryWithoutFollowingParameters). +# Change : Effective for new updates only +# Possible values: +# 0 - URLs are cleaned from the query string (ie: "/mypage.html") +# 1 - Full URL with query string is used (ie: "/mypage.html?p=x&q=y") +# Default: 0 +# +URLWithQuery=0 + + +# When URLWithQuery is on, you will get the full URL with all parameters in +# URL reports. But among thoose parameters, sometimes you don't need a +# particular parameter because it does not identify the page or because it's +# a random ID changing for each access even if URL points to same page. In +# such cases, it is higly recommanded to ask AWStats to keep only parameters +# you need (if you know them) before counting, manipulating and storing URL. +# Enter here list of wanted parameters. For example, with "param", one hit on +# /mypage.cgi?param=abc&id=Yo4UomP9d and /mypage.cgi?param=abc&id=Mu8fdxl3r +# will be reported as 2 hits on /mypage.cgi?param=abc +# This parameter is not used when URLWithQuery is 0 and can't be used with +# URLWithQueryWithoutFollowingParameters. +# Change : Effective for new updates only +# Example: "param" +# Default: "" +# +URLWithQueryWithOnlyFollowingParameters="" + + +# When URLWithQuery is on, you will get the full URL with all parameters in +# URL reports. But among thoose parameters, sometimes you don't need a +# particular parameter because it does not identify the page or because it's +# a random ID changing for each access even if URL points to same page. In +# such cases, it is higly recommanded to ask AWStats to remove such parameters +# from the URL before counting, manipulating and storing URL. Enter here list +# of all non wanted parameters. For example if you enter "id", one hit on +# /mypage.cgi?param=abc&id=Yo4UomP9d and /mypage.cgi?param=abc&id=Mu8fdxl3r +# will be reported as 2 hits on /mypage.cgi?param=abc +# This parameter is not used when URLWithQuery is 0 and can't be used with +# URLWithQueryWithOnlyFollowingParameters. +# Change : Effective for new updates only +# Example: "PHPSESSID jsessionid" +# Default: "" +# +URLWithQueryWithoutFollowingParameters="" + + +# Keep or remove the query string to the referrer URL in the statistics for +# external referrer pages. This is used to differentiate between the URLs of +# dynamic referrer pages. If set to 1, mypage.html?id=x and mypage.html?id=y +# are counted as two different referrer pages. +# Change : Effective for new updates only +# Possible values: +# 0 - Referrer URLs are cleaned from the query string (ie: "/mypage.html") +# 1 - Full URL with query string is used (ie: "/mypage.html?p=x&q=y") +# Default: 0 +# +URLReferrerWithQuery=0 + + +# AWStats can detect setup problems or show you important informations to have +# a better use. Keep this to 1, except if AWStats says you can change it. +# Change : Effective immediatly +# Possible values: 0 or 1 +# Default: 1 +# +WarningMessages=1 + + +# When an error occurs, AWStats outputs a message related to errors. If you +# want (in most cases for security reasons) to have no error messages, you +# can set this parameter to your personalized generic message. +# Change : Effective immediatly +# Example: "An error occurred. Contact your Administrator" +# Default: "" +# +ErrorMessages="" + + +# AWStat can be run with debug=x parameter to output various informations +# to help in debugging or solving troubles. If you want to allow this (not +# enabled by default for security reasons), set this parameter to 0. +# Change : Effective immediatly +# Possible values: 0 or 1 +# Default: 0 +# +DebugMessages=0 + + +# To help you to detect if your log format is good, AWStats reports an error +# if all the first NbOfLinesForCorruptedLog lines have a format that does not +# match the LogFormat parameter. +# However, some worm virus attack on your web server can result in a very high +# number of corrupted lines in your log. So if you experience awstats stop +# because of bad virus records at the beginning of your log file, you can +# increase this parameter (very rare). +# Change : Effective for new updates only +# Default: 50 +# +NbOfLinesForCorruptedLog=50 + + +# For some particular integration needs, you may want to have CGI links to +# point to another script than awstats.pl. +# Use the name of this script in WrapperScript parameter. +# Change : Effective immediatly +# Example: "awstatslauncher.pl" +# Example: "awstatswrapper.cgi?key=123" +# Default: "" +# +WrapperScript="" + + +# DecodeUA must be set to 1 if you use Roxen web server. This server converts +# all spaces in user agent field into %20. This make the AWStats robots, OS +# and browsers detection fail in some cases. Just change it to 1 if and only +# if your web server is Roxen. +# Change : Effective for new updates only +# Possible values: 0 or 1 +# Default: 0 +# +DecodeUA=0 + + +# MiscTrackerUrl can be used to make AWStats able to detect some miscellaneous +# things, that can not be tracked on other way, like: +# - Javascript disabled +# - Java enabled +# - Screen size +# - Color depth +# - Macromedia Director plugin +# - Macromedia Shockwave plugin +# - Realplayer G2 plugin +# - QuickTime plugin +# - Mediaplayer plugin +# - Acrobat PDF plugin +# To enable all these features, you must copy the awstats_misc_tracker.js file +# into a /js/ directory stored in your web document root and add the following +# HTML code at the end of your index page (but before ) : +# +# +# +# +# If code is not added in index page, all those detection capabilities will be +# disabled. You must also check that ShowScreenSizeStats and ShowMiscStats +# parameters are set to 1 to make results appear in AWStats report page. +# If you want to use another directory than /js/, you must also change the +# awstatsmisctrackerurl variable into the awstats_misc_tracker.js file. +# Change : Effective for new updates only. +# Possible value: URL of javascript tracker file added in your HTML code. +# Default: "/js/awstats_misc_tracker.js" +# +MiscTrackerUrl="/js/awstats_misc_tracker.js" + + + +#----------------------------------------------------------------------------- +# OPTIONAL ACCURACY SETUP SECTION (Not required but increase AWStats features) +#----------------------------------------------------------------------------- + +# The following values allow you to define accuracy of AWStats entities +# (robots, browsers, os, referers, file types) detection. +# It might be a good idea for large web sites or ISP that provides AWStats to +# high number of customers, to set this parameter to 1 (or 0), instead of 2. +# Possible values: +# 0 = No detection, +# 1 = Medium/Standard detection +# 2 = Full detection +# Change : Effective for new updates only +# Note : LevelForBrowsersDetection can also accept value "allphones". This +# enable detailed detection of phone/pda browsers. +# Default: 2 (0 for LevelForWormsDetection) +# +LevelForBrowsersDetection=2 # 0 disables Browsers detection. + # 2 reduces AWStats speed by 2% + # allphones reduces AWStats speed by 5% +LevelForOSDetection=2 # 0 disables OS detection. + # 2 reduces AWStats speed by 3% +LevelForRefererAnalyze=2 # 0 disables Origin detection. + # 2 reduces AWStats speed by 14% +LevelForRobotsDetection=2 # 0 disables Robots detection. + # 2 reduces AWStats speed by 2.5% +LevelForSearchEnginesDetection=2 # 0 disables Search engines detection. + # 2 reduces AWStats speed by 9% +LevelForKeywordsDetection=2 # 0 disables Keyphrases/Keywords detection. + # 2 reduces AWStats speed by 1% +LevelForFileTypesDetection=2 # 0 disables File types detection. + # 2 reduces AWStats speed by 1% +LevelForWormsDetection=0 # 0 disables Worms detection. + # 2 reduces AWStats speed by 15% + + + +#----------------------------------------------------------------------------- +# OPTIONAL APPEARANCE SETUP SECTION (Not required but increase AWStats features) +#----------------------------------------------------------------------------- + +# When you use AWStats as a CGI, you can have the reports shown in HTML frames. +# Frames are only available for report viewed dynamically. When you build +# pages from command line, this option is not used and no frames are built. +# Possible values: 0 or 1 +# Default: 1 +# +UseFramesWhenCGI=1 + + +# This parameter asks your browser to open detailed reports into a different +# window than the main page. +# Possible values: +# 0 - Open all in same browser window +# 1 - Open detailed reports in another window except if using frames +# 2 - Open always in a different window even if reports are framed +# Default: 1 +# +DetailedReportsOnNewWindows=1 + + +# You can add, in the HTML report page, a cache lifetime (in seconds) that +# will be returned to the browser in HTTP header answer by server. +# This parameter is not used when reports are built with -staticlinks option. +# Example: 3600 +# Default: 0 +# +Expires=3600 + + +# To avoid too large web pages, you can ask AWStats to limit number of rows of +# all reported charts to this number when no other limits apply. +# Default: 1000 +# +MaxRowsInHTMLOutput=1000 + + +# Set your primary language (ISO-639-1 language codes). +# Possible values: +# Albanian=al, Bosnian=ba, Bulgarian=bg, Catalan=ca, +# Chinese (Taiwan)=tw, Chinese (Simpliefied)=cn, Croatian=hr, Czech=cz, +# Danish=dk, Dutch=nl, English=en, Estonian=et, Euskara=eu, Finnish=fi, +# French=fr, Galician=gl, German=de, Greek=gr, Hebrew=he, Hungarian=hu, +# Icelandic=is, Indonesian=id, Italian=it, Japanese=jp, Korean=ko, +# Latvian=lv, Norwegian (Nynorsk)=nn, Norwegian (Bokmal)=nb, Polish=pl, +# Portuguese=pt, Portuguese (Brazilian)=br, Romanian=ro, Russian=ru, +# Serbian=sr, Slovak=sk, Slovenian=si, Spanish=es, Swedish=se, Turkish=tr, +# Ukrainian=ua, Welsh=cy. +# First available language accepted by browser=auto +# Default: "auto" +# +Lang="auto" + + +# Set the location of language files. +# Example: "/usr/share/awstats/lang" +# Default: "./lang" (means lang directory is in same location than awstats.pl) +# +DirLang="./lang" + + +# Show menu header with reports' links +# Possible values: 0 or 1 +# Default: 1 +# +ShowMenu=1 + + +# You choose here which reports you want to see in the main page and what you +# want to see in those reports. +# Possible values: +# 0 - Report is not shown at all +# 1 - Report is shown in main page with an entry in menu and default columns +# XYZ - Report shows column informations defined by code X,Y,Z... +# X,Y,Z... are code letters among the following: +# U = Unique visitors +# V = Visits +# P = Number of pages +# H = Number of hits (or mails) +# B = Bandwith (or total mail size for mail logs) +# L = Last access date +# E = Entry pages +# X = Exit pages +# C = Web compression (mod_gzip,mod_deflate) +# M = Average mail size (mail logs) +# + +# Show monthly summary +# Context: Web, Streaming, Mail, Ftp +# Default: UVPHB, Possible column codes: UVPHB +ShowSummary=UVPHB + +# Show monthly chart +# Context: Web, Streaming, Mail, Ftp +# Default: UVPHB, Possible column codes: UVPHB +ShowMonthStats=UVPHB + +# Show days of month chart +# Context: Web, Streaming, Mail, Ftp +# Default: VPHB, Possible column codes: VPHB +ShowDaysOfMonthStats=VPHB + +# Show days of week chart +# Context: Web, Streaming, Mail, Ftp +# Default: PHB, Possible column codes: PHB +ShowDaysOfWeekStats=PHB + +# Show hourly chart +# Context: Web, Streaming, Mail, Ftp +# Default: PHB, Possible column codes: PHB +ShowHoursStats=PHB + +# Show domains/country chart +# Context: Web, Streaming, Mail, Ftp +# Default: PHB, Possible column codes: PHB +ShowDomainsStats=PHB + +# Show hosts chart +# Context: Web, Streaming, Mail, Ftp +# Default: PHBL, Possible column codes: PHBL +ShowHostsStats=PHBL + +# Show authenticated users chart +# Context: Web, Streaming, Ftp +# Default: 0, Possible column codes: PHBL +ShowAuthenticatedUsers=0 + +# Show robots chart +# Context: Web, Streaming +# Default: HBL, Possible column codes: HBL +ShowRobotsStats=HBL + +# Show worms chart +# Context: Web, Streaming +# Default: 0 (If set to other than 0, see also LevelForWormsDetection), Possible column codes: HBL +ShowWormsStats=0 + +# Show email senders chart (For use when analyzing mail log files) +# Context: Mail +# Default: 0, Possible column codes: HBML +ShowEMailSenders=0 + +# Show email receivers chart (For use when analyzing mail log files) +# Context: Mail +# Default: 0, Possible column codes: HBML +ShowEMailReceivers=0 + +# Show session chart +# Context: Web, Streaming, Ftp +# Default: 1, Possible column codes: None +ShowSessionsStats=1 + +# Show pages-url chart. +# Context: Web, Streaming, Ftp +# Default: PBEX, Possible column codes: PBEX +ShowPagesStats=PBEX + +# Show file types chart. +# Context: Web, Streaming, Ftp +# Default: HB, Possible column codes: HBC +ShowFileTypesStats=HB + +# Show file size chart (Not yet available) +# Context: Web, Streaming, Mail, Ftp +# Default: 1, Possible column codes: None +ShowFileSizesStats=0 + +# Show downloads chart. +# Context: Web, Streaming, Ftp +# Default: HB, Possible column codes: HB +ShowDownloadsStats=HB + +# Show operating systems chart +# Context: Web, Streaming, Ftp +# Default: 1, Possible column codes: None +ShowOSStats=1 + +# Show browsers chart +# Context: Web, Streaming +# Default: 1, Possible column codes: None +ShowBrowsersStats=1 + +# Show screen size chart +# Context: Web, Streaming +# Default: 0 (If set to 1, see also MiscTrackerUrl), Possible column codes: None +ShowScreenSizeStats=0 + +# Show origin chart +# Context: Web, Streaming +# Default: PH, Possible column codes: PH +ShowOriginStats=PH + +# Show keyphrases chart +# Context: Web, Streaming +# Default: 1, Possible column codes: None +ShowKeyphrasesStats=1 + +# Show keywords chart +# Context: Web, Streaming +# Default: 1, Possible column codes: None +ShowKeywordsStats=1 + +# Show misc chart +# Context: Web, Streaming +# Default: a (See also MiscTrackerUrl parameter), Possible column codes: anjdfrqwp +ShowMiscStats=a + +# Show http errors chart +# Context: Web, Streaming +# Default: 1, Possible column codes: None +ShowHTTPErrorsStats=1 + +# Show smtp errors chart (For use when analyzing mail log files) +# Context: Mail +# Default: 0, Possible column codes: None +ShowSMTPErrorsStats=0 + +# Show the cluster report (Your LogFormat must contains the %cluster tag) +# Context: Web, Streaming, Ftp +# Default: 0, Possible column codes: PHB +ShowClusterStats=0 + + +# Some graphical reports are followed by the data array of values. +# If you don't want this array (to reduce the report size for example), you +# can set thoose options to 0. +# Possible values: 0 or 1 +# Default: 1 +# +# Data array values for the ShowMonthStats report +AddDataArrayMonthStats=1 +# Data array values for the ShowDaysOfMonthStats report +AddDataArrayShowDaysOfMonthStats=1 +# Data array values for the ShowDaysOfWeekStats report +AddDataArrayShowDaysOfWeekStats=1 +# Data array values for the ShowHoursStats report +AddDataArrayShowHoursStats=1 + + +# In the Origin chart, you have stats on where your hits came from. You can +# include hits on pages that come from pages of same sites in this chart. +# Possible values: 0 or 1 +# Default: 0 +# +IncludeInternalLinksInOriginSection=0 + + +# The following parameters can be used to choose the maximum number of lines +# shown for the particular following reports. +# +# Stats by countries/domains +MaxNbOfDomain = 10 +MinHitDomain = 1 +# Stats by hosts +MaxNbOfHostsShown = 10 +MinHitHost = 1 +# Stats by authenticated users +MaxNbOfLoginShown = 10 +MinHitLogin = 1 +# Stats by robots +MaxNbOfRobotShown = 10 +MinHitRobot = 1 +# Stats for Downloads +MaxNbOfDownloadsShown = 10 +MinHitDownloads = 1 +# Stats by pages +MaxNbOfPageShown = 10 +MinHitFile = 1 +# Stats by OS +MaxNbOfOsShown = 10 +MinHitOs = 1 +# Stats by browsers +MaxNbOfBrowsersShown = 10 +MinHitBrowser = 1 +# Stats by screen size +MaxNbOfScreenSizesShown = 5 +MinHitScreenSize = 1 +# Stats by window size (following 2 parameters are not yet used) +MaxNbOfWindowSizesShown = 5 +MinHitWindowSize = 1 +# Stats by referers +MaxNbOfRefererShown = 10 +MinHitRefer = 1 +# Stats for keyphrases +MaxNbOfKeyphrasesShown = 10 +MinHitKeyphrase = 1 +# Stats for keywords +MaxNbOfKeywordsShown = 10 +MinHitKeyword = 1 +# Stats for sender or receiver emails +MaxNbOfEMailsShown = 20 +MinHitEMail = 1 + + +# Choose if you want the week report to start on sunday or monday +# Possible values: +# 0 - Week starts on sunday +# 1 - Week starts on monday +# Default: 1 +# +FirstDayOfWeek=1 + + +# List of visible flags that link to other language translations. +# See Lang parameter for list of allowed flag/language codes. +# If you don't want any flag link, set ShowFlagLinks to "". +# This parameter is used only if ShowMenu parameter is set to 1. +# Possible values: "" or "language_codes_separated_by_space" +# Example: "en es fr nl de" +# Default: "" +# +ShowFlagLinks="" + + +# Each URL, shown in stats report views, are links you can click. +# Possible values: 0 or 1 +# Default: 1 +# +ShowLinksOnUrl=1 + + +# When AWStats builds HTML links in its report pages, it starts those links +# with "http://". However some links might be HTTPS links, so you can enter +# here the root of all your HTTPS links. If all your site is a SSL web site, +# just enter "/". +# This parameter is not used if ShowLinksOnUrl is 0. +# Example: "/shopping" +# Example: "/" +# Default: "" +# +UseHTTPSLinkForUrl="" + + +# Maximum length of URL part shown on stats page (number of characters). +# This affects only URL visible text, links still work. +# Default: 64 +# +MaxLengthOfShownURL=64 + + +# You can enter HTML code that will be added at the top of AWStats reports. +# Default: "" +# +HTMLHeadSection="" + + +# You can enter HTML code that will be added at the end of AWStats reports. +# Great to add advert ban. +# Default: "" +# +HTMLEndSection="" + + +# By default AWStats page contains meta tag robots=noindex,nofollow +# If you want to have your statistics to be indexed, set this option to 1. +# Default: 0 +# +MetaRobot=0 + + +# You can set Logo and LogoLink to use your own logo. +# Logo must be the name of image file (must be in $DirIcons/other directory). +# LogoLink is the expected URL when clicking on Logo. +# Default: "awstats_logo6.png" +# +Logo="awstats_logo6.png" +LogoLink="http://awstats.sourceforge.net" + + +# Value of maximum bar width/height for horizontal/vertical HTML graphics bars. +# Default: 260/90 +# +BarWidth = 260 +BarHeight = 90 + + +# You can ask AWStats to use a particular CSS (Cascading Style Sheet) to +# change its look. To create a style sheet, you can use samples provided with +# AWStats in wwwroot/css directory. +# Example: "/awstatscss/awstats_bw.css" +# Example: "/css/awstats_bw.css" +# Default: "" +# +StyleSheet="" + + +# Those color parameters can be used (if StyleSheet parameter is not used) +# to change AWStats look. +# Example: color_name="RRGGBB" # RRGGBB is Red Green Blue components in Hex +# +color_Background="FFFFFF" # Background color for main page (Default = "FFFFFF") +color_TableBGTitle="CCCCDD" # Background color for table title (Default = "CCCCDD") +color_TableTitle="000000" # Table title font color (Default = "000000") +color_TableBG="CCCCDD" # Background color for table (Default = "CCCCDD") +color_TableRowTitle="FFFFFF" # Table row title font color (Default = "FFFFFF") +color_TableBGRowTitle="ECECEC" # Background color for row title (Default = "ECECEC") +color_TableBorder="ECECEC" # Table border color (Default = "ECECEC") +color_text="000000" # Color of text (Default = "000000") +color_textpercent="606060" # Color of text for percent values (Default = "606060") +color_titletext="000000" # Color of text title within colored Title Rows (Default = "000000") +color_weekend="EAEAEA" # Color for week-end days (Default = "EAEAEA") +color_link="0011BB" # Color of HTML links (Default = "0011BB") +color_hover="605040" # Color of HTML on-mouseover links (Default = "605040") +color_u="FFAA66" # Background color for number of unique visitors (Default = "FFAA66") +color_v="F4F090" # Background color for number of visites (Default = "F4F090") +color_p="4477DD" # Background color for number of pages (Default = "4477DD") +color_h="66DDEE" # Background color for number of hits (Default = "66DDEE") +color_k="2EA495" # Background color for number of bytes (Default = "2EA495") +color_s="8888DD" # Background color for number of search (Default = "8888DD") +color_e="CEC2E8" # Background color for number of entry pages (Default = "CEC2E8") +color_x="C1B2E2" # Background color for number of exit pages (Default = "C1B2E2") + + + +#----------------------------------------------------------------------------- +# PLUGINS +#----------------------------------------------------------------------------- + +# Add here all plugin files you want to load. +# Plugin files must be .pm files stored in 'plugins' directory. +# Uncomment LoadPlugin lines to enable a plugin after checking that perl +# modules required by the plugin are installed. + +# PLUGIN: Tooltips +# REQUIRED MODULES: None +# PARAMETERS: None +# DESCRIPTION: Add tooltips pop-up help boxes to HTML report pages. +# NOTE: This will increased HTML report pages size, thus server load and bandwidth. +# +#LoadPlugin="tooltips" + +# PLUGIN: DecodeUTFKeys +# REQUIRED MODULES: Encode and URI::Escape +# PARAMETERS: None +# DESCRIPTION: Allow AWStats to show correctly (in language charset) +# keywords/keyphrases strings even if they were UTF8 coded by the +# referer search engine. +# +#LoadPlugin="decodeutfkeys" + +# PLUGIN: IPv6 +# PARAMETERS: None +# REQUIRED MODULES: Net::IP and Net::DNS +# DESCRIPTION: This plugin gives AWStats capability to make reverse DNS +# lookup on IPv6 addresses. +# +#LoadPlugin="ipv6" + +# PLUGIN: HashFiles +# REQUIRED MODULES: Storable +# PARAMETERS: None +# DESCRIPTION: AWStats DNS cache files are read/saved as native hash files. +# This increases DNS cache files loading speed, above all for very large web sites. +# +#LoadPlugin="hashfiles" + + +# PLUGIN: UserInfo +# REQUIRED MODULES: None +# PARAMETERS: None +# DESCRIPTION: Add a text (Firtname, Lastname, Office Department, ...) in +# authenticated user reports for each login value. +# A text file called userinfo.myconfig.txt, with two fields (first is login, +# second is text to show, separated by a tab char) must be created in DirData +# directory. +# +#LoadPlugin="userinfo" + +# PLUGIN: HostInfo +# REQUIRED MODULES: Net::XWhois +# PARAMETERS: None +# DESCRIPTION: Add a column into host chart with a link to open a popup window that shows +# info on host (like whois records). +# +#LoadPlugin="hostinfo" + +# PLUGIN: ClusterInfo +# REQUIRED MODULES: None +# PARAMETERS: None +# DESCRIPTION: Add a text (for example a full hostname) in cluster reports for each cluster +# number. A text file called clusterinfo.myconfig.txt, with two fields (first is +# cluster number, second is text to show) separated by a tab char. must be +# created into DirData directory. +# Note this plugin is useless if ShowClusterStats is set to 0 or if you don't +# use a personalized log format that contains %cluster tag. +# +#LoadPlugin="clusterinfo" + +# PLUGIN: UrlAliases +# REQUIRED MODULES: None +# PARAMETERS: None +# DESCRIPTION: Add a text (Page title, description...) in URL reports before URL value. +# A text file called urlalias.myconfig.txt, with two fields (first is URL, +# second is text to show, separated by a tab char) must be created into +# DirData directory. +# +#LoadPlugin="urlalias" + +# PLUGIN: TimeHiRes +# REQUIRED MODULES: Time::HiRes (if Perl < 5.8) +# PARAMETERS: None +# DESCRIPTION: Time reported by -showsteps option is in millisecond. For debug purpose. +# +#LoadPlugin="timehires" + +# PLUGIN: TimeZone +# REQUIRED MODULES: Time::Local +# PARAMETERS: [timezone offset] +# DESCRIPTION: Allow AWStats to adjust time stamps for a different timezone +# This plugin reduces AWStats speed of 10% !!!!!!! +# LoadPlugin="timezone" +# LoadPlugin="timezone +2" +# LoadPlugin="timezone CET" +# +#LoadPlugin="timezone +2" + +# PLUGIN: Rawlog +# REQUIRED MODULES: None +# PARAMETERS: None +# DESCRIPTION: This plugin adds a form in AWStats main page to allow users to see raw +# content of current log files. A filter is also available. +# +#LoadPlugin="rawlog" + +# PLUGIN: GraphApplet +# REQUIRED MODULES: None +# PARAMETERS: [CSS classes to override] +# DESCRIPTION: Supported charts are built by a 3D graphic applet. +# +#LoadPlugin="graphapplet /awstatsclasses" # EXPERIMENTAL FEATURE + +# PLUGIN: GraphGoogleChartAPI +# REQUIRED MODULES: None +# PARAMETERS: None +# DESCRIPTION: Replaces the standard charts with free Google API generated images +# in HTML reports. If country data is available and more than one country has hits, +# a map will be generated using Google Visualizations. +# Note: The machine where reports are displayed must have Internet access for the +# charts to be generated. The only data sent to Google includes the statistic numbers, +# legend names and country names. +# Warning: This plugin is not compatible with option BuildReportFormat=xhtml. +# +#LoadPlugin="graphgooglechartapi" + +# PLUGIN: GeoIPfree +# REQUIRED MODULES: Geo::IPfree version 0.2+ (from Graciliano M.P.) +# PARAMETERS: None +# DESCRIPTION: Country chart is built from an Internet IP-Country database. +# This plugin is useless for intranet only log files. +# Note: You must choose between using this plugin (need Perl Geo::IPfree +# module, database is free but not up to date) or the GeoIP plugin (need +# Perl Geo::IP module from Maxmind, database is also free and up to date). +# Note: Activestate provide a corrupted version of Geo::IPfree 0.2 Perl +# module, so install it from elsewhere (from www.cpan.org for example). +# This plugin reduces AWStats speed by up to 10% ! +# +#LoadPlugin="geoipfree" + +# MAXMIND GEO IP MODULES: Please see documentation for notes on all Maxmind modules + +# PLUGIN: GeoIP +# REQUIRED MODULES: Geo::IP or Geo::IP::PurePerl (from Maxmind) +# PARAMETERS: [GEOIP_STANDARD | GEOIP_MEMORY_CACHE] [/pathto/geoip.dat[+/pathto/override.txt]] +# DESCRIPTION: Builds a country chart and adds an entry to the hosts +# table with country name +# Replace spaces in the path of geoip data file with string "%20". +# +#LoadPlugin="geoip GEOIP_STANDARD /pathto/GeoIP.dat" + +# PLUGIN: GeoIP_City_Maxmind +# REQUIRED MODULES: Geo::IP or Geo::IP::PurePerl (from Maxmind) +# PARAMETERS: [GEOIP_STANDARD | GEOIP_MEMORY_CACHE] [/pathto/GeoIPCity.dat[+/pathto/override.txt]] +# DESCRIPTION: This plugin adds a column under the hosts field and tracks the pageviews +# and hits by city including regions. +# Replace spaces in the path of geoip data file with string "%20". +# +#LoadPlugin="geoip_city_maxmind GEOIP_STANDARD /pathto/GeoIPCity.dat" + +# PLUGIN: GeoIP_ASN_Maxmind +# REQUIRED MODULES: Geo::IP or Geo::IP::PurePerl (from Maxmind) +# PARAMETERS: [GEOIP_STANDARD | GEOIP_MEMORY_CACHE] [/pathto/GeoIPASN.dat[+/pathto/override.txt][+http://linktoASlookup]] +# DESCRIPTION: This plugin adds a chart of AS numbers where the host IP address is registered. +# This plugin can display some ISP information if included in the database. You can also provide +# a link that will be used to lookup additional registration data. Put the link at the end of +# the parameter string and the report page will include the link with the full AS number at the end. +# Replace spaces in the path of geoip data file with string "%20". +# +#LoadPlugin="geoip_asn_maxmind GEOIP_STANDARD /usr/local/geoip.dat+http://enc.com.au/itools/aut-num.php?autnum=" + +# PLUGIN: GeoIP_Region_Maxmind +# REQUIRED MODULES: Geo::IP or Geo::IP::PurePerl (from Maxmind) +# PARAMETERS: [GEOIP_STANDARD | GEOIP_MEMORY_CACHE] [/pathto/GeoIPRegion.dat[+/pathto/override.txt]] +# DESCRIPTION:This plugin adds a chart of hits by regions. Only regions for US and +# Canada can be detected. +# Replace spaces in the path of geoip data file with string "%20". +# +#LoadPlugin="geoip_region_maxmind GEOIP_STANDARD /pathto/GeoIPRegion.dat" + +# PLUGIN: GeoIP_ISP_Maxmind +# REQUIRED MODULES: Geo::IP or Geo::IP::PurePerl (from Maxmind) +# PARAMETERS: [GEOIP_STANDARD | GEOIP_MEMORY_CACHE] [/pathto/GeoIPISP.dat[+/pathto/override.txt]] +# DESCRIPTION: This plugin adds a chart of hits by ISP. +# Replace spaces in the path of geoip data file with string "%20". +# +#LoadPlugin="geoip_isp_maxmind GEOIP_STANDARD /pathto/GeoIPISP.dat" + +# PLUGIN: GeoIP_Org_Maxmind +# REQUIRED MODULES: Geo::IP or Geo::IP::PurePerl (from Maxmind) +# PARAMETERS: [GEOIP_STANDARD | GEOIP_MEMORY_CACHE] [/pathto/GeoIPOrg.dat[+/pathto/override.txt]] +# DESCRIPTION: This plugin add a chart of hits by Organization name +# Replace spaces in the path of geoip data file with string "%20". +# +#LoadPlugin="geoip_org_maxmind GEOIP_STANDARD /pathto/GeoIPOrg.dat" + + +#----------------------------------------------------------------------------- +# EXTRA SECTIONS +#----------------------------------------------------------------------------- + +# You can define your own charts, you choose here what are rows and columns +# keys. This feature is particularly useful for marketing purpose, tracking +# products orders for example. +# For this, edit all parameters of Extra section. Each set of parameter is a +# different chart. For several charts, duplicate section changing the number. +# Note: Each Extra section reduces AWStats speed by 8%. +# +# WARNING: A wrong setup of Extra section might result in too large arrays +# that will consume all your memory, making AWStats unusable after several +# updates, so be sure to setup it correctly. +# In most cases, you don't need this feature. +# +# ExtraSectionNameX is title of your personalized chart. +# ExtraSectionCodeFilterX is list of codes the record code field must match. +# Put an empty string for no test on code. +# ExtraSectionConditionX are conditions you can use to count or not the hit, +# Use one of the field condition +# (URL,URLWITHQUERY,QUERY_STRING,REFERER,UA,HOSTINLOG,HOST,VHOST,extraX) +# and a regex to match, after a coma. Use "||" for "OR". +# ExtraSectionFirstColumnTitleX is the first column title of the chart. +# ExtraSectionFirstColumnValuesX is a string to tell AWStats which field to +# extract value from +# (URL,URLWITHQUERY,QUERY_STRING,REFERER,UA,HOSTINLOG,HOST,VHOST,extraX) +# and how to extract the value (using regex syntax). Each different value +# found will appear in first column of report on a different row. Be sure +# that list of different possible values will not grow indefinitely. +# ExtraSectionFirstColumnFormatX is the string used to write value. +# ExtraSectionStatTypesX are things you want to count. You can use standard +# code letters (P for pages,H for hits,B for bandwidth,L for last access). +# ExtraSectionAddAverageRowX add a row at bottom of chart with average values. +# ExtraSectionAddSumRowX add a row at bottom of chart with sum values. +# MaxNbOfExtraX is maximum number of rows shown in chart. +# MinHitExtraX is minimum number of hits required to be shown in chart. +# + +# Example to report the 20 products the most ordered by "order.cgi" script +#ExtraSectionName1="Product orders" +#ExtraSectionCodeFilter1="200 304" +#ExtraSectionCondition1="URL,\/cgi\-bin\/order\.cgi||URL,\/cgi\-bin\/order2\.cgi" +#ExtraSectionFirstColumnTitle1="Product ID" +#ExtraSectionFirstColumnValues1="QUERY_STRING,productid=([^&]+)" +#ExtraSectionFirstColumnFormat1="%s" +#ExtraSectionStatTypes1=PL +#ExtraSectionAddAverageRow1=0 +#ExtraSectionAddSumRow1=1 +#MaxNbOfExtra1=20 +#MinHitExtra1=1 + + +# There is also a global parameter ExtraTrackedRowsLimit that limits the +# number of possible rows an ExtraSection can report. This parameter is +# here to protect too much memory use when you make a bad setup in your +# ExtraSection. It applies to all ExtraSection independently meaning that +# none ExtraSection can report more rows than value defined by ExtraTrackedRowsLimit. +# If you know an ExtraSection will report more rows than its value, you should +# increase this parameter or AWStats will stop with an error. +# Example: 2000 +# Default: 500 +# +ExtraTrackedRowsLimit=500 + + +#----------------------------------------------------------------------------- +# INCLUDES +#----------------------------------------------------------------------------- + +# You can include other config files using the directive with the name of the +# config file. +# This is particularly useful for users who have a lot of virtual servers, so +# a lot of config files and want to maintain common values in only one file. +# Note that when a variable is defined both in a config file and in an +# included file, AWStats will use the last value read for parameters that +# contains one value and AWStats will concat all values from both files for +# parameters that are lists of values. +# + +#Include "" + diff --git a/configuration/environments_scripts/central_reverse_proxy/files/etc/awstats/awstats.www.sapsailing.com.conf b/configuration/environments_scripts/central_reverse_proxy/files/etc/awstats/awstats.www.sapsailing.com.conf new file mode 100644 index 00000000000..2b669ada427 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/etc/awstats/awstats.www.sapsailing.com.conf @@ -0,0 +1,1572 @@ +# AWSTATS CONFIGURE FILE 7.0 +#----------------------------------------------------------------------------- +# Copy this file into awstats.www.mydomain.conf and edit this new config file +# to setup AWStats (See documentation in docs/ directory). +# The config file must be in /etc/awstats, /usr/local/etc/awstats or /etc (for +# Unix/Linux) or same directory than awstats.pl (Windows, Mac, Unix/Linux...) +# To include an environment variable in any parameter (AWStats will replace +# it with its value when reading it), follow the example: +# Parameter="__ENVNAME__" +# Note that environment variable AWSTATS_CURRENT_CONFIG is always defined with +# the config value in an AWStats running session and can be used like others. +#----------------------------------------------------------------------------- +# $Revision: 1.350 $ - $Author: eldy $ - $Date: 2010/09/29 19:16:21 $ + + + +#----------------------------------------------------------------------------- +# MAIN SETUP SECTION (Required to make AWStats work) +#----------------------------------------------------------------------------- + +# "LogFile" contains the web, ftp or mail server log file to analyze. +# Possible values: A full path, or a relative path from awstats.pl directory. +# Example: "/var/log/apache/access.log" +# Example: "../logs/mycombinedlog.log" +# You can also use tags in this filename if you need a dynamic file name +# depending on date or time (Replacement is made by AWStats at the beginning +# of its execution). This is available tags : +# %YYYY-n is replaced with 4 digits year we were n hours ago +# %YY-n is replaced with 2 digits year we were n hours ago +# %MM-n is replaced with 2 digits month we were n hours ago +# %MO-n is replaced with 3 letters month we were n hours ago +# %DD-n is replaced with day we were n hours ago +# %HH-n is replaced with hour we were n hours ago +# %NS-n is replaced with number of seconds at 00:00 since 1970 +# %WM-n is replaced with the week number in month (1-5) +# %Wm-n is replaced with the week number in month (0-4) +# %WY-n is replaced with the week number in year (01-52) +# %Wy-n is replaced with the week number in year (00-51) +# %DW-n is replaced with the day number in week (1-7, 1=sunday) +# use n=24 if you need (1-7, 1=monday) +# %Dw-n is replaced with the day number in week (0-6, 0=sunday) +# use n=24 if you need (0-6, 0=monday) +# Use 0 for n if you need current year, month, day, hour... +# Example: "/var/log/access_log.%YYYY-0%MM-0%DD-0.log" +# Example: "C:/WINNT/system32/LogFiles/W3SVC1/ex%YY-24%MM-24%DD-24.log" +# You can also use a pipe if log file come from a pipe : +# Example: "gzip -d outputpath/output.html"), enter +# path of icon directory relative to the output directory 'outputpath'. +# Example: "/awstatsicons" +# Example: "../icon" +# Default: "/icon" (means you must copy icon directories in "/mywwwroot/icon") +# +DirIcons="/awstatsicons" + + +# When this parameter is set to 1, AWStats adds a button on report page to +# allow to "update" statistics from a web browser. Warning, when "update" is +# made from a browser, AWStats is run as a CGI by the web server user defined +# in your web server (user "nobody" by default with Apache, "IUSR_XXX" with +# IIS), so the "DirData" directory and all already existing history files +# awstatsMMYYYY[.xxx].txt must be writable by this user. Change permissions if +# necessary to "Read/Write" (and "Modify" for Windows NTFS file systems). +# Warning: Update process can be long so you might experience "time out" +# browser errors if you don't launch AWStats frequently enough. +# When set to 0, update is only made when AWStats is run from the command +# line interface (or a task scheduler). +# Possible values: 0 or 1 +# Default: 0 +# +AllowToUpdateStatsFromBrowser=0 + + +# AWStats saves and sorts its database on a month basis (except if using +# databasebreak option from command line). +# However, if you choose the -month=all from command line or +# value '-Year-' from CGI combo form to have a report for all year, AWStats +# needs to reload all data for full year (each month), and sort them, +# requiring a large amount of time, memory and CPU. This might be a problem +# for web hosting providers that offer AWStats for large sites, on shared +# servers, to non CPU cautious customers. +# For this reason, the 'full year' is only enabled on Command Line by default. +# You can change this by setting this parameter to 0, 1, 2 or 3. +# Possible values: +# 0 - Never allowed +# 1 - Allowed on CLI only, -Year- value in combo is not visible +# 2 - Allowed on CLI only, -Year- value in combo is visible but not allowed +# 3 - Possible on CLI and CGI +# Default: 2 +# +AllowFullYearView=3 + + + +#----------------------------------------------------------------------------- +# OPTIONAL SETUP SECTION (Not required but increase AWStats features) +#----------------------------------------------------------------------------- + +# When the update process runs, AWStats can set a lock file in TEMP or TMP +# directory. This lock is to avoid to have 2 update processes running at the +# same time to prevent unknown conflicts problems and avoid DoS attacks when +# AllowToUpdateStatsFromBrowser is set to 1. +# Because, when you use lock file, you can experience sometimes problems in +# lock file not correctly removed (killed process for example requires that +# you remove the file manualy), this option is not enabled by default (Do +# not enable this option with no console server access). +# Change : Effective immediatly +# Possible values: 0 or 1 +# Default: 0 +# +EnableLockForUpdate=1 + + +# AWStats can do reverse DNS lookups through a static DNS cache file that was +# previously created manually. If no path is given in static DNS cache file +# name, AWStats will search DirData directory. This file is never changed. +# This option is not used if DNSLookup=0. +# Note: DNS cache file format is 'minsince1970 ipaddress resolved_hostname' +# or just 'ipaddress resolved_hostname' +# Change : Effective for new updates only +# Example: "/mydnscachedir/dnscache" +# Default: "dnscache.txt" +# +DNSStaticCacheFile="dnscache.txt" + + +# AWStats can do reverse DNS lookups through a DNS cache file that was created +# by a previous run of AWStats. This file is erased and recreated after each +# statistics update process. You don't need to create and/or edit it. +# AWStats will read and save this file in DirData directory. +# This option is used only if DNSLookup=1. +# Note: If a DNSStaticCacheFile is available, AWStats will check for DNS +# lookup in DNSLastUpdateCacheFile after checking into DNSStaticCacheFile. +# Change : Effective for new updates only +# Example: "/mydnscachedir/dnscachelastupdate" +# Default: "dnscachelastupdate.txt" +# +DNSLastUpdateCacheFile="dnscachelastupdate.txt" + + +# You can specify specific IP addresses that should NOT be looked up in DNS. +# This option is used only if DNSLookup=1. +# Note: Use space between each value. +# Note: You can use regular expression values writing value with REGEX[value]. +# Change : Effective for new updates only +# Example: "123.123.123.123 REGEX[^192\.168\.]" +# Default: "" +# +SkipDNSLookupFor="" + + +# The following two parameters allow you to protect a config file from being +# read by AWStats when called from a browser if web user has not been +# authenticated. Your AWStats program must be in a web protected "realm" (With +# Apache, you can use .htaccess files to do so. With other web servers, see +# your server setup manual). +# Change : Effective immediatly +# Possible values: 0 or 1 +# Default: 0 +# +AllowAccessFromWebToAuthenticatedUsersOnly=0 + + +# This parameter gives the list of all authorized authenticated users to view +# statistics for this domain/config file. This parameter is used only if +# AllowAccessFromWebToAuthenticatedUsersOnly is set to 1. +# Change : Effective immediatly +# Example: "user1 user2" +# Example: "__REMOTE_USER__" +# Default: "" +# +AllowAccessFromWebToFollowingAuthenticatedUsers="" + + +# When this parameter is defined to something, the IP address of the user that +# reads its statistics from a browser (when AWStats is used as a CGI) is +# checked and must match one of the IP address values or ranges. +# Change : Effective immediatly +# Example: "127.0.0.1 123.123.123.1-123.123.123.255" +# Default: "" +# +AllowAccessFromWebToFollowingIPAddresses="" + + +# If the "DirData" directory (see above) does not exist, AWStats return an +# error. However, you can ask AWStats to create it. +# This option can be used by some Web Hosting Providers that has defined a +# dynamic value for DirData (for example DirData="/home/__REMOTE_USER__") and +# don't want to have to create a new directory each time they add a new user. +# Change : Effective immediatly +# Possible values: 0 or 1 +# Default: 0 +# +CreateDirDataIfNotExists=0 + + +# You can choose in which format the Awstats history database is saved. +# Note: Using "xml" format make AWStats building database files three times +# larger than using "text" format. +# Change : Database format is switched after next update +# Possible values: text or xml +# Default: text +# +BuildHistoryFormat=text + + +# If you prefer having the report output pages be built as XML compliant pages +# instead of simple HTML pages, you can set this to 'xhtml' (May not work +# properly with old browsers). +# Change : Effective immediatly +# Possible values: html or xhtml +# Default: html +# +BuildReportFormat=html + + +# AWStats databases can be updated from command line of from a browser (when +# used as a cgi program). So AWStats database files need write permission +# for both command line user and default web server user (nobody for Unix, +# IUSR_xxx for IIS/Windows,...). +# To avoid permission problems between update process (run by an admin user) +# and CGI process (ran by a low level user), AWStats can save its database +# files with read and write permissions for everyone. +# By default, AWStats keeps default user permissions on updated files. If you +# set AllowToUpdateStatsFromBrowser to 1, you can change this parameter to 1. +# Change : Effective for new updates only +# Possible values: 0 or 1 +# Default: 0 +# +SaveDatabaseFilesWithPermissionsForEveryone=0 + + +# AWStats can purge log file, after analyzing it. Note that AWStats is able +# to detect new lines in a log file, to process only them, so you can launch +# AWStats as often as you want, even with this parameter to 0. +# With 0, no purge is made, so you must use a scheduled task or a web server +# that make this purge frequently. +# With 1, the purge of the log file is made each time AWStats update is run. +# This parameter doesn't work with IIS (This web server doesn't let its log +# file to be purged). +# Change : Effective for new updates only +# Possible values: 0 or 1 +# Default: 0 +# +PurgeLogFile=0 + + +# When PurgeLogFile is setup to 1, AWStats will clean your log file after +# processing it. You can however keep an archive file of all processed log +# records by setting this parameter (For example if you want to use another +# log analyzer). The archived log file is saved in "DirData" with name +# awstats_archive.configname[.suffix].log +# This parameter is not used if PurgeLogFile=0 +# Change : Effective for new updates only +# Possible values: 0, 1, or tags (See LogFile parameter) for suffix +# Example: 1 +# Example: %YYYY%MM%DD +# Default: 0 +# +ArchiveLogRecords=0 + + +# Each time you run the update process, AWStats overwrites the 'historic file' +# for the month (awstatsMMYYYY[.*].txt) with the updated one. +# When write errors occurs (IO, disk full,...), this historic file can be +# corrupted and must be deleted. Because this file contains information of all +# past processed log files, you will loose old stats if removed. So you can +# ask AWStats to save last non corrupted file in a .bak file. This file is +# stored in "DirData" directory with other 'historic files'. +# Change : Effective for new updates only +# Possible values: 0 or 1 +# Default: 0 +# +KeepBackupOfHistoricFiles=0 + + +# Default index page name for your web server. +# Change : Effective for new updates only +# Example: "index.php index.html default.html" +# Default: "index.php index.html" +# +DefaultFile="index.php index.html" + + +# Do not include access from clients that match following criteria. +# If your log file contains IP addresses in host field, you must enter here +# matching IP addresses criteria. +# If DNS lookup is already done in your log file, you must enter here hostname +# criteria, else enter ip address criteria. +# The opposite parameter of "SkipHosts" is "OnlyHosts". +# Note: Use space between each value. This parameter is not case sensitive. +# Note: You can use regular expression values writing value with REGEX[value]. +# Change : Effective for new updates only +# Example: "127.0.0.1 REGEX[^192\.168\.] REGEX[^10\.]" +# Example: "localhost REGEX[^.*\.localdomain$]" +# Default: "" +# +SkipHosts="" + + +# Do not include access from clients with a user agent that match following +# criteria. If you want to exclude a robot, you should update the robots.pm +# file instead of this parameter. +# The opposite parameter of "SkipUserAgents" is "OnlyUserAgents". +# Note: Use space between each value. This parameter is not case sensitive. +# Note: You can use regular expression values writing value with REGEX[value]. +# Change : Effective for new updates only +# Example: "konqueror REGEX[ua_test_v\d\.\d]" +# Default: "" +# +SkipUserAgents="" + + +# Use SkipFiles to ignore access to URLs that match one of following entries. +# You can enter a list of not important URLs (like framed menus, hidden pages, +# etc...) to exclude them from statistics. You must enter here exact relative +# URL as found in log file, or a matching REGEX value. Check apply on URL with +# all its query paramaters. +# For example, to ignore /badpage.php, just add "/badpage.php". To ignore all +# pages in a particular directory, add "REGEX[^\/directorytoexclude]". +# The opposite parameter of "SkipFiles" is "OnlyFiles". +# Note: Use space between each value. This parameter is or not case sensitive +# depending on URLNotCaseSensitive parameter. +# Note: You can use regular expression values writing value with REGEX[value]. +# Change : Effective for new updates only +# Example: "/badpage.php /page.php?param=x REGEX[^\/excludedirectory]" +# Default: "" +# +SkipFiles="" + + +# Use SkipReferrersBlackList if you want to exclude records coming from a SPAM +# referrer. Parameter must receive a local file name containing rules applied +# on referrer field. If parameter is empty, no filter is applied. +# An example of such a file is available in lib/blacklist.txt +# Change : Effective for new updates only +# Example: "/mylibpath/blacklist.txt" +# Default: "" +# +# WARNING!! Using this feature make AWStats running very slower (5 times slower +# with black list file provided with AWStats ! +# +SkipReferrersBlackList="" + + +# Include in stats, only accesses from hosts that match one of following +# entries. For example, if you want AWStats to filter access to keep only +# stats for visits from particular hosts, you can add those host names in +# this parameter. +# If DNS lookup is already done in your log file, you must enter here hostname +# criteria, else enter ip address criteria. +# The opposite parameter of "OnlyHosts" is "SkipHosts". +# Note: Use space between each value. This parameter is not case sensitive. +# Note: You can use regular expression values writing value with REGEX[value]. +# Change : Effective for new updates only +# Example: "127.0.0.1 REGEX[^192\.168\.] REGEX[^10\.]" +# Default: "" +# +OnlyHosts="" + + +# Include in stats, only accesses from user agent that match one of following +# entries. For example, if you want AWStats to filter access to keep only +# stats for visits from particular browsers, you can add their user agents +# string in this parameter. +# The opposite parameter of "OnlyUserAgents" is "SkipUserAgents". +# Note: Use space between each value. This parameter is not case sensitive. +# Note: You can use regular expression values writing value with REGEX[value]. +# Change : Effective for new updates only +# Example: "msie" +# Default: "" +# +OnlyUserAgents="" + + +# Include in stats, only accesses from authenticated users that match one of +# following entries. For example, if you want AWStats to filter access to keep +# only stats for authenticated users, you can add those users names in +# this parameter. Useful for statistics for per user ftp logs. +# Note: Use space between each value. This parameter is not case sensitive. +# Note: You can use regular expression values writing value with REGEX[value]. +# Change : Effective for new updates only +# Example: "john bob REGEX[^testusers]" +# Default: "" +# +OnlyUsers="" + + +# Include in stats, only accesses to URLs that match one of following entries. +# For example, if you want AWStats to filter access to keep only stats that +# match a particular string, like a particular directory, you can add this +# directory name in this parameter. +# The opposite parameter of "OnlyFiles" is "SkipFiles". +# Note: Use space between each value. This parameter is or not case sensitive +# depending on URLNotCaseSensitive parameter. +# Note: You can use regular expression values writing value with REGEX[value]. +# Change : Effective for new updates only +# Example: "REGEX[marketing_directory] REGEX[office\/.*\.(csv|sxw)$]" +# Default: "" +# +OnlyFiles="" + + +# Add here a list of kind of url (file extension) that must be counted as +# "Hit only" and not as a "Hit" and "Page/Download". You can set here all +# image extensions as they are hit downloaded that must be counted but they +# are not viewed pages. URLs with such extensions are not included in the TOP +# Pages/URL report. +# Note: If you want to exclude particular URLs from stats (No Pages and no +# Hits reported), you must use SkipFiles parameter. +# Change : Effective for new updates only +# Example: "css js class gif jpg jpeg png bmp ico rss xml swf zip arj rar gz z bz2 wav mp3 wma mpg avi" +# Example: "" +# Default: "css js class gif jpg jpeg png bmp ico rss xml swf" +# +NotPageList="css js class gif jpg jpeg png bmp ico rss xml swf" + + +# By default, AWStats considers that records found in web log file are +# successful hits if HTTP code returned by server is a valid HTTP code (200 +# and 304). Any other code are reported in HTTP status chart. +# Note that HTTP 'control codes', like redirection (302, 305) are not added by +# default in this list as they are not pages seen by a visitor but are +# protocol exchange codes to tell the browser to ask another page. Because +# this other page will be counted and seen with a 200 or 304 code, if you +# add such codes, you will have 2 pages viewed reported for only one in facts. +# Change : Effective for new updates only +# Example: "200 304 302 305" +# Default: "200 304" +# +ValidHTTPCodes="200 304" + + +# By default, AWStats considers that records found in mail log file are +# successful mail transfers if field that represent return code in analyzed +# log file match values defined by this parameter. +# Change : Effective for new updates only +# Example: "1 250 200" +# Default: "1 250" +# +ValidSMTPCodes="1 250" + + +# Some web servers on some Operating systems (IIS-Windows) consider that a +# login with same value but different case are the same login. To tell AWStats +# to also consider them as one, set this parameter to 1. +# Change : Effective for new updates only +# Possible values: 0 or 1 +# Default: 0 +# +AuthenticatedUsersNotCaseSensitive=0 + + +# Some web servers on some Operating systems (IIS-Windows) considers that two +# URLs with same value but different case are the same URL. To tell AWStats to +# also considers them as one, set this parameter to 1. +# Change : Effective for new updates only +# Possible values: 0 or 1 +# Default: 0 +# +URLNotCaseSensitive=0 + + +# Keep or remove the anchor string you can find in some URLs. +# Change : Effective for new updates only +# Possible values: 0 or 1 +# Default: 0 +# +URLWithAnchor=0 + + +# In URL links, "?" char is used to add parameter's list in URLs. Syntax is: +# /mypage.html?param1=value1¶m2=value2 +# However, some servers/sites use also other chars to isolate dynamic part of +# their URLs. You can complete this list with all such characters. +# Change : Effective for new updates only +# Example: "?;," +# Default: "?;" +# +URLQuerySeparators="?;" + + +# Keep or remove the query string to the URL in the statistics for individual +# pages. This is primarily used to differentiate between the URLs of dynamic +# pages. If set to 1, mypage.html?id=x and mypage.html?id=y are counted as two +# different pages. +# Warning, when set to 1, memory required to run AWStats is dramatically +# increased if you have a lot of changing URLs (for example URLs with a random +# id inside). Such web sites should not set this option to 1 or use seriously +# the next parameter URLWithQueryWithOnlyFollowingParameters (or eventually +# URLWithQueryWithoutFollowingParameters). +# Change : Effective for new updates only +# Possible values: +# 0 - URLs are cleaned from the query string (ie: "/mypage.html") +# 1 - Full URL with query string is used (ie: "/mypage.html?p=x&q=y") +# Default: 0 +# +URLWithQuery=0 + + +# When URLWithQuery is on, you will get the full URL with all parameters in +# URL reports. But among thoose parameters, sometimes you don't need a +# particular parameter because it does not identify the page or because it's +# a random ID changing for each access even if URL points to same page. In +# such cases, it is higly recommanded to ask AWStats to keep only parameters +# you need (if you know them) before counting, manipulating and storing URL. +# Enter here list of wanted parameters. For example, with "param", one hit on +# /mypage.cgi?param=abc&id=Yo4UomP9d and /mypage.cgi?param=abc&id=Mu8fdxl3r +# will be reported as 2 hits on /mypage.cgi?param=abc +# This parameter is not used when URLWithQuery is 0 and can't be used with +# URLWithQueryWithoutFollowingParameters. +# Change : Effective for new updates only +# Example: "param" +# Default: "" +# +URLWithQueryWithOnlyFollowingParameters="" + + +# When URLWithQuery is on, you will get the full URL with all parameters in +# URL reports. But among thoose parameters, sometimes you don't need a +# particular parameter because it does not identify the page or because it's +# a random ID changing for each access even if URL points to same page. In +# such cases, it is higly recommanded to ask AWStats to remove such parameters +# from the URL before counting, manipulating and storing URL. Enter here list +# of all non wanted parameters. For example if you enter "id", one hit on +# /mypage.cgi?param=abc&id=Yo4UomP9d and /mypage.cgi?param=abc&id=Mu8fdxl3r +# will be reported as 2 hits on /mypage.cgi?param=abc +# This parameter is not used when URLWithQuery is 0 and can't be used with +# URLWithQueryWithOnlyFollowingParameters. +# Change : Effective for new updates only +# Example: "PHPSESSID jsessionid" +# Default: "" +# +URLWithQueryWithoutFollowingParameters="" + + +# Keep or remove the query string to the referrer URL in the statistics for +# external referrer pages. This is used to differentiate between the URLs of +# dynamic referrer pages. If set to 1, mypage.html?id=x and mypage.html?id=y +# are counted as two different referrer pages. +# Change : Effective for new updates only +# Possible values: +# 0 - Referrer URLs are cleaned from the query string (ie: "/mypage.html") +# 1 - Full URL with query string is used (ie: "/mypage.html?p=x&q=y") +# Default: 0 +# +URLReferrerWithQuery=0 + + +# AWStats can detect setup problems or show you important informations to have +# a better use. Keep this to 1, except if AWStats says you can change it. +# Change : Effective immediatly +# Possible values: 0 or 1 +# Default: 1 +# +WarningMessages=1 + + +# When an error occurs, AWStats outputs a message related to errors. If you +# want (in most cases for security reasons) to have no error messages, you +# can set this parameter to your personalized generic message. +# Change : Effective immediatly +# Example: "An error occurred. Contact your Administrator" +# Default: "" +# +ErrorMessages="" + + +# AWStat can be run with debug=x parameter to output various informations +# to help in debugging or solving troubles. If you want to allow this (not +# enabled by default for security reasons), set this parameter to 0. +# Change : Effective immediatly +# Possible values: 0 or 1 +# Default: 0 +# +DebugMessages=0 + + +# To help you to detect if your log format is good, AWStats reports an error +# if all the first NbOfLinesForCorruptedLog lines have a format that does not +# match the LogFormat parameter. +# However, some worm virus attack on your web server can result in a very high +# number of corrupted lines in your log. So if you experience awstats stop +# because of bad virus records at the beginning of your log file, you can +# increase this parameter (very rare). +# Change : Effective for new updates only +# Default: 50 +# +NbOfLinesForCorruptedLog=50 + + +# For some particular integration needs, you may want to have CGI links to +# point to another script than awstats.pl. +# Use the name of this script in WrapperScript parameter. +# Change : Effective immediatly +# Example: "awstatslauncher.pl" +# Example: "awstatswrapper.cgi?key=123" +# Default: "" +# +WrapperScript="" + + +# DecodeUA must be set to 1 if you use Roxen web server. This server converts +# all spaces in user agent field into %20. This make the AWStats robots, OS +# and browsers detection fail in some cases. Just change it to 1 if and only +# if your web server is Roxen. +# Change : Effective for new updates only +# Possible values: 0 or 1 +# Default: 0 +# +DecodeUA=0 + + +# MiscTrackerUrl can be used to make AWStats able to detect some miscellaneous +# things, that can not be tracked on other way, like: +# - Javascript disabled +# - Java enabled +# - Screen size +# - Color depth +# - Macromedia Director plugin +# - Macromedia Shockwave plugin +# - Realplayer G2 plugin +# - QuickTime plugin +# - Mediaplayer plugin +# - Acrobat PDF plugin +# To enable all these features, you must copy the awstats_misc_tracker.js file +# into a /js/ directory stored in your web document root and add the following +# HTML code at the end of your index page (but before ) : +# +# +# +# +# If code is not added in index page, all those detection capabilities will be +# disabled. You must also check that ShowScreenSizeStats and ShowMiscStats +# parameters are set to 1 to make results appear in AWStats report page. +# If you want to use another directory than /js/, you must also change the +# awstatsmisctrackerurl variable into the awstats_misc_tracker.js file. +# Change : Effective for new updates only. +# Possible value: URL of javascript tracker file added in your HTML code. +# Default: "/js/awstats_misc_tracker.js" +# +MiscTrackerUrl="/js/awstats_misc_tracker.js" + + + +#----------------------------------------------------------------------------- +# OPTIONAL ACCURACY SETUP SECTION (Not required but increase AWStats features) +#----------------------------------------------------------------------------- + +# The following values allow you to define accuracy of AWStats entities +# (robots, browsers, os, referers, file types) detection. +# It might be a good idea for large web sites or ISP that provides AWStats to +# high number of customers, to set this parameter to 1 (or 0), instead of 2. +# Possible values: +# 0 = No detection, +# 1 = Medium/Standard detection +# 2 = Full detection +# Change : Effective for new updates only +# Note : LevelForBrowsersDetection can also accept value "allphones". This +# enable detailed detection of phone/pda browsers. +# Default: 2 (0 for LevelForWormsDetection) +# +LevelForBrowsersDetection=2 # 0 disables Browsers detection. + # 2 reduces AWStats speed by 2% + # allphones reduces AWStats speed by 5% +LevelForOSDetection=2 # 0 disables OS detection. + # 2 reduces AWStats speed by 3% +LevelForRefererAnalyze=2 # 0 disables Origin detection. + # 2 reduces AWStats speed by 14% +LevelForRobotsDetection=2 # 0 disables Robots detection. + # 2 reduces AWStats speed by 2.5% +LevelForSearchEnginesDetection=2 # 0 disables Search engines detection. + # 2 reduces AWStats speed by 9% +LevelForKeywordsDetection=2 # 0 disables Keyphrases/Keywords detection. + # 2 reduces AWStats speed by 1% +LevelForFileTypesDetection=2 # 0 disables File types detection. + # 2 reduces AWStats speed by 1% +LevelForWormsDetection=0 # 0 disables Worms detection. + # 2 reduces AWStats speed by 15% + + + +#----------------------------------------------------------------------------- +# OPTIONAL APPEARANCE SETUP SECTION (Not required but increase AWStats features) +#----------------------------------------------------------------------------- + +# When you use AWStats as a CGI, you can have the reports shown in HTML frames. +# Frames are only available for report viewed dynamically. When you build +# pages from command line, this option is not used and no frames are built. +# Possible values: 0 or 1 +# Default: 1 +# +UseFramesWhenCGI=1 + + +# This parameter asks your browser to open detailed reports into a different +# window than the main page. +# Possible values: +# 0 - Open all in same browser window +# 1 - Open detailed reports in another window except if using frames +# 2 - Open always in a different window even if reports are framed +# Default: 1 +# +DetailedReportsOnNewWindows=1 + + +# You can add, in the HTML report page, a cache lifetime (in seconds) that +# will be returned to the browser in HTTP header answer by server. +# This parameter is not used when reports are built with -staticlinks option. +# Example: 3600 +# Default: 0 +# +Expires=3600 + + +# To avoid too large web pages, you can ask AWStats to limit number of rows of +# all reported charts to this number when no other limits apply. +# Default: 1000 +# +MaxRowsInHTMLOutput=1000 + + +# Set your primary language (ISO-639-1 language codes). +# Possible values: +# Albanian=al, Bosnian=ba, Bulgarian=bg, Catalan=ca, +# Chinese (Taiwan)=tw, Chinese (Simpliefied)=cn, Croatian=hr, Czech=cz, +# Danish=dk, Dutch=nl, English=en, Estonian=et, Euskara=eu, Finnish=fi, +# French=fr, Galician=gl, German=de, Greek=gr, Hebrew=he, Hungarian=hu, +# Icelandic=is, Indonesian=id, Italian=it, Japanese=jp, Korean=ko, +# Latvian=lv, Norwegian (Nynorsk)=nn, Norwegian (Bokmal)=nb, Polish=pl, +# Portuguese=pt, Portuguese (Brazilian)=br, Romanian=ro, Russian=ru, +# Serbian=sr, Slovak=sk, Slovenian=si, Spanish=es, Swedish=se, Turkish=tr, +# Ukrainian=ua, Welsh=cy. +# First available language accepted by browser=auto +# Default: "auto" +# +Lang="auto" + + +# Set the location of language files. +# Example: "/usr/share/awstats/lang" +# Default: "./lang" (means lang directory is in same location than awstats.pl) +# +DirLang="./lang" + + +# Show menu header with reports' links +# Possible values: 0 or 1 +# Default: 1 +# +ShowMenu=1 + + +# You choose here which reports you want to see in the main page and what you +# want to see in those reports. +# Possible values: +# 0 - Report is not shown at all +# 1 - Report is shown in main page with an entry in menu and default columns +# XYZ - Report shows column informations defined by code X,Y,Z... +# X,Y,Z... are code letters among the following: +# U = Unique visitors +# V = Visits +# P = Number of pages +# H = Number of hits (or mails) +# B = Bandwith (or total mail size for mail logs) +# L = Last access date +# E = Entry pages +# X = Exit pages +# C = Web compression (mod_gzip,mod_deflate) +# M = Average mail size (mail logs) +# + +# Show monthly summary +# Context: Web, Streaming, Mail, Ftp +# Default: UVPHB, Possible column codes: UVPHB +ShowSummary=UVPHB + +# Show monthly chart +# Context: Web, Streaming, Mail, Ftp +# Default: UVPHB, Possible column codes: UVPHB +ShowMonthStats=UVPHB + +# Show days of month chart +# Context: Web, Streaming, Mail, Ftp +# Default: VPHB, Possible column codes: VPHB +ShowDaysOfMonthStats=VPHB + +# Show days of week chart +# Context: Web, Streaming, Mail, Ftp +# Default: PHB, Possible column codes: PHB +ShowDaysOfWeekStats=PHB + +# Show hourly chart +# Context: Web, Streaming, Mail, Ftp +# Default: PHB, Possible column codes: PHB +ShowHoursStats=PHB + +# Show domains/country chart +# Context: Web, Streaming, Mail, Ftp +# Default: PHB, Possible column codes: PHB +ShowDomainsStats=PHB + +# Show hosts chart +# Context: Web, Streaming, Mail, Ftp +# Default: PHBL, Possible column codes: PHBL +ShowHostsStats=PHBL + +# Show authenticated users chart +# Context: Web, Streaming, Ftp +# Default: 0, Possible column codes: PHBL +ShowAuthenticatedUsers=0 + +# Show robots chart +# Context: Web, Streaming +# Default: HBL, Possible column codes: HBL +ShowRobotsStats=HBL + +# Show worms chart +# Context: Web, Streaming +# Default: 0 (If set to other than 0, see also LevelForWormsDetection), Possible column codes: HBL +ShowWormsStats=0 + +# Show email senders chart (For use when analyzing mail log files) +# Context: Mail +# Default: 0, Possible column codes: HBML +ShowEMailSenders=0 + +# Show email receivers chart (For use when analyzing mail log files) +# Context: Mail +# Default: 0, Possible column codes: HBML +ShowEMailReceivers=0 + +# Show session chart +# Context: Web, Streaming, Ftp +# Default: 1, Possible column codes: None +ShowSessionsStats=1 + +# Show pages-url chart. +# Context: Web, Streaming, Ftp +# Default: PBEX, Possible column codes: PBEX +ShowPagesStats=PBEX + +# Show file types chart. +# Context: Web, Streaming, Ftp +# Default: HB, Possible column codes: HBC +ShowFileTypesStats=HB + +# Show file size chart (Not yet available) +# Context: Web, Streaming, Mail, Ftp +# Default: 1, Possible column codes: None +ShowFileSizesStats=0 + +# Show downloads chart. +# Context: Web, Streaming, Ftp +# Default: HB, Possible column codes: HB +ShowDownloadsStats=HB + +# Show operating systems chart +# Context: Web, Streaming, Ftp +# Default: 1, Possible column codes: None +ShowOSStats=1 + +# Show browsers chart +# Context: Web, Streaming +# Default: 1, Possible column codes: None +ShowBrowsersStats=1 + +# Show screen size chart +# Context: Web, Streaming +# Default: 0 (If set to 1, see also MiscTrackerUrl), Possible column codes: None +ShowScreenSizeStats=0 + +# Show origin chart +# Context: Web, Streaming +# Default: PH, Possible column codes: PH +ShowOriginStats=PH + +# Show keyphrases chart +# Context: Web, Streaming +# Default: 1, Possible column codes: None +ShowKeyphrasesStats=1 + +# Show keywords chart +# Context: Web, Streaming +# Default: 1, Possible column codes: None +ShowKeywordsStats=1 + +# Show misc chart +# Context: Web, Streaming +# Default: a (See also MiscTrackerUrl parameter), Possible column codes: anjdfrqwp +ShowMiscStats=a + +# Show http errors chart +# Context: Web, Streaming +# Default: 1, Possible column codes: None +ShowHTTPErrorsStats=1 + +# Show smtp errors chart (For use when analyzing mail log files) +# Context: Mail +# Default: 0, Possible column codes: None +ShowSMTPErrorsStats=0 + +# Show the cluster report (Your LogFormat must contains the %cluster tag) +# Context: Web, Streaming, Ftp +# Default: 0, Possible column codes: PHB +ShowClusterStats=0 + + +# Some graphical reports are followed by the data array of values. +# If you don't want this array (to reduce the report size for example), you +# can set thoose options to 0. +# Possible values: 0 or 1 +# Default: 1 +# +# Data array values for the ShowMonthStats report +AddDataArrayMonthStats=1 +# Data array values for the ShowDaysOfMonthStats report +AddDataArrayShowDaysOfMonthStats=1 +# Data array values for the ShowDaysOfWeekStats report +AddDataArrayShowDaysOfWeekStats=1 +# Data array values for the ShowHoursStats report +AddDataArrayShowHoursStats=1 + + +# In the Origin chart, you have stats on where your hits came from. You can +# include hits on pages that come from pages of same sites in this chart. +# Possible values: 0 or 1 +# Default: 0 +# +IncludeInternalLinksInOriginSection=0 + + +# The following parameters can be used to choose the maximum number of lines +# shown for the particular following reports. +# +# Stats by countries/domains +MaxNbOfDomain = 10 +MinHitDomain = 1 +# Stats by hosts +MaxNbOfHostsShown = 10 +MinHitHost = 1 +# Stats by authenticated users +MaxNbOfLoginShown = 10 +MinHitLogin = 1 +# Stats by robots +MaxNbOfRobotShown = 10 +MinHitRobot = 1 +# Stats for Downloads +MaxNbOfDownloadsShown = 10 +MinHitDownloads = 1 +# Stats by pages +MaxNbOfPageShown = 10 +MinHitFile = 1 +# Stats by OS +MaxNbOfOsShown = 10 +MinHitOs = 1 +# Stats by browsers +MaxNbOfBrowsersShown = 10 +MinHitBrowser = 1 +# Stats by screen size +MaxNbOfScreenSizesShown = 5 +MinHitScreenSize = 1 +# Stats by window size (following 2 parameters are not yet used) +MaxNbOfWindowSizesShown = 5 +MinHitWindowSize = 1 +# Stats by referers +MaxNbOfRefererShown = 10 +MinHitRefer = 1 +# Stats for keyphrases +MaxNbOfKeyphrasesShown = 10 +MinHitKeyphrase = 1 +# Stats for keywords +MaxNbOfKeywordsShown = 10 +MinHitKeyword = 1 +# Stats for sender or receiver emails +MaxNbOfEMailsShown = 20 +MinHitEMail = 1 + + +# Choose if you want the week report to start on sunday or monday +# Possible values: +# 0 - Week starts on sunday +# 1 - Week starts on monday +# Default: 1 +# +FirstDayOfWeek=1 + + +# List of visible flags that link to other language translations. +# See Lang parameter for list of allowed flag/language codes. +# If you don't want any flag link, set ShowFlagLinks to "". +# This parameter is used only if ShowMenu parameter is set to 1. +# Possible values: "" or "language_codes_separated_by_space" +# Example: "en es fr nl de" +# Default: "" +# +ShowFlagLinks="" + + +# Each URL, shown in stats report views, are links you can click. +# Possible values: 0 or 1 +# Default: 1 +# +ShowLinksOnUrl=1 + + +# When AWStats builds HTML links in its report pages, it starts those links +# with "http://". However some links might be HTTPS links, so you can enter +# here the root of all your HTTPS links. If all your site is a SSL web site, +# just enter "/". +# This parameter is not used if ShowLinksOnUrl is 0. +# Example: "/shopping" +# Example: "/" +# Default: "" +# +UseHTTPSLinkForUrl="" + + +# Maximum length of URL part shown on stats page (number of characters). +# This affects only URL visible text, links still work. +# Default: 64 +# +MaxLengthOfShownURL=64 + + +# You can enter HTML code that will be added at the top of AWStats reports. +# Default: "" +# +HTMLHeadSection="" + + +# You can enter HTML code that will be added at the end of AWStats reports. +# Great to add advert ban. +# Default: "" +# +HTMLEndSection="" + + +# By default AWStats page contains meta tag robots=noindex,nofollow +# If you want to have your statistics to be indexed, set this option to 1. +# Default: 0 +# +MetaRobot=0 + + +# You can set Logo and LogoLink to use your own logo. +# Logo must be the name of image file (must be in $DirIcons/other directory). +# LogoLink is the expected URL when clicking on Logo. +# Default: "awstats_logo6.png" +# +Logo="awstats_logo6.png" +LogoLink="http://awstats.sourceforge.net" + + +# Value of maximum bar width/height for horizontal/vertical HTML graphics bars. +# Default: 260/90 +# +BarWidth = 260 +BarHeight = 90 + + +# You can ask AWStats to use a particular CSS (Cascading Style Sheet) to +# change its look. To create a style sheet, you can use samples provided with +# AWStats in wwwroot/css directory. +# Example: "/awstatscss/awstats_bw.css" +# Example: "/css/awstats_bw.css" +# Default: "" +# +StyleSheet="" + + +# Those color parameters can be used (if StyleSheet parameter is not used) +# to change AWStats look. +# Example: color_name="RRGGBB" # RRGGBB is Red Green Blue components in Hex +# +color_Background="FFFFFF" # Background color for main page (Default = "FFFFFF") +color_TableBGTitle="CCCCDD" # Background color for table title (Default = "CCCCDD") +color_TableTitle="000000" # Table title font color (Default = "000000") +color_TableBG="CCCCDD" # Background color for table (Default = "CCCCDD") +color_TableRowTitle="FFFFFF" # Table row title font color (Default = "FFFFFF") +color_TableBGRowTitle="ECECEC" # Background color for row title (Default = "ECECEC") +color_TableBorder="ECECEC" # Table border color (Default = "ECECEC") +color_text="000000" # Color of text (Default = "000000") +color_textpercent="606060" # Color of text for percent values (Default = "606060") +color_titletext="000000" # Color of text title within colored Title Rows (Default = "000000") +color_weekend="EAEAEA" # Color for week-end days (Default = "EAEAEA") +color_link="0011BB" # Color of HTML links (Default = "0011BB") +color_hover="605040" # Color of HTML on-mouseover links (Default = "605040") +color_u="FFAA66" # Background color for number of unique visitors (Default = "FFAA66") +color_v="F4F090" # Background color for number of visites (Default = "F4F090") +color_p="4477DD" # Background color for number of pages (Default = "4477DD") +color_h="66DDEE" # Background color for number of hits (Default = "66DDEE") +color_k="2EA495" # Background color for number of bytes (Default = "2EA495") +color_s="8888DD" # Background color for number of search (Default = "8888DD") +color_e="CEC2E8" # Background color for number of entry pages (Default = "CEC2E8") +color_x="C1B2E2" # Background color for number of exit pages (Default = "C1B2E2") + + + +#----------------------------------------------------------------------------- +# PLUGINS +#----------------------------------------------------------------------------- + +# Add here all plugin files you want to load. +# Plugin files must be .pm files stored in 'plugins' directory. +# Uncomment LoadPlugin lines to enable a plugin after checking that perl +# modules required by the plugin are installed. + +# PLUGIN: Tooltips +# REQUIRED MODULES: None +# PARAMETERS: None +# DESCRIPTION: Add tooltips pop-up help boxes to HTML report pages. +# NOTE: This will increased HTML report pages size, thus server load and bandwidth. +# +#LoadPlugin="tooltips" + +# PLUGIN: DecodeUTFKeys +# REQUIRED MODULES: Encode and URI::Escape +# PARAMETERS: None +# DESCRIPTION: Allow AWStats to show correctly (in language charset) +# keywords/keyphrases strings even if they were UTF8 coded by the +# referer search engine. +# +#LoadPlugin="decodeutfkeys" + +# PLUGIN: IPv6 +# PARAMETERS: None +# REQUIRED MODULES: Net::IP and Net::DNS +# DESCRIPTION: This plugin gives AWStats capability to make reverse DNS +# lookup on IPv6 addresses. +# +#LoadPlugin="ipv6" + +# PLUGIN: HashFiles +# REQUIRED MODULES: Storable +# PARAMETERS: None +# DESCRIPTION: AWStats DNS cache files are read/saved as native hash files. +# This increases DNS cache files loading speed, above all for very large web sites. +# +#LoadPlugin="hashfiles" + + +# PLUGIN: UserInfo +# REQUIRED MODULES: None +# PARAMETERS: None +# DESCRIPTION: Add a text (Firtname, Lastname, Office Department, ...) in +# authenticated user reports for each login value. +# A text file called userinfo.myconfig.txt, with two fields (first is login, +# second is text to show, separated by a tab char) must be created in DirData +# directory. +# +#LoadPlugin="userinfo" + +# PLUGIN: HostInfo +# REQUIRED MODULES: Net::XWhois +# PARAMETERS: None +# DESCRIPTION: Add a column into host chart with a link to open a popup window that shows +# info on host (like whois records). +# +#LoadPlugin="hostinfo" + +# PLUGIN: ClusterInfo +# REQUIRED MODULES: None +# PARAMETERS: None +# DESCRIPTION: Add a text (for example a full hostname) in cluster reports for each cluster +# number. A text file called clusterinfo.myconfig.txt, with two fields (first is +# cluster number, second is text to show) separated by a tab char. must be +# created into DirData directory. +# Note this plugin is useless if ShowClusterStats is set to 0 or if you don't +# use a personalized log format that contains %cluster tag. +# +#LoadPlugin="clusterinfo" + +# PLUGIN: UrlAliases +# REQUIRED MODULES: None +# PARAMETERS: None +# DESCRIPTION: Add a text (Page title, description...) in URL reports before URL value. +# A text file called urlalias.myconfig.txt, with two fields (first is URL, +# second is text to show, separated by a tab char) must be created into +# DirData directory. +# +#LoadPlugin="urlalias" + +# PLUGIN: TimeHiRes +# REQUIRED MODULES: Time::HiRes (if Perl < 5.8) +# PARAMETERS: None +# DESCRIPTION: Time reported by -showsteps option is in millisecond. For debug purpose. +# +#LoadPlugin="timehires" + +# PLUGIN: TimeZone +# REQUIRED MODULES: Time::Local +# PARAMETERS: [timezone offset] +# DESCRIPTION: Allow AWStats to adjust time stamps for a different timezone +# This plugin reduces AWStats speed of 10% !!!!!!! +# LoadPlugin="timezone" +# LoadPlugin="timezone +2" +# LoadPlugin="timezone CET" +# +#LoadPlugin="timezone +2" + +# PLUGIN: Rawlog +# REQUIRED MODULES: None +# PARAMETERS: None +# DESCRIPTION: This plugin adds a form in AWStats main page to allow users to see raw +# content of current log files. A filter is also available. +# +#LoadPlugin="rawlog" + +# PLUGIN: GraphApplet +# REQUIRED MODULES: None +# PARAMETERS: [CSS classes to override] +# DESCRIPTION: Supported charts are built by a 3D graphic applet. +# +#LoadPlugin="graphapplet /awstatsclasses" # EXPERIMENTAL FEATURE + +# PLUGIN: GraphGoogleChartAPI +# REQUIRED MODULES: None +# PARAMETERS: None +# DESCRIPTION: Replaces the standard charts with free Google API generated images +# in HTML reports. If country data is available and more than one country has hits, +# a map will be generated using Google Visualizations. +# Note: The machine where reports are displayed must have Internet access for the +# charts to be generated. The only data sent to Google includes the statistic numbers, +# legend names and country names. +# Warning: This plugin is not compatible with option BuildReportFormat=xhtml. +# +#LoadPlugin="graphgooglechartapi" + +# PLUGIN: GeoIPfree +# REQUIRED MODULES: Geo::IPfree version 0.2+ (from Graciliano M.P.) +# PARAMETERS: None +# DESCRIPTION: Country chart is built from an Internet IP-Country database. +# This plugin is useless for intranet only log files. +# Note: You must choose between using this plugin (need Perl Geo::IPfree +# module, database is free but not up to date) or the GeoIP plugin (need +# Perl Geo::IP module from Maxmind, database is also free and up to date). +# Note: Activestate provide a corrupted version of Geo::IPfree 0.2 Perl +# module, so install it from elsewhere (from www.cpan.org for example). +# This plugin reduces AWStats speed by up to 10% ! +# +#LoadPlugin="geoipfree" + +# MAXMIND GEO IP MODULES: Please see documentation for notes on all Maxmind modules + +# PLUGIN: GeoIP +# REQUIRED MODULES: Geo::IP or Geo::IP::PurePerl (from Maxmind) +# PARAMETERS: [GEOIP_STANDARD | GEOIP_MEMORY_CACHE] [/pathto/geoip.dat[+/pathto/override.txt]] +# DESCRIPTION: Builds a country chart and adds an entry to the hosts +# table with country name +# Replace spaces in the path of geoip data file with string "%20". +# +#LoadPlugin="geoip GEOIP_STANDARD /pathto/GeoIP.dat" +LoadPlugin="geoip GEOIP_STANDARD /usr/share/GeoIP/GeoIP.dat" + +# PLUGIN: GeoIP_City_Maxmind +# REQUIRED MODULES: Geo::IP or Geo::IP::PurePerl (from Maxmind) +# PARAMETERS: [GEOIP_STANDARD | GEOIP_MEMORY_CACHE] [/pathto/GeoIPCity.dat[+/pathto/override.txt]] +# DESCRIPTION: This plugin adds a column under the hosts field and tracks the pageviews +# and hits by city including regions. +# Replace spaces in the path of geoip data file with string "%20". +# +#LoadPlugin="geoip_city_maxmind GEOIP_STANDARD /pathto/GeoIPCity.dat" + +# PLUGIN: GeoIP_ASN_Maxmind +# REQUIRED MODULES: Geo::IP or Geo::IP::PurePerl (from Maxmind) +# PARAMETERS: [GEOIP_STANDARD | GEOIP_MEMORY_CACHE] [/pathto/GeoIPASN.dat[+/pathto/override.txt][+http://linktoASlookup]] +# DESCRIPTION: This plugin adds a chart of AS numbers where the host IP address is registered. +# This plugin can display some ISP information if included in the database. You can also provide +# a link that will be used to lookup additional registration data. Put the link at the end of +# the parameter string and the report page will include the link with the full AS number at the end. +# Replace spaces in the path of geoip data file with string "%20". +# +#LoadPlugin="geoip_asn_maxmind GEOIP_STANDARD /usr/local/geoip.dat+http://enc.com.au/itools/aut-num.php?autnum=" + +# PLUGIN: GeoIP_Region_Maxmind +# REQUIRED MODULES: Geo::IP or Geo::IP::PurePerl (from Maxmind) +# PARAMETERS: [GEOIP_STANDARD | GEOIP_MEMORY_CACHE] [/pathto/GeoIPRegion.dat[+/pathto/override.txt]] +# DESCRIPTION:This plugin adds a chart of hits by regions. Only regions for US and +# Canada can be detected. +# Replace spaces in the path of geoip data file with string "%20". +# +#LoadPlugin="geoip_region_maxmind GEOIP_STANDARD /pathto/GeoIPRegion.dat" + +# PLUGIN: GeoIP_ISP_Maxmind +# REQUIRED MODULES: Geo::IP or Geo::IP::PurePerl (from Maxmind) +# PARAMETERS: [GEOIP_STANDARD | GEOIP_MEMORY_CACHE] [/pathto/GeoIPISP.dat[+/pathto/override.txt]] +# DESCRIPTION: This plugin adds a chart of hits by ISP. +# Replace spaces in the path of geoip data file with string "%20". +# +#LoadPlugin="geoip_isp_maxmind GEOIP_STANDARD /pathto/GeoIPISP.dat" + +# PLUGIN: GeoIP_Org_Maxmind +# REQUIRED MODULES: Geo::IP or Geo::IP::PurePerl (from Maxmind) +# PARAMETERS: [GEOIP_STANDARD | GEOIP_MEMORY_CACHE] [/pathto/GeoIPOrg.dat[+/pathto/override.txt]] +# DESCRIPTION: This plugin add a chart of hits by Organization name +# Replace spaces in the path of geoip data file with string "%20". +# +#LoadPlugin="geoip_org_maxmind GEOIP_STANDARD /pathto/GeoIPOrg.dat" + + +#----------------------------------------------------------------------------- +# EXTRA SECTIONS +#----------------------------------------------------------------------------- + +# You can define your own charts, you choose here what are rows and columns +# keys. This feature is particularly useful for marketing purpose, tracking +# products orders for example. +# For this, edit all parameters of Extra section. Each set of parameter is a +# different chart. For several charts, duplicate section changing the number. +# Note: Each Extra section reduces AWStats speed by 8%. +# +# WARNING: A wrong setup of Extra section might result in too large arrays +# that will consume all your memory, making AWStats unusable after several +# updates, so be sure to setup it correctly. +# In most cases, you don't need this feature. +# +# ExtraSectionNameX is title of your personalized chart. +# ExtraSectionCodeFilterX is list of codes the record code field must match. +# Put an empty string for no test on code. +# ExtraSectionConditionX are conditions you can use to count or not the hit, +# Use one of the field condition +# (URL,URLWITHQUERY,QUERY_STRING,REFERER,UA,HOSTINLOG,HOST,VHOST,extraX) +# and a regex to match, after a coma. Use "||" for "OR". +# ExtraSectionFirstColumnTitleX is the first column title of the chart. +# ExtraSectionFirstColumnValuesX is a string to tell AWStats which field to +# extract value from +# (URL,URLWITHQUERY,QUERY_STRING,REFERER,UA,HOSTINLOG,HOST,VHOST,extraX) +# and how to extract the value (using regex syntax). Each different value +# found will appear in first column of report on a different row. Be sure +# that list of different possible values will not grow indefinitely. +# ExtraSectionFirstColumnFormatX is the string used to write value. +# ExtraSectionStatTypesX are things you want to count. You can use standard +# code letters (P for pages,H for hits,B for bandwidth,L for last access). +# ExtraSectionAddAverageRowX add a row at bottom of chart with average values. +# ExtraSectionAddSumRowX add a row at bottom of chart with sum values. +# MaxNbOfExtraX is maximum number of rows shown in chart. +# MinHitExtraX is minimum number of hits required to be shown in chart. +# + +# Example to report the 20 products the most ordered by "order.cgi" script +#ExtraSectionName1="Product orders" +#ExtraSectionCodeFilter1="200 304" +#ExtraSectionCondition1="URL,\/cgi\-bin\/order\.cgi||URL,\/cgi\-bin\/order2\.cgi" +#ExtraSectionFirstColumnTitle1="Product ID" +#ExtraSectionFirstColumnValues1="QUERY_STRING,productid=([^&]+)" +#ExtraSectionFirstColumnFormat1="%s" +#ExtraSectionStatTypes1=PL +#ExtraSectionAddAverageRow1=0 +#ExtraSectionAddSumRow1=1 +#MaxNbOfExtra1=20 +#MinHitExtra1=1 + +# Report domain aliases based on virtual host name (Apache %v) collected by %extra1 in the LogFormat directive: +ExtraSectionName1="Domains aliases" +ExtraSectionCodeFilter1="200 304" +ExtraSectionCondition1="" +ExtraSectionFirstColumnTitle1="Domain alias" +ExtraSectionFirstColumnValues1="extra1,([^&]+)" +ExtraSectionFirstColumnFormat1="%s" +ExtraSectionStatTypes1=UVHL +ExtraSectionAddAverageRow1=0 +ExtraSectionAddSumRow1=1 +MaxNbOfExtra1=20 +MinHitExtra1=1 + + +# There is also a global parameter ExtraTrackedRowsLimit that limits the +# number of possible rows an ExtraSection can report. This parameter is +# here to protect too much memory use when you make a bad setup in your +# ExtraSection. It applies to all ExtraSection independently meaning that +# none ExtraSection can report more rows than value defined by ExtraTrackedRowsLimit. +# If you know an ExtraSection will report more rows than its value, you should +# increase this parameter or AWStats will stop with an error. +# Example: 2000 +# Default: 500 +# +ExtraTrackedRowsLimit=500 + + +#----------------------------------------------------------------------------- +# INCLUDES +#----------------------------------------------------------------------------- + +# You can include other config files using the directive with the name of the +# config file. +# This is particularly useful for users who have a lot of virtual servers, so +# a lot of config files and want to maintain common values in only one file. +# Note that when a variable is defined both in a config file and in an +# included file, AWStats will use the last value read for parameters that +# contains one value and AWStats will concat all values from both files for +# parameters that are lists of values. +# + +#Include "" + diff --git a/configuration/environments_scripts/central_reverse_proxy/files/etc/cron.daily/logrotate b/configuration/environments_scripts/central_reverse_proxy/files/etc/cron.daily/logrotate new file mode 100755 index 00000000000..c71ab48ad7e --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/etc/cron.daily/logrotate @@ -0,0 +1,8 @@ +#!/bin/sh + +/usr/sbin/logrotate /etc/logrotate.conf +EXITVALUE=$? +if [ $EXITVALUE != 0 ]; then + /usr/bin/logger -t logrotate "ALERT exited abnormally with [$EXITVALUE]" +fi +exit 0 \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/files/etc/cron.weekly/awstats b/configuration/environments_scripts/central_reverse_proxy/files/etc/cron.weekly/awstats new file mode 100755 index 00000000000..c59c91e855c --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/etc/cron.weekly/awstats @@ -0,0 +1,4 @@ +#!/bin/bash +su -l -c '/usr/share/awstats/tools/awstats_updateall.pl now -configdir="/etc/awstats" -awstatsprog="/usr/share/awstats/wwwroot/cgi-bin/awstats.pl" >>/var/log/awstats-cron.out 2>>/var/log/awstats-cron.err' +#exec /usr/share/awstats/tools/awstats_updateall.pl now -configdir="/etc/awstats" -awstatsprog="/usr/share/awstats/wwwroot/cgi-bin/awstats.pl" >>/var/log/awstats-cron.out 2>>/var/log/awstats-cron.err +exit 0 \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/files/etc/cron.weekly/letsencrypt b/configuration/environments_scripts/central_reverse_proxy/files/etc/cron.weekly/letsencrypt new file mode 100755 index 00000000000..581d840e997 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/etc/cron.weekly/letsencrypt @@ -0,0 +1,4 @@ +#!/bin/bash +# Check if sail-insight.com cert needs renewing +/usr/bin/sudo -u certbot docker run --rm --name certbot -v "/etc/letsencrypt:/etc/letsencrypt" -v "/var/lib/letsencrypt:/var/lib/letsencrypt" -v "/home/trac/sail-insight-website/:/home/trac/sail-insight-website" certbot/certbot renew && service httpd reload +exit 0 \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/files/etc/cron.weekly/unique_ips_per_referrer b/configuration/environments_scripts/central_reverse_proxy/files/etc/cron.weekly/unique_ips_per_referrer new file mode 100755 index 00000000000..2f1125dd2fc --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/etc/cron.weekly/unique_ips_per_referrer @@ -0,0 +1,4 @@ +#!/bin/bash +cd /var/log/old +unique_ips_per_referrer_fast `find . -name 'access_log*'` bundesliga2015_elb_access_log.gz tw2015_elb_access_log.gz >/dev/null 2>/dev/null +exit 0 diff --git a/configuration/environments_scripts/central_reverse_proxy/files/etc/dnf/vars/releasever b/configuration/environments_scripts/central_reverse_proxy/files/etc/dnf/vars/releasever new file mode 100644 index 00000000000..a0f9a4b4bc9 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/etc/dnf/vars/releasever @@ -0,0 +1 @@ +latest diff --git a/configuration/environments_scripts/central_reverse_proxy/files/etc/goaccess.conf b/configuration/environments_scripts/central_reverse_proxy/files/etc/goaccess.conf new file mode 100644 index 00000000000..dba864c3583 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/etc/goaccess.conf @@ -0,0 +1,720 @@ +###################################### +# Time Format Options (required) +###################################### +# +# The hour (24-hour clock) [00,23]; leading zeros are permitted but not required. +# The minute [00,59]; leading zeros are permitted but not required. +# The seconds [00,60]; leading zeros are permitted but not required. +# See `man strftime` for more details +# +# The following time format works with any of the +# Apache/NGINX's log formats below. +# +time-format %H:%M:%S +# +# Google Cloud Storage or +# The time in microseconds since the Unix epoch. +# +#time-format %f + +# Squid native log format +# +#time-format %s + +###################################### +# Date Format Options (required) +###################################### +# +# The date-format variable followed by a space, specifies +# the log format date containing any combination of regular +# characters and special format specifiers. They all begin with a +# percentage (%) sign. See `man strftime` +# +# The following date format works with any of the +# Apache/NGINX's log formats below. +# +date-format %d/%b/%Y +# +# AWS | Amazon CloudFront (Download Distribution) +# AWS | Elastic Load Balancing +# W3C (IIS) +# +#date-format %Y-%m-%d +# +# Google Cloud Storage or +# The time in microseconds since the Unix epoch. +# +#date-format %f + +# Squid native log format +# +#date-format %s + +###################################### +# Log Format Options (required) +###################################### +# +# The log-format variable followed by a space or \t for +# tab-delimited, specifies the log format string. +# +# NOTE: If the time/date is a timestamp in seconds or microseconds +# %x must be used instead of %d & %t to represent the date & time. + +# NCSA Combined Log Format +#log-format %h %^[%d:%t %^] "%r" %s %b "%R" "%u" +# NCSA Combined with virtual host name as prefix: +log-format %v %h %^[%d:%t %^] "%r" %s %b "%R" "%u" + +# NCSA Combined Log Format with Virtual Host +#log-format %v:%^ %h %^[%d:%t %^] "%r" %s %b "%R" "%u" + +# Common Log Format (CLF) +#log-format %h %^[%d:%t %^] "%r" %s %b + +# Common Log Format (CLF) with Virtual Host +#log-format %v:%^ %h %^[%d:%t %^] "%r" %s %b + +# W3C +#log-format %d %t %h %^ %^ %^ %^ %r %^ %s %b %^ %^ %u %R + +# Squid native log format +#log-format %^ %^ %^ %v %^: %x.%^ %~%L %h %^/%s %b %m %U + +# AWS | Amazon CloudFront (Download Distribution) +#log-format %d\t%t\t%^\t%b\t%h\t%m\t%^\t%r\t%s\t%R\t%u\t%^ + +# Google Cloud Storage +#log-format "%x","%h",%^,%^,"%m","%U","%s",%^,"%b","%D",%^,"%R","%u" + +# AWS | Elastic Load Balancing +#log-format %dT%t.%^ %^ %h:%^ %^ %T %^ %^ %^ %s %^ %b "%r" "%u" + +# AWSS3 | Amazon Simple Storage Service (S3) +#log-format %^[%d:%t %^] %h %^"%r" %s %^ %b %^ %L %^ "%R" "%u" + +# Virtualmin Log Format with Virtual Host +#log-format %h %^ %v %^[%d:%t %^] "%r" %s %b "%R" "%u" + +# In addition to specifying the raw log/date/time formats, for +# simplicity, any of the following predefined log format names can be +# supplied to the log/date/time-format variables. GoAccess can also +# handle one predefined name in one variable and another predefined +# name in another variable. +# +#log-format COMBINED +#log-format VCOMBINED +#log-format COMMON +#log-format VCOMMON +#log-format W3C +#log-format SQUID +#log-format CLOUDFRONT +#log-format CLOUDSTORAGE +#log-format AWSELB +#log-format AWSS3 + +###################################### +# UI Options +###################################### + +# Choose among color schemes +# 1 : Monochrome +# 2 : Green +# 3 : Monokai (if 256-colors supported) +# +#color-scheme 3 + +# Prompt log/date configuration window on program start. +# +config-dialog false + +# Color highlight active panel. +# +hl-header true + +# Specify a custom CSS file in the HTML report. +# +#html-custom-css /path/file.css + +# Specify a custom JS file in the HTML report. +# +#html-custom-js /path/file.js + +# Set default HTML preferences. +# +# NOTE: A valid JSON object is required. +# DO NOT USE A MULTILINE JSON OBJECT. +# The parser will only parse the value next to `html-prefs` (single line) +# It allows the ability to customize each panel plot. See example below. +# +#html-prefs {"theme":"bright","perPage":5,"layout":"horizontal","showTables":true,"visitors":{"plot":{"chartType":"bar"}}} + +# Set HTML report page title and header. +# +#html-report-title My Awesome Web Stats + +# Format JSON output using tabs and newlines. +# +json-pretty-print false + +# Turn off colored output. This is the default output on +# terminals that do not support colors. +# true : for no color output +# false : use color-scheme +# +no-color false + +# Don't write column names in the terminal output. By default, it displays +# column names for each available metric in every panel. +# +no-column-names false + +# Disable summary metrics on the CSV output. +# +no-csv-summary false + +# Disable progress metrics. +# +no-progress false + +# Disable scrolling through panels on TAB. +# +no-tab-scroll false + +# Do not show the last updated field displayed in the HTML generated report. +# +#no-html-last-updated true + +# Enable mouse support on main dashboard. +# +with-mouse false + +# Maximum number of items to show per panel. +# Note: Only the CSV and JSON outputs allow a maximum greater than the +# default value of 366. +# +#max-items 366 + +# Custom colors for the terminal output +# Tailor GoAccess to suit your own tastes. +# +# Color Syntax: +# DEFINITION space/tab colorFG#:colorBG# [[attributes,] PANEL] +# +# FG# = foreground color number [-1...255] (-1 = default terminal color) +# BG# = background color number [-1...255] (-1 = default terminal color) +# +# Optionally: +# +# It is possible to apply color attributes, such as: +# bold,underline,normal,reverse,blink. +# Multiple attributes are comma separated +# +# If desired, it is possible to apply custom colors per panel, that is, a +# metric in the REQUESTS panel can be of color A, while the same metric in the +# BROWSERS panel can be of color B. +# +# The following is a 256 color scheme (hybrid palette) +# +#color COLOR_MTRC_HITS color110:color-1 +#color COLOR_MTRC_VISITORS color173:color-1 +#color COLOR_MTRC_DATA color221:color-1 +#color COLOR_MTRC_BW color167:color-1 +#color COLOR_MTRC_AVGTS color143:color-1 +#color COLOR_MTRC_CUMTS color247:color-1 +#color COLOR_MTRC_MAXTS color186:color-1 +#color COLOR_MTRC_PROT color109:color-1 +#color COLOR_MTRC_MTHD color139:color-1 +#color COLOR_MTRC_HITS_PERC color186:color-1 +#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 +#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 VISITORS +#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 OS +#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 BROWSERS +#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 VISIT_TIMES +#color COLOR_MTRC_VISITORS_PERC color186:color-1 +#color COLOR_MTRC_VISITORS_PERC_MAX color139:color-1 +#color COLOR_PANEL_COLS color243:color-1 +#color COLOR_BARS color250:color-1 +#color COLOR_ERROR color231:color167 +#color COLOR_SELECTED color7:color167 +#color COLOR_PANEL_ACTIVE color7:color237 +#color COLOR_PANEL_HEADER color250:color235 +#color COLOR_PANEL_DESC color242:color-1 +#color COLOR_OVERALL_LBLS color243:color-1 +#color COLOR_OVERALL_VALS color167:color-1 +#color COLOR_OVERALL_PATH color186:color-1 +#color COLOR_ACTIVE_LABEL color139:color235 bold underline +#color COLOR_BG color250:color-1 +#color COLOR_DEFAULT color243:color-1 +#color COLOR_PROGRESS color7:color110 + +###################################### +# Server Options +###################################### + +# Specify IP address to bind server to. +# +#addr 0.0.0.0 + +# Run GoAccess as daemon (if --real-time-html enabled). +# +#daemonize false + +# Ensure clients send the specified origin header upon the WebSocket +# handshake. +# +#origin http://example.org + +# The port to which the connection is being attempted to connect. +# By default GoAccess' WebSocket server listens on port 7890 +# See man page or http://gwsocket.io for details. +# +#port 7890 + +# Enable real-time HTML output. +# +#real-time-html true + +# Path to TLS/SSL certificate. +# Note that ssl-cert and ssl-key need to be used to enable TLS/SSL. +# +#ssl-cert /path/ssl/domain.crt + +# Path to TLS/SSL private key. +# Note that ssl-cert and ssl-key need to be used to enable TLS/SSL. +# +#ssl-key /path/ssl/domain.key + +# URL to which the WebSocket server responds. This is the URL supplied +# to the WebSocket constructor on the client side. +# +# Optionally, it is possible to specify the WebSocket URI scheme, such as ws:// +# or wss:// for unencrypted and encrypted connections. +# e.g., ws-url wss://goaccess.io +# +# If GoAccess is running behind a proxy, you could set the client side +# to connect to a different port by specifying the host followed by a +# colon and the port. +# e.g., ws-url goaccess.io:9999 +# +# By default, it will attempt to connect to localhost. If GoAccess is +# running on a remote server, the host of the remote server should be +# specified here. Also, make sure it is a valid host and NOT an http +# address. +# +#ws-url goaccess.io + +# Path to read named pipe (FIFO). +# +#fifo-in /tmp/wspipein.fifo + +# Path to write named pipe (FIFO). +# +#fifo-in /tmp/wspipeout.fifo + +###################################### +# File Options +###################################### + +# Specify the path to the input log file. If set, it will take +# priority over -f from the command line. +# +#log-file /var/log/apache2/access.log + +# Send all debug messages to the specified file. +# +#debug-file debug.log + +# Specify a custom configuration file to use. If set, it will take +# priority over the global configuration file (if any). +# +#config-file + +# Log invalid requests to the specified file. +# +#invalid-requests + +# Do not load the global configuration file. +# +#no-global-config false + +###################################### +# Parse Options +###################################### + +# Enable a list of user-agents by host. For faster parsing, do not +# enable this flag. +# +agent-list false + +# Enable IP resolver on HTML|JSON|CSV output. +# +with-output-resolver false + +# Exclude an IPv4 or IPv6 from being counted. +# Ranges can be included as well using a dash in between +# the IPs (start-end). +# +#exclude-ip 127.0.0.1 +#exclude-ip 192.168.0.1-192.168.0.100 +#exclude-ip ::1 +#exclude-ip 0:0:0:0:0:ffff:808:804-0:0:0:0:0:ffff:808:808 + +# Include HTTP request method if found. This will create a +# request key containing the request method + the actual request. +# +# [default: yes] +# +http-method yes + +# Include HTTP request protocol if found. This will create a +# request key containing the request protocol + the actual request. +# +# [default: yes] +# +http-protocol yes + +# Write output to stdout given one of the following files and the +# corresponding extension for the output format: +# +# /path/file.csv - Comma-separated values (CSV) +# /path/file.json - JSON (JavaScript Object Notation) +# /path/file.html - HTML +# +#output-format /path/file.html + +# Ignore request's query string. +# i.e., www.google.com/page.htm?query => www.google.com/page.htm +# +# Note: Removing the query string can greatly decrease memory +# consumption, especially on timestamped requests. +# +no-query-string false + +# Disable IP resolver on terminal output. +# +no-term-resolver false + +# Treat non-standard status code 444 as 404. +# +444-as-404 false + +# Add 4xx client errors to the unique visitors count. +# +4xx-to-unique-count false + +# Include static files that contain a query string in the static files +# panel. +# e.g., /fonts/fontawesome-webfont.woff?v=4.0.3 +# +all-static-files false + +# Date specificity. Possible values: `date` (default), or `hr`. +# +#date-spec hr + +# Decode double-encoded values. +# +double-decode false + +# Enable parsing/displaying the given panel. +# +#enable-panel VISITORS +#enable-panel REQUESTS +#enable-panel REQUESTS_STATIC +#enable-panel NOT_FOUND +#enable-panel HOSTS +#enable-panel OS +#enable-panel BROWSERS +#enable-panel VISIT_TIMES +#enable-panel VIRTUAL_HOSTS +#enable-panel REFERRERS +#enable-panel REFERRING_SITES +#enable-panel KEYPHRASES +#enable-panel STATUS_CODES +#enable-panel REMOTE_USER +#enable-panel GEO_LOCATION + +# Hour specificity. Possible values: `hr` (default), or `min` (tenth +# of a minute). +# +#hour-spec min + +# Ignore crawlers from being counted. +# This will ignore robots listed under browsers.c +# Note that it will count them towards the total +# number of requests, but excluded from any of the panels. +# +ignore-crawlers false + +# Parse and display crawlers only. +# This will ignore robots listed under browsers.c +# Note that it will count them towards the total +# number of requests, but excluded from any of the panels. +# +crawlers-only false + +# Ignore parsing and displaying the given panel. +# +#ignore-panel VISITORS +#ignore-panel REQUESTS +#ignore-panel REQUESTS_STATIC +#ignore-panel NOT_FOUND +#ignore-panel HOSTS +#ignore-panel OS +#ignore-panel BROWSERS +#ignore-panel VISIT_TIMES +#ignore-panel VIRTUAL_HOSTS +ignore-panel REFERRERS +#ignore-panel REFERRING_SITES +ignore-panel KEYPHRASES +#ignore-panel STATUS_CODES +#ignore-panel REMOTE_USER +#ignore-panel GEO_LOCATION + +# Ignore referers from being counted. +# This supports wild cards. For instance, +# '*' matches 0 or more characters (including spaces) +# '?' matches exactly one character +# +#ignore-referer *.domain.com +#ignore-referer ww?.domain.* + +# Ignore parsing and displaying one or multiple status code(s) +# +#ignore-status 400 +#ignore-status 502 + +# Number of lines from the access log to test against the provided +# log/date/time format. By default, the parser is set to test 10 +# lines. If set to 0, the parser won't test any lines and will parse +# the whole access log. +# +#num-tests 10 + +# Parse log and exit without outputting data. +# +#process-and-exit false + +# Display real OS names. e.g, Windows XP, Snow Leopard. +# +real-os true + +# Sort panel on initial load. +# Sort options are separated by comma. +# Options are in the form: PANEL,METRIC,ORDER +# +# Available metrics: +# BY_HITS - Sort by hits +# BY_VISITORS - Sort by unique visitors +# BY_DATA - Sort by data +# BY_BW - Sort by bandwidth +# BY_AVGTS - Sort by average time served +# BY_CUMTS - Sort by cumulative time served +# BY_MAXTS - Sort by maximum time served +# BY_PROT - Sort by http protocol +# BY_MTHD - Sort by http method +# Available orders: +# ASC +# DESC +# +#sort-panel VISITORS,BY_DATA,ASC +#sort-panel REQUESTS,BY_HITS,ASC +#sort-panel REQUESTS_STATIC,BY_HITS,ASC +#sort-panel NOT_FOUND,BY_HITS,ASC +#sort-panel HOSTS,BY_HITS,ASC +#sort-panel OS,BY_HITS,ASC +#sort-panel BROWSERS,BY_HITS,ASC +#sort-panel VISIT_TIMES,BY_DATA,DESC +#sort-panel VIRTUAL_HOSTS,BY_HITS,ASC +#sort-panel REFERRERS,BY_HITS,ASC +#sort-panel REFERRING_SITES,BY_HITS,ASC +#sort-panel KEYPHRASES,BY_HITS,ASC +#sort-panel STATUS_CODES,BY_HITS,ASC +#sort-panel REMOTE_USER,BY_HITS,ASC +#sort-panel GEO_LOCATION,BY_HITS,ASC + +# Consider the following extensions as static files +# The actual '.' is required and extensions are case sensitive +# For a full list, uncomment the less common static extensions below. +# +static-file .css +static-file .js +static-file .jpg +static-file .png +static-file .gif +static-file .ico +static-file .jpeg +static-file .pdf +static-file .txt +static-file .csv +static-file .zip +static-file .mp3 +static-file .mp4 +static-file .mpeg +static-file .mpg +static-file .exe +static-file .swf +static-file .woff +static-file .woff2 +static-file .xls +static-file .xlsx +static-file .doc +static-file .docx +static-file .ppt +static-file .pptx +static-file .iso +static-file .gz +static-file .rar +static-file .svg +static-file .bmp +static-file .tar +static-file .tgz +static-file .tiff +static-file .tif +static-file .ttf +static-file .flv +#static-file .less +#static-file .ac3 +#static-file .avi +#static-file .bz2 +#static-file .class +#static-file .cue +#static-file .dae +#static-file .dat +#static-file .dts +#static-file .ejs +#static-file .eot +#static-file .eps +#static-file .img +#static-file .jar +#static-file .map +#static-file .mid +#static-file .midi +#static-file .mkv +#static-file .odp +#static-file .ods +#static-file .odt +#static-file .ogg +#static-file .otf +#static-file .pict +#static-file .pls +#static-file .ps +#static-file .qt +#static-file .rm +#static-file .svgz +#static-file .wav +#static-file .webp + +###################################### +# GeoIP Options +# Only if configured with --enable-geoip +###################################### + +# Standard GeoIP database for less memory usage. +# +#std-geoip false + +# Specify path to GeoIP database file. i.e., GeoLiteCity.dat +# .dat file needs to be downloaded from maxmind.com. +# +# For IPv4 City database: +# wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz +# gunzip GeoLiteCity.dat.gz +# +# For IPv6 City database: +# wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz +# gunzip GeoLiteCityv6.dat.gz +# +# For IPv6 Country database: +# wget -N http://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz +# gunzip GeoIPv6.dat.gz +# +# Note: `geoip-city-data` is an alias of `geoip-database` +# +#geoip-database /usr/local/share/GeoIP/GeoLiteCity.dat + +###################################### +# Tokyo Cabinet Options +# Only if configured with --enable-tcb=btree +###################################### + +# GoAccess has the ability to process logs incrementally through the on-disk +# B+Tree database. +# +# It works in the following way: +# - A data set must be persisted first with --keep-db-files, then the same data +# set can be loaded with --load-from-disk. +# - If new data is passed (piped or through a log file), it will append it to +# the original data set. +# - To preserve the data at all times, --keep-db-files must be used. +# - If --load-from-disk is used without --keep-db-files, database files will be +# deleted upon closing the program. + +# On-disk B+ Tree +# Persist parsed data into disk. This should be set to +# the first dataset prior to use `load-from-disk`. +# Setting it to false will delete all database files +# when exiting the program. +#keep-db-files true + +# On-disk B+ Tree +# Load previously stored data from disk. +# Database files need to exist. See `keep-db-files`. +#load-from-disk false + +# On-disk B+ Tree +# Path where the on-disk database files are stored. +# The default value is the /tmp/ directory +# Note the trailing forward-slash. +# +#db-path /tmp/ + +# On-disk B+ Tree +# Set the size in bytes of the extra mapped memory. +# The default value is 0. +# +#xmmap 0 + +# On-disk B+ Tree +# Max number of leaf nodes to be cached. +# Specifies the maximum number of leaf nodes to be cached. +# If it is not more than 0, the default value is specified. +# The default value is 1024. +# +#cache-lcnum 1024 + +# On-disk B+ Tree +# Specifies the maximum number of non-leaf nodes to be cached. +# If it is not more than 0, the default value is specified. +# The default value is 512. +# +#cache-ncnum 512 + +# On-disk B+ Tree +# Specifies the number of members in each leaf page. +# If it is not more than 0, the default value is specified. +# The default value is 128. +# +#tune-lmemb 128 + +# On-disk B+ Tree +# Specifies the number of members in each non-leaf page. +# If it is not more than 0, the default value is specified. +# The default value is 256. +# +#tune-nmemb 256 + +# On-disk B+ Tree +# Specifies the number of elements of the bucket array. +# If it is not more than 0, the default value is specified. +# The default value is 32749. +# Suggested size of the bucket array is about from 1 to 4 +# times of the number of all pages to be stored. +# +#tune-bnum 32749 + +# On-disk B+ Tree +# Specifies that each page is compressed with ZLIB|BZ2 encoding. +# Disabled by default. +# +#compression zlib diff --git a/configuration/environments_scripts/central_reverse_proxy/files/etc/logrotate.d/httpd b/configuration/environments_scripts/central_reverse_proxy/files/etc/logrotate.d/httpd new file mode 120000 index 00000000000..0f8227a0487 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/etc/logrotate.d/httpd @@ -0,0 +1 @@ +../../../../repo/etc/logrotate.d/httpd \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/files/etc/profile.d/sailing.sh b/configuration/environments_scripts/central_reverse_proxy/files/etc/profile.d/sailing.sh new file mode 100644 index 00000000000..da4d0734299 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/etc/profile.d/sailing.sh @@ -0,0 +1,2 @@ +ulimit -n 30000 +ulimit -u 40000 diff --git a/configuration/environments_scripts/central_reverse_proxy/files/etc/sudoers.d/trac_push_to_github_tools_sap b/configuration/environments_scripts/central_reverse_proxy/files/etc/sudoers.d/trac_push_to_github_tools_sap index 50326631bd3..e0abc98858d 100644 --- a/configuration/environments_scripts/central_reverse_proxy/files/etc/sudoers.d/trac_push_to_github_tools_sap +++ b/configuration/environments_scripts/central_reverse_proxy/files/etc/sudoers.d/trac_push_to_github_tools_sap @@ -1,2 +1,2 @@ -# Enables the trac user, on all hosts, to execute commands as the root user. It can do this without a password but only to run the given command. +# Enables the trac user, on all hosts, to execute commands as the root user. It can do this without a password but only to run the given command. trac ALL = (root) NOPASSWD: /usr/local/bin/push_hyperspace_to_github_tools_sap \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/files/etc/sudoers.d/www-for-wiki-git b/configuration/environments_scripts/central_reverse_proxy/files/etc/sudoers.d/www-for-wiki-git index 64fd1b25fe6..996587cc8df 100644 --- a/configuration/environments_scripts/central_reverse_proxy/files/etc/sudoers.d/www-for-wiki-git +++ b/configuration/environments_scripts/central_reverse_proxy/files/etc/sudoers.d/www-for-wiki-git @@ -1,3 +1,3 @@ -# Enables the apache user, on all hosts, to execute commands as the wiki user. This can be done without a password for -# running git, and all of its commands. +# Enables the apache user, on all hosts, to execute commands as the wiki user. This can be done without a password for +# running git, and all of its commands. This is used for the cgi-bin script on the central. apache ALL = (wiki) NOPASSWD: /usr/bin/git \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/create-internal-status-file.service b/configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/create-internal-status-file.service new file mode 120000 index 00000000000..5fa384f80bd --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/create-internal-status-file.service @@ -0,0 +1 @@ +../../../../../repo/etc/systemd/system/create-internal-status-file.service \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service b/configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service new file mode 120000 index 00000000000..2d5bf6279ff --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service @@ -0,0 +1 @@ +../../../../../repo/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/remove-welcome-conf.service b/configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/remove-welcome-conf.service new file mode 100644 index 00000000000..1900c57d2ef --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/remove-welcome-conf.service @@ -0,0 +1,12 @@ +[Unit] +Description=Removes welcome.conf. +Requires=-.mount network-online.target +After=-.mount network-online.target +Before=httpd.service get-latest-httpd-conf.service +[Install] +RequiredBy=multi-user.target +WantedBy=httpd.service +[Service] +Type=oneshot +RemainAfterExit=true +ExecStart=/bin/bash -c "cd /etc/httpd; [[ -e conf.d/welcome.conf ]] && rm -f conf.d/welcome.conf" \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/tmux-management-panel.service b/configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/tmux-management-panel.service new file mode 100644 index 00000000000..1a968607073 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/tmux-management-panel.service @@ -0,0 +1,12 @@ +[Unit] +Description=Starts up the tmux management panel +Requires=-.mount +After=-.mount + +[Install] +RequiredBy=multi-user.target + +[Service] +Type=oenshot +RemainAfterExit=true +ExecStart=/usr/local/bin/tmuxManagementConsole.sh diff --git a/configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/wiki.service b/configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/wiki.service new file mode 100644 index 00000000000..e017301e866 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/etc/systemd/system/wiki.service @@ -0,0 +1,13 @@ +[Unit] +Description=Starts up the wiki +Requires=-.mount +After=-.mount + +[Install] +RequiredBy=multi-user.target + +[Service] +Type=exec +RemainAfterExit=true +ExecStart=su - wiki -c "cd /home/wiki && rackup -p 4567 /home/wiki/config.ru" +ExecStop=/bin/kill -SIGTERM "$MAINPID" diff --git a/configuration/environments_scripts/central_reverse_proxy/files/etc/tmux.conf b/configuration/environments_scripts/central_reverse_proxy/files/etc/tmux.conf new file mode 100644 index 00000000000..c5ca9764b03 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/etc/tmux.conf @@ -0,0 +1,36 @@ +set-option -g prefix C-b + +# Start numbering at 0 +set -g base-index 0 + +# Allows for faster key repetition +set -s escape-time 0 + +# Set status bar +set -g status-bg black +set -g status-fg white +set -g status-left "" +set -g status-right "#[fg=green]#H" + +# set a short history limit to avoid using too much memory +set -g history-limit 1000 + +# Rather than constraining window size to the maximum size of any client +# connected to the *session*, constrain window size to the maximum size of any +# client connected to *that window*. Much more reasonable. +setw -g aggressive-resize on + +# Allows us to use C-b a to send commands to a TMUX session inside +# another TMUX session +bind-key a send-prefix + +# Activity monitoring +setw -g monitor-activity off +#set -g visual-activity on + +# Example of using a shell command in the status line +#set -g status-right "#[fg=yellow]#(uptime | cut -d ',' -f 2-)" + +# Highlight active window +set-window-option -g window-status-current-style bg=red +set-window-option -g default-shell /bin/bash diff --git a/configuration/environments_scripts/central_reverse_proxy/files/home/httpdConf/repo.git/hooks/post-receive b/configuration/environments_scripts/central_reverse_proxy/files/home/httpdConf/repo.git/hooks/post-receive index 53dbc0a49c4..7fed04c6bde 100755 --- a/configuration/environments_scripts/central_reverse_proxy/files/home/httpdConf/repo.git/hooks/post-receive +++ b/configuration/environments_scripts/central_reverse_proxy/files/home/httpdConf/repo.git/hooks/post-receive @@ -19,7 +19,7 @@ sync_repo() { # Gets all public ips for the instances with the chosen tag and iterates over the IPs. for IP in $(aws ec2 describe-instances --filters Name=tag-key,Values="${1}" | jq -r '.Reservations[].Instances[].PublicIpAddress'); do # strictHostKey... means no authenticity check. The sync-repo... script must be installed in the root user's home. - echo $IP + echo "Getting the latest changes on branch $2 for $IP ($1)" ssh -o "StrictHostKeyChecking=no" -o "ConnectTimeout=10" root@${IP} "cd ~ && sync-repo-and-execute-cmd.sh '${DIR}' '${COMMAND}' '${2}'"; done; } @@ -35,12 +35,12 @@ check_conflict() { merge_main_into_branch() { #$1 Branch name. Assumes you are in the git repo. git checkout "${1}" - check_conflict "$?" "Conflict checking out ${1}" + check_conflict "$?" "Conflict checking out ${1}, in the checked-out workspace." git pull - check_conflict "$?" "Conflict pulling ${1}" - echo "Merging ${MAIN_BRANCH_NAME} into ${1}" + check_conflict "$?" "Conflict pulling ${1}, in the checked-out workspace." + echo "Merging ${MAIN_BRANCH_NAME} into ${1}, in the checked-out workspace." git merge ${MAIN_BRANCH_NAME} - check_conflict "$?" "Conflict merging ${MAIN_BRANCH_NAME} into ${1}" + check_conflict "$?" "Conflict merging ${MAIN_BRANCH_NAME} into ${1}, in the checked-out workspace." git push } while read oldrev newrev refname; do # These vars are passed on stdin to this hook. @@ -55,14 +55,15 @@ while read oldrev newrev refname; do # These vars are passed on stdin to this echo "The central pulls the central branch" sync_repo "${CENTRAL_TAG}" "${CENTRAL_BRANCH_NAME}" elif [[ "$branch_name" == "$MAIN_BRANCH_NAME" ]]; then + echo "Performing manipulation in the checked-out workspace on the central, used for merges and manipulation." cd ~ unset GIT_DIR cd ${CHECKED_OUT_WORKSPACE_NAME} # Get the latest main git checkout ${MAIN_BRANCH_NAME} - check_conflict "$?" "Uncommitted files in the working directory" + check_conflict "$?" "Uncommitted files in the working directory, in the checked-out workspace." git pull - check_conflict "$?" "Merge conflict in main branch" + check_conflict "$?" "Merge conflict in main branch from the checked-out workspace." if [[ "$(git rev-parse ${MAIN_BRANCH_NAME})" != "$(git rev-parse origin/${MAIN_BRANCH_NAME})" ]]; then echo "Merge induced by pull. Pushing latest changes in ${MAIN_BRANCH_NAME} to origin" # In case of a merge from diverging branches. This seems cyclic as we are pushing to the same branch, but should resolve, when "Everything up-to-date". git push @@ -75,4 +76,4 @@ while read oldrev newrev refname; do # These vars are passed on stdin to this fi fi fi -done \ No newline at end of file +done diff --git a/configuration/environments_scripts/central_reverse_proxy/files/home/trac/git/hooks/post-receive b/configuration/environments_scripts/central_reverse_proxy/files/home/trac/git/hooks/post-receive index e5ea95a719d..9595b298680 100644 --- a/configuration/environments_scripts/central_reverse_proxy/files/home/trac/git/hooks/post-receive +++ b/configuration/environments_scripts/central_reverse_proxy/files/home/trac/git/hooks/post-receive @@ -1,11 +1,11 @@ -#!/bin/bash -# This hook executes once for the receive operation. It takes no arguments, but for each ref to be updated it receives on -# standard input a line of the format: -# -# SP SP LF -# -REFS=$( cat ) -curl https://postreceive:ohfa7083.98@hudson.sapsailing.com/git/notifyCommit?url=ssh://trac@sapsailing.com/home/trac/git -if echo "$REFS" | awk '{ print $3; }' | grep "\"; then - sudo /usr/local/bin/push_hyperspace_to_github_tools_sap -fi +#!/bin/bash +# This hook executes once for the receive operation. It takes no arguments, but for each ref to be updated it receives on +# standard input a line of the format: +# +# SP SP LF +# +REFS=$( cat ) +curl https://postreceive:ohfa7083.98@hudson.sapsailing.com/git/notifyCommit?url=ssh://trac@sapsailing.com/home/trac/git +if echo "$REFS" | awk '{ print $3; }' | grep "\"; then + sudo /usr/local/bin/push_hyperspace_to_github_tools_sap +fi diff --git a/configuration/environments_scripts/central_reverse_proxy/files/home/wiki/app.rb b/configuration/environments_scripts/central_reverse_proxy/files/home/wiki/app.rb new file mode 100644 index 00000000000..e53dc8c829c --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/home/wiki/app.rb @@ -0,0 +1,98 @@ +require 'gollum/app' +require 'digest/sha1' + + +#__DIR__ = File.expand_path(File.dirname(__FILE__)) +#$: << __DIR__ +class App < Precious::App + User = Struct.new(:name, :email, :password_hash, :can_write) + before { authenticate! } + before /edit/ do authorize_write! ; end + before do + session['gollum.author'] = { + :name => "%s" % settings.loggedInUser, + :email => "%s@example.com" % settings.loggedInUser, + } + end + + helpers do + def authenticate! + public_urls=IO.readlines 'public.txt' + public_urls.each {|url| + if self.env['PATH_INFO'] == url.slice(0, url.length-1) + puts "Allowing " + url + return + end + + if self.env['PATH_INFO'].start_with?('/wiki/images') || + self.env['PATH_INFO'].start_with?('/favicon.ico') + return + end + } + if self.env['PATH_INFO'].split('/')[1] == 'gollum' && self.env['PATH_INFO'].split('/')[2] == 'assets' + return + end + @_auth = Rack::Auth::Basic::Request.new(request.env) + if self.env['PATH_INFO'].split('/').length >=2 && + (self.env['PATH_INFO'].split('/')[1] != 'wiki' && + self.env['PATH_INFO'].split('/')[2] != 'wiki' && + self.env['PATH_INFO'].split('/')[1] != 'home' && + self.env['PATH_INFO'].split('/')[2] != 'home' && + self.env['PATH_INFO'].split('/')[1] != 'Home' && + self.env['PATH_INFO'].split('/')[2] != 'Home' && + self.env['PATH_INFO'].split('/')[1] != 'Home.md' && + self.env['PATH_INFO'].split('/')[1] != 'search' && + self.env['PATH_INFO'].split('/')[2] != 'search' && + self.env['PATH_INFO'].split('/')[1] != 'edit' && + self.env['PATH_INFO'].split('/')[2] != 'edit' && + self.env['PATH_INFO'].split('/')[1] != 'preview' && + self.env['PATH_INFO'].split('/')[2] != 'preview' && + (self.env['PATH_INFO'].split('/')[1] != 'gollum' || + (self.env['PATH_INFO'].split('/')[2] != 'create' && + (self.env['PATH_INFO'].split('/')[2] != 'overview' || self.env['PATH_INFO'].split('/')[3] != 'wiki')))) + throw(:halt, [403, 'Forbidden - You can not access anything outside wiki/ path.']) + end + if @_auth.provided? + end + if @_auth.provided? && @_auth.basic? && @_auth.credentials && @user = detected_user(@_auth.credentials) + Precious::App.set(:loggedInUser, @user.name) + return @user + else + response['WWW-Authenticate'] = %(Basic realm="Gollum Wiki") + throw(:halt, [401, "Not authorized\n"]) + end + end + + def authorize_write! + throw(:halt, [403, "Forbidden\n"]) unless @user.can_write + end + + def users + @_users ||= settings.authorized_users.map {|u| User.new(*u) } + end + + def detected_user(credentials) + users.detect do |u| + [u.email, u.password_hash] == + [credentials[0], Digest::SHA1.hexdigest(credentials[1])] + end + end + end + + def commit_ + { + :message => params[:message], + # :name => @user.name, + :email => @user.email + } + end +end +##set author +#class Precious::App +# before do +# session['gollum.author'] = { +# :name => "%s" % settings.loggedInUser, +# :email => "%s@example.com" % settings.loggedInUser, +# } +# end +#End diff --git a/configuration/environments_scripts/central_reverse_proxy/files/home/wiki/config.ru b/configuration/environments_scripts/central_reverse_proxy/files/home/wiki/config.ru new file mode 100644 index 00000000000..12f0110af24 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/home/wiki/config.ru @@ -0,0 +1,67 @@ +#!/usr/bin/env ruby +#require 'rubygems' +#require 'gollum/app' +## Define list of authorized users. +## Each user must have a username, password, name and email. +## +## Instead of a password you can also define a password_digest, which is the +## SHA-256 hash of a password. +## +## Example: +#users = YAML.load %q{ +#--- +#- username: rick +# password: asdf754&1129-@lUZw +# name: Rick Sanchez +# email: rick@example.com +#- username: morty +# password_digest: 5994471abb01112afcc18159f6cc74b4f511b99806da59b3caf5a9c173cacfc5 +# name: Morty Smith +# email: morty@example.com +#} +# +## Allow unauthenticated users to read the wiki (disabled by default). +#options = { allow_unauthenticated_readonly: true } +# +## Allow only authenticated users to change the wiki. +## (NOTE: This must be loaded *before* Precious::App!) +#use Gollum::Auth, users, options +# +## That's it. The rest is for gollum only. +#gollum_path = "/home/wiki/gitwiki" +#wiki_options = {:universal_toc => false} +#Precious::App.set(:gollum_path, gollum_path) +#Precious::App.set(:wiki_options, wiki_options) +#run Precious::App + +require 'rubygems' +require 'gollum/app' +require_relative 'app' +#require 'ruby-prof' +#require 'gollum/auth' # Don't forget to load the gem! +Gollum::Page.send :remove_const, :FORMAT_NAMES if defined? Gollum::Page::FORMAT_NAMES +#Gollum::Markup.formats.clear +#Gollum::Markup.formats[:markdown] = { +# :name => "MarkDown", +# :extensions => "md", +# :regexp => /md|mkdn?|mdown|markdown/ +#} + +gollum_path = "/home/wiki/gitwiki" +wiki_options = {:universal_toc => false} +Precious::App.set(:gollum_path, gollum_path) +Precious::App.set(:wiki_options, wiki_options) +Precious::App.set(:authorized_users, YAML.load_file(File.expand_path('users.yml', File.expand_path(File.dirname(__FILE__))))) +Precious::App.set(:loggedInUser, "anonymous"); +App.set(:default_markup, :markdown) # set your favorite markup language +run App + +#require 'rubygems' +#require 'gollum/app' +# +#gollum_path = "/home/wiki/gitwiki" +#wiki_options = {:universal_toc => false} +#Precious::App.set(:gollum_path, gollum_path) +#Precious::App.set(:default_markup, :markdown) # set your favorite markup language +#Precious::App.set(:wiki_options, wiki_options) +#run Precious::App diff --git a/configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/awsmfalogon.sh b/configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/awsmfalogon.sh new file mode 120000 index 00000000000..d002c903654 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/awsmfalogon.sh @@ -0,0 +1 @@ +../../../../../repo/usr/local/bin/awsmfalogon.sh \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/setupHttpdGitLocal.sh b/configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/setupHttpdGitLocal.sh new file mode 120000 index 00000000000..5c2ee809d4b --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/setupHttpdGitLocal.sh @@ -0,0 +1 @@ +../../../../../repo/usr/local/bin/setupHttpdGitLocal.sh \ No newline at end of file diff --git a/configuration/tmuxManagementConsole.sh b/configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/tmuxManagementConsole.sh similarity index 56% rename from configuration/tmuxManagementConsole.sh rename to configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/tmuxManagementConsole.sh index 4d4370a1a56..27b9324fdcc 100755 --- a/configuration/tmuxManagementConsole.sh +++ b/configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/tmuxManagementConsole.sh @@ -14,17 +14,8 @@ else cd /home/trac/git tmux new-session -s "$sn" -n "BUILD" -d - counter=1 - for dir in dev test prod1 prod2; do - cd $SERVERS_DIR/$dir - tmux new-window -t "$sn:$counter" -n `basename $dir` "bash -c './start'; bash" - counter=$[counter + 1] - done + counter=0 - cd /home/trac/servers/prod1 - tmux new-window -t "$sn:$counter" -n "UDP" "bash -c './udpmirror -v 2012 localhost 2010 localhost 2011 localhost 2013 localhost 2014'; bash" - - cd /opt/mongodb/bin tmux new-window -t "$sn:$[counter+1]" -n "GOAccess" "bash -c 'goaccess -f /var/log/httpd/access_log'; bash" cd /opt/ @@ -33,12 +24,6 @@ else cd /home/trac/servers tmux new-window -t "$sn:$[counter+3]" -n "Logs" "bash -c 'ls -lah'; bash" - cd /home/trac/servers/prod1 - tmux new-window -t "$sn:$[counter+4]" -n "STListener" "bash -c './swisstiminglistener 3500 3501'; bash" tmux select-window -t "$sn:0" -fi - -if [[ "$1" != "unattended" ]]; then - tmux -2 attach-session -t "$sn" -fi +fi \ No newline at end of file diff --git a/configuration/unique_ips_per_referrer b/configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/unique_ips_per_referrer similarity index 100% rename from configuration/unique_ips_per_referrer rename to configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/unique_ips_per_referrer diff --git a/configuration/unique_ips_per_referrer_fast b/configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/unique_ips_per_referrer_fast similarity index 100% rename from configuration/unique_ips_per_referrer_fast rename to configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/unique_ips_per_referrer_fast diff --git a/configuration/unique_ips_per_referrer_generate_month_results_only b/configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/unique_ips_per_referrer_generate_month_results_only similarity index 100% rename from configuration/unique_ips_per_referrer_generate_month_results_only rename to configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/unique_ips_per_referrer_generate_month_results_only diff --git a/configuration/unique_ips_per_referrer_generate_results_only b/configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/unique_ips_per_referrer_generate_results_only similarity index 100% rename from configuration/unique_ips_per_referrer_generate_results_only rename to configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/unique_ips_per_referrer_generate_results_only diff --git a/configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/update-tractrac-urls-to-archive.sh b/configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/update-tractrac-urls-to-archive.sh deleted file mode 100755 index 8dc5afa45b7..00000000000 --- a/configuration/environments_scripts/central_reverse_proxy/files/usr/local/bin/update-tractrac-urls-to-archive.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -if [[ $# -eq 0 ]]; then - GIT_ROOT=/home/wiki/gitwiki -else - GIT_ROOT=$1 -fi -PATH_TO_TRAC_TRAC_URLS="configuration/tractrac-json-urls" -urls=$(mongo --quiet "mongodb://dbserver.internal.sapsailing.com:10201/winddb?replicaSet=archive" --eval 'db.TRACTRAC_CONFIGURATIONS.find({}, {TT_CONFIG_JSON_URL : 1}).toArray()' | grep -v ObjectId | jq -r '.[].TT_CONFIG_JSON_URL' ) -if [[ $urls == "null" || $? -ne 0 ]]; then - echo "Mongo db returns null for tractrac url discovery" | notify-operators "MongoDB/tractrac urls issue" - exit 1 -else - echo "${urls}" | sort -u >"${GIT_ROOT}/${PATH_TO_TRAC_TRAC_URLS}" - cd "${GIT_ROOT}" - git add "${GIT_ROOT}/${PATH_TO_TRAC_TRAC_URLS}" - git commit -m "Updated tractrac-json-urls" - git push -fi diff --git a/configuration/environments_scripts/central_reverse_proxy/files/usr/share/bugzilla/data/params.json b/configuration/environments_scripts/central_reverse_proxy/files/usr/share/bugzilla/data/params.json new file mode 100644 index 00000000000..6ac423d1d33 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/usr/share/bugzilla/data/params.json @@ -0,0 +1,104 @@ +{ + "LDAPBaseDN" : "", + "LDAPbinddn" : "", + "LDAPfilter" : "", + "LDAPmailattribute" : "mail", + "LDAPserver" : "", + "LDAPstarttls" : "0", + "LDAPuidattribute" : "uid", + "RADIUS_NAS_IP" : "", + "RADIUS_email_suffix" : "", + "RADIUS_secret" : "", + "RADIUS_server" : "", + "ajax_user_autocompletion" : "1", + "allow_attachment_deletion" : "0", + "allow_attachment_display" : "0", + "allowbugdeletion" : "0", + "allowemailchange" : "1", + "allowuserdeletion" : "0", + "announcehtml" : "", + "attachment_base" : "", + "auth_env_email" : "", + "auth_env_id" : "", + "auth_env_realname" : "", + "chartgroup" : "editbugs", + "collapsed_comment_tags" : "obsolete, spam", + "comment_taggers_group" : "editbugs", + "commentonchange_resolution" : "0", + "commentonduplicate" : "0", + "confirmuniqueusermatch" : "1", + "cookiedomain" : "", + "cookiepath" : "/", + "createemailregexp" : ".*", + "debug_group" : "admin", + "default_search_limit" : "500", + "defaultopsys" : "", + "defaultplatform" : "", + "defaultpriority" : "P5", + "defaultquery" : "resolution=---&emailassigned_to1=1&emailassigned_to2=1&emailreporter2=1&emailcc2=1&emailqa_contact2=1&emaillongdesc3=1&order=Importance&long_desc_type=substring", + "defaultseverity" : "enhancement", + "duplicate_or_move_bug_status" : "RESOLVED", + "emailregexp" : "^[\\w\\.\\+\\-=']+@[\\w\\.\\-]+\\.[\\w\\-]+$", + "emailregexpdesc" : "A legal address must contain exactly one '@', and at least one '.' after the @.", + "emailsuffix" : "", + "font_file" : "", + "globalwatchers" : "", + "inbound_proxies" : "", + "insidergroup" : "", + "last_visit_keep_days" : "10", + "letsubmitterchoosemilestone" : "1", + "letsubmitterchoosepriority" : "1", + "mail_delivery_method" : "SMTP", + "mailfrom" : "bugzilla-daemon@sapsailing.com", + "maintainer" : "", + "makeproductgroups" : "0", + "max_search_results" : "10000", + "maxattachmentsize" : "1024", + "maxlocalattachment" : "0", + "maxusermatches" : "1000", + "memcached_namespace" : "bugzilla:", + "memcached_servers" : "", + "musthavemilestoneonaccept" : "0", + "mybugstemplate" : "buglist.cgi?resolution=---&emailassigned_to1=1&emailreporter1=1&emailtype1=exact&email1=%userid%", + "noresolveonopenblockers" : "0", + "or_groups" : "1", + "password_check_on_login" : "1", + "password_complexity" : "no_constraints", + "proxy_url" : "", + "querysharegroup" : "editbugs", + "quip_list_entry_control" : "open", + "rememberlogin" : "on", + "requirelogin" : "1", + "search_allow_no_criteria" : "1", + "shadowdb" : "", + "shadowdbhost" : "", + "shadowdbport" : "3306", + "shadowdbsock" : "", + "shutdownhtml" : "", + "smtp_debug" : "1", + "smtp_password" : "AkoTN6Upot/Sgc/JJD1VRUbqsM15eQ/5paY4FB/YkEkU", + "smtp_ssl" : "1", + "smtp_username" : "AKIAIHCQEFAZDLIK7SUQ", + "smtpserver" : "email-smtp.eu-west-1.amazonaws.com", + "ssl_redirect" : "0", + "sslbase" : "https://bugzilla.sapsailing.com/bugzilla/", + "strict_isolation" : "0", + "strict_transport_security" : "off", + "timetrackinggroup" : "editbugs", + "upgrade_notification" : "latest_stable_release", + "urlbase" : "https://bugzilla.sapsailing.com/bugzilla/", + "use_mailer_queue" : "0", + "use_see_also" : "1", + "useclassification" : "0", + "usemenuforusers" : "0", + "useqacontact" : "0", + "user_info_class" : "CGI", + "user_verify_class" : "DB", + "usestatuswhiteboard" : "0", + "usetargetmilestone" : "0", + "usevisibilitygroups" : "0", + "utf8" : "1", + "webdotbase" : "", + "webservice_email_filter" : "0", + "whinedays" : "7" +} diff --git a/configuration/environments_scripts/central_reverse_proxy/files/var/www/cgi-bin/github.cgi b/configuration/environments_scripts/central_reverse_proxy/files/var/www/cgi-bin/github.cgi index 75a7a891e7f..4ba836711d5 100755 --- a/configuration/environments_scripts/central_reverse_proxy/files/var/www/cgi-bin/github.cgi +++ b/configuration/environments_scripts/central_reverse_proxy/files/var/www/cgi-bin/github.cgi @@ -5,15 +5,19 @@ BODY=$( cat ) echo "${BODY}" >/tmp/github-hook-body REF=$( echo "${BODY}" | jq -r '.ref' ) PUSHER=$( echo "${BODY}" | jq -r '.pusher.email' ) -logger -t github-cgi "ref ia $REF, pusher was $PUSHER" +logger -t github-cgi "ref is $REF, pusher was $PUSHER" # For testing: #if [ "${PUSHER}" = "axel.uhl@sap.com" -a "${REF}" = "refs/heads/translation" ]; then -if [ "${PUSHER}" = "tmsatsls+github.tools.sap_service-tip-git@sap.com" -a "${REF}" = "refs/heads/translation" ]; then +tmsatsls+github.com_service-tip-git@sap.com +# The filter in case this is to be used for github.tools.sap: +#if [ "${PUSHER}" = "tmsatsls+github.tools.sap_service-tip-git@sap.com" -a "${REF}" = "refs/heads/translation" ]; then +# The filter for github.com: +if [ "${PUSHER}" = "tmsatsls+github.com_service-tip-git@sap.com" -a "${REF}" = "refs/heads/translation" ]; then echo "Identified a push to refs/heads/translation by ${PUSHER}." echo "Fetching translation branch from github.tools.sap and pushing it to ssh://trac@sapsailing.com/home/trac/git" logger -t github-cgi "fetching translation branch from github.tools.sap and pushing it to ssh://trac@sapsailing.com/home/trac/git" cd /home/wiki/gitwiki - sudo -u wiki git fetch sapsailing translation:translation 2>&1 + sudo -u wiki git fetch github translation:translation 2>&1 sudo -u wiki git push origin translation:translation 2>&1 else echo "Either pusher was not tmsatsls+github.tools.sap_service-tip-git@sap.com or ref was not refs/heads/translation. Not triggering." diff --git a/configuration/environments_scripts/central_reverse_proxy/files/var/www/cgi-bin/reverseProxyHealthcheck.sh b/configuration/environments_scripts/central_reverse_proxy/files/var/www/cgi-bin/reverseProxyHealthcheck.sh new file mode 120000 index 00000000000..76d5eae3f23 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/files/var/www/cgi-bin/reverseProxyHealthcheck.sh @@ -0,0 +1 @@ +../../../../../repo/var/www/cgi-bin/reverseProxyHealthcheck.sh \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/groups/static/gid b/configuration/environments_scripts/central_reverse_proxy/groups/static/gid new file mode 100644 index 00000000000..a141b9a0cb0 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/groups/static/gid @@ -0,0 +1 @@ +1006 \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/groups/static/users b/configuration/environments_scripts/central_reverse_proxy/groups/static/users new file mode 100644 index 00000000000..9343471b540 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/groups/static/users @@ -0,0 +1,2 @@ +wiki +trac \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh new file mode 100755 index 00000000000..90fe64f437d --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy-part-2.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# PART 2 +# Assumes to be run after or be invoked by setup-central-reverse-proxy.sh which +# is assumed to have prepared content for user home folders in /root/temporary_home_copy, +# has set up the software packages, and that the user has mounted the volumes with their +# original content to /home, /var/log, et cetera. +# +# Call as follows: +# setup-central-reverse-proxy-part-2.sh {external-ip-of-new-instance} {image-type} +# where {image-type} identifies the "environment" from configuration/environments_scripts +# which usually would be "central_reverse_proxy" here, passed in from the previous stage's +# script. +if [[ "$#" -ne 2 ]]; then + echo "2 arguments required. Check comment of scripts for more details." + exit 2 +fi +IP=$1 +IMAGE_TYPE="$2" +TEMPORARY_HOME_COPY_LOCATION="/root/temporary_home_copy" # home nested within this. +ssh -A "root@${IP}" "bash -s" << EOF +sudo systemctl start crond.service +sudo systemctl start tmux-management-panel.service +. imageupgrade_functions.sh +cp -r "$TEMPORARY_HOME_COPY_LOCATION"/home / +rm -rf "$TEMPORARY_HOME_COPY_LOCATION" +# Localhost works here as we are logged on as root and are using ssh agent forwarding. +if ! build_crontab_and_setup_files -h localhost -f "${IMAGE_TYPE}"; then # files have already been copied so -f is used. + exit 1 +fi +setup_keys -p "${IMAGE_TYPE}" +chown trac:static /var/www/static +# setup nfs +systemctl enable nfs-server +echo "/var/log/old 172.31.0.0/16(rw,nohide,no_root_squash) +/home/scores 172.31.0.0/16(rw,nohide,no_root_squash)" >>/etc/exports +systemctl start nfs-server +scp -p -o StrictHostKeyChecking=no -r root@sapsailing.com:/etc/ssh /etc +# append hostname to sysconfig +echo "HOSTNAME=sapsailing.com" >> /etc/sysconfig/network +sed -i "s/\(127.0.0.1 *\)/\1 sapsailing.com /" /etc/hosts +hostname sapsailing.com +hostnamectl set-hostname sapsailing.com +if sshd -t; then + systemctl restart sshd +fi +EOF +ssh -o StrictHostKeyChecking=no -A -f root@"$IP" "cd /var/log/old/cache/docker/registry && nohup docker-compose up &>/dev/null &" &> /dev/null +echo "PLEASE READ ALL OF THE FOLLOWING..." +echo "Remove the existing central reverse proxy from the SSH target groups." +echo "When ready, press a key to continue, which will run the script target-group-tag-route53-nfs-elasticIP-setup.sh." +echo "This configures the EC2 instance tags, adds to the " +echo "necessary target groups, modifies a few records in route53 (logs.internal.sapsailing.com" +echo "and smtp.internal.sapsailing.com), remounts those instances that are dependent on this, and sets the elastic IP." +echo "You will need to have the AWS CLI installed and have credentials that allow you to run all the commands in the script." +echo "Make sure you have an active session token in your shell's environment, e.g., obtained" +echo "through the awsmfalogon.sh script. The script requires that you have jq and host installed locally." +echo "If you haven't already authenticated, then stop this script; authenticate manually;" +echo "and then run the script named above with the remote IP as a parameter." +read -n 1 -p "Press a key to continue" key_pressed +"$(dirname $0)"/target-group-tag-route53-nfs-elasticIP-setup.sh \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh new file mode 100755 index 00000000000..5f1c8c690d3 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/setup-central-reverse-proxy.sh @@ -0,0 +1,231 @@ +#!/bin/bash + +# Setup script for a "Central Reverse HTTP Proxy" that runs Apache httpd, +# provides the Git repository, runs Bugzilla and Gollum for wiki access, +# furthermore AWStats, goaccess and apachetop support within a tmux session, +# releases.sapsailing.com, jobs.sapsailing.com content, and a Docker +# infrastructure for a self-hosted Docker image registry. It assumes the +# existence of a running webserver accessible under sapsailing.com. +# +# Start by launching a new instance, e.g., of type t3.xlarge, in the same AZ +# as the current Webserver / Central Reverse Proxy. This will become important +# as you will need to detach volumes from the latter to attach them to the +# new instance. +# +# Then, call this script with the new instance's external IP address as the first, +# and with a "bearer token" as a second argument, authenticating a user at +# security-service.sapsailing.com which needs to have the following permissions: +# USER:READ:* +# SSH_KEY_READ:* +# which will be used to determine the landscape management users, for example. In addition, +# add the IP/hostname of an instance with the Git repository checked out, containing +# the file configuration/imageupgrade_functions.sh, and the path to the repo on that instance. +# +# Example usage: +# setup-central-reverse-proxy.sh 1.2.3.4 0OcJ1938QE5it875kjlQe7HnzQ6740jsnMEVzowjZrs= 18.170.25.225 /home/sailing/code +# This will do all necessary set-up up to the point where the large volumes +# currently attached to and mounted on the current Central Reverse Proxy will +# need to be unmounted, detached, attached to the new instance, and mounted there. +terminationCheck() { + # $1 status returned by the last command. + # Exits the program if the passed argument is non zero. + # Used to ensure the program doesn't continue if a major error occurs. + [[ "$1" -eq 0 ]] || { echo "Non-zero exit code. Exiting to avoid damage" && exit 1; } +} +if [[ "$#" -ne 4 ]]; then + echo "4 arguments required. Please check comment description for further details." + echo "Example usage: setup-central-reverse-proxy.sh 1.2.3.4 0OcJ1938QE5it875kjlQe7HnzQ6740jsnMEVzowjZrs= 18.170.25.225 /home/sailing/code" + exit 2 +fi +echo "Make sure the instance is in the same AZ as the existing reverse proxy, so the volumes can be switched over." +IP=$1 +BEARER_TOKEN=$2 +IMAGEUPGRADE_FUNCTIONS_IP="$3" # can be a domain name, such as sapsailing.com +IMAGEUPGRADE_FUNCTIONS_PATH_ON_INSTANCE_TO_GIT="$4" +IMAGE_TYPE="central_reverse_proxy" +HTTP_LOGROTATE_ABSOLUTE=/etc/logrotate.d/httpd +# This authorized keys copying is essential as we rely on the ability to log into the root user. +ssh -A "ec2-user@${IP}" "bash -s" << FIRSTEOF +# Correct authorized keys. May not be necessary if update_authorized_keys is running. +sudo su - -c "cat ~ec2-user/.ssh/authorized_keys > /root/.ssh/authorized_keys" +FIRSTEOF +# writes std error to local text file +ssh -A "root@${IP}" "bash -s" << SECONDEOF +# update instance +yum update -y +yum install -y python httpd mod_proxy_html tmux nfs-utils git whois jq cronie iptables mailx nmap icu mariadb105-server tree #icu is a c/c++ library that provides unicode and globalisation support for software development. +# docker setup +yum install -y docker +sudo curl --silent -L "https://github.com/docker/compose/releases/download/v2.26.1/docker-compose-\$(uname -s)-\$(uname -m)" -o /usr/local/bin/docker-compose +chmod +x /usr/local/bin/docker-compose +yum install -y perl perl-CGI perl-Template-Toolkit perl-CPAN perl-DBD-MySQL mod_perl perl-GD gcc-c++ +# ruby and gollum for wiki +yum group install -y "Development Tools" +yum install -y ruby ruby-devel libicu libicu-devel zlib zlib-devel git cmake openssl-devel libyaml-devel +gem install gollum -v 5.3.2 +gem update --system 3.5.7 +cd /home +# copy bugzilla +scp -o StrictHostKeyChecking=no root@sapsailing.com:/var/www/static/bugzilla-5.0.4.tar.gz /usr/local/src +cd /usr/local/src +tar -xzvf bugzilla-5.0.4.tar.gz +mv bugzilla-5.0.4 /usr/share/bugzilla +cd /usr/share/bugzilla/ +scp -o StrictHostKeyChecking=no root@sapsailing.com:/usr/share/bugzilla/localconfig . +echo "Bugzilla has been copied. Now setting up bugzilla modules." +echo "This can take 5 minutes or so. The output is muted (but sent to log.txt) to prevent excessive warnings and clutter in the terminal." +SECONDEOF +terminationCheck "$?" +ssh -A "root@${IP}" "bash -s" << BUGZILLAEOF &>log.txt +cd /usr/share/bugzilla/ +# essentials bugzilla +/usr/bin/perl install-module.pl DateTime +/usr/bin/perl install-module.pl DateTime::TimeZone +/usr/bin/perl install-module.pl Email::Sender +/usr/bin/perl install-module.pl Email::MIME +/usr/bin/perl install-module.pl List::MoreUtils +/usr/bin/perl install-module.pl Math::Random::ISAAC +/usr/bin/perl install-module.pl JSON::XS + +# important bugzilla +/usr/bin/perl install-module.pl Email::Address +/usr/bin/perl install-module.pl autodie +/usr/bin/perl install-module.pl Class::XSAccessor +# nice to have for buzilla +/usr/bin/perl install-module.pl Date::Parse +/usr/bin/perl install-module.pl Email::Send +/usr/bin/perl install-module.pl DBI +/usr/bin/perl install-module.pl IO::Socket::SSL +/usr/bin/perl install-module.pl Chart::Lines +/usr/bin/perl install-module.pl Template::Plugin::GD::Image +/usr/bin/perl install-module.pl GD::Text +/usr/bin/perl install-module.pl GD::Graph +/usr/bin/perl install-module.pl PatchReader +/usr/bin/perl install-module.pl Authen::Radius +/usr/bin/perl install-module.pl JSON::RPC +/usr/bin/perl install-module.pl TheSchwartz +/usr/bin/perl install-module.pl Daemon::Generic +/usr/bin/perl install-module.pl File::MimeInfo::Magic +/usr/bin/perl install-module.pl File::Copy::Recursive +/usr/bin/perl install-module.pl MIME::Base64 +/usr/bin/perl install-module.pl Authen::SASL +BUGZILLAEOF +terminationCheck "$?" +read -n 1 -p "Bugzilla installation complete, when ready press a key to continue." key_pressed +# use the localconfig file to setup the bugzilla +# t forces tty allocation. +ssh root@"${IP}" -A -t 'cd /usr/share/bugzilla/; ./checksetup.pl' +terminationCheck "$?" +ssh -A -t "root@${IP}" "cpan install Geo::IP" +terminationCheck "$?" +ssh -A "root@${IP}" "bash -s" << THIRDEOF +# The following line is for production use: +scp -o StrictHostKeyChecking=no -p root@"$IMAGEUPGRADE_FUNCTIONS_IP":"$IMAGEUPGRADE_FUNCTIONS_PATH_ON_INSTANCE_TO_GIT"/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh /usr/local/bin +# The following line is for test use, copying from a test instance with a check-out Git workspace: +# scp -o StrictHostKeyChecking=no -p "root@13.40.100.54:/home/sailing/code/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh" /usr/local/bin +. imageupgrade_functions.sh +setup_cloud_cfg_and_root_login +# setup files +if ! build_crontab_and_setup_files -c -n "${IMAGE_TYPE}"; then # -c & -n mean only files are copied over. + exit 1 +fi +setup_swap 5000 +cd /home +for folder in * ; do + [[ -d "\$folder" ]] || continue + grep "\$folder" /etc/passwd || continue + chown -R "\$folder":"\$folder" "\$folder" +done +# setup mail +setup_mail_sending +# setup sshd config +setup_sshd_resilience +# setup goaccess and apachetop +setup_apachetop +setup_goaccess +echo $BEARER_TOKEN > /root/ssh-key-reader.token +# awstats - depends on some of the previous perl modules. +scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/usr/share/GeoIP /usr/share/GeoIP +cd /usr/local/src +wget http://prdownloads.sourceforge.net/awstats/awstats-7.0.tar.gz +tar -zvxf awstats-7.0.tar.gz +mv awstats-7.0/ /usr/share/awstats +mkdir /var/lib/awstats +# Copies the database for awstats over. +scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/var/lib/awstats /var/lib +chmod 755 /root +cd ~ +# Copies across the key vault and other relevant secrets from the existing +# central reverse proxy's /root folder: +rsync -a root@sapsailing.com:/root/{dev-secrets,github_tools_sap.pat,hudson-aws-credentials,key_vault,mail.properties,secrets,ssh-key-reader.token} /root +scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/etc/letsencrypt /etc +# add basic test page which won't cause redirect error code if used as a health check. +cat < /var/www/html/index.html +Health check

Test page

+EOF +echo "net.ipv4.ip_conntrac_max = 131072" >> /etc/sysctl.conf +# setup fail2ban +setup_fail2ban +# setup logrotate.d/httpd +# echo "Patching $HTTP_LOGROTATE_ABSOLUTE so that old logs go to /var/log/old/$IP" >>/var/log/sailing.out +# mkdir --parents "/var/log/old/REVERSE_PROXIES/${IP}" +# sed -i "s|/var/log/old|/var/log/old/REVERSE_PROXIES/${IP}|" $HTTP_LOGROTATE_ABSOLUTE +# logrotate.conf setup +mkdir /var/log/logrotate-target +sed -i 's/rotate 4/rotate 20 \n\nolddir \/var\/log\/logrotate-target/' /etc/logrotate.conf +sed -i "s/^#compress/compress/" /etc/logrotate.conf +# setup httpd git +(/usr/local/bin/setupHttpdGitLocal.sh "httpdConf@sapsailing.com:repo.git" central "Central Reverse Proxy") +scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/etc/httpd/conf/pass* /etc/httpd/conf/ +chown root:root /etc/httpd/conf/pass* +# create mountpoints (see part 2 for ownership changes) +mkdir /var/log/old +mkdir /var/www/static +download_and_install_latest_sap_jvm_8 +# enable units which build-crontab doesn't +systemctl enable httpd +systemctl start httpd +sudo systemctl enable crond.service +sudo systemctl enable postfix +sudo systemctl restart postfix +mkdir --parents /root/temporary_home_copy/home +mv /home/* /root/temporary_home_copy/home +scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/var/www/icons /var/www +scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/var/www/errorpages /var/www +scp -o StrictHostKeyChecking=no -r root@sapsailing.com:/var/www/error /var/www +echo "UUID=23d42c52-85ee-4f6d-bdfe-c62f69bb689f /home ext4 defaults,noatime,commit=30 0 0 +UUID=0b15f5cb-fd3e-48e6-8195-be248cd7726d /var/www/static ext3 defaults,noatime,commit=30 0 0 +UUID=ff598428-d380-4429-a690-3809157506b7 /var/log/old ext3 defaults,noatime,commit=30 0 0 +UUID=d371e530-c189-4012-ae57-45d67a690554 /var/log/old/cache ext4 defaults,noatime,commit=30 0 0" >>/etc/fstab +systemctl enable --now docker +THIRDEOF +terminationCheck "$?" +echo "" +echo "" +echo "Your turn! READ CAREFULLY BEFORE ACTING! The instance is now prepared." +echo "Ensure that there is at least 1 healthy disposable in the SAME availability zone as the archive," +echo "so there is no risk of all the targets being briefly unhealthy." +echo "Please remove the existing central reverse proxy from all target groups tagged with \"CentralReverseProxy\"" +echo "or \"allReverseProxies\" (draining can take 5 mins), this includes removing the private IP target from the NLB target group." +echo "But do not edit the SSH target group, otherwise the next part will fail." +echo "Then unmount the volumes /home, /var/www/static, /var/log/old and /var/log/old/cache from the existing reverse proxy," +echo "detach, reattach to the new instance and remount as follows:" +echo "First unmount, on the old instance, via" +echo " umount -l -f " +echo "Then begin detaching and attaching. This can be done in the AWS EC2 console by going to the old webserver" +echo "and clicking on the volumes in question (found within the storage tab)." +echo "Then click Detach from within the Actions dropdrown. Reattach to the new instance by clicking Attach from the Actions dropdown," +echo "then choose the new instance and the device name (which can be any value, because the mounting is done via UUID)." +echo "On the new instance itself, the remounting can be done with" +echo " mount -a" +echo "as /etc/fstab contains all the mount details." +echo "For further details, checkout this wiki page https://wiki.sapsailing.com/wiki/info/landscape/amazon-ec2#amazon-ec2-for-sap-sailing-analytics_landscape-overview_apache-httpd-the-central-reverse-proxy-webserver-and-disposable-reverse-proxies" +echo "Check that all these volumes were mounted successfully, e.g. by invoking" +echo " mount" +echo "without any arguments. If everything looks good, please press a key to trigger part 2, which" +echo "sets up the hostname, copies /etc/ssh and configures the users and crontabs." +read -n 1 -p "Press a key to continue" key_pressed +"$(dirname $0)"/setup-central-reverse-proxy-part-2.sh "$IP" "$IMAGE_TYPE" + +# anything in etc +#not available: perl-HTML-Template /usr/bin/perl install-module.pl GD diff --git a/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh new file mode 100755 index 00000000000..631af9669b4 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/target-group-tag-route53-nfs-elasticIP-setup.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# This script is to be run after part 1 and part 2. It should be run locally and +# requires the user to have the AWS CLI installed, as well as credentials. They must also have run the awsmfalogon.sh to authenticate. +# It will add the necessary tags, alter route 53 records, remount the nfs mounts that depend on these changes, add to the +# necessary target groups and switch the elastic IP. +if [[ "$#" -ne 1 ]]; then + echo "Please pass the remote IP address as the only parameter." + echo "Please check comment description for usage." + exit 2 +fi +target_groups=$(aws elbv2 describe-target-groups) +LOCAL_IPV4=$(ssh -o StrictHostKeyChecking=false root@"$1" "ec2-metadata --local-ipv4 2>/dev/null | sed \"s/local-ipv4: *//\"") +INSTANCE_ID=$(ssh -o StrictHostKeyChecking=false root@"$1" "ec2-metadata --instance-id 2>/dev/null | sed \"s/instance-id: *//\"") +if [[ -z "$LOCAL_IPV4" || -z "$INSTANCE_ID" ]]; then + echo "Null local IP or Instance ID" + exit 1 +fi +ELASTIC_IP="54.229.94.254" +INSTANCE_TAGS=("CentralReverseProxy" "ReverseProxy") +TARGET_GROUP_TAGS=("allReverseProxies" "CentralReverseProxy") +extract_public_ip() { + jq -r ' .Instances | .[] | .PublicIpAddress' | grep -v null +} +select_instances_by_tag() { + # $1: tag + jq -r '.Reservations | .[] | select(.Instances | .[] | .Tags| any (.Key=="'"$1"'"))' +} +cd $(dirname "$0") +# give the instance the necessary tags. +for tag in "${INSTANCE_TAGS[@]}"; do + aws ec2 create-tags --resources "$INSTANCE_ID" --tags Key="$tag",Value="" +done +# The nlb is the exception case as we use the load balancer arn to further identify it. +ip_based_target_groups_tags=$(aws elbv2 describe-tags --resource-arns $(echo "$target_groups" | jq -r '.TargetGroups | .[] | select(.LoadBalancerArns | any(contains("loadbalancer/net")) ) | .TargetGroupArn')) +nlb_target_group_arn=$(echo "$ip_based_target_groups_tags" | jq -r '.TagDescriptions | .[] | select(.Tags | any(.Key=="allReverseProxies") ) | .ResourceArn') +echo "Registering with nlb" +aws elbv2 register-targets --target-group-arn "$nlb_target_group_arn" --targets Id="${LOCAL_IPV4}",Port=80 +ssh_target_group_arn=$(echo "$ip_based_target_groups_tags" | jq -r '.TagDescriptions | .[] | select(.Tags | any(.Key=="CentralReverseProxy") ) | .ResourceArn') +echo "Registering with ssh target group" +aws elbv2 register-targets --target-group-arn "$ssh_target_group_arn" --targets Id="${LOCAL_IPV4}",Port=80 +echo "Fetching tags for all target groups to identify which groups to add the new central proxy to" +# We fetch the tags of the target groups to identify those which the central reverse proxy should be added to. +# We depend on SAILING_TARGET_GROUP_NAME_PREFIX to filter out all target groups which point to sailing servers, +# because describe tags can take a maximum of 20 resource-arns. +describe_tags=$(aws elbv2 describe-tags --resource-arns $(echo "$target_groups" | jq -r '.TargetGroups | .[] | select(.TargetGroupName | startswith("S-") | not ) | select(.LoadBalancerArns | any(contains("loadbalancer/net") | not) ) | .TargetGroupArn')) +for tag in "${TARGET_GROUP_TAGS[@]}"; do + echo "Adding to target groups with $tag" + for tgArn in $(echo "$describe_tags" | jq -r '.TagDescriptions | .[] | select(.Tags | any(.Key=="'"$tag"'") ) | .ResourceArn'); do + echo "Registering in $tgArn as it has the correct tag" + aws elbv2 register-targets --target-group-arn "$tgArn" --targets Id="${INSTANCE_ID}" + [[ "$?" -eq 0 ]] || echo "Registration of target was not successful" + done +done +# alter records using batch file. +sed -i "s/LOGFILES_INTERNAL_IP/$LOCAL_IPV4/" batch-for-route53-dns-record-update.json +sed -i "s/SMTP_INTERNAL_IP/$LOCAL_IPV4/" batch-for-route53-dns-record-update.json +HOSTED_ZONE_ID=$( aws route53 list-hosted-zones | \ + jq -r '.HostedZones[] | select(.Name == "sapsailing.com.").Id' | \ + sed -e 's|/hostedzone/||' ) +read -n 1 -p "Check the instance has the correct tags and is in the correct target group. +Furthermore, check the batch file batch-for-route53-dns-record-update.json has been modified correctly. +Press a key to continue.." key_pressed +aws route53 change-resource-record-sets --hosted-zone-id ${HOSTED_ZONE_ID} --change-batch file://batch-for-route53-dns-record-update.json +# reload the nfs mountpoints. +echo "Waiting for records to change." +until $(host "logfiles.internal.sapsailing.com" | grep -q "$LOCAL_IPV4") && $(host "smtp.internal.sapsailing.com" | grep -q "$LOCAL_IPV4"); do + sleep 5 +done +echo "Describing instances for remounting." +describe_instances=$(aws ec2 describe-instances) +echo "Sailing servers: " +for instanceIp in $(echo "$describe_instances" | select_instances_by_tag "sailing-analytics-server" | extract_public_ip); do + echo "Remounting on $instanceIp" + ssh -o StrictHostKeyChecking=false root@"${instanceIp}" "umount -l -f /home/scores; mount -a" +done +echo "Disposable reverse proxies: " +for instanceIp in $(echo "$describe_instances" | select_instances_by_tag "DisposableProxy" | extract_public_ip); do + echo "Remounting on $instanceIp" + ssh -o StrictHostKeyChecking=false root@"${instanceIp}" "umount -l -f /var/log/old; mount -a" +done +# Alter the elastic IP. +# WARNING: Will terminate connections via the existing public ip. +aws ec2 associate-address --instance-id "${INSTANCE_ID}" --public-ip "${ELASTIC_IP}" +git checkout main batch-for-route53-dns-record-update.json \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/users/certbot/gid b/configuration/environments_scripts/central_reverse_proxy/users/certbot/gid new file mode 100644 index 00000000000..a0ce719908c --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/users/certbot/gid @@ -0,0 +1 @@ +1016 \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/users/certbot/groups b/configuration/environments_scripts/central_reverse_proxy/users/certbot/groups new file mode 100644 index 00000000000..6d0eac4b37a --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/users/certbot/groups @@ -0,0 +1 @@ +docker \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/users/certbot/uid b/configuration/environments_scripts/central_reverse_proxy/users/certbot/uid new file mode 100644 index 00000000000..8c7b65e297f --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/users/certbot/uid @@ -0,0 +1 @@ +1014 \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/users/httpdConf/gid b/configuration/environments_scripts/central_reverse_proxy/users/httpdConf/gid new file mode 100644 index 00000000000..a804be795ba --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/users/httpdConf/gid @@ -0,0 +1 @@ +1017 \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/users/httpdConf/uid b/configuration/environments_scripts/central_reverse_proxy/users/httpdConf/uid new file mode 100644 index 00000000000..758690690ee --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/users/httpdConf/uid @@ -0,0 +1 @@ +1015 \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/users/root/crontab-update-authorized-keys b/configuration/environments_scripts/central_reverse_proxy/users/root/crontab-update-authorized-keys deleted file mode 120000 index d9b0fed0726..00000000000 --- a/configuration/environments_scripts/central_reverse_proxy/users/root/crontab-update-authorized-keys +++ /dev/null @@ -1 +0,0 @@ -../../../../crontabs/crontab-update-authorized-keys \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_home_httpdConf b/configuration/environments_scripts/central_reverse_proxy/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_home_httpdConf new file mode 120000 index 00000000000..cf597d8e543 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_home_httpdConf @@ -0,0 +1 @@ +../../../../crontabs/crontab-update-authorized-keys@USER_HOME_DIR \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_root b/configuration/environments_scripts/central_reverse_proxy/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_root new file mode 120000 index 00000000000..cf597d8e543 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_root @@ -0,0 +1 @@ +../../../../crontabs/crontab-update-authorized-keys@USER_HOME_DIR \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/users/scores/gid b/configuration/environments_scripts/central_reverse_proxy/users/scores/gid new file mode 100644 index 00000000000..9670e7bd0b0 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/users/scores/gid @@ -0,0 +1 @@ +1013 \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/users/scores/uid b/configuration/environments_scripts/central_reverse_proxy/users/scores/uid new file mode 100644 index 00000000000..0e4cd995bbd --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/users/scores/uid @@ -0,0 +1 @@ +1012 \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/users/trac/uid b/configuration/environments_scripts/central_reverse_proxy/users/trac/uid new file mode 100644 index 00000000000..f3d2a1110f1 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/users/trac/uid @@ -0,0 +1 @@ +1004 \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/users/wiki/crontab-download-new-archived-trac-trac-events b/configuration/environments_scripts/central_reverse_proxy/users/wiki/crontab-download-new-archived-trac-trac-events deleted file mode 120000 index f93858411b9..00000000000 --- a/configuration/environments_scripts/central_reverse_proxy/users/wiki/crontab-download-new-archived-trac-trac-events +++ /dev/null @@ -1 +0,0 @@ -../../../../crontabs/crontab-download-new-archived-trac-trac-events \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/users/wiki/crontab-download-new-archived-trac-trac-events b/configuration/environments_scripts/central_reverse_proxy/users/wiki/crontab-download-new-archived-trac-trac-events new file mode 100644 index 00000000000..858d1f80bca --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/users/wiki/crontab-download-new-archived-trac-trac-events @@ -0,0 +1 @@ +15 12 * * * export PATH=/bin:/usr/bin:/usr/local/bin; downloadNewArchivedTracTracEvents.sh /home/trac/static/TracTracTracks "/home/wiki/gitwiki" >/home/wiki/downloadNewArchivedTracTracEvents.out 2>/home/wiki/downloadNewArchivedTracTracEvents.err diff --git a/configuration/environments_scripts/central_reverse_proxy/users/wiki/crontab-syncgit b/configuration/environments_scripts/central_reverse_proxy/users/wiki/crontab-syncgit deleted file mode 120000 index 8ba5fde8354..00000000000 --- a/configuration/environments_scripts/central_reverse_proxy/users/wiki/crontab-syncgit +++ /dev/null @@ -1 +0,0 @@ -../../../../crontabs/crontab-syncgit \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/users/wiki/crontab-syncgit b/configuration/environments_scripts/central_reverse_proxy/users/wiki/crontab-syncgit new file mode 100644 index 00000000000..4c281ff9945 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/users/wiki/crontab-syncgit @@ -0,0 +1 @@ +*/10 * * * * export PATH=/bin:/usr/bin:/usr/local/bin; syncgit /home/wiki/gitwiki diff --git a/configuration/environments_scripts/central_reverse_proxy/users/wiki/groups b/configuration/environments_scripts/central_reverse_proxy/users/wiki/groups new file mode 100644 index 00000000000..ea6d9aebf83 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/users/wiki/groups @@ -0,0 +1 @@ +trac \ No newline at end of file diff --git a/configuration/environments_scripts/central_reverse_proxy/users/wiki/uid b/configuration/environments_scripts/central_reverse_proxy/users/wiki/uid new file mode 100644 index 00000000000..2478c955ec8 --- /dev/null +++ b/configuration/environments_scripts/central_reverse_proxy/users/wiki/uid @@ -0,0 +1 @@ +1003 \ No newline at end of file diff --git a/configuration/environments_scripts/hudson_slave/README b/configuration/environments_scripts/hudson_slave/README index b1a742db303..3ed7c3ea383 100755 --- a/configuration/environments_scripts/hudson_slave/README +++ b/configuration/environments_scripts/hudson_slave/README @@ -3,3 +3,5 @@ Deploy the imageupgrade script to /usr/local/bin, furthermore the ../imageupgrade_functions.sh has to go to /usr/local/bin, best as symbolic link to files in git. See the script in wiki/howto/development/ci.md that shows how to set up such an instance from scratch. + +With us moving our builds to Github, maintaining Hudson build slaves is DEPRECATED. diff --git a/configuration/environments_scripts/mongo_instance_setup/README b/configuration/environments_scripts/mongo_instance_setup/README deleted file mode 100644 index 289ba3f38e1..00000000000 --- a/configuration/environments_scripts/mongo_instance_setup/README +++ /dev/null @@ -1,16 +0,0 @@ -Deploy the .mount and .service units to /etc/systemd/system. -Deploy the ephemeralvolume and the patch-mongo-replicaset-name-from-ec2-metadata script to /usr/local/bin, -furthermore the ../imageupgrade_functions.sh has to go to /usr/local/bin. -Deploy mongod.conf to /etc and make sure that /root has a+r and a+x permissions because -otherwise the mongod user won't be able to read through the symbolic link -Link mongodb to /etc/logrotate.d -Utilise the build-crontab-and-cp-files script or the build_crontab_and_setup_files (in imageupgrade_functions.sh) -to create the crontab file and install it in the root user. -Run with optional EC2 user detail, e.g., as follows: - - REPLICA_SET_NAME=archive - REPLICA_SET_PRIMARY=dbserver.internal.sapsailing.com:10201 - -This will automatically patch /etc/mongod.conf such that the replSetName property -is set to the value of REPLICA_SET_NAME. Then, the instance will be added to -the REPLICA_SET_PRIMARY's replica set. diff --git a/configuration/environments_scripts/mongo_instance_setup/files/etc/logrotate.d/mongodb b/configuration/environments_scripts/mongo_instance_setup/files/etc/logrotate.d/mongodb deleted file mode 100644 index 4203c9a972c..00000000000 --- a/configuration/environments_scripts/mongo_instance_setup/files/etc/logrotate.d/mongodb +++ /dev/null @@ -1,9 +0,0 @@ -compress -/var/log/mongodb/mongod.log -{ - rotate 5 - weekly - postrotate - /usr/bin/killall -SIGUSR1 mongod - endscript -} diff --git a/configuration/environments_scripts/mongo_instance_setup/files/etc/logrotate.d/mongodb b/configuration/environments_scripts/mongo_instance_setup/files/etc/logrotate.d/mongodb new file mode 120000 index 00000000000..00ba3e765ea --- /dev/null +++ b/configuration/environments_scripts/mongo_instance_setup/files/etc/logrotate.d/mongodb @@ -0,0 +1 @@ +../../../../repo/etc/logrotate.d/mongodb \ No newline at end of file diff --git a/configuration/environments_scripts/mongo_instance_setup/files/etc/mongod.conf b/configuration/environments_scripts/mongo_instance_setup/files/etc/mongod.conf index 6373591def6..db4c61d58e8 100755 --- a/configuration/environments_scripts/mongo_instance_setup/files/etc/mongod.conf +++ b/configuration/environments_scripts/mongo_instance_setup/files/etc/mongod.conf @@ -16,7 +16,6 @@ storage: enabled: true directoryPerDB: true # engine: -# mmapv1: # wiredTiger: # how the process runs @@ -29,7 +28,6 @@ processManagement: net: port: 27017 # bindIp: 127.0.0.1 # Listen to local interface only, comment to listen on all interfaces. -# bindIp: 172.31.33.146 bindIp: 0.0.0.0 #security: diff --git a/configuration/environments_scripts/mongo_instance_setup/files/etc/systemd/system/chownvarlibmongo.service b/configuration/environments_scripts/mongo_instance_setup/files/etc/systemd/system/chownvarlibmongo.service index b4ea9c26d1f..c8623ab9475 100644 --- a/configuration/environments_scripts/mongo_instance_setup/files/etc/systemd/system/chownvarlibmongo.service +++ b/configuration/environments_scripts/mongo_instance_setup/files/etc/systemd/system/chownvarlibmongo.service @@ -10,5 +10,4 @@ RequiredBy=mongod.service [Service] Type=oneshot RemainAfterExit=true -ExecStart=/bin/chown -R mongod /var/lib/mongo/ -ExecStart=/bin/chgrp -R mongod /var/lib/mongo/ +ExecStart=/bin/chown -R mongod:mongod /var/lib/mongo/ diff --git a/configuration/environments_scripts/mongo_instance_setup/files/etc/systemd/system/ephemeralvolume.service b/configuration/environments_scripts/mongo_instance_setup/files/etc/systemd/system/ephemeralvolume.service deleted file mode 100644 index 657f8823da2..00000000000 --- a/configuration/environments_scripts/mongo_instance_setup/files/etc/systemd/system/ephemeralvolume.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Ensures /dev/nvme0n1 or /dev/xvdb is XFS-formatted -Requires=-.mount cloud-init.service network.service -After=-.mount cloud-init.service network.service - -[Install] - -[Service] -Type=oneshot -RemainAfterExit=true -ExecStart=/usr/local/bin/ephemeralvolume diff --git a/configuration/environments_scripts/mongo_instance_setup/files/etc/systemd/system/ephemeralvolume.service b/configuration/environments_scripts/mongo_instance_setup/files/etc/systemd/system/ephemeralvolume.service new file mode 120000 index 00000000000..f4392d61230 --- /dev/null +++ b/configuration/environments_scripts/mongo_instance_setup/files/etc/systemd/system/ephemeralvolume.service @@ -0,0 +1 @@ +../../../../../repo/etc/systemd/system/ephemeralvolume.service \ No newline at end of file diff --git a/configuration/environments_scripts/mongo_instance_setup/files/etc/systemd/system/imageupgrade.service b/configuration/environments_scripts/mongo_instance_setup/files/etc/systemd/system/imageupgrade.service new file mode 120000 index 00000000000..32775d542f4 --- /dev/null +++ b/configuration/environments_scripts/mongo_instance_setup/files/etc/systemd/system/imageupgrade.service @@ -0,0 +1 @@ +../../../../../repo/etc/systemd/system/imageupgrade.service \ No newline at end of file diff --git a/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/add-as-replica b/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/add-as-replica deleted file mode 100755 index 83e3e969a03..00000000000 --- a/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/add-as-replica +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -user_data=$( ec2-metadata -d | sed -e 's/^user-data: //' ) -if echo ${user_data} | grep -q "^image-upgrade$"; then - echo "Image upgrade... didn't expect to get this far because ephemeralvolume should have triggered upgrade and shutdown. Not registering MongoDB replica" -else - eval ${user_data} - if [ -z "$REPLICA_SET_NAME" ]; then - REPLICA_SET_NAME=live - fi - if [ -z "$REPLICA_SET_PRIMARY" ]; then - REPLICA_SET_PRIMARY=mongo0.internal.sapsailing.com:27017 - fi - if [ -z "$REPLICA_SET_PRIORITY" ]; then - REPLICA_SET_PRIORITY=1 - fi - if [ -z "$REPLICA_SET_VOTES" ]; then - REPLICA_SET_VOTES=0 - fi - if [ \! -z "REPLICA_SET_PRIMARY" ]; then - IP=$(ec2-metadata -o | sed -e 's/^local-ipv4: //') - echo "rs.add({host: \"$IP:27017\", priority: $REPLICA_SET_PRIORITY, votes: $REPLICA_SET_VOTES})" | mongo "mongodb://$REPLICA_SET_PRIMARY/?replicaSet=$REPLICA_SET_NAME&retryWrites=true" - else - echo "rs.initiate()" | mongo - fi -fi diff --git a/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/add-as-replica b/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/add-as-replica new file mode 120000 index 00000000000..5da30596a4f --- /dev/null +++ b/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/add-as-replica @@ -0,0 +1 @@ +../../../../../repo/usr/local/bin/add-as-replica \ No newline at end of file diff --git a/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/ephemeralvolume b/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/ephemeralvolume deleted file mode 100755 index 68518e31514..00000000000 --- a/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/ephemeralvolume +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -# Script to deploy on an instance that has an ephemeral volume as /dev/nvme0n1 (adjust env var PARTITION if different) -# Ensures the partition is xfs-formatted, any existing partition contents will be overwritten if formatted otherwise. -# An existing xfs partition will be left alone. - -METADATA=$( /bin/ec2-metadata -d | sed -e 's/^user-data: //' ) -echo "Metadata: ${METADATA}" -if echo "${METADATA}" | grep -q "^image-upgrade$"; then - echo "Image upgrade; not trying to mount/format ephemeral volume; calling imageupgrade.sh instead..." - imageupgrade.sh -else - echo "No image upgrade; looking for ephemeral volume and trying to format with xfs..." - PARTITION=/dev/nvme0n1 - if [ \! -e $PARTITION ]; then - PARTITION=/dev/xvdb - fi - if [ \! -e $PARTITION ]; then - echo "Neither /dev/nvme0n1 nor /dev/xvdb partition found; not formatting/mounting ephemeral volume" - elif cat /proc/mounts | awk '{print $1;}' | grep "${PARTITION}"; then - echo "Partition ${PARTITION} already mounted; not formatting/mounting ephemeral volume" - else - FSTYPE=$(blkid -p $PARTITION -s TYPE -o value) - if [ "$FSTYPE" != "xfs" ]; then - echo FSTYPE was "$FSTYPE" but should have been xfs. Formatting $PARTITION... - mkfs.xfs -f $PARTITION - else - echo FSTYPE was "$FSTYPE" which is just right :-\) - fi - # mount the thing to /var/lib/mongo - mount $PARTITION /var/lib/mongo - fi -fi diff --git a/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/ephemeralvolume b/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/ephemeralvolume new file mode 120000 index 00000000000..f23b5689cc4 --- /dev/null +++ b/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/ephemeralvolume @@ -0,0 +1 @@ +../../../../../repo/usr/local/bin/ephemeralvolume \ No newline at end of file diff --git a/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/imageupgrade b/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/imageupgrade new file mode 100755 index 00000000000..aca90f97879 --- /dev/null +++ b/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/imageupgrade @@ -0,0 +1,22 @@ +#!/bin/bash + +# Upgrades the AWS EC2 MongoDB instance that this script is assumed to be executed on. +# The steps are as follows: + +. imageupgrade_functions.sh + +clean_mongo_pid() { + rm -f /var/run/mongodb/mongod.pid +} + +LOGON_USER_HOME=/home/ec2-user +METADATA=$( get_ec2_user_data ) +echo "Metadata: ${METADATA}" +if echo "${METADATA}" | grep -q "^image-upgrade$"; then + echo "Image upgrade..." + run_yum_update + build_crontab_and_setup_files mongo_instance_setup + clean_startup_logs + clean_mongo_pid + finalize +fi diff --git a/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/imageupgrade.sh b/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/imageupgrade.sh deleted file mode 100755 index f237d47348a..00000000000 --- a/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/imageupgrade.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# Upgrades the AWS EC2 MongoDB instance that this script is assumed to be executed on. -# The steps are as follows: - -. imageupgrade_functions.sh - -run_git_pull_root() { - echo "Pulling git to /root/code" >>/var/log/sailing.err - cd /root/code - git pull -} - -clean_mongo_pid() { - rm -f /var/run/mongodb/mongod.pid -} - -LOGON_USER_HOME=/home/ec2-user - -run_yum_update -run_git_pull_root -clean_startup_logs -update_root_crontab -clean_mongo_pid -finalize diff --git a/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/patch-mongo-replicaset-name-from-ec2-metadata b/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/patch-mongo-replicaset-name-from-ec2-metadata deleted file mode 100755 index db43bf746d0..00000000000 --- a/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/patch-mongo-replicaset-name-from-ec2-metadata +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -REPLICA_SET_NAME=$(ec2-metadata | grep REPLICA_SET_NAME | sed -e 's/^user-data: //' | sed -e 's/^REPLICA_SET_NAME=//') -echo Replica set name: $REPLICA_SET_NAME -if [ \! -z "$REPLICA_SET_NAME" ]; then - echo "Not empty. Patching /etc/mongod.conf..." - sed -i -e "s/replSetName: .*$/replSetName: $REPLICA_SET_NAME/" /etc/mongod.conf - echo "Done" -fi diff --git a/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/patch-mongo-replicaset-name-from-ec2-metadata b/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/patch-mongo-replicaset-name-from-ec2-metadata new file mode 120000 index 00000000000..548951e4d32 --- /dev/null +++ b/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/patch-mongo-replicaset-name-from-ec2-metadata @@ -0,0 +1 @@ +../../../../../repo/usr/local/bin/patch-mongo-replicaset-name-from-ec2-metadata \ No newline at end of file diff --git a/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/remove-as-replica b/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/remove-as-replica deleted file mode 100755 index fdb821b8933..00000000000 --- a/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/remove-as-replica +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -eval $( ec2-metadata -d | sed -e 's/^user-data: //' ) -if [ \! -z "REPLICA_SET_PRIMARY" ]; then - IP=$(ec2-metadata -o | sed -e 's/^local-ipv4: //') - echo "rs.remove(\"${IP}:27017\")" | mongo "mongodb://${IP}:27017/?replicaSet=${REPLICA_SET_NAME}&retryWrites=true" -fi diff --git a/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/remove-as-replica b/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/remove-as-replica new file mode 120000 index 00000000000..cb04d966ff1 --- /dev/null +++ b/configuration/environments_scripts/mongo_instance_setup/files/usr/local/bin/remove-as-replica @@ -0,0 +1 @@ +../../../../../repo/usr/local/bin/remove-as-replica \ No newline at end of file diff --git a/configuration/environments_scripts/mongo_instance_setup/setup-mongo-instance.sh b/configuration/environments_scripts/mongo_instance_setup/setup-mongo-instance.sh new file mode 100755 index 00000000000..50e5b200480 --- /dev/null +++ b/configuration/environments_scripts/mongo_instance_setup/setup-mongo-instance.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Usage: Launch an Amazon EC2 instance from an Amazon Linux 2 AMI with +# 8GB of root partition size and the "MongoDB Replica Set" security group +# using an SSH key for which you have a working private key available. +# Then, run this script on your local computer, using the external IP address +# of the instance you just launched in AWS as only argument. This will then +# turn the instance into a MongoDB replica set node. +# When the script is done you may log in to look around and check +# things. When done, shut down the instance (Stop, not Terminate) and create +# an image off of it, naming it, e.g., "MongoDB Live Replica Set NVMe 2.0" and +# also tagging its root volume snapshot as, e.g., "MongoDB Live Replica Set NVMe 2.0 (Root)". +# If you want to use the resulting image in production, also tag it with +# tag key "image-type" and tag value "mongodb-server". +if [ $# != 0 ]; then + SERVER=$1 + scp "${0}" ec2-user@${SERVER}: + ssh -A ec2-user@${SERVER} ./$( basename "${0}" ) +else + if ec2-metadata | grep -q instance-id; then + echo "Running on an AWS EC2 instance as user ${USER} / $(whoami), starting setup..." + # Install standard packages: + sudo yum -y update + sudo yum -y install nvme-cli chrony cronie cronie-anacron jq mailx + # Copy imageupgrade_function.sh + scp -o StrictHostKeyChecking=no -p root@sapsailing.com:/home/wiki/gitwiki/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh . + sudo mv imageupgrade_functions.sh /usr/local/bin + # build-crontab + . imageupgrade_functions.sh + build_crontab_and_setup_files mongo_instance_setup + # obtain root SSH key from key vault: + setup_keys "mongo_instance_setup" + # Install MongoDB 4.4 and configure as replica set "live" + setup_mongo_4_4 + scp root@sapsailing.com:ssh-key-reader.token /tmp + sudo mv /tmp/ssh-key-reader.token /root + sudo chown root:root /root/ssh-key-reader.token + sudo chmod 600 /root/ssh-key-reader.token + setup_sshd_resilience + else + echo "Not running on an AWS instance; refusing to run setup!" >&2 + echo "To prepare an instance running in AWS, provide its external IP as argument to this script." >&2 + exit 2 + fi +fi diff --git a/configuration/environments_scripts/mongo_instance_setup/users/root/crontab-update-authorized-keys b/configuration/environments_scripts/mongo_instance_setup/users/root/crontab-update-authorized-keys deleted file mode 120000 index d9b0fed0726..00000000000 --- a/configuration/environments_scripts/mongo_instance_setup/users/root/crontab-update-authorized-keys +++ /dev/null @@ -1 +0,0 @@ -../../../../crontabs/crontab-update-authorized-keys \ No newline at end of file diff --git a/configuration/environments_scripts/mongo_instance_setup/users/root/crontab-update-authorized-keys-ec2-user b/configuration/environments_scripts/mongo_instance_setup/users/root/crontab-update-authorized-keys-ec2-user deleted file mode 120000 index 7a696ea0929..00000000000 --- a/configuration/environments_scripts/mongo_instance_setup/users/root/crontab-update-authorized-keys-ec2-user +++ /dev/null @@ -1 +0,0 @@ -../../../../crontabs/crontab-update-authorized-keys-ec2-user \ No newline at end of file diff --git a/configuration/environments_scripts/mongo_instance_setup/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_home_ec2-user b/configuration/environments_scripts/mongo_instance_setup/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_home_ec2-user new file mode 120000 index 00000000000..cf597d8e543 --- /dev/null +++ b/configuration/environments_scripts/mongo_instance_setup/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_home_ec2-user @@ -0,0 +1 @@ +../../../../crontabs/crontab-update-authorized-keys@USER_HOME_DIR \ No newline at end of file diff --git a/configuration/mysql_instance_setup/setup-mysql-server.sh b/configuration/environments_scripts/mysql_instance_setup/setup-mysql-server.sh similarity index 82% rename from configuration/mysql_instance_setup/setup-mysql-server.sh rename to configuration/environments_scripts/mysql_instance_setup/setup-mysql-server.sh index 939ed7b1e08..794de96860b 100755 --- a/configuration/mysql_instance_setup/setup-mysql-server.sh +++ b/configuration/environments_scripts/mysql_instance_setup/setup-mysql-server.sh @@ -32,11 +32,8 @@ else BACKUP_FILE=/home/ec2-user/backupdb.sql backupdbNOLOCK=/home/ec2-user/backupdbNOLOCK.sql # Install cron job for ssh key update for landscape managers - scp -o StrictHostKeyChecking=false root@sapsailing.com:/home/wiki/gitwiki/configuration/update_authorized_keys_for_landscape_managers /tmp - sudo mv /tmp/update_authorized_keys_for_landscape_managers /usr/local/bin - scp -o StrictHostKeyChecking=false root@sapsailing.com:/home/wiki/gitwiki/configuration/update_authorized_keys_for_landscape_managers_if_changed /tmp - sudo mv /tmp/update_authorized_keys_for_landscape_managers_if_changed /usr/local/bin - scp -o StrictHostKeyChecking=false root@sapsailing.com:/home/wiki/gitwiki/configuration/mysql_instance_setup/crontab-ec2-user /home/ec2-user/crontab + scp -o StrictHostKeyChecking=no -p root@sapsailing.com:/home/wiki/gitwiki/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh /home/ec2-user + sudo mv /home/ec2-user/imageupgrade_functions.sh /usr/local/bin/imageupgrade_functions.sh scp -o StrictHostKeyChecking=false root@sapsailing.com:ssh-key-reader.token /home/ec2-user sudo chown ec2-user /home/ec2-user/ssh-key-reader.token sudo chgrp ec2-user /home/ec2-user/ssh-key-reader.token @@ -49,7 +46,12 @@ else sudo systemctl start mariadb.service sudo systemctl enable crond.service sudo systemctl start crond.service - crontab /home/ec2-user/crontab + . imageupgrade_functions.sh + if ! build_crontab_and_setup_files mysql_instance_setup; then + exit 1 + fi + setup_sshd_resilience + sudo chown root:root /usr/local/bin/imageupgrade_functions.sh echo "Creating backup through mysql client on sapsailing.com..." ssh -o StrictHostKeyChecking=false root@sapsailing.com "mysqldump --all-databases -h mysql.internal.sapsailing.com --user=root --password=${ROOT_PW} --master-data --skip-lock-tables --lock-tables=0" >> ${BACKUP_FILE} # the two lock options are supposed to ignore table locks, but the following removes a problematic exception. @@ -62,6 +64,7 @@ else rm ${backupdbNOLOCK} sudo systemctl stop mariadb.service sudo systemctl start mariadb.service + echo "Showing bug count:" sudo mysql -u root -p${ROOT_PW} -e "select count(bug_id) from bugs.bugs;" echo 'Test your DB, e.g., by counting bugs: sudo mysql -u root -p -e "use bugs; select count(*) from bugs;"' echo "If you like what you see, switch to the new DB by updating the mysql.internal.sapsailing.com DNS record to this instance," diff --git a/configuration/environments_scripts/mysql_instance_setup/users/ec2-user/crontab-update-authorized-keys b/configuration/environments_scripts/mysql_instance_setup/users/ec2-user/crontab-update-authorized-keys deleted file mode 120000 index d9b0fed0726..00000000000 --- a/configuration/environments_scripts/mysql_instance_setup/users/ec2-user/crontab-update-authorized-keys +++ /dev/null @@ -1 +0,0 @@ -../../../../crontabs/crontab-update-authorized-keys \ No newline at end of file diff --git a/configuration/environments_scripts/mysql_instance_setup/users/ec2-user/crontab-update-authorized-keys@USER_HOME_DIR=_home_ec2-user b/configuration/environments_scripts/mysql_instance_setup/users/ec2-user/crontab-update-authorized-keys@USER_HOME_DIR=_home_ec2-user new file mode 120000 index 00000000000..cf597d8e543 --- /dev/null +++ b/configuration/environments_scripts/mysql_instance_setup/users/ec2-user/crontab-update-authorized-keys@USER_HOME_DIR=_home_ec2-user @@ -0,0 +1 @@ +../../../../crontabs/crontab-update-authorized-keys@USER_HOME_DIR \ No newline at end of file diff --git a/configuration/rabbitmq_instance_setup/setup-rabbitmq-server.sh b/configuration/environments_scripts/rabbitmq_instance_setup/setup-rabbitmq-server.sh similarity index 59% rename from configuration/rabbitmq_instance_setup/setup-rabbitmq-server.sh rename to configuration/environments_scripts/rabbitmq_instance_setup/setup-rabbitmq-server.sh index 91845d3d101..18081d9262d 100755 --- a/configuration/rabbitmq_instance_setup/setup-rabbitmq-server.sh +++ b/configuration/environments_scripts/rabbitmq_instance_setup/setup-rabbitmq-server.sh @@ -10,12 +10,6 @@ else # Fix the non-sensical use of "dash" as the default shell: sudo rm /usr/bin/sh sudo ln -s /usr/bin/bash /usr/bin/sh - # Install cron job for ssh key update for landscape managers - scp -o StrictHostKeyChecking=false root@sapsailing.com:/home/wiki/gitwiki/configuration/update_authorized_keys_for_landscape_managers /tmp - sudo mv /tmp/update_authorized_keys_for_landscape_managers /usr/local/bin - scp -o StrictHostKeyChecking=false root@sapsailing.com:/home/wiki/gitwiki/configuration/update_authorized_keys_for_landscape_managers_if_changed /tmp - sudo mv /tmp/update_authorized_keys_for_landscape_managers_if_changed /usr/local/bin - scp -o StrictHostKeyChecking=false root@sapsailing.com:/home/wiki/gitwiki/configuration/rabbitmq_instance_setup/crontab-admin /home/admin/crontab scp -o StrictHostKeyChecking=false root@sapsailing.com:ssh-key-reader.token /home/admin sudo chown admin /home/admin/ssh-key-reader.token sudo chgrp admin /home/admin/ssh-key-reader.token @@ -24,9 +18,16 @@ else sudo apt-get -y update sudo DEBIAN_FRONTEND=noninteractive apt-get -yq -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confnew upgrade sudo DEBIAN_FRONTEND=noninteractive apt-get -yq -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confnew install rabbitmq-server systemd-cron jq syslog-ng - sudo touch /var/run/last_change_aws_landscape_managers_ssh_keys - sudo chown admin:admin /var/run/last_change_aws_landscape_managers_ssh_keys - crontab /home/admin/crontab + sudo touch /var/run/last_change_aws_landscape_managers_ssh_keys__home_admin + sudo chown admin:admin /var/run/last_change_aws_landscape_managers_ssh_keys__home_admin + scp -o StrictHostKeyChecking=false -r root@sapsailing.com:/home/wiki/gitwiki/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh /home/admin + sudo mv imageupgrade_functions.sh /usr/local/bin + . imageupgrade_functions.sh + if ! build_crontab_and_setup_files 'rabbitmq_instance_setup'; then + exit 1 + fi + setup_sshd_resilience + sudo chown root:root /usr/local/bin/imageupgrade_functions.sh # Wait for RabbitMQ to become available; note that install under apt also means start... sleep 10 sudo rabbitmq-plugins enable rabbitmq_management @@ -36,8 +37,4 @@ loopback_users = none EOF " sudo systemctl restart rabbitmq-server.service - echo 'Test your DB, e.g., by counting bugs: sudo mysql -u root -p -e "use bugs; select count(*) from bugs;"' - echo "If you like what you see, switch to the new DB by updating the mysql.internal.sapsailing.com DNS record to this instance," - echo "make sure the instance has the \"Database and Messaging\" security group set," - echo "and tag the instance's root volume with the WeeklySailingInfrastructureBackup=Yes tag." fi diff --git a/configuration/environments_scripts/rabbitmq_instance_setup/users/admin/crontab-update-authorized-keys b/configuration/environments_scripts/rabbitmq_instance_setup/users/admin/crontab-update-authorized-keys deleted file mode 120000 index d9b0fed0726..00000000000 --- a/configuration/environments_scripts/rabbitmq_instance_setup/users/admin/crontab-update-authorized-keys +++ /dev/null @@ -1 +0,0 @@ -../../../../crontabs/crontab-update-authorized-keys \ No newline at end of file diff --git a/configuration/environments_scripts/repo/etc/init.d b/configuration/environments_scripts/repo/etc/init.d deleted file mode 120000 index 2f94ce46b4c..00000000000 --- a/configuration/environments_scripts/repo/etc/init.d +++ /dev/null @@ -1 +0,0 @@ -rc.d/init.d \ No newline at end of file diff --git a/configuration/environments_scripts/repo/etc/logrotate.d/httpd b/configuration/environments_scripts/repo/etc/logrotate.d/httpd new file mode 100644 index 00000000000..8b0d93d9d6b --- /dev/null +++ b/configuration/environments_scripts/repo/etc/logrotate.d/httpd @@ -0,0 +1,15 @@ +/var/log/httpd/*log { + missingok + notifempty + sharedscripts + olddir /var/log/logrotate-target + # use date as a suffix of the rotated file + dateext + compress + delaycompress + postrotate + /usr/bin/systemctl reload httpd.service > /dev/null 2>/dev/null || true + mv /var/log/logrotate-target/*.gz /var/log/old + endscript +} + diff --git a/configuration/environments_scripts/repo/etc/logrotate.d/mongodb b/configuration/environments_scripts/repo/etc/logrotate.d/mongodb new file mode 100644 index 00000000000..4203c9a972c --- /dev/null +++ b/configuration/environments_scripts/repo/etc/logrotate.d/mongodb @@ -0,0 +1,9 @@ +compress +/var/log/mongodb/mongod.log +{ + rotate 5 + weekly + postrotate + /usr/bin/killall -SIGUSR1 mongod + endscript +} diff --git a/configuration/environments_scripts/repo/etc/systemd/system/create-internal-status-file.service b/configuration/environments_scripts/repo/etc/systemd/system/create-internal-status-file.service new file mode 100644 index 00000000000..cc8f15841d3 --- /dev/null +++ b/configuration/environments_scripts/repo/etc/systemd/system/create-internal-status-file.service @@ -0,0 +1,14 @@ +[Unit] +Description=Creates httpd internal-status file. +Requires=-.mount network-online.target +After=-.mount network-online.target get-latest-httpd-conf.service +Before=httpd.service +# Staring before means no "systemctl start httpd" is needed. +[Install] +RequiredBy=multi-user.target +WantedBy=httpd.service + +[Service] +Type=oneshot +RemainAfterExit=true +ExecStart=setupHttpdGitLocal.sh diff --git a/configuration/environments_scripts/repo/etc/systemd/system/ephemeralvolume.service b/configuration/environments_scripts/repo/etc/systemd/system/ephemeralvolume.service new file mode 100644 index 00000000000..657f8823da2 --- /dev/null +++ b/configuration/environments_scripts/repo/etc/systemd/system/ephemeralvolume.service @@ -0,0 +1,11 @@ +[Unit] +Description=Ensures /dev/nvme0n1 or /dev/xvdb is XFS-formatted +Requires=-.mount cloud-init.service network.service +After=-.mount cloud-init.service network.service + +[Install] + +[Service] +Type=oneshot +RemainAfterExit=true +ExecStart=/usr/local/bin/ephemeralvolume diff --git a/configuration/environments_scripts/repo/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service b/configuration/environments_scripts/repo/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service new file mode 100644 index 00000000000..30e6e4367fc --- /dev/null +++ b/configuration/environments_scripts/repo/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service @@ -0,0 +1,19 @@ +[Unit] +Description=Force logrotate upon shutdown and clean files in var/cache +Requires=-.mount network-online.target +After=-.mount network-online.target +Before=shutdown.target +RequiresMountsFor=/ +SurviveFinalKillSignal=yes + +[Install] +RequiredBy=graphical.target +WantedBy=shutdown.target + +[Service] +Type=oneshot +RemainAfterExit=yes +KillMode=none +ExecStart=/bin/true +ExecStopPost=/bin/bash -c "logrotate --force -v /etc/logrotate.conf; /var/www/cgi-bin/reverseProxyHealthcheck.sh cleanup" +TimeoutStopSec=50 diff --git a/configuration/environments_scripts/repo/usr/local/bin/add-as-replica b/configuration/environments_scripts/repo/usr/local/bin/add-as-replica new file mode 100755 index 00000000000..6608a2cf544 --- /dev/null +++ b/configuration/environments_scripts/repo/usr/local/bin/add-as-replica @@ -0,0 +1,25 @@ +#!/bin/bash +user_data=$( ec2-metadata -d | sed -e 's/^user-data: //' ) +if echo ${user_data} | grep -q "^image-upgrade$"; then + echo "Image upgrade... didn't expect to get this far because ephemeralvolume should have triggered upgrade and shutdown. Not registering MongoDB replica" +else + eval ${user_data} + if [ -z "$REPLICA_SET_NAME" ]; then + REPLICA_SET_NAME=live + fi + if [ -z "$REPLICA_SET_PRIMARY" ]; then + REPLICA_SET_PRIMARY=mongo0.internal.sapsailing.com:27017 + fi + if [ -z "$REPLICA_SET_PRIORITY" ]; then + REPLICA_SET_PRIORITY=1 + fi + if [ -z "$REPLICA_SET_VOTES" ]; then + REPLICA_SET_VOTES=0 + fi + if [ -n "$REPLICA_SET_PRIMARY" ]; then + IP=$(ec2-metadata -o | sed -e 's/^local-ipv4: //') + echo "rs.add({host: \"$IP:27017\", priority: $REPLICA_SET_PRIORITY, votes: $REPLICA_SET_VOTES})" | mongo "mongodb://$REPLICA_SET_PRIMARY/?replicaSet=$REPLICA_SET_NAME&retryWrites=true" + else + echo "rs.initiate()" | mongo + fi +fi diff --git a/configuration/environments_scripts/repo/usr/local/bin/awsmfalogon.sh b/configuration/environments_scripts/repo/usr/local/bin/awsmfalogon.sh new file mode 100755 index 00000000000..797ece6d697 --- /dev/null +++ b/configuration/environments_scripts/repo/usr/local/bin/awsmfalogon.sh @@ -0,0 +1,15 @@ +# Use like this: +# . awsmfalogon.sh {mfaDeviceArn} {tokenCode} +# It will set the necessary environment variables that will allow the "aws" client to work +# with a session token +# Use with a bash alias definition like this, replacing {ARN-of-your-MFA-device} with the ARN of your MFA device: +# alias awsmfa='echo -n "Token: "; read aws_mfa_token; . awsmfalogon.sh "{ARN-of-your-MFA-device}" ${aws_mfa_token}' +# Then, you can invoke the alias "awsmfa" on your bash command line, and you will get prompted for an MFA token +# which, when entered, will add the necessary environment variables to your bash session that will allow your aws +# client to function with a valid session key. +mfaDeviceArn=$1 +tokenCode=$2 +jsonOutput="$(aws sts get-session-token --serial-number "${mfaDeviceArn}" --token-code ${tokenCode})" +export AWS_ACCESS_KEY_ID=$( echo "${jsonOutput}" | jq --raw-output '.Credentials.AccessKeyId' ) +export AWS_SECRET_ACCESS_KEY=$( echo "${jsonOutput}" | jq --raw-output '.Credentials.SecretAccessKey' ) +export AWS_SESSION_TOKEN=$( echo "${jsonOutput}" | jq --raw-output '.Credentials.SessionToken' ) diff --git a/configuration/environments_scripts/repo/usr/local/bin/ephemeralvolume b/configuration/environments_scripts/repo/usr/local/bin/ephemeralvolume new file mode 100755 index 00000000000..563d6127bad --- /dev/null +++ b/configuration/environments_scripts/repo/usr/local/bin/ephemeralvolume @@ -0,0 +1,32 @@ +#!/bin/bash + +# Script to deploy on an instance that has an ephemeral volume as /dev/nvme0n1 (adjust env var PARTITION if different) +# Ensures the partition is xfs-formatted, any existing partition contents will be overwritten if formatted otherwise. +# An existing xfs partition will be left alone. +METADATA=$( /bin/ec2-metadata -d | sed -e 's/^user-data: //' ) +echo "Metadata: ${METADATA}" +if echo "${METADATA}" | grep -q "^image-upgrade$"; then + echo "Image upgrade; not trying to mount/format ephemeral volume" + exit 0 +else + echo "No image upgrade; looking for ephemeral volume and trying to format with xfs..." + . imageupgrade_functions.sh + PARTITION_NAME=$(identify_suitable_partition_for_ephemeral_volume) + if [[ -n "${PARTITION_NAME}" ]]; then + PARTITION=/dev/${PARTITION_NAME} + if awk '{print $1;}' /proc/mounts | grep "${PARTITION}"; then + echo "Partition ${PARTITION} already mounted; not formatting/mounting ephemeral volume" + else + FSTYPE=$(blkid -p $PARTITION -s TYPE -o value) + if [[ "$FSTYPE" != "xfs" ]]; then + echo FSTYPE was "$FSTYPE" but should have been xfs. Formatting $PARTITION... + mkfs.xfs -f $PARTITION + else + echo FSTYPE was "$FSTYPE" which is just right :-\) + fi + # mount the thing to /var/lib/mongo + mount $PARTITION /var/lib/mongo + chown mongod:mongod /var/lib/mongo # Added for robustness + fi + fi +fi diff --git a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh index 52ca7a7c740..3d20e5f4e76 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh @@ -7,59 +7,64 @@ REBOOT_INDICATOR=/var/lib/sailing/is-rebooted LOGON_USER_HOME=/root run_yum_update() { - echo "Updating packages using yum" >>/var/log/sailing.err + echo "Updating packages using yum" yum -y update } +run_dnf_upgrade() { + echo "Upgrading using dnf" + dnf -y upgrade --releasever=latest +} + run_apt_update_upgrade() { - echo "Updating packages using apt" >>/var/log/sailing.err + echo "Updating packages using apt" apt-get -y update; apt-get -y upgrade apt-get -y install linux-image-cloud-amd64 apt-get -y autoremove } run_git_pull() { - echo "Pulling git to /home/sailing/code" >>/var/log/sailing.err + echo "Pulling git to /home/sailing/code" su - sailing -c "cd code; git pull" } download_and_install_latest_sap_jvm_8() { - echo "Downloading and installing latest SAP JVM 8 to /opt/sapjvm_8" >>/var/log/sailing.err - vmpath=$( curl -s --cookie eula_3_1_agreed=tools.hana.ondemand.com/developer-license-3_1.txt https://tools.hana.ondemand.com | grep additional/sapjvm-8\..*-linux-x64.zip | head -1 | sed -e 's/^.*a href="\(additional\/sapjvm-8\..*-linux-x64\.zip\)".*/\1/' ) + echo "Downloading and installing latest SAP JVM 8 to /opt/sapjvm_8" + vmpath=$( curl -s --cookie eula_3_2_agreed=tools.hana.ondemand.com/developer-license-3_2.txt https://tools.hana.ondemand.com | grep additional/sapjvm-8\..*-linux-x64.zip | head -1 | sed -e 's/^.*a href="\(additional\/sapjvm-8\..*-linux-x64\.zip\)".*/\1/' ) if [ -n "${vmpath}" ]; then - echo "Found VM version ${vmpath}; upgrading installation at /opt/sapjvm_8" >>/var/log/sailing.err + echo "Found VM version ${vmpath}; upgrading installation at /opt/sapjvm_8" if [ -z "${TMP}" ]; then TMP=/tmp fi - echo "Downloading SAP JVM 8 as ZIP file to ${TMP}/sapjvm8-linux-x64.zip" >>/var/log/sailing.err - curl --cookie eula_3_1_agreed=tools.hana.ondemand.com/developer-license-3_1.txt "https://tools.hana.ondemand.com/${vmpath}" > ${TMP}/sapjvm8-linux-x64.zip 2>>/var/log/sailing.err + echo "Downloading SAP JVM 8 as ZIP file to ${TMP}/sapjvm8-linux-x64.zip" + curl --cookie eula_3_2_agreed=tools.hana.ondemand.com/developer-license-3_2.txt "https://tools.hana.ondemand.com/${vmpath}" > ${TMP}/sapjvm8-linux-x64.zip cd /opt rm -rf sapjvm_8 if [ -f SIGNATURE.SMF ]; then rm -f SIGNATURE.SMF fi - unzip ${TMP}/sapjvm8-linux-x64.zip >>/var/log/sailing.err + unzip ${TMP}/sapjvm8-linux-x64.zip rm -f ${TMP}/sapjvm8-linux-x64.zip rm -f SIGNATURE.SMF else - echo "Did not find SAP JVM 8 at tools.hana.ondemand.com; not trying to upgrade" >>/var/log/sailing.err + echo "Did not find SAP JVM 8 at tools.hana.ondemand.com; not trying to upgrade" fi } clean_logrotate_target() { - echo "Clearing logrorate-targets" >>/var/log/sailing.err + echo "Clearing logrorate-targets" rm -rf /var/log/logrotate-target/* } clean_httpd_logs() { - echo "Clearing httpd logs" >>/var/log/sailing.err + echo "Clearing httpd logs" service httpd stop rm -rf /var/log/httpd/* rm -f /etc/httpd/conf.d/001-internals.conf } clean_startup_logs() { - echo "Clearing bootstrap logs" >>/var/log/sailing.err + echo "Clearing bootstrap logs" rm -f /var/log/sailing* # Ensure that upon the next boot the reboot indicator is not present, indicating that it's the first boot rm "${REBOOT_INDICATOR}" @@ -69,76 +74,212 @@ clean_servers_dir() { rm -rf /home/sailing/servers/* } -#DEPRECATED -update_root_crontab() { - # The following assumes that /root/crontab is a symbolic link to /home/sailing/code/configuration/crontabs/ - # which has previously been updated by a git pull: - cd /root - crontab crontab -} - build_crontab_and_setup_files() { - #1: Environment type. - local ENVIRONMENT_TYPE="$1" - #2 git copy user - local GIT_COPY_USER="$2" - #3 relative path to git within the git user - local RELATIVE_PATH_TO_GIT="$3" - if [[ "$#" -lt 3 || "$#" -gt 5 ]]; then - echo "Number of arguments is invalid" + if [[ "$#" -eq 0 ]]; then + echo "Number of arguments is invalid. Please use the options and arguments as follows." + echo "Options for this function:" + echo " -h is the hostname to fetch the configuration/environments_scripts from." + echo "The remaining (optional) args and options, if correct, are passed to the build_crontab_and_setup_files script." + echo " -c means no crontab file is created" + echo " -n means that if a crontab has been created, it isn't actually installed. This is useful for testing." + echo " -f means no files are copied over, which is useful if you have already copied files accross or don't want to override existing files" + echo "Then there are the arguments, where the order matters:" + echo " ENVIRONMENT_TYPE - the directory name in environments_scripts which will be used." else - TEMP_ENVIRONMENTS_SCRIPTS=$(mktemp -d /root/environments_scripts_XXX) - scp -o StrictHostKeyChecking=no -pr "wiki@sapsailing.com:~/gitwiki/configuration/environments_scripts/*" "${TEMP_ENVIRONMENTS_SCRIPTS}" - chown root:root "$TEMP_ENVIRONMENTS_SCRIPTS" + TEMP=$(getopt -o fnch: -n 'options checker' -- "$@") + [[ "$?" -eq 0 ]] || return 2 + eval set -- "$TEMP" + PASS_OPTIONS=() + HOSTNAME="sapsailing.com" + while :; do + case "$1" in + -c|-f|-n) + PASS_OPTIONS+=("$1") + ;; + -h) + if [[ "$2" ]]; then + HOSTNAME="$2" + shift + else + echo "hostname option requires argument" + fi + ;; + --) + shift + break + ;; + *) + echo "no more options" + break + esac + shift + done + TEMP_ENVIRONMENTS_SCRIPTS=$(mktemp -d /var/tmp/environments_scripts_XXX) + echo "Attempting access to the wiki, typically used by image upgrade. Otherwise, try root. NOTE: If the first command fails, there will be a warning message." + # During image upgrades, no environment type should have root access to the central server, but they do need + # access to the wiki copy. Therefore, the various keys for different environment types are in the authorized_keys + # of the wiki user. So, during image upgrade or to get the latest changes, the following command should succeed. + # The wiki user's authorized_keys is not updated automatically with landscape managers, so + # the below command may fail during initial image setup. In this scenario, the root user is instead the target + # user of the scp command (as seen in the second command below). + scp -o StrictHostKeyChecking=no -pr wiki@"$HOSTNAME":~/gitwiki/configuration/environments_scripts/* "${TEMP_ENVIRONMENTS_SCRIPTS}" + [[ "$?" -eq 0 ]] || scp -o StrictHostKeyChecking=no -pr root@"$HOSTNAME":/home/wiki/gitwiki/configuration/environments_scripts/* "${TEMP_ENVIRONMENTS_SCRIPTS}" # For initial setup as not all landscape managers have direct wiki access. + sudo chown root:root "${TEMP_ENVIRONMENTS_SCRIPTS}" + sudo chmod 777 "${TEMP_ENVIRONMENTS_SCRIPTS}" cd "${TEMP_ENVIRONMENTS_SCRIPTS}" - ./build-crontab-and-cp-files "${ENVIRONMENT_TYPE}" "${GIT_COPY_USER}" "${RELATIVE_PATH_TO_GIT}" + # Add all args to array, otherwise, if PASS_OPTIONS is empty, and we also pass $@ then argument $1 is in fact null, which would cause errors. + for option in "$@"; do + PASS_OPTIONS+=( "$option" ) + done + if ! sudo ./build-crontab-and-cp-files "${PASS_OPTIONS[@]}"; then + return 1 + fi cd .. - rm -rf "$TEMP_ENVIRONMENTS_SCRIPTS" + sudo rm -rf "$TEMP_ENVIRONMENTS_SCRIPTS" fi } +# Assumes a filename following the pattern "crontab*[@{param}={value} ...]" +# It prints an "sed" command on the standard output that callers can use to +# obtain a piping command through which to send the file's contents to achieve +# the text pattern replacements requested by the @-parts in the file/link +# name. +# If a value shall contain the @ literal, it needs to be doubled, as, e.g., in +# crontab-some-mail-thing@EMAIL_ADDRESS=john.doe@@example.com +# Multiple replacements may be requested by adding more @-phrases: +# crontab-something@A=x@B=y +# Example usage: +# echo "The user USER has e-mail address EMAIL_ADDRESS and home directory HOME_DIR" | eval $( get_replacements_from_crontab_filename_as_sed_command crontab-humba@USER=uhl@EMAIL_ADDRESS=axel.uhl@@sap.com@HOME_DIR=_home_axel__uhl_ ) +# will output +# The user uhl has e-mail address axel.uhl@sap.com and home directory /home/axel_uhl/ +get_replacements_from_crontab_filename_as_sed_command() { + LINK_NAME="${1}" + AT_SIGN_REPLACEMENT=`mktemp -u XXXXXXXXXXXXXXXX` + UNDERSCORE_REPLACEMENT=`mktemp -u XXXXXXXXXXXXXXXX` + LINK_NAME_WITH_UNDERSCORES_REPLACED=$( echo "${LINK_NAME}" | sed -e 's|__|'${UNDERSCORE_REPLACEMENT}'|g' ) + LINK_NAME_WITH_AT_LITERALS_REPLACED=$( echo "${LINK_NAME_WITH_UNDERSCORES_REPLACED}" | sed -e 's|@@|'${AT_SIGN_REPLACEMENT}'|g' ) + IFS='@' read -r -a BASENAME_AND_SUBSTITUTIONS <<<"${LINK_NAME_WITH_AT_LITERALS_REPLACED}" + # Now, BASENAME_AND_SUBSTITUTIONS[0] is the link's base name, 1..n are the replacements + SED_COMMAND="sed -e 's| | |'" # start with a no-op so that in case of no replacements we still have a valid sed command + for REPLACEMENT in ${BASENAME_AND_SUBSTITUTIONS[@]:1}; do + IFS='=' read -r -a PARAM_AND_VALUE <<<"${REPLACEMENT}" + PARAM="${PARAM_AND_VALUE[0]}" + VALUE_WITH_SLASHES=$( echo "${PARAM_AND_VALUE[1]}" | sed -e 's|_|/|g' ) + VALUE_WITH_UNDERSCORES=$( echo "${VALUE_WITH_SLASHES}" | sed -e 's|'${UNDERSCORE_REPLACEMENT}'|_|g' ) + VALUE=$( echo "${VALUE_WITH_UNDERSCORES}" | sed -e 's|'${AT_SIGN_REPLACEMENT}'|@|g' ) + SED_COMMAND="${SED_COMMAND} -e 's|${PARAM}|${VALUE}|g'" + done + echo "${SED_COMMAND}" +} + setup_keys() { - #1: Environment type. - TEMP_KEY_DIR=$(mktemp -d /root/keysXXXXX) - REGION=$(TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" --silent -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"` \ - && curl -H "X-aws-ec2-metadata-token: $TOKEN" --silent http://169.254.169.254/latest/meta-data/placement/region) + # Installs the necessary aws and ssh keys for a given environment type, by copying from the key vault. + # $1: Environment type. + # Optional parameter is -p which indicates that no permissions will be set or overwritten. + TEMP=$(getopt -o p -n 'options' -- "$@") + [[ "$?" -eq 0 ]] || return 2 + SET_PERMISSIONS="true" + eval set -- "$TEMP" + while true; do + case "$1" in + -p) + SET_PERMISSIONS="false" + ;; + --) + shift + break + ;; + *) + echo "Option not recognised" + return 2 + ;; + esac + shift + done + if [[ "$#" -ne 1 ]]; then + echo "Please specify the environment type and use the optional -p flag to indicate that no permissions will be set or overwritten." + return 2 + fi + pushd . + TEMP_KEY_DIR=$(mktemp -d /var/tmp/keysXXXXX) scp -o StrictHostKeyChecking=no -pr root@sapsailing.com:/root/key_vault/"${1}"/* "${TEMP_KEY_DIR}" + sudo su - -c "source imageupgrade_functions.sh; __setup_keys_using_local_copy $TEMP_KEY_DIR $SET_PERMISSIONS" + popd + rm -rf "${TEMP_KEY_DIR}" +} +__setup_keys_using_local_copy() { + # $1 the local location of the key_vault subdirectory corresponding to the image type this is run on. + # $2 a "true" or "false" string, indicating whether to override existing permissions. + # "true" indicates the permissions and ownership of the .ssh and .aws folders will not be set. + TEMP_KEY_DIR="$1" + SET_PERMISSIONS="$2" + REGION="$( ec2-metadata | grep "^placement:" | sed -e 's/^.*: \(.*\).$/\1/')" cd "${TEMP_KEY_DIR}" - for user in $(ls); do - if id -u "$user"; then + for user in *; do + [[ -e "$user" ]] || continue + if id -u "$user" > /dev/null; then user_home_dir=$(getent passwd $(id -u "$user") | cut -d: -f6) # getent searches for passwd based on user id, which the "id" command supplies. # aws setup if [[ -d "${user}/aws" ]]; then mkdir --parents "${user_home_dir}/.aws" - chmod 755 "${user_home_dir}/.aws" - \cp -r --preserve --dereference "${user}"/aws/* "${user_home_dir}/.aws" - echo "[default]" >> "${user_home_dir}/.aws/config" - echo "region = ${REGION}" >> "${user_home_dir}"/.aws/config - chown -R ${user}:${user} "${user_home_dir}/.aws" - chmod 600 "${user_home_dir}"/.aws/* + # Setup credentials + if [[ -d "${user}/aws/credentials" && ! -e "${user_home_dir}/.aws/credentials" ]]; then + > "${user_home_dir}"/.aws/credentials + for credentials in "${user}"/aws/credentials/*; do + [[ -f "$credentials" ]] || continue + cat "$credentials" >> "${user_home_dir}"/.aws/credentials + echo "" >> "${user_home_dir}"/.aws/credentials + done + fi + # Setup config + if [[ ! -e "${user_home_dir}/.aws/config" ]]; then + echo "[default]" >> "${user_home_dir}/.aws/config" + echo "region = ${REGION}" >> "${user_home_dir}"/.aws/config + echo "" >> "${user_home_dir}"/.aws/config + if [[ -d "${user}/aws/config" ]]; then + for config in "${user}"/aws/config/*; do + [[ -f "$config" ]] || continue + cat "$config" >> "${user_home_dir}"/.aws/config + echo "region = ${REGION}" >> "${user_home_dir}"/.aws/config + echo "" >> "${user_home_dir}"/.aws/config + done + fi + fi + if [[ "$SET_PERMISSIONS" == "true" ]]; then + chmod 755 "${user_home_dir}"/.aws + chown -R ${user}:${user} "${user_home_dir}/.aws" + chmod 600 "${user_home_dir}"/.aws/* + fi fi # ssh setup if [[ -d "${user}/ssh" ]]; then mkdir --parents "${user_home_dir}/.ssh" - chmod 700 "${user_home_dir}/.ssh" - \cp --preserve --dereference $(find ${user}/ssh -maxdepth 1 -type f) "${user_home_dir}/.ssh" - for key in $(find ${user}/ssh/authorized_keys -type f); do - cat "${key}" >> ${user_home_dir}/.ssh/authorized_keys + for key in "${user}"/ssh/*; do + [[ -f "$key" ]] || continue + [[ ! -f "$user_home_dir"/.ssh/"$(basename "$key")" ]] || { echo "$key not touched, as it is already present in ssh dir of $user. You may want to override it." && continue; } + \cp --preserve --dereference "$key" "$user_home_dir"/.ssh done - chown -R ${user}:${user} "${user_home_dir}/.ssh" - chmod 600 "${user_home_dir}"/.ssh/* + for key in "${user}"/ssh/authorized_keys/*; do + [[ -f "$key" ]] || continue + if ! grep -q "$(cat "$key")" "${user_home_dir}"/.ssh/authorized_keys; then + cat "${key}" >> "${user_home_dir}"/.ssh/authorized_keys + fi + done + if [[ "$SET_PERMISSIONS" == "true" ]]; then + chmod 700 "${user_home_dir}/.ssh" + chown -R ${user}:${user} "${user_home_dir}/.ssh" + chmod 600 "${user_home_dir}"/.ssh/* + fi fi fi done - cd / - rm -rf "${TEMP_KEY_DIR}" } clean_root_ssh_dir_and_tmp() { - echo "Cleaning up ${LOGON_USER_HOME}/.ssh" >>/var/log/sailing.err - rm -rf ${LOGON_USER_HOME}/.ssh/* - rm -f /var/run/last_change_aws_landscape_managers_ssh_keys + echo "Cleaning up ${LOGON_USER_HOME}/.ssh" + rm -rf ${LOGON_USER_HOME}/.ssh/authorized_keys + rm -rf ${LOGON_USER_HOME}/.ssh/known_hosts + rm -f /var/run/last_change_aws_landscape_managers_ssh_keys* rm -rf /tmp/image-upgrade-finished } @@ -160,14 +301,15 @@ finalize() { } setup_cloud_cfg_and_root_login() { - sed -i 's/#PermitRootLogin yes/PermitRootLogin without-password\nPermitRootLogin yes/' /etc/ssh/sshd_config - sed -i 's/^disable_root: true$/disable_root: false/' /etc/cloud/cloud.cfg - echo "preserve_hostname: true" >> /etc/cloud/cloud.cfg + sudo sed -i 's/#PermitRootLogin yes/PermitRootLogin without-password\nPermitRootLogin yes/' /etc/ssh/sshd_config + sudo sed -i 's/^disable_root: *true$/disable_root: false/' /etc/cloud/cloud.cfg } setup_fail2ban() { + pushd . if [[ ! -f "/etc/systemd/system/fail2ban.service" ]]; then yum install 2to3 -y + cd /usr/local/src wget https://github.com/fail2ban/fail2ban/archive/refs/tags/1.0.2.tar.gz tar -xvf 1.0.2.tar.gz cd fail2ban-1.0.2/ @@ -189,14 +331,32 @@ setup_fail2ban() { logpath = /var/log/fail2ban.log maxretry = 5 EOF + touch /var/log/fail2ban.log service fail2ban start yum remove -y firewalld + popd } setup_mail_sending() { - yum install -y mailx postfix - systemctl enable postfix - temp_mail_properties_location=$(mktemp /root/mail.properties_XXX) + # Sets up mail sending using Amazon Simple Email Service by configuring postfix. + # + # $1 is an optional argument which is prepended to the myorigin variable in the postfix conf. + # This variable controls the sender's email address. + # eg. "setup_mail_sending disposable" will result in mail from disposable.sapsailing.com. + # DO NOT include spaces in the parameter, even if you use quotes. + # The default is the local hostname. + if sudo grep "^relayhost\>" /etc/postfix/main.cf &>/dev/null; then #& here redirects stdout and stderr. \> means there must be a word boundary. + echo "Postfix is already installed and has some custom configuration. Please check it is configured correctly at /etc/postfix/main.cf" + echo "If you wish to change the sender's email address then edit the myorigin variable in /etc/postfix/main.cf" + return 0 + fi + subdomain_of_sender_address="$( ec2-metadata --local-hostname | sed "s/local-hostname: *//")" + if [[ -n "$1" ]]; then + subdomain_of_sender_address="$1" + fi + sudo yum install -y mailx postfix + sudo systemctl enable postfix + temp_mail_properties_location=$(mktemp /var/tmp/mail.properties_XXX) scp -o StrictHostKeyChecking=no -p root@sapsailing.com:mail.properties "${temp_mail_properties_location}" cd $(dirname "${temp_mail_properties_location}") local smtp_host="$(sed -n "s/mail.smtp.host \?= \?\(.*\)/\1/p" ${temp_mail_properties_location})" @@ -204,7 +364,7 @@ setup_mail_sending() { local smtp_user="$(sed -n "s/mail.smtp.user \?= \?\(.*\)/\1/p" ${temp_mail_properties_location})" local smtp_pass="$(sed -n "s/mail.smtp.password \?= \?\(.*\)/\1/p" ${temp_mail_properties_location})" local password_file_location="/etc/postfix/sasl_passwd" - echo "relayhost = [${smtp_host}]:${smtp_port} + sudo su -c "echo \"relayhost = [${smtp_host}]:${smtp_port} smtp_sasl_auth_enable = yes smtp_sasl_security_options = noanonymous smtp_sasl_password_maps = hash:${password_file_location} @@ -212,18 +372,97 @@ smtp_use_tls = yes smtp_tls_security_level = encrypt smtp_tls_note_starttls_offer = yes -myorigin =\$myhostname.sapsailing.com -" >> /etc/postfix/main.cf - sed -i "/smtp_tls_security_level = may/d" /etc/postfix/main.cf - echo "[${smtp_host}]:${smtp_port} ${smtp_user}:${smtp_pass}" >> ${password_file_location} - postmap hash:${password_file_location} - systemctl restart postfix +myorigin =${subdomain_of_sender_address}.sapsailing.com +\" >> /etc/postfix/main.cf" + sudo sed -i "/smtp_tls_security_level = may/d" /etc/postfix/main.cf + sudo su -c "echo \"[${smtp_host}]:${smtp_port} ${smtp_user}:${smtp_pass}\" >> ${password_file_location}" + sudo postmap hash:${password_file_location} + sudo systemctl restart postfix rm -f "${temp_mail_properties_location}" } setup_sshd_resilience() { - echo "ClientAliveInterval 3 + sudo su -c "echo 'ClientAliveInterval 3 ClientAliveCountMax 3 -GatewayPorts yes" >> /etc/ssh/sshd_config - systemctl reload sshd.service -} \ No newline at end of file +GatewayPorts yes +MaxStartups 100' >> /etc/ssh/sshd_config && systemctl reload sshd.service" +} + +identify_suitable_partition_for_ephemeral_volume() { + EPHEMERAL_VOLUME_NAME=$( + # List all block devices and find those named nvme... + for i in $(lsblk | grep -o "nvme[0-9][0-9]\?n[0-9]" | sort -u); do + # If they don't have any partitions, then... + if ! lsblk | grep -o "${i}p[0-9]\+" 2>&1 >/dev/null; then + # ...check whether they are EBS devices + /sbin/ebsnvme-id -u "/dev/$i" >/dev/null + # If not, list their name because then they must be ephemeral instance storage + if [[ $? -ne 0 ]]; then + echo "${i}" + fi + fi + done 2>/dev/null | head -n 1 ) + echo $EPHEMERAL_VOLUME_NAME +} + +setup_goaccess() { + # Compatible with Amazon Linux 2023 + pushd . + cd /usr/local/src + wget https://tar.goaccess.io/goaccess-1.9.1.tar.gz + tar -xzvf goaccess-1.9.1.tar.gz + cd goaccess-1.9.1/ + yum install -y gcc-c++ + yum install -y libmaxminddb-devel ncurses-devel + ./configure --enable-utf8 + make + make install + # old location: + # scp root@sapsailing.com:/etc/goaccess.conf /usr/local/etc/goaccess/goaccess.conf + # once we switch from amazon linux 1: + scp root@sapsailing.com:/usr/local/etc/goaccess/goaccess.conf /usr/local/etc/goaccess/goaccess.conf + popd +} +setup_apachetop() { + # Compatible with Amazon Linux 2023 + pushd . + yum install -y gcc-c++ + yum install -y ncurses-devel readline-devel + cd /usr/local/src + wget https://github.com/tessus/apachetop/releases/download/0.23.2/apachetop-0.23.2.tar.gz + tar -xvzf apachetop-0.23.2.tar.gz + cd apachetop-0.23.2 + ./configure + make + make install + popd +} + +setup_swap() { + # $1: size of swapspace in megabytes. + echo "Creating swapspace of $1 MBs" + local swapfile_location=/var/cache/swapfile + pushd . + sudo dd if=/dev/zero of="$swapfile_location" bs=1M count="$1" + sudo chmod 600 "$swapfile_location" + sudo chown root:root "$swapfile_location" + sudo mkswap "$swapfile_location" + sudo su - -c "echo \"$swapfile_location none swap pri=0 0 0\" >> /etc/fstab" + sudo swapon -a + popd +} + +setup_mongo_4_4() { + # Install MongoDB 4.4 and configure as replica set "live" + sudo su - -c "cat << EOF >/etc/yum.repos.d/mongodb-org.4.4.repo +[mongodb-org-4.4] +name=MongoDB Repository +baseurl=https://repo.mongodb.org/yum/amazon/2/mongodb-org/4.4/x86_64/ +gpgcheck=1 +enabled=1 +gpgkey=https://www.mongodb.org/static/pgp/server-4.4.asc +EOF +" + sudo yum -y update + sudo yum -y install mongodb-org-server mongodb-org-shell mongodb-org-tools +} diff --git a/configuration/environments_scripts/repo/usr/local/bin/mountnvmeswap b/configuration/environments_scripts/repo/usr/local/bin/mountnvmeswap index b94ea5a0c83..e900739a207 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/mountnvmeswap +++ b/configuration/environments_scripts/repo/usr/local/bin/mountnvmeswap @@ -2,30 +2,23 @@ # Script to deploy on an instance that has an ephemeral volume as /dev/nvme0n1 (adjust env var EPHEMERAL_VOLUME if different) # Ensures the partition is xfs-formatted, any existing partition contents will be overwritten if formatted otherwise. -# An existing xfs partition will be left alone. -EPHEMERAL_VOLUME_NAME=$( - # List all block devices and find those named nvme... - for i in $(lsblk | grep -o "nvme[0-9][0-9]\?n[0-9]" | sort -u); do - # If they don't have any partitions, then... - if ! lsblk | grep -o "${i}p[0-9]\+" 2>&1 >/dev/null; then - # ...check whether they are EBS devices - /sbin/ebsnvme-id -u "/dev/$i" >/dev/null - # If not, list their name because then they must be ephemeral instance storage - if [[ $? -ne 0 ]]; then - echo "${i}" - fi - fi - done 2>/dev/null | head -n 1 ) -if [ -n "${EPHEMERAL_VOLUME_NAME}" ]; then +# An existing xfs partition will be left alone. +. imageupgrade_functions.sh +EPHEMERAL_VOLUME_NAME=$(identify_suitable_partition_for_ephemeral_volume) +if [[ -n "${EPHEMERAL_VOLUME_NAME}" ]]; then EPHEMERAL_VOLUME=/dev/${EPHEMERAL_VOLUME_NAME} - FSTYPE=$(blkid -p $EPHEMERAL_VOLUME -s TYPE -o value) - if [ "$FSTYPE" = "" ]; then - echo FSTYPE was empty, creating swap partition - mkswap $EPHEMERAL_VOLUME - swapon --priority 2 -a $EPHEMERAL_VOLUME + if awk '{print $1;}' /proc/mounts | grep "${EPHEMERAL_VOLUME}"; then + echo "Partition ${PARTITION} already mounted; not formatting/mounting volume" else - echo "FSTYPE was $FSTYPE, not touching" + FSTYPE=$(blkid -p $EPHEMERAL_VOLUME -s TYPE -o value) + if [ "$FSTYPE" = "" ]; then + echo FSTYPE was empty, creating swap partition + mkswap $EPHEMERAL_VOLUME + swapon --priority 2 -a $EPHEMERAL_VOLUME + else + echo "FSTYPE was $FSTYPE, not touching" + fi fi else echo "No ephemeral partition found. Not creating any swap partition." -fi +fi \ No newline at end of file diff --git a/configuration/environments_scripts/repo/usr/local/bin/patch-mongo-replicaset-name-from-ec2-metadata b/configuration/environments_scripts/repo/usr/local/bin/patch-mongo-replicaset-name-from-ec2-metadata new file mode 100755 index 00000000000..db43bf746d0 --- /dev/null +++ b/configuration/environments_scripts/repo/usr/local/bin/patch-mongo-replicaset-name-from-ec2-metadata @@ -0,0 +1,8 @@ +#!/bin/bash +REPLICA_SET_NAME=$(ec2-metadata | grep REPLICA_SET_NAME | sed -e 's/^user-data: //' | sed -e 's/^REPLICA_SET_NAME=//') +echo Replica set name: $REPLICA_SET_NAME +if [ \! -z "$REPLICA_SET_NAME" ]; then + echo "Not empty. Patching /etc/mongod.conf..." + sed -i -e "s/replSetName: .*$/replSetName: $REPLICA_SET_NAME/" /etc/mongod.conf + echo "Done" +fi diff --git a/configuration/environments_scripts/repo/usr/local/bin/remove-as-replica b/configuration/environments_scripts/repo/usr/local/bin/remove-as-replica new file mode 100755 index 00000000000..cf00b796d75 --- /dev/null +++ b/configuration/environments_scripts/repo/usr/local/bin/remove-as-replica @@ -0,0 +1,11 @@ +#!/bin/bash +eval $( ec2-metadata -d | sed -e 's/^user-data: //' ) +if [ -n "$REPLICA_SET_PRIMARY" ]; then + IP=$(ec2-metadata -o | sed -e 's/^local-ipv4: //') + PRIMARY_IP=$(echo "rs.isMaster().primary" | mongo --quiet "mongodb://$IP" | grep -o "^[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+") + if [[ "$IP" == "$PRIMARY_IP" ]]; then + echo "rs.stepDown()" | mongo "mongodb://$PRIMARY_IP" + sleep 10 + fi + echo "rs.remove(\"$IP:27017\")" | mongo "mongodb://$IP/?replicaSet=$REPLICA_SET_NAME&retryWrites=true" +fi diff --git a/configuration/environments_scripts/reverse_proxy/files/root/setupHttpdGitLocal.sh b/configuration/environments_scripts/repo/usr/local/bin/setupHttpdGitLocal.sh similarity index 66% rename from configuration/environments_scripts/reverse_proxy/files/root/setupHttpdGitLocal.sh rename to configuration/environments_scripts/repo/usr/local/bin/setupHttpdGitLocal.sh index fc3471db9de..61d3c65bdf0 100755 --- a/configuration/environments_scripts/reverse_proxy/files/root/setupHttpdGitLocal.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/setupHttpdGitLocal.sh @@ -1,9 +1,15 @@ #!/bin/bash REMOTE=$1 +BRANCH=$2 +GIT_USERNAME="$3" STATUS_DEFINITION_FILE="internal-server-status.conf" SELF_IP=$( ec2-metadata --local-ipv4 | grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+\>") cd /etc/httpd -if ! git status; then +if ! git status &>/dev/null; then + if [[ "$#" -ne 3 ]]; then + echo "Incorrect number of arguments" + exit 1 + fi echo "Currently not a git repo, creating and pulling latest changes" rm -rf .git rm -rf conf # Perhaps, this script should instead mv to a backup location @@ -12,9 +18,8 @@ if ! git status; then git init git remote add origin "${REMOTE}" GIT_SSH_COMMAND="ssh -A -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git fetch - git checkout disposable + git checkout "$BRANCH" + git config user.name "$GIT_USERNAME" + git config user.email "nobody@$sapsailing.com" fi -echo "Use Status ${SELF_IP} internal-server-status" > /etc/httpd/conf.d/${STATUS_DEFINITION_FILE} -cd /etc/httpd -git config user.name "Disposable Reverse Proxy" -git config user.email "$(hostname)" \ No newline at end of file +echo "Use Status ${SELF_IP} internal-server-status" > /etc/httpd/conf.d/${STATUS_DEFINITION_FILE} \ No newline at end of file diff --git a/configuration/environments_scripts/repo/usr/local/bin/switchoverArchive.sh b/configuration/environments_scripts/repo/usr/local/bin/switchoverArchive.sh index a18fcd3b644..a52073a7e8d 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/switchoverArchive.sh +++ b/configuration/environments_scripts/repo/usr/local/bin/switchoverArchive.sh @@ -98,11 +98,11 @@ setFailoverIfNotSet() { logger -t archive "begin check" # --fail option ensures that, if a server error is returned (ie. 5xx/4xx status code), then the status code (stored in $?) will be non zero. # -L follows redirects -curl -s -L --fail --connect-timeout ${TIMEOUT1_IN_SECONDS} "http://${archiveIp}:${ARCHIVE_PORT}/gwt/status" >> /dev/null +curl -s -L --fail --max-time ${TIMEOUT1_IN_SECONDS} "http://${archiveIp}:${ARCHIVE_PORT}/gwt/status" >> /dev/null if [[ $? -ne 0 ]] then logger -t archive "first check failed" - curl -s -L --fail --connect-timeout ${TIMEOUT2_IN_SECONDS} "http://${archiveIp}:${ARCHIVE_PORT}/gwt/status" >> /dev/null + curl -s -L --fail --max-time ${TIMEOUT2_IN_SECONDS} "http://${archiveIp}:${ARCHIVE_PORT}/gwt/status" >> /dev/null if [[ $? -ne 0 ]] then setFailoverIfNotSet diff --git a/configuration/environments_scripts/repo/usr/local/bin/syncgit b/configuration/environments_scripts/repo/usr/local/bin/syncgit index a5f10a035a1..d21d700cdd9 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/syncgit +++ b/configuration/environments_scripts/repo/usr/local/bin/syncgit @@ -1,16 +1,35 @@ #!/bin/sh +# Syncs the necessary branches of a repo at a specified location or, as default, +# /home/wiki/gitwiki. The assumed checked-out branch is master and the tracking branch +# is assumed to be origin master. ADMIN_EMAIL="axel.uhl@sap.com jan.hamann@sapsailing.com" if [ $# -eq 0 ]; then GIT_PATH="/home/wiki/gitwiki" else - GIT_PATH=$1 + GIT_PATH="$1" +fi +cd "$GIT_PATH" +git fetch github main:main >/tmp/wiki-git.out 2>/tmp/wiki-git.err +if [ "$?" != "0" ]; then + cat /tmp/wiki-git.out /tmp/wiki-git.err | mail -s "Wiki git problem: could not fetch github/main into local main" $ADMIN_EMAIL +fi +git push origin main:main >/tmp/wiki-git.out 2>/tmp/wiki-git.err +if [ "$?" != "0" ]; then + cat /tmp/wiki-git.out /tmp/wiki-git.err | mail -s "Wiki git problem: could not push main into sapsailing/main" $ADMIN_EMAIL +fi +git pull github main >/tmp/wiki-git.out 2>/tmp/wiki-git.err +if [ "$?" != "0" ]; then + cat /tmp/wiki-git.out /tmp/wiki-git.err | mail -s "Wiki git problem: could not merge github/main into sapsailing/master" $ADMIN_EMAIL fi -cd $GIT_PATH git pull >/tmp/wiki-git.out 2>/tmp/wiki-git.err if [ "$?" != "0" ]; then - cat /tmp/wiki-git.out /tmp/wiki-git.err | mail -s "Wiki git problem" $ADMIN_EMAIL + cat /tmp/wiki-git.out /tmp/wiki-git.err | mail -s "Wiki git problem: could not pull origin/master" $ADMIN_EMAIL +fi +git push origin master:main >>/tmp/wiki-git.out 2>/tmp/wiki-git.err +if [ "$?" != "0" ]; then + cat /tmp/wiki-git.out /tmp/wiki-git.err | mail -s "Wiki git problem: could not push master to origin/main" $ADMIN_EMAIL fi git push >>/tmp/wiki-git.out 2>/tmp/wiki-git.err if [ "$?" != "0" ]; then - cat /tmp/wiki-git.out /tmp/wiki-git.err | mail -s "Wiki git problem" $ADMIN_EMAIL + cat /tmp/wiki-git.out /tmp/wiki-git.err | mail -s "Wiki git problem: could not push master to origin/master" $ADMIN_EMAIL fi diff --git a/configuration/environments_scripts/repo/usr/local/bin/update_authorized_keys_for_landscape_managers_if_changed b/configuration/environments_scripts/repo/usr/local/bin/update_authorized_keys_for_landscape_managers_if_changed index 245fb85c075..39c84272b05 100755 --- a/configuration/environments_scripts/repo/usr/local/bin/update_authorized_keys_for_landscape_managers_if_changed +++ b/configuration/environments_scripts/repo/usr/local/bin/update_authorized_keys_for_landscape_managers_if_changed @@ -2,7 +2,9 @@ BEARER_TOKEN="$1" BASE_URL="$2" LOGON_USER_HOME="$3" -LAST_CHANGE_FILE=/var/run/last_change_aws_landscape_managers_ssh_keys +# We append the user home to the LAST_CHANGE_FILE so that if the script is ran for multiple users, they each have their +# own LAST_CHANGE_FILE. Note, that we have to replace the slash character. +LAST_CHANGE_FILE=/var/run/last_change_aws_landscape_managers_ssh_keys_"$(echo "$LOGON_USER_HOME" | sed "s|/|_|g")" # Uncomment the following for production use, with no error output curl_output=$( curl -H 'X-SAPSSE-Forward-Request-To: master' -H 'Authorization: Bearer '${BEARER_TOKEN} "${BASE_URL}/landscape/api/landscape/get_time_point_of_last_change_in_ssh_keys_of_aws_landscape_managers" 2>/dev/null ) curl_exit_code=$? diff --git a/configuration/environments_scripts/repo/var/www/cgi-bin/reverseProxyHealthcheck.sh b/configuration/environments_scripts/repo/var/www/cgi-bin/reverseProxyHealthcheck.sh new file mode 100755 index 00000000000..76b5c80f0d7 --- /dev/null +++ b/configuration/environments_scripts/repo/var/www/cgi-bin/reverseProxyHealthcheck.sh @@ -0,0 +1,211 @@ +#!/bin/bash + +# Purpose: A smart health check for a Apache httpd reverse proxy server that takes three +# things into account: +# - /internal-server-status local technical health check +# - the availability zone (AZ) of the current live ARCHIVE application server +# - the health of all reverse proxies in a target group whose ARN is passed in the +# QUERY_STRINGS environments variable (assuming this script runs as a CGI-BIN script) +# as the "arn" query parameter, as in "...?arn=arn:aws:elasticloadbalancing:eu-west-1:017363970217:targetgroup/CentralWebServerHTTP-Dyn/32e57ea39e5fb165) +# The goal is to reduce cross-AZ traffic which produces extra cost and adds latency when +# a load balancer routes traffic to a reverse proxy in one AZ, and the application runs +# in a different AZ. So we'd like to have reverse proxies report an "unhealthy" status +# particularly if they run in an AZ different from the one in which the ARCHIVE application +# server runs, but only if there is at least one healthy reverse proxy in the ARCHIVE application +# server's AZ in the target group in question. +# +# To keep the health check swift, this script tries to keep the number of AWS API +# requests low. The first quick check is that for the local technical health, using the +# "/internal-server-status" endpoint. If that fails, this script will return an "unhealthy" +# status 500 and exit with code 1. +# +# When "/internal-server-status" reported a healthy status, the CIDRs of the VPC's subnets representing +# the AZs are cached persistently in a file once, using +# +# aws ec2 describe-subnets | jq -r '.Subnets[].CidrBlock' +# +# This produces, e.g., +# +# 172.31.16.0/20 +# 172.31.32.0/20 +# 172.31.0.0/20 +# +# These CIDRs are not assumed to change during the life-cycle of this instance, so this needs +# to happen only one time. Using +# +# nmap -sL -n | head -n -1 | tail -n +2 | grep -q " +# +# we can check quickly whether an address is within a CIDR. +# +# The ARCHIVE configuration with its production and failover instances can +# be determined from the /etc/httpd/conf.d/000-macros.conf file, telling the +# internal IP address of the ARCHIVE server currently used. From this, the AZ CIDR +# can be determined using the "nmap" approach described above. +# +# The same can be done for this instance's internal IP address. +# +# If the local instance is in the same AZ as the current live ARCHIVE server, +# we'll report "healthy" because our traffic to the ARCHIVE will not be cross-AZ. +# +# Otherwise (different AZ than live ARCHIVE server), we need to find out if at +# least one reverse proxy that is in the live ARCHIVE server's AZ is healthy. +# To determine the other targets in the target group specified through the "arn=" +# query parameter in the QUERY_STRING variable, an +# +# aws elbv2 describe-target-health +# +# call is made, and the instance IDs returned are mapped to their internal IP addresses +# using an "aws ec2 describe-instances" call. This is the list of all reverse proxies +# registered with the target group at that time. Since this list is not considered to +# change very often, and because the negative effects of an outdated list are mild +# (in the worst case causing an instance in the wrong AZ to report "healthy", causing +# some temporary cross-AZ traffic), we cache the results and update this list only every +# few minutes and not upon every health check. +# +# From the list of targets we focus on those that are in the same AZ as the live ARCHIVE +# and check their health using the "/internal-server-status" endpoint. As soon as one +# healthy reverse proxy in the same AZ as the live ARCHIVE is found, this script will +# return an unhealthy 503 status and exit with code 2. If no healthy target in the live +# ARCHIVE's AZ is found, healthy (200) is reported and the script exits with code 0. +# +# (Keep in mind that with a load balancer that has cross-AZ balancing activated, +# requests may come in to a load balancer node in one AZ, and the only health reverse +# proxy lives in a different AZ; while this kind of cross-AZ traffic also will add +# latency, it doesn't add cost as it is considered "intra-loadbalancer traffic.") +# +# The user it is run by must have aws credentials that don't need mfa. Install to /usr/share/httpd/.aws. +# +# Optional parameter: "cleanup"; this will clean the cache and exit +# +# Exit status: +# 0 means all necessary checks could be performed and we're healthy (status 200) +# 1 means we were technically not healthy because the internal-server-status check failed (500) +# 2 means all checks could be performed, but we're not healthy (503) because not in the right AZ +# and there is at least one healthy target in the correct AZ +# 3 means there was a problem determining our health; we may still report status 200 (healthy) + +outputMessage() { + # parameter 1: the message to display on the site + echo "Content-type: text/html" + echo "" + echo $1 +} + +status() { + # parameter 1: status code and messages + echo "Status: $1" +} + +randomise() { + # $1: A number to slightly randomise + echo $(($RANDOM % 5 + $1 )) +} + +getListOfIpsForCidr() { + # $1: a CIDR, such as 172.10.0.0/32 + # Outputs the list of IP addresses that match the CIDR. + # Tries to read this from a file in ${CACHE_LOCATION} + # whose name is inferred from the CIDR parameter. If not + # found, nmap is used to produce that list, and it is + # written to the file in the ${CACHE_LOCATION} so it can + # be found the next time requested + cidr="${1}" + FILE_WITH_IPS_FOR_CIDR=${CACHE_LOCATION}/$( echo "${cidr}" | tr '/' '_' ) + if [ -f ${FILE_WITH_IPS_FOR_CIDR} ]; then + cat ${FILE_WITH_IPS_FOR_CIDR} + else + nmap -sL -n ${cidr} | head -n -1 | tail -n +2 | sed -e 's/^.*\<\([0-9]*\.[0-9]*\.[0-9]*\.[0-9]*\)$/\1/' | tee ${FILE_WITH_IPS_FOR_CIDR} + fi +} + +getAzCidr() { + # $1: path to a file containing one CIDR per line + # $2: an IP address to obtain the subnet CIDR for + for cidr in $( cat "${1}" ); do + if getListOfIpsForCidr ${cidr} | grep -q ${2}; then + echo ${cidr} + fi + done +} + +# The regex to extract the ip from a line ending in an ip. +IP_REGEX="[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+$" +MY_IP=$( ec2-metadata --local-ipv4 | grep -o "${IP_REGEX}") +# First check, using /internal-server-status, if we're technically healthy, and abort if not. +curl --silent --location --fail "http://${MY_IP}/internal-server-status" >/dev/null +if [[ "$?" -ne 0 ]]; then + status "500 Reverse proxy itself is unhealthy" + outputMessage "Unhealthy: Reverse proxy is unhealthy." + exit 1 +fi +# Create cache location if it doesn't exist: +CACHE_LOCATION="/var/cache/httpd/healthcheck" # Folder storing all the cached info, which the Apache user can access. +mkdir --parents ${CACHE_LOCATION} +if [[ "$1" == "cleanup" ]]; then + rm -rf ${CACHE_LOCATION} + status "200" + outputMessage "cleanup complete" + exit 0 +fi +# Ensure we have a cached copy of the AZs' CIDRs: +AZ_CIDRS_FILENAME="${CACHE_LOCATION}/az_cidrs" +if [ ! -f "${AZ_CIDRS_FILENAME}" ]; then + aws ec2 describe-subnets | jq -r '.Subnets[].CidrBlock' >"${AZ_CIDRS_FILENAME}" +fi +# The names of the variables in the macros file. +MACROS_PATH="/etc/httpd/conf.d/000-macros.conf" +ARCHIVE_IP_NAME="ARCHIVE_IP" +ARCHIVE_FAILOVER_IP_NAME="ARCHIVE_FAILOVER_IP" +PRODUCTION_ARCHIVE_NAME="PRODUCTION_ARCHIVE" +# Extracts which IP is in production. +PRODUCTION_ARCHIVE=$(sed -n -e "s/^Define ${PRODUCTION_ARCHIVE_NAME}\> \(.*\)$/\1/p" ${MACROS_PATH}) +ARCHIVE_IP=$(grep -m 1 "^Define ${ARCHIVE_IP_NAME}\> .*" ${MACROS_PATH} | grep -o "${IP_REGEX}") +ARCHIVE_FAILOVER_IP=$(grep -m 1 "^Define ${ARCHIVE_FAILOVER_IP_NAME}\> .*" ${MACROS_PATH} | grep -o "${IP_REGEX}") +if [[ "$PRODUCTION_ARCHIVE" == "\${${ARCHIVE_IP_NAME}}" ]]; then + PRODUCTION_ARCHIVE_IP=${ARCHIVE_IP} +else + PRODUCTION_ARCHIVE_IP=${ARCHIVE_FAILOVER_IP} +fi +PRODUCTION_ARCHIVE_CIDR=$( getAzCidr "${AZ_CIDRS_FILENAME}" ${PRODUCTION_ARCHIVE_IP} ) +# AZ of instance +MY_AZ_CIDR=$( getAzCidr "${AZ_CIDRS_FILENAME}" ${MY_IP} ) +# Check if in the same AZ as the live ARCHIVE server and report healthy in that case +if [ "${PRODUCTION_ARCHIVE_CIDR}" = "${MY_AZ_CIDR}" ]; then + status "200" + outputMessage "Healthy: In the same AZ as the archive." + exit 0 +else + # Otherwise, get cached (or updated, if cache expired) list of reverse proxies in target group and + # search for targets in same AZ as live ARCHIVE: + TARGET_GROUP_ARN="${QUERY_STRING//arn=/}" + TARGET_GROUP_NAME=$( basename $( dirname "${TARGET_GROUP_ARN}" ) ) + LAST_TARGET_GROUP_IPS="${CACHE_LOCATION}/last_target_ips_${TARGET_GROUP_NAME}" + # Target group healthcheck timeout + TARGET_GROUP_HEALTHCHECK_TIMEOUT_SECONDS=300 + current_time=$( date +%s ) + if [[ ! -e "${LAST_TARGET_GROUP_IPS}" || "$(($current_time - $(stat --format '%Y' ${LAST_TARGET_GROUP_IPS}) ))" -gt "$(randomise $TARGET_GROUP_HEALTHCHECK_TIMEOUT_SECONDS)" ]]; then + # This branch runs if the cached timestamp, of the last target group healthcheck, doesn't exist, or if it exceeds TARGET_GROUP_HEALTHCHECK_TIMEOUT_SECONDS. + INSTANCE_IDS=$( aws elbv2 describe-target-health --target-group-arn ${TARGET_GROUP_ARN} | jq -r '.TargetHealthDescriptions[].Target.Id' ) + INSTANCE_PRIVATE_IPS=$( aws ec2 describe-instances --instance-ids $( echo "${INSTANCE_IDS}" | tr '\n' ' ' ) | jq -r '.Reservations[].Instances[].NetworkInterfaces[].PrivateIpAddress' ) + echo "${INSTANCE_PRIVATE_IPS}" >"${LAST_TARGET_GROUP_IPS}" + else + INSTANCE_PRIVATE_IPS=$( cat "${LAST_TARGET_GROUP_IPS}" ) + fi + for INSTANCE_PRIVATE_IP in ${INSTANCE_PRIVATE_IPS}; do + INSTANCE_CIDR=$( getAzCidr "${AZ_CIDRS_FILENAME}" $INSTANCE_PRIVATE_IP ) + if [ "${INSTANCE_CIDR}" = "${PRODUCTION_ARCHIVE_CIDR}" ]; then + # found a reverse proxy in the same AZ as the current live/production ARCHIVE; check its health: + curl --silent --location --fail "http://${INSTANCE_PRIVATE_IP}/internal-server-status" >/dev/null + if [[ "$?" = "0" ]]; then + # the reverse proxy in the same AZ as the current live/production ARCHIVE is healthy; then we're not: + status "503 Not in the same AZ" + outputMessage "Unhealthy: Not in the same AZ as the archive; healthy instance in the same AZ as the archive." + exit 2 + fi + fi + done + # No healthy reverse proxy found in the same AZ as live/production ARCHIVE, so we'll report healthy + status "200" + outputMessage "Healthy: No healthy instance in the same AZ as the archive" + exit 0 +fi diff --git a/configuration/environments_scripts/reverse_proxy/files/etc/dnf/vars/releasever b/configuration/environments_scripts/reverse_proxy/files/etc/dnf/vars/releasever new file mode 100644 index 00000000000..a0f9a4b4bc9 --- /dev/null +++ b/configuration/environments_scripts/reverse_proxy/files/etc/dnf/vars/releasever @@ -0,0 +1 @@ +latest diff --git a/configuration/environments_scripts/reverse_proxy/files/etc/logrotate.d/httpd b/configuration/environments_scripts/reverse_proxy/files/etc/logrotate.d/httpd deleted file mode 100644 index 6b1f0b00876..00000000000 --- a/configuration/environments_scripts/reverse_proxy/files/etc/logrotate.d/httpd +++ /dev/null @@ -1,15 +0,0 @@ -/var/log/httpd/*log { - missingok - notifempty - sharedscripts - olddir /var/log/logrotate-target - # use date as a suffix of the rotated file - dateext - compress - delaycompress - postrotate - /bin/systemctl reload httpd.service > /dev/null 2>/dev/null || true - mv /var/log/logrotate-target/*.gz /var/log/old - endscript -} - diff --git a/configuration/environments_scripts/reverse_proxy/files/etc/logrotate.d/httpd b/configuration/environments_scripts/reverse_proxy/files/etc/logrotate.d/httpd new file mode 120000 index 00000000000..0f8227a0487 --- /dev/null +++ b/configuration/environments_scripts/reverse_proxy/files/etc/logrotate.d/httpd @@ -0,0 +1 @@ +../../../../repo/etc/logrotate.d/httpd \ No newline at end of file diff --git a/configuration/environments_scripts/reverse_proxy/files/etc/systemd/system/create-internal-status-file.service b/configuration/environments_scripts/reverse_proxy/files/etc/systemd/system/create-internal-status-file.service deleted file mode 100644 index 79e9bfbcdee..00000000000 --- a/configuration/environments_scripts/reverse_proxy/files/etc/systemd/system/create-internal-status-file.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Creates httpd internal-status file. -Requires=-.mount network-online.target -After=-.mount network-online.target get-latest-httpd-conf.service -Before=httpd.service -# Staring before means no "systemctl start httpd" is needed. -[Install] -RequiredBy=multi-user.target -WantedBy=httpd.service - -[Service] -Type=oneshot -RemainAfterExit=true -ExecStart=/root/setupHttpdGitLocal.sh diff --git a/configuration/environments_scripts/reverse_proxy/files/etc/systemd/system/create-internal-status-file.service b/configuration/environments_scripts/reverse_proxy/files/etc/systemd/system/create-internal-status-file.service new file mode 120000 index 00000000000..5fa384f80bd --- /dev/null +++ b/configuration/environments_scripts/reverse_proxy/files/etc/systemd/system/create-internal-status-file.service @@ -0,0 +1 @@ +../../../../../repo/etc/systemd/system/create-internal-status-file.service \ No newline at end of file diff --git a/configuration/environments_scripts/reverse_proxy/files/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service b/configuration/environments_scripts/reverse_proxy/files/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service deleted file mode 100644 index 30e6e4367fc..00000000000 --- a/configuration/environments_scripts/reverse_proxy/files/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service +++ /dev/null @@ -1,19 +0,0 @@ -[Unit] -Description=Force logrotate upon shutdown and clean files in var/cache -Requires=-.mount network-online.target -After=-.mount network-online.target -Before=shutdown.target -RequiresMountsFor=/ -SurviveFinalKillSignal=yes - -[Install] -RequiredBy=graphical.target -WantedBy=shutdown.target - -[Service] -Type=oneshot -RemainAfterExit=yes -KillMode=none -ExecStart=/bin/true -ExecStopPost=/bin/bash -c "logrotate --force -v /etc/logrotate.conf; /var/www/cgi-bin/reverseProxyHealthcheck.sh cleanup" -TimeoutStopSec=50 diff --git a/configuration/environments_scripts/reverse_proxy/files/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service b/configuration/environments_scripts/reverse_proxy/files/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service new file mode 120000 index 00000000000..2d5bf6279ff --- /dev/null +++ b/configuration/environments_scripts/reverse_proxy/files/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service @@ -0,0 +1 @@ +../../../../../repo/etc/systemd/system/logrotate-and-general-clean-upon-shutdown.service \ No newline at end of file diff --git a/configuration/environments_scripts/reverse_proxy/files/usr/local/bin/awsmfalogon.sh b/configuration/environments_scripts/reverse_proxy/files/usr/local/bin/awsmfalogon.sh new file mode 120000 index 00000000000..d002c903654 --- /dev/null +++ b/configuration/environments_scripts/reverse_proxy/files/usr/local/bin/awsmfalogon.sh @@ -0,0 +1 @@ +../../../../../repo/usr/local/bin/awsmfalogon.sh \ No newline at end of file diff --git a/configuration/environments_scripts/reverse_proxy/files/usr/local/bin/imageupgrade b/configuration/environments_scripts/reverse_proxy/files/usr/local/bin/imageupgrade index db8a797ffcb..84b3de6131e 100755 --- a/configuration/environments_scripts/reverse_proxy/files/usr/local/bin/imageupgrade +++ b/configuration/environments_scripts/reverse_proxy/files/usr/local/bin/imageupgrade @@ -1,8 +1,5 @@ #!/bin/bash -# Script to deploy on an instance that has an ephemeral volume as /dev/nvme0n1 (adjust env var PARTITION if different) -# Ensures the partition is xfs-formatted, any existing partition contents will be overwritten if formatted otherwise. -# An existing xfs partition will be left alone. . imageupgrade_functions.sh get_ec2_user_data() { @@ -13,12 +10,10 @@ METADATA=$( get_ec2_user_data ) echo "Metadata: ${METADATA}" if echo "${METADATA}" | grep -q "^image-upgrade$"; then echo "Image upgrade..." - GIT_COPY_USER="trac" - RELATIVE_PATH_TO_GIT="gitcopy" LOGON_USER_HOME="ec2-user" ENVIRONMENT_TYPE="reverse_proxy" - run_yum_update - build_crontab_and_setup_files "${ENVIRONMENT_TYPE}" "${GIT_COPY_USER}" "${RELATIVE_PATH_TO_GIT}" + run_dnf_upgrade + build_crontab_and_setup_files "${ENVIRONMENT_TYPE}" finalize fi diff --git a/configuration/environments_scripts/reverse_proxy/files/usr/local/bin/setupHttpdGitLocal.sh b/configuration/environments_scripts/reverse_proxy/files/usr/local/bin/setupHttpdGitLocal.sh new file mode 120000 index 00000000000..5c2ee809d4b --- /dev/null +++ b/configuration/environments_scripts/reverse_proxy/files/usr/local/bin/setupHttpdGitLocal.sh @@ -0,0 +1 @@ +../../../../../repo/usr/local/bin/setupHttpdGitLocal.sh \ No newline at end of file diff --git a/configuration/environments_scripts/reverse_proxy/files/var/www/cgi-bin/reverseProxyHealthcheck.sh b/configuration/environments_scripts/reverse_proxy/files/var/www/cgi-bin/reverseProxyHealthcheck.sh deleted file mode 100755 index e721f964aac..00000000000 --- a/configuration/environments_scripts/reverse_proxy/files/var/www/cgi-bin/reverseProxyHealthcheck.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/bin/bash - -# Purpose: A smart health check for a Apache httpd reverse proxy server that takes three -# things into account: -# - /internal-server-status local technical health check -# - the availability zone (AZ) of the current live ARCHIVE application server -# - the health of all reverse proxies in a target group whose ARN is passed in the -# QUERY_STRINGS environments variable (assuming this script runs as a CGI-BIN script) -# as the "arn" query parameter, as in "...?arn=arn:aws:elasticloadbalancing:eu-west-1:017363970217:targetgroup/CentralWebServerHTTP-Dyn/32e57ea39e5fb165) -# The goal is to reduce cross-AZ traffic which produces extra cost and adds latency when -# a load balancer routes traffic to a reverse proxy in one AZ, and the application runs -# in a different AZ. So we'd like to have reverse proxies report an "unhealthy" status -# particularly if they run in an AZ different from the one in which the ARCHIVE application -# server runs, but only if there is at least one healthy reverse proxy in the ARCHIVE application -# server's AZ in the target group in question. -# -# To keep the health check swift, this script tries to keep the number of AWS API -# requests low. The first quick check is that for the local technical health, using the -# "/internal-server-status" endpoint. If that fails, this script will return an "unhealthy" -# status 500 and exit with code 1. -# -# When "/internal-server-status" reported a healthy status, the CIDRs of the VPC's subnets representing -# the AZs are cached persistently in a file once, using -# -# aws ec2 describe-subnets | jq -r '.Subnets[].CidrBlock' -# -# This produces, e.g., -# -# 172.31.16.0/20 -# 172.31.32.0/20 -# 172.31.0.0/20 -# -# These CIDRs are not assumed to change during the life-cycle of this instance, so this needs -# to happen only one time. Using -# -# nmap -sL -n | head -n -1 | tail -n +2 | grep -q " -# -# we can check quickly whether an address is within a CIDR. -# -# The ARCHIVE configuration with its production and failover instances can -# be determined from the /etc/httpd/conf.d/000-macros.conf file, telling the -# internal IP address of the ARCHIVE server currently used. From this, the AZ CIDR -# can be determined using the "nmap" approach described above. -# -# The same can be done for this instance's internal IP address. -# -# If the local instance is in the same AZ as the current live ARCHIVE server, -# we'll report "healthy" because our traffic to the ARCHIVE will not be cross-AZ. -# -# Otherwise (different AZ than live ARCHIVE server), we need to find out if at -# least one reverse proxy that is in the live ARCHIVE server's AZ is healthy. -# To determine the other targets in the target group specified through the "arn=" -# query parameter in the QUERY_STRING variable, an -# -# aws elbv2 describe-target-health -# -# call is made, and the instance IDs returned are mapped to their internal IP addresses -# using an "aws ec2 describe-instances" call. This is the list of all reverse proxies -# registered with the target group at that time. Since this list is not considered to -# change very often, and because the negative effects of an outdated list are mild -# (in the worst case causing an instance in the wrong AZ to report "healthy", causing -# some temporary cross-AZ traffic), we cache the results and update this list only every -# few minutes and not upon every health check. -# -# From the list of targets we focus on those that are in the same AZ as the live ARCHIVE -# and check their health using the "/internal-server-status" endpoint. As soon as one -# healthy reverse proxy in the same AZ as the live ARCHIVE is found, this script will -# return an unhealthy 503 status and exit with code 2. If no healthy target in the live -# ARCHIVE's AZ is found, healthy (200) is reported and the script exits with code 0. -# -# (Keep in mind that with a load balancer that has cross-AZ balancing activated, -# requests may come in to a load balancer node in one AZ, and the only health reverse -# proxy lives in a different AZ; while this kind of cross-AZ traffic also will add -# latency, it doesn't add cost as it is considered "intra-loadbalancer traffic.") -# -# The user it is run by must have aws credentials that don't need mfa. Install to /usr/share/httpd/.aws. -# -# Optional parameter: "cleanup"; this will clean the cache and exit -# -# Exit status: -# 0 means all necessary checks could be performed and we're healthy (status 200) -# 1 means we were technically not healthy because the internal-server-status check failed (500) -# 2 means all checks could be performed, but we're not healthy (503) because not in the right AZ -# and there is at least one healthy target in the correct AZ -# 3 means there was a problem determining our health; we may still report status 200 (healthy) - -outputMessage() { - # parameter 1: the message to display on the site - echo "Content-type: text/html" - echo "" - echo $1 -} - -status() { - # parameter 1: status code and messages - echo "Status: $1" -} - -randomise() { - # $1: A number to slightly randomise - echo $(($RANDOM % 5 + $1 )) -} - -getAzCidr() { - # $1: path to a file containing one CIDR per line - # $2: an IP address to obtain the subnet CIDR for - for cidr in $( cat "${1}" ); do - if nmap -sL -n ${cidr} | head -n -1 | tail -n +2 | grep -q ${2}; then - echo ${cidr} - fi - done -} - -# The regex to extract the ip from a line ending in an ip. -IP_REGEX="[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+$" -MY_IP=$( ec2-metadata --local-ipv4 | grep -o "${IP_REGEX}") -# First check, using /internal-server-status, if we're technically healthy, and abort if not. -curl --silent --location --fail "http://${MY_IP}/internal-server-status" >/dev/null -if [[ "$?" -ne 0 ]]; then - status "500 Reverse proxy itself is unhealthy" - outputMessage "Unhealthy: Reverse proxy is unhealthy." - exit 1 -fi -# Create cache location if it doesn't exist: -CACHE_LOCATION="/var/cache/httpd/healthcheck" # Folder storing all the cached info, which the Apache user can access. -mkdir --parents ${CACHE_LOCATION} -if [[ "$1" == "cleanup" ]]; then - rm -rf ${CACHE_LOCATION} - status "200" - outputMessage "cleanup complete" - exit 0 -fi -# Ensure we have a cached copy of the AZs' CIDRs: -AZ_CIDRS_FILENAME="${CACHE_LOCATION}/az_cidrs" -if [ ! -f "${AZ_CIDRS_FILENAME}" ]; then - aws ec2 describe-subnets | jq -r '.Subnets[].CidrBlock' >"${AZ_CIDRS_FILENAME}" -fi -# The names of the variables in the macros file. -MACROS_PATH="/etc/httpd/conf.d/000-macros.conf" -ARCHIVE_IP_NAME="ARCHIVE_IP" -ARCHIVE_FAILOVER_IP_NAME="ARCHIVE_FAILOVER_IP" -PRODUCTION_ARCHIVE_NAME="PRODUCTION_ARCHIVE" -# Extracts which IP is in production. -PRODUCTION_ARCHIVE=$(sed -n -e "s/^Define ${PRODUCTION_ARCHIVE_NAME}\> \(.*\)$/\1/p" ${MACROS_PATH}) -ARCHIVE_IP=$(grep -m 1 "^Define ${ARCHIVE_IP_NAME}\> .*" ${MACROS_PATH} | grep -o "${IP_REGEX}") -ARCHIVE_FAILOVER_IP=$(grep -m 1 "^Define ${ARCHIVE_FAILOVER_IP_NAME}\> .*" ${MACROS_PATH} | grep -o "${IP_REGEX}") -if [[ "$PRODUCTION_ARCHIVE" == "\${${ARCHIVE_IP_NAME}}" ]]; then - PRODUCTION_ARCHIVE_IP=${ARCHIVE_IP} -else - PRODUCTION_ARCHIVE_IP=${ARCHIVE_FAILOVER_IP} -fi -PRODUCTION_ARCHIVE_CIDR=$( getAzCidr "${AZ_CIDRS_FILENAME}" ${PRODUCTION_ARCHIVE_IP} ) -# AZ of instance -MY_AZ_CIDR=$( getAzCidr "${AZ_CIDRS_FILENAME}" ${MY_IP} ) -# Check if in the same AZ as the live ARCHIVE server and report healthy in that case -if [ "${PRODUCTION_ARCHIVE_CIDR}" = "${MY_AZ_CIDR}" ]; then - status "200" - outputMessage "Healthy: In the same AZ as the archive." - exit 0 -else - # Otherwise, get cached (or updated, if cache expired) list of reverse proxies in target group and - # search for targets in same AZ as live ARCHIVE: - TARGET_GROUP_ARN="${QUERY_STRING//arn=/}" - TARGET_GROUP_NAME=$( basename $( dirname "${TARGET_GROUP_ARN}" ) ) - LAST_TARGET_GROUP_IPS="${CACHE_LOCATION}/last_target_ips_${TARGET_GROUP_NAME}" - # Target group healthcheck timeout - TARGET_GROUP_HEALTHCHECK_TIMEOUT_SECONDS=300 - current_time=$( date +%s ) - if [[ ! -e "${LAST_TARGET_GROUP_IPS}" || "$(($current_time - $(stat --format '%Y' ${LAST_TARGET_GROUP_IPS}) ))" -gt "$(randomise $TARGET_GROUP_HEALTHCHECK_TIMEOUT_SECONDS)" ]]; then - # This branch runs if the cached timestamp, of the last target group healthcheck, doesn't exist, or if it exceeds TARGET_GROUP_HEALTHCHECK_TIMEOUT_SECONDS. - INSTANCE_IDS=$( aws elbv2 describe-target-health --target-group-arn ${TARGET_GROUP_ARN} | jq -r '.TargetHealthDescriptions[].Target.Id' ) - INSTANCE_PRIVATE_IPS=$( aws ec2 describe-instances --instance-ids $( echo "${INSTANCE_IDS}" | tr '\n' ' ' ) | jq -r '.Reservations[].Instances[].NetworkInterfaces[].PrivateIpAddress' ) - echo "${INSTANCE_PRIVATE_IPS}" >"${LAST_TARGET_GROUP_IPS}" - else - INSTANCE_PRIVATE_IPS=$( cat "${LAST_TARGET_GROUP_IPS}" ) - fi - for INSTANCE_PRIVATE_IP in ${INSTANCE_PRIVATE_IPS}; do - INSTANCE_CIDR=$( getAzCidr "${AZ_CIDRS_FILENAME}" $INSTANCE_PRIVATE_IP ) - if [ "${INSTANCE_CIDR}" = "${PRODUCTION_ARCHIVE_CIDR}" ]; then - # found a reverse proxy in the same AZ as the current live/production ARCHIVE; check its health: - curl --silent --location --fail "http://${INSTANCE_PRIVATE_IP}/internal-server-status" >/dev/null - if [[ "$?" = "0" ]]; then - # the reverse proxy in the same AZ as the current live/production ARCHIVE is healthy; then we're not: - status "503 Not in the same AZ" - outputMessage "Unhealthy: Not in the same AZ as the archive; healthy instance in the same AZ as the archive." - exit 2 - fi - fi - done - # No healthy reverse proxy found in the same AZ as live/production ARCHIVE, so we'll report healthy - status "200" - outputMessage "Healthy: No healthy instance in the same AZ as the archive" - exit 0 -fi diff --git a/configuration/environments_scripts/reverse_proxy/files/var/www/cgi-bin/reverseProxyHealthcheck.sh b/configuration/environments_scripts/reverse_proxy/files/var/www/cgi-bin/reverseProxyHealthcheck.sh new file mode 120000 index 00000000000..76d5eae3f23 --- /dev/null +++ b/configuration/environments_scripts/reverse_proxy/files/var/www/cgi-bin/reverseProxyHealthcheck.sh @@ -0,0 +1 @@ +../../../../../repo/var/www/cgi-bin/reverseProxyHealthcheck.sh \ No newline at end of file diff --git a/configuration/environments_scripts/reverse_proxy/setup-disposable-reverse-proxy.sh b/configuration/environments_scripts/reverse_proxy/setup-disposable-reverse-proxy.sh old mode 100644 new mode 100755 index daafc2a7dde..133856a2b97 --- a/configuration/environments_scripts/reverse_proxy/setup-disposable-reverse-proxy.sh +++ b/configuration/environments_scripts/reverse_proxy/setup-disposable-reverse-proxy.sh @@ -1,14 +1,16 @@ #!/bin/bash -# Setup script for Amazon Linux 2. May need to update macro definitions for the archive IP. +# Setup script for Amazon Linux 2023. May need to update macro definitions for the archive IP. # Parameter 1 is the IP and parameter 2 is the bearer token to be installed in the root home dir. # Ensure that the security for requesting the metadata uses IMDSv1 +if [[ "$#" -ne 2 ]]; then + echo "Incorrect number of args (2 required). See script header for further details." + exit 2 +fi IP=$1 BEARER_TOKEN=$2 IMAGE_TYPE="reverse_proxy" HTTP_LOGROTATE_ABSOLUTE=/etc/logrotate.d/httpd -GIT_COPY_USER="trac" -RELATIVE_PATH_TO_GIT="gitcopy" # the relative path to the repo within the git_copy_user ssh -A "ec2-user@${IP}" "bash -s" << FIRSTEOF # Correct authorized keys. May not be necessary if update_authorized_keys is running. sudo su - -c "cat ~ec2-user/.ssh/authorized_keys > /root/.ssh/authorized_keys" @@ -20,8 +22,8 @@ mkdir /var/log/old echo "logfiles.internal.sapsailing.com:/var/log/old /var/log/old nfs tcp,intr,timeo=100,retry=0" >> /etc/fstab mount -a # update instance -yum update -y -yum install -y httpd mod_proxy_html tmux nfs-utils git whois jq cronie iptables nmap +dnf upgrade -y --releasever=latest +dnf install -y httpd mod_proxy_html tmux nfs-utils git whois jq cronie iptables nmap sudo systemctl enable crond.service # setup other users and crontabs to keep repo updated cd /root @@ -31,7 +33,7 @@ scp -o StrictHostKeyChecking=no -p "root@sapsailing.com:/home/wiki/gitwiki/confi setup_keys "${IMAGE_TYPE}" setup_cloud_cfg_and_root_login # setup files and crontab for the required users, both dependent on the environment type. -build_crontab_and_setup_files "${IMAGE_TYPE}" "${GIT_COPY_USER}" "${RELATIVE_PATH_TO_GIT}" +build_crontab_and_setup_files "${IMAGE_TYPE}" # setup mail setup_mail_sending # setup sshd config @@ -48,6 +50,9 @@ systemctl enable httpd echo "net.ipv4.ip_conntrac_max = 131072" >> /etc/sysctl.conf # setup fail2ban setup_fail2ban +# goaccess and apachetop +setup_goaccess +setup_apachetop # mount nvme if available mountnvmeswap # setup logrotate.d/httpd @@ -59,7 +64,7 @@ sed -i "s|/var/log/old|/var/log/old/REVERSE_PROXIES/${IP}|" $HTTP_LOGROTATE_ABS sed -i 's/rotate 4/rotate 20 \n\nolddir \/var\/log\/logrotate-target/' /etc/logrotate.conf sed -i "s/^#compress/compress/" /etc/logrotate.conf # setup git -/root/setupHttpdGitLocal.sh "httpdConf@sapsailing.com:repo.git" +setupHttpdGitLocal.sh "httpdConf@sapsailing.com:repo.git" disposable "Disposable Reverse Proxy" # Final enabling and starting of services. systemctl start httpd sudo systemctl start crond.service diff --git a/configuration/environments_scripts/reverse_proxy/users/root/crontab-update-authorized-keys b/configuration/environments_scripts/reverse_proxy/users/root/crontab-update-authorized-keys deleted file mode 120000 index d9b0fed0726..00000000000 --- a/configuration/environments_scripts/reverse_proxy/users/root/crontab-update-authorized-keys +++ /dev/null @@ -1 +0,0 @@ -../../../../crontabs/crontab-update-authorized-keys \ No newline at end of file diff --git a/configuration/environments_scripts/reverse_proxy/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_root b/configuration/environments_scripts/reverse_proxy/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_root new file mode 120000 index 00000000000..cf597d8e543 --- /dev/null +++ b/configuration/environments_scripts/reverse_proxy/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_root @@ -0,0 +1 @@ +../../../../crontabs/crontab-update-authorized-keys@USER_HOME_DIR \ No newline at end of file diff --git a/configuration/environments_scripts/sailing_server/files/etc/logrotate.d/mongodb b/configuration/environments_scripts/sailing_server/files/etc/logrotate.d/mongodb new file mode 120000 index 00000000000..00ba3e765ea --- /dev/null +++ b/configuration/environments_scripts/sailing_server/files/etc/logrotate.d/mongodb @@ -0,0 +1 @@ +../../../../repo/etc/logrotate.d/mongodb \ No newline at end of file diff --git a/configuration/environments_scripts/sailing_server/files/etc/profile.d/sailing.sh b/configuration/environments_scripts/sailing_server/files/etc/profile.d/sailing.sh index c4f8f5f824f..9cb4414a866 100755 --- a/configuration/environments_scripts/sailing_server/files/etc/profile.d/sailing.sh +++ b/configuration/environments_scripts/sailing_server/files/etc/profile.d/sailing.sh @@ -1,7 +1,9 @@ # Script to be linked from /etc/profile.d # Appends to PATH, sets DISPLAY for VNC running on :2, exports JAVA_HOME and Amazon EC2 variables -ulimit -n 100000 +if [[ "$USER" == "root" ]]; then + ulimit -n 100000 +fi ulimit -u 40000 # SAP JVM diff --git a/configuration/environments_scripts/sailing_server/files/etc/security/limits.d/filedescriptorlimits.conf b/configuration/environments_scripts/sailing_server/files/etc/security/limits.d/filedescriptorlimits.conf new file mode 100644 index 00000000000..b3bb533d1ef --- /dev/null +++ b/configuration/environments_scripts/sailing_server/files/etc/security/limits.d/filedescriptorlimits.conf @@ -0,0 +1,2 @@ +* hard nofile 1000000 +* soft nofile 1000000 diff --git a/configuration/cp_root_mail_properties_sudoers b/configuration/environments_scripts/sailing_server/files/etc/sudoers.d/cp_root_mail_properties_sudoers similarity index 100% rename from configuration/cp_root_mail_properties_sudoers rename to configuration/environments_scripts/sailing_server/files/etc/sudoers.d/cp_root_mail_properties_sudoers diff --git a/configuration/environments_scripts/sailing_server/files/etc/systemd/system/imageupgrade.service b/configuration/environments_scripts/sailing_server/files/etc/systemd/system/imageupgrade.service new file mode 120000 index 00000000000..32775d542f4 --- /dev/null +++ b/configuration/environments_scripts/sailing_server/files/etc/systemd/system/imageupgrade.service @@ -0,0 +1 @@ +../../../../../repo/etc/systemd/system/imageupgrade.service \ No newline at end of file diff --git a/configuration/environments_scripts/sailing_server/files/etc/systemd/system/sailing.service b/configuration/environments_scripts/sailing_server/files/etc/systemd/system/sailing.service index 057bc0c5577..3a51513fbd6 100644 --- a/configuration/environments_scripts/sailing_server/files/etc/systemd/system/sailing.service +++ b/configuration/environments_scripts/sailing_server/files/etc/systemd/system/sailing.service @@ -9,5 +9,5 @@ RequiredBy=multi-user.target [Service] Type=oneshot RemainAfterExit=true -ExecStart=/etc/init.d/sailing start -ExecStop=/etc/init.d/sailing stop +ExecStart=/usr/local/bin/sailing start +ExecStop=/usr/local/bin/sailing stop diff --git a/configuration/cp_root_mail_properties b/configuration/environments_scripts/sailing_server/files/usr/local/bin/cp_root_mail_properties similarity index 100% rename from configuration/cp_root_mail_properties rename to configuration/environments_scripts/sailing_server/files/usr/local/bin/cp_root_mail_properties diff --git a/configuration/environments_scripts/sailing_server/files/usr/local/bin/imageupgrade b/configuration/environments_scripts/sailing_server/files/usr/local/bin/imageupgrade new file mode 100755 index 00000000000..7c428c20e10 --- /dev/null +++ b/configuration/environments_scripts/sailing_server/files/usr/local/bin/imageupgrade @@ -0,0 +1,20 @@ +#!/bin/bash + +# Upgrades the AWS EC2 instance that this script is assumed to be executed on. +# The steps are as follows: + +. `dirname $0`/imageupgrade_functions.sh + +METADATA=$( get_ec2_user_data ) +echo "Metadata: ${METADATA}" +if echo "${METADATA}" | grep -q "^image-upgrade$"; then + echo "Image upgrade..." + run_yum_update + download_and_install_latest_sap_jvm_8 + clean_logrotate_target + clean_httpd_logs + clean_servers_dir + clean_startup_logs + build_crontab_and_setup_files sailing_server + finalize +fi diff --git a/configuration/environments_scripts/sailing_server/files/usr/local/bin/imageupgrade.sh b/configuration/environments_scripts/sailing_server/files/usr/local/bin/imageupgrade.sh deleted file mode 100755 index dcac7e3e9f0..00000000000 --- a/configuration/environments_scripts/sailing_server/files/usr/local/bin/imageupgrade.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# Upgrades the AWS EC2 instance that this script is assumed to be executed on. -# The steps are as follows: - -. `dirname $0`/imageupgrade_functions.sh - -run_yum_update -run_git_pull -download_and_install_latest_sap_jvm_8 -clean_logrotate_target -clean_httpd_logs -clean_servers_dir -clean_startup_logs -update_root_crontab -finalize diff --git a/configuration/environments_scripts/sailing_server/files/usr/local/bin/refreshInstance.sh b/configuration/environments_scripts/sailing_server/files/usr/local/bin/refreshInstance.sh new file mode 100755 index 00000000000..ecf7e5c5303 --- /dev/null +++ b/configuration/environments_scripts/sailing_server/files/usr/local/bin/refreshInstance.sh @@ -0,0 +1,428 @@ +#!/bin/bash +ENV_SH="`pwd`/env.sh" +ENV_SH_DEFAULTS="`pwd`/env-default-rules.sh" +if [ -f "${ENV_SH}" ]; then + chmod a+x "${ENV_SH}" + source "${ENV_SH}" +fi +if [ -f "${ENV_SH_DEFAULTS}" ]; then + chmod a+x "${ENV_SH_DEFAULTS}" + source "${ENV_SH_DEFAULTS}" +fi +ON_AMAZON=`command -v ec2-metadata` +DATE_OF_EXECUTION=`date` + +# The following temporary file may be used by this script to dump EC2-provided user data +# variables to it; they can then be sourced from there and later appended to a new env.sh +ec2EnvVars_tmpFile=`mktemp /tmp/ec2EnvVars_XXX` + +find_project_home () +{ + if [[ $1 == '/' ]] || [[ $1 == "" ]]; then + echo "" + return 0 + fi + if [ ! -d "$1/.git" ]; then + PARENT_DIR=`cd $1/..;pwd` + OUTPUT=$(find_project_home $PARENT_DIR) + if [ "$OUTPUT" = "" ] && [ -d "$PARENT_DIR/$CODE_DIRECTORY" ] && [ -d "$PARENT_DIR/$CODE_DIRECTORY/.git" ]; then + OUTPUT="$PARENT_DIR/$CODE_DIRECTORY" + fi + echo $OUTPUT + return 0 + fi + echo $1 | sed -e 's/\/cygdrive\/\([a-zA-Z]\)/\1:/' +} + +checks () +{ + USER_HOME=~ + START_DIR=`pwd` + PROJECT_HOME=$(find_project_home $START_DIR) + # needed for maven on sapsailing.com to work correctly + if [ -f $USER_HOME/.bash_profile ]; then + source $USER_HOME/.bash_profile + fi + JAVA_BINARY=$JAVA_HOME/bin/java + if [[ ! -d "$JAVA_HOME" ]]; then + echo "Could not find $JAVA_BINARY set in env.sh. Trying to find the correct one..." + JAVA_VERSION=$(java -version 2>&1 | sed 's/^\(java version "\(.*\)\.\(.*\)\..*"\)\|\(openjdk version "\(.*\)\.\(.*\)\.\(.*\)" .*\)$/\3\5/; 1q') + if [ "$JAVA_VERSION" -lt 7 -o "$JAVA_VERSION" -gt 8 ]; then + echo "The current Java version ($JAVA_VERSION) does not match the requirements (>= Java 7, <= Java 8)." + exit 10 + fi + JAVA_BINARY=`which java` + echo "Using Java from $JAVA_BINARY" + fi + if [[ $DEPLOY_TO == "" ]]; then + SERVER_HOME=. + else + SERVER_HOME=$USER_HOME/servers/$DEPLOY_TO + fi + if [[ ! -d $SERVER_HOME ]]; then + SERVER_HOME=`pwd`/../../servers/$DEPLOY_TO + if [[ ! -d $SERVER_HOME ]]; then + echo "Could not find the correct directory for the server - assumed $SERVER_HOME. Adapt DEPLOY_TO in env.sh to point to the right server directory." + exit 10 + fi + fi +} + +copy_user_data_to_tmp_file () +{ + echo "Reading user-data provided by Amazon instance data to ${ec2EnvVars_tmpFile}" + VARS=$(ec2-metadata -d | sed "s/user-data\: //g") + if [[ "$VARS" != "not available" ]]; then + ec2-metadata -d | sed "s/user-data\: //g" >>"${ec2EnvVars_tmpFile}" + else + echo "No user data has been provided." + fi +} + +# loads the user data-provided variables by sourcing the script +activate_user_data () +{ + # make sure to reload data + source "${ec2EnvVars_tmpFile}" + INSTANCE_NAME=`ec2-metadata -i | cut -f2 -d " "` + INSTANCE_IP4=`ec2-metadata -v | cut -f2 -d " "` + INSTANCE_DNS=`ec2-metadata -p | cut -f2 -d " "` + INSTANCE_ID="$INSTANCE_NAME ($INSTANCE_IP4)" +} + +append_default_envsh_rules() +{ + echo " +# Default rules: START ($DATE_OF_EXECUTION)" >> $SERVER_HOME/env.sh + cat "${SERVER_HOME}/env-default-rules.sh" >>$SERVER_HOME/env.sh + echo " +# Default rules: END" >> $SERVER_HOME/env.sh + echo "Updated env.sh with data from env-default-rules.sh file!" +} + +append_user_data_to_envsh () +{ + mkdir -p $SERVER_HOME/environment 2>/dev/null >/dev/null + # make backup of original file + cp $SERVER_HOME/env.sh $SERVER_HOME/environment/env.sh.backup + + echo " +# User-Data: START ($DATE_OF_EXECUTION)" >> $SERVER_HOME/env.sh + echo "INSTANCE_NAME=`ec2-metadata -i | cut -f2 -d \" \"`" >> $SERVER_HOME/env.sh + echo "INSTANCE_IP4=`ec2-metadata -v | cut -f2 -d \" \"`" >> $SERVER_HOME/env.sh + echo "INSTANCE_INTERNAL_IP4=`ec2-metadata -o | cut -f2 -d \" \"`" >> $SERVER_HOME/env.sh + echo "INSTANCE_DNS=`ec2-metadata -p | cut -f2 -d \" \"`" >> $SERVER_HOME/env.sh + # Append EC2 user data to env.sh file: + cat "${ec2EnvVars_tmpFile}" >>$SERVER_HOME/env.sh + + echo "INSTANCE_ID=\"$INSTANCE_NAME ($INSTANCE_IP4)\"" >> $SERVER_HOME/env.sh + echo "# User-Data: END" >> $SERVER_HOME/env.sh + echo "Updated env.sh with data from user-data field!" +} + +install_environment () +{ + if [[ $USE_ENVIRONMENT != "" ]]; then + # clean up directory to really make sure that there are no files left + rm -rf ${SERVER_HOME}/environment + mkdir ${SERVER_HOME}/environment + if [[ ${INSTALL_FROM_SCP_USER_AT_HOST_AND_PORT} != "" ]]; then + SCP_PORT=$( echo ${INSTALL_FROM_SCP_USER_AT_HOST_AND_PORT} | sed -e 's/^[^:]*:\?\([0-9]*\)\?$/\1/' ) + if [ -n "${SCP_PORT}" ]; then + SCP_PORT_OPTION="-P ${SCP_PORT}" + fi + SCP_HOST=$( echo ${INSTALL_FROM_SCP_USER_AT_HOST_AND_PORT} | sed -e 's/^\([^:]*\):\?\([0-9]*\)\?$/\1/' ) + echo "Using environment ${SCP_HOST}:/home/trac/releases/environments/${USE_ENVIRONMENT}" + mkdir -p ./environment + scp ${SCP_PORT_OPTION} ${SCP_HOST}:/home/trac/releases/environments/${USE_ENVIRONMENT} ./environment + else + echo "Using environment https://releases.sapsailing.com/environments/$USE_ENVIRONMENT" + wget -P environment https://releases.sapsailing.com/environments/$USE_ENVIRONMENT + fi + echo " +# Environment ($USE_ENVIRONMENT): START ($DATE_OF_EXECUTION)" >> $SERVER_HOME/env.sh + cat ${SERVER_HOME}/environment/$USE_ENVIRONMENT >> $SERVER_HOME/env.sh + echo " +# Environment: END" >> ${SERVER_HOME}/env.sh + echo "Updated env.sh with data from environment file!" + else + echo "No environment file specified!" + fi +} + +load_from_release_file () +{ + if [[ ${INSTALL_FROM_RELEASE} == "" ]]; then + INSTALL_FROM_RELEASE="$(wget -O - https://releases.sapsailing.com/ 2>/dev/null | grep main- | tail -1 | sed -e 's/^.*\(main-[0-9]*\).*$/\1/')" + echo "You didn't provide a release. Defaulting to latest master build https://releases.sapsailing.com/$INSTALL_FROM_RELEASE" + # Alternatively, to download a release from Github, here is how the latest Java8/main-release can be obtained: + # curl -L -H 'Authorization: Bearer ***' https://api.github.com/repos/SAP/sailing-analytics/releases 2>/dev/null | jq -r 'sort_by(.created_at) | reverse | map(select(.name | startswith("main-")))[0].assets[] | select(.content_type=="application/x-tar").id' + # will output something like: + # 169233159 + # which can then be used in a request such as: + # curl -L -o main-1234567890.tar.gz -H 'Accept: application/octet-stream' -H 'Authorization: Bearer ***' 'https://api.github.com/repos/SAP/sailing-analytics/releases/assets/169233159' + # Or to obtain the latest docker-17 release, try this: + # curl -L -H 'Authorization: Bearer ***' https://api.github.com/repos/SAP/sailing-analytics/releases 2>/dev/null | jq -r 'sort_by(.created_at) | reverse | map(select(.name | startswith("docker-17-")))[0].assets[] | select(.content_type=="application/x-tar").id' + # and then on like above... + fi + if [ -n "${BUILD_COMPLETE_NOTIFY}" ]; then + echo "Build/Deployment process has been started - it can take 5 to 20 minutes until your instance is ready. " | mail -r axel.uhl@sap.com -s "Build or Deployment of $INSTANCE_ID to $SERVER_HOME for server $SERVER_NAME starting" ${BUILD_COMPLETE_NOTIFY} + fi + RELEASE_FILE_NAME=${INSTALL_FROM_RELEASE}.tar.gz + cd ${SERVER_HOME} + rm -f ${SERVER_HOME}/${INSTALL_FROM_RELEASE}.tar.gz* + rm -rf *.tar.gz + if [[ ${INSTALL_FROM_SCP_USER_AT_HOST_AND_PORT} != "" ]]; then + SCP_PORT=$( echo ${INSTALL_FROM_SCP_USER_AT_HOST_AND_PORT} | sed -e 's/^[^:]*:\?\([0-9]*\)\?$/\1/' ) + if [ -n "${SCP_PORT}" ]; then + SCP_PORT_OPTION="-P ${SCP_PORT}" + fi + SCP_HOST=$( echo ${INSTALL_FROM_SCP_USER_AT_HOST_AND_PORT} | sed -e 's/^\([^:]*\):\?\([0-9]*\)\?$/\1/' ) + scp ${SCP_PORT_OPTION} ${SCP_HOST}:/home/trac/releases/${INSTALL_FROM_RELEASE}/${RELEASE_FILE_NAME} . + else + echo "Loading from release file https://releases.sapsailing.com/${INSTALL_FROM_RELEASE}/${RELEASE_FILE_NAME}" + wget https://releases.sapsailing.com/${INSTALL_FROM_RELEASE}/${RELEASE_FILE_NAME} + fi + load_from_local_release_file +} + +load_from_local_release_file () +{ + if [[ ${INSTALL_FROM_RELEASE} != "" ]]; then + cd ${SERVER_HOME} + rm -rf plugins start stop status native-libraries org.eclipse.osgi + echo "Loading from release file ${INSTALL_FROM_RELEASE}" + mv env.sh env.sh.preserved + mv configuration/mail.properties configuration/mail.properties.preserved + mv configuration/debug.properties configuration/debug.properties.preserved + tar xvzf ${INSTALL_FROM_RELEASE}.tar.gz + mv env.sh.preserved env.sh + mv configuration/mail.properties.preserved configuration/mail.properties + mv configuration/debug.properties.preserved configuration/debug.properties + # Try to create the symbolic links to the /home/scores directories where uploaded results are mounted through NFS if in the right region + find /home/scores/* -type d -prune -exec ln -s {} \; 2>/dev/null >/dev/null + echo "Configuration for this server is unchanged - just binaries have been changed." + else + echo "The variable INSTALL_FROM_RELEASE has not been set, therefore no release file will be installed!" + fi +} + +checkout_code () +{ + cd $PROJECT_HOME + GIT_BINARY=`which git` + if [[ $COMPILE_GWT == "True" ]]; then + # only reset if GWT gets compiled + # if not p2build will not work + $GIT_BINARY reset --hard + fi + $GIT_BINARY checkout $BUILD_FROM + $GIT_BINARY pull +} + +build () +{ + # check for available memory - build can not be started with less than 1GB + MEM_TOTAL=`free -mt | grep Total | awk '{print $2}'` + if [ $MEM_TOTAL -lt 924 ]; then + echo "Could not start build process with less than 1GB of RAM!" + if [ -n "${BUILD_COMPLETE_NOTIFY}" ]; then + echo "Not enough RAM for completing the build process! You need at least 1GB. Instance NOT started!" | mail -r simon.marcel.pamies@sap.com -s "Build of $INSTANCE_ID failed" ${BUILD_COMPLETE_NOTIFY} + fi + else + if [[ $BUILD_BEFORE_START == "True" ]]; then + cd $PROJECT_HOME + TESTS="-t" + if [[ $RUN_TESTS == "True" ]]; then + TESTS="" + fi + GWT="-g" + if [[ $COMPILE_GWT == "True" ]]; then + GWT="" + fi + $PROJECT_HOME/configuration/buildAndUpdateProduct.sh $TESTS $GWT -u build + STATUS=$? + if [ $STATUS -eq 0 ]; then + echo "Build Successful" + else + echo "Build Failed" + exit 10 + fi + else + echo "The parameter BUILD_BEFORE_START is not set to True therefore no build will be executed!" + fi + fi +} + +deploy () +{ + cd $PROJECT_HOME + if [[ $DEPLOY_TO != "" ]]; then + DEPLOY="-s $DEPLOY_TO" + fi + + $PROJECT_HOME/configuration/buildAndUpdateProduct.sh -u $DEPLOY install > $SERVER_HOME/last_automatic_build.txt + STATUS=$? + if [ $STATUS -eq 0 ]; then + echo "Deployment Successful" + if [ -n "${BUILD_COMPLETE_NOTIFY}" ]; then + echo "OK - check the attachment for more information." | mail $MAIL_ATTACH_OPTION $SERVER_HOME/last_automatic_build.txt -r simon.marcel.pamies@sap.com -s "Build or Deployment of $INSTANCE_ID complete" ${BUILD_COMPLETE_NOTIFY} + fi + else + echo "Deployment Failed" + if [ -n "${BUILD_COMPLETE_NOTIFY}" ]; then + echo "ERROR - check the attachment for more information." | mail $MAIL_ATTACH_OPTION $SERVER_HOME/last_automatic_build.txt -r simon.marcel.pamies@sap.com -s "Build of $INSTANCE_ID failed" ${BUILD_COMPLETE_NOTIFY} + fi + fi +} + +auto_install () +{ + # activate everything found in user data + activate_user_data + # Now build or fetch the correct release, based on activated user data: + if [[ $BUILD_BEFORE_START = "True" ]]; then + checkout_code + build + deploy + else + load_from_release_file + fi + # then download and install environment and append to env.sh + install_environment + # then append user data to env.sh as it shall take precedence over the installed environment's defaults + append_user_data_to_envsh + # then append the rules that compute defaults for variables not set elsewhere; this has to come last: + append_default_envsh_rules + # make sure to reload data, this time including defaults from release's env.sh, environment settings and user data + source `pwd`/env.sh + # Append mail-related environment variables to configuration/mail.properties to override defaults + echo "mail.enabled = true" >>configuration/mail.properties + if [ -n "$MAIL_FROM" ]; then + echo "mail.from = $MAIL_FROM" >>configuration/mail.properties + fi + if [ -n "$MAIL_SMTP_HOST" ]; then + echo "mail.smtp.host = $MAIL_SMTP_HOST" >>configuration/mail.properties + fi + if [ -n "$MAIL_SMTP_PORT" ]; then + echo "mail.smtp.port = $MAIL_SMTP_PORT" >>configuration/mail.properties + fi + if [ -n "$MAIL_SMTP_AUTH" ]; then + echo "mail.smtp.auth = $MAIL_SMTP_AUTH" >>configuration/mail.properties + fi + if [ -n "$MAIL_SMTP_USER" ]; then + echo "mail.smtp.user = $MAIL_SMTP_USER" >>configuration/mail.properties + fi + if [ -n "$MAIL_SMTP_PASSWORD" ]; then + echo "mail.smtp.password = $MAIL_SMTP_PASSWORD" >>configuration/mail.properties + fi + echo "" + echo "INSTALL_FROM_RELEASE: $INSTALL_FROM_RELEASE" + echo "DEPLOY_TO: $DEPLOY_TO" + echo "BUILD_BEFORE_START: $BUILD_BEFORE_START" + echo "USE_ENVIRONMENT: $USE_ENVIRONMENT" + echo "" +} + +OPERATION=$1 +PARAM=$2 + +checks +if [[ $OPERATION == "auto-install" ]]; then + if [[ ! -z "$ON_AMAZON" ]]; then + # first check and activate everything found in user data + copy_user_data_to_tmp_file + auto_install + else + echo "This server does not seem to be running on Amazon! Automatic install only works on Amazon instances." + exit 1 + fi + rm "${ec2EnvVars_tmpFile}" + +elif [[ $OPERATION == "auto-install-from-stdin" ]]; then + # copy stdin to user data tmp file, + cat >"${ec2EnvVars_tmpFile}" + # then auto-install + auto_install + rm "${ec2EnvVars_tmpFile}" + +elif [[ $OPERATION == "install-release" ]]; then + INSTALL_FROM_RELEASE=$PARAM + # Honor the no-overrite setting if there is one + if [ -f $SERVER_HOME/no-overwrite ]; then + echo "Found a no-overwrite file in the servers directory. Please remove it to complete this operation!" + else + load_from_release_file + echo "ATTENTION: This new release code is not active yet. Make sure to restart the server if it is running!" + fi + +elif [[ $OPERATION == "install-local-release" ]]; then + INSTALL_FROM_RELEASE=$PARAM + if [[ $INSTALL_FROM_RELEASE == "" ]]; then + echo "You need to provide the file of a tar.gz release file, without the .tar.gz suffix" + exit 1 + fi + + # Honor the no-overrite setting if there is one + if [ -f $SERVER_HOME/no-overwrite ]; then + echo "Found a no-overwrite file in the servers directory. Please remove it to complete this operation!" + else + load_from_local_release_file + echo "ATTENTION: This new release code is not active yet. Make sure to restart the server if it is running!" + fi + +elif [[ $OPERATION == "install-env" ]]; then + USE_ENVIRONMENT=$PARAM + if [[ $USE_ENVIRONMENT == "" ]]; then + echo "You need to provide the name of an environment from https://releases.sapsailing.com/environments" + exit 1 + fi + + if [ -f $SERVER_HOME/no-overwrite ]; then + echo "Found a no-overwrite file in the servers directory. Please remove it to complete this operation!" + else + install_environment + # make sure to reload data + source `pwd`/env.sh + + echo "Configuration for this server is now:" + echo "" + echo "SERVER_NAME: $SERVER_NAME" + echo "MEMORY: $MEMORY" + echo "SERVER_PORT: $SERVER_PORT" + echo "TELNET_PORT: $TELNET_PORT" + echo "MONGODB_HOST: $MONGODB_HOST" + echo "MONGODB_PORT: $MONGODB_PORT" + echo "MONGODB_URI: $MONGODB_URI" + echo "EXPEDITION_PORT: $EXPEDITION_PORT" + echo "REPLICATION_HOST: $REPLICATION_HOST" + echo "REPLICATION_CHANNEL: $REPLICATION_CHANNEL" + echo "ADDITIONAL_ARGS: $ADDITIONAL_JAVA_ARGS" + echo "" + echo "INSTALL_FROM_RELEASE: $INSTALL_FROM_RELEASE" + echo "DEPLOY_TO: $DEPLOY_TO" + echo "BUILD_BEFORE_START: $BUILD_BEFORE_START" + echo "USE_ENVRIONMENT: $USE_ENVIRONMENT" + echo "" + echo "JAVA_HOME: $JAVA_HOME" + echo "INSTANCE_ID: $INSTANCE_ID" + echo "" + echo "ATTENTION: This new configuration is not active yet. Make sure to restart the server if it is running!" + fi +elif [[ $OPERATION == "install-user-data" ]]; then + if [ -f $SERVER_HOME/no-overwrite ]; then + echo "Found a no-overwrite file in the servers directory. Please remove it to complete this operation!" + else + append_user_data_to_envsh + fi +else + echo "Script to prepare a Java instance running on Amazon." + echo "" + echo "auto-install: downloads, builds, or takes from a local file a release, unpacks it in the current directory and applies the environment, user data, and environment defaults to the env.sh file" + echo "auto-install-from-stdin: like auto-install, only that the additional configuration data is sourced from standard input (stdin), not the AWS EC2 instance's user data." + echo "install-release : Downloads the release specified by the second option and overwrites all code for this server. Preserves env.sh." + echo "install-local-release : Installs the release file specified by the second option and overwrites all code for this server. Preserves env.sh." + echo "install-env : Downloads and updates the environment with the one specified as a second option. Does NOT take into account Amazon user-data!" + echo "install-user-data: appends the user data set for the EC2 instance to the env.sh file" + exit 0 +fi diff --git a/configuration/environments_scripts/sailing_server/files/usr/local/bin/sailing b/configuration/environments_scripts/sailing_server/files/usr/local/bin/sailing index 294b4d8b7c3..67ba3b239de 100755 --- a/configuration/environments_scripts/sailing_server/files/usr/local/bin/sailing +++ b/configuration/environments_scripts/sailing_server/files/usr/local/bin/sailing @@ -7,15 +7,15 @@ # -# Source function library. -. /etc/init.d/functions - +# Source function library. This alters the PATH variable. +if [ -f /etc/init.d/functions ]; then + . /etc/init.d/functions +fi RETVAL=0 SERVERS_DIR=/home/sailing/servers cd "${SERVERS_DIR}" JAVA_START_INSTANCES="$(find * -type d -prune)" -GIT_REPOSITORY=/home/sailing/code if [ -x /bin/ec2-metadata ]; then EC2_METADATA_CMD=/bin/ec2-metadata elif [ -x /usr/bin/ec2-metadata ]; then @@ -27,28 +27,24 @@ REBOOT_INDICATOR=/var/lib/sailing/is-rebooted mkdir --parents $( dirname ${REBOOT_INDICATOR} ) SSH_KEY_READER_BEARER_TOKEN=/root/ssh-key-reader.token -echo "Executing with $1 at `date`" >>/var/log/sailing.err +echo "Executing with $1 at `date`" start_servers() { - /usr/local/bin/update_authorized_keys_for_landscape_managers $( cat ${SSH_KEY_READER_BEARER_TOKEN} ) https://security-service.sapsailing.com /root 2>&1 >>/var/log/sailing.err - cp /home/sailing/code/configuration/cp_root_mail_properties /usr/local/bin - chown root /usr/local/bin/cp_root_mail_properties - chgrp root /usr/local/bin/cp_root_mail_properties + /usr/local/bin/update_authorized_keys_for_landscape_managers $( cat ${SSH_KEY_READER_BEARER_TOKEN} ) https://security-service.sapsailing.com /root + chown root:root /usr/local/bin/cp_root_mail_properties chmod 755 /usr/local/bin/cp_root_mail_properties - cp /home/sailing/code/configuration/cp_root_mail_properties_sudoers /etc/sudoers.d if which $EC2_METADATA_CMD && $EC2_METADATA_CMD -d | sed "s/user-data\: //g" | grep "^image-upgrade$"; then - echo "Found image-upgrade in EC2 user data; upgrading image, then probably shutting down for AMI creation depending on the no-shutdown user data string..." >>/var/log/sailing.err - $GIT_REPOSITORY/configuration/imageupgrade.sh + echo "Found image-upgrade in EC2 user data; not configuring/starting sailing server processes" else - echo "No image-upgrade request found in EC2 user data $($EC2_METADATA_CMD -d); proceeding with regular server launch..." >>/var/log/sailing.err - echo "Servers to launch: ${JAVA_START_INSTANCES}" >>/var/log/sailing.err + echo "No image-upgrade request found in EC2 user data $($EC2_METADATA_CMD -d); proceeding with regular server launch..." + echo "Servers to launch: ${JAVA_START_INSTANCES}" if [ -f "${REBOOT_INDICATOR}" ]; then - echo "This is a re-boot. No EC2 user data is evaluated for server configuration; no server configuration is performed. Only configured applications are launched." >>/var/log/sailing.err + echo "This is a re-boot. No EC2 user data is evaluated for server configuration; no server configuration is performed. Only configured applications are launched." for conf in ${JAVA_START_INSTANCES}; do - su - sailing -c "cd ${SERVERS_DIR}/${conf} && ./start" 2>>/var/log/sailing.err >>/var/log/sailing.err + su - sailing -c "cd ${SERVERS_DIR}/${conf} && ./start" done else - echo "This is a first-time boot. EC2 user data is evaluated for potential application deployment and configuration, and applications are launched." >>/var/log/sailing.err + echo "This is a first-time boot. EC2 user data is evaluated for potential application deployment and configuration, and applications are launched." echo "Initializing local MongoDB replica set \"replica\"..." while ! echo "rs.initiate()" | mongo; do echo "MongoDB not ready yet; waiting and trying again..." @@ -56,9 +52,9 @@ start_servers() { done FIRST_SERVER=$( eval $( ${EC2_METADATA_CMD} -d | sed -e 's/^user-data: //' ); echo $SERVER_NAME ) if [ "${FIRST_SERVER}" = "" ]; then - echo "No SERVER_NAME provided; not configuring/starting any application processes" >>/var/log/sailing.err + echo "No SERVER_NAME provided; not configuring/starting any application processes" else - echo "Server to configure and start: ${FIRST_SERVER}" >>/var/log/sailing.err + echo "Server to configure and start: ${FIRST_SERVER}" configure_and_start_server "${FIRST_SERVER}" fi echo 1 >"${REBOOT_INDICATOR}" @@ -77,26 +73,19 @@ configure_and_start_server() { chgrp sailing "${SERVERS_DIR}/${conf}" # If there is a secret /root/mail.properties, copy it into the default server's configuration directory: /usr/local/bin/cp_root_mail_properties "${conf}" - su - sailing -c "cd ${SERVERS_DIR}/${conf} && ${GIT_REPOSITORY}/java/target/refreshInstance.sh auto-install; ./start" 2>>/var/log/sailing.err >>/var/log/sailing.err - pushd ${SERVERS_DIR}/${conf} - ./defineReverseProxyMappings.sh 2>>/var/log/sailing.err >>/var/log/sailing.err - popd - RETVAL=$? - [ $RETVAL -eq 0 ] && success || failure + su - sailing -c "cd ${SERVERS_DIR}/${conf} && refreshInstance.sh auto-install; ./start" } stop_servers() { for conf in $JAVA_START_INSTANCES; do - echo "Stopping Java server $conf" >> /var/log/sailing.err + echo "Stopping Java server $conf" su - sailing -c "cd $SERVERS_DIR/$conf && ./stop" - RETVAL=$? - [ $RETVAL -eq 0 ] && success || failure sync_logs done } sync_logs() { - echo "Executing logrotate followed by a sync to ensure that logs are synchronized" >> /var/log/sailing.err + echo "Executing logrotate followed by a sync to ensure that logs are synchronized" logrotate -f /etc/logrotate.conf sync } diff --git a/configuration/sailing_server_setup/setup-sailing-server.sh b/configuration/environments_scripts/sailing_server/setup-sailing-server.sh similarity index 57% rename from configuration/sailing_server_setup/setup-sailing-server.sh rename to configuration/environments_scripts/sailing_server/setup-sailing-server.sh index 0e031b0abf5..c23c5b8c396 100755 --- a/configuration/sailing_server_setup/setup-sailing-server.sh +++ b/configuration/environments_scripts/sailing_server/setup-sailing-server.sh @@ -18,86 +18,54 @@ if [ $# != 0 ]; then else if ec2-metadata | grep -q instance-id; then echo "Running on an AWS EC2 instance as user ${USER} / $(whoami), starting setup..." + # Install standard packages: + sudo yum -y update + sudo yum -y install git tmux nvme-cli chrony cronie cronie-anacron jq telnet mailx # Allow root ssh login with the same key used for the ec2-user for now; # later, a cron job will be installed that keeps the /root/authorized_keys file # up to date with all landscape managers' public SSH keys sudo cp /home/ec2-user/.ssh/authorized_keys /root/.ssh sudo chown root /root/.ssh/authorized_keys sudo chgrp root /root/.ssh/authorized_keys - sudo adduser sailing - sudo su - sailing -c "mkdir servers" + # Copy imageupgrade_function.sh + scp -o StrictHostKeyChecking=no -p root@sapsailing.com:/home/wiki/gitwiki/configuration/environments_scripts/repo/usr/local/bin/imageupgrade_functions.sh . + sudo mv imageupgrade_functions.sh /usr/local/bin + # build-crontab + . imageupgrade_functions.sh + build_crontab_and_setup_files sailing_server # Create an SSH key pair with empty passphrase for ec2-user, deploy it to trac@sapsailing.com # and then move it to the sailing user's .ssh directory - ssh-keygen -t ed25519 -P '' -f /home/ec2-user/.ssh/id_ed25519 - cat /home/ec2-user/.ssh/id_ed25519.pub | ssh -o StrictHostKeyChecking=false root@sapsailing.com "cat >>/home/trac/.ssh/authorized_keys" - sudo mkdir /home/sailing/.ssh - sudo mv /home/ec2-user/.ssh/id* /home/sailing/.ssh - sudo chown -R sailing /home/sailing/.ssh - sudo chgrp -R sailing /home/sailing/.ssh - sudo chmod 700 /home/sailing/.ssh - # Install standard packages: - sudo yum -y update - sudo yum -y install git tmux nvme-cli chrony cronie cronie-anacron jq telnet mailx + setup_keys "sailing_server" + sudo su - sailing -c "mkdir servers" # Force acceptance of sapsailing.com's host key: sudo su - sailing -c "ssh -o StrictHostKeyChecking=false trac@sapsailing.com ls" >/dev/null - # Clone Git to /home/sailing/code - sudo su - sailing -c "git clone ssh://trac@sapsailing.com/home/trac/git code" + # Keep Amazon Linux from patching root's authorized_keys file and setup root login: + setup_cloud_cfg_and_root_login # Install SAP JVM 8: sudo mkdir -p /opt - sudo su - -c "source /home/sailing/code/configuration/imageupgrade_functions.sh; download_and_install_latest_sap_jvm_8" - # Install sailing.sh script to /etc/profile.d - sudo ln -s /home/sailing/code/configuration/sailing.sh /etc/profile.d - # Keep Amazon Linux from patching root's authorized_keys file: - sudo sed -i -e 's/disable_root: *true/disable_root: false/' /etc/cloud/cloud.cfg - # Configure SSH daemon: - sudo su - -c "cat << EOF >>/etc/ssh/sshd_config -PermitRootLogin without-password -PermitRootLogin Yes -MaxStartups 100 -EOF -" + sudo su - -c "source /usr/local/bin/imageupgrade_functions.sh; download_and_install_latest_sap_jvm_8" # Increase limits sudo su - -c "cat << EOF >>/etc/sysctl.conf # number of connections the firewall can track net.ipv4.ip_conntrac_max = 131072 EOF " - # Install mountnvmeswap stuff - sudo ln -s /home/sailing/code/configuration/sailing_server_setup/mountnvmeswap /usr/local/bin - sudo ln -s /home/sailing/code/configuration/sailing_server_setup/mountnvmeswap.service /etc/systemd/system sudo systemctl daemon-reload sudo systemctl enable mountnvmeswap.service # Install MongoDB 4.4 and configure as replica set "replica" - sudo su - -c "cat << EOF >/etc/yum.repos.d/mongodb-org.4.4.repo -[mongodb-org-4.4] -name=MongoDB Repository -baseurl=https://repo.mongodb.org/yum/amazon/2/mongodb-org/4.4/x86_64/ -gpgcheck=1 -enabled=1 -gpgkey=https://www.mongodb.org/static/pgp/server-4.4.asc -EOF -" - sudo yum -y update - sudo yum -y install mongodb-org-server mongodb-org-shell mongodb-org-tools + setup_mongo_4_4 sudo su - -c "cat << EOF >>/etc/mongod.conf replication: replSetName: replica EOF " sudo sed -i -e 's/bindIp: *[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+/bindIp: 0.0.0.0/' /etc/mongod.conf - # Install cron job for ssh key update for landscape managers - sudo ln -s /home/sailing/code/configuration/update_authorized_keys_for_landscape_managers /usr/local/bin - sudo ln -s /home/sailing/code/configuration/update_authorized_keys_for_landscape_managers_if_changed /usr/local/bin - sudo ln -s /home/sailing/code/configuration/sailing_server_setup/crontab-root /root/crontab - sudo su - -c "crontab /root/crontab" scp root@sapsailing.com:ssh-key-reader.token /tmp sudo mv /tmp/ssh-key-reader.token /root sudo chown root /root/ssh-key-reader.token sudo chgrp root /root/ssh-key-reader.token sudo chmod 600 /root/ssh-key-reader.token # Install /etc/init.d/sailing start-up / shut-down service - sudo ln -s /home/sailing/code/configuration/sailing /etc/init.d/sailing - sudo ln -s /home/sailing/code/configuration/sailing_server_setup/sailing.service /etc/systemd/system sudo systemctl daemon-reload sudo systemctl enable sailing.service # Install secrets @@ -112,15 +80,11 @@ EOF sudo chgrp root /root/mail.properties sudo chmod 600 /root/mail.properties # Create some swap space for the case mountnvmeswap hasn't created any - sudo dd if=/dev/zero of=/var/cache/swapfile bs=1M count=6000 - sudo chown root /var/cache/swapfile - sudo chgrp root /var/cache/swapfile - sudo chmod 600 /var/cache/swapfile - sudo mkswap /var/cache/swapfile - # And while adding to /etc/fstab, also add the NFS mount of /home/scores: + setup_swap 6000 + setup_sshd_resilience + # Add the NFS mount of /home/scores to /etc/fstab: sudo mkdir /home/scores - sudo su - -c 'echo "/var/cache/swapfile none swap pri=0 0 0 -logfiles.internal.sapsailing.com:/home/scores /home/scores nfs tcp,intr,timeo=100,retry=0" >>/etc/fstab' + sudo su - -c 'echo "logfiles.internal.sapsailing.com:/home/scores /home/scores nfs tcp,intr,timeo=100,retry=0" >>/etc/fstab' sudo swapon -a else echo "Not running on an AWS instance; refusing to run setup!" >&2 diff --git a/configuration/environments_scripts/sailing_server/users/root/crontab-update-authorized-keys b/configuration/environments_scripts/sailing_server/users/root/crontab-update-authorized-keys deleted file mode 120000 index d9b0fed0726..00000000000 --- a/configuration/environments_scripts/sailing_server/users/root/crontab-update-authorized-keys +++ /dev/null @@ -1 +0,0 @@ -../../../../crontabs/crontab-update-authorized-keys \ No newline at end of file diff --git a/configuration/environments_scripts/sailing_server/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_root b/configuration/environments_scripts/sailing_server/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_root new file mode 120000 index 00000000000..cf597d8e543 --- /dev/null +++ b/configuration/environments_scripts/sailing_server/users/root/crontab-update-authorized-keys@USER_HOME_DIR=_root @@ -0,0 +1 @@ +../../../../crontabs/crontab-update-authorized-keys@USER_HOME_DIR \ No newline at end of file diff --git a/configuration/environments_scripts/sailing_server/users/sailing/uid b/configuration/environments_scripts/sailing_server/users/sailing/uid new file mode 100644 index 00000000000..7cebf7db5f2 --- /dev/null +++ b/configuration/environments_scripts/sailing_server/users/sailing/uid @@ -0,0 +1 @@ +1001 \ No newline at end of file diff --git a/configuration/finduserbyemail.sh b/configuration/finduserbyemail.sh deleted file mode 100755 index a1bac2e4571..00000000000 --- a/configuration/finduserbyemail.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh - -# Executes the following script on the DB server: -##!/bin/sh -#useremail=$1 -#for port in 10201 10202 10203; do -# for db in `echo "rs.slaveOk() -# show dbs -# quit()" | mongo --port $port | tail -n +3 | grep -v "^bye$" | awk '{print $1;}'`; do -# match_1=`echo "rs.slaveOk() -# db.USERS.find({EMAIL: \"$useremail\"})" | mongo --quiet --port $port $db` -# match_2=`echo "rs.slaveOk() -# db.COMPETITORS.find({email: \"$useremail\"})" | mongo --quiet --port $port $db` -# if [ "$match_1" != "" -o "$match_2" != "" ]; then -# echo ${db} on port $port: $match_1 $match_2 -# fi -# done -#done - -ssh -A root@sapsailing.com ssh -A ec2-user@dbserver.internal.sapsailing.com finduserbyemail $1 diff --git a/configuration/github-copy-release-to-sapsailing-com.sh b/configuration/github-copy-release-to-sapsailing-com.sh new file mode 100755 index 00000000000..a098f70a189 --- /dev/null +++ b/configuration/github-copy-release-to-sapsailing-com.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Download release tar ball and release-notes.txt into the current working directory +# and echo the .tar.gz file name to the standard output if found. If no release is found +# for the prefix specified, nothing it sent to the standard output. The exit status will +# always be 0. +# +# Usage: +# ./github-copy-release-to-sapsailing-com.sh {BEARER_TOKEN} {release-name-prefix} +# For example: +# ./github-copy-release-to-sapsailing-com.sh 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 main- +# +# which will download the latest build of the main branch (main-xxxxxxxxxxx) and upload +# it to releases.sapsailing.com. +# Note the "-" at the end of the "main-" prefix specifier; this way we're making name +# clashes with releases whose name happens to start with "main" unlikely. This +# also suggests you shouldn't name releases "main-abcde-xxxxxxxxxxxx" because they would +# produce false matches for the "main-" prefix. +# For the "main" branch/release a special step is added for backward compatibility with +# java/target/refreshInstance.sh: the release is linked to from an additional folder +# called "build-xxxxxxxxxx" (using the same timestamp in its name), with the tar ball +# and the release notes being symbolic links to the corresponding files in the main-xxxxxxxxx +# folder. +BEARER_TOKEN="${1}" +RELEASE_NAME_PREFIX="${2}" +RELEASE_TAR_GZ_FILE_NAME=$( `dirname "${0}"`/github-download-release-assets.sh "${BEARER_TOKEN}" "${RELEASE_NAME_PREFIX}" ) +if [ "${RELEASE_TAR_GZ_FILE_NAME}" != "" ]; then + RELEASE_NAME=$( echo ${RELEASE_TAR_GZ_FILE_NAME} | sed -e 's/^\(.*\)-\([0-9]*\).tar.gz$/\1/' ) + RELEASE_TIMESTAMP=$( echo ${RELEASE_TAR_GZ_FILE_NAME} | sed -e 's/^\(.*\)-\([0-9]*\).tar.gz$/\2/' ) + RELEASE_FULL_NAME="${RELEASE_NAME}-${RELEASE_TIMESTAMP}" + echo "Found release ${RELEASE_FULL_NAME} with name ${RELEASE_NAME} and time stamp ${RELEASE_TIMESTAMP}" >&2 + FOLDER=releases/${RELEASE_FULL_NAME} + ssh trac@sapsailing.com mkdir -p ${FOLDER} + scp ${RELEASE_FULL_NAME}.tar.gz trac@sapsailing.com:${FOLDER} + scp release-notes.txt trac@sapsailing.com:${FOLDER} + if [ "${RELEASE_NAME}" = "main" ]; then + LINKING_FOLDER=releases/build-${RELEASE_TIMESTAMP} + ssh trac@sapsailing.com "mkdir -p ${LINKING_FOLDER}; ln -s ../${RELEASE_FULL_NAME}/${RELEASE_FULL_NAME}.tar.gz ${LINKING_FOLDER}/build-${RELEASE_TIMESTAMP}.tar.gz; ln -s ../${RELEASE_FULL_NAME}/release-notes.txt ${LINKING_FOLDER}" + fi + rm ${RELEASE_TAR_GZ_FILE_NAME} release-notes.txt +fi diff --git a/configuration/github-download-release-assets.sh b/configuration/github-download-release-assets.sh new file mode 100755 index 00000000000..bf50e6d7e4a --- /dev/null +++ b/configuration/github-download-release-assets.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Download release tar ball and release-notes.txt into the current working directory +# and echo the .tar.gz file name to the standard output if found. If no release is found +# for the prefix specified, nothing it sent to the standard output. The exit status will +# always be 0. +# +# Usage: +# ./github-download-release-assets.sh {BEARER_TOKEN} {release-name-prefix} +# For example: +# ./github-download-release-assets.sh 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 main- +# which will download the latest release tar.gz and release-notes.txt of the main branch (main-xxxxxxxxxxx). +# Note the "-" at the end of the "main-" prefix specifier; this way we're making name +# clashes with releases whose name happens to start with "main" unlikely. This +# also suggests you shouldn't name releases "main-abcde-xxxxxxxxxxxx" because they would +# produce false matches for the "main-" prefix. +BEARER_TOKEN="${1}" +RELEASE_NAME_PREFIX="${2}" +RELEASES=$( curl -L -H 'Authorization: Bearer '${BEARER_TOKEN} https://api.github.com/repos/SAP/sailing-analytics/releases 2>/dev/null ) +RELEASE_NOTES_TXT_ASSET_ID=$( echo "${RELEASES}" | jq -r 'sort_by(.published_at) | reverse | map(select(.name | startswith("'${RELEASE_NAME_PREFIX}'")))[0].assets[] | select(.content_type=="text/plain").id' 2>/dev/null) +if [ "$?" -ne "0" ]; then + echo "No release with prefix ${RELEASE_NAME_PREFIX} found. Not trying to download/upload anything." >&2 +else + RELEASE_TAR_GZ_ASSET_ID=$( echo "${RELEASES}" | jq -r 'sort_by(.published_at) | reverse | map(select(.name | startswith("'${RELEASE_NAME_PREFIX}'")))[0].assets[] | select(.content_type=="application/x-tar").id' ) + RELEASE_FULL_NAME=$( echo "${RELEASES}" | jq -r 'sort_by(.published_at) | reverse | map(select(.name | startswith("'${RELEASE_NAME_PREFIX}'")))[0].assets[] | select(.content_type=="application/x-tar").name' | sed -e 's/\.tar\.gz$//') + # For backward compatibility with deployed versions of java/target/refreshInstance.sh + # which may be looking for a default release named "build-...", in case the release + # is a "main-..." release, additionally create a corresponding folder and symbolic links named "build-..." + RELEASE_NAME=$( echo ${RELEASE_FULL_NAME} | sed -e 's/^\(.*\)-\([0-9]*\)$/\1/' ) + RELEASE_TIMESTAMP=$( echo ${RELEASE_FULL_NAME} | sed -e 's/^\(.*\)-\([0-9]*\)$/\2/' ) + echo "Found release ${RELEASE_FULL_NAME} with name ${RELEASE_NAME} and time stamp ${RELEASE_TIMESTAMP}, notes ID is ${RELEASE_NOTES_TXT_ASSET_ID}, tarball ID is ${RELEASE_TAR_GZ_ASSET_ID}" >&2 + RELEASE_TAR_GZ_FILE_NAME="${RELEASE_FULL_NAME}.tar.gz" + curl -o "${RELEASE_TAR_GZ_FILE_NAME}" -L -H 'Accept: application/octet-stream' -H 'Authorization: Bearer '${BEARER_TOKEN} 'https://api.github.com/repos/SAP/sailing-analytics/releases/assets/'${RELEASE_TAR_GZ_ASSET_ID} + curl -o release-notes.txt -L -H 'Accept: application/octet-stream' -H 'Authorization: Bearer '${BEARER_TOKEN} 'https://api.github.com/repos/SAP/sailing-analytics/releases/assets/'${RELEASE_NOTES_TXT_ASSET_ID} + echo "${RELEASE_TAR_GZ_FILE_NAME}" +fi diff --git a/configuration/github-download-workflow-artifacts.sh b/configuration/github-download-workflow-artifacts.sh new file mode 100755 index 00000000000..948b4ce97ba --- /dev/null +++ b/configuration/github-download-workflow-artifacts.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Call with two arguments: +# - the name of the branch for which you would like +# to download the latest workflow run's build artifacts +# - the Github PAT (personal access token) +# If found, the build.log.zip and test-result.zip files +# will be downloaded to the current working directory. +# The script will exit with status 0 if the workflow's +# conclusion was "success" and the artifacts downloaded +# fine. If the workflow's conclusion was not "success", +# an exit status of 1 is returned. If the downloads fail, +# an exit status of 2 is returned. +BRANCH="${1}" +BEARER_TOKEN="${2}" +# Get the artifacts URL of the last workflow run triggered by a branch push for ${BRANCH}: +LAST_WORKFLOW_FOR_BRANCH=$( curl --silent -L -H 'Authorization: Bearer '${BEARER_TOKEN} https://api.github.com/repos/SAP/sailing-analytics/actions/runs 2>/dev/null | jq -r '.workflow_runs | map(select(.status == "completed" and .name == "release" and ((.head_branch | startswith("'${BRANCH}'")) or (.head_branch | startswith("releases/'${BRANCH}'"))))) | sort_by(.updated_at) | reverse | .[0]' ) +ARTIFACTS_URL=$( echo "${LAST_WORKFLOW_FOR_BRANCH}" | jq -r '.artifacts_url' ) +CONCLUSION=$( echo "${LAST_WORKFLOW_FOR_BRANCH}" | jq -r '.conclusion' ) +ARTIFACTS_JSON=$( curl --silent -H 'Authorization: Bearer '${BEARER_TOKEN} "${ARTIFACTS_URL}" ) +if [ -z "${ARTIFACTS_JSON}" ]; then + echo "Workflow run or artifacts not found" + exit 1 +fi +BUILD_LOG_URL=$( echo "${ARTIFACTS_JSON}" | jq -r '.artifacts | map(select(.name == "build.log"))[0].archive_download_url' ) +if [ -z "${BUILD_LOG_URL}" ]; then + echo "build.log artifact not found" + exit 2 +fi +echo "Downloading build.log ZIP from ${BUILD_LOG_URL}" +curl --silent --output build.log.zip -L -H 'Authorization: Bearer '${BEARER_TOKEN} "${BUILD_LOG_URL}" +TEST_RESULTS_URL=$( echo "${ARTIFACTS_JSON}" | jq -r '.artifacts | map(select(.name == "test-results"))[0].archive_download_url' ) +if [ -z "${TEST_RESULTS_URL}" ]; then + echo "test-results artifact not found" + exit 2 +fi +echo "Downloading test-results ZIP from ${TEST_RESULTS_URL}" +curl --silent --output test-results.zip -L -H 'Authorization: Bearer '${BEARER_TOKEN} "${TEST_RESULTS_URL}" +if [ "${CONCLUSION}" != "success" ]; then + exit 1 +fi diff --git a/configuration/imageupgrade.sh b/configuration/imageupgrade.sh deleted file mode 120000 index be2deb37e8f..00000000000 --- a/configuration/imageupgrade.sh +++ /dev/null @@ -1 +0,0 @@ -environments_scripts/sailing_server/files/usr/local/bin/imageupgrade.sh \ No newline at end of file diff --git a/configuration/imageupgrade_functions.sh b/configuration/imageupgrade_functions.sh deleted file mode 120000 index 3929d31afe3..00000000000 --- a/configuration/imageupgrade_functions.sh +++ /dev/null @@ -1 +0,0 @@ -environments_scripts/sailing_server/files/usr/local/bin/imageupgrade_functions.sh \ No newline at end of file diff --git a/configuration/init.d/mongodb b/configuration/init.d/mongodb deleted file mode 100644 index 69fea4bd272..00000000000 --- a/configuration/init.d/mongodb +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# -# mongodb Starts sailing services like MongoDB -# -# chkconfig: 2345 95 10 -# description: Sailing contains all sailing services - -# Source function library. -. /etc/init.d/functions - -RETVAL=0 - -MONGODB_BINARY=/opt/mongodb-linux-x86_64-1.8.1/bin/mongod -MONGODB_CONF_DIR=/etc/mongodb -MONGODB_START_INSTANCES="default dev live archive racecommittee_dev" - -start_mongodb() { - for conf in $MONGODB_START_INSTANCES; do - echo "Starting MongoDB server $conf" - rm -f /var/lib/mongodb/$conf/mongod.lock - su - mongodb -c "$MONGODB_BINARY --quiet --fork -f $MONGODB_CONF_DIR/mongodb-$conf.cfg" - RETVAL=$? - [ $RETVAL -eq 0 ] && success || failure - done - - # give database servers some time to start - sleep 2 -} - -stop_mongodb() { - killall mongod - sleep 2 -} - -# See how we were called. -case "$1" in - start) - start_mongodb - ;; - stop) - stop_mongodb - ;; - start-mongo) - start_mongodb - ;; - start-wiki) - start_wiki - ;; - status) - status mongod - RETVAL=$? - ;; - *) - echo $"Usage: $0 {start|stop|status}" - RETVAL=3 -esac - -exit $RETVAL diff --git a/configuration/install-gwt-from-fork-releases b/configuration/install-gwt-from-fork-releases new file mode 100755 index 00000000000..fd524fbd3dd --- /dev/null +++ b/configuration/install-gwt-from-fork-releases @@ -0,0 +1,30 @@ +#!/bin/bash +# Downloads releases of GWT and the gwt-maven-plugin and installs them to the local Maven repo at ~/.m2 +# as well as copies the GWT JARs to the sailing Git workspace +# Usage: install-gwt-from-fork-releases {gwt-repo-URL} {gwt-maven-plugin-repo-URL} {gwt-version} {sailing-git-root} +# Example: install-gwt-from-fork-releases https://github.com/axeluhl/gwt https://github.com/axeluhl/gwt-maven-plugin 2.11.0 . +GWT_REPO_URL="${1}" +GWT_MAVEN_PLUG_REPO_URL="${2}" +GWT_VERSION="${3}" +SAILING_GIT_ROOT="$( cd ${4}; pwd )" +GWT="`mktemp -t -d gwt.XXXXX`" +cd "${GWT}" +echo "GWT directory: ${GWT}" +curl -L "${GWT_REPO_URL}/releases/download/gwt-${GWT_VERSION}/gwt-${GWT_VERSION}.zip" -o "gwt-${GWT_VERSION}.zip" +curl -L "${GWT_MAVEN_PLUG_REPO_URL=}/releases/download/gwt-${GWT_VERSION}/gwt-maven-plugin-${GWT_VERSION}.jar" -o "gwt-maven-plugin-${GWT_VERSION}.jar" +curl -L "${GWT_MAVEN_PLUG_REPO_URL=}/releases/download/gwt-${GWT_VERSION}/gwt-maven-plugin-${GWT_VERSION}.pom" -o "gwt-maven-plugin-${GWT_VERSION}.pom" +unzip gwt-${GWT_VERSION}.zip +cd gwt-${GWT_VERSION} +cp gwt-dev.jar ${SAILING_GIT_ROOT}/java/com.google.gwt.dev/lib/gwt-dev.jar +cp gwt-user.jar ${SAILING_GIT_ROOT}/java/com.google.gwt.user/lib/gwt-user.jar +cp gwt-servlet.jar ${SAILING_GIT_ROOT}/java/com.google.gwt.servlet/lib/gwt-servlet.jar +cp gwt-servlet-deps.jar ${SAILING_GIT_ROOT}/java/com.google.gwt.servlet/lib/gwt-servlet-deps.jar +for i in dev user servlet; do + mvn --batch-mode install:install-file -Dfile=gwt-${i}.jar -DpomFile="${SAILING_GIT_ROOT}/configuration/gwt-poms/gwt-$i-${GWT_VERSION}.pom" +done +mvn --batch-mode install:install-file -Dpackaging=pom -Dfile="${SAILING_GIT_ROOT}/configuration/gwt-poms/gwt-${GWT_VERSION}.pom" -DpomFile="${SAILING_GIT_ROOT}/configuration/gwt-poms/gwt-${GWT_VERSION}.pom" +mvn --batch-mode install:install-file -Dpackaging=pom -Dfile="${SAILING_GIT_ROOT}/configuration/gwt-poms/requestfactory-${GWT_VERSION}.pom" -DpomFile="${SAILING_GIT_ROOT}/configuration/gwt-poms/requestfactory-${GWT_VERSION}.pom" +# gwt-maven-plugin: +cd .. +mvn --batch-mode install:install-file -Dfile="gwt-maven-plugin-${GWT_VERSION}.jar" -DpomFile="gwt-maven-plugin-${GWT_VERSION}.pom" +rm -rf "${GWT}" diff --git a/configuration/mongo_instance_setup/README b/configuration/mongo_instance_setup/README deleted file mode 120000 index da42e16c027..00000000000 --- a/configuration/mongo_instance_setup/README +++ /dev/null @@ -1 +0,0 @@ -../environments_scripts/mongo_instance_setup/README \ No newline at end of file diff --git a/configuration/mongo_instance_setup/add-as-replica b/configuration/mongo_instance_setup/add-as-replica deleted file mode 120000 index 887b433c81d..00000000000 --- a/configuration/mongo_instance_setup/add-as-replica +++ /dev/null @@ -1 +0,0 @@ -../environments_scripts/mongo_instance_setup/files/usr/local/bin/add-as-replica \ No newline at end of file diff --git a/configuration/mongo_instance_setup/chownvarlibmongo.service b/configuration/mongo_instance_setup/chownvarlibmongo.service deleted file mode 120000 index 5b00b331375..00000000000 --- a/configuration/mongo_instance_setup/chownvarlibmongo.service +++ /dev/null @@ -1 +0,0 @@ -../environments_scripts/mongo_instance_setup/files/etc/systemd/system/chownvarlibmongo.service \ No newline at end of file diff --git a/configuration/mongo_instance_setup/crontab b/configuration/mongo_instance_setup/crontab deleted file mode 100644 index 9f4eeb382c9..00000000000 --- a/configuration/mongo_instance_setup/crontab +++ /dev/null @@ -1,3 +0,0 @@ -* * * * * export PATH=/bin:/usr/bin:/usr/local/bin; sleep $(( $RANDOM * 60 / 32768 )); update_authorized_keys_for_landscape_managers_if_changed $( cat /root/ssh-key-reader.token ) https://security-service.sapsailing.com /home/ec2-user 2>&1 >>/var/log/sailing.err -# NOTICE: Please try to reference the customised crontabs at $GIT_HOME/configuration/crontabs or use -# the build-crontab-and-cp-files script. This file has been maintained for continuity, but is deprecated. \ No newline at end of file diff --git a/configuration/mongo_instance_setup/ephemeralvolume b/configuration/mongo_instance_setup/ephemeralvolume deleted file mode 120000 index 461a6161b5e..00000000000 --- a/configuration/mongo_instance_setup/ephemeralvolume +++ /dev/null @@ -1 +0,0 @@ -../environments_scripts/mongo_instance_setup/files/usr/local/bin/ephemeralvolume \ No newline at end of file diff --git a/configuration/mongo_instance_setup/ephemeralvolume.service b/configuration/mongo_instance_setup/ephemeralvolume.service deleted file mode 120000 index 2167f128f54..00000000000 --- a/configuration/mongo_instance_setup/ephemeralvolume.service +++ /dev/null @@ -1 +0,0 @@ -../environments_scripts/mongo_instance_setup/files/etc/systemd/system/ephemeralvolume.service \ No newline at end of file diff --git a/configuration/mongo_instance_setup/imageupgrade.sh b/configuration/mongo_instance_setup/imageupgrade.sh deleted file mode 120000 index 329a0e6dccd..00000000000 --- a/configuration/mongo_instance_setup/imageupgrade.sh +++ /dev/null @@ -1 +0,0 @@ -../environments_scripts/mongo_instance_setup/files/usr/local/bin/imageupgrade.sh \ No newline at end of file diff --git a/configuration/mongo_instance_setup/mongo-replica-set.service b/configuration/mongo_instance_setup/mongo-replica-set.service deleted file mode 120000 index e24dc1470b6..00000000000 --- a/configuration/mongo_instance_setup/mongo-replica-set.service +++ /dev/null @@ -1 +0,0 @@ -../environments_scripts/mongo_instance_setup/files/etc/systemd/system/mongo-replica-set.service \ No newline at end of file diff --git a/configuration/mongo_instance_setup/mongod.conf b/configuration/mongo_instance_setup/mongod.conf deleted file mode 120000 index d1e8387ffda..00000000000 --- a/configuration/mongo_instance_setup/mongod.conf +++ /dev/null @@ -1 +0,0 @@ -../environments_scripts/mongo_instance_setup/files/etc/mongod.conf \ No newline at end of file diff --git a/configuration/mongo_instance_setup/mongodb b/configuration/mongo_instance_setup/mongodb deleted file mode 120000 index da48494d857..00000000000 --- a/configuration/mongo_instance_setup/mongodb +++ /dev/null @@ -1 +0,0 @@ -../environments_scripts/mongo_instance_setup/files/etc/logrotate.d/mongodb \ No newline at end of file diff --git a/configuration/mongo_instance_setup/patch-mongo-replicaset-name-from-ec2-metadata b/configuration/mongo_instance_setup/patch-mongo-replicaset-name-from-ec2-metadata deleted file mode 120000 index 934b72dce82..00000000000 --- a/configuration/mongo_instance_setup/patch-mongo-replicaset-name-from-ec2-metadata +++ /dev/null @@ -1 +0,0 @@ -../environments_scripts/mongo_instance_setup/files/usr/local/bin/patch-mongo-replicaset-name-from-ec2-metadata \ No newline at end of file diff --git a/configuration/mongo_instance_setup/patch-mongo-replicaset-name-from-ec2-metadata.service b/configuration/mongo_instance_setup/patch-mongo-replicaset-name-from-ec2-metadata.service deleted file mode 120000 index 10d95e0c525..00000000000 --- a/configuration/mongo_instance_setup/patch-mongo-replicaset-name-from-ec2-metadata.service +++ /dev/null @@ -1 +0,0 @@ -../environments_scripts/mongo_instance_setup/files/etc/systemd/system/patch-mongo-replicaset-name-from-ec2-metadata.service \ No newline at end of file diff --git a/configuration/mongo_instance_setup/remove-as-replica b/configuration/mongo_instance_setup/remove-as-replica deleted file mode 120000 index 506f1365f4e..00000000000 --- a/configuration/mongo_instance_setup/remove-as-replica +++ /dev/null @@ -1 +0,0 @@ -../environments_scripts/mongo_instance_setup/files/usr/local/bin/remove-as-replica \ No newline at end of file diff --git a/configuration/mysql_instance_setup/crontab-ec2-user b/configuration/mysql_instance_setup/crontab-ec2-user deleted file mode 100644 index 0e147adc31a..00000000000 --- a/configuration/mysql_instance_setup/crontab-ec2-user +++ /dev/null @@ -1,3 +0,0 @@ -* * * * * export PATH=/bin:/usr/bin:/usr/local/bin; sleep $(( $RANDOM * 60 / 32768 )); update_authorized_keys_for_landscape_managers_if_changed $( cat /home/ec2-user/ssh-key-reader.token ) https://security-service.sapsailing.com /home/ec2-user -# NOTICE: Please try to reference the customised crontabs at $GIT_HOME/configuration/crontabs or use -# the build_crontab_and_setup_files command in imageupgrade_functions.sh. This file has been maintained for continuity, but is deprecated. diff --git a/configuration/notify-unhealthy-mongodb b/configuration/notify-unhealthy-mongodb deleted file mode 120000 index 24b2aa93e61..00000000000 --- a/configuration/notify-unhealthy-mongodb +++ /dev/null @@ -1 +0,0 @@ -environments_scripts/central_reverse_proxy/files/home/trac/bin/notify-unhealthy-mongodb \ No newline at end of file diff --git a/configuration/on-site-scripts/paris2024/launch-replicas-in-region.sh b/configuration/on-site-scripts/paris2024/launch-replicas-in-region.sh index 619f5bac040..aa649a9f157 100755 --- a/configuration/on-site-scripts/paris2024/launch-replicas-in-region.sh +++ b/configuration/on-site-scripts/paris2024/launch-replicas-in-region.sh @@ -79,7 +79,7 @@ REPLICATE_MASTER_SERVLET_PORT=8888 REPLICATE_MASTER_EXCHANGE_NAME=paris2024 REPLICATE_MASTER_QUEUE_HOST=rabbit-eu-west-3.sapsailing.com REPLICATE_MASTER_BEARER_TOKEN=${BEARER_TOKEN} -ADDITIONAL_JAVA_ARGS=\"${ADDITIONAL_JAVA_ARGS} -Dcom.sap.sse.debranding=false\"" --ebs-optimized --key-name $KEY_NAME --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=SL Paris2024 (Upgrade Replica)},{Key=sailing-analytics-server,Value=paris2024}]" "ResourceType=volume,Tags=[{Key=Name,Value=SL Paris2024 (Upgrade Replica)}]" | jq -r '.Instances[].PrivateIpAddress + " " + .Instances[].InstanceId' ) +ADDITIONAL_JAVA_ARGS=\"${ADDITIONAL_JAVA_ARGS} -Dcom.sap.sse.debranding=true\"" --ebs-optimized --key-name $KEY_NAME --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=SL Paris2024 (Upgrade Replica)},{Key=sailing-analytics-server,Value=paris2024}]" "ResourceType=volume,Tags=[{Key=Name,Value=SL Paris2024 (Upgrade Replica)}]" | jq -r '.Instances[].PrivateIpAddress + " " + .Instances[].InstanceId' ) EXIT_CODE=$? if [ "${EXIT_CODE}" != "0" ]; then echo "Error launching instance in region ${REGION}. Exiting with status ${EXIT_CODE}" diff --git a/configuration/on-site-scripts/paris2024/monitor-paris2024-reachability-from-regions b/configuration/on-site-scripts/paris2024/monitor-paris2024-reachability-from-regions index ed22b6efe2e..66de7a53187 100755 --- a/configuration/on-site-scripts/paris2024/monitor-paris2024-reachability-from-regions +++ b/configuration/on-site-scripts/paris2024/monitor-paris2024-reachability-from-regions @@ -22,12 +22,12 @@ else echo "${message}" | notify-operators "${message}" else read first others <<<"${IPs}" - if ! ssh -o StrictHostKeyChecking=no root@${first} "curl https://palma2024.sailing.omegatiming.com/gwt/status 2>/dev/null >/dev/null"; then - echo "Problem reaching palma2024.sailing.omegatiming.com from instance ${first} in region ${i}" >&2 - message="Problem reaching palma2024.sailing.omegatiming.com from instance ${first} in region ${i}" - echo "${message}" | notify-operators "${message}" + if ! ssh -o StrictHostKeyChecking=no root@${first} "curl https://paris2024.sailing.omegatiming.com/gwt/status 2>/dev/null >/dev/null"; then + echo "Problem reaching paris2024.sailing.omegatiming.com from instance ${first} in region ${i}" >&2 + message="Problem reaching paris2024.sailing.omegatiming.com from instance ${first} in region ${i}" + echo "${message}" | notify-operators "${message}" else - echo "Access from region ${i}, IP ${first} OK." >&2 + echo "Access from region ${i}, IP ${first} OK." >&2 fi fi done diff --git a/configuration/on-site-scripts/paris2024/notify-operators b/configuration/on-site-scripts/paris2024/notify-operators index 9d841ea16aa..2b6e99bf7c4 100755 --- a/configuration/on-site-scripts/paris2024/notify-operators +++ b/configuration/on-site-scripts/paris2024/notify-operators @@ -1,5 +1,5 @@ #!/bin/bash -#OPERATORS="axel.uhl@sap.com jan.hamann@sapsailing.com alessandro.stoltenberg@sap.com alstolten@gmail.com" -OPERATORS="axel.uhl@sap.com thomasstokes@yahoo.co.uk" +OPERATORS="axel.uhl@sap.com jan.hamann@sapsailing.com alstolten@gmail.com" +#OPERATORS="axel.uhl@sap.com" logger -t sailing "Sending notification e-mail with subject $1 to ${OPERATORS}" mail -s "$1" ${OPERATORS} diff --git a/configuration/on-site-scripts/paris2024/regions.txt b/configuration/on-site-scripts/paris2024/regions.txt index dcd2bd407f4..cd6c8af475a 100644 --- a/configuration/on-site-scripts/paris2024/regions.txt +++ b/configuration/on-site-scripts/paris2024/regions.txt @@ -1,5 +1,5 @@ eu-west-3 -#ap-northeast-1 -#ap-southeast-2 -#us-west-1 -#us-east-1 +ap-northeast-1 +ap-southeast-2 +us-west-1 +us-east-1 diff --git a/configuration/on-site-scripts/paris2024/sap-p1-1/tractrac-A b/configuration/on-site-scripts/paris2024/sap-p1-1/tractrac-A index ea39a3d7d05..5263ebbde68 100755 --- a/configuration/on-site-scripts/paris2024/sap-p1-1/tractrac-A +++ b/configuration/on-site-scripts/paris2024/sap-p1-1/tractrac-A @@ -1,5 +1,5 @@ #!/bin/bash -pkill -f .autossh.*stso +pkill -f .autossh.*paris.*.tractrac.com sleep 2 # Tunnel to localhost for TracTrac port forwards -autossh -M 0 -f -A -N -L 4400:stso1:4400 -L 4401:stso1:4401 localhost +AUTOSSH_LOGLEVEL=7 AUTOSSH_POLL=5 autossh -M 0 -f -A -N -L 4400:paris1.tractrac.com:4400 -L 4401:paris1.tractrac.com:4401 localhost diff --git a/configuration/on-site-scripts/paris2024/sap-p1-1/tractrac-B b/configuration/on-site-scripts/paris2024/sap-p1-1/tractrac-B index 13f14f4ff19..e4148c10f1f 100755 --- a/configuration/on-site-scripts/paris2024/sap-p1-1/tractrac-B +++ b/configuration/on-site-scripts/paris2024/sap-p1-1/tractrac-B @@ -1,5 +1,5 @@ #!/bin/bash -pkill -f .autossh.*stso +pkill -f .autossh.*paris.*.tractrac.com sleep 2 # Tunnel to localhost for TracTrac port forwards -autossh -M 0 -f -A -N -L 4400:stso2:4400 -L 4401:stso2:4401 localhost +AUTOSSH_LOGLEVEL=7 AUTOSSH_POLL=5 autossh -M 0 -f -A -N -L 4400:paris2.tractrac.com:4400 -L 4401:paris2.tractrac.com:4401 localhost diff --git a/configuration/on-site-scripts/paris2024/sap-p1-1/tunnels b/configuration/on-site-scripts/paris2024/sap-p1-1/tunnels index b2aed5cd2b5..5703e83f988 100755 --- a/configuration/on-site-scripts/paris2024/sap-p1-1/tunnels +++ b/configuration/on-site-scripts/paris2024/sap-p1-1/tunnels @@ -2,7 +2,7 @@ killall autossh sleep 2 # Tunnel to cloud MongoDB on port 10203 with reverse tunnels for MongoDB on localhost:10201 and through tunnel on localhost:10202 to P1 #2 -autossh -M 20500 -f -A -N \ +AUTOSSH_LOGLEVEL=7 AUTOSSH_POLL=5 autossh -M 20500 -f -A -N \ -L 22443:sapsailing.com:443 \ -L 22222:sapsailing.com:22 \ -R 9443:localhost:9443 \ @@ -19,7 +19,7 @@ autossh -M 20500 -f -A -N \ ec2-user@paris-ssh.sapsailing.com sleep 2 # Tunnel to other laptop P1 #2, tunneling to MongoDB running on 10202, mapping to local port 10202 -autossh -M 20502 -f -A -N \ +AUTOSSH_LOGLEVEL=7 AUTOSSH_POLL=5 autossh -M 20502 -f -A -N \ -L 15674:localhost:15672 \ -L 10202:localhost:10202 \ -R 10201:localhost:10201 \ diff --git a/configuration/on-site-scripts/paris2024/sap-p1-1/tunnels-no-internet b/configuration/on-site-scripts/paris2024/sap-p1-1/tunnels-no-internet index 01c3799cb50..d0c4e5844a4 100755 --- a/configuration/on-site-scripts/paris2024/sap-p1-1/tunnels-no-internet +++ b/configuration/on-site-scripts/paris2024/sap-p1-1/tunnels-no-internet @@ -7,13 +7,13 @@ sleep 2 # localhost:9443 is established for access to this security-service.sapsailing.com backup replacement. #autossh -M 20500 -f -A -N -R 9443:localhost:9443 -R '*:8888:localhost:8888' -R 10201:localhost:10201 -L 10203:localhost:10203 -L 5673:rabbit-eu-west-3.sapsailing.com:5672 -L 15673:localhost:15673 -L 5675:rabbit.internal.sapsailing.com:5672 -L 15675:rabbit.internal.sapsailing.com:15672 -L 443:security-service.sapsailing.com:443 -R 18122:localhost:22 ec2-user@paris-ssh.sapsailing.com # Tunnel to other laptop P1 #2, tunneling to MongoDB running on 10202, mapping to local port 10202 -autossh -M 20502 -f -A -N \ +AUTOSSH_LOGLEVEL=7 AUTOSSH_POLL=5 autossh -M 20502 -f -A -N \ -L 10202:localhost:10202 \ -R 10201:localhost:10201 \ sailing@sap-p1-2 sleep 2 # An additional port forward from 443 to NGINX 9443 is required to reach a local backup copy of security-service.sapsailing.com -autossh -M 20504 -f -A -N \ +AUTOSSH_LOGLEVEL=7 AUTOSSH_POLL=5 autossh -M 20504 -f -A -N \ -L 443:localhost:9443 \ -L 5673:localhost:5672 \ -L 15673:localhost:15672 \ diff --git a/configuration/on-site-scripts/paris2024/sap-p1-1/tunnels-no-master b/configuration/on-site-scripts/paris2024/sap-p1-1/tunnels-no-master index 9bcdb70ba30..98574389d8e 100755 --- a/configuration/on-site-scripts/paris2024/sap-p1-1/tunnels-no-master +++ b/configuration/on-site-scripts/paris2024/sap-p1-1/tunnels-no-master @@ -2,7 +2,7 @@ killall autossh sleep 2 # Tunnel to cloud MongoDB on port 10203 with reverse tunnels for MongoDB on localhost:10201 and through tunnel on localhost:10202 to P1 #2 -autossh -M 20500 -f -A -N \ +AUTOSSH_LOGLEVEL=7 AUTOSSH_POLL=5 autossh -M 20500 -f -A -N \ -L 22443:sapsailing.com:443 \ -L 22222:sapsailing.com:22 \ -R 9443:localhost:9443 \ @@ -18,7 +18,7 @@ autossh -M 20500 -f -A -N \ ec2-user@paris-ssh.sapsailing.com sleep 2 # Tunnel to other laptop P1 #2, tunneling to MongoDB running on 10202, mapping to local port 10202 -autossh -M 20502 -f -A -N \ +AUTOSSH_LOGLEVEL=7 AUTOSSH_POLL=5 autossh -M 20502 -f -A -N \ -L 15674:localhost:15672 \ -L 10202:localhost:10202 \ -R 10201:localhost:10201 \ diff --git a/configuration/on-site-scripts/paris2024/sap-p1-2/tractrac-A b/configuration/on-site-scripts/paris2024/sap-p1-2/tractrac-A index ea39a3d7d05..6597f49719e 100755 --- a/configuration/on-site-scripts/paris2024/sap-p1-2/tractrac-A +++ b/configuration/on-site-scripts/paris2024/sap-p1-2/tractrac-A @@ -2,4 +2,4 @@ pkill -f .autossh.*stso sleep 2 # Tunnel to localhost for TracTrac port forwards -autossh -M 0 -f -A -N -L 4400:stso1:4400 -L 4401:stso1:4401 localhost +AUTOSSH_LOGLEVEL=7 AUTOSSH_POLL=5 autossh -M 0 -f -A -N -L 4400:stso1:4400 -L 4401:stso1:4401 localhost diff --git a/configuration/on-site-scripts/paris2024/sap-p1-2/tractrac-B b/configuration/on-site-scripts/paris2024/sap-p1-2/tractrac-B index 13f14f4ff19..8aaafaa978f 100755 --- a/configuration/on-site-scripts/paris2024/sap-p1-2/tractrac-B +++ b/configuration/on-site-scripts/paris2024/sap-p1-2/tractrac-B @@ -2,4 +2,4 @@ pkill -f .autossh.*stso sleep 2 # Tunnel to localhost for TracTrac port forwards -autossh -M 0 -f -A -N -L 4400:stso2:4400 -L 4401:stso2:4401 localhost +AUTOSSH_LOGLEVEL=7 AUTOSSH_POLL=5 autossh -M 0 -f -A -N -L 4400:stso2:4400 -L 4401:stso2:4401 localhost diff --git a/configuration/on-site-scripts/paris2024/sap-p1-2/tunnels b/configuration/on-site-scripts/paris2024/sap-p1-2/tunnels index 80ecea02e1e..517f7849bd0 100755 --- a/configuration/on-site-scripts/paris2024/sap-p1-2/tunnels +++ b/configuration/on-site-scripts/paris2024/sap-p1-2/tunnels @@ -7,7 +7,7 @@ # killall autossh sleep 2 -autossh -M 21504 -f -A -N \ +AUTOSSH_LOGLEVEL=7 AUTOSSH_POLL=5 autossh -M 21504 -f -A -N \ -L 22443:sapsailing.com:443 \ -L 22222:sapsailing.com:22 \ -R 9444:localhost:9443 \ @@ -20,7 +20,7 @@ autossh -M 21504 -f -A -N \ -R 18222:localhost:22 \ -L 588:email-smtp.eu-west-1.amazonaws.com:587 \ ec2-user@paris-ssh.sapsailing.com -autossh -M 0 -f -A -N \ +AUTOSSH_LOGLEVEL=7 AUTOSSH_POLL=5 autossh -M 0 -f -A -N \ -L 5673:localhost:5672 \ localhost # diff --git a/configuration/on-site-scripts/paris2024/sap-p1-2/tunnels-master b/configuration/on-site-scripts/paris2024/sap-p1-2/tunnels-master index a1b12452775..9957effb26e 100755 --- a/configuration/on-site-scripts/paris2024/sap-p1-2/tunnels-master +++ b/configuration/on-site-scripts/paris2024/sap-p1-2/tunnels-master @@ -7,7 +7,7 @@ # killall autossh # Tunnel to cloud MongoDB on port 10203; reverse tunnel to localhost:10202 for MongoDB -autossh -M 21504 -f -A -N \ +AUTOSSH_LOGLEVEL=7 AUTOSSH_POLL=5 autossh -M 21504 -f -A -N \ -L 22443:sapsailing.com:443 \ -L 22222:sapsailing.com:22 \ -R 9443:localhost:9443 \ diff --git a/configuration/on-site-scripts/paris2024/sap-p1-2/tunnels-master-no-internet b/configuration/on-site-scripts/paris2024/sap-p1-2/tunnels-master-no-internet index e8445bbb142..d762ea40c98 100755 --- a/configuration/on-site-scripts/paris2024/sap-p1-2/tunnels-master-no-internet +++ b/configuration/on-site-scripts/paris2024/sap-p1-2/tunnels-master-no-internet @@ -1,7 +1,7 @@ #!/bin/bash killall autossh # Tunnel to cloud MongoDB on port 10203; reverse tunnel to localhost:10202 for MongoDB -autossh -M 21504 -f -A -N \ +AUTOSSH_LOGLEVEL=7 AUTOSSH_POLL=5 autossh -M 21504 -f -A -N \ -L 443:localhost:9443 \ -L 5673:localhost:5672 \ -L 15673:localhost:15673 \ diff --git a/configuration/on-site-scripts/paris2024/sap-p1-2/tunnels-no-internet b/configuration/on-site-scripts/paris2024/sap-p1-2/tunnels-no-internet index c6cbce12061..6d0a3017763 100755 --- a/configuration/on-site-scripts/paris2024/sap-p1-2/tunnels-no-internet +++ b/configuration/on-site-scripts/paris2024/sap-p1-2/tunnels-no-internet @@ -13,7 +13,7 @@ sleep 2 # However, the RabbitMQ forwards to emulate rabbit.internal.sapsailing.com (eu-west-1) and rabbit-eu-west-3.sapsailing.com # with the local RabbitMQ need to go through a localhost port forward. Furthermore, the security-service.sapsailing.com tunnel # is re-directed to sap-p1-1's NGINX, assuming it has a replacement running if Internet fails. -autossh -M 21504 -f -A -N \ +AUTOSSH_LOGLEVEL=7 AUTOSSH_POLL=5 autossh -M 21504 -f -A -N \ -L 5673:localhost:5672 \ -L 15673:localhost:15672 \ -L 5675:sap-p1-1:5672 \ diff --git a/configuration/sailing b/configuration/sailing deleted file mode 120000 index 25a1c60a62f..00000000000 --- a/configuration/sailing +++ /dev/null @@ -1 +0,0 @@ -environments_scripts/sailing_server/files/usr/local/bin/sailing \ No newline at end of file diff --git a/configuration/sailing.sh b/configuration/sailing.sh deleted file mode 120000 index 648d1ba7071..00000000000 --- a/configuration/sailing.sh +++ /dev/null @@ -1 +0,0 @@ -environments_scripts/sailing_server/files/etc/profile.d/sailing.sh \ No newline at end of file diff --git a/configuration/sailing_server_setup/crontab-root b/configuration/sailing_server_setup/crontab-root deleted file mode 100644 index 68f39d7c136..00000000000 --- a/configuration/sailing_server_setup/crontab-root +++ /dev/null @@ -1,3 +0,0 @@ -* * * * * export PATH=/bin:/usr/bin:/usr/local/bin; sleep $(( $RANDOM * 60 / 32768 )); update_authorized_keys_for_landscape_managers_if_changed $( cat /root/ssh-key-reader.token ) https://security-service.sapsailing.com /root 2>&1 >>/var/log/sailing.err -# NOTICE: Please try to reference the customised crontabs at $GIT_HOME/configuration/crontabs or use -# the build_crontab_and_setup_files command in imageupgrade_functions.sh. This file has been maintained for continuity, but is deprecated. diff --git a/configuration/sailing_server_setup/mountnvmeswap b/configuration/sailing_server_setup/mountnvmeswap deleted file mode 120000 index cbfacbe095a..00000000000 --- a/configuration/sailing_server_setup/mountnvmeswap +++ /dev/null @@ -1 +0,0 @@ -../archive_instance_setup/mountnvmeswap \ No newline at end of file diff --git a/configuration/sailing_server_setup/mountnvmeswap.initd b/configuration/sailing_server_setup/mountnvmeswap.initd deleted file mode 120000 index 60a3925986e..00000000000 --- a/configuration/sailing_server_setup/mountnvmeswap.initd +++ /dev/null @@ -1 +0,0 @@ -../archive_instance_setup/mountnvmeswap.initd \ No newline at end of file diff --git a/configuration/sailing_server_setup/mountnvmeswap.service b/configuration/sailing_server_setup/mountnvmeswap.service deleted file mode 120000 index 69716204d64..00000000000 --- a/configuration/sailing_server_setup/mountnvmeswap.service +++ /dev/null @@ -1 +0,0 @@ -../archive_instance_setup/mountnvmeswap.service \ No newline at end of file diff --git a/configuration/sailing_server_setup/sailing.service b/configuration/sailing_server_setup/sailing.service deleted file mode 120000 index a551d92462c..00000000000 --- a/configuration/sailing_server_setup/sailing.service +++ /dev/null @@ -1 +0,0 @@ -../environments_scripts/sailing_server/files/etc/systemd/system/sailing.service \ No newline at end of file diff --git a/configuration/surefire-reports-resources.tar.gz b/configuration/surefire-reports-resources.tar.gz deleted file mode 100644 index 508760516f8..00000000000 Binary files a/configuration/surefire-reports-resources.tar.gz and /dev/null differ diff --git a/configuration/switchoverArchive.sh b/configuration/switchoverArchive.sh deleted file mode 120000 index a58dad37ae0..00000000000 --- a/configuration/switchoverArchive.sh +++ /dev/null @@ -1 +0,0 @@ -environments_scripts/repo/usr/local/bin/switchoverArchive.sh \ No newline at end of file diff --git a/configuration/syncgit b/configuration/syncgit deleted file mode 120000 index 8103b538c0d..00000000000 --- a/configuration/syncgit +++ /dev/null @@ -1 +0,0 @@ -environments_scripts/repo/usr/local/bin/syncgit \ No newline at end of file diff --git a/configuration/tractrac-json-urls b/configuration/tractrac-json-urls index 5e52076993a..86651f335a8 100644 --- a/configuration/tractrac-json-urls +++ b/configuration/tractrac-json-urls @@ -434,6 +434,7 @@ https://event3.tractrac.com/events/event_20231105_Europeaner/jsonservice.php https://event3.tractrac.com/events/event_20240129_HongKongRa/jsonservice.php https://event3.tractrac.com/events/event_20240223_WORLDCHAMP/jsonservice.php https://event3.tractrac.com/events/event_20240304_erWorldCha/jsonservice.php +https://event3.tractrac.com/events/event_20240606_HelgaCup/jsonservice.php https://event.tractrac.com/events/event_20170727_Travemnder2/jsonservice.php https://event.tractrac.com/events/event_20171013_DSLPokalGl/jsonservice.php https://event.tractrac.com/events/event_20181012_MstarnasMs/jsonservice.php @@ -512,6 +513,8 @@ https://event.tractrac.com/events/event_20230324_SAILINGCha/jsonservice.php https://event.tractrac.com/events/event_20230531_ERERFXNACR/jsonservice.php https://event.tractrac.com/events/event_20230611_KielerWoch/jsonservice.php https://event.tractrac.com/events/event_20230622_KielerWoch/jsonservice.php +https://event.tractrac.com/events/event_20240312_TrofeoPrin/jsonservice.php +https://event.tractrac.com/events/event_20240411_SemaineOly/jsonservice.php http://skitrac.traclive.dk/events/event_20140821_ESSCardiff/jsonservice.php http://skitrac.traclive.dk/events/event_20140909_ESSIstanbu/jsonservice.php http://skitrac.traclive.dk/events/event_20141001_ESSNice/jsonservice.php @@ -531,6 +534,8 @@ http://skitrac.traclive.dk/events/event_20160622_ESSCardiff/jsonservice.php http://skitrac.traclive.dk/events/event_20160727_ESSHamburg/jsonservice.php http://skitrac.traclive.dk/events/event_20160921_ESSInstanb/jsonservice.php http://skitrac.traclive.dk/events/event_20161005_ESSLisbon/jsonservice.php +https://paris1.tractrac.com/events/event_20240312_TrofeoPrin/jsonservice.php +https://paris1.tractrac.com/events/event_20240411_SemaineOly/jsonservice.php http://traclive.dk/events/event_20120905_erEuropean/jsonservice.php http://traclive.dk/events/event_20130302_ESSMuscat/jsonservice.php http://traclive.dk/events/event_20130401_STGTrainin/jsonservice.php diff --git a/configuration/update-tractrac-urls-to-archive.sh b/configuration/update-tractrac-urls-to-archive.sh deleted file mode 120000 index d3f7a97c311..00000000000 --- a/configuration/update-tractrac-urls-to-archive.sh +++ /dev/null @@ -1 +0,0 @@ -environments_scripts/central_reverse_proxy/files/usr/local/bin/update-tractrac-urls-to-archive.sh \ No newline at end of file diff --git a/configuration/update_authorized_keys_for_landscape_managers b/configuration/update_authorized_keys_for_landscape_managers deleted file mode 120000 index 21a778d16b9..00000000000 --- a/configuration/update_authorized_keys_for_landscape_managers +++ /dev/null @@ -1 +0,0 @@ -environments_scripts/repo/usr/local/bin/update_authorized_keys_for_landscape_managers \ No newline at end of file diff --git a/configuration/update_authorized_keys_for_landscape_managers_if_changed b/configuration/update_authorized_keys_for_landscape_managers_if_changed deleted file mode 120000 index 37944a8d022..00000000000 --- a/configuration/update_authorized_keys_for_landscape_managers_if_changed +++ /dev/null @@ -1 +0,0 @@ -environments_scripts/repo/usr/local/bin/update_authorized_keys_for_landscape_managers_if_changed \ No newline at end of file diff --git a/configuration/update_landscape_managers_mailing_list.sh b/configuration/update_landscape_managers_mailing_list.sh deleted file mode 120000 index d7306153c53..00000000000 --- a/configuration/update_landscape_managers_mailing_list.sh +++ /dev/null @@ -1 +0,0 @@ -environments_scripts/repo/usr/local/bin/update_landscape_managers_mailing_list.sh \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 796f0b19a12..9087d56d673 100755 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,16 +1,23 @@ -FROM docker.sapsailing.com/sapjvm8:8.1.097 +FROM eclipse-temurin:8-jdk +# This Dockerfile assumes that the release to use is provided as +# ${RELEASE}.tar.gz in the current Docker workspace. Use, e.g., +# the configuration/github-download-release-assets.sh script to +# obtain the tar.gz file for a specific or the latest "main" release. +# To build with an SAP JVM 8 base image built using Dockerfile_sapjvm, use something like: +#FROM sapjvm8:8.1.098 ARG RELEASE LABEL maintainer=axel.uhl@sap.com +LABEL org.opencontainers.image.description Sailing Analytics with Java 8 # Download and extract the release WORKDIR /home/sailing/servers/server RUN apt-get update \ && apt-get install -y vim telnet dnsutils net-tools jq COPY vimrc /root/.vimrc RUN wget -O /tmp/rds.pem https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem \ - && /opt/sapjvm_8/bin/keytool -importcert -alias AWSRDS -file /tmp/rds.pem -keystore /opt/sapjvm_8/jre/lib/security/cacerts -noprompt -storepass changeit \ + && ${JAVA_HOME}/bin/keytool -importcert -alias AWSRDS -file /tmp/rds.pem -keystore ${JAVA_HOME}/jre/lib/security/cacerts -noprompt -storepass changeit \ && rm /tmp/rds.pem -RUN wget -O /tmp/${RELEASE}.tar.gz http://releases.sapsailing.com/${RELEASE}/${RELEASE}.tar.gz \ - && tar xzvpf /tmp/${RELEASE}.tar.gz \ +COPY ${RELEASE}.tar.gz /tmp +RUN tar xzvpf /tmp/${RELEASE}.tar.gz \ && rm /tmp/${RELEASE}.tar.gz COPY env.sh . RUN cat env-default-rules.sh >>env.sh diff --git a/docker/Dockerfile-raspi.tpl b/docker/Dockerfile-raspi.tpl deleted file mode 100755 index 6cda42db1e4..00000000000 --- a/docker/Dockerfile-raspi.tpl +++ /dev/null @@ -1,18 +0,0 @@ -FROM donaldduck70/openjdk:1.8.0_242 -LABEL maintainer=axel.uhl@gmx.de -# Download and extract the release -WORKDIR /home/sailing/servers/server -RUN apt-get update \ - && apt-get install -y vim telnet dnsutils net-tools jq -COPY vimrc /root/.vimrc -RUN wget -O /tmp/rds.pem https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem \ - && keytool -importcert -alias AWSRDS -file /tmp/rds.pem -keystore /usr/lib/jvm/java-8-openjdk-arm64/jre/lib/security/cacerts -noprompt -storepass changeit \ - && rm /tmp/rds.pem -RUN wget -O /tmp/RELEASE.tar.gz http://releases.sapsailing.com/RELEASE/RELEASE.tar.gz \ - && tar xzvpf /tmp/RELEASE.tar.gz \ - && rm /tmp/RELEASE.tar.gz -COPY env.sh . -COPY start . -COPY JavaSE-11.profile . -EXPOSE 8888 14888 8000 7091 6666 -CMD [ "/home/sailing/servers/server/start", "fg" ] diff --git a/docker/Dockerfile_sapjvm b/docker/Dockerfile_sapjvm index ebdcddbd098..431ac6bef81 100644 --- a/docker/Dockerfile_sapjvm +++ b/docker/Dockerfile_sapjvm @@ -1,5 +1,5 @@ FROM buildpack-deps:bullseye -ARG SAPJVM_VERSION=8.1.097 +ARG SAPJVM_VERSION=8.1.098 LABEL maintainer=axel.uhl@sap.com # Download and extract the SAP JVM 8 ENV PATH=${PATH}:/opt/sapjvm_8/bin @@ -8,7 +8,7 @@ WORKDIR /opt RUN apt-get update \ && apt-get install -y wget apt-utils unzip \ && apt-get -y upgrade \ - && curl --cookie eula_3_1_agreed=tools.hana.ondemand.com/developer-license-3_1.txt "https://tools.hana.ondemand.com/additional/sapjvm-${SAPJVM_VERSION}-linux-x64.zip" --output sapjvm8-linux-x64.zip \ + && curl --cookie eula_3_2_agreed=tools.hana.ondemand.com/developer-license-3_2.txt "https://tools.hana.ondemand.com/additional/sapjvm-${SAPJVM_VERSION}-linux-x64.zip" --output sapjvm8-linux-x64.zip \ && unzip sapjvm8-linux-x64.zip \ && rm sapjvm8-linux-x64.zip \ && echo "export JAVA_HOME=/opt/sapjvm_8; export PATH=\${PATH}:/opt/sapjvm_8/bin" > /etc/profile.d/javahome.sh diff --git a/docker/Dockerfile_sapsailing_on_sapmachine11 b/docker/Dockerfile_sapsailing_on_sapmachine11 index 5fe836a9702..33e93a7756f 100644 --- a/docker/Dockerfile_sapsailing_on_sapmachine11 +++ b/docker/Dockerfile_sapsailing_on_sapmachine11 @@ -1,6 +1,11 @@ FROM sapmachine:11.0.17 +# This Dockerfile assumes that the release to use is provided as +# ${RELEASE}.tar.gz in the current Docker workspace. Use, e.g., +# the configuration/github-download-release-assets.sh script to +# obtain the tar.gz file for a specific or the latest "main" release. ARG RELEASE LABEL maintainer=axel.uhl@sap.com +LABEL org.opencontainers.image.description Sailing Analytics with Java 11 # Download and extract the release WORKDIR /home/sailing/servers/server RUN apt-get update \ @@ -9,8 +14,8 @@ COPY vimrc /root/.vimrc RUN wget -O /tmp/rds.pem https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem \ && ${JAVA_HOME}/bin/keytool -importcert -alias AWSRDS -file /tmp/rds.pem -keystore ${JAVA_HOME}/lib/security/cacerts -noprompt -storepass changeit \ && rm /tmp/rds.pem -RUN wget -O /tmp/${RELEASE}.tar.gz http://releases.sapsailing.com/${RELEASE}/${RELEASE}.tar.gz \ - && tar xzvpf /tmp/${RELEASE}.tar.gz \ +COPY ${RELEASE}.tar.gz /tmp +RUN tar xzvpf /tmp/${RELEASE}.tar.gz \ && rm /tmp/${RELEASE}.tar.gz COPY env.sh . RUN cat env-default-rules.sh >>env.sh diff --git a/docker/Dockerfile_sapsailing_on_sapmachine17 b/docker/Dockerfile_sapsailing_on_sapmachine17 index 7940296f8a5..bca49ad3fda 100644 --- a/docker/Dockerfile_sapsailing_on_sapmachine17 +++ b/docker/Dockerfile_sapsailing_on_sapmachine17 @@ -1,6 +1,11 @@ -FROM sapmachine:17.0.10 +FROM sapmachine:17.0.11 +# This Dockerfile assumes that the release to use is provided as +# ${RELEASE}.tar.gz in the current Docker workspace. Use, e.g., +# the configuration/github-download-release-assets.sh script to +# obtain the tar.gz file for a specific or the latest "main" release. ARG RELEASE LABEL maintainer=axel.uhl@sap.com +LABEL org.opencontainers.image.description Sailing Analytics with Java 17 # Download and extract the release WORKDIR /home/sailing/servers/server RUN apt-get update \ @@ -9,8 +14,8 @@ COPY vimrc /root/.vimrc RUN wget -O /tmp/rds.pem https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem \ && ${JAVA_HOME}/bin/keytool -importcert -alias AWSRDS -file /tmp/rds.pem -keystore ${JAVA_HOME}/lib/security/cacerts -noprompt -storepass changeit \ && rm /tmp/rds.pem -RUN wget -O /tmp/${RELEASE}.tar.gz http://releases.sapsailing.com/${RELEASE}/${RELEASE}.tar.gz \ - && tar xzvpf /tmp/${RELEASE}.tar.gz \ +COPY ${RELEASE}.tar.gz /tmp +RUN tar xzvpf /tmp/${RELEASE}.tar.gz \ && rm /tmp/${RELEASE}.tar.gz COPY env.sh . RUN cat env-default-rules.sh >>env.sh diff --git a/docker/Dockerfile_windestimation b/docker/Dockerfile_windestimation index fd1d577d4dc..3fd945bde5c 100755 --- a/docker/Dockerfile_windestimation +++ b/docker/Dockerfile_windestimation @@ -1,4 +1,4 @@ -ARG SAPJVM_VERSION=8.1.097 +ARG SAPJVM_VERSION=8.1.098 FROM docker.sapsailing.com/sapjvm8:${SAPJVM_VERSION} LABEL maintainer=axel.uhl@sap.com # Download and extract the release diff --git a/docker/Dockerfile_windestimation-on-sapmachine17 b/docker/Dockerfile_windestimation-on-sapmachine17 index 2ec6552f877..87d3c72747e 100755 --- a/docker/Dockerfile_windestimation-on-sapmachine17 +++ b/docker/Dockerfile_windestimation-on-sapmachine17 @@ -1,4 +1,4 @@ -FROM sapmachine:17.0.10 +FROM sapmachine:17.0.11 LABEL maintainer=axel.uhl@sap.com # Download and extract the release WORKDIR /home/sailing diff --git a/docker/docker-compose-17.yml b/docker/docker-compose-17.yml index 5b9d28c5d23..6de89609bf9 100644 --- a/docker/docker-compose-17.yml +++ b/docker/docker-compose-17.yml @@ -1,7 +1,7 @@ version: "3.10" services: sailing-analytics: - image: "docker.sapsailing.com/sapsailing_on_sapmachine17:latest" + image: "ghcr.io/sap/sailing-analytics:latest-17" ports: - "8888:8888" - "6666:6666" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index cb91494af67..d8d69cf1ae5 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.10" services: sailing-analytics: - image: "docker.sapsailing.com/sapsailing:latest" + image: "ghcr.io/sap/sailing-analytics:latest" ports: - "8888:8888" - "6666:6666" diff --git a/docker/makeImageForLatestRelease b/docker/makeImageForLatestRelease index 381858cacc3..daaccfeb9b4 100755 --- a/docker/makeImageForLatestRelease +++ b/docker/makeImageForLatestRelease @@ -1,24 +1,27 @@ #!/bin/bash -release=$1 -if [ "${release}" = "" ]; then - SET_LATEST=1 - release=$(wget -O - http://releases.sapsailing.com/ 2>/dev/null | grep 'build-[0-9]\{10\}' | tail -1 | sed -e 's/^.*\(build-[0-9]*\).*$/\1/') - echo Latest release is ${release} -fi +release_prefix=$1 GITROOT="`dirname $0`/.." DOCKERDIR="${GITROOT}/docker" DOCKERFILE="$DOCKERDIR/Dockerfile" -echo "Copying files from GITROOT $GITROOT into Docker workspace" -cp "$GITROOT/java/target/env.sh" "$DOCKERDIR" -cp "$GITROOT/java/target/start" "$DOCKERDIR" -cp "$GITROOT/java/target/configuration/JavaSE-11.profile" "$DOCKERDIR" -cd "$DOCKERDIR" -docker build --build-arg RELEASE=${release} -t sapsailing:${release} . -echo "Cleaning up..." -rm start env.sh JavaSE-11.profile Dockerfile -docker tag sapsailing:${release} docker.sapsailing.com/sapsailing:${release} -docker push docker.sapsailing.com/sapsailing:${release} -if [ "$SET_LATEST" = "1" ]; then - docker tag sapsailing:${release} docker.sapsailing.com/sapsailing:latest - docker push docker.sapsailing.com/sapsailing:latest +if [ "${release_prefix}" = "" ]; then + SET_LATEST=1 + release_prefix="main-" fi +pushd "${DOCKERDIR}" +RELEASE_TAR_GZ_FILENAME=$( ${GITROOT}/configuration/github-download-release-assets.sh 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 "${release_prefix}" ) +if [ "${RELEASE_TAR_GZ_FILENAME}" = "" ]; then + echo "No release with prefix ${release_prefix} found" >&2 +else + release=$( echo ${RELEASE_TAR_GZ_FILENAME} | sed -e 's/\.tar\.gz//' ) + echo Release is ${release} + echo "Copying files from GITROOT $GITROOT into Docker workspace" + cp "$GITROOT/java/target/env.sh" "$DOCKERDIR" + cp "$GITROOT/java/target/start" "$DOCKERDIR" + cp "$GITROOT/java/target/configuration/JavaSE-11.profile" "$DOCKERDIR" + cd "$DOCKERDIR" + docker buildx create --name=container --driver=docker-container --use --bootstrap + docker buildx build --builder=container --platform=linux/amd64,linux/arm64 --build-arg RELEASE=${release} -t ghcr.io/sap/sailing-analytics:${release} --push . + echo "Cleaning up..." + rm start env.sh JavaSE-11.profile ${RELEASE_TAR_GZ_FILENAME} release-notes.txt +fi +popd diff --git a/docker/makeImageForLatestRelease-arm b/docker/makeImageForLatestRelease-arm new file mode 100755 index 00000000000..45ee6485b36 --- /dev/null +++ b/docker/makeImageForLatestRelease-arm @@ -0,0 +1,31 @@ +#!/bin/bash +release_prefix=$1 +GITROOT="`dirname $0`/.." +DOCKERDIR="${GITROOT}/docker" +DOCKERFILE="$DOCKERDIR/Dockerfile" +if [ "${release_prefix}" = "" ]; then + SET_LATEST=1 + release_prefix="main-" +fi +pushd "${DOCKERDIR}" +RELEASE_TAR_GZ_FILENAME=$( ${GITROOT}/configuration/github-download-release-assets.sh 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 "${release_prefix}" ) +if [ "${RELEASE_TAR_GZ_FILENAME}" = "" ]; then + echo "No release with prefix ${release_prefix} found" >&2 +else + release=$( echo ${RELEASE_TAR_GZ_FILENAME} | sed -e 's/\.tar\.gz//' ) + echo Release is ${release} + echo "Copying files from GITROOT $GITROOT into Docker workspace" + cp "$GITROOT/java/target/env.sh" "$DOCKERDIR" + cp "$GITROOT/java/target/start" "$DOCKERDIR" + cp "$GITROOT/java/target/configuration/JavaSE-11.profile" "$DOCKERDIR" + cd "$DOCKERDIR" + docker build --build-arg RELEASE=${release} -t ghcr.io/sap/sailing-analytics:${release}-arm . + echo "Cleaning up..." + rm start env.sh JavaSE-11.profile ${RELEASE_TAR_GZ_FILENAME} release-notes.txt + docker push ghcr.io/sap/sailing-analytics:${release}-arm + if [ "$SET_LATEST" = "1" ]; then + docker tag ghcr.io/sap/sailing-analytics:${release}-arm ghcr.io/sap/sailing-analytics:latest-arm + docker push ghcr.io/sap/sailing-analytics:latest-arm + fi +fi +popd diff --git a/docker/makeImageForLatestRelease-on-sapmachine11 b/docker/makeImageForLatestRelease-on-sapmachine11 index 2c5ae81b834..7cf27720483 100755 --- a/docker/makeImageForLatestRelease-on-sapmachine11 +++ b/docker/makeImageForLatestRelease-on-sapmachine11 @@ -1,27 +1,31 @@ #!/bin/bash -release=$1 -if [ "${release}" = "" ]; then - SET_LATEST=1 - release=$(wget -O - http://releases.sapsailing.com/ 2>/dev/null | grep 'sapmachine11-[0-9]\{10\}' | tail -1 | sed -e 's/^.*\(sapmachine11-[0-9]*\).*$/\1/') - echo Latest release is ${release} -fi +release_prefix=$1 GITROOT="`dirname $0`/.." DOCKERDIR="${GITROOT}/docker" DOCKERFILE="$DOCKERDIR/Dockerfile" -FILES_TO_COPY_FROM_JAVA_TARGET="env.sh start configuration/JavaSE-11.profile" -echo "Copying files from GITROOT $GITROOT into Docker workspace" -for f in ${FILES_TO_COPY_FROM_JAVA_TARGET}; do - cp "${GITROOT}/java/target/${f}" "$DOCKERDIR" -done -cd "$DOCKERDIR" -docker build --build-arg RELEASE=${release} -t sapsailing_on_sapmachine11:${release} -f Dockerfile_sapsailing_on_sapmachine11 . -echo "Cleaning up..." -for f in ${FILES_TO_COPY_FROM_JAVA_TARGET}; do - rm "`basename ${f}`" -done -docker tag sapsailing_on_sapmachine11:${release} docker.sapsailing.com/sapsailing_on_sapmachine11:${release} -docker push docker.sapsailing.com/sapsailing_on_sapmachine11:${release} -if [ "$SET_LATEST" = "1" ]; then - docker tag sapsailing_on_sapmachine11:${release} docker.sapsailing.com/sapsailing_on_sapmachine11:latest - docker push docker.sapsailing.com/sapsailing_on_sapmachine11:latest +if [ "${release_prefix}" = "" ]; then + SET_LATEST=1 + release_prefix="docker-11-" fi +pushd "${DOCKERDIR}" +RELEASE_TAR_GZ_FILENAME=$( ${GITROOT}/configuration/github-download-release-assets.sh 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 "${release_prefix}" ) +if [ "${RELEASE_TAR_GZ_FILENAME}" = "" ]; then + echo "No release with prefix ${release_prefix} found" >&2 +else + release=$( echo ${RELEASE_TAR_GZ_FILENAME} | sed -e 's/\.tar\.gz//' ) + echo Release is ${release} + echo "Copying files from GITROOT $GITROOT into Docker workspace" + cp "$GITROOT/java/target/env.sh" "$DOCKERDIR" + cp "$GITROOT/java/target/start" "$DOCKERDIR" + cp "$GITROOT/java/target/configuration/JavaSE-11.profile" "$DOCKERDIR" + cd "$DOCKERDIR" + docker build --build-arg RELEASE=${release} -t ghcr.io/sap/sailing-analytics:${release} -f Dockerfile_sapsailing_on_sapmachine11 . + echo "Cleaning up..." + rm start env.sh JavaSE-11.profile ${RELEASE_TAR_GZ_FILENAME} release-notes.txt + docker push ghcr.io/sap/sailing-analytics:${release} + if [ "$SET_LATEST" = "1" ]; then + docker tag ghcr.io/sap/sailing-analytics:${release} ghcr.io/sap/sailing-analytics:latest-11 + docker push ghcr.io/sap/sailing-analytics:latest-11 + fi +fi +popd diff --git a/docker/makeImageForLatestRelease-on-sapmachine17 b/docker/makeImageForLatestRelease-on-sapmachine17 index 3513d69dbe3..0faf536d237 100755 --- a/docker/makeImageForLatestRelease-on-sapmachine17 +++ b/docker/makeImageForLatestRelease-on-sapmachine17 @@ -1,27 +1,31 @@ #!/bin/bash -release=$1 -if [ "${release}" = "" ]; then - SET_LATEST=1 - release=$(wget -O - http://releases.sapsailing.com/ 2>/dev/null | grep 'sapmachine17-[0-9]\{10\}' | tail -1 | sed -e 's/^.*\(sapmachine17-[0-9]*\).*$/\1/') - echo Latest release is ${release} -fi +release_prefix=$1 GITROOT="`dirname $0`/.." DOCKERDIR="${GITROOT}/docker" DOCKERFILE="$DOCKERDIR/Dockerfile" -FILES_TO_COPY_FROM_JAVA_TARGET="env.sh start configuration/JavaSE-11.profile" -echo "Copying files from GITROOT $GITROOT into Docker workspace" -for f in ${FILES_TO_COPY_FROM_JAVA_TARGET}; do - cp "${GITROOT}/java/target/${f}" "$DOCKERDIR" -done -cd "$DOCKERDIR" -docker build --build-arg RELEASE=${release} -t sapsailing_on_sapmachine17:${release} -f Dockerfile_sapsailing_on_sapmachine17 . -echo "Cleaning up..." -for f in ${FILES_TO_COPY_FROM_JAVA_TARGET}; do - rm "`basename ${f}`" -done -docker tag sapsailing_on_sapmachine17:${release} docker.sapsailing.com/sapsailing_on_sapmachine17:${release} -docker push docker.sapsailing.com/sapsailing_on_sapmachine17:${release} -if [ "$SET_LATEST" = "1" ]; then - docker tag sapsailing_on_sapmachine17:${release} docker.sapsailing.com/sapsailing_on_sapmachine17:latest - docker push docker.sapsailing.com/sapsailing_on_sapmachine17:latest +if [ "${release_prefix}" = "" ]; then + SET_LATEST=1 + release_prefix="docker-17-" fi +pushd "${DOCKERDIR}" +RELEASE_TAR_GZ_FILENAME=$( ${GITROOT}/configuration/github-download-release-assets.sh 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 "${release_prefix}" ) +if [ "${RELEASE_TAR_GZ_FILENAME}" = "" ]; then + echo "No release with prefix ${release_prefix} found" >&2 +else + release=$( echo ${RELEASE_TAR_GZ_FILENAME} | sed -e 's/\.tar\.gz//' ) + echo Release is ${release} + echo "Copying files from GITROOT $GITROOT into Docker workspace" + cp "$GITROOT/java/target/env.sh" "$DOCKERDIR" + cp "$GITROOT/java/target/start" "$DOCKERDIR" + cp "$GITROOT/java/target/configuration/JavaSE-11.profile" "$DOCKERDIR" + cd "$DOCKERDIR" + docker build --build-arg RELEASE=${release} -t ghcr.io/sap/sailing-analytics:${release} -f Dockerfile_sapsailing_on_sapmachine17 . + echo "Cleaning up..." + rm start env.sh JavaSE-11.profile ${RELEASE_TAR_GZ_FILENAME} release-notes.txt + docker push ghcr.io/sap/sailing-analytics:${release} + if [ "$SET_LATEST" = "1" ]; then + docker tag ghcr.io/sap/sailing-analytics:${release} ghcr.io/sap/sailing-analytics:latest-17 + docker push ghcr.io/sap/sailing-analytics:latest-17 + fi +fi +popd diff --git a/docker/makeImageForLatestRelease-raspi b/docker/makeImageForLatestRelease-raspi deleted file mode 100755 index a5b0477f3fb..00000000000 --- a/docker/makeImageForLatestRelease-raspi +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -release=$1 -if [ "${release}" = "" ]; then - SET_LATEST=1 - release=$(wget -O - http://releases.sapsailing.com/ 2>/dev/null | grep build- | tail -1 | sed -e 's/^.*\(build-[0-9]*\).*$/\1/') - echo Latest release is ${release} -fi -GITROOT="`dirname $0`/.." -DOCKERDIR="${GITROOT}/docker" -DOCKERFILE="$DOCKERDIR/Dockerfile-raspi" -echo "Copying files from GITROOT $GITROOT into Docker workspace" -cp "$GITROOT/java/target/env.sh" "$DOCKERDIR" -cp "$GITROOT/java/target/start" "$DOCKERDIR" -cp "$GITROOT/java/target/configuration/JavaSE-11.profile" "$DOCKERDIR" -cat ${DOCKERFILE}.tpl | sed -e "s/RELEASE/${release}/g" >${DOCKERFILE} -cd "$DOCKERDIR" -docker build -t sapsailing:${release}-raspi -f ${DOCKERFILE} . -echo "Cleaning up..." -rm start env.sh JavaSE-11.profile ${Dockerfile} -docker tag sapsailing:${release}-raspi donaldduck70/sapsailing:${release}-raspi -docker push donaldduck70/sapsailing:${release}-raspi -if [ "$SET_LATEST" = "1" ]; then - docker tag sapsailing:${release}-raspi donaldduck70/sapsailing:latest-raspi - docker push donaldduck70/sapsailing:latest-raspi -fi diff --git a/dummy/README b/dummy/README new file mode 100644 index 00000000000..de447822b75 --- /dev/null +++ b/dummy/README @@ -0,0 +1,5 @@ +This folder contains what is needed to satisfy our Azure Build Pipeline +for our Android apps, in particular a dummy coverage test result where +there are no real coverage tests run during the Android build, and a +dummy Surefire test report where no surefire/JUnit tests are run during +our Android builds. diff --git a/java/com.github.branflake2267.gwt-maps-api/.settings/com.gwtplugins.gwt.eclipse.core.prefs b/java/com.github.branflake2267.gwt-maps-api/.settings/com.gwtplugins.gwt.eclipse.core.prefs index 275d767bfd2..595ad7a3a24 100644 --- a/java/com.github.branflake2267.gwt-maps-api/.settings/com.gwtplugins.gwt.eclipse.core.prefs +++ b/java/com.github.branflake2267.gwt-maps-api/.settings/com.gwtplugins.gwt.eclipse.core.prefs @@ -1,2 +1,3 @@ //gwtVersion_/com.google.gwt.user/lib= eclipse.preferences.version=1 +gwtVersion_\\com.google.gwt.user\\lib= diff --git a/java/com.github.branflake2267.gwt-maps-api/LICENSE b/java/com.github.branflake2267.gwt-maps-api/LICENSE new file mode 100644 index 00000000000..ebc92a22ec6 --- /dev/null +++ b/java/com.github.branflake2267.gwt-maps-api/LICENSE @@ -0,0 +1,4 @@ +License +======= + +GWT-Maps-V3-Api is freely distributable under the [Apache 2.0 License](http://www.apache.org/licenses/LICENSE-2.0.html). \ No newline at end of file diff --git a/java/com.google.gwt.ajaxloader/LICENSE b/java/com.google.gwt.ajaxloader/LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/java/com.google.gwt.ajaxloader/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/java/com.google.gwt.dev/LICENSE b/java/com.google.gwt.dev/LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/java/com.google.gwt.dev/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/java/com.google.gwt.servlet/LICENSE b/java/com.google.gwt.servlet/LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/java/com.google.gwt.servlet/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/java/com.google.gwt.user/LICENSE b/java/com.google.gwt.user/LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/java/com.google.gwt.user/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/java/com.googlecode.java-diff-utils.test/LICENSE b/java/com.googlecode.java-diff-utils.test/LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/java/com.googlecode.java-diff-utils.test/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/java/com.googlecode.java-diff-utils/LICENSE b/java/com.googlecode.java-diff-utils/LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/java/com.googlecode.java-diff-utils/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/java/com.sap.sailing.dashboards.gwt/.settings/com.gwtplugins.gwt.eclipse.core.prefs b/java/com.sap.sailing.dashboards.gwt/.settings/com.gwtplugins.gwt.eclipse.core.prefs index 5066c713fac..0a7fed401d0 100644 --- a/java/com.sap.sailing.dashboards.gwt/.settings/com.gwtplugins.gwt.eclipse.core.prefs +++ b/java/com.sap.sailing.dashboards.gwt/.settings/com.gwtplugins.gwt.eclipse.core.prefs @@ -5,3 +5,4 @@ eclipse.preferences.version=1 entryPointModules= filesCopiedToWebInfLib= +gwtVersion_C\:\\Program\ Files\\GWT\\gwt-2.11.0=2.11.0 diff --git a/java/com.sap.sailing.dashboards.gwt/WEB-INF/web.xml b/java/com.sap.sailing.dashboards.gwt/WEB-INF/web.xml index c3fc8ac7d1d..f6635893f4e 100644 --- a/java/com.sap.sailing.dashboards.gwt/WEB-INF/web.xml +++ b/java/com.sap.sailing.dashboards.gwt/WEB-INF/web.xml @@ -44,15 +44,6 @@ *.html - - LocaleInjectionFilter - com.sap.sse.security.ui.server.LocaleInjectionFilter - - - LocaleInjectionFilter - *.html - - GWTCacheControlServletFilter com.sap.sse.gwt.server.GWTCacheControlServletFilter diff --git a/java/com.sap.sailing.datamining.shared/.settings/com.gwtplugins.gwt.eclipse.core.prefs b/java/com.sap.sailing.datamining.shared/.settings/com.gwtplugins.gwt.eclipse.core.prefs index 7515a4a21eb..5aaa6c9593e 100644 --- a/java/com.sap.sailing.datamining.shared/.settings/com.gwtplugins.gwt.eclipse.core.prefs +++ b/java/com.sap.sailing.datamining.shared/.settings/com.gwtplugins.gwt.eclipse.core.prefs @@ -3,3 +3,4 @@ eclipse.preferences.version=1 entryPointModules= filesCopiedToWebInfLib= +gwtVersion_C\:\\Program\ Files\\GWT\\gwt-2.11.0=2.11.0 diff --git a/java/com.sap.sailing.datamining/.settings/com.gwtplugins.gwt.eclipse.core.prefs b/java/com.sap.sailing.datamining/.settings/com.gwtplugins.gwt.eclipse.core.prefs index 7515a4a21eb..5aaa6c9593e 100644 --- a/java/com.sap.sailing.datamining/.settings/com.gwtplugins.gwt.eclipse.core.prefs +++ b/java/com.sap.sailing.datamining/.settings/com.gwtplugins.gwt.eclipse.core.prefs @@ -3,3 +3,4 @@ eclipse.preferences.version=1 entryPointModules= filesCopiedToWebInfLib= +gwtVersion_C\:\\Program\ Files\\GWT\\gwt-2.11.0=2.11.0 diff --git a/java/com.sap.sailing.domain.common/.settings/com.gwtplugins.gwt.eclipse.core.prefs b/java/com.sap.sailing.domain.common/.settings/com.gwtplugins.gwt.eclipse.core.prefs index 7515a4a21eb..5aaa6c9593e 100644 --- a/java/com.sap.sailing.domain.common/.settings/com.gwtplugins.gwt.eclipse.core.prefs +++ b/java/com.sap.sailing.domain.common/.settings/com.gwtplugins.gwt.eclipse.core.prefs @@ -3,3 +3,4 @@ eclipse.preferences.version=1 entryPointModules= filesCopiedToWebInfLib= +gwtVersion_C\:\\Program\ Files\\GWT\\gwt-2.11.0=2.11.0 diff --git a/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/BoatClassMasterdata.java b/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/BoatClassMasterdata.java index 25767ae0304..a53c2001028 100755 --- a/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/BoatClassMasterdata.java +++ b/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/BoatClassMasterdata.java @@ -123,7 +123,8 @@ public enum BoatClassMasterdata { RS_AERO ("RS Aero", true, 4.00, 1.40, BoatHullType.MONOHULL, false, "RSAERO", "RS_Aero"), RS_X ("RS:X", true, 2.86, 0.93, BoatHullType.SURFERBOARD, false, "RS-X", "RSX", "RS:X", "RS:X Men", "RS:X Woman", "RS:X Women"), RS_FEVA ("RS Feva", true, 3.64, 1.42, BoatHullType.MONOHULL, true, "RSFeva"), - RS_TERA ("RS Tera", true, 2.87, 1.23, BoatHullType.MONOHULL, true, "RSTera"), + RS_TERA ("RS Tera", true, 2.87, 1.23, BoatHullType.MONOHULL, false, "RSTera"), + RS_VAREO ("RS Vareo", true, 4.25, 1.57, BoatHullType.MONOHULL, true, "RS_VAREO", "RSVareo", "RS Vareo"), RS_VENTURE ("RS Venture", true, 4.9, 2.0, BoatHullType.MONOHULL, true, "RSVenture", "RS Venture Connect"), SALONA_46 ("Salona 46", true, 14.14, 4.2, BoatHullType.MONOHULL, true, "Salona-46"), SCAN_KAP_99 ("Scan-kap 99", true, 9.86, 2.61, BoatHullType.MONOHULL, true, "Scan Kap 99"), @@ -158,6 +159,7 @@ public enum BoatClassMasterdata { WASZP ("WASZP", true, 3.35, 2.25, BoatHullType.MONOHULL, false, "WASZPs"), WAYFARER ("Wayfarer", true, 4.82, 1.85, BoatHullType.MONOHULL, false), WETA ("Weta", true, 4.4, 3.5, BoatHullType.TRIMARAN, true, "Weta Trimaran"), + WINGFOIL ("Wingfoil", true, 1.95, 0.8, BoatHullType.SURFERBOARD, false, "Wing Foil", "X-15"), X_332 ("X-332", true, 10.06, 3.30, BoatHullType.MONOHULL, true, "X332"), X_99 ("X-99", true, 9.96, 2.95, BoatHullType.MONOHULL, true, "X99"), diff --git a/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/LeaderboardNameConstants.java b/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/LeaderboardNameConstants.java index 259243684c7..85400964a56 100755 --- a/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/LeaderboardNameConstants.java +++ b/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/LeaderboardNameConstants.java @@ -14,6 +14,7 @@ public interface LeaderboardNameConstants { static final String SCORINGSCHEME = "scoringScheme"; static final String SCORINGCOMMENT = "scoringComment"; static final String LASTSCORINGUPDATE = "lastScoringUpdate"; + static final String LASTSCORINGUPDATE_MILLIS = "lastScoringUpdate-ms"; static final String REGATTANAME = "regattaName"; static final String SERIES = "series"; static final String ISMEDALSERIES = "isMedalSeries"; @@ -29,5 +30,5 @@ public interface LeaderboardNameConstants { static final String RACEVIEWERURLS = "raceViewerUrls"; static final String HASGPSDATA = "hasGpsData"; static final String HASWINDDATA = "hasWindData"; - + static final String DISCARDS = "discardIndexResultsStartingWithHowManyRaces"; } \ No newline at end of file diff --git a/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/security/SecuredDomainType.java b/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/security/SecuredDomainType.java index f812bbdb880..920f7ce8d28 100644 --- a/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/security/SecuredDomainType.java +++ b/java/com.sap.sailing.domain.common/src/com/sap/sailing/domain/common/security/SecuredDomainType.java @@ -101,7 +101,19 @@ public class SecuredDomainType extends HasPermissionsImpl { "SWISS_TIMING_ARCHIVE_ACCOUNT"); public static final HasPermissions TRACTRAC_ACCOUNT = new SecuredDomainType("TRACTRAC_ACCOUNT"); public static final HasPermissions YELLOWBRICK_ACCOUNT = new SecuredDomainType("YELLOWBRICK_ACCOUNT"); + + /** + * The type-relative object identifier is expected to name the "SERVER" object to which the request to read + * or update the wind estimation models is sent. + */ public static final HasPermissions WIND_ESTIMATION_MODELS = new SecuredDomainType("WIND_ESTIMATION_MODELS"); + + /** + * The type-relative object identifier is expected to name the "SERVER" object to which the request to read + * the polar data is sent. + */ + public static final HasPermissions POLAR_DATA = new SecuredDomainType("POLAR_DATA"); + public static final HasPermissions MARK_PROPERTIES = new SecuredDomainType("MARK_PROPERTIES"); public static final HasPermissions MARK_TEMPLATE = new SecuredDomainType("MARK_TEMPLATE"); public static final HasPermissions COURSE_TEMPLATE = new SecuredDomainType("COURSE_TEMPLATE"); diff --git a/java/com.sap.sailing.domain.igtimiadapter.gateway/WEB-INF/web.xml b/java/com.sap.sailing.domain.igtimiadapter.gateway/WEB-INF/web.xml index c69e58f5cf4..e8836000d22 100644 --- a/java/com.sap.sailing.domain.igtimiadapter.gateway/WEB-INF/web.xml +++ b/java/com.sap.sailing.domain.igtimiadapter.gateway/WEB-INF/web.xml @@ -27,7 +27,7 @@ ShiroFilter - org.apache.shiro.web.servlet.ShiroFilter + com.sap.sse.security.shiro.ShiroFilterForAllButOptionsRequests CORSFilter - com.sap.sailing.www.CORSFilter + com.sap.sse.rest.CORSFilter CORSFilter @@ -36,7 +36,7 @@ ShiroFilter - org.apache.shiro.web.servlet.ShiroFilter + com.sap.sse.security.shiro.ShiroFilterForAllButOptionsRequests diff --git a/java/com.sap.sailing.server.gateway.test/src/com/sap/sailing/server/gateway/jaxrs/api/CompareServersTest.java b/java/com.sap.sailing.server.gateway.test/src/com/sap/sailing/server/gateway/jaxrs/api/CompareServersTest.java index 468b1eb28c0..ad30421a7fa 100755 --- a/java/com.sap.sailing.server.gateway.test/src/com/sap/sailing/server/gateway/jaxrs/api/CompareServersTest.java +++ b/java/com.sap.sailing.server.gateway.test/src/com/sap/sailing/server/gateway/jaxrs/api/CompareServersTest.java @@ -247,8 +247,8 @@ public class CompareServersTest { @Test public void testComparingLeaderboardGroupOutputWithEventOrderChangeOnly() throws IOException, ParseException { - final Object oldLg = new JSONParser().parse(new InputStreamReader(getClass().getResourceAsStream("/LeaderboardGroup1_old.json"))); - final Object newLg = new JSONParser().parse(new InputStreamReader(getClass().getResourceAsStream("/LeaderboardGroup1_new.json"))); + final JSONObject oldLg = (JSONObject) new JSONParser().parse(new InputStreamReader(getClass().getResourceAsStream("/LeaderboardGroup1_old.json"))); + final JSONObject newLg = (JSONObject) new JSONParser().parse(new InputStreamReader(getClass().getResourceAsStream("/LeaderboardGroup1_new.json"))); final Pair result = resource.removeUnnecessaryAndDuplicateFields(oldLg, newLg); assertNull(result.getA()); assertNull(result.getB()); @@ -256,8 +256,8 @@ public class CompareServersTest { @Test public void testComparingLeaderboardGroupOutputWithRaceHavingLostWindAndGPS() throws IOException, ParseException { - final Object oldLg = new JSONParser().parse(new InputStreamReader(getClass().getResourceAsStream("/LeaderboardGroup2_old.json"))); - final Object newLg = new JSONParser().parse(new InputStreamReader(getClass().getResourceAsStream("/LeaderboardGroup2_new.json"))); + final JSONObject oldLg = (JSONObject) new JSONParser().parse(new InputStreamReader(getClass().getResourceAsStream("/LeaderboardGroup2_old.json"))); + final JSONObject newLg = (JSONObject) new JSONParser().parse(new InputStreamReader(getClass().getResourceAsStream("/LeaderboardGroup2_new.json"))); final Pair result = resource.removeUnnecessaryAndDuplicateFields(oldLg, newLg); assertNotNull(result.getA()); assertNotNull(result.getB()); diff --git a/java/com.sap.sailing.server.gateway/.settings/org.eclipse.core.resources.prefs b/java/com.sap.sailing.server.gateway/.settings/org.eclipse.core.resources.prefs index 99f26c0203a..d8fd4cd2088 100644 --- a/java/com.sap.sailing.server.gateway/.settings/org.eclipse.core.resources.prefs +++ b/java/com.sap.sailing.server.gateway/.settings/org.eclipse.core.resources.prefs @@ -1,2 +1,3 @@ eclipse.preferences.version=1 +encoding//resources/stringmessages/StringMessages_cs.properties=UTF-8 encoding/=UTF-8 diff --git a/java/com.sap.sailing.server.gateway/WEB-INF/web.xml b/java/com.sap.sailing.server.gateway/WEB-INF/web.xml index c26b3c3c18c..41d75509c56 100644 --- a/java/com.sap.sailing.server.gateway/WEB-INF/web.xml +++ b/java/com.sap.sailing.server.gateway/WEB-INF/web.xml @@ -25,7 +25,7 @@ ShiroFilter - org.apache.shiro.web.servlet.ShiroFilter + com.sap.sse.security.shiro.ShiroFilterForAllButOptionsRequests diff --git a/java/com.sap.sailing.www/release_notes_admin.html b/java/com.sap.sailing.www/release_notes_admin.html index d53628ae7b8..73c7d6502d4 100755 --- a/java/com.sap.sailing.www/release_notes_admin.html +++ b/java/com.sap.sailing.www/release_notes_admin.html @@ -23,6 +23,80 @@

Release Notes - Administration Console

+

July 2024

+
    +
  • A regatta's end date is now considered for deciding whether to display the regatta as still + "in progress." A regatta will be shown as "finished" now even if not all its races (as may occur + particlarly in split-fleet scenarios) have been completed, as soon as the regatta's end date + has passed.
    • +
  • Upgraded TracAPI to version 3.15.13, resolving issues with deadlocks based on HTTP ports + being blocked.
    • +
+

June 2024

+
    +
  • Upgraded to TracAPI 3.15.8, supporting more TracTrac invalid result markers (IRMs), + in particular DCT, DNE, and RCT.
    • +
  • For raw data output of wind sensor and object positions, where permitted, use seven instead of + six decimal places in lat/lng output to increase resolution to approximately 1cm.
    • +
  • Fixed concurrency issue with subscription webhook callbacks which in rare cases led + expiring subscriptions or failing payments to not properly revoke user permissions.
    • +
  • When archiving a replica set, any remote reference pointing to it from the archive running + at www.sapsailing.com is being removed. Now, the same reference will also be removed + from the failover archive at archive-failover.sapsailing.com to reduce inconsistency + and a circular reference structure in case we need to switch to the failover.
    • +
  • Upgraded to TracAPI 3.15.9, filtering mark positions by course area on the server side now and + offering the client archived MTB files instead of live DB access where available.
    • +
  • Upgraded to TracAPI 3.15.10, fixing a bug introduced by TracAPI 3.15.9 regarding locally + overriding the so-called "stored data URI"
    • +
  • Added class "Wingfoil"
    • +
  • TracAPI 3.15.12 fixes the MTB file reader that was not releasing all the resources once + the reading process was over.
    • +
  • Added class "RS Vareo"
    • +
  • Fixed Czech translation for the headerLogo of the web interface: 'Sailing Analytics' is now + used instead of czech phrase.
    • +
+

May 2024

+
    +
  • Reading the wind estimation data from a server during start-up, using the + windestimation.source.url system property, now requires authentication. + The java/target/start script now reads the WINDESTIMATION_BEARER_TOKEN + variable and passes it as the windestimation.source.bearertoken system property + to the application. The user authenticated by that token must have the + WIND_ESTIMATION_MODELS:READ permission for the server identified by the + URL provided in the windestimation.source.url property, for example + WIND_ESTIMATION_MODELS:READ:ARCHIVE if the URL is https://sapsailing.com
    • +
  • Reading the polar data from a server during start-up, using the + polardata.source.url system property, now requires authentication. + The java/target/start script now reads the POLAR_DATA_BEARER_TOKEN + variable and passes it as the polardata.source.bearertoken system property + to the application. The user authenticated by that token must have the + POLAR_DATA:READ permission for the server identified by the + URL provided in the polardata.source.url property, for example + POLAR_DATA:READ:ARCHIVE if the URL is https://sapsailing.com
    • +
+

April 2024

+
    +
  • The TracTrac connector now pools and re-uses event subscriber threads for live connections + for the same event with equal stored and live URIs. This helps reduce the number of threads + running in the system and is "friendlier" on the TracTrac infrastructure regarding the + number of concurrent connections the system needs to manage.
    • +
  • During live TracTrac connections, mark positions are now filtered by the race's course area if + both, mark and race, specify one. This way, redundant and excessive mark fixes can be avoided, + don't consume extra memory and don't waste replication bandwidth. See also + bug 5919.
    • +
  • Fixed file upload for Chrome browsers after Chrome made an incompatible change, appending + a combination of <div> tags after the already <pre>-enclosed + form submission completion payload.
    • +
  • The TracTrac connector will no longer disconnect from a race when it is abandoned. It is now + assumed the race will be re-sailed, and so the connector will keep tracking the race.
    • +
  • Fixed handling of null values in "compareservers" end point.
    • +
  • Upgraded to TracAPI 3.15.7, fixing an issue with static mark positions not transmitted during live races.
    • +
  • The LOW_POINT_WITH_ELIMINATING_MEDAL_SERIES_PROMOTING_ONE_TO_FINAL_AND_TWO_TO_SEMIFINAL scoring scheme, + as used typically for iQFOil formats with quarter/semi/grand finals, now does no longer require explicit + column factors of 1.0 for those medal-series race columns. The medal series-based doubling is deactivated + now for this scoring scheme. Should you have relied on this scheme doubling your medal series scores, + you now have to adjust the column factors for those race columns to 2.0 explicitly.
    • +

March 2024