mirror of
https://github.com/eclipse-sailing-analytics/sailing-analytics.git
synced 2026-09-23 14:08:40 +00:00
56 lines
2.3 KiB
Bash
Executable File
56 lines
2.3 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
CACHE=/var/log/old/cache/unique-ips-per-referrer
|
|
VISITED_FILES=$CACHE/visited
|
|
STATS=$CACHE/stats
|
|
MONTHS=$STATS/months
|
|
|
|
mkdir -p "$STATS"
|
|
|
|
# groups Apache / httpd log entries into "${referrer}.ips" files, where ${referrer} is the
|
|
# referrer URL identifying the "event"; the lines written to the .ips files hold the IP
|
|
# address of the requestor, the date (not the time) and the user agent string.
|
|
# Sorting for unique entries should give a count similar to what goaccess is using to
|
|
# determine "unique visitors."
|
|
|
|
# A sample line:
|
|
# 505Worlds2012.sapsailing.com 66.249.73.53 - - [12/Jan/2014:03:33:11 +0000] "GET /robots.txt HTTP/1.1" 404 238 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
|
|
|
echo Starting "$0" at `date` on file set "$*"
|
|
|
|
for i in $*; do
|
|
grep -q "^$i\$" $VISITED_FILES
|
|
if [ "$?" = "0" ]; then
|
|
echo "Already visited $i; ignoring (edit $VISITED_FILES to change this)."
|
|
else
|
|
echo "Analyzing log file $i"
|
|
if [ ${i: -3} == ".gz" ]; then
|
|
gzip -cd $i
|
|
else
|
|
cat $i
|
|
fi | recode ISO-8859-1..UTF-8 | sed -e 's/^\([^ ]*\) \([^ ]*\) \([^ ]*\) \([^ ]*\) \[\([^:]*\):\([^]]*\)\] \"[^"]*\" [^ ]* [^ ]* \"[^"]*\" \"\([^"]*\)\"/\1 \2 \5 \7/' | while read referrer hit; do
|
|
echo "$hit" >>${STATS}/${referrer}.ips
|
|
done
|
|
echo "$i" >>$VISITED_FILES
|
|
fi
|
|
done
|
|
for i in ${STATS}/*.ips; do
|
|
echo -n "`basename $i .ips` "
|
|
cat $i | sort -u | wc | awk '{ print $1; }'
|
|
# TODO consider producing an AWStats configuration file for each virtual host name and splitting all new logs into virtual host name-specific log files
|
|
done | tee $STATS/unique-ips-days-useragents-per-event
|
|
echo "Wrote total results to $STATS/unique-ips-days-useragents-per-event"
|
|
|
|
|
|
# Now filter and group by month
|
|
cat ${STATS}/*.ips | awk '{ print $2; }' | sed -e 's/^[0-9]*\///' | sort -u >$MONTHS
|
|
for month in `cat $MONTHS`; do
|
|
for i in ${STATS}/*.ips; do
|
|
echo -n "`basename $i .ips` "
|
|
cat $i | grep "^[^ ]* [0-9]*/$month .*" | sort -u | wc | awk '{ print $1; }'
|
|
# TODO consider producing an AWStats configuration file for each virtual host name and splitting all new logs into virtual host name-specific log files
|
|
done | tee $STATS/unique-ips-days-useragents-per-event-`echo $month | tr / -`
|
|
echo "Wrote per-month results to $STATS/unique-ips-days-useragents-per-event-`echo $month | tr / -`"
|
|
done
|
|
echo "Done at `date`."
|