eclipse-sailing-analytics/sailing-analytics
1
0 Fork 0
mirror of https://github.com/eclipse-sailing-analytics/sailing-analytics.git synced 2026-10-02 10:23:51 +00:00
Code Activity

filter and group unique referrers in logs by months

Browse Source
This commit is contained in:
Wiki User committed 2015-11-13 09:19:05 +00:00
1 parent 3cd2058cd8
commit 2d38da1fb1
1 file changed
+39 -6
Download Patch File Download Diff File Expand all files Collapse all files
configuration/unique_ips_per_referrer
+39 -6
View File
@@ -1,5 +1,12 @@
#!/bin/bash
CACHE=/var/log/old/cache/unique-ips-per-referrer
VISITED_FILES=$CACHE/visited
STATS=$CACHE/stats
MONTHS=$STATS/months
mkdir -p "$STATS"
# groups Apache / httpd log entries into "${referrer}.ips" files, where ${referrer} is the
# referrer URL identifying the "event"; the lines written to the .ips files hold the IP
# address of the requestor, the date (not the time) and the user agent string.
@@ -9,11 +16,37 @@
# A sample line:
# 505Worlds2012.sapsailing.com 66.249.73.53 - - [12/Jan/2014:03:33:11 +0000] "GET /robots.txt HTTP/1.1" 404 238 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
cat $* | sed -e 's/^\([^ ]*\) \([^ ]*\) \([^ ]*\) \([^ ]*\) \[\([^:]*\):\([^]]*\)\] \"[^"]*\" [^ ]* [^ ]* \"[^"]*\" \"\([^"]*\)\"/\1 \2 \5 \7/' | while read referrer hit; do
echo "$hit" >>${referrer}.ips
for i in $*; do
grep -q "^$i\$" $VISITED_FILES
if [ "$?" = "0" ]; then
echo "Already visited $i; ignoring (edit $VISITED_FILES to change this)." >&2
else
echo "Analyzing log file $i" >&2
if [ ${i: -3} == ".gz" ]; then
gzip -cd $i
else
cat $i
fi | sed -e 's/^\([^ ]*\) \([^ ]*\) \([^ ]*\) \([^ ]*\) \[\([^:]*\):\([^]]*\)\] \"[^"]*\" [^ ]* [^ ]* \"[^"]*\" \"\([^"]*\)\"/\1 \2 \5 \7/' | while read referrer hit; do
echo "$hit" >>${STATS}/${referrer}.ips
done
echo "$i" >>$VISITED_FILES
fi
done
for i in *.ips; do
echo -n "$i "
for i in ${STATS}/*.ips; do
echo -n "`basename $i .ips` "
cat $i | sort -u | wc | awk '{ print $1; }'
done | tee unique-ips-days-useragents-per-event
echo "Done. Wrote results to `pwd`/unique-ips-days-useragents-per-event"
# TODO consider producing an AWStats configuration file for each virtual host name and splitting all new logs into virtual host name-specific log files
done | tee $STATS/unique-ips-days-useragents-per-event
echo "Done. Wrote results to $STATS/unique-ips-days-useragents-per-event"
# Now filter and group by month
cat ${STATS}/*.ips | awk '{ print $2; }' | sed -e 's/^[0-9]*\///' | sort -u >$MONTHS
for month in `cat $MONTHS`; do
for i in ${STATS}/*.ips; do
echo -n "`basename $i .ips` "
cat $i | grep "^[^ ]* [0-9]*/$month .*" | sort -u | wc | awk '{ print $1; }'
# TODO consider producing an AWStats configuration file for each virtual host name and splitting all new logs into virtual host name-specific log files
done | tee $STATS/unique-ips-days-useragents-per-event-$month
echo "Done. Wrote results to $STATS/unique-ips-days-useragents-per-event-$month"
done