script that tries to avoid external commands for log file mass analysis

configuration/unique_ips_per_referrer_fast

@@ -8,6 +8,7 @@ VISITED_FILES=$CACHE/visited
 STATS=$CACHE/stats
 OUTPUT=$STATS/results
 MONTHS=$OUTPUT/permonth
+YEARS=$OUTPUT/peryear
 TOTALS=$OUTPUT/totals
 EVENTTOTALS=$OUTPUT/eventtotals
 
@@ -53,32 +54,120 @@ function monthMapper() {
   esac
 }
 
+mkdir -p $STATS
 mkdir -p $OUTPUT
-rm $OUTPUT/*
 mkdir -p $MONTHS
-rm $MONTHS/*
+mkdir -p $YEARS
+
+# groups Apache / httpd log entries into "${referrer}.ips" files, where ${referrer} is the
+# referrer URL identifying the "event"; the lines written to the .ips files hold the IP
+# address of the requestor, the date (not the time) and the user agent string.
+# Sorting for unique entries should give a count similar to what goaccess is using to
+# determine "unique visitors."
+
+# A sample line:
+# 505Worlds2012.sapsailing.com 66.249.73.53 - - [12/Jan/2014:03:33:11 +0000] "GET /robots.txt HTTP/1.1" 404 238 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+
+echo Starting "$0" at `date` on file set "$*"
+
+for i in $*; do
+  grep -q "^$i\$" $VISITED_FILES
+  if [ "$?" = "0" ]; then
+    echo "Already visited $i; ignoring (edit $VISITED_FILES to change this)."
+  else
+    echo "Analyzing log file $i"
+    if [ ${i: -3} == ".gz" ]; then
+      gzip -cd $i
+    else
+      cat $i
+    fi | recode ISO-8859-1..UTF-8 | sed -e 's/^\([^ ]*\) \([^ ]*\) \([^ ]*\) \([^ ]*\) \[\([^:]*\):\([^]]*\)\] \"[^"]*\" [^ ]* [^ ]* \"[^"]*\" \"\([^"]*\)\"/\1 \2 \5 \7/' | while read referrer hit; do
+      echo "$hit" >>${STATS}/${referrer}.ips
+    done
+    echo "$i" >>$VISITED_FILES
+  fi
+done
+
+# Clean up old, orphaned temporary files in $OUTPUT because we'd like to append only,
+# but leave the ".unique" results files there for now; they will be overwritten at the end
+echo Cleaning up output from previous runs
+rm -v $MONTHS/????-?? $MONTHS/-
+rm -v $YEARS/????
+rm -v $EVENTTOTALS
+rm -v $TOTALS
 
 # Now filter and group by month
+# Use file handles 3/4 for $EVENTTOTALS/$TOTALS to avoid having to open the files over and over
+exec 3>$EVENTTOTALS
+exec 4>$TOTALS
+# Furthermore, we'll have a file descriptor for each monthfile and each yearfile.
+# These file descriptors will be held in variables named like the basename of
+# the respective file, prefixed with "month" and "year" respectively. For example,
+# ${month2017_09} and ${year2018}
 for i in ${STATS}/*.ips; do
+  echo Handling file $i
   hostname=`basename $i .ips`
-  cat $i | while read line; do
+  while read line; do
     #echo line is $line
     read ipaddress dayandmonth useragent <<< "$line"
     read day month <<< "${dayandmonth/\// }"
     read monthname year <<< "${month//\// }"
-    monthfile=$MONTHS/${year}-$(monthMapper $monthname)
+    if [ "$year" != "" ]; then
-    #echo ipaddress is $ipaddress and dayandmonth is $dayandmonth and month is $month and monthname is $monthname and year is $year and monthfile is $monthfile
+      monthfilebasename=${year}_$(monthMapper $monthname)
-    echo "$hostname $line" >>$monthfile
+      monthfile=$MONTHS/$monthfilebasename
-    echo "$hostname $line" >>$EVENTTOTALS
+      month_fd_varname=month$monthfilebasename
-    echo "$line" >>$TOTALS
+      #echo month_fd_varname=$month_fd_varname, variable value: ${!month_fd_varname}
-  done
+      if [ "${!month_fd_varname}" = "" ]; then
+	exec {m}>$monthfile
+	declare $month_fd_varname=$m
+	echo Creating per-month file $monthfile with file descriptor ${month_fd_varname}=${!month_fd_varname}
+      fi
+      yearfile=$YEARS/${year}
+      year_fd_varname=year$year
+      #echo year_fd_varname=$year_fd_varname, variable value: ${!year_fd_varname}
+      if [ "${!year_fd_varname}" = "" ]; then
+	exec {m}>$yearfile
+	declare $year_fd_varname=$m
+	echo Creating per-year file $yearfile with file descriptor ${year_fd_varname}=${!year_fd_varname}
+      fi
+      #echo ipaddress is $ipaddress and dayandmonth is $dayandmonth and month is $month and monthname is $monthname and year is $year and monthfile is $monthfile
+      echo "$hostname $line" >&${!month_fd_varname}
+      echo "$hostname $line" >&${!year_fd_varname}
+    fi
+    # append to $EVENTTOTALS
+    echo "$hostname $line" >&3
+    # append to $TOTALS
+    echo "$line" >&4
+  done <$i
 done
+
+# Close file handles for month/year files and $EVENTTOTALS and $TOTALS again:
+exec 3>&-
+exec 4>&-
+for i in $MONTHS/*; do
+  monthfilebasename=`basename $i`
+  month_fd_varname=month$monthfilebasename
+  month_fd=${!month_fs_varname}
+  echo Closing month file descriptor ${month_fd} for file $i
+  exec {month_fd}>&-
+done
+for i in $YEARS/*; do
+  yearfilebasename=`basename $i`
+  year_fd_varname=month$yearfilebasename
+  year_fd=${!year_fs_varname}
+  echo Closing year file descriptor ${year_fd} for file $i
+  exec {year_fd}>&-
+done
+
 cat $EVENTTOTALS | sort -u | awk '{ print $1; }' | uniq -c | awk '{ print $2, $1; }' >${EVENTTOTALS}.unique
 rm $EVENTTOTALS
 cat $TOTALS | sort -u | wc | awk '{ print $1; }' >${TOTALS}.unique
 rm $TOTALS
-for monthfile in $MONTHS/*; do
+for monthfile in $MONTHS/????-??; do
   cat $monthfile | sort -u | awk '{ print $1; }' | uniq -c | awk '{ print $2, $1; }' >$monthfile.unique
   rm $monthfile
 done
+for yearfile in $YEARS/????; do
+  cat $yearfile | sort -u | awk '{ print $1; }' | uniq -c | awk '{ print $2, $1; }' >$yearfile.unique
+  rm $yearfile
+done
 echo Done with $0 at `date`