From 20e5fd44e1d1b8095ca8085d65158359779840f8 Mon Sep 17 00:00:00 2001 From: Axel Uhl Date: Wed, 24 Jan 2018 18:24:54 +0100 Subject: [PATCH] added logging and fixed newlines in output Change-Id: I7bbc5b03f3429f95d7af2fe010169862470c2712 --- configuration/unique_ips_per_referrer_fast | 4 +- .../util/apachelog/ApacheLogAnalysisTest.java | 38 +++ .../com/sap/sse/util/apachelog/LogEntry.java | 117 +++++++ .../sse/util/apachelog/PerHostnameEntry.java | 70 ++++ .../util/apachelog/UniqueIPsPerReferrer.java | 313 ++++++++++++++++++ 5 files changed, 540 insertions(+), 2 deletions(-) create mode 100755 java/com.sap.sse.test/src/com/sap/sse/util/apachelog/ApacheLogAnalysisTest.java create mode 100755 java/com.sap.sse/src/com/sap/sse/util/apachelog/LogEntry.java create mode 100755 java/com.sap.sse/src/com/sap/sse/util/apachelog/PerHostnameEntry.java create mode 100755 java/com.sap.sse/src/com/sap/sse/util/apachelog/UniqueIPsPerReferrer.java diff --git a/configuration/unique_ips_per_referrer_fast b/configuration/unique_ips_per_referrer_fast index aa7e4074ccd..093e4d9c881 100755 --- a/configuration/unique_ips_per_referrer_fast +++ b/configuration/unique_ips_per_referrer_fast @@ -1,9 +1,9 @@ #!/bin/bash # Production: -CACHE=/var/log/old/cache/unique-ips-per-referrer +#CACHE=/var/log/old/cache/unique-ips-per-referrer # Test: -#CACHE=/var/log/old/cache/unique-ips-per-referrer/test +CACHE=/var/log/old/cache/unique-ips-per-referrer/test VISITED_FILES=$CACHE/visited STATS=$CACHE/stats OUTPUT=$STATS/results diff --git a/java/com.sap.sse.test/src/com/sap/sse/util/apachelog/ApacheLogAnalysisTest.java b/java/com.sap.sse.test/src/com/sap/sse/util/apachelog/ApacheLogAnalysisTest.java new file mode 100755 index 00000000000..6eea3264f86 --- /dev/null +++ b/java/com.sap.sse.test/src/com/sap/sse/util/apachelog/ApacheLogAnalysisTest.java @@ -0,0 +1,38 @@ +package com.sap.sse.util.apachelog; + +import static org.junit.Assert.assertEquals; + +import java.text.ParseException; +import java.util.Calendar; +import java.util.GregorianCalendar; +import java.util.TimeZone; + +import org.junit.Test; + +public class ApacheLogAnalysisTest { + @Test + public void testParseSingleLogLine() throws ParseException { + final LogEntry entry = new LogEntry("ess2017.sapsailing.com 91.7.27.233 - - [15/Dec/2017:16:49:41 +0000] \"POST /gwt/service/sailing HTTP/1.1\" 200 437 \"https://ess2017.sapsailing.com/gwt/RaceBoard.html?regattaName=ESS+2017+Los+Cabos&raceName=Race+1&leaderboardName=ESS+2017+Los+Cabos&leaderboardGroupName=Extreme+Sailing+Series+2017&eventId=6eb987e5-1334-45f9-b54a-8b772a3fbb5b&mode=PLAYER\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36\""); + assertEquals("ess2017.sapsailing.com", entry.getHostname()); + assertEquals("91.7.27.233", entry.getRequestorIpString()); + assertEquals("-", entry.getRemoteLogname()); + assertEquals("-", entry.getRemoteUser()); + assertEquals("15/Dec/2017:16:49:41 +0000", entry.getTimestampString()); + assertEquals("15/Dec/2017", entry.getDateString()); + final Calendar cal = new GregorianCalendar(); + cal.setTimeZone(TimeZone.getTimeZone("UTC")); + cal.set(Calendar.DAY_OF_MONTH, 15); + cal.set(Calendar.MONTH, 11); + cal.set(Calendar.YEAR, 2017); + cal.set(Calendar.HOUR_OF_DAY, 16); + cal.set(Calendar.MINUTE, 49); + cal.set(Calendar.SECOND, 41); + cal.set(Calendar.MILLISECOND, 0); + assertEquals(cal.getTime(), entry.getTimepoint().asDate()); + assertEquals("POST /gwt/service/sailing HTTP/1.1", entry.getRequest()); + assertEquals(200, entry.getHttpStatusCode()); + assertEquals(437, entry.getSize()); + assertEquals("https://ess2017.sapsailing.com/gwt/RaceBoard.html?regattaName=ESS+2017+Los+Cabos&raceName=Race+1&leaderboardName=ESS+2017+Los+Cabos&leaderboardGroupName=Extreme+Sailing+Series+2017&eventId=6eb987e5-1334-45f9-b54a-8b772a3fbb5b&mode=PLAYER", entry.getReferrer()); + assertEquals("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36", entry.getUserAgent()); + } +} diff --git a/java/com.sap.sse/src/com/sap/sse/util/apachelog/LogEntry.java b/java/com.sap.sse/src/com/sap/sse/util/apachelog/LogEntry.java new file mode 100755 index 00000000000..01f885db337 --- /dev/null +++ b/java/com.sap.sse/src/com/sap/sse/util/apachelog/LogEntry.java @@ -0,0 +1,117 @@ +package com.sap.sse.util.apachelog; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Iterator; + +import com.sap.sse.common.TimePoint; +import com.sap.sse.common.Util; +import com.sap.sse.common.impl.MillisecondsTimePoint; + +/** + * Represents a line of the form + * 
+ * ess2017.sapsailing.com 91.7.27.233 - - [15/Dec/2017:16:49:41 +0000] "POST /gwt/service/sailing HTTP/1.1" 200 437 "https://ess2017.sapsailing.com/gwt/RaceBoard.html?regattaName=ESS+2017+Los+Cabos&raceName=Race+1&leaderboardName=ESS+2017+Los+Cabos&leaderboardGroupName=Extreme+Sailing+Series+2017&eventId=6eb987e5-1334-45f9-b54a-8b772a3fbb5b&mode=PLAYER" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36"
+ *
+ * with its constituents. It is parsed by splitting by space, respecting double-quoted strings which then may contain spaces. + * + * @author Axel Uhl (D043530) + * + */ +public class LogEntry { + private final SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ss ZZZZZ"); + + private final String hostname; + private final String requestorIpString; + private final String remoteLogname; + private final String remoteUser; + private final String timestampString; + private final String request; + private final String httpStatusCodeString; + private final String sizeString; + private final String referrer; + private final String userAgent; + + public LogEntry(String logEntry) { + final Iterator i = Util.splitAlongWhitespaceRespectingDoubleQuotedPhrases(logEntry).iterator(); + hostname = i.next(); + requestorIpString = i.next(); + remoteLogname = i.next(); + remoteUser = i.next(); + final String dateTimeWithOpeningBracket = i.next(); + final String timeZoneOffsetWithClosingBracket = i.next(); + timestampString = dateTimeWithOpeningBracket.substring(1) + " " + timeZoneOffsetWithClosingBracket.substring(0, timeZoneOffsetWithClosingBracket.length()-1); + request = i.next(); + httpStatusCodeString = i.next(); + sizeString = i.next(); + referrer = i.next(); + userAgent = i.next(); + } + + public String getHostname() { + return hostname; + } + + public String getRequestorIpString() { + return requestorIpString; + } + + public InetAddress getRequestorIp() throws UnknownHostException { + return InetAddress.getByName(getRequestorIpString()); + } + + public String getRemoteLogname() { + return remoteLogname; + } + + public String getRemoteUser() { + return remoteUser; + } + + public String getTimestampString() { + return timestampString; + } + + /** + * Returns only the date-specific portion of the {@link #getTimestampString() timestamp string}. For example, + * "15/Dec/2017:16:49:41 +0000" will result in "15/Dec/2017". + */ + public String getDateString() { + return getTimestampString().substring(0, getTimestampString().indexOf(':')); + } + + public TimePoint getTimepoint() throws ParseException { + return new MillisecondsTimePoint(dateFormat.parse(getTimestampString())); + } + + public String getRequest() { + return request; + } + + public String getHttpStatusCodeString() { + return httpStatusCodeString; + } + + public int getHttpStatusCode() { + return Integer.valueOf(getHttpStatusCodeString()); + } + + public String getSizeString() { + return sizeString; + } + + public long getSize() { + return Long.valueOf(getSizeString()); + } + + public String getReferrer() { + return referrer; + } + + public String getUserAgent() { + return userAgent; + } + +} diff --git a/java/com.sap.sse/src/com/sap/sse/util/apachelog/PerHostnameEntry.java b/java/com.sap.sse/src/com/sap/sse/util/apachelog/PerHostnameEntry.java new file mode 100755 index 00000000000..2876b70a39e --- /dev/null +++ b/java/com.sap.sse/src/com/sap/sse/util/apachelog/PerHostnameEntry.java @@ -0,0 +1,70 @@ +package com.sap.sse.util.apachelog; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.GregorianCalendar; +import java.util.Iterator; +import java.util.TimeZone; + +import com.sap.sse.common.Util; + +public class PerHostnameEntry { + private final static SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MMM/yyyy"); + private final String requestorIpString; + private final String dateString; + private final String userAgent; + private final int year; + private final int month; + + /** + * Parses a line of format "66.249.66.14 17/Dec/2017 Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" + * assuming that IP and date and useragent are separated by spaces and the user agent string fills up the remainder of the line. + */ + public PerHostnameEntry(String line) { + final Iterator i = Util.splitAlongWhitespaceRespectingDoubleQuotedPhrases(line).iterator(); + requestorIpString = i.next(); + dateString = i.next(); + final StringBuilder userAgentBuilder = new StringBuilder(); + while (i.hasNext()) { + userAgentBuilder.append(i.next()); + if (i.hasNext()) { + userAgentBuilder.append(' '); + } + } + userAgent = userAgentBuilder.toString(); + final Calendar cal = new GregorianCalendar(TimeZone.getTimeZone("UTC")); + int tmpYear; + int tmpMonth; + try { + cal.setTime(dateFormat.parse(dateString)); + tmpYear = cal.get(Calendar.YEAR); + tmpMonth = cal.get(Calendar.MONTH); + } catch (ParseException e) { + tmpYear = 0; + tmpMonth = 0; + } + month = tmpMonth; + year = tmpYear; + } + + public String getRequestorIpString() { + return requestorIpString; + } + + public String getDateString() { + return dateString; + } + + public String getUserAgent() { + return userAgent; + } + + public int getYear() { + return year; + } + + public int getMonth() { + return month; + } +} diff --git a/java/com.sap.sse/src/com/sap/sse/util/apachelog/UniqueIPsPerReferrer.java b/java/com.sap.sse/src/com/sap/sse/util/apachelog/UniqueIPsPerReferrer.java new file mode 100755 index 00000000000..0bf59e21d98 --- /dev/null +++ b/java/com.sap.sse/src/com/sap/sse/util/apachelog/UniqueIPsPerReferrer.java @@ -0,0 +1,313 @@ +package com.sap.sse.util.apachelog; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.ParseException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.function.Function; +import java.util.logging.Logger; +import java.util.zip.GZIPInputStream; + +import com.sap.sse.common.TimePoint; +import com.sap.sse.common.impl.MillisecondsTimePoint; + +/** + * From a set of Apache log files, a set of already visited files and the pre-existing results gathered from the files + * already visited, computes an updated version of the stats that tell which hostname received how many unique visitors + * per month and per year. This is intended to replace Bash script that has horrible performance. We're hoping to speed + * things up considerably by using Java for this task. + * + * @author Axel Uhl (D043530) + * + */ +public class UniqueIPsPerReferrer { + private static final Logger logger = Logger.getLogger(UniqueIPsPerReferrer.class.getName()); + private static final String HOSTNAME_FILE_EXTENSION = ".ips"; + private static final String UNIQUE_SUFFIX = ".unique"; + private final File CACHE; + private final File VISITED_FILES; + private final File STATS; + private final File OUTPUT; + private final File MONTHS; + private final File YEARS; + private final File TOTALS; + private final File EVENTTOTALS; + + public UniqueIPsPerReferrer() { + this("/var/log/old/cache/unique-ips-per-referrer/test"); + } + + public UniqueIPsPerReferrer(final String cacheDir) { + CACHE = new File(cacheDir); + VISITED_FILES = new File(CACHE + "/visited"); + STATS = new File(CACHE + "/stats"); + OUTPUT = new File(STATS + "/results"); + MONTHS = new File(OUTPUT + "/permonth"); + YEARS = new File(OUTPUT + "/peryear"); + TOTALS = new File(OUTPUT + "/totals"); + EVENTTOTALS = new File(OUTPUT + "/eventtotals"); + ensureDirectoriesExist(); + } + + /** + * The first argument is the base directory for where to place the outputs, used for {@link #CACHE}. + * All subsequent arguments are considered to be file names of logfiles that are supposed to be + * analyzed. + */ + public static void main(String[] args) throws IOException, ParseException { + final TimePoint started = MillisecondsTimePoint.now(); + final String[] logfilenames = new String[args.length-1]; + System.arraycopy(args, 1, logfilenames, 0, logfilenames.length); + new UniqueIPsPerReferrer(args[0]).analyze(logfilenames); + logger.info("Finished. Took "+started.until(MillisecondsTimePoint.now())); + } + + public void analyze(String... filenames) throws IOException, ParseException { + appendHitsToHostnameSpecificFiles(filenames); + cleanUpOldResults(); + computePerMonthPerYearPerEventAndTotals(); + countUniqueEventOccurrencesPerMonthAndPerYearAndOverall(); + countTotalHits(); + } + + private void ensureDirectoriesExist() { + CACHE.mkdirs(); + STATS.mkdirs(); + OUTPUT.mkdirs(); + MONTHS.mkdirs(); + YEARS.mkdirs(); + } + + private Set getVisitedFileNames() throws IOException { + final Set result = new HashSet<>(); + if (VISITED_FILES.exists()) { + final BufferedReader br = new BufferedReader(new FileReader(VISITED_FILES)); + String visitedFileName; + while ((visitedFileName = br.readLine()) != null) { + result.add(visitedFileName); + } + br.close(); + } + return result; + } + + private void appendHitsToHostnameSpecificFiles(String... filenames) throws IOException { + final Set visitedFileNames = getVisitedFileNames(); + for (final String filename : filenames) { + logger.info("Analyzing log file "+filename); + if (!visitedFileNames.contains(filename)) { + appendHitsToHostnameSpecificFiles(new File(filename)); + visitedFileNames.add(filename); + addVisitedFile(filename); + } + } + } + + /** + * Removes the interim files for per-month and per-year stats, as well as the interim + * files for totals and per-event totals. + */ + private void cleanUpOldResults() throws IOException { + logger.info("Cleanign up old results"); + deleteDirectoryContentsRecursively(MONTHS.toPath(), "....-.."); + deleteDirectoryContentsRecursively(YEARS.toPath(), "...."); + EVENTTOTALS.delete(); + TOTALS.delete(); + } + + private void computePerMonthPerYearPerEventAndTotals() throws IOException, ParseException { + final Map monthFileWritersPerFileBaseName = new HashMap<>(); + final Map yearFileWritersPerFileBaseName = new HashMap<>(); + final FileWriter eventTotalsWriter = new FileWriter(EVENTTOTALS); + final FileWriter totalsWriter = new FileWriter(TOTALS); + for (final File perHostnameFile : getAllHostnameFiles()) { + final String hostname = perHostnameFile.getName().substring(0, perHostnameFile.getName().length()-HOSTNAME_FILE_EXTENSION.length()); + logger.info("Extracting per-month and per-year stats for hostname "+hostname); + final BufferedReader r = new BufferedReader(new FileReader(perHostnameFile)); + try { + String line; + while ((line=r.readLine()) != null) { + final PerHostnameEntry entry = new PerHostnameEntry(line); + if (entry.getYear() != 0) { + final String monthFileBasename = ""+entry.getYear()+"_"+entry.getMonth(); + final FileWriter monthFileWriter = getFileWriter(monthFileBasename, monthFileWritersPerFileBaseName, this::getMonthFile); + final String combinedLine = hostname+" "+line+"\n"; + monthFileWriter.write(combinedLine); + final String yearFileBasename = ""+entry.getYear(); + final FileWriter yearFileWriter = getFileWriter(yearFileBasename, yearFileWritersPerFileBaseName, this::getYearFile); + yearFileWriter.write(combinedLine); + } + eventTotalsWriter.write(hostname+" "+line+"\n"); + totalsWriter.write(line+"\n"); + } + } finally { + r.close(); + } + } + for (final FileWriter monthFileWriter : monthFileWritersPerFileBaseName.values()) { + monthFileWriter.close(); + } + for (final FileWriter yearFileWriter : yearFileWritersPerFileBaseName.values()) { + yearFileWriter.close(); + } + eventTotalsWriter.close(); + totalsWriter.close(); + } + + private File getMonthFile(String monthFileBaseName) { + return new File(MONTHS, monthFileBaseName); + } + + private File getYearFile(String yearFileBaseName) { + return new File(YEARS, yearFileBaseName); + } + + private void countUniqueEventOccurrencesPerMonthAndPerYearAndOverall() throws IOException { + logger.info("Counting unique visitors per host for per-month files"); + countUniqueVisitorsPerHostname(MONTHS.listFiles((dir, filename)->filename.matches("...._.."))); + logger.info("Counting unique visitors per host for per-year files"); + countUniqueVisitorsPerHostname(YEARS.listFiles((dir, filename)->filename.matches("...."))); + logger.info("Counting unique visitors per host for event totals"); + countUniqueVisitorsPerHostname(new File[] { EVENTTOTALS }); + } + + /** + * For each file in {@code files} produces a ".unique" file next to it that counts the distinct unique visitors for + * each hostname found in the first column of the respective file. + * + * @param files + * files that contain lines of the form "yes2015.sapsailing.com 66.249.66.14 17/Dec/2017 Mozilla/5.0 + * (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" where the first element is the hostname + * and the rest is the "unique visitor" information consisting of the IP address, day and user agent + * string. + */ + private void countUniqueVisitorsPerHostname(final File[] files) throws IOException { + for (final File file : files) { + final File uniqueFile = new File(file.getCanonicalFile()+UNIQUE_SUFFIX); + final Set uniqueLinesFromFile = readDistinctLines(file); + final FileWriter uniqueFileWriter = new FileWriter(uniqueFile); + final Map hostnameCounts = new HashMap<>(); + for (final String uniqueLineFromMonthFile : uniqueLinesFromFile) { + final String hostname = uniqueLineFromMonthFile.split(" ")[0]; + inc(hostnameCounts, hostname); + } + for (final Entry e : hostnameCounts.entrySet()) { + uniqueFileWriter.write(e.getKey()+" "+e.getValue()+"\n"); + } + uniqueFileWriter.close(); + } + } + + private void inc(Map hostnameCountsForMonth, String hostname) { + Integer count = hostnameCountsForMonth.get(hostname); + count = count == null ? 1 : count+1; + hostnameCountsForMonth.put(hostname, count); + } + + private void countTotalHits() throws IOException { + logger.info("Counting total number of unique visitors"); + final FileWriter totalsCountWriter = new FileWriter(new File(TOTALS.getCanonicalPath()+UNIQUE_SUFFIX)); + final Set uniqueTotalLines = readDistinctLines(TOTALS); + totalsCountWriter.write(uniqueTotalLines.size()+"\n"); + totalsCountWriter.close(); + } + + private Set readDistinctLines(File fromFile) throws IOException { + final Set result = new HashSet<>(); + final BufferedReader br = new BufferedReader(new FileReader(fromFile)); + try { + String line; + while ((line = br.readLine()) != null) { + result.add(line); + } + } finally { + br.close(); + } + return result; + } + + private void deleteDirectoryContentsRecursively(final Path directory, String fileNameRegexpPattern) throws IOException { + Files.walkFileTree(directory, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (fileNameRegexpPattern == null || file.getName(file.getNameCount()-1).toString().matches(fileNameRegexpPattern)) { + Files.delete(file); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + if (!dir.equals(directory)) { + Files.delete(dir); + } + return FileVisitResult.CONTINUE; + } + }); + } + + private void addVisitedFile(String filename) throws IOException { + final FileWriter visitedFilesWriter = new FileWriter(VISITED_FILES, /* append */ true); + visitedFilesWriter.write(filename+"\n"); + visitedFilesWriter.close(); + } + + private void appendHitsToHostnameSpecificFiles(File logfileWhichMayUseGzipCompression) throws IOException { + final Reader reader; + if (logfileWhichMayUseGzipCompression.getName().toLowerCase().endsWith(".gz")) { + reader = new InputStreamReader(new GZIPInputStream(new FileInputStream(logfileWhichMayUseGzipCompression))); + } else { + reader = new FileReader(logfileWhichMayUseGzipCompression); + } + appendHitsToHostnameSpecificFiles(reader); + } + + private void appendHitsToHostnameSpecificFiles(Reader logfileReader) throws IOException { + final Map fileWritersPerHostname = new HashMap<>(); + final BufferedReader br = new BufferedReader(logfileReader); + String line; + while ((line=br.readLine()) != null) { + final LogEntry entry = new LogEntry(line); + final String hostname = entry.getHostname(); + final FileWriter fileWriterForHostname = getFileWriter(hostname, fileWritersPerHostname, this::getHostnameSpecificFile); + fileWriterForHostname.write(entry.getRequestorIpString()+" "+entry.getDateString()+" "+entry.getUserAgent()+"\n"); + } + for (final FileWriter fw : fileWritersPerHostname.values()) { + fw.close(); + } + } + + private FileWriter getFileWriter(String key, Map fileWriterCache, Function fileConstructor) throws IOException { + final FileWriter fileWriterForHostname; + if (fileWriterCache.containsKey(key)) { + fileWriterForHostname = fileWriterCache.get(key); + } else { + fileWriterForHostname = new FileWriter(fileConstructor.apply(key), /* append */ true); + fileWriterCache.put(key, fileWriterForHostname); + } + return fileWriterForHostname; + } + + private File getHostnameSpecificFile(String hostname) { + return new File(STATS, hostname+HOSTNAME_FILE_EXTENSION); + } + + private File[] getAllHostnameFiles() { + return STATS.listFiles((dir, filename)->filename.endsWith(HOSTNAME_FILE_EXTENSION)); + } +}