eclipse-sailing-analytics/sailing-analytics
1
0 Fork 0
mirror of https://github.com/eclipse-sailing-analytics/sailing-analytics.git synced 2026-11-15 14:56:41 +00:00
Code Activity

added logging and fixed newlines in output

Browse Source 
Change-Id: I7bbc5b03f3429f95d7af2fe010169862470c2712
This commit is contained in:
Axel Uhl committed 2018-01-24 18:24:54 +01:00
1 parent 804cffb297
commit 20e5fd44e1
5 files changed
+540 -2
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

configuration/unique_ips_per_referrer_fast
+2 -2
View File
@@ -1,9 +1,9 @@
#!/bin/bash
# Production:
CACHE=/var/log/old/cache/unique-ips-per-referrer
#CACHE=/var/log/old/cache/unique-ips-per-referrer
# Test:
#CACHE=/var/log/old/cache/unique-ips-per-referrer/test
CACHE=/var/log/old/cache/unique-ips-per-referrer/test
VISITED_FILES=$CACHE/visited
STATS=$CACHE/stats
OUTPUT=$STATS/results
java/com.sap.sse.test/src/com/sap/sse/util/apachelog/ApacheLogAnalysisTest.java
Executable
+38
View File
@@ -0,0 +1,38 @@
package com.sap.sse.util.apachelog;
import static org.junit.Assert.assertEquals;
import java.text.ParseException;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.TimeZone;
import org.junit.Test;
public class ApacheLogAnalysisTest {
@Test
public void testParseSingleLogLine() throws ParseException {
final LogEntry entry = new LogEntry("ess2017.sapsailing.com 91.7.27.233 - - [15/Dec/2017:16:49:41 +0000] \"POST /gwt/service/sailing HTTP/1.1\" 200 437 \"https://ess2017.sapsailing.com/gwt/RaceBoard.html?regattaName=ESS+2017+Los+Cabos&raceName=Race+1&leaderboardName=ESS+2017+Los+Cabos&leaderboardGroupName=Extreme+Sailing+Series+2017&eventId=6eb987e5-1334-45f9-b54a-8b772a3fbb5b&mode=PLAYER\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36\"");
assertEquals("ess2017.sapsailing.com", entry.getHostname());
assertEquals("91.7.27.233", entry.getRequestorIpString());
assertEquals("-", entry.getRemoteLogname());
assertEquals("-", entry.getRemoteUser());
assertEquals("15/Dec/2017:16:49:41 +0000", entry.getTimestampString());
assertEquals("15/Dec/2017", entry.getDateString());
final Calendar cal = new GregorianCalendar();
cal.setTimeZone(TimeZone.getTimeZone("UTC"));
cal.set(Calendar.DAY_OF_MONTH, 15);
cal.set(Calendar.MONTH, 11);
cal.set(Calendar.YEAR, 2017);
cal.set(Calendar.HOUR_OF_DAY, 16);
cal.set(Calendar.MINUTE, 49);
cal.set(Calendar.SECOND, 41);
cal.set(Calendar.MILLISECOND, 0);
assertEquals(cal.getTime(), entry.getTimepoint().asDate());
assertEquals("POST /gwt/service/sailing HTTP/1.1", entry.getRequest());
assertEquals(200, entry.getHttpStatusCode());
assertEquals(437, entry.getSize());
assertEquals("https://ess2017.sapsailing.com/gwt/RaceBoard.html?regattaName=ESS+2017+Los+Cabos&raceName=Race+1&leaderboardName=ESS+2017+Los+Cabos&leaderboardGroupName=Extreme+Sailing+Series+2017&eventId=6eb987e5-1334-45f9-b54a-8b772a3fbb5b&mode=PLAYER", entry.getReferrer());
assertEquals("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36", entry.getUserAgent());
}
}
java/com.sap.sse/src/com/sap/sse/util/apachelog/LogEntry.java
Executable
+117
View File
@@ -0,0 +1,117 @@
package com.sap.sse.util.apachelog;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Iterator;
import com.sap.sse.common.TimePoint;
import com.sap.sse.common.Util;
import com.sap.sse.common.impl.MillisecondsTimePoint;
/**
* Represents a line of the form
* <pre>
* ess2017.sapsailing.com 91.7.27.233 - - [15/Dec/2017:16:49:41 +0000] "POST /gwt/service/sailing HTTP/1.1" 200 437 "https://ess2017.sapsailing.com/gwt/RaceBoard.html?regattaName=ESS+2017+Los+Cabos&raceName=Race+1&leaderboardName=ESS+2017+Los+Cabos&leaderboardGroupName=Extreme+Sailing+Series+2017&eventId=6eb987e5-1334-45f9-b54a-8b772a3fbb5b&mode=PLAYER" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36"
* </pre>
* with its constituents. It is parsed by splitting by space, respecting double-quoted strings which then may contain spaces.
*
* @author Axel Uhl (D043530)
*
*/
public class LogEntry {
private final SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ss ZZZZZ");
private final String hostname;
private final String requestorIpString;
private final String remoteLogname;
private final String remoteUser;
private final String timestampString;
private final String request;
private final String httpStatusCodeString;
private final String sizeString;
private final String referrer;
private final String userAgent;
public LogEntry(String logEntry) {
final Iterator<String> i = Util.splitAlongWhitespaceRespectingDoubleQuotedPhrases(logEntry).iterator();
hostname = i.next();
requestorIpString = i.next();
remoteLogname = i.next();
remoteUser = i.next();
final String dateTimeWithOpeningBracket = i.next();
final String timeZoneOffsetWithClosingBracket = i.next();
timestampString = dateTimeWithOpeningBracket.substring(1) + " " + timeZoneOffsetWithClosingBracket.substring(0, timeZoneOffsetWithClosingBracket.length()-1);
request = i.next();
httpStatusCodeString = i.next();
sizeString = i.next();
referrer = i.next();
userAgent = i.next();
}
public String getHostname() {
return hostname;
}
public String getRequestorIpString() {
return requestorIpString;
}
public InetAddress getRequestorIp() throws UnknownHostException {
return InetAddress.getByName(getRequestorIpString());
}
public String getRemoteLogname() {
return remoteLogname;
}
public String getRemoteUser() {
return remoteUser;
}
public String getTimestampString() {
return timestampString;
}
/**
* Returns only the date-specific portion of the {@link #getTimestampString() timestamp string}. For example,
* "15/Dec/2017:16:49:41 +0000" will result in "15/Dec/2017".
*/
public String getDateString() {
return getTimestampString().substring(0, getTimestampString().indexOf(':'));
}
public TimePoint getTimepoint() throws ParseException {
return new MillisecondsTimePoint(dateFormat.parse(getTimestampString()));
}
public String getRequest() {
return request;
}
public String getHttpStatusCodeString() {
return httpStatusCodeString;
}
public int getHttpStatusCode() {
return Integer.valueOf(getHttpStatusCodeString());
}
public String getSizeString() {
return sizeString;
}
public long getSize() {
return Long.valueOf(getSizeString());
}
public String getReferrer() {
return referrer;
}
public String getUserAgent() {
return userAgent;
}
}
java/com.sap.sse/src/com/sap/sse/util/apachelog/PerHostnameEntry.java
Executable
+70
View File
@@ -0,0 +1,70 @@
package com.sap.sse.util.apachelog;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.Iterator;
import java.util.TimeZone;
import com.sap.sse.common.Util;
public class PerHostnameEntry {
private final static SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MMM/yyyy");
private final String requestorIpString;
private final String dateString;
private final String userAgent;
private final int year;
private final int month;
/**
* Parses a line of format "66.249.66.14 17/Dec/2017 Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
* assuming that IP and date and useragent are separated by spaces and the user agent string fills up the remainder of the line.
*/
public PerHostnameEntry(String line) {
final Iterator<String> i = Util.splitAlongWhitespaceRespectingDoubleQuotedPhrases(line).iterator();
requestorIpString = i.next();
dateString = i.next();
final StringBuilder userAgentBuilder = new StringBuilder();
while (i.hasNext()) {
userAgentBuilder.append(i.next());
if (i.hasNext()) {
userAgentBuilder.append(' ');
}
}
userAgent = userAgentBuilder.toString();
final Calendar cal = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
int tmpYear;
int tmpMonth;
try {
cal.setTime(dateFormat.parse(dateString));
tmpYear = cal.get(Calendar.YEAR);
tmpMonth = cal.get(Calendar.MONTH);
} catch (ParseException e) {
tmpYear = 0;
tmpMonth = 0;
}
month = tmpMonth;
year = tmpYear;
}
public String getRequestorIpString() {
return requestorIpString;
}
public String getDateString() {
return dateString;
}
public String getUserAgent() {
return userAgent;
}
public int getYear() {
return year;
}
public int getMonth() {
return month;
}
}
java/com.sap.sse/src/com/sap/sse/util/apachelog/UniqueIPsPerReferrer.java
Executable
+313
View File
@@ -0,0 +1,313 @@
package com.sap.sse.util.apachelog;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.text.ParseException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.function.Function;
import java.util.logging.Logger;
import java.util.zip.GZIPInputStream;
import com.sap.sse.common.TimePoint;
import com.sap.sse.common.impl.MillisecondsTimePoint;
/**
* From a set of Apache log files, a set of already visited files and the pre-existing results gathered from the files
* already visited, computes an updated version of the stats that tell which hostname received how many unique visitors
* per month and per year. This is intended to replace Bash script that has horrible performance. We're hoping to speed
* things up considerably by using Java for this task.
*
* @author Axel Uhl (D043530)
*
*/
public class UniqueIPsPerReferrer {
private static final Logger logger = Logger.getLogger(UniqueIPsPerReferrer.class.getName());
private static final String HOSTNAME_FILE_EXTENSION = ".ips";
private static final String UNIQUE_SUFFIX = ".unique";
private final File CACHE;
private final File VISITED_FILES;
private final File STATS;
private final File OUTPUT;
private final File MONTHS;
private final File YEARS;
private final File TOTALS;
private final File EVENTTOTALS;
public UniqueIPsPerReferrer() {
this("/var/log/old/cache/unique-ips-per-referrer/test");
}
public UniqueIPsPerReferrer(final String cacheDir) {
CACHE = new File(cacheDir);
VISITED_FILES = new File(CACHE + "/visited");
STATS = new File(CACHE + "/stats");
OUTPUT = new File(STATS + "/results");
MONTHS = new File(OUTPUT + "/permonth");
YEARS = new File(OUTPUT + "/peryear");
TOTALS = new File(OUTPUT + "/totals");
EVENTTOTALS = new File(OUTPUT + "/eventtotals");
ensureDirectoriesExist();
}
/**
* The first argument is the base directory for where to place the outputs, used for {@link #CACHE}.
* All subsequent arguments are considered to be file names of logfiles that are supposed to be
* analyzed.
*/
public static void main(String[] args) throws IOException, ParseException {
final TimePoint started = MillisecondsTimePoint.now();
final String[] logfilenames = new String[args.length-1];
System.arraycopy(args, 1, logfilenames, 0, logfilenames.length);
new UniqueIPsPerReferrer(args[0]).analyze(logfilenames);
logger.info("Finished. Took "+started.until(MillisecondsTimePoint.now()));
}
public void analyze(String... filenames) throws IOException, ParseException {
appendHitsToHostnameSpecificFiles(filenames);
cleanUpOldResults();
computePerMonthPerYearPerEventAndTotals();
countUniqueEventOccurrencesPerMonthAndPerYearAndOverall();
countTotalHits();
}
private void ensureDirectoriesExist() {
CACHE.mkdirs();
STATS.mkdirs();
OUTPUT.mkdirs();
MONTHS.mkdirs();
YEARS.mkdirs();
}
private Set<String> getVisitedFileNames() throws IOException {
final Set<String> result = new HashSet<>();
if (VISITED_FILES.exists()) {
final BufferedReader br = new BufferedReader(new FileReader(VISITED_FILES));
String visitedFileName;
while ((visitedFileName = br.readLine()) != null) {
result.add(visitedFileName);
}
br.close();
}
return result;
}
private void appendHitsToHostnameSpecificFiles(String... filenames) throws IOException {
final Set<String> visitedFileNames = getVisitedFileNames();
for (final String filename : filenames) {
logger.info("Analyzing log file "+filename);
if (!visitedFileNames.contains(filename)) {
appendHitsToHostnameSpecificFiles(new File(filename));
visitedFileNames.add(filename);
addVisitedFile(filename);
}
}
}
/**
* Removes the interim files for per-month and per-year stats, as well as the interim
* files for totals and per-event totals.
*/
private void cleanUpOldResults() throws IOException {
logger.info("Cleanign up old results");
deleteDirectoryContentsRecursively(MONTHS.toPath(), "....-..");
deleteDirectoryContentsRecursively(YEARS.toPath(), "....");
EVENTTOTALS.delete();
TOTALS.delete();
}
private void computePerMonthPerYearPerEventAndTotals() throws IOException, ParseException {
final Map<String, FileWriter> monthFileWritersPerFileBaseName = new HashMap<>();
final Map<String, FileWriter> yearFileWritersPerFileBaseName = new HashMap<>();
final FileWriter eventTotalsWriter = new FileWriter(EVENTTOTALS);
final FileWriter totalsWriter = new FileWriter(TOTALS);
for (final File perHostnameFile : getAllHostnameFiles()) {
final String hostname = perHostnameFile.getName().substring(0, perHostnameFile.getName().length()-HOSTNAME_FILE_EXTENSION.length());
logger.info("Extracting per-month and per-year stats for hostname "+hostname);
final BufferedReader r = new BufferedReader(new FileReader(perHostnameFile));
try {
String line;
while ((line=r.readLine()) != null) {
final PerHostnameEntry entry = new PerHostnameEntry(line);
if (entry.getYear() != 0) {
final String monthFileBasename = ""+entry.getYear()+"_"+entry.getMonth();
final FileWriter monthFileWriter = getFileWriter(monthFileBasename, monthFileWritersPerFileBaseName, this::getMonthFile);
final String combinedLine = hostname+" "+line+"\n";
monthFileWriter.write(combinedLine);
final String yearFileBasename = ""+entry.getYear();
final FileWriter yearFileWriter = getFileWriter(yearFileBasename, yearFileWritersPerFileBaseName, this::getYearFile);
yearFileWriter.write(combinedLine);
}
eventTotalsWriter.write(hostname+" "+line+"\n");
totalsWriter.write(line+"\n");
}
} finally {
r.close();
}
}
for (final FileWriter monthFileWriter : monthFileWritersPerFileBaseName.values()) {
monthFileWriter.close();
}
for (final FileWriter yearFileWriter : yearFileWritersPerFileBaseName.values()) {
yearFileWriter.close();
}
eventTotalsWriter.close();
totalsWriter.close();
}
private File getMonthFile(String monthFileBaseName) {
return new File(MONTHS, monthFileBaseName);
}
private File getYearFile(String yearFileBaseName) {
return new File(YEARS, yearFileBaseName);
}
private void countUniqueEventOccurrencesPerMonthAndPerYearAndOverall() throws IOException {
logger.info("Counting unique visitors per host for per-month files");
countUniqueVisitorsPerHostname(MONTHS.listFiles((dir, filename)->filename.matches("...._..")));
logger.info("Counting unique visitors per host for per-year files");
countUniqueVisitorsPerHostname(YEARS.listFiles((dir, filename)->filename.matches("....")));
logger.info("Counting unique visitors per host for event totals");
countUniqueVisitorsPerHostname(new File[] { EVENTTOTALS });
}
/**
* For each file in {@code files} produces a ".unique" file next to it that counts the distinct unique visitors for
* each hostname found in the first column of the respective file.
*
* @param files
* files that contain lines of the form "yes2015.sapsailing.com 66.249.66.14 17/Dec/2017 Mozilla/5.0
* (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" where the first element is the hostname
* and the rest is the "unique visitor" information consisting of the IP address, day and user agent
* string.
*/
private void countUniqueVisitorsPerHostname(final File[] files) throws IOException {
for (final File file : files) {
final File uniqueFile = new File(file.getCanonicalFile()+UNIQUE_SUFFIX);
final Set<String> uniqueLinesFromFile = readDistinctLines(file);
final FileWriter uniqueFileWriter = new FileWriter(uniqueFile);
final Map<String, Integer> hostnameCounts = new HashMap<>();
for (final String uniqueLineFromMonthFile : uniqueLinesFromFile) {
final String hostname = uniqueLineFromMonthFile.split(" ")[0];
inc(hostnameCounts, hostname);
}
for (final Entry<String, Integer> e : hostnameCounts.entrySet()) {
uniqueFileWriter.write(e.getKey()+" "+e.getValue()+"\n");
}
uniqueFileWriter.close();
}
}
private void inc(Map<String, Integer> hostnameCountsForMonth, String hostname) {
Integer count = hostnameCountsForMonth.get(hostname);
count = count == null ? 1 : count+1;
hostnameCountsForMonth.put(hostname, count);
}
private void countTotalHits() throws IOException {
logger.info("Counting total number of unique visitors");
final FileWriter totalsCountWriter = new FileWriter(new File(TOTALS.getCanonicalPath()+UNIQUE_SUFFIX));
final Set<String> uniqueTotalLines = readDistinctLines(TOTALS);
totalsCountWriter.write(uniqueTotalLines.size()+"\n");
totalsCountWriter.close();
}
private Set<String> readDistinctLines(File fromFile) throws IOException {
final Set<String> result = new HashSet<>();
final BufferedReader br = new BufferedReader(new FileReader(fromFile));
try {
String line;
while ((line = br.readLine()) != null) {
result.add(line);
}
} finally {
br.close();
}
return result;
}
private void deleteDirectoryContentsRecursively(final Path directory, String fileNameRegexpPattern) throws IOException {
Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
if (fileNameRegexpPattern == null || file.getName(file.getNameCount()-1).toString().matches(fileNameRegexpPattern)) {
Files.delete(file);
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
if (!dir.equals(directory)) {
Files.delete(dir);
}
return FileVisitResult.CONTINUE;
}
});
}
private void addVisitedFile(String filename) throws IOException {
final FileWriter visitedFilesWriter = new FileWriter(VISITED_FILES, /* append */ true);
visitedFilesWriter.write(filename+"\n");
visitedFilesWriter.close();
}
private void appendHitsToHostnameSpecificFiles(File logfileWhichMayUseGzipCompression) throws IOException {
final Reader reader;
if (logfileWhichMayUseGzipCompression.getName().toLowerCase().endsWith(".gz")) {
reader = new InputStreamReader(new GZIPInputStream(new FileInputStream(logfileWhichMayUseGzipCompression)));
} else {
reader = new FileReader(logfileWhichMayUseGzipCompression);
}
appendHitsToHostnameSpecificFiles(reader);
}
private void appendHitsToHostnameSpecificFiles(Reader logfileReader) throws IOException {
final Map<String, FileWriter> fileWritersPerHostname = new HashMap<>();
final BufferedReader br = new BufferedReader(logfileReader);
String line;
while ((line=br.readLine()) != null) {
final LogEntry entry = new LogEntry(line);
final String hostname = entry.getHostname();
final FileWriter fileWriterForHostname = getFileWriter(hostname, fileWritersPerHostname, this::getHostnameSpecificFile);
fileWriterForHostname.write(entry.getRequestorIpString()+" "+entry.getDateString()+" "+entry.getUserAgent()+"\n");
}
for (final FileWriter fw : fileWritersPerHostname.values()) {
fw.close();
}
}
private FileWriter getFileWriter(String key, Map<String, FileWriter> fileWriterCache, Function<String, File> fileConstructor) throws IOException {
final FileWriter fileWriterForHostname;
if (fileWriterCache.containsKey(key)) {
fileWriterForHostname = fileWriterCache.get(key);
} else {
fileWriterForHostname = new FileWriter(fileConstructor.apply(key), /* append */ true);
fileWriterCache.put(key, fileWriterForHostname);
}
return fileWriterForHostname;
}
private File getHostnameSpecificFile(String hostname) {
return new File(STATS, hostname+HOSTNAME_FILE_EXTENSION);
}
private File[] getAllHostnameFiles() {
return STATS.listFiles((dir, filename)->filename.endsWith(HOSTNAME_FILE_EXTENSION));
}
}