[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [metrics-lib/master] Add log line interfaces and access methods.



commit 59689a9fa4c162378f347902eb68e4c21ccf0043
Author: iwakeh <iwakeh@xxxxxxxxxxxxxx>
Date:   Tue Feb 6 14:59:05 2018 +0000

    Add log line interfaces and access methods.
    
    For both the general LogDescriptor and extension WebServerAccessLog.
    Include some new tests.
---
 .../org/torproject/descriptor/LogDescriptor.java   | 12 ++++++++++
 .../torproject/descriptor/{log => }/Method.java    |  5 ++--
 .../torproject/descriptor/WebServerAccessLog.java  | 28 ++++++++++++++++++++++
 .../descriptor/log/WebServerAccessLogImpl.java     | 20 ++++++++++++++++
 .../descriptor/log/WebServerAccessLogLine.java     | 15 ++++++++++--
 .../descriptor/log/LogDescriptorTest.java          | 15 +++++++-----
 6 files changed, 85 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/torproject/descriptor/LogDescriptor.java b/src/main/java/org/torproject/descriptor/LogDescriptor.java
index ff02cae..6a6bf84 100644
--- a/src/main/java/org/torproject/descriptor/LogDescriptor.java
+++ b/src/main/java/org/torproject/descriptor/LogDescriptor.java
@@ -43,5 +43,17 @@ public interface LogDescriptor extends Descriptor {
   @Override
   public List<String> getUnrecognizedLines();
 
+  /**
+   * Returns a list of all parseable log lines.
+   * <p>Might require a lot of memory depending on log size.</p>
+   */
+  public List<? extends Line> logLines() throws DescriptorParseException;
+
+  public interface Line {
+
+    /** Returns a log line string. */
+    public String toLogString();
+
+  }
 }
 
diff --git a/src/main/java/org/torproject/descriptor/log/Method.java b/src/main/java/org/torproject/descriptor/Method.java
similarity index 50%
rename from src/main/java/org/torproject/descriptor/log/Method.java
rename to src/main/java/org/torproject/descriptor/Method.java
index c29d495..9135fe2 100644
--- a/src/main/java/org/torproject/descriptor/log/Method.java
+++ b/src/main/java/org/torproject/descriptor/Method.java
@@ -1,8 +1,9 @@
 /* Copyright 2018 The Tor Project
  * See LICENSE for licensing information */
 
-package org.torproject.descriptor.log;
+package org.torproject.descriptor;
 
-public  enum Method {
+/** Enum for web server access log methods. */
+public enum Method {
     GET, HEAD, POST;
 }
diff --git a/src/main/java/org/torproject/descriptor/WebServerAccessLog.java b/src/main/java/org/torproject/descriptor/WebServerAccessLog.java
index b94bc30..b4f1940 100644
--- a/src/main/java/org/torproject/descriptor/WebServerAccessLog.java
+++ b/src/main/java/org/torproject/descriptor/WebServerAccessLog.java
@@ -5,6 +5,7 @@ package org.torproject.descriptor;
 
 import java.time.LocalDate;
 import java.util.List;
+import java.util.Optional;
 
 /**
  * Contains a sanitized web server access log file from a {@code torproject.org}
@@ -61,5 +62,32 @@ public interface WebServerAccessLog extends LogDescriptor {
   @Override
   public List<String> getUnrecognizedLines();
 
+  public interface Line extends LogDescriptor.Line {
+
+    /** Returns the IP address of the requesting host. */
+    public String getIp();
+
+    /** Returns the HTTP method, e.g., GET. */
+    public Method getMethod();
+
+    /** Returns the protocol and version, e.g., HTTP/1.1. */
+    public String getProtocol();
+
+    /** Returns the requested resource. */
+    public String getRequest();
+
+    /** Returns the size of the response in bytes, if available. */
+    public Optional<Integer> getSize();
+
+    /** Returns the final status code, e.g., 200. */
+    public int getResponse();
+
+    /** Returns the date when the request was received. */
+    public LocalDate getDate();
+
+    /** True, if this is a valid web server access log line. */
+    public boolean isValid();
+  }
+
 }
 
diff --git a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java
index f02b1d7..7b56528 100644
--- a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java
+++ b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java
@@ -10,12 +10,17 @@ import org.torproject.descriptor.internal.FileType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
 import java.io.File;
+import java.io.InputStreamReader;
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.Collection;
+import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 /**
  * Implementation of web server access log descriptors.
@@ -126,5 +131,20 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl
     return this.logDate;
   }
 
+  /** Returns a list of all valid log lines. */
+  @Override
+  public List<WebServerAccessLog.Line> logLines()
+      throws DescriptorParseException {
+    try (BufferedReader br
+        = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(
+        this.getRawDescriptorBytes())))) {
+      return br.lines().map(line
+          -> (WebServerAccessLog.Line) WebServerAccessLogLine.makeLine(line))
+        .filter(line -> line.isValid()).collect(Collectors.toList());
+    } catch (Exception ex) {
+      throw new DescriptorParseException("Cannot retrieve log lines.", ex);
+    }
+  }
+
 }
 
diff --git a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java
index c9d73cc..8a17230 100644
--- a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java
+++ b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java
@@ -3,6 +3,9 @@
 
 package org.torproject.descriptor.log;
 
+import org.torproject.descriptor.Method;
+import org.torproject.descriptor.WebServerAccessLog;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -18,7 +21,7 @@ import java.util.Optional;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-public class WebServerAccessLogLine {
+public class WebServerAccessLogLine implements WebServerAccessLog.Line {
 
   private static final Logger log = LoggerFactory
       .getLogger(WebServerAccessLogLine.class);
@@ -54,6 +57,7 @@ public class WebServerAccessLogLine {
   private String protocol;
 
   /** Returns a log line string. Possibly empty. */
+  @Override
   public String toLogString() {
     if (!this.valid) {
       return "";
@@ -74,7 +78,7 @@ public class WebServerAccessLogLine {
     return this.date.format(DateTimeFormatter.ofPattern(DATE_PATTERN));
   }
 
-  /** Returns a string containing the ip. */
+  @Override
   public String getIp() {
     return this.ip;
   }
@@ -84,22 +88,27 @@ public class WebServerAccessLogLine {
     this.ip = fromMap(ip, ipMap);
   }
 
+  @Override
   public Method getMethod() {
     return this.method;
   }
 
+  @Override
   public String getProtocol() {
     return this.protocol;
   }
 
+  @Override
   public String getRequest() {
     return this.request;
   }
 
+  @Override
   public Optional<Integer> getSize() {
     return this.size < 0 ? Optional.empty() : Optional.of(this.size);
   }
 
+  @Override
   public int getResponse() {
     return this.response;
   }
@@ -109,10 +118,12 @@ public class WebServerAccessLogLine {
     this.request = fromMap(request, requestMap);
   }
 
+  @Override
   public LocalDate getDate() {
     return this.date;
   }
 
+  @Override
   public boolean isValid() {
     return this.valid;
   }
diff --git a/src/test/java/org/torproject/descriptor/log/LogDescriptorTest.java b/src/test/java/org/torproject/descriptor/log/LogDescriptorTest.java
index b12cfc0..a871791 100644
--- a/src/test/java/org/torproject/descriptor/log/LogDescriptorTest.java
+++ b/src/test/java/org/torproject/descriptor/log/LogDescriptorTest.java
@@ -1,4 +1,3 @@
-
 /* Copyright 2017--2018 The Tor Project
  * See LICENSE for licensing information */
 
@@ -51,6 +50,7 @@ public class LogDescriptorTest {
   protected String[] pan;
   protected Class<LogDescriptor> type;
   protected boolean isDecompressionTest;
+  protected int lineCount;
 
   /** All types of data that can be encountered during sync. */
   @Parameters
@@ -60,29 +60,30 @@ public class LogDescriptorTest {
             "metrics.torproject.org_meronense.torproject.org_access.log"
             + "_20170530.gz",
             "metrics.torproject.org", "20170530", "gz"},
-         WebServerAccessLog.class},
+         WebServerAccessLog.class, 24},
         {Boolean.FALSE, 1878, new String[]{"meronense.torproject.org",
             "xy.host.org_meronense.torproject.org_access.log_20170530.log",
             "metrics.torproject.org", "20170530", "xz"},
-         WebServerAccessLog.class},
+         WebServerAccessLog.class, 24},
         {Boolean.TRUE, 70730, new String[]{"archeotrichon.torproject.org",
             "archive.torproject.org_archeotrichon.torproject.org_access.log_"
             + "20151007.xz",
             "archive.torproject.org", "20151007", "xz"},
-         WebServerAccessLog.class},
+         WebServerAccessLog.class, 655},
         {Boolean.TRUE, 0, new String[]{"dummy.host.net",
             "nix.server.org_dummy.host.net_access.log_20111111.bz2",
             "nix.server.org", "20111111", "bz2"},
-         WebServerAccessLog.class}});
+         WebServerAccessLog.class, 0}});
   }
 
   /** This constructor receives the above defined data for each run. */
   public LogDescriptorTest(boolean decompression, int size, String[] pan,
-      Class<LogDescriptor> type) {
+        Class<LogDescriptor> type, int lineCount) {
     this.pan = pan;
     this.size = size;
     this.type = type;
     this.isDecompressionTest = decompression;
+    this.lineCount = lineCount;
   }
 
   /** Prepares the temporary folder and writes files to it for this test. */
@@ -129,6 +130,8 @@ public class LogDescriptorTest {
     InternalLogDescriptor ld = (InternalLogDescriptor) descs.get(0);
     assertEquals("Wrong compression type string. " + dataUsed(),
         pan[4], ld.getCompressionType());
+    List<? extends LogDescriptor.Line> lines = ld.logLines();
+    assertEquals(this.lineCount, lines.size());
   }
 
   private String dataUsed() {

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits