[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [collector/master] Remove dependency on metrics-lib's log package (2/4).



commit ea1b1b4f6ab11e7ac933b0e00d5f8c040e4cc11e
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Sat Nov 23 17:43:45 2019 +0100

    Remove dependency on metrics-lib's log package (2/4).
    
     - Remove unused code.
---
 .../persist/WebServerAccessLogPersistence.java     |  5 --
 .../collector/webstats/InternalLogDescriptor.java  | 35 --------------
 .../collector/webstats/LogDescriptorImpl.java      | 53 +---------------------
 .../metrics/collector/webstats/LogFileMap.java     | 26 -----------
 .../collector/webstats/SanitizeWeblogs.java        |  2 +-
 .../collector/webstats/WebServerAccessLogImpl.java | 38 +---------------
 6 files changed, 4 insertions(+), 155 deletions(-)

diff --git a/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java
index 0f862b4..6afb97e 100644
--- a/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java
+++ b/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java
@@ -7,9 +7,6 @@ import org.torproject.descriptor.WebServerAccessLog;
 import org.torproject.metrics.collector.webstats.FileType;
 import org.torproject.metrics.collector.webstats.InternalWebServerAccessLog;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
 import java.time.format.DateTimeFormatter;
@@ -19,8 +16,6 @@ public class WebServerAccessLogPersistence
 
   public static final String SEP = InternalWebServerAccessLog.SEP;
   public static final FileType COMPRESSION = FileType.XZ;
-  private static final Logger log
-      = LoggerFactory.getLogger(WebServerAccessLogPersistence.class);
 
   private DateTimeFormatter yearPattern = DateTimeFormatter.ofPattern("yyyy");
   private DateTimeFormatter monthPattern = DateTimeFormatter.ofPattern("MM");
diff --git a/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java b/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java
index 3a8a1f0..ac7d689 100644
--- a/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java
+++ b/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java
@@ -3,7 +3,6 @@
 
 package org.torproject.metrics.collector.webstats;
 
-import org.torproject.descriptor.DescriptorParseException;
 import org.torproject.descriptor.LogDescriptor;
 
 /**
@@ -17,23 +16,6 @@ public interface InternalLogDescriptor extends LogDescriptor {
   String SEP = "_";
 
   /**
-   * Validate log lines.
-   *
-   * @since 2.2.0
-   */
-  void validate() throws DescriptorParseException;
-
-  /**
-   * Set the {@code Validator} that will perform the validation on log
-   * lines.
-   *
-   * <p>Usually set by the implementing class.</p>
-   *
-   * @since 2.2.0
-   */
-  void setValidator(Validator validator);
-
-  /**
    * Set the descriptor's bytes.
    *
    * @since 2.2.0
@@ -42,22 +24,5 @@ public interface InternalLogDescriptor extends LogDescriptor {
 
   /** Return the descriptor's preferred compression. */
   String getCompressionType();
-
-  /**
-   * Provides a single function for validating a single log line.
-   *
-   * @since 2.2.0
-   */
-  interface Validator {
-
-    /**
-     * Verifies a log line.
-     *
-     * @since 2.2.0
-     */
-    boolean validate(String line);
-
-  }
-
 }
 
diff --git a/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java b/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java
index d13c85a..c0b0c69 100644
--- a/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java
+++ b/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java
@@ -3,25 +3,17 @@
 
 package org.torproject.metrics.collector.webstats;
 
-import org.torproject.descriptor.Descriptor;
 import org.torproject.descriptor.DescriptorParseException;
 import org.torproject.descriptor.LogDescriptor;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.stream.Collectors;
 
 /**
  * Base class for log descriptors.
@@ -34,11 +26,6 @@ public abstract class LogDescriptorImpl
   /** The log's file name should contain this string. */
   public static final String MARKER = ".log";
 
-  private static final int unrecognizedLinesLimit = 3;
-
-  private static final Logger log
-      = LoggerFactory.getLogger(LogDescriptorImpl.class);
-
   private static Pattern filenamePattern = Pattern.compile(
       "(?:\\S*)" + MARKER + SEP + "(?:[0-9a-zA-Z]*)(?:\\.?)([a-zA-Z2]*)");
 
@@ -51,8 +38,6 @@ public abstract class LogDescriptorImpl
 
   private List<String> unrecognizedLines = new ArrayList<>();
 
-  private Validator validator = (String line) -> true;
-
   /**
    * This constructor performs basic operations on the given bytes.
    *
@@ -63,8 +48,7 @@ public abstract class LogDescriptorImpl
    * @since 2.2.0
    */
   protected LogDescriptorImpl(byte[] logBytes, File descriptorFile,
-       String logName, FileType defaultCompression)
-       throws DescriptorParseException {
+      String logName) throws DescriptorParseException {
     this.logBytes = logBytes;
     this.descriptorFile = descriptorFile;
     try {
@@ -75,7 +59,7 @@ public abstract class LogDescriptorImpl
       }
       this.fileType = FileType.findType(mat.group(1).toUpperCase());
       if (FileType.PLAIN == this.fileType) {
-        this.fileType = defaultCompression;
+        this.fileType = FileType.XZ;
         this.logBytes = this.fileType.compress(this.logBytes);
       }
     } catch (Exception ex) {
@@ -95,39 +79,6 @@ public abstract class LogDescriptorImpl
   }
 
   @Override
-  public void validate() throws DescriptorParseException {
-    try (BufferedReader br = new BufferedReader(
-        new InputStreamReader(decompressedByteStream()))) {
-      this.unrecognizedLines.addAll(br.lines().parallel().filter((line)
-          -> null != line && !line.isEmpty() && !validator.validate(line))
-          .limit(unrecognizedLinesLimit).collect(Collectors.toList()));
-    } catch (Exception ex) {
-      throw new DescriptorParseException("Cannot validate log lines.", ex);
-    }
-  }
-
-  /**
-   * Assemble a LogDescriptor.
-   *
-   * @since 2.2.0
-   */
-  public static List<Descriptor> parse(byte[] logBytes,
-      File descriptorFile, String logName) throws DescriptorParseException {
-    if (logName.contains(InternalWebServerAccessLog.MARKER)) {
-      return Arrays.asList(new Descriptor[]{
-          new WebServerAccessLogImpl(logBytes, descriptorFile, logName)});
-    } else {
-      throw new DescriptorParseException("Cannot parse file " + logName
-          + " from file " + descriptorFile.getName());
-    }
-  }
-
-  @Override
-  public void setValidator(Validator validator) {
-    this.validator = validator;
-  }
-
-  @Override
   public String getCompressionType() {
     return this.fileType.name().toLowerCase();
   }
diff --git a/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java b/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java
index 2036b1f..d3e7dcd 100644
--- a/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java
+++ b/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java
@@ -15,7 +15,6 @@ import java.nio.file.attribute.BasicFileAttributes;
 import java.time.LocalDate;
 import java.util.Optional;
 import java.util.TreeMap;
-import java.util.stream.Stream;
 
 /**
  * Processes the given path and stores metadata for log files.
@@ -74,30 +73,5 @@ public class LogFileMap
           k -> new TreeMap<>());
     physicalHosts.put(metadata.date, metadata);
   }
-
-  /**
-   * Takes the given metadata and returns the LogMetadata for the entry
-   * of the next day.
-   */
-  public Optional<LogMetadata> nextDayLogFor(LogMetadata metadata) {
-    TreeMap<String, TreeMap<LocalDate, LogMetadata>> virtualHosts
-        = this.get(metadata.virtualHost);
-    if (null == virtualHosts) {
-      return Optional.empty();
-    }
-    TreeMap<LocalDate, LogMetadata> physicalHosts
-        = virtualHosts.get(metadata.physicalHost);
-    if (null == physicalHosts) {
-      return Optional.empty();
-    }
-    return Optional.ofNullable(physicalHosts.get(metadata.date.plusDays(1)));
-  }
-
-  /** Returns a stream of all contained log metadata. */
-  public Stream<LogMetadata> metadataStream() {
-    return this.values().stream()
-        .flatMap((virtualHosts) -> virtualHosts.values().stream())
-        .flatMap((physicalHosts) -> physicalHosts.values().stream());
-  }
 }
 
diff --git a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
index 74bd741..b2b4c49 100644
--- a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
+++ b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
@@ -147,7 +147,7 @@ public class SanitizeWeblogs extends CollecTorMain {
       WebServerAccessLogPersistence walp
           = new WebServerAccessLogPersistence(
           new WebServerAccessLogImpl(toCompressedBytes(retainedLines),
-          new File(name), name, false));
+          new File(name), name));
       log.debug("Storing {}.", name);
       walp.storeOut(this.outputPathName);
       walp.storeRecent(this.recentPathName);
diff --git a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java
index 2ca608a..548fbf6 100644
--- a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java
+++ b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java
@@ -6,9 +6,6 @@ package org.torproject.metrics.collector.webstats;
 import org.torproject.descriptor.DescriptorParseException;
 import org.torproject.descriptor.WebServerAccessLog;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.InputStreamReader;
@@ -32,9 +29,6 @@ import java.util.stream.Stream;
 public class WebServerAccessLogImpl extends LogDescriptorImpl
     implements InternalWebServerAccessLog, WebServerAccessLog {
 
-  private static final Logger log
-      = LoggerFactory.getLogger(WebServerAccessLogImpl.class);
-
   /** The log's name should include this string. */
   public static final String MARKER = InternalWebServerAccessLog.MARKER;
 
@@ -49,8 +43,6 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl
 
   private final LocalDate logDate;
 
-  private boolean validate = true;
-
   /**
    * Creates a WebServerAccessLog from the given bytes and filename.
    *
@@ -68,30 +60,7 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl
    */
   protected WebServerAccessLogImpl(byte[] logBytes, File file, String logName)
       throws DescriptorParseException {
-    this(logBytes, file, logName, FileType.XZ);
-  }
-
-  /** For internal use only. */
-  public WebServerAccessLogImpl(byte[] bytes, String filename,
-      boolean validate) throws DescriptorParseException {
-    this(bytes, null, filename, FileType.XZ, validate);
-  }
-
-  /** For internal use only. */
-  public WebServerAccessLogImpl(byte[] bytes, File sourceFile, String filename,
-      boolean validate) throws DescriptorParseException {
-    this(bytes, sourceFile, filename, FileType.XZ, validate);
-  }
-
-  private WebServerAccessLogImpl(byte[] logBytes, File file, String logName,
-      FileType defaultCompression) throws DescriptorParseException {
-    this(logBytes, file, logName, defaultCompression, true);
-  }
-
-  private WebServerAccessLogImpl(byte[] logBytes, File file, String logName,
-      FileType defaultCompression, boolean validate)
-      throws DescriptorParseException {
-    super(logBytes, file, logName, defaultCompression);
+    super(logBytes, file, logName);
     try {
       String fn = Paths.get(logName).getFileName().toString();
       Matcher mat = filenamePattern.matcher(fn);
@@ -108,11 +77,6 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl
       }
       String ymd = mat.group(3);
       this.logDate = LocalDate.parse(ymd, DateTimeFormatter.BASIC_ISO_DATE);
-      this.setValidator((line)
-          -> WebServerAccessLogLine.makeLine(line).isValid());
-      if (validate) {
-        this.validate();
-      }
     } catch (DescriptorParseException dpe) {
       throw dpe; // escalate
     } catch (Exception pe) {



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits