[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [onionoo/release] Replace Gson with Jackson.



commit 73f8dedfc994dd6476570798c576060bcffc6fa6
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Wed May 23 10:15:36 2018 +0200

    Replace Gson with Jackson.
    
    Update metrics-lib dependency to 2.4.0.
    
    Adapt touched logging statements to our standards.
    
    Implements #25848.
---
 CHANGELOG.md                                       |  6 +++
 build.xml                                          |  6 ++-
 .../org/torproject/onionoo/docs/DetailsStatus.java |  6 +--
 .../org/torproject/onionoo/docs/DocumentStore.java | 57 +++++++++++++-------
 .../torproject/onionoo/docs/SummaryDocument.java   | 63 ++++++++++------------
 .../torproject/onionoo/server/ResponseBuilder.java | 26 ++++++---
 .../onionoo/server/ResourceServletTest.java        | 20 +++++--
 .../onionoo/writer/GraphHistoryCompilerTest.java   | 17 ++++--
 8 files changed, 126 insertions(+), 75 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 199cf6f..7a792fc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,9 @@
+# Changes in version 6.0-1.??.? - 2018-05-??
+
+ * Medium changes
+   - Replace Gson with Jackson.
+
+
 # Changes in version 6.0-1.13.0 - 2018-04-17
 
  * Medium changes
diff --git a/build.xml b/build.xml
index 557297d..7d752e2 100644
--- a/build.xml
+++ b/build.xml
@@ -11,7 +11,7 @@
   <property name="onionoo.protocol.version" value="6.0"/>
   <property name="release.version"
             value="${onionoo.protocol.version}-1.13.0-dev"/>
-  <property name="metricslibversion" value="2.2.0"/>
+  <property name="metricslibversion" value="2.4.0"/>
   <property name="jetty.version" value="-9.2.21.v20170120" />
   <property name="warfile"
             value="onionoo-${release.version}.war"/>
@@ -47,7 +47,9 @@
     <include name="commons-codec-1.10.jar"/>
     <include name="commons-compress-1.13.jar"/>
     <include name="commons-lang3-3.5.jar"/>
-    <include name="gson-2.4.jar"/>
+    <include name="jackson-annotations-2.8.6.jar"/>
+    <include name="jackson-core-2.8.6.jar"/>
+    <include name="jackson-databind-2.8.6.jar"/>
     <include name="logback-classic-1.1.9.jar"/>
     <include name="logback-core-1.1.9.jar"/>
     <include name="slf4j-api-1.7.22.jar"/>
diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java
index f838ec0..7ee0bb6 100644
--- a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java
+++ b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java
@@ -34,12 +34,12 @@ public class DetailsStatus extends Document {
   private String desc_published;
 
   public void setDescPublished(Long descPublished) {
-    this.desc_published = DateTimeHelper.format(descPublished);
+    this.desc_published = null == descPublished ? null
+        : DateTimeHelper.format(descPublished);
   }
 
   public Long getDescPublished() {
-    return this.desc_published == null ? null :
-        DateTimeHelper.parse(this.desc_published);
+    return DateTimeHelper.parse(this.desc_published);
   }
 
   private String last_restarted;
diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java
index f1f3803..3fac5c9 100644
--- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java
+++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java
@@ -5,9 +5,12 @@ package org.torproject.onionoo.docs;
 
 import org.torproject.onionoo.util.FormattingUtils;
 
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonParseException;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.PropertyNamingStrategy;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -41,6 +44,12 @@ public class DocumentStore {
   private static Logger log = LoggerFactory.getLogger(
       DocumentStore.class);
 
+  private static ObjectMapper objectMapper = new ObjectMapper()
+      .setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE)
+      .setSerializationInclusion(JsonInclude.Include.NON_EMPTY)
+      .setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE)
+      .setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
+
   private final File statusDir = new File("status");
 
   private File outDir = null;
@@ -162,12 +171,11 @@ public class DocumentStore {
         String line = null;
         try (BufferedReader br = new BufferedReader(new FileReader(
             summaryFile))) {
-          Gson gson = new Gson();
           while ((line = br.readLine()) != null) {
             if (line.length() == 0) {
               continue;
             }
-            SummaryDocument summaryDocument = gson.fromJson(line,
+            SummaryDocument summaryDocument = objectMapper.readValue(line,
                 SummaryDocument.class);
             if (summaryDocument != null) {
               parsedSummaryDocuments.put(summaryDocument.getFingerprint(),
@@ -178,11 +186,8 @@ public class DocumentStore {
           this.listedFiles += parsedSummaryDocuments.size();
           this.listOperations++;
         } catch (IOException e) {
-          log.error("Could not read file '"
-              + summaryFile.getAbsolutePath() + "'.", e);
-        } catch (JsonParseException e) {
-          log.error("Could not parse summary document '" + line
-              + "' in file '" + summaryFile.getAbsolutePath() + "'.", e);
+          log.error("Could not parse summary document '{}' from file '{}'.",
+              line, summaryFile.getAbsolutePath(), e);
         }
       }
     }
@@ -303,13 +308,15 @@ public class DocumentStore {
         || document instanceof WeightsDocument
         || document instanceof ClientsDocument
         || document instanceof UptimeDocument) {
-      Gson gson = new Gson();
-      documentString = gson.toJson(document);
+      try {
+        documentString = objectMapper.writeValueAsString(document);
+      } catch (JsonProcessingException e) {
+        log.error("Serializing failed for type {}.",
+            document.getClass().getName(), e);
+        return false;
+      }
     } else if (document instanceof DetailsStatus
         || document instanceof DetailsDocument) {
-      /* Don't escape HTML characters, like < and >, contained in
-       * strings. */
-      Gson gson = new GsonBuilder().disableHtmlEscaping().create();
       /* We must ensure that details files only contain ASCII characters
        * and no UTF-8 characters.  While UTF-8 characters are perfectly
        * valid in JSON, this would break compatibility with existing files
@@ -317,7 +324,14 @@ public class DocumentStore {
        * objects are escaped JSON, e.g., \u00F2.  When Gson serlializes
        * this string, it escapes the \ to \\, hence writes \\u00F2.  We
        * need to undo this and change \\u00F2 back to \u00F2. */
-      documentString = FormattingUtils.replaceValidUtf(gson.toJson(document));
+      try {
+        documentString = FormattingUtils.replaceValidUtf(
+            objectMapper.writeValueAsString(document));
+      } catch (JsonProcessingException e) {
+        log.error("Serializing failed for type {}.",
+            document.getClass().getName(), e);
+        return false;
+      }
       /* Existing details statuses don't contain opening and closing curly
        * brackets, so we should remove them from new details statuses,
        * too. */
@@ -536,9 +550,8 @@ public class DocumentStore {
   private <T extends Document> T retrieveParsedDocumentFile(
       Class<T> documentType, String documentString) {
     T result = null;
-    Gson gson = new Gson();
     try {
-      result = gson.fromJson(documentString, documentType);
+      result = objectMapper.readValue(documentString, documentType);
     } catch (Throwable e) {
       /* Handle below. */
       log.error(documentString);
@@ -777,10 +790,14 @@ public class DocumentStore {
       return;
     }
     StringBuilder sb = new StringBuilder();
-    Gson gson = new Gson();
     for (SummaryDocument summaryDocument :
         this.cachedSummaryDocuments.values()) {
-      String line = gson.toJson(summaryDocument);
+      String line = null;
+      try {
+        line = objectMapper.writeValueAsString(summaryDocument);
+      } catch (JsonProcessingException e) {
+        line = null;
+      }
       if (line != null) {
         sb.append(line + "\n");
       } else {
diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java
index 42a5a64..31cd84c 100644
--- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java
+++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java
@@ -3,8 +3,8 @@
 
 package org.torproject.onionoo.docs;
 
-import com.google.gson.annotations.Expose;
-import com.google.gson.annotations.SerializedName;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.commons.codec.DecoderException;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.codec.binary.Hex;
@@ -20,8 +20,7 @@ import java.util.regex.Pattern;
 
 public class SummaryDocument extends Document {
 
-  @Expose
-  @SerializedName("t")
+  @JsonProperty("t")
   private boolean isRelay;
 
   public void setRelay(boolean isRelay) {
@@ -32,8 +31,7 @@ public class SummaryDocument extends Document {
     return this.isRelay;
   }
 
-  @Expose
-  @SerializedName("f")
+  @JsonProperty("f")
   private String fingerprint;
 
   /** Sets the fingerprint to the given 40 hex characters and clears
@@ -57,6 +55,7 @@ public class SummaryDocument extends Document {
     return this.fingerprint;
   }
 
+  @JsonIgnore
   private transient String hashedFingerprint = null;
 
   /** Returns the SHA1-hashed fingerprint, or <code>null</code> if no
@@ -73,6 +72,7 @@ public class SummaryDocument extends Document {
     return this.hashedFingerprint;
   }
 
+  @JsonIgnore
   private transient String base64Fingerprint = null;
 
   /** Returns the base64-encoded fingerprint, or <code>null</code> if no
@@ -89,6 +89,7 @@ public class SummaryDocument extends Document {
     return this.base64Fingerprint;
   }
 
+  @JsonIgnore
   private transient String[] fingerprintSortedHexBlocks = null;
 
   /** Returns a sorted array containing blocks of 4 upper-case hex
@@ -109,8 +110,7 @@ public class SummaryDocument extends Document {
     return this.fingerprintSortedHexBlocks;
   }
 
-  @Expose
-  @SerializedName("n")
+  @JsonProperty("n")
   private String nickname;
 
   @SuppressWarnings("checkstyle:javadocmethod")
@@ -126,8 +126,7 @@ public class SummaryDocument extends Document {
     return this.nickname == null ? "Unnamed" : this.nickname;
   }
 
-  @Expose
-  @SerializedName("ad")
+  @JsonProperty("ad")
   private String[] addresses;
 
   public void setAddresses(List<String> addresses) {
@@ -169,8 +168,7 @@ public class SummaryDocument extends Document {
     return sortedSet;
   }
 
-  @Expose
-  @SerializedName("cc")
+  @JsonProperty("cc")
   private String countryCode;
 
   public void setCountryCode(String countryCode) {
@@ -181,8 +179,7 @@ public class SummaryDocument extends Document {
     return this.countryCode;
   }
 
-  @Expose
-  @SerializedName("as")
+  @JsonProperty("as")
   private String asNumber;
 
   public void setAsNumber(String asNumber) {
@@ -193,8 +190,7 @@ public class SummaryDocument extends Document {
     return this.asNumber;
   }
 
-  @Expose
-  @SerializedName("fs")
+  @JsonProperty("fs")
   private String firstSeenMillis;
 
   public void setFirstSeenMillis(long firstSeenMillis) {
@@ -205,8 +201,7 @@ public class SummaryDocument extends Document {
     return DateTimeHelper.parse(this.firstSeenMillis);
   }
 
-  @Expose
-  @SerializedName("ls")
+  @JsonProperty("ls")
   private String lastSeenMillis;
 
   public void setLastSeenMillis(long lastSeenMillis) {
@@ -217,8 +212,7 @@ public class SummaryDocument extends Document {
     return DateTimeHelper.parse(this.lastSeenMillis);
   }
 
-  @Expose
-  @SerializedName("rf")
+  @JsonProperty("rf")
   private String[] relayFlags;
 
   public void setRelayFlags(SortedSet<String> relayFlags) {
@@ -229,8 +223,7 @@ public class SummaryDocument extends Document {
     return this.stringArrayToSortedSet(this.relayFlags);
   }
 
-  @Expose
-  @SerializedName("cw")
+  @JsonProperty("cw")
   private long consensusWeight;
 
   public void setConsensusWeight(long consensusWeight) {
@@ -241,8 +234,7 @@ public class SummaryDocument extends Document {
     return this.consensusWeight;
   }
 
-  @Expose
-  @SerializedName("r")
+  @JsonProperty("r")
   private boolean running;
 
   public void setRunning(boolean isRunning) {
@@ -253,8 +245,7 @@ public class SummaryDocument extends Document {
     return this.running;
   }
 
-  @Expose
-  @SerializedName("c")
+  @JsonProperty("c")
   private String contact;
 
   @SuppressWarnings("checkstyle:javadocmethod")
@@ -273,8 +264,7 @@ public class SummaryDocument extends Document {
   /* This attribute can go away once all Onionoo services had their hourly
    * updater write effective families to summary documents at least once.
    * Remove this code after September 8, 2015. */
-  @Expose
-  @SerializedName("ff")
+  @JsonProperty("ff")
   private String[] familyFingerprints;
 
   public void setFamilyFingerprints(
@@ -286,8 +276,7 @@ public class SummaryDocument extends Document {
     return this.stringArrayToSortedSet(this.familyFingerprints);
   }
 
-  @Expose
-  @SerializedName("ef")
+  @JsonProperty("ef")
   private String[] effectiveFamily;
 
   public void setEffectiveFamily(SortedSet<String> effectiveFamily) {
@@ -298,8 +287,7 @@ public class SummaryDocument extends Document {
     return this.stringArrayToSortedSet(this.effectiveFamily);
   }
 
-  @Expose
-  @SerializedName("v")
+  @JsonProperty("v")
   private String version;
 
   public void setVersion(String version) {
@@ -310,8 +298,7 @@ public class SummaryDocument extends Document {
     return this.version;
   }
 
-  @Expose
-  @SerializedName("h")
+  @JsonProperty("h")
   private String hostName;
 
   public void setHostName(String hostName) {
@@ -322,8 +309,7 @@ public class SummaryDocument extends Document {
     return this.hostName;
   }
 
-  @Expose
-  @SerializedName("rv")
+  @JsonProperty("rv")
   private Boolean recommendedVersion;
 
   public void setRecommendedVersion(Boolean recommendedVersion) {
@@ -334,6 +320,11 @@ public class SummaryDocument extends Document {
     return this.recommendedVersion;
   }
 
+  /** Instantiate an empty summary document. */
+  public SummaryDocument() {
+    /* empty */
+  }
+
   /* The familyFingerprints parameter can go away after September 8, 2015.
    * See above. */
   /** Instantiates a summary document with all given properties. */
diff --git a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java
index 64e23ed..1dcfeeb 100644
--- a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java
+++ b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java
@@ -14,8 +14,12 @@ import org.torproject.onionoo.docs.UptimeDocument;
 import org.torproject.onionoo.docs.WeightsDocument;
 import org.torproject.onionoo.util.FormattingUtils;
 
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.PropertyNamingStrategy;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -31,6 +35,12 @@ public class ResponseBuilder {
   private static final Logger logger =
       LoggerFactory.getLogger(ResponseBuilder.class);
 
+  private static ObjectMapper objectMapper = new ObjectMapper()
+      .setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE)
+      .setSerializationInclusion(JsonInclude.Include.NON_EMPTY)
+      .setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE)
+      .setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
+
   private DocumentStore documentStore;
   private String buildRevision;
 
@@ -343,13 +353,15 @@ public class ResponseBuilder {
             dd.setVersionStatus(detailsDocument.getVersionStatus());
           }
         }
-        /* Don't escape HTML characters, like < and >, contained in
-         * strings. */
-        Gson gson = new GsonBuilder().disableHtmlEscaping().create();
-        /* Whenever we provide Gson with a string containing an escaped
+        /* Whenever we provide Jackson with a string containing an escaped
          * non-ASCII character like \u00F2, it escapes the \ to \\, which
          * we need to undo before including the string in a response. */
-        return FormattingUtils.replaceValidUtf(gson.toJson(dd));
+        try {
+          return FormattingUtils.replaceValidUtf(
+              objectMapper.writeValueAsString(dd));
+        } catch (JsonProcessingException e) {
+          return "";
+        }
       } else {
         // TODO We should probably log that we didn't find a details
         // document that we expected to exist.
diff --git a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java
index c371d71..d64d59d 100644
--- a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java
+++ b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java
@@ -13,7 +13,11 @@ import org.torproject.onionoo.docs.DocumentStoreFactory;
 import org.torproject.onionoo.docs.DummyDocumentStore;
 import org.torproject.onionoo.docs.UpdateStatus;
 
-import com.google.gson.Gson;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.PropertyNamingStrategy;
 
 import org.junit.Before;
 import org.junit.Test;
@@ -36,6 +40,12 @@ import java.util.TreeSet;
  * which tests servlet specifics. */
 public class ResourceServletTest {
 
+  private static ObjectMapper objectMapper = new ObjectMapper()
+      .setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE)
+      .setSerializationInclusion(JsonInclude.Include.NON_EMPTY)
+      .setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE)
+      .setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
+
   private SortedMap<String, org.torproject.onionoo.docs.SummaryDocument> relays;
   private SortedMap<String, org.torproject.onionoo.docs.SummaryDocument>
       bridges;
@@ -263,11 +273,10 @@ public class ResourceServletTest {
     rs.doGet(this.request, this.response, TEST_TIME);
   }
 
-  private void parseResponse() {
+  private void parseResponse() throws IOException {
     this.responseString = this.response.getWrittenContent();
     if (this.responseString != null) {
-      Gson gson = new Gson();
-      this.summaryDocument = gson.fromJson(this.responseString,
+      this.summaryDocument = objectMapper.readValue(this.responseString,
           SummaryDocument.class);
     }
   }
@@ -344,6 +353,9 @@ public class ResourceServletTest {
 
   @SuppressWarnings("MemberName")
   private static class SummaryDocument {
+    private String version;
+    private String next_major_version_scheduled;
+    private String build_revision;
     private String relays_published;
     private int relays_skipped;
     private int relays_truncated;
diff --git a/src/test/java/org/torproject/onionoo/writer/GraphHistoryCompilerTest.java b/src/test/java/org/torproject/onionoo/writer/GraphHistoryCompilerTest.java
index 6d9c461..b055426 100644
--- a/src/test/java/org/torproject/onionoo/writer/GraphHistoryCompilerTest.java
+++ b/src/test/java/org/torproject/onionoo/writer/GraphHistoryCompilerTest.java
@@ -9,7 +9,11 @@ import static org.junit.Assert.assertNotNull;
 import org.torproject.onionoo.docs.DateTimeHelper;
 import org.torproject.onionoo.docs.GraphHistory;
 
-import com.google.gson.Gson;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.PropertyNamingStrategy;
 
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -17,6 +21,7 @@ import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameter;
 import org.junit.runners.Parameterized.Parameters;
 
+import java.io.IOException;
 import java.time.Period;
 import java.util.Arrays;
 import java.util.Collection;
@@ -25,6 +30,12 @@ import java.util.Map;
 @RunWith(Parameterized.class)
 public class GraphHistoryCompilerTest {
 
+  private static ObjectMapper objectMapper = new ObjectMapper()
+      .setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE)
+      .setSerializationInclusion(JsonInclude.Include.NON_EMPTY)
+      .setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE)
+      .setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
+
   /** Provide test data. */
   @Parameters
   public static Collection<Object[]> data() {
@@ -156,7 +167,7 @@ public class GraphHistoryCompilerTest {
       DateTimeHelper.TEN_DAYS };
 
   @Test
-  public void test() {
+  public void test() throws IOException {
     GraphHistoryCompiler ghc = new GraphHistoryCompiler(DateTimeHelper.parse(
         "2018-01-01 00:00:00"));
     ghc.setDivisible(this.divisible);
@@ -172,7 +183,7 @@ public class GraphHistoryCompilerTest {
     Map<String, GraphHistory> compiledGraphHistories =
         ghc.compileGraphHistories();
     String message = this.testDescription + "; "
-        + new Gson().toJson(compiledGraphHistories);
+        + objectMapper.writeValueAsString(compiledGraphHistories);
     assertEquals(message, this.expectedGraphs, compiledGraphHistories.size());
     if (null != this.expectedGraphName) {
       GraphHistory gh = compiledGraphHistories.get(this.expectedGraphName);



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits