[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [metrics-web/master] Remove long unused code from legacy module.
commit ca5fa45df0cfb14801e3556caa93f2cc6d26d790
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Tue Nov 13 18:10:18 2018 +0100
Remove long unused code from legacy module.
This includes the lock file, the option to write raw output files for
importing into the database, and a couple boolean config options that
have always been true.
Required changes to existing legacy.config (removals):
ImportDirectoryArchives
KeepDirectoryArchiveImportHistory
WriteRelayDescriptorDatabase
WriteRelayDescriptorsRawFiles
RelayDescriptorRawFilesDirectory
Part of #28116.
---
.../metrics/stats/servers/Configuration.java | 46 +-------
.../torproject/metrics/stats/servers/LockFile.java | 61 ----------
.../org/torproject/metrics/stats/servers/Main.java | 33 +-----
.../servers/RelayDescriptorDatabaseImporter.java | 126 +--------------------
src/main/resources/legacy.config.template | 20 ----
5 files changed, 10 insertions(+), 276 deletions(-)
diff --git a/src/main/java/org/torproject/metrics/stats/servers/Configuration.java b/src/main/java/org/torproject/metrics/stats/servers/Configuration.java
index 76788df..b6ee397 100644
--- a/src/main/java/org/torproject/metrics/stats/servers/Configuration.java
+++ b/src/main/java/org/torproject/metrics/stats/servers/Configuration.java
@@ -24,21 +24,11 @@ public class Configuration {
private static Logger log = LoggerFactory.getLogger(Configuration.class);
- private boolean importDirectoryArchives = false;
-
private List<File> directoryArchivesDirectories = new ArrayList<>();
- private boolean keepDirectoryArchiveImportHistory = false;
-
- private boolean writeRelayDescriptorDatabase = false;
-
private String relayDescriptorDatabaseJdbc =
"jdbc:postgresql://localhost/tordir?user=metrics&password=password";
- private boolean writeRelayDescriptorsRawFiles = false;
-
- private String relayDescriptorRawFilesDirectory = "pg-import/";
-
/** Initializes this configuration class. */
public Configuration() {
@@ -51,24 +41,10 @@ public class Configuration {
String line = null;
try (BufferedReader br = new BufferedReader(new FileReader(configFile))) {
while ((line = br.readLine()) != null) {
- if (line.startsWith("ImportDirectoryArchives")) {
- this.importDirectoryArchives = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DirectoryArchivesDirectory")) {
+ if (line.startsWith("DirectoryArchivesDirectory")) {
this.directoryArchivesDirectories.add(new File(line.split(" ")[1]));
- } else if (line.startsWith("KeepDirectoryArchiveImportHistory")) {
- this.keepDirectoryArchiveImportHistory = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("WriteRelayDescriptorDatabase")) {
- this.writeRelayDescriptorDatabase = Integer.parseInt(
- line.split(" ")[1]) != 0;
} else if (line.startsWith("RelayDescriptorDatabaseJDBC")) {
this.relayDescriptorDatabaseJdbc = line.split(" ")[1];
- } else if (line.startsWith("WriteRelayDescriptorsRawFiles")) {
- this.writeRelayDescriptorsRawFiles = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("RelayDescriptorRawFilesDirectory")) {
- this.relayDescriptorRawFilesDirectory = line.split(" ")[1];
} else if (!line.startsWith("#") && line.length() > 0) {
log.error("Configuration file contains unrecognized "
+ "configuration key in line '{}'! Exiting!", line);
@@ -93,10 +69,6 @@ public class Configuration {
}
}
- public boolean getImportDirectoryArchives() {
- return this.importDirectoryArchives;
- }
-
/** Returns directories containing archived descriptors. */
public List<File> getDirectoryArchivesDirectories() {
if (this.directoryArchivesDirectories.isEmpty()) {
@@ -108,24 +80,8 @@ public class Configuration {
}
}
- public boolean getKeepDirectoryArchiveImportHistory() {
- return this.keepDirectoryArchiveImportHistory;
- }
-
- public boolean getWriteRelayDescriptorDatabase() {
- return this.writeRelayDescriptorDatabase;
- }
-
public String getRelayDescriptorDatabaseJdbc() {
return this.relayDescriptorDatabaseJdbc;
}
-
- public boolean getWriteRelayDescriptorsRawFiles() {
- return this.writeRelayDescriptorsRawFiles;
- }
-
- public String getRelayDescriptorRawFilesDirectory() {
- return this.relayDescriptorRawFilesDirectory;
- }
}
diff --git a/src/main/java/org/torproject/metrics/stats/servers/LockFile.java b/src/main/java/org/torproject/metrics/stats/servers/LockFile.java
deleted file mode 100644
index c6063d1..0000000
--- a/src/main/java/org/torproject/metrics/stats/servers/LockFile.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/* Copyright 2011--2018 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.metrics.stats.servers;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-
-public class LockFile {
-
- private File lockFile;
-
- private static Logger log = LoggerFactory.getLogger(LockFile.class);
-
- public LockFile() {
- this.lockFile = new File("lock");
- }
-
- /** Acquires the lock by checking whether a lock file already exists,
- * and if not, by creating one with the current system time as
- * content. */
- public boolean acquireLock() {
- log.debug("Trying to acquire lock...");
- try {
- if (this.lockFile.exists()) {
- BufferedReader br = new BufferedReader(new FileReader("lock"));
- long runStarted = Long.parseLong(br.readLine());
- br.close();
- if (System.currentTimeMillis() - runStarted
- < 23L * 60L * 60L * 1000L) {
- return false;
- }
- }
- BufferedWriter bw = new BufferedWriter(new FileWriter("lock"));
- bw.append("").append(String.valueOf(System.currentTimeMillis()))
- .append("\n");
- bw.close();
- log.debug("Acquired lock.");
- return true;
- } catch (IOException e) {
- log.warn("Caught exception while trying to acquire "
- + "lock!");
- return false;
- }
- }
-
- /** Releases the lock by deleting the lock file, if present. */
- public void releaseLock() {
- log.debug("Releasing lock...");
- this.lockFile.delete();
- log.debug("Released lock.");
- }
-}
-
diff --git a/src/main/java/org/torproject/metrics/stats/servers/Main.java b/src/main/java/org/torproject/metrics/stats/servers/Main.java
index 4d349bc..1454418 100644
--- a/src/main/java/org/torproject/metrics/stats/servers/Main.java
+++ b/src/main/java/org/torproject/metrics/stats/servers/Main.java
@@ -24,38 +24,15 @@ public class Main {
// Initialize configuration
Configuration config = new Configuration();
- // Use lock file to avoid overlapping runs
- LockFile lf = new LockFile();
- if (!lf.acquireLock()) {
- log.error("Warning: ERNIE is already running or has not exited "
- + "cleanly! Exiting!");
- System.exit(1);
- }
-
// Define stats directory for temporary files
File statsDirectory = new File("stats");
// Import relay descriptors
- if (config.getImportDirectoryArchives()) {
- RelayDescriptorDatabaseImporter rddi =
- config.getWriteRelayDescriptorDatabase()
- || config.getWriteRelayDescriptorsRawFiles()
- ? new RelayDescriptorDatabaseImporter(
- config.getWriteRelayDescriptorDatabase()
- ? config.getRelayDescriptorDatabaseJdbc() : null,
- config.getWriteRelayDescriptorsRawFiles()
- ? config.getRelayDescriptorRawFilesDirectory() : null,
- config.getDirectoryArchivesDirectories(),
- statsDirectory,
- config.getKeepDirectoryArchiveImportHistory()) : null;
- if (null != rddi) {
- rddi.importRelayDescriptors();
- rddi.closeConnection();
- }
- }
-
- // Remove lock file
- lf.releaseLock();
+ RelayDescriptorDatabaseImporter rddi = new RelayDescriptorDatabaseImporter(
+ config.getRelayDescriptorDatabaseJdbc(),
+ config.getDirectoryArchivesDirectories(), statsDirectory);
+ rddi.importRelayDescriptors();
+ rddi.closeConnection();
log.info("Terminating ERNIE.");
}
diff --git a/src/main/java/org/torproject/metrics/stats/servers/RelayDescriptorDatabaseImporter.java b/src/main/java/org/torproject/metrics/stats/servers/RelayDescriptorDatabaseImporter.java
index 2d1ae47..d1ae43c 100644
--- a/src/main/java/org/torproject/metrics/stats/servers/RelayDescriptorDatabaseImporter.java
+++ b/src/main/java/org/torproject/metrics/stats/servers/RelayDescriptorDatabaseImporter.java
@@ -10,15 +10,10 @@ import org.torproject.descriptor.ExtraInfoDescriptor;
import org.torproject.descriptor.NetworkStatusEntry;
import org.torproject.descriptor.RelayNetworkStatusConsensus;
-import org.postgresql.util.PGbytea;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
import java.sql.CallableStatement;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -95,21 +90,6 @@ public final class RelayDescriptorDatabaseImporter {
= LoggerFactory.getLogger(RelayDescriptorDatabaseImporter.class);
/**
- * Directory for writing raw import files.
- */
- private String rawFilesDirectory;
-
- /**
- * Raw import file containing status entries.
- */
- private BufferedWriter statusentryOut;
-
- /**
- * Raw import file containing bandwidth histories.
- */
- private BufferedWriter bwhistOut;
-
- /**
* Date format to parse timestamps.
*/
private SimpleDateFormat dateTimeFormat;
@@ -126,30 +106,24 @@ public final class RelayDescriptorDatabaseImporter {
*/
private Set<String> insertedStatusEntries = new HashSet<>();
- private boolean importIntoDatabase;
-
- private boolean writeRawImportFiles;
+ private boolean importIntoDatabase = true;
private List<File> archivesDirectories;
private File statsDirectory;
- private boolean keepImportHistory;
-
/**
* Initialize database importer by connecting to the database and
* preparing statements.
*/
public RelayDescriptorDatabaseImporter(String connectionUrl,
- String rawFilesDirectory, List<File> archivesDirectories,
- File statsDirectory, boolean keepImportHistory) {
+ List<File> archivesDirectories, File statsDirectory) {
if (archivesDirectories == null || statsDirectory == null) {
throw new IllegalArgumentException();
}
this.archivesDirectories = archivesDirectories;
this.statsDirectory = statsDirectory;
- this.keepImportHistory = keepImportHistory;
if (connectionUrl != null) {
try {
@@ -175,18 +149,11 @@ public final class RelayDescriptorDatabaseImporter {
this.psU = conn.prepareStatement("INSERT INTO scheduled_updates "
+ "(date) VALUES (?)");
this.scheduledUpdates = new HashSet<>();
- this.importIntoDatabase = true;
} catch (SQLException e) {
log.warn("Could not connect to database or prepare statements.", e);
}
}
- /* Remember where we want to write raw import files. */
- if (rawFilesDirectory != null) {
- this.rawFilesDirectory = rawFilesDirectory;
- this.writeRawImportFiles = true;
- }
-
/* Initialize date format, so that we can format timestamps. */
this.dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -278,53 +245,6 @@ public final class RelayDescriptorDatabaseImporter {
this.importIntoDatabase = false;
}
}
- if (this.writeRawImportFiles) {
- try {
- if (this.statusentryOut == null) {
- new File(rawFilesDirectory).mkdirs();
- this.statusentryOut = new BufferedWriter(new FileWriter(
- rawFilesDirectory + "/statusentry.sql"));
- this.statusentryOut.write(" COPY statusentry (validafter, "
- + "nickname, fingerprint, descriptor, published, address, "
- + "orport, dirport, isauthority, isbadExit, "
- + "isbaddirectory, isexit, isfast, isguard, ishsdir, "
- + "isnamed, isstable, isrunning, isunnamed, isvalid, "
- + "isv2dir, isv3dir, version, bandwidth, ports, rawdesc) "
- + "FROM stdin;\n");
- }
- this.statusentryOut.write(
- this.dateTimeFormat.format(validAfter) + "\t" + nickname
- + "\t" + fingerprint.toLowerCase() + "\t"
- + descriptor.toLowerCase() + "\t"
- + this.dateTimeFormat.format(published) + "\t" + address
- + "\t" + orPort + "\t" + dirPort + "\t"
- + (flags.contains("Authority") ? "t" : "f") + "\t"
- + (flags.contains("BadExit") ? "t" : "f") + "\t"
- + (flags.contains("BadDirectory") ? "t" : "f") + "\t"
- + (flags.contains("Exit") ? "t" : "f") + "\t"
- + (flags.contains("Fast") ? "t" : "f") + "\t"
- + (flags.contains("Guard") ? "t" : "f") + "\t"
- + (flags.contains("HSDir") ? "t" : "f") + "\t"
- + (flags.contains("Named") ? "t" : "f") + "\t"
- + (flags.contains("Stable") ? "t" : "f") + "\t"
- + (flags.contains("Running") ? "t" : "f") + "\t"
- + (flags.contains("Unnamed") ? "t" : "f") + "\t"
- + (flags.contains("Valid") ? "t" : "f") + "\t"
- + (flags.contains("V2Dir") ? "t" : "f") + "\t"
- + (flags.contains("V3Dir") ? "t" : "f") + "\t"
- + (version != null ? version : "\\N") + "\t"
- + (bandwidth >= 0 ? bandwidth : "\\N") + "\t"
- + (ports != null ? ports : "\\N") + "\t");
- this.statusentryOut.write(PGbytea.toPGString(rawDescriptor)
- .replaceAll("\\\\", "\\\\\\\\") + "\n");
- } catch (IOException e) {
- log.warn("Could not write network status "
- + "consensus entry to raw database import file. We won't "
- + "make any further attempts to write raw import files in "
- + "this execution.", e);
- this.writeRawImportFiles = false;
- }
- }
}
/**
@@ -552,26 +472,6 @@ public final class RelayDescriptorDatabaseImporter {
this.importIntoDatabase = false;
}
}
- if (this.writeRawImportFiles) {
- try {
- if (this.bwhistOut == null) {
- new File(rawFilesDirectory).mkdirs();
- this.bwhistOut = new BufferedWriter(new FileWriter(
- rawFilesDirectory + "/bwhist.sql"));
- }
- this.bwhistOut.write("SELECT insert_bwhist('" + fingerprint
- + "','" + lastDate + "','" + readIntArray.toString()
- + "','" + writtenIntArray.toString() + "','"
- + dirreadIntArray.toString() + "','"
- + dirwrittenIntArray.toString() + "');\n");
- } catch (IOException e) {
- log.warn("Could not write bandwidth "
- + "history to raw database import file. We won't make "
- + "any further attempts to write raw import files in "
- + "this execution.", e);
- this.writeRawImportFiles = false;
- }
- }
readArray = writtenArray = dirreadArray = dirwrittenArray = null;
}
if (historyLine.equals("EOL")) {
@@ -628,9 +528,7 @@ public final class RelayDescriptorDatabaseImporter {
reader.setMaxDescriptorsInQueue(10);
File historyFile = new File(statsDirectory,
"database-importer-relay-descriptor-history");
- if (keepImportHistory) {
- reader.setHistoryFile(historyFile);
- }
+ reader.setHistoryFile(historyFile);
for (Descriptor descriptor : reader.readDescriptors(
this.archivesDirectories.toArray(
new File[this.archivesDirectories.size()]))) {
@@ -641,9 +539,7 @@ public final class RelayDescriptorDatabaseImporter {
this.addExtraInfoDescriptor((ExtraInfoDescriptor) descriptor);
}
}
- if (keepImportHistory) {
- reader.saveHistoryFile(historyFile);
- }
+ reader.saveHistoryFile(historyFile);
}
log.info("Finished importing relay descriptors.");
@@ -728,20 +624,6 @@ public final class RelayDescriptorDatabaseImporter {
log.warn("Could not close database connection.", e);
}
}
-
- /* Close raw import files. */
- try {
- if (this.statusentryOut != null) {
- this.statusentryOut.write("\\.\n");
- this.statusentryOut.close();
- }
- if (this.bwhistOut != null) {
- this.bwhistOut.write("\\.\n");
- this.bwhistOut.close();
- }
- } catch (IOException e) {
- log.warn("Could not close one or more raw database import files.", e);
- }
}
}
diff --git a/src/main/resources/legacy.config.template b/src/main/resources/legacy.config.template
index 5475c1e..e2e0dac 100644
--- a/src/main/resources/legacy.config.template
+++ b/src/main/resources/legacy.config.template
@@ -1,28 +1,8 @@
-## Import directory archives from disk, if available
-#ImportDirectoryArchives 0
-#
## Relative paths to directories to import directory archives from
#DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/consensuses/
#DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/server-descriptors/
#DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/extra-infos/
#
-## Keep a history of imported directory archive files to know which files
-## have been imported before. This history can be useful when importing
-## from a changing source to avoid importing descriptors over and over
-## again, but it can be confusing to users who don't know about it.
-#KeepDirectoryArchiveImportHistory 0
-#
-## Write relay descriptors to the database
-#WriteRelayDescriptorDatabase 0
-#
## JDBC string for relay descriptor database
#RelayDescriptorDatabaseJDBC jdbc:postgresql://localhost/tordir?user=metrics&password=password
#
-## Write relay descriptors to raw text files for importing them into the
-## database using PostgreSQL's \copy command
-#WriteRelayDescriptorsRawFiles 0
-#
-## Relative path to directory to write raw text files; note that existing
-## files will be overwritten!
-#RelayDescriptorRawFilesDirectory pg-import/
-#
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits