[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [metrics-web/master] Remove psql dependency for producing torperf.csv.
commit 9c22116574b8dd79e82bb8c59314bce569794f71
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Sun Jan 10 17:38:49 2016 +0100
Remove psql dependency for producing torperf.csv.
---
modules/legacy/db/tordir.sql | 24 ---
.../legacy/src/org/torproject/ernie/cron/Main.java | 2 +-
.../ernie/cron/performance/TorperfProcessor.java | 167 +++++---------------
shared/bin/50-run-legacy-stats.sh | 1 -
4 files changed, 41 insertions(+), 153 deletions(-)
diff --git a/modules/legacy/db/tordir.sql b/modules/legacy/db/tordir.sql
index e5f02f3..2c97829 100644
--- a/modules/legacy/db/tordir.sql
+++ b/modules/legacy/db/tordir.sql
@@ -807,20 +807,6 @@ CREATE TABLE dirreq_stats (
PRIMARY KEY (source, statsend, seconds, country)
);
--- TABLE torperf_stats
--- Quantiles and medians of daily torperf results.
-CREATE TABLE torperf_stats (
- "date" DATE NOT NULL,
- source CHARACTER VARYING(32) NOT NULL,
- q1 INTEGER NOT NULL,
- md INTEGER NOT NULL,
- q3 INTEGER NOT NULL,
- timeouts INTEGER NOT NULL,
- failures INTEGER NOT NULL,
- requests INTEGER NOT NULL,
- CONSTRAINT torperf_stats_pkey PRIMARY KEY("date", source)
-);
-
-- Refresh all statistics in the database.
CREATE OR REPLACE FUNCTION refresh_all() RETURNS INTEGER AS $$
BEGIN
@@ -966,16 +952,6 @@ UNION ALL
current_date - 3)
ORDER BY 1, 2, 3;
--- View for exporting torperf statistics.
-CREATE VIEW stats_torperf AS
-SELECT date, CASE WHEN source LIKE '%-50kb' THEN 50 * 1024
- WHEN source LIKE '%-1mb' THEN 1024 * 1024
- WHEN source LIKE '%-5mb' THEN 5 * 1024 * 1024 END AS size,
- CASE WHEN source NOT LIKE 'all-%'
- THEN split_part(source, '-', 1) END AS source, q1, md, q3, timeouts,
- failures, requests FROM torperf_stats WHERE date < current_date - 1
- ORDER BY 1, 2, 3;
-
-- View for exporting connbidirect statistics.
CREATE VIEW stats_connbidirect AS
SELECT DATE(statsend) AS date, source, belownum AS below, readnum AS read,
diff --git a/modules/legacy/src/org/torproject/ernie/cron/Main.java b/modules/legacy/src/org/torproject/ernie/cron/Main.java
index 5d561a6..9bd2d34 100644
--- a/modules/legacy/src/org/torproject/ernie/cron/Main.java
+++ b/modules/legacy/src/org/torproject/ernie/cron/Main.java
@@ -88,7 +88,7 @@ public class Main {
// Import and process torperf stats
if (config.getImportWriteTorperfStats()) {
new TorperfProcessor(new File(config.getTorperfDirectory()),
- statsDirectory, config.getRelayDescriptorDatabaseJDBC());
+ statsDirectory);
}
// Remove lock file
diff --git a/modules/legacy/src/org/torproject/ernie/cron/performance/TorperfProcessor.java b/modules/legacy/src/org/torproject/ernie/cron/performance/TorperfProcessor.java
index d7322db..ed3a0af 100644
--- a/modules/legacy/src/org/torproject/ernie/cron/performance/TorperfProcessor.java
+++ b/modules/legacy/src/org/torproject/ernie/cron/performance/TorperfProcessor.java
@@ -8,21 +8,12 @@ import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.SortedMap;
import java.util.TimeZone;
import java.util.TreeMap;
@@ -36,8 +27,7 @@ import org.torproject.descriptor.DescriptorSourceFactory;
import org.torproject.descriptor.TorperfResult;
public class TorperfProcessor {
- public TorperfProcessor(File torperfDirectory, File statsDirectory,
- String connectionURL) {
+ public TorperfProcessor(File torperfDirectory, File statsDirectory) {
if (torperfDirectory == null || statsDirectory == null) {
throw new IllegalArgumentException();
@@ -45,7 +35,7 @@ public class TorperfProcessor {
Logger logger = Logger.getLogger(TorperfProcessor.class.getName());
File rawFile = new File(statsDirectory, "torperf-raw");
- File statsFile = new File(statsDirectory, "torperf-stats");
+ File statsFile = new File(statsDirectory, "torperf.csv");
SortedMap<String, String> rawObs = new TreeMap<String, String>();
SortedMap<String, String> stats = new TreeMap<String, String>();
int addedRawObs = 0;
@@ -76,7 +66,9 @@ public class TorperfProcessor {
BufferedReader br = new BufferedReader(new FileReader(statsFile));
String line = br.readLine(); // ignore header
while ((line = br.readLine()) != null) {
- String key = line.split(",")[0] + "," + line.split(",")[1];
+ String[] parts = line.split(",");
+ String key = String.format("%s,%s,%s", parts[0], parts[1],
+ parts[2]);
stats.put(key, line);
}
br.close();
@@ -165,18 +157,27 @@ public class TorperfProcessor {
long q1 = dlTimes.get(dlTimes.size() / 4 - 1);
long md = dlTimes.get(dlTimes.size() / 2 - 1);
long q3 = dlTimes.get(dlTimes.size() * 3 / 4 - 1);
- stats.put(tempSourceDate, tempSourceDate + "," + q1 + ","
- + md + "," + q3 + "," + timeouts + "," + failures + ","
- + requests);
- String allSourceDate = "all" + tempSourceDate.substring(
- tempSourceDate.indexOf("-"));
- if (dlTimesAllSources.containsKey(allSourceDate)) {
- dlTimesAllSources.get(allSourceDate).addAll(dlTimes);
+ String[] tempParts = tempSourceDate.split("[-,]", 3);
+ String tempDate = tempParts[2];
+ int tempSize = Integer.parseInt(
+ tempParts[1].substring(0, tempParts[1].length() - 2))
+ * 1024 * (tempParts[1].endsWith("mb") ? 1024 : 1);
+ String tempSource = tempParts[0];
+ String tempDateSizeSource = String.format("%s,%d,%s",
+ tempDate, tempSize, tempSource);
+ stats.put(tempDateSizeSource,
+ String.format("%s,%s,%s,%s,%s,%s,%s",
+ tempDateSizeSource, q1, md, q3, timeouts, failures,
+ requests));
+ String allDateSizeSource = String.format("%s,%d,",
+ tempDate, tempSize);
+ if (dlTimesAllSources.containsKey(allDateSizeSource)) {
+ dlTimesAllSources.get(allDateSizeSource).addAll(dlTimes);
} else {
- dlTimesAllSources.put(allSourceDate, dlTimes);
+ dlTimesAllSources.put(allDateSizeSource, dlTimes);
}
- if (statusesAllSources.containsKey(allSourceDate)) {
- long[] status = statusesAllSources.get(allSourceDate);
+ if (statusesAllSources.containsKey(allDateSizeSource)) {
+ long[] status = statusesAllSources.get(allDateSizeSource);
status[0] += timeouts;
status[1] += failures;
status[2] += requests;
@@ -185,7 +186,7 @@ public class TorperfProcessor {
status[0] = timeouts;
status[1] = failures;
status[2] = requests;
- statusesAllSources.put(allSourceDate, status);
+ statusesAllSources.put(allDateSizeSource, status);
}
}
dlTimes = new ArrayList<Long>();
@@ -212,19 +213,20 @@ public class TorperfProcessor {
bw.close();
for (Map.Entry<String, List<Long>> e :
dlTimesAllSources.entrySet()) {
- String allSourceDate = e.getKey();
+ String allDateSizeSource = e.getKey();
dlTimes = e.getValue();
Collections.sort(dlTimes);
long q1 = dlTimes.get(dlTimes.size() / 4 - 1);
long md = dlTimes.get(dlTimes.size() / 2 - 1);
long q3 = dlTimes.get(dlTimes.size() * 3 / 4 - 1);
- long[] status = statusesAllSources.get(allSourceDate);
+ long[] status = statusesAllSources.get(allDateSizeSource);
timeouts = status[0];
failures = status[1];
requests = status[2];
- stats.put(allSourceDate, allSourceDate + "," + q1 + "," + md
- + "," + q3 + "," + timeouts + "," + failures + ","
- + requests);
+ stats.put(allDateSizeSource,
+ String.format("%s,%s,%s,%s,%s,%s,%s",
+ allDateSizeSource, q1, md, q3, timeouts, failures,
+ requests));
}
logger.fine("Finished writing file " + rawFile.getAbsolutePath()
+ ".");
@@ -232,11 +234,18 @@ public class TorperfProcessor {
if (stats.size() > 0) {
logger.fine("Writing file " + statsFile.getAbsolutePath()
+ "...");
+ SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
+ dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String yesterday = dateFormat.format(System.currentTimeMillis()
+ - 86400000L);
statsFile.getParentFile().mkdirs();
BufferedWriter bw = new BufferedWriter(new FileWriter(statsFile));
- bw.append("source,date,q1,md,q3,timeouts,failures,requests\n");
+ bw.append("date,size,source,q1,md,q3,timeouts,failures,"
+ + "requests\n");
for (String s : stats.values()) {
- bw.append(s + "\n");
+ if (s.compareTo(yesterday) < 0) {
+ bw.append(s + "\n");
+ }
}
bw.close();
logger.fine("Finished writing file " + statsFile.getAbsolutePath()
@@ -273,102 +282,6 @@ public class TorperfProcessor {
dumpStats.append("\n" + lastSource + " " + lastKnownObservation);
}
logger.info(dumpStats.toString());
-
- /* Write results to database. */
- if (connectionURL != null) {
- try {
- Map<String, String> insertRows = new HashMap<String, String>();
- insertRows.putAll(stats);
- Set<String> updateRows = new HashSet<String>();
- Connection conn = DriverManager.getConnection(connectionURL);
- conn.setAutoCommit(false);
- Statement statement = conn.createStatement();
- ResultSet rs = statement.executeQuery(
- "SELECT date, source, q1, md, q3, timeouts, failures, "
- + "requests FROM torperf_stats");
- while (rs.next()) {
- String date = rs.getDate(1).toString();
- String source = rs.getString(2);
- String key = source + "," + date;
- if (insertRows.containsKey(key)) {
- String insertRow = insertRows.remove(key);
- String[] newStats = insertRow.split(",");
- long newQ1 = Long.parseLong(newStats[2]);
- long newMd = Long.parseLong(newStats[3]);
- long newQ3 = Long.parseLong(newStats[4]);
- long newTimeouts = Long.parseLong(newStats[5]);
- long newFailures = Long.parseLong(newStats[6]);
- long newRequests = Long.parseLong(newStats[7]);
- long oldQ1 = rs.getLong(3);
- long oldMd = rs.getLong(4);
- long oldQ3 = rs.getLong(5);
- long oldTimeouts = rs.getLong(6);
- long oldFailures = rs.getLong(7);
- long oldRequests = rs.getLong(8);
- if (newQ1 != oldQ1 || newMd != oldMd || newQ3 != oldQ3 ||
- newTimeouts != oldTimeouts ||
- newFailures != oldFailures ||
- newRequests != oldRequests) {
- updateRows.add(insertRow);
- }
- }
- }
- PreparedStatement psU = conn.prepareStatement(
- "UPDATE torperf_stats SET q1 = ?, md = ?, q3 = ?, "
- + "timeouts = ?, failures = ?, requests = ? "
- + "WHERE date = ? AND source = ?");
- for (String row : updateRows) {
- String[] newStats = row.split(",");
- String source = newStats[0];
- java.sql.Date date = java.sql.Date.valueOf(newStats[1]);
- long q1 = Long.parseLong(newStats[2]);
- long md = Long.parseLong(newStats[3]);
- long q3 = Long.parseLong(newStats[4]);
- long timeouts = Long.parseLong(newStats[5]);
- long failures = Long.parseLong(newStats[6]);
- long requests = Long.parseLong(newStats[7]);
- psU.clearParameters();
- psU.setLong(1, q1);
- psU.setLong(2, md);
- psU.setLong(3, q3);
- psU.setLong(4, timeouts);
- psU.setLong(5, failures);
- psU.setLong(6, requests);
- psU.setDate(7, date);
- psU.setString(8, source);
- psU.executeUpdate();
- }
- PreparedStatement psI = conn.prepareStatement(
- "INSERT INTO torperf_stats (q1, md, q3, timeouts, failures, "
- + "requests, date, source) VALUES (?, ?, ?, ?, ?, ?, ?, ?)");
- for (String row : insertRows.values()) {
- String[] newStats = row.split(",");
- String source = newStats[0];
- java.sql.Date date = java.sql.Date.valueOf(newStats[1]);
- long q1 = Long.parseLong(newStats[2]);
- long md = Long.parseLong(newStats[3]);
- long q3 = Long.parseLong(newStats[4]);
- long timeouts = Long.parseLong(newStats[5]);
- long failures = Long.parseLong(newStats[6]);
- long requests = Long.parseLong(newStats[7]);
- psI.clearParameters();
- psI.setLong(1, q1);
- psI.setLong(2, md);
- psI.setLong(3, q3);
- psI.setLong(4, timeouts);
- psI.setLong(5, failures);
- psI.setLong(6, requests);
- psI.setDate(7, date);
- psI.setString(8, source);
- psI.executeUpdate();
- }
- conn.commit();
- conn.close();
- } catch (SQLException e) {
- logger.log(Level.WARNING, "Failed to add torperf stats to "
- + "database.", e);
- }
- }
}
}
diff --git a/shared/bin/50-run-legacy-stats.sh b/shared/bin/50-run-legacy-stats.sh
index b2b9f1a..75fe66c 100755
--- a/shared/bin/50-run-legacy-stats.sh
+++ b/shared/bin/50-run-legacy-stats.sh
@@ -5,7 +5,6 @@ psql -U metrics tordir -c 'SELECT * FROM refresh_all();'
mkdir -p stats
psql -c 'COPY (SELECT * FROM stats_servers) TO STDOUT WITH CSV HEADER;' tordir > stats/servers.csv
psql -c 'COPY (SELECT * FROM stats_bandwidth) TO STDOUT WITH CSV HEADER;' tordir > stats/bandwidth.csv
-psql -c 'COPY (SELECT * FROM stats_torperf) TO STDOUT WITH CSV HEADER;' tordir > stats/torperf.csv
psql -c 'COPY (SELECT * FROM stats_connbidirect) TO STDOUT WITH CSV HEADER;' tordir > stats/connbidirect.csv
cd ../../
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits