[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[or-cvs] [metrics-web/master 2/2] Remove file system support for relay search.
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Thu, 14 Oct 2010 10:37:50 +0200
Subject: Remove file system support for relay search.
Commit: c3743d8377254aaa349877a7934807ab4e89957f
---
.../torproject/ernie/web/RelaySearchServlet.java | 440 +++++---------------
1 files changed, 115 insertions(+), 325 deletions(-)
diff --git a/src/org/torproject/ernie/web/RelaySearchServlet.java b/src/org/torproject/ernie/web/RelaySearchServlet.java
index 2471a16..c938bba 100644
--- a/src/org/torproject/ernie/web/RelaySearchServlet.java
+++ b/src/org/torproject/ernie/web/RelaySearchServlet.java
@@ -84,8 +84,6 @@ public class RelaySearchServlet extends HttpServlet {
}
private void writeHeader(PrintWriter out) throws IOException {
-
-
out.println("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0 "
+ "Transitional//EN\">\n"
+ "<html>\n"
@@ -173,27 +171,15 @@ public class RelaySearchServlet extends HttpServlet {
PrintWriter out = new PrintWriter(response.getWriter(), true);
writeHeader(out);
- /* If we don't have a database, see if we have consensus on disk. */
- SortedSet<File> consensusDirectories = new TreeSet<File>();
+ /* Check if we have a database connection. */
if (conn == null) {
- /* Check if we have a consensuses directory. */
- File consensusDirectory = new File(CONSENSUS_DIRECTORY);
- if (consensusDirectory.exists() && consensusDirectory.isDirectory()) {
- for (File yearFile : consensusDirectory.listFiles()) {
- for (File monthFile : yearFile.listFiles()) {
- consensusDirectories.add(monthFile);
- }
- }
- }
- if (consensusDirectories.isEmpty()) {
- out.println("<p><font color=\"red\"><b>Warning: </b></font>This "
- + "server doesn't have any relay lists available. If this "
- + "problem persists, please "
- + "<a href=\"mailto:tor-assistants@xxxxxxxxxxxxx\";>let us "
- + "know</a>!</p>\n");
- writeFooter(out);
- return;
- }
+ out.println("<p><font color=\"red\"><b>Warning: </b></font>This "
+ + "server doesn't have any relay lists available. If this "
+ + "problem persists, please "
+ + "<a href=\"mailto:tor-assistants@xxxxxxxxxxxxx\";>let us "
+ + "know</a>!</p>\n");
+ writeFooter(out);
+ return;
}
/* Read search parameter, if any. */
@@ -451,329 +437,133 @@ public class RelaySearchServlet extends HttpServlet {
out.write(" ...</p>\n");
out.flush();
- /* If we have a database connection, search relays in the database. */
- if (conn != null) {
-
- StringBuilder query = new StringBuilder("SELECT validafter, "
- + "rawdesc FROM statusentry WHERE ");
- boolean addAnd = false;
- if (searchDayTimestamps.size() > 0 ||
- searchMonthTimestamps.size() > 0) {
- boolean addOr = false;
- query.append("(");
- for (long searchTimestamp : searchDayTimestamps) {
- query.append((addOr ? "OR " : "") + "(validafter >= '"
- + dateTimeFormat.format(searchTimestamp) + "' AND "
- + "validafter < '" + dateTimeFormat.format(searchTimestamp
- + 24L * 60L * 60L * 1000L) + "') ");
- addOr = true;
- }
- for (long searchTimestamp : searchMonthTimestamps) {
- Calendar firstOfNextMonth = Calendar.getInstance(
- TimeZone.getTimeZone("UTC"));
- firstOfNextMonth.setTimeInMillis(searchTimestamp);
- firstOfNextMonth.add(Calendar.MONTH, 1);
- query.append((addOr ? "OR " : "") + "(validafter >= '"
- + dateTimeFormat.format(searchTimestamp) + "' AND "
- + "validafter < '" + dateTimeFormat.format(
- firstOfNextMonth.getTimeInMillis()) + "') ");
- addOr = true;
- }
- query.append(") ");
- } else {
- query.append("validafter >= '" + dateTimeFormat.format(
- started - 30L * 24L * 60L * 60L * 1000L) + "' ");
- }
- if (searchNickname.length() > 0) {
- query.append("AND LOWER(nickname) LIKE '"
- + searchNickname.toLowerCase() + "%' ");
- }
- if (searchFingerprint.length() > 0) {
- query.append("AND fingerprint LIKE '"
- + searchFingerprint.toLowerCase() + "%' ");
+ /* Search relays in the database. */
+ StringBuilder query = new StringBuilder("SELECT validafter, rawdesc "
+ + "FROM statusentry WHERE ");
+ boolean addAnd = false;
+ if (searchDayTimestamps.size() > 0 ||
+ searchMonthTimestamps.size() > 0) {
+ boolean addOr = false;
+ query.append("(");
+ for (long searchTimestamp : searchDayTimestamps) {
+ query.append((addOr ? "OR " : "") + "(validafter >= '"
+ + dateTimeFormat.format(searchTimestamp) + "' AND "
+ + "validafter < '" + dateTimeFormat.format(searchTimestamp
+ + 24L * 60L * 60L * 1000L) + "') ");
+ addOr = true;
}
- if (searchIPAddress.length() > 0) {
- query.append("AND address LIKE '" + searchIPAddress + "%' ");
+ for (long searchTimestamp : searchMonthTimestamps) {
+ Calendar firstOfNextMonth = Calendar.getInstance(
+ TimeZone.getTimeZone("UTC"));
+ firstOfNextMonth.setTimeInMillis(searchTimestamp);
+ firstOfNextMonth.add(Calendar.MONTH, 1);
+ query.append((addOr ? "OR " : "") + "(validafter >= '"
+ + dateTimeFormat.format(searchTimestamp) + "' AND "
+ + "validafter < '" + dateTimeFormat.format(
+ firstOfNextMonth.getTimeInMillis()) + "') ");
+ addOr = true;
}
- for (String search : searchFingerprintOrNickname) {
- query.append("AND (LOWER(nickname) LIKE '" + search.toLowerCase()
- + "%' OR fingerprint LIKE '" + search.toLowerCase() + "%') ");
- }
- query.append("ORDER BY validafter DESC, fingerprint LIMIT 31");
- out.println("<!-- " + query.toString() + " -->");
- int matches = 0;
- long startedQuery = System.currentTimeMillis();
- try {
- Statement statement = conn.createStatement();
- ResultSet rs = statement.executeQuery(query.toString());
- String lastValidAfter = null;
- while (rs.next()) {
- matches++;
- if (matches > 30) {
- break;
- }
- String validAfter = rs.getTimestamp(1).toString().
- substring(0, 19);
- if (!validAfter.equals(lastValidAfter)) {
- out.println(" <br><tt>valid-after "
- + "<a href=\"consensus?valid-after="
- + validAfter.replaceAll(":", "-").replaceAll(" ", "-")
- + "\" target=\"_blank\">" + validAfter + "</a></tt><br>");
- lastValidAfter = validAfter;
- out.flush();
- }
- byte[] rawStatusEntry = rs.getBytes(2);
- try {
- String statusEntryLines = new String(rawStatusEntry,
- "US-ASCII");
- String[] lines = statusEntryLines.split("\n");
- for (String line : lines) {
- if (line.startsWith("r ")) {
- String[] parts = line.split(" ");
- String descriptor = String.format("%040x",
- new BigInteger(1, Base64.decodeBase64(parts[3]
- + "==")));
- out.println(" <tt>r " + parts[1] + " " + parts[2] + " "
- + "<a href=\"descriptor.html?desc-id=" + descriptor
- + "\" target=\"_blank\">" + parts[3] + "</a> "
- + parts[4] + " " + parts[5] + " " + parts[6] + " "
- + parts[7] + " " + parts[8] + "</tt><br>");
- } else {
- out.println(" <tt>" + line + "</tt><br>");
- }
- }
- out.println(" <br>");
- out.flush();
- } catch (UnsupportedEncodingException e) {
- /* This shouldn't happen, because we know that ASCII is
- * supported. */
- }
- }
- statement.close();
- } catch (SQLException e) {
- out.println("<p><font color=\"red\"><b>Warning: </b></font>We "
- + "experienced an unknown database problem while running the "
- + "search. The query was '" + query + "'. If this problem "
- + "persists, please "
- + "<a href=\"mailto:tor-assistants@xxxxxxxxxxxxx\";>let us "
- + "know</a>!</p>\n");
- writeFooter(out);
- return;
- }
-
- /* Display total search time on the results page. */
- long searchTime = System.currentTimeMillis() - started;
- long queryTime = System.currentTimeMillis() - startedQuery;
- out.write(" <br><p>Found " + (matches > 30 ? "more than 30"
- : "" + matches) + " relays " + (matches > 30 ?
- "(displaying only the first 30 hits) " : "") + "in "
- + String.format("%d.%03d", searchTime / 1000, searchTime % 1000)
- + " seconds.</p>\n");
- if (searchTime > 10L * 1000L) {
- out.write(" <p>In theory, search time should not exceed "
- + "10 seconds. The query was '" + query + "'. If this or "
- + "similar searches remain slow, please "
- + "<a href=\"mailto:tor-assistants@xxxxxxxxxxxxx\";>let us "
- + "know</a>!</p>\n");
- }
-
- /* Finish writing response. */
- writeFooter(out);
- return;
+ query.append(") ");
+ } else {
+ query.append("validafter >= '" + dateTimeFormat.format(
+ started - 30L * 24L * 60L * 60L * 1000L) + "' ");
}
-
- /* Compile a regular expression pattern to parse r lines more
- * quickly. */
- StringBuilder patternBuilder = new StringBuilder("r ");
- if (searchNickname.length() > 0 || searchFingerprint.length() > 0) {
- if (searchNickname.length() > 0) {
- patternBuilder.append(searchNickname);
- }
- if (searchFingerprint.length() > 0) {
- try {
- patternBuilder.append(".*" + Base64.encodeBase64String(
- Hex.decodeHex((searchFingerprint
- + (searchFingerprint.length() % 2 == 1 ? "0" : "")).
- toCharArray())).substring(0, searchFingerprint.length() *
- 2 / 3));
- } catch (DecoderException e) {
- /* We make sure this exception is never thrown by passing an
- * even number of only hex characters to Hex.decodeHex(). */
- }
- }
- } else if (searchFingerprintOrNickname.size() > 0) {
- List<String> searchTermsCopy = new ArrayList<String>(
- searchFingerprintOrNickname);
- if (searchTermsCopy.size() < 2) {
- searchTermsCopy.add("");
- }
- patternBuilder.append("(");
- for (int i = 0; i < 2; i++) {
- patternBuilder.append(searchTermsCopy.get(i));
- String searchTerm = searchTermsCopy.get((i + 1) % 2);
- if (searchTerm.length() > 0) {
- try{
- patternBuilder.append(".*" + Base64.encodeBase64String(
- Hex.decodeHex((searchTerm + (searchTerm.length()
- % 2 == 1 ? "0" : "")).toCharArray())).substring(0,
- searchTerm.length() * 2 / 3));
- } catch (DecoderException e) {
- /* We make sure this exception is never thrown by passing an
- * even number of only hex characters to Hex.decodeHex(). */
- }
- }
- if (i == 0) {
- patternBuilder.append("|");
- }
- }
- patternBuilder.append(")");
+ if (searchNickname.length() > 0) {
+ query.append("AND LOWER(nickname) LIKE '"
+ + searchNickname.toLowerCase() + "%' ");
+ }
+ if (searchFingerprint.length() > 0) {
+ query.append("AND fingerprint LIKE '"
+ + searchFingerprint.toLowerCase() + "%' ");
}
if (searchIPAddress.length() > 0) {
- patternBuilder.append(".* " + searchIPAddress.replaceAll("\\.",
- "\\\\."));
+ query.append("AND address LIKE '" + searchIPAddress + "%' ");
}
- patternBuilder.append(".*");
- String pattern = patternBuilder.toString();
- Pattern searchPattern = Pattern.compile(pattern);
-
- /* While parsing, memorize the r lines of the last 24 parsed
- * consensuses, so that we don't have to parse them again. */
- Set<String> failedRLines = new HashSet<String>();
- List<Set<String>> addedFailedRLines = new ArrayList<Set<String>>();
-
- /* Parse consensus files from newest to oldest. Stop after either
- * parsing 31 * 24 consensuses, finding 30 hits, or running out of
- * consensuses. */
- SortedSet<File> consensusDirsToParse = new TreeSet<File>();
- consensusDirsToParse.addAll(consensusDirectories);
- SortedSet<File> consensusesToParse = new TreeSet<File>();
- int matches = 0, consensusesParsed = 0;
- while (consensusesParsed < 31 * 24 && matches < 30 &&
- !(consensusDirsToParse.isEmpty() &&
- consensusesToParse.isEmpty())) {
-
- /* Only put consensuses of one month in the queue at the same
- * time. */
- while (consensusesToParse.isEmpty() &&
- !consensusDirsToParse.isEmpty()) {
- Stack<File> parse = new Stack<File>();
- File dir = consensusDirsToParse.last();
- parse.add(dir);
- consensusDirsToParse.remove(dir);
- while (!parse.isEmpty()) {
- File pop = parse.remove(0);
- if (pop.isDirectory()) {
- for (File file : pop.listFiles()) {
- parse.add(file);
- }
- } else {
- consensusesToParse.add(pop);
- }
+ for (String search : searchFingerprintOrNickname) {
+ query.append("AND (LOWER(nickname) LIKE '" + search.toLowerCase()
+ + "%' OR fingerprint LIKE '" + search.toLowerCase() + "%') ");
+ }
+ query.append("ORDER BY validafter DESC, fingerprint LIMIT 31");
+ out.println("<!-- " + query.toString() + " -->");
+ int matches = 0;
+ long startedQuery = System.currentTimeMillis();
+ try {
+ Statement statement = conn.createStatement();
+ ResultSet rs = statement.executeQuery(query.toString());
+ String lastValidAfter = null;
+ while (rs.next()) {
+ matches++;
+ if (matches > 30) {
+ break;
}
- }
- if (consensusesToParse.isEmpty()) {
- break;
- }
-
- /* Parse consensus at the head of the queue. */
- File consensus = consensusesToParse.last();
- consensusesToParse.remove(consensus);
- BufferedReader br = new BufferedReader(new FileReader(consensus));
- String line = null, validAfterLine = null;
- Set<String> currentlyAddedFailedRLines = new HashSet<String>();
- addedFailedRLines.add(currentlyAddedFailedRLines);
- while ((line = br.readLine()) != null) {
- if (line.startsWith("r ")) {
-
- /* If we already know this r line doesn't match our regular
- * expression, ignore it. */
- if (failedRLines.contains(line)) {
-
- /* If we don't know this r line yet, but it doesn't match our
- * regular expression, memorize it. */
- } else if (!searchPattern.matcher(line).matches()) {
- currentlyAddedFailedRLines.add(line);
- failedRLines.add(line);
-
- /* If this r line matches our regular expression, compare fields
- * to be certain we want this relay. */
- } else {
- String[] parts = line.split(" ");
- String nickname = parts[1];
- String address = parts[6];
- if (searchNickname.length() > 0 &&
- !nickname.startsWith(searchNickname)) {
- continue;
- }
- if (searchIPAddress.length() > 0 &&
- !address.startsWith(searchIPAddress)) {
- continue;
- }
- String fingerprint = String.format("%040x", new BigInteger(1,
- Base64.decodeBase64(parts[2] + "=="))).toLowerCase();
- if (searchFingerprint.length() > 0 && !fingerprint.startsWith(
- searchFingerprint.toLowerCase())) {
- continue;
- }
- boolean skip = false;
- for (String searchTerm : searchFingerprintOrNickname) {
- if (!nickname.startsWith(searchTerm) &&
- !fingerprint.startsWith(searchTerm.toLowerCase())) {
- skip = true;
- break;
- }
- }
- if (skip) {
- continue;
- }
-
- /* This r line matches the search criteria. If this is the
- * first match in this consensus, print the valid-after
- * line. */
- if (validAfterLine != null) {
- out.println(" <br><tt>valid-after "
- + "<a href=\"consensus?valid-after="
- + validAfterLine.substring("valid-after ".length()).
- replaceAll(":", "-").replaceAll(" ", "-")
- + "\" target=\"_blank\">"
- + validAfterLine.substring("valid-after ".length())
- + "</a></tt><br>");
- validAfterLine = null;
+ String validAfter = rs.getTimestamp(1).toString().
+ substring(0, 19);
+ if (!validAfter.equals(lastValidAfter)) {
+ out.println(" <br><tt>valid-after "
+ + "<a href=\"consensus?valid-after="
+ + validAfter.replaceAll(":", "-").replaceAll(" ", "-")
+ + "\" target=\"_blank\">" + validAfter + "</a></tt><br>");
+ lastValidAfter = validAfter;
+ out.flush();
+ }
+ byte[] rawStatusEntry = rs.getBytes(2);
+ try {
+ String statusEntryLines = new String(rawStatusEntry,
+ "US-ASCII");
+ String[] lines = statusEntryLines.split("\n");
+ for (String line : lines) {
+ if (line.startsWith("r ")) {
+ String[] parts = line.split(" ");
+ String descriptor = String.format("%040x",
+ new BigInteger(1, Base64.decodeBase64(parts[3]
+ + "==")));
+ out.println(" <tt>r " + parts[1] + " " + parts[2] + " "
+ + "<a href=\"descriptor.html?desc-id=" + descriptor
+ + "\" target=\"_blank\">" + parts[3] + "</a> "
+ + parts[4] + " " + parts[5] + " " + parts[6] + " "
+ + parts[7] + " " + parts[8] + "</tt><br>");
+ } else {
+ out.println(" <tt>" + line + "</tt><br>");
}
-
- /* And print the r line. */
- String descriptor = String.format("%040x", new BigInteger(1,
- Base64.decodeBase64(parts[3] + "==")));
- out.println(" <tt>r " + parts[1] + " " + parts[2] + " "
- + "<a href=\"descriptor.html?desc-id=" + descriptor
- + "\" " + "target=\"_blank\">" + parts[3] + "</a> "
- + parts[4] + " " + parts[5] + " " + parts[6] + " "
- + parts[7] + " " + parts[8] + "</tt><br>");
- matches++;
}
- } else if (line.startsWith("valid-after ")) {
- validAfterLine = line;
+ out.println(" <br>");
+ out.flush();
+ } catch (UnsupportedEncodingException e) {
+ /* This shouldn't happen, because we know that ASCII is
+ * supported. */
}
}
- br.close();
- consensusesParsed++;
-
- /* Forget about failed r lines if they are 24 consensuses apart. */
- while (addedFailedRLines.size() >= 24) {
- Set<String> removeFailedRLines = addedFailedRLines.remove(0);
- failedRLines.removeAll(removeFailedRLines);
- }
+ statement.close();
+ } catch (SQLException e) {
+ out.println("<p><font color=\"red\"><b>Warning: </b></font>We "
+ + "experienced an unknown database problem while running the "
+ + "search. The query was '" + query + "'. If this problem "
+ + "persists, please "
+ + "<a href=\"mailto:tor-assistants@xxxxxxxxxxxxx\";>let us "
+ + "know</a>!</p>\n");
+ writeFooter(out);
+ return;
}
/* Display total search time on the results page. */
long searchTime = System.currentTimeMillis() - started;
- out.write(" <br><p>Found " + matches + " relays in the last "
- + consensusesParsed + " known consensuses in "
+ long queryTime = System.currentTimeMillis() - startedQuery;
+ out.write(" <br><p>Found " + (matches > 30 ? "more than 30"
+ : "" + matches) + " relays " + (matches > 30 ?
+ "(displaying only the first 30 hits) " : "") + "in "
+ String.format("%d.%03d", searchTime / 1000, searchTime % 1000)
+ " seconds.</p>\n");
+ if (searchTime > 10L * 1000L) {
+ out.write(" <p>In theory, search time should not exceed "
+ + "10 seconds. The query was '" + query + "'. If this or "
+ + "similar searches remain slow, please "
+ + "<a href=\"mailto:tor-assistants@xxxxxxxxxxxxx\";>let us "
+ + "know</a>!</p>\n");
+ }
/* Finish writing response. */
writeFooter(out);
+ return;
}
}
--
1.7.1