[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[or-cvs] [metrics-db/master] Make secrets persistent.
commit 1bdf6a5f78012539ae58a506c8b5e337ea43c1fe
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Wed Feb 9 17:46:10 2011 +0100
Make secrets persistent.
---
.../ernie/db/SanitizedBridgesWriter.java | 146 ++++++++++++++++----
1 files changed, 120 insertions(+), 26 deletions(-)
diff --git a/src/org/torproject/ernie/db/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/SanitizedBridgesWriter.java
index 0d08b76..122aae8 100644
--- a/src/org/torproject/ernie/db/SanitizedBridgesWriter.java
+++ b/src/org/torproject/ernie/db/SanitizedBridgesWriter.java
@@ -167,12 +167,16 @@ public class SanitizedBridgesWriter {
private boolean replaceIPAddressesWithHashes;
+ private boolean persistenceProblemWithSecrets;
+
private SortedMap<String, byte[]> secretsForHashingIPAddresses;
private String bridgeDescriptorMappingsCutOffTimestamp;
private boolean haveWarnedAboutLimitedMapping;
+ private File bridgeIpSecretsFile;
+
/**
* Initializes this class, including reading in the known descriptor
* mapping.
@@ -199,9 +203,48 @@ public class SanitizedBridgesWriter {
DescriptorMapping>();
this.descriptorPublicationTimes = new TreeSet<String>();
- /* Read secrets for replacing IP addresses with hashes from disk. */
- // TODO actually implement reading from disk
+ /* Read hex-encoded secrets for replacing IP addresses with hashes
+ * from disk. */
this.secretsForHashingIPAddresses = new TreeMap<String, byte[]>();
+ this.bridgeIpSecretsFile = new File(statsDirectory,
+ "bridge-ip-secrets");
+ if (this.bridgeIpSecretsFile.exists()) {
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.bridgeIpSecretsFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (line.length() != ("yyyy-MM,".length() + 31 * 2) ||
+ line.split(",").length != 2) {
+ this.logger.warning("Invalid line in bridge-ip-secrets file "
+ + "starting with '" + line.substring(0, 7) + "'! "
+ + "Not calculating any IP address hashes in this "
+ + "execution!");
+ this.persistenceProblemWithSecrets = true;
+ break;
+ }
+ String[] parts = line.split(",");
+ String month = parts[0];
+ byte[] secret = Hex.decodeHex(parts[1].toCharArray());
+ this.secretsForHashingIPAddresses.put(month, secret);
+ }
+ if (!this.persistenceProblemWithSecrets) {
+ this.logger.fine("Read "
+ + this.secretsForHashingIPAddresses.size() + " secrets for "
+ + "hashing bridge IP addresses.");
+ }
+ } catch (DecoderException e) {
+ this.logger.log(Level.WARNING, "Failed to decode hex string in "
+ + this.bridgeIpSecretsFile + "! Not calculating any IP "
+ + "address hashes in this execution!", e);
+ this.persistenceProblemWithSecrets = true;
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed to read "
+ + this.bridgeIpSecretsFile + "! Not calculating any IP "
+ + "address hashes in this execution!", e);
+ this.persistenceProblemWithSecrets = true;
+ }
+ }
/* If we're configured to keep descriptor mappings only for a limited
* time, define the cut-off day and time. */
@@ -258,8 +301,13 @@ public class SanitizedBridgesWriter {
}
private String scrubAddress(String address, byte[] fingerprintBytes,
- String published) {
+ String published) throws IOException {
if (this.replaceIPAddressesWithHashes) {
+ if (this.persistenceProblemWithSecrets) {
+ /* There's a persistence problem, so we shouldn't scrub more IP
+ * addresses in this execution. */
+ return null;
+ }
byte[] hashInput = new byte[4 + 20 + 31];
String[] ipParts = address.split("\\.");
for (int i = 0; i < 4; i++) {
@@ -270,17 +318,33 @@ public class SanitizedBridgesWriter {
if (!this.secretsForHashingIPAddresses.containsKey(month)) {
// TODO implement generating secrets using a secure random
// generator
- this.secretsForHashingIPAddresses.put(month,
- ("secret for hashing IPs: " + month).getBytes());
+ byte[] secret = ("secret for hashing IPs: " + month).getBytes();
if (month.compareTo(
this.bridgeDescriptorMappingsCutOffTimestamp) < 0) {
this.logger.warning("Generated a secret that we won't make "
+ "persistent, because it's outside our bridge descriptors "
+ "mapping interval.");
} else {
- // TODO append secrets to file on disk immediately before using
- // it, or we might end with inconsistently sanitized bridges
+ /* Append secret to file on disk immediately before using it, or
+ * we might end with inconsistently sanitized bridges. */
+ try {
+ if (!this.bridgeIpSecretsFile.exists()) {
+ this.bridgeIpSecretsFile.getParentFile().mkdirs();
+ }
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.bridgeIpSecretsFile,
+ this.bridgeIpSecretsFile.exists()));
+ bw.write(month + "," + Hex.encodeHexString(secret) + "\n");
+ bw.close();
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not store new secret "
+ + "to disk! Not calculating any IP address hashes in "
+ + "this execution!", e);
+ this.persistenceProblemWithSecrets = true;
+ throw new IOException(e);
+ }
}
+ this.secretsForHashingIPAddresses.put(month, secret);
}
byte[] secret = this.secretsForHashingIPAddresses.get(month);
System.arraycopy(secret, 0, hashInput, 24, 31);
@@ -302,6 +366,12 @@ public class SanitizedBridgesWriter {
public void sanitizeAndStoreNetworkStatus(byte[] data,
String publicationTime) {
+ if (this.persistenceProblemWithSecrets) {
+ /* There's a persistence problem, so we shouldn't scrub more IP
+ * addresses in this execution. */
+ return;
+ }
+
if (this.bridgeDescriptorMappingsCutOffTimestamp.
compareTo(publicationTime) > 0) {
this.logger.log(!this.haveWarnedAboutLimitedMapping ? Level.WARNING
@@ -356,9 +426,14 @@ public class SanitizedBridgesWriter {
String sdi = Base64.encodeBase64String(Hex.decodeHex(
mapping.serverDescriptorIdentifier.toCharArray())).
substring(0, 27);
- String scrubbedAddress = scrubAddress(address,
- Base64.decodeBase64(bridgeIdentity + "=="),
- descPublicationTime);
+ String scrubbedAddress = null;
+ try {
+ scrubbedAddress = scrubAddress(address,
+ Base64.decodeBase64(bridgeIdentity + "=="),
+ descPublicationTime);
+ } catch (IOException e) {
+ return;
+ }
if (scrubbed.length() > 0) {
String scrubbedLine = scrubbed.toString();
scrubbedLines.put(scrubbedLine.split(" ")[2], scrubbedLine);
@@ -444,6 +519,12 @@ public class SanitizedBridgesWriter {
*/
public void sanitizeAndStoreServerDescriptor(byte[] data) {
+ if (this.persistenceProblemWithSecrets) {
+ /* There's a persistence problem, so we shouldn't scrub more IP
+ * addresses in this execution. */
+ return;
+ }
+
/* Parse descriptor to generate a sanitized version and to look it up
* in the descriptor mapping. */
String scrubbedDesc = null;
@@ -511,8 +592,14 @@ public class SanitizedBridgesWriter {
fingerprint.toCharArray());
hashedBridgeIdentity = DigestUtils.shaHex(fingerprintBytes).
toLowerCase();
- scrubbedAddress = scrubAddress(address, fingerprintBytes,
- published);
+ try {
+ scrubbedAddress = scrubAddress(address, fingerprintBytes,
+ published);
+ } catch (IOException e) {
+ /* There's a persistence problem, so we shouldn't scrub more
+ * IP addresses in this execution. */
+ return;
+ }
scrubbed.append("opt fingerprint");
for (int i = 0; i < hashedBridgeIdentity.length() / 4; i++)
scrubbed.append(" " + hashedBridgeIdentity.substring(4 * i,
@@ -1171,22 +1258,29 @@ public class SanitizedBridgesWriter {
if (!this.secretsForHashingIPAddresses.isEmpty() &&
this.secretsForHashingIPAddresses.firstKey().compareTo(
this.bridgeDescriptorMappingsCutOffTimestamp) < 0) {
- int kept = 0, deleted = 0;
- SortedMap<String, byte[]> secretsStoredOnDisk =
- new TreeMap<String, byte[]>();
- for (Map.Entry<String, byte[]> e :
- this.secretsForHashingIPAddresses.entrySet()) {
- if (e.getKey().compareTo(
- this.bridgeDescriptorMappingsCutOffTimestamp) < 0) {
- deleted++;
- } else {
- secretsStoredOnDisk.put(e.getKey(), e.getValue());
- kept++;
+ try {
+ int kept = 0, deleted = 0;
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.bridgeIpSecretsFile));
+ for (Map.Entry<String, byte[]> e :
+ this.secretsForHashingIPAddresses.entrySet()) {
+ if (e.getKey().compareTo(
+ this.bridgeDescriptorMappingsCutOffTimestamp) < 0) {
+ deleted++;
+ } else {
+ bw.write(e.getKey() + "," + Hex.encodeHexString(e.getValue())
+ + "\n");
+ kept++;
+ }
}
+ bw.close();
+ this.logger.info("Deleted " + deleted + " secrets that we don't "
+ + "need anymore and kept " + kept + ".");
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not store reduced set of "
+ + "secrets to disk! This is a bad sign, better check what's "
+ + "going on!", e);
}
- // TODO write reduced set of secrets to disk
- this.logger.info("Deleted " + deleted + " secrets that we don't "
- + "need anymore and kept " + kept + ".");
}
}
}