[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [torperf/master] Truncate .data and .extradata files to contain only the last 4 days.



commit 178f753a451959cb78c6d0bf13a929ed1940a2e1
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Mon Mar 5 16:33:41 2012 +0100

    Truncate .data and .extradata files to contain only the last 4 days.
---
 extra_stats.py     |   57 +++++++++++++++++++++++++++++++++++----
 measurements-HOWTO |   18 ++++++++----
 truncate-data.py   |   75 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 138 insertions(+), 12 deletions(-)

diff --git a/extra_stats.py b/extra_stats.py
index 69bb8a5..662cf43 100755
--- a/extra_stats.py
+++ b/extra_stats.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python
 
-import sys, time
+import os, re, sys, time
 import TorCtl.TorUtil as TorUtil
 import TorCtl.TorCtl as TorCtl
 
@@ -22,15 +22,18 @@ class Circuit:
     self.stream_fail_reason = None
 
 class WriteStats(TorCtl.PostEventListener):
-  def __init__(self, port, filename):
+  def __init__(self, port, filename, truncate):
     TorCtl.PostEventListener.__init__(self)
     self._port = int(port)
     self._filename = filename
+    self.truncate = truncate
+    self.first_launched = None
     self._conn = None
     self.all_circs = {}
     self.ignore_streams = {}
     self.current_timeout = None
     self.current_quantile = None
+    self.truncate_statsfile()
 
   def connect(self):
     self._conn = TorCtl.connect(HOST, self._port)
@@ -52,6 +55,7 @@ class WriteStats(TorCtl.PostEventListener):
     self.current_quantile = b.cutoff_quantile
     result = b.event_name + " " +b.body
     self.write_result(result)
+    self.truncate_statsfile()
 
   def circ_status_event(self, c):
     if c.status == "LAUNCHED":
@@ -94,6 +98,7 @@ class WriteStats(TorCtl.PostEventListener):
          (self.current_timeout, self.current_quantile)
 
     self.write_result(result)
+    self.truncate_statsfile()
 
   def stream_status_event(self, event):
     if event.status == "NEW":
@@ -140,19 +145,59 @@ class WriteStats(TorCtl.PostEventListener):
 
   def write_result(self, result):
     # XXX: hrmm. seems wasteful to keep opening+closing..
+    # XXX: When changing this, also change truncated_statsfile().
     statsfile = open(self._filename, 'a')
     statsfile.write(result+"\n")
     statsfile.close()
 
+  def truncate_statsfile(self):
+    if not self.truncate:
+      return
+    launched_str = "^.*LAUNCH=([\\d]*).*$"
+    if not self.first_launched:
+      if os.path.isfile(self._filename):
+        launched_re = re.compile(launched_str)
+        with open(self._filename) as statsfile:
+          for line in statsfile:
+            m = launched_re.match(line)
+            if m:
+              self.first_launched = int(m.group(1))
+              break
+      if not self.first_launched:
+        self.first_launched = time.time()
+    now = time.time()
+    if self.first_launched < now - 7 * 24 * 60 * 60:
+      copylines = False
+      statsfilebak_path = self._filename + ".bak"
+      statsfilebak_file = open(statsfilebak_path, "w")
+      launched_re = re.compile(launched_str)
+      with open(self._filename) as statsfile:
+        for line in statsfile:
+          if copylines:
+            statsfilebak_file.write(line)
+          else:
+            m = launched_re.match(line)
+            if m and int(m.group(1)) >= now - 4 * 24 * 60 * 60:
+              statsfilebak_file.write(line)
+              copylines = True
+      statsfilebak_file.close()
+      os.rename(statsfilebak_path, self._filename)
+
 def main():
-  if len(sys.argv) < 3:
+  if len(sys.argv) < 3 or len(sys.argv) > 4:
     print "Bad arguments"
     sys.exit(1)
 
-  port = sys.argv[1]
-  filename = sys.argv[2]
+  truncate = False
+  if (sys.argv[1] == "--truncate"):
+    truncate = True
+    port = sys.argv[2]
+    filename = sys.argv[3]
+  else:
+    port = sys.argv[1]
+    filename = sys.argv[2]
 
-  stats = WriteStats(port, filename)
+  stats = WriteStats(port, filename, truncate)
   stats.connect()
   stats.setup_listener()
   try:
diff --git a/measurements-HOWTO b/measurements-HOWTO
index 67de0c5..2cefb68 100644
--- a/measurements-HOWTO
+++ b/measurements-HOWTO
@@ -103,14 +103,17 @@ cd ~/torperf/torclient50kb && tor -f ~/torperf/torclient50kb/torrc
 cd ~/torperf/torclient1mb && tor -f ~/torperf/torclient1mb/torrc
 cd ~/torperf/torclient5mb && tor -f ~/torperf/torclient5mb/torrc
 sleep 5
-cd ~/torperf/torclient50kb && python ../extra_stats.py 10020
+cd ~/torperf/torclient50kb && python ../extra_stats.py --truncate 10020
   ../50kb.extradata &
-cd ~/torperf/torclient1mb && python ../extra_stats.py 10021
+cd ~/torperf/torclient1mb && python ../extra_stats.py --truncate 10021
   ../1mb.extradata &
-cd ~/torperf/torclient5mb && python ../extra_stats.py 10022
+cd ~/torperf/torclient5mb && python ../extra_stats.py --truncate 10022
   ../5mb.extradata &
 EOF
 
+(Omit the --truncate switch if you don't want .extradata files to be
+truncated once per week to contain only the last 4 days of data.)
+
 $ chmod a+x start-tors
 $ ./start-tors
 
@@ -125,13 +128,16 @@ $ crontab -e
 
 */5 * * * * timeout -s2 295 ~/torperf/trivsocks-client
   torperf.torproject.org 127.0.0.1:9020 /.50kbfile >> ~/torperf/50kb.data
-  2>/dev/null
+  2>/dev/null; ~/torperf/truncate-data.py ~/torperf/50kb.data
 2,32 * * * * timeout -s2 1795 ~/torperf/trivsocks-client
   torperf.torproject.org 127.0.0.1:9021 /.1mbfile >> ~/torperf/1mb.data
-  2>/dev/null
+  2>/dev/null; ~/torperf/truncate-data.py ~/torperf/1mb.data
 8 * * * * timeout -s2 3595 ~/torperf/trivsocks-client
   torperf.torproject.org 127.0.0.1:9022 /.5mbfile >> ~/torperf/5mb.data
-  2>/dev/null
+  2>/dev/null; ~/torperf/truncate-data.py ~/torperf/5mb.data
+
+(Omit the truncate-data.py command if you don't want .data files to be
+truncated once per week to contain only the last 4 days of data.)
 
 From now on, the three files 50kb.data, 1mb.data, and 5mb.data should
 accumulate lines like this (50kb.data shown here; line breaks are only for
diff --git a/truncate-data.py b/truncate-data.py
new file mode 100755
index 0000000..a70e3fc
--- /dev/null
+++ b/truncate-data.py
@@ -0,0 +1,75 @@
+#!/usr/bin/python
+import os
+import re
+import sys
+import time
+
+# Truncate Torperf .data file by deleting lines older than 4 days, but
+# only truncate once a week.
+def main():
+
+  # Check usage.
+  if len(sys.argv) != 2:
+    print "Usage: ./truncate.py <.data file>"
+    return
+  data_path = sys.argv[1]
+  if not os.path.isfile(data_path):
+    print "%s is not a .data file." % data_path
+    return
+
+  # Prepare for parsing.
+  parselines = False
+  copylines = False
+  databak_path = data_path + ".bak"
+  databak_file = None
+  started_re = re.compile('(^[\\d]*) .*')
+  now = time.time()
+
+  # Parse the .data file line by line, possibly stopping early if the
+  # first timestamp we find isn't older than a week.
+  with open(data_path) as data_file:
+    for line in data_file:
+
+      # Copy lines written in the past 4 days.  We have already decided to
+      # copy this part of the .data file before, so just copy the line and
+      # continue.
+      if copylines:
+        databak_file.write(line)
+        continue
+
+      # Skip empty lines.
+      if line.strip() == "":
+        continue
+
+      # Extract the first timestamp from the current line.
+      m = started_re.match(line)
+      if not m:
+        print "%s is not a valid .data file." % data_path
+        return
+      started_ts = int(m.group(1))
+
+      # Decide whether to start copying lines.  We have already decided to
+      # truncate this file before.
+      if parselines:
+        if started_ts >= now - 4 * 24 * 60 * 60:
+          databak_file.write(line)
+          copylines = True
+        continue
+
+      # Decide whether to truncate this file at all.
+      if started_ts >= now - 7 * 24 * 60 * 60:
+        return
+
+      # Open a .bak file to write into and start parsing lines to copy in
+      # the next iteration.
+      databak_file = open(databak_path, "w")
+      parselines = True
+
+  # Close the .bak file and replace the original .data file with it.
+  if databak_file:
+    databak_file.close()
+    os.rename(databak_path, data_path)
+
+if __name__ == "__main__":
+  main()
+

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits