[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [onionperf/develop] Update do_simple analysis param to new do_complete tgen semantics



commit 05eb9cdf56f6ae275ace65a0bdbdcf2c3b5e1c40
Author: Ana Custura <ana@xxxxxxxxxxxxxx>
Date:   Fri Jun 26 10:52:46 2020 +0100

    Update do_simple analysis param to new do_complete tgen semantics
---
 onionperf/analysis.py     | 42 +++++++++++++++++++++---------------------
 onionperf/onionperf       |  8 ++++----
 onionperf/reprocessing.py |  8 ++++----
 3 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/onionperf/analysis.py b/onionperf/analysis.py
index 20ca354..eaacbb9 100644
--- a/onionperf/analysis.py
+++ b/onionperf/analysis.py
@@ -58,7 +58,7 @@ class Analysis(object):
         except:
             return None
 
-    def analyze(self, do_simple=True, date_filter=None):
+    def analyze(self, do_complete=False, date_filter=None):
         if self.did_analysis:
             return
 
@@ -70,7 +70,7 @@ class Analysis(object):
             if len(filepaths) > 0:
                 for filepath in filepaths:
                     logging.info("parsing log file at {0}".format(filepath))
-                    parser.parse(util.DataSource(filepath), do_simple=do_simple)
+                    parser.parse(util.DataSource(filepath), do_complete=do_complete)
 
                 if self.nickname is None:
                     parsed_name = parser.get_name()
@@ -150,13 +150,13 @@ class Analysis(object):
 def subproc_analyze_func(analysis_args):
     signal(SIGINT, SIG_IGN)  # ignore interrupts
     a = analysis_args[0]
-    do_simple = analysis_args[1]
-    a.analyze(do_simple=do_simple)
+    do_complete = analysis_args[1]
+    a.analyze(do_complete=do_complete)
     return a
 
 class ParallelAnalysis(Analysis):
 
-    def analyze(self, search_path, do_simple=True, nickname=None, tgen_search_expressions=["tgen.*\.log"],
+    def analyze(self, search_path, do_complete=False, nickname=None, tgen_search_expressions=["tgen.*\.log"],
                 torctl_search_expressions=["torctl.*\.log"], num_subprocs=cpu_count()):
 
         pathpairs = util.find_file_paths_pairs(search_path, tgen_search_expressions, torctl_search_expressions)
@@ -169,7 +169,7 @@ class ParallelAnalysis(Analysis):
                 a.add_tgen_file(tgen_filepath)
             for torctl_filepath in torctl_filepaths:
                 a.add_torctl_file(torctl_filepath)
-            analysis_args = [a, do_simple]
+            analysis_args = [a, do_complete]
             analysis_jobs.append(analysis_args)
 
         analyses = None
@@ -293,7 +293,7 @@ class Transfer(object):
 
 class Parser(object, metaclass=ABCMeta):
     @abstractmethod
-    def parse(self, source, do_simple):
+    def parse(self, source, do_complete):
         pass
     @abstractmethod
     def get_data(self):
@@ -321,7 +321,7 @@ class TGenParser(Parser):
             # both the filter and the unix timestamp should be in UTC at this point
             return util.do_dates_match(self.date_filter, date_to_check)
 
-    def __parse_line(self, line, do_simple):
+    def __parse_line(self, line, do_complete):
         if self.name is None and re.search("Initializing traffic generator on host", line) is not None:
             self.name = line.strip().split()[11]
 
@@ -334,7 +334,7 @@ class TGenParser(Parser):
             if not self.__is_date_valid(line_date):
                 return True
 
-        if not do_simple and re.search("state\sRESPONSE\sto\sstate\sPAYLOAD", line) is not None:
+        if do_complete and re.search("state\sRESPONSE\sto\sstate\sPAYLOAD", line) is not None:
             # another run of tgen starts the id over counting up from 1
             # if a prev transfer with the same id did not complete, we can be sure it never will
             parts = line.strip().split()
@@ -343,7 +343,7 @@ class TGenParser(Parser):
             if transfer_id in self.state:
                 self.state.pop(transfer_id)
 
-        elif not do_simple and re.search("transfer-status", line) is not None:
+        elif do_complete and re.search("transfer-status", line) is not None:
             status = TransferStatusEvent(line)
             xfer = self.state.setdefault(status.transfer_id, Transfer(status.transfer_id))
             xfer.add_event(status)
@@ -351,7 +351,7 @@ class TGenParser(Parser):
         elif re.search("transfer-complete", line) is not None:
             complete = TransferSuccessEvent(line)
 
-            if not do_simple:
+            if do_complete:
                 xfer = self.state.setdefault(complete.transfer_id, Transfer(complete.transfer_id))
                 xfer.add_event(complete)
                 self.transfers[xfer.id] = xfer.get_data()
@@ -369,7 +369,7 @@ class TGenParser(Parser):
         elif re.search("transfer-error", line) is not None:
             error = TransferErrorEvent(line)
 
-            if not do_simple:
+            if do_complete:
                 xfer = self.state.setdefault(error.transfer_id, Transfer(error.transfer_id))
                 xfer.add_event(error)
                 self.transfers[xfer.id] = xfer.get_data()
@@ -382,12 +382,12 @@ class TGenParser(Parser):
 
         return True
 
-    def parse(self, source, do_simple=True):
+    def parse(self, source, do_complete=False):
         source.open()
         for line in source:
             # ignore line parsing errors
             try:
-                if not self.__parse_line(line, do_simple):
+                if not self.__parse_line(line, do_complete):
                     break
             except:
                 logging.warning("TGenParser: skipping line due to parsing error: {0}".format(line))
@@ -526,7 +526,7 @@ class TorCtlParser(Parser):
 
     def __init__(self, date_filter=None):
         ''' date_filter should be given in UTC '''
-        self.do_simple = True
+        self.do_complete = False
         self.bandwidth_summary = {'bytes_read':{}, 'bytes_written':{}}
         self.circuits_state = {}
         self.circuits = {}
@@ -580,11 +580,11 @@ class TorCtlParser(Parser):
                         self.circuits_summary['buildtimes'].append(built - started)
                     if ended is not None and started is not None:
                         self.circuits_summary['lifetimes'].append(ended - started)
-                    if not self.do_simple:
+                    if self.do_complete:
                         self.circuits[cid] = data
                 self.circuits_state.pop(cid)
 
-        elif not self.do_simple and isinstance(event, CircMinorEvent):
+        elif self.do_complete and isinstance(event, CircMinorEvent):
             if event.purpose != event.old_purpose or event.event != CircEvent.PURPOSE_CHANGED:
                 key = "{0}:{1}".format(event.event, event.purpose)
                 circ.add_event(key, arrival_dt)
@@ -620,7 +620,7 @@ class TorCtlParser(Parser):
 
             data = strm.get_data()
             if data is not None:
-                if not self.do_simple:
+                if self.do_complete:
                     self.streams[sid] = data
                 self.streams_summary['lifetimes'].setdefault(stream_type, []).append(ended - started)
             self.streams_state.pop(sid)
@@ -664,7 +664,7 @@ class TorCtlParser(Parser):
             elif re.search("BOOTSTRAP", line) is not None and re.search("PROGRESS=100", line) is not None:
                 self.boot_succeeded = True
 
-        if self.do_simple is False or (self.do_simple is True and re.search("650\sBW", line) is not None):
+        if self.do_complete or (self.do_complete is False and re.search("650\sBW", line) is not None):
             # parse with stem
             timestamps, sep, raw_event_str = line.partition(" 650 ")
             if sep == '':
@@ -683,8 +683,8 @@ class TorCtlParser(Parser):
             self.__handle_event(event, unix_ts)
         return True
 
-    def parse(self, source, do_simple=True):
-        self.do_simple = do_simple
+    def parse(self, source, do_complete=False):
+        self.do_complete = do_complete
         source.open(newline='\r\n')
         for line in source:
             # ignore line parsing errors
diff --git a/onionperf/onionperf b/onionperf/onionperf
index a7d32f6..ddbeaf1 100755
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@ -281,8 +281,8 @@ files generated by this script will be written""",
 
     analyze_parser.add_argument('-s', '--do-simple-parse',
         help="""parse and export only summary statistics rather than full transfer/circuit/stream data""",
-        action="store_true", dest="do_simple",
-        default=False)
+        action="store_false", dest="do_complete",
+        default=True)
 
     # visualize
     visualize_parser = sub_parser.add_parser('visualize', description=DESC_VISUALIZE, help=HELP_VISUALIZE,
@@ -387,7 +387,7 @@ def analyze(args):
             analysis.add_tgen_file(args.tgen_logpath)
         if args.torctl_logpath is not None:
             analysis.add_torctl_file(args.torctl_logpath)
-        analysis.analyze(args.do_simple, date_filter=args.date_filter)
+        analysis.analyze(args.do_complete, date_filter=args.date_filter)
         analysis.save(output_prefix=args.prefix, date_prefix=args.date_prefix)
 
     elif args.tgen_logpath is not None and os.path.isdir(args.tgen_logpath) and args.torctl_logpath is not None and os.path.isdir(args.torctl_logpath):
@@ -396,7 +396,7 @@ def analyze(args):
         torctl_logs = reprocessing.collect_logs(args.torctl_logpath, '*torctl.log*')
         log_pairs = reprocessing.match(tgen_logs, torctl_logs, args.date_filter)
         logging.info("Found {0} matching log pairs to be reprocessed".format(len(log_pairs)))
-        reprocessing.multiprocess_logs(log_pairs, args.prefix, args.nickname, args.do_simple)
+        reprocessing.multiprocess_logs(log_pairs, args.prefix, args.nickname, args.do_complete)
 
     else:
         logging.error("Given paths were an unrecognized mix of file and directory paths, nothing will be analyzed")
diff --git a/onionperf/reprocessing.py b/onionperf/reprocessing.py
index f88f311..ad0308f 100644
--- a/onionperf/reprocessing.py
+++ b/onionperf/reprocessing.py
@@ -46,21 +46,21 @@ def match(tgen_logs, tor_logs, date_filter):
     return log_pairs
 
 
-def analyze_func(prefix, nick, do_simple, pair):
+def analyze_func(prefix, nick, do_complete, pair):
     analysis = Analysis(nickname=nick)
     logging.info('Analysing pair for date {0}'.format(pair[2]))
     analysis.add_tgen_file(pair[0])
     analysis.add_torctl_file(pair[1])
-    analysis.analyze(do_simple=do_simple, date_filter=pair[2])
+    analysis.analyze(do_complete=do_complete, date_filter=pair[2])
     analysis.save(output_prefix=prefix)
     return 1
 
 
-def multiprocess_logs(log_pairs, prefix, nick=None, do_simple=False):
+def multiprocess_logs(log_pairs, prefix, nick=None, do_complete=False):
     pool = Pool(cpu_count())
     analyses = None
     try:
-        func = partial(analyze_func, prefix, nick, do_simple)
+        func = partial(analyze_func, prefix, nick, do_complete)
         mr = pool.map_async(func, log_pairs)
         pool.close()
         while not mr.ready():



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits