[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] r19229: {torflow} Make js differ record all parse errors instead of just the f (in torflow/trunk/NetworkScanners: . libs/jsparser)



Author: mikeperry
Date: 2009-04-06 19:34:15 -0400 (Mon, 06 Apr 2009)
New Revision: 19229

Modified:
   torflow/trunk/NetworkScanners/libs/jsparser/test.py
   torflow/trunk/NetworkScanners/libsoat.py
   torflow/trunk/NetworkScanners/snakeinspector.py
   torflow/trunk/NetworkScanners/soat.py
Log:

Make js differ record all parse errors instead of just the
first.  Also change timeout logging to not report (incorrect)
total attempts.



Modified: torflow/trunk/NetworkScanners/libs/jsparser/test.py
===================================================================
--- torflow/trunk/NetworkScanners/libs/jsparser/test.py	2009-04-06 19:35:59 UTC (rev 19228)
+++ torflow/trunk/NetworkScanners/libs/jsparser/test.py	2009-04-06 23:34:15 UTC (rev 19229)
@@ -17,7 +17,7 @@
 class ExceptionalJSLexer(JavaScriptLexer):
   def displayRecognitionError(self, tokens, e): raise LexerError(tokens, e) 
 
-input = 'var foo = function() { var foo = "h\'i"; return foo+2; };'
+input = 'var foo = function() { var foo = document.cookie; return foo+2; };;;\n;'
 char_stream = antlr3.ANTLRStringStream(input)
 # or to parse a file:
 # char_stream = antlr3.ANTLRFileStream(path_to_input)

Modified: torflow/trunk/NetworkScanners/libsoat.py
===================================================================
--- torflow/trunk/NetworkScanners/libsoat.py	2009-04-06 19:35:59 UTC (rev 19228)
+++ torflow/trunk/NetworkScanners/libsoat.py	2009-04-06 23:34:15 UTC (rev 19229)
@@ -29,23 +29,21 @@
 from JavaScriptLexer import JavaScriptLexer
 from JavaScriptParser import JavaScriptParser
 
-class ParseError(Exception): 
-  def __init__(self, tokens, e):
-    Exception.__init__(self, str(e))
-    self.tokens = tokens
-    self.e = e
+class LoggingJSParser(JavaScriptParser):
+  def __init__(self, tokens):
+    JavaScriptParser.__init__(self, tokens)
+    self.parse_errors__ = []
+  def displayRecognitionError(self, tokens, e):
+    self.parse_errors__.append(e)
+    JavaScriptParser.displayRecognitionError(self, tokens, e)
+class LoggingJSLexer(JavaScriptLexer):
+  def __init__(self, tokens):
+    JavaScriptLexer.__init__(self, tokens)
+    self.lex_errors__ = []
+  def displayRecognitionError(self, tokens, e):
+    self.lex_errors__.append(e)
+    JavaScriptLexer.displayRecognitionError(self, tokens, e)
 
-class LexerError(Exception): 
-  def __init__(self, tokens, e):
-    Exception.__init__(self, str(e))
-    self.tokens = tokens
-    self.e = e
-
-class ExceptionalJSParser(JavaScriptParser):
-  def displayRecognitionError(self, tokens, e): raise ParseError(tokens, e) 
-class ExceptionalJSLexer(JavaScriptLexer):
-  def displayRecognitionError(self, tokens, e): raise LexerError(tokens, e) 
-
 # constants
 
 TEST_SUCCESS = 0
@@ -851,38 +849,39 @@
 
   def _antlr_parse(self, js_string):
     char_stream = antlr3.ANTLRStringStream(js_string)
-    lexer = ExceptionalJSLexer(char_stream)
+    lexer = LoggingJSLexer(char_stream)
     tokens = antlr3.CommonTokenStream(lexer)
-    parser = ExceptionalJSParser(tokens)
+    parser = LoggingJSParser(tokens)
     program = parser.program()
+    program.tree.parse_errors = parser.parse_errors__
+    program.tree.lex_errors = lexer.lex_errors__
     return program.tree
-                                              
+                            
   def _count_ast_elements(self, js_string, name="global"):
     ast_cnts = {}
     try:
-      js_string = js_string.replace("\n\r","\n").replace("\r\n","\n").replace("\r","\n")
+      js_string = js_string.replace("\n\r","\n").replace("\r\n","\n").replace("\r","\n")+";"
       
       ast = self._antlr_parse(js_string)
       JSDiffer._ast_recursive_worker(ast, ast_cnts)
+      for e in ast.lex_errors+ast.parse_errors:
+        name+=":"+e.__class__.__name__
+        if "line" in e.__dict__: 
+          name+=":"+str(e.line)
+        if "token" in e.__dict__ and e.token \
+            and "type" in e.token.__dict__: 
+          name+=":"+JSTokenNames[e.token.type]
+        # XXX: Any other things we want to add?
+        plog("INFO", "Parse error "+name+" on "+js_string)
+        if not "ParseError:"+name in ast_cnts:
+          ast_cnts["ParseError:"+name] = 1
+        else: ast_cnts["ParseError:"+name] += 1
     except UnicodeDecodeError, e:
       name+=":"+e.__class__.__name__
       plog("INFO", "Unicode error "+name+" on "+js_string)
       if not "ParseError:"+name in ast_cnts:
         ast_cnts["ParseError:"+name] = 1
       else: ast_cnts["ParseError:"+name] +=1
-    except (LexerError, ParseError), e:
-      # Store info about the name and type of parse error
-      # so we can match that up too.
-      name+=":"+e.__class__.__name__
-      if "line" in e.e.__dict__: 
-        name+=":"+str(e.e.line)
-      if "token" in e.e.__dict__ and "type" in e.e.token.__dict__: 
-        name+=":"+JSTokenNames[e.e.token.type]
-      # XXX: Any other things we want to add?
-      plog("INFO", "Parse error "+name+" on "+js_string)
-      if not "ParseError:"+name in ast_cnts:
-        ast_cnts["ParseError:"+name] = 1
-      else: ast_cnts["ParseError:"+name] +=1
     return ast_cnts
 
   def _difference_pruner(self, other_cnts):

Modified: torflow/trunk/NetworkScanners/snakeinspector.py
===================================================================
--- torflow/trunk/NetworkScanners/snakeinspector.py	2009-04-06 19:35:59 UTC (rev 19228)
+++ torflow/trunk/NetworkScanners/snakeinspector.py	2009-04-06 23:34:15 UTC (rev 19229)
@@ -28,6 +28,7 @@
   print "  --file <.result file>"
   print "  --exit <idhex>"
   print "  --reason <soat failure reason>"
+  print "  --noreason <soat failure reason>"
   print "  --proto <protocol>"
   print "  --resultfilter <TestResult class name>"
   print "  --statuscode <'Failure' or 'Inconclusive'>"
@@ -37,16 +38,17 @@
 
 def getargs(argv):
   try:
-    opts,args = getopt.getopt(argv[1:],"d:f:e:r:vt:p:s:o:", 
+    opts,args = getopt.getopt(argv[1:],"d:f:e:r:vt:p:s:o:n:", 
              ["dir=", "file=", "exit=", "reason=", "resultfilter=", "proto=", 
-              "verbose", "statuscode=", "sortby="])
+              "verbose", "statuscode=", "sortby=", "noreason="])
   except getopt.GetoptError,err:
     print str(err)
     usage(argv)
   use_dir="./data/"
   use_file=None
   node=None
-  reason=None
+  reasons=[]
+  noreasons=[]
   result=2
   verbose=1
   proto=None
@@ -60,7 +62,9 @@
     elif o == '-e' or o == '--exit': 
       node = a
     elif o == '-r' or o == '--reason': 
-      reason = a
+      reasons.append(a)
+    elif o == '-r' or o == '--noreason': 
+      noreasons.append(a)
     elif o == '-v' or o == '--verbose': 
       verbose += 1
     elif o == '-t' or o == '--resultfilter':
@@ -76,10 +80,10 @@
         result = int(a)
       except ValueError:
         result = RESULT_CODES[a]
-  return use_dir,use_file,node,reason,result,verbose,resultfilter,proto,sortby
+  return use_dir,use_file,node,reasons,noreasons,result,verbose,resultfilter,proto,sortby
  
 def main(argv):
-  use_dir,use_file,node,reason,result,verbose,resultfilter,proto,sortby=getargs(argv)
+  use_dir,use_file,node,reasons,noreasons,result,verbose,resultfilter,proto,sortby=getargs(argv)
   dh = DataHandler(use_dir)
   print dh.data_dir
 
@@ -99,8 +103,9 @@
 
   for r in results:
     r.verbose = verbose
+    if r.reason in noreasons: continue
+    if reasons and r.reason not in reasons: continue
     if (not result or r.status == result) and \
-       (not reason or r.reason == reason) and \
        (not proto or r.proto == proto) and \
        (not resultfilter or r.__class__.__name__ == resultfilter):
       try:

Modified: torflow/trunk/NetworkScanners/soat.py
===================================================================
--- torflow/trunk/NetworkScanners/soat.py	2009-04-06 19:35:59 UTC (rev 19228)
+++ torflow/trunk/NetworkScanners/soat.py	2009-04-06 23:34:15 UTC (rev 19229)
@@ -378,15 +378,14 @@
     self.timeout_fails[result.exit_node] += 1
 
     t_cnt = self.timeout_fails[result.exit_node]
-    tot_cnt = self.site_tests(result.site)
    
     if t_cnt > num_timeouts_per_node:
-      result.extra_info = str(t_cnt)+"/"+str(tot_cnt)
+      result.extra_info = str(t_cnt)
       self.register_exit_failure(result)
       del self.timeout_fails[result.exit_node]
       return TEST_FAILURE
     else:
-      plog("NOTICE", self.proto+" timeout at "+result.exit_node+". This makes "+str(t_cnt)+"/"+str(tot_cnt)+" timeouts")
+      plog("NOTICE", self.proto+" timeout at "+result.exit_node+". This makes "+str(t_cnt)+" timeouts")
       return TEST_INCONCLUSIVE
 
   def register_exit_failure(self, result):