[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [gettor/develop] Deleted database interaction from channels and core. Added new logging strings for stats purposes (See #17588).



commit 553519a8d4fd0f1514cca164def386a96e393328
Author: ilv <ilv@xxxxxxxxxxxxxxxxxxxxxxxx>
Date:   Mon Nov 16 15:19:44 2015 -0300

    Deleted database interaction from channels and core. Added new logging strings for stats purposes (See #17588).
---
 gettor/core.py    |   11 -----------
 gettor/smtp.py    |   40 +++++++++-------------------------------
 gettor/twitter.py |   51 ++++++++++++++++++++-------------------------------
 gettor/utils.py   |   15 +++++++++------
 gettor/xmpp.py    |   28 ++++++++--------------------
 log/smtp.log      |    3 +++
 6 files changed, 49 insertions(+), 99 deletions(-)

diff --git a/gettor/core.py b/gettor/core.py
index 6336e04..3d946c9 100644
--- a/gettor/core.py
+++ b/gettor/core.py
@@ -469,14 +469,3 @@ class Core(object):
         else:
             self.log.debug("FAILED (links file doesn't seem legit)")
             raise LinkFileError("No links file for %s" % provider)
-
-    def add_request_to_db(self):
-        """Add request to database."""
-        self.log.debug("Trying to add request to database")
-        try:
-            self.db.connect()
-            self.db.add_request()
-            self.log.debug("Request added!")
-        except db.DBError as e:
-            self.log.debug("FAILED %s" % str(e))
-            raise InternalError("Couldn't add request to database %s" % str(e))
diff --git a/gettor/smtp.py b/gettor/smtp.py
index 3697786..9203039 100644
--- a/gettor/smtp.py
+++ b/gettor/smtp.py
@@ -461,24 +461,15 @@ class SMTP(object):
                 self.log.debug("Normalizing address...")
                 norm_from_addr = self._get_normalized_address(from_addr)
             except AddressError as e:
-                status = 'malformed'
                 bogus_request = True
-                self.log.debug("Address is malformed!")
-                # it might be interesting to know what triggered this
-                # we are not logging this for now
-                # logfile = self._log_email('malformed', content)
+                self.log.info('REQUEST: invalid; OS: none; LC: none')
 
             if norm_from_addr:
-                self.log.debug("Anonymizing address...")
                 anon_addr = utils.get_sha256(norm_from_addr)
 
                 if self._is_blacklisted(anon_addr):
-                    status = 'blacklisted'
                     bogus_request = True
-                    self.log.debug("Address is blacklisted!")
-                    # it might be interesting to know extra info
-                    # we are not logging this for now
-                    # logfile = self._log_email(anon_addr, content)
+                    self.log.info('REQUEST: blacklist; OS: none; LC: none')
 
             if not bogus_request:
                 # try to figure out what the user is asking
@@ -487,45 +478,43 @@ class SMTP(object):
 
                 # our address should have the locale requested
                 our_addr = "gettor+%s@%s" % (req['lc'], self.our_domain)
-                self.log.debug("Replying from %s" % our_addr)
 
                 # possible options: help, links, mirrors
                 if req['type'] == 'help':
-                    self.log.debug("Type of request: help")
                     self.log.debug("Trying to send help...")
+                    self.log.info('REQUEST: help; OS: none; LC: %s' %
+                                  req['lc'])
                     # make sure we can send emails
                     try:
                         self._send_help('en', our_addr, norm_from_addr)
-                        status = 'success'
                     except SendEmailError as e:
-                        status = 'internal_error'
                         self.log.debug("FAILED: %s" % str(e))
                         raise InternalError("Something's wrong with the SMTP "
                                             "server: %s" % str(e))
 
                 elif req['type'] == 'mirrors':
-                    self.log.debug("Type of request: mirrors")
                     self.log.debug("Trying to send the mirrors...")
+                    self.log.info('REQUEST: mirrors; OS: none; LC: %s' %
+                                  req['lc'])
                     # make sure we can send emails
                     try:
                         self._send_mirrors('en', our_addr, norm_from_addr)
-                        status = 'success'
                     except SendEmailError as e:
-                        status = 'internal_error'
                         self.log.debug("FAILED: %s" % str(e))
                         raise SendEmailError("Something's wrong with the SMTP "
                                              "server: %s" % str(e))
 
                 elif req['type'] == 'links':
-                    self.log.debug("Type of request: links")
                     self.log.debug("Trying to obtain the links...")
+                    self.log.info('REQUEST: links; OS: %s; LC: %s' %
+                                  req['os '], req['lc'])
+
                     try:
                         links = self.core.get_links(
                             'SMTP', req['os'], req['lc']
                         )
                     # if core fails, we fail too
                     except (core.InternalError, core.ConfigError) as e:
-                        status = 'core_error'
                         self.log.debug("FAILED: %s" % str(e))
                         # something went wrong with the core
                         raise InternalError("Error obtaining the links")
@@ -535,19 +524,8 @@ class SMTP(object):
                     try:
                         self._send_links(links, req['lc'], req['os'], our_addr,
                                          norm_from_addr)
-                        status = 'success'
                     except SendEmailError as e:
-                        status = 'internal_error'
                         self.log.debug("FAILED: %s" % str(e))
                         raise SendEmailError("Something's wrong with the SMTP "
                                              "server: %s" % str(e))
                 self.log.debug("Mail sent!")
-        finally:
-            # keep stats
-            if req:
-                self.log.debug("Adding request to database... ")
-                try:
-                    self.core.add_request_to_db()
-                    self.log.debug("Request added")
-                except InternalError as e:
-                    self.log.debug("FAILED: %s" % str(e))
diff --git a/gettor/twitter.py b/gettor/twitter.py
index 13666a3..85cad47 100644
--- a/gettor/twitter.py
+++ b/gettor/twitter.py
@@ -24,6 +24,7 @@ import blacklist
 
 """Twitter channel for distributing links to download Tor Browser."""
 
+
 class ConfigError(Exception):
     pass
 
@@ -51,7 +52,7 @@ class TwitterBot(object):
 
         :param: cfg (string) the path of the configuration file.
         """
-        
+
         default_cfg = 'twitter.cfg'
         config = ConfigParser.ConfigParser()
 
@@ -107,10 +108,8 @@ class TwitterBot(object):
         logfileh.setLevel(logging.getLevelName(loglevel))
         log.addHandler(logfileh)
 
-        # stop logging on stdout from now on
-        #log.propagate = False
         self.log = log
-    
+
     def _is_blacklisted(self, username):
         """Check if a user is blacklisted.
 
@@ -141,7 +140,6 @@ class TwitterBot(object):
         Return: a string containing the given message.
 
         """
-        #self.log.debug("Getting message '%s' for locale %s" % (msgid, lc))
         try:
             t = gettext.translation(lc, self.i18ndir, languages=[lc])
             _ = t.ugettext
@@ -161,7 +159,7 @@ class TwitterBot(object):
 
         :return: request (list) 3-tuple with locale, os and type of request.
         """
-        
+
         # core knows what OS are supported
         supported_os = self.core.get_supported_os()
         supported_lc = self.core.get_supported_lc()
@@ -198,7 +196,7 @@ class TwitterBot(object):
                     req['type'] = 'mirrors'
             if (found_lc and found_os) or (found_lc and found_mirrors):
                 break
-            
+
         return req
 
     def parse_request(self, dm):
@@ -220,31 +218,30 @@ class TwitterBot(object):
 
         try:
             if self._is_blacklisted(str(sender_id)):
-                self.log.info("Request from blacklisted account!")
-                status = 'blacklisted'
+                self.log.info('REQUEST: blacklist; OS: none; LC: none')
                 bogus_request = True
 
             if not bogus_request:
                 self.log.debug("Request seems legit, let's parse it")
                 # let's try to guess what the user is asking
                 request = self.parse_text(str(msg))
-        
+
                 # possible options: links, mirrors, help
                 if request['type'] == 'links':
-                    self.log.debug("Type of request: links")
+                    self.log.info('REQUEST: links; OS: %s; LC: %s' %
+                                  (req['os'], req['lc']))
                     links = self.core.get_links(
                         'Twitter',
                         request['os'],
                         request['lc']
                     )
-                    
+
                     reply = self._get_msg('links', 'en')
                     reply = reply % (request['os'], request['lc'], links)
-                    
 
                 elif request['type'] == 'mirrors':
-                    self.log.debug("Type of request: mirrors")
-                    status = 'success'
+                    self.log.info('REQUEST: mirrors; OS: none; LC: %s' %
+                                  req['lc'])
                     reply = self._get_msg('mirrors', 'en')
                     try:
                         with open(self.mirrors, "r") as list_mirrors:
@@ -255,10 +252,10 @@ class TwitterBot(object):
                         reply = self._get_msg('mirrors_unavailable', 'en')
 
                 else:
-                    self.log.debug("Type of request: help")
-                    status = 'success'
+                    self.log.info('REQUEST: help; OS: none; LC: %s' %
+                                  req['lc'])
                     reply = self._get_msg('help', 'en')
-                
+
                 self.api.send_direct_message(
                     user_id=sender_id,
                     text=reply
@@ -267,25 +264,18 @@ class TwitterBot(object):
         except (core.ConfigError, core.InternalError) as e:
             # if core failes, send the user an error message, but keep going
             self.log.error("Something went wrong internally: %s" % str(e))
-            status = 'core_error'
             reply = self._get_msg('internal_error', 'en')
 
-        finally:
-            # keep stats
-            if request:
-                self.log.debug("Adding request to database... ")
-                self.core.add_request_to_db()    
-
     def start(self):
         """ Start the bot for handling requests.
 
         Start a new Twitter bot.
         """
         self.auth = tweepy.OAuthHandler(
-            self.api_key, 
+            self.api_key,
             self.api_secret
         )
-        
+
         self.auth.set_access_token(
             self.access_token,
             self.token_secret
@@ -293,11 +283,10 @@ class TwitterBot(object):
 
         self.api = tweepy.API(self.auth)
         self.bot_info = self.api.me()
-         
+
         stream = tweepy.Stream(
-            auth = self.api.auth,
+            auth=self.api.auth,
             listener=GetTorStreamListener(self)
         )
-        
-        stream.userstream()
 
+        stream.userstream()
diff --git a/gettor/utils.py b/gettor/utils.py
index b4ab9c6..8224f25 100644
--- a/gettor/utils.py
+++ b/gettor/utils.py
@@ -16,25 +16,28 @@ import hashlib
 """Common utilities for GetTor modules."""
 
 
-LOGGING_FORMAT = "[%(levelname)s] %(asctime)s - %(message)s"
-DATE_FORMAT = "%Y-%m-%d" # %H:%M:%S
+LOGGING_FORMAT = "[%(levelname)s] DATE: %(asctime)s; %(message)s"
+DATE_FORMAT = "%Y-%m-%d"  # %H:%M:%S
+
 
 def get_logging_format():
     """Get the logging format.
-    
+
     :return: (string) the logging format.
-    
+
     """
     return LOGGING_FORMAT
 
+
 def get_date_format():
     """Get the date format for logging.
-    
+
     :return: (string) the date format for logging.
-    
+
     """
     return DATE_FORMAT
 
+
 def get_sha256(string):
     """Get sha256 of a string.
 
diff --git a/gettor/xmpp.py b/gettor/xmpp.py
index a605f50..072d988 100644
--- a/gettor/xmpp.py
+++ b/gettor/xmpp.py
@@ -277,8 +277,7 @@ class XMPP(object):
         self.log.debug("Parsing request")
         try:
             if self._is_blacklisted(str(account)):
-                self.log.info("Request from blacklisted account!")
-                status = 'blacklisted'
+                self.log.info('REQUEST: blacklist; OS: none; LC: none')
                 bogus_request = True
 
             # first let's find out how many words are in the message
@@ -287,7 +286,7 @@ class XMPP(object):
             if len(words) > self.max_words:
                 bogus_request = True
                 self.log.info("Message way too long")
-                status = 'error'
+                self.log.info('REQUEST: invalid; OS: none; LC: none')
                 reply = self._get_msg('message_error', 'en')
 
             if not bogus_request:
@@ -296,13 +295,13 @@ class XMPP(object):
                 req = self._parse_text(str(msg))
 
                 if req['type'] == 'help':
-                    self.log.debug("Type of request: help")
-                    status = 'success'
+                    self.log.info('REQUEST: help; OS: none; LC: %s' %
+                                  req['lc'])
                     reply = self._get_msg('help', 'en')
 
                 elif req['type'] == 'mirrors':
-                    self.log.debug("Type of request: mirrors")
-                    status = 'success'
+                    self.log.info('REQUEST: mirrors; OS: none; LC: %s' %
+                                  req['lc'])
                     reply = self._get_msg('mirrors', 'en')
                     try:
                         with open(self.mirrors, "r") as list_mirrors:
@@ -312,7 +311,8 @@ class XMPP(object):
                         reply = self._get_msg('mirrors_unavailable', 'en')
 
                 elif req['type'] == 'links':
-                    self.log.debug("Type of request: help")
+                    self.log.info('REQUEST: links; OS: %s; LC: %s' %
+                                  (req['os'], req['lc']))
                     links = self.core.get_links(
                         "XMPP",
                         req['os'],
@@ -321,22 +321,10 @@ class XMPP(object):
                     reply = self._get_msg('links', 'en')
                     reply = reply % (req['os'], req['lc'], links)
 
-                    status = 'success'
-
         except (core.ConfigError, core.InternalError) as e:
             # if core failes, send the user an error message, but keep going
             self.log.error("Something went wrong internally: %s" % str(e))
-            status = 'core_error'
             reply = self._get_msg('internal_error', req['lc'])
 
         finally:
-            # keep stats
-            if req:
-                self.log.debug("Adding request to database... ")
-                self.core.add_request_to_db()
-
-            if reply:
-                self.log.debug("Everything seems OK. Sending back the reply")
-            else:
-                self.log.debug("Nothing to reply!")
             return reply
diff --git a/log/smtp.log b/log/smtp.log
index e69de29..3950925 100644
--- a/log/smtp.log
+++ b/log/smtp.log
@@ -0,0 +1,3 @@
+[INFO] DATE: 2015-11-16; REQUEST: help; OS: none; LC: fa
+[INFO] DATE: 2015-11-16; REQUEST: links; OS: windows; LC: zh
+[INFO] DATE: 2015-11-16; REQUEST: mirrors; OS: none; LC: tr

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits