[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [gettor/master] Create script to add links to db



commit da53fc70b6871c13c5821bf588113943a49baeb3
Author: hiro <hiro@xxxxxxxxxxxxxx>
Date:   Fri Mar 8 10:52:25 2019 +0100

    Create script to add links to db
---
 bin/gettor_service                 |  8 +++++
 gettor/__init__.py                 |  4 ++-
 gettor/main.py                     |  6 ++--
 gettor/services/email/sendmail.py  | 21 +++++--------
 gettor/services/twitter/twitter.py |  5 ++-
 gettor/services/xmpp/xmpp.py       |  5 ++-
 gettor/utils/db.py                 | 36 +++++++++++++++------
 gettor/utils/options.py            |  6 +++-
 gettor/utils/strings.py            |  9 +++++-
 gettor/web/http.py                 | 27 +++++++++-------
 scripts/add_lins_to_db             | 64 ++++++++++++++++++++++++++++++++++++++
 share/locale/en.json               |  2 ++
 12 files changed, 152 insertions(+), 41 deletions(-)

diff --git a/bin/gettor_service b/bin/gettor_service
index ae85f56..3c1ac04 100755
--- a/bin/gettor_service
+++ b/bin/gettor_service
@@ -10,6 +10,14 @@
 #
 # :license: This is Free Software. See LICENSE for license information.
 
+
+################################################################################
+#                                                                              #
+# This is how GetTor is started as a twisted application.                      #
+# The script allows to start, stop, restart GetTor and get its status.         #
+#                                                                              #
+################################################################################
+
 case "$1" in
 start)
    twistd --python=scripts/gettor --logfile=log/gettor.log --pidfile=gettor.pid
diff --git a/gettor/__init__.py b/gettor/__init__.py
index bf73506..74a0363 100644
--- a/gettor/__init__.py
+++ b/gettor/__init__.py
@@ -12,6 +12,8 @@ the Tor Browser.
 
 from .utils import strings
 
-
+"""
+This is where version and available locales get loaded.
+"""
 __version__ = strings.get_version()
 __locales__ = strings.get_locales()
diff --git a/gettor/main.py b/gettor/main.py
index 53fbf61..0932ed5 100644
--- a/gettor/main.py
+++ b/gettor/main.py
@@ -10,7 +10,9 @@ the Tor Browser.
 :license: see included LICENSE for information
 """
 
-"""This module sets up GetTor and starts the servers running."""
+"""
+This sets up GetTor and starts the servers running.
+"""
 
 import sys
 
@@ -22,7 +24,7 @@ from .services.email.sendmail import Sendmail
 
 def run(gettor, app):
     """
-        This is GetTor's main entry point and main runtime loop.
+    This is GetTor's main entry point and main runtime loop.
     """
     settings = options.parse_settings()
 
diff --git a/gettor/services/email/sendmail.py b/gettor/services/email/sendmail.py
index 8c86bdc..92aa766 100644
--- a/gettor/services/email/sendmail.py
+++ b/gettor/services/email/sendmail.py
@@ -103,7 +103,7 @@ class Sendmail(object):
 
     @defer.inlineCallbacks
     def get_new(self):
-        """strings.load_strings("en")
+        """
         Get new requests to process. This will define the `main loop` of
         the Sendmail service.
         """
@@ -117,18 +117,16 @@ class Sendmail(object):
             status="ONHOLD", command="links", service="email"
         )
 
+        """
+        Load strings for translations
+        """
+        # for now just english
+        strings.load_strings("en")
 
         if help_requests:
             try:
                 log.info("Got new help request.")
 
-                # for now just english
-                en = gettext.translation(
-                    'email', localedir='locales', languages=['en']
-                )
-                en.install()
-                _ = en.gettext
-
                 for request in help_requests:
                     id = request[0]
                     date = request[4]
@@ -142,8 +140,8 @@ class Sendmail(object):
 
                     yield self.sendmail(
                         email_addr=id,
-                        subject=_("help_subject"),
-                        body=_("help_body")
+                        subject=strings._("help_subject"),
+                        body=strings._("help_body")
                     )
 
                     yield self.conn.update_stats(
@@ -162,9 +160,6 @@ class Sendmail(object):
             try:
                 log.info("Got new links request.")
 
-                # for now just english
-                strings.load_strings("en")
-
                 for request in link_requests:
                     id = request[0]
                     date = request[4]
diff --git a/gettor/services/twitter/twitter.py b/gettor/services/twitter/twitter.py
index b5e08ba..fcdc353 100644
--- a/gettor/services/twitter/twitter.py
+++ b/gettor/services/twitter/twitter.py
@@ -22,7 +22,10 @@ import core
 import utils
 import blacklist
 
-"""Twitter channel for distributing links to download Tor Browser."""
+"""
+Twitter channel for distributing links to download Tor Browser.
+Needs to be refactored to work with twisted and updates to twitter apis
+"""
 
 OS = {
     'osx': 'Mac OS X',
diff --git a/gettor/services/xmpp/xmpp.py b/gettor/services/xmpp/xmpp.py
index ff5e207..cfcf8bf 100644
--- a/gettor/services/xmpp/xmpp.py
+++ b/gettor/services/xmpp/xmpp.py
@@ -28,7 +28,10 @@ import utils
 import blacklist
 
 
-"""XMPP module for processing requests."""
+"""
+XMPP module for processing requests.
+Needs to be refactored to work with twisted
+"""
 
 OS = {
     'osx': 'Mac OS X',
diff --git a/gettor/utils/db.py b/gettor/utils/db.py
index ee4c398..99ae081 100644
--- a/gettor/utils/db.py
+++ b/gettor/utils/db.py
@@ -16,7 +16,7 @@ from twisted.enterprise import adbapi
 
 class SQLite3(object):
 	"""
-
+	This class handles the database connections and operations.
 	"""
 	def __init__(self, dbname):
 		"""Constructor."""
@@ -25,18 +25,26 @@ class SQLite3(object):
 		)
 
 	def query_callback(self, results=None):
-		""" """
+		"""
+		Query callback
+		Log that the database query has been executed and return results
+		"""
 		log.msg("Database query executed successfully.")
 		return results
 
 	def query_errback(self, error=None):
-		""" """
+		"""
+        Query error callback
+		Logs database error
+		"""
 		if error:
 			log.msg("Database error: {}".format(error))
 		return None
 
 	def new_request(self, id, command, service, platform, date, status):
-		""" """
+		"""
+		Perform a new request to the database
+		"""
 		query = "INSERT INTO requests VALUES(?, ?, ?, ?, ?, ?)"
 
 		return self.dbpool.runQuery(
@@ -44,7 +52,9 @@ class SQLite3(object):
 		).addCallback(self.query_callback).addErrback(self.query_errback)
 
 	def get_requests(self, status, command, service):
-		""" """
+		"""
+		Perform a SELECT request to the database
+		"""
 		query = "SELECT * FROM requests WHERE service=? AND command=? AND "\
 		"status = ?"
 
@@ -53,7 +63,9 @@ class SQLite3(object):
 		).addCallback(self.query_callback).addErrback(self.query_errback)
 
 	def get_num_requests(self, id, service):
-		""" """
+		"""
+		Get number of requests for statistics
+		"""
 		query = "SELECT COUNT(rowid) FROM requests WHERE id=? AND service=?"
 
 		return self.dbpool.runQuery(
@@ -61,7 +73,9 @@ class SQLite3(object):
 		).addCallback(self.query_callback).addErrback(self.query_errback)
 
 	def update_request(self, id, hid, status, service, date):
-		""" """
+		"""
+		Update request record in the database
+		"""
 		query = "UPDATE requests SET id=?, status=? WHERE id=? AND "\
 		"service=? AND date=?"
 
@@ -70,7 +84,9 @@ class SQLite3(object):
 		).addCallback(self.query_callback).addErrback(self.query_errback)
 
 	def update_stats(self, command, service, platform=None):
-		""" """
+		"""
+		Update statistics to the database
+		"""
 		now_str = datetime.now().strftime("%Y%m%d")
 		query = "REPLACE INTO stats(num_requests, platform, "\
 		"command, service, date) VALUES(COALESCE((SELECT num_requests FROM stats "\
@@ -81,7 +97,9 @@ class SQLite3(object):
 		).addCallback(self.query_callback).addErrback(self.query_errback)
 
 	def get_links(self, platform, status):
-		""" """
+		"""
+		Get links from the database per platform
+		"""
 		query = "SELECT * FROM links WHERE platform=? AND status=?"
 		return self.dbpool.runQuery(
 			query, (platform, status)
diff --git a/gettor/utils/options.py b/gettor/utils/options.py
index e8e2996..ef4b346 100644
--- a/gettor/utils/options.py
+++ b/gettor/utils/options.py
@@ -25,6 +25,10 @@ def load_settings(config=None):
     return settings
 
 def parse_settings():
+    """
+    Parse settings and loads strings in a given locale
+    This function needs to be rewritten considering passing a locale and
+    returing translated strings
+    """
     strings.load_strings("en")
-
     return load_settings(config=False)
diff --git a/gettor/utils/strings.py b/gettor/utils/strings.py
index 1cf0003..5231dc7 100644
--- a/gettor/utils/strings.py
+++ b/gettor/utils/strings.py
@@ -72,13 +72,20 @@ def get_resource_path(filename, path):
     return os.path.join(prefix, filename)
 
 def get_version():
-    # The current version
+    """
+    The current version
+    """
+
     version = ""
     with open(get_resource_path('version.txt', '../share')) as f:
         version = f.read().strip()
     return version
 
 def get_locales():
+    """
+    Get available_locales
+    """
+    
     filename = get_resource_path("available_locales.json", '../share/locale')
     locales = {}
     with open(filename, encoding='utf-8') as f:
diff --git a/gettor/web/http.py b/gettor/web/http.py
index 77120d6..067aef8 100644
--- a/gettor/web/http.py
+++ b/gettor/web/http.py
@@ -22,7 +22,10 @@ from time import gmtime, strftime
 import core
 import utils
 
-"""GetTor RESTful API"""
+"""
+GetTor RESTful API
+This part of GetTor has not been integrated into twisted-gettor just yet.
+"""
 
 # currently supported locales for Tor Browser
 LC = ['ar', 'de', 'en-US', 'es-ES', 'fa', 'fr', 'it', 'ko', 'nl', 'pl',
@@ -123,7 +126,7 @@ class HTTP(object):
         except ValueError, e:
             return False
         return True
-    
+
     def _write_json(self, path, content):
         """
         """
@@ -376,9 +379,9 @@ class HTTP(object):
 
     def build(self):
         """ Build RESTful API. """
-        
+
         print "Building API"
-        
+
         # resources
         self._write_json(
             os.path.join(self.tree, 'api'),
@@ -388,7 +391,7 @@ class HTTP(object):
         api_path = os.path.join(self.tree, 'api-content')
         if not os.path.isdir(api_path):
             os.mkdir(api_path)
-        
+
         # providers
         self._write_json(
             os.path.join(api_path, 'providers'),
@@ -415,7 +418,7 @@ class HTTP(object):
 
             if not os.path.isdir(provider_path):
                 os.mkdir(provider_path)
-            
+
             for osys in self.links[provider]:
                 self._write_json(
                     os.path.join(provider_path, osys),
@@ -424,11 +427,11 @@ class HTTP(object):
 
                 provider_os_path = os.path.join(
                     provider_path, "%s-content" % osys
-                )            
+                )
 
                 if not os.path.isdir(provider_os_path):
                     os.mkdir(provider_os_path)
-                
+
                 for lc in self.links[provider][osys]:
                     self._write_json(
                         os.path.join(provider_os_path, lc),
@@ -440,7 +443,7 @@ class HTTP(object):
             os.path.join(api_path, 'latest'),
             self.lv
         )
-        
+
         lv_path = os.path.join(api_path, 'latest-content')
         if not os.path.isdir(lv_path):
             os.mkdir(lv_path)
@@ -453,7 +456,7 @@ class HTTP(object):
                 os.path.join(lv_path, release),
                 self.lv[release]
             )
-            
+
             release_path = os.path.join(
                 lv_path,
                 "%s-content" % release
@@ -461,7 +464,7 @@ class HTTP(object):
 
             if not os.path.isdir(release_path):
                 os.mkdir(release_path)
-            
+
             for osys in self.lv[release]['downloads']:
                 self._write_json(
                     os.path.join(release_path, osys),
@@ -475,7 +478,7 @@ class HTTP(object):
 
                 if not os.path.isdir(release_os_path):
                     os.mkdir(release_os_path)
-                
+
                 for lc in self.lv[release]['downloads'][osys]:
                     self._write_json(
                         os.path.join(release_os_path, lc),
diff --git a/scripts/add_lins_to_db b/scripts/add_lins_to_db
new file mode 100644
index 0000000..5da92b3
--- /dev/null
+++ b/scripts/add_lins_to_db
@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# This file is part of GetTor, a Tor Browser distribution system.
+#
+# :authors: hiro <hiro@xxxxxxxxxxxxxx>
+#           see also AUTHORS file
+#
+# :license: This is Free Software. See LICENSE for license information.
+
+import os
+import sys
+import sqlite3
+import urllib import request
+
+def main():
+  args = parser.parse_args()
+  abs_filename = os.path.abspath(args.filename)
+
+  webFile = request.urlopen("https://lektor-staging.torproject.org/tpo/staging/projects/torbrowser/RecommendedTBBVersions/";)
+  versions = webfile.read().decode('utf-8')
+  version = versions.split("&#34;")[1]
+
+  gitlab = "https://gitlab.com/hiromipaw/torbrowser/raw/releases/";
+
+  prefixes = {
+    "osx": "TorBrowser-",
+    "windows": "torbrowser-install-",
+    "linux": "tor-browser-linux64-"
+  }
+
+  versions = {"windows": version, 'linux': version, 'osx': version}
+
+  suffixes = {
+    "osx": "-osx64_en-US.dmg",
+    "windows": "_en-US.exe",
+    "linux": "_en-US.tar.xz"
+  }
+
+  keys = set().union(suffixes, versions, prefixes)
+  releases = {k: "".join(dic.get(k, version) for dic in (prefixes, versions, suffixes))  for k in keys}
+
+
+  if not abs_filename:
+      print("Missing database filename.")
+  elif args.new and not args.overwrite and os.path.isfile(abs_filename):
+      print("Database file already exists.")
+  elif args.new:
+      conn = sqlite3.connect(abs_filename)
+      with conn:
+          c = conn.cursor()
+          """
+          Here we drop previous links TABLE but probably it would be better to just update old links to INACTIVE
+          """
+          c.execute("DROP TABLE IF EXISTS links")
+          c.execute(
+              "CREATE TABLE links(link TEXT, platform TEXT, arch TEXT,"
+              " version TEXT, provider TEXT, status TEXT)"
+          )
+          for k in keys:
+              c.execute(
+                  "INSERT INTO links(link, platform, arch, version, provider, status)"
+                  "VALUES ('%s', '%s' '64', '%s', 'gitlab', 'ACTIVE')" %(releases.get(key), k, version)
+              )
diff --git a/share/locale/en.json b/share/locale/en.json
index 0e9d47c..8824c2c 100644
--- a/share/locale/en.json
+++ b/share/locale/en.json
@@ -1,6 +1,8 @@
 {
   "links_body": "GetTor Test. Please be kind.",
   "links_subject": "GetTor Email Test",
+  "help_body": "GetTor Help Test. Please be kind.",
+  "help_subject": "GetTor Help Email Test",
   "help_debug": "Log application errors to stdout",
   "help_config": "Custom config file location (optional)",
   "smtp_links_subject": "[GetTor] Links for your request",



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits