[tor-commits] [ooni-probe/master] Add support for downloading GeoIP files via the ooniresource tool.

[art@xxxxxxxxxxxxxx]

commit a61e3361a0bd3d931ab539301963940154b4277a
Author: Arturo Filastò <art@xxxxxxxxx>
Date:   Tue Aug 12 21:43:46 2014 +0200

    Add support for downloading GeoIP files via the ooniresource tool.
    
    * Add better exception handling to ooniresource.
    * Add gunzip to utils.
---
 ooni/resources/__init__.py |   21 ++++++++++++++++++++-
 ooni/resources/cli.py      |   34 ++++++++++++++++++++++++++--------
 ooni/resources/update.py   |   15 +++++++++++----
 ooni/utils/__init__.py     |   13 +++++++++++++
 4 files changed, 70 insertions(+), 13 deletions(-)

diff --git a/ooni/resources/__init__.py b/ooni/resources/__init__.py
index 0afc0af..e60784f 100644
--- a/ooni/resources/__init__.py
+++ b/ooni/resources/__init__.py
@@ -1,9 +1,11 @@
 from ooni.settings import config
-from ooni.utils import unzip
+from ooni.utils import unzip, gunzip
 
 from ooni.deckgen.processors import citizenlab_test_lists
 from ooni.deckgen.processors import namebench_dns_servers
 
+config.read_config_file()
+
 __version__ = "0.0.1"
 
 inputs = {
@@ -20,3 +22,20 @@ inputs = {
         "processor": citizenlab_test_lists
     }
 }
+
+geoip = {
+    "GeoIPASNum.dat.gz": {
+        "url": "http://www.maxmind.com/download/";
+               "geoip/database/asnum/GeoIPASNum.dat.gz",
+        "action": gunzip,
+        "action_args": [config.advanced.geoip_data_dir],
+        "processor": None
+    },
+    "GeoIP.dat.gz": {
+        "url": "http://geolite.maxmind.com/";
+               "download/geoip/database/GeoLiteCountry/GeoIP.dat.gz",
+        "action": gunzip,
+        "action_args": [config.advanced.geoip_data_dir],
+        "processor": None
+    }
+}
diff --git a/ooni/resources/cli.py b/ooni/resources/cli.py
index 3ee7945..a140c8d 100644
--- a/ooni/resources/cli.py
+++ b/ooni/resources/cli.py
@@ -1,7 +1,9 @@
 import sys
 
+from twisted.internet import defer
 from twisted.python import usage
 
+from ooni.utils import log
 from ooni.settings import config
 
 from ooni.resources import __version__
@@ -9,10 +11,11 @@ from ooni.resources import update
 
 
 class Options(usage.Options):
-    synopsis = """%s"""
+    synopsis = """%s""" % sys.argv[0]
 
     optFlags = [
-        ["update-inputs", None, "Update the resources needed for inputs"]
+        ["update-inputs", None, "Update the resources needed for inputs."],
+        ["update-geoip", None, "Update the geoip related resources."]
     ]
     optParameters = []
 
@@ -21,9 +24,10 @@ class Options(usage.Options):
         sys.exit(0)
 
 
+@xxxxxxxxxxxxxxxxxxxxx
 def run():
-    options = Options()
     config.read_config_file()
+    options = Options()
     try:
         options.parseOptions()
     except usage.UsageError as error_message:
@@ -31,9 +35,23 @@ def run():
         print "%s: Try --help for usage details." % (sys.argv[0])
         sys.exit(1)
 
-    if options['update-inputs']:
-        return update.download_inputs()
+    if not any(options.values()):
+        print("%s: no command specified" % sys.argv[0])
+        print options
+        sys.exit(1)
 
-    print "%s: no command specified" % sys.argv[0]
-    print "%s: Try --help for usage details." % (sys.argv[0])
-    sys.exit(1)
+    if options['update-inputs']:
+        print "Downloading inputs"
+        try:
+            yield update.download_inputs()
+        except Exception as exc:
+            log.err("failed to download geoip files")
+            log.exception(exc)
+
+    if options['update-geoip']:
+        print "Downloading geoip files"
+        try:
+            yield update.download_geoip()
+        except Exception as exc:
+            log.err("failed to download geoip files")
+            log.exception(exc)
diff --git a/ooni/resources/update.py b/ooni/resources/update.py
index 28a6ec7..4cb0f49 100644
--- a/ooni/resources/update.py
+++ b/ooni/resources/update.py
@@ -4,7 +4,7 @@ from twisted.internet import reactor, defer, protocol
 from twisted.web.client import RedirectAgent, Agent
 
 from ooni.settings import config
-from ooni.resources import inputs
+from ooni.resources import inputs, geoip
 
 agent = RedirectAgent(Agent(reactor))
 
@@ -29,12 +29,11 @@ class SaveToFile(protocol.Protocol):
 
 
 @defer.inlineCallbacks
-def download_inputs():
-    for filename, resource in inputs.items():
+def download_resource(resources):
+    for filename, resource in resources.items():
         print "Downloading %s" % filename
 
         filename = os.path.join(config.resources_directory, filename)
-
         response = yield agent.request("GET", resource['url'])
         finished = defer.Deferred()
         response.deliverBody(SaveToFile(finished, response.length, filename))
@@ -45,3 +44,11 @@ def download_inputs():
                                       filename,
                                       *resource['action_args'])
         print "%s written." % filename
+
+
+def download_inputs():
+    return download_resource(inputs)
+
+
+def download_geoip():
+    return download_resource(geoip)
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index 5050e14..30a93d5 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -1,8 +1,10 @@
+import shutil
 import string
 import random
 import glob
 import os
 
+import gzip
 from zipfile import ZipFile
 
 from ooni import otime
@@ -156,3 +158,14 @@ def unzip(filename, dst):
         zip_file = ZipFile(zfp)
         zip_file.extractall(dst_path)
     return dst_path
+
+def gunzip(filename, dst):
+    assert filename.endswith(".gz")
+    dst_path = os.path.join(
+        dst,
+        os.path.basename(filename).replace(".gz", "")
+    )
+    with open(dst_path, "w+") as fw:
+        gzip_file = gzip.open(filename)
+        shutil.copyfileobj(gzip_file, fw)
+        gzip_file.close()



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits