[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Include the GeoIP data files as part of the source distribution



commit 6c3accabfe4c7f465afc02593d46bab4b29026a2
Author: Arturo Filastò <art@xxxxxxxxx>
Date:   Fri Jan 3 16:05:19 2014 +0100

    Include the GeoIP data files as part of the source distribution
    
    * Install the geoip data files to /usr/share/geoip
---
 MANIFEST.in                |    7 ++--
 data/ooniprobe.conf.sample |    4 +--
 requirements.txt           |    4 +--
 setup.py                   |   77 ++++++++++++++++++++++++++++++++++++++------
 4 files changed, 76 insertions(+), 16 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index adcc182..92586f8 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,6 +1,7 @@
 include README.md ChangeLog.md requirements.txt
-recursive-include data/ui/app *
 recursive-include data/decks *
-recursive-include data/inputs *
-include data/Makefile
+include data/inputs/
+include data/GeoIP.dat
+include data/GeoIPASNum.dat
+include data/GeoLiteCity.dat
 include data/ooniprobe.conf.sample
diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample
index 83c1ebc..8a4e0f7 100644
--- a/data/ooniprobe.conf.sample
+++ b/data/ooniprobe.conf.sample
@@ -21,7 +21,7 @@ reports:
     pcap: null
     collector: 'httpo://nkvphnp3p6agi5qq.onion'
 advanced:
-    geoip_data_dir: /usr/share/ooni/
+    geoip_data_dir: /usr/share/geoip
     debug: false
     # enable if auto detection fails
     #tor_binary: /usr/sbin/tor
@@ -46,7 +46,7 @@ advanced:
     # How many reports to perform concurrently
     reporting_concurrency: 15
     # Specify here a custom data_dir path
-    data_dir: /usr/share/ooni/
+    data_dir: /usr/share/ooni
     oonid_api_port: 8042
 tor:
     #socks_port: 8801
diff --git a/requirements.txt b/requirements.txt
index 5fb1ff9..638b33b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,8 +5,8 @@ docutils>=0.9.1
 ipaddr>=2.1.10
 pyOpenSSL>=0.13
 # XXX fix this once the following issue is resolved: https://github.com/appliedsec/pygeoip/issues/56
-https://github.com/appliedsec/pygeoip/archive/master.zip
-#pygeoip>=0.3.1
+https://github.com/appliedsec/pygeoip/archive/master.zip#egg=pygeoip-0.3.1
+pygeoip>=0.3.1
 txtorcon>=0.7
 txsocksx>=0.0.2
 Pyrex>=0.9.8.6
diff --git a/setup.py b/setup.py
index 8eccd4e..f761f30 100644
--- a/setup.py
+++ b/setup.py
@@ -2,41 +2,101 @@
 #-*- coding: utf-8 -*-
 
 from ooni import __version__
+import urllib2
 import os
+import gzip
 from os.path import join as pj
 import sys
 from setuptools import setup
 
-dependency_links = [
-    'https://people.torproject.org/~ioerror/src/mirrors/ooniprobe'
-]
+def download_geoip_files():
+    urls = [
+        'http://www.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz',
+        'http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz',
+        'http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz'
+    ]
+    for url in urls:
+        target_gz_file = pj('data', os.path.basename(url))
+        target_file = target_gz_file.replace('.gz', '')
+
+        if os.path.isfile(target_file):
+            print "%s already exists. Skipping." % target_file
+            continue
+
+        print "Downloading %s" % url
+        response = urllib2.urlopen(url)
+
+        with open(target_gz_file, 'w+') as f:
+            f.write(response.read())
+        
+        with open(target_file, 'w+') as f:
+            gf = gzip.open(target_gz_file, 'rb')
+            f.write(gf.read())
+            gf.close()
+
+        os.unlink(target_gz_file)
+
+download_geoip_files()
 
 usr_share_path = '/usr/share/ooni'
 # If this is true then it means we are in a virtualenv
+# therefore we should not place our data files inside /usr/share/ooni, but
+# place them inside the virtual env system prefix.
 if hasattr(sys, 'real_prefix'):
-    usr_share_path = pj(sys.prefix, 'share', 'ooni')
+    usr_share_path = os.path.abspath(pj(sys.prefix, 'share', 'ooni'))
+    if not os.path.isdir(usr_share_path):
+        os.makedirs(usr_share_path)
     with open(pj('data', 'ooniprobe.conf.sample.new'), 'w+') as w:
         with open(pj('data', 'ooniprobe.conf.sample')) as f:
             for line in f:
                 if line.startswith('    data_dir: /usr/share/ooni'):
                     w.write('    data_dir: %s\n' % usr_share_path)
+                elif line.startswith('    geoip_data_dir: /usr/share/'):
+                    w.write('    geoip_data_dir: %s\n' % usr_share_path)
                 else:
                     w.write(line)
     os.rename(pj('data', 'ooniprobe.conf.sample.new'),
               pj('data', 'ooniprobe.conf.sample'))
 
-data_files = []
+    data_files = [(
+        usr_share_path + '/', 
+        [
+            'data/GeoIP.dat',
+            'data/GeoIPASNum.dat',
+            'data/GeoLiteCity.dat'
+        ]
+    )]
+else:
+    data_files = [(
+        '/usr/share/geoip/', 
+        [
+            'data/GeoIP.dat',
+            'data/GeoIPASNum.dat',
+            'data/GeoLiteCity.dat'
+        ]
+    )]
+
 for root, dirs, file_names in os.walk('data/'):
     files = []
     for file_name in file_names:
-        if not file_name.endswith('.pyc'):
-            files.append(pj(root, file_name))
+        if file_name.endswith('.pyc'):
+            continue
+        elif file_name.endswith('.dat') and \
+                file_name.startswith('Geo'):
+            continue
+        files.append(pj(root, file_name))
     data_files.append([pj(usr_share_path, root.replace('data/', '')), files])
 
 install_requires = []
+dependency_links = [
+    'https://people.torproject.org/~ioerror/src/mirrors/ooniprobe'
+]
 with open('requirements.txt') as f:
     for line in f:
-        if line.startswith("#") or line.startswith('http'):
+        if line.startswith("#"):
+            continue
+        if line.startswith('https'):
+            dependency_links.append(line)
             continue
         install_requires.append(line)
 
@@ -54,7 +114,6 @@ setup(
         'ooni.nettests.blocking',
         'ooni.nettests.third_party',
         'ooni.templates', 'ooni.tests', 'ooni.utils'],
-
     scripts=["bin/ooniprobe"],
     dependency_links=dependency_links,
     install_requires=install_requires



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits