[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [ooni-probe/master] Restructure directories where ooni software writes/reads from
commit 2db26224cc0e79a8b144bd590287cd688c7891e0
Author: Arturo Filastò <art@xxxxxxxxx>
Date: Sat Jan 3 17:48:18 2015 +0100
Restructure directories where ooni software writes/reads from
This implements: https://trac.torproject.org/projects/tor/ticket/14086
---
data/ooniprobe.conf.sample | 2 -
ooni/deckgen/cli.py | 4 +-
ooni/deckgen/processors/citizenlab_test_lists.py | 16 ++--
ooni/deckgen/processors/namebench_dns_servers.py | 8 +-
ooni/geoip.py | 6 +-
ooni/resources/__init__.py | 24 ++++--
ooni/resources/update.py | 5 +-
ooni/settings.py | 88 +++++++++++++++-------
setup.py | 25 +++++-
9 files changed, 118 insertions(+), 60 deletions(-)
diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample
index 831525a..3c9cea9 100644
--- a/data/ooniprobe.conf.sample
+++ b/data/ooniprobe.conf.sample
@@ -45,8 +45,6 @@ advanced:
reporting_retries: 3
# How many reports to perform concurrently
reporting_concurrency: 15
- # Specify here a custom data_dir path
- data_dir: /usr/share/ooni
oonid_api_port: 8042
tor:
#socks_port: 8801
diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py
index 216b9a7..3f4342a 100644
--- a/ooni/deckgen/cli.py
+++ b/ooni/deckgen/cli.py
@@ -118,6 +118,7 @@ def generate_deck(options):
@defer.inlineCallbacks
def get_user_country_code():
+ config.privacy.includecountry = True
probe_ip = ProbeIP()
yield probe_ip.lookup()
defer.returnValue(probe_ip.geodata['countrycode'])
@@ -155,9 +156,6 @@ def run():
"deck-%s" % options['country-code'])
options['output'] = output_dir
- config.initialize_ooni_home()
- config.read_config_file()
-
try:
os.makedirs(options['output'])
except OSError as exception:
diff --git a/ooni/deckgen/processors/citizenlab_test_lists.py b/ooni/deckgen/processors/citizenlab_test_lists.py
index b36e726..b455008 100644
--- a/ooni/deckgen/processors/citizenlab_test_lists.py
+++ b/ooni/deckgen/processors/citizenlab_test_lists.py
@@ -26,10 +26,10 @@ def generate_country_input(country_code, dst):
country_code = country_code.lower()
filename = os.path.join(dst, "citizenlab-urls-%s.txt" % country_code)
- input_list = os.path.join(config.resources_directory,
- "citizenlab-test-lists",
- "test-lists-master",
- "csv", country_code + ".csv")
+ input_list = config.get_data_file_path("resources/"
+ "citizenlab-test-lists/"
+ "test-lists-master/csv/"
+ + country_code + ".csv")
if not os.path.exists(input_list):
raise Exception("Could not find list for country %s" % country_code)
@@ -42,10 +42,10 @@ def generate_country_input(country_code, dst):
def generate_global_input(dst):
filename = os.path.join(dst, "citizenlab-urls-global.txt")
- input_list = os.path.join(config.resources_directory,
- "citizenlab-test-lists",
- "test-lists-master",
- "csv", "global.csv")
+ input_list = config.get_data_file_path("resources/"
+ "citizenlab-test-lists/"
+ "test-lists-master/csv/"
+ "global.csv")
load_input(input_list, filename)
diff --git a/ooni/deckgen/processors/namebench_dns_servers.py b/ooni/deckgen/processors/namebench_dns_servers.py
index e434f72..f35ca93 100644
--- a/ooni/deckgen/processors/namebench_dns_servers.py
+++ b/ooni/deckgen/processors/namebench_dns_servers.py
@@ -14,8 +14,7 @@ class GeoIPDB(object):
self.__dict__ = self._borg
if not self.country:
try:
- country_file = os.path.join(config.advanced.geoip_data_dir,
- 'GeoIP.dat')
+ country_file = config.get_data_file_path('GeoIP/GeoIP.dat')
self.country = GeoIP.open(country_file,
GeoIP.GEOIP_STANDARD)
except:
@@ -25,8 +24,9 @@ class GeoIPDB(object):
def generate_country_input(country_code, dst):
- csv_file = os.path.join(config.resources_directory,
- "namebench-dns-servers.csv")
+ csv_file = config.get_data_file_path("resources/"
+ "namebench-dns-servers.csv")
+
filename = os.path.join(dst, "dns-server-%s.txt" % country_code)
fw = open(filename, "w")
geoip_db = GeoIPDB()
diff --git a/ooni/geoip.py b/ooni/geoip.py
index 48e1134..9c3ee31 100644
--- a/ooni/geoip.py
+++ b/ooni/geoip.py
@@ -30,9 +30,9 @@ class GeoIPDataFilesNotFound(Exception):
def IPToLocation(ipaddr):
from ooni.settings import config
- city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat')
- country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat')
- asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat')
+ city_file = config.get_data_file_path('GeoIP/GeoLiteCity.dat')
+ country_file = config.get_data_file_path('GeoIP/GeoIP.dat')
+ asn_file = config.get_data_file_path('GeoIP/GeoIPASNum.dat')
location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'}
diff --git a/ooni/resources/__init__.py b/ooni/resources/__init__.py
index 71e0a95..46fad05 100644
--- a/ooni/resources/__init__.py
+++ b/ooni/resources/__init__.py
@@ -1,14 +1,24 @@
+import os
+
from ooni.settings import config
from ooni.utils import unzip, gunzip
from ooni.deckgen.processors import citizenlab_test_lists
from ooni.deckgen.processors import namebench_dns_servers
-config.initialize_ooni_home()
-config.read_config_file()
-
__version__ = "0.0.1"
+if os.access(config.var_lib_path, os.W_OK):
+ resources_directory = os.path.join(config.var_lib_path,
+ "resources")
+ geoip_directory = os.path.join(config.var_lib_path,
+ "GeoIP")
+else:
+ resources_directory = os.path.join(config.ooni_home,
+ "resources")
+ geoip_directory = os.path.join(config.ooni_home,
+ "GeoIP")
+
inputs = {
"namebench-dns-servers.csv": {
"url": "https://namebench.googlecode.com/svn/trunk/config/servers.csv";,
@@ -19,7 +29,7 @@ inputs = {
"citizenlab-test-lists.zip": {
"url": "https://github.com/citizenlab/test-lists/archive/master.zip";,
"action": unzip,
- "action_args": [config.resources_directory],
+ "action_args": [resources_directory],
"processor": citizenlab_test_lists
}
}
@@ -29,21 +39,21 @@ geoip = {
"url": "https://geolite.maxmind.com/download/";
"geoip/database/GeoLiteCity.dat.gz",
"action": gunzip,
- "action_args": [config.advanced.geoip_data_dir],
+ "action_args": [geoip_directory],
"processor": None
},
"GeoIPASNum.dat.gz": {
"url": "https://geolite.maxmind.com/download/";
"geoip/database/asnum/GeoIPASNum.dat.gz",
"action": gunzip,
- "action_args": [config.advanced.geoip_data_dir],
+ "action_args": [geoip_directory],
"processor": None
},
"GeoIP.dat.gz": {
"url": "https://geolite.maxmind.com/";
"download/geoip/database/GeoLiteCountry/GeoIP.dat.gz",
"action": gunzip,
- "action_args": [config.advanced.geoip_data_dir],
+ "action_args": [geoip_directory],
"processor": None
}
}
diff --git a/ooni/resources/update.py b/ooni/resources/update.py
index 4017965..7a949c9 100644
--- a/ooni/resources/update.py
+++ b/ooni/resources/update.py
@@ -3,8 +3,7 @@ import os
from twisted.internet import defer
from twisted.web.client import downloadPage
-from ooni.settings import config
-from ooni.resources import inputs, geoip
+from ooni.resources import inputs, geoip, resources_directory
from ooni.utils import unzip, gunzip
@@ -17,7 +16,7 @@ def download_resource(resources):
dirname = resource["action_args"][0]
filename = os.path.join(dirname, filename)
else:
- filename = os.path.join(config.resources_directory, filename)
+ filename = os.path.join(resources_directory, filename)
if not os.path.exists(filename):
directory = os.path.dirname(filename)
if not os.path.isdir(directory):
diff --git a/ooni/settings.py b/ooni/settings.py
index 5ad622d..71b449e 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -32,21 +32,63 @@ class OConfig(object):
self.privacy = Storage()
self.set_paths()
- @property
- def data_directory(self):
+ def embedded_settings(self, category, option):
embedded_settings = os.path.join(get_ooni_root(), 'settings.ini')
- if os.getenv("OONI_DATA_DIR"):
- return os.getenv("OONI_DATA_DIR")
- elif self.global_options.get('datadir'):
- return abspath(expanduser(self.global_options['datadir']))
- elif self.advanced.get('data_dir'):
- return self.advanced['data_dir']
- elif os.path.isfile(embedded_settings):
+ if os.path.isfile(embedded_settings):
settings = SafeConfigParser()
with open(embedded_settings) as fp:
settings.readfp(fp)
- return os.path.abspath(settings.get("directories", "data_dir"))
- return abspath(os.path.join(get_ooni_root(), '..', 'data'))
+ return settings.get(category, option)
+ return None
+
+ @property
+ def var_lib_path(self):
+ var_lib_path = self.embedded_settings("directories", "var_lib")
+ if var_lib_path:
+ return os.path.abspath(var_lib_path)
+ return "/var/lib/ooni"
+
+ @property
+ def usr_share_path(self):
+ usr_share_path = self.embedded_settings("directories", "usr_share")
+ if usr_share_path:
+ return os.path.abspath(usr_share_path)
+ return "/usr/share/ooni"
+
+ @property
+ def data_directory_candidates(self):
+ dirs = [
+ os.path.join(expanduser('~'+self.current_user), '.ooni'),
+ self.var_lib_path,
+ self.usr_share_path,
+ os.path.join(get_ooni_root(), '..', 'data'),
+ '/usr/share/'
+ ]
+ if os.getenv("OONI_DATA_DIR"):
+ dirs.insert(0, os.getenv("OONI_DATA_DIR"))
+ if self.global_options.get('datadir'):
+ dirs.insert(0, abspath(expanduser(self.global_options['datadir'])))
+ return dirs
+
+ @property
+ def data_directory(self):
+ for target_dir in self.data_directory_candidates:
+ if os.path.isdir(target_dir):
+ return target_dir
+ return self.var_lib_path
+
+ @property
+ def ooni_home(self):
+ if self._custom_home:
+ return self._custom_home
+ return os.path.join(expanduser('~'+self.current_user),
+ '.ooni')
+
+ def get_data_file_path(self, file_name):
+ for target_dir in self.data_directory_candidates:
+ file_path = os.path.join(target_dir, file_name)
+ if os.path.isfile(file_path):
+ return file_path
def set_paths(self, ooni_home=None):
if ooni_home:
@@ -54,15 +96,12 @@ class OConfig(object):
self.nettest_directory = os.path.join(get_ooni_root(), 'nettests')
- self.ooni_home = os.path.join(expanduser('~'+self.current_user),
- '.ooni')
- if self._custom_home:
- self.ooni_home = self._custom_home
self.inputs_directory = os.path.join(self.ooni_home, 'inputs')
self.decks_directory = os.path.join(self.ooni_home, 'decks')
self.reports_directory = os.path.join(self.ooni_home, 'reports')
self.report_log_file = os.path.join(self.ooni_home, 'reporting.yml')
- self.resources_directory = os.path.join(self.data_directory, "resources")
+ self.resources_directory = os.path.join(self.data_directory,
+ 'resources')
if self.global_options.get('configfile'):
config_file = self.global_options['configfile']
@@ -71,12 +110,8 @@ class OConfig(object):
self.config_file = os.path.join(self.ooni_home, 'ooniprobe.conf')
if 'logfile' in self.basic:
- self.basic.logfile = expanduser(self.basic.logfile.replace('~','~'+self.current_user))
-
- if not os.path.exists(self.data_directory):
- log.err("Data directory %s does not exists" % self.data_directory)
- log.err("Edit data_dir inside of %s" % self.config_file)
-
+ self.basic.logfile = expanduser(self.basic.logfile.replace(
+ '~', '~'+self.current_user))
def initialize_ooni_home(self, ooni_home=None):
if ooni_home:
@@ -94,14 +129,15 @@ class OConfig(object):
def _create_config_file(self):
target_config_file = self.config_file
print "Creating it for you in '%s'." % target_config_file
- sample_config_file = os.path.join(self.data_directory,
- 'ooniprobe.conf.sample')
+ sample_config_file = self.get_data_file_path('ooniprobe.conf.sample')
with open(sample_config_file) as f:
with open(target_config_file, 'w+') as w:
for line in f:
if line.startswith(' logfile: '):
- w.write(' logfile: %s\n' % os.path.join(self.ooni_home, 'ooniprobe.log'))
+ w.write(' logfile: %s\n' % (
+ os.path.join(self.ooni_home, 'ooniprobe.log'))
+ )
else:
w.write(line)
@@ -176,4 +212,4 @@ config = OConfig()
if not os.path.isfile(config.config_file) \
and os.path.isfile('/etc/ooni/ooniprobe.conf'):
config.global_options['configfile'] = '/etc/ooniprobe.conf'
- config.set_paths(ooni_home=config.advanced.data_dir)
+ config.set_paths()
diff --git a/setup.py b/setup.py
index 9b66169..62335d9 100644
--- a/setup.py
+++ b/setup.py
@@ -24,7 +24,13 @@ class install(_st_install):
o.close()
return config_file
- def set_data_files(self, share_path):
+ def set_data_files(self, prefix):
+ share_path = pj(prefix, 'share')
+ if prefix.startswith("/usr"):
+ var_path = "/var/lib/"
+ else:
+ var_path = pj(prefix, 'var', 'lib')
+
for root, dirs, file_names in os.walk('data/'):
files = []
for file_name in file_names:
@@ -45,14 +51,25 @@ class install(_st_install):
)
settings = SafeConfigParser()
settings.add_section("directories")
- settings.set("directories", "data_dir",
+ settings.set("directories", "usr_share",
os.path.join(share_path, "ooni"))
+ settings.set("directories", "var_lib",
+ os.path.join(var_path, "ooni"))
with open("ooni/settings.ini", "w+") as fp:
settings.write(fp)
+ try:
+ os.makedirs(pj(var_path, 'ooni'))
+ except OSError:
+ pass
+ try:
+ os.makedirs(pj(share_path, 'ooni'))
+ except OSError:
+ pass
+
def run(self):
- share_path = os.path.abspath(pj(self.prefix, 'share'))
- self.set_data_files(share_path)
+ prefix = os.path.abspath(self.prefix)
+ self.set_data_files(prefix)
self.do_egg_install()
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits