[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [ooni-probe/master] Change the default measurement concurrency to 4 and reporting to 7.
commit 71e2b411377319f88b5c19f25e0339158dfcc677
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date: Fri Jul 8 17:09:11 2016 +0200
Change the default measurement concurrency to 4 and reporting to 7.
Rewrite the configuration for lepidopter installations of ooniprobe.
* Clean up the data/ tree
---
MANIFEST.in | 2 ++
data/Makefile | 10 ----------
data/inputs/.gitignore | 2 ++
data/inputs/Makefile | 13 -------------
data/inputs/README | 47 ----------------------------------------------
data/ooniprobe.conf.sample | 10 +++++-----
data/resources/README | 3 ++-
setup.py | 22 ++++++++++++++++++++--
8 files changed, 31 insertions(+), 78 deletions(-)
diff --git a/MANIFEST.in b/MANIFEST.in
index 920f933..e2903fb 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -6,4 +6,6 @@ include data/ooniprobe.1
include data/oonireport.1
include data/ooniresources.1
include data/ooniprobe.conf.sample
+include data/configs/lepidopter-ooniprobe.conf
+include data/configs/lepidopter-oonireport.conf
include ooni/settings.ini
diff --git a/data/Makefile b/data/Makefile
deleted file mode 100644
index 969cc65..0000000
--- a/data/Makefile
+++ /dev/null
@@ -1,10 +0,0 @@
-geoip:
- wget -N http://www.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz
- gunzip -f GeoIPASNum.dat.gz
- wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
- gunzip -f GeoIP.dat.gz
- wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz
- gunzip -f GeoLiteCity.dat.gz
-clean:
- rm -f *.dat
-
diff --git a/data/inputs/.gitignore b/data/inputs/.gitignore
new file mode 100644
index 0000000..d6b7ef3
--- /dev/null
+++ b/data/inputs/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/data/inputs/Makefile b/data/inputs/Makefile
deleted file mode 100644
index ff74cf6..0000000
--- a/data/inputs/Makefile
+++ /dev/null
@@ -1,13 +0,0 @@
-# This file will fetch a set of inputs that are of use to ooni-probe
-all: whatheaders lists
-
-lists:
- curl -O https://ooni.torproject.org/inputs/input-pack.tar.gz
- tar xzf input-pack.tar.gz
- rm input-pack.tar.gz
-
-whatheaders:
- wget http://s3.amazonaws.com/data.whatheaders.com/whatheaders-latest.xml.zip
- unzip whatheaders-latest.xml.zip
- mv whatheaders*.xml whatheaders.xml
- rm whatheaders-latest.xml.zip
diff --git a/data/inputs/README b/data/inputs/README
deleted file mode 100644
index 9a55f67..0000000
--- a/data/inputs/README
+++ /dev/null
@@ -1,47 +0,0 @@
-In here you will find some very simple input lists that are useful for testing
-the correct functionality of the various OONIProbe tests.
-
-# DNS Consistency
-
-./ooniprobe-dev -o dns_tamper_test.yamloo data/nettests/blocking/dns_consistency.py -t
-example_inputs/dns_tamper_test_resolvers.txt -f example_inputs/dns_tamper_file.txt
-
-less dns_tamper_test.yamloo
-
-# Captive Portal
-
-./ooniprobe-dev -o captive_portal_test.yamloo data/nettests/core/captiveportal.py
-
-less captive_portal_test.yamloo
-
-# HTTP Host
-
-./ooniprobe-dev -o http_host.yamloo data/nettests/manipulation/http_host.py -b http://ooni.nu/test -f example_inputs/http_host_file.txt
-
-less http_host.yamloo
-
-# Keyword filtering
-
-./ooniprobe-dev -o keyword_filtering.yamloo data/nettests/core/keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt
-
-less keyword_filtering.yamloo
-
-# URL List
-
-./ooniprobe-dev -o url_lists.yamloo data/nettests/core/url_list.py -f test_inputs/url_lists_file.txt
-
-less url_lists.yamloo
-
-# Squid transparent proxy
-
-./ooniprobe-dev -o squid.yamloo data/nettests/core/squid.py
-
-less squid.yamloo
-
-# HTTP Requests
-
-Not Implemented
-
-# Traceroute
-
-Not Implemented
diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample
index e310ddc..cde4c00 100644
--- a/data/ooniprobe.conf.sample
+++ b/data/ooniprobe.conf.sample
@@ -33,17 +33,17 @@ advanced:
# explicitly set the control port and SOCKS port
start_tor: true
# After how many seconds we should give up on a particular measurement
- measurement_timeout: 60
+ measurement_timeout: 120
# After how many retries we should give up on a measurement
measurement_retries: 2
# How many measurements to perform concurrently
- measurement_concurrency: 10
+ measurement_concurrency: 4
# After how may seconds we should give up reporting
- reporting_timeout: 80
+ reporting_timeout: 360
# After how many retries to give up on reporting
- reporting_retries: 3
+ reporting_retries: 5
# How many reports to perform concurrently
- reporting_concurrency: 15
+ reporting_concurrency: 7
oonid_api_port: 8042
report_log_file: null
inputs_dir: null
diff --git a/data/resources/README b/data/resources/README
index 5157b33..fb768f4 100644
--- a/data/resources/README
+++ b/data/resources/README
@@ -1 +1,2 @@
-In here shall be stored all the resources needed for using ooniresources and oonideckgen.
+In here shall be stored all the resources downloaded via ooniresources to be
+used by oonideckgen.
diff --git a/setup.py b/setup.py
index 1138ee3..1a8138c 100644
--- a/setup.py
+++ b/setup.py
@@ -86,6 +86,8 @@ To setup a daily cronjob run this:
Have fun!
"""
+from __future__ import print_function
+
from ooni import __version__, __author__
import os
import sys
@@ -106,13 +108,18 @@ TEST_LISTS_URL = "https://github.com/citizenlab/test-lists/archive/master.zip";
def run_command(args, cwd=None):
try:
p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
- except EnvironmentError as e:
+ except EnvironmentError:
return None
stdout = p.communicate()[0].strip()
if p.returncode != 0:
return None
return stdout
+def is_lepidopter():
+ if os.path.exists("/etc/default/lepidopter"):
+ return True
+ return False
+
class OoniInstall(install):
def gen_config(self, share_path):
config_file = pj(tempfile.mkdtemp(), "ooniprobe.conf.sample")
@@ -175,11 +182,22 @@ class OoniInstall(install):
stderr=sys.stderr.fileno())
process.wait()
+ def update_lepidopter_config(self):
+ try:
+ shutil.copyfile("data/configs/lepidopter-ooniprobe.conf",
+ "/etc/ooniprobe/ooniprobe.conf")
+ shutil.copyfile("data/configs/lepidopter-oonireport.conf",
+ "/etc/ooniprobe/oonireport.conf")
+ except Exception:
+ print("ERR: Failed to copy configuration files to /etc/ooniprobe/")
+
def run(self):
prefix = os.path.abspath(self.prefix)
self.set_data_files(prefix)
self.do_egg_install()
self.ooniresources()
+ if is_lepidopter():
+ self.update_lepidopter_config()
class ExecutableNotFound(Exception):
pass
@@ -226,7 +244,7 @@ class CreateOoniResources(Command):
try:
self.find_executables()
except ExecutableNotFound as enf:
- print("Could not find '{0}'".format(enf.message))
+ print("ERR: Could not find '{0}'".format(enf.message))
return
tmp_dir = tempfile.mkdtemp()
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits