[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [metrics-cloud/master] Initial CloudFormation template and Ansible for exit scanner
commit c4aaea174394a8ba612ca898a443fc07c4813bbf
Author: Iain R. Learmonth <irl@xxxxxxxxxxxxxx>
Date: Thu Feb 20 14:11:21 2020 +0000
Initial CloudFormation template and Ansible for exit scanner
The exitmap module used for the exit scanner is maintained as part of
metrics-cloud. If it were rewritten to be less of a hack, it might be
imported into the upstream exitmap repository.
---
ansible/exit-scanners-aws.yml | 9 ++
ansible/roles/exit-scanner-sys/tasks/main.yml | 83 ++++++++++++++++
ansible/roles/exit-scanner/files/exitscan.py | 105 +++++++++++++++++++++
.../roles/exit-scanner/files/exitscanner.service | 10 ++
ansible/roles/exit-scanner/files/ipscan.py | 95 +++++++++++++++++++
ansible/roles/exit-scanner/tasks/main.yml | 53 +++++++++++
cloudformation/exit-scanner-dev.yml | 27 ++++++
7 files changed, 382 insertions(+)
diff --git a/ansible/exit-scanners-aws.yml b/ansible/exit-scanners-aws.yml
new file mode 100644
index 0000000..72ce0c6
--- /dev/null
+++ b/ansible/exit-scanners-aws.yml
@@ -0,0 +1,9 @@
+---
+- hosts: exit-scanners
+ user: admin
+ vars:
+ onionoo_version: 7.0-1.21.0
+ roles:
+ - tor-client
+ - exit-scanner-sys
+ - exit-scanner
diff --git a/ansible/roles/exit-scanner-sys/tasks/main.yml b/ansible/roles/exit-scanner-sys/tasks/main.yml
new file mode 100644
index 0000000..78916d8
--- /dev/null
+++ b/ansible/roles/exit-scanner-sys/tasks/main.yml
@@ -0,0 +1,83 @@
+---
+- name: disable system tor
+ systemd:
+ name: tor.service
+ enabled: false
+ state: stopped
+ become: true
+- name: install stem for py2 from backports
+ apt:
+ pkg: python-stem
+ state: latest
+ default_release: buster-backports
+ become: true
+- name: install stem for py3 from backports
+ apt:
+ pkg: python3-stem
+ state: latest
+ default_release: buster-backports
+ become: true
+- name: install exitmap requirements
+ apt:
+ pkg:
+ - git
+ - python-dnspython
+ update_cache: yes
+ become: yes
+- name: create check account
+ user:
+ name: check
+ comment: "Check Service User"
+ #uid: 1547
+ state: present
+ become: yes
+- name: create tordnsel account
+ user:
+ name: tordnsel
+ comment: "Exit Scanner Service User"
+ #uid: 1547
+ state: present
+ become: yes
+- name: create service directory
+ file:
+ path: /srv/exitscanner.torproject.org
+ state: directory
+ become: yes
+- name: link /home in /srv
+ file:
+ src: /home
+ dest: /srv/home
+ state: link
+ become: yes
+- name: link home directories /home
+ file:
+ src: "{{ item.src }}"
+ dest: "{{ item.dest }}"
+ state: link
+ force: yes
+ with_items:
+ - { src: /home/tordnsel, dest: /srv/exitscanner.torproject.org/home }
+ - { src: /home/check, dest: /srv/exitscanner.torproject.org/check-home }
+ become: yes
+- name: create exit scanner runtime directory
+ file:
+ path: /srv/exitscanner.torproject.org/exitscanner
+ owner: tordnsel
+ group: tordnsel
+ mode: 0755
+ state: directory
+ become: yes
+- name: create check runtime directory
+ file:
+ path: /srv/exitscanner.torproject.org/check
+ owner: check
+ group: check
+ mode: 0755
+ state: directory
+ become: yes
+- name: enable lingering for service users
+ shell: "loginctl enable-linger {{ item }}"
+ with_items:
+ - tordnsel
+ - check
+ become: yes
diff --git a/ansible/roles/exit-scanner/files/exitscan.py b/ansible/roles/exit-scanner/files/exitscan.py
new file mode 100644
index 0000000..14c0b17
--- /dev/null
+++ b/ansible/roles/exit-scanner/files/exitscan.py
@@ -0,0 +1,105 @@
+
+import collections
+import datetime
+import glob
+import json
+import os
+import os.path
+import re
+import subprocess
+
+import stem.descriptor
+
+fortyeighthoursago = datetime.datetime.utcnow() - datetime.timedelta(hours=48)
+
+Measurement = collections.namedtuple("Measurement", ["address", "date"])
+exits = dict()
+
+
+def merge_addresses(fp, new):
+ addresses = exits[fp].exit_addresses
+ addresses.extend(new)
+ addresses.sort(key=lambda x: x[1], reverse=True)
+ uniq_addresses = []
+ while len(uniq_addresses) < len(addresses):
+ if addresses[len(uniq_addresses)][0] in uniq_addresses:
+ addresses.remove(addresses[len(uniq_addresses)])
+ continue
+ uniq_addresses.append(addresses[len(uniq_addresses)][0])
+ return [
+ a for a in addresses
+ if a[1] > fortyeighthoursago
+ ]
+
+
+def merge(desc):
+ if desc.fingerprint not in exits:
+ exits[desc.fingerprint] = desc
+ return
+ fp = desc.fingerprint
+ exits[fp].published = max(exits[fp].published, desc.published)
+ exits[fp].last_status = max(exits[fp].last_status, desc.last_status)
+ exits[fp].exit_addresses = merge_addresses(fp, desc.exit_addresses)
+
+
+def run():
+ exit_lists = list(glob.iglob('lists/2*')) # fix this glob before 23:59 on 31st Dec 2999
+
+ # Import latest exit list from disc
+ if exit_lists:
+ latest_exit_list = max(exit_lists, key=os.path.getctime)
+ for desc in stem.descriptor.parse_file(latest_exit_list,
+ descriptor_type="tordnsel 1.0"):
+ merge(desc)
+
+ # Import new measurements
+ with subprocess.Popen(["./bin/exitmap", "ipscan", "-o", "/dev/stdout"],
+ cwd="/srv/exitscanner.torproject.org/exitscanner/exitmap",
+ stdout=subprocess.PIPE,
+ encoding='utf-8') as p:
+ for line in p.stdout:
+ print(line)
+ result = re.match(
+ r"^([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}),[0-9]{3} modules\.ipscan \[INFO\] (\{.*\})$",
+ line)
+ if result:
+ print(result)
+ check_result = json.loads(result.group(2))
+ desc = stem.descriptor.tordnsel.TorDNSEL("", False)
+ desc.fingerprint = check_result["Fingerprint"]
+ desc.last_status = datetime.datetime.utcnow().replace(minute=0, second=0, microsecond=0)
+ desc.published = datetime.datetime.strptime(
+ check_result["DescPublished"], "%Y-%m-%dT%H:%M:%S")
+ desc.exit_addresses = [
+ (check_result["IP"],
+ datetime.datetime.strptime(result.group(1),
+ "%Y-%m-%d %H:%M:%S"))
+ ]
+ merge(desc)
+
+ # Format exit list filename
+ now = datetime.datetime.utcnow()
+ filename = (f"{now.year}-{now.month:02d}-"
+ f"{now.day:02d}-{now.hour:02d}-"
+ f"{now.minute:02d}-{now.second:02d}")
+
+ # Format an exit list
+ with open(f"lists/{filename}", "w") as out:
+ for desc in exits.values():
+ if desc.exit_addresses:
+ out.write(f"ExitNode {desc.fingerprint}\n")
+ out.write(f"Published {desc.published}\n")
+ out.write(f"LastStatus {desc.last_status}\n")
+ for a in desc.exit_addresses:
+ out.write(f"ExitAddress {a[0]} {a[1]}\n")
+
+ # Provide the snapshot emulation
+ os.unlink("lists/latest")
+ os.symlink(os.path.abspath(f"lists/{filename}"), "lists/latest")
+
+if __name__ == "__main__":
+ while True:
+ start = datetime.datetime.utcnow()
+ run()
+ while datetime.datetime.utcnow() < start + datetime.timedelta(minutes=40):
+ pass
diff --git a/ansible/roles/exit-scanner/files/exitscanner.service b/ansible/roles/exit-scanner/files/exitscanner.service
new file mode 100644
index 0000000..012d8b7
--- /dev/null
+++ b/ansible/roles/exit-scanner/files/exitscanner.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=Exit Scanner
+
+[Service]
+Type=simple
+WorkingDirectory=/srv/exitscanner.torproject.org/exitscanner
+ExecStart=/usr/bin/python3 /srv/exitscanner.torproject.org/exitscanner/exitscan.py
+
+[Install]
+WantedBy=default.target
diff --git a/ansible/roles/exit-scanner/files/ipscan.py b/ansible/roles/exit-scanner/files/ipscan.py
new file mode 100644
index 0000000..d59ce4c
--- /dev/null
+++ b/ansible/roles/exit-scanner/files/ipscan.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python2
+
+# Copyright 2013-2017 Philipp Winter <phw@xxxxxxxxx>
+#
+# This file is part of exitmap.
+#
+# exitmap is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# exitmap is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with exitmap. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Module to detect false negatives for <https://check.torproject.org>.
+"""
+
+import sys
+import json
+import logging
+try:
+ import urllib2
+except ImportError:
+ import urllib.request as urllib2
+
+from util import exiturl
+
+import stem.descriptor.server_descriptor as descriptor
+
+log = logging.getLogger(__name__)
+
+# exitmap needs this variable to figure out which relays can exit to the given
+# destination(s).
+
+destinations = [("check.torproject.org", 443)]
+
+
+def fetch_page(exit_desc):
+ """
+ Fetch check.torproject.org and see if we are using Tor.
+ """
+
+ data = None
+ url = exiturl(exit_desc.fingerprint)
+
+ try:
+ data = urllib2.urlopen("https://check.torproject.org/api/ip";,
+ timeout=10).read()
+ except Exception as err:
+ log.debug("urllib2.urlopen says: %s" % err)
+ return
+
+ if not data:
+ return
+
+ try:
+ check_answer = json.loads(data)
+ except ValueError as err:
+ log.warning("Couldn't parse JSON over relay %s: %s" % (url, data))
+ return
+
+ check_answer["DescPublished"] = exit_desc.published.isoformat()
+ check_answer["Fingerprint"] = exit_desc.fingerprint
+
+ log.info(json.dumps(check_answer))
+
+def probe(exit_desc, run_python_over_tor, run_cmd_over_tor, **kwargs):
+ """
+ Probe the given exit relay and look for check.tp.o false negatives.
+ """
+
+ run_python_over_tor(fetch_page, exit_desc)
+
+
+def main():
+ """
+ Entry point when invoked over the command line.
+ """
+
+ desc = descriptor.ServerDescriptor("")
+ desc.fingerprint = "bogus"
+ desc.address = "0.0.0.0"
+ fetch_page(desc)
+
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/ansible/roles/exit-scanner/tasks/main.yml b/ansible/roles/exit-scanner/tasks/main.yml
new file mode 100644
index 0000000..d80edb5
--- /dev/null
+++ b/ansible/roles/exit-scanner/tasks/main.yml
@@ -0,0 +1,53 @@
+---
+- name: clone the sources
+ git:
+ repo: https://github.com/NullHypothesis/exitmap.git
+ dest: /srv/exitscanner.torproject.org/exitscanner/exitmap
+ become: true
+ become_user: tordnsel
+- name: install the ipscan module
+ copy:
+ src: ipscan.py
+ dest: /srv/exitscanner.torproject.org/exitscanner/exitmap/src/modules/ipscan.py
+ mode: 0755
+ become: true
+ become_user: tordnsel
+- name: install the exit scanner script
+ copy:
+ src: exitscan.py
+ dest: /srv/exitscanner.torproject.org/exitscanner/exitscan.py
+ mode: 0755
+ become: true
+ become_user: tordnsel
+- name: create systemd user directory for exitscanner
+ file:
+ path: /srv/exitscanner.torproject.org/home/.config/systemd/user
+ state: directory
+ become: true
+ become_user: tordnsel
+- name: create exit lists directory
+ file:
+ path: /srv/exitscanner.torproject.org/exitscanner/lists
+ state: directory
+ become: true
+ become_user: tordnsel
+- name: install exit scanner service file
+ copy:
+ src: exitscanner.service
+ dest: "/srv/exitscanner.torproject.org/home/.config/systemd/user/exitscanner.service"
+ become: true
+ become_user: tordnsel
+- name: reload systemd daemon
+ systemd:
+ scope: user
+ daemon_reload: yes
+ become: true
+ become_user: tordnsel
+- name: enable and start exitscanner service
+ systemd:
+ scope: user
+ name: exitscanner
+ state: started
+ enabled: yes
+ become: yes
+ become_user: tordnsel
diff --git a/cloudformation/exit-scanner-dev.yml b/cloudformation/exit-scanner-dev.yml
new file mode 100644
index 0000000..2ee4259
--- /dev/null
+++ b/cloudformation/exit-scanner-dev.yml
@@ -0,0 +1,27 @@
+---
+# CloudFormation Stack for Exit Scanner development instance
+# This stack will only deploy on us-east-1 and will deploy in the Metrics VPC
+# aws cloudformation deploy --region us-east-1 --stack-name `whoami`-exit-scanner-dev --template-file exit-scanner-dev.yml --parameter-overrides myKeyPair="$(./identify_user.sh)"
+AWSTemplateFormatVersion: 2010-09-09
+Parameters:
+ myKeyPair:
+ Description: Amazon EC2 Key Pair
+ Type: "AWS::EC2::KeyPair::KeyName"
+Resources:
+ Instance:
+ Type: AWS::EC2::Instance
+ Properties:
+ AvailabilityZone: us-east-1a
+ ImageId: ami-01db78123b2b99496
+ InstanceType: t2.large
+ SubnetId:
+ Fn::ImportValue: 'MetricsSubnet'
+ KeyName: !Ref myKeyPair
+ SecurityGroupIds:
+ - Fn::ImportValue: 'MetricsInternetSecurityGroup'
+ - Fn::ImportValue: 'MetricsPingableSecurityGroup'
+ - Fn::ImportValue: 'MetricsHTTPSSecurityGroup'
+Outputs:
+ PublicIp:
+ Description: "Instance public IP"
+ Value: !GetAtt Instance.PublicIp
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits