[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [support-tools/master] Add scripts to monitor changes to RT articles



commit 26e45154a20650fa8d2ae2c2891f7aa864d7d750
Author: Lunar <lunar@xxxxxxxxxxxxxx>
Date:   Fri Jul 18 15:54:13 2014 +0200

    Add scripts to monitor changes to RT articles
---
 rt-articles/README              |   69 +++++++++++++++++++++++++++++++++++++++
 rt-articles/dump_rt_articles    |   69 +++++++++++++++++++++++++++++++++++++++
 rt-articles/push_rt_articles    |   11 +++++++
 rt-articles/receive_rt_articles |   19 +++++++++++
 4 files changed, 168 insertions(+)

diff --git a/rt-articles/README b/rt-articles/README
new file mode 100644
index 0000000..7fb39c2
--- /dev/null
+++ b/rt-articles/README
@@ -0,0 +1,69 @@
+This is a set of script to extract RT articles from the database and
+track when they change.
+
+Setup notes
+===========
+
+Two systems are involved. rude.torproject.org which hosts RT and
+perdulce.torproject.org which hosts people.torproject.org. We are going
+to allow a very restricted push of data from the formor to the latter.
+
+On rude.torproject.org
+----------------------
+
+Create SSH key:
+
+    ssh-keygen -t rsa -b 4096 -N '' -f .ssh/id_rsa_rt_articles
+
+Configure SSH (through `~/.ssh/config`):
+
+    Host perdulce.torproject.org
+            BatchMode yes
+            IdentityFile ~/.ssh/id_rsa_rt_articles
+
+Add cron job:
+
+    17,47 * * * * /srv/rtstuff/support-tools/rt-articles/push_rt_articles
+
+On perdulce.torproject.org
+--------------------------
+
+Install the public key from rude. To do that, use email interface of
+ud-ldap: <https://db.debian.org/doc-mail.html>. *But replace
+`debian.org` with `torproject.org`!*
+
+    ssh-rsa AAAAâ?¦== usual key
+    allowed_hosts=perdulce.debian.org from="rude.torproject.org",command="~/bin/receive_rt_articles",no-pty,no-port-forwarding ssh-rsa AAAAâ?¦== key from rude
+
+Create the Git repository:
+
+    cd public_html
+    mkdir rt-articles.git && cd rt-articles.git
+    git init --bare
+    vi description
+    mv hooks/post-update.sample hooks/post-update
+    chmod +x hooks/post-update
+
+Configure the email notification:
+
+    echo 'sh /usr/share/git-core/contrib/hooks/post-receive-email' >> hooks/post-receive
+    chmod +x hooks/post-receive
+    git config hooks.mailinglist lunar@xxxxxxxxxxxxxx
+    git config hooks.envelopesender lunar+rt-articles@xxxxxxxxxxxxxx
+    git config hooks.emailprefix '[RT Articles] '
+    git config hooks.diffopts '--stat --summary --find-copies-harder -p'
+
+Do the initial clone:
+
+    cd $HOME
+    git clone public_html/rt-articles.git
+
+Also clone the script repository:
+
+    cd $HOME
+    git clone https://git.torproject.org/support-tools.git
+
+And add the symlink:
+
+    mkdir -p bin
+    ln -s ../support-tools/receive_rt_articles bin
diff --git a/rt-articles/dump_rt_articles b/rt-articles/dump_rt_articles
new file mode 100755
index 0000000..b47bf2f
--- /dev/null
+++ b/rt-articles/dump_rt_articles
@@ -0,0 +1,69 @@
+#!/usr/bin/python
+#
+# This program is free software. It comes without any warranty, to
+# the extent permitted by applicable law. You can redistribute it
+# and/or modify it under the terms of the Do What The Fuck You Want
+# To Public License, Version 2, as published by Sam Hocevar. See
+# http://sam.zoy.org/wtfpl/COPYING for more details.
+#
+# This script will dump all RT articles to a given directory.
+# Ideally, the directory should be empty.
+
+import psycopg2
+import re
+import sys
+import os
+import os.path
+
+RT_CONNINFO = "host=drobovi.torproject.org sslmode=require user=rtreader dbname=rt"
+
+SELECT_TRANSACTIONS_QUERY = """
+SELECT classes.name AS class,
+       articles.name AS title,
+       CASE WHEN objectcustomfieldvalues.content != '' THEN objectcustomfieldvalues.content
+            ELSE objectcustomfieldvalues.largecontent
+       END AS content,
+       objectcustomfieldvalues.lastupdated,
+       articles.id
+  FROM classes, articles, objectcustomfieldvalues
+ WHERE articles.class = classes.id
+   AND objectcustomfieldvalues.objecttype = 'RT::Article'
+   AND objectcustomfieldvalues.objectid = articles.id
+   AND objectcustomfieldvalues.id = (
+           SELECT objectcustomfieldvalues.id
+             FROM objectcustomfieldvalues
+            WHERE objectcustomfieldvalues.objectid = articles.id
+              AND objectcustomfieldvalues.disabled = 0
+            ORDER BY objectcustomfieldvalues.lastupdated DESC
+            LIMIT 1)
+ ORDER BY classes.id, articles.id
+"""
+
+def slugify(title):
+    return re.sub(r'\W+', '-', title)
+
+def dump_rt_articles(con, path):
+    cur = con.cursor()
+    cur.execute(SELECT_TRANSACTIONS_QUERY)
+    for article_class, title, content, last_updated, article_id in cur:
+        class_slug = slugify(article_class)
+        class_dir = os.path.join(path, class_slug)
+        if not os.path.isdir(class_dir):
+            os.mkdir(class_dir)
+        filename = os.path.join(class_dir, "%03d_%s" % (article_id, slugify(title)))
+        with open(filename, 'w') as f:
+            f.write('Title: %s\n' % title)
+            f.write('Last-updated: %s\n\n' % last_updated)
+            f.write(content)
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        print >>sys.stderr, "Usage: %s DIR" % (sys.argv[0])
+        sys.exit(1)
+    path = sys.argv[1]
+    if not os.path.isdir(path):
+        print >>sys.stderr, "Usage: %s DIR" % (sys.argv[0])
+        sys.exit(2)
+    con = psycopg2.connect(RT_CONNINFO)
+    dump_rt_articles(con, path)
+    con.close()
diff --git a/rt-articles/push_rt_articles b/rt-articles/push_rt_articles
new file mode 100755
index 0000000..a48b801
--- /dev/null
+++ b/rt-articles/push_rt_articles
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# To be run on rude.torproject.org from a cronjob
+
+set -e
+
+DIR=$(mktemp -d)
+trap "rm -r '$DIR'" EXIT
+
+$(dirname "$0")/dump_rt_articles "$DIR"
+tar -C "$DIR" -zcf - . | ssh perdulce.torproject.org bin/receive_rt_articles
diff --git a/rt-articles/receive_rt_articles b/rt-articles/receive_rt_articles
new file mode 100755
index 0000000..9e1fe68
--- /dev/null
+++ b/rt-articles/receive_rt_articles
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# To be run from perdule.torproject.org through a restricted SSH command.
+# stdin will be a gzipped tarball with the articles
+
+# Must be a Git repository
+DEST="$HOME/rt-articles"
+
+set -e
+
+(
+cd "$DEST"
+git rm --quiet --force -r . || true
+tar -zxf -
+git add .
+git diff --cached --quiet ||
+	git commit --quiet --author='An automated script <lunar+rt-articles@xxxxxxxxxxxxxx>' -m "Update RT articles"
+git push --quiet origin master
+)

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits