[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [tor-browser-bundle/master] fetch-inputs: implement proper gpg checking and partial script cleanup

To: tor-commits@xxxxxxxxxxxxxxxxxxxx
Subject: [tor-commits] [tor-browser-bundle/master] fetch-inputs: implement proper gpg checking and partial script cleanup
From: mikeperry@xxxxxxxxxxxxxx
Date: Wed, 10 Jul 2013 18:00:12 +0000 (UTC)
Delivered-to: archiver@xxxxxxxx
Delivery-date: Wed, 10 Jul 2013 14:00:28 -0400
List-archive: <http://lists.torproject.org/pipermail/tor-commits/>
List-help: <mailto:tor-commits-request@lists.torproject.org?subject=help>
List-id: "auto: code repository commits" <tor-commits.lists.torproject.org>
List-post: <mailto:tor-commits@lists.torproject.org>
List-subscribe: <https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits>, <mailto:tor-commits-request@lists.torproject.org?subject=subscribe>
List-unsubscribe: <https://lists.torproject.org/cgi-bin/mailman/options/tor-commits>, <mailto:tor-commits-request@lists.torproject.org?subject=unsubscribe>
Patch-author: Peter Palfrader <peter@xxxxxxxxxxxxx>
Sender: "tor-commits" <tor-commits-bounces@xxxxxxxxxxxxxxxxxxxx>

commit 1913aee57276738ad65bfc34d3177f375b8e0d90
Author: Peter Palfrader <peter@xxxxxxxxxxxxx>
Date:   Wed Jul 10 18:56:59 2013 +0200

    fetch-inputs: implement proper gpg checking and partial script cleanup
---
 gitian/fetch-inputs.sh |  269 +++++++++++++++++++++++-------------------------
 gitian/gpg/OPENSSL.gpg |  Bin 0 -> 4316 bytes
 2 files changed, 126 insertions(+), 143 deletions(-)

diff --git a/gitian/fetch-inputs.sh b/gitian/fetch-inputs.sh
index 21728e5..cd8b4cd 100755
--- a/gitian/fetch-inputs.sh
+++ b/gitian/fetch-inputs.sh
@@ -3,90 +3,143 @@
 # fetch-inputs.sh - Fetch our inputs from the source mirror
 #
 
-. ./versions
-
+MIRROR_URL=https://people.torproject.org/~mikeperry/mirrors/sources/
+set -e
+set -u
 umask 0022
 
-export WRAPPER_DIR=$PWD
+if ! [ -e ./versions ]; then
+  echo >&2 "Error: ./versions file does not exist"
+  exit 1
+fi
+
+. ./versions
+
+WRAPPER_DIR=$(dirname "$0")
+WRAPPER_DIR=$(readlink -f "$WRAPPER_DIR")
 
-if [ -z "$1" ]; then
-  INPUTS_DIR=$PWD/../../gitian-builder/inputs
+if [ "$#" -gt 1 ]; then
+  echo >&2 "Usage: $0 [<inputsdir>]"
+  exit 1
+elif [ "$#" = 1 ]; then
+  INPUTS_DIR="$1"
 else
-  INPUTS_DIR=$1
+  INPUTS_DIR="$PWD/../../gitian-builder/inputs"
 fi
 
-if [ -n $INPUTS_DIR -a ! -d $INPUTS_DIR ];
-then
-  mkdir $INPUTS_DIR
-fi
+mkdir -p "$INPUTS_DIR"
+cd "$INPUTS_DIR"
 
-if [ -n $INPUTS_DIR -a -d $INPUTS_DIR ]; then
-  cd $INPUTS_DIR
-fi
 
-MIRROR_URL=https://people.torproject.org/~mikeperry/mirrors/sources/
+##############################################################################
+CLEANUP=$(tempfile)
+trap "bash '$CLEANUP'; rm -f '$CLEANUP'" EXIT
 
-gpg --import $WRAPPER_DIR/gpg/*
+verify() {
+  local file="$1"; shift
+  local keyring="$1"; shift
 
-# Get package files from mirror
-for i in OPENSSL TOOLCHAIN4 OSXSDK # OBFSPROXY
-do
-  PACKAGE=${i}"_PACKAGE"
-  URL=${MIRROR_URL}${!PACKAGE}
-  wget -N ${URL} #>& /dev/null
-  if [ $? -ne 0 ]; then
-    echo "$i url ${URL} is broken!"
-    mv ${!PACKAGE} ${!PACKAGE}".removed"
+  local f
+  for f in "$file" "$file.asc" "$keyring"; do
+    if ! [ -e "$f" ]; then
+      echo >&2 "Error: Required file $f does not exist."; exit 1
+    fi
+  done
+
+  local tmpfile=$(tempfile)
+  echo "rm -f '$tmpfile'" >> "$CLEANUP"
+  local gpghome=$(mktemp -d)
+  echo "rm -rf '$gpghome'" >> "$CLEANUP"
+  exec 3> "$tmpfile"
+
+  GNUPGHOME="$gpghome" gpg --no-options --no-default-keyring --trust-model=always --keyring="$keyring" --status-fd=3 --verify "$file.asc" "$file" >/dev/null 2>&1
+  if grep -q '^\[GNUPG:\] GOODSIG ' "$tmpfile"; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+get() {
+  local file="$1"; shift
+  local url="$1"; shift
+
+  if ! wget -N "$url" >& /dev/null; then
+    echo >&2 "Error: Cannot download $url"
+    mv "${file}" "${file}.DLFAILED"
     exit 1
   fi
-done
+}
+
+update_git() {
+  local dir="$1"; shift
+  local url="$1"; shift
+  local tag="${1:-}"
+
+  if [ -d "$dir/.git" ];
+  then
+    (cd "$dir" && git fetch origin && git fetch --tags origin)
+  else
+    if ! git clone "$url"; then
+      echo >&2 "Error: Cloning $url failed"
+      exit 1
+    fi
+  fi
+
+  if [ -n "$tag" ]; then
+    (cd "$dir" && git checkout "$tag")
+  fi
+}
+
+##############################################################################
+# Get package files from mirror
 
 # Get+verify sigs that exist
-# XXX: This doesn't cover everything. See #8525
 for i in OPENSSL # OBFSPROXY
 do
-  PACKAGE=${i}"_PACKAGE"
-  URL=${MIRROR_URL}${!PACKAGE}
-  if [ ! -f ${!PACKAGE}".asc" ]; then
-    wget -N ${URL}".asc" >& /dev/null
-    if [ $? -ne 0 ]; then
-      echo "$i GPG sig url ${URL} is broken!"
-      mv ${!PACKAGE} ${!PACKAGE}".nogpg"
-      exit 1
-    fi
-  fi
-  gpg ${!PACKAGE}".asc" >& /dev/null
-  if [ $? -ne 0 ]; then
-    echo "$i GPG signature is broken for ${URL}"
-    mv ${!PACKAGE} ${!PACKAGE}".badgpg"
+  PACKAGE="${i}_PACKAGE"
+  URL="${MIRROR_URL}${!PACKAGE}"
+  get "${!PACKAGE}" "$URL"
+  get "${!PACKAGE}.asc" "$URL.asc"
+
+  if ! verify "${!PACKAGE}" "$WRAPPER_DIR/gpg/$i.gpg"; then
+    echo "$i: GPG signature is broken for ${URL}"
+    mv "${!PACKAGE}" "${!PACKAGE}.badgpg"
     exit 1
   fi
 done
 
+# XXX: This doesn't cover everything. See #8525
+for i in TOOLCHAIN4 OSXSDK
+do
+  PACKAGE="${i}_PACKAGE"
+  URL="${MIRROR_URL}${!PACKAGE}"
+  get "${!PACKAGE}" "${MIRROR_URL}${!PACKAGE}"
+  echo >&2 "Warning, not verifying signature for $i"
+done
+
 # Verify packages with weak or no signatures via multipath downloads
 # (OpenSSL is signed with MD5, and OSXSDK is not signed at all)
 mkdir -p verify
 cd verify
 for i in OPENSSL OSXSDK
 do
-  URL=${i}"_URL"
-  PACKAGE=${i}"_PACKAGE"
-  wget -N --no-remove-listing ${!URL} >& /dev/null
-  if [ $? -ne 0 ]; then
+  URL="${i}_URL"
+  PACKAGE="${i}_PACKAGE"
+  if ! wget -N --no-remove-listing "${!URL}"; then
     echo "$i url ${!URL} is broken!"
-    mv ${!PACKAGE} ${!PACKAGE}".removed"
+    mv "${!PACKAGE}" "${!PACKAGE}.removed"
     exit 1
   fi
 done
 # XXX: Google won't allow wget -N.. We need to re-download the whole
 # TOOLCHAIN4 each time :/
-rm -f $TOOLCHAIN4_PACKAGE
-wget $TOOLCHAIN4_URL
+rm -f "$TOOLCHAIN4_PACKAGE"
+wget "$TOOLCHAIN4_URL"
 for i in OPENSSL OSXSDK TOOLCHAIN4
 do
-   PACKAGE=${i}"_PACKAGE"
-   diff ${!PACKAGE} ../${!PACKAGE}
-   if [ $? -ne 0 ]; then
+   PACKAGE="${i}_PACKAGE"
+   if ! diff "${!PACKAGE}" "../${!PACKAGE}"; then
      echo "Package ${!PACKAGE} differs from our mirror's version!"
      exit 1
    fi
@@ -102,7 +155,7 @@ if [ ! -f mingw-w64-svn-snapshot-r5830.zip ];
 then
   svn co -r 5830 https://mingw-w64.svn.sourceforge.net/svnroot/mingw-w64/trunk mingw-w64-svn || exit 1
   # XXX: Path
-  ZIPOPTS="-x*/.svn/*" $WRAPPER_DIR/build-helpers/dzip.sh mingw-w64-svn-snapshot-r5830.zip mingw-w64-svn
+  ZIPOPTS="-x*/.svn/*" "$WRAPPER_DIR/build-helpers/dzip.sh" mingw-w64-svn-snapshot-r5830.zip mingw-w64-svn
 fi
 
 mkdir -p linux-langpacks
@@ -112,25 +165,26 @@ mkdir -p mac-langpacks
 for i in $BUNDLE_LOCALES
 do
   cd linux-langpacks
-  wget -N https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$FIREFOX_LANG_VER/linux-i686/xpi/$i.xpi
+  wget -N "https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$FIREFOX_LANG_VER/linux-i686/xpi/$i.xpi";
   cd ..
   cd win32-langpacks
-  wget -N https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$FIREFOX_LANG_VER/win32/xpi/$i.xpi
+  wget -N "https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$FIREFOX_LANG_VER/win32/xpi/$i.xpi";
   cd ..
   cd mac-langpacks
-  wget -N https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$FIREFOX_LANG_VER/mac/xpi/$i.xpi
+  wget -N "https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$FIREFOX_LANG_VER/mac/xpi/$i.xpi";
   cd ..
 done
 
-$WRAPPER_DIR/build-helpers/dzip.sh win32-langpacks.zip win32-langpacks
-$WRAPPER_DIR/build-helpers/dzip.sh linux-langpacks.zip linux-langpacks
-$WRAPPER_DIR/build-helpers/dzip.sh mac-langpacks.zip mac-langpacks
+"$WRAPPER_DIR/build-helpers/dzip.sh" win32-langpacks.zip win32-langpacks
+"$WRAPPER_DIR/build-helpers/dzip.sh" linux-langpacks.zip linux-langpacks
+"$WRAPPER_DIR/build-helpers/dzip.sh" mac-langpacks.zip mac-langpacks
 
-ln -sf $NOSCRIPT_PACKAGE noscript@xxxxxxxxxxxxxxxx
-ln -sf $PDFJS_PACKAGE uriloader@xxxxxxxxxx
-ln -sf $OPENSSL_PACKAGE openssl.tar.gz
+ln -sf "$NOSCRIPT_PACKAGE" noscript@xxxxxxxxxxxxxxxx
+ln -sf "$PDFJS_PACKAGE" uriloader@xxxxxxxxxx
+ln -sf "$OPENSSL_PACKAGE" openssl.tar.gz
 
 # Fetch latest gitian-builder itself
+# XXX - this is broken if a non-standard inputs dir is selected using the command line flag.
 cd ..
 git remote set-url origin https://git.torproject.org/builders/gitian-builder.git
 git fetch origin
@@ -138,89 +192,18 @@ git fetch --tags origin
 git checkout tor-browser-builder-2
 cd inputs
 
-if [ -d tbb-windows-installer/.git ];
-then
-  cd tbb-windows-installer
-  git fetch origin
-  git fetch --tags origin
-  cd ..
-else
-  git clone https://github.com/moba/tbb-windows-installer.git || exit 1
-fi
-
-if [ -d zlib/.git ];
-then
-  cd zlib
-  git fetch origin
-  git fetch --tags origin
-  cd ..
-else
-  git clone https://github.com/madler/zlib.git || exit 1
-fi
-
-if [ -d libevent/.git ];
-then
-  cd libevent
-  git fetch origin
-  git fetch --tags origin
-  cd ..
-else
-  git clone https://github.com/libevent/libevent.git || exit 1
-fi
-
-if [ -d tor-launcher/.git ];
-then
-  cd tor-launcher
-  git fetch origin
-  git fetch --tags origin
-  cd ..
-else
-  git clone https://git.torproject.org/tor-launcher.git || exit 1
-fi
-
-if [ -d tor/.git ];
-then
-  cd tor
-  git fetch origin
-  git fetch --tags origin
-  cd ..
-else
-  git clone https://git.torproject.org/tor.git || exit 1
-fi
-
-if [ -d torbutton/.git ];
-then
-  cd torbutton
-  git fetch origin
-  git fetch --tags origin
-  cd ..
-else
-  git clone https://git.torproject.org/torbutton.git || exit 1
-fi
-
-if [ -d https-everywhere/.git ];
-then
-  cd https-everywhere
-  git fetch origin
-  git fetch --tags origin
-  cd ..
-else
-  git clone https://git.torproject.org/https-everywhere.git || exit 1
-fi
-
-if [ -d tor-browser/.git ];
-then
-  cd tor-browser
-  git fetch origin
-  git fetch --tags origin
-  git checkout $TORBROWSER_TAG
-  cd ..
-else
-  git clone https://git.torproject.org/tor-browser.git || exit 1
-  cd tor-browser
-  git checkout $TORBROWSER_TAG
-  cd ..
-fi
+while read dir url tag; do
+  update_git "$dir" "$url" "$tag"
+done << EOF
+tbb-windows-installer https://github.com/moba/tbb-windows-installer.git
+zlib                  https://github.com/madler/zlib.git
+libevent              https://github.com/libevent/libevent.git
+tor-launcher          https://git.torproject.org/tor-launcher.git
+tor                   https://git.torproject.org/tor.git
+torbutton             https://git.torproject.org/torbutton.git
+https-everywhere      https://git.torproject.org/https-everywhere.git
+tor-browser           https://git.torproject.org/tor-browser.git          $TORBROWSER_TAG
+EOF
 
 exit 0
 
diff --git a/gitian/gpg/OPENSSL.gpg b/gitian/gpg/OPENSSL.gpg
new file mode 100644
index 0000000..1b282b7
Binary files /dev/null and b/gitian/gpg/OPENSSL.gpg differ

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

Prev by Author: [tor-commits] [tor-browser-bundle/master] Fix incorrect empty string tests.
Next by Author: [tor-commits] [tor-launcher/master] Use new processID API if available.
Previous by thread: [tor-commits] [tor-browser-bundle/master] Fix incorrect empty string tests.
Next by thread: [tor-commits] r26251: {website} update contact info (website/trunk/about/en)
Index(es):
- Author
- Thread