[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [gitian-builder/tor-browser-builder-3] Bug 15578: Switch over to Wheezy for Linux builds



commit 15d166d65d006f564bf3c7dbb8780ed0649352ba
Author: Georg Koppen <gk@xxxxxxxxxxxxxx>
Date:   Thu Nov 26 06:34:42 2015 +0000

    Bug 15578: Switch over to Wheezy for Linux builds
    
    Support for Debian guest VMs was developed by Joseph Bisch. This
    commit is largely a backport of this feature adapted to our needs
    allowing us to get rid of Ubuntu Lucid which is EOL for a while now.
---
 README.md                  |   22 +++++++++++-
 bin/gbuild                 |    3 ++
 bin/make-base-vm           |   81 +++++++++++++++++++++++++++++++++++++-------
 libexec/copy-from-target   |    2 +-
 libexec/copy-to-target     |    2 +-
 libexec/on-target          |    2 +-
 target-bin/bootstrap-fixup |   18 ++++++++--
 7 files changed, 111 insertions(+), 19 deletions(-)

diff --git a/README.md b/README.md
index 2f07c6a..c78e231 100644
--- a/README.md
+++ b/README.md
@@ -32,6 +32,22 @@ This performs a build inside a VM, with deterministic inputs and outputs.  If th
 
 Install virtualbox from http://www.virtualbox.org, and make sure `VBoxManage` is in your `$PATH`.
 
+## Debian Guests
+
+Gitian now supports Debian guests in addition to Ubuntu guests. Note that this doesn't mean you can allow the builders to choose to use either Debian or Ubuntu guests. The person creating the Gitian descriptor will need to choose a particular distro and suite for the guest and all builders must use that particular distro and suite, otherwise the software won't reproduce for everyone.
+
+The official vmbuilder only includes support for Ubuntu guests, so you need to install [Joseph Bisch's fork of vmbuilder](https://github.com/josephbisch/vmbuilder), which adds a Debian plugin.
+
+To create a Debian guest:
+
+    bin/make-base-vm --distro debian --suite jessie
+
+There is currently no support for LXC Debian guests. There is just KVM support. LXC support for Debian guests is planned to be added soon.
+
+Only Debian Jessie guests have been tested with Gitian. Debian Jessie is the current stable release of Debian at this time. If you have success (or trouble) with other versions of Debian, please let us know.
+
+If you are creating a Gitian descriptor, you can now specify a distro. If no distro is provided, the default is to assume Ubuntu. Since Ubuntu is assumed, older Gitian descriptors that don't specify a distro will still work as they always have.
+
 ## Create the base VM for use in further builds
 **NOTE:** requires `sudo`, please review the script
 
@@ -88,6 +104,10 @@ If you have everything set-up properly, you should be able to:
     PATH=$PATH:$(pwd)/libexec
     make-clean-vm --suite lucid --arch i386
 
+    # on-target needs $DISTRO to be set to debian if using a Debian guest
+    # (when running gbuild, $DISTRO is set based on the descriptor, so this line isn't needed)
+    DiSTRO=debian
+
     # For LXC:
     LXC_ARCH=i386 LXC_SUITE=lucid on-target ls -la
 
@@ -128,7 +148,7 @@ After you've merged everybody's signatures, verify them:
 * Log files are captured to the _var_ directory
 * You can run the utilities in libexec by running `PATH="libexec:$PATH"`
 * To start the target VM run `start-target 32 lucid-i386` or `start-target 64 lucid-amd64`
-* To ssh into the target run `on-target` or `on-target -u root`
+* To ssh into the target run `on-target` (after setting $DISTRO to debian if using a Debian guest) or `on-target -u root`
 * On the target, the _build_ directory contains the code as it is compiled and _install_ contains intermediate libraries
 * By convention, the script in `<package>.yml` starts with any environment setup you would need to manually compile things on the target
 
diff --git a/bin/gbuild b/bin/gbuild
index d256986..0171ccf 100755
--- a/bin/gbuild
+++ b/bin/gbuild
@@ -180,10 +180,13 @@ FileUtils.mkdir_p(result_dir)
 package_name = build_desc["name"] or raise "must supply name"
 package_name = sanitize(package_name, "package name")
 
+distro = build_desc["distro"] || "ubuntu"
 suites = build_desc["suites"] or raise "must supply suites"
 archs = build_desc["architectures"] or raise "must supply architectures"
 reference_datetime = build_desc["reference_datetime"] or raise "must supply reference_datetime"
 
+ENV['DISTRO'] = distro
+
 desc_sum = `sha256sum #{build_desc_file}`
 desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml")
 in_sums << desc_sum
diff --git a/bin/make-base-vm b/bin/make-base-vm
index 66a3704..1aa8eac 100755
--- a/bin/make-base-vm
+++ b/bin/make-base-vm
@@ -1,10 +1,10 @@
 #!/bin/sh
 set -e
 
+DISTRO=ubuntu
 SUITE=lucid
 ARCH=amd64
-MIRROR=http://${MIRROR_HOST:-127.0.0.1}:3142/archive.ubuntu.com/ubuntu
-SECURITY_MIRROR=http://${MIRROR_HOST:-127.0.0.1}:3142/security.ubuntu.com/ubuntu
+MIRROR_BASE=http://${MIRROR_HOST:-127.0.0.1}:3142
 LXC=0
 VBOX=0
 
@@ -13,11 +13,12 @@ usage() {
   echo "Make a base client."
   echo
   cat << EOF
-  --help     display this help and exit
-  --suite U  build suite U instead of lucid
-  --arch A   build architecture A (e.g. i386) instead of amd64
-  --lxc      use lxc instead of kvm
-  --vbox     use VirtualBox instead of kvm
+  --help      display this help and exit
+  --distro D  build distro D (e.g. debian) instead of ubuntu
+  --suite U   build suite U instead of lucid
+  --arch A    build architecture A (e.g. i386) instead of amd64
+  --lxc       use lxc instead of kvm
+  --vbox      use VirtualBox instead of kvm
 EOF
 }
 
@@ -28,6 +29,10 @@ if [ $# != 0 ] ; then
         usage
         exit 0
         ;;
+      --distro|-d)
+        DISTRO="$2"
+        shift 2
+        ;;
       --suite|-s)
         SUITE="$2"
         shift 2
@@ -55,6 +60,19 @@ if [ $# != 0 ] ; then
   done
 fi
 
+if [ $DISTRO = "debian" -a $LXC = "1" ]; then
+  echo "There is no support for Debian guests using LXC currently. Please use KVM or another distro for now."
+  exit 1
+fi
+
+if [ $DISTRO = "ubuntu" ]; then
+  MIRROR=$MIRROR_BASE/archive.ubuntu.com/ubuntu
+  SECURITY_MIRROR=$MIRROR_BASE/security.ubuntu.com/ubuntu
+elif [ $DISTRO = "debian" ]; then
+  MIRROR=$MIRROR_BASE/ftp.debian.org/debian
+  SECURITY_MIRROR=$MIRROR_BASE/security.debian.org/
+fi
+
 mkdir -p var
 
 if [ ! -e var/id_dsa ]; then
@@ -68,7 +86,47 @@ if [ $ARCH = "amd64" -a $SUITE = "hardy" ]; then
   FLAVOUR=server
 fi
 
-addpkg=openssh-server,pciutils,build-essential,git-core,subversion,lxc
+if [ $DISTRO = "debian" -a $ARCH = "amd64" ]; then
+  FLAVOUR=amd64
+elif [ $DISTRO = "debian" -a $ARCH = "i386" -a \($SUITE = "squeeze" -o $SUITE = "lenny" -o $SUITE = "etch" -o $SUITE = "sarge" -o $SUITE = "woody" -o $SUITE = "potato" -o $SUITE = "slink" -o $SUITE = "hamm" -o $SUITE = "bo" -o $SUITE = "rex" -o $SUITE = "buzz"\) ]; then
+  FLAVOUR=686
+elif [ $DISTRO = "debian" ]; then
+  FLAVOUR=686-pae
+fi
+
+LOCALE_PKG=language-pack-en
+if [ $DISTRO = "debian" ]; then
+  LOCALE_PKG=locales
+fi
+
+addpkg=pciutils,build-essential,git-core,subversion,$LOCALE_PKG,wget,lsb-release
+
+if [ $DISTRO = "ubuntu" ]; then
+  # Need comma at end to work around an issue with apt for Debian <= Wheezy regarding empty strings
+  #
+  # If we left the comma down below when adding KERNEL_PKG to addpkg, the fact that KERNEL_PKG is undefined
+  # if DISTRO is debian would result in two commas in a row (,,), which is interpreted by apt-get as the
+  # package with the name empty string (""). This triggers a bug with apt versions < 1.0.3. So by adding the
+  # comma to the end of KERNEL_PKG, we are including that comma if the distro is ubuntu (and therefore we do
+  # have a kernel package that needs to be installed). If KERNEL_PKG is not set (i.e. we have Debian as the
+  # distro), then we don't add that extra comma and therefore, we don't end up with two commas in a row.
+  #
+  # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=744940
+  # http://anonscm.debian.org/cgit/apt/apt.git/commit/?h=1.0.3&id=d99854cac4065bc7b337815fb2116269d58dab73
+  KERNEL_PKG=linux-image-generic,
+fi
+
+GRUB_PKG=grub
+if [ $DISTRO = "ubuntu" ]; then
+  GRUB_PKG=grub-pc
+fi
+
+if [ $LXC = "1" ]; then
+  addpkg=$addpkg,lxc
+else
+  # Lack of comma after KERNEL_PKG is not a typo
+  addpkg=$addpkg,${KERNEL_PKG}${GRUB_PKG},openssh-server
+fi
 
 # Remove cron to work around vmbuilder issue when umounting /dev on target
 removepkg=cron
@@ -84,8 +142,8 @@ if [ $VBOX = "1" ]; then
 
   vagrant ssh "$NAME" -c "sudo mkdir -p /root/.ssh && sudo chmod 700 /root/.ssh"
   vagrant ssh "$NAME" -c "sudo sh -c 'cat >> /root/.ssh/authorized_keys'" < var/id_dsa.pub
-  vagrant ssh "$NAME" -c "sudo -u ubuntu mkdir -p /home/ubuntu/.ssh && sudo -u ubuntu chmod 700 /home/ubuntu/.ssh"
-  vagrant ssh "$NAME" -c "sudo sh -c 'cat >> /home/ubuntu/.ssh/authorized_keys'" < var/id_dsa.pub
+  vagrant ssh "$NAME" -c "sudo -u $DISTRO mkdir -p /home/$DISTRO/.ssh && sudo -u $DISTRO chmod 700 /home/$DISTRO/.ssh"
+  vagrant ssh "$NAME" -c "sudo sh -c 'cat >> /home/$DISTRO/.ssh/authorized_keys'" < var/id_dsa.pub
 
   VBoxManage snapshot "Gitian-$NAME" take "Gitian-Clean"
   vagrant suspend "$NAME"
@@ -99,7 +157,7 @@ if [ -e $OUT.qcow2 ]; then
 fi
 
 rm -rf $OUT
-sudo vmbuilder kvm ubuntu --rootsize 15360 --arch=$ARCH --suite=$SUITE --addpkg=$addpkg --removepkg=$removepkg --ssh-key=var/id_dsa.pub --ssh-user-key=var/id_dsa.pub --mirror=$MIRROR --security-mirror=$SECURITY_MIRROR --dest=$OUT --flavour=$FLAVOUR --firstboot=`pwd`/target-bin/bootstrap-fixup
+sudo vmbuilder kvm $DISTRO --rootsize 15360 --arch=$ARCH --suite=$SUITE --addpkg=$addpkg --removepkg=$removepkg --ssh-key=var/id_dsa.pub --ssh-user-key=var/id_dsa.pub --mirror=$MIRROR --security-mirror=$SECURITY_MIRROR --dest=$OUT --flavour=$FLAVOUR --firstboot=`pwd`/target-bin/bootstrap-fixup
 mv $OUT/*.qcow2 $OUT.qcow2
 rm -rf $OUT
 
@@ -114,4 +172,3 @@ if [ $LXC = "1" ]; then
     rm -f $OUT.raw
     # bootstrap-fixup is done in libexec/make-clean-vm
 fi
-
diff --git a/libexec/copy-from-target b/libexec/copy-from-target
index f82da66..5372bb9 100755
--- a/libexec/copy-from-target
+++ b/libexec/copy-from-target
@@ -2,7 +2,7 @@
 
 . gconfig
 
-TUSER=ubuntu
+TUSER=${DISTRO:-ubuntu}
 QUIET_FLAG=
 
 usage() {
diff --git a/libexec/copy-to-target b/libexec/copy-to-target
index f9d900e..df45812 100755
--- a/libexec/copy-to-target
+++ b/libexec/copy-to-target
@@ -2,7 +2,7 @@
 
 . gconfig
 
-TUSER=ubuntu
+TUSER=${DISTRO:-ubuntu}
 QUIET_FLAG=
 
 usage() {
diff --git a/libexec/on-target b/libexec/on-target
index ae653bd..fe56fee 100755
--- a/libexec/on-target
+++ b/libexec/on-target
@@ -4,7 +4,7 @@ set -e
 
 . gconfig
 
-TUSER=ubuntu
+TUSER=${DISTRO:-ubuntu}
 
 usage() {
   echo "Usage: ${0##*/} [OPTION]... <command>"
diff --git a/target-bin/bootstrap-fixup b/target-bin/bootstrap-fixup
index ccbb977..d371072 100755
--- a/target-bin/bootstrap-fixup
+++ b/target-bin/bootstrap-fixup
@@ -2,7 +2,8 @@
 
 set -e
 
-. /etc/lsb-release
+DISTRIB_NAME=`lsb_release -is`
+DISTRIB_CODENAME=`lsb_release -cs`
 
 ip=`hostname --all-ip-addresses | cut -d ' ' -f1 | cut -d. -f1-3`
 
@@ -14,5 +15,16 @@ else
     MIRROR_HOST_ON_GUEST=${MIRROR_HOST_ON_GUEST:-10.0.2.2}
 fi
 
-echo "deb http://$MIRROR_HOST_ON_GUEST:3142/archive.ubuntu.com/ubuntu $DISTRIB_CODENAME main universe" > $1/etc/apt/sources.list
-echo "deb http://$MIRROR_HOST_ON_GUEST:3142/archive.ubuntu.com/ubuntu $DISTRIB_CODENAME-updates main universe" >> $1/etc/apt/sources.list
+if [ $DISTRIB_NAME = "Ubuntu" ]; then
+  echo "deb http://$MIRROR_HOST_ON_GUEST:3142/archive.ubuntu.com/ubuntu $DISTRIB_CODENAME main universe" > $1/etc/apt/sources.list
+  echo "deb http://$MIRROR_HOST_ON_GUEST:3142/security.ubuntu.com/ubuntu $DISTRIB_CODENAME-security main universe" >> $1/etc/apt/sources.list
+  echo "deb http://$MIRROR_HOST_ON_GUEST:3142/archive.ubuntu.com/ubuntu $DISTRIB_CODENAME-updates main universe" >> $1/etc/apt/sources.list
+elif [ $DISTRIB_NAME = "Debian" ]; then
+  echo "deb http://$MIRROR_HOST_ON_GUEST:3142/ftp.debian.org/debian $DISTRIB_CODENAME main" > $1/etc/apt/sources.list
+  echo "deb http://$MIRROR_HOST_ON_GUEST:3142/security.debian.org/ $DISTRIB_CODENAME/updates main" >> $1/etc/apt/sources.list
+  echo "deb http://$MIRROR_HOST_ON_GUEST:3142/ftp.debian.org/debian $DISTRIB_CODENAME-updates main" >> $1/etc/apt/sources.list
+  # grub-legacy conflicts grub-pc dependencies
+  # No grub-legacy on Ubuntu, just on Debian
+  # Work around bcron-run conflict due to cron being removed
+  apt-get purge -y grub-legacy bcron-run &> /dev/null
+fi

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits