[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [tor-browser-build/master] Bug 28803: Integrate building Pluggable Transports for Android



commit eee5d30a9ab1d727caac262cb62f72aaab75e0a0
Author: Matthew Finkel <sysrqb@xxxxxxxxxxxxxx>
Date:   Thu Oct 31 01:52:21 2019 +0000

    Bug 28803: Integrate building Pluggable Transports for Android
    
    We integrate building Pluggable Transports for Android to solve #32303
    in particular later on but it is important for providing snowflake
    support as well at some point (see: #28672)
    
    The bulk of the patch got written by boklm when trying to provide
    snowflake support for Android (#28672).
---
 .../go/0001-Use-fixed-go-build-tmp-directory.patch | 33 ++++++++++++++++++++
 projects/go/build                                  | 12 ++++----
 projects/go/config                                 | 30 +++++++++++++++++--
 projects/gobsaes/config                            |  9 ++++++
 ...BILE_TMPDIR-env-variable-is-defined-use-t.patch | 35 ++++++++++++++++++++++
 projects/gomobile/config                           | 33 ++++++++++++++++++++
 projects/goptlib/config                            |  8 +++++
 projects/goutls/config                             |  9 ++++++
 projects/goxexp/config                             | 19 ++++++++++++
 projects/goximage/config                           | 28 +++++++++++++++++
 projects/goxnet/config                             |  9 ++++++
 projects/obfs4/build                               |  4 +++
 projects/obfs4/config                              |  3 ++
 projects/tor-onion-proxy-library/build             | 10 +++++++
 projects/tor-onion-proxy-library/config            |  2 ++
 rbm.conf                                           |  6 ++++
 16 files changed, 241 insertions(+), 9 deletions(-)

diff --git a/projects/go/0001-Use-fixed-go-build-tmp-directory.patch b/projects/go/0001-Use-fixed-go-build-tmp-directory.patch
new file mode 100644
index 0000000..a10e059
--- /dev/null
+++ b/projects/go/0001-Use-fixed-go-build-tmp-directory.patch
@@ -0,0 +1,33 @@
+From 44e3391a5a3d9a2c4b57f78e43414465ab84681e Mon Sep 17 00:00:00 2001
+From: Nicolas Vigier <boklm@xxxxxxxxxxxxxx>
+Date: Thu, 27 Jun 2019 19:21:05 +0200
+Subject: [PATCH] Use fixed go-build tmp directory
+
+Use fixed go-build tmp directory, when the directory does not exist, in
+order to avoid some reproducibility issues where build directory gets
+embedded in generated binaries.
+---
+ src/cmd/go/internal/work/action.go | 10 +++++++---
+ 1 file changed, 7 insertions(+), 3 deletions(-)
+
+diff --git a/src/cmd/go/internal/work/action.go b/src/cmd/go/internal/work/action.go
+index 1f91046eb1..ba74b26a38 100644
+--- a/src/cmd/go/internal/work/action.go
++++ b/src/cmd/go/internal/work/action.go
+@@ -224,9 +224,13 @@ func (b *Builder) Init() {
+ 	if cfg.BuildN {
+ 		b.WorkDir = "$WORK"
+ 	} else {
+-		tmp, err := ioutil.TempDir(os.Getenv("GOTMPDIR"), "go-build")
+-		if err != nil {
+-			base.Fatalf("go: creating work dir: %v", err)
++		tmp := filepath.Join(os.Getenv("GOTMPDIR"), "go-build-workdir")
++		_, err := os.Stat(tmp)
++		if !os.IsNotExist(err) {
++			tmp, err = ioutil.TempDir(os.Getenv("GOTMPDIR"), "go-build")
++			if err != nil {
++				base.Fatalf("go: creating work dir: %v", err)
++			}
+ 		}
+ 		if !filepath.IsAbs(tmp) {
+ 			abs, err := filepath.Abs(tmp)
diff --git a/projects/go/build b/projects/go/build
index 4945c88..aa092d6 100644
--- a/projects/go/build
+++ b/projects/go/build
@@ -28,6 +28,9 @@ export GOROOT_BOOTSTRAP="/var/tmp/build/go1.4"
 cd $rootdir
 [% IF ! c("var/linux") %]
   [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+  # http://golang.org/doc/install/source#environment
+  export GOOS=[% c("var/GOOS") %]
+  export GOARCH=[% c("var/GOARCH") %]
 [% END %]
 
 # Building go
@@ -48,9 +51,6 @@ cd /var/tmp/dist/go/src
   echo "#!/bin/sh" > "$CC_FOR_TARGET"
   echo "exec $CC $CFLAGS $LDFLAGS \"\$@\"" >> "$CC_FOR_TARGET"
   chmod +x "$CC_FOR_TARGET"
-  # http://golang.org/doc/install/source#environment
-  export GOOS=darwin
-  export GOARCH=amd64
   # faketime is needed because clang 3.9.1 on Darwin embeds the timestamps of
   # certain intermediate object files (including those that arise here while
   # compiling the Go runtime itself). Without this, variable timestamps would
@@ -65,10 +65,10 @@ cd /var/tmp/dist/go/src
   echo "#!/bin/sh" > "$CC_FOR_TARGET"
   echo "exec [% c("arch") %]-w64-mingw32-gcc [% c("var/CFLAGS") %] [% c("var/LDFLAGS") %] \"\$@\"" >> "$CC_FOR_TARGET"
   chmod +x "$CC_FOR_TARGET"
-  # http://golang.org/doc/install/source#environment
-  export GOOS=windows
-  export GOARCH=[% c("var/GOARCH") %]
   CGO_ENABLED=1 CC_FOR_TARGET="$CC_FOR_TARGET" CC= CFLAGS= LDFLAGS= ./make.bash
+[% ELSIF c("var/android") -%]
+  patch -p2 < $rootdir/0001-Use-fixed-go-build-tmp-directory.patch
+  CGO_ENABLED=1 CC_FOR_TARGET=[% c("var/CC") %] CGO_CFLAGS='-D__ANDROID_API__=[% c("var/android_min_api") %]' CC= CFLAGS= LDFLAGS= ./make.bash
 [% END -%]
 
 cd /var/tmp/dist
diff --git a/projects/go/config b/projects/go/config
index ffe07ea..31c510a 100644
--- a/projects/go/config
+++ b/projects/go/config
@@ -10,8 +10,13 @@ var:
   setup: |
     mkdir -p /var/tmp/dist
     tar -C /var/tmp/dist -xf $rootdir/[% c("go_tarfile") %]
-    export GOOS=[% c("var/GOOS") %]
-    export GOARCH=[% c("var/GOARCH") %]
+    [% IF pc(c("origin_project"), "var/no_crosscompile") -%]
+      export GOOS=linux
+      export GOARCH=amd64
+    [% ELSE -%]
+      export GOOS=[% c("var/GOOS") %]
+      export GOARCH=[% c("var/GOARCH") %]
+    [% END -%]
     export GOPATH=/var/tmp/dist/gopath
     export PATH=/var/tmp/dist/go/bin:/var/tmp/dist/gopath/bin:"$PATH"
     [% IF c("var/linux") -%]
@@ -21,6 +26,8 @@ var:
     [% IF c("var/linux-i686") -%]
       export CGO_ENABLED=1
     [% END -%]
+    export GOTMPDIR=/var/tmp/build/go-tmp
+    mkdir -p "$GOTMPDIR"
 
   # Template build script for building a go library.
   # This can be called as projects/go/var/build_go_lib.
@@ -82,15 +89,32 @@ targets:
   linux-i686:
     var:
       GOARCH: 386
+  android:
+    var:
+      GOOS: android
+  android-x86:
+    var:
+      GOARCH: 386
+  android-x86_64:
+    var:
+      GOARCH: amd64
+  android-armv7:
+    var:
+      GOARCH: arm
+  android-aarch64:
+    var:
+      GOARCH: arm64
 
 input_files:
   - project: container-image
   - name: '[% c("var/compiler") %]'
     project: '[% c("var/compiler") %]'
-    enable: '[% c("var/windows") || c("var/osx") %]'
+    enable: '[% ! c("var/linux") %]'
   - URL: 'https://golang.org/dl/go[% c("version") %].src.tar.gz'
     name: go
     sha256sum: ab0e56ed9c4732a653ed22e232652709afbf573e710f56a07f7fdeca578d62fc
   - URL: 'https://golang.org/dl/go[% c("var/go14_version") %].src.tar.gz'
     name: go14
     sha256sum: 9947fc705b0b841b5938c48b22dc33e9647ec0752bae66e50278df4f23f64959
+  - filename: 0001-Use-fixed-go-build-tmp-directory.patch
+    enable: '[% c("var/android") %]'
diff --git a/projects/gobsaes/config b/projects/gobsaes/config
index 4da5828..0ced258 100644
--- a/projects/gobsaes/config
+++ b/projects/gobsaes/config
@@ -11,7 +11,16 @@ var:
     use_container: 1
   go_lib: git.schwanenlied.me/yawning/bsaes.git
 
+targets:
+  android:
+    var:
+      build_go_lib_pre: |
+        [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+
 input_files:
   - project: container-image
   - name: go
     project: go
+  - name: '[% c("var/compiler") %]'
+    project: '[% c("var/compiler") %]'
+    enable: '[% c("var/android") %]'
diff --git a/projects/gomobile/0001-If-the-GOMOBILE_TMPDIR-env-variable-is-defined-use-t.patch b/projects/gomobile/0001-If-the-GOMOBILE_TMPDIR-env-variable-is-defined-use-t.patch
new file mode 100644
index 0000000..770b22a
--- /dev/null
+++ b/projects/gomobile/0001-If-the-GOMOBILE_TMPDIR-env-variable-is-defined-use-t.patch
@@ -0,0 +1,35 @@
+From 188307b6acd5003e82cb68872da9463f5f707515 Mon Sep 17 00:00:00 2001
+From: Nicolas Vigier <boklm@xxxxxxxxxxxxxx>
+Date: Thu, 27 Jun 2019 12:46:38 +0200
+Subject: [PATCH] If the GOMOBILE_TMPDIR env variable is defined, use that as
+ tmpdir
+
+Having the option to select a fixed tmpdir makes it possible to have
+reproducible builds, as the build directory is embedded in the generated
+executables.
+---
+ cmd/gomobile/env.go | 11 ++++++++---
+ 1 file changed, 8 insertions(+), 3 deletions(-)
+
+diff --git a/cmd/gomobile/env.go b/cmd/gomobile/env.go
+index 471b009..b97141b 100644
+--- a/cmd/gomobile/env.go
++++ b/cmd/gomobile/env.go
+@@ -61,9 +61,14 @@ func buildEnvInit() (cleanup func(), err error) {
+ 		tmpdir = "$WORK"
+ 		cleanupFn = func() {}
+ 	} else {
+-		tmpdir, err = ioutil.TempDir("", "gomobile-work-")
+-		if err != nil {
+-			return nil, err
++		gomobile_tmpdir := os.Getenv("GOMOBILE_TMPDIR")
++		if gomobile_tmpdir != "" {
++			tmpdir = gomobile_tmpdir
++		} else {
++			tmpdir, err = ioutil.TempDir("", "gomobile-work-")
++			if err != nil {
++				return nil, err
++			}
+ 		}
+ 	}
+ 	if buildX {
diff --git a/projects/gomobile/config b/projects/gomobile/config
new file mode 100644
index 0000000..9dbfc29
--- /dev/null
+++ b/projects/gomobile/config
@@ -0,0 +1,33 @@
+# vim: filetype=yaml sw=2
+version: '[% c("abbrev") %]'
+git_url: https://github.com/golang/mobile
+git_hash: 92f3b9caf7ba8f4f9c10074225afcba0cba47a62
+filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz'
+
+build: '[% c("projects/go/var/build_go_lib") %]'
+
+var:
+  container:
+    use_container: 1
+  no_crosscompile: 1
+  go_lib: golang.org/x/mobile
+  go_lib_deps:
+    - goximage
+    - goxexp
+  go_lib_install:
+    - golang.org/x/mobile/cmd/gobind
+    - golang.org/x/mobile/cmd/gomobile
+  build_go_lib_pre: |
+    [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+
+input_files:
+  - project: container-image
+  - name: go
+    project: go
+  - name: goximage
+    project: goximage
+  - name: goxexp
+    project: goxexp
+  - name: '[% c("var/compiler") %]'
+    project: '[% c("var/compiler") %]'
+  - filename: 0001-If-the-GOMOBILE_TMPDIR-env-variable-is-defined-use-t.patch
diff --git a/projects/goptlib/config b/projects/goptlib/config
index 8c2a95d..4dfc6ff 100644
--- a/projects/goptlib/config
+++ b/projects/goptlib/config
@@ -18,7 +18,15 @@ targets:
     version: master
     tag_gpg_id: 0
 
+  android:
+    var:
+      build_go_lib_pre: |
+        [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+
 input_files:
   - project: container-image
   - name: go
     project: go
+  - name: '[% c("var/compiler") %]'
+    project: '[% c("var/compiler") %]'
+    enable: '[% c("var/android") %]'
diff --git a/projects/goutls/config b/projects/goutls/config
index fa4b5d8..7a46517 100644
--- a/projects/goutls/config
+++ b/projects/goutls/config
@@ -17,6 +17,12 @@ var:
     - gocompress
     - gobsaes
 
+targets:
+  android:
+    var:
+      build_go_lib_pre: |
+        [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+
 input_files:
   - project: container-image
   - name: go
@@ -28,3 +34,6 @@ input_files:
   - name: gobsaes
     project: gobsaes
   - filename: sessionid.patch
+  - name: '[% c("var/compiler") %]'
+    project: '[% c("var/compiler") %]'
+    enable: '[% c("var/android") %]'
diff --git a/projects/goxexp/config b/projects/goxexp/config
new file mode 100644
index 0000000..c81a65f
--- /dev/null
+++ b/projects/goxexp/config
@@ -0,0 +1,19 @@
+# vim: filetype=yaml sw=2
+version: '[% c("abbrev") %]'
+git_url: https://github.com/golang/exp
+git_hash: a1d7652674e8f269b17d9be90d3c4c1d0e8830e8
+filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz'
+
+build: '[% c("projects/go/var/build_go_lib") %]'
+
+var:
+  container:
+    use_container: 1
+  no_crosscompile: 1
+  go_lib: golang.org/x/exp
+  go_lib_install: []
+
+input_files:
+  - project: container-image
+  - name: go
+    project: go
diff --git a/projects/goximage/config b/projects/goximage/config
new file mode 100644
index 0000000..308a1ba
--- /dev/null
+++ b/projects/goximage/config
@@ -0,0 +1,28 @@
+# vim: filetype=yaml sw=2
+version: '[% c("abbrev") %]'
+git_url: https://github.com/golang/image
+git_hash: 0694c2d4d067f97ebef574d63a763ee8ab559da7
+filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz'
+
+build: '[% c("projects/go/var/build_go_lib") %]'
+
+var:
+  container:
+    use_container: 1
+  no_crosscompile: 1
+  go_lib: golang.org/x/image
+  go_lib_install:
+    - golang.org/x/image/webp
+    - golang.org/x/image/bmp
+    - golang.org/x/image/draw
+    - golang.org/x/image/font
+    - golang.org/x/image/math/fixed
+    - golang.org/x/image/math/f32
+    - golang.org/x/image/math/f64
+    - golang.org/x/image/vp8
+    - golang.org/x/image/vp8l
+
+input_files:
+  - project: container-image
+  - name: go
+    project: go
diff --git a/projects/goxnet/config b/projects/goxnet/config
index f6a2c29..5394875 100644
--- a/projects/goxnet/config
+++ b/projects/goxnet/config
@@ -16,9 +16,18 @@ var:
   go_lib_deps:
     - goxsys
 
+targets:
+  android:
+    var:
+      build_go_lib_pre: |
+        [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+
 input_files:
   - project: container-image
   - name: go
     project: go
   - name: goxsys
     project: goxsys
+  - name: '[% c("var/compiler") %]'
+    project: '[% c("var/compiler") %]'
+    enable: '[% c("var/android") %]'
diff --git a/projects/obfs4/build b/projects/obfs4/build
index 33b0add..53b83df 100644
--- a/projects/obfs4/build
+++ b/projects/obfs4/build
@@ -5,6 +5,10 @@ distdir=/var/tmp/dist/[% project %]
 [% c("var/set_PTDIR_DOCSDIR") -%]
 mkdir -p $PTDIR
 
+[% IF c("var/android") -%]
+  [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+[% END -%]
+
 tar -C /var/tmp/dist -xf [% c('input_files_by_name/goptlib') %]
 tar -C /var/tmp/dist -xf [% c('input_files_by_name/ed25519') %]
 tar -C /var/tmp/dist -xf [% c('input_files_by_name/siphash') %]
diff --git a/projects/obfs4/config b/projects/obfs4/config
index d9357bb..21bdab6 100644
--- a/projects/obfs4/config
+++ b/projects/obfs4/config
@@ -36,3 +36,6 @@ input_files:
     project: goutls
   - name: goxtext
     project: goxtext
+  - name: '[% c("var/compiler") %]'
+    project: '[% c("var/compiler") %]'
+    enable: '[% c("var/android") %]'
diff --git a/projects/tor-onion-proxy-library/build b/projects/tor-onion-proxy-library/build
index 0b949a3..0f2d976 100644
--- a/projects/tor-onion-proxy-library/build
+++ b/projects/tor-onion-proxy-library/build
@@ -22,6 +22,16 @@ cd /var/tmp/build/[% project %]-[% c('version') %]
 patch -p1 < $rootdir/gradle.patch
 patch -p1 < $rootdir/canceldormant.patch
 
+# Extract obfs4proxy from TorBrowser/Tor/PluggableTransports/obfs4proxy
+tar --strip-components=4 -xf $rootdir/[% c('input_files_by_name/obfs4') %]
+
+# Overwrite the obfs4proxy binary provided by Pluto
+for d in external/pluto/bin/*; do
+  cp obfs4proxy $d/
+done
+
+rm obfs4proxy
+
 # Build Android Libraries and Apps
 $GRADLE_HOME/gradle-4.10.2/bin/gradle --offline --no-daemon -P androidplugin=3.1.0 -Dmaven.repo.local=$gradle_repo assembleRelease -x lint
 
diff --git a/projects/tor-onion-proxy-library/config b/projects/tor-onion-proxy-library/config
index c2bd1a4..ff3432c 100644
--- a/projects/tor-onion-proxy-library/config
+++ b/projects/tor-onion-proxy-library/config
@@ -25,6 +25,8 @@ input_files:
   - project: container-image
   - name: '[% c("var/compiler") %]'
     project: '[% c("var/compiler") %]'
+  - project: obfs4
+    name: obfs4
   - filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]'
     name: gradle-dependencies
     exec: '[% INCLUDE "fetch-gradle-dependencies" %]'
diff --git a/rbm.conf b/rbm.conf
index 727c09f..aaa6e3a 100644
--- a/rbm.conf
+++ b/rbm.conf
@@ -183,6 +183,7 @@ targets:
     var:
       android-armv7: 1
       osname: android-armv7
+      toolchain_arch: arm
   torbrowser-android-x86:
     - android-x86
     - android
@@ -191,6 +192,7 @@ targets:
     var:
       android-x86: 1
       osname: android-x86
+      toolchain_arch: x86
   torbrowser-android-x86_64:
     - android-x86_64
     - android
@@ -199,6 +201,7 @@ targets:
     var:
       android-x86_64: 1
       osname: android-x86_64
+      toolchain_arch: x86_64
   torbrowser-android-aarch64:
     - android-aarch64
     - android
@@ -207,6 +210,7 @@ targets:
     var:
       android-aarch64: 1
       osname: android-aarch64
+      toolchain_arch: arm64
   android:
     var:
       android: 1
@@ -219,6 +223,8 @@ targets:
       android_min_api_x86: 16
       android_min_api_x86_64: 21
       snowflake: 0
+      CC: '$ANDROID_NDK_HOME/[% c("var/toolchain_arch") %]/bin/clang'
+      CXX: '$ANDROID_NDK_HOME/[% c("var/toolchain_arch") %]/bin/clang++'
       container:
         suite: stretch
         arch: amd64



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits