[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [tor-browser-build/master] Bug 40270: Improve process to generate gradle-dependencies-list.txt

To: tor-commits@xxxxxxxxxxxxxxxxxxxx, tbb-commits@xxxxxxxxxxxxxxxxxxxx
Subject: [tor-commits] [tor-browser-build/master] Bug 40270: Improve process to generate gradle-dependencies-list.txt
From: sysrqb@xxxxxxxxxxxxxx
Date: Tue, 20 Apr 2021 15:30:33 +0000 (UTC)
Delivered-to: archiver@xxxxxxxx
Delivery-date: Tue, 20 Apr 2021 11:31:04 -0400
List-archive: <http://lists.torproject.org/pipermail/tor-commits/>
List-help: <mailto:tor-commits-request@lists.torproject.org?subject=help>
List-id: "auto: code repository commits" <tor-commits.lists.torproject.org>
List-post: <mailto:tor-commits@lists.torproject.org>
List-subscribe: <https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits>, <mailto:tor-commits-request@lists.torproject.org?subject=subscribe>
List-unsubscribe: <https://lists.torproject.org/cgi-bin/mailman/options/tor-commits>, <mailto:tor-commits-request@lists.torproject.org?subject=unsubscribe>
Patch-author: Nicolas Vigier <boklm@xxxxxxxxxxxxxx>
Sender: "tor-commits" <tor-commits-bounces@xxxxxxxxxxxxxxxxxxxx>

commit 8ba33c11bb66c6ef775155542bba3f24706a71d1
Author: Nicolas Vigier <boklm@xxxxxxxxxxxxxx>
Date:   Tue Apr 13 17:59:12 2021 +0200

    Bug 40270: Improve process to generate gradle-dependencies-list.txt
---
 Makefile                                           |  9 +++++
 projects/android-components/build                  |  4 ++-
 projects/android-components/config                 | 14 ++++++--
 projects/application-services/build                |  4 ++-
 projects/application-services/config               | 23 +++++++++++--
 {tools => projects/common}/gen_gradle_deps_file.sh |  0
 .../how-to-create-gradle-dependencies-list.txt     | 40 ++++++++++++++++------
 projects/fenix/build                               |  7 +++-
 projects/fenix/config                              | 33 ++++++++++++------
 9 files changed, 105 insertions(+), 29 deletions(-)

diff --git a/Makefile b/Makefile
index 35e8834..1d16fa4 100644
--- a/Makefile
+++ b/Makefile
@@ -219,6 +219,15 @@ create_glean_deps_tarball: submodule-update
 create_glean_deps_tarball-with_torsocks: submodule-update
 	$(rbm) build glean --step create_glean_deps_tarball --target alpha --target torbrowser-android-armv7 --target with_torsocks
 
+get_gradle_dependencies_list-fenix: submodule-update
+	$(rbm) build fenix --step get_gradle_dependencies_list --target nightly --target torbrowser-android-armv7
+
+get_gradle_dependencies_list-application-services: submodule-update
+	$(rbm) build application-services --step get_gradle_dependencies_list --target nightly --target torbrowser-android-armv7
+
+get_gradle_dependencies_list-android-components: submodule-update
+	$(rbm) build android-components --step get_gradle_dependencies_list --target nightly --target torbrowser-android-armv7
+
 submodule-update:
 	git submodule update --init
 
diff --git a/projects/android-components/build b/projects/android-components/build
index 896ba3a..e14b803 100644
--- a/projects/android-components/build
+++ b/projects/android-components/build
@@ -41,7 +41,9 @@ patch -p1 < $rootdir/git.patch
   # XXX: fetching deps for `assembleGeckoBeta -x lint` by using that same target
   # results in some missing dependencies for yet unknown reasons. Thus, we use
   # `assemble` instead for now.
-  gradle --debug --no-daemon assemble
+  gradle --debug --no-daemon assemble > gradle.log 2>&1 || ( cat gradle.log; exit 1 )
+  $rootdir/gen_gradle_deps_file.sh gradle.log
+  mv gradle-dependencies-list.txt '[% dest_dir _ "/" _ c("filename") %]'
 [% ELSE %]
   # Prepare Glean dependencies for offline build
   tar -xf $rootdir/[% c('input_files_by_name/glean') %]
diff --git a/projects/android-components/config b/projects/android-components/config
index e3068b6..5284ce7 100644
--- a/projects/android-components/config
+++ b/projects/android-components/config
@@ -14,9 +14,6 @@ var:
     use_container: 1
   # This should be updated when the list of gradle dependencies is changed.
   gradle_dependencies_version: 22
-  # Switch to make it easier to grab all dependencies during a dry-run.
-  # Note: Use the commit before support for new GeckoView interfaces gets added.
-  fetch_gradle_dependencies: 0
   gradle_version: 6.6.1
   glean_parser: 2.2.0
 
@@ -27,10 +24,13 @@ targets:
 
 input_files:
   - project: container-image
+    pkg_type: build
   - name: '[% c("var/compiler") %]'
     project: '[% c("var/compiler") %]'
+    pkg_type: build
   - project: gradle
     name: gradle
+    pkg_type: build
   - name: geckoview
     project: geckoview
     pkg_type: merge_aars
@@ -52,6 +52,8 @@ input_files:
     name: glean
     sha256sum: 24ceaaadaf155445e8ad135173d894e05c0745b41ab5fee150f9548550acf2a6
     enable: '[% !c("var/fetch_gradle_dependencies") %]'
+  - filename: gen_gradle_deps_file.sh
+    enable: '[% c("var/fetch_gradle_dependencies") %]'
 
 steps:
   list_toolchain_updates:
@@ -75,3 +77,9 @@ steps:
       - name: glean
         project: glean
         pkg_type: src
+
+  get_gradle_dependencies_list:
+    filename: 'gradle-dependencies-list-[% c("var/android_components_version") %].txt'
+    get_gradle_dependencies_list: '[% INCLUDE build %]'
+    var:
+      fetch_gradle_dependencies: 1
diff --git a/projects/application-services/build b/projects/application-services/build
index b7461bf..40001cd 100644
--- a/projects/application-services/build
+++ b/projects/application-services/build
@@ -83,7 +83,9 @@ patch -p1 < $rootdir/target.patch
 [% IF c('var/fetch_gradle_dependencies') %]
   # XXX: `assemble` is still not enough to see all fetched dependencies via
   # Gradle's --debug. See: tor-browser-build#40056.
-  gradle --debug --no-daemon assemble
+  gradle --debug --no-daemon assemble > gradle.log 2>&1 || ( cat gradle.log; exit 1 )
+  $rootdir/gen_gradle_deps_file.sh gradle.log
+  mv gradle-dependencies-list.txt '[% dest_dir _ "/" _ c("filename") %]'
 [% ELSE %]
   # Prepare Glean dependencies for offline build
   tar -xjf $rootdir/glean-parser-[% c('var/glean_parser') %].tar.bz2
diff --git a/projects/application-services/config b/projects/application-services/config
index 588d573..456ce87 100644
--- a/projects/application-services/config
+++ b/projects/application-services/config
@@ -12,50 +12,61 @@ var:
   gradle_dependencies_version: 5
   # This should be updated when the list of rust dependencies is changed.
   rust_vendor_version: 5
-  # Switch to make it easier to grab all dependencies during a dry-run.
-  fetch_gradle_dependencies: 0
   gradle_version: 6.5
 
 input_files:
   - project: container-image
+    pkg_type: build
   - project: '[% c("var/compiler") %]'
     name: '[% c("var/compiler") %]'
+    pkg_type: build
   - project: gradle
     name: gradle
+    pkg_type: build
   - project: rust
     name: rust
+    pkg_type: build
   - project: uniffi-rs
     name: uniffi-rs
+    pkg_type: build
   - project: nss
     name: nss-armv7
+    pkg_type: build
     target_prepend:
       - android-armv7
   - project: nss
     name: nss-aarch64
+    pkg_type: build
     target_prepend:
       - android-aarch64
   - project: nss
     name: nss-x86
+    pkg_type: build
     target_prepend:
       - android-x86
   - project: nss
     name: nss-x86_64
+    pkg_type: build
     target_prepend:
       - android-x86_64
   - project: sqlcipher
     name: sqlcipher-armv7
+    pkg_type: build
     target_prepend:
       - android-armv7
   - project: sqlcipher
     name: sqlcipher-aarch64
+    pkg_type: build
     target_prepend:
       - android-aarch64
   - project: sqlcipher
     name: sqlcipher-x86
+    pkg_type: build
     target_prepend:
       - android-x86
   - project: sqlcipher
     name: sqlcipher-x86_64
+    pkg_type: build
     target_prepend:
       - android-x86_64
   - name: python
@@ -79,6 +90,8 @@ input_files:
   - filename: target.patch
   - filename: viaduct-workaround.patch
   - filename: update-cargo-lock.patch
+  - filename: gen_gradle_deps_file.sh
+    enable: '[% c("var/fetch_gradle_dependencies") %]'
 
 steps:
   list_toolchain_updates:
@@ -87,3 +100,9 @@ steps:
     var:
       container:
         use_container: 0
+
+  get_gradle_dependencies_list:
+    filename: 'gradle-dependencies-list-[% c("version") %].txt'
+    get_gradle_dependencies_list: '[% INCLUDE build %]'
+    var:
+      fetch_gradle_dependencies: 1
diff --git a/tools/gen_gradle_deps_file.sh b/projects/common/gen_gradle_deps_file.sh
similarity index 100%
rename from tools/gen_gradle_deps_file.sh
rename to projects/common/gen_gradle_deps_file.sh
diff --git a/projects/common/how-to-create-gradle-dependencies-list.txt b/projects/common/how-to-create-gradle-dependencies-list.txt
index 940f810..b233eec 100644
--- a/projects/common/how-to-create-gradle-dependencies-list.txt
+++ b/projects/common/how-to-create-gradle-dependencies-list.txt
@@ -3,9 +3,17 @@ If additional Android dependencies are required by the project's build, then
 the Gradle build will fail due to missing dependencies. To find out what the
 missing dependencies are take the following steps.
 
-If the dependencies for `application-services`, `android-components`, or `fenix`
-are needed, set the `fetch_gradle_dependencies` flag in their `config` files to
-`1`.
+1) Updating gradle dependencies for `application-services`, `android-components`
+   or `fenix`
+
+The following makefile rules can be used:
+
+  make get_gradle_dependencies_list-application-services
+  make get_gradle_dependencies_list-android-components
+  make get_gradle_dependencies_list-fenix
+
+which should create the gradle-dependencies-list.txt files in the
+corresponding out/$project directory, using nightly's branch.
 
 Note: `android-components` and `fenix` require modified `geckoview` and
 `android-components` artifacts to compile successfully. In order to generate the
@@ -18,7 +26,16 @@ breaks). For `fenix` dependencies there is still a manual processing required
 afterwards right now as e.g. our tor-android-service related artifacts are not
 picked up.
 
-For the firefox project, comment out the following line in the project's build file:
+
+2) Updating gradle dependencies for `geckoview`, `tor-android-service`
+   or `tor-onion-proxy-library`
+
+The `geckoview`, `tor-android-service` and `tor-onion-proxy-library`
+projects don't have (yet) a makefile rule to generate their
+gradle-dependencies-list.txt file, so a few more steps are needed.
+
+For the geckoview project, comment out the following line in the project's
+build file:
 
    export GRADLE_MAVEN_REPOSITORIES="file://$gradle_repo"
 
@@ -34,11 +51,14 @@ Finally, allow network access during the build by setting
 
 Dependent artifacts will show up as downloads in the logs. You can pull out
 these dependencies into a list by passing the log file to the gradle dependency
-list script in the tools directory:
+list script in the projects/common directory:
+
+`projects/common/gen_gradle_deps_file.sh /path/to/log/file`
+
 
-`./gen_gradle_deps_file.sh /path/to/log/file`
+3) Copying the resulting `gradle-dependencies-list.txt`
 
-Copy the resulting `gradle-dependencies-list.txt` over the one in the respective
-project. Then, in the project's config file, increment the
-`var/gradle_dependencies_version` and make sure to restore the project's build
-file back to original or set `fetch_gradle_dependencies` to `0` again.
+Copy the resulting `gradle-dependencies-list.txt` (from step 1. or 2.)
+over the one in the respective project. Then, in the project's config
+file, increment the `var/gradle_dependencies_version` and make sure to
+restore the project's build file back to original if you modified it.
diff --git a/projects/fenix/build b/projects/fenix/build
index b118cdc..6de75cf 100644
--- a/projects/fenix/build
+++ b/projects/fenix/build
@@ -42,7 +42,12 @@ cp $rootdir/[% c('input_files_by_name/tor-android-service') %]/* app/
 # XXX We need the build variant in lower case. Do something smarter here.
 v=[% c("variant") %]
 [% IF c("var/fetch_gradle_dependencies") %]
-  $GRADLE_HOME/gradle/bin/gradle --debug --no-daemon app:assemble[% c('variant') %] app:assembleAndroidTest -x lint -PtestBuildType=${v,} -PdisableOptimization
+  $GRADLE_HOME/gradle/bin/gradle --debug --no-daemon app:assemble[% c('variant') %] \
+    app:assembleAndroidTest -x lint -PtestBuildType=${v,} -PdisableOptimization \
+    > gradle.log 2>&1 \
+    || ( cat gradle.log; exit 1 )
+  $rootdir/gen_gradle_deps_file.sh gradle.log
+  mv gradle-dependencies-list.txt '[% dest_dir _ "/" _ c("filename") %]'
 [% ELSE %]
   # Add our localized strings
   [% FOREACH lang = c('var/locales_mobile');
diff --git a/projects/fenix/config b/projects/fenix/config
index 46cbc1e..ff87d4b 100644
--- a/projects/fenix/config
+++ b/projects/fenix/config
@@ -15,9 +15,6 @@ var:
     use_container: 1
   # This should be updated when the list of gradle dependencies is changed.
   gradle_dependencies_version: 24
-  # Switch to make it easier to grab all dependencies during a dry-run.
-  # Note: Use the commit that integrates Tor.
-  fetch_gradle_dependencies: 0
   gradle_version: 6.5.1
   glean_parser: 2.2.0
 
@@ -35,30 +32,22 @@ input_files:
     name: gradle
   - name: android-components
     project: android-components
-    enable: '[% !c("var/fetch_gradle_dependencies") %]'
   - name: application-services
     project: application-services
-    enable: '[% !c("var/fetch_gradle_dependencies") %]'
   - name: geckoview
     project: geckoview
     pkg_type: merge_aars
-    enable: '[% !c("var/fetch_gradle_dependencies") %]'
   - name: python
     project: python
-    enable: '[% !c("var/fetch_gradle_dependencies") %]'
   - name: tba-translation
     project: tba-translation
-    enable: '[% !c("var/fetch_gradle_dependencies") %]'
   - filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]'
     name: gradle-dependencies
-    enable: '[% !c("var/fetch_gradle_dependencies") %]'
     exec: '[% INCLUDE "fetch-gradle-dependencies" %]'
   - URL: https://people.torproject.org/~boklm/mirrors/sources/glean-wheels-[% c('var/glean_parser') %].tar.xz
     name: glean
     sha256sum: 24ceaaadaf155445e8ad135173d894e05c0745b41ab5fee150f9548550acf2a6
-    enable: '[% !c("var/fetch_gradle_dependencies") %]'
   - filename: mavenLocal.patch
-    enable: '[% !c("var/fetch_gradle_dependencies") %]'
   - project: tor-android-service
     name: tor-android-service
   - project: tor-onion-proxy-library
@@ -73,3 +62,25 @@ steps:
     var:
       container:
         use_container: 0
+
+  get_gradle_dependencies_list:
+    filename: 'gradle-dependencies-list-[% c("var/fenix_version") %].txt'
+    get_gradle_dependencies_list: '[% INCLUDE build %]'
+    var:
+      fetch_gradle_dependencies: 1
+    input_files:
+      - project: container-image
+        pkg_type: build
+      - filename: gen_gradle_deps_file.sh
+      - name: '[% c("var/compiler") %]'
+        project: '[% c("var/compiler") %]'
+        pkg_type: build
+      - project: gradle
+        name: gradle
+        pkg_type: build
+      - project: tor-android-service
+        name: tor-android-service
+        pkg_type: build
+      - project: tor-onion-proxy-library
+        name: topl
+        pkg_type: build

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

Prev by Author: [tor-commits] [tor-browser-build/maint-10.0-android] Bug 40241: Update components for mozilla87-based Fenix (application-services part)
Next by Author: [tor-commits] [tor-browser-build/master] Release preparations for 10.5a15
Previous by thread: [tor-commits] [tor-browser-build/master] Bug 40259: Update components for mozilla88-based Fenix (fenix part)
Next by thread: [tor-commits] [translation/tbmanual-contentspot] remove dropped locale
Index(es):
- Author
- Thread