Commits:
-
5f6737f0
by Beatriz Rizental at 2025-08-21T15:13:13+02:00
BB 43564: Modify ./mach bootstrap for Base Browser
-
2b7a0550
by Beatriz Rizental at 2025-08-21T15:13:16+02:00
fixup! BB 43564: Modify ./mach bootstrap for Base Browser
EXTRA: Stop asking to configure git during bootstrap.
-
c1827fba
by Beatriz Rizental at 2025-08-21T16:01:36+02:00
TB 43564: Modify ./mach bootstrap for Tor Browser
10 changed files:
Changes:
build/moz.configure/basebrowser-resources.configure
|
|
1
|
+# Helpers
|
|
|
2
|
+# -------------------------------------------------
|
|
|
3
|
+
|
|
|
4
|
+
|
|
|
5
|
+@depends(build_project)
|
|
|
6
|
+def is_desktop_build(build_project):
|
|
|
7
|
+ return build_project == "browser"
|
|
|
8
|
+
|
|
|
9
|
+
|
|
|
10
|
+# Bootstrap resources
|
|
|
11
|
+# -------------------------------------------------
|
|
|
12
|
+
|
|
|
13
|
+
|
|
|
14
|
+option(
|
|
|
15
|
+ "--with-noscript",
|
|
|
16
|
+ env="NOSCRIPT",
|
|
|
17
|
+ nargs=1,
|
|
|
18
|
+ default=None,
|
|
|
19
|
+ help="Path to noscript .xpi extension archive.",
|
|
|
20
|
+)
|
|
|
21
|
+
|
|
|
22
|
+
|
|
|
23
|
+@depends(
|
|
|
24
|
+ "--with-noscript",
|
|
|
25
|
+ mozbuild_state_path,
|
|
|
26
|
+ bootstrap_path(
|
|
|
27
|
+ "noscript", no_unpack=True, when=depends("--with-noscript")(lambda x: not x)
|
|
|
28
|
+ ),
|
|
|
29
|
+)
|
|
|
30
|
+@checking("for noscript")
|
|
|
31
|
+@imports(_from="pathlib", _import="Path")
|
|
|
32
|
+def noscript(value, mozbuild_state_path, _bootstrapped):
|
|
|
33
|
+ if value:
|
|
|
34
|
+ path = Path(value[0])
|
|
|
35
|
+ if path.is_file() and path.suffix == ".xpi":
|
|
|
36
|
+ return value[0]
|
|
|
37
|
+ else:
|
|
|
38
|
+ die("--with-noscript must be an existing .xpi file")
|
|
|
39
|
+
|
|
|
40
|
+ bootstrapped_location = Path(mozbuild_state_path) / "browser"
|
|
|
41
|
+ for file in bootstrapped_location.glob(f"*.xpi"):
|
|
|
42
|
+ if "noscript" in file.name:
|
|
|
43
|
+ return str(bootstrapped_location / file)
|
|
|
44
|
+
|
|
|
45
|
+ # noscript is not required for building.
|
|
|
46
|
+ return None
|
|
|
47
|
+
|
|
|
48
|
+
|
|
|
49
|
+set_config("NOSCRIPT", noscript)
|
|
|
50
|
+
|
|
|
51
|
+
|
|
|
52
|
+option(
|
|
|
53
|
+ "--with-tor-browser-fonts",
|
|
|
54
|
+ env="TOR_BROWSER_FONTS",
|
|
|
55
|
+ nargs=1,
|
|
|
56
|
+ default=None,
|
|
|
57
|
+ help="Path to location of fonts directory.",
|
|
|
58
|
+)
|
|
|
59
|
+
|
|
|
60
|
+
|
|
|
61
|
+@depends(
|
|
|
62
|
+ "--with-tor-browser-fonts",
|
|
|
63
|
+ mozbuild_state_path,
|
|
|
64
|
+ bootstrap_path(
|
|
|
65
|
+ "fonts",
|
|
|
66
|
+ when=depends("--with-tor-browser-fonts")(lambda x: not x) & is_desktop_build,
|
|
|
67
|
+ ),
|
|
|
68
|
+)
|
|
|
69
|
+@checking("for tor-browser fonts directory")
|
|
|
70
|
+@imports(_from="pathlib", _import="Path")
|
|
|
71
|
+def tor_browser_fonts(value, mozbuild_state_path, _bootstrapped):
|
|
|
72
|
+ if value:
|
|
|
73
|
+ path = Path(value[0])
|
|
|
74
|
+ # TODO: Do a more thorough check on the directory.
|
|
|
75
|
+ if path.is_dir():
|
|
|
76
|
+ return value[0]
|
|
|
77
|
+ else:
|
|
|
78
|
+ die("--with-tor-browser-fonts must point to a real directory.")
|
|
|
79
|
+
|
|
|
80
|
+ bootstrapped_location = Path(mozbuild_state_path) / "fonts"
|
|
|
81
|
+ if bootstrapped_location.is_dir():
|
|
|
82
|
+ return str(bootstrapped_location)
|
|
|
83
|
+
|
|
|
84
|
+ # tor browser fonts directory is not required for building.
|
|
|
85
|
+ return None
|
|
|
86
|
+
|
|
|
87
|
+
|
|
|
88
|
+set_config("TOR_BROWSER_FONTS", tor_browser_fonts) |
build/moz.configure/bootstrap.configure
| ... |
... |
@@ -4,6 +4,29 @@ |
|
4
|
4
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
5
|
5
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
6
|
6
|
|
|
|
7
|
+option(
|
|
|
8
|
+ "--with-tor-browser-build-out",
|
|
|
9
|
+ env="TOR_BROWSER_BUILD_OUT",
|
|
|
10
|
+ nargs=1,
|
|
|
11
|
+ default="https://tb-build-06.torproject.org/~tb-builder/tor-browser-build/out",
|
|
|
12
|
+ help="URL pointing to a Tor Browser Build out folder, served over HTTP[S].",
|
|
|
13
|
+)
|
|
|
14
|
+
|
|
|
15
|
+
|
|
|
16
|
+@depends("--with-tor-browser-build-out")
|
|
|
17
|
+def tor_browser_build_out(value):
|
|
|
18
|
+ if value:
|
|
|
19
|
+ return value[0]
|
|
|
20
|
+
|
|
|
21
|
+
|
|
|
22
|
+option(
|
|
|
23
|
+ "--enable-tor-browser-build-only-bootstrap",
|
|
|
24
|
+ env="TBB_ONLY_BOOTSTRAP",
|
|
|
25
|
+ default=False,
|
|
|
26
|
+ help="Flag that disables bootstrapping any artifact from Mozilla's Taskcluster. Will only bootstrap artifacts from tor-browser-build.",
|
|
|
27
|
+)
|
|
|
28
|
+
|
|
|
29
|
+
|
|
7
|
30
|
option(
|
|
8
|
31
|
env="MOZ_FETCHES_DIR",
|
|
9
|
32
|
nargs=1,
|
| ... |
... |
@@ -115,9 +138,10 @@ def bootstrap_toolchain_tasks(host): |
|
115
|
138
|
def bootstrap_path(path, **kwargs):
|
|
116
|
139
|
when = kwargs.pop("when", None)
|
|
117
|
140
|
allow_failure = kwargs.pop("allow_failure", None)
|
|
|
141
|
+ no_unpack = kwargs.pop("no_unpack", False)
|
|
118
|
142
|
if kwargs:
|
|
119
|
143
|
configure_error(
|
|
120
|
|
- "bootstrap_path only takes `when` and `allow_failure` as a keyword argument"
|
|
|
144
|
+ "bootstrap_path only takes `when`, `allow_failure` and `no_unpack` as keyword arguments"
|
|
121
|
145
|
)
|
|
122
|
146
|
|
|
123
|
147
|
@depends(
|
| ... |
... |
@@ -129,11 +153,16 @@ def bootstrap_path(path, **kwargs): |
|
129
|
153
|
build_environment,
|
|
130
|
154
|
dependable(path),
|
|
131
|
155
|
dependable(allow_failure),
|
|
|
156
|
+ dependable(no_unpack),
|
|
|
157
|
+ tor_browser_build_out,
|
|
|
158
|
+ "--enable-tor-browser-build-only-bootstrap",
|
|
|
159
|
+ target,
|
|
132
|
160
|
when=when,
|
|
133
|
161
|
)
|
|
134
|
162
|
@imports("os")
|
|
135
|
163
|
@imports("subprocess")
|
|
136
|
164
|
@imports("sys")
|
|
|
165
|
+ @imports("mozbuild.tbbutils")
|
|
137
|
166
|
@imports(_from="mozbuild.dirutils", _import="ensureParentDir")
|
|
138
|
167
|
@imports(_from="importlib", _import="import_module")
|
|
139
|
168
|
@imports(_from="shutil", _import="rmtree")
|
| ... |
... |
@@ -148,6 +177,10 @@ def bootstrap_path(path, **kwargs): |
|
148
|
177
|
build_env,
|
|
149
|
178
|
path,
|
|
150
|
179
|
allow_failure,
|
|
|
180
|
+ no_unpack,
|
|
|
181
|
+ tor_browser_build_out,
|
|
|
182
|
+ tbb_only_bootstrap,
|
|
|
183
|
+ target,
|
|
151
|
184
|
):
|
|
152
|
185
|
if not path:
|
|
153
|
186
|
return
|
| ... |
... |
@@ -158,6 +191,81 @@ def bootstrap_path(path, **kwargs): |
|
158
|
191
|
if path_parts[0] == "clang-tools":
|
|
159
|
192
|
path_prefix = path_parts.pop(0)
|
|
160
|
193
|
|
|
|
194
|
+ # Small hack because noscript is inside the browser folder.
|
|
|
195
|
+ if path_parts[0] == "noscript":
|
|
|
196
|
+ path_prefix = "browser"
|
|
|
197
|
+
|
|
|
198
|
+ def try_tbb_bootstrap(exists):
|
|
|
199
|
+ if not tor_browser_build_out:
|
|
|
200
|
+ return False
|
|
|
201
|
+
|
|
|
202
|
+ # Tor browser build doesn't have artifacts for all targets supported
|
|
|
203
|
+ # by the Firefox build system. When this is empty it means we are
|
|
|
204
|
+ # building for a platform which tbb doesn't support.
|
|
|
205
|
+ if not target.tor_browser_build_alias:
|
|
|
206
|
+ return False
|
|
|
207
|
+
|
|
|
208
|
+ artifact = mozbuild.tbbutils.get_artifact_name(
|
|
|
209
|
+ path_parts[0], target, tasks.prefix
|
|
|
210
|
+ )
|
|
|
211
|
+ if not artifact:
|
|
|
212
|
+ log.info("%s is not mapped to a tbb artifact", path_parts[0])
|
|
|
213
|
+ return False
|
|
|
214
|
+
|
|
|
215
|
+ artifact_path = mozbuild.tbbutils.get_artifact_path(
|
|
|
216
|
+ tor_browser_build_out, artifact, target, prefix=path_prefix
|
|
|
217
|
+ )
|
|
|
218
|
+ if not artifact_path:
|
|
|
219
|
+ log.info("no path found in tbb/out for %s", artifact)
|
|
|
220
|
+ return False
|
|
|
221
|
+
|
|
|
222
|
+ # We will use the name of the artifact as the index.
|
|
|
223
|
+ #
|
|
|
224
|
+ # It's usually unique to the artifact version, but each artifact follows
|
|
|
225
|
+ # a different naming convention, so we can't really get more specific here.
|
|
|
226
|
+ artifact_index = artifact_path.rsplit("/", 1)[-1]
|
|
|
227
|
+ index_file = os.path.join(toolchains_base_dir, "indices", artifact)
|
|
|
228
|
+ try:
|
|
|
229
|
+ with open(index_file) as fh:
|
|
|
230
|
+ index = fh.read().strip()
|
|
|
231
|
+ except Exception:
|
|
|
232
|
+ index = None
|
|
|
233
|
+ if index == artifact_index and exists:
|
|
|
234
|
+ log.debug("%s is up-to-date", artifact)
|
|
|
235
|
+ return True
|
|
|
236
|
+
|
|
|
237
|
+ command = ["artifact", "toolchain", "--from-url", artifact_path]
|
|
|
238
|
+
|
|
|
239
|
+ if no_unpack:
|
|
|
240
|
+ command.append("--no-unpack")
|
|
|
241
|
+
|
|
|
242
|
+ # Note to rebasers:
|
|
|
243
|
+ # From here on, it's a slightly modified copy/paste
|
|
|
244
|
+ # from the end of the try_bootstrap function
|
|
|
245
|
+ log.info(
|
|
|
246
|
+ "%s bootstrapped toolchain from TBB in %s",
|
|
|
247
|
+ "Updating" if exists else "Installing",
|
|
|
248
|
+ os.path.join(toolchains_base_dir, path_prefix, artifact),
|
|
|
249
|
+ )
|
|
|
250
|
+ os.makedirs(os.path.join(toolchains_base_dir, path_prefix), exist_ok=True)
|
|
|
251
|
+ proc = subprocess.run(
|
|
|
252
|
+ [
|
|
|
253
|
+ sys.executable,
|
|
|
254
|
+ os.path.join(build_env.topsrcdir, "mach"),
|
|
|
255
|
+ "--log-no-times",
|
|
|
256
|
+ ]
|
|
|
257
|
+ + command,
|
|
|
258
|
+ cwd=os.path.join(toolchains_base_dir, path_prefix),
|
|
|
259
|
+ check=not allow_failure,
|
|
|
260
|
+ )
|
|
|
261
|
+ if proc.returncode != 0 and allow_failure:
|
|
|
262
|
+ return False
|
|
|
263
|
+ ensureParentDir(index_file)
|
|
|
264
|
+ with open(index_file, "w") as fh:
|
|
|
265
|
+ fh.write(artifact_index)
|
|
|
266
|
+
|
|
|
267
|
+ return True
|
|
|
268
|
+
|
|
161
|
269
|
def try_bootstrap(exists):
|
|
162
|
270
|
if not tasks:
|
|
163
|
271
|
return False
|
| ... |
... |
@@ -280,9 +388,10 @@ def bootstrap_path(path, **kwargs): |
|
280
|
388
|
try:
|
|
281
|
389
|
# With --enable-bootstrap=no-update, we don't `try_bootstrap`, except
|
|
282
|
390
|
# when the toolchain can't be found.
|
|
283
|
|
- if (
|
|
284
|
|
- "no-update" not in enable_bootstrap or not exists
|
|
285
|
|
- ) and not try_bootstrap(exists):
|
|
|
391
|
+ if ("no-update" not in enable_bootstrap or not exists) and not (
|
|
|
392
|
+ try_tbb_bootstrap(exists)
|
|
|
393
|
+ or (not tbb_only_bootstrap and try_bootstrap(exists))
|
|
|
394
|
+ ):
|
|
286
|
395
|
# If there aren't toolchain artifacts to use for this build,
|
|
287
|
396
|
# don't return a path.
|
|
288
|
397
|
return None
|
build/moz.configure/init.configure
| ... |
... |
@@ -590,6 +590,21 @@ def split_triplet(triplet, allow_wasi=False): |
|
590
|
590
|
else:
|
|
591
|
591
|
toolchain = "%s-%s" % (cpu, os)
|
|
592
|
592
|
|
|
|
593
|
+ # In tor-browser-build we use slightly different terminology for
|
|
|
594
|
+ # the supported platforms. Let's prepare that OS string here.
|
|
|
595
|
+ #
|
|
|
596
|
+ # Not all possible platforms listed here are supported in tbb,
|
|
|
597
|
+ # so this value will be empty sometimes.
|
|
|
598
|
+ tor_browser_build_alias = None
|
|
|
599
|
+ if canonical_os == "Android" and canonical_kernel == "Linux":
|
|
|
600
|
+ tor_browser_build_alias = f"android"
|
|
|
601
|
+ elif canonical_os == "GNU" and canonical_kernel == "Linux":
|
|
|
602
|
+ tor_browser_build_alias = f"linux"
|
|
|
603
|
+ elif canonical_os == "OSX" and canonical_kernel == "Darwin":
|
|
|
604
|
+ tor_browser_build_alias = f"macos"
|
|
|
605
|
+ elif canonical_os == "WINNT" and canonical_kernel == "WINNT":
|
|
|
606
|
+ tor_browser_build_alias = f"windows"
|
|
|
607
|
+
|
|
593
|
608
|
return namespace(
|
|
594
|
609
|
alias=triplet,
|
|
595
|
610
|
cpu=CPU(canonical_cpu),
|
| ... |
... |
@@ -604,6 +619,7 @@ def split_triplet(triplet, allow_wasi=False): |
|
604
|
619
|
toolchain=toolchain,
|
|
605
|
620
|
vendor=vendor,
|
|
606
|
621
|
sub_configure_alias=sub_configure_alias,
|
|
|
622
|
+ tor_browser_build_alias=tor_browser_build_alias,
|
|
607
|
623
|
)
|
|
608
|
624
|
|
|
609
|
625
|
|
build/moz.configure/torbrowser-resources.configure
|
|
1
|
+option(
|
|
|
2
|
+ "--with-tor-expert-bundle",
|
|
|
3
|
+ env="TOR_EXPERT_BUNDLE",
|
|
|
4
|
+ nargs=1,
|
|
|
5
|
+ default=None,
|
|
|
6
|
+ help="Path to location of tor-expert-bundle directory.",
|
|
|
7
|
+)
|
|
|
8
|
+
|
|
|
9
|
+
|
|
|
10
|
+@depends(
|
|
|
11
|
+ "--with-tor-expert-bundle",
|
|
|
12
|
+ mozbuild_state_path,
|
|
|
13
|
+ bootstrap_path(
|
|
|
14
|
+ "tor-expert-bundle",
|
|
|
15
|
+ when=depends("--with-tor-expert-bundle")(lambda x: not x) & is_desktop_build,
|
|
|
16
|
+ ),
|
|
|
17
|
+)
|
|
|
18
|
+@checking("for tor-expert-bundle")
|
|
|
19
|
+@imports(_from="pathlib", _import="Path")
|
|
|
20
|
+def tor_expert_bundle(value, mozbuild_state_path, _bootstrapped):
|
|
|
21
|
+ if value:
|
|
|
22
|
+ path = Path(value[0])
|
|
|
23
|
+ # TODO: Do a more thorough check on the directory.
|
|
|
24
|
+ if path.is_dir():
|
|
|
25
|
+ return value[0]
|
|
|
26
|
+ else:
|
|
|
27
|
+ die("--with-tor-expert-bundle must point to a real directory.")
|
|
|
28
|
+
|
|
|
29
|
+ bootstrapped_location = Path(mozbuild_state_path) / "tor-expert-bundle"
|
|
|
30
|
+ if bootstrapped_location.is_dir():
|
|
|
31
|
+ return str(bootstrapped_location)
|
|
|
32
|
+
|
|
|
33
|
+ # tor-expert-bundle is not required for building.
|
|
|
34
|
+ return None
|
|
|
35
|
+
|
|
|
36
|
+
|
|
|
37
|
+set_config("TOR_EXPERT_BUNDLE", tor_expert_bundle) |
moz.configure
| ... |
... |
@@ -229,6 +229,8 @@ check_prog("WGET", ("wget",), allow_missing=True) |
|
229
|
229
|
|
|
230
|
230
|
|
|
231
|
231
|
include("build/moz.configure/toolchain.configure", when="--enable-compile-environment")
|
|
|
232
|
+include("build/moz.configure/basebrowser-resources.configure")
|
|
|
233
|
+include("build/moz.configure/torbrowser-resources.configure")
|
|
232
|
234
|
|
|
233
|
235
|
include("build/moz.configure/pkg.configure")
|
|
234
|
236
|
include("build/moz.configure/memory.configure", when="--enable-compile-environment")
|
python/mozboot/mozboot/bootstrap.py
| ... |
... |
@@ -52,21 +52,28 @@ Note on Artifact Mode: |
|
52
|
52
|
Artifact builds download prebuilt C++ components rather than building
|
|
53
|
53
|
them locally. Artifact builds are faster!
|
|
54
|
54
|
|
|
55
|
|
-Artifact builds are recommended for people working on Firefox or
|
|
56
|
|
-Firefox for Android frontends, or the GeckoView Java API. They are unsuitable
|
|
|
55
|
+Artifact builds are recommended for people working on Tor Browser or
|
|
|
56
|
+Tor Browser for Android frontends, or the GeckoView Java API. They are unsuitable
|
|
57
|
57
|
for those working on C++ code. For more information see:
|
|
58
|
58
|
https://firefox-source-docs.mozilla.org/contributing/build/artifact_builds.html.
|
|
59
|
59
|
|
|
60
|
|
-Please choose the version of Firefox you want to build (see note above):
|
|
|
60
|
+# Note to Tor Browser developers
|
|
|
61
|
+
|
|
|
62
|
+This is still highly experimental. Expect bugs!
|
|
|
63
|
+
|
|
|
64
|
+Please choose the version of Tor Browser you want to build (see note above):
|
|
61
|
65
|
%s
|
|
62
|
66
|
Your choice: """
|
|
63
|
67
|
|
|
64
|
68
|
APPLICATIONS = OrderedDict(
|
|
65
|
69
|
[
|
|
66
|
|
- ("Firefox for Desktop Artifact Mode", "browser_artifact_mode"),
|
|
67
|
|
- ("Firefox for Desktop", "browser"),
|
|
68
|
|
- ("GeckoView/Firefox for Android Artifact Mode", "mobile_android_artifact_mode"),
|
|
69
|
|
- ("GeckoView/Firefox for Android", "mobile_android"),
|
|
|
70
|
+ ("Tor Browser for Desktop Artifact Mode", "browser_artifact_mode"),
|
|
|
71
|
+ ("Tor Browser for Desktop", "browser"),
|
|
|
72
|
+ (
|
|
|
73
|
+ "GeckoView/Tor Browser for Android Artifact Mode",
|
|
|
74
|
+ "mobile_android_artifact_mode",
|
|
|
75
|
+ ),
|
|
|
76
|
+ ("GeckoView/Tor Browser for Android", "mobile_android"),
|
|
70
|
77
|
("SpiderMonkey _javascript_ engine", "js"),
|
|
71
|
78
|
]
|
|
72
|
79
|
)
|
| ... |
... |
@@ -360,6 +367,8 @@ class Bootstrapper: |
|
360
|
367
|
getattr(self.instance, "ensure_%s_packages" % application)()
|
|
361
|
368
|
|
|
362
|
369
|
def check_code_submission(self, checkout_root: Path):
|
|
|
370
|
+ return
|
|
|
371
|
+
|
|
363
|
372
|
if self.instance.no_interactive or which("moz-phab"):
|
|
364
|
373
|
return
|
|
365
|
374
|
|
| ... |
... |
@@ -474,8 +483,7 @@ class Bootstrapper: |
|
474
|
483
|
configure_mercurial(hg, state_dir)
|
|
475
|
484
|
|
|
476
|
485
|
# Offer to configure Git, if the current checkout or repo type is Git.
|
|
477
|
|
- elif git and checkout_type == "git":
|
|
478
|
|
- should_configure_git = False
|
|
|
486
|
+ elif False and git and checkout_type == "git":
|
|
479
|
487
|
if not self.instance.no_interactive:
|
|
480
|
488
|
should_configure_git = self.instance.prompt_yesno(prompt=CONFIGURE_GIT)
|
|
481
|
489
|
else:
|
python/mozbuild/mozbuild/action/tooltool.py
| ... |
... |
@@ -1029,14 +1029,29 @@ def unpack_file(filename): |
|
1029
|
1029
|
"""Untar `filename`, assuming it is uncompressed or compressed with bzip2,
|
|
1030
|
1030
|
xz, gzip, zst, or unzip a zip file. The file is assumed to contain a single
|
|
1031
|
1031
|
directory with a name matching the base of the given filename.
|
|
1032
|
|
- Xz support is handled by shelling out to 'tar'."""
|
|
|
1032
|
+ Xz support is handled by shelling out to 'tar'.
|
|
|
1033
|
+
|
|
|
1034
|
+ tor-browser#41564 - For supporting tor-browser-build artifacts that contain
|
|
|
1035
|
+ multiple directories, the archive is extracted into a directory with the
|
|
|
1036
|
+ same name as the base of the filename. This modification is only applied to
|
|
|
1037
|
+ tar archives, because that is all that was necessary.
|
|
|
1038
|
+ """
|
|
1033
|
1039
|
if os.path.isfile(filename) and tarfile.is_tarfile(filename):
|
|
1034
|
1040
|
tar_file, zip_ext = os.path.splitext(filename)
|
|
1035
|
1041
|
base_file, tar_ext = os.path.splitext(tar_file)
|
|
1036
|
1042
|
clean_path(base_file)
|
|
1037
|
1043
|
log.info('untarring "%s"' % filename)
|
|
1038
|
1044
|
with TarFile.open(filename) as tar:
|
|
1039
|
|
- safe_extract(tar)
|
|
|
1045
|
+ top_level_directories = set()
|
|
|
1046
|
+ for name in tar.getnames():
|
|
|
1047
|
+ dir = name.split("/", 1)[0]
|
|
|
1048
|
+ top_level_directories.add(dir)
|
|
|
1049
|
+ if len(top_level_directories) == 1:
|
|
|
1050
|
+ safe_extract(tar)
|
|
|
1051
|
+ else:
|
|
|
1052
|
+ safe_extract(
|
|
|
1053
|
+ tar, path=os.path.join(os.path.dirname(filename), base_file)
|
|
|
1054
|
+ )
|
|
1040
|
1055
|
elif os.path.isfile(filename) and filename.endswith(".tar.zst"):
|
|
1041
|
1056
|
import zstandard
|
|
1042
|
1057
|
|
python/mozbuild/mozbuild/artifact_commands.py
| ... |
... |
@@ -244,6 +244,12 @@ def artifact_clear_cache(command_context, tree=None, job=None, verbose=False): |
|
244
|
244
|
nargs="+",
|
|
245
|
245
|
help="Download toolchain artifact from a given task.",
|
|
246
|
246
|
)
|
|
|
247
|
+@CommandArgument(
|
|
|
248
|
+ "--from-url",
|
|
|
249
|
+ metavar="URL",
|
|
|
250
|
+ nargs="+",
|
|
|
251
|
+ help="Download toolchain artifact from an arbitrary address.",
|
|
|
252
|
+)
|
|
247
|
253
|
@CommandArgument(
|
|
248
|
254
|
"--tooltool-manifest",
|
|
249
|
255
|
metavar="MANIFEST",
|
| ... |
... |
@@ -273,6 +279,7 @@ def artifact_toolchain( |
|
273
|
279
|
skip_cache=False,
|
|
274
|
280
|
from_build=(),
|
|
275
|
281
|
from_task=(),
|
|
|
282
|
+ from_url=[],
|
|
276
|
283
|
tooltool_manifest=None,
|
|
277
|
284
|
no_unpack=False,
|
|
278
|
285
|
retry=0,
|
| ... |
... |
@@ -504,6 +511,13 @@ def artifact_toolchain( |
|
504
|
511
|
record = ArtifactRecord(task_id, name)
|
|
505
|
512
|
records[record.filename] = record
|
|
506
|
513
|
|
|
|
514
|
+ if from_url:
|
|
|
515
|
+ for file in from_url:
|
|
|
516
|
+ record = DownloadRecord(
|
|
|
517
|
+ file, file.rsplit("/", 1)[-1], None, None, None, True
|
|
|
518
|
+ )
|
|
|
519
|
+ records[record.filename] = record
|
|
|
520
|
+
|
|
507
|
521
|
for record in records.values():
|
|
508
|
522
|
command_context.log(
|
|
509
|
523
|
logging.INFO,
|
python/mozbuild/mozbuild/backend/base.py
| ... |
... |
@@ -2,11 +2,14 @@ |
|
2
|
2
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
3
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
4
|
4
|
|
|
|
5
|
+import errno
|
|
5
|
6
|
import itertools
|
|
|
7
|
+import logging
|
|
6
|
8
|
import os
|
|
7
|
9
|
import time
|
|
8
|
10
|
from abc import ABCMeta, abstractmethod
|
|
9
|
11
|
from contextlib import contextmanager
|
|
|
12
|
+from pathlib import Path
|
|
10
|
13
|
|
|
11
|
14
|
import mozpack.path as mozpath
|
|
12
|
15
|
from mach.mixin.logging import LoggingMixin
|
| ... |
... |
@@ -239,6 +242,129 @@ class BuildBackend(LoggingMixin): |
|
239
|
242
|
with open(mozpath.join(dir, ".purgecaches"), "w") as f:
|
|
240
|
243
|
f.write("\n")
|
|
241
|
244
|
|
|
|
245
|
+ def _setup_tor_browser_environment(self, config):
|
|
|
246
|
+ app = config.substs["MOZ_BUILD_APP"]
|
|
|
247
|
+
|
|
|
248
|
+ noscript_target_filename = "{73a6fe31-595d-460b-a920-fcc0f8843232}.xpi"
|
|
|
249
|
+ noscript_location = Path(config.substs["NOSCRIPT"])
|
|
|
250
|
+
|
|
|
251
|
+ def _infallible_symlink(src, dst):
|
|
|
252
|
+ try:
|
|
|
253
|
+ os.symlink(src, dst)
|
|
|
254
|
+ except OSError as e:
|
|
|
255
|
+ if e.errno == errno.EEXIST:
|
|
|
256
|
+ # If the symlink already exists, remove it and try again.
|
|
|
257
|
+ os.remove(dst)
|
|
|
258
|
+ os.symlink(src, dst)
|
|
|
259
|
+ else:
|
|
|
260
|
+ return
|
|
|
261
|
+
|
|
|
262
|
+ if app == "browser":
|
|
|
263
|
+ tbdir = Path(config.topobjdir) / "dist" / "bin"
|
|
|
264
|
+
|
|
|
265
|
+ if config.substs.get("OS_TARGET") == "Darwin":
|
|
|
266
|
+ tbdir = next(tbdir.glob("*.app"))
|
|
|
267
|
+ paths = {
|
|
|
268
|
+ "docs": tbdir / "Contents/Resources/TorBrowser/Docs",
|
|
|
269
|
+ "exts": tbdir / "Contents/Resources/distribution/extensions",
|
|
|
270
|
+ "tor_bin": tbdir / "Contents/MacOS/tor",
|
|
|
271
|
+ "tor_config": tbdir / "Contents/Resources/TorBrowser/Tor",
|
|
|
272
|
+ "fonts": tbdir / "Resources/fonts",
|
|
|
273
|
+ }
|
|
|
274
|
+ else:
|
|
|
275
|
+ paths = {
|
|
|
276
|
+ "docs": tbdir / "TorBrowser/Docs",
|
|
|
277
|
+ "exts": tbdir / "distribution/extensions",
|
|
|
278
|
+ "tor_bin": tbdir / "TorBrowser/Tor",
|
|
|
279
|
+ "tor_config": tbdir / "TorBrowser/Data/Tor",
|
|
|
280
|
+ "fonts": tbdir / "fonts",
|
|
|
281
|
+ }
|
|
|
282
|
+
|
|
|
283
|
+ fonts_location = Path(config.substs["TOR_BROWSER_FONTS"])
|
|
|
284
|
+ if fonts_location.is_dir():
|
|
|
285
|
+ self.log(
|
|
|
286
|
+ logging.INFO,
|
|
|
287
|
+ "_setup_tor_browser_environment",
|
|
|
288
|
+ {
|
|
|
289
|
+ "fonts_location": str(fonts_location),
|
|
|
290
|
+ "fonts_target": str(paths["fonts"]),
|
|
|
291
|
+ },
|
|
|
292
|
+ "Creating symlink for fonts files from {fonts_location} to {fonts_target}",
|
|
|
293
|
+ )
|
|
|
294
|
+
|
|
|
295
|
+ for file in fonts_location.iterdir():
|
|
|
296
|
+ target = paths["fonts"] / file.name
|
|
|
297
|
+ _infallible_symlink(file, target)
|
|
|
298
|
+
|
|
|
299
|
+ # Set up NoScript extension
|
|
|
300
|
+ if noscript_location.is_file():
|
|
|
301
|
+ noscript_target = paths["exts"] / noscript_target_filename
|
|
|
302
|
+ self.log(
|
|
|
303
|
+ logging.INFO,
|
|
|
304
|
+ "_setup_tor_browser_environment",
|
|
|
305
|
+ {
|
|
|
306
|
+ "noscript_location": str(noscript_location),
|
|
|
307
|
+ "noscript_target": str(noscript_target),
|
|
|
308
|
+ },
|
|
|
309
|
+ "Creating symlink for NoScript from {noscript_location} to {noscript_target}",
|
|
|
310
|
+ )
|
|
|
311
|
+
|
|
|
312
|
+ paths["exts"].mkdir(parents=True, exist_ok=True)
|
|
|
313
|
+ _infallible_symlink(noscript_location, noscript_target)
|
|
|
314
|
+
|
|
|
315
|
+ expert_bundle_location = Path(config.substs["TOR_EXPERT_BUNDLE"])
|
|
|
316
|
+ if expert_bundle_location.is_dir():
|
|
|
317
|
+ self.log(
|
|
|
318
|
+ logging.INFO,
|
|
|
319
|
+ "_setup_tor_browser_environment",
|
|
|
320
|
+ {
|
|
|
321
|
+ "expert_bundle_location": str(expert_bundle_location),
|
|
|
322
|
+ },
|
|
|
323
|
+ "Setting up tor-expert-bundle resources from {expert_bundle_location}",
|
|
|
324
|
+ )
|
|
|
325
|
+
|
|
|
326
|
+ # Set up Tor configuration files
|
|
|
327
|
+ paths["tor_config"].mkdir(parents=True, exist_ok=True)
|
|
|
328
|
+ for file in ["geoip", "geoip6"]:
|
|
|
329
|
+ target = paths["tor_config"] / file
|
|
|
330
|
+ _infallible_symlink(expert_bundle_location / "data" / file, target)
|
|
|
331
|
+
|
|
|
332
|
+ # Set up Conjure documentation
|
|
|
333
|
+ conjust_docs_location = paths["docs"] / "conjure"
|
|
|
334
|
+ conjust_docs_location.mkdir(parents=True, exist_ok=True)
|
|
|
335
|
+ conjure_readme = conjust_docs_location / "README.CONJURE.md"
|
|
|
336
|
+ _infallible_symlink(
|
|
|
337
|
+ expert_bundle_location
|
|
|
338
|
+ / "tor/pluggable_transports/README.CONJURE.md",
|
|
|
339
|
+ conjure_readme,
|
|
|
340
|
+ )
|
|
|
341
|
+
|
|
|
342
|
+ # Set up pluggable transports
|
|
|
343
|
+ paths["tor_bin"].mkdir(parents=True, exist_ok=True)
|
|
|
344
|
+ pluggable_transports_location = (
|
|
|
345
|
+ expert_bundle_location / "tor/pluggable_transports"
|
|
|
346
|
+ )
|
|
|
347
|
+ pluggable_transports_target = paths["tor_bin"] / "PluggableTransports"
|
|
|
348
|
+ pluggable_transports_target.mkdir(parents=True, exist_ok=True)
|
|
|
349
|
+ for file in pluggable_transports_location.iterdir():
|
|
|
350
|
+ # We only want the PT executables.
|
|
|
351
|
+ if os.access(file, os.X_OK) or file.suffix.lower() == ".exe":
|
|
|
352
|
+ target = pluggable_transports_target / file.name
|
|
|
353
|
+ _infallible_symlink(file, target)
|
|
|
354
|
+
|
|
|
355
|
+ # Setup Tor binary
|
|
|
356
|
+ for item in (expert_bundle_location / "tor").iterdir():
|
|
|
357
|
+ target = paths["tor_bin"] / item.name
|
|
|
358
|
+ if target.is_file():
|
|
|
359
|
+ _infallible_symlink(item, target)
|
|
|
360
|
+
|
|
|
361
|
+ # Set up licenses
|
|
|
362
|
+ licenses_location = paths["docs"] / "Licenses"
|
|
|
363
|
+ licenses_location.mkdir(parents=True, exist_ok=True)
|
|
|
364
|
+ for item in (expert_bundle_location / "docs").iterdir():
|
|
|
365
|
+ target = licenses_location / item.name
|
|
|
366
|
+ _infallible_symlink(item, target)
|
|
|
367
|
+
|
|
242
|
368
|
def post_build(self, config, output, jobs, verbose, status):
|
|
243
|
369
|
"""Called late during 'mach build' execution, after `build(...)` has finished.
|
|
244
|
370
|
|
| ... |
... |
@@ -257,6 +383,9 @@ class BuildBackend(LoggingMixin): |
|
257
|
383
|
"""
|
|
258
|
384
|
self._write_purgecaches(config)
|
|
259
|
385
|
|
|
|
386
|
+ if status == 0:
|
|
|
387
|
+ self._setup_tor_browser_environment(config)
|
|
|
388
|
+
|
|
260
|
389
|
return status
|
|
261
|
390
|
|
|
262
|
391
|
@contextmanager
|
python/mozbuild/mozbuild/tbbutils.py
|
|
1
|
+import re
|
|
|
2
|
+from urllib.request import Request, urlopen
|
|
|
3
|
+
|
|
|
4
|
+
|
|
|
5
|
+def list_files_http(url):
|
|
|
6
|
+ try:
|
|
|
7
|
+ req = Request(url, method="GET")
|
|
|
8
|
+ with urlopen(req) as response:
|
|
|
9
|
+ if response.status != 200:
|
|
|
10
|
+ return []
|
|
|
11
|
+ html = response.read().decode()
|
|
|
12
|
+ except Exception:
|
|
|
13
|
+ return []
|
|
|
14
|
+
|
|
|
15
|
+ links = []
|
|
|
16
|
+ for href in re.findall(r'<a href="">"([^"]+)"', html):
|
|
|
17
|
+ if href == "../":
|
|
|
18
|
+ continue
|
|
|
19
|
+
|
|
|
20
|
+ if "tor-expert-bundle" in href:
|
|
|
21
|
+ href = f"{href}/tor-expert-bundle.tar.gz"
|
|
|
22
|
+
|
|
|
23
|
+ links.append(href)
|
|
|
24
|
+
|
|
|
25
|
+ return links
|
|
|
26
|
+
|
|
|
27
|
+
|
|
|
28
|
+TOR_BROWSER_BUILD_ARTIFACTS = [
|
|
|
29
|
+ # Tor Browser Build-only artifacts, these artifacts are not common with Firefox.
|
|
|
30
|
+ "noscript",
|
|
|
31
|
+ "fonts",
|
|
|
32
|
+ "tor-expert-bundle",
|
|
|
33
|
+]
|
|
|
34
|
+
|
|
|
35
|
+# Mapping of artifacts from taskcluster to tor-browser-build.
|
|
|
36
|
+ARTIFACT_NAME_MAP = {
|
|
|
37
|
+ "cbindgen": "cbindgen",
|
|
|
38
|
+ # FIXME (tor-browser-build#41471): nasm is more or less ready to go, but it needs to have the
|
|
|
39
|
+ # executable in the root of the artifact folder instead of nasm/bin.
|
|
|
40
|
+ # "nasm": "nasm",
|
|
|
41
|
+ # FIXME (tor-browser-build#41421): the clang project as is, is not ready to use. It needs
|
|
|
42
|
+ # to be repackaged with a bunch of things that differ per platform. Fun stuff.
|
|
|
43
|
+ # "clang": "clang",
|
|
|
44
|
+ "node": "node",
|
|
|
45
|
+}
|
|
|
46
|
+
|
|
|
47
|
+
|
|
|
48
|
+def get_artifact_name(original_artifact_name, target, host):
|
|
|
49
|
+ # These are not build artifacts, they are pre-built artifacts to be added to the final build,
|
|
|
50
|
+ # therefore this check can come before the host check.
|
|
|
51
|
+ if original_artifact_name in TOR_BROWSER_BUILD_ARTIFACTS:
|
|
|
52
|
+ return original_artifact_name
|
|
|
53
|
+
|
|
|
54
|
+ if host != "linux64":
|
|
|
55
|
+ # Tor browser build only has development artifacts for linux64 host systems.
|
|
|
56
|
+ return None
|
|
|
57
|
+
|
|
|
58
|
+ return ARTIFACT_NAME_MAP.get(original_artifact_name)
|
|
|
59
|
+
|
|
|
60
|
+
|
|
|
61
|
+def get_artifact_path(url, artifact, target, prefix=""):
|
|
|
62
|
+ if prefix:
|
|
|
63
|
+ path = prefix
|
|
|
64
|
+ else:
|
|
|
65
|
+ path = artifact
|
|
|
66
|
+
|
|
|
67
|
+ # The `?C=M;O=D` parameters make it so links are ordered by
|
|
|
68
|
+ # the last modified date. This here to make us get the latest
|
|
|
69
|
+ # version of file in the case there are multiple and we just
|
|
|
70
|
+ # grab the first one.
|
|
|
71
|
+ files = list_files_http(f"{url}/{path}?C=M;O=D")
|
|
|
72
|
+
|
|
|
73
|
+ if not files:
|
|
|
74
|
+ return None
|
|
|
75
|
+
|
|
|
76
|
+ def filter_files(files, keyword):
|
|
|
77
|
+ return [file for file in files if keyword in file]
|
|
|
78
|
+
|
|
|
79
|
+ artifact_files = filter_files(files, artifact)
|
|
|
80
|
+
|
|
|
81
|
+ if len(artifact_files) == 1:
|
|
|
82
|
+ return f"{url}/{path}/{artifact_files[0]}"
|
|
|
83
|
+
|
|
|
84
|
+ files_per_os = filter_files(artifact_files, target.tor_browser_build_alias)
|
|
|
85
|
+
|
|
|
86
|
+ # If there are files in the folder, but they don't have the OS in the name
|
|
|
87
|
+ # it means we can get any of them because they can be used to build for any OS.
|
|
|
88
|
+ # So let's just get the first one.
|
|
|
89
|
+ if len(files_per_os) == 0:
|
|
|
90
|
+ return f"{url}/{artifact}/{artifact_files[0]}"
|
|
|
91
|
+
|
|
|
92
|
+ elif len(files_per_os) == 1:
|
|
|
93
|
+ return f"{url}/{artifact}/{files_per_os[0]}"
|
|
|
94
|
+
|
|
|
95
|
+ matches = filter_files(files_per_os, target.cpu)
|
|
|
96
|
+
|
|
|
97
|
+ return f"{url}/{artifact}/{matches[0]}" if matches else None |
|