Commits:
-
b0b67b81
by Pier Angelo Vendrame at 2025-12-10T21:38:25+01:00
Bug 41627: Build OpenSSL in the Python package.
Build OpenSSL with Python, so that the ssl module can be imported.
We do not use the openssl project to avoid re-building a consistent
part of the Linux toolchain every time we bump it.
Also, bump the Python version since we are at it (we downgraded to be
able to build it with OpenSSL 1.1.x, now we don't have that requirement
anymore), and bump ninja, as the old version didn't build with our new
version of Python.
-
75825497
by Pier Angelo Vendrame at 2025-12-10T21:39:51+01:00
Bug 41662: Add python-zstandard to desktop containers.
They are needed when we build artifacts.
12 changed files:
Changes:
projects/common/list_toolchain_updates-common-firefox-geckoview
| ... |
... |
@@ -159,7 +159,7 @@ if (m/^\\s*MINIMUM_MINOR_VERSION = ([0-9]+)/) { |
|
159
|
159
|
}
|
|
160
|
160
|
EOF
|
|
161
|
161
|
needed=3.$(cat python/mozboot/bin/bootstrap.py | perl -ne "$p")
|
|
162
|
|
-current="3.9" # 3.11.x on Debian bookworm, 3.9.20 on our python project
|
|
|
162
|
+current="3.9" # 3.11.x on Debian bookworm, 3.13.11 on our python project
|
|
163
|
163
|
check_update_needed python "$needed" "$current"
|
|
164
|
164
|
|
|
165
|
165
|
|
projects/firefox/build
| ... |
... |
@@ -34,6 +34,8 @@ export PATH="/var/tmp/dist/rust/bin:/var/tmp/dist/cbindgen:/var/tmp/dist/node/bi |
|
34
|
34
|
tar -C /var/tmp/dist -xf [% c('input_files_by_name/clang') %]
|
|
35
|
35
|
tar -C /var/tmp/dist -xf [% c('input_files_by_name/python') %]
|
|
36
|
36
|
export PATH="/var/tmp/dist/python/bin:$PATH"
|
|
|
37
|
+ # For OpenSSL, see Python's README.md.
|
|
|
38
|
+ export LD_LIBRARY_PATH=/var/tmp/dist/python/lib:$LD_LIBRARY_PATH
|
|
37
|
39
|
[% IF ! c("var/linux-cross") -%]
|
|
38
|
40
|
tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/binutils') %]
|
|
39
|
41
|
export PATH="/var/tmp/dist/binutils/bin:$PATH"
|
| ... |
... |
@@ -48,6 +50,9 @@ export PATH="/var/tmp/dist/rust/bin:/var/tmp/dist/cbindgen:/var/tmp/dist/node/bi |
|
48
|
50
|
# down properly in that case. Thus, we set it here in the build script.
|
|
49
|
51
|
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:/usr/lib/[% c("var/crosstarget") %]/pkgconfig"
|
|
50
|
52
|
[% END -%]
|
|
|
53
|
+ [% IF c("var/dev_artifacts") -%]
|
|
|
54
|
+ python3 -m pip install $rootdir/[% c('input_files_by_name/python-zstandard') %]/*.whl
|
|
|
55
|
+ [% END -%]
|
|
51
|
56
|
[% END -%]
|
|
52
|
57
|
|
|
53
|
58
|
[% IF c("var/macos") && c("var/dev_artifacts") %]
|
projects/firefox/config
| ... |
... |
@@ -154,6 +154,7 @@ targets: |
|
154
|
154
|
arch_deps:
|
|
155
|
155
|
- python3
|
|
156
|
156
|
- python3-distutils
|
|
|
157
|
+ - python3-zstandard
|
|
157
|
158
|
- rsync
|
|
158
|
159
|
|
|
159
|
160
|
windows:
|
| ... |
... |
@@ -161,6 +162,7 @@ targets: |
|
161
|
162
|
arch_deps:
|
|
162
|
163
|
- python3
|
|
163
|
164
|
- python3-distutils
|
|
|
165
|
+ - python3-zstandard
|
|
164
|
166
|
- wine
|
|
165
|
167
|
|
|
166
|
168
|
input_files:
|
| ... |
... |
@@ -228,6 +230,9 @@ input_files: |
|
228
|
230
|
enable: '[% c("var/mullvad-browser") && c("var/has_l10n") %]'
|
|
229
|
231
|
- filename: marsigner.der
|
|
230
|
232
|
enable: '[% c("var/override_updater_url") %]'
|
|
|
233
|
+ - project: python-zstandard
|
|
|
234
|
+ enable: '[% c("var/linux") && c("var/dev_artifacts") %]'
|
|
|
235
|
+ name: python-zstandard
|
|
231
|
236
|
- filename: dmg-root
|
|
232
|
237
|
enable: '[% c("var/macos") && c("var/dev_artifacts") %]'
|
|
233
|
238
|
- project: hfsplus-tools
|
projects/ninja/build
| ... |
... |
@@ -12,8 +12,6 @@ mkdir -p /var/tmp/build |
|
12
|
12
|
tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
|
|
13
|
13
|
cd /var/tmp/build/[% project %]-[% c('version') %]
|
|
14
|
14
|
|
|
15
|
|
-patch -p1 < "$rootdir/python3.patch"
|
|
16
|
|
-
|
|
17
|
15
|
./configure.py --bootstrap
|
|
18
|
16
|
|
|
19
|
17
|
mkdir -p $distdir
|
projects/ninja/config
|
1
|
1
|
# vim: filetype=yaml sw=2
|
|
2
|
|
-version: '[% c("abbrev") %]'
|
|
|
2
|
+version: '1.13.2'
|
|
3
|
3
|
git_url: https://github.com/ninja-build/ninja.git
|
|
4
|
|
-git_hash: a524bf3f6bacd1b4ad85d719eed2737d8562f27a #v1.11.1
|
|
|
4
|
+git_hash: 3441b633c2fe2c494e958780ba0f4227b1327634 # v1.13.2
|
|
5
|
5
|
filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.[% c("compress_tar") %]'
|
|
6
|
6
|
container:
|
|
7
|
7
|
use_container: 1
|
| ... |
... |
@@ -16,4 +16,3 @@ input_files: |
|
16
|
16
|
- name: python
|
|
17
|
17
|
project: python
|
|
18
|
18
|
enable: '[% c("var/linux") %]' |
|
19
|
|
- - filename: python3.patch |
projects/ninja/python3.patch
deleted
|
1
|
|
-From 6a17e84370064eec6f22cfb1717ab80cf898d82b Mon Sep 17 00:00:00 2001
|
|
2
|
|
-From: Nico Weber <thakis@xxxxxxxxxxxx>
|
|
3
|
|
-Date: Tue, 10 Jan 2023 10:46:45 -0500
|
|
4
|
|
-Subject: [PATCH] Use python3 in all run lines
|
|
5
|
|
-
|
|
6
|
|
-We already did this in some, this converts the rest.
|
|
7
|
|
-Also chmod +x on write_fake_manifests.py while here.
|
|
8
|
|
----
|
|
9
|
|
- configure.py | 4 +---
|
|
10
|
|
- misc/measure.py | 4 +---
|
|
11
|
|
- misc/ninja_syntax_test.py | 2 +-
|
|
12
|
|
- misc/write_fake_manifests.py | 2 +-
|
|
13
|
|
- src/browse.py | 4 +---
|
|
14
|
|
- 5 files changed, 5 insertions(+), 11 deletions(-)
|
|
15
|
|
- mode change 100644 => 100755 misc/write_fake_manifests.py
|
|
16
|
|
-
|
|
17
|
|
-diff --git a/configure.py b/configure.py
|
|
18
|
|
-index 09c5b283e0..588250aa8a 100755
|
|
19
|
|
---- a/configure.py
|
|
20
|
|
-+++ b/configure.py
|
|
21
|
|
-@@ -1,4 +1,4 @@
|
|
22
|
|
--#!/usr/bin/env python
|
|
23
|
|
-+#!/usr/bin/env python3
|
|
24
|
|
- #
|
|
25
|
|
- # Copyright 2001 Google Inc. All Rights Reserved.
|
|
26
|
|
- #
|
|
27
|
|
-@@ -19,8 +19,6 @@
|
|
28
|
|
- Projects that use ninja themselves should either write a similar script
|
|
29
|
|
- or use a meta-build system that supports Ninja output."""
|
|
30
|
|
-
|
|
31
|
|
--from __future__ import print_function
|
|
32
|
|
--
|
|
33
|
|
- from optparse import OptionParser
|
|
34
|
|
- import os
|
|
35
|
|
- import pipes
|
|
36
|
|
-diff --git a/misc/measure.py b/misc/measure.py
|
|
37
|
|
-index 8ce95e696b..f3825efbb0 100755
|
|
38
|
|
---- a/misc/measure.py
|
|
39
|
|
-+++ b/misc/measure.py
|
|
40
|
|
-@@ -1,4 +1,4 @@
|
|
41
|
|
--#!/usr/bin/env python
|
|
42
|
|
-+#!/usr/bin/env python3
|
|
43
|
|
-
|
|
44
|
|
- # Copyright 2011 Google Inc. All Rights Reserved.
|
|
45
|
|
- #
|
|
46
|
|
-@@ -17,8 +17,6 @@
|
|
47
|
|
- """measure the runtime of a command by repeatedly running it.
|
|
48
|
|
- """
|
|
49
|
|
-
|
|
50
|
|
--from __future__ import print_function
|
|
51
|
|
--
|
|
52
|
|
- import time
|
|
53
|
|
- import subprocess
|
|
54
|
|
- import sys
|
|
55
|
|
-diff --git a/misc/ninja_syntax_test.py b/misc/ninja_syntax_test.py
|
|
56
|
|
-index 90ff9c6bdb..61fb177d43 100755
|
|
57
|
|
---- a/misc/ninja_syntax_test.py
|
|
58
|
|
-+++ b/misc/ninja_syntax_test.py
|
|
59
|
|
-@@ -1,4 +1,4 @@
|
|
60
|
|
--#!/usr/bin/env python
|
|
61
|
|
-+#!/usr/bin/env python3
|
|
62
|
|
-
|
|
63
|
|
- # Copyright 2011 Google Inc. All Rights Reserved.
|
|
64
|
|
- #
|
|
65
|
|
-diff --git a/misc/write_fake_manifests.py b/misc/write_fake_manifests.py
|
|
66
|
|
-old mode 100644
|
|
67
|
|
-new mode 100755
|
|
68
|
|
-index abcb677e18..bf9cf7de92
|
|
69
|
|
---- a/misc/write_fake_manifests.py
|
|
70
|
|
-+++ b/misc/write_fake_manifests.py
|
|
71
|
|
-@@ -1,4 +1,4 @@
|
|
72
|
|
--#!/usr/bin/env python
|
|
73
|
|
-+#!/usr/bin/env python3
|
|
74
|
|
-
|
|
75
|
|
- """Writes large manifest files, for manifest parser performance testing.
|
|
76
|
|
-
|
|
77
|
|
-diff --git a/src/browse.py b/src/browse.py
|
|
78
|
|
-index 653cbe91f6..b125e805a9 100755
|
|
79
|
|
---- a/src/browse.py
|
|
80
|
|
-+++ b/src/browse.py
|
|
81
|
|
-@@ -1,4 +1,4 @@
|
|
82
|
|
--#!/usr/bin/env python
|
|
83
|
|
-+#!/usr/bin/env python3
|
|
84
|
|
- #
|
|
85
|
|
- # Copyright 2001 Google Inc. All Rights Reserved.
|
|
86
|
|
- #
|
|
87
|
|
-@@ -20,8 +20,6 @@
|
|
88
|
|
- it when needed.
|
|
89
|
|
- """
|
|
90
|
|
-
|
|
91
|
|
--from __future__ import print_function
|
|
92
|
|
--
|
|
93
|
|
- try:
|
|
94
|
|
- import http.server as httpserver
|
|
95
|
|
- import socketserver |
projects/python-zstandard/README.md
|
|
1
|
+Mozilla uses the python-zstandard module for various tasks in their CI, but
|
|
|
2
|
+they do not vendor it in Firefox's source tree.
|
|
|
3
|
+
|
|
|
4
|
+Since we build our own Python on Linux, we also need to build this module. |
projects/python-zstandard/build
|
|
1
|
+#!/bin/bash
|
|
|
2
|
+[% c("var/set_default_env") -%]
|
|
|
3
|
+outdir=[% dest_dir _ '/' _ c('filename') %]
|
|
|
4
|
+mkdir -p /var/tmp/dist $outdir
|
|
|
5
|
+
|
|
|
6
|
+tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/python') %]
|
|
|
7
|
+export PATH=/var/tmp/dist/python/bin:$PATH
|
|
|
8
|
+
|
|
|
9
|
+python3 -m pip install *.whl
|
|
|
10
|
+
|
|
|
11
|
+tar -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
|
|
|
12
|
+cd [% project %]-[% c('version') %]
|
|
|
13
|
+python3 setup.py bdist_wheel
|
|
|
14
|
+mv dist/*.whl $outdir/ |
projects/python-zstandard/config
|
|
1
|
+version: 0.24.0
|
|
|
2
|
+filename: 'python-zstandard-[% c("version") %]-[% c("var/build_id") %]'
|
|
|
3
|
+git_url: https://github.com/indygreg/python-zstandard.git
|
|
|
4
|
+git_hash: 9223924b1db8f36cf1c7c2dcd55232093890d145 # 0.24.0
|
|
|
5
|
+container:
|
|
|
6
|
+ use_container: 1
|
|
|
7
|
+
|
|
|
8
|
+input_files:
|
|
|
9
|
+ - project: container-image
|
|
|
10
|
+ - project: python
|
|
|
11
|
+ name: python
|
|
|
12
|
+ # This contains only binaries for Windows.
|
|
|
13
|
+ - name: setuptools
|
|
|
14
|
+ URL: https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl
|
|
|
15
|
+ sha256sum: 062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922
|
|
|
16
|
+ # This contains only Python.
|
|
|
17
|
+ - name: packaging
|
|
|
18
|
+ URL: https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl
|
|
|
19
|
+ sha256sum: 29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 |
projects/python/README.md
| ... |
... |
@@ -17,3 +17,17 @@ image. |
|
17
|
17
|
|
|
18
|
18
|
`browser` is a notable exception: we redefine `var/deps` for all platforms and
|
|
19
|
19
|
already add `python3` there.
|
|
|
20
|
+
|
|
|
21
|
+## OpenSSL
|
|
|
22
|
+
|
|
|
23
|
+Some Python module complain about the OpenSSL version of the container being
|
|
|
24
|
+too old. Therefore, we also build OpenSSL in this project, and other projects
|
|
|
25
|
+needing it must add `/var/tmp/dist/python/lib` to `LD_LIBRARY_PATH`.
|
|
|
26
|
+
|
|
|
27
|
+We do it here instead of using the `openssl` project because we do not want to
|
|
|
28
|
+rebuild a big part of the toolchain for each OpenSSL update (the module would
|
|
|
29
|
+be used mostly for HTTP requests, which will not go through in our builds,
|
|
|
30
|
+since they happen offline).
|
|
|
31
|
+
|
|
|
32
|
+When updating to a newever version of Debian for Linux containers, we might
|
|
|
33
|
+stop building OpenSSL and go back to using the system library. |
projects/python/build
| ... |
... |
@@ -6,14 +6,28 @@ |
|
6
|
6
|
[% END -%]
|
|
7
|
7
|
|
|
8
|
8
|
distdir=/var/tmp/dist/[% project %]
|
|
9
|
|
-mkdir -p $distdir
|
|
|
9
|
+openssldir=/var/tmp/dist/openssl
|
|
|
10
|
+mkdir -p $distdir $openssldir
|
|
|
11
|
+
|
|
|
12
|
+tar -xf $rootdir/[% c('input_files_by_name/openssl') %]
|
|
|
13
|
+pushd openssl-3.5.4
|
|
|
14
|
+./Configure --prefix=$openssldir --libdir=lib
|
|
|
15
|
+make -j[% c("num_procs") %]
|
|
|
16
|
+make -j[% c("num_procs") %] install
|
|
|
17
|
+# Python will try to load OpenSSL during its build process.
|
|
|
18
|
+export LD_LIBRARY_PATH=$openssldir/lib:$LD_LIBRARY_PATH
|
|
|
19
|
+popd
|
|
|
20
|
+
|
|
10
|
21
|
tar xf [% c('input_files_by_name/python') %]
|
|
11
|
22
|
cd Python-[% c('version') %]
|
|
12
|
|
-./configure --prefix=$distdir --enable-optimizations
|
|
|
23
|
+./configure --prefix=$distdir --enable-optimizations --with-openssl=$openssldir
|
|
13
|
24
|
make -j[% c("num_procs") %]
|
|
14
|
25
|
make prefix=$distdir install
|
|
|
26
|
+
|
|
|
27
|
+cp -a $openssldir/lib/lib*.so* $distdir/lib/
|
|
|
28
|
+
|
|
15
|
29
|
cd /var/tmp/dist
|
|
16
|
30
|
[% c('tar', {
|
|
17
|
|
- tar_src => 'python',
|
|
18
|
|
- tar_args => '-caf ' _ dest_dir _ '/' _ c('filename'),
|
|
19
|
|
- }) %] |
|
|
31
|
+ tar_src => 'python',
|
|
|
32
|
+ tar_args => '-caf ' _ dest_dir _ '/' _ c('filename'),
|
|
|
33
|
+ }) %] |
projects/python/config
|
1
|
1
|
# vim: filetype=yaml sw=2
|
|
2
|
|
-version: 3.9.20
|
|
3
|
|
-filename: 'python-[% c("var/build_id") %].tar.[% c("compress_tar") %]'
|
|
|
2
|
+version: 3.13.11
|
|
|
3
|
+filename: 'python-[% c("version") %]-[% c("var/build_id") %].tar.[% c("compress_tar") %]'
|
|
4
|
4
|
container:
|
|
5
|
5
|
use_container: 1
|
|
6
|
6
|
var:
|
| ... |
... |
@@ -12,7 +12,6 @@ var: |
|
12
|
12
|
- libffi-dev
|
|
13
|
13
|
- libncurses-dev
|
|
14
|
14
|
- libsqlite3-dev
|
|
15
|
|
- - libssl-dev
|
|
16
|
15
|
- zlib1g-dev
|
|
17
|
16
|
|
|
18
|
17
|
setup: |
|
| ... |
... |
@@ -24,7 +23,14 @@ input_files: |
|
24
|
23
|
- project: container-image
|
|
25
|
24
|
- name: python
|
|
26
|
25
|
URL: 'https://www.python.org/ftp/python/[% c("version") %]/Python-[% c("version") %].tar.xz'
|
|
27
|
|
- sha256sum: 6b281279efd85294d2d6993e173983a57464c0133956fbbb5536ec9646beaf0c
|
|
|
26
|
+ sha256sum: 16ede7bb7cdbfa895d11b0642fa0e523f291e6487194d53cf6d3b338c3a17ea2
|
|
28
|
27
|
- name: '[% c("var/compiler") %]'
|
|
29
|
28
|
project: '[% c("var/compiler") %]'
|
|
30
|
29
|
enable: '[% c("var/linux") %]'
|
|
|
30
|
+ # We do not use the same we use as tor dependency because we do not want to
|
|
|
31
|
+ # rebuild Clang, Rust and other projects at each OpenSSL update.
|
|
|
32
|
+ # FWIW, these OpenSSL would be used for requests that will not go through,
|
|
|
33
|
+ # as our builds run offline.
|
|
|
34
|
+ - name: openssl
|
|
|
35
|
+ URL: 'https://github.com/openssl/openssl/releases/download/openssl-3.5.4/openssl-3.5.4.tar.gz'
|
|
|
36
|
+ sha256sum: 967311f84955316969bdb1d8d4b983718ef42338639c621ec4c34fddef355e99 |
|