[tor-commits] [Git][tpo/applications/tor-browser][base-browser-128.1.0esr-14.0-1] fixup! Add CI for Base Browser

["Pier Angelo Vendrame \(@pierov\) via tor-commits" <tor-commits@xxxxxxxxxxxxxxxxxxxx>]

Title: GitLab

Pier Angelo Vendrame pushed to branch base-browser-128.1.0esr-14.0-1 at The Tor Project / Applications / Tor Browser

Commits:

• 8f26cea2
  by Beatriz Rizental at 2024-08-12T16:32:16+02:00

  fixup! Add CI for Base Browser
  
  Bug 42997: Reduce the number of git fetches by fetching during a setup
  job and passing that initial fetch around as a git bundle to other jobs
  through the Gitlab artifacts. Further reduce the number of fetches by
  creating a list of changed files in the setup job and passing that
  around as well as an artifact.
  

4 changed files:

• .gitlab-ci.yml
• .gitlab/ci/lint.yml
• .gitlab/ci/scripts/run_linters.py → .gitlab/ci/scripts/helpers.py
• + .gitlab/ci/setup.yml

Changes:

.gitlab-ci.yml

---

1	1	stages:
	2	+ - setup
2	3	- lint
3	4
4	5	variables:
5	6	IMAGE_PATH: containers.torproject.org/tpo/applications/tor-browser/base:latest
6	7
7	8	include:
	9	+ - local: '.gitlab/ci/setup.yml'
8	10	- local: '.gitlab/ci/lint.yml'

.gitlab/ci/lint.yml

---

1	1	.base:
2	2	stage: lint
	3	+ image: $IMAGE_PATH
3	4	interruptible: true
	5	+ needs:
	6	+ - job: setup-env
	7	+ artifacts: true
	8	+ - job: create-bundle
	9	+ artifacts: true
4	10	variables:
5	11	MOZBUILD_STATE_PATH: "$CI_PROJECT_DIR/.cache/mozbuild"
	12	+ GIT_STRATEGY: "none"
6	13	cache:
7	14	paths:
8	15	- node_modules
...	...	@@ -11,12 +18,26 @@
11	18	when: 'always'
12	19	# Share the cache throughout all pipelines running for a given branch
13	20	key: $CI_COMMIT_REF_SLUG
	21	+ before_script:
	22	+ # DEBUG: Are all artifacts here?
	23	+ - ls -a
	24	+ - mkdir app && cd app
	25	+ # Initialize a fresh git repo
	26	+ - git init
	27	+ # Add app.bundle as the remote. All operations that communicate with the remote will be local.
	28	+ - git remote add origin ../app.bundle
	29	+ # shallow.txt contains the SHA of the base commit of the bundle.
	30	+ # The bundle is shallow, therefore it's base commit will not have a parent.
	31	+ # Adding the SHA of the base commit to .git/shallow tells git that it doesn't need
	32	+ # to crash when it realizes said base commit doesn't have a parent.
	33	+ - cp ../shallow.txt .git/shallow
	34	+ # Finally, unpack the bundle. Time it for debugging purposes.
	35	+ - time git pull origin $BRANCH_NAME
14	36
15	37	eslint:
16	38	extends: .base
17		- image: $IMAGE_PATH
18	39	script:
19		- - .gitlab/ci/scripts/run_linters.py eslint
	40	+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l eslint
20	41	rules:
21	42	- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
22	43	changes:
...	...	@@ -44,9 +65,8 @@ eslint:
44	65
45	66	stylelint:
46	67	extends: .base
47		- image: $IMAGE_PATH
48	68	script:
49		- - .gitlab/ci/scripts/run_linters.py stylelint
	69	+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l stylelint
50	70	rules:
51	71	- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
52	72	changes:
...	...	@@ -63,9 +83,8 @@ stylelint:
63	83
64	84	py-black:
65	85	extends: .base
66		- image: $IMAGE_PATH
67	86	script:
68		- - .gitlab/ci/scripts/run_linters.py black
	87	+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l black
69	88	rules:
70	89	- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
71	90	changes:
...	...	@@ -83,9 +102,8 @@ py-black:
83	102
84	103	py-ruff:
85	104	extends: .base
86		- image: $IMAGE_PATH
87	105	script:
88		- - .gitlab/ci/scripts/run_linters.py ruff
	106	+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l ruff
89	107	rules:
90	108	- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
91	109	changes:
...	...	@@ -103,9 +121,8 @@ py-ruff:
103	121
104	122	yaml:
105	123	extends: .base
106		- image: $IMAGE_PATH
107	124	script:
108		- - .gitlab/ci/scripts/run_linters.py yaml
	125	+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l yaml
109	126	rules:
110	127	- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
111	128	changes:
...	...	@@ -119,9 +136,8 @@ yaml:
119	136
120	137	shellcheck:
121	138	extends: .base
122		- image: $IMAGE_PATH
123	139	script:
124		- - .gitlab/ci/scripts/run_linters.py shellcheck
	140	+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l shellcheck
125	141	rules:
126	142	- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
127	143	changes:
...	...	@@ -134,10 +150,9 @@ shellcheck:
134	150
135	151	clang-format:
136	152	extends: .base
137		- image: $IMAGE_PATH
138	153	script:
139	154	- ./mach configure --without-wasm-sandboxed-libraries --with-base-browser-version=0.0.0
140		- - .gitlab/ci/scripts/run_linters.py clang-format
	155	+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l clang-format
141	156	rules:
142	157	- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
143	158	changes:
...	...	@@ -155,9 +170,8 @@ clang-format:
155	170
156	171	rustfmt:
157	172	extends: .base
158		- image: $IMAGE_PATH
159	173	script:
160		- - .gitlab/ci/scripts/run_linters.py rustfmt
	174	+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l rustfmt
161	175	rules:
162	176	- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
163	177	changes:
...	...	@@ -170,9 +184,8 @@ rustfmt:
170	184
171	185	fluent-lint:
172	186	extends: .base
173		- image: $IMAGE_PATH
174	187	script:
175		- - .gitlab/ci/scripts/run_linters.py fluent-lint
	188	+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l fluent-lint
176	189	rules:
177	190	- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
178	191	changes:
...	...	@@ -186,9 +199,8 @@ fluent-lint:
186	199
187	200	localization:
188	201	extends: .base
189		- image: $IMAGE_PATH
190	202	script:
191		- - .gitlab/ci/scripts/run_linters.py l10n
	203	+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l l10n
192	204	rules:
193	205	- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
194	206	changes:
...	...	@@ -204,9 +216,8 @@ localization:
204	216
205	217	mingw-capitalization:
206	218	extends: .base
207		- image: $IMAGE_PATH
208	219	script:
209		- - .gitlab/ci/scripts/run_linters.py mingw-capitalization
	220	+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l mingw-capitalization
210	221	rules:
211	222	- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
212	223	changes:
...	...	@@ -222,9 +233,8 @@ mingw-capitalization:
222	233
223	234	mscom-init:
224	235	extends: .base
225		- image: $IMAGE_PATH
226	236	script:
227		- - .gitlab/ci/scripts/run_linters.py mscom-init
	237	+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l mscom-init
228	238	rules:
229	239	- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
230	240	changes:
...	...	@@ -240,9 +250,8 @@ mscom-init:
240	250
241	251	file-whitespace:
242	252	extends: .base
243		- image: $IMAGE_PATH
244	253	script:
245		- - .gitlab/ci/scripts/run_linters.py file-whitespace
	254	+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l file-whitespace
246	255	rules:
247	256	- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
248	257	changes:
...	...	@@ -270,9 +279,8 @@ file-whitespace:
270	279
271	280	test-manifest:
272	281	extends: .base
273		- image: $IMAGE_PATH
274	282	script:
275		- - .gitlab/ci/scripts/run_linters.py test-manifest-alpha test-manifest-disable test-manifest-skip-if
	283	+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l test-manifest-alpha -l test-manifest-disable -l test-manifest-skip-if
276	284	rules:
277	285	- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
278	286	changes:
...	...	@@ -286,9 +294,8 @@ test-manifest:
286	294
287	295	trojan-source:
288	296	extends: .base
289		- image: $IMAGE_PATH
290	297	script:
291		- - .gitlab/ci/scripts/run_linters.py trojan-source
	298	+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l trojan-source
292	299	rules:
293	300	- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
294	301	changes:

.gitlab/ci/scripts/run_linters.py → .gitlab/ci/scripts/helpers.py

---

...	...	@@ -5,7 +5,6 @@ import os
5	5	import re
6	6	import shlex
7	7	import subprocess
8		-import sys
9	8
10	9
11	10	def git(command):
...	...	@@ -15,8 +14,8 @@ def git(command):
15	14	return result.stdout.strip()
16	15
17	16
18		-def get_firefox_tag_from_branch_name(branch_name):
19		- """Extracts the Firefox tag associated with a branch name.
	17	+def get_firefox_tag(reference):
	18	+ """Extracts the Firefox tag associated with a branch or tag name.
20	19
21	20	The "firefox tag" is the tag that marks
22	21	the end of the Mozilla commits and the start of the Tor Project commits.
...	...	@@ -25,26 +24,26 @@ def get_firefox_tag_from_branch_name(branch_name):
25	24	this function may return the incorrect reference number.
26	25
27	26	Args:
28		- branch_name: The branch name to extract the tag from.
	27	+ reference: The branch or tag name to extract the Firefox tag from.
29	28	Expected format is tor-browser-91.2.0esr-11.0-1,
30	29	where 91.2.0esr is the Firefox version.
31	30
32	31	Returns:
33	32	The reference specifier of the matching Firefox tag.
34		- An exception wil be raised if anything goes wrong.
	33	+ An exception will be raised if anything goes wrong.
35	34	"""
36	35
37		- # Extracts the version number from a branch name.
	36	+ # Extracts the version number from a branch or tag name.
38	37	firefox_version = ""
39		- match = re.search(r"(?<=browser-)([^-]+)", branch_name)
	38	+ match = re.search(r"(?<=browser-)([^-]+)", reference)
40	39	if match:
41	40	# TODO: Validate that what we got is actually a valid semver string?
42	41	firefox_version = match.group(1)
43	42	else:
44		- raise ValueError(f"Failed to extract version from branch name '{branch_name}'.")
	43	+ raise ValueError(f"Failed to extract version from reference '{reference}'.")
45	44
46	45	tag = f"FIREFOX_{firefox_version.replace('.', '_')}_"
47		- remote_tags = git("ls-remote --tags")
	46	+ remote_tags = git("ls-remote --tags origin")
48	47
49	48	# Each line looks like:
50	49	# 9edd658bfd03a6b4743ecb75fd4a9ad968603715 refs/tags/FIREFOX_91_9_0esr_BUILD1
...	...	@@ -54,7 +53,7 @@ def get_firefox_tag_from_branch_name(branch_name):
54	53	return match.group(0).split()[0]
55	54	else:
56	55	raise ValueError(
57		- f"Failed to find reference specifier for Firefox tag '{tag}' in branch '{branch_name}'."
	56	+ f"Failed to find reference specifier for Firefox tag '{tag}' from '{reference}'."
58	57	)
59	58
60	59
...	...	@@ -74,37 +73,42 @@ def get_list_of_changed_files():
74	73	base_reference = ""
75	74
76	75	if os.getenv("CI_PIPELINE_SOURCE") == "merge_request_event":
77		- # For merge requests, the base_reference is the common ancestor between the MR and the target branch.
	76	+ # For merge requests, the base_reference is the common ancestor between the MR and the target branch
78	77	base_reference = os.getenv("CI_MERGE_REQUEST_DIFF_BASE_SHA")
79	78	else:
80	79	# When not in merge requests, the base reference is the Firefox tag
81		- base_reference = get_firefox_tag_from_branch_name(os.getenv("CI_COMMIT_BRANCH"))
	80	+ base_reference = get_firefox_tag(os.getenv("CI_COMMIT_BRANCH"))
82	81
83	82	if not base_reference:
84	83	raise RuntimeError("No base reference found. There might be more errors above.")
85	84
86	85	# Fetch the tag reference
87	86	git(f"fetch origin {base_reference} --depth=1 --filter=blob:none")
88		- # Return the list of changed files
89		- return git(f"diff --diff-filter=d --name-only {base_reference} HEAD").split("\n")
	87	+ # Return but filter the issue_templates files because those file names have spaces which can cause issues
	88	+ return git("diff --diff-filter=d --name-only FETCH_HEAD HEAD").split("\n")
90	89
91	90
92	91	if __name__ == "__main__":
93		- parser = argparse.ArgumentParser(
94		- description="Run ./mach linters in CI. Warning: if you run this in your local environment it might mess up your git history."
	92	+ parser = argparse.ArgumentParser(description="")
	93	+
	94	+ parser.add_argument(
	95	+ "--get-firefox-tag",
	96	+ help="Get the Firefox tag related to a given (tor-mullvad-base)-browser tag or branch name.",
	97	+ type=str,
95	98	)
96	99	parser.add_argument(
97		- "linters", metavar="L", type=str, nargs="+", help="A list of linters to run."
	100	+ "--get-changed-files",
	101	+ help="Get list of changed files."
	102	+ "When running from a merge request get sthe list of changed files since the merge-base of the current branch."
	103	+ "When running from a protected branch i.e. any branch that starts with <something>-browser-, gets the list of files changed since the FIREFOX_ tag.",
	104	+ action="store_true",
98	105	)
	106	+
99	107	args = parser.parse_args()
100	108
101		- changed_files = get_list_of_changed_files()
102		- if changed_files:
103		- command = ["./mach", "lint", "-v"]
104		- for linter in args.linters:
105		- command.extend(["-l", linter])
106		- command.extend(changed_files)
107		- result = subprocess.run(command, text=True)
108		- sys.exit(result.returncode)
	109	+ if args.get_firefox_tag:
	110	+ print(get_firefox_tag(args.get_firefox_tag))
	111	+ elif args.get_changed_files:
	112	+ print("\n".join(get_list_of_changed_files()))
109	113	else:
110		- print("No files changed, skipping linting.")
	114	+ print("No valid option provided.")

.gitlab/ci/setup.yml

---

	1	+setup-env:
	2	+ stage: setup
	3	+ interruptible: true
	4	+ variables:
	5	+ GIT_STRATEGY: "none"
	6	+ rules:
	7	+ - if: $CI_PIPELINE_SOURCE == 'merge_request_event' || $CI_COMMIT_REF_PROTECTED == 'true'
	8	+ script:
	9	+ - |
	10	+ if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ]; then
	11	+ echo "BRANCH_NAME=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" > build.env
	12	+ else
	13	+ echo "BRANCH_NAME=$CI_COMMIT_REF_NAME" > build.env
	14	+ fi
	15	+ artifacts:
	16	+ reports:
	17	+ dotenv:
	18	+ - build.env
	19	+
	20	+create-bundle:
	21	+ stage: setup
	22	+ # TODO: Find a better suited image, this one just has git.
	23	+ image: python
	24	+ needs:
	25	+ - job: setup-env
	26	+ artifacts: true
	27	+ rules:
	28	+ - if: $CI_PIPELINE_SOURCE == 'merge_request_event' || $CI_COMMIT_REF_PROTECTED == 'true'
	29	+ variables:
	30	+ GIT_DEPTH: "1"
	31	+ interruptible: true
	32	+ script:
	33	+ # DEBUG: Check repository status.
	34	+ - git status
	35	+ # Switch to a named branch.
	36	+ - git switch -c $BRANCH_NAME
	37	+ # Create a git bundle -- this will generate the app.bundle file,
	38	+ # which can be used as a git remote for offline fetching.
	39	+ - git bundle create app.bundle --all
	40	+ # Retain the SHA of the base of this shallow repository.
	41	+ - cat .git/shallow > shallow.txt
	42	+ # DEBUG: Check sizes.
	43	+ - du -sh .git
	44	+ - du -sh app.bundle
	45	+ # Since this is the only job we have access to the Gitlab remote,
	46	+ # let's get a list of changed files to use in the next jobs.
	47	+ - .gitlab/ci/scripts/helpers.py --get-changed-files > changedfiles.txt
	48	+ artifacts:
	49	+ paths:
	50	+ - app.bundle
	51	+ - shallow.txt
	52	+ - changedfiles.txt
	53	+ expire_in: 1 hour

—
View it on GitLab.
You're receiving this email because of your account on gitlab.torproject.org. Manage all notifications · Help

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits