[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [Git][tpo/applications/tor-browser][base-browser-115.13.0esr-13.5-1] Add CI for Base Browser

Title: GitLab

Pier Angelo Vendrame pushed to branch base-browser-115.13.0esr-13.5-1 at The Tor Project / Applications / Tor Browser

Commits:

  • 7b915bf9
    by Beatriz Rizental at 2024-07-11T11:20:16+02:00 
    Add CI for Base Browser

3 changed files:

Changes:

  • .gitlab-ci.yml
    1 
    +stages:
    2 
    +  - lint
    3 
    +
    4 
    +include:
    5 
    +  - local: '.gitlab/ci/lint.yml'

  • .gitlab/ci/lint.yml
    1 
    +variables:
    2 
    +  # This needs to be kept in sync with the max Python version accepted by ./mach
    3 
    +  PYTHON_VERSION: "3.11.7"
    4 
    +
    5 
    +.base:
    6 
    +  stage: lint
    7 
    +  interruptible: true
    8 
    +  variables:
    9 
    +    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
    10 
    +  cache:
    11 
    +    paths:
    12 
    +      - node_modules
    13 
    +      - .cache/pip
    14 
    +
    15 
    +eslint:
    16 
    +  extends: .base
    17 
    +  image: cimg/python:$PYTHON_VERSION-node
    18 
    +  script:
    19 
    +    - .gitlab/ci/scripts/run_linters.py eslint
    20 
    +  rules:
    21 
    +    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    22 
    +      changes:
    23 
    +        # List copied from: taskcluster/ci/source-test/mozlint.yml
    24 
    +        #
    25 
    +        # Files that are likely audited.
    26 
    +        - '**/*.js'
    27 
    +        - '**/*.jsm'
    28 
    +        - '**/*.json'
    29 
    +        - '**/*.jsx'
    30 
    +        - '**/*.mjs'
    31 
    +        - '**/*.sjs'
    32 
    +        - '**/*.html'
    33 
    +        - '**/*.xhtml'
    34 
    +        - '**/*.xml'
    35 
    +        - 'tools/lint/eslint.yml'
    36 
    +        # Run when eslint policies change.
    37 
    +        - '**/.eslintignore'
    38 
    +        - '**/*eslintrc*'
    39 
    +        # The plugin implementing custom checks.
    40 
    +        - 'tools/lint/eslint/eslint-plugin-mozilla/**'
    41 
    +        - 'tools/lint/eslint/eslint-plugin-spidermonkey-js/**'
    42 
    +    # Run job whenever a new tag is created
    43 
    +    # or whenever a commit is merged to a protected branch
    44 
    +    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
    45 
    +
    46 
    +stylelint:
    47 
    +  extends: .base
    48 
    +  image: cimg/python:$PYTHON_VERSION-node
    49 
    +  script:
    50 
    +    - .gitlab/ci/scripts/run_linters.py stylelint
    51 
    +  rules:
    52 
    +    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    53 
    +      changes:
    54 
    +        # List copied from: taskcluster/ci/source-test/mozlint.yml
    55 
    +        #
    56 
    +        # Files that are likely audited.
    57 
    +        - '**/*.css'
    58 
    +        - 'tools/lint/styleint.yml'
    59 
    +        # Run when stylelint policies change.
    60 
    +        - '**/.stylelintignore'
    61 
    +        - '**/*stylelintrc*'
    62 
    +    # Run job whenever a new tag is created
    63 
    +    # or whenever a commit is merged to a protected branch
    64 
    +    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
    65 
    +
    66 
    +py-black:
    67 
    +  extends: .base
    68 
    +  image: cimg/python:$PYTHON_VERSION
    69 
    +  script:
    70 
    +    - .gitlab/ci/scripts/run_linters.py black
    71 
    +  rules:
    72 
    +    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    73 
    +      changes:
    74 
    +        # List copied from: taskcluster/ci/source-test/mozlint.yml
    75 
    +        #
    76 
    +        # The list of extensions should match tools/lint/black.yml
    77 
    +        - '**/*.py'
    78 
    +        - '**/moz.build'
    79 
    +        - '**/*.configure'
    80 
    +        - '**/*.mozbuild'
    81 
    +        - 'pyproject.toml'
    82 
    +        - 'tools/lint/black.yml'
    83 
    +    # Run job whenever a new tag is created
    84 
    +    # or whenever a commit is merged to a protected branch
    85 
    +    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
    86 
    +
    87 
    +py-ruff:
    88 
    +  extends: .base
    89 
    +  image: cimg/python:$PYTHON_VERSION
    90 
    +  script:
    91 
    +    - .gitlab/ci/scripts/run_linters.py ruff
    92 
    +  rules:
    93 
    +    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    94 
    +      changes:
    95 
    +        # List copied from: taskcluster/ci/source-test/mozlint.yml
    96 
    +        #
    97 
    +        - '**/*.py'
    98 
    +        - '**/*.configure'
    99 
    +        - '**/.ruff.toml'
    100 
    +        - 'pyproject.toml'
    101 
    +        - 'tools/lint/ruff.yml'
    102 
    +        - 'tools/lint/python/ruff.py'
    103 
    +        - 'tools/lint/python/ruff_requirements.txt'
    104 
    +    # Run job whenever a new tag is created
    105 
    +    # or whenever a commit is merged to a protected branch
    106 
    +    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
    107 
    +
    108 
    +yaml:
    109 
    +  extends: .base
    110 
    +  image: cimg/python:$PYTHON_VERSION
    111 
    +  script:
    112 
    +    - .gitlab/ci/scripts/run_linters.py yaml
    113 
    +  rules:
    114 
    +    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    115 
    +      changes:
    116 
    +        # List copied from: taskcluster/ci/source-test/mozlint.yml
    117 
    +        #
    118 
    +        - '**/*.yml'
    119 
    +        - '**/*.yaml'
    120 
    +        - '**/.ymllint'
    121 
    +    # Run job whenever a new tag is created
    122 
    +    # or whenever a commit is merged to a protected branch
    123 
    +    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
    124 
    +
    125 
    +shellcheck:
    126 
    +  extends: .base
    127 
    +  image: cimg/python:$PYTHON_VERSION
    128 
    +  script:
    129 
    +    - .gitlab/ci/scripts/run_linters.py shellcheck
    130 
    +  rules:
    131 
    +    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    132 
    +      changes:
    133 
    +        # List copied from: taskcluster/ci/source-test/mozlint.yml
    134 
    +        #
    135 
    +        - '**/*.sh'
    136 
    +        - 'tools/lint/shellcheck.yml'
    137 
    +    # Run job whenever a new tag is created
    138 
    +    # or whenever a commit is merged to a protected branch
    139 
    +    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
    140 
    +
    141 
    +clang-format:
    142 
    +  extends: .base
    143 
    +  image: cimg/python:$PYTHON_VERSION
    144 
    +  script:
    145 
    +    - .gitlab/ci/scripts/run_linters.py clang-format
    146 
    +  rules:
    147 
    +    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    148 
    +      changes:
    149 
    +        # List copied from: taskcluster/ci/source-test/mozlint.yml
    150 
    +        #
    151 
    +        - '**/*.cpp'
    152 
    +        - '**/*.c'
    153 
    +        - '**/*.cc'
    154 
    +        - '**/*.h'
    155 
    +        - '**/*.m'
    156 
    +        - '**/*.mm'
    157 
    +        - 'tools/lint/clang-format.yml'
    158 
    +    # Run job whenever a new tag is created
    159 
    +    # or whenever a commit is merged to a protected branch
    160 
    +    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
    161 
    +
    162 
    +rustfmt:
    163 
    +  extends: .base
    164 
    +  image: cimg/python:$PYTHON_VERSION
    165 
    +  script:
    166 
    +    - .gitlab/ci/scripts/run_linters.py rustfmt
    167 
    +  rules:
    168 
    +    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    169 
    +      changes:
    170 
    +        # List copied from: taskcluster/ci/source-test/mozlint.yml
    171 
    +        #
    172 
    +        - '**/*.rs'
    173 
    +        - 'tools/lint/rustfmt.yml'
    174 
    +    # Run job whenever a new tag is created
    175 
    +    # or whenever a commit is merged to a protected branch
    176 
    +    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
    177 
    +
    178 
    +fluent-lint:
    179 
    +  extends: .base
    180 
    +  image: cimg/python:$PYTHON_VERSION
    181 
    +  script:
    182 
    +    - .gitlab/ci/scripts/run_linters.py fluent-lint
    183 
    +  rules:
    184 
    +    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    185 
    +      changes:
    186 
    +        # List copied from: taskcluster/ci/source-test/mozlint.yml
    187 
    +        #
    188 
    +        - '**/*.ftl'
    189 
    +        - 'tools/lint/fluent-lint.yml'
    190 
    +        - 'tools/lint/fluent-lint/exclusions.yml'
    191 
    +    # Run job whenever a new tag is created
    192 
    +    # or whenever a commit is merged to a protected branch
    193 
    +    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
    194 
    +
    195 
    +localization:
    196 
    +  extends: .base
    197 
    +  image: cimg/python:$PYTHON_VERSION
    198 
    +  script:
    199 
    +    - .gitlab/ci/scripts/run_linters.py l10n
    200 
    +  rules:
    201 
    +    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    202 
    +      changes:
    203 
    +        # List copied from: taskcluster/ci/source-test/mozlint.yml
    204 
    +        #
    205 
    +        - '**/locales/en-US/**'
    206 
    +        - '**/l10n.toml'
    207 
    +        - 'third_party/python/compare-locales/**'
    208 
    +        - 'third_party/python/fluent/**'
    209 
    +        - 'tools/lint/l10n.yml'
    210 
    +    # Run job whenever a new tag is created
    211 
    +    # or whenever a commit is merged to a protected branch
    212 
    +    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
    213 
    +
    214 
    +mingw-capitalization:
    215 
    +  extends: .base
    216 
    +  image: cimg/python:$PYTHON_VERSION
    217 
    +  script:
    218 
    +    - .gitlab/ci/scripts/run_linters.py mingw-capitalization
    219 
    +  rules:
    220 
    +    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    221 
    +      changes:
    222 
    +        # List copied from: taskcluster/ci/source-test/mozlint.yml
    223 
    +        #
    224 
    +        - '**/*.cpp'
    225 
    +        - '**/*.cc'
    226 
    +        - '**/*.c'
    227 
    +        - '**/*.h'
    228 
    +        - 'tools/lint/mingw-capitalization.yml'
    229 
    +    # Run job whenever a new tag is created
    230 
    +    # or whenever a commit is merged to a protected branch
    231 
    +    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
    232 
    +
    233 
    +mscom-init:
    234 
    +  extends: .base
    235 
    +  image: cimg/python:$PYTHON_VERSION
    236 
    +  script:
    237 
    +    - .gitlab/ci/scripts/run_linters.py mscom-init
    238 
    +  rules:
    239 
    +    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    240 
    +      changes:
    241 
    +        # List copied from: taskcluster/ci/source-test/mozlint.yml
    242 
    +        #
    243 
    +        - '**/*.cpp'
    244 
    +        - '**/*.cc'
    245 
    +        - '**/*.c'
    246 
    +        - '**/*.h'
    247 
    +        - 'tools/lint/mscom-init.yml'
    248 
    +    # Run job whenever a new tag is created
    249 
    +    # or whenever a commit is merged to a protected branch
    250 
    +    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
    251 
    +
    252 
    +file-whitespace:
    253 
    +  extends: .base
    254 
    +  image: cimg/python:$PYTHON_VERSION
    255 
    +  script:
    256 
    +    - .gitlab/ci/scripts/run_linters.py file-whitespace
    257 
    +  rules:
    258 
    +    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    259 
    +      changes:
    260 
    +        # List copied from: taskcluster/ci/source-test/mozlint.yml
    261 
    +        #
    262 
    +        - '**/*.c'
    263 
    +        - '**/*.cc'
    264 
    +        - '**/*.cpp'
    265 
    +        - '**/*.css'
    266 
    +        - '**/*.dtd'
    267 
    +        - '**/*.idl'
    268 
    +        - '**/*.ftl'
    269 
    +        - '**/*.h'
    270 
    +        - '**/*.html'
    271 
    +        - '**/*.md'
    272 
    +        - '**/*.properties'
    273 
    +        - '**/*.py'
    274 
    +        - '**/*.rs'
    275 
    +        - '**/*.rst'
    276 
    +        - '**/*.webidl'
    277 
    +        - '**/*.xhtml'
    278 
    +        - 'tools/lint/file-whitespace.yml'
    279 
    +    # Run job whenever a new tag is created
    280 
    +    # or whenever a commit is merged to a protected branch
    281 
    +    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
    282 
    +
    283 
    +test-manifest:
    284 
    +  extends: .base
    285 
    +  image: cimg/python:$PYTHON_VERSION
    286 
    +  script:
    287 
    +    - .gitlab/ci/scripts/run_linters.py test-manifest-alpha test-manifest-disable test-manifest-skip-if
    288 
    +  rules:
    289 
    +    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    290 
    +      changes:
    291 
    +        # List copied from: taskcluster/ci/source-test/mozlint.yml
    292 
    +        #
    293 
    +        - '**/*.ini'
    294 
    +        - 'python/mozlint/**'
    295 
    +        - 'tools/lint/**'
    296 
    +    # Run job whenever a new tag is created
    297 
    +    # or whenever a commit is merged to a protected branch
    298 
    +    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
    299 
    +
    300 
    +trojan-source:
    301 
    +  extends: .base
    302 
    +  image: cimg/python:$PYTHON_VERSION
    303 
    +  script:
    304 
    +    - .gitlab/ci/scripts/run_linters.py trojan-source
    305 
    +  rules:
    306 
    +    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    307 
    +      changes:
    308 
    +        # List copied from: taskcluster/ci/source-test/mozlint.yml
    309 
    +        #
    310 
    +        - '**/*.c'
    311 
    +        - '**/*.cc'
    312 
    +        - '**/*.cpp'
    313 
    +        - '**/*.h'
    314 
    +        - '**/*.py'
    315 
    +        - '**/*.rs'
    316 
    +        - 'tools/lint/trojan-source.yml'
    317 
    +    # Run job whenever a new tag is created
    318 
    +    # or whenever a commit is merged to a protected branch
    319 
    +    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'

  • .gitlab/ci/scripts/run_linters.py
    1 
    +#!/usr/bin/env python3
    2 
    +
    3 
    +import argparse
    4 
    +import os
    5 
    +import re
    6 
    +import shlex
    7 
    +import subprocess
    8 
    +import sys
    9 
    +
    10 
    +
    11 
    +def git(command):
    12 
    +    result = subprocess.run(
    13 
    +        ["git"] + shlex.split(command), check=True, capture_output=True, text=True
    14 
    +    )
    15 
    +    return result.stdout.strip()
    16 
    +
    17 
    +
    18 
    +def get_firefox_tag_from_branch_name(branch_name):
    19 
    +    """Extracts the Firefox tag associated with a branch name.
    20 
    +
    21 
    +       The "firefox tag" is the tag that marks
    22 
    +       the end of the Mozilla commits and the start of the Tor Project commits.
    23 
    +
    24 
    +       Know issue: If ever there is more than one tag per Firefox ESR version,
    25 
    +       this function may return the incorrect reference number.
    26 
    +
    27 
    +    Args:
    28 
    +        branch_name: The branch name to extract the tag from.
    29 
    +        Expected format is tor-browser-91.2.0esr-11.0-1,
    30 
    +        where 91.2.0esr is the Firefox version.
    31 
    +
    32 
    +    Returns:
    33 
    +        The reference specifier of the matching Firefox tag.
    34 
    +        An exception wil be raised if anything goes wrong.
    35 
    +    """
    36 
    +
    37 
    +    # Extracts the version number from a branch name.
    38 
    +    firefox_version = ""
    39 
    +    match = re.search(r"(?<=browser-)([^-]+)", branch_name)
    40 
    +    if match:
    41 
    +        # TODO: Validate that what we got is actually a valid semver string?
    42 
    +        firefox_version = match.group(1)
    43 
    +    else:
    44 
    +        raise ValueError(f"Failed to extract version from branch name '{branch_name}'.")
    45 
    +
    46 
    +    tag = f"FIREFOX_{firefox_version.replace('.', '_')}_"
    47 
    +    remote_tags = git("ls-remote --tags")
    48 
    +
    49 
    +    # Each line looks like:
    50 
    +    # 9edd658bfd03a6b4743ecb75fd4a9ad968603715  refs/tags/FIREFOX_91_9_0esr_BUILD1
    51 
    +    pattern = rf"(.*){re.escape(tag)}(.*)$"
    52 
    +    match = re.search(pattern, remote_tags, flags=re.MULTILINE)
    53 
    +    if match:
    54 
    +        return match.group(0).split()[0]
    55 
    +    else:
    56 
    +        raise ValueError(
    57 
    +            f"Failed to find reference specifier for Firefox tag in branch '{branch_name}'."
    58 
    +        )
    59 
    +
    60 
    +
    61 
    +def get_list_of_changed_files():
    62 
    +    """Gets a list of files changed in the working directory.
    63 
    +
    64 
    +       This function is meant to be run inside the Gitlab CI environment.
    65 
    +
    66 
    +       When running in a default branch, get the list of changed files since the last Firefox tag.
    67 
    +       When running for a new MR commit, get a list of changed files in the current MR.
    68 
    +
    69 
    +    Returns:
    70 
    +        A list of filenames of changed files (excluding deleted files).
    71 
    +        An exception wil be raised if anything goes wrong.
    72 
    +    """
    73 
    +
    74 
    +    base_reference = ""
    75 
    +
    76 
    +    if os.getenv("CI_PIPELINE_SOURCE") == "merge_request_event":
    77 
    +        # For merge requests, the base_reference is the common ancestor between the MR and the target branch.
    78 
    +        base_reference = os.getenv("CI_MERGE_REQUEST_DIFF_BASE_SHA")
    79 
    +    else:
    80 
    +        # When not in merge requests, the base reference is the Firefox tag
    81 
    +        base_reference = get_firefox_tag_from_branch_name(os.getenv("CI_COMMIT_BRANCH"))
    82 
    +
    83 
    +    if not base_reference:
    84 
    +        raise RuntimeError("No base reference found. There might be more errors above.")
    85 
    +
    86 
    +    # Fetch the tag reference
    87 
    +    git(f"fetch origin {base_reference} --depth=1 --filter=blob:none")
    88 
    +    # Return the list of changed files
    89 
    +    return git(f"diff --diff-filter=d --name-only {base_reference} HEAD").split("\n")
    90 
    +
    91 
    +
    92 
    +if __name__ == "__main__":
    93 
    +    parser = argparse.ArgumentParser(
    94 
    +        description="Run ./mach linters in CI. Warning: if you run this in your local environment it might mess up your git history."
    95 
    +    )
    96 
    +    parser.add_argument(
    97 
    +        "linters", metavar="L", type=str, nargs="+", help="A list of linters to run."
    98 
    +    )
    99 
    +    args = parser.parse_args()
    100 
    +
    101 
    +    command = [
    102 
    +        "./mach",
    103 
    +        "lint",
    104 
    +        "-v",
    105 
    +        *(s for l in args.linters for s in ("-l", l)),
    106 
    +        *get_list_of_changed_files(),
    107 
    +    ]
    108 
    +    result = subprocess.run(command, text=True)
    109 
    +
    110 
    +    sys.exit(result.returncode)

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.torproject.org. Manage all notifications · Help 

    _______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits