[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [Git][tpo/applications/tor-browser-build][main] Bug 41713: Triage: find components for bugs found from commits.



Title: GitLab

Pier Angelo Vendrame pushed to branch main at The Tor Project / Applications / tor-browser-build

Commits:

  • 3086604b
    by Pier Angelo Vendrame at 2026-02-11T15:50:54+00:00
    Bug 41713: Triage: find components for bugs found from commits.
    
    In the Bugzilla triage script, run a second query for the bugs that
    were not found in the first query and instead have been found with
    git commits.
    

2 changed files:

Changes:

  • tools/browser/generate-bugzilla-triage-csv deleted
    1
    -#!/usr/bin/env bash
    
    2
    -
    
    3
    -# gitlab labels for review tickets
    
    4
    -browser_label="Apps::Product::TorBrowser"
    
    5
    -esr_label="esr-next"
    
    6
    -priority_label="Priority::Medium"
    
    7
    -impact_label="Apps::Impact::Unknown"
    
    8
    -type_label="Apps::Type::Audit"
    
    9
    -
    
    10
    -# milestone for the next major Tor Browser release
    
    11
    -milestone="Browser 16.0"
    
    12
    -
    
    13
    -# prints to stderr
    
    14
    -function echoerr() { echo "$@" 1>&2; }
    
    15
    -
    
    16
    -script_dir=$(dirname "${BASH_ARGV0:-$0}")
    
    17
    -
    
    18
    -# help dialog
    
    19
    -if [ "$#" -ne 2 ]; then
    
    20
    -    echoerr "Usage: $0 <ff-version> <gitlab-audit-issue>"
    
    21
    -    echoerr
    
    22
    -    echoerr "ff-version             rapid-release Firefox version to audit"
    
    23
    -    echoerr "gitlab-audit-issue     tor-browser GitLab issue number for this audit"
    
    24
    -    echoerr ""
    
    25
    -    echoerr "Example:"
    
    26
    -    echoerr ""
    
    27
    -    echoerr "$0 129 43303"
    
    28
    -    exit 1
    
    29
    -fi
    
    30
    -
    
    31
    -# exit on error
    
    32
    -set -e
    
    33
    -
    
    34
    -
    
    35
    -# Ensure various required tools are available
    
    36
    -function check_exists() {
    
    37
    -    local cmd=$1
    
    38
    -    if ! which ${cmd} > /dev/null ; then
    
    39
    -        echoerr "missing ${cmd} dependency"
    
    40
    -        exit 1
    
    41
    -    fi
    
    42
    -}
    
    43
    -
    
    44
    -check_exists git
    
    45
    -check_exists jq
    
    46
    -check_exists mktemp
    
    47
    -check_exists perl
    
    48
    -check_exists printf
    
    49
    -check_exists sed
    
    50
    -check_exists sort
    
    51
    -check_exists touch
    
    52
    -check_exists uniq
    
    53
    -check_exists wget
    
    54
    -
    
    55
    -# Assign arguments to named variables
    
    56
    -firefox_version=$1
    
    57
    -firefox_old_version=$((firefox_version - 1))
    
    58
    -audit_issue=$2
    
    59
    -git_begin="FIREFOX_NIGHTLY_${firefox_old_version}_END"
    
    60
    -git_end="FIREFOX_NIGHTLY_${firefox_version}_END"
    
    61
    -
    
    62
    -# Check valid Firefox version
    
    63
    -if ! [[ "${firefox_version}" =~ ^[1-9][0-9]{2}$ ]]; then
    
    64
    -    echoerr "invalid Firefox version (probably)"
    
    65
    -    exit 1
    
    66
    -fi
    
    67
    -
    
    68
    -# Check valid Gitlab issue number
    
    69
    -if ! [[ "${audit_issue}" =~ ^[1-9][0-9]{4}$ ]]; then
    
    70
    -    echoerr "invalid gitlab audit issue number (probably)"
    
    71
    -    exit 1
    
    72
    -fi
    
    73
    -
    
    74
    -#
    
    75
    -# Encoding/Decoding Functions
    
    76
    -#
    
    77
    -
    
    78
    -# escape " and \
    
    79
    -function json_escape() {
    
    80
    -    local input="$1"
    
    81
    -    echo "${input}" | sed 's/["\]/\\"/g'
    
    82
    -}
    
    83
    -
    
    84
    -
    
    85
    -# un-escape \"
    
    86
    -function jq_unescape() {
    
    87
    -    local input="$1"
    
    88
    -    echo "${input}" | sed 's/\\"/"/g'
    
    89
    -}
    
    90
    -
    
    91
    -# change quotes to double-quotes
    
    92
    -function csv_escape() {
    
    93
    -    local input="$1"
    
    94
    -    echo "${input}" | sed 's/"/""/g'
    
    95
    -}
    
    96
    -
    
    97
    -# we need to urlencode the strings used in the new issue link
    
    98
    -function url_encode() {
    
    99
    -    local input="$1"
    
    100
    -    echo "${input}" | perl -MURI::Escape -wlne 'print uri_escape $_'
    
    101
    -}
    
    102
    -
    
    103
    -
    
    104
    -#
    
    105
    -# Create temp json files
    
    106
    -#
    
    107
    -git_json=$(mktemp -t git-audit-${firefox_version}-XXXXXXXXXXX.json)
    
    108
    -bugzilla_json=$(mktemp -t bugzilla-audit-${firefox_version}-XXXXXXXXXXX.json)
    
    109
    -union_json=$(mktemp -t union-audit-${firefox_version}-XXXXXXXXXXX.json)
    
    110
    -touch "${git_json}"
    
    111
    -touch "${bugzilla_json}"
    
    112
    -touch "${union_json}"
    
    113
    -
    
    114
    -function json_cleanup {
    
    115
    -    rm -f "${git_json}"
    
    116
    -    rm -f "${bugzilla_json}"
    
    117
    -    rm -f "${union_json}"
    
    118
    -    popd > /dev/null
    
    119
    -}
    
    120
    -pushd "${script_dir}/torbrowser" > /dev/null
    
    121
    -trap json_cleanup EXIT
    
    122
    -
    
    123
    -#
    
    124
    -# Generate Git Commit Triage List
    
    125
    -#
    
    126
    -
    
    127
    -# Try and extract bug id and summary from git log
    
    128
    -# Mozilla's commits are not always 100% consistently named, so this
    
    129
    -# regex is a bit flexible to handle various inputs such as:
    
    130
    -# "Bug 1234 -", "Bug 1234:", "Bug Bug 1234 -", "[Bug 1234] -", " bug 1234 -".
    
    131
    -sed_extract_id_summary="s/^[[ ]*[bug –-]+ ([1-9][0-9]*)[]:\., –-]*(.*)\$/\\1 \\2/pI"
    
    132
    -
    
    133
    -# Generate a json array of objects in the same format as bugzilla: {component: string, id: number, summary: string}
    
    134
    -printf "[\n" >> "${git_json}"
    
    135
    -
    
    136
    -first_object=true
    
    137
    -git log --format='%s' $git_begin..$git_end  \
    
    138
    -| sed -En "${sed_extract_id_summary}" \
    
    139
    -| sort -h \
    
    140
    -| uniq \
    
    141
    -| while IFS= read -r line; do
    
    142
    -    read -r id summary <<< "${line}"
    
    143
    -    summary=$(json_escape "${summary}")
    
    144
    -
    
    145
    -    # json does not allow trailing commas
    
    146
    -    if [[ "${first_object}" = true ]]; then
    
    147
    -        first_object=false
    
    148
    -    else
    
    149
    -        printf ",\n" >> "${git_json}"
    
    150
    -    fi
    
    151
    -
    
    152
    -    product="Unknown"
    
    153
    -    # Bugs that include webcompat-reviewers and no other additional -reviewers
    
    154
    -    # from other teams are *likely* in the Web Compatibility component. But they
    
    155
    -    # won't show in the bugzilla query because the component doesn't tend to
    
    156
    -    # have any set milestones.
    
    157
    -    if [[ "${summary}" =~ r=.*webcompat-reviewers && ! "${summary}" =~ r=.*-reviewers.*-reviewers ]]; then
    
    158
    -        product="Web Compatibility"
    
    159
    -    fi
    
    160
    -
    
    161
    -    printf "  { \"product\": \"%s\", \"component\": \"Unknown\", \"id\": %s, \"summary\": \"%s\" }" "${product}" ${id} "${summary}" >> "${git_json}"
    
    162
    -done
    
    163
    -printf "\n]\n" >> "${git_json}"
    
    164
    -
    
    165
    -#
    
    166
    -# Download Bugzilla Triage List
    
    167
    -#
    
    168
    -
    
    169
    -# search for:
    
    170
    -# + Product is NOT "Thunderbird,Calander,Chat Core,MailNews Core" (&f1=product&n1=1&o1=anyexact&v1=Thunderbird%2CCalendar%2CChat%20Core%2CMailNews%20Core). AND
    
    171
    -# + Target Milestone contains "${firefox_version}" (e.g. 115 Branch or Firefox 115) (&f2=target_milestone&o2=substring&v2=${firefox_version}).
    
    172
    -# "&limit=0" shows all matching bugs.
    
    173
    -
    
    174
    -query_tail="&f1=product&n1=1&o1=anyexact&v1=Thunderbird%2CCalendar%2CChat%20Core%2CMailNews%20Core&f2=target_milestone&o2=substring&v2=${firefox_version}&limit=0"
    
    175
    -
    
    176
    -bugzilla_query="https://bugzilla.mozilla.org/buglist.cgi?${query_tail}"
    
    177
    -bugzilla_json_query="https://bugzilla.mozilla.org/rest/bug?include_fields=id,product,component,summary${query_tail}"
    
    178
    -
    
    179
    -wget "${bugzilla_json_query}" -O ${bugzilla_json}
    
    180
    -
    
    181
    -#
    
    182
    -# Create Union of these two sets of issues
    
    183
    -#
    
    184
    -
    
    185
    -# bugzilla array is actually on a root object: { bugs: [...] }
    
    186
    -jq -s '[ (.[0].bugs)[], (.[1])[] ] | group_by(.id) | map(.[0])' "${bugzilla_json}" "${git_json}" > "${union_json}"
    
    187
    -
    
    188
    -#
    
    189
    -# Generate Triage CSV
    
    190
    -#
    
    191
    -
    
    192
    -echo "\"Review\",,\"Product :: Component\",\"Bugzilla Bug\""
    
    193
    -
    
    194
    -jq '. | sort_by([.product, .component, .id])[] | "\(.id)|\(.product)|\(.component)|\(.summary)"' ${union_json} \
    
    195
    -| while IFS='|' read -r id product component summary; do
    
    196
    -
    
    197
    -    # bugzilla info
    
    198
    -    id="${id:1}"
    
    199
    -    component="${component:0}"
    
    200
    -    summary="${summary:0:-1}"
    
    201
    -    summary=$(jq_unescape "${summary}")
    
    202
    -    # short summary for gitlab issue title
    
    203
    -    [[ ${#summary} -gt 80 ]] && summary_short="${summary:0:77}..." || summary_short="${summary}"
    
    204
    -
    
    205
    -    # filter out some issue types that we never care about
    
    206
    -    skip_issue=false
    
    207
    -
    
    208
    -    # skip `[wpt-sync] Sync PR`
    
    209
    -    if [[ "${summary}" =~ ^\[wpt-sync\]\ Sync\ PR.*$ ]]; then
    
    210
    -        skip_issue=true
    
    211
    -    # skip `Crash in [@` and variants
    
    212
    -    elif [[ "${summary}" =~ ^Crash[esin\ ]*\ \[\@.*$ ]]; then
    
    213
    -        skip_issue=true
    
    214
    -    # skip `Assertion failuire: `
    
    215
    -    elif [[ "${summary}" =~ ^Assertion\ failure:\ .*$ ]]; then
    
    216
    -        skip_issue=true
    
    217
    -    # skip `Hit MOZ_CRASH`
    
    218
    -    elif [[ "${summary}" =~ ^Hit\ MOZ_CRASH.*$ ]]; then
    
    219
    -        skip_issue=true
    
    220
    -    fi
    
    221
    -
    
    222
    -    if [[ "${skip_issue}" = true ]]; then
    
    223
    -        echoerr "Skipped Bugzilla ${id}: ${summary_short}"
    
    224
    -    else
    
    225
    -        csv_summary=$(csv_escape "${summary}")
    
    226
    -        csv_product_component=$(csv_escape "${product} :: ${component}")
    
    227
    -
    
    228
    -        # parent issue
    
    229
    -        bugzilla_url="https://bugzilla.mozilla.org/show_bug.cgi?id=${id}"
    
    230
    -        # review issue title
    
    231
    -        new_issue_title=$(url_encode "Review Mozilla ${id}: ${summary_short}")
    
    232
    -        # review issue description + labeling (14.0 stable, FF128-esr, Next)
    
    233
    -        new_issue_description=$(url_encode "### Bugzilla: ${bugzilla_url}")%0A$(url_encode "/label ~\"${browser_label}\" ~\"${esr_label}\" ~\"${priority_label}\" ~\"${impact_label}\" ~\"${type_label}\"")%0A$(url_encode "/milestone %\"${milestone}\"")%0A$(url_encode "/relate tpo/applications/tor-browser#${audit_issue}")%0A%0A$(url_encode "<!-- briefly describe why this issue needs further review -->")%0A
    
    234
    -
    
    235
    -        # url which create's new issue with title and description pre-populated
    
    236
    -        new_issue_url="https://gitlab.torproject.org/tpo/applications/tor-browser/-/issues/new?issue[title]=${new_issue_title}&issue[description]=${new_issue_description}"
    
    237
    -
    
    238
    -        # this link will start the creation of a new gitlab issue to review
    
    239
    -        create_issue=$(csv_escape "=HYPERLINK(\"${new_issue_url}\", \"New Issue\")")
    
    240
    -        bugzilla_link=$(csv_escape "=HYPERLINK(\"${bugzilla_url}\", \"Bugzilla ${id}: ${csv_summary}\")")
    
    241
    -
    
    242
    -        echo "FALSE,\"${create_issue}\",\"${csv_product_component}\",\"${bugzilla_link}\","
    
    243
    -    fi
    
    244
    -done
    
    245
    -
    
    246
    -echo
    
    247
    -
    
    248
    -bugzilla_query="=HYPERLINK(\"${bugzilla_query}\", \"Bugzilla query\")"
    
    249
    -echo \"$(csv_escape "${bugzilla_query}")\"

  • tools/browser/generate-bugzilla-triage-csv.py
    1
    +#!/usr/bin/env python3
    
    2
    +import argparse
    
    3
    +import csv
    
    4
    +import re
    
    5
    +import sys
    
    6
    +import urllib.parse
    
    7
    +from pathlib import Path
    
    8
    +
    
    9
    +import requests
    
    10
    +from git import Repo
    
    11
    +
    
    12
    +
    
    13
    +GITLAB_LABELS = [
    
    14
    +    "Apps::Product::TorBrowser",
    
    15
    +    "esr-next",
    
    16
    +    "Priority::Medium",
    
    17
    +    "Apps::Impact::Unknown",
    
    18
    +    "Apps::Type::Audit",
    
    19
    +]
    
    20
    +GITLAB_MILESTONE = "Browser 16.0"
    
    21
    +
    
    22
    +
    
    23
    +def download_bugs(url):
    
    24
    +    url += "&" if "?" in url else "?"
    
    25
    +    url += "include_fields=id,product,component,summary"
    
    26
    +    r = requests.get(url)
    
    27
    +    r.raise_for_status()
    
    28
    +    # {"bugs": [ {...}, ... ]}
    
    29
    +    bugs = r.json().get("bugs", [])
    
    30
    +    results = {}
    
    31
    +    for bug in bugs:
    
    32
    +        product = bug.get("product", "")
    
    33
    +        component = bug.get("component", "")
    
    34
    +        results[bug.get("id")] = {
    
    35
    +            "id": bug.get("id"),
    
    36
    +            "component": f"{product} :: {component}",
    
    37
    +            "summary": bug.get("summary", ""),
    
    38
    +        }
    
    39
    +    return results
    
    40
    +
    
    41
    +
    
    42
    +def extract_from_git(repo_path, ff_version, bugs):
    
    43
    +    subj_regex = re.compile(
    
    44
    +        r"^[\s*[bug –-]+\s*([1-9][0-9]*)[\]:\., –-]*(.*)", re.VERBOSE
    
    45
    +    )
    
    46
    +    # Keep these lowercase
    
    47
    +    skip = ["no bug", "merge ", "revert "]
    
    48
    +
    
    49
    +    repo = Repo(repo_path)
    
    50
    +    seen_ids = set()
    
    51
    +    results = {}
    
    52
    +    for commit in repo.iter_commits(
    
    53
    +        f"FIREFOX_NIGHTLY_{ff_version - 1}_END..FIREFOX_NIGHTLY_{ff_version}_END",
    
    54
    +        reverse=True,
    
    55
    +    ):
    
    56
    +        subject = commit.message.splitlines()[0]
    
    57
    +        subj_low = subject.lower()
    
    58
    +        m = subj_regex.search(subj_low)
    
    59
    +        if not m:
    
    60
    +            if not any(subj_low.startswith(p) for p in skip):
    
    61
    +                print(f"Could not match {subject}", file=sys.stderr)
    
    62
    +            continue
    
    63
    +        bug_id = int(m.group(1))
    
    64
    +        if bug_id in bugs or bug_id in seen_ids:
    
    65
    +            continue
    
    66
    +        seen_ids.add(bug_id)
    
    67
    +        summary = m.group(2).strip()
    
    68
    +        results[bug_id] = {
    
    69
    +            "id": bug_id,
    
    70
    +            "component": "(git commit)",
    
    71
    +            "summary": summary,
    
    72
    +        }
    
    73
    +
    
    74
    +    # Notice that seen_ids contains only the bugs we have seen in the
    
    75
    +    # commit range but we did not find earlier.
    
    76
    +    # So, we expect this number to be quite low and that it will be
    
    77
    +    # possible to fetch all these bugs in a single pass.
    
    78
    +    if seen_ids:
    
    79
    +        url = "https://bugzilla.mozilla.org/rest/bug?id="
    
    80
    +        url += ",".join([str(i) for i in seen_ids])
    
    81
    +        results.update(download_bugs(url))
    
    82
    +    bugs.update(results)
    
    83
    +
    
    84
    +
    
    85
    +def make_hyperlink(url, label):
    
    86
    +    label = label.replace('"', '""')
    
    87
    +    return f'=HYPERLINK("{url}", "{label}")'
    
    88
    +
    
    89
    +
    
    90
    +def url_encode(s):
    
    91
    +    return urllib.parse.quote(s)
    
    92
    +
    
    93
    +
    
    94
    +def generate_csv(bugs, audit_issue, query, output_path):
    
    95
    +    bugs = sorted(bugs.values(), key=lambda x: (x["component"], x["id"]))
    
    96
    +
    
    97
    +    with output_path.open("w") as f:
    
    98
    +        writer = csv.writer(f, quoting=csv.QUOTE_ALL)
    
    99
    +        writer.writerow(["Review", "", "Bugzilla Component", "Bugzilla Bug"])
    
    100
    +
    
    101
    +        # Keep these lowercase!
    
    102
    +        skip = [
    
    103
    +            "[wpt-sync] sync pr",
    
    104
    +            "assertion failure: ",
    
    105
    +            "crash in ",
    
    106
    +            "crash [@",
    
    107
    +            "hit moz_crash",
    
    108
    +            "update android nightly application-services",
    
    109
    +            "update web-platform-tests",
    
    110
    +        ]
    
    111
    +        for bug in bugs:
    
    112
    +            bug_id = bug["id"]
    
    113
    +            component = bug["component"]
    
    114
    +            summary = bug["summary"]
    
    115
    +            summary_lower = summary.lower()
    
    116
    +            short = (summary[:77] + "...") if len(summary) > 80 else summary
    
    117
    +
    
    118
    +            if any(summary_lower.startswith(p) for p in skip):
    
    119
    +                print(f"Skipped Bugzilla {bug_id}: {summary}", file=sys.stderr)
    
    120
    +                continue
    
    121
    +
    
    122
    +            bugzilla_url = (
    
    123
    +                f"https://bugzilla.mozilla.org/show_bug.cgi?id={bug_id}"
    
    124
    +            )
    
    125
    +            new_title = url_encode(f"Review Mozilla {bug_id}: {short}")
    
    126
    +            new_desc = (
    
    127
    +                url_encode(f"### Bugzilla: {bugzilla_url}")
    
    128
    +                + "%0A"
    
    129
    +                + url_encode("/label ~" + (" ~".join(GITLAB_LABELS)))
    
    130
    +                + "%0A"
    
    131
    +                + url_encode(f'/milestone %"{GITLAB_MILESTONE}"')
    
    132
    +                + "%0A"
    
    133
    +                + url_encode(
    
    134
    +                    f"/relate tpo/applications/tor-browser#{audit_issue}"
    
    135
    +                )
    
    136
    +                + "%0A%0A"
    
    137
    +                + url_encode(
    
    138
    +                    "<!-- briefly describe why this issue needs further review -->"
    
    139
    +                )
    
    140
    +            )
    
    141
    +            new_issue_url = f"https://gitlab.torproject.org/tpo/applications/tor-browser/-/issues/new?issue[title]={new_title}&issue[description]={new_desc}"
    
    142
    +
    
    143
    +            create_link = make_hyperlink(new_issue_url, "New Issue")
    
    144
    +            bugzilla_link = make_hyperlink(
    
    145
    +                bugzilla_url, f"Bugzilla {bug_id}: {summary}"
    
    146
    +            )
    
    147
    +            writer.writerow(["FALSE", create_link, component, bugzilla_link])
    
    148
    +
    
    149
    +        writer.writerow([])
    
    150
    +        writer.writerow([make_hyperlink(query, "Bugzilla query")])
    
    151
    +
    
    152
    +
    
    153
    +def main():
    
    154
    +    parser = argparse.ArgumentParser(
    
    155
    +        description="Create a triage CSV for a Firefox version."
    
    156
    +    )
    
    157
    +    parser.add_argument(
    
    158
    +        "-r",
    
    159
    +        "--repo-path",
    
    160
    +        help="Path to a tor-browser.git clone",
    
    161
    +        type=Path,
    
    162
    +        default=Path(__file__).parent / "torbrowser",
    
    163
    +    )
    
    164
    +    parser.add_argument(
    
    165
    +        "-o",
    
    166
    +        "--output-file",
    
    167
    +        help="Path to the output file",
    
    168
    +        type=Path,
    
    169
    +    )
    
    170
    +    parser.add_argument(
    
    171
    +        "ff_version",
    
    172
    +        help="Firefox rapid-release version (e.g. 150)",
    
    173
    +        type=int,
    
    174
    +    )
    
    175
    +    parser.add_argument(
    
    176
    +        "gitlab_audit_issue",
    
    177
    +        help="Tor-Browser-Spec GitLab issue number",
    
    178
    +        type=int,
    
    179
    +    )
    
    180
    +    known_args, remaining = parser.parse_known_args()
    
    181
    +    parser.set_defaults(
    
    182
    +        output_file=Path(f"triage-{known_args.ff_version}.csv")
    
    183
    +    )
    
    184
    +    args = parser.parse_args()
    
    185
    +
    
    186
    +    if args.ff_version < 140:
    
    187
    +        print("Wrong Firefox version (probably)!", file=sys.stderr)
    
    188
    +        sys.exit(1)
    
    189
    +    if args.gitlab_audit_issue < 40000:
    
    190
    +        print("Wrong GitLab issue number (probably)!", file=sys.stderr)
    
    191
    +        sys.exit(1)
    
    192
    +
    
    193
    +    excluded_products = "Thunderbird,Calendar,Chat%20Core,MailNews%20Core"
    
    194
    +    query = f"https://bugzilla.mozilla.org/rest/bug?f1=product&n1=1&o1=anyexact&v1={excluded_products}&f2=target_milestone&o2=substring&v2={args.ff_version}&limit=0"
    
    195
    +    bugs = download_bugs(query)
    
    196
    +    extract_from_git(args.repo_path, args.ff_version, bugs)
    
    197
    +
    
    198
    +    generate_csv(
    
    199
    +        bugs,
    
    200
    +        args.gitlab_audit_issue,
    
    201
    +        query,
    
    202
    +        args.output_file,
    
    203
    +    )
    
    204
    +
    
    205
    +
    
    206
    +if __name__ == "__main__":
    
    207
    +    main()

  • _______________________________________________
    tor-commits mailing list -- tor-commits@xxxxxxxxxxxxxxxxxxxx
    To unsubscribe send an email to tor-commits-leave@xxxxxxxxxxxxxxxxxxxx