[tor-commits] [Git][tpo/applications/rbm][main] 2 commits: Fix "mercurial repo build" test

richard pushed to branch main at The Tor Project / Applications / RBM

Commits:

45dcee8b

by Nicolas Vigier at 2024-06-12T11:39:32+02:00

Fix "mercurial repo build" test

Depending on the mercurial or tar version installed, the generated tar
file can be different. We change the test to check the checksum of the
files included in the tarball instead of the tar file.

1e0cfb68

by Nicolas Vigier at 2024-06-26T13:22:00+02:00

Bug 40077: Allow to specify sha512sum for input_files

Also add the sha512 and sha512file template functions.

We also add tests to `test.pl` to check that we exit with an error when
sha256sum or sha512sum is wrong (and that we don't when it's the
expected one).

6 changed files:

doc/rbm_input_files.asc
doc/rbm_templates.asc
lib/RBM.pm
test.pl
test/projects/mozmill-automation/config
+ test/projects/shasum/config

Changes:

doc/rbm_input_files.asc

@@ -26,7 +26,7 @@ The following input sources are available:
  The file that has been retrieved can be verified with:
 -- matching a specific sha256 checksum
 +- matching a specific sha256 or sha512 checksum
  - a gpg signature file from a specific key or keyring
@@ -104,17 +104,22 @@ enable::
  refresh_input::
          By default, if the file is already present, it is not downloaded
 -        or created again, except when an sha256sum is defined and the
 -        file present is not matching. If this option is set to a true
 -        value, the file will be removed and created again on each run,
 -        except when an sha256sum is defined and the file present is
 -        matching.
 +        or created again, except when an sha256sum or sha512sum is
 +        defined and the file present is not matching. If this option is
 +        set to a true value, the file will be removed and created again
 +        on each run, except when an sha256sum or sha512sum is defined
 +        and the file present is matching.
  sha256sum::
          The sha256 checksum of the file. The build will fail with an
          error if the file does not match the expected sha256 checksum.
          If the value is empty, the checksum is not checked.
 +sha512sum::
 +        The sha512 checksum of the file. The build will fail with an
 +        error if the file does not match the expected sha512 checksum.
 +        If the value is empty, the checksum is not checked.
++
  file_gpg_id::
          If this option is set to 1, the file is checked for a gpg
          signature. If it is set to an other non zero value, or an array

doc/rbm_templates.asc

@@ -138,10 +138,18 @@ sha256::
          A function returning the sha256 digest of its argument as an
          hexadecimal string.
 +sha512::
 +        A function returning the sha512 digest of its argument as an
 +        hexadecimal string.
++
  sha256file::
          A function returning the sha256 digest of a file as an hexadecimal
          string. If the file does not exist, an empty string is returned.
 +sha512file::
 +        A function returning the sha512 digest of a file as an hexadecimal
 +        string. If the file does not exist, an empty string is returned.
++
  fileparse::
          A function to parse a path. Returns an array containing the
          filename, and the directory path. This is the fileparse routine

lib/RBM.pm

@@ -19,7 +19,7 @@ use String::ShellQuote;
  use Sort::Versions;
  use RBM::CaptureExec qw(capture_exec);
  use RBM::DefaultConfig;
 -use Digest::SHA qw(sha256_hex);
 +use Digest::SHA qw(sha256_hex sha512_hex);
  use Data::UUID;
  use Data::Dump qw(dd pp);
  use FindBin;
@@ -673,16 +673,22 @@ sub maketar {
      return $tar_file;
+ }
 -sub sha256file {
 +sub shafile {
      CORE::state %res;
 +    my $type = shift;
      my $f = rbm_path(shift);
      my $opt = shift;
 +    my %sha_hex = (
 +        sha256sum => \&sha256_hex,
 +        sha512sum => \&sha512_hex,
 +    );
 +    exit_error "Unknown sha type $type" unless $sha_hex{$type};
      if (ref $opt eq 'HASH' && $opt->{remove_cache}) {
 -        delete $res{$f};
 +        delete $res{$type}{$f};
          return;
+     }
 -    return $res{$f} if exists $res{$f};
 -    return $res{$f} = -f $f ? sha256_hex(path($f)->slurp_raw) : '';
 +    return $res{$type}{$f} if exists $res{$type}{$f};
 +    return $res{$type}{$f} = -f $f ? $sha_hex{$type}->(path($f)->slurp_raw) : '';
+ }
  sub process_template_opt {
@@ -731,7 +737,11 @@ sub process_template {
          sha256      => sub {
              return sha256_hex(encode("utf8", $_[0]));
          },
 -        sha256file  => \&sha256file,
 +        sha512      => sub {
 +            return sha512_hex(encode("utf8", $_[0]));
 +        },
 +        sha256file  => sub { return shafile('sha256sum', @_) },
 +        sha512file  => sub { return shafile('sha512sum', @_) },
          fileparse   => \&fileparse,
          ENV         => \%ENV,
      };
@@ -782,18 +792,22 @@ sub file_in_dir {
  sub input_file_need_dl {
      my ($input_file, $t, $fname, $action) = @_;
      return undef if $action eq 'getfpaths';
 -    if ($fname
 -        && ($input_file->{sha256sum} || $input_file->{norec}{sha256sum})
 -        && $t->('sha256sum')
 -        && $t->('sha256sum') ne sha256file($fname)) {
 -        sha256file($fname, { remove_cache => 1 });
 -        $fname = undef;
 +    for my $checksum (qw/sha512sum sha256sum/) {
 +        if ($fname
 +            && ($input_file->{$checksum} || $input_file->{norec}{$checksum})
 +            && $t->($checksum)
 +            && $t->($checksum) ne shafile($checksum, $fname)) {
 +            shafile($checksum, $fname, { remove_cache => 1 });
 +            $fname = undef;
 +        }
+     }
      if ($action eq 'input_files_id') {
          return undef if $input_file->{input_file_id};
 -        if ( ($input_file->{sha256sum} || $input_file->{norec}{sha256sum})
 -             && $t->('sha256sum') ) {
 -            return undef;
 +        for my $checksum (qw/sha512sum sha256sum/) {
 +            if ( ($input_file->{$checksum} || $input_file->{norec}{$checksum})
 +                && $t->($checksum) ) {
 +                return undef;
 +            }
+         }
          return undef if $input_file->{exec};
          return undef if ($fname && !$t->('refresh_input'));
@@ -808,8 +822,8 @@ sub input_file_need_dl {
  sub input_file_id_hash {
      my ($fname, $filename) = @_;
      exit_error "input_file_id: file $filename is missing" unless $fname;
 -    return $filename . ':' . sha256file($fname) if -f $fname;
 -    return $filename . ':' . sha256file(readlink $fname) if -l $fname;
 +    return $filename . ':' . shafile('sha256sum', $fname) if -f $fname;
 +    return $filename . ':' . shafile('sha256sum', readlink $fname) if -l $fname;
      my @subdirs = sort(map { $_->basename } path($fname)->children);
      my @hashes = map { input_file_id_hash("$fname/$_", "$filename/$_") } @subdirs;
      return join("\n", @hashes);
@@ -819,9 +833,11 @@ sub input_file_id {
      my ($input_file, $t, $fname, $filename) = @_;
      return $t->('input_file_id') if $input_file->{input_file_id};
      return $input_file->{project} . ':' . $filename if $input_file->{project};
 -    if ( ($input_file->{sha256sum} || $input_file->{norec}{sha256sum})
 -         && $t->('sha256sum') ) {
 -        return $filename . ':' . $t->('sha256sum');
 +    for my $checksum (qw/sha512sum sha256sum/) {
 +        if ( ($input_file->{$checksum} || $input_file->{norec}{$checksum})
 +            && $t->($checksum) ) {
 +            return $filename . ':' . $t->($checksum);
 +        }
+     }
      my $opts = { norec => { output_dir => '/out', getting_id => 1, }};
      return $filename . ':' . sha256_hex($t->('exec', $opts))
@@ -1057,11 +1073,13 @@ sub input_files {
              next;
+         }
          exit_error "Missing file $name" unless $fname;
 -        if ($t->('sha256sum')
 -            && $t->('sha256sum') ne sha256file($fname)) {
 -            exit_error "Can't have sha256sum on directory: $fname" if -d $fname;
 -            exit_error "Wrong sha256sum for $fname.\n" .
 -                       "Expected sha256sum: " . $t->('sha256sum');
 +        for my $checksum (qw/sha512sum sha256sum/) {
 +            if ($t->($checksum)
 +                && $t->($checksum) ne shafile($checksum, $fname)) {
 +                exit_error "Can't have $checksum on directory: $fname" if -d $fname;
 +                exit_error "Wrong $checksum for $fname.\n" .
 +                           "Expected $checksum: " . $t->($checksum);
 +            }
+         }
          if ($file_gpg_id) {
              exit_error "Can't have gpg sig on directory: $fname" if -d $fname;

test.pl

  #!/usr/bin/perl -w
  use strict;
  use Path::Tiny;
 -use Test::More tests => 41;
 +use Test::More tests => 45;
  use lib 'lib/';
  sub set_target {
@@ -248,8 +248,7 @@ my @tests = (
          build => [ 'mozmill-automation', 'build' ],
          files => {
              'out/mozmill-automation-bbad7215c713_sha256sum.txt' =>
 -            '13660d3f3ebbc363056ccbd3794f8f78a940dd394a464093bee5fc0575ee4090  '
 -            . "mozmill-automation-bbad7215c713.tar\n",
 +            "ceeda3cd3285b6ed53233dc65e3beac82f2b284402a80ef6c1fcdf5b9861f068  s.txt\n",
          },
      },
+     {
@@ -261,6 +260,28 @@ my @tests = (
              "1\n2\n3\n4\n1\n2\n",
          },
      },
 +    {
 +        name => 'sha256sum input_files',
 +        target => [ 'sha256sum' ],
 +        build  => [ 'shasum', 'build' ],
 +        files  => {},
 +    },
 +    {
 +        name => 'sha512sum input_files',
 +        target => [ 'sha512sum' ],
 +        build  => [ 'shasum', 'build' ],
 +        files  => {},
 +    },
 +    {
 +        name => 'wrong sha256sum input_files',
 +        target => [ 'wrong_sha256sum' ],
 +        fail_build  => [ 'shasum', 'build' ],
 +    },
 +    {
 +        name => 'wrong sha512sum input_files',
 +        target => [ 'wrong_sha512sum' ],
 +        fail_build  => [ 'shasum', 'build' ],
 +    },
  );
  foreach my $test (@tests) {
@@ -279,4 +300,16 @@ foreach my $test (@tests) {
          my $res = grep { path($_)->slurp_utf8 ne $test->{files}{$_} } keys %{$test->{files}};
          ok(!$res, $test->{name});
+     }
 +    if ($test->{fail_build}) {
 +        my $pid = fork;
 +        if (!$pid) {
 +            close STDOUT;
 +            close STDERR;
 +            RBM::build_run(@{$test->{fail_build}});
 +            exit 0;
 +        }
 +        wait;
 +        my $exit_code = $?;
 +        ok($exit_code, $test->{name});
 +    }
+ }

test/projects/mozmill-automation/config

@@ -5,4 +5,11 @@ compress_tar: ''
  t: '[% sha256(exec("cat testrun_remote.py")) %]'
  build: |
    #!/bin/sh
 -  sha256sum [% project %]-[% c("version") %].tar > [% dest_dir %]/[% project %]-[% c("version") %]_sha256sum.txt
 +  mkdir t
 +  tar -C t -xf mozmill-automation-bbad7215c713.tar
 +  files=$(find t | sort)
 +  for file in $files
 +  do
 +    sha256sum $file >> s.txt
 +  done
 +  sha256sum s.txt > [% dest_dir %]/[% project %]-[% c("version") %]_sha256sum.txt

test/projects/shasum/config

 +# vim: filetype=yaml sw=2
 +debug: 1
 +filename: 'shasum_project-[% c("input_files_id") %]'
 +build: |
 +  #!/bin/sh
 +  echo ok > [% dest_dir _ '/' _ c("filename") %]
++
 +targets:
 +  sha256sum:
 +    input_files:
 +      - filename: sha256sums-signed-build.txt
 +        URL: https://archive.torproject.org/tor-package-archive/torbrowser/13.0.15/sha256sums-signed-build.txt
 +        sha256sum: 380c611762cf02a89a5885e7182ce17fc653f6b910c00ce50295c03c488b13ac
 +  sha512sum:
 +    input_files:
 +      - filename: sha256sums-signed-build.txt
 +        URL: https://archive.torproject.org/tor-package-archive/torbrowser/13.0.15/sha256sums-signed-build.txt
 +        sha512sum: 5a1a5199f2135dd75bfeddafc25a62ce473083d371b13f90582b5faf3a3e7c415c4b4990d4927d8a468dca88bc8376fb55143020e7dadcc69b316f6212a7f825
 +  wrong_sha256sum:
 +    input_files:
 +      - filename: sha256sums-signed-build.txt
 +        URL: https://archive.torproject.org/tor-package-archive/torbrowser/13.0.15/sha256sums-signed-build.txt
 +        sha256sum: aaa
 +  wrong_sha512sum:
 +    input_files:
 +      - filename: sha256sums-signed-build.txt
 +        URL: https://archive.torproject.org/tor-package-archive/torbrowser/13.0.15/sha256sums-signed-build.txt
 +        sha512sum: aaa