[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [tor-browser-build/master] Bug 21655: add a script to clean old built components



commit 5d046ea3c4af539e36a886ee3a7b03c07f5d7b48
Author: Nicolas Vigier <boklm@xxxxxxxxxxxxxx>
Date:   Mon Mar 20 18:29:03 2017 +0100

    Bug 21655: add a script to clean old built components
---
 Makefile               |   5 ++-
 README                 |  10 ++++-
 rbm                    |   2 +-
 rbm.local.conf.example |  28 ++++++++++++
 tools/clean-old        | 114 +++++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 155 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 9481537..b82309c 100644
--- a/Makefile
+++ b/Makefile
@@ -86,6 +86,9 @@ submodule-update:
 fetch: submodule-update
 	$(rbm) fetch
 
-clean-old: submodule-update
+clean: submodule-update
 	./tools/clean-old
 
+clean-dry-run: submodule-update
+	./tools/clean-old --dry-run
+
diff --git a/README b/README
index a14fb42..f4a74e0 100644
--- a/README
+++ b/README
@@ -171,8 +171,14 @@ file.
 Cleaning obsolete files and containers images
 ---------------------------------------------
 
-There will be a script to clean old build files and containers that are
-no longer used, but it has not been added yet.
+You can run 'make clean' to clean old build files and containers that
+are no longer used in current builds. Before doing that, you need to
+configure the branches and build targets you are using in the
+rbm.local.conf file. The cleaning script will checkout all the configured
+branches to create a list of used build files, and delete the files
+from the 'out' directory that are not used. If you want to see the list
+of files and containers that would be removed without doing it, you can
+use 'make clean-dry-run'.
 
 
 Multiple build directories on the same host
diff --git a/rbm b/rbm
index 845bb90..3f3886e 160000
--- a/rbm
+++ b/rbm
@@ -1 +1 @@
-Subproject commit 845bb904fb87b6c56ab789f35cd02bce32453f52
+Subproject commit 3f3886e1f210ad2853209c5aecd0951350a6f758
diff --git a/rbm.local.conf.example b/rbm.local.conf.example
index 2a86647..78de08f 100644
--- a/rbm.local.conf.example
+++ b/rbm.local.conf.example
@@ -39,6 +39,34 @@ var:
   ### file.
   #sign_build_gpg_opts: '--local-user XXXXXXXX'
 
+  ### The clean configuration is used by the cleaning script to find the
+  ### branches and build targets you are using, to compute the list of
+  ### files that should be kept.
+  ###
+  ### If you only do alpha builds for all platforms, you can use the
+  ### following configuration:
+  #clean:
+  #  HEAD:
+  #    - project: release
+  #      target:
+  #        - alpha
+  #        - torbrowser-all
+  #
+  ### If you are doing 'release' builds in the maint-7.0 branch and
+  ### 'alpha' builds in the master branch, you can use the following
+  ### configuration:
+  #clean:
+  #  master:
+  #    - project: release
+  #      target:
+  #        - alpha
+  #        - torbrowser-all
+  #  maint-7.0:
+  #    - project: release
+  #      target:
+  #        - release
+  #        - torbrowser-all
+
 targets:
 
   ### testbuild is based on alpha by default. Uncomment this if you want it
diff --git a/tools/clean-old b/tools/clean-old
new file mode 100755
index 0000000..4d603fc
--- /dev/null
+++ b/tools/clean-old
@@ -0,0 +1,114 @@
+#!/usr/bin/perl -w
+use strict;
+use IO::CaptureOutput qw(capture_exec);
+use FindBin;
+use lib "$FindBin::Bin/../rbm/lib";
+use RBM;
+use File::Slurp qw(read_dir write_file read_file);
+use Getopt::Long;
+
+my %options;
+my @options_list = qw(dry-run! list-used-files=s project=s target=s@);
+Getopt::Long::GetOptionsFromArray(\@ARGV, \%options, @options_list) || exit 1;
+
+sub clean_file {
+    my ($file, $used_files) = @_;
+    return if $used_files->{$file};
+    if (-d $file) {
+        my @l = read_dir($file);
+        foreach my $subfile (@l) {
+            clean_file("$file/$subfile", $used_files);
+        }
+        @l = read_dir($file);
+        rmdir $file unless @l;
+    } else {
+        print "Removing file $file\n";
+        unlink $file unless $options{'dry-run'};
+    }
+}
+
+sub clean_docker_images {
+    my ($dockerdir, $used_files) = @_;
+    my $imgprefix = RBM::project_config('docker-image', 'docker_image_prefix');
+    my @imgs = read_dir($dockerdir);
+    foreach my $dockerimage (@imgs) {
+        next if $used_files->{"$dockerdir/$dockerimage"};
+        my $img = "$imgprefix:$dockerimage";
+        print "Cleaning docker image $img\n";
+        next if $options{'dry-run'};
+        my ($out, $err, $success) = capture_exec('docker', 'rmi', '-f', $img);
+        if (!$success) {
+            print STDERR "Error removing docker image $img:\n$err\n";
+            exit 1;
+        }
+        unlink "$dockerdir/$dockerimage";
+    }
+}
+
+sub get_project_input_files {
+    my ($project, @targets) = @_;
+    print "Getting input files for $project ", join(' ', @targets), "\n";
+    $RBM::config->{run}{target} = \@targets;
+    my $res = RBM::project_config($project, 'input_files_paths',
+                                        {error_if_undef => 1});
+    return ref $res eq 'ARRAY' ? @$res : ();
+}
+
+sub get_input_files {
+    my ($c) = @_;
+    my @files;
+    foreach my $p (@$c) {
+        my $tmp = File::Temp->new();
+        my @args = ("$FindBin::Bin/$FindBin::Script", '--list-used-files', $tmp,
+                    '--project', $p->{project});
+        foreach my $target (@{$p->{target}}) {
+            push @args, ('--target', $target);
+        }
+        RBM::exit_error 'Error getting used files' unless system(@args) == 0;
+        push @files, map { chomp; $_ } read_file($tmp);
+    }
+    return @files;
+}
+
+RBM::load_config;
+RBM::load_system_config;
+RBM::load_local_config;
+RBM::set_default_env;
+
+if ($options{'list-used-files'}) {
+    if (!$options{project} || !$options{target}) {
+        RBM::exit_error 'Missing option project or target';
+    }
+    my @files = get_project_input_files($options{project}, @{$options{target}});
+    write_file($options{'list-used-files'}, join("\n", @files));
+    exit 0;
+}
+
+my $clean = RBM::project_config('clean', 'var/clean');
+if (!$clean) {
+    print STDERR "Clean configuration is missing. ",
+                 "You should add it to rbm.local.conf.\n";
+    exit 1;
+}
+
+my @files = get_input_files($clean->{HEAD}) if $clean->{HEAD};
+
+foreach my $branch (keys %$clean) {
+    next if $branch eq 'HEAD';
+    print "Checking out $branch branch\n";
+    RBM::exit_error("Error checking out $branch branch")
+        unless system('git', 'checkout', $branch) == 0;
+    RBM::exit_error("Error running git submodule update --init")
+        unless system('git', 'submodule', 'update', '--init') == 0;
+    push @files, get_input_files($clean->{$branch});
+    RBM::exit_error('Error checking out @{-1}')
+        unless system('git', 'checkout', '@{-1}') == 0;
+    RBM::exit_error("Error running git submodule update --init")
+        unless system('git', 'submodule', 'update', '--init') == 0;
+}
+my %used_files = map { $_ => 1 } @files;
+my $outdir = $RBM::config->{basedir} . '/out';
+# Don't clean docker-image files yet
+$used_files{"$outdir/docker-image"} = 1;
+clean_file($outdir, \%used_files);
+clean_docker_images("$outdir/docker-image", \%used_files);

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits