X-Git-Url: https://git.madduck.net/code/myrepos.git/blobdiff_plain/1610ff290fb8fd39be43f5063163751a36e17f1e..e7a82f206d14cbaad72df80de8844feadbe5a24b:/mr diff --git a/mr b/mr index a3d3a45..f5c37af 100755 --- a/mr +++ b/mr @@ -30,9 +30,9 @@ B [options] action [params ...] B is a Multiple Repository management tool. It can checkout, update, or perform other actions on a set of repositories as if they were one combined -respository. It supports any combination of subversion, git, cvs, mecurial and -bzr repositories, and support for other revision control systems can easily be -added. +respository. It supports any combination of subversion, git, cvs, mecurial, +bzr and darcs repositories, and support for other revision control systems can +easily be added. B cds into and operates on all registered repositories at or below your working directory. Or, if you are in a subdirectory of a repository that @@ -157,6 +157,12 @@ about exactly which repositories failed and were skipped, if any. Just operate on the repository for the current directory, do not recurse into deeper repositories. +=item -j number + +Run the specified number of jobs in parallel. This can greatly speed up +operations such as updates. It is not recommended for interactive +operations. + =back =head1 FILES @@ -223,7 +229,17 @@ mr is run by joey. The second uses the hours_since function been at least 12 hours since the last update. skip = test $(whoami) != joey - skip = [ "$1" = update ] && [ $(hours_since "$1") -lt 12 ] + skip = [ "$1" = update ] && ! hours_since "$1" 12 + +=item order + +The "order" parameter can be used to override the default ordering of +repositories. The default order value is 10. Use smaller values to make +repositories be processed earlier, and larger values to make repositories +be processed later. + +Note that if a repository is located in a subdirectory of another +repository, ordering it to be processed earlier is not recommended. =item chain @@ -255,6 +271,13 @@ use warnings; use strict; use Getopt::Long; use Cwd qw(getcwd abs_path); +use POSIX "WNOHANG"; +use constant { + OK => 0, + FAILED => 1, + SKIPPED => 2, + ABORT => 3, +}; $SIG{INT}=sub { print STDERR "mr: interrupted\n"; @@ -263,14 +286,15 @@ $SIG{INT}=sub { $ENV{MR_CONFIG}="$ENV{HOME}/.mrconfig"; my $config_overridden=0; -my $directory=getcwd(); my $verbose=0; my $stats=0; my $no_recurse=0; +my $jobs=1; my %config; my %configfiles; my %knownactions; my %alias; +my $directory=getcwd(); Getopt::Long::Configure("no_permute"); my $result=GetOptions( @@ -279,12 +303,16 @@ my $result=GetOptions( "v|verbose" => \$verbose, "s|stats" => \$stats, "n|no-recurse" => \$no_recurse, + "j|jobs=i" => \$jobs, ); if (! $result || @ARGV < 1) { die("Usage: mr [-d directory] action [params ...]\n". "(Use mr help for man page.)\n"); } +if (! defined $directory) { + die("mr: failed to determine working directory\n"); +} # Make sure MR_CONFIG is an absolute path, but don't use abs_path since # the config file might be a symlink to elsewhere, and the directory it's @@ -292,17 +320,17 @@ if (! $result || @ARGV < 1) { if ($ENV{MR_CONFIG} !~ /^\//) { $ENV{MR_CONFIG}=getcwd()."/".$ENV{MR_CONFIG}; } +# Try to set MR_PATH to the path to the program. +eval { + use FindBin qw($Bin $Script); + $ENV{MR_PATH}=$Bin."/".$Script; +}; loadconfig(\*DATA); loadconfig($ENV{MR_CONFIG}); #use Data::Dumper; #print Dumper(\%config); -eval { - use FindBin qw($Bin $Script); - $ENV{MR_PATH}=$Bin."/".$Script; -}; - # alias expansion and command stemming my $action=shift @ARGV; if (exists $alias{$action}) { @@ -385,43 +413,93 @@ elsif ($action eq 'register') { exec($command) || die "exec: $!"; } +# an ordered list of repos +my @list; +foreach my $topdir (sort keys %config) { + foreach my $subdir (sort keys %{$config{$topdir}}) { + push @list, { + topdir => $topdir, + subdir => $subdir, + order => $config{$topdir}{$subdir}{order}, + }; + } +} +@list = sort { + $a->{order} <=> $b->{order} + || + $a->{topdir} cmp $b->{topdir} + || + $a->{subdir} cmp $b->{subdir} + } @list; + # work out what repos to act on my @repos; my $nochdir=0; -foreach my $topdir (sort keys %config) { - foreach my $subdir (sort keys %{$config{$topdir}}) { +foreach my $repo (@list) { + my $topdir=$repo->{topdir}; + my $subdir=$repo->{subdir}; + + next if $subdir eq 'DEFAULT'; + my $dir=($subdir =~/^\//) ? $subdir : $topdir.$subdir; + my $d=$directory; + $dir.="/" unless $dir=~/\/$/; + $d.="/" unless $d=~/\/$/; + next if $no_recurse && $d ne $dir; + next if $dir ne $d && $dir !~ /^\Q$d\E/; + push @repos, [$dir, $topdir, $subdir]; +} +if (! @repos) { + # fallback to find a leaf repo + foreach my $repo (reverse @list) { + my $topdir=$repo->{topdir}; + my $subdir=$repo->{subdir}; + next if $subdir eq 'DEFAULT'; my $dir=($subdir =~/^\//) ? $subdir : $topdir.$subdir; my $d=$directory; $dir.="/" unless $dir=~/\/$/; $d.="/" unless $d=~/\/$/; - next if $no_recurse && $d ne $dir; - next if $dir ne $d && $dir !~ /^\Q$d\E/; - push @repos, [$dir, $topdir, $subdir]; - } -} -if (! @repos) { - # fallback to find a leaf repo - LEAF: foreach my $topdir (reverse sort keys %config) { - foreach my $subdir (reverse sort keys %{$config{$topdir}}) { - next if $subdir eq 'DEFAULT'; - my $dir=($subdir =~/^\//) ? $subdir : $topdir.$subdir; - my $d=$directory; - $dir.="/" unless $dir=~/\/$/; - $d.="/" unless $d=~/\/$/; - if ($d=~/^\Q$dir\E/) { - push @repos, [$dir, $topdir, $subdir]; - last LEAF; - } + if ($d=~/^\Q$dir\E/) { + push @repos, [$dir, $topdir, $subdir]; + last; } } $nochdir=1; } -my (@failed, @ok, @skipped); -foreach my $repo (@repos) { - action($action, @$repo); +# run the action on each repository and print stats +my (@ok, @failed, @skipped); +if ($jobs > 1) { + mrs(@repos); } +else { + foreach my $repo (@repos) { + record($repo, action($action, @$repo)); + } +} +if (! @ok && ! @failed && ! @skipped) { + die "mr $action: no repositories found to work on\n"; +} +print "mr $action: finished (".join("; ", + showstat($#ok+1, "ok", "ok"), + showstat($#failed+1, "failed", "failed"), + showstat($#skipped+1, "skipped", "skipped"), +).")\n"; +if ($stats) { + if (@skipped) { + print "mr $action: (skipped: ".join(" ", @skipped).")\n"; + } + if (@failed) { + print STDERR "mr $action: (failed: ".join(" ", @failed).")\n"; + } +} +if (@failed) { + exit 1; +} +elsif (! @ok && @skipped) { + exit 1; +} +exit 0; sub action { #{{{ my ($action, $dir, $topdir, $subdir) = @_; @@ -433,18 +511,17 @@ sub action { #{{{ if ($action eq 'checkout') { if (-d $dir) { print "mr $action: $dir already exists, skipping checkout\n" if $verbose; - push @skipped, $dir; - return; + return SKIPPED; } $dir=~s/^(.*)\/[^\/]+\/?$/$1/; if (! -d $dir) { print "mr $action: creating parent directory $dir\n" if $verbose; - my $ret=system("mkdir", "-p", $dir); + system("mkdir", "-p", $dir); } } - elsif ($action eq 'update') { + elsif ($action =~ /update/) { if (! -d $dir) { return action("checkout", $dir, $topdir, $subdir); } @@ -460,34 +537,35 @@ sub action { #{{{ if ($ret != 0) { if (($? & 127) == 2) { print STDERR "mr $action: interrupted\n"; - exit 2; + return ABORT; } elsif ($? & 127) { print STDERR "mr $action: skip test received signal ".($? & 127)."\n"; - exit 1; + return ABORT; } } if ($ret >> 8 == 0) { print "mr $action: $dir skipped per config file\n" if $verbose; - push @skipped, $dir; - return; + return SKIPPED; } } if (! $nochdir && ! chdir($dir)) { print STDERR "mr $action: failed to chdir to $dir: $!\n"; - push @failed, $dir; + return FAILED; } elsif (! exists $config{$topdir}{$subdir}{$action}) { print STDERR "mr $action: no defined $action command for $topdir$subdir, skipping\n"; - push @skipped, $dir; + return SKIPPED; } else { if (! $nochdir) { print "mr $action: $topdir$subdir\n"; } else { - print "mr $action: $topdir$subdir (in subdir $directory)\n"; + my $s=$directory; + $s=~s/^\Q$topdir$subdir\E\/?//; + print "mr $action: $topdir$subdir (in subdir $s)\n"; } my $command="set -e; ".$lib. "my_action(){ $config{$topdir}{$subdir}{$action}\n }; my_action ". @@ -497,32 +575,120 @@ sub action { #{{{ if ($ret != 0) { if (($? & 127) == 2) { print STDERR "mr $action: interrupted\n"; - exit 2; + return ABORT; } elsif ($? & 127) { print STDERR "mr $action: received signal ".($? & 127)."\n"; + return ABORT; } print STDERR "mr $action: failed ($ret)\n" if $verbose; - push @failed, $dir; if ($ret >> 8 != 0) { print STDERR "mr $action: command failed\n"; } elsif ($ret != 0) { print STDERR "mr $action: command died ($ret)\n"; } + return FAILED; } else { if ($action eq 'checkout' && ! -d $dir) { print STDERR "mr $action: $dir missing after checkout\n";; - push @failed, $dir; - return; + return FAILED; } - push @ok, $dir; + return OK; } + } +} #}}} +# run actions on multiple repos, in parallel +sub mrs { #{{{ + $| = 1; + my @active; + my @fhs; + my @out; + my $running=0; + while (@fhs or @repos) { + while ($running < $jobs && @repos) { + $running++; + my $repo = shift @repos; + pipe(my $outfh, CHILD_STDOUT); + pipe(my $errfh, CHILD_STDERR); + my $pid; + unless ($pid = fork) { + die "mr $action: cannot fork: $!" unless defined $pid; + open(STDOUT, ">&CHILD_STDOUT") || die "mr $action cannot reopen stdout: $!"; + open(STDERR, ">&CHILD_STDERR") || die "mr $action cannot reopen stderr: $!"; + close CHILD_STDOUT; + close CHILD_STDERR; + close $outfh; + close $errfh; + exit action($action, @$repo); + } + close CHILD_STDOUT; + close CHILD_STDERR; + push @active, [$pid, $repo]; + push @fhs, [$outfh, $errfh]; + push @out, ['', '']; + } + my ($rin, $rout) = ('',''); + my $nfound; + foreach my $fh (@fhs) { + next unless defined $fh; + vec($rin, fileno($fh->[0]), 1) = 1 if defined $fh->[0]; + vec($rin, fileno($fh->[1]), 1) = 1 if defined $fh->[1]; + } + $nfound = select($rout=$rin, undef, undef, 1); + foreach my $channel (0, 1) { + foreach my $i (0..$#fhs) { + next unless defined $fhs[$i]; + my $fh = $fhs[$i][$channel]; + next unless defined $fh; + if (vec($rout, fileno($fh), 1) == 1) { + my $r = ''; + if (sysread($fh, $r, 1024) == 0) { + close($fh); + $fhs[$i][$channel] = undef; + if (! defined $fhs[$i][0] && + ! defined $fhs[$i][1]) { + waitpid($active[$i][0], 0); + print STDOUT $out[$i][0]; + print STDERR $out[$i][1]; + record($active[$i][1], $? >> 8); + splice(@fhs, $i, 1); + splice(@active, $i, 1); + splice(@out, $i, 1); + $running--; + } + } + $out[$i][$channel] .= $r; + } + } + } + } +} #}}} + +sub record { #{{{ + my $dir=shift()->[0]; + my $ret=shift; + + if ($ret == OK) { + push @ok, $dir; print "\n"; } + elsif ($ret == FAILED) { + push @failed, $dir; + print "\n"; + } + elsif ($ret == SKIPPED) { + push @skipped, $dir; + } + elsif ($ret == ABORT) { + exit 1; + } + else { + die "unknown exit status $ret"; + } } #}}} sub showstat { #{{{ @@ -534,29 +700,6 @@ sub showstat { #{{{ } return; } #}}} -if (! @ok && ! @failed && ! @skipped) { - die "mr $action: no repositories found to work on\n"; -} -print "mr $action: finished (".join("; ", - showstat($#ok+1, "ok", "ok"), - showstat($#failed+1, "failed", "failed"), - showstat($#skipped+1, "skipped", "skipped"), -).")\n"; -if ($stats) { - if (@skipped) { - print "mr $action: (skipped: ".join(" ", @skipped).")\n"; - } - if (@failed) { - print STDERR "mr $action: (failed: ".join(" ", @failed).")\n"; - } -} -if (@failed) { - exit 1; -} -elsif (! @ok && @skipped) { - exit 1; -} -exit 0; my %loaded; sub loadconfig { #{{{ @@ -786,13 +929,24 @@ ci = commit ls = list [DEFAULT] +order = 10 lib = + PWD="$pwd" error() { echo "mr: $@" >&2 exit 1 } + warning() { + echo "mr (warning): $@" >&2 + } + info() { + echo "mr: $@" >&2 + } hours_since() { - for dir in .git .svn .bzr CVS .hg; do + if [ -z "$1" ] || [ -z "$2" ]; then + error "mr: usage: hours_since action num" + fi + for dir in .git .svn .bzr CVS .hg _darcs; do if [ -e "$MR_REPO/$dir" ]; then flagfile="$MR_REPO/$dir/.mr_last$1" break @@ -801,127 +955,205 @@ lib = if [ -z "$flagfile" ]; then error "cannot determine flag filename" fi - perl -wle 'print -f shift() ? int((-M _) * 24) : 9999' "$flagfile" - touch "$flagfile" + delta=$(perl -wle 'print -f shift() ? int((-M _) * 24) : 9999' "$flagfile") + if [ "$delta" -lt "$2" ]; then + exit 0 + else + touch "$flagfile" + exit 1 + fi } - -update = - if [ -d "$MR_REPO"/.svn ]; then - svn update "$@" - elif [ -d "$MR_REPO"/.git ]; then - if [ -z "$@" ]; then - git pull -t origin master + get_git_repo_type() + { + if [ -d "$1"/.git ] && [ -d "$1"/.git/refs/heads ] && + [ -d "$1"/.git/objects ] && [ -f "$1"/.git/config ]; + then + echo non-bare + elif [ -d "$1"/refs/heads ] && [ -d "$1"/refs/tags ] && + [ -d "$1"/objects ] && [ -f "$1"/config ]; then + local bare + bare="$(GIT_CONFIG="$1"/config git-config --get core.bare)" + case "$bare" in + true) echo bare;; + false) echo fake-bare;; + *) return 255;; + esac else - git pull "$@" + return 1 fi - elif [ -d "$MR_REPO"/.bzr ]; then - bzr merge "$@" - elif [ -d "$MR_REPO"/CVS ]; then - cvs update "$@" - elif [ -d "$MR_REPO"/.hg ]; then - hg pull "$@" && hg update "$@" - else - error "unknown repo type" - fi + } + is_git_repo() { + get_git_repo_type "$1" >/dev/null + } + get_repo_type() { + if [ -d "$1"/.svn ]; then + echo svn + elif is_git_repo "$1"; then + echo git + elif [ -d "$1"/.bzr ]; then + echo bzr + elif [ -d "$1"/CVS ]; then + echo CVS + elif [ -d "$1"/.hg ]; then + echo hg + elif [ -d "$1"/_darcs ]; then + echo darcs + else + echo unknown + fi + } + +update = + case "$(get_repo_type "$MR_REPO")" in + svn) svn update "$@";; + git) + # all this is because of a bug in git-fetch, which requires GIT_DIR set + local git_dir_override; git_dir_override=.git + case "$(get_git_repo_type "$MR_REPO")" in + fake-bare) git_dir_override="$MR_REPO";; + esac + args="$@" + [ -z "$args" ] && args="-t origin master" + eval GIT_DIR="$git_dir_override" git pull "$args" + ;; + bzr) bzr merge "$@";; + CVS) cvs update "$@";; + hg) hg pull "$@" && hg update "$@";; + darcs) darcs pull -a "$@";; + *) error "unknown repo type";; + esac + status = - if [ -d "$MR_REPO"/.svn ]; then - svn status "$@" - elif [ -d "$MR_REPO"/.git ]; then - git status "$@" || true - elif [ -d "$MR_REPO"/.bzr ]; then - bzr status "$@" - elif [ -d "$MR_REPO"/CVS ]; then - cvs status "$@" - elif [ -d "$MR_REPO"/.hg ]; then - hg status "$@" - else - error "unknown repo type" - fi + case "$(get_repo_type "$MR_REPO")" in + svn) svn status "$@";; + git) git status "$@" || :;; + bzr) bzr status "$@";; + CVS) cvs status "$@";; + hg) hg status "$@";; + darcs) darcs whatsnew -ls "$@";; + *) error "unknown repo type";; + esac + commit = - if [ -d "$MR_REPO"/.svn ]; then - svn commit "$@" - elif [ -d "$MR_REPO"/.git ]; then + case "$(get_repo_type "$MR_REPO")" in + svn) svn commit "$@";; + git) + case "$(get_git_repo_type "$MR_REPO")" in + bare) error "cannot commit to bare git repositories";; + fake-bare) error "commit does not work for fake bare git repositories (yet).";; + esac git commit -a "$@" && git push --all - elif [ -d "$MR_REPO"/.bzr ]; then - bzr commit "$@" && bzr push - elif [ -d "$MR_REPO"/CVS ]; then - cvs commit "$@" - elif [ -d "$MR_REPO"/.hg ]; then - hg commit -m "$@" && hg push - else - error "unknown repo type" - fi + ;; + bzr) bzr commit "$@" && bzr push;; + CVS) cvs commit "$@";; + hg) hg commit -m "$@" && hg push;; + darcs) darcs commit -a -m "$@" && darcs push -a;; + *) error "unknown repo type";; + esac + diff = - if [ -d "$MR_REPO"/.svn ]; then - svn diff "$@" - elif [ -d "$MR_REPO"/.git ]; then + case "$(get_repo_type "$MR_REPO")" in + svn) svn diff "$@";; + git) + case "$(get_git_repo_type "$MR_REPO")" in + bare) error "cannot diff in bare git repositories";; + fake-bare) error "diff does not work for fake bare git repositories (yet).";; + esac git diff "$@" - elif [ -d "$MR_REPO"/.bzr ]; then - bzr diff "$@" - elif [ -d "$MR_REPO"/CVS ]; then - cvs diff "$@" - elif [ -d "$MR_REPO"/.hg ]; then - hg diff "$@" - else - error "unknown repo type" - fi + ;; + bzr) bzr diff "$@";; + CVS) cvs diff "$@";; + hg) hg diff "$@";; + darcs) darcs diff "$@";; + *) error "unknown repo type";; + esac + log = - if [ -d "$MR_REPO"/.svn ]; then - svn log"$@" - elif [ -d "$MR_REPO"/.git ]; then - git log "$@" - elif [ -d "$MR_REPO"/.bzr ]; then - bzr log "$@" - elif [ -d "$MR_REPO"/CVS ]; then - cvs log "$@" - elif [ -d "$MR_REPO"/.hg ]; then - hg log "$@" - else - error "unknown repo type" - fi + case "$(get_repo_type "$MR_REPO")" in + svn) svn log"$@";; + git) git log "$@";; + bzr) bzr log "$@";; + CVS) cvs log "$@";; + hg) hg log "$@";; + darcs) darcs changes "$@";; + *) error "unknown repo type";; + esac + register = if [ -n "$1" ]; then cd "$1" fi - basedir="$(basename $(pwd))" - if [ -d .svn ]; then + basedir="${PWD##*/}" + case "$(get_repo_type .)" in + svn) url=$(LANG=C svn info . | grep -i ^URL: | cut -d ' ' -f 2) if [ -z "$url" ]; then error "cannot determine svn url" fi echo "Registering svn url: $url in $MR_CONFIG" - mr -c "$MR_CONFIG" config "$(pwd)" checkout="svn co $url $basedir" - elif [ -d .git ]; then - url=$(LANG=C git-config --get remote.origin.url) + mr -c "$MR_CONFIG" config "$PWD" checkout="svn co $url $basedir" + ;; + git) + local repo_type; repo_type="$(get_git_repo_type .)" + local config; + case "$repo_type" in + non-bare) config=.git/config;; + bare|fake-bare) config=config;; + esac + url="$(LANG=C GIT_CONFIG="$config" git-config --get remote.origin.url)" || : if [ -z "$url" ]; then error "cannot determine git url" fi - echo "Registering git url: $url in $MR_CONFIG" - mr -c "$MR_CONFIG" config "$(pwd)" checkout="git clone $url $basedir" - elif [ -d .bzr ]; then + case "$repo_type" in + bare|fake-bare) + # this seems like a bare repo as it has no + # worktree. + local work_tree + work_tree="$(git-config --get core.worktree)" || : + if [ ! -d "$work_tree" ]; then + error "git worktree $work_tree does not exist" + fi + suffix=" (with worktree $work_tree)" + mr -c "$MR_CONFIG" config "$PWD" \ + lib="GIT_WORK_TREE=$work_tree; export GIT_WORK_TREE" + ;; + esac + echo "Registering git url: $url in $MR_CONFIG${suffix:-}" + mr -c "$MR_CONFIG" config "$PWD" checkout="git clone $url $basedir" + ;; + bzr) url=$(cat .bzr/branch/parent) if [ -z "$url" ]; then error "cannot determine bzr url" fi echo "Registering bzr url: $url in $MR_CONFIG" - mr -c "$MR_CONFIG" config "$(pwd)" checkout="bzr clone $url $basedir" - elif [ -d CVS ]; then + mr -c "$MR_CONFIG" config "$PWD" checkout="bzr clone $url $basedir" + ;; + CVS) repo=$(cat CVS/Repository) root=$(cat CVS/Root) if [ -z "$root" ]; then error "cannot determine cvs root" fi echo "Registering cvs repository $repo at root $root" - mr -c "$MR_CONFIG" config "$(pwd)" \ + mr -c "$MR_CONFIG" config "$PWD" \ checkout="cvs -d '$root' co -d $basedir $repo" - elif [ -d .hg ]; then + ;; + hg) url=$(hg showconfig paths.default) echo "Registering mercurial repo url: $url in $MR_CONFIG" - mr -c "$MR_CONFIG" config "$(pwd)" \ + mr -c "$MR_CONFIG" config "$PWD" \ checkout="hg clone $url $basedir" - else - error "unable to register this repo type" - fi + ;; + darcs) + url=$(cat _darcs/prefs/defaultrepo) + echo "Registering darcs repository $url in $MR_CONFIG" + mr -c "$MR_CONFIG" config "$PWD" \ + checkout="darcs get $url $basedir" + ;; + *) error "unable to register this repo type";; + esac + help = if [ ! -e "$MR_PATH" ]; then error "cannot find program path" @@ -934,3 +1166,5 @@ ed = echo "A horse is a horse, of course, of course.." T = echo "I pity the fool." right = echo "Not found." #}}} + +# vim:sw=8:sts=0:ts=8:noet