dm6: Detect when rename fails to clean up $tmp_to

[dupemerge] / dm6

diff --git a/dm6 b/dm6
index a5d56a6545a59258d2742e7dcfcedd0a4be5d843..8638e28c55e8d76e9772dde939a22d60af98616e 100755 (executable)
--- a/dm6
+++ b/dm6
@@ -7,8 +7,9 @@ use File::Compare;
 use File::Path;
 use File::Temp;
 use File::stat;
+use MIME::Base64;
 
-# Copyright (C) 2010 Zygo Blaxell <dm5@mailtoo.hungrycats.org>
+# Copyright (C) 2010 Zygo Blaxell <dupemerge@mailtoo.hungrycats.org>
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -40,7 +41,7 @@ sub usage {
 Usage: $0 link-dir
 Hashes a NUL-separated list of files on stdin into link-dir.
 
-Version: 20100513.0
+Version: 0.20100514
 USAGE
 }
 
@@ -57,39 +58,70 @@ sub link_files {
        print STDERR "\bL";
        link($from, $tmp_to) or die "link: $from -> $tmp_to: $!";
        print STDERR "\bR";
-       unless (rename($tmp_to, $to)) {
-               my $saved_bang = $!;
-               print STDERR "\bU";
-               unlink($tmp_to) or warn "unlink: $tmp_to: $!";  # Try, possibly in vain, to clean up
-               die "rename: $tmp_to -> $from: $saved_bang";
-       }
-       print STDERR "\b \b";
+       my $saved_bang;
+       $saved_bang = $! unless rename($tmp_to, $to);
+
+       # If $to exists and is a hardlink to $tmp_to (or $from),
+       # rename returns success but $tmp_to still exists.
+       print STDERR "\bU";
+       unlink($tmp_to) or warn "unlink: $tmp_to: $!" if -e $tmp_to;
+
+       die "rename: $tmp_to -> $from: $saved_bang" if $saved_bang;
+       print STDERR "\b";
 }
 
 my $link_dir = shift @ARGV;
-(-d $link_dir) or usage;
+usage unless $link_dir;
+
+my $prefix_length = 3;
 
 sub slash_prefix {
        my ($file) = @_;
-       my $prefix = substr($file, 0, 3);
-       my $suffix = substr($file, 3);
+       $file .= '_' x (length($file) + 1 - $prefix_length) if length($file) + 1 < $prefix_length;
+       my $prefix = substr($file, 0, $prefix_length);
+       my $suffix = substr($file, $prefix_length);
        $prefix =~ s:(.):$1/:osg;
        chop($prefix);
        return ($prefix, $suffix);
 }
 
+sub mkdir_p {
+       my ($dir) = @_;
+       return if -d $dir;
+       $dir =~ s:/+$::os;
+       my $parent;
+       ($parent = $dir) =~ s:[^/]+$::os;
+       if ($parent ne $dir) {
+               mkdir_p($parent);
+               print STDERR 'm';
+               mkdir($dir) or die "mkdir: $dir: $!";
+       }
+       die "mkdir: $dir: $!" unless -d $dir;
+}
+
 sub prepare_parents {
        my ($link_dir, $file) = @_;
        my ($prefix, $suffix) = slash_prefix($file);
        my $parent = "$link_dir/$prefix";
-       print STDERR 'm';
-       mkpath($parent, { verbose => 0 });
-       print STDERR "\b";
-       die "mkpath: $parent: $!" unless -d $parent;
+       mkdir_p($parent);
        return "$parent/$suffix";
 }
 
+sub name_ino {
+       my ($int64) = @_;
+       my $packed = pack('Q>', $int64);
+       $packed =~ s/^\0+//os;
+       my $base64_packed = encode_base64($packed, '');
+       $base64_packed =~ y:/:_:;
+       # Don't strip off the trailing padding since it makes the string
+       # so short we end up just putting it back on again.
+       # $base64_packed =~ s/=+$//os;
+       return $base64_packed;
+}
+
 # ext3 cannot handle more than 32000 links to a file.  Leave some headroom.
+# Arguably this should be configurable, but the losses are miniscule and
+# the coding for option support is not.
 my $link_count_max = 31990;
 
 $/ = "\0";
@@ -111,7 +143,7 @@ while (<STDIN>) {
                        next if ($st->nlink > $link_count_max);
 
                        # Check link to inode
-                       my $inode_link = prepare_parents("$link_dir/inode", $st->ino);
+                       my $inode_link = prepare_parents($link_dir, name_ino($st->ino));
                        print STDERR 'I';
                        my $inode_st = lstat($inode_link);
                        my $update_links;
@@ -142,7 +174,7 @@ while (<STDIN>) {
                        $digest =~ y:/:_:;
 
                        # Check link to digest
-                       my $digest_link = prepare_parents("$link_dir/digest", $digest);
+                       my $digest_link = prepare_parents($link_dir, $digest);
                        print STDERR 'D';
                        my $digest_st = lstat($digest_link);
                        if ($digest_st) {
@@ -153,6 +185,10 @@ while (<STDIN>) {
                                        undef $digest_st;
                                }
                        }
+                       print STDERR "\b";
+
+                       # Which file are we keeping?
+                       my $keep_ino;
 
                        # If digest link exists, link it to file
                        if ($digest_st) {
@@ -162,20 +198,33 @@ while (<STDIN>) {
                                # Old, replace input with old file
                                print STDERR '-';
                                link_files($digest_link, $file);
+                               $keep_ino = $digest_st->ino;
                        } else {
                                # New, add input to digest
                                print STDERR '+';
                                link_files($file, $digest_link);
+                               $keep_ino = $st->ino;
                        }
 
                        # A link to the inode indicates we are done, so do it last
-                       link_files($file, $inode_link);
+                       $inode_link = prepare_parents($link_dir, name_ino($keep_ino));
+                       print STDERR ' ';
+                       link_files($digest_link, $inode_link);
 
                }
        };
        warn "$file: $@" if $@;
 }
 
+# Garbage collection
+print STDERR "\nGarbage collection in '$link_dir'...";
+chdir($link_dir) || die "chdir: $link_dir: $!";
+print STDERR "\nRemoving files with link count < 3...";
+system("find . -type f -links -3 -print0 | xargs -0rt rm -f") and die "system: exit status $?";
+print STDERR "\nRemoving empty directories...";
+system("find . -type d -empty -print0 | xargs -0rt rmdir -p --ignore-fail-on-non-empty") and die "system: exit status $?";
+print STDERR "\nDone.\n";
+
 exit(0);
 
 __END__