X-Git-Url: http://git.hungrycats.org/cgi-bin/gitweb.cgi?p=dupemerge;a=blobdiff_plain;f=dm6;h=a3c7b9ea1541662ef0f5825b21a8f599b2d5bb58;hp=26546983bccc5b0b1e1b3ae97e584df831163f85;hb=4f5affec900edd9b4b9acc4c390bfed4c91fb248;hpb=181753a22ed5f794f52bc049be170dbdccc9e845

diff --git a/dm6 b/dm6
index 2654698..a3c7b9e 100755
--- a/dm6
+++ b/dm6
@@ -1,14 +1,15 @@
 #!/usr/bin/perl
 use warnings;
 use strict;
-use Digest::SHA1 qw(sha1 sha1_hex sha1_base64);
+use Digest::SHA qw(sha1 sha1_hex sha1_base64);
 use Fcntl qw(:DEFAULT :flock);
 use File::Compare;
 use File::Path;
 use File::Temp;
 use File::stat;
+use MIME::Base64;
 
-# Copyright (C) 2010 Zygo Blaxell <dm5@mailtoo.hungrycats.org>
+# Copyright (C) 2010-2012 Zygo Blaxell <dupemerge@mailtoo.hungrycats.org>
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -27,7 +28,7 @@ use File::stat;
 sub digest {
 	my ($filename) = (@_);
 	die "'$filename' is not a plain file" if (-l $filename) || ! (-f _);
-	my $ctx = Digest::SHA1->new;
+	my $ctx = Digest::SHA->new;
 	sysopen(FILE, $filename, O_RDONLY|O_NONBLOCK) or die "open: $filename: $!";
 	binmode(FILE);		# FIXME:  Necessary?  Probably harmless...
 	$ctx->addfile(\*FILE);
@@ -40,7 +41,7 @@ sub usage {
 Usage: $0 link-dir
 Hashes a NUL-separated list of files on stdin into link-dir.
 
-Version: 20100513.0
+Version: 0.20120914
 USAGE
 }
 
@@ -57,17 +58,20 @@ sub link_files {
 	print STDERR "\bL";
 	link($from, $tmp_to) or die "link: $from -> $tmp_to: $!";
 	print STDERR "\bR";
-	unless (rename($tmp_to, $to)) {
-		my $saved_bang = $!;
-		print STDERR "\bU";
-		unlink($tmp_to) or warn "unlink: $tmp_to: $!";	# Try, possibly in vain, to clean up
-		die "rename: $tmp_to -> $from: $saved_bang";
-	}
+	my $saved_bang;
+	$saved_bang = $! unless rename($tmp_to, $to);
+
+	# If $to exists and is a hardlink to $tmp_to (or $from),
+	# rename returns success but $tmp_to still exists.
+	print STDERR "\bU";
+	unlink($tmp_to) or warn "unlink: $tmp_to: $!" if -e $tmp_to;
+
+	die "rename: $tmp_to -> $from: $saved_bang" if $saved_bang;
 	print STDERR "\b";
 }
 
 my $link_dir = shift @ARGV;
-(-d $link_dir) or usage;
+usage unless $link_dir;
 
 my $prefix_length = 3;
 
@@ -103,7 +107,21 @@ sub prepare_parents {
 	return "$parent/$suffix";
 }
 
+sub name_ino {
+	my ($int64) = @_;
+	my $packed = pack('Q>', $int64);
+	$packed =~ s/^\0+//os;
+	my $base64_packed = encode_base64($packed, '');
+	$base64_packed =~ y:/:_:;
+	# Don't strip off the trailing padding since it makes the string
+	# so short we end up just putting it back on again.
+	# $base64_packed =~ s/=+$//os;
+	return $base64_packed;
+}
+
 # ext3 cannot handle more than 32000 links to a file.  Leave some headroom.
+# Arguably this should be configurable, but the losses are miniscule and
+# the coding for option support is not.
 my $link_count_max = 31990;
 
 $/ = "\0";
@@ -125,7 +143,7 @@ while (<STDIN>) {
 			next if ($st->nlink > $link_count_max);
 
 			# Check link to inode
-			my $inode_link = prepare_parents("$link_dir/inode", $st->ino);
+			my $inode_link = prepare_parents($link_dir, name_ino($st->ino));
 			print STDERR 'I';
 			my $inode_st = lstat($inode_link);
 			my $update_links;
@@ -150,13 +168,12 @@ while (<STDIN>) {
 			# Compute digest
 			print STDERR 'd';
 			my $digest = digest($file);
-			print STDERR "\b";
 
 			# Base64 uses /, we prefer _
 			$digest =~ y:/:_:;
 
 			# Check link to digest
-			my $digest_link = prepare_parents("$link_dir/digest", $digest);
+			my $digest_link = prepare_parents($link_dir, $digest);
 			print STDERR 'D';
 			my $digest_st = lstat($digest_link);
 			if ($digest_st) {
@@ -189,9 +206,9 @@ while (<STDIN>) {
 			}
 
 			# A link to the inode indicates we are done, so do it last
-			my $keep_link = prepare_parents("$link_dir/inode", $keep_ino);
-			print STDERR '_';
-			link_files($keep_link, $inode_link);
+			$inode_link = prepare_parents($link_dir, name_ino($keep_ino));
+			print STDERR ' ';
+			link_files($digest_link, $inode_link);
 
 		}
 	};
@@ -199,13 +216,44 @@ while (<STDIN>) {
 }
 
 # Garbage collection
-print STDERR "\nGarbage collection in '$link_dir'...";
+print STDERR "\nGarbage collection in '$link_dir'...\n";
 chdir($link_dir) || die "chdir: $link_dir: $!";
-print STDERR "\nRemoving files with link count < 3...";
-system("find digest inode -type f -links -3 -print0 | xargs -0 rm -f") and die "system: exit status $?";
-print STDERR "\nRemoving empty directories...";
-system("find digest inode -type d -empty -print0 | xargs -0r rmdir -p --ignore-fail-on-non-empty") and die "system: exit status $?";
-print STDERR "\nDone.\n";
+
+my ($last_inode) = '';
+my @last_links;
+
+sub handle_gc_file {
+	my ($line) = @_;
+	my ($inode, $link) = ($line =~ /^(\S+) (.+)\0$/os);
+	$inode ||= '';
+	if ($inode ne $last_inode) {
+		my ($dev, $ino, $links) = ($last_inode =~ /^(\d+):(\d+):(\d+)$/os);
+		if (defined($links)) {
+			if ($links && $links == @last_links) {
+				print STDERR "rm -f @last_links\n";
+				for my $unlink (@last_links) {
+					unlink($unlink) or warn "unlink: $unlink: $!";
+				}
+			}
+		} else {
+			warn "Could not parse '$last_inode' in '$line'" unless $last_inode eq '';
+		}
+		@last_links = ();
+	}
+	$last_inode = $inode;
+	push(@last_links, $link);
+}
+
+print STDERR "Removing files contained entirely in '$link_dir'...\n";
+open(FIND, "find . -type f -printf '%D:%i:%n %p\\0' | sort -z --compress-program=gzip |") or die "open: find: $!";
+while (<FIND>) {
+	handle_gc_file($_);
+}
+handle_gc_file('');
+
+print STDERR "Removing empty directories...\n";
+system("find . -type d -empty -print0 | xargs -0rt rmdir -p --ignore-fail-on-non-empty") and die "system: exit status $?";
+print STDERR "Done.\n";
 
 exit(0);