X-Git-Url: http://git.hungrycats.org/cgi-bin/gitweb.cgi?p=dupemerge;a=blobdiff_plain;f=faster-dupemerge;h=f1413b47e10a7d60364c14d532efdcdb049a76f0;hp=6223ab0adc2d7cacb4dee7e7c4f4e71a2aa3a1d4;hb=1c211f4055c67cfbf832d057d5b8bb3c7bf6d21c;hpb=95742bfa9d5a1f33ce77689f0b2fb93bdf888e85 diff --git a/faster-dupemerge b/faster-dupemerge index 6223ab0..f1413b4 100755 --- a/faster-dupemerge +++ b/faster-dupemerge @@ -4,7 +4,7 @@ use Fcntl qw(:DEFAULT :flock); use File::Compare; use File::Temp; -# Copyright (C) 2003-2006 Zygo Blaxell +# Copyright (C) 2003-2010 Zygo Blaxell # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -87,6 +87,7 @@ my $collapse_timestamp = 0; my $collapse_zero = 0; my $skip_compares = 0; my $skip_hashes = 0; +my $progress = 0; my $verbose = 0; my $debug = 0; my $dry_run = 0; @@ -102,6 +103,7 @@ sub digest { if ($skip_hashes) { return "SKIPPING HASHES"; } else { + print STDERR 'H' if $progress; my $digest = &really_digest($filename); $hash_bytes += -s $filename; $hash_files++; @@ -172,6 +174,8 @@ while ($#ARGV >= 0) { $skip_compares = 1; } elsif ($arg eq '--skip-hash') { $skip_hashes = 1; + } elsif ($arg eq '--progress') { + $progress = 1; } elsif ($arg eq '--verbose') { $verbose = 1; } elsif ($arg eq '--lock-rm') { @@ -238,7 +242,7 @@ my @find_command = ('find', @directories, @extra_find_opts, '-type', 'f'); my $printf_string = '%s ' . ($collapse_access ? '0 0 0 ' : '%U %G %m ') . ($collapse_timestamp ? '0 ' : '%T@ ') . - '%D:%i %p\0'; + '%D %i %p\0'; push(@find_command, '!', '-empty') unless $collapse_zero; push(@find_command, '-printf', $printf_string); @@ -269,6 +273,7 @@ sub link_files { my $quoted_from = tick_quote($from); my $quoted_to = tick_quote($to); + print STDERR "\n" if $progress; print STDERR "ln -f $quoted_from $quoted_to\n"; return if $dry_run; @@ -279,15 +284,25 @@ sub link_files { $inode_base =~ s:^.*/::os; my $tmp_to = File::Temp::tempnam($inode_dir, ".$inode_base."); print STDERR "\tlink: $from -> $tmp_to\n" if $debug; + print STDERR 'L' if $progress; link($from, $tmp_to) or die "link: $from -> $tmp_to: $!"; print STDERR "\trename: $tmp_to -> $to\n" if $debug; + print STDERR 'R' if $progress; unless (rename($tmp_to, $to)) { my $saved_bang = $!; + print STDERR 'U' if $progress; unlink($tmp_to) or warn "unlink: $tmp_to: $!"; # Try, possibly in vain, to clean up die "rename: $tmp_to -> $from: $saved_bang"; } } +# Convert $dev,$ino into a single string where lexical and numeric orderings are equivalent +sub format_inode ($$) { + my ($dev, $ino) = @_; + # 64 bits ought to be enough for everybody! + return sprintf('%016x:%016x', $dev, $ino); +} + # Process all known files so far. sub merge_files { $merges_attempted++; @@ -333,8 +348,9 @@ hash_file: if ($ok) { print STDERR "\t\tDigest is $digest\n" if $debug; - my $incumbent = $hash_to_inode{$digest}; - if (defined($incumbent)) { + my $incumbent_list = ($hash_to_inode{$digest} ||= []); + my $incumbent_matched = 0; + for my $incumbent (sort @$incumbent_list) { print STDERR "\t\tInodes $incumbent and $candidate have same hash\n" if $debug; my $finished = 0; @@ -344,12 +360,13 @@ link_start: until ($finished) { my @incumbent_names = sort keys(%{$inode_to_file_name{$incumbent}}); my @candidate_names = sort keys(%{$inode_to_file_name{$candidate}}); - print STDERR "\t\tLinks to $incumbent:", join("\n\t\t\t", '', @incumbent_names), "\n" if $debug; + print STDERR "\t\tLinks to $incumbent:", join("\n\t\t\t", '', @incumbent_names), "\n" if $debug; print STDERR "\t\tLinks to $candidate:", join("\n\t\t\t", '', @candidate_names), "\n" if $debug; incumbent_file: foreach my $incumbent_file (@incumbent_names) { + print STDERR 'S' if $progress; my ($incumbent_dev,$incumbent_ino,$incumbent_mode,$incumbent_nlink,$incumbent_uid,$incumbent_gid,$incumbent_rdev,$incumbent_size,$incumbent_atime,$incumbent_mtime,$incumbent_ctime,$incumbent_blksize,$incumbent_blocks) = lstat($incumbent_file); print STDERR "\t\tINCUMBENT dev=$incumbent_dev ino=$incumbent_ino mode=$incumbent_mode nlink=$incumbent_nlink uid=$incumbent_uid gid=$incumbent_gid rdev=$incumbent_rdev size=$incumbent_size atime=$incumbent_atime mtime=$incumbent_mtime ctime=$incumbent_ctime blksize=$incumbent_blksize blocks=$incumbent_blocks _=$incumbent_file\n" if $debug; @@ -359,8 +376,8 @@ incumbent_file: next incumbent_file; } - if ($incumbent_ino != $incumbent) { - warn "$incumbent_file: expected inode $incumbent, found $incumbent_ino"; + if (format_inode($incumbent_dev, $incumbent_ino) ne $incumbent) { + warn "$incumbent_file: expected inode $incumbent, found $incumbent_dev:$incumbent_ino"; $surprises++; next incumbent_file; } @@ -370,6 +387,7 @@ incumbent_file: candidate_file: foreach my $candidate_file (@candidate_names) { + print STDERR 's' if $progress; my ($candidate_dev,$candidate_ino,$candidate_mode,$candidate_nlink,$candidate_uid,$candidate_gid,$candidate_rdev,$candidate_size,$candidate_atime,$candidate_mtime,$candidate_ctime,$candidate_blksize,$candidate_blocks) = lstat($candidate_file); print STDERR "\t\t\tCANDIDATE dev=$candidate_dev ino=$candidate_ino mode=$candidate_mode nlink=$candidate_nlink uid=$candidate_uid gid=$candidate_gid rdev=$candidate_rdev size=$candidate_size atime=$candidate_atime mtime=$candidate_mtime ctime=$candidate_ctime blksize=$candidate_blksize blocks=$candidate_blocks _=$candidate_file\n" if $debug; @@ -379,8 +397,8 @@ candidate_file: next candidate_file; } - if ($candidate_ino != $candidate) { - warn "$candidate_file: expected inode $candidate, found $candidate_ino"; + if (format_inode($candidate_dev, $candidate_ino) ne $candidate) { + warn "$candidate_file: expected inode $candidate, found $candidate_dev:$candidate_ino"; $surprises++; next candidate_file; } @@ -401,6 +419,7 @@ candidate_file: my $quoted_incumbent_file = tick_quote($incumbent_file); my $quoted_candidate_file = tick_quote($candidate_file); print STDERR "cmp $quoted_incumbent_file $quoted_candidate_file\n" if $debug; + print STDERR 'C' if $progress; if (compare($incumbent_file, $candidate_file)) { $compare_differences++; $identical = 0; @@ -411,6 +430,7 @@ candidate_file: print STDERR "$quoted_incumbent_file and $quoted_candidate_file have same hash but do not compare equal!\n" unless $skip_hashes; } else { $identical = 1; + $incumbent_matched = 1; } $compare_count++; $compare_bytes += $incumbent_size; @@ -493,13 +513,9 @@ candidate_file: # My random number generator chooses the incumbent's size. if ($link_done) { - # Since we're in a dry run, the filesystem doesn't change. - # Our notion of what the filesystem should look like should not change either. delete $inode_to_file_name{$to_inode}->{$to_file}; - unless ($dry_run) { - $inode_to_file_name{$from_inode}->{$to_file} = undef; - $hash_to_inode{$digest} = $from_inode; - } + $inode_to_file_name{$from_inode}->{$to_file} = undef; + $hash_to_inode{$digest} = [ $from_inode ]; $hard_links++; if ($to_nlink == 1) { @@ -527,9 +543,10 @@ candidate_file: } $finished = 1; } - } else { + } + unless ($incumbent_matched) { print STDERR "\t\tNew hash entered\n" if $debug; - $hash_to_inode{$digest} = $candidate; + push(@$incumbent_list, $candidate); } } else { warn "No digests found for inode $candidate\n"; @@ -539,14 +556,17 @@ candidate_file: end_merge: + print STDERR '.' if $progress; print STDERR "Merge done.\n" if $debug; undef %inode_to_file_name; } while () { - my ($weak_key, $inode, $name) = m/^(\d+ \d+ \d+ \d+ -?\d+) (\d+:\d+) (.+)\0$/so; + my ($weak_key, $dev, $ino, $name) = m/^(\d+ \d+ \d+ \d+ -?[\d.]+) (\d+) (\d+) (.+)\0$/so; die "read error: $!\nLast input line was '$_'" unless defined($name); + my $inode = format_inode($dev, $ino); + print STDERR "weak_key=$weak_key inode=$inode name=$name\n" if $debug; unless (! (-l $name) && (-f _)) {