4 use Digest::SHA1 qw(sha1 sha1_hex sha1_base64);
5 use Fcntl qw(:DEFAULT :flock);
12 # Copyright (C) 2010 Zygo Blaxell <dupemerge@mailtoo.hungrycats.org>
14 # This program is free software; you can redistribute it and/or modify
15 # it under the terms of the GNU General Public License as published by
16 # the Free Software Foundation; either version 2 of the License, or
17 # (at your option) any later version.
19 # This program is distributed in the hope that it will be useful,
20 # but WITHOUT ANY WARRANTY; without even the implied warranty of
21 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 # GNU General Public License for more details.
24 # You should have received a copy of the GNU General Public License
25 # along with this program; if not, write to the Free Software
26 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
29 my ($filename) = (@_);
30 die "'$filename' is not a plain file" if (-l $filename) || ! (-f _);
31 my $ctx = Digest::SHA1->new;
32 sysopen(FILE, $filename, O_RDONLY|O_NONBLOCK) or die "open: $filename: $!";
33 binmode(FILE); # FIXME: Necessary? Probably harmless...
34 $ctx->addfile(\*FILE);
35 close(FILE) or die "close: $filename: $!";
36 return $ctx->b64digest;
42 Hashes a NUL-separated list of files on stdin into link-dir.
50 my ($from, $to) = (@_);
55 $inode_dir =~ s:[^/]*$::o;
56 $inode_base =~ s:^.*/::os;
57 my $tmp_to = File::Temp::tempnam($inode_dir, ".$inode_base.");
59 link($from, $tmp_to) or die "link: $from -> $tmp_to: $!";
62 $saved_bang = $! unless rename($tmp_to, $to);
64 # If $to exists and is a hardlink to $tmp_to (or $from),
65 # rename returns success but $tmp_to still exists.
67 unlink($tmp_to) or warn "unlink: $tmp_to: $!" if -e $tmp_to;
69 die "rename: $tmp_to -> $from: $saved_bang" if $saved_bang;
73 my $link_dir = shift @ARGV;
74 usage unless $link_dir;
76 my $prefix_length = 3;
80 $file .= '_' x (length($file) + 1 - $prefix_length) if length($file) + 1 < $prefix_length;
81 my $prefix = substr($file, 0, $prefix_length);
82 my $suffix = substr($file, $prefix_length);
83 $prefix =~ s:(.):$1/:osg;
85 return ($prefix, $suffix);
93 ($parent = $dir) =~ s:[^/]+$::os;
94 if ($parent ne $dir) {
97 mkdir($dir) or die "mkdir: $dir: $!";
99 die "mkdir: $dir: $!" unless -d $dir;
102 sub prepare_parents {
103 my ($link_dir, $file) = @_;
104 my ($prefix, $suffix) = slash_prefix($file);
105 my $parent = "$link_dir/$prefix";
107 return "$parent/$suffix";
112 my $packed = pack('Q>', $int64);
113 $packed =~ s/^\0+//os;
114 my $base64_packed = encode_base64($packed, '');
115 $base64_packed =~ y:/:_:;
116 # Don't strip off the trailing padding since it makes the string
117 # so short we end up just putting it back on again.
118 # $base64_packed =~ s/=+$//os;
119 return $base64_packed;
122 # ext3 cannot handle more than 32000 links to a file. Leave some headroom.
123 # Arguably this should be configurable, but the losses are miniscule and
124 # the coding for option support is not.
125 my $link_count_max = 31990;
136 my $st = lstat($file);
137 die "lstat: $file: $!" unless $st;
142 # Skip the file if it has far too many links already
143 next if ($st->nlink > $link_count_max);
145 # Check link to inode
146 my $inode_link = prepare_parents($link_dir, name_ino($st->ino));
148 my $inode_st = lstat($inode_link);
151 my $inode_dev = $inode_st->dev;
152 my $inode_ino = $inode_st->ino;
153 my $file_dev = $st->dev;
154 my $file_ino = $st->ino;
155 if ($inode_ino != $file_ino || $inode_dev != $file_dev) {
156 warn "inode link '$inode_link' is wrong (inode $inode_ino should be $file_ino)" if $inode_ino != $file_ino;
157 warn "inode link '$inode_link' is wrong (dev $inode_dev should be $file_dev)" if $inode_dev != $file_dev;
165 # If neither criteria for updating link is met, leave it as-is
166 next unless $update_links;
170 my $digest = digest($file);
173 # Base64 uses /, we prefer _
176 # Check link to digest
177 my $digest_link = prepare_parents($link_dir, $digest);
179 my $digest_st = lstat($digest_link);
181 my $digest_nlink = $digest_st->nlink;
182 if ($digest_nlink > 31990) {
184 unlink($digest_link) or die "unlink: $digest_link: $!";
190 # Which file are we keeping?
193 # If digest link exists, link it to file
196 die "NOT identical!" if compare($digest_link, $file);
198 # Old, replace input with old file
200 link_files($digest_link, $file);
201 $keep_ino = $digest_st->ino;
203 # New, add input to digest
205 link_files($file, $digest_link);
206 $keep_ino = $st->ino;
209 # A link to the inode indicates we are done, so do it last
210 $inode_link = prepare_parents($link_dir, name_ino($keep_ino));
212 link_files($digest_link, $inode_link);
216 warn "$file: $@" if $@;
220 print STDERR "\nGarbage collection in '$link_dir'...";
221 chdir($link_dir) || die "chdir: $link_dir: $!";
222 print STDERR "\nRemoving files with link count < 3 and temporary links...";
223 system('find . -type f \( -links -3 -o -name ".*" \) -print0 | xargs -0rt rm -f') and die "system: exit status $?";
224 print STDERR "\nRemoving empty directories...";
225 system("find . -type d -empty -print0 | xargs -0rt rmdir -p --ignore-fail-on-non-empty") and die "system: exit status $?";
226 print STDERR "\nDone.\n";
232 #################################################################################
233 # GNU GENERAL PUBLIC LICENSE #
234 # Version 2, June 1991 #
236 # Copyright (C) 1989, 1991 Free Software Foundation, Inc. #
237 # 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA #
238 # Everyone is permitted to copy and distribute verbatim copies #
239 # of this license document, but changing it is not allowed. #
243 # The licenses for most software are designed to take away your #
244 # freedom to share and change it. By contrast, the GNU General Public #
245 # License is intended to guarantee your freedom to share and change free #
246 # software--to make sure the software is free for all its users. This #
247 # General Public License applies to most of the Free Software #
248 # Foundation's software and to any other program whose authors commit to #
249 # using it. (Some other Free Software Foundation software is covered by #
250 # the GNU Library General Public License instead.) You can apply it to #
251 # your programs, too. #
253 # When we speak of free software, we are referring to freedom, not #
254 # price. Our General Public Licenses are designed to make sure that you #
255 # have the freedom to distribute copies of free software (and charge for #
256 # this service if you wish), that you receive source code or can get it #
257 # if you want it, that you can change the software or use pieces of it #
258 # in new free programs; and that you know you can do these things. #
260 # To protect your rights, we need to make restrictions that forbid #
261 # anyone to deny you these rights or to ask you to surrender the rights. #
262 # These restrictions translate to certain responsibilities for you if you #
263 # distribute copies of the software, or if you modify it. #
265 # For example, if you distribute copies of such a program, whether #
266 # gratis or for a fee, you must give the recipients all the rights that #
267 # you have. You must make sure that they, too, receive or can get the #
268 # source code. And you must show them these terms so they know their #
271 # We protect your rights with two steps: (1) copyright the software, and #
272 # (2) offer you this license which gives you legal permission to copy, #
273 # distribute and/or modify the software. #
275 # Also, for each author's protection and ours, we want to make certain #
276 # that everyone understands that there is no warranty for this free #
277 # software. If the software is modified by someone else and passed on, we #
278 # want its recipients to know that what they have is not the original, so #
279 # that any problems introduced by others will not reflect on the original #
280 # authors' reputations. #
282 # Finally, any free program is threatened constantly by software #
283 # patents. We wish to avoid the danger that redistributors of a free #
284 # program will individually obtain patent licenses, in effect making the #
285 # program proprietary. To prevent this, we have made it clear that any #
286 # patent must be licensed for everyone's free use or not licensed at all. #
288 # The precise terms and conditions for copying, distribution and #
289 # modification follow. #
291 # GNU GENERAL PUBLIC LICENSE #
292 # TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION #
294 # 0. This License applies to any program or other work which contains #
295 # a notice placed by the copyright holder saying it may be distributed #
296 # under the terms of this General Public License. The "Program", below, #
297 # refers to any such program or work, and a "work based on the Program" #
298 # means either the Program or any derivative work under copyright law: #
299 # that is to say, a work containing the Program or a portion of it, #
300 # either verbatim or with modifications and/or translated into another #
301 # language. (Hereinafter, translation is included without limitation in #
302 # the term "modification".) Each licensee is addressed as "you". #
304 # Activities other than copying, distribution and modification are not #
305 # covered by this License; they are outside its scope. The act of #
306 # running the Program is not restricted, and the output from the Program #
307 # is covered only if its contents constitute a work based on the #
308 # Program (independent of having been made by running the Program). #
309 # Whether that is true depends on what the Program does. #
311 # 1. You may copy and distribute verbatim copies of the Program's #
312 # source code as you receive it, in any medium, provided that you #
313 # conspicuously and appropriately publish on each copy an appropriate #
314 # copyright notice and disclaimer of warranty; keep intact all the #
315 # notices that refer to this License and to the absence of any warranty; #
316 # and give any other recipients of the Program a copy of this License #
317 # along with the Program. #
319 # You may charge a fee for the physical act of transferring a copy, and #
320 # you may at your option offer warranty protection in exchange for a fee. #
322 # 2. You may modify your copy or copies of the Program or any portion #
323 # of it, thus forming a work based on the Program, and copy and #
324 # distribute such modifications or work under the terms of Section 1 #
325 # above, provided that you also meet all of these conditions: #
327 # a) You must cause the modified files to carry prominent notices #
328 # stating that you changed the files and the date of any change. #
330 # b) You must cause any work that you distribute or publish, that in #
331 # whole or in part contains or is derived from the Program or any #
332 # part thereof, to be licensed as a whole at no charge to all third #
333 # parties under the terms of this License. #
335 # c) If the modified program normally reads commands interactively #
336 # when run, you must cause it, when started running for such #
337 # interactive use in the most ordinary way, to print or display an #
338 # announcement including an appropriate copyright notice and a #
339 # notice that there is no warranty (or else, saying that you provide #
340 # a warranty) and that users may redistribute the program under #
341 # these conditions, and telling the user how to view a copy of this #
342 # License. (Exception: if the Program itself is interactive but #
343 # does not normally print such an announcement, your work based on #
344 # the Program is not required to print an announcement.) #
346 # These requirements apply to the modified work as a whole. If #
347 # identifiable sections of that work are not derived from the Program, #
348 # and can be reasonably considered independent and separate works in #
349 # themselves, then this License, and its terms, do not apply to those #
350 # sections when you distribute them as separate works. But when you #
351 # distribute the same sections as part of a whole which is a work based #
352 # on the Program, the distribution of the whole must be on the terms of #
353 # this License, whose permissions for other licensees extend to the #
354 # entire whole, and thus to each and every part regardless of who wrote it. #
356 # Thus, it is not the intent of this section to claim rights or contest #
357 # your rights to work written entirely by you; rather, the intent is to #
358 # exercise the right to control the distribution of derivative or #
359 # collective works based on the Program. #
361 # In addition, mere aggregation of another work not based on the Program #
362 # with the Program (or with a work based on the Program) on a volume of #
363 # a storage or distribution medium does not bring the other work under #
364 # the scope of this License. #
366 # 3. You may copy and distribute the Program (or a work based on it, #
367 # under Section 2) in object code or executable form under the terms of #
368 # Sections 1 and 2 above provided that you also do one of the following: #
370 # a) Accompany it with the complete corresponding machine-readable #
371 # source code, which must be distributed under the terms of Sections #
372 # 1 and 2 above on a medium customarily used for software interchange; or, #
374 # b) Accompany it with a written offer, valid for at least three #
375 # years, to give any third party, for a charge no more than your #
376 # cost of physically performing source distribution, a complete #
377 # machine-readable copy of the corresponding source code, to be #
378 # distributed under the terms of Sections 1 and 2 above on a medium #
379 # customarily used for software interchange; or, #
381 # c) Accompany it with the information you received as to the offer #
382 # to distribute corresponding source code. (This alternative is #
383 # allowed only for noncommercial distribution and only if you #
384 # received the program in object code or executable form with such #
385 # an offer, in accord with Subsection b above.) #
387 # The source code for a work means the preferred form of the work for #
388 # making modifications to it. For an executable work, complete source #
389 # code means all the source code for all modules it contains, plus any #
390 # associated interface definition files, plus the scripts used to #
391 # control compilation and installation of the executable. However, as a #
392 # special exception, the source code distributed need not include #
393 # anything that is normally distributed (in either source or binary #
394 # form) with the major components (compiler, kernel, and so on) of the #
395 # operating system on which the executable runs, unless that component #
396 # itself accompanies the executable. #
398 # If distribution of executable or object code is made by offering #
399 # access to copy from a designated place, then offering equivalent #
400 # access to copy the source code from the same place counts as #
401 # distribution of the source code, even though third parties are not #
402 # compelled to copy the source along with the object code. #
404 # 4. You may not copy, modify, sublicense, or distribute the Program #
405 # except as expressly provided under this License. Any attempt #
406 # otherwise to copy, modify, sublicense or distribute the Program is #
407 # void, and will automatically terminate your rights under this License. #
408 # However, parties who have received copies, or rights, from you under #
409 # this License will not have their licenses terminated so long as such #
410 # parties remain in full compliance. #
412 # 5. You are not required to accept this License, since you have not #
413 # signed it. However, nothing else grants you permission to modify or #
414 # distribute the Program or its derivative works. These actions are #
415 # prohibited by law if you do not accept this License. Therefore, by #
416 # modifying or distributing the Program (or any work based on the #
417 # Program), you indicate your acceptance of this License to do so, and #
418 # all its terms and conditions for copying, distributing or modifying #
419 # the Program or works based on it. #
421 # 6. Each time you redistribute the Program (or any work based on the #
422 # Program), the recipient automatically receives a license from the #
423 # original licensor to copy, distribute or modify the Program subject to #
424 # these terms and conditions. You may not impose any further #
425 # restrictions on the recipients' exercise of the rights granted herein. #
426 # You are not responsible for enforcing compliance by third parties to #
429 # 7. If, as a consequence of a court judgment or allegation of patent #
430 # infringement or for any other reason (not limited to patent issues), #
431 # conditions are imposed on you (whether by court order, agreement or #
432 # otherwise) that contradict the conditions of this License, they do not #
433 # excuse you from the conditions of this License. If you cannot #
434 # distribute so as to satisfy simultaneously your obligations under this #
435 # License and any other pertinent obligations, then as a consequence you #
436 # may not distribute the Program at all. For example, if a patent #
437 # license would not permit royalty-free redistribution of the Program by #
438 # all those who receive copies directly or indirectly through you, then #
439 # the only way you could satisfy both it and this License would be to #
440 # refrain entirely from distribution of the Program. #
442 # If any portion of this section is held invalid or unenforceable under #
443 # any particular circumstance, the balance of the section is intended to #
444 # apply and the section as a whole is intended to apply in other #
447 # It is not the purpose of this section to induce you to infringe any #
448 # patents or other property right claims or to contest validity of any #
449 # such claims; this section has the sole purpose of protecting the #
450 # integrity of the free software distribution system, which is #
451 # implemented by public license practices. Many people have made #
452 # generous contributions to the wide range of software distributed #
453 # through that system in reliance on consistent application of that #
454 # system; it is up to the author/donor to decide if he or she is willing #
455 # to distribute software through any other system and a licensee cannot #
456 # impose that choice. #
458 # This section is intended to make thoroughly clear what is believed to #
459 # be a consequence of the rest of this License. #
461 # 8. If the distribution and/or use of the Program is restricted in #
462 # certain countries either by patents or by copyrighted interfaces, the #
463 # original copyright holder who places the Program under this License #
464 # may add an explicit geographical distribution limitation excluding #
465 # those countries, so that distribution is permitted only in or among #
466 # countries not thus excluded. In such case, this License incorporates #
467 # the limitation as if written in the body of this License. #
469 # 9. The Free Software Foundation may publish revised and/or new versions #
470 # of the General Public License from time to time. Such new versions will #
471 # be similar in spirit to the present version, but may differ in detail to #
472 # address new problems or concerns. #
474 # Each version is given a distinguishing version number. If the Program #
475 # specifies a version number of this License which applies to it and "any #
476 # later version", you have the option of following the terms and conditions #
477 # either of that version or of any later version published by the Free #
478 # Software Foundation. If the Program does not specify a version number of #
479 # this License, you may choose any version ever published by the Free Software #
482 # 10. If you wish to incorporate parts of the Program into other free #
483 # programs whose distribution conditions are different, write to the author #
484 # to ask for permission. For software which is copyrighted by the Free #
485 # Software Foundation, write to the Free Software Foundation; we sometimes #
486 # make exceptions for this. Our decision will be guided by the two goals #
487 # of preserving the free status of all derivatives of our free software and #
488 # of promoting the sharing and reuse of software generally. #
492 # 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY #
493 # FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN #
494 # OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES #
495 # PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED #
496 # OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF #
497 # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS #
498 # TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE #
499 # PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, #
500 # REPAIR OR CORRECTION. #
502 # 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING #
503 # WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR #
504 # REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, #
505 # INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING #
506 # OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED #
507 # TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY #
508 # YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER #
509 # PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE #
510 # POSSIBILITY OF SUCH DAMAGES. #
512 # END OF TERMS AND CONDITIONS #
514 # How to Apply These Terms to Your New Programs #
516 # If you develop a new program, and you want it to be of the greatest #
517 # possible use to the public, the best way to achieve this is to make it #
518 # free software which everyone can redistribute and change under these terms. #
520 # To do so, attach the following notices to the program. It is safest #
521 # to attach them to the start of each source file to most effectively #
522 # convey the exclusion of warranty; and each file should have at least #
523 # the "copyright" line and a pointer to where the full notice is found. #
525 # <one line to give the program's name and a brief idea of what it does.> #
526 # Copyright (C) <year> <name of author> #
528 # This program is free software; you can redistribute it and/or modify #
529 # it under the terms of the GNU General Public License as published by #
530 # the Free Software Foundation; either version 2 of the License, or #
531 # (at your option) any later version. #
533 # This program is distributed in the hope that it will be useful, #
534 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
535 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
536 # GNU General Public License for more details. #
538 # You should have received a copy of the GNU General Public License #
539 # along with this program; if not, write to the Free Software #
540 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA #
543 # Also add information on how to contact you by electronic and paper mail. #
545 # If the program is interactive, make it output a short notice like this #
546 # when it starts in an interactive mode: #
548 # Gnomovision version 69, Copyright (C) year name of author #
549 # Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. #
550 # This is free software, and you are welcome to redistribute it #
551 # under certain conditions; type `show c' for details. #
553 # The hypothetical commands `show w' and `show c' should show the appropriate #
554 # parts of the General Public License. Of course, the commands you use may #
555 # be called something other than `show w' and `show c'; they could even be #
556 # mouse-clicks or menu items--whatever suits your program. #
558 # You should also get your employer (if you work as a programmer) or your #
559 # school, if any, to sign a "copyright disclaimer" for the program, if #
560 # necessary. Here is a sample; alter the names: #
562 # Yoyodyne, Inc., hereby disclaims all copyright interest in the program #
563 # `Gnomovision' (which makes passes at compilers) written by James Hacker. #
565 # <signature of Ty Coon>, 1 April 1989 #
566 # Ty Coon, President of Vice #
568 # This General Public License does not permit incorporating your program into #
569 # proprietary programs. If your program is a subroutine library, you may #
570 # consider it more useful to permit linking proprietary applications with the #
571 # library. If this is what you want to do, use the GNU Library General #
572 # Public License instead of this License. #
573 #################################################################################