X-Git-Url: http://git.hungrycats.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=hacks%2Fwebcollage;h=330a363d5d395848565071aa945af2acaddc8cd2;hb=8eb2873d7054e705c4e83f22d18c40946a9e2529;hp=f77c6153500a83fdfacf12671d7e118390083077;hpb=3c58fb6311db49c46f1670922933b27c6ea0c065;p=xscreensaver diff --git a/hacks/webcollage b/hacks/webcollage index f77c6153..330a363d 100755 --- a/hacks/webcollage +++ b/hacks/webcollage @@ -9,7 +9,7 @@ # the above copyright notice appear in all copies and that both that # copyright notice and this permission notice appear in supporting # documentation. No representations are made about the suitability of this -# software for any purpose. It is provided "as is" without express or +# software for any purpose. It is provided "as is" without express or # implied warranty. # To run this as a display mode with xscreensaver, add this to `programs': @@ -35,18 +35,19 @@ use POSIX qw(strftime); my $progname = $0; $progname =~ s@.*/@@g; -my $version = q{ $Revision: 1.77 $ }; $version =~ s/^[^0-9]+([0-9.]+).*$/$1/; +my $version = q{ $Revision: 1.82 $ }; $version =~ s/^[^0-9]+([0-9.]+).*$/$1/; my $copyright = "WebCollage $version, Copyright (c) 1999-2001" . " Jamie Zawinski \n" . " http://www.jwz.org/xscreensaver/\n"; -my @search_methods = ( 35, "imagevista", \&pick_from_alta_vista_images, - 33, "altavista", \&pick_from_alta_vista_text, - 20, "yahoorand", \&pick_from_yahoo_random_link, +my @search_methods = ( 30, "imagevista", \&pick_from_alta_vista_images, + 28, "altavista", \&pick_from_alta_vista_text, + 18, "yahoorand", \&pick_from_yahoo_random_link, + 14, "googleimgs", \&pick_from_google_images, 2, "yahoonews", \&pick_from_yahoo_news_text, - 10, "lycos", \&pick_from_lycos_text, + 8, "lycos", \&pick_from_lycos_text, # Hotbot gives me "no matches" just about every time. # Then I try the same URL again, and it works. I guess @@ -55,11 +56,11 @@ my @search_methods = ( 35, "imagevista", \&pick_from_alta_vista_images, # 0, "hotbot", \&pick_from_hotbot_text, ); -#@search_methods=(100, "lycos", \&pick_from_lycos_text); +#@search_methods=(100, "googleimgs",\&pick_from_google_images); # programs we can use to write to the root window (tried in ascending order.) # -my @root_displayers = ( +my @root_displayers = ( "xloadimage -quiet -onroot -center -border black", "xli -quiet -onroot -center -border black", "xv -root -quit -viewonly +noresetroot -rmode 5" . @@ -77,10 +78,10 @@ my %cookies = ( "www.altavista.com" => "AV_ALL=1", # request uncensored searches "web.altavista.com" => "AV_ALL=1", - # log in as "cpunks" - "www.nytimes.com" => "NYT-S=104nv1sChNnnWAvTLGx6eiDhzQcbSoN" . - "6zOMB7s0Qm8MlMaa8It.2/BlXTrpbBk" . - "jinV68IcqxOvAABDyKdciIJ8O000", + # log in as "cipherpunk" + "www.nytimes.com" => 'NYT-S=18cHMIlJOn2Y1bu5xvEG3Ufuk6E1oJ.' . + 'FMxWaQV0igaB5Yi/Q/guDnLeoL.pe7i1oakSb' . + '/VqfdUdb2Uo27Vzt1jmPn3cpYRlTw9', ); @@ -100,6 +101,9 @@ my %poisoners = ( # Since Akamai is super-expensive, let's # go out on a limb and assume that all of # their customers are rich-and-boring. + "bartleby.com" => 1, # Dictionary, cluttering altavista. + "encyclopedia.com" => 1, # Dictionary, cluttering altavista. + "onlinedictionary.datasegment.com" => 1, # Dictionary, cluttering altavista. ); @@ -153,18 +157,13 @@ my $min_width = 50; my $min_height = 50; my $min_ratio = 1/5; +my $min_gif_area = (120 * 120); + + my $no_output_p = 0; my $urls_only_p = 0; -my $wordlist = "/usr/dict/words"; - -if (!-r $wordlist) { - $wordlist = "/usr/share/dict/words"; # BSD -} -if (!-r $wordlist) { - $wordlist = "/usr/share/lib/dict/words"; # Irix -} -die "$progname: $wordlist doesn't exist!\n" unless (-r $wordlist); +my $wordlist; my %rejected_urls; my @tripwire_words = ("aberrate", "abode", "amorphous", "antioch", @@ -255,16 +254,21 @@ sub get_document_1 { my $cookie = $cookies{$them}; + my $user_agent = "$progname/$version"; + if ($url =~ m@^http://www\.altavista\.com/@) { + # block this, you turkeys. + $user_agent = "Mozilla/4.76 [en] (X11; U; Linux 2.2.16-22 i686; Nav)"; + } + my $hdrs = "GET " . ($http_proxy ? $url : "/$path") . " HTTP/1.0\r\n" . "Host: $them\r\n" . - "User-Agent: $progname/$version\r\n"; + "User-Agent: $user_agent\r\n"; if ($referer) { $hdrs .= "Referer: $referer\r\n"; } if ($cookie) { - foreach (split(/\r?\n/, $cookie)) { - $hdrs .= "Cookie: $_\r\n"; - } + my @cc = split(/\r?\n/, $cookie); + $hdrs .= "Cookie: " . join('; ', @cc) . "\r\n"; } $hdrs .= "\r\n"; @@ -581,6 +585,15 @@ sub pick_image_from_body { next; } + # skip GIFs with a small number of pixels -- those usually suck. + if ($width && $height && + m/\.gif$/io && + ($width * $height) < $min_gif_area) { + LOG ($verbose_filter, " skip small GIF $_ (${width}x$height)"); + next; + } + + my $url = $_; if ($unique_urls{$url}) { @@ -589,7 +602,7 @@ sub pick_image_from_body { } LOG ($verbose_filter, - " image $url" . + " image $url" . ($width && $height ? " (${width}x${height})" : "") . ($was_inline ? " (inline)" : "")); @@ -615,13 +628,13 @@ sub pick_image_from_body { $_ = undef; $body = undef; + @urls = depoison (@urls); + if ( $#urls < 0 ) { LOG ($verbose_load, "no images on $base" . ($fsp ? " (frameset)" : "")); return (); } - @urls = depoison (@urls); - # pick a random element of the table my $i = int(rand($#urls+1)); $url = $urls[$i]; @@ -640,10 +653,22 @@ sub pick_image_from_body { ############################################################################ +sub pick_dictionary { + my @dicts = ("/usr/dict/words", + "/usr/share/dict/words", + "/usr/share/lib/dict/words"); + foreach my $f (@dicts) { + if (-f $f) { + $wordlist = $f; + last; + } + } + error ("$dicts[0] does not exist") unless defined($wordlist); +} + # returns a random word from the dictionary # sub random_word { - my $word = 0; if (open (IN, "<$wordlist")) { my $size = (stat(IN))[7]; @@ -687,6 +712,20 @@ sub random_words { } +sub url_quote { + my ($s) = @_; + $s =~ s|([^-a-zA-Z0-9.\@/_\r\n])|sprintf("%%%02X", ord($1))|ge; + return $s; +} + +sub url_unquote { + my ($s) = @_; + $s =~ s/[+]/ /g; + $s =~ s/%([a-z0-9]{2})/chr(hex($1))/ige; + return $s; +} + + # Loads the given URL (a search on some search engine) and returns: # - the total number of hits the search engine claimed it had; # - a list of URLs from the page that the search engine returned; @@ -737,6 +776,12 @@ sub pick_from_search_engine { $search_count = $1; } elsif ($body =~ m@About ((\d{1,3})(,\d{3})*) images@i) { # imagevista $search_count = $1; + } elsif ($body =~ m@We found ((\d{1,3})(,\d{3})*|\d+) results@i) { # *vista + $search_count = $1; + } elsif ($body =~ m@ of about ((\d{1,3})(,\d{3})*)<@i) { # googleimages + $search_count = $1; + } elsif ($body =~ m@((\d{1,3})(,\d{3})*) Web sites were found@i) { + $search_count = $1; # lycos } elsif ($body =~ m@WEB.*?RESULTS.*?\b((\d{1,3})(,\d{3})*)\b.*?Matches@i) { $search_count = $1; # hotbot } elsif ($body =~ m@no photos were found containing@i) { # imagevista @@ -896,7 +941,7 @@ sub pick_from_yahoo_random_link { return; } - LOG ($verbose_load, "redirected to: $base"); + LOG ($verbose_load, "redirected to: $base"); my $img = pick_image_from_body ($base, $body); $body = undef; @@ -927,7 +972,6 @@ my $alta_vista_images_url = "http://www.altavista.com/cgi-bin/query" . "&mmW=1" . # unknown, but required "&q="; - # imagevista sub pick_from_alta_vista_images { my ( $timeout ) = @_; @@ -946,6 +990,12 @@ sub pick_from_alta_vista_images { my @candidates = (); foreach my $u (@subpages) { + + # altavista is encoding their URLs now. + next unless ($u =~ + m@^/r\?ck_sm=[a-zA-Z0-9]+\&ref=[a-zA-Z0-9]+(\&uid=[a-zA-Z0-9]+)?\&r=(.*)@); + $u = url_unquote($2); + next unless ($u =~ m@^http://@i); # skip non-HTTP or relative URLs next if ($u =~ m@[/.]altavista\.com\b@i); # skip altavista builtins next if ($u =~ m@[/.]doubleclick\.net\b@i); # you cretins @@ -963,6 +1013,56 @@ sub pick_from_alta_vista_images { } + +############################################################################ +# +# Pick images by feeding random words into Google Image Search +# By Charles Gales +# +############################################################################ + + +my $google_images_url = "http://images.google.com/images" . + "?site=images" . # photos + "&btnG=Search" . # graphics + "&safe=off" . # no screening + "&imgsafe=off" . + "&q="; + +# googleimgs +sub pick_from_google_images { + my ( $timeout ) = @_; + + my $words = random_word; # only one word for Google + my $page = (int(rand(9)) + 1); + my $num = 20; # 20 images per page + my $search_url = $google_images_url . $words; + + if ($page > 1) { + $search_url .= "&start=" . $page*$num; # page number + $search_url .= "&num=" . $num; #images per page + } + + my ($search_hit_count, @subpages) = + pick_from_search_engine ($timeout, $search_url, $words); + + my @candidates = (); + foreach my $u (@subpages) { + next unless ($u =~ m@imgres\?imgurl@i); # All pics start with this + next if ($u =~ m@[/.]google\.com\b@i); # skip google builtins + + if ($u =~ m@^/imgres\?imgurl=(.*?)\&imgrefurl=(.*?)\&@) { + my $urlf = $2; + LOG ($verbose_filter, " candidate: $urlf"); + push @candidates, $urlf; + } + } + + return pick_image_from_pages ($search_url, $search_hit_count, $#subpages+1, + $timeout, @candidates); +} + + ############################################################################ # @@ -971,8 +1071,12 @@ sub pick_from_alta_vista_images { ############################################################################ -my $alta_vista_url = "http://www.altavista.com/cgi-bin/query?pg=q" . - "&text=yes&kl=XX&stype=stext&q="; +my $alta_vista_url_1 = "http://www.altavista.com/cgi-bin/query?pg=q" . + "&text=yes&kl=XX&stype=stext&q="; +my $alta_vista_url_2 = "http://www.altavista.com/sites/search/web?pg=q" . + "&kl=XX&search=Search&q="; + +my $alta_vista_url = $alta_vista_url_2; # altavista sub pick_from_alta_vista_text { @@ -998,8 +1102,9 @@ sub pick_from_alta_vista_text { # onMouseOver to make it look like they're not! Well, it makes it # easier for us to identify search results... # - next unless ($u =~ m@^/r\?ck_sm=[a-zA-Z0-9]+\&ref=[a-zA-Z0-9]+\&r=(.*)@); - $u = $1; + next unless ($u =~ + m@^/r\?ck_sm=[a-zA-Z0-9]+\&ref=[a-zA-Z0-9]+\&uid=[a-zA-Z0-9]+\&r=(.*)@); + $u = url_unquote($1); LOG ($verbose_filter, " candidate: $u"); push @candidates, $u; @@ -1138,7 +1243,7 @@ sub pick_from_yahoo_news_text { # Picks a random image on a random page, and returns two URLs: -# the page containing the image, and the image. +# the page containing the image, and the image. # Returns () if nothing found this time. # Uses the url-randomizer 1 time in 5, else the image randomizer. # @@ -1191,8 +1296,8 @@ sub blurb { } sub error { - ($_) = @_; - print STDERR blurb() . "$_\n"; + my ($err) = @_; + print STDERR blurb() . "$err\n"; exit 1; } @@ -1471,7 +1576,7 @@ sub jpeg_size { # We must skip variables, since FFs in variable names aren't # valid JPEG markers. my $s = substr($body, $i, 2); $i += 2; - my ($c1, $c2) = unpack ("C"x2, $s); + my ($c1, $c2) = unpack ("C"x2, $s); my $length = ($c1 << 8) | $c2; return () if ($length < 2); $i += $length-2; @@ -1563,7 +1668,7 @@ sub x_cleanup { # sub nontrapping_system { $! = 0; - + $_ = join(" ", @_); s/\"[^\"]+\"/\"...\"/g; @@ -1591,7 +1696,7 @@ sub nontrapping_system { # Given the URL of a GIF or JPEG image, and the body of that image, writes a -# PPM to the given output file. Returns the width/height of the image if +# PPM to the given output file. Returns the width/height of the image if # successful. # sub image_to_pnm { @@ -1932,10 +2037,10 @@ sub paste_image { } if (rand() < $crop_chance) { - + my $ow = $crop_w; my $oh = $crop_h; - + if ($crop_w > $min_width) { # if it's a banner, select the width linearly. # otherwise, select a bell. @@ -1950,7 +2055,7 @@ sub paste_image { $crop_h = $min_height + int (bellrand() * ($crop_h - $min_height)); $crop_y = int (rand() * ($oh - $crop_h)); } - + if ($crop_x != 0 || $crop_y != 0 || $crop_w != $iw || $crop_h != $ih) { LOG ($verbose_pbm, @@ -1970,13 +2075,13 @@ sub paste_image { $y < 0 || $x + $crop_w > $img_width || $y + $crop_h > $img_height) { - + LOG ($verbose_pbm, "cropping for effective paste of ${crop_w}x$crop_h \@ $x,$y"); - + if ($x < 0) { $crop_x -= $x; $crop_w += $x; $x = 0; } if ($y < 0) { $crop_y -= $y; $crop_h += $y; $y = 0; } - + if ($x + $crop_w >= $img_width) { $crop_w = $img_width - $x - 1; } if ($y + $crop_h >= $img_height) { $crop_h = $img_height - $y - 1; } } @@ -2034,7 +2139,7 @@ sub paste_image { my $tsize = (stat($target))[7]; if ($tsize > 200) { $cmd = "$ppm_to_root_window_cmd $target"; - + # xv seems to hate being killed. it tends to forget to clean # up after itself, and leaves windows around and colors allocated. # I had this same problem with vidwhacker, and I'm not entirely @@ -2046,14 +2151,14 @@ sub paste_image { # to do anyway. # $cmd .= " &"; - + $rc = nontrapping_system ($cmd); - + if ($rc != 0) { LOG (($verbose_pbm || $verbose_load), "display failed: \"$cmd\""); return; } - + } else { LOG ($verbose_pbm, "$target size is $tsize"); } @@ -2074,6 +2179,7 @@ sub main { srand(time ^ $$); my $verbose = 0; + my $dict; $current_state = "init"; $load_method = "none"; @@ -2123,10 +2229,13 @@ sub main { } } elsif ($_ eq "-proxy" || $_ eq "-http-proxy") { $http_proxy = shift @ARGV; + } elsif ($_ eq "-dictionary" || $_ eq "-dict") { + $dict = shift @ARGV; } else { print STDERR "$copyright\nusage: $progname [-root]" . " [-display dpy] [-root] [-verbose] [-timeout secs]\n" . "\t\t [-delay secs] [-filter cmd] [-filter2 cmd]\n" . + "\t\t [-dictionary dictionary-file]\n" . "\t\t [-http-proxy host[:port]]\n"; exit 1; } @@ -2200,6 +2309,12 @@ sub main { $verbose_exec = 1; } + if ($dict) { + error ("$dict does not exist") unless (-f $dict); + $wordlist = $dict; + } else { + pick_dictionary(); + } if ($urls_only_p) { url_only_output;