perl爬虫

最新推荐文章于 2023-10-26 08:49:42 发布

weixin_33904756

最新推荐文章于 2023-10-26 08:49:42 发布

阅读量107

点赞数

文章标签：爬虫

爬行深度之类的我没加，加上也容易，几句话的事情。直接代码描述吧。我C写得多一些，所以perl代码的风格不怎么perl。

#d:\perl\bin\perl.exe -w
use warnings;

use Data::Dumper;
use threads;
use threads::shared;
use Thread::Queue;
use Thread::Semaphore;

use Bloom::Filter;
use URI::URL;
use LWP::Simple;
use LWP::UserAgent;
use HTTP::Cookies;
use Web::Scraper;

my $max_threads = 30;
my $base_url = $ARGV[0] || 'http://www.icylife.net/';
my $host = URI->new($base_url)->host;

my $queue = Thread::Queue->new( );
my $semaphore = Thread::Semaphore->new( $max_threads );
my $filter= shared_clone(Bloom::Filter->new(capacity => 50000, error_rate => 0.001) );

$queue->enqueue( $base_url );
$filter->add( $base_url );

while( 1 )
{
# join all threads which can be joined
foreach ( threads->list(threads::joinable) )
{
$_->join( );
}

# if there are no url need process.
my $item = $queue->pending();
if( $item == 0 )
{
# there are no active thread, we finish the job
if( threads->list(threads::running) == 0 )
{
   print "All done!n";
   last;
}
# we will get some more url if there are some active threads, just wait for them
else
{
   sleep 1;
   next;
}
}

# if there are some url need process
while( $semaphore->down )
{
threads->create( &ProcessUrl );
}
}

# join all threads which can be joined
foreach ( threads->list() )
{
$_->join( );
}

sub ProcessUrl
{
my $scraper = scraper
{
process '//a', 'links[]' => '@href';
};

my $res;
my $link;

while( my $url = $queue->dequeue_nb() )
{
    eval
    {
        $res = $scraper->scrape( URI->new($url) )->{'links'};
    };
    if( $@ )
    {
        warn "$@\n";
        next;
    }
    next if (! defined $res );

#print "there are ".scalar(threads->list(threads::running))." threads, ", $queue->pending(), " urls need process.n";

foreach( @{$res} )
{
   $link = $_->as_string;
   $link = URI::URL->new($link, $url);

   # not http and not https?
   next if( $link->scheme ne 'http' && $link->scheme ne 'https' );
   # another domain?
   next if( $link->host ne $host );

   $link = $link->abs->as_string;

   if( $link =~ /(.*?)#(.*)/ )
   {
    $link = $1;
   }

   next if( $link =~ /.(jpg|png|zip|rar|iso)$/i );

   if( ! $filter->check($link) )
   {
    print $filter->key_count(), " ", $link, "n";
    $filter->add($link);
    $queue->enqueue($link);
   }
}
}
$semaphore->up( );
}