# -*- Mode: cperl; coding: utf-8; cperl-indent-level: 4 -*-
# vim: ts=4 sts=4 sw=4:
=head1 NAME
CPAN::Mirrors - Get CPAN miror information and select a fast one
=head1 SYNOPSIS
use CPAN::Mirrors;
my $mirrors = CPAN::Mirrors->new;
$mirrors->parse_from_file( $mirrored_by_file );
my $seen = {};
my $best_continent = $mirrors->find_best_continents( { seen => $seen } );
my @mirrors = $mirrors->get_mirrors_by_continents( $best_continent );
my $callback = sub {
my( $m ) = @_;
printf "%s = %s\n", $m->hostname, $m->rtt
};
$mirrors->get_mirrors_timings( \@mirrors, $seen, $callback );
@mirrors = sort { $a->rtt <=> $b->rtt } @mirrors;
print "Best mirrors are ", map( { $_->rtt } @mirrors[0..3] ), "\n";
=head1 DESCRIPTION
=over
=cut
package CPAN::Mirrors;
use strict;
use vars qw($VERSION $urllist $silent);
$VERSION = "1.9600";
use Carp;
use FileHandle;
use Fcntl ":flock";
use Net::Ping ();
=item new( LOCAL_FILE_NAME )
=cut
sub new {
my ($class, $file) = @_;
my $self = bless {
mirrors => [],
geography => {},
}, $class;
if( defined $file ) {
$self->parse_mirrored_by( $file );
}
return $self
}
sub parse_mirrored_by {
my ($self, $file) = @_;
my $handle = FileHandle->new;
$handle->open($file)
or croak "Couldn't open $file: $!";
flock $handle, LOCK_SH;
$self->_parse($file,$handle);
flock $handle, LOCK_UN;
$handle->close;
}
=item continents()
Return a list of continents based on those defined in F<MIRRORED.BY>.
=cut
sub continents {
my ($self) = @_;
return keys %{$self->{geography}};
}
=item countries( [CONTINENTS] )
Return a list of countries based on those defined in F<MIRRORED.BY>.
It only returns countries for the continents you specify (as defined
in C<continents>). If you don't specify any continents, it returns all
of the countries listed in F<MIRRORED.BY>.
=cut
sub countries {
my ($self, @continents) = @_;
@continents = $self->continents unless @continents;
my @countries;
for my $c (@continents) {
push @countries, keys %{ $self->{geography}{$c} };
}
return @countries;
}
=item mirrors( [COUNTRIES] )
Return a list of mirrors based on those defined in F<MIRRORED.BY>.
It only returns mirrors for the countries you specify (as defined
in C<countries>). If you don't specify any countries, it returns all
of the mirrors listed in F<MIRRORED.BY>.
=cut
sub mirrors {
my ($self, @countries) = @_;
return @{$self->{mirrors}} unless @countries;
my %wanted = map { $_ => 1 } @countries;
my @found;
for my $m (@{$self->{mirrors}}) {
push @found, $m if exists $wanted{$m->country};
}
return @found;
}
=item get_mirrors_by_countries( [COUNTRIES] )
A more sensible synonym for mirrors.
=cut
sub get_mirrors_by_countries { &mirrors }
=item get_mirrors_by_continents( [CONTINENTS] )
Return a list of mirrors for all of continents you specify. If you don't
specify any continents, it returns all of the mirrors.
=cut
sub get_mirrors_by_continents {
my ($self, $continents ) = @_;
$self->mirrors( $self->get_countries_by_continents( @$continents ) );
}
=item get_countries_by_continents( [CONTINENTS] )
A more sensible synonym for countries.
=cut
sub get_countries_by_continents { &countries }
=item best_mirrors
C<best_mirrors> checks for the best mirrors based on the list of
continents you pass, or, without that, all continents, as defined
by C<CPAN::Mirrored::By>. It pings each mirror, up to the value of
C<how_many>. In list context, it returns up to C<how_many> mirror.
In scalar context, it returns the single best mirror.
Arguments
how_many - the number of mirrors to return. Default: 1
callback - a callback for find_best_continents
verbose - true or false on all the whining and moaning. Default: false
continents - an array ref of the continents to check
If you don't specify the continents, C<best_mirrors> calls
C<find_best_continents> to get the list of continents to check.
=cut
sub best_mirrors {
my ($self, %args) = @_;
my $how_many = $args{how_many} || 1;
my $callback = $args{callback};
my $verbose = defined $args{verbose} ? $args{verbose} : 0;
my $continents = $args{continents} || [];
$continents = [$continents] unless ref $continents;
# Old Net::Ping did not do timings at all
return "http://www.cpan.org/" unless Net::Ping->VERSION gt '2.13';
my $seen = {};
if ( ! @$continents ) {
print "Searching for the best continent ...\n" if $verbose;
my @best_continents = $self->find_best_continents(
seen => $seen,
verbose => $verbose,
callback => $callback,
);
# Only add enough continents to find enough mirrors
my $count = 0;
for my $continent ( @best_continents ) {
push @$continents, $continent;
$count += $self->mirrors( $self->countries($continent) );
last if $count >= $how_many;
}
}
print "Scanning " . join(", ", @$continents) . " ...\n" if $verbose;
my $trial_mirrors = $self->get_n_random_mirrors_by_continents( 3 * $how_many, $continents->[0] );
my $timings = $self->get_mirrors_timings( $trial_mirrors, $seen, $callback );
return [] unless @$timings;
$how_many = @$timings if $how_many > @$timings;
return wantarray ? @{$timings}[0 .. $how_many-1] : $timings->[0];
}
=item get_n_random_mirrors_by_continents( N, [CONTINENTS]
Returns up to N random mirrors for the specified continents. Specify the
continents as an array reference.
=cut
sub get_n_random_mirrors_by_continents {
my( $self, $n, $continents ) = @_;
$n ||= 3;
$continents = [ $continents ] unless ref $continents;
if ( $n <= 0 ) {
return wantarray ? () : [];
}
my @long_list = $self->get_mirrors_by_continents( $continents );
if ( $n eq '*' or $n > @long_list ) {
return wantarray ? @long_list : \@long_list;
}
@long_list = map {$_->[0]}
sort {$a->[1] <=> $b->[1]}
map {[$_, rand]} @long_list;
splice @long_list, $n; # truncate
\@long_list;
}
=item get_mirrors_timings( MIRROR_LIST, SEEN, CALLBACK );
Pings the listed mirrors and returns a list of mirrors sorted
in ascending ping times.
=cut
sub get_mirrors_timings {
my( $self, $mirror_list, $seen, $callback ) = @_;
$seen = {} unless defined $seen;
croak "The mirror list argument must be an array reference"
unless ref $mirror_list eq ref [];
croak "The seen argument must be a hash reference"
unless ref $seen eq ref {};
croak "callback must be a subroutine"
if( defined $callback and ref $callback ne ref sub {} );
my $timings = [];
for my $m ( @$mirror_list ) {
$seen->{$m->hostname} = $m;
next unless eval{ $m->http };
if( $self->_try_a_ping( $seen, $m, ) ) {
my $ping = $m->ping;
next unless defined $ping;
push @$timings, $m;
$callback->( $m ) if $callback;
}
else {
push @$timings, $seen->{$m->hostname}
if defined $seen->{$m->hostname}->rtt;
}
}
my @best = sort {
if( defined $a->rtt and defined $b->rtt ) {
$a->rtt <=> $b->rtt
}
elsif( defined $a->rtt and ! defined $b->rtt ) {
return -1;
}
elsif( ! defined $a->rtt and defined $b->rtt ) {
return 1;
}
elsif( ! defined $a->rtt and ! defined $b->rtt ) {
return 0;
}
} @$timings;
return wantarray ? @best : \@best;
}
=item find_best_continents( HASH_REF );
C<find_best_continents> goes through each continent and pings C<N> random
mirrors on that continent. It then orders the continents by ascending
median ping time. In list context, it returns the ordered list of
continent. In scalar context, it returns the same list as an anonymous
array.
Arguments:
n - the number of hosts to ping for each continent. Default: 3
seen - a hashref of cached hostname ping times
verbose - true or false for noisy or quiet. Default: false
callback - a subroutine to run after each ping.
ping_cache_limit - how long, in seconds, to reuse previous ping times.
Default: 1 day
The C<seen> hash has hostnames as keys and anonymous arrays as values. The
anonymous array is a triplet of a C<CPAN::Mirrored::By> object, a ping
time, and the epoch time for the measurement.
The callback subroutine gets the C<CPAN::Mirrored::By> object, the ping
time, and measurement time (the same things in the C<seen> hashref) as arguments.
C<find_best_continents> doesn't care what the callback does and ignores the return
value.
=cut
sub find_best_continents {
my ($self, %args) = @_;
$args{n} ||= 3;
$args{verbose} = 0 unless defined $args{verbose};
$args{seen} = {} unless defined $args{seen};
croak "The seen argument must be a hash reference"
unless ref $args{seen} eq ref {};
$args{ping_cache_limit} = 24 * 60 * 60
unless defined $args{ping_cache_time};
croak "callback must be a subroutine"
if( defined $args{callback} and ref $args{callback} ne ref sub {} );
my %medians;
CONT: for my $c ( $self->continents ) {
print "Testing $c\n" if $args{verbose};
my @mirrors = $self->mirrors( $self->countries($c) );
next CONT unless @mirrors;
my $n = (@mirrors < $args{n}) ? @mirrors : $args{n};
my @tests;
my $tries = 0;
RANDOM: while ( @mirrors && @tests < $n && $tries++ < 15 ) {
my $m = splice( @mirrors, int(rand(@mirrors)), 1 );
if( $self->_try_a_ping( $args{seen}, $m, $args{ping_cache_limit} ) ) {
$self->get_mirrors_timings( [ $m ], @args{qw(seen callback)} );
next RANDOM unless defined $args{seen}{$m->hostname}->rtt;
}
printf "\t%s -> %0.2f ms\n",
$m->hostname,
join ' ', 1000 * $args{seen}{$m->hostname}->rtt
if $args{verbose};
push @tests, $args{seen}{$m->hostname}->rtt;
}
my $median = $self->_get_median_ping_time( \@tests, $args{verbose} );
$medians{$c} = $median if defined $median;
}
my @best_cont = sort { $medians{$a} <=> $medians{$b} } keys %medians;
if ( $args{verbose} ) {
print "Median result by continent:\n";
for my $c ( @best_cont ) {
printf( " %4d ms %s\n", int($medians{$c}*1000+.5), $c );
}
}
return wantarray ? @best_cont : $best_cont[0];
}
# retry if
sub _try_a_ping {
my ($self, $seen, $mirror, $ping_cache_limit ) = @_;
( ! exists $seen->{$mirror->hostname} )
or
(
! defined $seen->{$mirror->hostname}->rtt
or
time - $seen->{$mirror->hostname}->rtt > $ping_cache_limit
)
}
sub _get_median_ping_time {
my ($self, $tests, $verbose ) = @_;
my @sorted = sort { $a <=> $b } @$tests;
my $median = do {
if ( @sorted == 0 ) { undef }
elsif ( @sorted == 1 ) { $sorted[0] }
elsif ( @sorted % 2 ) { $sorted[ int(@sorted / 2) ] }
else {
my $mid_high = int(@sorted/2);
($sorted[$mid_high-1] + $sorted[$mid_high])/2;
}
};
printf "\t-->median time: %0.2f ms\n", $median * 1000 if $verbose;
return $median;
}
# Adapted from Parse::CPAN::MirroredBy by Adam Kennedy
sub _parse {
my ($self, $file, $handle) = @_;
my $output = $self->{mirrors};
my $geo = $self->{geography};
local $/ = "\012";
my $line = 0;
my $mirror = undef;
while ( 1 ) {
# Next line
my $string = <$handle>;
last if ! defined $string;
$line = $line + 1;
# Remove the useless lines
chomp( $string );
next if $string =~ /^\s*$/;
next if $string =~ /^\s*#/;
# Hostname or property?
if ( $string =~ /^\s/ ) {
# Property
unless ( $string =~ /^\s+(\w+)\s+=\s+\"(.*)\"$/ ) {
croak("Invalid property on line $line");
}
my ($prop, $value) = ($1,$2);
$mirror ||= {};
if ( $prop eq 'dst_location' ) {
my (@location,$continent,$country);
@location = (split /\s*,\s*/, $value)
and ($continent, $country) = @location[-1,-2];
$continent =~ s/\s\(.*//;
$continent =~ s/\W+$//; # if Jarkko doesn't know latitude/longitude
$geo->{$continent}{$country} = 1 if $continent && $country;
$mirror->{continent} = $continent || "unknown";
$mirror->{country} = $country || "unknown";
}
elsif ( $prop eq 'dst_http' ) {
$mirror->{http} = $value;
}
elsif ( $prop eq 'dst_ftp' ) {
$mirror->{ftp} = $value;
}
elsif ( $prop eq 'dst_rsync' ) {
$mirror->{rsync} = $value;
}
else {
$prop =~ s/^dst_//;
$mirror->{$prop} = $value;
}
} else {
# Hostname
unless ( $string =~ /^([\w\.-]+)\:\s*$/ ) {
croak("Invalid host name on line $line");
}
my $current = $mirror;
$mirror = { hostname => "$1" };
if ( $current ) {
push @$output, CPAN::Mirrored::By->new($current);
}
}
}
if ( $mirror ) {
push @$output, CPAN::Mirrored::By->new($mirror);
}
return;
}
#--------------------------------------------------------------------------#
package CPAN::Mirrored::By;
use strict;
use Net::Ping ();
sub new {
my($self,$arg) = @_;
$arg ||= {};
bless $arg, $self;
}
sub hostname { shift->{hostname} }
sub continent { shift->{continent} }
sub country { shift->{country} }
sub http { shift->{http} || '' }
sub ftp { shift->{ftp} || '' }
sub rsync { shift->{rsync} || '' }
sub rtt { shift->{rtt} }
sub ping_time { shift->{ping_time} }
sub url {
my $self = shift;
return $self->{http} || $self->{ftp};
}
sub ping {
my $self = shift;
my $ping = Net::Ping->new("tcp",1);
my ($proto) = $self->url =~ m{^([^:]+)};
my $port = $proto eq 'http' ? 80 : 21;
return unless $port;
if ( $ping->can('port_number') ) {
$ping->port_number($port);
}
else {
$ping->{'port_num'} = $port;
}
$ping->hires(1) if $ping->can('hires');
my ($alive,$rtt) = $ping->ping($self->hostname);
$self->{rtt} = $alive ? $rtt : undef;
$self->{ping_time} = time;
$self->rtt;
}
1;
=back
=head1 AUTHOR
Andreas Koenig C<< <andk@cpan.org> >>, David Golden C<< <dagolden@cpan.org> >>,
brian d foy C<< <bdfoy@cpan.org> >>
=head1 LICENSE
This program is free software; you can redistribute it and/or
modify it under the same terms as Perl itself.
See L<http://www.perl.com/perl/misc/Artistic.html>
=cut
Mini Shell