#!/usr/bin/perl

use strict;
use warnings;

use LWP::RobotUA;
use HTML::SimpleLinkExtor;

sub grab_links {

  my ( $ua, $url ) = @_;

  my @links;
  my $response = $ua->get($url);

  if ($response->is_success) {
    my $extor   = HTML::SimpleLinkExtor->new();
    my $content = $response->content;

    $extor->parse($content);
    @links = $extor->a;   # get a ref links. Check docs - these are relative paths.

  } else {
    die $response->status_line;
  }

  return @links;
}

my $visit = $ARGV[0];

my $ua = LWP::RobotUA->new('my-robot/0.1', 'me@foo.com');  # Change this to suit.
$ua->delay( 0.1 );  # hit every 1/10 second

my @links = grab_links($ua, $visit);

my %uniq;

foreach ( @links ) {
  $uniq{$_}++;
}

print "Visited: ", $visit, " found these links:\n", join( "\n", keys %uniq), "\n";