Hello
I am attempting to fetch some .gz files in a small web spider.
First I get the robots.txt file and look for a sitemap entry. --- This works fine
Next, I'll download sitemap.xml, and parsing that fetch the files that are in the links --- Works great for text based files, and although it appears as though it works (using getstore), for binary files, after it finishes, the files aren't there
I,m not sure what's wrong, or even if getstore can handle binary files (seems to me I have used it for .gifs, etc. with success)
The getstore is in the text subroutine
Here's the code
#!/usr/bin/env perl
#
# Name: TestFetch.pl
#
# Requires Internet access
#
use strict;
use warnings;
use LWP::Simple;
use HTML::Parser;
# Global Variables
my $debug = 1;
package MyParser;
my $sitetrigger = 0;
my $lastmodtrigger = 0;
my $tofile = "";
my $pos = -1;
use base qw(HTML::Parser);
sub start {
my ($self, $tagname, $attr, $attrseq, $origtext) = @_;
$sitetrigger = 0;
$lastmodtrigger = 0;
if((index $tagname, "loc") ne -1) { $sitetrigger = 1; }
if((index $tagname, "lastmod") ne -1) { $lastmodtrigger = 1; }
if($debug == 1) {
print "------------START-----------\n";
print "tagname: $tagname\n";
}
}
sub text {
my ($self, $text) = @_;
my $filename = "";
if($sitetrigger == 1) {
$filename = "";
$pos = rindex($text, '/', );
if($pos ne -1) {
$filename = substr($text, ($pos + 1));
}
print "fetching: $text into $filename\n";
LWP::Simple->getstore ($text, $filename);
sleep(6);
}
if($debug == 1) {
print "------------TEXT-----------\n";
print "sitetrigger: $sitetrigger\n";
print "lastmodtrigger: $lastmodtrigger\n";
print "filename: $filename\n";
print "text: $text\n";
}
}
sub end {
my($self, $end, $origtext) = @_;
if($debug == 1) {
print "------------END-----------\n";
print "end: $end\n";
}
}
package main;
my $htmlparse = new MyParser;
my $loc = "";
my $siteurl;
my $filefound = 0;
my $pos1 = -1;
my $content = "";
my $url = $ARGV[0];
$loc = $url . '/robots.txt';
if($loc ne "") {
if ($debug == 1) { print "loc: $loc\n"; }
getstore($loc, 'robots.txt') or die "Couldn't get robots.txt";
open IN, 'robots.txt' or die $!;
while (<IN>) {
$pos1 = index (uc $_, 'SITEMAP');
if($pos1 ne -1) {
$siteurl = substr($_, ($pos1 + 8));
if ($debug == 1) { print "siteurl: $siteurl\n"; }
$content = get($siteurl);
$filefound = 1;
last;
}
}
close IN or die "IN: $!";
if($filefound == 1) { $htmlparse->parse($content); }
}
Any assistance would be appreciated.
Thanks
Largins#!/usr/bin/env perl
#
# Name: TestFetch.pl
#
# Requires Internet access
#
use strict;
use warnings;
use LWP::Simple;
use HTML::Parser;
# Global Variables
my $debug = 1;
package MyParser;
my $sitetrigger = 0;
my $lastmodtrigger = 0;
my $tofile = lastmodtrigger: $lastmodtrigger\n
