use strict;
use warnings;

#Hello,
#
#Currently I'm facing problem with comparing two huges files on a particular key
#column. One file consists of 10k records and other one 18million records. Both
#files are | (pipe) delimited. I am comparing based on the first column in the
#two files and redirecting them to two separate files.
#If Key columns are same it has to pick the record from 10k records file and send
#it to one file.
#If the Key columns are not matching ie., the key column is present in 18 million
#records file but not in 10k records file, it has to go into another file.
#
#Here I'm pasting the query what I have written, taking more time.

my $oldFile1 = <<DAT;
1|oldFile|another field
5|oldFile
DAT
my $newFile1 = <<DAT;
1|newFile1|z
2|newFile1|x
3|newFile1|y
4|newFile1|p
DAT
my $oldFile2;
my $changes1;


open OLDFILE1, '<', \$oldFile1;

# Build the reference hash from the 'small' file
my %oldKeys;

while (<OLDFILE1>) {
    chomp;
    my ($key, $tail) = split /\|/, $_, 2;

    if (exists $oldKeys{$key}) {
        warn "Key $key duplicated. Duplicate ignored!\n";
        next;
    }

    $oldKeys{$key} = $tail;
}

close OLDFILE1;

# Process the new file
open NEWFILE1, '<', \$newFile1;
open OLDFILE2, '>', \$oldFile2;
open CHANGES1, '>', \$changes1;
while (<NEWFILE1>) {
    chomp;
    my ($key, $tail) = split /\|/, $_, 2;

    if (exists $oldKeys{$key}) {
        print OLDFILE2 "$key|$oldKeys{$key}\n";
    } else {
        print CHANGES1 "$key|$tail\n";
    }
}

close (NEWFILE1);
close (OLDFILE2);
close (CHANGES1);

print "OLDFILE2:\n$oldFile2\n\n";
print "CHANGES1:\n$changes1\n\n";