my $dir = '/path/to/the/files';
opendir(my $d, $dir) or die("opendir() failed: $!");

my @sort_files = sort { -M($b) <=> -M($a) }
    grep { -f($_) } readdir($d);

unlink(map { "$dir/$_" } @f[0, 1]) if (@sort_files > 20);