in reply to Merging multidimensional hashes from forks to parent hash
I believe I understood what you were trying to do, but as I had not looked at Hash::Merge, I guess I went about things the long way. The sample code below ran a maximum of 65 children at a rate of a maximum of 5 simultaneous children. To simulate a workload, each generates a "filename" and picks 4 keys from a set of 8 possible, and generates a value for each key. When returned to the parent, the data is merged into a single HoH (level 1: filename; level 2: key), with a "total" of the keys is also generated. The data is then dumped at the end of the script. (Sample data run follows in <readmore></readmore>.)
#!/usr/bin/env perl use strict; use warnings; use Data::Dumper; use Getopt::Long; use Parallel::ForkManager; $Data::Dumper::Deepcopy = 1; $Data::Dumper::Sortkeys = 1; $| = 1; my $max_child = 5; my %parent_hash = (); my $pm = Parallel::ForkManager->new($max_child); $pm->run_on_finish( sub { my ( $pid, $exit_code, $process_ident, $exit_signal, $core_dump, $ds_ref, ) = @_; if ( defined $ds_ref ) { foreach my $k1 ( sort { $a cmp $b } keys %{$ds_ref} ) { foreach my $k2 ( keys %{ $ds_ref->{$k1} } ) { $parent_hash{$k1}{$k2} += $ds_ref->{$k1}{$k2}; $parent_hash{total}{$k2} += $ds_ref->{$k1}{$k2}; } } } }, ); foreach ( my $i = 0 ; $i < 128 ; $i += 8 ) { $pm->start and next; sleep $i; srand(); # So children have different random seeds my $child_hash; foreach my $j ( 0 .. int( rand() * 5 + 1 ) ) { my $fn = sprintf qq{file%05d.dat}, $i + $j; my @letters = ( 'a' .. 'h', ); foreach my $k ( 0 .. 3 ) { my $l; while ( $l = int( rand() * scalar @letters ) ) { last unless ( exists $child_hash->{$fn}{ $letters[$l] } ); } my $m = int( rand() * 20 ); $child_hash->{$fn}{ $letters[$l] } = $m; } } print Data::Dumper->Dump( [ \$i, \$child_hash, ], [qw( *i *child_hash )] ), qq{\n}; $pm->finish( 0, $child_hash, ); } $pm->wait_all_children; print Data::Dumper->Dump( [ \%parent_hash, ], [qw( *parent_hash )] ), qq{\n};
Test run output:
$ perl test-20180712.pl
$i = \0;
$child_hash = \{
'file00000.dat' => {
'c' => 1,
'd' => 12,
'g' => 19,
'h' => 13
},
'file00001.dat' => {
'a' => 14,
'e' => 3,
'f' => 17
},
'file00002.dat' => {
'b' => 8,
'd' => 19,
'g' => 17,
'h' => 17
},
'file00003.dat' => {
'a' => 15,
'd' => 11,
'g' => 1
},
'file00004.dat' => {
'a' => 12,
'b' => 3,
'c' => 17,
'e' => 2
},
'file00005.dat' => {
'a' => 15,
'c' => 9,
'e' => 18,
'g' => 1
}
};
$i = \8;
$child_hash = \{
'file00008.dat' => {
'c' => 9,
'f' => 15,
'g' => 19,
'h' => 11
},
'file00009.dat' => {
'b' => 4,
'c' => 16,
'd' => 6,
'f' => 4
},
'file00010.dat' => {
'a' => 6,
'c' => 9,
'g' => 8,
'h' => 17
}
};
$i = \16;
$child_hash = \{
'file00016.dat' => {
'a' => 12,
'd' => 13,
'g' => 12,
'h' => 16
},
'file00017.dat' => {
'a' => 18,
'c' => 12,
'd' => 16
},
'file00018.dat' => {
'a' => 16,
'b' => 12,
'd' => 1,
'f' => 17
},
'file00019.dat' => {
'a' => 4,
'e' => 18,
'h' => 6
},
'file00020.dat' => {
'b' => 8,
'd' => 7,
'e' => 6,
'g' => 1
},
'file00021.dat' => {
'a' => 16,
'c' => 17,
'd' => 13,
'h' => 2
}
};
$i = \24;
$child_hash = \{
'file00024.dat' => {
'c' => 0,
'e' => 13,
'f' => 19,
'h' => 3
},
'file00025.dat' => {
'a' => 7,
'e' => 13,
'f' => 12,
'h' => 16
}
};
$i = \32;
$child_hash = \{
'file00032.dat' => {
'a' => 12,
'b' => 15,
'g' => 6,
'h' => 16
},
'file00033.dat' => {
'a' => 4,
'c' => 5,
'f' => 17,
'h' => 12
},
'file00034.dat' => {
'a' => 18,
'd' => 18,
'f' => 0,
'h' => 5
},
'file00035.dat' => {
'b' => 13,
'f' => 15,
'g' => 5,
'h' => 13
},
'file00036.dat' => {
'a' => 9,
'c' => 6,
'e' => 16,
'f' => 17
}
};
$i = \40;
$child_hash = \{
'file00040.dat' => {
'c' => 10,
'f' => 0,
'g' => 3,
'h' => 18
},
'file00041.dat' => {
'a' => 14,
'd' => 1,
'e' => 9,
'g' => 19
}
};
$i = \48;
$child_hash = \{
'file00048.dat' => {
'a' => 18,
'b' => 12,
'c' => 0,
'e' => 0
},
'file00049.dat' => {
'a' => 16,
'b' => 10,
'c' => 4,
'e' => 16
},
'file00050.dat' => {
'b' => 12,
'e' => 8,
'f' => 2,
'h' => 1
},
'file00051.dat' => {
'c' => 16,
'd' => 1,
'g' => 19,
'h' => 1
}
};
$i = \56;
$child_hash = \{
'file00056.dat' => {
'b' => 6,
'd' => 4,
'e' => 19,
'f' => 15
},
'file00057.dat' => {
'a' => 4,
'f' => 8,
'h' => 11
},
'file00058.dat' => {
'b' => 12,
'c' => 18,
'f' => 3,
'h' => 16
},
'file00059.dat' => {
'a' => 14,
'b' => 14,
'c' => 4,
'd' => 10
},
'file00060.dat' => {
'a' => 17,
'b' => 9,
'e' => 6,
'f' => 1
}
};
$i = \64;
$child_hash = \{
'file00064.dat' => {
'b' => 19,
'c' => 7,
'e' => 5,
'g' => 9
},
'file00065.dat' => {
'a' => 14,
'c' => 17,
'e' => 8,
'g' => 16
},
'file00066.dat' => {
'e' => 4,
'f' => 1,
'g' => 10,
'h' => 0
}
};
$i = \72;
$child_hash = \{
'file00072.dat' => {
'b' => 7,
'd' => 12,
'g' => 9,
'h' => 5
},
'file00073.dat' => {
'b' => 11,
'd' => 18,
'e' => 9,
'g' => 15
},
'file00074.dat' => {
'b' => 3,
'd' => 15,
'e' => 13,
'h' => 1
},
'file00075.dat' => {
'a' => 6,
'c' => 5,
'e' => 2,
'g' => 17
}
};
$i = \80;
$child_hash = \{
'file00080.dat' => {
'a' => 8,
'b' => 9,
'd' => 1,
'g' => 4
},
'file00081.dat' => {
'a' => 18,
'e' => 10,
'g' => 15
},
'file00082.dat' => {
'a' => 8,
'c' => 7,
'f' => 12,
'g' => 15
},
'file00083.dat' => {
'b' => 2,
'c' => 9,
'e' => 15,
'f' => 1
},
'file00084.dat' => {
'b' => 10,
'd' => 0,
'f' => 2,
'g' => 7
},
'file00085.dat' => {
'a' => 12,
'b' => 2,
'e' => 12,
'h' => 13
}
};
$i = \88;
$child_hash = \{
'file00088.dat' => {
'b' => 5,
'd' => 3,
'e' => 8,
'f' => 11
},
'file00089.dat' => {
'a' => 10,
'e' => 13,
'f' => 13,
'g' => 12
},
'file00090.dat' => {
'b' => 3,
'f' => 11,
'g' => 10,
'h' => 19
},
'file00091.dat' => {
'c' => 13,
'd' => 18,
'f' => 5,
'h' => 16
},
'file00092.dat' => {
'a' => 8,
'b' => 12,
'd' => 18,
'g' => 15
},
'file00093.dat' => {
'c' => 4,
'e' => 17,
'g' => 2,
'h' => 14
}
};
$i = \96;
$child_hash = \{
'file00096.dat' => {
'a' => 12,
'c' => 19,
'd' => 4,
'e' => 13
},
'file00097.dat' => {
'a' => 7,
'd' => 14,
'e' => 8,
'f' => 18
},
'file00098.dat' => {
'a' => 13,
'f' => 0,
'g' => 18,
'h' => 13
},
'file00099.dat' => {
'a' => 19,
'c' => 13,
'd' => 12,
'g' => 2
},
'file00100.dat' => {
'c' => 14,
'd' => 10,
'e' => 6,
'f' => 11
},
'file00101.dat' => {
'd' => 9,
'e' => 17,
'f' => 0,
'h' => 5
}
};
$i = \104;
$child_hash = \{
'file00104.dat' => {
'b' => 6,
'c' => 19,
'd' => 17,
'e' => 13
},
'file00105.dat' => {
'b' => 19,
'd' => 10,
'f' => 7,
'h' => 5
},
'file00106.dat' => {
'a' => 5,
'b' => 2,
'c' => 5,
'g' => 10
},
'file00107.dat' => {
'a' => 18,
'c' => 4,
'e' => 17,
'f' => 16
},
'file00108.dat' => {
'b' => 3,
'c' => 12,
'e' => 5,
'h' => 4
},
'file00109.dat' => {
'a' => 8,
'b' => 3,
'c' => 2,
'f' => 0
}
};
$i = \112;
$child_hash = \{
'file00112.dat' => {
'a' => 17,
'd' => 7,
'f' => 13,
'h' => 6
},
'file00113.dat' => {
'a' => 18,
'd' => 6,
'h' => 15
},
'file00114.dat' => {
'c' => 6,
'd' => 19,
'f' => 10,
'g' => 8
}
};
$i = \120;
$child_hash = \{
'file00120.dat' => {
'a' => 2,
'b' => 17,
'c' => 10,
'd' => 7
},
'file00121.dat' => {
'c' => 13,
'd' => 19,
'e' => 10,
'g' => 14
},
'file00122.dat' => {
'a' => 6,
'b' => 12,
'f' => 17,
'h' => 9
},
'file00123.dat' => {
'b' => 6,
'c' => 19,
'd' => 4,
'f' => 13
}
};
%parent_hash = (
'file00000.dat' => {
'c' => 1,
'd' => 12,
'g' => 19,
'h' => 13
},
'file00001.dat' => {
'a' => 14,
'e' => 3,
'f' => 17
},
'file00002.dat' => {
'b' => 8,
'd' => 19,
'g' => 17,
'h' => 17
},
'file00003.dat' => {
'a' => 15,
'd' => 11,
'g' => 1
},
'file00004.dat' => {
'a' => 12,
'b' => 3,
'c' => 17,
'e' => 2
},
'file00005.dat' => {
'a' => 15,
'c' => 9,
'e' => 18,
'g' => 1
},
'file00008.dat' => {
'c' => 9,
'f' => 15,
'g' => 19,
'h' => 11
},
'file00009.dat' => {
'b' => 4,
'c' => 16,
'd' => 6,
'f' => 4
},
'file00010.dat' => {
'a' => 6,
'c' => 9,
'g' => 8,
'h' => 17
},
'file00016.dat' => {
'a' => 12,
'd' => 13,
'g' => 12,
'h' => 16
},
'file00017.dat' => {
'a' => 18,
'c' => 12,
'd' => 16
},
'file00018.dat' => {
'a' => 16,
'b' => 12,
'd' => 1,
'f' => 17
},
'file00019.dat' => {
'a' => 4,
'e' => 18,
'h' => 6
},
'file00020.dat' => {
'b' => 8,
'd' => 7,
'e' => 6,
'g' => 1
},
'file00021.dat' => {
'a' => 16,
'c' => 17,
'd' => 13,
'h' => 2
},
'file00024.dat' => {
'c' => 0,
'e' => 13,
'f' => 19,
'h' => 3
},
'file00025.dat' => {
'a' => 7,
'e' => 13,
'f' => 12,
'h' => 16
},
'file00032.dat' => {
'a' => 12,
'b' => 15,
'g' => 6,
'h' => 16
},
'file00033.dat' => {
'a' => 4,
'c' => 5,
'f' => 17,
'h' => 12
},
'file00034.dat' => {
'a' => 18,
'd' => 18,
'f' => 0,
'h' => 5
},
'file00035.dat' => {
'b' => 13,
'f' => 15,
'g' => 5,
'h' => 13
},
'file00036.dat' => {
'a' => 9,
'c' => 6,
'e' => 16,
'f' => 17
},
'file00040.dat' => {
'c' => 10,
'f' => 0,
'g' => 3,
'h' => 18
},
'file00041.dat' => {
'a' => 14,
'd' => 1,
'e' => 9,
'g' => 19
},
'file00048.dat' => {
'a' => 18,
'b' => 12,
'c' => 0,
'e' => 0
},
'file00049.dat' => {
'a' => 16,
'b' => 10,
'c' => 4,
'e' => 16
},
'file00050.dat' => {
'b' => 12,
'e' => 8,
'f' => 2,
'h' => 1
},
'file00051.dat' => {
'c' => 16,
'd' => 1,
'g' => 19,
'h' => 1
},
'file00056.dat' => {
'b' => 6,
'd' => 4,
'e' => 19,
'f' => 15
},
'file00057.dat' => {
'a' => 4,
'f' => 8,
'h' => 11
},
'file00058.dat' => {
'b' => 12,
'c' => 18,
'f' => 3,
'h' => 16
},
'file00059.dat' => {
'a' => 14,
'b' => 14,
'c' => 4,
'd' => 10
},
'file00060.dat' => {
'a' => 17,
'b' => 9,
'e' => 6,
'f' => 1
},
'file00064.dat' => {
'b' => 19,
'c' => 7,
'e' => 5,
'g' => 9
},
'file00065.dat' => {
'a' => 14,
'c' => 17,
'e' => 8,
'g' => 16
},
'file00066.dat' => {
'e' => 4,
'f' => 1,
'g' => 10,
'h' => 0
},
'file00072.dat' => {
'b' => 7,
'd' => 12,
'g' => 9,
'h' => 5
},
'file00073.dat' => {
'b' => 11,
'd' => 18,
'e' => 9,
'g' => 15
},
'file00074.dat' => {
'b' => 3,
'd' => 15,
'e' => 13,
'h' => 1
},
'file00075.dat' => {
'a' => 6,
'c' => 5,
'e' => 2,
'g' => 17
},
'file00080.dat' => {
'a' => 8,
'b' => 9,
'd' => 1,
'g' => 4
},
'file00081.dat' => {
'a' => 18,
'e' => 10,
'g' => 15
},
'file00082.dat' => {
'a' => 8,
'c' => 7,
'f' => 12,
'g' => 15
},
'file00083.dat' => {
'b' => 2,
'c' => 9,
'e' => 15,
'f' => 1
},
'file00084.dat' => {
'b' => 10,
'd' => 0,
'f' => 2,
'g' => 7
},
'file00085.dat' => {
'a' => 12,
'b' => 2,
'e' => 12,
'h' => 13
},
'file00088.dat' => {
'b' => 5,
'd' => 3,
'e' => 8,
'f' => 11
},
'file00089.dat' => {
'a' => 10,
'e' => 13,
'f' => 13,
'g' => 12
},
'file00090.dat' => {
'b' => 3,
'f' => 11,
'g' => 10,
'h' => 19
},
'file00091.dat' => {
'c' => 13,
'd' => 18,
'f' => 5,
'h' => 16
},
'file00092.dat' => {
'a' => 8,
'b' => 12,
'd' => 18,
'g' => 15
},
'file00093.dat' => {
'c' => 4,
'e' => 17,
'g' => 2,
'h' => 14
},
'file00096.dat' => {
'a' => 12,
'c' => 19,
'd' => 4,
'e' => 13
},
'file00097.dat' => {
'a' => 7,
'd' => 14,
'e' => 8,
'f' => 18
},
'file00098.dat' => {
'a' => 13,
'f' => 0,
'g' => 18,
'h' => 13
},
'file00099.dat' => {
'a' => 19,
'c' => 13,
'd' => 12,
'g' => 2
},
'file00100.dat' => {
'c' => 14,
'd' => 10,
'e' => 6,
'f' => 11
},
'file00101.dat' => {
'd' => 9,
'e' => 17,
'f' => 0,
'h' => 5
},
'file00104.dat' => {
'b' => 6,
'c' => 19,
'd' => 17,
'e' => 13
},
'file00105.dat' => {
'b' => 19,
'd' => 10,
'f' => 7,
'h' => 5
},
'file00106.dat' => {
'a' => 5,
'b' => 2,
'c' => 5,
'g' => 10
},
'file00107.dat' => {
'a' => 18,
'c' => 4,
'e' => 17,
'f' => 16
},
'file00108.dat' => {
'b' => 3,
'c' => 12,
'e' => 5,
'h' => 4
},
'file00109.dat' => {
'a' => 8,
'b' => 3,
'c' => 2,
'f' => 0
},
'file00112.dat' => {
'a' => 17,
'd' => 7,
'f' => 13,
'h' => 6
},
'file00113.dat' => {
'a' => 18,
'd' => 6,
'h' => 15
},
'file00114.dat' => {
'c' => 6,
'd' => 19,
'f' => 10,
'g' => 8
},
'file00120.dat' => {
'a' => 2,
'b' => 17,
'c' => 10,
'd' => 7
},
'file00121.dat' => {
'c' => 13,
'd' => 19,
'e' => 10,
'g' => 14
},
'file00122.dat' => {
'a' => 6,
'b' => 12,
'f' => 17,
'h' => 9
},
'file00123.dat' => {
'b' => 6,
'c' => 19,
'd' => 4,
'f' => 13
},
'total' => {
'a' => 470,
'b' => 289,
'c' => 351,
'd' => 355,
'e' => 352,
'f' => 323,
'g' => 338,
'h' => 319
}
);
$
Hope that helps.
|
|---|
In Section
Seekers of Perl Wisdom