in reply to Merging multidimensional hashes from forks to parent hash
I believe I understood what you were trying to do, but as I had not looked at Hash::Merge, I guess I went about things the long way. The sample code below ran a maximum of 65 children at a rate of a maximum of 5 simultaneous children. To simulate a workload, each generates a "filename" and picks 4 keys from a set of 8 possible, and generates a value for each key. When returned to the parent, the data is merged into a single HoH (level 1: filename; level 2: key), with a "total" of the keys is also generated. The data is then dumped at the end of the script. (Sample data run follows in <readmore></readmore>.)
#!/usr/bin/env perl use strict; use warnings; use Data::Dumper; use Getopt::Long; use Parallel::ForkManager; $Data::Dumper::Deepcopy = 1; $Data::Dumper::Sortkeys = 1; $| = 1; my $max_child = 5; my %parent_hash = (); my $pm = Parallel::ForkManager->new($max_child); $pm->run_on_finish( sub { my ( $pid, $exit_code, $process_ident, $exit_signal, $core_dump, $ds_ref, ) = @_; if ( defined $ds_ref ) { foreach my $k1 ( sort { $a cmp $b } keys %{$ds_ref} ) { foreach my $k2 ( keys %{ $ds_ref->{$k1} } ) { $parent_hash{$k1}{$k2} += $ds_ref->{$k1}{$k2}; $parent_hash{total}{$k2} += $ds_ref->{$k1}{$k2}; } } } }, ); foreach ( my $i = 0 ; $i < 128 ; $i += 8 ) { $pm->start and next; sleep $i; srand(); # So children have different random seeds my $child_hash; foreach my $j ( 0 .. int( rand() * 5 + 1 ) ) { my $fn = sprintf qq{file%05d.dat}, $i + $j; my @letters = ( 'a' .. 'h', ); foreach my $k ( 0 .. 3 ) { my $l; while ( $l = int( rand() * scalar @letters ) ) { last unless ( exists $child_hash->{$fn}{ $letters[$l] } ); } my $m = int( rand() * 20 ); $child_hash->{$fn}{ $letters[$l] } = $m; } } print Data::Dumper->Dump( [ \$i, \$child_hash, ], [qw( *i *child_hash )] ), qq{\n}; $pm->finish( 0, $child_hash, ); } $pm->wait_all_children; print Data::Dumper->Dump( [ \%parent_hash, ], [qw( *parent_hash )] ), qq{\n};
Test run output:
$ perl test-20180712.pl $i = \0; $child_hash = \{ 'file00000.dat' => { 'c' => 1, 'd' => 12, 'g' => 19, 'h' => 13 }, 'file00001.dat' => { 'a' => 14, 'e' => 3, 'f' => 17 }, 'file00002.dat' => { 'b' => 8, 'd' => 19, 'g' => 17, 'h' => 17 }, 'file00003.dat' => { 'a' => 15, 'd' => 11, 'g' => 1 }, 'file00004.dat' => { 'a' => 12, 'b' => 3, 'c' => 17, 'e' => 2 }, 'file00005.dat' => { 'a' => 15, 'c' => 9, 'e' => 18, 'g' => 1 } }; $i = \8; $child_hash = \{ 'file00008.dat' => { 'c' => 9, 'f' => 15, 'g' => 19, 'h' => 11 }, 'file00009.dat' => { 'b' => 4, 'c' => 16, 'd' => 6, 'f' => 4 }, 'file00010.dat' => { 'a' => 6, 'c' => 9, 'g' => 8, 'h' => 17 } }; $i = \16; $child_hash = \{ 'file00016.dat' => { 'a' => 12, 'd' => 13, 'g' => 12, 'h' => 16 }, 'file00017.dat' => { 'a' => 18, 'c' => 12, 'd' => 16 }, 'file00018.dat' => { 'a' => 16, 'b' => 12, 'd' => 1, 'f' => 17 }, 'file00019.dat' => { 'a' => 4, 'e' => 18, 'h' => 6 }, 'file00020.dat' => { 'b' => 8, 'd' => 7, 'e' => 6, 'g' => 1 }, 'file00021.dat' => { 'a' => 16, 'c' => 17, 'd' => 13, 'h' => 2 } }; $i = \24; $child_hash = \{ 'file00024.dat' => { 'c' => 0, 'e' => 13, 'f' => 19, 'h' => 3 }, 'file00025.dat' => { 'a' => 7, 'e' => 13, 'f' => 12, 'h' => 16 } }; $i = \32; $child_hash = \{ 'file00032.dat' => { 'a' => 12, 'b' => 15, 'g' => 6, 'h' => 16 }, 'file00033.dat' => { 'a' => 4, 'c' => 5, 'f' => 17, 'h' => 12 }, 'file00034.dat' => { 'a' => 18, 'd' => 18, 'f' => 0, 'h' => 5 }, 'file00035.dat' => { 'b' => 13, 'f' => 15, 'g' => 5, 'h' => 13 }, 'file00036.dat' => { 'a' => 9, 'c' => 6, 'e' => 16, 'f' => 17 } }; $i = \40; $child_hash = \{ 'file00040.dat' => { 'c' => 10, 'f' => 0, 'g' => 3, 'h' => 18 }, 'file00041.dat' => { 'a' => 14, 'd' => 1, 'e' => 9, 'g' => 19 } }; $i = \48; $child_hash = \{ 'file00048.dat' => { 'a' => 18, 'b' => 12, 'c' => 0, 'e' => 0 }, 'file00049.dat' => { 'a' => 16, 'b' => 10, 'c' => 4, 'e' => 16 }, 'file00050.dat' => { 'b' => 12, 'e' => 8, 'f' => 2, 'h' => 1 }, 'file00051.dat' => { 'c' => 16, 'd' => 1, 'g' => 19, 'h' => 1 } }; $i = \56; $child_hash = \{ 'file00056.dat' => { 'b' => 6, 'd' => 4, 'e' => 19, 'f' => 15 }, 'file00057.dat' => { 'a' => 4, 'f' => 8, 'h' => 11 }, 'file00058.dat' => { 'b' => 12, 'c' => 18, 'f' => 3, 'h' => 16 }, 'file00059.dat' => { 'a' => 14, 'b' => 14, 'c' => 4, 'd' => 10 }, 'file00060.dat' => { 'a' => 17, 'b' => 9, 'e' => 6, 'f' => 1 } }; $i = \64; $child_hash = \{ 'file00064.dat' => { 'b' => 19, 'c' => 7, 'e' => 5, 'g' => 9 }, 'file00065.dat' => { 'a' => 14, 'c' => 17, 'e' => 8, 'g' => 16 }, 'file00066.dat' => { 'e' => 4, 'f' => 1, 'g' => 10, 'h' => 0 } }; $i = \72; $child_hash = \{ 'file00072.dat' => { 'b' => 7, 'd' => 12, 'g' => 9, 'h' => 5 }, 'file00073.dat' => { 'b' => 11, 'd' => 18, 'e' => 9, 'g' => 15 }, 'file00074.dat' => { 'b' => 3, 'd' => 15, 'e' => 13, 'h' => 1 }, 'file00075.dat' => { 'a' => 6, 'c' => 5, 'e' => 2, 'g' => 17 } }; $i = \80; $child_hash = \{ 'file00080.dat' => { 'a' => 8, 'b' => 9, 'd' => 1, 'g' => 4 }, 'file00081.dat' => { 'a' => 18, 'e' => 10, 'g' => 15 }, 'file00082.dat' => { 'a' => 8, 'c' => 7, 'f' => 12, 'g' => 15 }, 'file00083.dat' => { 'b' => 2, 'c' => 9, 'e' => 15, 'f' => 1 }, 'file00084.dat' => { 'b' => 10, 'd' => 0, 'f' => 2, 'g' => 7 }, 'file00085.dat' => { 'a' => 12, 'b' => 2, 'e' => 12, 'h' => 13 } }; $i = \88; $child_hash = \{ 'file00088.dat' => { 'b' => 5, 'd' => 3, 'e' => 8, 'f' => 11 }, 'file00089.dat' => { 'a' => 10, 'e' => 13, 'f' => 13, 'g' => 12 }, 'file00090.dat' => { 'b' => 3, 'f' => 11, 'g' => 10, 'h' => 19 }, 'file00091.dat' => { 'c' => 13, 'd' => 18, 'f' => 5, 'h' => 16 }, 'file00092.dat' => { 'a' => 8, 'b' => 12, 'd' => 18, 'g' => 15 }, 'file00093.dat' => { 'c' => 4, 'e' => 17, 'g' => 2, 'h' => 14 } }; $i = \96; $child_hash = \{ 'file00096.dat' => { 'a' => 12, 'c' => 19, 'd' => 4, 'e' => 13 }, 'file00097.dat' => { 'a' => 7, 'd' => 14, 'e' => 8, 'f' => 18 }, 'file00098.dat' => { 'a' => 13, 'f' => 0, 'g' => 18, 'h' => 13 }, 'file00099.dat' => { 'a' => 19, 'c' => 13, 'd' => 12, 'g' => 2 }, 'file00100.dat' => { 'c' => 14, 'd' => 10, 'e' => 6, 'f' => 11 }, 'file00101.dat' => { 'd' => 9, 'e' => 17, 'f' => 0, 'h' => 5 } }; $i = \104; $child_hash = \{ 'file00104.dat' => { 'b' => 6, 'c' => 19, 'd' => 17, 'e' => 13 }, 'file00105.dat' => { 'b' => 19, 'd' => 10, 'f' => 7, 'h' => 5 }, 'file00106.dat' => { 'a' => 5, 'b' => 2, 'c' => 5, 'g' => 10 }, 'file00107.dat' => { 'a' => 18, 'c' => 4, 'e' => 17, 'f' => 16 }, 'file00108.dat' => { 'b' => 3, 'c' => 12, 'e' => 5, 'h' => 4 }, 'file00109.dat' => { 'a' => 8, 'b' => 3, 'c' => 2, 'f' => 0 } }; $i = \112; $child_hash = \{ 'file00112.dat' => { 'a' => 17, 'd' => 7, 'f' => 13, 'h' => 6 }, 'file00113.dat' => { 'a' => 18, 'd' => 6, 'h' => 15 }, 'file00114.dat' => { 'c' => 6, 'd' => 19, 'f' => 10, 'g' => 8 } }; $i = \120; $child_hash = \{ 'file00120.dat' => { 'a' => 2, 'b' => 17, 'c' => 10, 'd' => 7 }, 'file00121.dat' => { 'c' => 13, 'd' => 19, 'e' => 10, 'g' => 14 }, 'file00122.dat' => { 'a' => 6, 'b' => 12, 'f' => 17, 'h' => 9 }, 'file00123.dat' => { 'b' => 6, 'c' => 19, 'd' => 4, 'f' => 13 } }; %parent_hash = ( 'file00000.dat' => { 'c' => 1, 'd' => 12, 'g' => 19, 'h' => 13 }, 'file00001.dat' => { 'a' => 14, 'e' => 3, 'f' => 17 }, 'file00002.dat' => { 'b' => 8, 'd' => 19, 'g' => 17, 'h' => 17 }, 'file00003.dat' => { 'a' => 15, 'd' => 11, 'g' => 1 }, 'file00004.dat' => { 'a' => 12, 'b' => 3, 'c' => 17, 'e' => 2 }, 'file00005.dat' => { 'a' => 15, 'c' => 9, 'e' => 18, 'g' => 1 }, 'file00008.dat' => { 'c' => 9, 'f' => 15, 'g' => 19, 'h' => 11 }, 'file00009.dat' => { 'b' => 4, 'c' => 16, 'd' => 6, 'f' => 4 }, 'file00010.dat' => { 'a' => 6, 'c' => 9, 'g' => 8, 'h' => 17 }, 'file00016.dat' => { 'a' => 12, 'd' => 13, 'g' => 12, 'h' => 16 }, 'file00017.dat' => { 'a' => 18, 'c' => 12, 'd' => 16 }, 'file00018.dat' => { 'a' => 16, 'b' => 12, 'd' => 1, 'f' => 17 }, 'file00019.dat' => { 'a' => 4, 'e' => 18, 'h' => 6 }, 'file00020.dat' => { 'b' => 8, 'd' => 7, 'e' => 6, 'g' => 1 }, 'file00021.dat' => { 'a' => 16, 'c' => 17, 'd' => 13, 'h' => 2 }, 'file00024.dat' => { 'c' => 0, 'e' => 13, 'f' => 19, 'h' => 3 }, 'file00025.dat' => { 'a' => 7, 'e' => 13, 'f' => 12, 'h' => 16 }, 'file00032.dat' => { 'a' => 12, 'b' => 15, 'g' => 6, 'h' => 16 }, 'file00033.dat' => { 'a' => 4, 'c' => 5, 'f' => 17, 'h' => 12 }, 'file00034.dat' => { 'a' => 18, 'd' => 18, 'f' => 0, 'h' => 5 }, 'file00035.dat' => { 'b' => 13, 'f' => 15, 'g' => 5, 'h' => 13 }, 'file00036.dat' => { 'a' => 9, 'c' => 6, 'e' => 16, 'f' => 17 }, 'file00040.dat' => { 'c' => 10, 'f' => 0, 'g' => 3, 'h' => 18 }, 'file00041.dat' => { 'a' => 14, 'd' => 1, 'e' => 9, 'g' => 19 }, 'file00048.dat' => { 'a' => 18, 'b' => 12, 'c' => 0, 'e' => 0 }, 'file00049.dat' => { 'a' => 16, 'b' => 10, 'c' => 4, 'e' => 16 }, 'file00050.dat' => { 'b' => 12, 'e' => 8, 'f' => 2, 'h' => 1 }, 'file00051.dat' => { 'c' => 16, 'd' => 1, 'g' => 19, 'h' => 1 }, 'file00056.dat' => { 'b' => 6, 'd' => 4, 'e' => 19, 'f' => 15 }, 'file00057.dat' => { 'a' => 4, 'f' => 8, 'h' => 11 }, 'file00058.dat' => { 'b' => 12, 'c' => 18, 'f' => 3, 'h' => 16 }, 'file00059.dat' => { 'a' => 14, 'b' => 14, 'c' => 4, 'd' => 10 }, 'file00060.dat' => { 'a' => 17, 'b' => 9, 'e' => 6, 'f' => 1 }, 'file00064.dat' => { 'b' => 19, 'c' => 7, 'e' => 5, 'g' => 9 }, 'file00065.dat' => { 'a' => 14, 'c' => 17, 'e' => 8, 'g' => 16 }, 'file00066.dat' => { 'e' => 4, 'f' => 1, 'g' => 10, 'h' => 0 }, 'file00072.dat' => { 'b' => 7, 'd' => 12, 'g' => 9, 'h' => 5 }, 'file00073.dat' => { 'b' => 11, 'd' => 18, 'e' => 9, 'g' => 15 }, 'file00074.dat' => { 'b' => 3, 'd' => 15, 'e' => 13, 'h' => 1 }, 'file00075.dat' => { 'a' => 6, 'c' => 5, 'e' => 2, 'g' => 17 }, 'file00080.dat' => { 'a' => 8, 'b' => 9, 'd' => 1, 'g' => 4 }, 'file00081.dat' => { 'a' => 18, 'e' => 10, 'g' => 15 }, 'file00082.dat' => { 'a' => 8, 'c' => 7, 'f' => 12, 'g' => 15 }, 'file00083.dat' => { 'b' => 2, 'c' => 9, 'e' => 15, 'f' => 1 }, 'file00084.dat' => { 'b' => 10, 'd' => 0, 'f' => 2, 'g' => 7 }, 'file00085.dat' => { 'a' => 12, 'b' => 2, 'e' => 12, 'h' => 13 }, 'file00088.dat' => { 'b' => 5, 'd' => 3, 'e' => 8, 'f' => 11 }, 'file00089.dat' => { 'a' => 10, 'e' => 13, 'f' => 13, 'g' => 12 }, 'file00090.dat' => { 'b' => 3, 'f' => 11, 'g' => 10, 'h' => 19 }, 'file00091.dat' => { 'c' => 13, 'd' => 18, 'f' => 5, 'h' => 16 }, 'file00092.dat' => { 'a' => 8, 'b' => 12, 'd' => 18, 'g' => 15 }, 'file00093.dat' => { 'c' => 4, 'e' => 17, 'g' => 2, 'h' => 14 }, 'file00096.dat' => { 'a' => 12, 'c' => 19, 'd' => 4, 'e' => 13 }, 'file00097.dat' => { 'a' => 7, 'd' => 14, 'e' => 8, 'f' => 18 }, 'file00098.dat' => { 'a' => 13, 'f' => 0, 'g' => 18, 'h' => 13 }, 'file00099.dat' => { 'a' => 19, 'c' => 13, 'd' => 12, 'g' => 2 }, 'file00100.dat' => { 'c' => 14, 'd' => 10, 'e' => 6, 'f' => 11 }, 'file00101.dat' => { 'd' => 9, 'e' => 17, 'f' => 0, 'h' => 5 }, 'file00104.dat' => { 'b' => 6, 'c' => 19, 'd' => 17, 'e' => 13 }, 'file00105.dat' => { 'b' => 19, 'd' => 10, 'f' => 7, 'h' => 5 }, 'file00106.dat' => { 'a' => 5, 'b' => 2, 'c' => 5, 'g' => 10 }, 'file00107.dat' => { 'a' => 18, 'c' => 4, 'e' => 17, 'f' => 16 }, 'file00108.dat' => { 'b' => 3, 'c' => 12, 'e' => 5, 'h' => 4 }, 'file00109.dat' => { 'a' => 8, 'b' => 3, 'c' => 2, 'f' => 0 }, 'file00112.dat' => { 'a' => 17, 'd' => 7, 'f' => 13, 'h' => 6 }, 'file00113.dat' => { 'a' => 18, 'd' => 6, 'h' => 15 }, 'file00114.dat' => { 'c' => 6, 'd' => 19, 'f' => 10, 'g' => 8 }, 'file00120.dat' => { 'a' => 2, 'b' => 17, 'c' => 10, 'd' => 7 }, 'file00121.dat' => { 'c' => 13, 'd' => 19, 'e' => 10, 'g' => 14 }, 'file00122.dat' => { 'a' => 6, 'b' => 12, 'f' => 17, 'h' => 9 }, 'file00123.dat' => { 'b' => 6, 'c' => 19, 'd' => 4, 'f' => 13 }, 'total' => { 'a' => 470, 'b' => 289, 'c' => 351, 'd' => 355, 'e' => 352, 'f' => 323, 'g' => 338, 'h' => 319 } ); $
Hope that helps.
|
---|
In Section
Seekers of Perl Wisdom