Beefy Boxes and Bandwidth Generously Provided by pair Networks
Do you know where your variables are?
 
PerlMonks  

Re: Merging multidimensional hashes from forks to parent hash

by atcroft (Abbot)
on Jul 12, 2018 at 20:01 UTC ( #1218403=note: print w/replies, xml ) Need Help??


in reply to Merging multidimensional hashes from forks to parent hash

I believe I understood what you were trying to do, but as I had not looked at Hash::Merge, I guess I went about things the long way. The sample code below ran a maximum of 65 children at a rate of a maximum of 5 simultaneous children. To simulate a workload, each generates a "filename" and picks 4 keys from a set of 8 possible, and generates a value for each key. When returned to the parent, the data is merged into a single HoH (level 1: filename; level 2: key), with a "total" of the keys is also generated. The data is then dumped at the end of the script. (Sample data run follows in <readmore></readmore>.)

#!/usr/bin/env perl use strict; use warnings; use Data::Dumper; use Getopt::Long; use Parallel::ForkManager; $Data::Dumper::Deepcopy = 1; $Data::Dumper::Sortkeys = 1; $| = 1; my $max_child = 5; my %parent_hash = (); my $pm = Parallel::ForkManager->new($max_child); $pm->run_on_finish( sub { my ( $pid, $exit_code, $process_ident, $exit_signal, $core_dump, $ds_ref, ) = @_; if ( defined $ds_ref ) { foreach my $k1 ( sort { $a cmp $b } keys %{$ds_ref} ) { foreach my $k2 ( keys %{ $ds_ref->{$k1} } ) { $parent_hash{$k1}{$k2} += $ds_ref->{$k1}{$k2}; $parent_hash{total}{$k2} += $ds_ref->{$k1}{$k2}; } } } }, ); foreach ( my $i = 0 ; $i < 128 ; $i += 8 ) { $pm->start and next; sleep $i; srand(); # So children have different random seeds my $child_hash; foreach my $j ( 0 .. int( rand() * 5 + 1 ) ) { my $fn = sprintf qq{file%05d.dat}, $i + $j; my @letters = ( 'a' .. 'h', ); foreach my $k ( 0 .. 3 ) { my $l; while ( $l = int( rand() * scalar @letters ) ) { last unless ( exists $child_hash->{$fn}{ $letters[$l] } ); } my $m = int( rand() * 20 ); $child_hash->{$fn}{ $letters[$l] } = $m; } } print Data::Dumper->Dump( [ \$i, \$child_hash, ], [qw( *i *child_hash )] ), qq{\n}; $pm->finish( 0, $child_hash, ); } $pm->wait_all_children; print Data::Dumper->Dump( [ \%parent_hash, ], [qw( *parent_hash )] ), qq{\n};

Test run output:

$ perl test-20180712.pl
$i = \0;
$child_hash = \{
                  'file00000.dat' => {
                                       'c' => 1,
                                       'd' => 12,
                                       'g' => 19,
                                       'h' => 13
                                     },
                  'file00001.dat' => {
                                       'a' => 14,
                                       'e' => 3,
                                       'f' => 17
                                     },
                  'file00002.dat' => {
                                       'b' => 8,
                                       'd' => 19,
                                       'g' => 17,
                                       'h' => 17
                                     },
                  'file00003.dat' => {
                                       'a' => 15,
                                       'd' => 11,
                                       'g' => 1
                                     },
                  'file00004.dat' => {
                                       'a' => 12,
                                       'b' => 3,
                                       'c' => 17,
                                       'e' => 2
                                     },
                  'file00005.dat' => {
                                       'a' => 15,
                                       'c' => 9,
                                       'e' => 18,
                                       'g' => 1
                                     }
                };

$i = \8;
$child_hash = \{
                  'file00008.dat' => {
                                       'c' => 9,
                                       'f' => 15,
                                       'g' => 19,
                                       'h' => 11
                                     },
                  'file00009.dat' => {
                                       'b' => 4,
                                       'c' => 16,
                                       'd' => 6,
                                       'f' => 4
                                     },
                  'file00010.dat' => {
                                       'a' => 6,
                                       'c' => 9,
                                       'g' => 8,
                                       'h' => 17
                                     }
                };

$i = \16;
$child_hash = \{
                  'file00016.dat' => {
                                       'a' => 12,
                                       'd' => 13,
                                       'g' => 12,
                                       'h' => 16
                                     },
                  'file00017.dat' => {
                                       'a' => 18,
                                       'c' => 12,
                                       'd' => 16
                                     },
                  'file00018.dat' => {
                                       'a' => 16,
                                       'b' => 12,
                                       'd' => 1,
                                       'f' => 17
                                     },
                  'file00019.dat' => {
                                       'a' => 4,
                                       'e' => 18,
                                       'h' => 6
                                     },
                  'file00020.dat' => {
                                       'b' => 8,
                                       'd' => 7,
                                       'e' => 6,
                                       'g' => 1
                                     },
                  'file00021.dat' => {
                                       'a' => 16,
                                       'c' => 17,
                                       'd' => 13,
                                       'h' => 2
                                     }
                };

$i = \24;
$child_hash = \{
                  'file00024.dat' => {
                                       'c' => 0,
                                       'e' => 13,
                                       'f' => 19,
                                       'h' => 3
                                     },
                  'file00025.dat' => {
                                       'a' => 7,
                                       'e' => 13,
                                       'f' => 12,
                                       'h' => 16
                                     }
                };

$i = \32;
$child_hash = \{
                  'file00032.dat' => {
                                       'a' => 12,
                                       'b' => 15,
                                       'g' => 6,
                                       'h' => 16
                                     },
                  'file00033.dat' => {
                                       'a' => 4,
                                       'c' => 5,
                                       'f' => 17,
                                       'h' => 12
                                     },
                  'file00034.dat' => {
                                       'a' => 18,
                                       'd' => 18,
                                       'f' => 0,
                                       'h' => 5
                                     },
                  'file00035.dat' => {
                                       'b' => 13,
                                       'f' => 15,
                                       'g' => 5,
                                       'h' => 13
                                     },
                  'file00036.dat' => {
                                       'a' => 9,
                                       'c' => 6,
                                       'e' => 16,
                                       'f' => 17
                                     }
                };

$i = \40;
$child_hash = \{
                  'file00040.dat' => {
                                       'c' => 10,
                                       'f' => 0,
                                       'g' => 3,
                                       'h' => 18
                                     },
                  'file00041.dat' => {
                                       'a' => 14,
                                       'd' => 1,
                                       'e' => 9,
                                       'g' => 19
                                     }
                };

$i = \48;
$child_hash = \{
                  'file00048.dat' => {
                                       'a' => 18,
                                       'b' => 12,
                                       'c' => 0,
                                       'e' => 0
                                     },
                  'file00049.dat' => {
                                       'a' => 16,
                                       'b' => 10,
                                       'c' => 4,
                                       'e' => 16
                                     },
                  'file00050.dat' => {
                                       'b' => 12,
                                       'e' => 8,
                                       'f' => 2,
                                       'h' => 1
                                     },
                  'file00051.dat' => {
                                       'c' => 16,
                                       'd' => 1,
                                       'g' => 19,
                                       'h' => 1
                                     }
                };

$i = \56;
$child_hash = \{
                  'file00056.dat' => {
                                       'b' => 6,
                                       'd' => 4,
                                       'e' => 19,
                                       'f' => 15
                                     },
                  'file00057.dat' => {
                                       'a' => 4,
                                       'f' => 8,
                                       'h' => 11
                                     },
                  'file00058.dat' => {
                                       'b' => 12,
                                       'c' => 18,
                                       'f' => 3,
                                       'h' => 16
                                     },
                  'file00059.dat' => {
                                       'a' => 14,
                                       'b' => 14,
                                       'c' => 4,
                                       'd' => 10
                                     },
                  'file00060.dat' => {
                                       'a' => 17,
                                       'b' => 9,
                                       'e' => 6,
                                       'f' => 1
                                     }
                };

$i = \64;
$child_hash = \{
                  'file00064.dat' => {
                                       'b' => 19,
                                       'c' => 7,
                                       'e' => 5,
                                       'g' => 9
                                     },
                  'file00065.dat' => {
                                       'a' => 14,
                                       'c' => 17,
                                       'e' => 8,
                                       'g' => 16
                                     },
                  'file00066.dat' => {
                                       'e' => 4,
                                       'f' => 1,
                                       'g' => 10,
                                       'h' => 0
                                     }
                };

$i = \72;
$child_hash = \{
                  'file00072.dat' => {
                                       'b' => 7,
                                       'd' => 12,
                                       'g' => 9,
                                       'h' => 5
                                     },
                  'file00073.dat' => {
                                       'b' => 11,
                                       'd' => 18,
                                       'e' => 9,
                                       'g' => 15
                                     },
                  'file00074.dat' => {
                                       'b' => 3,
                                       'd' => 15,
                                       'e' => 13,
                                       'h' => 1
                                     },
                  'file00075.dat' => {
                                       'a' => 6,
                                       'c' => 5,
                                       'e' => 2,
                                       'g' => 17
                                     }
                };

$i = \80;
$child_hash = \{
                  'file00080.dat' => {
                                       'a' => 8,
                                       'b' => 9,
                                       'd' => 1,
                                       'g' => 4
                                     },
                  'file00081.dat' => {
                                       'a' => 18,
                                       'e' => 10,
                                       'g' => 15
                                     },
                  'file00082.dat' => {
                                       'a' => 8,
                                       'c' => 7,
                                       'f' => 12,
                                       'g' => 15
                                     },
                  'file00083.dat' => {
                                       'b' => 2,
                                       'c' => 9,
                                       'e' => 15,
                                       'f' => 1
                                     },
                  'file00084.dat' => {
                                       'b' => 10,
                                       'd' => 0,
                                       'f' => 2,
                                       'g' => 7
                                     },
                  'file00085.dat' => {
                                       'a' => 12,
                                       'b' => 2,
                                       'e' => 12,
                                       'h' => 13
                                     }
                };

$i = \88;
$child_hash = \{
                  'file00088.dat' => {
                                       'b' => 5,
                                       'd' => 3,
                                       'e' => 8,
                                       'f' => 11
                                     },
                  'file00089.dat' => {
                                       'a' => 10,
                                       'e' => 13,
                                       'f' => 13,
                                       'g' => 12
                                     },
                  'file00090.dat' => {
                                       'b' => 3,
                                       'f' => 11,
                                       'g' => 10,
                                       'h' => 19
                                     },
                  'file00091.dat' => {
                                       'c' => 13,
                                       'd' => 18,
                                       'f' => 5,
                                       'h' => 16
                                     },
                  'file00092.dat' => {
                                       'a' => 8,
                                       'b' => 12,
                                       'd' => 18,
                                       'g' => 15
                                     },
                  'file00093.dat' => {
                                       'c' => 4,
                                       'e' => 17,
                                       'g' => 2,
                                       'h' => 14
                                     }
                };

$i = \96;
$child_hash = \{
                  'file00096.dat' => {
                                       'a' => 12,
                                       'c' => 19,
                                       'd' => 4,
                                       'e' => 13
                                     },
                  'file00097.dat' => {
                                       'a' => 7,
                                       'd' => 14,
                                       'e' => 8,
                                       'f' => 18
                                     },
                  'file00098.dat' => {
                                       'a' => 13,
                                       'f' => 0,
                                       'g' => 18,
                                       'h' => 13
                                     },
                  'file00099.dat' => {
                                       'a' => 19,
                                       'c' => 13,
                                       'd' => 12,
                                       'g' => 2
                                     },
                  'file00100.dat' => {
                                       'c' => 14,
                                       'd' => 10,
                                       'e' => 6,
                                       'f' => 11
                                     },
                  'file00101.dat' => {
                                       'd' => 9,
                                       'e' => 17,
                                       'f' => 0,
                                       'h' => 5
                                     }
                };

$i = \104;
$child_hash = \{
                  'file00104.dat' => {
                                       'b' => 6,
                                       'c' => 19,
                                       'd' => 17,
                                       'e' => 13
                                     },
                  'file00105.dat' => {
                                       'b' => 19,
                                       'd' => 10,
                                       'f' => 7,
                                       'h' => 5
                                     },
                  'file00106.dat' => {
                                       'a' => 5,
                                       'b' => 2,
                                       'c' => 5,
                                       'g' => 10
                                     },
                  'file00107.dat' => {
                                       'a' => 18,
                                       'c' => 4,
                                       'e' => 17,
                                       'f' => 16
                                     },
                  'file00108.dat' => {
                                       'b' => 3,
                                       'c' => 12,
                                       'e' => 5,
                                       'h' => 4
                                     },
                  'file00109.dat' => {
                                       'a' => 8,
                                       'b' => 3,
                                       'c' => 2,
                                       'f' => 0
                                     }
                };

$i = \112;
$child_hash = \{
                  'file00112.dat' => {
                                       'a' => 17,
                                       'd' => 7,
                                       'f' => 13,
                                       'h' => 6
                                     },
                  'file00113.dat' => {
                                       'a' => 18,
                                       'd' => 6,
                                       'h' => 15
                                     },
                  'file00114.dat' => {
                                       'c' => 6,
                                       'd' => 19,
                                       'f' => 10,
                                       'g' => 8
                                     }
                };

$i = \120;
$child_hash = \{
                  'file00120.dat' => {
                                       'a' => 2,
                                       'b' => 17,
                                       'c' => 10,
                                       'd' => 7
                                     },
                  'file00121.dat' => {
                                       'c' => 13,
                                       'd' => 19,
                                       'e' => 10,
                                       'g' => 14
                                     },
                  'file00122.dat' => {
                                       'a' => 6,
                                       'b' => 12,
                                       'f' => 17,
                                       'h' => 9
                                     },
                  'file00123.dat' => {
                                       'b' => 6,
                                       'c' => 19,
                                       'd' => 4,
                                       'f' => 13
                                     }
                };

%parent_hash = (
                 'file00000.dat' => {
                                      'c' => 1,
                                      'd' => 12,
                                      'g' => 19,
                                      'h' => 13
                                    },
                 'file00001.dat' => {
                                      'a' => 14,
                                      'e' => 3,
                                      'f' => 17
                                    },
                 'file00002.dat' => {
                                      'b' => 8,
                                      'd' => 19,
                                      'g' => 17,
                                      'h' => 17
                                    },
                 'file00003.dat' => {
                                      'a' => 15,
                                      'd' => 11,
                                      'g' => 1
                                    },
                 'file00004.dat' => {
                                      'a' => 12,
                                      'b' => 3,
                                      'c' => 17,
                                      'e' => 2
                                    },
                 'file00005.dat' => {
                                      'a' => 15,
                                      'c' => 9,
                                      'e' => 18,
                                      'g' => 1
                                    },
                 'file00008.dat' => {
                                      'c' => 9,
                                      'f' => 15,
                                      'g' => 19,
                                      'h' => 11
                                    },
                 'file00009.dat' => {
                                      'b' => 4,
                                      'c' => 16,
                                      'd' => 6,
                                      'f' => 4
                                    },
                 'file00010.dat' => {
                                      'a' => 6,
                                      'c' => 9,
                                      'g' => 8,
                                      'h' => 17
                                    },
                 'file00016.dat' => {
                                      'a' => 12,
                                      'd' => 13,
                                      'g' => 12,
                                      'h' => 16
                                    },
                 'file00017.dat' => {
                                      'a' => 18,
                                      'c' => 12,
                                      'd' => 16
                                    },
                 'file00018.dat' => {
                                      'a' => 16,
                                      'b' => 12,
                                      'd' => 1,
                                      'f' => 17
                                    },
                 'file00019.dat' => {
                                      'a' => 4,
                                      'e' => 18,
                                      'h' => 6
                                    },
                 'file00020.dat' => {
                                      'b' => 8,
                                      'd' => 7,
                                      'e' => 6,
                                      'g' => 1
                                    },
                 'file00021.dat' => {
                                      'a' => 16,
                                      'c' => 17,
                                      'd' => 13,
                                      'h' => 2
                                    },
                 'file00024.dat' => {
                                      'c' => 0,
                                      'e' => 13,
                                      'f' => 19,
                                      'h' => 3
                                    },
                 'file00025.dat' => {
                                      'a' => 7,
                                      'e' => 13,
                                      'f' => 12,
                                      'h' => 16
                                    },
                 'file00032.dat' => {
                                      'a' => 12,
                                      'b' => 15,
                                      'g' => 6,
                                      'h' => 16
                                    },
                 'file00033.dat' => {
                                      'a' => 4,
                                      'c' => 5,
                                      'f' => 17,
                                      'h' => 12
                                    },
                 'file00034.dat' => {
                                      'a' => 18,
                                      'd' => 18,
                                      'f' => 0,
                                      'h' => 5
                                    },
                 'file00035.dat' => {
                                      'b' => 13,
                                      'f' => 15,
                                      'g' => 5,
                                      'h' => 13
                                    },
                 'file00036.dat' => {
                                      'a' => 9,
                                      'c' => 6,
                                      'e' => 16,
                                      'f' => 17
                                    },
                 'file00040.dat' => {
                                      'c' => 10,
                                      'f' => 0,
                                      'g' => 3,
                                      'h' => 18
                                    },
                 'file00041.dat' => {
                                      'a' => 14,
                                      'd' => 1,
                                      'e' => 9,
                                      'g' => 19
                                    },
                 'file00048.dat' => {
                                      'a' => 18,
                                      'b' => 12,
                                      'c' => 0,
                                      'e' => 0
                                    },
                 'file00049.dat' => {
                                      'a' => 16,
                                      'b' => 10,
                                      'c' => 4,
                                      'e' => 16
                                    },
                 'file00050.dat' => {
                                      'b' => 12,
                                      'e' => 8,
                                      'f' => 2,
                                      'h' => 1
                                    },
                 'file00051.dat' => {
                                      'c' => 16,
                                      'd' => 1,
                                      'g' => 19,
                                      'h' => 1
                                    },
                 'file00056.dat' => {
                                      'b' => 6,
                                      'd' => 4,
                                      'e' => 19,
                                      'f' => 15
                                    },
                 'file00057.dat' => {
                                      'a' => 4,
                                      'f' => 8,
                                      'h' => 11
                                    },
                 'file00058.dat' => {
                                      'b' => 12,
                                      'c' => 18,
                                      'f' => 3,
                                      'h' => 16
                                    },
                 'file00059.dat' => {
                                      'a' => 14,
                                      'b' => 14,
                                      'c' => 4,
                                      'd' => 10
                                    },
                 'file00060.dat' => {
                                      'a' => 17,
                                      'b' => 9,
                                      'e' => 6,
                                      'f' => 1
                                    },
                 'file00064.dat' => {
                                      'b' => 19,
                                      'c' => 7,
                                      'e' => 5,
                                      'g' => 9
                                    },
                 'file00065.dat' => {
                                      'a' => 14,
                                      'c' => 17,
                                      'e' => 8,
                                      'g' => 16
                                    },
                 'file00066.dat' => {
                                      'e' => 4,
                                      'f' => 1,
                                      'g' => 10,
                                      'h' => 0
                                    },
                 'file00072.dat' => {
                                      'b' => 7,
                                      'd' => 12,
                                      'g' => 9,
                                      'h' => 5
                                    },
                 'file00073.dat' => {
                                      'b' => 11,
                                      'd' => 18,
                                      'e' => 9,
                                      'g' => 15
                                    },
                 'file00074.dat' => {
                                      'b' => 3,
                                      'd' => 15,
                                      'e' => 13,
                                      'h' => 1
                                    },
                 'file00075.dat' => {
                                      'a' => 6,
                                      'c' => 5,
                                      'e' => 2,
                                      'g' => 17
                                    },
                 'file00080.dat' => {
                                      'a' => 8,
                                      'b' => 9,
                                      'd' => 1,
                                      'g' => 4
                                    },
                 'file00081.dat' => {
                                      'a' => 18,
                                      'e' => 10,
                                      'g' => 15
                                    },
                 'file00082.dat' => {
                                      'a' => 8,
                                      'c' => 7,
                                      'f' => 12,
                                      'g' => 15
                                    },
                 'file00083.dat' => {
                                      'b' => 2,
                                      'c' => 9,
                                      'e' => 15,
                                      'f' => 1
                                    },
                 'file00084.dat' => {
                                      'b' => 10,
                                      'd' => 0,
                                      'f' => 2,
                                      'g' => 7
                                    },
                 'file00085.dat' => {
                                      'a' => 12,
                                      'b' => 2,
                                      'e' => 12,
                                      'h' => 13
                                    },
                 'file00088.dat' => {
                                      'b' => 5,
                                      'd' => 3,
                                      'e' => 8,
                                      'f' => 11
                                    },
                 'file00089.dat' => {
                                      'a' => 10,
                                      'e' => 13,
                                      'f' => 13,
                                      'g' => 12
                                    },
                 'file00090.dat' => {
                                      'b' => 3,
                                      'f' => 11,
                                      'g' => 10,
                                      'h' => 19
                                    },
                 'file00091.dat' => {
                                      'c' => 13,
                                      'd' => 18,
                                      'f' => 5,
                                      'h' => 16
                                    },
                 'file00092.dat' => {
                                      'a' => 8,
                                      'b' => 12,
                                      'd' => 18,
                                      'g' => 15
                                    },
                 'file00093.dat' => {
                                      'c' => 4,
                                      'e' => 17,
                                      'g' => 2,
                                      'h' => 14
                                    },
                 'file00096.dat' => {
                                      'a' => 12,
                                      'c' => 19,
                                      'd' => 4,
                                      'e' => 13
                                    },
                 'file00097.dat' => {
                                      'a' => 7,
                                      'd' => 14,
                                      'e' => 8,
                                      'f' => 18
                                    },
                 'file00098.dat' => {
                                      'a' => 13,
                                      'f' => 0,
                                      'g' => 18,
                                      'h' => 13
                                    },
                 'file00099.dat' => {
                                      'a' => 19,
                                      'c' => 13,
                                      'd' => 12,
                                      'g' => 2
                                    },
                 'file00100.dat' => {
                                      'c' => 14,
                                      'd' => 10,
                                      'e' => 6,
                                      'f' => 11
                                    },
                 'file00101.dat' => {
                                      'd' => 9,
                                      'e' => 17,
                                      'f' => 0,
                                      'h' => 5
                                    },
                 'file00104.dat' => {
                                      'b' => 6,
                                      'c' => 19,
                                      'd' => 17,
                                      'e' => 13
                                    },
                 'file00105.dat' => {
                                      'b' => 19,
                                      'd' => 10,
                                      'f' => 7,
                                      'h' => 5
                                    },
                 'file00106.dat' => {
                                      'a' => 5,
                                      'b' => 2,
                                      'c' => 5,
                                      'g' => 10
                                    },
                 'file00107.dat' => {
                                      'a' => 18,
                                      'c' => 4,
                                      'e' => 17,
                                      'f' => 16
                                    },
                 'file00108.dat' => {
                                      'b' => 3,
                                      'c' => 12,
                                      'e' => 5,
                                      'h' => 4
                                    },
                 'file00109.dat' => {
                                      'a' => 8,
                                      'b' => 3,
                                      'c' => 2,
                                      'f' => 0
                                    },
                 'file00112.dat' => {
                                      'a' => 17,
                                      'd' => 7,
                                      'f' => 13,
                                      'h' => 6
                                    },
                 'file00113.dat' => {
                                      'a' => 18,
                                      'd' => 6,
                                      'h' => 15
                                    },
                 'file00114.dat' => {
                                      'c' => 6,
                                      'd' => 19,
                                      'f' => 10,
                                      'g' => 8
                                    },
                 'file00120.dat' => {
                                      'a' => 2,
                                      'b' => 17,
                                      'c' => 10,
                                      'd' => 7
                                    },
                 'file00121.dat' => {
                                      'c' => 13,
                                      'd' => 19,
                                      'e' => 10,
                                      'g' => 14
                                    },
                 'file00122.dat' => {
                                      'a' => 6,
                                      'b' => 12,
                                      'f' => 17,
                                      'h' => 9
                                    },
                 'file00123.dat' => {
                                      'b' => 6,
                                      'c' => 19,
                                      'd' => 4,
                                      'f' => 13
                                    },
                 'total' => {
                              'a' => 470,
                              'b' => 289,
                              'c' => 351,
                              'd' => 355,
                              'e' => 352,
                              'f' => 323,
                              'g' => 338,
                              'h' => 319
                            }
               );

$ 

Hope that helps.

Log In?
Username:
Password:

What's my password?
Create A New User
Node Status?
node history
Node Type: note [id://1218403]
help
Chatterbox?
and the web crawler heard nothing...

How do I use this? | Other CB clients
Other Users?
Others pondering the Monastery: (6)
As of 2020-09-24 09:12 GMT
Sections?
Information?
Find Nodes?
Leftovers?
    Voting Booth?
    If at first I donít succeed, I Ö










    Results (132 votes). Check out past polls.

    Notices?