Thanks to LeoNerd who kindly responded to my desperate queries at https://kiwiirc.com/nextclient/#irc://irc.perl.org/#perl the script now works. And confidence to calling-by-reference has been restored, Balans returned to the Universe.

LeoNerd suggested and implemented a new sv_setrv() which is how you stuff an array into an SV. He also suggested replacing av_push(av, (SV *)av2); with av_push(av, newRV_noinc((SV *)av2)); which pushes an arrayref to an array.

The following script now passes all cases and shows how to call an Inline::C function and returning results by ref with either an arrayref (Case1), a scalarref which we assign an arrayref to it (Case3) or a scalar which we assign an arrayref to it (Case2).

Definetely an addition to the sparse XS documentation and even sparser examples. A big thank you to LeoNerd.

use strict;
use warnings;

use Test::More;

use Inline C => Config =>
    BUILD_NOISY => 1,
    clean_after_build => 0,
    warnings => 10,
;
use Inline C => <<'EOC';
#include <stdio.h>

/************************************************************/

/* Monkeypatch by LeoNerd to set an arrayref into a scalarref
   As posted on https://kiwiirc.com/nextclient/#irc://irc.perl.org/#pe
+rl
   at 10:50 23/07/2021
   A BIG THANK YOU LeoNerd
*/

#define HAVE_PERL_VERSION(R, V, S) \
    (PERL_REVISION > (R) || (PERL_REVISION == (R) && (PERL_VERSION > (
+V) || (PERL_VERSION == (V) && (PERL_SUBVERSION >= (S))))))

#define sv_setrv(s, r)  S_sv_setrv(aTHX_ s, r)
static void S_sv_setrv(pTHX_ SV *sv, SV *rv)
{
  sv_setiv(sv, (IV)rv);
#if !HAVE_PERL_VERSION(5, 24, 0)
  SvIOK_off(sv);
#endif
  SvROK_on(sv);
}
/************************************************************/
int is_array_ref(
    SV *array,
    size_t *array_sz
){
    if( ! SvROK(array) ){ fprintf(stderr, "is_array_ref() : warning, i
+nput '%p' is not a reference.\n", array); return 0; }
    if( SvTYPE(SvRV(array)) != SVt_PVAV ){ fprintf(stderr, "is_array_r
+ef() : warning, input ref '%p' is not an ARRAY reference.\n", array);
+ return 0; }
    // it's an array, cast it to AV to get its len via av_len();
    // yes, av_len needs to be bumped up
    int asz = 1+av_len((AV *)SvRV(array));
    if( asz < 0 ){ fprintf(stderr, "is_array_ref() : error, input arra
+y ref '%p' has negative size!\n", array); return 0; }
    *array_sz = (size_t )asz;
    return 1; // success, it is an array and size returned by ref, abo
+ve
}

int func(
    SV *inp,
    SV *out
){
    AV *av, *av2;
    size_t i, j, asz;
    if( is_array_ref(out, &asz) ){
        printf("Case1: @out\n");
        // we have an \@R, e.g. func(\@R)
         av = (AV *)SvRV(out);
        // but first clear any contents
        if( asz > 0 ) av_clear(av);
    } else if( SvROK(out) ){
        printf("Case3: \\$out\n");
        // we have a scalar ref, e.g. func(\$x)
        av = newAV();
        sv_setrv(SvRV(out), (SV *)av);
    } else {
        printf("Case2: $out\n");
        // we have a scalar e.g func($x);
        av = newAV();
        sv_setrv(out, (SV *)av);
    }

    for(i=0;i<5;i++){
        av2 = newAV();
        av_extend(av2, 3);
        //av_push(av, (SV *)av2);
        // LeoNerd suggestion:
        av_push(av, newRV_noinc((SV *)av2));
        for(j=0;j<3;j++){
            av_store(av2, j, newSViv(42));
        }
    }
    return 0; // success
}
EOC

my @inp = (1..5);
my @out;
my $T = 'Case1';
is(func(\@inp, \@out),0, "$T: called success.");
is(scalar(@out), 5, "$T: rows are 5");
for(my $i=0;$i<5;$i++){
    ok(ref($out[$i])eq'ARRAY', "$T : item $i is ARRAYref.");
    is(scalar(@{$out[$i]}), 3, "$T : it has 3 elements: ".scalar(@{$ou
+t[$i]}));
        for(my $j=0;$j<3;$j++){
          is($out[$i]->[$j], 42, "$T : it's value is 42.");
        }
}

$T = 'Case2';
my $out;
is(func(\@inp, $out),0, "$T: called success.");
ok(ref($out)eq'ARRAY', "$T: it is now an ARRAYref.");
@out = @$out;
is(scalar(@out), 5, "$T: rows are 5");
for(my $i=0;$i<5;$i++){
    ok(ref($out[$i])eq'ARRAY', "$T : item $i is ARRAYref.");
    is(scalar(@{$out[$i]}), 3, "$T : it has 3 elements.");
        for(my $j=0;$j<3;$j++){
          is($out[$i]->[$j], 42, "$T : it's value is 42.");
        }
}

$T = 'Case3';
$out = undef;
is(func(\@inp, \$out),0, "$T: called success.");
ok(ref($out)eq'ARRAY', "$T: it is now an ARRAYref: ".ref($out));
@out = @$out;
is(scalar(@out), 5, "$T: rows are 5");
for(my $i=0;$i<5;$i++){
    ok(ref($out[$i])eq'ARRAY', "$T : item $i is ARRAYref.");
    is(scalar(@{$out[$i]}), 3, "$T : it has 3 elements.");
        for(my $j=0;$j<3;$j++){
          is($out[$i]->[$j], 42, "$T : it's value is 42.");
        }
}

done_testing();
[download]

Update: minor code-fixing (and spelling) after 1hr of posting

bw, bliako