NB. -----------------------------------------------------------
NB. --- This file is "llil4.ijs"
NB. --- Run as e.g.:
NB.
NB. jconsole.exe llil4.ijs big1.txt big2.txt big3.txt out.txt
NB.
NB. --- (NOTE: last arg is output filename, file is overwritten)
NB. -----------------------------------------------------------

pattern =: 0 1

NB. ========> This line has a star in its right margin =======> NB. *

args   =: 2 }. ARGV
fn_out =: {: args
fn_in  =: }: args

NB. PAD_CHAR =: ' '

filter_CR       =: #~ ~: & CR
make_more_space =: ' ' I. @ ((LF = ]) +. (TAB = ])) } ]
find_spaces     =: I. @: = & ' '

read_file =: {{
  'fname pattern' =. y

  text =. make_more_space filter_CR fread fname
  selectors =. (|.!.0 , {:) >: find_spaces text

  width  =. # pattern
  height =. width <. @ %~ # selectors

  append_diffs =. }: , 2& (-~/\)
  shuffle_dims =. (1 0 3 & |:) @ ((2, height, width, 1) & $)

  selectors =. append_diffs selectors
  selectors =. shuffle_dims selectors

  literal  =. < @: (}:"1) @: (];. 0)        & text "_1
  numeric  =. < @: (0&".) @: (; @: (<;. 0)) & text "_1
  extract  =. pattern & {
  using    =. 1 & \
  or_maybe =. `

  ,(extract literal or_maybe numeric) using selectors
}}

read_many_files =: {{
  'fnames pattern' =. y

  ,&.>/"2 (-#pattern) ]\ ,(read_file @:(; &pattern)) "0 fnames  NB. *
}}

'words nums' =: read_many_files fn_in ; pattern

t1 =: (6!:1) ''         NB. time since engine start

'words nums' =: (~. words) ; words +//. nums                    NB. *
'words nums' =: (\: nums)& { &.:>"_1 words ; nums
words =: ; nums < @ /:~/. words

t2 =: (6!:1) ''         NB. time since engine start

text =: , words ,. TAB ,. (": ,. nums) ,. LF
erase 'words' ; 'nums'
text =: (#~ ~: & ' ') text
text fwrite fn_out
erase < 'text'

t3 =: (6!:1) ''         NB. time since engine start

echo 'Read and parse input:    ' , ": t1
echo 'Classify, sum, sort:     ' , ": t2 - t1
echo 'Format and write output: ' , ": t3 - t2
echo 'Total time:              ' , ": t3
echo ''
echo 'Finished. Waiting for a key...'
stdin ''
exit 0