#! /bin/bash -eu 
# Last edited on 2026-06-29 13:31:44 by stolfi

# Creates "in/${version}-starps-wc.ivp" and "in/${version}-starps-wp.ivp"
# from Notes/074/star25e1.ivt".

version="$1"; shift

for utype in wp wc; do
  ifile="074/star25e1.ivt"
  ofile="in/${version}-starps-${utype}.ivp"
  cat ${ifile} \
    | convert_starps_raw_to_lin_ivt.py ${utype} \
    | convert_starps_lin_to_par_ivt.py ${utype} \
    | sed \
       -e 's:<[^<>]*>::g' \
       -e 's:[?][?]+:?:g' \
       -e 's:[.]: :g' \
       -e 's:^[ ]*: :g' \
       -e 's:[ ]*$: :g' \
       -e 's:[ ][ ]+: :g' \
    > ${ofile}
  pfile="in/.lins-${utype}"
  tfile="in/.toks-${utype}"
  wfile="in/.wcts-${utype}"
  cat ${ofile} | egrep -v -e '^[ ]*([#]|$)' > ${pfile}
  cat ${pfile} | tr ' ' '\012'| egrep -e '.' > ${tfile}
  cat ${tfile} | sort | uniq -c > ${wfile}
  printf "%6d parags\n" $( cat ${pfile} | wc -l ) 1>&2
  printf "%6d tokens\n" $( cat ${tfile} | wc -l ) 1>&2
  printf "%6d lexemes\n" $( cat ${wfile} | wc -l ) 1>&2
  for n in 1 2 3 ; do 
    printf "entropy (min ct = ${n}): " 1>&2
    cat ${wfile} \
      | gawk -v n=${n} '//{ if ($1 >= n) { printf "%6d 1 %s\n", $1, $2 }}' \
      | work/compute_cond_entropy.gawk \
      1>&2
  done
done