#! /bin/bash -eu 
# Last edited on 2026-06-30 23:39:59 by stolfi

# Creates "in/${version}-bencao-py.ivp"
# from Notes/077/in/bencao-4.uts".

export LC_ALL=en_US.utf8
export LANG=en_US.utf8
export LC_CTYPE=en_US.utf8

version="$1"; shift

for utype in py; do
  ifile="077/in/bencao-4.uts"
  ofile="in/${version}-bencao-${utype}.ivp"
  cat ${ifile} \
    | 077/remove_vms_omitted_fields_from_sbj.py \
    > .foo
  cat .foo \
    | work/remove_chinese_punctuation.py \
    | sed \
       -e 's:<[^<>]*>::g' \
    | pypinyin --style TONE \
    | sed \
       -e 's:<[^<>]*>::g' \
       -e 's:^[ ]*: :g' \
       -e 's:[ ]*$: :g' \
       -e 's:[ ][ ]+: :g' \
    > ${ofile}
  pfile="in/.lins-${utype}"
  tfile="in/.toks-${utype}"
  wfile="in/.wcts-${utype}"
  cat ${ofile} | egrep -v -e '^[ ]*([#]|$)' > ${pfile}
  cat ${pfile} | tr ' ' '\012'| egrep -e '.' > ${tfile}
  cat ${tfile} | sort | uniq -c > ${wfile}
  printf "%6d parags\n" $( cat ${pfile} | wc -l ) 1>&2
  printf "%6d tokens\n" $( cat ${tfile} | wc -l ) 1>&2
  printf "%6d lexemes\n" $( cat ${wfile} | wc -l ) 1>&2
  for n in 1 2 3 ; do 
    printf "entropy (min ct = ${n}): " 1>&2
    cat ${wfile} \
      | gawk -v n=${n} '//{ if ($1 >= n) { printf "%6d 1 %s\n", $1, $2 }}' \
      | work/compute_cond_entropy.gawk \
      1>&2
  done
done