#! /bin/bash -eu
# Last edited on 2026-03-14 11:40:34 by stolfi

# For each (section,text type) pair {st}, like "hea-parags", reads the
# word counts and freqs file "st_words/{st}.wcf". Then reduces each word
# to its "root" and saves the occurrence counts and freqs of the roots
# to "st_roots/{st}.wcf".
# 
# The counts are assumed to be fractional.
#
# Words and roots with '?' characters are excluded before cmputing the
# frequencies and saving in the ".wcf" files.

mkdir -p .errors

# Directory with transcription files per (section,txtype):
tr_dir="st_files"

# Directory with word counts and freqa per (section,txtype):
wd_dir="st_words"

# Directory with root counts and freqs per (section,txtype):
rt_dir="st_roots"
mkdir -p ${rt_dir}

rm -f ${rt_dir}/*.wcf

mkdir -p .errors

temp="/tmp/$$"

# Get list {sts} of (section,txtype) pairs:
# sts=( $( cd ${tr_dir}/ && ls *.ivt | sed -e 's:[.]ivt::g' | sort -t- -k2,2 -k1,1 ) )
sts=( $( cd ${tr_dir}/ && ls *.ivt | sed -e 's:[.]ivt::g' ) )
for st in ${sts[@]}; do
  echo "extracting root freqs for ${st} ..."
  # Extract fractional root counts and freqs 
  # "${rt_dir}/{st}.wcf" of (section,txtype) pair {st}:
  wcf_file_words="st_words/${st}.wcf"
  wcf_file_roots="st_roots/${st}.wcf"
  cat ${wcf_file_words} \
    | gawk \
        -i root_from_word_funcs.gawk  \
        ' // { ct = $1; fr = $2; wd = $3;
            rt = root_from_word(wd);
            print ct, rt
          }
        ' \
    | combine_counts.gawk \
    | compute_freqs.gawk \
    | sort -b -k1,1gr \
    > ${wcf_file_roots}
done

# Get word and root count+frequency files for sections "hea" and "heb" together, txtype "parags":
for dir in ${wd_dir} ${rt_dir}; do
  wcf_both_file="${dir}/hea-heb-parags.wcf"
  echo "creating ${wcf_both_file} ..." 1>&2
  cat ${dir}/{hea,heb}-parags.wcf \
    | gawk '//{ print $1, $3 }' \
    | combine_counts.gawk \
    | compute_freqs.gawk \
    | sort -b -k1,1gr \
    > ${wcf_both_file}
done

# Compute paired counts and freqs for Herbal A and B:
for kind in words roots; do 
  ct_dir="st_${kind}"
  for crit in freq item; do
    if [[ ${crit} == "item" ]]; then
      wct_min_a=0; wct_min_b=0
    else
      wct_min_a=8; wct_min_b=3
    fi
    echo "pairing up ${kind} of hea (ct >= ${wct_min_a}) and heb (ct >= ${wct_min_b}) by criterion '${crit}' ..." 1>&2
    wcf_file_a="${ct_dir}/hea-parags.wcf"
    wcf_file_b="${ct_dir}/heb-parags.wcf"
    oct_file="${ct_dir}/hea-${wct_min_a}-heb-${wct_min_b}-parags-${kind}-paired-by-${crit}.oct"
    pair_item_count_and_freq_files.sh \
        ${crit} ${wcf_file_a} ${wct_min_a} ${wcf_file_b} ${wct_min_b} \
      > ${oct_file}
  done
done