#! /bin/bash -e
# Last edited on 2026-01-16 18:45:25 by stolfi

# Extracting the sequence of line-initial words:

src=text25e1rz-f83.ivt
cat ${src} \
  | sed \
      -e 's:^[0-9][0-9] *[.]::g' \
      -e 's:[.].*$::g' \
  > .first-words.txt

# Extracting the sequence of non-line-initial words:

cat ${src} \
  | sed \
      -e 's:^[0-9][0-9] *[.]:@:g' \
      -e 's:^[@][^.][^.]*[.]::g' \
  | tr '.' '\012' \
  | egrep -e '[a-z]' \
  > .middle-words.txt

# Computing frequencies:

for w in first middle ; do
  cat .${w}-words.txt \
    | sort | uniq -c \
    | compute_freqs.gawk \
    | sort -k1,1r -k3,3 \
    > .${w}-words.wfr
done

# Comparing the frequencies:


for w in middle first; do 
  cat .${w}-words.wfr | sort -k3,3 > .${w}.srt
done
join -j1 3 -j2 3 \
    -o 1.1,1.2,2.1,2.2,0 \
    -a 1 -a 2 -e '0' \
    .{first,middle}.srt \
  | gawk \
      ' // { 
          nf = $1; ff = $2; nm = $3; fm = $4; wd = $5;
          nt = nf + nm; 
          printf "%02d %02d %02d %s\n", 
            nt, nf, nm, wd
        }
      ' \
  > .cmp-words.wfrfr

cat .cmp-words.wfrfr \
  | gawk \
      ' // { 
          nt = $1; nf = $2; nm = $3; wd = $4;
          if (nt >= 2) {
            if (nf/nt >= 0.8) { k = "F"; } 
            else if (nm/nt >= 0.8) { k = "M"; } 
            else { k = "B"; }
          } else {
            k = "?"
          }
          print $0, k
        }
      ' \
  > .tag-words.wfrfr
  
cat .tag-words.wfrfr \
  | egrep -e ' F' \
  | sed -e 's: [A-Z?]$::g' \
  | sort -k2,2nr -k4,4 \
  > .only-first.wfrfr
  
cat .tag-words.wfrfr \
  | egrep -e ' M' \
  | sed -e 's: [A-Z?]$::g' \
  | sort -k3,3nr -k4,4 \
  > .only-middle.wfrfr
  
cat .tag-words.wfrfr \
  | egrep -e ' B' \
  | sed -e 's: [A-Z?]$::g' \
  | sort -k1,1nr -k4,4 \
  > .both.wfrfr
   
cat .tag-words.wfrfr \
  | egrep -e ' [?]' \
  | sed -e 's: [A-Z?]$::g' \
  | sort -k1,1nr -k4,4 \
  > .rare.wfrfr