#! /usr/bin/gawk -f
# Last edited on 2026-04-25 05:43:52 by stolfi

# Reads a file with counts of word pairs {W1,W2},
# sorted by {W1}.
# 
# Computes from that data the predictability {H21} of {W2} given {W1},
# expressed as the entropy {E21} of the distribution of {W2} given {W1}
# minus the entropy {E2} of {W2} in general, 
# averaged over {W1}.
# 
# User must define (with "-v") variables {kind}, {size} and {skip}.
# These are just written out. The output is a single
# line with "{kind} {size} {skip} {E1} {E2} {E21}"
# where {E1} and {E2} are the entropies of {W1} and {W2} 
# in general and {H21} is the predictability above.
#

BEGIN {
  nt = 0            # Number of data rescords read.
  nW1 = 0           # Number of distinct words {W1}.
  nW2 = 0           # Number of distinct words {W2}.
  nW1W2 = 0         # Number of distinct pairs {W1,W2}.
  
  split("", CtW1)   # Count of each {W1}, indexed by it.
  split("", CtW2)   # Count of each {W2}, indexed by it.
  split("", CtW1W2) # Count of each {W1,W2}, indexed by "{W1}_{W2}".
  
  totCt = 0         # Total count of pairs.
  debug = 0
}

/^[ ]*[0-9]/ {
  if (NF != 3) { data_error(("bad NF = " NF)) }
  ct = $1; fr = $2; W1W2 = $3
  if (W1W2 == "~TOT~") { 
    totCt = ct
    next
  }
  
  nr += 1
  
  if (debug) { printf "!= W1W2 = '%s'\n", W1W2 > "/dev/stderr" }
  if (! (W1W2 in CtW1W2)) { CtW1W2[W1W2] = 0; nW1W2 += 1}
  CtW1W2[W1W2] += ct

  tmp = W1W2
  gsub(/[_]/, " ", tmp)
  nn = split(tmp, Wk)
  if (nn != 2) { data_error(("bad W1W2 = '" W1W2 "'")) }

  W1 = Wk[1]
  if (! (W1 in CtW1)) { CtW1[W1] = 0; nW1 += 1}
  CtW1[W1] += ct

  W2 = Wk[2]
  if (! (W2 in CtW2)) { CtW2[W2] = 0; nW2 += 1}
  CtW2[W2] += ct
  next;
} 

// { data_error("bad line format") }

END { 
  printf "%d data records read\n", nr > "/dev/stderr"
  printf "%d distinct pairs {W1,W2} seen\n", nW1W2 > "/dev/stderr"
  printf "%d distinct words {W1} seen\n", nW1 > "/dev/stderr"
  printf "%d distinct words {W2} seen\n", nW2 > "/dev/stderr"
  
  E1 = compute_entropy_of_distr(CtW1, totCt)
  E2 = compute_entropy_of_distr(CtW2, totCt)
  sum_fr1_H21 = 0
  sum_fr1 = 0
  totCt1 = 0
  maxH21 = -999.99
  maxH21_W1 = ""
  maxH21_W2 = ""
  for (W1 in CtW1) {
    sz1 = length(W1)
    if (sz1 != size) { arg_error(("bad word size = " sz1)) }
    ct1 = CtW1[W1]; totCt1 += ct1
    fr1 = ct1/totCt
    split("", CtW21)
    maxCt21 = 0
    maxCt21_W2 = ""
    for (W1W2 in CtW1W2) {
      if (substr(W1W2, 1, sz1) == W1) {
        W2 = substr(W1W2, sz1+2)
        if (W2 in CtW21) { prog_error(("bug '" W1 "' '" W2 "'")) }
        CtW21[W2] = CtW1W2[W1W2]
        if (CtW21[W2] > maxCt21) { maxCt21 = CtW21[W2]; maxCt21_W2 = W2 }
      }
    }
    E21 = compute_entropy_of_distr(CtW21, CtW1[W1])
    H21 = E2 - E21
    if (H21 > maxH21) { maxH21 = H21; maxH21_W1 = W1; maxH21_W2 = maxCt21_W2 }
    sum_fr1_H21 += fr1 * H21
    sum_fr1 += fr1
  }
  if (totCt1 != totCt) { prog_error(("sum of ct1 " totCt1 " " totCt)) }
  if (fabs(sum_fr1 - 1.0) > 0.1e-5) { prog_error(("sum of fr1 " sum_fr1)) }
  avgH21 = sum_fr1_H21/sum_fr1
  printf "%d %d %d %6.3f %6.3f %6.3f %s %s %6.3f\n", \
    kind, size, skip, E1, E2, avgH21, maxH21_W1, maxH21_W2, maxH21
}

function compute_entropy_of_distr(Ct,tCt, W,sum_ct,fr,E) {
  sum_ct = 0
  for (W in Ct) { sum_ct += Ct[W] }
  if (sum_ct != tCt) { prog_error(("sums dont match")) }
  E = 0
  for (W in Ct) {
    fr = Ct[W]/sum_ct
    E += - (fr * log(fr)/log(2))
  }
  return E
}

function fabs(x) { return ( x < 0 ? -x : x ) }

function map_text(tx,  rt) {
  rt = tolower(tx)
  # General cleanup, just in case:
  gsub(/[<][!][^<>]*[>]/, "", rt)
  gsub(/[<].[>]/, "", rt)
  gsub(/[«=»]/, "", rt)
  gsub(/[&][0-9][0-9][0-9][;]?/, "?", rt)
  rt = gensub(/[{]([^{}]*)[}]/, "\\1", "g", rt)
  
  # Delete all spaces:
  gsub(/[,.-]/, "", rt)

  # Map rare characters @b, @j, etc to '?':
  gsub(/[bjuvxy]/, "?", rt)

  # Map circles @q, @a, @o, @y:
  gsub(/[q]/, "Q", rt)
  gsub(/[oay]/, "O", rt)
  
  # Map codas: 
  gsub(/[i]*[nmg]/, "N", rt)
  gsub(/[i][i]?r/, "N", rt)
  
  # Map benches:
  gsub(/ee[e]?/, "B", rt)
  gsub(/[ics]h[e]?/, "B", rt)
  gsub(/c'h[e]?/, "B", rt)
  
  # Collapse all gallows to 'K':
  # Assume 'w' and 'z' are puffs with hooks.
  gsub(/[wztkpf][e]?/, "K", rt)
  
  # Split platform gallows:
  gsub(/[ic]Kh[he]?/, "BK", rt)
  
  # Map all dealers to D:
  gsub(/[i]*[dlrs]/, "D", rt)

  if (match(rt, /[^QOBKDN]/)) { 
    printf "** tx = [[%s]]\n", tx > "/dev/stderr"
    ch = substr(rt, RSTART, RLENGTH)
    prog_error(("mapping failed at '" ch "'"))
  }
  return rt
}