#! /usr/bin/python3
# Last edited on 2026-05-20 18:30:48 by stolfi

import os, sys, re
from sys import stderr as err, stdout as out
from math import sqrt, hypot, log
from error_funcs import file_line_error as ferr

def main():
  # Read SPS word count file:
  wfile = f"res/starps-fu-wc-par-1.tfr"
  with open(wfile, "r") as rd:
    lines = rd.readlines()
  nl = len(lines)
  err.write(f"read {nl} lines.\n")
  
  # Extract word, prefix, core, and suffix counts from file:
  sz_pref = 1
  sz_suff = 1
  ct_word = dict() # Couns per word.
  ct_pref = dict() # Count per prefix.
  ct_core = dict() # Counts per core.
  ct_suff = dict() # count per suffix.
  ct_TOTAL = None  # Total count from file
  ct_sum = 0       # Computed total count
  for kl in range(nl):
    line = lines[kl].strip()
    if re.match(r"[ ]*([#]|$)", line):
      err.write(f"ignored {line = !r}\n")
      continue
    m = re.fullmatch(r"[ ]*([0-9]+)[ ]+[0-9.]+[ ]([-0-9A-Za-z?]+)[ ]*", line)
    if m == None: ferr(wfile, kl+1, "bad line format", line)
    ct = int(m.group(1))
    word = m.group(2)
    if word == "TOTAL":
      ct_TOTAL = ct
    elif word == "utf-8":
      # Bug workaround
      ct_sum += ct
      continue
    else:
      if re.fullmatch(r"[a-z?]+", word) == None:
        ferr(wfile, kl+1, "bad word", line)
      assert not word in ct_word, "dup word" 
      ct_sum += ct
      sz = len(word)
      if sz >= sz_pref + 2 + sz_suff:
        pref = word[0:sz_pref]
        core = word[sz_pref:sz-sz_suff]
        suff = word[sz-sz_suff:]
        ct_word[word] = ct
        ct_pref[pref] = ct + (ct_pref[pref] if pref in ct_pref else 0)
        ct_core[core] = ct + (ct_core[core] if core in ct_core else 0)
        ct_suff[suff] = ct + (ct_suff[suff] if suff in ct_suff else 0)
  err.write(f"{ct_sum = } {ct_TOTAL = }\n")
  assert ct_sum == ct_TOTAL
  
  def winnow(ct_dic, ct_min):
    keys = list(ct_dic.keys())
    for key in keys:
      if ct_dic[key] < ct_min:
        del ct_dic[key]
    return 
    # ............................
  
  winnow(ct_pref, 20)
  winnow(ct_core, 20)
  winnow(ct_suff, 20)
  
  def show_dict(ct_dic, name):
    items = list(ct_dic.keys()); nk = len(items)
    err.write(f"got {nk} {name}s\n")
    for item in items: err.write(f"  {ct_dic[item]} {item}\n")
    return items, nk
    # ..............................
  
  prefs, np = show_dict(ct_pref, "pref")
  cores, nc = show_dict(ct_core, "core")
  suffs, ns = show_dict(ct_suff, "suff")
  
  def skewness(ct11, ct12, ct21, ct22):
    nonlocal ct_TOTAL
    fr11 = (ct11 + 1)/(ct_TOTAL + 1)
    fr12 = (ct12 + 1)/(ct_TOTAL + 1)
    fr21 = (ct21 + 1)/(ct_TOTAL + 1)
    fr22 = (ct22 + 1)/(ct_TOTAL + 1)
    skew = (log(fr12) - log(fr11)) - (log(fr22) - log(fr21))
    return skew
    # .......................................
  
  skew_min = 0
  ct_row_min = 10
  ct_col_min = 10
  for kc in range(nc):
    for kp1 in range(np):
      for kp2 in range(kp1+1,np):
        for ks1 in range(ns):
          for ks2 in range(ks1+1,ns):
            word11 = prefs[kp1] + cores[kc] + suffs[ks1]; ct11 = ct_word[word11] if word11 in ct_word else 0
            word12 = prefs[kp1] + cores[kc] + suffs[ks2]; ct12 = ct_word[word12] if word12 in ct_word else 0
            word21 = prefs[kp2] + cores[kc] + suffs[ks1]; ct21 = ct_word[word21] if word21 in ct_word else 0
            word22 = prefs[kp2] + cores[kc] + suffs[ks2]; ct22 = ct_word[word22] if word22 in ct_word else 0
            ct_square = ct11 + ct12 + ct21 + ct22
            ct_row1 = ct11 + ct12
            ct_row2 = ct21 + ct22
            ct_col1 = ct11 + ct21
            ct_col2 = ct12 + ct22
            if min(ct_row1,ct_row2) >= ct_row_min and min(ct_col1,ct_col2) >= ct_col_min:
              skew = skewness(ct11, ct12, ct21, ct22)
              if skew >= skew_min:
                err.write(f"  {ct11:3d} {word11:<20s}   {ct12:3d} {word12:<20s}\n")
                err.write(f"  {ct21:3d} {word21:<20s}   {ct22:3d} {word22:<20s}  {skew:+8.3f}\n")
                err.write("\n")

  return 
  # ----------------------------------------------------------------------
  
main()