#! /usr/bin/python3
# -*- coding: utf-8 -*-

last_edit = "Last edited on 2026-09-09 04:15:05 by stolfi"

import sys, re, os, string, glob
from sys import stderr as err
import html_gen as h
import bimatching_eval_funcs as bef
from math import sqrt, hypot, exp, log, floor, ceil, isfinite, isnan, inf, nan

def format_starps_parag_evaluation_INFO(st):
  # Appends to {st] the explanation for the output of
  # {format_starps_parag_evaluation}.
  
  info = """The first line of each block is the page and line number of
  the head line of the candidate SPS parag and a tag that identifies the
  cleaning and normalizations applied, followed by its its badness
  score, by the total length and its discrepancy, and by the total
  keyword penalty, and the EVA per hanzi ratio.
  
  The length discrepancy is the difference between the actual length of
  the parag (in EVA letters, ignoring word spaces, after any applicable
  trimming and normalization) and the length predicted from the count of
  hanzi in the SBJ entry, using the listed EVA to hanzi ratio.
  
  The key penalty is the the sum of of the penalty points associated
  with the specific alternatives of the keywords that were used.
  
  That line is followed by {N+1} lines showing the assumed parsing of
  the parag's text by the specified EVA keyword patterns. There is one
  line for each EVA gap of that parsing. Each of these lines has five
  fields: 
  
  "{hit} {kpena} {gsz}({gszerr}) {gscore} {gap}"
  
  where 
  
  {hit}    the EVA keyword hit before the gap (blank for the first gap).<br/>
  {kpena}  the penalty for that particular alternative of the keyword.<br/>
  {gsz}    the actual length of the gap<br/>
  {gszerr} the signed difference between the actual and predicted gap length.<br/>
  {gscore} the contribution of that gap to the parag's score,<br/>
  {gap}    the EVA text of the gap
  
  The gap's score is the squared relative discrepancy between the gap's
  size in the SPS entry and the size expected from the corresponding gap
  in the SBJ entry, multiplied by a weight that depends on the number of
  gaps and the position of the gap in the list (smaller for the first
  and last gaps). The discrepancy is the difference between the two,
  computed in log scale, and divided by the estimated deviation of that
  difference.
  
  Within the gaps, any substrings that matches any of the possible EVA keywords
  is highighted in boldface, even though they were not considered hits
  for the purposes of parsing or computation.
  
  All size discrepancies are in EVA characters. The badness score is the
  sum of the gap scores and the keyword hit penalties, and would be zero
  for a perfect match (all gap sizes match their predicted values and
  all keyword hits are canonical)."""
  
  return info
  # ----------------------------------------------------------------------

def format_starps_parag_evaluation(pev, hipat_ec):
  # Formats the SPS parag evaluation tuple {pev} as a multiline
  # string. Each line of it ends with "\n".
  # 
  # Note that this function uses the ratio {pev['eva_per_hanzi'] to
  # compute both the the global and per-gap size errors, even though a
  # distinct default ratio was used for the size exclusion criteria
  #
  # Let {ns} be {len(segs_ch) = len(segs_ec)}. Let {nh = ns//2} be the
  # number of hits in both parsings, and {ng = nh+1} be the number of
  # gaps.
  # 
  # If {hipat_ec} is not {None}, it must be an EVA RE pattern
  # used to highlight substrings of the gaps of the EVA macro-parsing {segs-ec}
  # in {pev}.
  # 
  # The size discrepancies shown in the output are computed as the
  # difference between the observed EVA sizes in the SPS text and and
  # the expected values computed from the observed hanzi sizes in the
  # SBJ text.
  # 
  # For the total size errors, the format uses the {eva_per_hanz} ratio
  # stored in {pev}
  # 
  # See {format_starps_parag_evaluation_INFO} for the output format.
  
  debug = False

  score = pev['score']
  loc_ch = pev['loc_ch']; tvar_ch = pev['tvar_ch']; segs_ch = pev['segs_ch']
  loc_ec = pev['loc_ec']; tvar_ec = pev['tvar_ec']; segs_ec = pev['segs_ec']
  kwords_en = pev['kwords_en']; eva_per_hanzi = pev['eva_per_hanzi']; 
  hit_penalties = pev['hit_penalties']

  assert isfinite(score) and score >= 0, "parev with bad score"
  assert segs_ec != None, "parev with null SPS parag"

  ns = len(segs_ec); assert len(segs_ch) == ns
  nh = ns//2; ng = nh + 1; assert ns == ng + nh
 
  # Gap and total bencao sizes:
  gsizes_ch = bef.get_gap_sizes(segs_ch)
  tsize_ch = 0;
  for ks in range(ns): tsize_ch += len(segs_ch[ks])

  if debug:
    err.write(f"!> >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n")
    err.write(f"!> {segs_ch = !r}\n")
    err.write(f"!> {gsizes_ch = }\n")
    err.write(f"!> {loc_ec = }\n")

  # Actual starps gap, hit, and text size:
  gsizes_ec = bef.get_gap_sizes(segs_ec)
  hsizes_ec = bef.get_hit_sizes(segs_ec)
  if debug:
    err.write(f"!> {loc_ec = }\n")
    err.write(f"!> {segs_ec = !r}\n")
    err.write(f"!> {gsizes_ec = }\n")
  
  # Total text sizes minus punctutaion:
  tsize_ch = sum(len(s) for s in segs_ch); 
  tsize_ec = sum(len(s) for s in segs_ec); 

  # Expected number of EVA letters per hanzi character:
  ch_trimmed = True   # Assumed when this function is called.

  def pcterr(s, es):
    eps = 1
    if s < es[0]:
      ferr = (s - es[0])/hypot(es[0], eps)
    elif s > es[1]:
      ferr = (s - es[1])/hypot(es[1], eps)
    else:
      ferr = 0
    return 100*ferr
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

  cbits = []
  
  tot_hit_penalty = 0;
  for hp in hit_penalties:
    assert 0.000 <= hp and hp <= 9.999 
    tot_hit_penalty += hp
  assert tot_hit_penalty <= score

  # SPS parag locus ID, badness score, total text size error, key penalty:
  xloc_ec = f"<{loc_ec}>"
  cbits.append(f"  {xloc_ec:<10s}")
  xtvar_ec = f"({tvar_ec})"
  cbits.append(f"  {xtvar_ec:<10s}")
  cbits.append(f" <b>{score:7.3f}</b>")
  cbits.append(f"  {tsize_ec:3d}")
  terr = size_error(tsize_ch, tsize_ec, eva_per_hanzi)
  xterr = f"({terr:+d})"
  cbits.append(f"{xterr:<6s}")
  cbits.append(f"  {tot_hit_penalty:6.3f}")
  cbits.append(f"  {eva_per_hanzi:5.3f} e/h")

  cbits.append("\n")

  # EVA hits and gaps:
  hwd = 0
  for hsz in hsizes_ec: hwd = max(hwd, hsz)
  for ig in range(ng):
    hit_ec = f"{segs_ec[2*ig-1]}" if ig > 0 else ""
    hit_pena = f"{hit_penalties[ig-1]:5.3f}" if ig > 0 else " "*5
    gap_ec = segs_ec[2*ig]

    cbits.append("  ");
    # The hit string and its penalty:
    hit_ec = hit_ec.ljust(hwd, " ")
    cbits.append(f"<b>{hit_ec}</b>")
    cbits.append(" ");
    cbits.append(hit_pena)
    
    # The gap size error and score:
    gsz_ec = gsizes_ec[ig]
    gsz_ch = gsizes_ch[ig]
    assert gsz_ec == len(gap_ec);
    cbits.append(f" {gsz_ec:3d}")
    gerr_ec = size_error(gsz_ch, gsz_ec, eva_per_hanzi)
    gerr_ec_str = f"({gerr_ec:+d})"
    cbits.append(f"{gerr_ec_str:<6s}")
    gsc = bef.compute_single_gap_score(gsz_ch, gsz_ec, eva_per_hanzi, ig, ng)
    gsc_str = f" {gsc:6.3f} "
    cbits.append(gsc_str)
    
    # The gap text, with highlights:
    if hipat_ec != None:
      gap_ec = highlight_keywords_in_text(gap_ec, hipat_ec, "<b>", "</b>")
    cbits.append(gap_ec)
    cbits.append("\n")

  pev_str = "".join(cbits)
  return pev_str
  # ----------------------------------------------------------------------

def size_error(size_ch, size_ec, eva_per_hanzi):
  # Signed integer difference between and EVA size {size_ec} and the 
  # value predicted from the hanzi size {size_ch}.
  
  debug = False
  esz_lo, esz_hi = bef.expected_size1_from_size0(size_ch, eva_per_hanzi)
  if debug:
    err.write(f"!: esz = {esz_lo}..{esz_hi}\n")
  eps = 1
  if size_ec < esz_lo:
    size_err = size_ec - int(floor((esz_lo + esz_hi)/2))
  elif size_ec > esz_hi:
    size_err = size_ec - int(ceil((esz_lo + esz_hi)/2))
  else:
    size_err = 0
  return size_err
  # ----------------------------------------------------------------------

def highlight_keywords_in_text(text, hipat, hbeg, hend):
  # Insert {hbeg} and {hend} around all substrings of {text} that match the
  # pattern {hipat}, which should not match the empty string.
  #
  # The substrings that match may overlapm but the marekrs
  # {hbeg} and {hend} will be simplified so that they
  # are never nested or tangled. 
  #
  str_hi = ""  # Highlighted string.
  end_hi = -1  # End of current highlight, or {-1}.
  nt = len(text)
  for it in range(nt + 1):
    matched = False
    # The "(?b)" says use the longest match:
    m = rex.match("(?b)" + hipat, text[it:], )
    if m != None:
      end_this = it + m.end(0)
      if end_this > it:
        # Matched a non-empty prefix:
        matched = True
        if end_hi < it: str_hi += hbeg
        end_hi = max(end_hi, end_this)
    if not matched and end_hi == it: str_hi += hend
    if it < nt: str_hi += text[it]
  return str_hi
  # ----------------------------------------------------------------------

def format_macro_parsing_ch(loc_ch, tvar_ch, segs_ch, hipat_ch):
  # Formats the macro-parsing {segs_ch} of the pure hanzi 
  # text of an SBJ entry. Each line ends with "\n".
  # 
  # If {hipat_ch} is not {None}, it must be a hanzi RE pattern
  # used to highlight substrings of the gaps of the macro-parsing {segs_ch}.
  
  debug = False

  assert segs_ch != None

  ns = len(segs_ch)
  nh = ns//2; ng = nh+1; assert ns == ng + nh

  tsize_ch = 0
  for ks in range(ns): tsize_ch += len(segs_ch[ks])
  
  gsizes_ch = bef.get_gap_sizes(segs_ch)
  hsizes_ch = bef.get_hit_sizes(segs_ch)
  if debug:
    err.write(f"!< <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n")
    err.write(f"!< {segs_ch = !r}\n")
    err.write(f"!< {tsize_ch = }\n")
    err.write(f"!< {gsizes_ch = }\n")
    err.write(f"!< {hsizes_ch = }\n")

  cbits = []

  # SBJ entry locus ID and variant tag:
  loc_ch = f"<{loc_ch}>"
  cbits.append(f"  {loc_ch:<10s}")
  cbits.append(f" ({tvar_ch})")
  cbits.append(f" {tsize_ch:3d}")
  cbits.append("\n")

  # Gaps and hits:
  hwd = 0
  for hs in hsizes_ch: hwd = max(hwd, hs)
  for ig in range(ng):
    hit_ch = segs_ch[2*ig-1] if ig > 0 else ""
    gap_ch = segs_ch[2*ig] 

    cbits.append("  ");
    # Hit and gap sizes in hanzi:
    hsz_str = f"{hsizes_ch[ig-1]:2d}" if ig > 0 else "  "
    cbits.append(hsz_str);
    gsz_str = f" {gsizes_ch[ig]:2d}"
    cbits.append(gsz_str); 
    
    # The hit string:
    cbits.append(" ")
    hit_ch = hit_ch.ljust(hwd, "　")
    cbits.append(f"<b>{hit_ch}</b> ")
    
    # The gap string:
    if hipat_ch != None: 
      gap_ch = highlight_keywords_in_text(gap_ch, hipat_ch, "<b>", "</b>")
    cbits.append(gap_ch)
    cbits.append("\n")

  entry_str = "".join(cbits)
  return entry_str
  # ----------------------------------------------------------------------