#! /bin/python3
# Last edited on 2026-09-09 08:56:06 by stolfi

import sys, os, re
import regex as rex
from sys import stderr as err, stdin as inp, stdout as out
from process_funcs import bash, basic_line_loop
from error_funcs import arg_error, file_line_error, prog_error
from chinese_funcs import read_chinese_char_set
import size_position_funcs as spf
import match_multi_funcs as mmf
import bimatching_eval_funcs as bef
import format_matching_funcs as fmf
from math import sqrt, hypot, exp, log, pi, inf, nan, floor, ceil, isfinite
import html_gen as htg

def evaluate_starps_parags(ivt_file, locs_to_try, kwords_en, eval_func):
  # Compares all parags of the SPS with an entry of the SBJ, 
  # according to a specific macro-parsing of the latter into
  # keyword hits and gaps.
  # 
  # Arguments:
  # 
  #   {ivt_file}      name of input file with all candidate parags, or "-".
  #   {kwords_en}     list of crib IDs like {('USES','QI','QI')} that will be looked.
  #   {locs_to_try}   set of loc IDs of SPS parags to consider, or {None} for all. 
  #   {eval_func}     an SBJ-SPS matching function.
  #
  # The function reads from {ivt_file} (assumed to be in Unicode UTF-8
  # encoding) the transcription of one or more SPS parags, which should
  # by an IVTFF-like format, with one parag per line in the format
  # "<{LOC}> {TEXT}". The {TEXT} should be the complete text of one
  # parag EVA encoding.
  #
  # The function then calls {eval_func(loc_ec,dirtytx_ec,data_error,stats)}, which must
  # return 
  #
  #   {score}           a numeric match badness score.
  #
  #   {loc_ch}          the ID of the SBJ entry that was compared. 
  #   {tvar_ch}         tag identifying the trim/adjust variant of that entry. 
  #   {cleantx_ch}      the cleaned/trimmed/adjusted text of that entry. 
  #   {segs_ch}         a macro-parsing {segs_ch[0..ns-1]} of the same.
  # 
  #   {loc_ec}          the ID of the parag (same as the given one).      
  #   {tvar_ec}         tag identifying the trim/adjust applied to {dirtytx_ec}. 
  #   {cleantx_ec}      the cleaned/trimmed/adjusted text of that parag. 
  #   {segs_ec}         a macro-parsing {segs_ec[0..ns-1]} of the same. 
  #   {eva_per_hanzi    the ratio of EVA letters per hansi assumed for this func.
  #   {hit_penalties}   the penalties due to keyword variants used.
  # 
  # The {eval_func} should call {data_error} if the arguments {loc_ec}
  # or {dirtytx_ec} are invalid for some reason. The {stats} argument to
  # {eval_func} is a dict with various counters (see below).
  #
  # The macro_parsings {segs_ch,segs_ec} must split the clean texts
  # {cleantx_ch,cleantx_ec} of the SBJ entry and of the SPS parag into
  # the same number {nh} of hits and same number {ng=nh+1} gaps. The
  # {hit_penalties} should be a list of {nh} floats with the
  # contributions to {score} that are due to non-canonical keyword
  # choices for the hits, like @daiin instead of @daiin and @chedo
  # instead of @chedy. If not available, {hit_penalties} should be set
  # to zero.
  # 
  # If the {eval_func} returns {+inf} as the score, it is assumed that that
  # {dirtytx_ec} cannot be matched. This function then assigns a score of
  # {+inf} to the parag and sets {segs_ec} to {None}.
  #
  # The {eval_func} should increment {stats['too_large']} and
  # {stats['too_small']} when appropriate. It should increment
  # {stats['npar_with']} iff the parag has all the keywords requedged by
  # {kwords_en}, irrespective of badness. It should also imcrement
  # {stats['npar_bima']} iff the parag is submitted to
  # {match_bitemplate}. Other {stats} fields are incremented outside
  # {eval_func}.
  #
  # At the end, this function returns a list {pevs} of tuples with the
  # parameters and results of the matching including the badness score,
  # sorted by increasing score; and the dictionary {stats} with various
  # counts of the operation.
  #
  # If {locs_to_try} is not {None}, it must be a set with the location
  # IDS of the SPS parags to be considered for the match. If
  # {locs_to_try} is None, considers all parags.

  debug_file = False
  
  nh = len(kwords_en)
  ng = nh+1; ns = ng + nh
  
  if debug_file: err.write(f"!a {ivt_file = !r}\n")
  rd = inp if ivt_file == "-" else open(ivt_file, "r")
  rd.reconfigure(encoding='utf-8')
  
  utype_ec = "ec"
  pat_line, pat_unit, pat_sepa, clean_sepa = spf.get_parsing_patterns(utype_ec)
  
  stats = dict() # Counts of various things.

  stats['npar_read'] = 0 # Count of SPS parags read.
  stats['npar_excl'] = 0 # Count of SPS parags excluded by {locs_to_try}.
  stats['npar_eval'] = 0 # Count of SPS parags submitted to {eval_func}.
  stats['npar_bima'] = 0 # Count of SPS parags submitted to {match_bitemplate}.
  stats['npar_with'] = 0 # Count of SPS parags with all the keywords.
  
  stats['too_small'] = 0   # Count of parags rejected for being too short.
  stats['too_large'] = 0   # Count of parags rejected for being too long.
  stats['min_matched_size'] = +inf # Minimum size of parags that matched.
  stats['max_matched_size'] = 0    # Maximum size of parags that matched.
  
  pevs = [] # Candidates after analysis.

  def process_input_line(nline, line):
    nonlocal stats, pevs
    # 
    # Parses a line {line} assuming it is line {nline} of the file. The
    # {line} is always a string (never {None}), but may be "" if the
    # line is empty.
    # 
    # Ignores the line if it is a blank or #-comment.
    # 
    # Otherwise the line must be a data line, matching {pat_line}
    # 
    # Increments {stats['npar_read']} for each data line.
    # 
    # Calls {eval_func} to try to obtain viable parallel macro-pasings
    # of the texts of the SBJ entry and of the SPS parag, after
    # applicable trimmings and adjustments.
    #
    # Increments {stats['npar_eval']} for each parag submitted to
    # {eval_func}. Also increments {stats['npar_excl']} for
    # each parag excuded by {locs_to_try}.
    # 
    # If {eval_func} returned a finite score, updates
    # {stats['min_matched_size'],stats['max_matched_size']} creates the
    # candidate evaluation tuple {pev} and appends it to the list
    # {pevs}.

    def data_error(msg):
      nonlocal ivt_file, nline, line
      file_line_error(ivt_file, nline, msg, line)
      assert False
      # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
    assert line != None, "The {line} arg must not be {None}" 

    # Ignore comments and blank lines:
    if re.match(r" *([#]|$)", line): return

    # Just in case, ignore IVTFF page headers:
    if re.match(r"<f[0-9]+[rv][0-9]*>", line): return
    
    stats['npar_read'] += 1

    m = re.match(pat_line, line)
    if m is None: 
      # Invalid line format.
      data_error("invalid line format")

    # Parse the line into locus ID {loc_ec} and dirty text:
    loc_ec = m.group(1)
    dirtytx_ec = m.group(2) 
    
    if locs_to_try != None and loc_ec not in locs_to_try:
      stats['npar_excl'] += 1
      return

    score, \
        loc_ch_e, tvar_ch_e, cleantx_ch_e, segs_ch_e, \
        loc_ec_e, tvar_ec_e, cleantx_ec_e, segs_ec_e, \
        eva_per_hanzi_e, hit_penalties_e = \
      eval_func(loc_ec, dirtytx_ec, data_error, stats)
    err.write(f"score {score:.8f}.\n")
    stats['npar_eval'] += 1

    if isfinite(score):
      # Was able to match:
      assert loc_ec_e == loc_ec
      assert tvar_ch_e != None and tvar_ec_e != None
      assert cleantx_ch_e != None and cleantx_ec_e != None
      assert segs_ch_e != None and segs_ec_e != None
      assert hit_penalties_e != None and len(hit_penalties_e) == nh
      assert len(segs_ec_e) == ns
      for sg in segs_ch_e: assert isinstance(sg, str)
      for sg in segs_ec_e: assert isinstance(sg, str)

      # Collect statistics of parag lengths:
      psize_ec = len(cleantx_ec_e)
      if psize_ec < stats['min_matched_size']: stats['min_matched_size'] = psize_ec
      if psize_ec > stats['max_matched_size']: stats['max_matched_size'] = psize_ec
      pev = \
        { 'score': score, 
          'loc_ch': loc_ch_e, 'tvar_ch': tvar_ch_e, 'segs_ch': segs_ch_e, 
          'loc_ec': loc_ec, 'tvar_ec': tvar_ec, 'segs_ec': segs_ec_e, 
          'kwords_en': kwords_en, 'eva_per_hanzi': eva_per_hanzi_e,
          'hit_penalties': hit_penalties_e
        }
      pevs.append(pev)
   
    return      
    # ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

  err.write(f"reading file '{ivt_file}' ...\n")
  nread = basic_line_loop(rd, process_input_line)
  rd.close()
  err.write(f"{nread:6d} lines read from {ivt_file}\n")
  
  err.write(f"sorting parag evaluation records ...\n")
  pevs.sort(key = lambda x : x[0])
  
  return pevs, stats
  # ----------------------------------------------------------------------

def output_parev(wr, pev, hipat_ec):
  # Writes to {wr} the parag evaluation tuple {pev} in the 
  # format suitable for the report.
  # The {hipat_ec} should be an EVA pattern used to highlight
  # keywrds in the gaps of the EVA macro-parsing.

  pev_str = fmf.format_starps_parag_evaluation(pev, hipat_ec)
  wr.write(pev_str);
  return
  # ----------------------------------------------------------------------
  
def add_bencao_starps_matching_INFO(st):
  # Appends to {st] the explanation for how SPS parags are evaluated.

  htg.parags(st, """The subpages about individual SBJ entries contain the
  results of structural matching of the entry in question against the
  SPS parags. These reports consist of one or more /parag evaluation
  blocks/ or /parevs/, each evaluating one parag of the SPS, which is
  summarized in a numeric badness score.
  
  The reports assume that the SBJ entry, minus all markup and
  puntuation, is parsed according to a given list of {N} /hanzi keyword
  patterns/, for example [ '主', '久服|久食' ]. None of these patterns
  should match the empty string. The parsing splits the entry (cleaned
  of all punctuation and metadata) into {N} non-overlapping hanzi
  strings that match those patterns (the /hanzi hits/), in the given
  order, and {N+1} hanzi strings before, between, and after those hits
  (the /hanzi gaps/). This parsing is shown before all the parevs.
  
  Each SPS parag is evauated by deleting all word space markers [,.-]
  and other markup, and parsing the resulting EVA string too into {N}
  /EVA hits/ and {N+1} /EVA gaps/, by another list of {N} /EVA keyword
  patterns/, for example [ 'daiin|dain|laiin', 'chedy|chedo' ]. None of
  these patterns should match the empty string. Then the badness score
  is computed by comparing actual and predicted lengths (in EVA letters)
  of the whole SPS parag and of the {N+1} EVA gaps. The prediction for
  each of these substrings is based on the number of hanzi in the
  corresponding substring of the SBJ entry, multiplied by a fixed scale
  factor.
  
  If there are multiple ways to match the {N} EVA keywords within the
  parag's text, the parev data reflects the choice of the {N} matches
  that gave the lowest badness score.
  
  Conversely, if there is no way to match {N} EVA keywords within the
  parag's text, the parag is not considered to be a candidate 
  match and is omitted from the matching report.""")

  return
  # ----------------------------------------------------------------------
  
def test_stuff():
  err.write("TESTING\n")
  
  err.write("----------------------------------------\n")
  test_other_stuff()
  err.write("----------------------------------------\n")

  return
  # ---------------------------------------------------------------------- 

def test_other_stuff():

  err.write("----------------------------------------\n")

  gsizes_ch_str = "20..30,7,15,12..13,8"
  gsizes_ch = spf.parse_size_ranges(gsizes_ch_str)
  err.write(f"{gsizes_ch_str =!r}\n")
  err.write(f"{gsizes_ch     =!r}\n")

  err.write("----------------------------------------\n")
  loc_ch = "TEST"
  tvar_ch = "chop"
  segs_ch = [ '黍米无毒', '主', '益', '气', '补中多热', '令', '人烦' ]
  err.write(f"{segs_ch = !r}\n")

  loc_ec = "f117r.1"
  tvar_ec = "norm"
  segs_ec = [ 'psheodalodarchydaltedyqote', 'saiin', 'okalal', 'shdy', 'otaiinarorshedydaiint', 'cheod', 'lchy' ]
  err.write(f"{segs_ec = !r}\n")
  kwords_en = ( 'USES', 'QI', 'MAKES' )
  hit_penalties = [ 0.200, 0.400, 0.800 ]
  eva_per_hanzi = 5.000;
  score = bef.compute_full_score_from_macro_parsings \
    ( segs_ch, segs_ec, eva_per_hanzi, hit_penalties )
  err.write(f"{score = :6.1f}\n")
  
  err.write("----------------------------------------\n")

  pev = \
    { 'score': score, 
      'loc_ch': loc_ch, 'tvar_ch': tvar_ch, 'segs_ch': segs_ch, 
      'loc_ec': loc_ec, 'tvar_ec': tvar_ec, 'segs_ec': segs_ec, 
      'kwords_en': kwords_en, 'eva_per_hanzi': eva_per_hanzi, 
      'hit_penalties': hit_penalties
    }
  hipat = [ 'tedy+', 'dal', 'taiin', 'daiin', ]
  hipat = '|'.join(hipat)
  output_parev(err, pev, hipat)

  # Paranoia:
  score_check = bef.compute_full_score_from_macro_parsings \
    ( segs_ch, segs_ec, eva_per_hanzi, hit_penalties )
  if score != score_check:
    err.write(f"{score =       :24.16e}\n")
    err.write(f"{score_check = :24.16e}\n")
  assert score == score_check
  
  return
  # ----------------------------------------------------------------------

if len(sys.argv) == 2 and sys.argv[1] == "ANN.TEST":
  test_stuff()