#! /usr/bin/python3
# -*- coding: utf-8 -*-

last_edit = "Last edited on 2026-11-03 11:38:18 by stolfi"

import sys, re, os, string, glob
from sys import stderr as err
import html_gen as h
from process_funcs import bash, run_command, basic_line_loop
import html_report_funcs as hr
import match_multi_funcs as mmf
import analyze_starps_parag_funcs as anf
import align_bencao_starps_items_funcs as alf
import report_077_alt_matching_funcs as r77alt
import bencao_starps_report_funcs as repf
import bencao_starps_standard_bipatterns as stdbip
import bencao_starps_convert_funcs as convf
import bencao_starps_normalize_funcs as normf
import bencao_starps_size_position_funcs as spf
import bimatching_eval_funcs as bef
import format_matching_funcs as fmf
from math import sqrt, hypot, exp, log, floor, ceil, isfinite, isnan, inf, nan

def add_intro(code_ch, loc_ch, name_ch, name_py, name_en):
  name_en = re.sub(r"_", " ", name_en.lower())
  name_en_uscore = re.sub(r"[ ]+", "_", name_en)
  name_en_caps = string.capwords(name_en)
  
  bencao_clean_dir = "bencao/clean"
  tvar_ch, source_ch, itms_tg, itms_wh, itms_py, itms_en = \
    repf.fetch_bencao_entry_wh_py_en(bencao_clean_dir, code_ch, loc_ch)
  assert tvar_ch == "full" # For now.
  assert source_ch == 'ZHB' or source_ch == 'CTP'

  title = f"[{code_ch}.077] The '{name_en_caps}' entry of the SBJ [{source_ch}]"
  color = "#ddeeff" if source_ch == 'ZHB' else "#eeffdd"
  st = h.new_doc(title, color, text_width = 1600)
  
  h.section(st, 2, "Summary")

  h.parags(st, f"""This webpage discusses the SBJ entry titled {name_ch}
  {name_py} = "{name_en}", parsed into its sub-entries and their fields, and
  possible correspondences to parags of the SPS. The modern Mandarin
  reading of the text and an English translation are also shown. This
  entry may be referred as "{code_ch}" in tables.""")
  
  h.section(st, 2, "The Shennong Bencaojing entry")
  
  repf.add_formatted_entry_table(st, itms_tg, itms_wh, itms_py, itms_en)

  return \
    st, code_ch, loc_ch, tvar_ch, source_ch, \
    name_ch, name_py, name_en, \
    itms_tg, itms_wh, itms_py, itms_en
  # ----------------------------------------------------------------------

def add_chinese_text_trimming_section \
  ( st, loc_ch, name_ch, name_py, name_en, nch_long, nch_midi, itms_wh ):

  h.section(st, 2, "The SBJ entry")
  
  h.section(st, 3, "Full entry")
  text_ch_full = "".join(itms_wh)
  text_ch_full = re.sub(r"[　]", "", text_ch_full)
  
  text_ch_long = f"<{loc_ch}> (full) " + text_ch_full
  text_ch_long, tvar_ch_long = \
    display_hanzi_pure_text(st, loc_ch, nch_long, text_ch_long)
  assert tvar_ch_long == "full"
  
  h.section(st, 3, "Simplified entry")

  h.parags(st, """As discussed elsewhere, it seems that the
  SPS transcription systematically omits the fields [Nature],
  [Another name], ans [Provenance] of every entry,
  as well as those comments that are marked by 《》 in the 
  SBJ transcription files.""")

  text_ch_midi = f"<{loc_ch}> (midi) " + text_ch_full
  text_ch_midi = re.sub(r"［］", "", text_ch_midi);
  text_ch_midi = re.sub(r"［味］[^［］《》主。]*[。]", "", text_ch_midi);
  text_ch_midi = re.sub(r"［一名］[^［］《》名主。]*[。]", "", text_ch_midi);
  text_ch_midi = re.sub(r"［生］[^［］《》生主。]*[。]", "", text_ch_midi);
  text_ch_midi = re.sub(r"[《][^［］《》生主]*[》]", "", text_ch_midi);
  text_ch_midi, tvar_ch_midi = \
    display_hanzi_pure_text(st, loc_ch, nch_midi, text_ch_midi)

  if (m := re.search(r"［[^］]*(味|一名|生)", text_ch_midi)): 
    assert False, f"entry trimming faile at '{m.group(0)}'"
  
  return text_ch_long, tvar_ch_long, text_ch_midi, tvar_ch_midi
  # ----------------------------------------------------------------------

def display_hanzi_pure_text(st, loc_ch, nch, hanzi_text):
  # Takes a {hanzi_text} of the form "<{LOC}> ({TVAR}) {TEXT}"
  # where {LOC} must match {loc_ch} and {TEXT} must be a hanzi
  # string possibly including hanzi punctuation.
  #.
  # Extracts a pure hanzi string {hanzi_pure} by removing all 
  # hanzi punctuation from {TEXT}. The result must contain only 
  # non-punctuation hanzi and its length (in hanzi) must be {nch}.
  # 
  # Appends to {st} a parag with {loc_ch}, {tvar_ch},{nch}
  # and the {hanzi_pure} text.  Returns {hanzi_pure} and the {TVAR} as 
  # the result.
  
  hanzi_text = hanzi_text.strip()
  hanzi_text = re.sub(r"[ \012]", "", hanzi_text)
  m = re.fullmatch(r"<([a-z0-9.]+)> *[(]([^()]+)[)] *(.*)", hanzi_text)
  assert m is not None, f"invalid chinese entry format {hanzi_text!r}"
  assert m.group(1) == loc_ch, f"loc ID mismtch: {loc_ch} != {m.group(1)}"
  tvar_ch = m.group(2)
  hanzi_body = m.group(3)
  hanzi_pure = re.sub(r"[：［］《》、，。；　]", "", hanzi_body)
  nch_real = len(hanzi_pure)
  if nch_real != nch:
    err.write(f"** trimming length error: {nch = } actual {nch_real}\n")
    err.write(f"  text = '{hanzi_body}'\n")
    err.write(f"  pure = '{hanzi_pure}'\n")
    assert False
    
  if nch < 45:
    hanzi_chops = [ hanzi_pure, ]
  else:
    hanzi_chops = [ hanzi_pure[k:k+40] for k in range(0, nch, 40) ]
  
  hanzi_display = f"<{loc_ch}> ({tvar_ch}) {nch:2d} hanzi\n" + "\n".join(hanzi_chops)
  h.append_preformatted(st, h.protect_html(hanzi_display), ind = 4, centered = False)
  
  return hanzi_pure, tvar_ch
  # ----------------------------------------------------------------------

def add_starps_matching_section \
  ( st, code_ch, loc_ch, itms_wh, variants, 
    del_qo, exp_irm, max_score, locs_to_try, locs_to_show,
    by_parag = True 
  ):
  # Appends to document {st} the body of a section that searches the SPS
  # file for parags matching a given SBJ entry in various ways.
  #
  # The {variants} argument must be a list of pairs of the form
  # 
  #   {((tvar_ch, text_ch), kwords_en)}
  #
  # where {kwords_en} is a list of strings identifying the "cribs" to
  # use in the matching, {text_ch} is a hanzi text of type "ch" which is
  # the suitably normalized and trimmed variant for the entry to be
  # matched, and {tvar_ch} is a short string that identifies {text_ch}
  # among other variants of the same SBJ recipe that differ in trimming
  # or other modifications.
  #
  # Calls {add_starps_matching_subsection} for each tuple
  # in {variants}, to compare that variant to all SPS parags.
  # 
  # Returns a list of of the results of those calls. Each element of
  # this list is a list of matching results. Each matching result is a
  # tuple as returned by {r77alt.search_bencao_entry_in_starps_parags_file}
  # (quod videt).
  #
  # Also adds a summary of the matches at the end.  If {by_parag} is true,
  # the summary is sorted and sectioned by the SPS parag loc ID, else it is sorted 
  # and sectioned by crib list and the SBJ text variant.
  # 
  # If {locs_to_try} is not {None}, it must be a set with the location IDS
  # of the SPS parags to be considered for the match. If {locs_to_try} is None,
  # considers all parags.
  #
  # If {locs_to_show} is not {None}, it must be a set or list of parag
  # locus IDs that should be listed in the summary, in addition to the
  # best ones in each variant.
  
  h.section(st, 2, f"Matching results")
  
  words_wh = [ wd.strip('　') for wd in itms_wh ]
  text_wh = ''.join(words_wh)
  
  pevs_list = []
  for tvtx_ch, kwords_en in variants:
    tvar_ch, text_ch = tvtx_ch
    cribs = stdbip.keywords_list_to_string(kwords_en)
    title = f"SBJ variant: {tvar_ch} Cribs: {cribs}"
    pevs = add_starps_matching_subsection \
      ( st, code_ch, loc_ch, tvar_ch, text_ch, text_wh, 
        kwords_en, del_qo, exp_irm, title, max_score, locs_to_try, locs_to_show
      )
    pevs_list.append(pevs)
    
  add_summary_of_matching_section \
    ( st, code_ch, loc_ch, text_wh, del_qo, exp_irm, 
      variants, pevs_list, locs_to_show, 
      by_parag
    )
  return pevs_list
  # ----------------------------------------------------------------------

def add_starps_matching_subsection \
  ( st, code_ch, loc_ch, tvar_ch, text_ch, text_wh, 
    kwords_en, del_qo, exp_irm, title, max_score, locs_to_try, locs_to_show
  ):
  # Appends to document {st} a section that searches the SPS
  # file for parags matching a given SBJ entry in various ways.
  # The parameters are:
  #
  #   {code_ch}       four-letter code of the SBJ entry.
  #   {loc_ch}        loc ID of the entry to be matched in the SBJ file.
  #   {tvar_ch}       a tag that identifies the variant (trimming etc.) of that entry.
  #   {text_ch}       the text of the variant to be matched, trimmed, without punctuation.
  #   {text_wh}       the full text of the entry, untrimmed, with punctuation.
  #   {kwords_en}     is a list of strings.
  #   {del_qo}        should we delete the @qo prefixes in the starps parags?
  #   {exp_irm}       should we expand the @ir, @is, @m finales in the starps parags?
  #   {title}         a title fof the section
  #   {max_score}     maximum interesting badness score.
  #   {locs_to_trty}  set of loc IDs of SPS parags to consider, or {None} for all.
  #   {locs_to_show}  list or set of parag loc IDs to show whatever their scores.
  #
  # Returns a list of matching results. Each matching result is a tuple
  # as returned by {r77alt.search_bencao_entry_in_starps_parags_file}
  # (quod videt).
  #
  # Each element of {kwords_en} should be a /keyword code/, a string,
  # like 'USES' or 'QI' that specifies an abstract keyword. This
  # parameter is converted to a /bitemplate/ {bitemp} as expected by
  # {btmpf.match_bitemplate}
  #
  # The procedure finds a macro-parsing {segs_ch} of {text_ch} by side 0
  # of {bitemp}. It then calls
  # {r77alt.search_bencao_entry_in_starps_parags_file} on the file of
  # good SPS parags {ivt_file} and evaluates each parag for how well it
  # matches the SBJ entry.
  # 
  # Let {nh} be the number of pattern codes in {kwords}. Let {ng} be
  # {nh+1}, and {ns} be {ng + nh}.
  # 
  # For each parag considered, the procedure creates a version {text_ec}
  # of its text, converted to "ec" format and suitably normalized. See
  # {spf.normalize_text} for how {del_qo} and {exp_irm} affect this
  # step. The preocedure then splits {text_ec} into a macro-parsings
  # {segs_ec[0..ns-1]}, consisting of {nh} /hits/ (strings matched by
  # the keyword templates of the hits in {segs_ch}) and {ng} /gaps/ (the
  # strings before, between, and after the hits).
  #
  # For each parag the procedure also computes a badness {score} that
  # combines penalties for the use of non-perfect hits (like 'laiin' 
  # instead of 'daiin') and discrepancies between the sizes of the 
  # gaps in {segs_ch} and {segs_ec}. 
  # 
  # The result is a list of parag evaluation tuples (/parevs/), one for
  # each parag that may possibly match, with its badness score, the
  # locus ID of the parag, the macro-parsing {segs_ch[0..ns-1]} of the SBJ entry,
  # and the matching macro-parsing {segs_ec[0..ns-1]} of the SPS parag.
  # 
  # This procedure then trims that list of parevs after the first one with
  # score exceeding {max_score}, and inserts the list into the document
  # {st}, formatted as described in {fmf.format_starps_parag_evaluation}.
  # However, parags whose locs are in {locs_to_show are shown in any case.
  #
  # The result of the call is that trimmed list.
  
  debug = False
  verbose = True
  
  ctsize_ch = len(text_ch)
  h.section(st, 3, f"{title}")
 
  h.parags(st, f"Trimmed SBJ entry ({tvar_ch} - {ctsize_ch} hanzi):")

  temptx_ch = f"<{loc_ch}> ({tvar_ch}) " + text_ch
  text_check, tvar_ch_check = display_hanzi_pure_text \
    ( st, loc_ch, ctsize_ch, temptx_ch )
  assert text_check == text_ch
  assert tvar_ch_check == tvar_ch
  
  # The result of comparing an SBJ entry with clean text {text_ch}
  # and and an SPS parag with clean normalized text {text_ec} is either {None}
  # or a parag evaluation record (/parev/). 
  # See {format_starps_parag_evaluation} for the contents of a parev.
  
  nh = len(kwords_en); ng = nh + 1; ns = ng + nh

  h.parags(st, "Keyword patterns:")
  
  bitemp = stdbip.build_bitemplate(kwords_en)
  r77alt.add_bitemplate_description(st, kwords_en, bitemp)
  
  hipat_ch, hipat_ec = r77alt.get_keyword_highlight_patterns_alt(bitemp)
  if debug: err.write(f"!@ SBJ highlight pattern = {hipat_ch!r}\n")
  if debug: err.write(f"!@ SPS highlight pattern = {hipat_ec!r}\n")

  # Scan the SPS parags collecting reasonable matches:
  use_python = False
  pevs, data = r77alt.search_bencao_entry_in_starps_parags_file \
    ( code_ch, loc_ch, tvar_ch, text_ch, kwords_en, bitemp, 
      locs_to_try, del_qo, exp_irm, use_python )

  # Discard totally bad candidates:
  nc = len(pevs)
  nc_good = 0  # Parevs with acceptable score.
  while nc_good < nc and pevs[nc_good]['score'] <= max_score: nc_good += 1
  err.write(f"  {nc_good = }\n")

  xcounts = []
  xcounts.append(f"{data['npar_read']:6d} parags read")
  if data['npar_excl'] > 0:
    xcounts.append(f"<b>{data['npar_excl']:6d} were were explicitly excluded.</b>")
  xcounts.append(f"{data['npar_eval']:6d} were evaluated.")
  min_sz_ok = data['min_valid_size']
  max_sz_ok = data['max_valid_size']
  xcounts.append(f"{data['too_small']:6d} were rejected for having less than {min_sz_ok} letters.")
  xcounts.append(f"{data['too_large']:6d} were rejected for having more than {max_sz_ok} letters.")
  xcounts.append(f"{data['npar_bima']:6d} were examined for the requested cribs")
  if data['npar_bima'] > 0:
    min_sz_match = data['min_matched_size']
    max_sz_match = data['max_matched_size']
    xszrange = f"{min_sz_match}..{max_sz_match}"
    xcounts.append(f"{data['npar_with']:6d} had all the cribs -- sizes {xszrange}")
    if nc != data['npar_with']:
      xcounts.append(f"{nc:6d} of them got a finite badness score.")
    xcounts.append(f"{nc_good:6d} parags had badness {max_score} or less.")
  else:
    assert nc == 0

  h.append_preformatted(st, "\n".join(xcounts), ind=0, centered=False)

  nc_show = nc_good
  
  # First parev with unacceptable score:
  if nc_show < nc: nc_show += 1 
  
  # Discard excessive parevs:
  max_show = 30
  nc_show = min(nc_show, max_show)
  
  if nc_show < max_show:
    # Ensure that some parevs are shown:
    nc_show = max(nc_show, min(nc, 3))
    
  # Salutar paranoia:
  err.write(f"  {nc_show = }\n")
  assert nc_show <= max_show + 1
  
  validate_parevs \
    ( pevs, loc_ch, tvar_ch, text_ch, 
      del_qo, exp_irm, max_score, nc_good, use_python, verbose
    )

  # Show the candidates:
  
  h.parags(st, f"Showing {nc_show} matches:")
 
  # Ensure that {locs_to_show} is a set:
  if locs_to_show is None:
    locs_to_show = set()
  elif isinstance(locs_to_show, list) or isinstance(locs_to_show, tuple):
    locs_to_show = set(locs_to_show)
  elif isinstance(locs_to_show, set):
    pass
  else:
    assert False, f"invalid {locs_to_show = !r}"

  ec_list_blocks = []
  prev_segs_ch = None
  pevs_out = []
  for ic in range(nc):
    pev = pevs[ic]
    score = pev['score']
    loc_ch_1 = pev['loc_ch']; tvar_ch_1 = pev['tvar_ch']; segs_ch = pev['segs_ch']
    loc_ec = pev['loc_ec']; tvar_ec = pev['tvar_ec']; segs_ec = pev['segs_ec']
    kwords_en = pev['kwords_en']; eva_per_hanzi= pev['eva_per_hanzi']
    hit_penalties = pev['hit_penalties']
    # !!! check unused !!!
  
    assert loc_ch_1 == loc_ch
    assert tvar_ch_1 == tvar_ch
    show = ic < nc_show or loc_ec in locs_to_show

    if show:
      if segs_ch != prev_segs_ch:
        # Must show the SBJ entry parsing:
        ec_list_blocks.append("\n")
        ch_str = fmf.format_macro_parsing_ch(loc_ch, tvar_ch, segs_ch, hipat_ch)
        ch_str = h.indent_lines(4, ch_str)
        ec_list_blocks.append(ch_str)
        ec_list_blocks.append("\n")
        prev_segs_ch = segs_ch
        
      ideal_eph = spf.expected_eva_letters_per_hanzi(del_qo, exp_irm)
      ec_str = fmf.format_starps_parag_evaluation(pev, hipat_ec, ideal_eph)
      ec_list_blocks.append(ec_str)
      pevs_out.append(pev)

  ec_list_str = "\n".join(ec_list_blocks)
  ec_list_str = h.protect_html(ec_list_str)
  h.append_preformatted(st, ec_list_str, ind = 2, centered = False)

  return pevs_out
  # ----------------------------------------------------------------------

def validate_parevs \
  ( pevs, loc_ch, tvar_ch, text_ch, 
    del_qo, exp_irm, max_score, nc_good, use_python, verbose
  ):
  prev_score_p = -inf; prev_loc_ec_p = "NONE"
  nc = len(pevs)
  for ic in range(nc):
    pev = pevs[ic]

    score_p = pev['score']
    loc_ch_p = pev['loc_ch']; tvar_ch_p = pev['tvar_ch']; segs_ch_p = pev['segs_ch']
    loc_ec_p = pev['loc_ec']; tvar_ec_p = pev['tvar_ec']; segs_ec_p = pev['segs_ec']
    kwords_en_p = pev['kwords_en']; eva_per_hanzi_p = pev['eva_per_hanzi']
    hit_penalties_p = pev['hit_penalties']
    # !!! check unused !!!
 
    assert loc_ch_p == loc_ch, \
      f"wrong SBJ locus {loc_ch_p = !r} {loc_ch = !r}"
    assert tvar_ch_p == tvar_ch, \
      f"wrong SBJ variant {tvar_ch_p = !r} {tvar_ch = !r}"
    assert loc_ec_p != prev_loc_ec_p, \
      f"dup SPS parag {loc_ec_p = !r}"

    text_ch_p = "".join(segs_ch_p)
    assert text_ch_p == text_ch, \
      f"SBJ text/segs mismatch {text_ch_p = !r} {text_ch = !r}"

    text_ec_p = "".join(segs_ec_p)
    
    if verbose: 
      err.write(f"  parag {loc_ec_p:<12s} {score_p = :6.2f}\n")
    assert isfinite(score_p) and score_p > 0, \
      f"bad {score_p = }"
    assert score_p >= prev_score_p, \
      f"scores out of order {prev_score_p = } {score_p = !r}"
    if ic < nc_good: 
      assert score_p <= max_score, f"{score_p = } exceeds max_score"
      
    assert isfinite(eva_per_hanzi_p)
    if use_python:
      score_check = bef.compute_full_score_from_macro_parsings \
        (segs_ch_p, segs_ec_p, eva_per_hanzi_p, hit_penalties_p)
      if score_p != score_check:
        err.write(f"{score_p =     :24.16e}\n")
        err.write(f"{score_check = :24.16e}\n")
      assert abs(score_p - score_check) < 1.0e-6
    prev_score_p = score_p
    prev_loc_ec_p = loc_ec_p
  return
  # ----------------------------------------------------------------------
  
def write_dics_from_parev(st, code_ch, pev):
  # Writes a set of hanzi-to-EVA dictionaries based on the locus ID {loc_ch} and
  # text {text_ch} of an SBJ entry, the locus ID {loc_ec} of an SPS parag,
  # and the macro-parsings of the two parsed texts.
  #
  # The dictionary for SBJ text fragments of length {fsize_ch}
  # is written to file "dics/{code_ch}_{loc_ec}_{fsize_ch}.dic".
  
  score = pev['score']
  loc_ch = pev['loc_ch']; tvar_ch = pev['tvar_ch']; segs_ch = pev['segs_ch']
  loc_ec = pev['loc_ec']; tvar_ec = pev['tvar_ec']; segs_ec = pev['segs_ec']
  kwords_en = pev['kwords_en']; eva_per_hanzi = pev['eva_per_hanzi']
  hit_penalties = pev['hit_penalties']
  # !!! check unused !!!

  # Score and locus ID of parag:
  loc_ec = re.sub(r"<[^<>]*>", "", loc_ec) # Just in case:
  loc_ec = re.sub(r"[.]([0-9])$", r".0\1", loc_ec) # Zero-pad the line number.
  
  max_fsize_ch = 4
  for fsize_ch in range(max_fsize_ch + 1):
    vms_dic = make_dic_from_parev(code_ch, loc_ch, pev, fsize_ch)
    assert vms_dic is not None

    dic_file = f"dics/{code_ch}_{loc_ec}_{fsize_ch}.dic"
    wr = open(dic_file, "w")
    wr.reconfigure(encoding='utf-8')
    wr.write("# -*- coding: utf-8 -*-\n")
    wr.write(f"# {loc_ch = }\n")
    wr.write(f"# {loc_ec = }\n")

    pref = f"{code_ch}:{tvar_ch} | {loc_ch:<8s} | {loc_ec:<8s} |"

    for frag_ch, frag_ec in vms_dic:
      wr.write(pref)
      assert fsize_ch == 0 or len(frag_ch) == fsize_ch
      frag_ch = frag_ch.ljust(10,"　")
      frag_ec = frag_ec.ljust(50," ")
      wr.write(f" {frag_ch} | {frag_ec} |\n")

    wr.close()
    
  return
  # ----------------------------------------------------------------------

def add_summary_of_matching_section \
  ( st, code_ch, loc_ch, text_wh, del_qo, exp_irm,
    variants, pevs_list, locs_to_show,
    by_parag
  ):
  # The {pevs_list} must be a lits of lists of parevs, where
  # {pevs_list[iv]} is the result of matching according to variant
  # {variants[iv]}.  
  #
  # Selects a few parags that look like valid matches, or best
  # approximations thereof. For each variant, and each of those parags,
  # plus the parags in {locs_to_show}, prints a line with a terse
  # summary: badness, EVA keywords matched, and gap errors, followed by
  # the graphic plot.
  #
  # If {by_parag} is true, the summary is sorted and sectioned by the
  # SPS parag loc ID, else it is sorted and sectioned by the 
  # crib list and then by SBJ text varant.
  
  h.section(st, 2, "Matching summary")

  nv = len(variants); assert nv == len(pevs_list)
  
  # First, select the best matches from each variant, and any 
  # other parags that have similar badness:
  #
  cands = locs_to_show # Set of loc ids of candidates.
  for iv in range(nv):
    cands |= select_best_cands(pevs_list[iv])
  
  # Second, for each cand, get the best score among all the 
  # variants:
  best_loc_scores = [] # List of pairs {(loc_ec, score_min)}
  for loc_ec in cands:
    score_min = +inf
    for iv in range(nv):
      tvtx_ch, kwords_en = variants[iv]
      tvar_ch, text_ch = tvtx_ch
      for pev in pevs_list[iv]:
        score_p = pev['score']
        loc_ch_p = pev['loc_ch']; tvar_ch_p = pev['tvar_ch']
        loc_ec_p = pev['loc_ec']; tvar_ec_p = pev['tvar_ec']
        assert loc_ch_p == loc_ch
        assert tvar_ch_p == tvar_ch
        if loc_ec == loc_ec_p:
          if score_p < score_min: score_min = score_p
    best_loc_scores.append((loc_ec, score_min, ))
  best_loc_scores.sort(key = lambda x: x[1])
  
  if by_parag:
    for loc_ec, score_min in best_loc_scores:
      add_summary_of_parag_matching \
        ( st, loc_ec, del_qo, exp_irm, 
          code_ch, loc_ch, text_wh, variants,
          pevs_list )
  else:
    # Now print the summaries per variant:
    for iv in range(nv):
      pevs = pevs_list[iv]
      tvtx_ch, kwords_en = variants[iv]
      tvar_ch, text_ch = tvtx_ch
      cribs = stdbip.keywords_list_to_string(kwords_en)
      title = f"Trim: {tvar_ch} Cribs: {cribs}"
      add_summary_of_variant_matching \
        ( st, code_ch, loc_ch, tvar_ch, text_wh, del_qo, exp_irm,
          best_loc_scores, pevs, title )

  return
  # ----------------------------------------------------------------------
    
def select_best_cands(pevs):
  # Given a list {pevs} of parevs, selects the one with minimum badness score
  # and a few more with similar scores, if any. Returns the /set/
  # of the locus IDs of the parags selected.
  # Expects the list {pevs} to be sorted by non-decreasing 
  # badness scores.
  
  max_cands = 5
  cands = set()
  if pevs is not None:
    score_min = None
    tol = 0.2
    for pev in pevs:
      score = pev['score']
      loc_ch = pev['loc_ch']; tvar_ch = pev['tvar_ch']; segs_ch = pev['segs_ch']
      loc_ec = pev['loc_ec']; tvar_ec = pev['tvar_ec']; segs_ec = pev['segs_ec']
      kwords_en = pev['kwords_en']
      eva_per_hanzi = pev['eva_per_hanzi']; hit_penalties = pev['hit_penalties']
      if score < +inf:
        if score_min is None: score_min = score
        assert score >= score_min
        if score <= (1 + tol)*score_min: 
          cands.add(loc_ec)
          tol = 0 if len(cands) >= max_cands else tol/2
  return cands
  # ----------------------------------------------------------------------

def add_summary_of_variant_matching \
  ( st, code_ch, loc_ch, tvar_ch, text_wh, del_qo, exp_irm, 
    best_loc_scores, pevs, title ):
  # Appends to {st} a summary of the matching attempts of one variant.
  # 
  # The {best_loc_scores} must be a list of pairs {(loc_ec, score_min)}
  # where {loc_ec} is the locus ID of an SPS parag and {score_min}
  # is its lowest badness score over all the variants.
  # The order of these pairs defines the order in which the 
  # parags will be listed.
  # 
  # The {pevs} must be the list of parevs that resulted from 
  # the matching of this variant.
  #
  # The {title} should be an explanatory title for the varaint.

  def find_parev(loc_ec, pevs):
    # Finds the parev of parag {loc_ec} in {pevs}, or {None}:
    the_pev = None
    for pev in pevs:
      if pev['loc_ec'] == loc_ec: # !!! Maybe check {tvar_ec}, {kwords_en} !!!
        the_pev = pev
        break
    return the_pev
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

  h.section(st, 3, title)
  rows = []
  matched = False
  rows.append([ "<hr style=\"border:0;\">" ])
  for loc_ec, score_min in best_loc_scores:
    tvar_ec = None
    pev = find_parev(loc_ec, pevs)
    if pev is not None:
      score_p = pev['score']
      if score_p == +inf:
        pev = None
      else:
        assert isfinite(score_p) and score_p >= score_min
        tvar_ec = pev['tvar_ec']
    sum_rows = make_rows_of_summary_of_matching \
      ( st, code_ch, loc_ch, tvar_ch, loc_ec, tvar_ec, pev )
    for row in sum_rows:
      if row is not None: rows.append(row)
    if pev is not None:
      row = make_plot_row_for_summary_of_matching \
        ( st, code_ch, text_wh, del_qo, exp_irm, pev )
      rows.append(row)
    rows.append([ "<hr style=\"border:0;\">", ])
  add_match_summary_table(st, rows)
  return
  # ----------------------------------------------------------------------

def add_match_summary_table(st, rows):
  # Adds a tabel to a match summary section with data of one or more
  # parevs.  Each parev is assumed to be formatted as one or more rows
  # with the following general columns:
  #
  #   {code_ch} {tvar_ch} {loc_ec} {tvar_ec} {eph} {score} {tsz} {obs}
  #
  # where {eph} is the {eva_per_hanzi} used when computing the parev's score,
  # {totsz} is the total parag or total gap size, its error, and implied e/h,
  # and {obs} is some string.
  
  sty_left =      "padding-left:1ch; padding-right:1ch; text-align:left;"
  sty_left_bold = "padding-left:1ch; padding-right:1ch; text-align:left; font-weight:bold;"
  sty_right =     "padding-left:1ch; padding-right:1ch; text-align:right;"

  sty_code_ch = sty_left_bold
  sty_tvar_ch = sty_left_bold
  sty_loc_ec =  sty_left_bold
  sty_tvar_ec = sty_left_bold
  sty_eph =     sty_right
  sty_score =   sty_right
  sty_tsz =     sty_left
  sty_obs =     sty_left
  
  col_mods = [
     f"style='{sty_code_ch}'",  # {code_ch}
     f"style='{sty_tvar_ch}'",  # {tvar_ch}
     f"style='{sty_loc_ec}'",   # {loc_ec}
     f"style='{sty_tvar_ec}'",  # {tvar_ec}
     f"style='{sty_eph}'",      # {eva_per_hanzi}
     f"style='{sty_score}'",    # {score} 
     f"style='{sty_tsz}'",      # {tot size info} 
     f"style='{sty_obs}'",      # {misc info}
   ]
  html_tb = h.make_table(rows, by_rows = True, col_mods = col_mods)
  h.append_centered(st, html_tb, centered = False)
  return
  # ----------------------------------------------------------------------

def add_summary_of_parag_matching \
  ( st, loc_ec, del_qo, exp_irm, 
    code_ch, loc_ch, text_wh, variants,
    pevs_list ):
  # Prints a summary of the matching attempts for parag {loc_ec}.
  # They are sorted and sectioned by crib list and then by SBJ text variant.
  
  nv = len(variants); assert nv == len(pevs_list)
  
  # Collect all parevs of parag {loc_ec}:
  this_pevs = []
  for pevs in pevs_list:
    for pev in pevs:
      if pev['loc_ec'] == loc_ec:
        this_pevs.append(pev)
        
  title3 = f"Parag: {loc_ec}"
  if len(this_pevs) == 0:
    title3 += " -- No match"
  
  h.section(st, 3, title3)
  
  if len(this_pevs) == 0: return
  
  # Sort the parevs by crib list and then by SBJ text variant:
  this_pevs.sort(key = lambda pev: (tuple(pev['kwords_en']), pev['tvar_ch']) )
  
  rows = []
  prev_kwords_en = [ ]
  for pev in this_pevs:
    kwords_en = pev['kwords_en']
    if kwords_en != prev_kwords_en:
      # New sub-section - close off the previous one:
      if len(rows) != 0:
        add_match_summary_table(st, rows)
        rows = []
      # New sub-section - print header:
      cribs = stdbip.keywords_list_to_string(kwords_en)
      title4 = f"Cribs: {cribs}"
      h.section(st, 4, title4)
      prev_kwords_en = kwords_en
    tvar_ch = pev['tvar_ch']
    tvar_ec = pev['tvar_ec']
    sum_rows = make_rows_of_summary_of_matching \
      ( st, code_ch, loc_ch, tvar_ch, loc_ec, tvar_ec, pev )
    for row in sum_rows:
      if row is not None: rows.append(row)
    if pev is not None:
      row = make_plot_row_for_summary_of_matching \
        ( st, code_ch, text_wh, del_qo, exp_irm, pev )
      rows.append(row)
    rows.append([ "<hr style=\"border:0;\">", ])

  # Finish off last table:
  if len(rows) != 0:
    add_match_summary_table(st, rows)
  return
  # ----------------------------------------------------------------------

def make_rows_of_summary_of_matching \
  (st, code_ch, loc_ch, tvar_ch, loc_ec, tvar_ec, pev):
  # Returns a row of the table of summary of variant matchings, for 
  # the parev {pev}, which must be for the given .
  # 
  # The row always has the fllowing fields extracted from {pev}: 
  #
  #   the variant of the SBJ text {tvar_ch}.
  #   the starps parag locus ID {loc_ec}.
  #   its  variant {tvar_ec}.
  #   the badness score of the bimatch.
  #   the assumed EVA per hanzi ratio.
  #   the hit strings used in the SPS side.
  #   the total hit penalty.
  #   the total SPS text length error
  #   the individual SPS gap errors.
  #
  # However if {pev} is {None} then a "No match" replaces the score, and
  # the other fields are empty strings.
  
  # First row -- recipe and parag IDs and variants, tot score, assumed and actual e/h.
  rows = [ ]
  row1 = [ ]

  row1.append(f"{code_ch}")
  
  tvar_ch_str = f"({tvar_ch})"
  row1.append(f"{tvar_ch_str:<10s}")
  
  row1.append(f"{loc_ec:<7s}")
  
  tvar_ec_str = f"({tvar_ec})" if tvar_ec is not None else "----"
  row1.append(f"{tvar_ec_str:<8s}")
  
  if pev is None:
    row1.append("No match")
    rows.append(row1)
  else:
    score = pev['score']
    assert isfinite(score) and score >= 0
    assert pev['loc_ch'] == loc_ch
    assert pev['tvar_ch'] == tvar_ch 
    assert pev['loc_ec'] == loc_ec
    assert pev['tvar_ec'] == tvar_ec
    segs_ec = pev['segs_ec']
    segs_ch = pev['segs_ch']
    kwords_en = pev['kwords_en']
    eva_per_hanzi = pev['eva_per_hanzi']
    hit_penalties = pev['hit_penalties']
    
    eph_str = f"{eva_per_hanzi:7.3f} e/h"
    row1.append(eph_str)
    
    score_str = f"<b>{score:7.3f}</b>"
    row1.append(score_str)
    
    tsz_all_ec, tsz_gap_ec = spf.total_seg_sizes(segs_ec)
    tsz_all_ch, tsz_gap_ch = spf.total_seg_sizes(segs_ch)

    # Total parag size and error for it:
    tsz_all_str = fmf.size_ec_summary(tsz_all_ec, tsz_all_ch, eva_per_hanzi)
    row1.append(f"{tsz_all_str}")
    
    rows.append(row1)
    
    # Second row -- gap score, gap sizes, tot gap size, errors, actual gap e/h
    row2 = [ ]

    row2.append(f" ") # {code}
    row2.append(f" ") # {tvar_ch}
    row2.append(f" ") # {loc_ec}
    row2.append(f" ") # {tvar_ec}
    
    row2.append(f"gaps:") # {eva_per_hanzi}
    
    hit_score = 0
    for hp in hit_penalties: hit_score += hp
    gap_score = score - hit_score
    
    gap_score_str = f"{gap_score:7.3f}"
    row2.append(gap_score_str)
     
    # Total gap size and error for it:
    tsz_gap_str = fmf.size_ec_summary(tsz_gap_ec, tsz_gap_ch, eva_per_hanzi)
    row2.append(f"{tsz_gap_str}")
     
    hits_ec, tot_err, gap_errs = short_parev_summary \
      ( segs_ch, segs_ec, eva_per_hanzi )
    
    gap_errs_str = ",".join([ "00" if ger == 0 else f"{ger:+d}" for ger in gap_errs ])
    row2.append(gap_errs_str)
    
    rows.append(row2)

    # Third row -- hit score, hits.
    row3 = [ ]

    row3.append(f" ") # {code}
    row3.append(f" ") # {tvar_ch}
    row3.append(f" ") # {loc_ec}
    row3.append(f" ") # {tvar_ec}
    row3.append(f"hits:") # {eva_per_hanzi}

    hit_score_str = f"{hit_score:7.3f}"
    row3.append(f"{hit_score_str:<13s}")
    
    hits_ec_str = ",".join([ "@" + ht for ht in hits_ec ])
    row3.append(hits_ec_str)
    
    rows.append(row3)
    
  return tuple(rows)
  # ----------------------------------------------------------------------
      
def make_plot_row_for_summary_of_matching \
  ( st, code_ch, text_wh, del_qo, exp_irm, pev ):
  assert pev is not None
  score_p = pev['score']
  loc_ch_p = pev['loc_ch']; tvar_ch_p = pev['tvar_ch']; segs_ch_p = pev['segs_ch']
  loc_ec_p = pev['loc_ec']; tvar_ec_p = pev['tvar_ec']; segs_ec_p = pev['segs_ec']
  kwords_en_p = pev['kwords_en']; eva_per_hanzi_p = pev['eva_per_hanzi']
  hit_penalties_p = pev['hit_penalties']
  # !!! check unused !!!
  assert isfinite(score_p)
  
  text_ch = "".join(segs_ch_p)
  
  text_ec = "".join(segs_ec_p)
  text_wc = get_clean_normalized_starps_text_wc(loc_ec_p, del_qo, exp_irm)
  
  # Create the plot files:
  fname_plot_png, fname_plot_jpg = create_graphical_alignment_plot \
    (code_ch, text_wh, text_wc, pev)

  img_size = st['text_width']-200
  img_url = fname_plot_png
  img_thumb_url = fname_plot_jpg
  link_text = f"<a href=\"{fname_plot_png}\">{fname_plot_png}</a>"
  link_url = img_url
  html_plot = h.make_link(st, link_url, link_text, img_thumb_url, img_size, img_size)
  html_plot = f"<span style=\"display:inline-block;padding-bottom:2ex;\">{html_plot}</span>"
  row = [ html_plot ]
  return row
  # ----------------------------------------------------------------------

def short_parev_summary(segs_ch, segs_ec, eva_per_hanzi):
  # Returns a list of the hits in {segs_ec}, the total size error,
  # and a list of gap errors compared to the 
  # predictions from {segs_ch}, in EVA letters.
  
  ns = len(segs_ch); assert len(segs_ec) == ns
  nh = ns//2; ng = nh + 1; assert ns == nh + ng

  # Total text sizes:
  tsz_ch = sum(len(s) for s in segs_ch)
  tsz_ec = sum(len(s) for s in segs_ec)

  hits_ec = [ segs_ec[2*ih + 1] for ih in range(nh) ]

  tot_err = fmf.size_error(tsz_ch, tsz_ec, eva_per_hanzi)
  gap_errs = [ ]
  for ig in range(ng):
    gsz_ch = len(segs_ch[2*ig])
    gsz_ec = len(segs_ec[2*ig])
    gap_errs.append(fmf.size_error(gsz_ch, gsz_ec, eva_per_hanzi))
  return hits_ec, tot_err, gap_errs
  # ----------------------------------------------------------------------

def add_chosen_starps_parag_section \
  ( st, itms_tg, loc_ch, code_ch, itms_wh,
    chosen_loc_ec, chosen_tvar_ch, chosen_kwords_en, 
    del_qo, exp_irm, pevs_list, itms_en
  ):
  # Inserts a parag saying that the SPS parag chosen to match 
  # the SBJ entry {loc_ch} (code {code_ch}) is {chosen_loc_ec}.
  #
  # The {chosen_loc_ec} may be {None} to say that the SBJ entry will not be
  # assigned to any SPS parag. In that case, the procedure ignores 
  # the parameters {chosen_tvar_ch}, and {chosen_kwords_en} are ignored, 
  # inserta a note in the report, and returns {None}.
  #
  # If {chosen_loc_ec} is not {None}, the procedure hooses a parev in
  # {pevs_list} by caling {find_best_starps_parag} (q.v.) with arguments
  # {chosen_loc_ec}, {chosen_tvar_ch}, {chosen_kwords_en}, {pevs_list}
  #
  # If the search in {pevs_list} fails to find the requested parev,
  # displays a warning on the report and returns {None}.
  #
  # If the search returns a parev {best_pev}, the procedure displays
  # the parsings of the SBJ entry and of the SPS parag described
  # therein, both textually and graphically.
  #
  # The textual display includes the "macro-parsing" of the EVA
  # characters of the SPS parag into gaps and hit, as contained in the
  # {best_pev}.
  #
  # The textual display also shows a "micro-parsing" where the gaps are
  # subdivided according to given parsings of the hanzi text into
  # syntactic items. This display also shows the english translation.
  #
  # The graphical display plots the "ch" and "ec" text as two horz lines
  # with rungs connecting the hits.
  #
  # Also writes the hanzi-eva dictionary files implied by the chosen parev.
  # 
  # Returns that {best_pev}.
  #
  # Also writes the data files for the graphical match figure to 
  # files "fig/bencao/{code_ch}.utp" and "fig/starps/{chosen_loc_ec}.evp"
  # then runs the plotting program and includes the plot in the HTML report.
    
  debug = False
  
  h.section(st, 2, "Chosen match")
  
  text_wh = "".join(itms_wh)

  pev = None
  if chosen_loc_ec is not None:
    h.parags(st, f"""We will tentatively assign {code_ch}
    <{loc_ch}> (variant {chosen_tvar_ch}) to {chosen_loc_ec}.  However we must be aware
    that the true match may not have made it into the "good" subset.""")
    pev = find_best_starps_parag(chosen_tvar_ch, chosen_loc_ec, chosen_kwords_en, pevs_list)
    if pev is None:
      msg_tvar_ch = "" if chosen_tvar_ch is None else f" FOR VARIANT {chosen_tvar_ch}"
      if chosen_kwords_en is None:
        msg_kwords_en = "" 
      else:
        kwords_en_str = stdbip.keywords_list_to_string(chosen_kwords_en)
        msg_kwords_en = f" WITH CRIBS {kwords_en_str}"
      msg = \
        f"WARNING - MISSING EVAL RECORD" + \
        f" FOR {code_ch}{msg_tvar_ch}" + \
        f" AGAINST {chosen_loc_ec}{msg_kwords_en}"
      h.parags(st, f"<b>{msg}</b>")
      err.write(f"!! {msg}\n")
    else:
      if debug: err.write(f"!c \n{pev = !r}\n")
  else:
    h.parags(st, f"""We will not assign {code_ch} <{loc_ch}> to any SPS parag.""")
  
  itms_wc = None
  if pev is not None:
    score = pev['score']
    loc_ch_p = pev['loc_ch']; tvar_ch = pev['tvar_ch']; segs_ch = pev['segs_ch']
    loc_ec = pev['loc_ec']; tvar_ec = pev['tvar_ec']; segs_ec = pev['segs_ec']
    kwords_en = pev['kwords_en']; eva_per_hanzi = pev['eva_per_hanzi']
    hit_penalties = pev['hit_penalties']
    if debug:
      err.write(f"!& ### macro-parsing ch, ec from parev ###\n")
      alf.write_wh_ec_wc_macro_parsings(err, "!&", segs_ch, segs_ec, None)
    assert loc_ch_p == loc_ch
    if chosen_loc_ec is not None: assert loc_ec == chosen_loc_ec 
    if chosen_tvar_ch is not None: assert tvar_ch == chosen_tvar_ch 
    if chosen_kwords_en is not None: assert kwords_en == chosen_kwords_en 
    
    kwords_en_str = stdbip.keywords_list_to_string(kwords_en)
    h.parags(st, f"Requested cribs: {kwords_en_str}:")
    
    h.parags(st, f"SBJ entry parsing:")
    ch_str = fmf.format_macro_parsing_ch(loc_ch, tvar_ch, segs_ch, None)
    ch_str = h.protect_html(ch_str)
    h.append_preformatted(st, ch_str, ind = 4, centered = False)

    h.parags(st, f"SPS entry parsing:")
    
    ideal_eph = spf.expected_eva_letters_per_hanzi(del_qo, exp_irm)
    ec_str = fmf.format_starps_parag_evaluation(pev, None, ideal_eph)
    ec_str = h.protect_html(ec_str)
    h.append_preformatted(st, ec_str, ind = 4, centered = False)
    
    # Compute the ideal EVA/hanzi ratio from the gap sizes:
    tot_eph, gap_eph, avg_eph = compute_observed_eva_per_hanzi(segs_ch, segs_ec)
    h.parags(st, f"EVA/hanzi ratio from text lengths = {tot_eph:5.3f}")
    if isnan(gap_eph):
      assert avg_eph == tot_eph
    else:
      h.parags(st, f"EVA/hanzi ratio from gap lengths = {gap_eph:5.3f}")
      h.parags(st, f"Suggested EVA/hanzi ratio = {avg_eph:5.3f}")

    write_dics_from_parev(st, code_ch, pev)
    
    write_cribs_used(st, segs_ch, segs_ec)

    #  Fetch the parag from the word-split SPS good parags file:
    text_wc = get_clean_normalized_starps_text_wc(chosen_loc_ec, del_qo, exp_irm)

    add_chosen_textual_alignment_subsection \
      ( st, itms_tg, itms_wh, itms_en, segs_ch, 
        text_wc, chosen_loc_ec, segs_ec )
      
    add_chosen_graphical_alignment_subsection \
      ( st, code_ch, text_wh, text_wc, pev )

  return pev
  # ----------------------------------------------------------------------
    
def compute_observed_eva_per_hanzi(segs_ch, segs_ec):
  ns = len(segs_ch); assert len(segs_ec) == ns
  nh = ns//2; ng = nh + 1; assert ns == ng + nh
  
  # Compute the EVA/hanzi ratio using the total text lengths:
  len_ch = sum(len(s) for s in segs_ch)
  len_ec = sum(len(s) for s in segs_ec)
  tot_eva_per_hanzi = len_ec/len_ch

  # Compute the bencao and starps total gap sizes, biased by gap weight:
  sum_wt_gsz_ch = 0
  sum_wt_gsz_ec = 0
  sum_wt = 0
  for ig in range(ng):
    ks = 2*ig
    wt = bef.gap_score_weight(ig, ng)
    sum_wt_gsz_ch += wt * len(segs_ch[ks])
    sum_wt_gsz_ec += wt * len(segs_ec[ks])
    sum_wt += wt
  assert sum_wt > 0
  tot_gsz_ch = ng * sum_wt_gsz_ch/sum_wt
  tot_gsz_ec = ng * sum_wt_gsz_ec/sum_wt
  
  # If the bencao gaps are a small fraction of the entry, use tot length:
  if tot_gsz_ch == 0:
    # The bencao text i all hits, no gaps:
    gap_eva_per_hanzi = nan  
    avg_eva_per_hanzi = tot_eva_per_hanzi
  else:
    gap_eva_per_hanzi = tot_gsz_ec / tot_gsz_ch
    # If the bencao gaps are too small, bias towards text length rato:
    fgap_ch = tot_gsz_ch/len_ch
    s = fgap_ch * fgap_ch
    avg_eva_per_hanzi = (1-s) * tot_eva_per_hanzi + s * gap_eva_per_hanzi
  
  return tot_eva_per_hanzi, gap_eva_per_hanzi, avg_eva_per_hanzi
  # ----------------------------------------------------------------------
    
def write_cribs_used(st, segs_ch, segs_ec):
  # Extracts and prints from {segs_ch,segs_ec} a 
  # table of the cribs actually used in the match.
  ns = len(segs_ch); assert len(segs_ec) == ns
  nh = ns//2; ng = nh + 1; assert ns == ng + nh

  # Collect the cribs used:
  cribs = [] 
  for ih in range(nh):
    ks = 2*ih + 1
    cri = [ segs_ch[ks], segs_ec[ks], 1 ] # !!! PENALTY TOO? !!!
    cribs.append(cri)
  # Sort them lexicographically:
  cribs.sort()
  # Combine repeated uses into multiplicities:
  kc = 0
  for ic in range(len(cribs)):
    err.write(f"!C  {kc} {cribs[kc]!r}  {ic} {cribs[ic]!r}\n")
    if kc < ic:
      if cribs[kc][0] == cribs[ic][0] and cribs[kc][1] == cribs[ic][1]:
        cribs[kc][2] += 1
      else:
        kc += 1
        cribs[kc] = cribs[ic]
  nc = kc+1 # Number of cribs.
  cribs = cribs[:nc]
  # Format ane print them:
  h.parags(st, "Cribs actually used in this match:")
  hwd_ch = max(len(c[0]) for c in cribs)
  hwd_ec = max(len(c[1]) for c in cribs)
  lines = []
  for ic in range(nc):
    cri = cribs[ic]
    hit_ch = cri[0].ljust(hwd_ch,'　')
    hit_ec = cri[1].ljust(hwd_ch,' ')
    xcriwdwd = f"{cri[2]:2d} {hit_ch} {hit_ec}"
    lines.append(xcriwdwd)
  lines = "\n".join(lines)
  h.append_preformatted(st, lines, ind = 2, centered = False)
  return 
  # ----------------------------------------------------------------------
    
def get_clean_normalized_starps_text_wc(loc_ec, del_qo, exp_irm):
  ivt_file = "res/starps-gd-wc-par.ivt"
  rawtx_wc, nlin = fetch_starps_line(ivt_file, loc_ec)
  assert rawtx_wc is not None, f"** cannot find {loc_ec} in the starps file"

  def data_error(msg):
    file_line_error(ivt_file, nlin, msg, f"<{loc_ec}> {rawtx_wc}")
    assert False
    # ..................................................................

  utype = "wc"
  text_wc, head, tail = convf.convert_starps_raw_text(rawtx_wc, utype, data_error)
  text_wc = normf.normalize_starps_text(text_wc, utype, del_qo, exp_irm, data_error)

  return text_wc
  # ----------------------------------------------------------------------

def add_chosen_textual_alignment_subsection \
  ( st, itms_tg, itms_wh, itms_en, segs_ch, text_wc, loc_ec, segs_ec ):
  # Adds a subsection with a three-column table that displays the
  # "micro-parsings" {itms_tg}, {itms_wh}, {itms_en} and the punctuated
  # EVA text {txt_wc}, refined to make them compatible with the
  # macro-parsings {segs_ch} and {segs_ec}and with each other.
  #
  # The english text strings from {itms_en} are also aligned but not split.

  bites_tg, bites_wh, bites_wc, bites_en = \
    alf.align_text_wc_with_micro_parsing_tg_wh_en_and_macro_parsing_ec \
      ( itms_tg, itms_wh, itms_en, segs_ch, text_wc, segs_ec )

  assert bites_wc is not None

  h.section(st, 3, "Aligning the two versions")

  h.parags(st, f""" Here is the same text with the conjectured
  correspondence with parag {loc_ec} of the
  SPS:""")

  repf.add_three_column_entry_table(st, bites_tg, bites_wh, bites_wc, bites_en)

  h.parags(st, """Note that the alignment of the Voynichese column 
  is only a rough guess based on the hanzi and EVA letter counts.""")      

  return 
  # ----------------------------------------------------------------------

def add_chosen_graphical_alignment_subsection(st, code_ch, text_wh, text_wc, pev):
  # Adds a subsection with a figure showing the
  # hanzi text of {itms_wh} and {text_wc} 
  # with the bimatching implied by {pev}.

  h.section(st, 3, "Graphical alignment")

  fname_plot_png, fname_plot_jpg = create_graphical_alignment_plot \
    (code_ch, text_wh, text_wc, pev)

  img_wd = st['text_width']-200
  hr.image_link_parag \
    ( st, img_url = fname_plot_png, img_size = img_wd, 
      link_text = fname_plot_png, img_thumb_url = fname_plot_jpg
    )

  return 
  # ----------------------------------------------------------------------

def create_graphical_alignment_plot(code_ch, text_wh, text_wc, pev):
  # Runs an external program that creates a graphical image of the 
  # {pev}.  Returns the name of the image file.
  
  debug = False
  
  def data_error(msg):
    err.write(f"** {msg}\n")
    assert False
  
  score = pev['score']
  loc_ch = pev['loc_ch']; tvar_ch = pev['tvar_ch']; segs_ch = pev['segs_ch']
  loc_ec = pev['loc_ec']; tvar_ec = pev['tvar_ec']; segs_ec = pev['segs_ec']
  kwords_en = pev['kwords_en']; eva_per_hanzi = pev['eva_per_hanzi']
  hit_penalties = pev['hit_penalties']

  if debug: err.write(f"!g {segs_ch = !r}\n")
  text_ch = ''.join(segs_ch)
  punct_ch = set(r"［《》］、，。；：　～ ")
  text_wh = repf.remove_omitted_chars(text_wh, text_ch, punct_ch)
  if debug: err.write(f"!g {text_wh = !r}\n")

  if debug: err.write(f"!g {segs_ec = !r}\n")
  text_ec = ''.join(segs_ec) 
  punct_ec = set(r"-.,")
  text_wc = repf.remove_omitted_chars(text_wc, text_ec, punct_ec)
  if debug: err.write(f"!g {text_wc = !r}\n")

  len_ch = len(text_ch)
  len_wh = len(text_wh)

  name_bencao = f"{code_ch}-{tvar_ch}"
  dir_in_ch = "plot/in/ch"
  os.makedirs(dir_in_ch, exist_ok=True)
  text_wh_header = f"<{loc_ch}> {tvar_ch} {len_ch}"
  htext_wh = text_wh_header + "\n" + text_wh
  write_text_for_plot(dir_in_ch, name_bencao, "text", htext_wh)
  segs_ch_str = fmf.format_macro_parsing_ch(loc_ch, tvar_ch, segs_ch, None)
  write_text_for_plot(dir_in_ch, name_bencao, "segs", segs_ch_str)

  name_starps = f"{loc_ec}-{tvar_ec}"
  dir_in_ec = "plot/in/ec"
  os.makedirs(dir_in_ec, exist_ok=True)
  text_wc_header = f"<{loc_ec}> {tvar_ec} {len(text_ec)}"
  htext_wc = text_wc_header + "\n" + text_wc
  write_text_for_plot(dir_in_ec, name_starps, "text", htext_wc)
  
  # The {ideal_eph} parameter of {format_starps_parag_evaluation}
  # hould be the ideal one but we can't compute it here.
  # Anywat that parameter is used only to compute the total size
  # error {terr}, and the plot program does not use {terr}.
  ideal_eph = eva_per_hanzi 
  segs_ec_str = fmf.format_starps_parag_evaluation(pev, None, ideal_eph)
  write_text_for_plot(dir_in_ec, name_starps, "segs", segs_ec_str)
  
  kwords_tag = make_tag_from_kwords_en(kwords_en)
  name_plot = f"{name_bencao}-{name_starps}-{kwords_tag}"
  dir_out = "plot/out"
  os.makedirs(dir_out, exist_ok=True)
  
  command = [
      "plot_alignment.sh",
      f"{eva_per_hanzi:5.3f}",
      f"{dir_in_ch}/{name_bencao}", 
      f"{dir_in_ec}/{name_starps}",
      f"{dir_out}/{name_plot}"
    ]
  err.write(f"plot command = \"{command}\"\n")
  run_command(command)
  
  fname_plot_png = f"{dir_out}/{name_plot}-sma.png"
  fname_plot_jpg = f"{dir_out}/{name_plot}-sma.jpg"
  return fname_plot_png, fname_plot_jpg 
  # ----------------------------------------------------------------------
  
def make_tag_from_kwords_en(kwords_en):
  kwords_en_str = stdbip.keywords_list_to_string(kwords_en)
  # Fletcher's checksum:
  sum1a = 0; sum2a = 0
  sum1b = 0; sum2b = 0
  bytes_en = kwords_en_str.encode('utf-8')
  for bt in bytes_en:
    sum1a = (sum1a + bt) % 255
    sum2a = (sum2a + sum1a) % 255
    sum1b = (3*sum1b + 5*bt) % 255
    sum2b = (sum2b + sum1b) % 255
  chka = (sum2a << 8) | sum1a
  chkb = (sum2b << 8) | sum1b
  return f"{chka:04X}-{chkb:04X}"
  # ----------------------------------------------------------------------

def write_text_for_plot(odir, name, tag, text):
  fname = f"{odir}/{name}-{tag}.txt"
  wr = open(fname, "w")
  date = h.get_current_date()
  wr.write(f"# Created by h77.write_text_for_plot on {date}\n")
  text = re.sub(r"<[/]?[bi]>", "", text)
  wr.write(text)
  wr.write("\n")
  wr.close()
  return
  # ----------------------------------------------------------------------

def fetch_starps_line(ivt_file, loc_starps):
  # Reads fle {ivt_file} (assumed to be UTF-8) and looks for a line with
  # locus ID "<{loc_starps}>". Returns that line, minus the locus D, stripped.
  # Also returns the line number (from 1). 
  #
  # If the {loc_starps} is not found, returns {None,None}.
  
  # Just in case:
  loc_starps = re.sub(r"[<>]", "", loc_starps)
  
  loc_pat = f"<{loc_starps}>"
  
  rd = open(ivt_file, "r")
  rd.reconfigure(encoding='utf-8')
  text = None
  nlin = 0
  for line in rd:
    nlin += 1
    if re.match(loc_pat, line):
      text = re.sub(loc_pat, "", line)
      text = text.strip()
      break
  rd.close()
  return text, nlin
  # ----------------------------------------------------------------------

def find_best_starps_parag(tvar_ch, loc_ec, kwords_en, pevs_list):
  # Scans a bunch of lists of parevs {pevs_list}, selecting the one
  # with smallest badness score that has the given values of the fields 
  # {tvar_ch}, {loc_ec}, and {kwords_en}.
  #
  # The {pevs_list} must be a list of lists, each one of them being
  # either {None} or a list of tuples as returned by
  # {analyze_and_show_starps_parags}. The procedure ignores elements of
  # {pevs_list} that are {None}.
  #
  # If {loc_ec} is a string, considers only parevs that have that value
  # in the field 'loc_ec' (the SPS locus ID). If {loc_ec} is a set,
  # considers only parevs whose SPS locus ID is in that set. If {loc_ec}
  # is {None}, accepts any parag.
  #
  # If {tvar_ch} is not {None}, considers only parevs that have that
  # specific 'tvar_ch' (trim variant tag) field. If {tvar_ch} is a set,
  # considers only parevs whose trimming tag in that set. If {tvar_ch}
  # is {None}, consideres parevs with any trim variant.
  #
  # If {kwords_en} is not {None}, considers only parevs that have
  # that specific set of keywords as the field 'kwords_en'. If
  # {kwords_en} is {None}, considers parevs with any keyword
  # set.
  # 
  # If it finds at least one entry in the {pevs_list} with the specified
  # {loc_ec}, {tvar_ch}, and {kwords_en}, chooses
  # the parev {best_pev} in those lists that has the specified and
  # minimum badness score.
  #
  # Then returned result is the best parev among the parevs that were considered.
  #
  # If it cannot find any parev as requested, returns {None}.
  
  gud_loc_ec = set((loc_ec,)) if isinstance(loc_ec, str) else loc_ec
  gud_tvar_ch = set((tvar_ch,)) if isinstance(tvar_ch, str) else tvar_ch
  
  best_pev = None
  if pevs_list is not None:
    # Find the best candidate record:
    min_score = +inf
    for pevs in pevs_list:
      if pevs is not None:
        for pev in pevs:
          score_p = pev['score'];  tvar_ch_p = pev['tvar_ch'];
          loc_ec_p = pev['loc_ec']; kwords_en_p = pev['kwords_en']
          loc_ec_ok = (gud_loc_ec is None) or (loc_ec_p in gud_loc_ec)
          tvar_ch_ok = (gud_tvar_ch is None) or (tvar_ch_p in gud_tvar_ch)
          kwords_en_ok = (kwords_en is None) or (kwords_en == kwords_en_p)
          if loc_ec_ok and tvar_ch_ok and kwords_en_ok:
            if score_p < min_score:
              best_pev = pev; min_score = score_p
  return best_pev
  # ----------------------------------------------------------------------

def make_dic_from_parev(code_ch, loc_ch, pev, fsize_ch):
  # Returns a list of pairs {frag_ch,frag_ec} of hanzi and EVA fragments that 
  # are the conjectured matching parts of the SBJ entry {loc_ch}.
  # Each is given the badness {score} of the pairing.
  #
  # If {fsize_ch} is positive, the hanzi fragments {frag_ch} will be
  # all (overlapping) substrings of length {fsize_ch} of all the hanzi gaps in that
  # macro-parsing.  Specifically, if {segs_ch[ks]} is a gap (even {ks}), then, if {frag_ch} is
  # centered at character position {kch} of that gap, the correspondng
  # EVA fragment {frag_ec} is taken from {segs_ec[ks]} centered at a location
  # {kec} that is {kch} scaled by the ratiof of the lengths of the two
  # gaps. The fragment {frag_ec} is padded if needed with '·' (centered
  # dots) to size {fsize.
  #
  # If {fsize_ch} is zero, the fragments {frag_ch} will be all the hits
  # in the macro-parsing {segs_ch}, whole; and they will be paired
  # with fragments {frag-ec} which are the corresponding hits of {segs_ec}.
    
  score = pev['score']; loc_ch_p = pev['loc_ch']
  segs_ch = pev['segs_ch']; segs_ec = pev['segs_ec']

  # tvar_ch = pev['tvar_ch']
  # loc_ec = pev['loc_ec']; tvar_ec = pev['tvar_ec']
  # kwords_en = pev['kwords_en']; eva_per_hanzi = pev['eva_per_hanzi']
  # hit_penalties = pev['hit_penalties']

  assert loc_ch_p == loc_ch
  assert segs_ch is not None
  ns = len(segs_ch); assert len(segs_ec) == ns
  nh = ns//2; ng = nh+1; assert ns == ng + nh
  
  def data_error(msg):
    assert False, msg
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

  vms_dic = []

  def extract_frag_pair(it_ch, fsz_ch, str_ch, str_ec):
    # Extracts a fragment {frag_ch} of length {fsz_ch} at index {it_ch}
    # from {str_ch}.  Then extracts from {str_ec}
    # the corresponding fragment {frag_ec}, assuming that 
    # the whole of {str_ch} maps to the whole of {str_ec}. 
    
    nonlocal vms_dic
    # Define the number of EVA letters to take on each side of frag center:
    mrg_ec = 15 + int(ceil(2.5*fsz_ch)) 

    nt_ch = len(str_ch)
    nt_ec = len(str_ec)
    scale = (nt_ec+1)/(nt_ch+1)
    
    # Limit of fragment on the SBJ gap:
    jt_ch = it_ch + fsz_ch
    
    # Character indices {kt_ch,kt_ec} of the frag centers:
    kt_ch = (it_ch + jt_ch)/2
    kt_ec = int(floor(scale * kt_ch + 0.5))
   
    # Start of fragment on the SPS gap, and necessary padding:
    it_ec = kt_ec - mrg_ec
    lpad = 0 if it_ec >= 0 else -it_ec
    it_ec = min(nt_ec-1, max(0, it_ec))

    # Limit of fragment on the SPS gap, and necessary padding: 
    jt_ec = kt_ec + mrg_ec
    rpad = 0 if jt_ec <= nt_ec else jt_ec - nt_ec
    jt_ec = min(nt_ec, max(1, jt_ec))

    frag_ch = str_ch[it_ch:jt_ch]
    frag_ec = ("·" * lpad) + str_ec[it_ec:jt_ec] + ("·" * rpad)
    return frag_ch, frag_ec
    # ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

  if fsize_ch == 0:
    # Report the keyword hits:
    for ih in range(nh):
      frag_ch = segs_ch[2*ih+1]
      frag_ec = re.sub(r'[\[\]]', "", segs_ec[2*ih+1])
      vms_dic.append((frag_ch, frag_ec,))
  else:
    # Report substrings of the gaps with length {fsize_ch}:
    for ig in range(ng):
      gap_ch = segs_ch[2*ig]
      gap_ec = re.sub(r'[\[\]]', "", segs_ec[2*ig])
      nt_ch = len(gap_ch)
      for it_ch in range(nt_ch + 1 - fsize_ch):
        frag_ch, frag_ec = extract_frag_pair(it_ch, fsize_ch, gap_ch, gap_ec)
        vms_dic.append((frag_ch, frag_ec,))
       
  return vms_dic
  # ----------------------------------------------------------------------

def test_stuff():
  err.write("BEGIN TESTING\n")
  err.write("----------------------------------------\n")
  test_add_chosen_graphical_alignment_subsection()
  err.write("----------------------------------------\n")
  test_add_starps_matching_section()
  err.write("----------------------------------------\n")
  err.write("END TESTING\n")

  return
  # ---------------------------------------------------------------------- 

def test_add_starps_matching_section():
  
  err.write("----------------------------------------\n")
  err.write(f"@@@ testing add_starps_matching_section\n")
  st, code_ch, loc_ch, tvar_ch, source_ch, \
      name_ch, name_py, name_en, \
      tags1, itms_ch, itms_py, itms_en = \
    add_intro( "WHOP", "b2.4.094", '白马茎', 'bái mǎ jīng', 'white horse penis' )

  err.write("@@@ calling display_hanzi_pure_text ...\n")

  itms_wh = [ 
      "白马茎：　　　　",
      "　［味］咸，平。",
      "　［主］　　　　",
      "　　伤中，　　　",
      "　　脉绝　　　　",
      "　　　阴不起，　",
      "　　强志，　　　",
      "　　益气，　　　",
      "　　长肌肉　　　",
      "　　　肥健，　　",
      "　　生子。　　　",
      "眼：　　　　　　",
      "　［主］　　　　",
      "　　惊痫，　　　",
      "　　腹满，　　　",
      "　　疟疾。　　　",
      "悬蹄：　　　　　",
      "　［主］　　　　",
      "　　惊邪，　　　",
      "　　瘈疭，　　　",
      "　　乳难；　　　",
      "　　辟　　　　　",
      "　　　恶气，　　",
      "　　　鬼毒，　　",
      "　　　蛊注，　　",
      "　　　不祥。　　",
      "　［生］平泽。　",
    ]
  
  text_wh_trim = """
  <b2.4.094> (trim)
  白马茎：［主］伤中脉绝，阴不起，强志，益气，长肌肉。肥健，生子。
  眼：［主］惊痫，腹满，疟疾。
  悬蹄：［主］惊邪，瘈疭，乳难。辟恶气，鬼毒，蛊注，不祥。
  """
  text_ch_trim, tvar_ch_trim = display_hanzi_pure_text(st, loc_ch, 48, text_wh_trim)
  assert tvar_ch_trim == "trim"

  kwords_en_A = ( 'USES',       'USES',  'USES',       )
  kwords_en_B = ( 'USES', 'QI', 'USES',  'USES', 'QI', )
  
  max_score = 8.0

  variants = \
    (
      ( (tvar_ch_trim, text_ch_trim), kwords_en_A, ),
      ( (tvar_ch_trim, text_ch_trim), kwords_en_B, ),
    )
  
  err.write("@@@ calling add_starps_matching_section ...\n")

  locs_to_show = set()
  locs_to_try = set( [ "f114r.4", "f114v.1", "f106v.42", ] )
  del_qo = True
  exp_irm = True
  pevs_list = add_starps_matching_section \
    ( st, code_ch, loc_ch, itms_wh, 
      variants, del_qo, exp_irm, max_score, 
      locs_to_try, locs_to_show
    )

  err.write("@@@ finishing document ...\n")
  h.output_doc(st, sys.stdout, 0, last_edit)
  sys.stdout.flush()

  err.write("@@@ done.\n")
  err.write("----------------------------------------\n")

  return
  # ----------------------------------------------------------------------

def test_add_chosen_graphical_alignment_subsection():
  
  err.write("----------------------------------------\n")
  err.write(f"@@@ testing add_chosen_graphical_alignment_subsection\n")
  st, code_ch, loc_ch, tvar_ch, source_ch, \
      name_ch, name_py, name_en, \
      tags1, itms_ch, itms_py, itms_en = \
    add_intro( "WHOP", "b2.4.094", '白马茎', 'bái mǎ jīng', 'white_horse_penis' )
    
  itms_wh = [ 
    "白马茎：",
    "　［味］咸，平。",
    "　［主］",
    "　　伤中，",
    "　　脉绝",
    "　　　阴不起，",
    "　　强志，",
    "　　益气，",
    "　　长肌肉",
    "　　　肥健，",
    "　　生子。",
    "眼：",
    "　［主］",
    "　　惊痫，",
    "　　腹满，",
    "　　疟疾。",
    "悬蹄：",
    "　［主］",
    "　　惊邪，",
    "　　瘈疭，",
    "　　乳难；",
    "　　辟",
    "　　　恶气，",
    "　　　鬼毒，",
    "　　　蛊注，",
    "　　　不祥。",
    "　［生］平泽。",
    ]
  words_wh = [ wd.strip('　') for wd in itms_wh ]
  text_wh = ''.join(words_wh)
  
  tvar_ch = "trim"

  segs_ch = [ \
            "白马茎",
      "主", "伤中脉绝阴不起强志益",
      "气", "长肌肉肥健生子眼",
      "主", "惊痫腹满疟疾悬蹄",
      "主", "惊邪瘈疭乳难辟恶",
      "气", "鬼毒蛊注不祥",
    ]
  
  kwords_en = ( 'USES', 'QI', 'USES', 'USES', 'QI' )

  loc_ec = "f114r.4"
  tvar_ec = "norm"
  text_wc = \
    "fdeechdy.opchedaiin.ypchedy.odaly.chedy.p.cheokaiin." + \
    "shedy.podaiin.ochedal.loiin.chedy.kaiin.chdy.daiin." + \
    "dchdos.eedol.chdol.kchedy.cho.kaiin.chdy.eedy." + \
    "okchedy.daiiin.chedy.daiin.ykeedy.okeeedy.chedol." + \
    "chdaiin.ykar.dary.cheol.dchedy.dkchs.aiin.chdedy." + \
    "daiin.okchedaiin.chain"
  
  score = 2.492
  hit_penalties = [ 0.000, 0.000, 0.140, 0.000, 0.000 ]

  segs_ec = [
                "fdeechdyopche",
       "daiin", "ypchedyodalychedypcheokaiinshedypodaiinochedalloiin",
       "chedy", "kaiinchdydaiindchdoseedolchdolkchedycho",
       "kaiin", "chdyeedyokchedydaiiinchedy",
       "daiin", "ykeedyokeeedychedolchdaiinykardarycheold",
       "chedy", "dkchsaiinchdedydaiinokchedaiinchain",
   ]
  
  len_ch = sum(len(s) for s in segs_ch)
  len_ec = sum(len(s) for s in segs_ec)
  del_qo = True
  exp_irm = True
  eva_per_hanzi = spf.expected_eva_letters_per_hanzi(del_qo, exp_irm)
  pev = \
    { 'score': score, 
      'loc_ch': loc_ch, 'tvar_ch': tvar_ch, 'segs_ch': segs_ch, 
      'loc_ec': loc_ec, 'tvar_ec': tvar_ec, 'segs_ec': segs_ec, 
      'kwords_en': kwords_en, 'eva_per_hanzi': eva_per_hanzi,
      'hit_penalties': hit_penalties
    }
  
  add_chosen_graphical_alignment_subsection(st, code_ch, text_wh, text_wc, pev)

  err.write("@@@ finishing document ...\n")
  h.output_doc(st, sys.stdout, 0, last_edit)
  sys.stdout.flush()

  err.write("@@@ done.\n")
  err.write("----------------------------------------\n")

  return
  # ----------------------------------------------------------------------

if len(sys.argv) == 2 and sys.argv[1] == "R77.TEST":
  test_stuff()