#! /usr/bin/python3
# -*- coding: utf-8 -*-
# Functions common Note/071/report, Note/077/report, and maybe others.

last_edit = "Last edited on 2026-10-08 00:35:03 by stolfi"

import sys, re, os, string, glob
from sys import stderr as err
import html_gen as h
from process_funcs import bash, run_command, basic_line_loop
import html_report_funcs as hr
import bencao_starps_convert_funcs as convf 
import bencao_starps_normalize_funcs as normf 
from math import sqrt, hypot, exp, log, floor, ceil, isfinite, isnan, inf, nan

def fetch_bencao_entry_wh_py_en(top_dir, code_ch, loc_ch):
  # Fetches from the directory {top_dir} the hanzi, pinyin, and English
  # versions of the SBJ entry with the four-letter code {code_ch} and
  # SBJ location ID {loc_ch}.
  #
  # The three versions are read from three separate files,
  # "{top_dir}/{utype}/{code_ch}-{loc_ch}.utf" where {utype} is "wh" for
  # the hanzi version, "py" for the pinyin version, and {en} for the
  # English version. The contents of each file must be as described in
  # {split_formatted_entry} and {split_formatted_entry_hanzi}.
  #
  # Returns {TVAR}, {SOURCE}, the list {FRTAGS} of fragment labels, then
  # the three lists {FRTEXTS_WH}, {FRTEXTS_PY}, {FRTEXTS_EN} of the
  # fragments of each {utype}.
  #
  # The field {TVAR} and the list {FRTAGS} must be exactly the same 
  # in all three files.  The {SOURCE} may be different in each file;
  # the procedure returns the one from the "wh" file.
  # 

  def fetch_entry(utype):
    # Reads the hanzi (if {utype} is "wh"), pinyin (if {utype} is {py}),
    # or English (if {utype} is "en") of the SBJ recipe defined by its 
    # four-letter code {code_ch} and location ID {loc_ch}. Returns the file
    # contents as a single string with lines separated by end-of-line.
    fname = f"{top_dir}/{utype}/{code_ch}-{loc_ch}.utf"
    rd = open(fname, "r")
    text = rd.read()
    rd.close()
    return text
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

  code1, loc1, tvar1, source1, tags1, itms_wh = \
    split_formatted_entry_hanzi(fetch_entry("wh"), "wh")
    
  assert code1 == code_ch
  assert loc1 == loc_ch
  assert tvar1 == "full"
  
  tvar = tvar1
  source = source1
  tags = tags1
    
  code2, loc2, tvar2, source2, tags2, itms_py = \
    split_formatted_entry(fetch_entry("py"), "py")

  assert code2 == code1
  assert loc2 == loc1
  assert tvar2 == tvar1
  assert tags2 == tags1
  # {source2} can be different from {source}.

  code3, loc3, tvar3, source3, tags3, itms_en = \
    split_formatted_entry(fetch_entry("en"), "en")

  assert code3 == code1
  assert loc3 == loc1
  assert tvar3 == tvar1
  assert tags3 == tags1
  # {source3} can be different from {source}.
  
  return tvar, source, tags, itms_wh, itms_py, itms_en
  # ----------------------------------------------------------------------

def split_formatted_entry(entry, utype):
  # Parses an SBJ entry (hanzi, pinyin, translation, or Voynichese)
  # that has been cast in multiline format.
  #
  # The {entry} must be a multiline string. Any #-comments are
  # deleted, and blank lines. are ignored.
  # 
  # The first non-blank, non-comment line must be a header in the format
  #
  #   "{CODE} <{LOC}> ({TVAR}) {SOURCE}"
  #
  # where {CODE} and {LOC} are the same as in the file name, {TVAR} is a
  # tag that identifies a variant (currently must be "full") and
  # {SOURCE} is the source of the entry (currently "ZHB" for the Zhenghe
  # Bencao, or "CTP" for a corrected merge of the Chinese Texts Project and
  # Chinese Wikisource "reconstruction" files.
  # 
  # Subsequent lines must be in the format
  #
  #   "{FRTAG} | {FRTEXT}" 
  #
  # where {FRTAG} is a label that matches the RE pattern
  # [(][A-Za-z0-9]+[)], and {FRTEXT} is a fragment of the text 
  # of the entry, possibly empty. 
  #
  # Blanks before or after these two fields are
  # ignored. The {FRTEXT} is submitted to {convf.convert_raw_text}
  # with the given {utype} parameter. 
  #
  # If {utype} is "wh" or "ch", the {FRTEXT} must consist of hanzi
  # letters and punctuation. In the first case the cleanup will only
  # remove some redundant ideographic punctuation, like leading and
  # trailing blanks. In the second case it will remove all punctuation.
  #
  # If {utype} is "py", the {FRTEXT} must consist of zero or more
  # pinyin syllables with tone diacritics separated by ASCII blanks,
  # with ASCII punctuation, possibly with '·' as visible indentation.
  # The cleanup will remove leading, trailing, and redundant blanks,
  # leaving the other punctuation.
  #
  # If {utype} is "en", the {FRTEXT} must consist of ASCII letters
  # and punctuation, possibly with '·' as visible indentation.
  # The cleanup will remove leading, trailing, and redundant blanks.
  #
  # The {utypes} "wc", "wp", and "ec" are currenly not supported.
  #
  # Returns {LOC}, {TVAR}, {SOURCE}, a list of all {FRTAGS}s, and a
  # list of all the matching {FRTEXT}s.

  def cleanup_line(line):
    line = re.sub(r"[#].*$", "", line)
    # Must keep ideographic spaces.
    line = line.strip(' \011\015\012')
    return line
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

  lines = entry.splitlines()
  lines = [ cleanup_line(line) for line in lines ]
  lines = [ line for line in lines if line != "" ]
  
  nline = 0
  code = None; loc = None; tvar = None; source = None
  frtags = []
  frtexts = []
  
  pat_code = r"([A-Z][A-Z][A-Z][A-Z])"
  pat_loc = r"[<]([a-z0-9.]+)[>]"
  pat_tvar = r"[(]([a-z0-9]+)[)]"
  pat_source = r"([A-Z][A-Z][A-Z])"
  pat_header = f"[ ]*{pat_code}[ ]*{pat_loc}[ ]*{pat_tvar}[ ]*{pat_source}[ ]*"
  
  pat_frtag = r"([(][A-Za-z0-9]+[)])"
  pat_frtext = r"([^\012\015]+)"
  pat_frag = f"[ ]*{pat_frtag}[ ]*[|][ ]*{pat_frtext}[ ]*"
  
  for line in lines:
    nline += 1
    
    def data_error(msg):
      err.write(f"** entry line {nline}: {msg}\n  line = [[{line}]]\n")
      assert False
      # ----------------------------------------------------------------------
    
    line = cleanup_line(line)
    if line == "": continue
    
    if loc is None:
      m = re.fullmatch(pat_header, line)
      assert m != None, f"bad header line format [[{line}]]"
      code = m.group(1)
      loc = m.group(2)
      tvar = m.group(3)
      source = m.group(4)
    else:
      assert loc != None, f"missing header line"
      m = re.fullmatch(pat_frag, line)
      assert m != None, f"bad fragment line format '{line}'"
      frtag = m.group(1)
      frtext = m.group(2)
      frtags.append(frtag)
      frtext = convf.convert_raw_text(frtext, utype, data_error)
      frtexts.append(frtext)

  return code, loc, tvar, source, frtags, frtexts
  # ----------------------------------------------------------------------

def split_formatted_entry_hanzi(entry, utype):
  # Parses an SBJ entry hanzi that has been cast in multiline format.
  # See {split_formatted_entry} for the
  # format of {entry}. Then does some checking and cleanup of the items.
  # 
  # The items must contain only hanzi, ideographic blanks and
  # punctuation (which are retained), and leading or trailing ASCII
  # blanks (which are stripped). Pads all items with ideographic blanks
  # to the same width.
  #
  # Returns {LOC}, {TVAR}, {SOURCE}, the list of all {TAG}s, and the list of all
  # {ITEM}s.
  
  code, loc_ch, tvar_ch, source_ch, frtags, itms_wh = \
    split_formatted_entry(entry, utype)
  
  # Pad all items>
  max_item_sz = 0
  for item in itms_wh:
    if re.search(r"[\001-\377]", item):
      assert False, f"ascii character in hanzi item '{item}'"
    max_item_sz = max(len(item), max_item_sz)
  itms_wh = [ item.ljust(max_item_sz, "　") for item in itms_wh ]
  return code, loc_ch, tvar_ch, source_ch, frtags, itms_wh
  # ----------------------------------------------------------------------

def make_three_column_entry_table(frtags, itms_wh, itms_aa, itms_bb):
  # Returns the HTML of a table with the given {frtags} on column 1, the
  # given {itms_wh} in column 2, and arbitrary ascii entries{itms_aa,itms_bb} in
  # columns 3 and 4.
  # 
  # All four lists must have the same length, with corresponding 
  # elements in the same positions.
  
  N = len(frtags)
  assert len(itms_wh) == N, f"{N = } {len(itms_wh) = }"
  assert len(itms_aa) == N, f"{N = } {len(itms_aa) = }"
  assert len(itms_bb) == N, f"{N = } {len(itms_bb) = }"
  
  bars = [ '&nbsp;|&nbsp;' ] * N

  rows = list(zip(frtags, bars, itms_wh, bars, itms_aa, bars, itms_bb))
  
  col_mods = [
     "style='padding-left:4ch; padding-right:4ch; text-align:left; font-weight:bold;'", 
     "align=left",  
     "align=left", 
     "align=left",  
     "align=left", 
     "align=left", 
     "align=left", 
   ]
  html_tb = h.make_table(rows, by_rows = True, col_mods = col_mods)
  return html_tb
  # ----------------------------------------------------------------------

def add_three_column_entry_table(st, frtags, itms_wh, itms_aa, itms_bb):
  # Appends to {st} a table with the given {frtags} on column 1, 
  # the given {itms_wh}
  # in column 2, and arbitrary ascii entries{itms_aa,itms_bb} in columns 3 and 4.
  #
  # The elements of {itms_wh} are assumed to consist of hanzi and/or 
  # ideographic punctuation.  The other columns are supposed to be 
  # Latin (or pinyin) letters with ISO-Latin punctuation.
  # 
  # All four lists must have the same length, with corresponding 
  # elements in the same positions.
  
  html_tb = make_three_column_entry_table(frtags, itms_wh, itms_aa, itms_bb)
  h.append_centered(st, html_tb, centered = False)
  return
  # ----------------------------------------------------------------------

def entry_align_table(st, rows):
  # Prints a table with the frtags on column 1, hanzi in column2, and
  # arbitrary ascii entries in columns 3 and 4.
  
  ch_ps_wp_en_wcol_mods = [
     "style='padding-left:4ch; padding-right:4ch; text-align:left; font-weight:bold;'", 
     "align=left", 
     "align=left", 
     "align=left", 
     "align=left",
   ]
   
  h.table(st, rows, col_mods = col_mods, centered = False)
  
  return
  # ----------------------------------------------------------------------

def read_parms_from_file_header(rd):
  # Reads {rd} and looks for lines of the form "# {KEY} = {VALUE}".
  # Returns a dict with those keys and values.
  # The {KEY} may be any python3-style identifier.
  # The {VALUE} for now may be an integer, a float, or a string.
  
  vms_dic = dict()
  
  err.write("!= beg\n")
  
  def process_line(nread, line):
    nonlocal vms_dic
    line = line.strip()
    err.write(f"!= {nread:5d} {line = !r}\n")
    m = re.fullmatch(r"# *([a-zA-Z][a-zA-Z_0-9]*) *[=] *(.*)", line)
    if m == None: return
    key = m.group(1)
    val = m.group(2).strip()
    err.write(f"!= {key = !r} {val = !r}\n")
    if re.fullmatch(r"[-+]?[0-9]+", val):
      # Integer
      val = int(val)
    elif re.fullmatch(r"[(][-+0-9, ]+[)]", val):
      # Integer tuple; assume pair:
      m = re.fullmatch(r"[(]([-+]?[0-9]+)[ ,]+([-+]?[0-9]+)[)]", val)
      val = (int(m.group(1)), int(m.group(2)),) 
    elif re.fullmatch(r"[-+]?[0-9]*([.][0-9]|[0-9][.])[0-9]*([Ee][-+]?[0-9]+)?", val):
      val = float(val)
    elif re.fullmatch(r"['][^']*[']", val):
      val = re.sub(r"[']", "", val)
    elif re.fullmatch(r'["][^"]*["]', val):
      val = re.sub(r'["]', "", val)
    elif re.fullmatch(r'\[.*\]', val):
      val = re.sub(r'^\[', "", val)
      val = re.sub(r'\]$', "", val)
      elems = re.split(r'[, ]+', val)
      items = []
      for el in elems:
        if el != "":
          if el[0] == '"':
            el = re.sub(r'"', "", el); 
          elif el[0] == "'":
            el = re.sub(r"^'", "", el); 
          items.append(el)
      val = items
    else:
      assert False, f"** bad value «{val}»"
    vms_dic[key] = val
    return 
    # :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
  
  nread = basic_line_loop(rd, process_line)
  return vms_dic
  # ----------------------------------------------------------------------

def add_formatted_entry_table(st, frtags, itms_wh, itms_py, itms_en):

  h.section(st, 3, "Pinyin and translation")

  h.parags(st, """Here is the same entry, with punctuation added according
  to this parsing, the modern Mandarin readings in pinyin, and a somewhat
  literal English translation:""")
  
  add_three_column_entry_table(st, frtags, itms_wh, itms_py, itms_en)
  return
  # ----------------------------------------------------------------------

def fetch_starps_line(ivt_file, loc_starps):
  # Reads fle {ivt_file} (assumed to be UTF-8) and looks for a line with
  # locus ID "<{loc_starps}>". Returns that line, minus the locus D, stripped.
  # Also returns the line number (from 1). 
  #
  # If the {loc_starps} is not found, returns {None,None}.
  
  # Just in case:
  loc_starps = re.sub(r"[<>]", "", loc_starps)
  
  loc_pat = f"<{loc_starps}>"
  
  rd = open(ivt_file, "r")
  rd.reconfigure(encoding='utf-8')
  text = None
  nlin = 0
  for line in rd:
    nlin += 1
    if re.match(loc_pat, line):
      text = re.sub(loc_pat, "", line)
      text = text.strip()
      break
  rd.close()
  return text, nlin
  # ----------------------------------------------------------------------
    
def get_clean_normalized_starps_text_wc(loc_ec, del_qo, exp_irm):
  utype = "wc"
  ivt_file = f"res/starps-gd-{utype}-par.ivt"
  rawtx_wc, nlin = fetch_starps_line(ivt_file, loc_ec)
  assert rawtx_wc != None, f"** cannot find {loc_ec} in the starps file"

  def data_error(msg):
    file_line_error(ivt_file, nlin, msg, f"<{loc_ec}> {rawtx_wc}")
    assert False
    # ..................................................................

  text, head, tail = spf.convert_starps_raw_text(rawtx, utype, data_error)
  text = spf.normalize_starps_text(text, utype, del_qo, exp_irm, data_error)

  return text
  # ----------------------------------------------------------------------

def remove_omitted_chars(textw, textc, punct):
  # Given a hanzi or EVA string {textw} possibly with with punctuations,
  # and the same string {textc} without punctuations and possibly without some
  # omitted substrings, returns a copy {textr} of {textw}, still with
  # all the punctuations in the set {punct}, but without the omitted substrings.
  #
  # Assumes that the first char after an omitted substring does not
  # occur in the substring.
  
  debug = False
  
  if debug: err.write(f"!o {textw = !r}\n");
  if debug: err.write(f"!o {textc = !r}\n");

  nw = len(textw)
  nc = len(textc)
  
  tableau = [ None ] * nw;
  for i in range(nw): tableau[i] = [ None ] * nc
  # For {ic > 0} and {iw >= 0}, the entry {tableau[iw][ic]} is {(jw,jc)}
  # iff {0 <= jw < iw}, {jc == ic-1}, and {textc[ic] == textw[iw]}, and
  # {textc[0..jc]} matches a subsequence of {textw[0..jw]} that ends
  # with {textw[jw]}.  Otherwise {tableau[iw][ic]} is {None}.
  
  # The entry {tableau[iw][0]} is {(-1,-1)} if {0 <= jw < iw}
  # and {textc[0] == textw[iw]}.
  
  # Get indices in {textw} of chars that are not punct:
  iws = [ iw for iw in range(nw) if textw[iw] not in punct ]
  
  # Fill the tableau:
  for ic in range(nc):
    chic = textc[ic];
    assert chic not in punct, f"punct '{chic}' in \"{textc}\"" 
    matched = False
    for iw in iws:
      chiw = textw[iw];
      if chiw == chic:
        if ic == 0:
          tableau[iw][ic] = (-1,-1)
          matched = True
          if debug: err.write(f"!o {iw = } {ic = } {tableau[iw][ic] = !r}\n");
        else:
          for jw in iws:
            if jw < iw and tableau[jw][ic-1] is not None:
              tableau[iw][ic] = (jw,ic-1)
              matched = True
              if debug: err.write(f"!o {iw = } {ic = } {tableau[iw][ic] = !r}\n");
    assert matched, f"could not match text[{ic}] = {chic!r}" 
  
  # Extract the result:
  textr = [ ]
  kw = nw
  for jc in range(nc):
    kc = nc - jc - 1
    chkc = textc[kc]
    if debug: err.write(f"!o {kc = } looking for {chkc = !r}\n");
    found = False
    while kw > 0:
      kw -= 1;
      chkw = textw[kw]
      if debug: err.write(f"!o {kw = } {chkw = !r}\n");
      if tableau[kw][kc] != None:
        assert chkw == chkc
        textr.append(chkw)
        if debug: err.write(f"!o found {chkc!r} at {kw = }\n");
        found = True
        break
      elif chkw in punct:
        textr.append(chkw)
    assert found, f"could not find match for {chkc!r}"
  while kw > 0:
    kw -= 1
    chkw = textw[kw]
    if debug: err.write(f"!o {kw = } {chkw = !r}\n");
    if chkw in punct:
      textr.append(chkw)
  textr.reverse()
  return "".join(textr)
  # ----------------------------------------------------------------------

def get_recipe_pages_in_dir(dir):
  # Scans the folder "{dir}" for files called "*_entry.html" or "*_entry_src.py".
  # 
  # For every {name} such that either "{dir}/{name}_entry_src.py" or
  # "{dir}/{name}_entry.html" exist, the resulting list will have
  # "{dir}/{name}_entry.html".
  #
  assert os.path.exists(dir), f"folder {dir} does not exist"
  src_html_files = \
    glob.glob(f"./{dir}/*_entry_src.py") + \
    glob.glob(f"./{dir}/*_entry.html")
  hnames = map(lambda x: re.sub(r"(_entry_src[.]py|_entry[.]html)$", "", x), src_html_files)
  hnames = map(lambda x: re.sub(r"^([.]/)*", "", x), hnames)
  hnames = map(lambda x: re.sub(r"/([.]/)*", "/", x), hnames)
  hnames = [ x for x in hnames if x != "" and not re.search(r"\b(work|JUNK|SAVE|AAAA)\b", x) ]
  hnames = sorted(list(set(hnames)))
  hfiles = list()
  for hname in hnames:
    hfiles.append(f"{hname}_entry.html")
  return hfiles
  # ----------------------------------------------------------------------

def test_stuff():
  err.write("TESTING\n")
  
  err.write("----------------------------------------\n")
  test_remove_omitted_chars()
  err.write("----------------------------------------\n")

  return
  # ---------------------------------------------------------------------- 

def test_remove_omitted_chars():
  err.write("----------------------------------------\n")
  err.write(f"@@@ testing remove_omitted_chars\n")
  
  text_wh = "丹雄鸡：　［味］甘，微温。　［主］　［女人］　崩中，　漏下，　赤白沃。　补虚，　温中，　止血，　通神，　杀毒，　辟不祥。　头：　［主］　杀鬼。　《东门上者　尤良。》　肪：　［主］　耳聋。　肠：　［主］　遗溺。　肶胵裹黄皮：　［主］　泄利。　屎白：　［主］　消渴，　伤寒寒热。　翮羽：　［主］　下血闭。　鸡子：　［主］　除　热火疮，　痫痉；　［可作］　虎魄，　神物。　鸡白蠹：　能肥脂。　［生］平泽。"
  err.write(f"  {text_wh = !r}\n")

  text_ch = "丹雄鸡主崩中漏下赤白沃补虚温中止血通神杀毒辟不祥头主杀鬼肪主耳聋肠主遗溺肶胵裹黄皮主泄利屎白主消渴伤寒寒热翮羽主下血闭鸡子主除热火疮痫痉可作虎魄"
  err.write(f"  {text_ch = !r}\n")

  text_wh_r = remove_omitted_chars(text_wh, text_ch, set(r"［《》］、，。；：　～ "))
  nc_r = len(text_wh_r)
  err.write(f"  {nc_r = } {text_wh_r = !r}\n")
  
  text_wh_r_exp = "丹雄鸡：　［］，。　［主］　［］　崩中，　漏下，　赤白沃。　补虚，　温中，　止血，　通神，　杀毒，　辟不祥。　头：　［主］　杀鬼。　《　。》　肪：　［主］　耳聋。　肠：　［主］　遗溺。　肶胵裹黄皮：　［主］　泄利。　屎白：　［主］　消渴，　伤寒寒热。　翮羽：　［主］　下血闭。　鸡子：　［主］　除　热火疮，　痫痉；　［可作］　虎魄，　。　：　。　［］。"
  nc_r_exp = len(text_wh_r_exp)
  err.write(f"  {nc_r_exp = } {text_wh_r_exp = !r}\n")
  
  if text_wh_r != text_wh_r_exp:
    for ic in range(max(nc_r, nc_r_exp)):
      cr = "#" if ic >= nc_r else text_wh_r[ic]
      cr_exp = "#" if ic >= nc_r_exp else text_wh_r_exp[ic]
      if cr != cr_exp:
        err.write(f"  {ic = :3d} {cr = !r} {cr_exp = !r}\n")
    assert(False)

  err.write("OK!\n")
  return
  # ----------------------------------------------------------------------

  return
  # ----------------------------------------------------------------------

if len(sys.argv) == 2 and sys.argv[1] == "BSRF.TEST":
  test_stuff()