#! /usr/bin/python3
# Last edited on 2026-10-07 09:05:58 by stolfi

# Functions for cleaning bencao (hanzi) and starps (EVA) text.

import sys, os, re
from sys import stdout as out, stderr as err, stdin as inp
from error_funcs import arg_error, file_line_error, prog_error
from process_funcs import bash, basic_line_loop
from chinese_funcs import read_chinese_char_set
from compute_stats_funcs import compute_and_print_stats
from math import sqrt, hypot, exp, log, pi, inf, nan, floor, ceil

def convert_raw_text(text, utype, data_error):
  # Cleans up a raw SBJ or SPS {text} so as to produce a valid text of
  # the specified {utype}.
  #
  # Cleanup means removing extraneous stuff, like inline comments
  # and alignment markers. It also entails applying some transformations
  # like replacing EVA Weirdos and rare letters by "?".
  #
  # See {convert_bencao_raw_text} for details when {utype} is "ch" or "wh".
  # 
  # See {convert_starps_raw_text} for details when {utype} is "ec",
  # "wp", and "wc".
  # 
  if utype == "ch" or utype == "wh":
    text = convert_bencao_raw_text(text, utype, data_error)
  elif utype == "ec" or utype == "wc" or utype == "wp":
    text, head, tail = convert_starps_raw_text(text, utype, data_error)
    # Discards the booleans {head,tail}.
  return text
  # ----------------------------------------------------------------------

def convert_starps_raw_text(text, utype, data_error):
  # Cleans up the raw {text} from an SPS (EVA) text to produce
  # a text of the specfied {utype}, which must be "ec", "wc", or "wp".
  #
  # The function returns the converted {text} and booleans {head} and
  # {tail} that specify whether the line was a parag head line or parag
  # tail line, respectively. Note that both may be true for a one-line
  # parag.
  #
  # INPUT TEXT EXPECTED
  # 
  #   The raw {text} given must be a non-empty string that may contain
  #   any of these:
  #
  #     Prefix "<%>" marking the line as the head of a paragraph.
  #
  #     Prefix [«=»] to indicate the alignment of the start of the line
  #       relative to the left rail.
  #
  #     EVA letters [A_Za-z].
  #
  #     Invalid EVA code '?'.
  #
  #     Weirdo codes "&{NNN};" (possibly without the ';') 
  #       where {NNN} is three decimal digits.
  #
  #     Ligatures consisting of two or more EVA letters, '?', or
  #       weirdo codes enclosed in braces "{...}".
  #
  #     Word separators [-.,].
  #
  #     Suffix [«=»] to indicate the alignment of the end of the line
  #       relative to the right rail.
  #
  #     Suffix "<$>" to mark the line as the tail of a parag.
  #
  #     Inline comments "<!...>", including various special comments
  #       to indicate stars in the margin, wide linegaps, figure intrusions.
  #       vellum folds, etc. These may appear anywhere in the line
  #       including before or after all the {text} and inside other markup
  #       tags.
  #
  #   Example:
  #
  #     "<%><!S09><!WGP>=w{Ch}edair.{Ch}otaropdaiin.otol,dair.aiir,aim="
  #     "={Ch}eeo,daiin.{Sh}eedy&162&211am.{Ch}odal.ol{Ch}edy«<!WGN><$>"
  #     "<%>={Ch}ol.oeedy.keedy.{Ch}{Ch}ekyair.{Ch}edy.ot{Ch}edy.{Qo}ty=" 
  #
  #   Note the parag breaks at the start and between the second and
  #   third line.
  #     
  # CLEANUP PERFORMED
  # 
  #   For all {utype}s, the cleanup entails:
  #     
  #     Removing all inline comments.
  #     
  #     Removing all markup, including the braces [{}], parag markers
  #     "<%>" and "<$>", and rail alignment markers [«=»].
  #
  #     Mapping 'w' and 'z', used in my transcription to encode hooked
  #     puffs, to 'p' and 'f', respctively.
  #
  #     Mapping the uncertain reading charaters [bj] to '?'.
  #
  #     Mapping all weirdo codes to '?'.
  #
  #     Mapping everything to lowercase. 
  #
  #   Moreover, 
  #
  #     If {utype} is "ec", all punctuation [-,.] is deleted.
  #
  #     If {utype} is "wp", every ',' is deleted, and '-' is mapped to '.'
  #
  #     If {utype} is "wc", every ',' and '-' are mapped to '.'
  #   
  # CLEANED TEXT
  # 
  #   After this cleanup, if {utype} is "ec", the text must  have only
  #   characters [ac-fhik-tvxy?]. Example:
  #
  #     "choloeedykeedychchekycheod?eyke??edeedydaiiinaldair"
  #
  #   If {utype} is "wc" or "wp", the raw text will also have '.'
  #   before, between, and after every token. Example:
  #   
  #     ".chol.oeedy.keedy.chcheky.cheod?ey.ke??edeedy.daiiin.ald.air."
  #   
  #   For more restrictions, see {check_starps_conversion} below.
  #
  
  debug = False

  text = text.strip()
  if debug: err.write(f"!c text (raw)       = [[{text}]]\n")
  if utype == "ec" or utype == "wc" or utype == "wp":
    # Remove inline comments:
    text = re.sub(r"[<][!][^<>]*[>]", "", text)
    
    # Map weirdo codes to '?':
    text = re.sub(r"[&][0-9][0-9][0-9][;]?", "?", text)
    
    # Remove ligature marks '{}':
    text = re.sub(r"[{]([^{}]*)[}]", r"\1", text)
    
    # Map to lowercase:
    text = text.lower()
    
    # Map hooked puffs to normal ones:
    text = re.sub(r"w", "p", text)
    text = re.sub(r"z", "f", text)
    
    # Map rare uncertain characters to '?':
    text = re.sub(r"[bj]", "?", text)
      
    # Check for parag markers:
    head = ( re.search(r"^[<][%][>]", text) != None )
    tail = ( re.search(r"[<][$][>]$", text) != None )
    
    # Remove all parag markers for now:
    text = re.sub(r"[<][%$][>]", "", text)

    # Remove alignment markers:
    text = re.sub(r"^[«=»]", "", text)
    text = re.sub(r"[«=»]$", "", text)

    # Check for invalid characters in input:
    m_bad = re.search(r"[^-.,ac-ik-z?]", text)
    if m_bad is not None:
      ibeg = m_bad.start()
      data_error(f"invalid character '{text[ibeg]}' at {ibeg}")
      
    # Check for irregular spaces in input:
    m_bad = re.search(r"[-,.][-,.]", text)
    if m_bad is not None:
      ibeg = m_bad.start(); iend = m_bad.end()
      data_error(f"improper punctuation '{text[ibeg:iend]}' at {ibeg}")

    # Replace punctuation:
    text = re.sub(r"-", ".", text)
    if utype == "ec":
      text = re.sub(r"[,.]", "", text)
    elif utype == "wc":
      text = re.sub(r"[,]", ".", text)
    elif utype == "wp":
      text = re.sub(r"[,]", "", text)
    else:
      assert False # Should not happen.
  else:
    arg_error(f"invalid {utype = !r}")
    
  if debug: err.write(f"!c text (midway)    = [[{text}]]\n")

  if utype == "wp" or utype == "wc":
    # Ensure single period before and after every token:
    text = f".{text}."
    text = re.sub(r"[.][.]+", ".", text)
    text = re.sub(r"^[.][.]+", ".", text)
    text = re.sub(r"[.][.]+$", ".", text)
    
  if debug: err.write(f"!c text (converted) = [[{text}]]\n")
  check_starps_conversion(text, utype, data_error)
  
  return text, head, tail
  # ----------------------------------------------------------------------

def convert_bencao_raw_text(text, utype, data_error):
  # Cleans up the raw {text} from an SBJ transcription file,
  # returnin a text of the specified {utype} (which must be "wh" or "ch").
  #
  # INPUT TEXT EXPECTED
  #
  #   For both "ch" and "wh",
  #   the input raw {text} must consist of simplified hanzi
  #   (Chinese characters) in Unicode, plus ideographic punctuation
  #   and ideographic spaces. Like:  
  #
  #       "冬葵子：［味］甘寒。［主］五藏六腑寒热羸瘦。［久服］坚骨长肌肉。《草头》" Like:
  #
  #   In either case the raw {text} should NOT have embedded ASCII
  #   blanks or other ASCII chars.
  #
  # CLEANUP
  #
  #   Cleanup for {utype} "ch" implies removing all punctuation.
  #
  #   Cleanup for {utype} "wh" implies removing all
  #   ideographic spaces.
  #
  # CLEANED TEXT
  # 
  #   After this cleanup, if {utype} is "ch", the raw text must
  #   consist entirely of simplified hanzi. Example:
  #
  #     "冬葵子味甘寒主五藏六腑寒热羸瘦久服坚骨长肌肉"
  # 
  if utype == "ch":
    # Remove all punctuation: */
    text = re.sub(r"[］［；：、，。［］《》　]", "", text)
  elif utype == "wh":
    text = re.sub(r"[　]", "", text)
  else:
    arg_error(f"invalid text unit type {utype = !r}")
    
  check_bencao_conversion(text, utype, data_error)

  return text
  # ----------------------------------------------------------------------

def check_starps_conversion(text, utype, data_error):
  # Checks whether {text} is a valid "clean" SPS text according
  # to the given {utype}. Calls {data-error} if not.
  
  # Check for invalid characters:
  if utype == "ec":
    m_bad = re.search(r"[^ac-ik-vxy?]", text)
  elif utype == "wc" or utype == "wp":
    m_bad = re.search(r"[^.ac-ik-vxy?]", text)
  else:
    arg_error(f"invalid {utype = !r}")
  if m_bad is not None:
    ibeg = m_bad.start()
    data_error(f"invalid text char '{text[ibeg]}' at pos {ibeg}")

  if utype == "wc" or utype == "wp":
    # Check for improper spacing:
    pat_bad_punc = r"[.][.]"
    m_bad = re.search(pat_bad_punc, text)
    if m_bad != None:
      ibeg = m_bad.start()
      data_error(f"improper punctuation '{text[ibeg:ibeg+5]}' at pos {ibeg}")
  
  return
  # ----------------------------------------------------------------------

def check_bencao_conversion(text, utype, data_error):
  # Checks whether {text} is a valid "clean" SBJ text according
  # to the given {utype}. Calls {data-error} if not.
  #
  # !!! Currently does not do a thorough check. Improve! !!!

  m_bad = None
  if utype == "ch":
    # Check for ASCII characters:
    m_bad = re.search(r"[\001-\377]", text)
  if m_bad is not None:
    ibeg = m_bad.start()
    data_error(f"invalid text char '{text[ibeg]}' at {ibeg}")

  return
  # ----------------------------------------------------------------------

def test_stuff():

  def data_error(msg):
    err.write(f"** {msg}\n"); assert False
    # ....................................................................

  return
  # ----------------------------------------------------------------------

if len(sys.argv) > 1 and sys.argv[1] == "BSCF.TEST":
  test_stuff()