#! /usr/bin/python3
# Last edited on 2026-02-26 07:38:50 by stolfi

# Takes as arguments a {book} ("bencao" or "starps"), a subset {bsub}
# (e.g. "fu" or "gd"), and a unit type {unit} ("ch", "wc", etc.).
#
# Reads a transcription file "res/{book}-{bsub}-{enc}.ivp" in generic
# IVTFF-like format, with one paragraph per line. Writes a file with the
# size of each paragraph.
#
# Ignores spaces, punctuation, symbols, etc. Ignores #-comments and
# blank lines.
#
# Each non-ignored input line must have the format "<{loc}> {text}"
# where {loc} is a locus ID like "f105v.20" or "b2.1.050",
# and {text} is a line of the transcribed text.
#
# Each output line has the format "{loc} {psize} {npunc}" where
# {psize} is the number of units of a specified type (hanzi characters,
# EVA characters, EVA words, pinyin syllables, ...) in the paragraph,
# and {npunc} is the number of punctuation characters in it.
#
# The program takes two command line arguments, the text encoding {enc}
# of the input file, and the {unit} for {psize}. The {unit} may be:
# 
#   "ch" the {enc} must be {chu} and the input must be hanzi
#       characters in Unicode, like
#         "<b1.1.001> 玉泉：味甘平。主治五脏百病。"
#       The size is the number of characters, excluding ideographic
#       punctuation.
#
#   "ps" the {enc} must be "pys" and the input text must be in
#       pinyin, with isolated syllables, like
#         "<b1.1.001> yù quán : wèi gān píng. zhǔ zhì wǔ zàng bǎi bìng."
#       The size of a parag is the number of syllables. 
#
#   "ec" the {enc} must be "eva" abd the input must be Voynichese
#       in EVA, like
#         "<f105v.32 ytchedy.qokchedy-qotchy.qokche,dy."
#       The size is the number of EVA characters [a-z?] after removing
#       all punctuation and markup, maping weirdos to '?', and mapping 
#       everything to lowercase.
# 
#   "wc" the {enc} must be "eva" and the input must be as above. The size is the
#       number of EVA words, counting period '.', comma ',', and line break
#       '-' as word spaces.
# 
#   "wp" same as "wc" but comma is ignored rather than considered
#       word space.
# 
# If {enc} is "eva" the input file must be in the ISO-Latin-1 encoding.
# Otherwise it must be in the Unicode UTF-8 encoding.

import sys, os, re
from sys import stderr as err
from process_funcs import bash, basic_line_loop
from error_funcs import arg_error, file_line_error, prog_error
from chinese_funcs import read_chinese_char_set
from note_077_funcs import compute_and_print_stats, enc_from_unit, name_for_tex_macro
import size_position_funcs as spf

from note_077_funcs import enc_from_unit

def main(book, bsub, unit):

  enc = enc_from_unit(unit)
  ivp_file = f"res/{book}-{bsub}-{enc}.ivp"
  ivp_rd = open(ivp_file, "r")
  ivp_rd.reconfigure(encoding='utf-8')

  upp_file = f"res/{book}-{bsub}-{unit}.upp"
  upp_wr = open(upp_file, "w")
  upp_wr.reconfigure(encoding='utf-8')
  upp_wr.write("# -*- coding: utf-8 -*-\n")

  pat_line, pat_unit, pat_junk, pat_sepa, clean_sepa = spf.get_parsing_patterns(enc, unit)

  tot_para = 0 # Count of data lines.
  tot_unit = 0 # Total count of units in input file.
  tot_sepa = 0 # Total chars removed in cleanup.

  psizes = []   # List of raw sizes of parags, for statistics.

  def process_input_line(nline, line):
    nonlocal enc, unit, ivp_file, upp_wr
    nonlocal pat_line, pat_unit, pat_junk, pat_sepa, clean_sepa
    nonlocal tot_para, tot_unit, tot_sepa, psizes
    #
    # Parses a line {line} assuming it is line {nline} of the file.
    # The {line} is always a string (never {None}), but may be "" if the line
    # is empty.
    # 
    # Ignores the line if it is a blank or #-comment.
    # 
    # Otherwise the line must be a data line, matching {pat_line}.
    # 
    # Increments {tot_para} for each data line.
    # 
    # Cleans the {text} according to {enc} and {unit}, updating
    # {tot_sepa}, {tot_unit}. Then counts the number {psize_raw} of
    # units if type {unit} in it.
    # 
    # Writes to stdout one output line for each input data line,
    # and appends {psize_raw} to {psizes}.
    
    # Should we debug the parag?
    debug = False
    
    def data_error(msg):
      nonlocal ivp_file, nline, line
      file_line_error(ivp_file, nline, msg, line)
      assert False
      # ----------------------------------------------------------------------

    assert line != None, "The {line} arg must not be {None}" 

    # Ignore comments and blank lines:
    if re.match(r" *([#]|$)", line): return

    # Just in case, ignore IVTFF page headers:
    if re.match(r"<f[0-9]+[rv][0-9]*>", line): return

    tot_para += 1

    m = re.match(pat_line, line)
    if m is None: 
      # Invalid line format.
      data_error(f"invalid line format for {pat_line = !r}")

    # Parse the line into locus ID and text:
    assert m.lastindex == 2, f"bug {m.lastindex = }"
    loc = m.group(1)
    text = m.group(2) 
    
    units, ct_sepa = spf.split_text_into_units(text, enc, unit, pat_unit, pat_junk, pat_sepa, data_error)
    ct_unit = len(units)
    tot_unit += ct_unit
    tot_sepa += ct_sepa

    psize_raw = ct_unit

    psizes.append(psize_raw)
    if debug: err.write(f"!~ {loc:<12s} psize = {psize_raw}\n")

    upp_wr.write(f"{loc:<12s} {psize_raw:5d} {ct_sepa:5d}\n")
    return      
    # ......................................................................

  err.write(f"reading file '{ivp_file}' ...\n")
  nread = basic_line_loop(ivp_rd, process_input_line)
  ivp_rd.close()
  upp_wr.close()

  err.write(f"{nread:5d} total lines\n")
  err.write(f"{tot_para:5d} parags found\n")
  err.write(f"{tot_sepa:6d} total separator chars\n")
  err.write(f"{tot_unit:5d} total units\n")
  err.write(f"{tot_unit/tot_para:8.2f} avg units/parag\n")
  
  write_TeX_parms_file(book, bsub, unit, psizes, tot_para, tot_unit)
  return
  # ----------------------------------------------------------------------

def write_TeX_parms_file(book, bsub, unit, psizes, tot_para, tot_unit):
  # Writes the file "res/{book}-{bsub}-{unit}-upp-parms.tex" with 
  # parameter defintions for LaTeX.
  
  sznum, sztot, szmin, szsin, szmax, szsax, szavg, szdev = \
    compute_and_print_stats("parag sizes", psizes)

  assert sznum == tot_para  
  assert sztot == tot_unit  

  szmax_rel = szmax/szavg
  err.write(f"{szmax_rel:6.2f} max relative parag size\n")

  szmin_rel = szmin/szavg
  err.write(f"{szmin_rel:6.2f} min relative parag size\n")

  pref = f"{book}-{bsub}-{unit}"
  txpref = name_for_tex_macro(pref)
  
  tex_file = f"res/{pref}-upp-parms.tex"
  tex_wr = open(tex_file, "w")
  
  tex_wr.write(f"\\def\\{txpref}NumParags{{{tot_para}}}\n")
  tex_wr.write(f"\\def\\{txpref}TotUnits{{{tot_unit}}}\n")
  tex_wr.write(f"\n") 
  tex_wr.write(f"\\def\\{txpref}PerParagMinUnits{{{szmin}}}\n")
  tex_wr.write(f"\\def\\{txpref}PerParagMaxUnits{{{szmax}}}\n")
  tex_wr.write(f"\\def\\{txpref}PerParagAvgUnits{{{szavg:.2f}}}\n")
  tex_wr.write(f"\\def\\{txpref}PerParagDevUnits{{{szdev:.2f}}}\n")
  tex_wr.write(f"\n") 
  tex_wr.write(f"\\def\\{txpref}PerParagSecMinUnits{{{szsin}}}\n")
  tex_wr.write(f"\\def\\{txpref}PerParagSecMaxUnits{{{szsax}}}\n")
  tex_wr.write(f"\n") 

  tex_wr.write(f"\\def\\{txpref}PerParagMinRelSize{{{szmin_rel:.2f}}}\n")
  tex_wr.write(f"\\def\\{txpref}PerParagMaxRelSize{{{szmax_rel:.2f}}}\n")

  tex_wr.close()
  return
  # ----------------------------------------------------------------------

def test_stuff():
  arg_error("no tests yet\n")
  return
  # ----------------------------------------------------------------------

if sys.argv[1] == "test":
  test_stuff()
else:
  book = sys.argv[1]
  bsub = sys.argv[2]
  unit = sys.argv[3]
  main(book, bsub, unit)