#! /usr/bin/python3
# -*- coding: utf-8 -*-
last_edit = "Last edited on 2026-09-09 04:12:57 by stolfi"
import sys, re, os, string, glob
from sys import stderr as err
import html_gen as h
from process_funcs import bash, run_command, basic_line_loop
import html_report_funcs as hr
import size_position_funcs as spf
import match_multi_funcs as mmf
import analyze_starps_parag_funcs as anf
import align_bencao_starps_items_funcs as alf
import report_077_alt_matching_funcs as r77alt
import standard_bipatterns as stdbip
import bimatching_eval_funcs as bef
import format_matching_funcs as fmf
from math import sqrt, hypot, exp, log, floor, ceil, isfinite, isnan, inf, nan
def split_formatted_entry(entry):
# Parses an SBJ entry (hanzi, pinyin, translation, or Voynichese)
# that has been cast in multiline format.
#
# The {entry} must be a multiline string where the first line has the
# format "<{LOC}>" and each subsequent line has the format
# "{TAG}{SEP}{ITEM}" where the {TAG} is a string of [A-Z0-9] in parens
# '()', {SEP} is one or more blanks or '|'s, {ITEM} is any string.
#
# Removes leading and traling ASCII spaces from {ITEM} (but not
# ideographic spaces). Removes #-comments and ignores blank lines.
#
# Returns the {LOC}, the list of all {TAG}s, and the list of all
# {ITEM}s.
def cleanup_line(line):
line = re.sub(r"[#].*$", "", line)
# Must keep ideographic spaces.
line = line.strip(' \011\015\012')
return line
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
lines = entry.splitlines()
lines = [ cleanup_line(lin) for lin in lines ]
lines = [ lin for lin in lines if lin != "" ]
nline = 0
loc = None
itms_tg = []
itms_wh = []
for line in lines:
nline += 1
line = cleanup_line(line)
if line == "": continue
if loc is None:
m = re.fullmatch(r"[ ]*[<]([a-z0-9.]+)[>][ ]*", line)
assert m != None, f"bad header line format [[{line}]]"
loc = m.group(1)
else:
assert loc != None, f"missing header line"
m = re.fullmatch(r"[ \012]*([(][A-Z0-9a-z]+[)])[ |]+(.+)[ \012]*", line)
assert m != None, f"bad entry line format '{line}'"
tag = m.group(1)
item = m.group(2)
itms_tg.append(tag)
# Cannot strip -- ust preserve ideographic spaces.
item = re.sub(r"^[ ]+", "", item)
item = re.sub(r"[ ]+$", "", item)
itms_wh.append(item)
return loc, itms_tg, itms_wh
# ----------------------------------------------------------------------
def split_formatted_entry_hanzi(entry):
# Parses an SBJ entry hanzi that has been cast in multiline format.
# See {split_formatted_entry} for the format of {entry}.
# Then does some checking and cleanup of the items.
#
# The items must contain only hanzi, ideographic blanks and
# punctuation (which are retained), and leading or trailing ASCII
# blanks (which are stripped). Pads all items with ideographic blanks
# to the same width.
#
# Returns the {LOC}, the list of all {TAG}s, and the list of all
# {ITEM}s.
loc_ch, itms_tg, itms_wh = split_formatted_entry(entry)
# Pad all items>
max_item_sz = 0
for item in itms_wh:
if re.search(r"[\001-\377]", item):
assert False, f"ascii character in hanzi item '{item}'"
max_item_sz = max(len(item), max_item_sz)
itms_wh = [ item.ljust(max_item_sz, " ") for item in itms_wh ]
return loc_ch, itms_tg, itms_wh
# ----------------------------------------------------------------------
def make_three_column_entry_table(itms_tg, itms_wh, itms_aa, itms_bb):
# Returns the HTML of a table with the given {itms_tg} on column 1, the
# given {itms_wh} in column 2, and arbitrary ascii entries{itms_aa,itms_bb} in
# columns 3 and 4.
#
# All four lists must have the same length, with corresponding
# elements in the same positions.
N = len(itms_tg)
assert len(itms_wh) == N, f"{N = } {len(itms_wh) = }"
assert len(itms_aa) == N, f"{N = } {len(itms_aa) = }"
assert len(itms_bb) == N, f"{N = } {len(itms_bb) = }"
bars = [ ' | ' ] * N
rows = list(zip(itms_tg, bars, itms_wh, bars, itms_aa, bars, itms_bb))
col_mods = [
"style='padding-left:4ch; padding-right:4ch; text-align:left; font-weight:bold;'",
"align=left",
"align=left",
"align=left",
"align=left",
"align=left",
"align=left",
]
html_tb = h.make_table(rows, by_rows = True, col_mods = col_mods)
return html_tb
# ----------------------------------------------------------------------
def add_three_column_entry_table(st, itms_tg, itms_wh, itms_aa, itms_bb):
# Appends to {st} a table with the given {itms_tg} on column 1,
# the given {itms_wh}
# in column 2, and arbitrary ascii entries{itms_aa,itms_bb} in columns 3 and 4.
#
# The elements of {itms_wh} are assumed to consist of hanzi and/or
# ideographic punctuation. The other columns are supposed to be
# Latin (or pinyin) letters with ISO-Latin punctuation.
#
# All four lists must have the same length, with corresponding
# elements in the same positions.
html_tb = make_three_column_entry_table(itms_tg, itms_wh, itms_aa, itms_bb)
h.append_centered(st, html_tb, centered = False)
return
# ----------------------------------------------------------------------
def entry_align_table(st, rows):
# Prints a table with the itms_tg on column 1, hanzi in column2, and
# arbitrary ascii entries in columns 3 and 4.
ch_ps_wp_en_wcol_mods = [
"style='padding-left:4ch; padding-right:4ch; text-align:left; font-weight:bold;'",
"align=left",
"align=left",
"align=left",
"align=left",
]
h.table(st, rows, col_mods = col_mods, centered = False)
return
# ----------------------------------------------------------------------
def read_parms_from_file_header(rd):
# Reads {rd} and looks for lines of the form "# {KEY} = {VALUE}".
# Returns a dict with those keys and values.
# The {KEY} may be any python3-style identifier.
# The {VALUE} for now may be an integer, a float, or a string.
vms_dic = dict()
err.write("!= beg\n")
def process_line(nread, line):
nonlocal vms_dic
line = line.strip()
err.write(f"!= {nread:5d} {line = !r}\n")
m = re.fullmatch(r"# *([a-zA-Z][a-zA-Z_0-9]*) *[=] *(.*)", line)
if m == None: return
key = m.group(1)
val = m.group(2).strip()
err.write(f"!= {key = !r} {val = !r}\n")
if re.fullmatch(r"[-+]?[0-9]+", val):
# Integer
val = int(val)
elif re.fullmatch(r"[(][-+0-9, ]+[)]", val):
# Integer tuple; assume pair:
m = re.fullmatch(r"[(]([-+]?[0-9]+)[ ,]+([-+]?[0-9]+)[)]", val)
val = (int(m.group(1)), int(m.group(2)),)
elif re.fullmatch(r"[-+]?[0-9]*([.][0-9]|[0-9][.])[0-9]*([Ee][-+]?[0-9]+)?", val):
val = float(val)
elif re.fullmatch(r"['][^']*[']", val):
val = re.sub(r"[']", "", val)
elif re.fullmatch(r'["][^"]*["]', val):
val = re.sub(r'["]', "", val)
elif re.fullmatch(r'\[.*\]', val):
val = re.sub(r'^\[', "", val)
val = re.sub(r'\]$', "", val)
elems = re.split(r'[, ]+', val)
items = []
for el in elems:
if el != "":
if el[0] == '"':
el = re.sub(r'"', "", el);
elif el[0] == "'":
el = re.sub(r"^'", "", el);
items.append(el)
val = items
else:
assert False, f"** bad value «{val}»"
vms_dic[key] = val
return
# :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
nread = basic_line_loop(rd, process_line)
return vms_dic
# ----------------------------------------------------------------------
def add_intro(code_ch, loc_ch, name_ch, name_py, name_en, source):
name_en = re.sub(r"_", " ", name_en.lower())
name_en_uscore = re.sub(r"[ ]+", "_", name_en)
name_en_caps = string.capwords(name_en)
title = f"[{code_ch}.077] The '{name_en_caps}' entry of the SBJ [{source}]"
assert source == 'ZHB' or source == 'CTP'
color = "#ddeeff" if source == 'ZHB' else "#eeffdd"
st = h.new_doc(title, color, text_width = 1600)
h.section(st, 2, "Summary")
h.parags(st, f"""This webpage discusses the SBJ entry titled {name_ch}
{name_py} = "{name_en}", parsed into its sub-entries and their fields, and
possible correspondences to parags of the SPS. The modern Mandarin
reading of the text and an English translation are also shown. This
entry may be referred as "{code_ch}" in tables.""")
def fetch_entry(utype):
# Reads the hanzi (if {utype} is "ch") or pinyin (if {utype} is
# {py}) of the SBJ recipe defined by {code_ch}, assuming its locus
# ID is {loc_ch}. Returns the file contents as a single string with
# lines separated by end-of-line.
fname = f"in/{utype}/{code_ch}-{loc_ch}.utf"
rd = open(fname, "r")
text = rd.read()
rd.close()
return text
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
h.section(st, 2, "The Shennong Bencaojing entry")
loc1, tags1, itms_wh = split_formatted_entry_hanzi(fetch_entry("ch"))
loc2, tags2, itms_py = split_formatted_entry(fetch_entry("py"))
assert loc2 == loc1
assert tags2 == tags1
loc3, tags3, itms_en = split_formatted_entry(fetch_entry("en"))
assert loc3 == loc1
assert tags3 == tags1
add_formatted_entry_table(st, tags1, itms_wh, itms_py, itms_en)
return st, code_ch, loc_ch, name_ch, name_py, name_en, tags1, itms_wh, itms_py, itms_en
# ----------------------------------------------------------------------
def add_chinese_text_section(st, loc_ch, name_ch, name_py, name_en, nch, itms_wh):
h.section(st, 3, "The full entry")
hanzi_full_text = f"<{loc_ch}> (full) " + "".join(itms_wh)
hanzi_full, tvar_ch_full = display_hanzi_pure_text(st, loc_ch, nch, hanzi_full_text)
assert tvar_ch_full == "full"
return hanzi_full
# ----------------------------------------------------------------------
def display_hanzi_pure_text(st, loc_ch, nch, hanzi_text):
# Takes a {hanzi_text} of the form "<{LOC}> ({TVAR}) {TEXT}"
# where {LOC} must match {loc_ch} and {TEXT} must be a hanzi
# string possibly including hanzi punctuation.
#.
# Extracts a pure hanzi string {hanzi_pure} by removing all
# hanzi punctuation from {TEXT}. The result must contain only
# non-punctuation hanzi and its length (in hanzi) must be {nch}.
#
# Appends to {st} a parag with {loc_ch}, {tvar_ch},{nch}
# and the {hanzi_pure} text. Returns {hanzi_pure} and the {TVAR} as
# the result.
hanzi_text = hanzi_text.strip()
hanzi_text = re.sub(r"[ \012]", "", hanzi_text)
m = re.fullmatch(r"<([a-z0-9.]+)> *[(]([^()]+)[)] *(.*)", hanzi_text)
assert m != None, f"invalid chinese entry format {hanzi_text!r}"
assert m.group(1) == loc_ch, f"loc ID mismtch: {loc_ch} != {m.group(1)}"
tvar_ch = m.group(2)
hanzi_body = m.group(3)
hanzi_pure = re.sub(r"[:[]()、,。; ]", "", hanzi_body)
nch_real = len(hanzi_pure)
assert nch_real == nch, f"length error: {nch = } actual {nch_real}"
if nch < 45:
hanzi_chops = [ hanzi_pure, ]
else:
hanzi_chops = [ hanzi_pure[k:k+40] for k in range(0, nch, 40) ]
hanzi_display = f"<{loc_ch}> ({tvar_ch}) {nch:2d} hanzi\n" + "\n".join(hanzi_chops)
h.append_preformatted(st, h.protect_html(hanzi_display), ind = 4, centered = False)
return hanzi_pure, tvar_ch
# ----------------------------------------------------------------------
def add_formatted_entry_table(st, itms_tg, itms_wh, itms_py, itms_en):
h.section(st, 3, "Pinyin and translation")
h.parags(st, """Here is the same entry, with punctuation added according
to this parsing, the modern Mandarin readings in pinyin, and a somewhat
literal English translation:""")
add_three_column_entry_table(st, itms_tg, itms_wh, itms_py, itms_en)
return
# ----------------------------------------------------------------------
def add_starps_matching_section \
( st, code_ch, loc_ch, itms_wh, variants,
del_qo, exp_irm, max_score, locs_to_try, locs_to_show ):
# Appends to document {st} the body of a section that searches the SPS
# file for parags matching a given SBJ entry in various ways.
#
# The {variants} argument must be a list of pairs of the form
#
# {((tvar_ch, text_ch), kwords_en)}
#
# where {kwords_en} is a list of strings identifying the "cribs" to use
# in the matching, {text_ch} is a suitably cleaned and
# trimmed hanzi text of the variant entry to be matched, without punctuation,
# and {tvar_ch} is a short string
# that identifies {text_ch} among other variants of the same SBJ recipe
# that differ in trimming or other modifications,
# and.
#
# Calls {add_starps_matching_subsection} for each tuple
# in {variants}, to compare that variant to all SPS parags.
#
# Returns a list of of the results of those calls. Each element of
# this list is a list of matching results. Each matching result is a
# tuple as returned by {r77alt.search_bencao_entry_in_starps_parags_file}
# (quod videt).
#
# Also adds a summary of the matches at the end.
#
# If {locs_to_try} is not {None}, it must be a set with the location IDS
# of the SPS parags to be considered for the match. If {locs_to_try} is None,
# considers all parags.
#
# If {locs_to_show} is not {None}, it must be a set or list of parag
# locus IDs that should be listed in the summary, in addition to the
# best ones in each variant.
h.section(st, 2, f"Matching results")
words_wh = [ wd.strip(' ') for wd in itms_wh ]
text_wh = ' '.join(words_wh)
pevs_list = []
for tvtx_ch, kwords_en in variants:
tvar_ch, text_ch = tvtx_ch
cribs = ",".join(kwords_en)
title = f"SBJ variant: {tvar_ch} Cribs: {cribs}"
pevs = add_starps_matching_subsection \
( st, code_ch, loc_ch, tvar_ch, text_ch, text_wh,
kwords_en, del_qo, exp_irm, title, max_score, locs_to_try, locs_to_show
)
pevs_list.append(pevs)
add_summary_of_matching_section \
( st, code_ch, loc_ch, text_wh,
variants, pevs_list, locs_to_show )
return pevs_list
# ----------------------------------------------------------------------
def add_starps_matching_subsection \
( st, code_ch, loc_ch, tvar_ch, text_ch, text_wh,
kwords_en, del_qo, exp_irm, title, max_score, locs_to_try, locs_to_show
):
# Appends to document {st} a section that searches the SPS
# file for parags matching a given SBJ entry in various ways.
# The parameters are:
#
# {code_ch} four-letter code of the SBJ entry.
# {loc_ch} loc ID of the entry to be matched in the SBJ file.
# {tvar_ch} a tag that identifies the variant (trimming etc.) of that entry.
# {text_ch} the text of the variant to be matched, trimmed, without punctuation.
# {text_wh} the full text of the entry, untrimmed, with punctuation.
# {kwords_en} is a list of strings.
# {del_qo} should we delete the @qo prefixes in the starps parags?
# {exp_irm} should we expand the @ir, @is, @m finales in the starps parags?
# {title} a title fof the section
# {max_score} maximum interesting badness score.
# {locs_to_trty} set of loc IDs of SPS parags to consider, or {None} for all.
# {locs_to_show} list or set of parag loc IDs to show whatever their scores.
#
# Returns a list of matching results. Each matching result is a tuple
# as returned by {r77alt.search_bencao_entry_in_starps_parags_file}
# (quod videt).
#
# Each element of {kwords_en} should be a /keyword code/, a string,
# like 'USES' or 'QI' that specifies an abstract keyword. This
# parameter is converted to a /bitemplate/ {bitemp} as expected by
# {bmf.match_bitemplate}
#
# The procedure finds a macro-parsing {segs_ch} of {text_ch} by side 0
# of {bitemp}. It then calls
# {r77alt.search_bencao_entry_in_starps_parags_file} on the file of
# good SPS parags {ivt_file} and evaluates each parag for how well it
# matches the SBJ entry.
#
# Let {nh} be the number of pattern codes in {kwords}. Let {ng} be
# {nh+1}, and {ns} be {ng + nh}.
#
# For each parag considered, the procedure creates a version {text_ec}
# of its text, cleaned and normalized. See {spf.normalize_text} for
# how {del_qo} and {exp_irm} affect this step. The preocedure then splits
# {text_ec} into a macro-parsings {segs_ec[0..ns-1]}, consisting of
# {nh} /hits/ (strings matched by the keyword templates of the hits in
# {segs_ch}) and {ng} /gaps/ (the strings before, between, and after
# the hits).
#
# For each parag the procedure also computes a badness {score} that
# combines penalties for the use of non-perfect hits (like 'laiin'
# instead of 'daiin') and discrepancies between the sizes of the
# gaps in {segs_ch} and {segs_ec}.
#
# The result is a list of parag evaluation tuples (/parevs/), one for
# each parag that may possibly match, with its badness score, the
# locus ID of the parag, the macro-parsing {segs_ch[0..ns-1]} of the SBJ entry,
# and the matching macro-parsing {segs_ec[0..ns-1]} of the SPS parag.
#
# This procedure then trims that list of parevs after the first one with
# score exceeding {max_score}, and inserts the list into the document
# {st}, formatted as described in {fmf.format_starps_parag_evaluation}.
# However, parags whose locs are in {locs_to_show are shown in any case.
#
# The result of the call is that trimmed list.
debug = False
verbose = True
ctsize_ch = len(text_ch)
kwords_str = ", ".join(kwords_en)
h.section(st, 3, f"{title}")
h.parags(st, f"Trimmed SBJ entry ({tvar_ch} - {ctsize_ch} hanzi):")
temptx_ch = f"<{loc_ch}> ({tvar_ch}) " + text_ch
text_check, tvar_ch_check = display_hanzi_pure_text \
( st, loc_ch, ctsize_ch, temptx_ch )
assert text_check == text_ch
assert tvar_ch_check == tvar_ch
# The result of comparing an SBJ entry with clean text {text_ch}
# and and an SPS parag with clean normalized text {text_ec} is either {None}
# or a parag evaluation record (/parev/).
# See {format_starps_parag_evaluation} for the contents of a parev.
nh = len(kwords_en); ng = nh + 1; ns = ng + nh
h.parags(st, "Keyword patterns:")
bitemp = build_bitemplate(kwords_en)
r77alt.add_bitemplate_description(st, kwords_en, bitemp)
hipat_ch, hipat_ec = r77alt.get_keyword_highlight_patterns_alt(bitemp)
if debug: err.write(f"!@ SBJ highlight pattern = {hipat_ch!r}\n")
if debug: err.write(f"!@ SPS highlight pattern = {hipat_ec!r}\n")
# Scan the SPS parags collecting reasonable matches:
pevs, data = r77alt.search_bencao_entry_in_starps_parags_file \
( code_ch, loc_ch, tvar_ch, text_ch, kwords_en, bitemp,
locs_to_try, del_qo, exp_irm )
# Discard totally bad candidates:
nc = len(pevs)
nc_good = 0; # Parevs with acceptable score.
while nc_good < nc and pevs[nc_good][0] <= max_score: nc_good += 1
err.write(f" {nc_good = }\n")
xcounts = []
xcounts.append(f"{data['npar_read']:6d} parags read")
if data['npar_excl'] > 0:
xcounts.append(f"{data['npar_excl']:6d} were were explicitly excluded.")
xcounts.append(f"{data['npar_eval']:6d} were evaluated.")
min_sz_ok = data['min_valid_size']
max_sz_ok = data['max_valid_size']
xcounts.append(f"{data['too_small']:6d} were rejected for having less than {min_sz_ok} letters.")
xcounts.append(f"{data['too_large']:6d} were rejected for having more than {max_sz_ok} letters.")
xcounts.append(f"{data['npar_bima']:6d} were examined for the requested cribs")
if data['npar_bima'] > 0:
min_sz_match = data['min_matched_size']
max_sz_match = data['max_matched_size']
xszrange = f"{min_sz_match}..{max_sz_match}"
xcounts.append(f"{data['npar_with']:6d} had all the cribs -- sizes {xszrange}")
if nc != data['npar_with']:
xcounts.append(f"{nc:6d} of them got a finite badness score.")
xcounts.append(f"{nc_good:6d} parags had badness {max_score} or less.")
else:
assert nc == 0
h.append_preformatted(st, "\n".join(xcounts), ind=0, centered=False)
nc_show = nc_good
# First parev with unacceptable score:
if nc_show < nc: nc_show += 1
# Discard excessive parevs:
max_show = 30
nc_show = min(nc_show, max_show)
if nc_show < max_show:
# Ensure that some parevs are shown:
nc_show = max(nc_show, min(nc, 3))
# Salutar paranoia:
err.write(f" {nc_show = }\n")
assert nc_show <= max_show + 1
validate_parevs \
( pevs, loc_ch, tvar_ch, text_ch, del_qo, exp_irm, max_score, nc_good, verbose )
# Show the candidates:
h.parags(st, f"Showing {nc_show} matches:")
# Ensure that {locs_to_show} is a set:
if locs_to_show is None:
locs_to_show = set()
elif isinstance(locs_to_show, list) or isinstance(locs_to_show, tuple):
locs_to_show = set(locs_to_show)
elif isinstance(locs_to_show, set):
pass
else:
assert False, f"invalid {locs_to_show = !r}"
ec_list_blocks = []
prev_segs_ch = None
pevs_out = []
for ic in range(nc):
pev = pevs[ic]
score = pev['score']
loc_ch_1 = pev['loc_ch']; tvar_ch_1 = pev['tvar_ch']; segs_ch = pev['segs_ch']
loc_ec = pev['loc_ec']; tvar_ec = pev['tvar_ec']; segs_ec = pev['segs_ec']
kwords_en = pev['kwords_en']; eva_per_hanzi= pev['eva_per_hanzi'];
hit_penalties = pev['hit_penalties']
# !!! check unused !!!
assert loc_ch_1 == loc_ch
assert tvar_ch_1 == tvar_ch
show = ic < nc_show or loc_ec in locs_to_show
if show:
if segs_ch != prev_segs_ch:
# Must show the SBJ entry parsing:
ec_list_blocks.append("\n")
ch_str = fmf.format_macro_parsing_ch(loc_ch, tvar_ch, segs_ch, hipat_ch)
ch_str = h.indent_lines(4, ch_str)
ec_list_blocks.append(ch_str)
ec_list_blocks.append("\n")
prev_segs_ch = segs_ch
ec_str = fmf.format_starps_parag_evaluation(pev, hipat_ec)
ec_list_blocks.append(ec_str)
pevs_out.append(pev)
ec_list_str = "\n".join(ec_list_blocks)
ec_list_str = h.protect_html(ec_list_str)
h.append_preformatted(st, ec_list_str, ind = 2, centered = False)
return pevs_out
# ----------------------------------------------------------------------
def build_bitemplate(kwords_en):
# Builds a bitemplate {bitemp} from the list {kwords_en} of
# abstract bipattern names like "USES", "QI", etc.
bitemp = []
for kw_en in kwords_en:
bipat = stdbip.get_bencao_starps_bipattern(kw_en)
bitemp.append(bipat)
return bitemp
# ----------------------------------------------------------------------
def validate_parevs \
(pevs, loc_ch, tvar_ch, text_ch, del_qo, exp_irm, max_score, nc_good, verbose ):
prev_score_p = -inf; prev_loc_ec_p = "NONE"
nc = len(pevs)
for ic in range(nc):
pev = pevs[ic]
score_p = pev['score']
loc_ch_p = pev['loc_ch']; tvar_ch_p = pev['tvar_ch']; segs_ch_p = pev['segs_ch']
loc_ec_p = pev['loc_ec']; tvar_ec_p = pev['tvar_ec']; segs_ec_p = pev['segs_ec']
kwords_en_p = pev['kwords_en']; eva_per_hanzi_p = pev['eva_per_hanzi'];
hit_penalties_p = pev['hit_penalties']
# !!! check unused !!!
assert loc_ch_p == loc_ch, \
f"wrong SBJ locus {loc_ch_p = !r} {loc_ch = !r}"
assert tvar_ch_p == tvar_ch, \
f"wrong SBJ variant {tvar_ch_p = !r} {tvar_ch = !r}"
assert loc_ec_p != prev_loc_ec_p, \
f"dup SPS parag {loc_ec_p = !r}"
text_ch_p = "".join(segs_ch_p)
assert text_ch_p == text_ch, \
f"SBJ text/segs mismatch {text_ch_p = !r} {text_ch = !r}"
text_ec_p = "".join(segs_ec_p)
if verbose:
err.write(f" parag {loc_ec_p:<12s} {score_p = :6.2f}\n")
assert isfinite(score_p) and score_p > 0, \
f"bad {score_p = }"
assert score_p >= prev_score_p, \
f"scores out of order {prev_score_p = } {score_p = !r}"
if ic < nc_good:
assert score_p <= max_score, f"{score_p = } exceeds max_score"
spc_eva_per_hanzi = spf.specific_eva_per_hanzi \
( len(text_ch), len(text_ec_p), del_qo, exp_irm )
assert isfinite(spc_eva_per_hanzi)
assert abs(spc_eva_per_hanzi - spc_eva_per_hanzi_p) < 1.0e-6
score_check = bef.compute_full_score_from_macro_parsings \
(segs_ch_p, segs_ec_p, eva_per_hanzi_p, hit_penalties_p)
if score_p != score_check:
err.write(f"{score_p = :24.16e}\n")
err.write(f"{score_check = :24.16e}\n")
assert abs(score_p - score_check) < 1.0e-6
prev_score_p = score_p
prev_loc_ec_p = loc_ec_p
return
# ----------------------------------------------------------------------
def write_dics_from_parev(st, code_ch, pev):
# Writes a set of hanzi-to-EVA dictionaries based on the locus ID {loc_ch} and
# text {text_ch} of an SBJ entry, the locus ID {loc_ec} of an SPS parag,
# and the macro-parsings of the two parsed texts.
#
# The dictionary for SBJ text fragments of length {fsize_ch}
# is written to file "dics/{code_ch}_{loc_ec}_{fsize_ch}.dic".
score = pev['score']
loc_ch = pev['loc_ch']; tvar_ch = pev['tvar_ch']; segs_ch = pev['segs_ch']
loc_ec = pev['loc_ec']; tvar_ec = pev['tvar_ec']; segs_ec = pev['segs_ec']
kwords_en = pev['kwords_en']; eva_per_hanzi = pev['eva_per_hanzi'];
hit_penalties = pev['hit_penalties']
# !!! check unused !!!
# Score and locus ID of parag:
loc_ec = re.sub(r"<[^<>]*>", "", loc_ec) # Just in case:
loc_ec = re.sub(r"[.]([0-9])$", r".0\1", loc_ec) # Zero-pad the line number.
max_fsize_ch = 4
for fsize_ch in range(max_fsize_ch + 1):
vms_dic = make_dic_from_parev(code_ch, loc_ch, pev, fsize_ch)
assert vms_dic != None
dic_file = f"dics/{code_ch}_{loc_ec}_{fsize_ch}.dic"
wr = open(dic_file, "w")
wr.reconfigure(encoding='utf-8')
wr.write("# -*- coding: utf-8 -*-\n")
wr.write(f"# {loc_ch = }\n")
wr.write(f"# {loc_ec = }\n")
pref = f"{code_ch}:{tvar_ch} | {loc_ch:<8s} | {loc_ec:<8s} |"
for frag_ch, frag_ec in vms_dic:
wr.write(pref)
assert fsize_ch == 0 or len(frag_ch) == fsize_ch
frag_ch = frag_ch.ljust(10," ")
frag_ec = frag_ec.ljust(50," ")
wr.write(f" {frag_ch} | {frag_ec} |\n")
wr.close()
return
# ----------------------------------------------------------------------
def add_summary_of_matching_section \
( st, code_ch, loc_ch, text_wh,
variants, pevs_list, locs_to_show ):
# The {pevs_list} must be a lits of lists of parevs, where
# {pevs_list[iv]} is the result of matching according to variant
# {variants[iv]}.
#
# Selects a few parags that look like valid matches, or best
# approximations thereof. For each variant, and each of
# those parags, prints a line with a terse summary:
# badness, EVA keywords matched, and gap errors.
h.section(st, 2, "Matching summary")
nv = len(variants); assert nv == len(pevs_list)
# First, select the best matches from each variant, and any
# other parags that have similar badness:
#
cands = locs_to_show # Set of loc ids of candidates.
for iv in range(nv):
cands |= select_best_cands(pevs_list[iv])
# Second, for each cand, get the best score among all the
# variants:
best_loc_scores = [] # List of pairs {(loc_ec, score_min)}
for loc_ec in cands:
score_min = +inf
for iv in range(nv):
tvtx_ch, kwords_en = variants[iv]
tvar_ch, text_ch = tvtx_ch
for pev in pevs_list[iv]:
score_p = pev['score']
loc_ch_p = pev['loc_ch']; tvar_ch_p = pev['tvar_ch'];
loc_ec_p = pev['loc_ec']; tvar_ec_p = pev['tvar_ec'];
assert loc_ch_p == loc_ch
assert tvar_ch_p == tvar_ch
if loc_ec == loc_ec_p:
if score_p < score_min: score_min = score_p
best_loc_scores.append((loc_ec, score_min, ))
best_loc_scores.sort(key = lambda x: x[1])
# Now print the summaries per variant:
for iv in range(nv):
pevs = pevs_list[iv]
tvtx_ch, kwords_en = variants[iv]
tvar_ch, text_ch = tvtx_ch
cribs = ",".join(kwords_en)
title = f"Trim: {tvar_ch} Cribs: {cribs}"
add_summary_of_variant_matching \
( st, code_ch, loc_ch, tvar_ch, text_wh, best_loc_scores, pevs, title )
return
# ----------------------------------------------------------------------
def select_best_cands(pevs):
# Given a list {pevs} of parevs, selects the one with minimum badness score
# and a few more with similar scores, if any. Returns the /set/
# of the locus IDs of the parags selected.
# Expects the list {pevs} to be sorted by non-decreasing
# badness scores.
max_cands = 5
cands = set()
if pevs != None:
score_min = None
tol = 0.2
for pev in pevs:
score = pev['score']
loc_ch = pev['loc_ch']; tvar_ch = pev['tvar_ch']; segs_ch = pev['segs_ch']
loc_ec = pev['loc_ec']; tvar_ec = pev['tvar_ec']; segs_ec = pev['segs_ec']
kwords_en = pev['kwords_en']; eva_per_hanzi = pev['eva_per_hanzi']; hit_penalties = pev['hit_penalties']
score, loc_ch, tvar_ch, loc_ec, tvar_ec, kwords_en, \
segs_ch, segs_ec, hit_penalties = pev
if score < +inf:
if score_min == None: score_min = score
assert score >= score_min
if score <= (1 + tol)*score_min:
cands.add(loc_ec)
tol = 0 if len(cands) >= max_cands else tol/2
return cands
# ----------------------------------------------------------------------
def add_summary_of_variant_matching \
( st, code_ch, loc_ch, tvar_ch, text_wh,
best_loc_scores, pevs, title ):
# Appends to {st} a summary of the matching attempts of one variant.
#
# The {best_loc_scores} must be a list of pairs {(loc_ec, score_min)}
# where {loc_ec} is the locus ID of an SPS parag and {score_min}
# is its lowest badness score over all the variants.
# The order of these pairs defines the order in which the
# parags will be listed.
#
# The {pevs} must be the list of parevs that resulted from
# the matching of this variant.
#
# The {title} should be an explanatory title for the varaint.
def find_parev(loc_ec, pevs):
# Finds the parev of parag {loc_ec} in {pevs}, or {None}:
the_pev = None
for pev in pevs:
if pev[3] == loc_ec: # !!! Maybe check {tvar_ec}, {kwords_en} !!!
the_pev = pev
break
return the_pev
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
h.section(st, 3, title)
rows = []
matched = False
rows.append([ "
" ])
for loc_ec, score_min in best_loc_scores:
pev = find_parev(loc_ec, pevs)
if pev is not None:
score_p = pev[0];
if score_p == +inf:
pev = None
else:
assert isfinite(score_p) and score_p >= score_min
row = make_row_of_summary_of_variant_matching(st, pev)
rows.append(row)
if pev is not None:
row = make_plot_row_for_summary_of_variant_matching(st, pev, text_wh)
rows.append(row)
rows.append([ "
", ])
sty_loc_ec = "padding-left:1ch; padding-right:2ch; text-align:left; font-weight:bold;"
sty_score = "padding-left:1ch; padding-right:1ch; text-align:right;"
sty_left = "padding-left:1ch; padding-right:1ch; text-align:left;"
col_mods = [
f"style='{sty_loc_ec}'",
f"style='{sty_loc_ec}'",
f"style='{sty_score}'",
f"style='{sty_left}'",
f"style='{sty_score}'",
f"style='{sty_left}'",
f"style='{sty_left}'",
]
html_tb = h.make_table(rows, by_rows = True, col_mods = col_mods)
h.append_centered(st, html_tb, centered = False)
return
# ----------------------------------------------------------------------
def make_row_of_summary_of_variant_matching(st, pev):
# Returns a row of the table of summary of variant matchings, for
# the parev {pev}, which must be for the given .
#
# The row always has the fllowing fields extracted from {pev}:
#
# the variant of the SBJ text {tvar_ch}.
# the starps parag locus ID {loc_ec}.
# its variant {tvar_ec}.
# the badness score of the bimatch.
# the assumed EVA per hanzi ratio.
# the hit strings used in the SPS side.
# the total hit penalty.
# the total SPS text length error
# the individual SPS gap errors.
#
# However if {pev} is {None} then a "No match" replaces the score, and
# the other fields are empty strings.
row = [ ]
tvar_ch_str = f"({tvar_ch})"
row.append(f"{tvar_ch_str:<10s}")
row.append(f"{loc_ec:<7s}")
tvar_ec_str = f"({tvar_ec})"
row.append(f"{tvar_ec_str:<8s}")
if pev is None:
row.append("No match")
else:
score = pev['score']
loc_ch = pev['loc_ch']; tvar_ch = pev['tvar_ch']; segs_ch = pev['segs_ch']
loc_ec = pev['loc_ec']; tvar_ec = pev['tvar_ec']; segs_ec = pev['segs_ec']
kwords_en = pev['kwords_en']; eva_per_hanzi = pev['eva_per_hanzi'];
hit_penalties = pev['hit_penalties']
assert isfinite(score_p)
score_str = f"{score:7.3f}"
row.append(score_str)
row_append(f" {eva_per_hanzi:5.3f} e/h")
hits_ec, tot_err, gap_errs = short_parev_summary \
( segs_ch, segs_ec, eva_per_hanzi )
hits_ec_str = ",".join([ "@" + ht for ht in hits_ec ])
row.append(hits_ec_str)
tot_hit_penalty = 0
for hp in hit_penalties: tot_hit_penalty += hp
kpen_str = f"({tot_hit_penalty:.3f})"
row.append(kpen_str)
tot_err_str = "00" if tot_err == 0 else f"{tot_err:+d}"
row.append(tot_err_str)
gap_errs_str = ",".join([ "00" if ger == 0 else f"{ger:+d}" for ger in gap_errs ])
row.append(gap_errs_str)
while len(row) < 7: row.append("")
return row
# ----------------------------------------------------------------------
def make_plot_row_for_summary_of_variant_matching(st, pev, text_wh):
assert pev != None
score_p = pev['score']
loc_ch_p = pev['loc_ch']; tvar_ch_p = pev['tvar_ch']; segs_ch_p = pev['segs_ch']
loc_ec_p = pev['loc_ec']; tvar_ec_p = pev['tvar_ec']; segs_ec_p = pev['segs_ec']
kwords_en_p = pev['kwords_en']; eva_per_hanzi_p = pev['eva_per_hanzi'];
hit_penalties_p = pev['hit_penalties']
# !!! check unused !!!
assert isfinite(score_p)
text_ch = "".join(segs_ch_p)
text_ec = "".join(segs_ec_p)
# Create the plot files:
fname_plot_png, fname_plot_jpg = create_graphical_alignment_plot(pev)
img_size = st['text_width']-200
img_url = fname_plot_png
img_thumb_url = fname_plot_jpg
link_text = f"{fname_plot_png}"
link_url = img_url
html_plot = h.make_link(st, link_url, link_text, img_thumb_url, img_size, img_size)
html_plot = f"{html_plot}"
row = [ html_plot ]
return row
# ----------------------------------------------------------------------
def short_parev_summary(segs_ch, segs_ec, eva_per_hanzi):
# Returns a list of the hits in {segs_ec}, the total size error,
# and a list of gap errors compared to the
# predictions from {segs_ch}, in EVA letters.
ns = len(segs_ch); assert len(segs_ec) == ns
nh = ns//2; ng = nh + 1; assert ns == nh + ng
# Total text sizes:
tsz_ch = sum(len(s) for s in segs_ch);
tsz_ec = sum(len(s) for s in segs_ec);
hits_ec = [ segs_ec[2*ih + 1] for ih in range(nh) ]
tot_err = anf.size_error(tsz_ch, tsz_ec, eva_per_hanzi)
gap_errs = [ ]
for ig in range(ng):
gsz_ch = len(segs_ch[2*ig])
gsz_ec = len(segs_ec[2*ig])
gap_errs.append(anf.size_error(gsz_ch, gsz_ec, eva_per_hanzi))
return hits_ec, tot_err, gap_errs
# ----------------------------------------------------------------------
def add_chosen_starps_parag_section \
( st, itms_tg, loc_ch, code_ch, itms_wh, \
chosen_loc_ec, chosen_tvar_ch, pevs_list, itms_en ):
# Inserts a parag saying that the SPS parag chosen to match
# the SBJ entry {loc_ch} (code {code_ch}) is {chosen_loc_ec},
#
# The {chosen_loc_ec} may be {None} to say that the SBJ entry will not
# be assigned to any SPS parag.
#
# If {chosen_loc_ec} is not {None}, the procedure looks it up in the
# evauation results {pevs_list}. The latter must be a list of lists,
# each one of them being either {None} or a list of tuples as returned
# by {analyze_and_show_starps_parags}.
#
# If {chosen_tvar_ch} is not {None}, considers only parevs that have
# that specific trim variant tag. If {chosen_tvar_ch} is {None},
# considers all parevs, of any trim variant.
#
# If it finds the {chosen_loc_ec} in the {pevs_list}, chooses the
# parev {best_pev} in those lists that has the specified
# {chosen_loc_ec}, and minimum badness score. Then displays the
# parsings of the SBJ entry and of the SPS parag described therein,
# both textually and graphically.
#
# The textual display includes the "macro-parsing" of the EVA
# characters of the SPS parag into gaps and hit, as contained in the
# {best_pev}.
#
# The textual display also shows a "micro-parsing" where the gaps are
# subdivided according to given parsings of the hanzi text into
# syntactic items. This display also shows the english translation.
#
# The graphical display plots the "ch" and "ec" text as two horz lines
# with rungs connecting the hits.
#
# Also writes the hanzi-eva dictionary files implied by the chosen parev.
#
# Returns that {best_pev}.
#
# If it cannot find the {chosen_loc_ec} in the {pevs_list}, displays a
# warning and returns {None}.
#
# Also writes the data files for the graphical match figure to
# files "fig/bencao/{code_ch}.utp" and "fig/starps/{chosen_loc_ec}.evp"
# then runs the plotting program and includes the plot in the HTML report.
debug = False
h.section(st, 2, "Chosen match")
best_pev = None
if chosen_loc_ec != None:
h.parags(st, f"""We will tentatively assign {code_ch} <{loc_ch}>
(variant {chosen_tvar_ch}) to {chosen_loc_ec}. However we must be aware
that the true match may not have made it into the "good" subset.""")
best_pev = find_best_starps_parag(chosen_tvar_ch, chosen_loc_ec, pevs_list)
if best_pev == None:
msg = f"WARNING - MISSING EVAL RECORD FOR {chosen_tvar_ch}:{chosen_loc_ec}"
h.parags(st, f"{msg}")
err.write(f"!! {msg}\n")
else:
if (debug): err.write(f"!c \n{best_pev = !r}\n")
else:
h.parags(st, f"""We will not assign {code_ch} <{loc_ch}> to any SPS parag.""")
itms_wc = None
if best_pev != None:
score, loc_ch, tvar_ch, loc_ec, tvar_ec, kwords_en, \
segs_ch, segs_ec, key_penalty = best_pev
if debug:
err.write(f"!& ### macro-parsing ch, ec from parev ###\n")
alf.write_wh_ec_wc_macro_parsings(err, "!&", segs_ch, segs_ec, None)
assert loc_ec == chosen_loc_ec
if chosen_tvar_ch != None: assert tvar_ch == chosen_tvar_ch
h.parags(st, f"SBJ entry parsing:")
ch_str = fmf.format_macro_parsing_ch(loc_ch, tvar_ch, segs_ch, None)
ch_str = h.protect_html(ch_str)
h.append_preformatted(st, ch_str, ind = 4, centered = False)
h.parags(st, f"SPS entry parsing:")
ec_str = fmf.format_starps_parag_evaluation(best_pev, None)
ec_str = h.protect_html(ec_str)
h.append_preformatted(st, ec_str, ind = 4, centered = False)
# Compute the ideal EVA/hanzi ratio from the gap sizes:
tot_eph, gap_eph, avg_eph = compute_observed_eva_per_hanzi(segs_ch, segs_ec)
h.parags(st, f"EVA/hanzi ratio from text lengths = {tot_eph:5.3f}")
if isnan(gap_eph):
assert avg_eph == tot_eph
else:
h.parags(st, f"EVA/hanzi ratio from gap lengths = {gap_eph:5.3f}")
h.parags(st, f"Suggested EVA/hanzi ratio = {avg_eph:5.3f}")
write_dics_from_parev(st, code_ch, best_pev)
write_cribs_used(st, segs_ch, segs_ec)
# Fetch the parag from the word-split SPS good parags file:
text_wc = get_clean_normalized_starps_text_wc(chosen_loc_ec, del_qo, exp_irm);
add_chosen_textual_alignment_subsection \
( st, itms_tg, itms_wh, itms_en, segs_ch,
text_wc, chosen_loc_ec, segs_ec )
add_chosen_graphical_alignment_subsection(st, code_ch, best_pev)
return best_pev
# ----------------------------------------------------------------------
def compute_observed_eva_per_hanzi(segs_ch, segs_ec):
ns = len(segs_ch); assert len(segs_ec) == ns
nh = ns//2; ng = nh + 1; assert ns == ng + nh
# Compute the EVA/hanzi ratio using the total text lengths:
len_ch = sum(len(s) for s in segs_ch)
len_ec = sum(len(s) for s in segs_ec)
tot_eva_per_hanzi = len_ec/len_ch
# Compute the bencao and starps total gap sizes, biasied by gap weight:
sum_wt_gsz_ch = 0
sum_wt_gsz_ec = 0
sum_wt = 0
for ig in range(ng):
ks = 2*ig
wt = bef.gap_score_weight(ig, ng)
sum_wt_gsz_ch += wt * len(segs_ch[ks])
sum_wt_gsz_ec += wt * len(segs_ec[ks])
sum_wt += wt
assert sum_wt > 0
tot_gsz_ch = ng * sum_wt_gsz_ch/sum_wt
tot_gsz_ec = ng * sum_wt_gsz_ec/sum_wt
# If the bencao gaps are a small fraction of the entry, use tot length:
if tot_gsz_ch == 0:
# The bencao text i all hits, no gaps:
gap_eva_per_hanzi = nan
avg_eva_per_hanzi = tot_eva_per_hanzi
else:
gap_eva_per_hanzi = tot_gsz_ec / tot_gsz_ch
# If the bencao gaps are too small, bias towards text length rato:
fgap_ch = tot_gsz_ch/len_ch
s = fgap_ch * fgap_ch
avg_eva_per_hanzi = (1-s) * tot_eva_per_hanzi + s * gap_eva_per_hanzi
return tot_eva_per_hanzi, gap_eva_per_hanzi, avg_eva_per_hanzi
# ----------------------------------------------------------------------
def write_cribs_used(st, segs_ch, segs_ec):
# Extracts and prints from {segs_ch,segs_ec} a
# table of the cribs actually used in the match.
ns = len(segs_ch); assert len(segs_ec) == ns
nh = ns//2; ng = nh + 1; assert ns == ng + nh
# Collect the cribs used:
cribs = []
for ih in range(nh):
ks = 2*ih + 1
cri = [ segs_ch[ks], segs_ec[ks], 1 ] # ??? PENALTY TOO ???
cribs.append(cri)
# Sort them lexicographically:
cribs.sort()
# Combine repeated uses into multiplicities:
kc = 0
for ic in range(len(cribs)):
if kc > ic and cribs[kc][0] == cribs[ic][0] and cribs[kc][1] == cribs[ic][1]:
cribs[kc][2] += 1
else:
kc += 1
nc = kc # Number of cribs.
cribs = cribs[:nc]
# Format ane print them:
h.parags(st, "Cribs used in this match:")
hwd_ch = max(len(c[0]) for c in cribs)
hwd_ec = max(len(c[1]) for c in cribs)
xcribs = []
for ic in range(nc):
cri = cribs[ic]
hit_ch = cri[0].ljust(hwd_ch,' ')
hit_ec = cri[1].ljust(hwd_ch,' ')
xcri = f"{hit_ch} {hit_ec} {cri[2]:2d}"
xcribs.append(xcri)
xcribs = "\n".join(xcribs)
h.append_preformatted(st, xcribs, ind = 2, centered = False)
return
# ----------------------------------------------------------------------
def get_clean_normalized_starps_text_wc(loc_ec, del_qo, exp_irm):
ivt_file = "res/starps-gd-wc-par.ivt"
rawtx_wc, nlin = fetch_starps_line(ivt_file, loc_ec)
assert rawtx_wc != None, f"** cannot find {loc_ec} in the starps file"
def data_error(msg):
file_line_error(ivt_file, nlin, msg, f"<{loc_ec}> {rawtx_wc}")
assert False
# ..................................................................
utype = "wc"
text_wc, head, tail = spf.clean_up_starps_raw_text(rawtx_wc, utype, data_error)
text_wc = spf.normalize_starps_text(text_wc, utype, del_qo, exp_irm, data_error)
return text_wc
# ----------------------------------------------------------------------
def add_chosen_textual_alignment_subsection \
( st, itms_tg, itms_wh, itms_en, segs_ch, text_wc, loc_ec, segs_ec ):
# Adds a subsection with a three-column table that displays the
# "micro-parsings" {itms_tg}, {itms_wh}, {itms_en} and the punctuated
# EVA text {txt_wc}, refined to make them compatible with the
# macro-parsings {segs_ch} and {segs_ec}and with each other.
#
# The english text strings from {itms_en} are also aligned but not split.
bites_tg, bites_wh, bites_wc, bites_en = \
alf.align_text_wc_with_micro_parsing_tg_wh_en_and_macro_parsing_ec \
( itms_tg, itms_wh, itms_en, segs_ch, text_wc, segs_ec )
assert bites_wc != None;
h.section(st, 3, "Aligning the two versions")
h.parags(st, f""" Here is the same text with the conjectured
correspondence with parag {loc_ec} of the
SPS:""")
add_three_column_entry_table(st, bites_tg, bites_wh, bites_wc, bites_en)
h.parags(st, """Note that the alignment of the Voynichese column
is only a rough guess based on the hanzi and EVA letter counts.""")
return
# ----------------------------------------------------------------------
def add_chosen_graphical_alignment_subsection(st, code_ch, pev):
# Adds a subsection with a figure showing the
# hanzi text of {itms_wh} and {text_wc}
# with the bimatching implied by {pev}.
h.section(st, 3, "Graphical alignment")
fname_plot_png, fname_plot_jpg = create_graphical_alignment_plot(code_ch, pev)
img_wd = st['text_width']-200
hr.image_link_parag \
( st, img_url = fname_plot_png, img_size = img_wd,
link_text = fname_plot_png, img_thumb_url = fname_plot_jpg
)
return
# ----------------------------------------------------------------------
def create_graphical_alignment_plot(code_ch, pev):
# Runs an external program that creates a graphical image of the
# {pev}. Returns the name of the image file.
debug = False
def data_error(msg):
err.write(f"** {msg}\n")
assert False
score = pev['score']
loc_ch = pev['loc_ch']; tvar_ch = pev['tvar_ch']; segs_ch = pev['segs_ch']
loc_ec = pev['loc_ec']; tvar_ec = pev['tvar_ec']; segs_ec = pev['segs_ec']
kwords_en = pev['kwords_en']; eva_per_hanzi = pev['eva_per_hanzi'];
hit_penalties = pev['hit_penalties']
if debug: err.write(f"!g {segs_ch = !r}\n");
text_ch = ''.join(segs_ch)
text_wh = ' '.join(segs_ch)
if debug: err.write(f"!g {text_wh = !r}\n");
if debug: err.write(f"!g {segs_ec = !r}\n");
text_ec = ''.join(segs_ec)
text_wc = ' '.join(segs_ec)
if debug: err.write(f"!g {text_wc = !r}\n");
len_ch = len(text_ch)
len_wh = len(text_wh)
name_bencao = f"{code_ch}-{tvar_ch}"
dir_in_ch = "plot/in/ch"
os.makedirs(dir_in_ch, exist_ok=True)
text_wh_header = f"<{loc_ch}> {tvar_ch} {len_ch}"
htext_wh = text_wh_header + "\n" + text_wh
write_text_for_plot(dir_in_ch, name_bencao, "text", htext_wh)
segs_ch_str = fmf.format_macro_parsing_ch(loc_ch, tvar_ch, segs_ch, None)
write_text_for_plot(dir_in_ch, name_bencao, "segs", segs_ch_str)
name_starps = f"{loc_ec}-{tvar_ec}"
dir_in_ec = "plot/in/ec"
os.makedirs(dir_in_ec, exist_ok=True)
text_wc_header = f"<{loc_ec}> {tvar_ec} {len(text_ec)}"
htext_wc = text_wc_header + "\n" + text_wc
write_text_for_plot(dir_in_ec, name_starps, "text", htext_wc)
segs_ec_str = fmf.format_starps_parag_evaluation(pev, None)
write_text_for_plot(dir_in_ec, name_starps, "segs", segs_ec_str)
kwords_tag = make_tag_from_kwords_en(kwords_en)
name_plot = f"{name_bencao}-{name_starps}-{kwords_tag}"
dir_out = "plot/out"
os.makedirs(dir_out, exist_ok=True)
command = [
"plot_alignment.sh",
f"{eva_per_hanzi:5.3f}",
f"{dir_in_ch}/{name_bencao}",
f"{dir_in_ec}/{name_starps}",
f"{dir_out}/{name_plot}"
]
err.write(f"plot command = \"{command}\"\n")
run_command(command)
fname_plot_png = f"{dir_out}/{name_plot}-sma.png"
fname_plot_jpg = f"{dir_out}/{name_plot}-sma.jpg"
return fname_plot_png, fname_plot_jpg
# ----------------------------------------------------------------------
def make_tag_from_kwords_en(kwords_en):
# Fletcher's checksum:
sum1a = 0; sum2a = 0
sum1b = 0; sum2b = 0
for kw_en in kwords_en:
bytes_en = (kw_en + ",").encode('utf-8')
for bt in bytes_en:
sum1a = (sum1a + bt) % 255
sum2a = (sum2a + sum1a) % 255
sum1b = (3*sum1b + 5*bt) % 255
sum2b = (sum2b + sum1b) % 255
chka = (sum2a << 8) | sum1a
chkb = (sum2b << 8) | sum1b
return f"{chka:04X}-{chkb:04X}"
# ----------------------------------------------------------------------
def remove_omitted_chars(textw, textc, punct):
# Given a hanzi or EVA string {textw} possibly with with punctuations,
# and the same string {textc} without punctuations and possibly without some
# omitted substrings, returns a copy {textr} of {textw} with
# all the punctuations in the set {punct but without the omitted substrings.
#
# Assumes that the first char after an omitted substring does not
# occur in the substring.
debug = False
if debug: err.write(f"!o {textw = !r}\n");
if debug: err.write(f"!o {textc = !r}\n");
nw = len(textw)
nc = len(textc)
tableau = [ None ] * nw;
for i in range(nw): tableau[i] = [ None ] * nc
# For {ic > 0} and {iw >= 0}, the entry {tableau[iw][ic]} is {(jw,jc)}
# iff {0 <= jw < iw}, {jc == ic-1}, and {textc[ic] == textw[iw]}, and
# {textc[0..jc]} matches a subsequence of {textw[0..jw]} that ends
# with {textw[jw]}. Otherwise {tableau[iw][ic]} is {None}.
# The entry {tableau[iw][0]} is {(-1,-1)} if {0 <= jw < iw}
# and {textc[0] == textw[iw]}.
# Get indices in {textw} of chars that are not punct:
iws = [ iw for iw in range(nw) if textw[iw] not in punct ]
# Fill the tableau:
for ic in range(nc):
chic = textc[ic];
assert chic not in punct, f"punct '{chic}' in \"{textc}\""
matched = False
for iw in iws:
chiw = textw[iw];
if chiw == chic:
if ic == 0:
tableau[iw][ic] = (-1,-1)
matched = True
if debug: err.write(f"!o {iw = } {ic = } {tableau[iw][ic] = !r}\n");
else:
for jw in iws:
if jw < iw and tableau[jw][ic-1] is not None:
tableau[iw][ic] = (jw,ic-1)
matched = True
if debug: err.write(f"!o {iw = } {ic = } {tableau[iw][ic] = !r}\n");
assert matched, f"could not match text[{ic}] = {chic!r}"
# Extract the result:
textr = [ ]
kw = nw
for jc in range(nc):
kc = nc - jc - 1
chkc = textc[kc]
if debug: err.write(f"!o {kc = } looking for {chkc = !r}\n");
found = False
while kw > 0:
kw -= 1;
chkw = textw[kw]
if debug: err.write(f"!o {kw = } {chkw = !r}\n");
if tableau[kw][kc] != None:
assert chkw == chkc
textr.append(chkw)
if debug: err.write(f"!o found {chkc!r} at {kw = }\n");
found = True
break
elif chkw in punct:
textr.append(chkw)
assert found, f"could not find match for {chkc!r}"
while kw > 0:
kw -= 1
chkw = textw[kw]
if debug: err.write(f"!o {kw = } {chkw = !r}\n");
if chkw in punct:
textr.append(chkw)
textr.reverse()
return "".join(textr)
# ----------------------------------------------------------------------
def write_text_for_plot(odir, name, tag, text):
fname = f"{odir}/{name}-{tag}.txt"
wr = open(fname, "w")
date = h.get_current_date()
wr.write(f"# Created by h77.write_text_for_plot on {date}\n")
text = re.sub(r"<[/]?[bi]>", "", text)
wr.write(text)
wr.write("\n")
wr.close()
return
# ----------------------------------------------------------------------
def fetch_starps_line(ivt_file, loc_starps):
# Reads fle {ivt_file} (assumed to be UTF-8) and looks for a line with
# locus ID "<{loc_starps}>". Returns that line, minus the locus D, stripped.
# Also returns the line number (from 1).
#
# If the {loc_starps} is not found, returns {None,None}.
# Just in case:
loc_starps = re.sub(r"[<>]", "", loc_starps)
loc_pat = f"<{loc_starps}>"
rd = open(ivt_file, "r")
rd.reconfigure(encoding='utf-8')
text = None
nlin = 0
for line in rd:
nlin += 1
if re.match(loc_pat, line):
text = re.sub(loc_pat, "", line)
text = text.strip()
break
rd.close()
return text, nlin
# ----------------------------------------------------------------------
def find_best_starps_parag(tvar_ch, loc_ec, pevs_list):
# Scans a bunch of lists of parevs (parag evaluations), selecting the
# with smallest badness score.
#
# If not {none}, the {pevs_list} must be a list of lists, each one
# of them being either {None} or a list of tuples as returned by
# {analyze_and_show_starps_parags}. The procedure ignores elements of
# {pevs_list} that are {None}.
#
# If {loc_ec} is a string, considers only parevs that have that SPS
# locus ID. If {loc_ec} is a set, considers only parevs whose SPS
# locus ID is in that set. If {loc_ec} is {None}, accepts any parag.
#
# If {tvar_ch} is a string, considers only parevs that have that SBJ
# trimming variant tag. If {tvar_ch} is a set, considers only parevs
# whose trimming tag in that set. If {tvar_ch} is {None},
# acepts any variant tag.
#
# Then returned result is the best parev among the parevs that were considered.
#
# If it cannot find any parev as requested, returns {None}.
gud_loc_ec = set((loc_ec,)) if isinstance(loc_ec, str) else loc_ec
gud_tvar_ch = set((tvar_ch,)) if isinstance(tvar_ch, str) else tvar_ch
best_pev = None;
if pevs_list != None:
# Find the best candidate record:
min_score = +inf
for pevs in pevs_list:
if pevs != None:
for pev in pevs:
score_p, loc_ch_p, tvar_ch_p, loc_ec_p, tvar_ec_p, kwords_en_p, \
segs_ch_p, segs_ec_p, key_penalty_p = pev
loc_ec_ok = (gud_loc_ec == None) or (loc_ec_p in gud_loc_ec)
tvar_ch_ok = (gud_tvar_ch == None) or (tvar_ch_p in gud_tvar_ch)
if loc_ec_ok and tvar_ch_ok:
if score_p < min_score:
best_pev = pev; min_score = score_p
return best_pev
# ----------------------------------------------------------------------
def make_dic_from_parev(code_ch, loc_ch, pev, fsize_ch):
# Returns a list of pairs {frag_ch,frag_ec} of hanzi and EVA fragments that
# are the conjectured matching parts of the SBJ entry {loc_ch}.
# Each is given the badness {score} of the pairing.
#
# If {fsize_ch} is positive, the hanzi fragments {frag_ch} will be
# all (overlapping) substrings of length {fsize_ch} of all the hanzi gaps in that
# macro-parsing. Specifically, if {segs_ch[ks]} is a gap (even {ks}), then, if {frag_ch} is
# centered at character position {kch} of that gap, the correspondng
# EVA fragment {frag_ec} is taken from {segs_ec[ks]} centered at a location
# {kec} that is {kch} scaled by the ratiof of the lengths of the two
# gaps. The fragment {frag_ec} is padded if needed with '·' (centered
# dots) to size {fsize.
#
# If {fsize_ch} is zero, the fragments {frag_ch} will be all the hits
# in the macro-parsing {segs_ch}, whole; and they will be paired
# with fragments {frag-ec} which are the corresponding hits of {segs_ec}.
score, loc_ch_p, tvar_ch, loc_ec, tvar_ec, kwords_en, \
segs_ch, segs_ec, key_penalty = pev
assert loc_ch_p == loc_ch
assert segs_ch != None
ns = len(segs_ch); assert len(segs_ec) == ns
nh = ns//2; ng = nh+1; assert ns == ng + nh
def data_error(msg):
assert False, msg
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
vms_dic = []
def extract_frag_pair(it_ch, fsz_ch, str_ch, str_ec):
# Extracts a fragment {frag_ch} of length {fsz_ch} at index {it_ch}
# from {str_ch}. Then extracts from {str_ec}
# the corresponding fragment {frag_ec}, assuming that
# the whole of {str_ch} maps to the whole of {str_ec}.
nonlocal vms_dic
# Define the number of EVA letters to take on each side of frag center:
mrg_ec = 15 + int(ceil(2.5*fsz_ch))
nt_ch = len(str_ch)
nt_ec = len(str_ec)
scale = (nt_ec+1)/(nt_ch+1)
# Limit of fragment on the SBJ gap:
jt_ch = it_ch + fsz_ch
# Character indices {kt_ch,kt_ec} of the frag centers:
kt_ch = (it_ch + jt_ch)/2
kt_ec = int(floor(scale * kt_ch + 0.5))
# Start of fragment on the SPS gap, and necessary padding:
it_ec = kt_ec - mrg_ec
lpad = 0 if it_ec >= 0 else -it_ec
it_ec = min(nt_ec-1, max(0, it_ec))
# Limit of fragment on the SPS gap, and necessary padding:
jt_ec = kt_ec + mrg_ec
rpad = 0 if jt_ec <= nt_ec else jt_ec - nt_ec
jt_ec = min(nt_ec, max(1, jt_ec))
frag_ch = str_ch[it_ch:jt_ch]
frag_ec = ("·" * lpad) + str_ec[it_ec:jt_ec] + ("·" * rpad)
return frag_ch, frag_ec
# ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
if fsize_ch == 0:
# Report the keyword hits:
for ih in range(nh):
frag_ch = segs_ch[2*ih+1]
frag_ec = re.sub(r'[\[\]]', "", segs_ec[2*ih+1])
vms_dic.append((frag_ch, frag_ec,))
else:
# Report substrings of the gaps with length {fsize_ch}:
for ig in range(ng):
gap_ch = segs_ch[2*ig]
gap_ec = re.sub(r'[\[\]]', "", segs_ec[2*ig])
nt_ch = len(gap_ch)
for it_ch in range(nt_ch + 1 - fsize_ch):
frag_ch, frag_ec = extract_frag_pair(it_ch, fsize_ch, gap_ch, gap_ec)
vms_dic.append((frag_ch, frag_ec,))
return vms_dic
# ----------------------------------------------------------------------
def get_recipe_pages_in_dir(dir):
# Scans the folder "{dir}" for files called "*_entry.html" or "*_entry_src.py".
#
# For every {name} such that either "{dir}/{name}_entry_src.py" or
# "{dir}/{name}_entry.html" exist, the resulting list will have
# "{dir}/{name}_entry.html".
#
assert os.path.exists(dir), f"folder {dir} does not exist"
src_html_files = \
glob.glob(f"./{dir}/*_entry_src.py") + \
glob.glob(f"./{dir}/*_entry.html")
hnames = map(lambda x: re.sub(r"(_entry_src[.]py|_entry[.]html)$", "", x), src_html_files)
hnames = map(lambda x: re.sub(r"^([.]/)*", "", x), hnames)
hnames = map(lambda x: re.sub(r"/([.]/)*", "/", x), hnames)
hnames = [ x for x in hnames if x != "" and not re.search(r"\b(work|JUNK|SAVE|AAAA)\b", x) ]
hnames = sorted(list(set(hnames)))
hfiles = list()
for hname in hnames:
hfiles.append(f"{hname}_entry.html")
return hfiles
# ----------------------------------------------------------------------
def test_stuff():
err.write("TESTING\n")
err.write("----------------------------------------\n")
test_remove_omitted_chars()
err.write("----------------------------------------\n")
test_add_chosen_graphical_alignment_subsection()
err.write("----------------------------------------\n")
test_add_starps_matching_section()
err.write("----------------------------------------\n")
return
# ----------------------------------------------------------------------
def test_remove_omitted_chars():
err.write("----------------------------------------\n")
err.write(f"@@@ testing remove_omitted_chars\n")
text_wh = "丹雄鸡: [味]甘,微温。 [主] (女人) 崩中, 漏下, 赤白沃。 补虚, 温中, 止血, 通神, 杀毒, 辟不祥。 头: [主] 杀鬼。 (东门上者 尤良。) 肪: [主] 耳聋。 肠: [主] 遗溺。 肶胵裹黄皮: [主] 泄利。 屎白: [主] 消渴, 伤寒寒热。 翮羽: [主] 下血闭。 鸡子: [主] 除 热火疮, 痫痉; [可作] 虎魄, 神物。 鸡白蠹: 能肥脂。 [生]平泽。"
err.write(f" {text_wh = !r}\n")
text_ch = "丹雄鸡主崩中漏下赤白沃补虚温中止血通神杀毒辟不祥头主杀鬼肪主耳聋肠主遗溺肶胵裹黄皮主泄利屎白主消渴伤寒寒热翮羽主下血闭鸡子主除热火疮痫痉可作虎魄"
err.write(f" {text_ch = !r}\n")
text_wh_r = remove_omitted_chars(text_wh, text_ch, set(r"[()]、,。;: ~ "))
nc_r = len(text_wh_r)
err.write(f" {nc_r = } {text_wh_r = !r}\n")
text_wh_r_exp = "丹雄鸡: [],。 [主] () 崩中, 漏下, 赤白沃。 补虚, 温中, 止血, 通神, 杀毒, 辟不祥。 头: [主] 杀鬼。 ( 。) 肪: [主] 耳聋。 肠: [主] 遗溺。 肶胵裹黄皮: [主] 泄利。 屎白: [主] 消渴, 伤寒寒热。 翮羽: [主] 下血闭。 鸡子: [主] 除 热火疮, 痫痉; [可作] 虎魄, 。 : 。 []。"
nc_r_exp = len(text_wh_r_exp)
err.write(f" {nc_r_exp = } {text_wh_r_exp = !r}\n")
if text_wh_r != text_wh_r_exp:
for ic in range(max(nc_r, nc_r_exp)):
cr = "#" if ic > nc_r else text_wh_r[ic]
cr_exp = "#" if ic > nc_r_exp else text_wh_r_exp[ic]
if cr != cr_exp:
err.write(f" {cr = !r} {cr_exp = !r}\n")
assert(False)
err.write("OK!\n")
return
# ----------------------------------------------------------------------
def test_add_starps_matching_section():
err.write("----------------------------------------\n")
err.write(f"@@@ testing add_starps_matching_section\n")
st, code_ch, loc_ch, name_ch, name_py, name_en, tags1, itms_ch, itms_py, itms_en = \
add_intro ( "WHOP", "b2.4.094", '白马茎', 'bái mǎ jīng', 'white horse penis', 'ZHB' )
err.write("@@@ calling display_hanzi_pure_text ...\n")
itms_wh = [
"白马茎: ",
" [味]咸,平。",
" [主] ",
" 伤中, ",
" 脉绝 ",
" 阴不起, ",
" 强志, ",
" 益气, ",
" 长肌肉 ",
" 肥健, ",
" 生子。 ",
"眼: ",
" [主] ",
" 惊痫, ",
" 腹满, ",
" 疟疾。 ",
"悬蹄: ",
" [主] ",
" 惊邪, ",
" 瘈疭, ",
" 乳难; ",
" 辟 ",
" 恶气, ",
" 鬼毒, ",
" 蛊注, ",
" 不祥。 ",
" [生]平泽。 ",
]
text_wh_trim = """
(trim)
白马茎:[主]伤中脉绝,阴不起,强志,益气,长肌肉。肥健,生子。
眼:[主]惊痫,腹满,疟疾。
悬蹄:[主]惊邪,瘈疭,乳难。辟恶气,鬼毒,蛊注,不祥。
"""
text_ch_trim, tvar_ch_trim = display_hanzi_pure_text(st, loc_ch, 48, text_wh_trim)
assert tvar_ch_trim == "trim"
kwords_en_A = ( 'USES', 'USES', 'USES', )
kwords_en_B = ( 'USES', 'QI', 'USES', 'USES', 'QI', )
max_score = 8.0
variants = \
(
( (tvar_ch_trim, text_ch_trim), kwords_en_A, ),
( (tvar_ch_trim, text_ch_trim), kwords_en_B, ),
)
err.write("@@@ calling add_starps_matching_section ...\n")
locs_to_show = set()
locs_to_try = set( [ "f114r.4", "f114v.1", "f106v.42", ] )
del_qo = True
exp_irm = True
pevs_list = add_starps_matching_section \
( st, code_ch, loc_ch, itms_wh, variants, del_qo, exp_irm, max_score, locs_to_try, locs_to_show )
err.write("@@@ finishing document ...\n")
h.output_doc(st, sys.stdout, 0, last_edit)
sys.stdout.flush()
err.write("@@@ done.\n")
err.write("----------------------------------------\n")
return
# ----------------------------------------------------------------------
def test_add_chosen_graphical_alignment_subsection():
err.write("----------------------------------------\n")
err.write(f"@@@ testing add_chosen_graphical_alignment_subsection\n")
st, code_ch, loc_ch, name_ch, name_py, name_en, tags1, itms_ch, itms_py, itms_en = \
add_intro ( "WHOP", "b2.4.094", '白马茎', 'bái mǎ jīng', 'white_horse_penis', 'ZHB' )
itms_wh = [
"白马茎:",
" [味]咸,平。",
" [主]",
" 伤中,",
" 脉绝",
" 阴不起,",
" 强志,",
" 益气,",
" 长肌肉",
" 肥健,",
" 生子。",
"眼:",
" [主]",
" 惊痫,",
" 腹满,",
" 疟疾。",
"悬蹄:",
" [主]",
" 惊邪,",
" 瘈疭,",
" 乳难;",
" 辟",
" 恶气,",
" 鬼毒,",
" 蛊注,",
" 不祥。",
" [生]平泽。",
]
tvar_ch = "trim"
segs_ch = [ \
"白马茎",
"主", "伤中脉绝阴不起强志益",
"气", "长肌肉肥健生子眼",
"主", "惊痫腹满疟疾悬蹄",
"主", "惊邪瘈疭乳难辟恶",
"气", "鬼毒蛊注不祥",
]
kwords_en = ( 'USES', 'QI', 'USES', 'USES', 'QI' )
loc_ec = "f114r.4"
tvar_ec = "norm"
text_wc = \
"fdeechdy.opchedaiin.ypchedy.odaly.chedy.p.cheokaiin." + \
"shedy.podaiin.ochedal.loiin.chedy.kaiin.chdy.daiin." + \
"dchdos.eedol.chdol.kchedy.cho.kaiin.chdy.eedy." + \
"okchedy.daiiin.chedy.daiin.ykeedy.okeeedy.chedol." + \
"chdaiin.ykar.dary.cheol.dchedy.dkchs.aiin.chdedy." + \
"daiin.okchedaiin.chain"
score = 2.492
hit_penalties = [ 0.000, 0.000, 0.140, 0.000, 0.000 ]
segs_ec = [
"fdeechdyopche",
"daiin", "ypchedyodalychedypcheokaiinshedypodaiinochedalloiin",
"chedy", "kaiinchdydaiindchdoseedolchdolkchedycho",
"kaiin", "chdyeedyokchedydaiiinchedy",
"daiin", "ykeedyokeeedychedolchdaiinykardarycheold",
"chedy", "dkchsaiinchdedydaiinokchedaiinchain",
]
len_ch = sum(len(s) for s in segs_ch)
len_ec = sum(len(s) for s in segs_ec)
del_qo = True
exp_irm = True
eva_per_hanzi = spf.specific_eva_per_hanzi(len_ch, len_ec, del_qo, exp_irm)
pev = \
{ 'score': score,
'loc_ch': loc_ch, 'tvar_ch': tvar_ch, 'segs_ch': segs_ch,
'loc_ec': loc_ec, 'tvar_ec': tvar_ec, 'segs_ec': segs_ec,
'kwords_en': kwords_en, 'eva_per_hanzi': eva_per_hanzi,
'hit_penalties': hit_penalties
}
add_chosen_graphical_alignment_subsection(st, code_ch, pev)
err.write("@@@ finishing document ...\n")
h.output_doc(st, sys.stdout, 0, last_edit)
sys.stdout.flush()
err.write("@@@ done.\n")
err.write("----------------------------------------\n")
return
# ----------------------------------------------------------------------
if len(sys.argv) == 2 and sys.argv[1] == "R77.TEST":
test_stuff()