#! /usr/bin/python3 # Last edited on 2026-09-09 08:58:58 by stolfi # Alternative matching sections for the report -- using # a single bitemplate with penalties for non-standard keywords. import sys, re, string from sys import stderr as err import html_gen as h import html_report_funcs as hr import size_position_funcs as spf import bitemplate_match_funcs as bmf import write_parsing_funcs as wpf import analyze_starps_parag_funcs as anf import bimatching_eval_funcs as bef from math import sqrt, hypot, exp, log, floor, ceil, isfinite, isnan, inf, nan def search_bencao_entry_in_starps_parags_file \ ( code_ch, loc_ch, tvar_ch, cleantx_ch, kwords_en, bitemp, locs_to_try, del_qo, exp_irm ): # # Arguments: # # {code_ch} four-letter code of the SBJ entry. # {loc_ch} locus ID of theSBJ entry in the CTP file. # {tvar_ch} tag that identifies the variant of that entry (trimming etc). # {cleantx_ch} text of that entry, suitably cleaned and trimmed. # {kwords_en} list of crib tags like {('USES','QI')} # {bitemp} the bitemplate determined by . # {locs_to_try} set of loc IDs of SPS parags to consider, or {None} for all. # {del_qo} should we delete the @qo prefixes in the starps parags? # {exp_irm} should we expand the @ir, @is, @m finales in the starps parags? # # The parameter {bitemp} must be a bitemplate suitable for # {bmf.match_bitemplate} # # Let {nh} be the number of pattern codes in {kwords_en}. Let {ng} be # {nh+1}, and {ns} be {ng + nh}. The procedure calls # {anf.evaluate_starps_parags} with arguments {code_ch}, {tvar_ch}, # {ivt_file}, {locs_to_try}, {kwords_en}, and {match_func} as the eval # function, to scan the file of good SPS parags {ivt_file} and # evaluate each parag for how well it matches the SBJ. # # The result of this procedure is a list {parevs} of parag evaluation # tuples (/parevs/) and a dictionary {stats} with various counts of the # operation. There will be a parev for each parag that may possibly # match. # # Each parev is a dict with the following keys: # # {'score'} badness score of the match. # {'loc_ch'} locus ID of the SBJ entry. # {'tvar_ch'} a short tag that identifies the variant of the same. # {'segs_ch'} macro-parsing of the cleaned SBJ hanzi text. # {'loc_ec'} locus ID of the head line of the SPS parag. # {'tvar_ec'} a short tag that identifies the variant of the same. # {'segs_ec'} macro-parsing of pure normalized SPS EVA text. # {'kwords_en'} the list of crib codes used, like {('USES','QI')'}. # {'eva_per_hanzi'} rato of EVA letter per hanzi assumed when comparing sizes. # {'hit_penalties'} list of contribs to {score} due to bad keywords. debug = False # Expected counts: nr = len(bitemp); # Rungs in bimatching. nh = nr # Hits. ng = nh + 1; # Gaps. ns = ng + nh # Segments. len_ch = len(cleantx_ch) min_valid_size, max_valid_size = \ spf.acceptable_starps_size_range(len_ch, del_qo, exp_irm) def match_func(loc_ec, dirtytx_ec, data_error, stats): nonlocal len_ch, max_valid_size, min_valid_size # SBJ-SPS entry matching function. # # This function receives the location ID and the raw text {dirtytx_ec} # of an SPS parag as read from the SPS transcription file. # It applies {dirtytx_ec} the cleanup and normalization # appropriate for {utype = "ec"} and as requested by {del_qo} and {exp_irm}, # obtaining a cleaned EVA text {cleantx_ec}. This text has only # lowercase EVA letters [a-z?] and is normalized by undoing presumed # errors, deleting presumed additions, etc. See # {clean_up_starps_raw_text} and {normalize_starps_text} in # {size_position_funcs.py} for details. # # The {match_func} function then computes the ratio of EVA letters # in that text to hanzi in {cleantx_ch} and rejects the parag if # that ratio is too different from the ideal. # # If the size is OK, the {match_func} function chooses the EVA # letter to hanzi ratio {eva_per_hanzi}, and calls # {bmf.match_bitemplate} with the specified {bitemp} to find the # best bimatching (pair of corresponding macro-parsing) for # {cleantx_ch} and {cleantx_ec}. # # The gap size error # penalties are computed using the estimated {eva_per_hanzi ratio. # The result is parallel # macro-parsings {segs_ch[0..ns-1]} and {segs_ec[0..ns-1]} of # {cleantx_ch} and {cleantx_ec}. # # The matching also computes a badness {score} for the parag as a # candidate for the SPS version of the SBJ entry. The sizes of the # gaps (even-indexed elements) in the SPS parag macro-parsing # {segs_ec[0..ns-1]} are compared with the sizes of the # corresponding gaps in the macro-parsing {segs_ch[0..ns-1]} of # the SBJ entry, individually and as total, taking into account the # average hanzi-to-EVA conversion factor and roundoff errors due to # the sizes being integers. # # The returned result is the {score} and the macro-parsings # {segs_ch[0..ns-1]}, {segs_ec[0..ns-1]}. # # The procedure increments {stats['npar_with']} as specified in # {anf.evaluate_starps_parags} debug_line = (loc_ec == "f105v.32") len_dirty_ec = len(dirtytx_ec) err.write(f"recipe {code_ch} {loc_ch} {tvar_ch} - {len_ch} hanzi\n") # Normalize and cleanup the raw text: utype = "ec" cleantx_ec, head, tail = spf.clean_up_starps_raw_text(dirtytx_ec, utype, data_error) cleantx_ec = spf.normalize_starps_text(cleantx_ec, utype, del_qo, exp_irm, data_error) len_clean_ec = len(cleantx_ec) tvar_ec = f"q{int(not del_qo)}r{int(not exp_irm)}" err.write(f"······································································\n") err.write(f"parag {loc_ec} {utype} size dirty = {len_dirty_ec} normalized = {len_clean_ec}\n") err.write(f"kwords {','.join(kwords_en)}\n") if debug_line: err.write(f"!a dirty text len = {len_dirty_ec:3d} {dirtytx_ec!r}\n") err.write(f"!a clean text len = {len_clean_ec:3d} {cleantx_ec!r}\n") dirty_eva_per_hanzi = len_dirty_ec/len(cleantx_ch) err.write(f"raw eva per hanzi = {dirty_eva_per_hanzi:5.3f}\n") score = +inf segs_ch = None segs_ec = None spc_eva_per_hanzi = nan hit_penalties = None # Save for callers: stats['min_valid_size'] = min_valid_size stats['max_valid_size'] = max_valid_size # Quick reject if sizes are too far off: if len_clean_ec < min_valid_size: err.write(f"SPS parag skipped - has {len_clean_ec} letters, min {min_valid_size}\n") stats['too_small'] += 1 elif len_clean_ec > max_valid_size: err.write(f"SPS parag skipped - has {len_clean_ec} letters, max {max_valid_size}\n") stats['too_large'] += 1 else: spc_eva_per_hanzi = spf.specific_eva_per_hanzi(len_ch, len_clean_ec, del_qo, exp_irm) assert isfinite(spc_eva_per_hanzi) err.write(f"assumed EVA per hanzi = {spc_eva_per_hanzi:5.3f}\n") score, segs_ch, segs_ec, hit_penalties = call_bimatch(cleantx_ec, spc_eva_per_hanzi, stats) if debug_line: err.write(f"!a {loc_ec:<12s} ({tvar_ec}) {score = :+8.3f}\n") return \ score, \ loc_ch, tvar_ch, cleantx_ch, segs_ch, \ loc_ec, tvar_ec, cleantx_ec, segs_ec, eva_per_hanzi, hit_penalties # :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: def call_bimatch(cleantx_ec, spc_eva_per_hanzi, stats): nonlocal len_ch def eval_one_gap(gsize_ch, gsize_ec, ig, ng): nonlocal spc_eva_per_hanzi debug = False if debug: err.write("!E {gsize_ch = !r} {gsize_ec = !r}\n") gap_score = bef.compute_single_gap_score \ ( gsize_ch, gsize_ec, spc_eva_per_hanzi, ig, ng ) if debug: err.write(f"!E {gap_score = :.3f}\n") return gap_score # .................................................................. # Tries to match {cleantx_ch} with {cleantx_ec} as per # bitemplate {bitemp}: debug_level = 0 rungs, score = bmf.match_bitemplate \ ( cleantx_ch, cleantx_ec, bitemp, eval_one_gap, debug_level ) stats['npar_bima'] += 1 if rungs == None: assert score == +inf segs_ch = None; segs_ec = None; hit_penalties = None else: assert len(rungs) == nr; assert isfinite(score) and score >= 0 stats['npar_with'] += 1 # Extract the segments defined by the rungs: segs_ch, segs_ec, hit_penalties = \ bmf.extract_segments(cleantx_ch, cleantx_ec, rungs) assert len(segs_ch) == ns and len(segs_ec) == ns; if debug: err.write(f"!e {ns = } {hit_penalties = !r}\n") mpars = ((segs_ch, '?', ' ',), (segs_ec, '?', ' ',),) wpf.write_parsings(err, "!e", True, mpars) assert len(segs_ec) == ns; if segs_ec != None: assert len(segs_ec) == ns return score, segs_ch, segs_ec, hit_penalties # :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: ivt_file = f"res/starps-gd-ec-par.ivt" parevs, stats = anf.evaluate_starps_parags \ ( ivt_file, locs_to_try, kwords_en, match_func ) min_ok = stats['min_valid_size'] max_ok = stats['max_valid_size'] err.write(f"{stats['npar_read']:6d} parags read.\n") if stats['npar_excl'] >0: err.write(f"{stats['npar_excl']:6d} parags excluded by {{locs_to_try}}.\n") err.write(f"{stats['npar_eval']:6d} were given to {{eval_func}}.\n") err.write(f"{stats['too_small']:6d} were rejected for having less than {min_ok} letters.\n") err.write(f"{stats['too_large']:6d} were rejected for having more than {max_ok} letters.\n") err.write(f"{stats['npar_bima']:6d} were examined for the requested cribs.\n") err.write(f"{stats['npar_with']:6d} of them had all the cribs") err.write(f" -- sizes {stats['min_matched_size']}..{stats['max_matched_size']}\n") return parevs, stats # ---------------------------------------------------------------------- def add_bitemplate_description(st, kwords_en, bitemp): nr = len(bitemp) assert len(kwords_en) == nr bars = '||' rows = [] for ir in range(nr): if ir > 0: row = tuple([ " " ]*7) rows.append(row) bipat = bitemp[ir] nv = len(bipat) for iv in range(nv): pvar = bipat[iv] pena, pat_ch, pat_ec = pvar row = ( \ (kwords_en[ir] if iv == 0 else ""), bars, f"{pena:.3f}", bars, pat_ch.ljust(3, '　'), bars, pat_ec ) rows.append(row) col_mods = [ "style='padding-left:4ch; padding-right:2ch; text-align:left;'", "style='padding-left:1ch; padding-right:1ch;'", "style='text-align:right;white-space:nowrap;'", "style='padding-left:1ch; padding-right:1ch;'", "style='text-align:left;white-space:nowrap;'", "style='padding-left:1ch; padding-right:1ch;'", "style='text-align:left;white-space:nowrap;'", ] h.table(st, rows, col_mods = col_mods, centered = False) return # ---------------------------------------------------------------------- def get_keyword_highlight_patterns_alt(bitemp): # Extracts from the bitemplate {bitemp} the two keyword patterns # {hipat_ch,hipat_ec} useful for # highlighting potential keywords in gaps of the SBJ and SPS macro. debug = False pats_ch = [] pats_ec = [] for bipat in bitemp: if debug: err.write(f"!/ {bipat = !r}\n") for pvar in bipat: if debug: err.write(f"!/ {pvar = !r}\n") pena, pat_ch, pat_ec = pvar pats_ch += pat_ch.split('|') pats_ec += pat_ec.split('|') hipat_ch = "|".join(tuple(set(tuple(pats_ch)))) hipat_ec = "|".join(tuple(set(tuple(pats_ec)))) assert re.match(hipat_ch, "") == None, f"hipat_ch matches empty" assert re.match(hipat_ec, "") == None, f"hipat_ec matches empty" return hipat_ch, hipat_ec # ----------------------------------------------------------------------