#! /usr/bin/python3 # Last edited on 2026-10-12 09:38:59 by stolfi # Functions for measuring size and listing word position in a # parag. To be included in other python scripts. import sys, os, re from sys import stdout as out, stderr as err, stdin as inp from error_funcs import arg_error, file_line_error, prog_error from process_funcs import bash, basic_line_loop from chinese_funcs import read_chinese_char_set import bencao_starps_convert_funcs as convf from math import sqrt, hypot, exp, log, pi, inf, nan, floor, ceil def get_parsing_patterns(utype): # Returns # # {pat_line} RE pattern matching a data line, where group 1 # is the locator (minus '<>') and group 2 is the text. # # {pat_unit} RE pattern matching one unit in the raw text. # # {pat_sepa} RE pattern matching any char that serves # as unit separator in the clean text. # # {clean_sepa} String that should separate units in clean text. # # The pattern {pat_sepa} may be {None} # if there are supposed to be no chars in that role. # # See {convert_starps_raw_text} and {convert_raw_bencao_text} for # the semantics of {utype}. # # The locus ID format {loc} depends on {utype}: # # "ch": the {loc} must be "b{sec}.{subsec}.{lseq}". # # "ec", "wp", or "wc": the {loc} must be "f[0-9]*[rv][0-9]*". # # Unit of text size/position: # # "ch": The units of counting is one hanzi. # # "ec": a single eva character [?a-z]. # # "wc" and "wc": one EVA token, a string [?a-z]+ delimited by '.' # if utype == "ch": pat_loc = r"<(b[1-3][.][0-9][.][0-9][0-9][0-9])>" pat_unit = r"." # Each char is a unit. pat_sepa = None # There are no unit separators on input. clean_sepa = "" # There are no unit separators in the converted text. elif utype == "ec" or utype == "wc" or utype == "wp": pat_loc = r"<(f[0-9]+[rv][0-9]*[.][0-9]+)>" if utype == "ec": pat_unit = r"[a-z?]" # Unit is a single EVA char. pat_sepa = None # There are no unit separators on input. clean_sepa = '' # There are no unit separators in the converted text. elif utype == "wc" or utype == "wp": pat_unit = r"[a-z?]+" # Unit is one or more EVA chars. pat_sepa = r"[.]" # In input text only '.' should be is separator. clean_sepa = '.' # Unit separator is '.' in cleaned text. else: arg_error(f"invalid {utype = !r}") pat_line = f"^[ ]*{pat_loc}[ ]+(.*)$" return pat_line, pat_unit, pat_sepa, clean_sepa # ---------------------------------------------------------------------- def acceptable_starps_size_range(len_ch, del_qo, exp_irm): # The minimum and maximum number of EVA letters in an SPS parag # that is worth evaluating, given the length {len_ch} # of the SBJ recipe in hanzi (trimmed and nomalized, sans punctuation). # # See {normalize_starps_text} for the meaning of {del_qo} and {exp_irm}. debug = False ideal_ratio = expected_eva_letters_per_hanzi(del_qo, exp_irm); max_rel_ratio = 1.15; # Relative tolerance for rejecting. min_size = int(floor(max(0, len_ch - 2)*ideal_ratio/max_rel_ratio)) max_size = int(ceil((len_ch + 2)*ideal_ratio*max_rel_ratio)) return min_size, max_size # ---------------------------------------------------------------------- def expected_eva_letters_per_hanzi(utype, del_qo, exp_irm): # A priori average number of EVA unitse for a Chinese character. # # If {utype} is "ec", the units are EVA letters, not counting # punctuation. If {utype} is "wc" or "wp", the units are EVA tokens. # Assumes that the Chinese text has no omitted parts (Nature, Alias, # etc.) if utype == "ec": eph = expected_eva_letters_per_hanzi(del_qo, exp_irm) elif utype == "wc": eph = expected_eva_tokens_per_hanzi(has_commas = True) elif utype == "wp": eph = expected_eva_tokens_per_hanzi(has_commas = False) else: assert False, f"invalid {utype = !r}" return # ---------------------------------------------------------------------- def expected_eva_letters_per_hanzi(del_qo, exp_irm): # A priori average number of EVA letters for a Chinese character. # # Assumes that the Chinese text has no omitted parts (Nature, Alias, etc.) # See {normalize_starps_text} for the meaning of {del_qo} and {exp_irm}. # # ROOS BLAR # ---- ---- # full 95 54 # trim - 44 # w0a0 74 - # w0a1 78 - # w1a0 76 - # w1a1 80 - # dqF xmF 361 236 # dqF xmT 372 237 # dqT xmF 345 228 # dqT xmT 356 229 eph = 4.933 # (361+236)/(77+44) if del_qo: eph *= 0.962 # (345+228+356+229)/(361+236+372+237) if exp_irm: eph *= 1.020 # (372+237+356+229)/(345+228+361+236) return eph # ---------------------------------------------------------------------- def expected_eva_tokens_per_hanzi(has_commas): # A priori average number of EVA tokens for a Chinese character. # # Assumes that the Chinese text has no omitted parts (Nature, Alias, etc.) # If {has_commas} is true, assumes that commas are treated as token # separators, like '.' or '-'. Otherwise assumes that commas are ignored. # tph = 0.950 if has_commas else 0.938 # !!! CONFIRM !!! return tph # ---------------------------------------------------------------------- def split_text_into_units(text, utype, pat_unit, pat_sepa, data_error): # Splits the {text}, assumed to be of type {utype}, into units of the type # implied by the {utype}. # # Returns a list {units} with the units, and a count {ct_sepa} of # chars in {text} that were matched by {pat_sepa} (and thus not counted as units). # # Currenly the {utype} must be "ch" (Chinese characters in Unicode) # "ec", "wc", or "wp" (EVA-encoded Voynichese). # # Loop on units and separators: ct_sepa = 0 units = [] while len(text) > 0: if pat_sepa is not None: m = re.match(pat_sepa, text) if m is not None: assert m.start() == 0 n = m.end() assert n > 0, "pattern {pat_sepa} matched empty" ct_sepa += n text = text[n:] continue m = re.match(pat_unit, text) if m is not None: assert m.start() == 0 n = m.end() assert n > 0, "pattern {pat_unit} matched empty" units.append(text[0:n]) text = text[n:] continue data_error(f"invalid chars = '{text}'") return units, ct_sepa # ---------------------------------------------------------------------- def list_occurrences_in_units(word, units, clean_sepa, utype, data_error): # Returns a list of the occurrences of {word} in # the text that consists of the list {units} of units of type {utype} # preceded, separated, and terminated by {clean_sepa}. # # Note that the bracketing {clean_sepa} make the text not valid. # They are added to simplify matching whole words. # # The {word} may be an RE pattern and/or may include {clean_sepa} # to control matches. # # !!! Misses overlapping occurrences. Is that OK? !!! if len(units) == 0: return [] btext = clean_sepa + clean_sepa.join(units) + clean_sepa if utype == "ch": oclist = list_ch_occurrences(word, btext) elif utype == "ec": oclist = list_ec_occurrences(word, btext) elif utype == "wc" or utype == "wp": oclist = list_wc_or_wp_occurrences(word, btext) else: arg_error(f"invalid {utype = !r}") return oclist # ---------------------------------------------------------------------- def list_ch_occurrences(word, btext): # Finds occurrences of {word} as string in the Unicode CHU-8 hanzi # {btext}, returning their positions. The text {btext} should have no # iso-latin-1 chars or hanzi punctuation. # # Positions are measured in hanzi chars. oclist = list(( p.start() for p in re.finditer(word, btext) )) return oclist # ---------------------------------------------------------------------- def list_ec_occurrences(word, btext): # Finds occurrences of {word} as string in the EVA {btext}, returning # their positions. Assumes that all blanks and EVA puntuation [-,.] # have been deleted. # # Positions are measured in EVA character counts oclist = list(( p.start() for p in re.finditer(word, btext) )) return oclist # ---------------------------------------------------------------------- def list_wc_or_wp_occurrences(word, btext): # Finds occurrences of {word} in the EVA {btext}, returning their # positions. # # Assumes that blanks, other EVA junk, and unwanted separators ([,] # for "wp") have been deleted, relevant unit separators ([-,.] for # "wc", [-.] for "wp") have been mapped to '.', and there is a single # '.' before, between, and after every unit, # # Positions are measured in EVA words. Unmatched EVA word suffixes or # prefizes are counted as 0.5 word. assert btext[0] == '.' and btext[-1] == '.', "cleanup failed" debug = False # We must loop on words: rest = btext oclist = [] pref = "" while True: if debug: err.write(f"!* '{pref}|{rest}'\n") m = re.search(word, rest) if m is None: break ix = m.start(); fx = m.end() if debug: err.write(f"!* {ix = } {fx = }\n") assert ix < len(rest) # If {word} matched just before '.', adjust {ix} to exclude the '.': if rest[ix] == '.': ix = ix + 1 # If {word} matched just after '.', adjust {fx} to exclude the '.': if fx < len(rest) and rest[fx-1] == ' ': fx = fx - 1 if debug: err.write(f"!* adjusted {ix = } {fx = }\n") # Count whole words before occurrence of {word}:] pref = pref + rest[0:ix]; core = rest[ix:fx]; rest = rest[fx:] if debug: err.write(f"!* '{pref}[{core}]{rest}'\n") oc = len(re.findall(r"[.]", pref)) - 1 if debug: err.write(f"!* {oc = }'\n") # If {word} matched only a suffix, count the prefix as half a word: if pref != '' and pref[-1] != '.': oc += 0.5 oclist.append(oc) pref = pref + core return oclist # ---------------------------------------------------------------------- def total_seg_sizes(segs): # Given a list of strings {segs}, # returns the {tot_size}, the total size of all the strings in {segs}, and # the {tot_gsz}, the totla size of the even-indexed ones. tot_size = 0; tot_gsz = 0 for i in range(len(segs)): s = segs[i] assert isinstance(s, str) tot_size += len(s) if i % 2 == 0: tot_gsz += len(s) return tot_size, tot_gsz # ---------------------------------------------------------------------- def compute_string_sizes(strings): # Given a list {strings} of strings, returns a list of the lengths of # those strings, and their total. sizes = [ len(st) for st in strings ] tsize = 0; for sz in sizes: tsize += sz return sizes, tsize # ---------------------------------------------------------------------- def round_est_size(sz, eps_log): # Rounds an estimated size after adding # {eps_log} in log scale. Rounds down if {eps_log} # is negative, up otherwise. tiny = 1.0e-100 if eps_log < 0: rsz = int(floor(exp(log(sz+tiny) + eps_log))) assert rsz <= sz elif eps_log > 0: rsz = int(ceil(exp(log(sz+tiny) + eps_log))) assert rsz >= sz else: rsz = int(sz + 0.5) return rsz # ---------------------------------------------------------------------- def scale_size_fuzzy(size, scale, eps): # Scales the given {size} by the given {scale} with some # slack up or down. Specifically {±eps} times an estimate of the # uncertainty in log scale. # # The {size} may be an integer or a range represented as a # pair of integers {(lo,hi)}. The result will always be a range # Accept integer as singleton ranges. if isinstance(size, int): size = (size,size,) loz = size[0]; lo_dev_log = compute_dev_of_log(loz) los = round_est_size(scale * loz, -eps * lo_dev_log) hiz = size[1] hi_dev_log = compute_dev_of_log(hiz) his = round_est_size(scale * hiz, +eps * hi_dev_log) return (los,his,) # ---------------------------------------------------------------------- def scale_sizes_fuzzy(sizes, scale, eps): # Given a list {sizes} of integers or integer ranges, # scales each element {sz} with {scale_size_fuzzy(sz,scale,eps} # (quod videt). new_sizes = [] for sz in sizes: new_sizes.append(scale_size_fuzzy(sz, scale, eps)) return new_sizes # ---------------------------------------------------------------------- def compute_dev_of_log(es): # Assumes that {es} is the expected value of a size variable with a # log-normal distribution with deviation proportional to {sqrt(es)}. # Returns the approximate deviation of the distributon of the log of # the size. debug = False if debug: err.write(f"!@ {es = :3d}") eps = 1 es0 = hypot(es, eps) # In case {es} is zero. es0_log = log(es0) if debug: err.write(f" {es0 = :5.1f} {es0_log = :+7.3f}") var_abs = 0.10 * es0 # Rough guess - to be improved. dev_abs = sqrt(var_abs) if debug: err.write(f" {dev_abs = :7.3f}") es1 = hypot(es0 + dev_abs, eps) es1_log = log(es1) if debug: err.write(f" {es1 = :5.1f} {es1_log = :+7.3f}") dev_log = es1_log - es0_log if debug: err.write(f" {dev_log = :+7.3f}\n") return dev_log # ---------------------------------------------------------------------- def compute_frac_error(s, es): # Fractional error between a size {s} and the expected size range {es}, # relative to the estimated deviation of the latter, in log scale. debug = False if debug: err.write(f"!~ {s = :3d} es = {es[0]:3d}..{es[1]:3d}") assert len(es) == 2 and es[0] <= es[1] avg = es[0] if s < es[0] else es[1] if s > es[1] else s if debug: err.write(f" {avg = :3d}") if s == avg: frac = 0 else: eps = 1 s0 = hypot(s, eps) s0_log = log(s0) if debug: err.write(f" {s0 = :5.1f} {s0_log = :+7.3f}") avg0 = hypot(avg, eps) avg0_log = log(avg0) if debug: err.write(f" {avg0 = :5.1f} {avg0_log = :+7.3f}") dev_log = compute_dev_of_log(avg) if debug: err.write(f" {dev_log = :7.3f}") frac = (s0_log - avg0_log)/dev_log if debug: err.write(f" {frac = :7.3f}\n") return frac # ---------------------------------------------------------------------- def compute_rorre_carf(frac, es): # Inverse of fractional error eps = 1 if frac < 0: dev_log = compute_dev_of_log(es[0]) y = frac*dev_log + log(hypot(es[0], eps)) z = exp(y) s = 0 if z <= eps else sqrt(z*z - eps*eps) elif frac > 0: dev_log = compute_dev_of_log(es[1]) y = frac*dev_log + log(hypot(es[1], eps)) z = exp(y) s = 0 if z <= eps else sqrt(z*z - eps*eps) else: # Geometric mean of the range end values: s = sqrt(hypot(es[0], eps)*hypot(es[1], eps)) assert s >= eps s = sqrt(s*s - eps*eps) return s # ---------------------------------------------------------------------- def parse_size_ranges(szranges_str): # The list {szranges_str} must be a string consisting of {neg} items # separated by commas, where each item is either an integer or a pair of # integers separated by '..'. They are converted to a list # {szranges[0..neg-1]} of integer pairs {(lo,hi)}. # xitems = re.split(r"[,]", szranges_str) szranges = [] for xit in xitems: xit = xit.strip() if xit != "": m = re.fullmatch(r"([0-9]+)[.][.]([0-9]+)", xit) if m != None: it_min = int(m.group(1)) it_max = int(m.group(2)) assert 1 <= it_min and it_min <= it_max, f"bad expected range {it_min}..{it_max}" szranges.append((it_min,it_max,)) else: it_val = int(xit) szranges.append((it_val,it_val,)) return szranges # ---------------------------------------------------------------------- def format_range(szrange): los = szrange[0]; his = szrange[1] if los == his: return str(los) else: return str(los) + ".." + str(his) # ---------------------------------------------------------------------- def format_size_ranges(szranges): szranges_str = '' for szrange in szranges: if szranges_str != '': szranges_str += ',' egs_str = format_range(szrange) szranges_str += egs_str return szranges_str # ---------------------------------------------------------------------- def test_stuff(): def data_error(msg): err.write(f"** {msg}\n"); assert False # .................................................................... err.write("----------------------------------------\n") err.write("testing compute_frac_error, compute_rorre_carf ...\n") for gs in range(30): fr = compute_frac_error(gs, (15,20,)) rf = compute_rorre_carf(fr, (15,20,)) err.write(f"{gs = :4d} {fr = :7.3f} {rf = :7.3f}\n") err.write("----------------------------------------\n") err.write("testing compute_dev_of_log ...\n") for k in range(5): esk = 5*k esk_log = log(hypot(esk, 1)) dev_log = compute_dev_of_log(esk) err.write(f"{esk = :4d} {esk_log = :7.3f} {dev_log = :7.3f}\n") err.write("----------------------------------------\n") err.write("testing get_parsing_patterns, split_text_into_units ...\n") for utype in ( "wc", "wp", "ec", "ch", ): err.write(f"!* counting '{utype}' units\n") pat_line, pat_unit, pat_sepa, clean_sepa = \ get_parsing_patterns(utype) if utype == "ec" or utype == "wc" or utype == "wp": text = "foo,par.paz-q.?ofoo,parrifoo" words = [ "foo", "par", ] if utype == "wc" or utype == "wp": words += [ "[.]foo", "foo[.]", "[.]foo[.]", "foo[.]par", ] assert clean_sepa == '.' elif utype == "ec": assert clean_sepa == '' elif utype == "ch": text = "白石英：［味］甘微温。［主］消渴，阴痿不足，欬逆白石，青石黑石脂等：［味］甘平。［主］黄疸，泄利" words = [ "白石", "主" ] assert clean_sepa == '' else: assert False err.write(f"!* raw text = '{text}')\n") text = convf.convert_raw_text(text, utype, data_error) err.write(f"!* converted text = '{text}')\n") units, ct_sepa = \ split_text_into_units(text, utype, pat_unit, pat_sepa, data_error) err.write(f"!* {ct_sepa = } {units = !r}\n") err.write("\n") for word in words: err.write(f"!* looking up '{word}'\n") oclist = list_occurrences_in_units(word, units, clean_sepa, utype, data_error) err.write(f"!* found = {oclist}\n") err.write("\n") return # ---------------------------------------------------------------------- if len(sys.argv) > 1 and sys.argv[1] == "BSPF.TEST": test_stuff()