#! /usr/bin/python3 # Last edited on 2026-09-09 04:06:58 by stolfi # Functions for measuring size and listing word position in a # parag. To be included in other python scripts. import sys, os, re from sys import stdout as out, stderr as err, stdin as inp from error_funcs import arg_error, file_line_error, prog_error from process_funcs import bash, basic_line_loop from chinese_funcs import read_chinese_char_set from note_077_funcs import compute_and_print_stats from math import sqrt, hypot, exp, log, pi, inf, nan, floor, ceil def clean_up_raw_text(text, utype, data_error): # Cleans up the raw {text} from an SBJ or SPS transcription file # according to {utype} for use in Note 077 analysis. # # See {clean_up_bencao_raw_text} for the meaning of {utype} "ch". # # See {clean_up_starps_raw_text} for the meaning of {utype}s "ec", # "wp", and "wc". # if utype == "ch": text = clean_up_bencao_raw_text(text, utype, data_error) elif utype == "ec" or utype == "wc" or utype == "wp": text, head, tail = clean_up_starps_raw_text(text, utype, data_error) return text # ---------------------------------------------------------------------- def normalize_text(text, utype, del_qo, exp_irm, data_error): # Modifies a {text} (typically after {cleanup_text}) from an SBJ or # SPS transcription file according to {utype} for use in Note 077 # analysis, by replacing certain substrings that are assumed to be # scribal errors, abbreviations, or non-significant variants. # # See {normalize_bencao_text} for the meaning of {utype} "ch". # # See {normalize_starps_text} for the meaning of {utype}s "ec", "wp", # and "wc". The parameters {del_qo} and {exp_irm} are used only for # these {utype}s. # if utype == "ch": text = normalize_bencao_text(text, utype, data_error) elif utype == "ec" or utype == "wc" or utype == "wp": text = normalize_starps_text(text, utype, del_qo, exp_irm, data_error) return text # ---------------------------------------------------------------------- def clean_up_starps_raw_text(text, utype, data_error): # Cleans up the raw {text} from an SPS transcription file according to # {utype} for use in Note 077 analysis. # # The function returns the cleaned {text} and booleans {head} and # {tail} that specify whether the line was a parag head line or parag # tail line, respectively. Note that both may be true for a one-line # parag. # # INPUT TEXT EXPECTED # # The raw {text} given must be a non-empty string that may contain # any of these: # # Prefix "<%>" marking the line as the head of a paragraph. # # Prefix [«=»] to indicate the alignment of the start of the line # relative to the left rail. # # EVA letters [A_Za-z]. # # Invalid EVA code '?'. # # Weirdo codes "&{NNN};" (possibly without the ';') # where {NNN} is three decimal digits. # # Ligatures consisting of two or more EVA letters, '?', or # weirdo codes enclosed in braces "{...}". # # Word separators [-.,]. # # Suffix [«=»] to indicate the alignment of the end of the line # relative to the right rail. # # Suffix "<$>" to mark the line as the tail of a parag. # # Inline comments "", including various special comments # to indicate stars in the margin, wide linegaps, figure intrusions. # vellum folds, etc. These may appear anywhere in the line # including before or after all the {text} and inside other markup # tags. # # Example: # # "<%>=w{Ch}edair.{Ch}otaropdaiin.otol,dair.aiir,aim=" # "={Ch}eeo,daiin.{Sh}eedy&162&211am.{Ch}odal.ol{Ch}edy«<$>" # "<%>={Ch}ol.oeedy.keedy.{Ch}{Ch}ekyair.{Ch}edy.ot{Ch}edy.{Qo}ty=" # # Note the parag breaks at the start and between the second and # third line. # # CLEANUP PERFORMED # # For all {utype}s, the cleanup entails: # # Removing all inline comments. # # Removing all markup, including the braces [{}], parag markers # "<%>" and "<$>", and rail alignment markers [«=»]. # # Mapping 'w' and 'z', used in my transcription to encode hooked # puffs, to 'p' and 'f', respctively. # # Mapping the uncertain reading charaters [bj] to '?'. # # Mapping all weirdo codes to '?'. # # Mapping everything to lowercase. # # Moreover, # # If {utype} is "ec", all punctuation [-,.] is deleted. # # If {utype} is "wp", every ',' is deleted, and '-' is mapped to '.' # # If {utype} is "wc", every ',' and '-' are mapped to '.' # # CLEANED TEXT # # After this cleanup, if {utype} is "ec", the text must have only # characters [ac-fhik-tvxy?]. Example: # # "choloeedykeedychchekycheod?eyke??edeedydaiiinaldair" # # If {utype} is "wc" or "wp", the raw text may also have '.' # separators. Example: # # "chol.oeedy.keedy.chcheky.cheod?ey.ke??edeedy.daiiin.ald.air" # # For more restrictions, see {check_starps_cleanup} below. # text = text.strip() if utype == "ec" or utype == "wc" or utype == "wp": # Remove inline comments: text = re.sub(r"[<][!][^<>]*[>]", "", text) # Map weirdo codes to '?': text = re.sub(r"[&][0-9][0-9][0-9][;]?", "?", text) # Remove ligature marks '{}': text = re.sub(r"[{]([^{}]*)[}]", r"\1", text) # Map to lowercase: text = text.lower() # Map hooked puffs to normal ones: text = re.sub(r"w", "p", text) text = re.sub(r"z", "f", text) # Map rare uncertain characters to '?': text = re.sub(r"[bj]", "?", text) # Check for parag markers: head = ( re.search(r"^[<][%][>]", text) != None ) tail = ( re.search(r"[<][$][>]$", text) != None ) # Remove all parag markers for now: text = re.sub(r"[<][%$][>]", "", text) # Remove alignment markers: text = re.sub(r"^[«=»]", "", text) text = re.sub(r"[«=»]$", "", text) # Check for invalid characters in input: m_bad = re.search(r"[^-.,ac-ik-z?]", text) if m_bad is not None: ibeg = m_bad.start() data_error(f"invalid character '{text[ibeg]}' at {ibeg}") # Check for irregular spaces in input: m_bad = re.search(r"^[-,.]|[-,.][-,.]|[-,.]$", text) if m_bad is not None: ibeg = m_bad.start(); iend = m_bad.end() data_error(f"improper punctuation '{text[ibeg:iend]}' at {ibeg}") # Replace punctuation: text = re.sub(r"-", ".", text) if utype == "ec": text = re.sub(r"[,.]", "", text) elif utype == "wc": text = re.sub(r"[,]", ".", text) elif utype == "wp": text = re.sub(r"[,]", "", text) else: assert False # Should not happen. else: arg_error(f"invalid {utype = !r}") check_starps_cleanup(text, utype, data_error) return text, head, tail # ---------------------------------------------------------------------- def clean_up_bencao_raw_text(text, utype, data_error): # Cleans up the raw {text} from an SBJ transcription file according to # {utype} for use in Note 077 analysis. Returns the cleaned {text}. # # INPUT TEXT EXPECTED # # For any {utype} the input text may contain editorial annotations # and markup. For both "ch" and "wh", # the input raw {text} must consist of simplified hanzi # (Chinese characters) in Unicode, plus ideographic punctuation # nd markup characters. Like: # # "冬葵子：［味］甘寒。［主］五藏六腑寒热羸瘦。［久服］坚骨长肌肉。（草头）" Like: # # In either case the raw {text} should not have embedded ASCII # blanks or other ASCII chars. Example: # # CLEANUP # # Cleanup for {utype} "ch" implies removong all punctuation. # # Cleanup for {utype} "wh" is currently a no-op. # # CLEANED TEXT # # After this cleanup, if {utype} is "ch", the raw text must # consist entirely of simplified hanzi. Example: # # "冬葵子味甘寒主五藏六腑寒热羸瘦久服坚骨长肌肉" # if utype == "ch": # Remove all punctuation: */ text = re.sub(r"[］［；：、，。（）　]", "", text) elif utype == "wh": # Nothing for now: pass else: arg_error(f"invalid text unit type {utype = !r}") check_bencao_cleanup(text, utype, data_error) return text # ---------------------------------------------------------------------- def normalize_starps_text(text, utype, del_qo, exp_irm, data_error): # Normalizes the raw {text} from an SPS transcription file according # to {utype} for use in Note 077 analysis. Returns the normalized text # # For all {utype}s, the normalization entails fixing assumed scribal # mistakes and assumed non-significant handwriting variation. # # Specifically, if {del_qo} is true, deletes all "qo". # If {exp_irm} is true, expands all "ir", "is", and "m". # if utype == "ec" or utype == "wc" or utype == "wp": # Reduce presumed handwriting variants: text = re.sub(r"g", "m", text) text = re.sub(r"u", "n", text) # Correct presumed scribal and transcription errors and abbreviations: text = re.sub(r"hh", "he", text) text = re.sub(r"ih", "ch", text) text = re.sub(r"i([ktpf])h", r"c\1h", text) if exp_irm: # Expand presumed scribal abbreviations text = re.sub(r"i([-,.]*)r", "i\1in", text) text = re.sub(r"i([-,.]*)s", "i\1in", text) text = re.sub(r"m", "iin", text) if del_qo: # Remove the "qo" glyphs. text = re.sub(r"qo", "", text) # Remove double punct that may have been created by normalization: if utype == "wc" or utype == "wp": text = re.sub(r"[,.]*[.][,.]*", ".", text) else: arg_error(f"invalid {utype = !r}") return text # ---------------------------------------------------------------------- def normalize_bencao_text(text, utype, data_error): # Normalizes the raw {text} from an SBJ transcription file according to {utype} # for use in Note 077 analysis. Returns the normalized {text}. # # If {utype} is "ch", the input raw {text} must have no punctuation, # just the hanzi letters. If the {utype} is "wh", it may have # punctuation, including ideographic brackets '' around keywords 味, # 主, 久服, etc. # # For both {utype}s, normalizing entails replacing some keys: # # 主治 [zhǔ zhì] to 主 [zhǔ] # 久服之 [jiǔ fú] zhī to 久服 [jiǔ fú] # 久食 [jiǔ shí] to 久服 [jiǔ fú] # check_bencao_cleanup(text, utype, data_error) if utype == "ch": # Regularize field keywords: text = re.sub(r"主治", "主", text) text = re.sub(r"久食", "久服", text) text = re.sub(r"久服之", "久服", text) elif utype == "wh": # Regularize field keywords: text = re.sub(r"主治", "主", text) text = re.sub(r"久食", "久服", text) text = re.sub(r"久服\］之", "久服］", text) else: arg_error(f"invalid text unit type {utype = !r}") return text # ---------------------------------------------------------------------- def check_starps_cleanup(text, utype, data_error): # Checks whether {text} is a valid "clean" SPS text according # to the given {utype}. Calls {data-error} if not. # Check for invalid characters: if utype == "ec": m_bad = re.search(r"[^ac-ik-vxy?]", text) elif utype == "wc" or utype == "wp": m_bad = re.search(r"[^.ac-ik-vxy?]", text) else: arg_error(f"invalid {utype = !r}") if m_bad is not None: ibeg = m_bad.start() data_error(f"invalid text char '{text[ibeg]}' at pos {ibeg}") if utype == "wc" or utype == "wp": # Check for improper spacing: pat_bad_punc = r"^[.]|[.][.]|[.]$" m_bad = re.search(pat_bad_punc, text) if m_bad != None: ibeg = m_bad.start() data_error(f"improper punctuation '{text[ibeg]}' at pos {ibeg}") return # ---------------------------------------------------------------------- def check_bencao_cleanup(text, utype, data_error): # Checks whether {text} is a valid "clean" SBJ text according # to the given {utype}. Calls {data-error} if not. # # !!! Currently does not do a thorough check. Improve! !!! m_bad = None if utype == "ch": # Check for ASCII characters: m_bad = re.search(r"[\001-\377]", text) if m_bad is not None: ibeg = m_bad.start() data_error(f"invalid text char '{text[ibeg]}' at {ibeg}") return # ---------------------------------------------------------------------- def get_parsing_patterns(utype): # Returns # # {pat_line} RE pattern matching a data line, where group 1 # is the locator (minus '<>') and group 2 is the text. # # {pat_unit} RE pattern matching one unit in the raw text. # # {pat_sepa} RE pattern matching any char that serves # as unit separator in the clean text. # # {clean_sepa} String that should separate units in clean text. # # The pattern {pat_sepa} may be {None} # if there are supposed to be no chars in that role. # # See {clean_up_starps_raw_text} and {clean_up_raw_bencao_text} for # the semantics of {utype}. # # The locus ID format {loc} depends on {utype}: # # "ch": the {loc} must be "b{sec}.{subsec}.{lseq}". # # "ec", "wp", or "wc": the {loc} must be "f[0-9]*[rv][0-9]*". # # Unit of text size/position: # # "ch": The units of counting is one hanzi. # # "ec": a single eva character [?a-z]. # # "wc" and "wc": one EVA token, a string [?a-z]+ delimited by '.' # if utype == "ch": pat_loc = r"<(b[1-3][.][0-9][.][0-9][0-9][0-9])>" pat_unit = r"." # Each char is a unit. pat_sepa = None # There are no unit separators on input. clean_sepa = "" # No separators in cleaned text. elif utype == "ec" or utype == "wc" or utype == "wp": pat_loc = r"<(f[0-9]+[rv][0-9]*[.][0-9]+)>" if utype == "ec": pat_unit = r"[a-z?]" # Unit is a single EVA char. pat_sepa = None # There are no unit separators on input. clean_sepa = '' # No unit separator in cleaned text. elif utype == "wc" or utype == "wp": pat_unit = r"[a-z?]+" # Unit is one or more EVA chars. pat_sepa = r"[.]" # In clean text only '.' is separator. clean_sepa = '.' # Unit separator is '.' in cleaned text. else: arg_error(f"invalid {utype = !r}") pat_line = f"{pat_loc}[ ]+(.*)" return pat_line, pat_unit, pat_sepa, clean_sepa # ---------------------------------------------------------------------- def acceptable_starps_size_range(len_ch, del_qo, exp_irm): # The minimum and maximum number of EVA letters in an SPS parag # that is worth evaluating, given the length {len_ch} # of the SBJ recipe in hanzi (trimmed and nomalized, sans punctuation). # # See {normalize_starps_text} for the meaning of {del_qo} and {exp_irm}. debug = False ideal_ratio = ideal_eva_per_hanzi \ (ch_trimmed = True, del_qo = del_qo, exp_irm = exp_irm); max_rel_ratio = 1.07; # Relative tolerance for rejecting. min_size = int(floor(max(0, len_ch - 2)*ideal_ratio/max_rel_ratio)) max_size = int(ceil((len_ch + 2)*ideal_ratio*max_rel_ratio)) return min_size, max_size # ---------------------------------------------------------------------- def specific_eva_per_hanzi(len_ch, len_ec, del_qo, exp_irm): # The ratio of EVA letters per hanzi character ratio to assume when # computing gap size penalties. Currently based on total lengths of # the two texts (trimmed and nomalized, sans punctuation). # # See {normalize_starps_text} for the meaning of {del_qo} and {exp_irm}. # # The result is {len_ec/len_ch} clipped to a # range close to the ideal ratio as defined by {ideal_eva_per_hanzi}, # then rounded to 3 decimal places (apart from float roundoff). debug = False assert len_ch >= 1, f"invalid recipe length {len_ch}" assert len_ec >= 1, f"invalid parag length {len_ec}" eph = len_ec/len_ch ideal_ratio = ideal_eva_per_hanzi \ (ch_trimmed = True, del_qo = del_qo, exp_irm = exp_irm); dev = eph/ideal_ratio if debug: err.write(f"!r {len_ec = } {len_ch = }\n") if debug: err.write(f"!r {eph = :8.6f} {ideal_ratio = :8.6f} {dev = :8.6f}\n") # Don't let the assumed ratio be too far from ideal: gud_rel_ratio = 1.05; # Relative tolerance for clipping. if dev > gud_rel_ratio: eph = ideal_ratio * gud_rel_ratio elif dev < 1/gud_rel_ratio: eph = ideal_ratio / gud_rel_ratio # Round to three decmals for consistency with print formats: eph = floor(eph*1000 + 0.5)/1000 if debug: err.write(f"!r {eph = :8.6f}\n") return eph # ---------------------------------------------------------------------- def ideal_eva_per_hanzi(ch_trimmed, del_qo, exp_irm): # A priori average number of EVA letters for a Chinese character. # # If {ch_trimmed} assumes that all SBJ fields that are omitted # in the SPS have been deleted from the SBJ text. # # See {normalize_starps_text} for the meaning of {del_qo} and {exp_irm}. # eph = 5.084 if del_qo: eph *= 1.031 # !!! ??? Confirm ??? !!! if exp_irm: eph *= 0.990 # !!! ??? Confirm ??? !!! if not ch_trimmed: eph = eph / 1.150 # !!! ??? Confirm ??? !!! return eph # ---------------------------------------------------------------------- def ideal_eva_token_per_hanzi(ch_trimmed, has_commas): # A priori average number of EVA tokens for a Chinese character. # # If {ch_trimmed} assumes that all SBJ fields that are omitted # in the SPS have been deleted from the SBJ text. # tph = 0.950 if has_commas else 0.938 if ch_trimmed: tph = tph * 1.200 return tph # ---------------------------------------------------------------------- def generic_hanzi_per_unit(utype, del_qo, exp_irm): # A priori average number of Chinese characters for each units # of type {utype}. # # If {utype} is "ch" the result is 1.0. The other params are ignored. # # Otherwise {utype} must be "ec", "wc", or "wp". The # parameter {del_qo} and {exp_irm} are used only if {utype} = "ec". # See {normalize_starps_text} for their meaning. hpu = nan if utype == "ch": hpu = 1.000 elif utype == "ec" : hpu = 1.0/ideal_eva_per_hanzi \ (ch_trimmed = False, del_qo = del_qo, exp_irm = exp_irm) elif utype == "wc": hpu = 1.0/ideal_eva_token_per_hanzi(ch_trimmed = False, has_comma = True) elif utype == "wp": hpu = 1.0/ideal_eva_token_per_hanzi(ch_trimmed = False, has_comma = False) else: assert False, f"invalid {utype = !r}" return hpu # ---------------------------------------------------------------------- def split_text_into_units(text, utype, pat_unit, pat_sepa, data_error): # Splits cleaned {text} into units of the type specified by {utype}. # Assumes that the txt was cleaned according to {utype}. # # Returns a list {units} with the units, and a count {ct_sepa} of # chars in {text} that were matched by {pat_sepa} (and thus not counted as units). # # Currenly the {utype} must be "ch" (Chinese characters in Unicode) # , "ec", "wc", or "wp" (EVA-encoded Voynichese). # # Loop on units and separators: ct_sepa = 0 units = [] while len(text) > 0: if pat_sepa is not None: m = re.match(pat_sepa, text) if m is not None: assert m.start() == 0 n = m.end() assert n > 0, "pattern {pat_sepa} matched empty" ct_sepa += n text = text[n:] continue m = re.match(pat_unit, text) if m is not None: assert m.start() == 0 n = m.end() assert n > 0, "pattern {pat_unit} matched empty" units.append(text[0:n]) text = text[n:] continue data_error(f"invalid chars = '{text}'") return units, ct_sepa # ---------------------------------------------------------------------- def list_occurrences_in_units(word, units, clean_sepa, utype, data_error): # Returns a list of the occurrences of {word} in # the text that consists of the list {units} of units of type {utype} # preceded, separated, and terminated by {clean_sepa}. # # Note that the bracketing {clean_sepa} make the text not valid. # They are added to simplify matching whole words. # # The {word} may be an RE pattern and/or may include {clean_sepa} # to control matches. # # !!! Misses overlapping occurrences. Is that OK? !!! if len(units) == 0: return [] btext = clean_sepa + clean_sepa.join(units) + clean_sepa if utype == "ch": oclist = list_ch_occurrences(word, btext) elif utype == "ec": oclist = list_ec_occurrences(word, btext) elif utype == "wc" or utype == "wp": oclist = list_wc_or_wp_occurrences(word, btext) else: arg_error(f"invalid {utype = !r}") return oclist # ---------------------------------------------------------------------- def list_ch_occurrences(word, btext): # Finds occurrences of {word} as string in the Unicode CHU-8 hanzi # {btext}, returning their positions. The text {btext} should have no # iso-latin-1 chars or hanzi punctuation. # # Positions are measured in hanzi chars. oclist = list(( p.start() for p in re.finditer(word, btext) )) return oclist # ---------------------------------------------------------------------- def list_ec_occurrences(word, btext): # Finds occurrences of {word} as string in the EVA {btext}, returning # their positions. Assumes that all blanks and EVA puntuation [-,.] # have been deleted. # # Positions are measured in EVA character counts oclist = list(( p.start() for p in re.finditer(word, btext) )) return oclist # ---------------------------------------------------------------------- def list_wc_or_wp_occurrences(word, btext): # Finds occurrences of {word} in the EVA {btext}, returning their # positions. # # Assumes that blanks, other EVA junk, and unwanted separators ([,] # for "wp") have been deleted, relevant unit separators ([-,.] for # "wc", [-.] for "wp") have been mapped to '.', and there is a single # '.' before, between, and after every unit, # # Positions are measured in EVA words. Unmatched EVA word suffixes or # prefizes are counted as 0.5 word. assert btext[0] == '.' and btext[-1] == '.', "cleanup failed" debug = False # We must loop on words: rest = btext oclist = [] pref = "" while True: if debug: err.write(f"!* '{pref}|{rest}'\n") m = re.search(word, rest) if m is None: break ix = m.start(); fx = m.end() if debug: err.write(f"!* {ix = } {fx = }\n") assert ix < len(rest) # If {word} matched just before '.', adjust {ix} to exclude the '.': if rest[ix] == '.': ix = ix + 1 # If {word} matched just after '.', adjust {fx} to exclude the '.': if fx < len(rest) and rest[fx-1] == ' ': fx = fx - 1 if debug: err.write(f"!* adjusted {ix = } {fx = }\n") # Count whole words before occurrence of {word}:] pref = pref + rest[0:ix]; core = rest[ix:fx]; rest = rest[fx:] if debug: err.write(f"!* '{pref}[{core}]{rest}'\n") oc = len(re.findall(r"[.]", pref)) - 1 if debug: err.write(f"!* {oc = }'\n") # If {word} matched only a suffix, count the prefix as half a word: if pref != '' and pref[-1] != '.': oc += 0.5 oclist.append(oc) pref = pref + core return oclist # ---------------------------------------------------------------------- def compute_string_sizes(strings): # Given a list {strings} of strings, returns a list of the lengths of # those strings, and their total. sizes = [ len(st) for st in strings ] tsize = 0; for sz in sizes: tsize += sz return sizes, tsize # ---------------------------------------------------------------------- def round_est_size(sz, eps_log): # Rounds an estimated size after adding # {eps_log} in log scale. Rounds down if {eps_log} # is negative, up otherwise. tiny = 1.0e-100 if eps_log < 0: rsz = int(floor(exp(log(sz+tiny) + eps_log))) assert rsz <= sz elif eps_log > 0: rsz = int(ceil(exp(log(sz+tiny) + eps_log))) assert rsz >= sz else: rsz = int(sz + 0.5) return rsz # ---------------------------------------------------------------------- def scale_size_fuzzy(size, scale, eps): # Scales the given {size} by the given {scale} with some # slack up or down. Specifically {±eps} times an estimate of the # uncertainty in log scale. # # The {size} may be an integer or a range represented as a # pair of integers {(lo,hi)}. The result will always be a range # Accept integer as singleton ranges. if isinstance(size, int): size = (size,size,) loz = size[0]; lo_dev_log = compute_dev_of_log(loz) los = round_est_size(scale * loz, -eps * lo_dev_log) hiz = size[1] hi_dev_log = compute_dev_of_log(hiz) his = round_est_size(scale * hiz, +eps * hi_dev_log) return (los,his,) # ---------------------------------------------------------------------- def scale_sizes_fuzzy(sizes, scale, eps): # Given a list {sizes} of integers or integer ranges, # scales each element {sz} with {scale_size_fuzzy(sz,scale,eps} # (quod videt). new_sizes = [] for sz in sizes: new_sizes.append(scale_size_fuzzy(sz, scale, eps)) return new_sizes # ---------------------------------------------------------------------- def compute_dev_of_log(es): # Assumes that {es} is the expected value of a size variable with a # log-normal distribution with deviation proportional to {sqrt(es)}. # Returns the approximate deviation of the distributon of the log of # the size. debug = False if debug: err.write(f"!@ {es = :3d}") eps = 1 es0 = hypot(es, eps) # In case {es} is zero. es0_log = log(es0) if debug: err.write(f" {es0 = :5.1f} {es0_log = :+7.3f}") var_abs = 0.10 * es0 # Rough guess - to be improved. dev_abs = sqrt(var_abs) if debug: err.write(f" {dev_abs = :7.3f}") es1 = hypot(es0 + dev_abs, eps) es1_log = log(es1) if debug: err.write(f" {es1 = :5.1f} {es1_log = :+7.3f}") dev_log = es1_log - es0_log if debug: err.write(f" {dev_log = :+7.3f}\n") return dev_log # ---------------------------------------------------------------------- def compute_frac_error(s, es): # Fractional error between a size {s} and the expected size range {es}, # relative to the estimated deviation of the latter, in log scale. debug = False if debug: err.write(f"!~ {s = :3d} es = {es[0]:3d}..{es[1]:3d}") assert len(es) == 2 and es[0] <= es[1] avg = es[0] if s < es[0] else es[1] if s > es[1] else s if debug: err.write(f" {avg = :3d}") if s == avg: frac = 0 else: eps = 1 s0 = hypot(s, eps) s0_log = log(s0) if debug: err.write(f" {s0 = :5.1f} {s0_log = :+7.3f}") avg0 = hypot(avg, eps) avg0_log = log(avg0) if debug: err.write(f" {avg0 = :5.1f} {avg0_log = :+7.3f}") dev_log = compute_dev_of_log(avg) if debug: err.write(f" {dev_log = :7.3f}") frac = (s0_log - avg0_log)/dev_log if debug: err.write(f" {frac = :7.3f}\n") return frac # ---------------------------------------------------------------------- def compute_rorre_carf(frac, es): # Inverse of fractional error eps = 1 if frac < 0: dev_log = compute_dev_of_log(es[0]) y = frac*dev_log + log(hypot(es[0], eps)) z = exp(y) s = 0 if z <= eps else sqrt(z*z - eps*eps) elif frac > 0: dev_log = compute_dev_of_log(es[1]) y = frac*dev_log + log(hypot(es[1], eps)) z = exp(y) s = 0 if z <= eps else sqrt(z*z - eps*eps) else: # Geometric mean of the range end values: s = sqrt(hypot(es[0], eps)*hypot(es[1], eps)) assert s >= eps s = sqrt(s*s - eps*eps) return s # ---------------------------------------------------------------------- def parse_size_ranges(szranges_str): # The list {szranges_str} must be a string consisting of {neg} items # separated by commas, where each item is either an integer or a pair of # integers separated by '..'. They are converted to a list # {szranges[0..neg-1]} of integer pairs {(lo,hi)}. # xitems = re.split(r"[,]", szranges_str) szranges = [] for xit in xitems: xit = xit.strip() if xit != "": m = re.fullmatch(r"([0-9]+)[.][.]([0-9]+)", xit) if m != None: it_min = int(m.group(1)) it_max = int(m.group(2)) assert 1 <= it_min and it_min <= it_max, f"bad expected range {it_min}..{it_max}" szranges.append((it_min,it_max,)) else: it_val = int(xit) szranges.append((it_val,it_val,)) return szranges # ---------------------------------------------------------------------- def format_range(szrange): los = szrange[0]; his = szrange[1] if los == his: return str(los) else: return str(los) + ".." + str(his) # ---------------------------------------------------------------------- def format_size_ranges(szranges): szranges_str = '' for szrange in szranges: if szranges_str != '': szranges_str += ',' egs_str = format_range(szrange) szranges_str += egs_str return szranges_str # ---------------------------------------------------------------------- def test_stuff(): def data_error(msg): err.write(f"** {msg}\n"); assert False # .................................................................... err.write("----------------------------------------\n") err.write("testing compute_frac_error, compute_rorre_carf ...\n") for gs in range(30): fr = compute_frac_error(gs, (15,20,)) rf = compute_rorre_carf(fr, (15,20,)) err.write(f"{gs = :4d} {fr = :7.3f} {rf = :7.3f}\n") err.write("----------------------------------------\n") err.write("testing compute_dev_of_log ...\n") for k in range(5): esk = 5*k esk_log = log(hypot(esk, 1)) dev_log = compute_dev_of_log(esk) err.write(f"{esk = :4d} {esk_log = :7.3f} {dev_log = :7.3f}\n") err.write("----------------------------------------\n") err.write("testing get_parsing_patterns, split_text_into_units ...\n") for utype in ( "wc", "wp", "ec", "ch", ): err.write(f"!* counting '{utype}' units\n") pat_line, pat_unit, pat_sepa, clean_sepa = \ get_parsing_patterns(utype) if utype == "ec" or utype == "wc" or utype == "wp": text = "foo,par.paz-q.?ofoo,parrifoo" words = [ "foo", "par", ] if utype == "wc" or utype == "wp": words += [ "[.]foo", "foo[.]", "[.]foo[.]", "foo[.]par", ] assert clean_sepa == '.' elif utype == "ec": assert clean_sepa == '' elif utype == "ch": text = "白石英：［味］甘微温。［主］消渴，阴痿不足，欬逆白石，青石黑石脂等：［味］甘平。［主］黄疸，泄利" words = [ "白石", "主" ] assert clean_sepa == '' else: assert False err.write(f"!* raw text = '{text}')\n") text = clean_up_raw_text(text, utype, data_error) err.write(f"!* cleaned text = '{text}')\n") units, ct_sepa = \ split_text_into_units(text, utype, pat_unit, pat_sepa, data_error) err.write(f"!* {ct_sepa = } {units = !r}\n") err.write("\n") for word in words: err.write(f"!* looking up '{word}'\n") oclist = list_occurrences_in_units(word, units, clean_sepa, utype, data_error) err.write(f"!* found = {oclist}\n") err.write("\n") return # ---------------------------------------------------------------------- if len(sys.argv) > 1 and sys.argv[1] == "SPF.TEST": test_stuff()